Dhalang 0.5.0 → 0.6.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4bd71cb9e8eb643b85d2d2e68909db725b088b10ef29cd717634bc5429067ed4
-  data.tar.gz: 77b10c45d0c30132e3888c2cc04998502b34ae24a0e363e70b971987ec6a722c
+  metadata.gz: affc5a7a677cade8adf37073ab71155ad812fdb0765c947c560ecb8de50dafc3
+  data.tar.gz: 72a7122ea73080759aef9d5ccd375167da9efffee1836f92df990c7046df08d7
 SHA512:
-  metadata.gz: d36638644a8b02e167b76f4b943b527fd06dd20cdc6ef4c77ce37af1e41bb24f4f03f13549b0b3cfcd9a1693b7aad535448ebbfd01b6a54cb53be39d32bbd063
-  data.tar.gz: e31806a5c74557130c558eda263a0b72045814db398c1554fe5bfaa5ce0240f14b22b52ce1afd1d8c488936097a8c407ff255c53c166cf346054d57cff749c80
+  metadata.gz: a007aa999b92d55b8d10bf6e0683d9f5ddf833f5a3f26914abfbdda562a25bb197ba29595f4d55edbcd9c11ea15eb0a2c57da8bb5595e38f521e5a465c13aa24
+  data.tar.gz: 2bdf23451af29feaa0ba70b78c7af7ca603bc02db2dfffcbbd815dd766b4fc99972328a7cf2d6b15dad23dadfc51aa8f3ea10713652ed087df4fa46890211f3e

data/.travis.yml

@@ -1,13 +1,21 @@
 language: ruby
 node_js:
- - node
+- node
 rvm:
- - 2.6
+- 2.6
 addons:
   chrome: stable
 before_install:
- - nvm install v15.0.1
+- nvm install v15.0.1
 install:
- - bundle install
- - npm install
-script: bundle exec rake spec
+- bundle install
+- npm install
+script: bundle exec rake spec
+deploy:
+  provider: rubygems    
+  gemspec: Dhalang.gemspec
+  api_key:
+    secure: PMiaYF8Xtyyl9Kuhf9WwDRaUzUcqbe430DF8Rwk2qYM1oI8eiYbESswHVavAnrT+ZLKZVzp0+avXJ1TFjoz2tHBa7YVmmz76+RvECozwBXFVofK0GX8c6PVZTuuHgoanr1RVv0KTJOZXs5Aa+372Er31ESuwaaRPU2y/d8aotxtoe7MUoVle67S7Z28H53jwDCut54WFq6hKi+gNO/B+7y/e2efPgrqSktHxaPDGcaUaX9v8fAYBJgAulJLYMeRj+m7IQccajIjppr7vWDE1m9XEDAK71i3EZis96/z+KexB2bo3+Qc+TvpTWP5NeKw61uhtPn9HU4+XCaOV8Vz0e6PH2JfS82h333BcSifXjcFb9iQU8aEMpSpcjraUvvvu8JZepUUO//vtGEJLyOJ0L4f9XiUPhQ15OYjBsJ2ADFdNcqyFeASDuLoeKbAQrWunyTStb+wFrr5TzAoQp9ItZbEt6eHaI6WSd4qhJfzqU7NOzE06Ns03J1jMdmzMxREl8ENJFle65fQouQfRnfcZTS6qW7UX/7YZUiMw60Cpa15qHW+92glCkOuiZUhr4D999ejqMFY5Jd3tHGVBX0/1MWc2jl0fPk2/fueA/YW8xLd3xKP9p0KhbqWZhiMxIct5w718nfOLuSoL1VLJRImj4HAKmoCFKP4RE+1Vl95UwEY=
+  on:
+    branch: master
+    tags: true

data/Dhalang.gemspec

@@ -22,9 +22,9 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
-  spec.add_development_dependency "bundler", "~> 1.16"
-  spec.add_development_dependency "rake", "~> 13.0.1"
+  spec.add_development_dependency "bundler", "~> 2.3.6"
+  spec.add_development_dependency "fastimage", "~> 2.2.6"
+  spec.add_development_dependency "pdf-reader", "~> 2.9.0"
+  spec.add_development_dependency "rake", "~> 13.0.6"
   spec.add_development_dependency "rspec", "~> 3.0"
-  spec.add_development_dependency "pdf-reader", "~> 2.2"
-  spec.add_development_dependency "fastimage", "~> 1.8"
 end

data/Gemfile.lock

@@ -1,52 +1,49 @@
 PATH
   remote: .
   specs:
-    Dhalang (0.3.1)
+    Dhalang (0.6.2)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    Ascii85 (1.0.3)
-    addressable (2.3.8)
+    Ascii85 (1.1.0)
     afm (0.2.2)
-    diff-lcs (1.3)
-    fastimage (1.9.0)
-      addressable (~> 2.3.5)
+    diff-lcs (1.5.0)
+    fastimage (2.2.6)
     hashery (2.1.2)
-    pdf-reader (2.4.0)
-      Ascii85 (~> 1.0.0)
+    pdf-reader (2.9.0)
+      Ascii85 (~> 1.0)
       afm (~> 0.2.1)
       hashery (~> 2.0)
       ruby-rc4
       ttfunk
-    rake (13.0.1)
-    rspec (3.9.0)
-      rspec-core (~> 3.9.0)
-      rspec-expectations (~> 3.9.0)
-      rspec-mocks (~> 3.9.0)
-    rspec-core (3.9.1)
-      rspec-support (~> 3.9.1)
-    rspec-expectations (3.9.0)
+    rake (13.0.6)
+    rspec (3.10.0)
+      rspec-core (~> 3.10.0)
+      rspec-expectations (~> 3.10.0)
+      rspec-mocks (~> 3.10.0)
+    rspec-core (3.10.2)
+      rspec-support (~> 3.10.0)
+    rspec-expectations (3.10.2)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.9.0)
-    rspec-mocks (3.9.1)
+      rspec-support (~> 3.10.0)
+    rspec-mocks (3.10.3)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.9.0)
-    rspec-support (3.9.2)
+      rspec-support (~> 3.10.0)
+    rspec-support (3.10.3)
     ruby-rc4 (0.1.5)
-    ttfunk (1.6.2.1)
+    ttfunk (1.7.0)
 
 PLATFORMS
-  ruby
-  x64-mingw32
+  x86-mingw32
 
 DEPENDENCIES
   Dhalang!
-  bundler (~> 1.16)
-  fastimage (~> 1.8)
-  pdf-reader (~> 2.2)
-  rake (~> 13.0.1)
+  bundler (~> 2.3.6)
+  fastimage (~> 2.2.6)
+  pdf-reader (~> 2.9.0)
+  rake (~> 13.0.6)
   rspec (~> 3.0)
 
 BUNDLED WITH
-   1.17.3
+   2.3.6

data/LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2019 NielsS
+Copyright (c) 2021 NielsS
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

data/README.md

@@ -27,18 +27,25 @@ Install puppeteer in your application's root directory:
 <sub>NodeJS v10.18.1 or greater is required</sub>
 ## Usage
 __Get a PDF of a website url__  
-`Dhalang::PDF.get_from_url("https://www.google.com")`  
+```ruby
+Dhalang::PDF.get_from_url("https://www.google.com")
+```
 It is important to pass the complete url, leaving out https://, http:// or www. will result in an error.
-  
+
 __Get a PDF of a HTML string__  
-`Dhalang::PDF.get_from_html("<html><head></head><body><h1>examplestring</h1></body></html>")`  
+```ruby
+Dhalang::PDF.get_from_html("<html><head></head><body><h1>examplestring</h1></body></html>") 
+```
 
 __Get a PNG screenshot of a website__  
-`Dhalang::Screenshot.get_from_url_as_png("https://www.google.com")`  
+```ruby
+Dhalang::Screenshot.get_from_url_as_png("https://www.google.com")  
+```
 
 __Get a JPEG screenshot of a website__  
-`Dhalang::Screenshot.get_from_url_as_jpeg("https://www.google.com")`  
-
+```ruby
+Dhalang::Screenshot.get_from_url_as_jpeg("https://www.google.com")  
+```
 All methods return a string containing the PDF or JPEG/PNG in binary.   
   
   
@@ -47,59 +54,70 @@ All methods return a string containing the PDF or JPEG/PNG in binary.
 To override the default options that are set by Dhalang you can pass as last argument a hash with the custom options you want to set.
 
 For example to set custom margins for PDFs:
-
-`Dhalang::PDF.get_from_url("https://www.google.com", {margin: { top: 100, right: 100, bottom: 100, left: 100}})
-`
+```ruby
+Dhalang::PDF.get_from_url("https://www.google.com", {margin: { top: 100, right: 100, bottom: 100, left: 100}})
+```
 
 For example to only take a screenshot of the visible part of the page:
-`Dhalang::Screenshot.get_from_url_as_png("https://www.google.com", {fullPage: false})
-`
+```ruby
+Dhalang::Screenshot.get_from_url_as_png("https://www.google.com", {fullPage: false})
+```
 
 A list of all possible PDF options that can be set, can be found at: https://github.com/puppeteer/puppeteer/blob/main/docs/api.md#pagepdfoptions
 
 A list of all possible screenshot options that can be set, can be found at: https://github.com/puppeteer/puppeteer/blob/main/docs/api.md#pagescreenshotoptions
 
-
+> The default Puppeteer options contain the options `headerTemplate` and `footerTemplate`. Puppeteer expects these to be HTML strings. By default, the Dhalang
+> gem passes all options as arguments in a `node ...` shell command.  In case the HTML strings are too long they might surpass the maximum
+> argument length of the host.  For example, on Linux the `MAX_ARG_LEN` is 128kB. Therefore, you can also pass the headers and footers as file path using the
+> options `headerTemplateFile` and `footerTemplateFile`. These non-Puppeteer-options will be used to populate the Puppeteer-options `headerTemplate` and `footerTemplate`.
+>
+> For example: `Dhalang::PDF.get_from_url("https://www.google.com", {headerTemplateFile: '/tmp/header.html', footerTemplateFile: '/tmp/footer.html'})`
 
 
 ## Custom user options
 You may want to change the way Dhalang interacts with Puppeteer in general. User options can be set by providing them in a hash as last argument to any calls you make to the library. Are you setting both custom PDF and user options? Then they should be passed as a single hash. 
 
 For example to set a custom navigation timeout:
-`Dhalang::Screenshot.get_from_url_as_jpeg("https://www.google.com", {navigationTimeout: 20000})`  
+```ruby
+Dhalang::Screenshot.get_from_url_as_jpeg("https://www.google.com", {navigationTimeout: 20000})  
+```
 
 Below table lists all possible configuration parameters that can be set:
 | Key                | Description                                                                             | Default                         |
 |--------------------|-----------------------------------------------------------------------------------------|---------------------------------|
 | navigationTimeout  | Amount of milliseconds until Puppeteer while timeout when navigating to the given page  | 10000                           |
+| navigationWaitForSelector | If set, Dhalang will wait for the specified selector to appear before creating the screenshot or PDF | None        |
+| navigationWaitForXPath | If set, Dhalang will wait for the specified XPath to appear before creating the screenshot or PDF | None              |
 | userAgent          | User agent to send with the request                                                     | Default Puppeteer one           |
 | isHeadless         | Indicates if Chromium should be launched headless                                       | true                            |
+| isAutoHeight       | When set to true the height of generated PDFs will be based on the scrollHeight property of the document body | false     |
 | viewPort           | Custom viewport to use for the request                                                  | Default Puppeteer one           |
 | httpAuthenticationCredentials | Custom HTTP authentication credentials to use for the request                | None                            |
-
+| chromeOptions  | A array of [options](https://peter.sh/experiments/chromium-command-line-switches/) that can be passed to puppeteer in addition to the mandatory `['--no-sandbox', '--disable-setuid-sandbox']` | []                           |
 
 
 ## Examples of using Dhalang
 To return a PDF from a Rails controller you can do the following:  
-```
-def example_controller_method  
-    binary_pdf = Dhalang::PDF.get_from_url("https://www.google.com")  
-    send_data(binary_pdf, filename: 'pdfofgoogle.pdf', type: 'application/pdf')  
+```ruby
+def example_controller_method
+  binary_pdf = Dhalang::PDF.get_from_url("https://www.google.com")  
+  send_data(binary_pdf, filename: 'pdfofgoogle.pdf', type: 'application/pdf')  
 end
 ```
 
 To return a PNG from a Rails controller you can do the following:  
-```
-def example_controller_method  
-    binary_png = Dhalang::Screenshot.get_from_url_as_png("https://www.google.com")
-    send_data(binary_png, filename: 'screenshotofgoogle.png', type: 'image/png')   
+```ruby
+def example_controller_method
+  binary_png = Dhalang::Screenshot.get_from_url_as_png("https://www.google.com")
+  send_data(binary_png, filename: 'screenshotofgoogle.png', type: 'image/png')   
 end
 ```
 
 To return a JPEG from a Rails controller you can do the following:  
-```
-def example_controller_method  
-    binary_jpeg = Dhalang::Screenshot.get_from_url_as_jpeg("https://www.google.com")
-    send_data(binary_jpeg, filename: 'screenshotofgoogle.jpeg', type: 'image/jpeg')   
+```ruby
+def example_controller_method
+  binary_jpeg = Dhalang::Screenshot.get_from_url_as_jpeg("https://www.google.com")
+  send_data(binary_jpeg, filename: 'screenshotofgoogle.jpeg', type: 'image/jpeg')   
 end
 ```

data/lib/Dhalang/puppeteer.rb

@@ -7,10 +7,14 @@ module Dhalang
         USER_OPTIONS = {
                 navigationTimeout: 10000,
                 navigationWaitUntil: 'load',
+                navigationWaitForSelector: '',
+                navigationWaitForXPath: '',
                 userAgent: '',
                 isHeadless: true,
                 viewPort: '',
-                httpAuthenticationCredentials: ''
+                httpAuthenticationCredentials: '',
+                isAutoHeight: false,
+                chromeOptions: []
         }
         private_constant :USER_OPTIONS
 
@@ -19,6 +23,8 @@ module Dhalang
             displayHeaderFooter: false,
             headerTemplate: '',
             footerTemplate: '',
+            headerTemplateFile: '',
+            footerTemplateFile: '',
             printBackground: true,
             landscape: false,
             pageRanges: '',
@@ -26,7 +32,8 @@ module Dhalang
             width: '',
             height: '',
             margin: { top: 36, right: 36, bottom: 20, left: 36 },
-            preferCSSPageSiz: false
+            preferCSSPageSize: true,
+            omitBackground: false
         }
         private_constant :DEFAULT_PDF_OPTIONS
 

data/lib/Dhalang/version.rb

@@ -1,3 +1,3 @@
 module Dhalang
-  VERSION = "0.5.0"
+  VERSION = "0.6.3"
 end

data/lib/js/dhalang.js

@@ -1,3 +1,5 @@
+const fs = require('fs')
+
 /**
  * @typedef {Object} Configuration
  * @property {string} webPageUrl            - The url of the webpage to visit.
@@ -5,7 +7,7 @@
  * @property {string} puppeteerModulePath   - The path of the Puppeteer module.
  * @property {string} imageType             - The type of image to save ( undefined for pdfgenerator ).
  * @property {UserOptions} userOptions      - User defined and default parameters to use when navigating to pages.
- * @property {Object} pdfOptions            - User defined and default parameters to use when creating PDFs.
+ * @property {Object} pdfOptions            - User defined and default parameters to use when creating PDFs. Note: Do not use directly, rather use {@link getConfiguredPdfOptions}.
  * @property {Object} pngOptions            - User defined and default parameters to use when creating PNGs.
  * @property {Object} jpegOptions           - User defined and default parameters to use when creating JPEGs.
  */
@@ -14,10 +16,13 @@
  * @typedef {Object} UserOptions
  * @property {number} navigationTimeout             - Maximum in milliseconds until navigation times out, we use a default of 10 seconds as timeout.
  * @property {string} navigationWaitUntil           - Determines when the navigation was finished, we wait here until the Window.load event is fired ( meaning all images, stylesheet, etc was loaded ).
+ * @property {string} navigationWaitForSelector     - If set, specifies the selector Puppeteer should wait for to appear before continuing.
+ * @property {string} navigationWaitForXPath        - If set, specifies the XPath Puppeteer should wait for to appear before continuing.
  * @property {string} userAgent                     - The user agent to send with requests.
  * @property {boolean} isHeadless                   - Indicates if Puppeteer should launch Chromium in headless mode.
  * @property {Object} viewPort                      - The view port to use.
  * @property {Object} httpAuthenticationCredentials - The credentials to use for HTTP authentication.
+ * @property {boolean} isAutoHeight                 - The height is automatically set
  */
 
 /**
@@ -26,6 +31,11 @@
  * @property {string} waituntil             - Determines when the navigation was finished, we wait here until the Window.load event is fired ( meaning all images, stylesheet, etc was loaded ).
  */
 
+/**
+ * @typedef {Object} WaitingParameters
+ * @property {number} timeout               - Maximum in milliseconds until navigation times out, we use a default of 10 seconds as timeout.
+ */
+
 /**
  * Parses the given configuration process argument from Ruby to a JS object.
  * @returns {Configuration}
@@ -38,13 +48,13 @@ exports.getConfiguration = function () {
 /**
  * Launches Puppeteer and returns its instance.
  * @param {UserOptions} configuration - The configuration to use.
- * @returns {Promise<Object>} 
+ * @returns {Promise<Object>}
  * The launched instance of Puppeteer.
  */
 exports.launchPuppeteer = async function (configuration) {
     module.paths.push(configuration.puppeteerPath);
     const puppeteer = require('puppeteer');
-    const launchArgs = ['--no-sandbox', '--disable-setuid-sandbox'];
+    const launchArgs = ['--no-sandbox', '--disable-setuid-sandbox'].concat(configuration.userOptions.chromeOptions).filter((item, index, self) => self.indexOf(item) === index);
     return await puppeteer.launch({
         args: launchArgs,
         headless: configuration.userOptions.isHeadless
@@ -52,28 +62,84 @@ exports.launchPuppeteer = async function (configuration) {
 }
 
 /**
- * Configures the given Puppeteer page.
+ * Configures the given Puppeteer page object.
+ * @param {Object} page - The Puppeteer page object to configure.
+ * @param {UserOptions} userOptions - The user options to use.
+ */
+exports.configure = async function (page, userOptions) {
+    if (userOptions.userAgent !== "") {
+        await page.setUserAgent(userOptions.userAgent)
+    }
+
+    if (userOptions.viewPort !== "") {
+        await page.setViewport(userOptions.viewPort)
+    }
+
+    if (userOptions.httpAuthenticationCredentials !== "") {
+        await page.authenticate(userOptions.httpAuthenticationCredentials)
+    }
+}
+
+/**
+ * Makes the Puppeteer page object open the url with the specified navigation logic as specified in the given configuration.
+ * @param {Object} page - The Puppeteer page object to use for navigation.
+ * @param {Configuration} configuration - The configuration to use. 
+ */
+exports.navigate = async function (page, configuration) {
+    const navigationWaitForSelector = configuration.userOptions.navigationWaitForSelector;
+    const navigationWaitForXPath = configuration.userOptions.navigationWaitForXPath;
+
+    await page.goto(configuration.webPageUrl, this.getNavigationParameters(configuration));
+
+    if (navigationWaitForSelector !== "") {
+        await page.waitForSelector(navigationWaitForSelector, this.getWaitingParameters(configuration));
+    } else if (navigationWaitForXPath !== "") {
+        await page.waitForXPath(navigationWaitForXPath, this.getWaitingParameters(configuration));
+    } else {
+        await page.waitForTimeout(250);
+    }
+}
+
+/**
+ * Returns the PDF options to pass to Puppeteer based on the set user options and the documents body.  
  * @param {Object} page - The Puppeteer page to configure.
  * @param {UserOptions} configuration - The configuration to use.
+ * @returns {Object} - pdfOptions
  */
-exports.configurePage = async function (page, configuration) {
-    if (configuration.userAgent !== "") {
-        await page.setUserAgent(configuration.userAgent)
-    }
+exports.getConfiguredPdfOptions = async function (page, configuration) {
+    const pdfOptions = configuration.pdfOptions
 
-    if (configuration.viewPort !== "") {
-        await page.setViewport(configuration.viewPort)
+    if (pdfOptions['headerTemplateFile'] !== '') {
+        try {
+            pdfOptions['headerTemplate'] = fs.readFileSync(pdfOptions['headerTemplateFile'], 'utf8')
+        } catch(err) {
+            // ignoring file related errors
+        }
+    }
+    if (pdfOptions['footerTemplateFile'] !== '') {
+        try {
+            pdfOptions['footerTemplate'] = fs.readFileSync(pdfOptions['footerTemplateFile'], 'utf8')
+        } catch(err) {
+            // ignoring file related errors
+        }
     }
 
-    if (configuration.httpAuthenticationCredentials !== "") {
-        await page.authenticate(configuration.authenticationCredentials)
+    if (configuration.userOptions.isAutoHeight === true) {
+        const pageHeight = await page.evaluate(() => {
+            return Math.max(document.body.scrollHeight, document.body.offsetHeight);
+        })
+        if (pageHeight) {
+            pdfOptions['height'] = pageHeight + 1 + 'px'
+        }
     }
+
+    return pdfOptions
 }
 
 /**
  * Extracts the navigation parameters from the configuration in a format that is usable by Puppeteer.
  * @param {Configuration} configuration - The configuration to extract the navigation parameters from.
- * @returns {NavigationParameters} 
+ * @returns {NavigationParameters}
  * The extracted navigation parameters.
  */
 exports.getNavigationParameters = function (configuration) {
@@ -81,4 +147,17 @@ exports.getNavigationParameters = function (configuration) {
         timeout: configuration.userOptions.navigationTimeout,
         waituntil: configuration.userOptions.navigationWaitUntil
     }
-}
+}
+
+
+/**
+ * Extracts the waiting parameters from the configuration in a format that is usable by Puppeteer.
+ * @param {Configuration} configuration - The configuration to extract the waiting parameters from.
+ * @returns {WaitingParameters} 
+ * The extracted waiting parameters.
+ */
+exports.getWaitingParameters = function (configuration) {
+    return {
+        timeout: configuration.userOptions.navigationTimeout
+    }
+}

data/lib/js/pdf-generator.js

@@ -8,14 +8,14 @@ const createPdf = async () => {
     try {
         browser = await dhalang.launchPuppeteer(configuration);
         const page = await browser.newPage();
-        await dhalang.configurePage(page, configuration.userOptions);
-        await page.goto(configuration.webPageUrl, dhalang.getNavigationParameters(configuration));
-        await page.waitForTimeout(250);
+        await dhalang.configure(page, configuration.userOptions);
+        await dhalang.navigate(page, configuration);
+        const pdfOptions = await dhalang.getConfiguredPdfOptions(page, configuration);
         await page.pdf({
             ...{
                 path: configuration.tempFilePath
             },
-            ...configuration.pdfOptions
+            ...pdfOptions
         });
     } catch (error) {
         console.error(error.message);

data/lib/js/screenshot-generator.js

@@ -1,16 +1,16 @@
 'use strict';
 const dhalang = require('./dhalang')
 
-const createPdf = async () => {
+const createScreenshot = async () => {
     const configuration = dhalang.getConfiguration();
 
     let browser;
     try {
         browser = await dhalang.launchPuppeteer(configuration);
         const page = await browser.newPage();
-        await dhalang.configurePage(page, configuration.userOptions);
-        await page.goto(configuration.webPageUrl, dhalang.getNavigationParameters(configuration));
-        await page.waitForTimeout(250);
+        await dhalang.configure(page, configuration.userOptions);
+        await dhalang.navigate(page, configuration);
+
         const screenshotOptions = configuration.imageType === "png" ? configuration.pngOptions : configuration.jpegOptions
         await page.screenshot({
             ...{
@@ -29,4 +29,4 @@ const createPdf = async () => {
         process.exit();
     }
 };
-createPdf();
+createScreenshot();