Dhalang 0.3.0 → 0.6.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0e81627f9004f346443d46b9ab0befaec1c11c95a6c81521c9e1340901d97427
-  data.tar.gz: adb156ead06f2b4886f7ce0972b4477f56a1a793d3fa4fd9fe289e899088f309
+  metadata.gz: c68cc819480b82b9d2693cc4689e54237650e2cc2f4d1e6e40297b711e4e9c76
+  data.tar.gz: c5af3754067a98ccc3af4c93266827e498888d26c16557bae82b6defa3c818b3
 SHA512:
-  metadata.gz: 6019c5635f2020d8c49c5dfc4cab09f4bd59c5445402016e894579fd4a68097b2675559950aa7f2a8cdf014db4ae1d0381682ecbcc04e7de6bf0e57195caf1fa
-  data.tar.gz: c833cdfb54c3f4f99b3e047fa1381555025c1faf48f723ec38eff8a29ad572b20b22d9dfada76fda0b71d9c45b734ea3fbae378ef5d4ca7872aa07f1abd698e6
+  metadata.gz: b483a6aa15f278dd3d49529e539160b999ed30cfc26a8e32ec77b9a862a7b4ae67c274f9fc7a6d46c207f204ed0882d3badf2a1491aca2b46ba6ae7504074190
+  data.tar.gz: e432a18e55b081e3eafbb4085483b406f879be835c0d00647ca38a38f9d7ec2ed3caefbc6b2f7a78362422eb6290b9473202aa5a3c27a6976baeef9784e58728

data/.travis.yml

@@ -1,8 +1,12 @@
 language: ruby
+node_js:
+ - node
 rvm:
  - 2.6
 addons:
   chrome: stable
+before_install:
+ - nvm install v15.0.1
 install:
  - bundle install
  - npm install

data/Dhalang.gemspec

@@ -13,8 +13,6 @@ Gem::Specification.new do |spec|
   spec.summary       = "Ruby wrapper for Puppeteer. Generate screenshots and PDF's from HTML!"
   spec.homepage      = "https://github.com/NielsSteensma/Dhalang"
 
-
-
   # Specify which files should be added to the gem when it is released.
   # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
   spec.files         = Dir.chdir(File.expand_path('..', __FILE__)) do

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    Dhalang (0.2.0)
+    Dhalang (0.6.0)
 
 GEM
   remote: https://rubygems.org/
@@ -49,4 +49,4 @@ DEPENDENCIES
   rspec (~> 3.0)
 
 BUNDLED WITH
-   1.17.2
+   1.17.3

data/LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2019 NielsS
+Copyright (c) 2021 NielsS
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

data/README.md

@@ -3,11 +3,14 @@
 > Dhalang is a Ruby wrapper for Google's Puppeteer.
 
 
+
 ## Features
 * Generate PDFs from pages
 * Generate PDFs from html ( external images/stylesheets supported )  
 * Capture a screenshot of a webpage
 
+
+
 ## Installation
 Add this line to your application's Gemfile:
 
@@ -21,7 +24,7 @@ Install puppeteer in your application's root directory:
 
     $ npm install puppeteer
 
-<sub>NodeJS v7.6.0 or greater is required</sub>
+<sub>NodeJS v10.18.1 or greater is required</sub>
 ## Usage
 __Get a PDF of a website url__  
 `Dhalang::PDF.get_from_url("https://www.google.com")`  
@@ -38,7 +41,49 @@ __Get a JPEG screenshot of a website__
 
 All methods return a string containing the PDF or JPEG/PNG in binary.   
   
-To return the PDF from a Rails controller you can do the following:  
+  
+  
+## Custom PDF/screenshot options
+To override the default options that are set by Dhalang you can pass as last argument a hash with the custom options you want to set.
+
+For example to set custom margins for PDFs:
+
+`Dhalang::PDF.get_from_url("https://www.google.com", {margin: { top: 100, right: 100, bottom: 100, left: 100}})
+`
+
+For example to only take a screenshot of the visible part of the page:
+`Dhalang::Screenshot.get_from_url_as_png("https://www.google.com", {fullPage: false})
+`
+
+A list of all possible PDF options that can be set, can be found at: https://github.com/puppeteer/puppeteer/blob/main/docs/api.md#pagepdfoptions
+
+A list of all possible screenshot options that can be set, can be found at: https://github.com/puppeteer/puppeteer/blob/main/docs/api.md#pagescreenshotoptions
+
+
+
+
+## Custom user options
+You may want to change the way Dhalang interacts with Puppeteer in general. User options can be set by providing them in a hash as last argument to any calls you make to the library. Are you setting both custom PDF and user options? Then they should be passed as a single hash. 
+
+For example to set a custom navigation timeout:
+`Dhalang::Screenshot.get_from_url_as_jpeg("https://www.google.com", {navigationTimeout: 20000})`  
+
+Below table lists all possible configuration parameters that can be set:
+| Key                | Description                                                                             | Default                         |
+|--------------------|-----------------------------------------------------------------------------------------|---------------------------------|
+| navigationTimeout  | Amount of milliseconds until Puppeteer while timeout when navigating to the given page  | 10000                           |
+| navigationWaitForSelector | If set, Dhalang will wait for the specified selector to appear before creating the screenshot or PDF | None        |
+| navigationWaitForXPath | If set, Dhalang will wait for the specified XPath to appear before creating the screenshot or PDF | None              |
+| userAgent          | User agent to send with the request                                                     | Default Puppeteer one           |
+| isHeadless         | Indicates if Chromium should be launched headless                                       | true                            |
+| isAutoHeight       | When set to true the height of generated PDFs will be based on the scrollHeight property of the document body | false     |
+| viewPort           | Custom viewport to use for the request                                                  | Default Puppeteer one           |
+| httpAuthenticationCredentials | Custom HTTP authentication credentials to use for the request                | None                            |
+
+
+
+## Examples of using Dhalang
+To return a PDF from a Rails controller you can do the following:  
 ```
 def example_controller_method  
     binary_pdf = Dhalang::PDF.get_from_url("https://www.google.com")  
@@ -46,7 +91,7 @@ def example_controller_method
 end
 ```
 
-To return the PNG from a Rails controller you can do the following:  
+To return a PNG from a Rails controller you can do the following:  
 ```
 def example_controller_method  
     binary_png = Dhalang::Screenshot.get_from_url_as_png("https://www.google.com")
@@ -54,7 +99,7 @@ def example_controller_method
 end
 ```
 
-To return the JPEG from a Rails controller you can do the following:  
+To return a JPEG from a Rails controller you can do the following:  
 ```
 def example_controller_method  
     binary_jpeg = Dhalang::Screenshot.get_from_url_as_jpeg("https://www.google.com")

data/lib/Dhalang.rb

@@ -4,8 +4,11 @@ module Dhalang
   require_relative 'Dhalang/version'
   require_relative 'Dhalang/url_utils'
   require_relative 'Dhalang/file_utils'
+  require_relative 'Dhalang/error'
   require_relative 'Dhalang/puppeteer'
   require 'uri'
   require 'tempfile'
   require 'shellwords'
-end
+  require 'json'
+  require 'open3'
+end

data/lib/Dhalang/error.rb

@@ -0,0 +1 @@
+class DhalangError < StandardError; end

data/lib/Dhalang/puppeteer.rb

@@ -3,15 +3,94 @@ module Dhalang
     class Puppeteer
         NODE_MODULES_PATH = Dir.pwd + '/node_modules/'.freeze
         private_constant :NODE_MODULES_PATH
-        
+
+        USER_OPTIONS = {
+                navigationTimeout: 10000,
+                navigationWaitUntil: 'load',
+                navigationWaitForSelector: '',
+                navigationWaitForXPath: '',
+                userAgent: '',
+                isHeadless: true,
+                viewPort: '',
+                httpAuthenticationCredentials: '',
+                isAutoHeight: false
+        }
+        private_constant :USER_OPTIONS
+
+        DEFAULT_PDF_OPTIONS = {
+            scale: 1,
+            displayHeaderFooter: false,
+            headerTemplate: '',
+            footerTemplate: '',
+            printBackground: true,
+            landscape: false,
+            pageRanges: '',
+            format: 'A4',
+            width: '',
+            height: '',
+            margin: { top: 36, right: 36, bottom: 20, left: 36 },
+            preferCSSPageSize: true,
+            omitBackground: false
+        }
+        private_constant :DEFAULT_PDF_OPTIONS
+
+        DEFAULT_PNG_OPTIONS = {
+            fullPage: true,
+            clip: nil,
+            omitBackground: false
+        }
+        private_constant :DEFAULT_PNG_OPTIONS
+
+        DEFAULT_JPEG_OPTIONS = {
+            quality: 100,
+            fullPage: true,
+            clip: nil,
+            omitBackground: false
+        }
+        private_constant :DEFAULT_JPEG_OPTIONS
+
+
         # Launches a new Node process, executing the (Puppeteer) script under the given script_path.
         #
         # @param [String] page_url              The url to pass to the goTo method of Puppeteer.
         # @param [String] script_path           The absolute path of the JS script to execute.
-        # @param [String] temp_file_path        The absolute path of the temp file to use to write any actions tom from Puppeteer.
+        # @param [String] temp_file_path        The absolute path of the temp file to use to write any actions from Puppeteer.
         # @param [String] temp_file_extension   The extension of the temp file.
-        def self.visit(page_url, script_path, temp_file_path, temp_file_extension)
-            system("node #{script_path} #{Shellwords.escape(NODE_MODULES_PATH)} #{page_url} #{Shellwords.escape(temp_file_path)} #{Shellwords.escape(temp_file_extension)}")
+        # @param [Object] options               Set of options to use, configurable by the user.
+        def self.visit(page_url, script_path, temp_file_path, temp_file_extension, options)
+            configuration = create_configuration(page_url, script_path, temp_file_path, temp_file_extension, options)
+
+            command = "node #{script_path} #{Shellwords.escape(configuration)}"
+
+            Open3.popen2e(command) do |_stdin, stdouterr, wait|
+                return nil if wait.value.success?
+
+                output = stdouterr.read.strip
+                output = nil if output == ''
+                message = output || "Exited with status #{wait.value.exitstatus}"
+                raise DhalangError, message
+            end
+        end
+
+
+        # Returns a JSON string with the configuration to use within the Puppeteer script.
+        #
+        # @param [String] page_url              The url to pass to the goTo method of Puppeteer.
+        # @param [String] script_path           The absolute path of the JS script to execute.
+        # @param [String] temp_file_path        The absolute path of the temp file to use to write any actions from Puppeteer.
+        # @param [String] temp_file_extension   The extension of the temp file.
+        # @param [Hash]   options               Set of options to use, configurable by the user.
+        private_class_method def self.create_configuration(page_url, script_path, temp_file_path, temp_file_extension, options)
+            {
+                webPageUrl: page_url,
+                tempFilePath: temp_file_path,
+                puppeteerPath: NODE_MODULES_PATH,
+                imageType: temp_file_extension,
+                userOptions: USER_OPTIONS.map { |option, value| [option, options.has_key?(option) ? options[option] : value]}.to_h,
+                pdfOptions: DEFAULT_PDF_OPTIONS.map { |option, value| [option, options.has_key?(option) ? options[option] : value] }.to_h,
+                pngOptions: DEFAULT_PNG_OPTIONS.map { |option, value| [option, options.has_key?(option) ? options[option] : value] }.to_h,
+                jpegOptions: DEFAULT_JPEG_OPTIONS.map { |option, value| [option, options.has_key?(option) ? options[option] : value] }.to_h
+            }.to_json
         end
     end
-end
+end

data/lib/Dhalang/version.rb

@@ -1,3 +1,3 @@
 module Dhalang
-  VERSION = "0.3.0"
+  VERSION = "0.6.1"
 end

data/lib/PDF.rb

@@ -6,25 +6,27 @@ module Dhalang
     
     # Captures the full webpage under the given url as PDF.
     #
-    # @param  [String] url    The url to get as PDF.
+    # @param  [String] url      The url to get as PDF.
+    # @param  [Hash]   options  User configurable options.
     #
     # @return [String] The PDF that was created as binary.
-    def self.get_from_url(url)
+    def self.get_from_url(url, options = {})
       UrlUtils.validate(url)
-      get(url)
+      get(url, options)
     end
 
     # Captures the full HTML as PDF.
     # Useful when creating dynamic content, for example invoices.
     #
-    # @param  [String] html   The html to get as PDF.
+    # @param  [String]  html     The html to get as PDF.
+    # @param  [Hash]    options  User configurable options.
     #
     # @return [String] The PDF that was created as binary.
-    def self.get_from_html(html)
+    def self.get_from_html(html, options = {})
       html_file = FileUtils.create_temp_file("html", html)
       url = "file://" + html_file.path
       begin
-          binary_pdf_content = get(url)
+          binary_pdf_content = get(url, options)
       ensure
         FileUtils.delete(html_file)
       end
@@ -34,13 +36,14 @@ module Dhalang
     
     # Groups and executes the logic for creating a PDF of a webpage.
     #
-    # @param  [String] url    The url to create a PDF for.
+    # @param  [String]  url      The url to create a PDF for.
+    # @param  [Hash]    options  Set of options to use, passed by the user of this library.
     #
     # @return [String] The PDF that was created as binary.
-    private_class_method def self.get(url)
+    private_class_method def self.get(url, options)
       temp_file = FileUtils.create_temp_file("pdf")
       begin
-        Puppeteer.visit(url, PUPPETEER_SCRIPT_PATH, temp_file.path, "pdf")
+        Puppeteer.visit(url, PUPPETEER_SCRIPT_PATH, temp_file.path, "pdf", options)
         binary_pdf_content = FileUtils.read_binary(temp_file.path)
       ensure
         FileUtils.delete(temp_file)

data/lib/Screenshot.rb

@@ -7,37 +7,51 @@ module Dhalang
     # Captures a full JPEG screenshot of the webpage under the given url.
     #
     # @param  [String] url        The url to take a screenshot of.
+    # @param  [Hash]   options    User configurable options.
     #
     # @return [String] the screenshot that was taken as binary.
-    def self.get_from_url_as_jpeg(url)
-      get(url, "jpeg")
+    def self.get_from_url_as_jpeg(url, options = {})
+      get(url, "jpeg", options)
     end
 
     # Captures a full PNG screenshot of the webpage under the given url.
     #
     # @param  [String] url        The url to take a screenshot of.
+    # @param  [Hash]   options    User configurable options.
     #
     # @return [String] The screenshot that was taken as binary.
-    def self.get_from_url_as_png(url)
-      get(url, "png")
+    def self.get_from_url_as_png(url, options = {})
+      get(url, "png", options)
     end
     
     # Groups and executes the logic for taking a screenhot of a webpage.
     #
     # @param  [String] url        The url to take a screenshot of.
     # @param  [String] image_type The image type to use for storing the screenshot.
+    # @param  [Hash]   options    Set of options to use, passed by the user of this library.
     #
     # @return [String] The screenshot that was taken as binary.
-    private_class_method def self.get(url, image_type)
+    private_class_method def self.get(url, image_type, options)
       UrlUtils.validate(url)
+      validate_options(options)
       temp_file = FileUtils.create_temp_file(image_type)
       begin
-        Puppeteer.visit(url, PUPPETEER_SCRIPT_PATH, temp_file.path, image_type)
+        Puppeteer.visit(url, PUPPETEER_SCRIPT_PATH, temp_file.path, image_type, options)
         binary_image_content = FileUtils.read_binary(temp_file.path)
       ensure
         FileUtils.delete(temp_file)
       end
       return binary_image_content
     end
+
+    # Raises an error if the given options might conflict with the Puppeteer configuration.
+    #
+    # @param [Hash] options The options to validate
+    private_class_method def self.validate_options(options)
+      symbolized_options = options.transform_keys(&:to_sym)
+      if symbolized_options.has_key?(:type)
+        raise DhalangError, 'Invalid option set: "type"'
+      end
+    end
   end
 end

data/lib/js/dhalang.js

@@ -1,54 +1,146 @@
 /**
  * @typedef {Object} Configuration
- * @property {string} webPageUrl - The url of the webpage to visit.
- * @property {string} tempFilePath - The path of the tempfile to write the screenshot/pdf to.
- * @property {string} puppeteerModulePath - The path of the Puppeteer module.
- * @property {string} imageType - The type of image to save ( undefined for pdfgenerator ).
+ * @property {string} webPageUrl            - The url of the webpage to visit.
+ * @property {string} tempFilePath          - The path of the tempfile to write the screenshot/pdf to.
+ * @property {string} puppeteerModulePath   - The path of the Puppeteer module.
+ * @property {string} imageType             - The type of image to save ( undefined for pdfgenerator ).
+ * @property {UserOptions} userOptions      - User defined and default parameters to use when navigating to pages.
+ * @property {Object} pdfOptions            - User defined and default parameters to use when creating PDFs. Note: Do not use directly, rather use {@link getConfiguredPdfOptions}.
+ * @property {Object} pngOptions            - User defined and default parameters to use when creating PNGs.
+ * @property {Object} jpegOptions           - User defined and default parameters to use when creating JPEGs.
+ */
+
+/**
+ * @typedef {Object} UserOptions
+ * @property {number} navigationTimeout             - Maximum in milliseconds until navigation times out, we use a default of 10 seconds as timeout.
+ * @property {string} navigationWaitUntil           - Determines when the navigation was finished, we wait here until the Window.load event is fired ( meaning all images, stylesheet, etc was loaded ).
+ * @property {string} navigationWaitForSelector     - If set, specifies the selector Puppeteer should wait for to appear before continuing.
+ * @property {string} navigationWaitForXPath        - If set, specifies the XPath Puppeteer should wait for to appear before continuing.
+ * @property {string} userAgent                     - The user agent to send with requests.
+ * @property {boolean} isHeadless                   - Indicates if Puppeteer should launch Chromium in headless mode.
+ * @property {Object} viewPort                      - The view port to use.
+ * @property {Object} httpAuthenticationCredentials - The credentials to use for HTTP authentication.
+ * @property {boolean} isAutoHeight                 - The height is automatically set
  */
 
 /**
  * @typedef {Object} NavigationParameters
- * @property {number} timeout - Maximum in milliseconds until navigation times out, we use a default of 10 seconds as timeout.
- * @property {string} waituntil - Determines when the navigation was finished, we wait here until the Window.load event is fired ( meaning all images, stylesheet, etc was loaded ).
+ * @property {number} timeout               - Maximum in milliseconds until navigation times out, we use a default of 10 seconds as timeout.
+ * @property {string} waituntil             - Determines when the navigation was finished, we wait here until the Window.load event is fired ( meaning all images, stylesheet, etc was loaded ).
+ */
+
+/**
+ * @typedef {Object} WaitingParameters
+ * @property {number} timeout               - Maximum in milliseconds until navigation times out, we use a default of 10 seconds as timeout.
  */
 
 /**
- * Generates a configuration object based on the given process arguments from Ruby.
- * @param {Boolean} isForScreenshotGenerator - Indicates if this configuration is for a screenshot generator.
+ * Parses the given configuration process argument from Ruby to a JS object.
  * @returns {Configuration}
- * The generated configuration object.
+ * The configuration object.
  */
-exports.getConfiguration = function (isForScreenshotGenerator) {
-    return {
-        puppeteerPath: process.argv[2],
-        webPageUrl: process.argv[3],
-        tempFilePath: process.argv[4],
-        imageType: isForScreenshotGenerator ? process.argv[5] : undefined
-    }
+exports.getConfiguration = function () {
+    return JSON.parse(process.argv[2])
 }
 
 /**
  * Launches Puppeteer and returns its instance.
- * @param {string} puppeteerModulePath - The path puppeteer is under.
- * @returns {Promise<Object>} 
+ * @param {UserOptions} configuration - The configuration to use.
+ * @returns {Promise<Object>}
  * The launched instance of Puppeteer.
  */
-exports.launchPuppeteer = async function (puppeteerModulePath) {
-    module.paths.push(puppeteerModulePath);
+exports.launchPuppeteer = async function (configuration) {
+    module.paths.push(configuration.puppeteerPath);
     const puppeteer = require('puppeteer');
+    const launchArgs = ['--no-sandbox', '--disable-setuid-sandbox'];
     return await puppeteer.launch({
-        args: ['--no-sandbox', '--disable-setuid-sandbox']
+        args: launchArgs,
+        headless: configuration.userOptions.isHeadless
     });
 }
 
 /**
- * Returns a new object containing the navigation parameters to use when navigating with Puppeteer to web pages.
+ * Configures the given Puppeteer page object.
+ * @param {Object} page - The Puppeteer page object to configure.
+ * @param {UserOptions} userOptions - The user options to use.
+ */
+exports.configure = async function (page, userOptions) {
+    if (userOptions.userAgent !== "") {
+        await page.setUserAgent(userOptions.userAgent)
+    }
+
+    if (userOptions.viewPort !== "") {
+        await page.setViewport(userOptions.viewPort)
+    }
+
+    if (userOptions.httpAuthenticationCredentials !== "") {
+        await page.authenticate(userOptions.authenticationCredentials)
+    }
+}
+
+/**
+ * Makes the Puppeteer page object open the url with the specified navigation logic as specified in the given configuration.
+ * @param {Object} page - The Puppeteer page object to use for navigation.
+ * @param {Configuration} configuration - The configuration to use. 
+ */
+exports.navigate = async function (page, configuration) {
+    const navigationWaitForSelector = configuration.userOptions.navigationWaitForSelector;
+    const navigationWaitForXPath = configuration.userOptions.navigationWaitForXPath;
+
+    await page.goto(configuration.webPageUrl, this.getNavigationParameters(configuration));
+
+    if (navigationWaitForSelector !== "") {
+        await page.waitForSelector(navigationWaitForSelector, this.getWaitingParameters(configuration));
+    } else if (navigationWaitForXPath !== "") {
+        await page.waitForXPath(navigationWaitForXPath, this.getWaitingParameters(configuration));
+    } else {
+        await page.waitForTimeout(250);
+    }
+}
+
+/**
+ * Returns the PDF options to pass to Puppeteer based on the set user options and the documents body.  
+ * @param {Object} page - The Puppeteer page to configure.
+ * @param {UserOptions} configuration - The configuration to use.
+ * @returns {Object} - pdfOptions
+ */
+exports.getConfiguredPdfOptions = async function (page, configuration) {
+    const pdfOptions = configuration.pdfOptions
+
+    if (configuration.userOptions.isAutoHeight === true) {
+        const pageHeight = await page.evaluate(() => {
+            return Math.max(document.body.scrollHeight, document.body.offsetHeight);
+        })
+        if (pageHeight) {
+            pdfOptions['height'] = pageHeight + 1 + 'px'
+        }
+    }
+
+    return pdfOptions
+}
+
+/**
+ * Extracts the navigation parameters from the configuration in a format that is usable by Puppeteer.
+ * @param {Configuration} configuration - The configuration to extract the navigation parameters from.
  * @returns {NavigationParameters}
- * The navigation parameters to use.
+ * The extracted navigation parameters.
+ */
+exports.getNavigationParameters = function (configuration) {
+    return {
+        timeout: configuration.userOptions.navigationTimeout,
+        waituntil: configuration.userOptions.navigationWaitUntil
+    }
+}
+
+
+/**
+ * Extracts the waiting parameters from the configuration in a format that is usable by Puppeteer.
+ * @param {Configuration} configuration - The configuration to extract the waiting parameters from.
+ * @returns {WaitingParameters} 
+ * The extracted waiting parameters.
  */
-exports.getNavigationParameters = function () {
+exports.getWaitingParameters = function (configuration) {
     return {
-        timeout: 10000,
-        waitUntil: 'load'
+        timeout: configuration.userOptions.navigationTimeout
     }
 }