Dhalang 0.4.0 → 0.6.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 835393ff0f501220a6c71edd20db19897e874b578f0921b37f70b5403840c928
-  data.tar.gz: 285639cb8e59c0416d82a15c15ef7e656c00e5d2498b19f885d8635dafc05314
+  metadata.gz: 0b5a8f2427673c1a21cbbfcc3f9b3119c11890b9dbb21a1656ec970c14b4faef
+  data.tar.gz: d98cac7a86e5c12c0fc845e71a8810c3adaa3c583fb53a4139f2e1408231cceb
 SHA512:
-  metadata.gz: aeacd63e036366dcd4f61da1b06fe65965378c6d4dba61de5401daa67f7f3f06744f01e7e4d24bd0d28a157e052eb5ff597695e978f73b9290594787ef48a482
-  data.tar.gz: 1888bee0d744f67529fa7927740fbe057ceb73c571c3d06d7de74e24ee028f0769592bc7107bb1696e31e448fd02f7ead676b57f0a94bb2d8ee68192e1382c18
+  metadata.gz: 142fcef44cc80374790a915356c7eda5b5a5d91e532c44a4a81515f07454b9c2a7a04d34d53b3cb481fb0ed2f58cc6a0ed974a0ad4d244e9622f754e32e47624
+  data.tar.gz: 8379ed46f315e7cf3cfa657c2972fbdc3e34bee4255f2be741e84835c82536d4ace52290fb0197e1638a77667fa0a59309357ce62bca3887ba746c44665d8e9c

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    Dhalang (0.3.1)
+    Dhalang (0.6.0)
 
 GEM
   remote: https://rubygems.org/

data/LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2019 NielsS
+Copyright (c) 2021 NielsS
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

data/README.md

@@ -3,11 +3,14 @@
 > Dhalang is a Ruby wrapper for Google's Puppeteer.
 
 
+
 ## Features
 * Generate PDFs from pages
 * Generate PDFs from html ( external images/stylesheets supported )  
 * Capture a screenshot of a webpage
 
+
+
 ## Installation
 Add this line to your application's Gemfile:
 
@@ -40,41 +43,66 @@ All methods return a string containing the PDF or JPEG/PNG in binary.
   
   
   
-## Custom configuration
-Depending on your use case you may want to change the way Dhalang interacts with Puppeteer. You can do this by passing a Hash with custom configuration parameters as last argument to any calls you make to the library.
+## Custom PDF/screenshot options
+To override the default options that are set by Dhalang you can pass as last argument a hash with the custom options you want to set.
+
+For example to set custom margins for PDFs:
+
+`Dhalang::PDF.get_from_url("https://www.google.com", {margin: { top: 100, right: 100, bottom: 100, left: 100}})
+`
+
+For example to only take a screenshot of the visible part of the page:
+`Dhalang::Screenshot.get_from_url_as_png("https://www.google.com", {fullPage: false})
+`
+
+A list of all possible PDF options that can be set, can be found at: https://github.com/puppeteer/puppeteer/blob/main/docs/api.md#pagepdfoptions
+
+A list of all possible screenshot options that can be set, can be found at: https://github.com/puppeteer/puppeteer/blob/main/docs/api.md#pagescreenshotoptions
+
 
-So for example:
-`Dhalang::Screenshot.get_from_url_as_jpeg("https://www.google.com", {navigation_timeout: 20000})`  
 
-Below table list the possible configuration parameters you can set:
+
+## Custom user options
+You may want to change the way Dhalang interacts with Puppeteer in general. User options can be set by providing them in a hash as last argument to any calls you make to the library. Are you setting both custom PDF and user options? Then they should be passed as a single hash. 
+
+For example to set a custom navigation timeout:
+`Dhalang::Screenshot.get_from_url_as_jpeg("https://www.google.com", {navigationTimeout: 20000})`  
+
+Below table lists all possible configuration parameters that can be set:
 | Key                | Description                                                                             | Default                         |
 |--------------------|-----------------------------------------------------------------------------------------|---------------------------------|
-| navigation_timeout | Amount of milliseconds until Puppeteer while timeout when navigating to the given page  | 10000                           |
-| user_agent         | User agent to send with the request                                                     | Default Puppeteer one           |
-| view_port          | Custom viewport to use for the request                                                  | Default Puppeteer one           |
-| http_authentication_credentials | Custom HTTP authentication credentials to use for the request              | None                            |
+| navigationTimeout  | Amount of milliseconds until Puppeteer while timeout when navigating to the given page  | 10000                           |
+| navigationWaitForSelector | If set, Dhalang will wait for the specified selector to appear before creating the screenshot or PDF | None        |
+| navigationWaitForXPath | If set, Dhalang will wait for the specified XPath to appear before creating the screenshot or PDF | None              |
+| userAgent          | User agent to send with the request                                                     | Default Puppeteer one           |
+| isHeadless         | Indicates if Chromium should be launched headless                                       | true                            |
+| isAutoHeight       | When set to true the height of generated PDFs will be based on the scrollHeight property of the document body | false     |
+| viewPort           | Custom viewport to use for the request                                                  | Default Puppeteer one           |
+| httpAuthenticationCredentials | Custom HTTP authentication credentials to use for the request                | None                            |
+
 
-## Examples
+
+## Examples of using Dhalang
 To return a PDF from a Rails controller you can do the following:  
-```
-def example_controller_method  
-    binary_pdf = Dhalang::PDF.get_from_url("https://www.google.com")  
-    send_data(binary_pdf, filename: 'pdfofgoogle.pdf', type: 'application/pdf')  
+```ruby
+def example_controller_method
+  binary_pdf = Dhalang::PDF.get_from_url("https://www.google.com")  
+  send_data(binary_pdf, filename: 'pdfofgoogle.pdf', type: 'application/pdf')  
 end
 ```
 
 To return a PNG from a Rails controller you can do the following:  
-```
-def example_controller_method  
-    binary_png = Dhalang::Screenshot.get_from_url_as_png("https://www.google.com")
-    send_data(binary_png, filename: 'screenshotofgoogle.png', type: 'image/png')   
+```ruby
+def example_controller_method
+  binary_png = Dhalang::Screenshot.get_from_url_as_png("https://www.google.com")
+  send_data(binary_png, filename: 'screenshotofgoogle.png', type: 'image/png')   
 end
 ```
 
 To return a JPEG from a Rails controller you can do the following:  
-```
-def example_controller_method  
-    binary_jpeg = Dhalang::Screenshot.get_from_url_as_jpeg("https://www.google.com")
-    send_data(binary_jpeg, filename: 'screenshotofgoogle.jpeg', type: 'image/jpeg')   
+```ruby
+def example_controller_method
+  binary_jpeg = Dhalang::Screenshot.get_from_url_as_jpeg("https://www.google.com")
+  send_data(binary_jpeg, filename: 'screenshotofgoogle.jpeg', type: 'image/jpeg')   
 end
 ```

data/lib/Dhalang/error.rb

@@ -0,0 +1 @@
+class DhalangError < StandardError; end

data/lib/Dhalang/puppeteer.rb

@@ -4,22 +4,20 @@ module Dhalang
         NODE_MODULES_PATH = Dir.pwd + '/node_modules/'.freeze
         private_constant :NODE_MODULES_PATH
 
-        NAVIGATION_TIMEOUT = 10000
-        private_constant :NAVIGATION_TIMEOUT
-
-        NAVIGATION_WAIT_UNTIL = 'load'
-        private_constant :NAVIGATION_WAIT_UNTIL
-
-        USER_AGENT = ''
-        private_constant :USER_AGENT
-
-        VIEW_PORT = ''
-        private_constant :VIEW_PORT
-
-        HTTP_AUTHENTICATION_CREDENTIALS = ''
-        private_constant :HTTP_AUTHENTICATION_CREDENTIALS
+        USER_OPTIONS = {
+                navigationTimeout: 10000,
+                navigationWaitUntil: 'load',
+                navigationWaitForSelector: '',
+                navigationWaitForXPath: '',
+                userAgent: '',
+                isHeadless: true,
+                viewPort: '',
+                httpAuthenticationCredentials: '',
+                isAutoHeight: false
+        }
+        private_constant :USER_OPTIONS
 
-        DEFAULT_OPTIONS = {
+        DEFAULT_PDF_OPTIONS = {
             scale: 1,
             displayHeaderFooter: false,
             headerTemplate: '',
@@ -31,9 +29,25 @@ module Dhalang
             width: '',
             height: '',
             margin: { top: 36, right: 36, bottom: 20, left: 36 },
-            preferCSSPageSiz: false
+            preferCSSPageSize: true,
+            omitBackground: false
+        }
+        private_constant :DEFAULT_PDF_OPTIONS
+
+        DEFAULT_PNG_OPTIONS = {
+            fullPage: true,
+            clip: nil,
+            omitBackground: false
+        }
+        private_constant :DEFAULT_PNG_OPTIONS
+
+        DEFAULT_JPEG_OPTIONS = {
+            quality: 100,
+            fullPage: true,
+            clip: nil,
+            omitBackground: false
         }
-        private_constant :DEFAULT_OPTIONS
+        private_constant :DEFAULT_JPEG_OPTIONS
 
 
         # Launches a new Node process, executing the (Puppeteer) script under the given script_path.
@@ -45,7 +59,17 @@ module Dhalang
         # @param [Object] options               Set of options to use, configurable by the user.
         def self.visit(page_url, script_path, temp_file_path, temp_file_extension, options)
             configuration = create_configuration(page_url, script_path, temp_file_path, temp_file_extension, options)
-            Kernel.system("node #{script_path} #{Shellwords.escape(configuration)}")
+
+            command = "node #{script_path} #{Shellwords.escape(configuration)}"
+
+            Open3.popen2e(command) do |_stdin, stdouterr, wait|
+                return nil if wait.value.success?
+
+                output = stdouterr.read.strip
+                output = nil if output == ''
+                message = output || "Exited with status #{wait.value.exitstatus}"
+                raise DhalangError, message
+            end
         end
 
 
@@ -62,16 +86,10 @@ module Dhalang
                 tempFilePath: temp_file_path,
                 puppeteerPath: NODE_MODULES_PATH,
                 imageType: temp_file_extension,
-                userOptions: {
-                    navigationParameters: {
-                        timeout: options.has_key?(:navigation_timeout) ? options[:navigation_timeout] : NAVIGATION_TIMEOUT,
-                        waitUntil: NAVIGATION_WAIT_UNTIL
-                    },
-                    userAgent: options.has_key?(:user_agent) ? options[:user_agent] : USER_AGENT,
-                    viewPort: options.has_key?(:view_port) ? options[:view_port] : VIEW_PORT,
-                    httpAuthenticationCredentials: options.has_key?(:http_authentication_credentials) ? options[:http_authentication_credentials] : HTTP_AUTHENTICATION_CREDENTIALS
-                },
-                pdfOptions: DEFAULT_OPTIONS.map { |option, value| [option, options.has_key?(option) ? options[option] : value] }.to_h
+                userOptions: USER_OPTIONS.map { |option, value| [option, options.has_key?(option) ? options[option] : value]}.to_h,
+                pdfOptions: DEFAULT_PDF_OPTIONS.map { |option, value| [option, options.has_key?(option) ? options[option] : value] }.to_h,
+                pngOptions: DEFAULT_PNG_OPTIONS.map { |option, value| [option, options.has_key?(option) ? options[option] : value] }.to_h,
+                jpegOptions: DEFAULT_JPEG_OPTIONS.map { |option, value| [option, options.has_key?(option) ? options[option] : value] }.to_h
             }.to_json
         end
     end

data/lib/Dhalang/version.rb

@@ -1,3 +1,3 @@
 module Dhalang
-  VERSION = "0.4.0"
+  VERSION = "0.6.2"
 end

data/lib/Dhalang.rb

@@ -4,9 +4,11 @@ module Dhalang
   require_relative 'Dhalang/version'
   require_relative 'Dhalang/url_utils'
   require_relative 'Dhalang/file_utils'
+  require_relative 'Dhalang/error'
   require_relative 'Dhalang/puppeteer'
   require 'uri'
   require 'tempfile'
   require 'shellwords'
   require 'json'
-end
+  require 'open3'
+end

data/lib/Screenshot.rb

@@ -33,6 +33,7 @@ module Dhalang
     # @return [String] The screenshot that was taken as binary.
     private_class_method def self.get(url, image_type, options)
       UrlUtils.validate(url)
+      validate_options(options)
       temp_file = FileUtils.create_temp_file(image_type)
       begin
         Puppeteer.visit(url, PUPPETEER_SCRIPT_PATH, temp_file.path, image_type, options)
@@ -42,5 +43,15 @@ module Dhalang
       end
       return binary_image_content
     end
+
+    # Raises an error if the given options might conflict with the Puppeteer configuration.
+    #
+    # @param [Hash] options The options to validate
+    private_class_method def self.validate_options(options)
+      symbolized_options = options.transform_keys(&:to_sym)
+      if symbolized_options.has_key?(:type)
+        raise DhalangError, 'Invalid option set: "type"'
+      end
+    end
   end
 end

data/lib/js/dhalang.js

@@ -4,14 +4,23 @@
  * @property {string} tempFilePath          - The path of the tempfile to write the screenshot/pdf to.
  * @property {string} puppeteerModulePath   - The path of the Puppeteer module.
  * @property {string} imageType             - The type of image to save ( undefined for pdfgenerator ).
- * @property {UserOptions} userOptions      - User defined and default parameters to use when navigating to pages with Puppeteer.
+ * @property {UserOptions} userOptions      - User defined and default parameters to use when navigating to pages.
+ * @property {Object} pdfOptions            - User defined and default parameters to use when creating PDFs. Note: Do not use directly, rather use {@link getConfiguredPdfOptions}.
+ * @property {Object} pngOptions            - User defined and default parameters to use when creating PNGs.
+ * @property {Object} jpegOptions           - User defined and default parameters to use when creating JPEGs.
  */
+
 /**
  * @typedef {Object} UserOptions
- * @property {NavigationParameters} navigationParameters    - The parameters to use when navigating to pages with Puppeteer.
- * @property {string} userAgent                             - The user agent to send with requests.
- * @property {Object} viewPort                              - The view port to use.
- * @property {Object} httpAuthenticationCredentials         - The credentials to use for HTTP authentication.
+ * @property {number} navigationTimeout             - Maximum in milliseconds until navigation times out, we use a default of 10 seconds as timeout.
+ * @property {string} navigationWaitUntil           - Determines when the navigation was finished, we wait here until the Window.load event is fired ( meaning all images, stylesheet, etc was loaded ).
+ * @property {string} navigationWaitForSelector     - If set, specifies the selector Puppeteer should wait for to appear before continuing.
+ * @property {string} navigationWaitForXPath        - If set, specifies the XPath Puppeteer should wait for to appear before continuing.
+ * @property {string} userAgent                     - The user agent to send with requests.
+ * @property {boolean} isHeadless                   - Indicates if Puppeteer should launch Chromium in headless mode.
+ * @property {Object} viewPort                      - The view port to use.
+ * @property {Object} httpAuthenticationCredentials - The credentials to use for HTTP authentication.
+ * @property {boolean} isAutoHeight                 - The height is automatically set
  */
 
 /**
@@ -20,6 +29,11 @@
  * @property {string} waituntil             - Determines when the navigation was finished, we wait here until the Window.load event is fired ( meaning all images, stylesheet, etc was loaded ).
  */
 
+/**
+ * @typedef {Object} WaitingParameters
+ * @property {number} timeout               - Maximum in milliseconds until navigation times out, we use a default of 10 seconds as timeout.
+ */
+
 /**
  * Parses the given configuration process argument from Ruby to a JS object.
  * @returns {Configuration}
@@ -31,33 +45,102 @@ exports.getConfiguration = function () {
 
 /**
  * Launches Puppeteer and returns its instance.
- * @param {string} puppeteerModulePath - The path puppeteer is under.
- * @returns {Promise<Object>} 
+ * @param {UserOptions} configuration - The configuration to use.
+ * @returns {Promise<Object>}
  * The launched instance of Puppeteer.
  */
-exports.launchPuppeteer = async function (puppeteerModulePath) {
-    module.paths.push(puppeteerModulePath);
+exports.launchPuppeteer = async function (configuration) {
+    module.paths.push(configuration.puppeteerPath);
     const puppeteer = require('puppeteer');
+    const launchArgs = ['--no-sandbox', '--disable-setuid-sandbox'];
     return await puppeteer.launch({
-        args: ['--no-sandbox', '--disable-setuid-sandbox']
+        args: launchArgs,
+        headless: configuration.userOptions.isHeadless
     });
 }
 
 /**
- * Configures the given Puppeteer page.
+ * Configures the given Puppeteer page object.
+ * @param {Object} page - The Puppeteer page object to configure.
+ * @param {UserOptions} userOptions - The user options to use.
+ */
+exports.configure = async function (page, userOptions) {
+    if (userOptions.userAgent !== "") {
+        await page.setUserAgent(userOptions.userAgent)
+    }
+
+    if (userOptions.viewPort !== "") {
+        await page.setViewport(userOptions.viewPort)
+    }
+
+    if (userOptions.httpAuthenticationCredentials !== "") {
+        await page.authenticate(userOptions.httpAuthenticationCredentials)
+    }
+}
+
+/**
+ * Makes the Puppeteer page object open the url with the specified navigation logic as specified in the given configuration.
+ * @param {Object} page - The Puppeteer page object to use for navigation.
+ * @param {Configuration} configuration - The configuration to use. 
+ */
+exports.navigate = async function (page, configuration) {
+    const navigationWaitForSelector = configuration.userOptions.navigationWaitForSelector;
+    const navigationWaitForXPath = configuration.userOptions.navigationWaitForXPath;
+
+    await page.goto(configuration.webPageUrl, this.getNavigationParameters(configuration));
+
+    if (navigationWaitForSelector !== "") {
+        await page.waitForSelector(navigationWaitForSelector, this.getWaitingParameters(configuration));
+    } else if (navigationWaitForXPath !== "") {
+        await page.waitForXPath(navigationWaitForXPath, this.getWaitingParameters(configuration));
+    } else {
+        await page.waitForTimeout(250);
+    }
+}
+
+/**
+ * Returns the PDF options to pass to Puppeteer based on the set user options and the documents body.  
  * @param {Object} page - The Puppeteer page to configure.
  * @param {UserOptions} configuration - The configuration to use.
+ * @returns {Object} - pdfOptions
  */
-exports.configurePage = async function (page, configuration) {
-    if (configuration.userAgent !== "") {
-        await page.setUserAgent(configuration.userAgent)
+exports.getConfiguredPdfOptions = async function (page, configuration) {
+    const pdfOptions = configuration.pdfOptions
+
+    if (configuration.userOptions.isAutoHeight === true) {
+        const pageHeight = await page.evaluate(() => {
+            return Math.max(document.body.scrollHeight, document.body.offsetHeight);
+        })
+        if (pageHeight) {
+            pdfOptions['height'] = pageHeight + 1 + 'px'
+        }
     }
 
-    if (configuration.viewPort !== "") {
-        await page.setViewport(configuration.viewPort)
+    return pdfOptions
+}
+
+/**
+ * Extracts the navigation parameters from the configuration in a format that is usable by Puppeteer.
+ * @param {Configuration} configuration - The configuration to extract the navigation parameters from.
+ * @returns {NavigationParameters}
+ * The extracted navigation parameters.
+ */
+exports.getNavigationParameters = function (configuration) {
+    return {
+        timeout: configuration.userOptions.navigationTimeout,
+        waituntil: configuration.userOptions.navigationWaitUntil
     }
+}
 
-    if (configuration.httpAuthenticationCredentials !== "") {
-        await page.authenticate(configuration.authenticationCredentials)
+
+/**
+ * Extracts the waiting parameters from the configuration in a format that is usable by Puppeteer.
+ * @param {Configuration} configuration - The configuration to extract the waiting parameters from.
+ * @returns {WaitingParameters} 
+ * The extracted waiting parameters.
+ */
+exports.getWaitingParameters = function (configuration) {
+    return {
+        timeout: configuration.userOptions.navigationTimeout
     }
-}
+}

data/lib/js/pdf-generator.js

@@ -6,19 +6,20 @@ const createPdf = async () => {
 
     let browser;
     try {
-        browser = await dhalang.launchPuppeteer(configuration.puppeteerPath);
+        browser = await dhalang.launchPuppeteer(configuration);
         const page = await browser.newPage();
-        await dhalang.configurePage(page, configuration.userOptions);
-        await page.goto(configuration.webPageUrl, configuration.userOptions.navigationParameters);
-        await page.waitForTimeout(250);
+        await dhalang.configure(page, configuration.userOptions);
+        await dhalang.navigate(page, configuration);
+        const pdfOptions = await dhalang.getConfiguredPdfOptions(page, configuration);
         await page.pdf({
             ...{
                 path: configuration.tempFilePath
             },
-            ...configuration.pdfOptions
+            ...pdfOptions
         });
     } catch (error) {
-        console.log(error.message);
+        console.error(error.message);
+        process.exit(1);
     } finally {
         if (browser) {
             browser.close();

data/lib/js/screenshot-generator.js

@@ -1,23 +1,27 @@
 'use strict';
 const dhalang = require('./dhalang')
 
-const createPdf = async () => {
+const createScreenshot = async () => {
     const configuration = dhalang.getConfiguration();
 
     let browser;
     try {
-        browser = await dhalang.launchPuppeteer(configuration.puppeteerPath);
+        browser = await dhalang.launchPuppeteer(configuration);
         const page = await browser.newPage();
-        await dhalang.configurePage(page, configuration.userOptions);
-        await page.goto(configuration.webPageUrl, configuration.userOptions.navigationParameters);
-        await page.waitForTimeout(250);
+        await dhalang.configure(page, configuration.userOptions);
+        await dhalang.navigate(page, configuration);
+
+        const screenshotOptions = configuration.imageType === "png" ? configuration.pngOptions : configuration.jpegOptions
         await page.screenshot({
-            path: configuration.tempFilePath,
-            type: configuration.imageType,
-            fullPage: true
+            ...{
+                path: configuration.tempFilePath,
+                type: configuration.imageType,
+            },
+            ...screenshotOptions
         });
     } catch (error) {
-        console.log(error.message);
+        console.error(error.message);
+        process.exit(1);
     } finally {
         if (browser) {
             browser.close();
@@ -25,4 +29,4 @@ const createPdf = async () => {
         process.exit();
     }
 };
-createPdf();
+createScreenshot();