Dhalang 0.7.0 → 0.7.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7392a94eca6e888d2a81b779cde341fd7a9be0bbddd2d254e708e93aa18b7fe3
-  data.tar.gz: dc520a25fcaf30ad5584820bb45b9bcf72b35531024676e77d9ce0f54291f91f
+  metadata.gz: f8181be50a21d8c3b688b3ba8aa7f683f1bd6101acc36b6623b2948ffd0462ca
+  data.tar.gz: 7f04e3437befb446f2e2a45b90f46ffe17e97c56d570299ac1852c4d3fd2ed9a
 SHA512:
-  metadata.gz: 3b75659edee50ba18a726be62a32d39336d5cb8ce6ac94625e529a75015ba7dee45fb396bc292a34604c602369ebba39842c127753d537fc882665a48aacf249
-  data.tar.gz: 965cbdae8bc88057c4d7a7a366940a5bb8deedf367893fab5d2c7c437bf96dc9530caf96a138ae7b19ef87ce190c9e8740d83f36cf2a411ec745c3c18b5d208b
+  metadata.gz: 5a45db0cb7cbf08828e99ebd1a86fdc8e3dc8ac1fb01d10d3c521b2bbe38168fdd93e79271fe766504a1868d9cec124390ec3bf65cf513354ee6c974f0e50e37
+  data.tar.gz: 6f56ec00f075b58242fdf55c5d49ca2cb545fcf3b249942c1c4a0f1b102623ce7d633b7db25bb5f04450b25b599580fd6a81c502e434710e282cfcd6ed1af8f0

data/Gemfile.lock

@@ -1,14 +1,14 @@
 PATH
   remote: .
   specs:
-    Dhalang (0.7.0)
+    Dhalang (0.7.2)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    Ascii85 (1.1.0)
+    Ascii85 (1.1.1)
     afm (0.2.2)
-    bigdecimal (3.1.7)
+    bigdecimal (3.1.8)
     diff-lcs (1.5.1)
     fastimage (2.2.7)
     hashery (2.1.2)
@@ -23,15 +23,15 @@ GEM
       rspec-core (~> 3.13.0)
       rspec-expectations (~> 3.13.0)
       rspec-mocks (~> 3.13.0)
-    rspec-core (3.13.0)
+    rspec-core (3.13.2)
       rspec-support (~> 3.13.0)
-    rspec-expectations (3.13.0)
+    rspec-expectations (3.13.3)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.13.0)
-    rspec-mocks (3.13.0)
+    rspec-mocks (3.13.2)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.13.0)
-    rspec-support (3.13.1)
+    rspec-support (3.13.2)
     ruby-rc4 (0.1.5)
     ttfunk (1.8.0)
       bigdecimal (~> 3.1)

data/README.md

@@ -1,4 +1,4 @@
-# Dhalang [![Build](https://github.com/NielsSteensma/Dhalang/actions/workflows/build.yml/badge.svg)](https://github.com/NielsSteensma/Dhalang/actions/workflows/build.yml)
+# Dhalang [![Build](https://github.com/NielsSteensma/Dhalang/actions/workflows/build.yml/badge.svg)](https://github.com/NielsSteensma/Dhalang/actions/workflows/build.yml) [![Gem Version](https://badge.fury.io/rb/Dhalang.svg)](https://badge.fury.io/rb/Dhalang)
 
 > Dhalang is a Ruby wrapper for Google's Puppeteer.
 
@@ -11,7 +11,11 @@
 * Scrape HTML from webpages
 
 
-
+## Prerequisites
+* Node ≥ 18
+* Puppeteer ≥ 22
+* Unix shell ( Dhalang will not work on Windows shells )
+  
 ## Installation
 Add this line to your application's Gemfile:
 
@@ -21,11 +25,12 @@ And then execute:
 
     $ bundle update
 
-Install puppeteer in your application's root directory:
+Install puppeteer or puppeteer-core in your application's root directory:
 
-    $ npm install puppeteer
+    $ npm install puppeteer 
+    or
+    $ npm install puppeteer-core
 
-<sub>Dhalang and Puppeteer require Node ≥ 18 and Puppeteer ≥ 22</sub>
 ## Usage
 __PDF of a website url__  
 ```ruby
@@ -75,10 +80,9 @@ For example to only take a screenshot of the visible part of the page:
 Dhalang::Screenshot.get_from_url("https://www.google.com", :webp, {fullPage: false})
 ```
 
-A list of all possible PDF options that can be set, can be found at: https://github.com/puppeteer/puppeteer/blob/main/docs/api.md#pagepdfoptions
-
-A list of all possible screenshot options that can be set, can be found at: https://github.com/puppeteer/puppeteer/blob/main/docs/api.md#pagescreenshotoptions
+A list of all possible PDF options that can be set, can be found at: https://github.com/puppeteer/puppeteer/blob/main/docs/api/puppeteer.pdfoptions.md
 
+A list of all possible screenshot options that can be set, can be found at: https://github.com/puppeteer/puppeteer/blob/main/docs/api/puppeteer.screenshotoptions.md
 > The default Puppeteer options contain the options `headerTemplate` and `footerTemplate`. Puppeteer expects these to be HTML strings. By default, the Dhalang
 > gem passes all options as arguments in a `node ...` shell command.  In case the HTML strings are too long they might surpass the maximum
 > argument length of the host.  For example, on Linux the `MAX_ARG_LEN` is 128kB. Therefore, you can also pass the headers and footers as file path using the
@@ -86,24 +90,17 @@ A list of all possible screenshot options that can be set, can be found at: http
 >
 > For example: `Dhalang::PDF.get_from_url("https://www.google.com", {headerTemplateFile: '/tmp/header.html', footerTemplateFile: '/tmp/footer.html'})`
 
-
-## Custom user options
-You may want to change the way Dhalang interacts with Puppeteer in general. User options can be set by providing them in a hash as last argument to any calls you make to the library. Are you setting both custom PDF and user options? Then they should be passed as a single hash. 
-
-For example to set a custom navigation timeout:
-```ruby
-Dhalang::Screenshot.get_from_url("https://www.google.com", :jpeg, {navigationTimeout: 20000})  
-```
-
-Below table lists all possible configuration parameters that can be set:
+Below table lists more configuration parameters that can be set:
 | Key                | Description                                                                             | Default                         |
 |--------------------|-----------------------------------------------------------------------------------------|---------------------------------|
+| isHeadless         | Indicates if Chromium should be launched headless (useful for debugging)                | true                            |
+| slowMo             | Amount of milliseconds to slow down Puppeteer operations (useful for debugging)         | 0                               |
+| browserWebsocketUrl | Websocket url of remote chromium browser to use                                        | None                            |
 | navigationTimeout  | Amount of milliseconds until Puppeteer while timeout when navigating to the given page  | 10000                           |
 | printToPDFTimeout  | Amount of milliseconds until Puppeteer while timeout when calling Page.printToPDF       | 0 (unlimited)                   |
 | navigationWaitForSelector | If set, Dhalang will wait for the specified selector to appear before creating the screenshot or PDF | None        |
 | navigationWaitForXPath | If set, Dhalang will wait for the specified XPath to appear before creating the screenshot or PDF | None              |
 | userAgent          | User agent to send with the request                                                     | Default Puppeteer one           |
-| isHeadless         | Indicates if Chromium should be launched headless                                       | true                            |
 | isAutoHeight       | When set to true the height of generated PDFs will be based on the scrollHeight property of the document body | false     |
 | viewPort           | Custom viewport to use for the request                                                  | Default Puppeteer one           |
 | httpAuthenticationCredentials | Custom HTTP authentication credentials to use for the request                | None                            |

data/lib/Dhalang/configuration.rb

@@ -3,6 +3,7 @@ module Dhalang
   class Configuration
     NODE_MODULES_PATH = Dir.pwd + '/node_modules/'.freeze
     USER_OPTIONS = {
+      browserWebsocketUrl: '',
       navigationTimeout: 10000,
       printToPDFTimeout: 0, # unlimited
       navigationWaitUntil: 'load',
@@ -13,7 +14,8 @@ module Dhalang
       viewPort: '',
       httpAuthenticationCredentials: '',
       isAutoHeight: false,
-      chromeOptions: []
+      chromeOptions: [],
+      slowMo: 0
     }.freeze
     DEFAULT_PDF_OPTIONS = {
       scale: 1,
@@ -48,6 +50,7 @@ module Dhalang
     private_constant :DEFAULT_JPEG_OPTIONS
 
     private attr_accessor :page_url
+    private attr_accessor :browser_websocket_url
     private attr_accessor :temp_file_path
     private attr_accessor :temp_file_extension
     private attr_accessor :user_options

data/lib/Dhalang/url_utils.rb

@@ -6,9 +6,10 @@ module Dhalang
         #
         # @param [String] url The url to validate
         def self.validate(url)
-            if (url !~ URI::DEFAULT_PARSER.regexp[:ABS_URI])
-                raise URI::InvalidURIError, 'The given url was invalid, use format http://www.example.com'
-            end
+            parsed = URI.parse(url) # Raise URI::InvalidURIError on invalid URLs
+            return true if parsed.absolute?
+
+            raise URI::InvalidURIError, 'The given url was invalid, use format http://www.example.com'
         end
     end
-end
+end

data/lib/Dhalang/version.rb

@@ -1,3 +1,3 @@
 module Dhalang
-  VERSION = "0.7.0"
+  VERSION = "0.7.2"
 end

data/lib/js/dhalang.js

@@ -14,6 +14,7 @@ const fs = require('fs')
 
 /**
  * @typedef {Object} UserOptions
+ * @property {string} browserWebsocketUrl           - The websocket url of remote Chromium browser to use.
  * @property {number} navigationTimeout             - Maximum in milliseconds until navigation times out, we use a default of 10 seconds as timeout.
  * @property {string} navigationWaitUntil           - Determines when the navigation was finished, we wait here until the Window.load event is fired ( meaning all images, stylesheet, etc was loaded ).
  * @property {string} navigationWaitForSelector     - If set, specifies the selector Puppeteer should wait for to appear before continuing.
@@ -23,6 +24,7 @@ const fs = require('fs')
  * @property {Object} viewPort                      - The view port to use.
  * @property {Object} httpAuthenticationCredentials - The credentials to use for HTTP authentication.
  * @property {boolean} isAutoHeight                 - The height is automatically set
+ * @property {number} slowMo                        - Amount of milliseconds to slow down Puppeteer operations.
  */
 
 /**
@@ -47,7 +49,7 @@ exports.getConfiguration = function () {
 
 /**
  * Launches Puppeteer and returns its instance.
- * @param {UserOptions} configuration - The configuration to use.
+ * @param {Configuration} configuration - The configuration to use.
  * @returns {Promise<Object>}
  * The launched instance of Puppeteer.
  */
@@ -55,10 +57,18 @@ exports.launchPuppeteer = async function (configuration) {
     module.paths.push(configuration.puppeteerPath);
     const puppeteer = require('puppeteer');
     const launchArgs = ['--no-sandbox', '--disable-setuid-sandbox'].concat(configuration.userOptions.chromeOptions).filter((item, index, self) => self.indexOf(item) === index);
-    return await puppeteer.launch({
-        args: launchArgs,
-        headless: configuration.userOptions.isHeadless
-    });
+
+    if (configuration.userOptions['browserWebsocketUrl'] !== "") {
+        return await puppeteer.connect( {
+            "browserWSEndpoint": configuration.userOptions.browserWebsocketUrl
+        })
+    } else {
+        return await puppeteer.launch({
+            args: launchArgs,
+            headless: configuration.userOptions.isHeadless,
+            slowMo: configuration.userOptions.slowMo
+        });
+    }
 }
 
 /**
@@ -148,7 +158,7 @@ exports.getConfiguredPdfOptions = async function (page, configuration) {
 exports.getNavigationParameters = function (configuration) {
     return {
         timeout: configuration.userOptions.navigationTimeout,
-        waituntil: configuration.userOptions.navigationWaitUntil
+        waitUntil: configuration.userOptions.navigationWaitUntil
     }
 }
 

data/lib/js/html-scraper.js

@@ -6,9 +6,10 @@ const scrapeHtml = async () => {
     const configuration = dhalang.getConfiguration();
 
     let browser;
+    let page;
     try {
         browser = await dhalang.launchPuppeteer(configuration);
-        const page = await browser.newPage();
+        page = await browser.newPage();
         await dhalang.configure(page, configuration.userOptions);
         await dhalang.navigate(page, configuration);
         const html = await page.content();
@@ -17,8 +18,10 @@ const scrapeHtml = async () => {
         console.error(error.message);
         process.exit(1);
     } finally {
-        if (browser) {
+        if (browser && configuration.userOptions['browserWebsocketUrl'] === "") {
             browser.close();
+        } else {
+            page.close();
         }
         process.exit(0);
     }

data/lib/js/pdf-generator.js

@@ -5,9 +5,10 @@ const createPdf = async () => {
     const configuration = dhalang.getConfiguration();
 
     let browser;
+    let page;
     try {
         browser = await dhalang.launchPuppeteer(configuration);
-        const page = await browser.newPage();
+        page = await browser.newPage();
         await dhalang.configure(page, configuration.userOptions);
         await dhalang.navigate(page, configuration);
         const pdfOptions = await dhalang.getConfiguredPdfOptions(page, configuration);
@@ -21,8 +22,10 @@ const createPdf = async () => {
         console.error(error.message);
         process.exit(1);
     } finally {
-        if (browser) {
+        if (browser && configuration.userOptions['browserWebsocketUrl'] === "") {
             browser.close();
+        } else {
+            page.close();
         }
         process.exit();
     }

data/lib/js/screenshot-generator.js

@@ -5,9 +5,10 @@ const createScreenshot = async () => {
     const configuration = dhalang.getConfiguration();
 
     let browser;
+    let page;
     try {
         browser = await dhalang.launchPuppeteer(configuration);
-        const page = await browser.newPage();
+        page = await browser.newPage();
         await dhalang.configure(page, configuration.userOptions);
         await dhalang.navigate(page, configuration);
 
@@ -23,8 +24,10 @@ const createScreenshot = async () => {
         console.error(error.message);
         process.exit(1);
     } finally {
-        if (browser) {
+        if (browser && configuration.userOptions['browserWebsocketUrl'] === "") {
             browser.close();
+        } else {
+            page.close();
         }
         process.exit();
     }