Dhalang 0.6.5 → 0.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 35f7dcd3cc7787c9e13429033b9c177d6f5c82f0b94364a2d4efbe3ffe601a16
-  data.tar.gz: b204fe4df6c7b3e0da1e7d59407a711a9bd1d9f41c7828b05eccc478499e2b0b
+  metadata.gz: 7392a94eca6e888d2a81b779cde341fd7a9be0bbddd2d254e708e93aa18b7fe3
+  data.tar.gz: dc520a25fcaf30ad5584820bb45b9bcf72b35531024676e77d9ce0f54291f91f
 SHA512:
-  metadata.gz: 25c582e213b4b26851e59859ab24115f136ea02b4b2889eb99502a9b3d3ddc7688f667ffe656d4f79afb81a65dda0b81a2e482a8d310890a1baf5c11fd17dbc0
-  data.tar.gz: 3cd5d4ece1b4e8f87a59abeb781ae241e8b7a515665f6b5c57cc636ccb767a98ebc7a7be71f7746f379bf71d083aba33eb7003c524e9efa1d652d77a940b2656
+  metadata.gz: 3b75659edee50ba18a726be62a32d39336d5cb8ce6ac94625e529a75015ba7dee45fb396bc292a34604c602369ebba39842c127753d537fc882665a48aacf249
+  data.tar.gz: 965cbdae8bc88057c4d7a7a366940a5bb8deedf367893fab5d2c7c437bf96dc9530caf96a138ae7b19ef87ce190c9e8740d83f36cf2a411ec745c3c18b5d208b

data/.github/workflows/build.yml

@@ -0,0 +1,20 @@
+name: Build
+on:
+  - push
+  - pull_request
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: 3.1.3
+          bundler-cache: true
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 18
+          cache: 'npm'
+          cache-dependency-path: '**/package-lock.json'
+      - run: npm ci
+      - run: bundle exec rake spec

data/.github/workflows/publish.yml

@@ -0,0 +1,23 @@
+name: Publish
+on:
+  release:
+    types:
+      - published
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+    env:
+      GEM_HOST_API_KEY: ${{ secrets.RUBYGEMS_API_KEY }}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: 3.1.3
+          bundler-cache: true
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 18
+      - run: npm ci
+      - run: bundle exec rake spec
+      - run: gem build -o Dhalang.gem Dhalang.gemspec
+      - run: gem push Dhalang.gem

data/.gitignore

@@ -50,3 +50,4 @@ build-iPhoneSimulator/
 
 # unless supporting rvm < 1.11.0 or doing something fancy, ignore this:
 .rvmrc
+.npmrc

data/Gemfile.lock

@@ -1,15 +1,16 @@
 PATH
   remote: .
   specs:
-    Dhalang (0.6.4)
+    Dhalang (0.7.0)
 
 GEM
   remote: https://rubygems.org/
   specs:
     Ascii85 (1.1.0)
     afm (0.2.2)
-    diff-lcs (1.5.0)
-    fastimage (2.2.6)
+    bigdecimal (3.1.7)
+    diff-lcs (1.5.1)
+    fastimage (2.2.7)
     hashery (2.1.2)
     pdf-reader (2.9.2)
       Ascii85 (~> 1.0)
@@ -18,24 +19,26 @@ GEM
       ruby-rc4
       ttfunk
     rake (13.0.6)
-    rspec (3.11.0)
-      rspec-core (~> 3.11.0)
-      rspec-expectations (~> 3.11.0)
-      rspec-mocks (~> 3.11.0)
-    rspec-core (3.11.0)
-      rspec-support (~> 3.11.0)
-    rspec-expectations (3.11.0)
+    rspec (3.13.0)
+      rspec-core (~> 3.13.0)
+      rspec-expectations (~> 3.13.0)
+      rspec-mocks (~> 3.13.0)
+    rspec-core (3.13.0)
+      rspec-support (~> 3.13.0)
+    rspec-expectations (3.13.0)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.11.0)
-    rspec-mocks (3.11.1)
+      rspec-support (~> 3.13.0)
+    rspec-mocks (3.13.0)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.11.0)
-    rspec-support (3.11.0)
+      rspec-support (~> 3.13.0)
+    rspec-support (3.13.1)
     ruby-rc4 (0.1.5)
-    ttfunk (1.7.0)
+    ttfunk (1.8.0)
+      bigdecimal (~> 3.1)
 
 PLATFORMS
-  x86_64-darwin-20
+  arm64-darwin-21
+  x86_64-linux
 
 DEPENDENCIES
   Dhalang!
@@ -46,4 +49,4 @@ DEPENDENCIES
   rspec (~> 3.0)
 
 BUNDLED WITH
-   2.3.15
+   2.3.7

data/README.md

@@ -1,13 +1,14 @@
-# Dhalang [![Build Status](https://travis-ci.com/NielsSteensma/Dhalang.svg?token=XZgKAByw2KZjcrsCh8gW&branch=master)](https://travis-ci.com/NielsSteensma/Dhalang)
+# Dhalang [![Build](https://github.com/NielsSteensma/Dhalang/actions/workflows/build.yml/badge.svg)](https://github.com/NielsSteensma/Dhalang/actions/workflows/build.yml)
 
 > Dhalang is a Ruby wrapper for Google's Puppeteer.
 
 
 
 ## Features
-* Generate PDFs from pages
-* Generate PDFs from html ( external images/stylesheets supported )  
-* Capture a screenshot of a webpage
+* Generate PDFs from webpages
+* Generate PDFs from HTML ( external images/stylesheets supported )  
+* Capture screenshots from webpages
+* Scrape HTML from webpages
 
 
 
@@ -24,39 +25,44 @@ Install puppeteer in your application's root directory:
 
     $ npm install puppeteer
 
-<sub>NodeJS v10.18.1 or greater is required</sub>
+<sub>Dhalang and Puppeteer require Node ≥ 18 and Puppeteer ≥ 22</sub>
 ## Usage
-__Get a PDF of a website url__  
+__PDF of a website url__  
 ```ruby
 Dhalang::PDF.get_from_url("https://www.google.com")
 ```
 It is important to pass the complete url, leaving out https://, http:// or www. will result in an error.
 
-__Get a PDF of a HTML string__  
+__PDF of a HTML string__  
 ```ruby
 Dhalang::PDF.get_from_html("<html><head></head><body><h1>examplestring</h1></body></html>") 
 ```
 
-__Get a PNG screenshot of a website__  
+__PNG screenshot of a website__  
 ```ruby
 Dhalang::Screenshot.get_from_url("https://www.google.com", :png)  
 ```
 
-__Get a JPEG screenshot of a website__  
+__JPEG screenshot of a website__  
 ```ruby
 Dhalang::Screenshot.get_from_url("https://www.google.com", :jpeg)  
 ```
 
-__Get a WEBP screenshot of a website__  
+__WEBP screenshot of a website__  
 ```ruby
 Dhalang::Screenshot.get_from_url("https://www.google.com", :webp)  
 ```
 
-All methods return a string containing the PDF or JPEG/PNG/WEBP in binary.   
+__HTML of a website__
+```ruby
+Dhalang::Scraper.html("https://www.google.com")  
+```
+
+Above methods either return a string containing the PDF/JPEG/PNG/WEBP in binary or the scraped HTML.   
   
   
   
-## Custom PDF/screenshot options
+## Custom options
 To override the default options that are set by Dhalang you can pass as last argument a hash with the custom options you want to set.
 
 For example to set custom margins for PDFs:

data/lib/Dhalang/configuration.rb

@@ -0,0 +1,87 @@
+module Dhalang
+  # Groups Puppeteer and Dhalang configuration.
+  class Configuration
+    NODE_MODULES_PATH = Dir.pwd + '/node_modules/'.freeze
+    USER_OPTIONS = {
+      navigationTimeout: 10000,
+      printToPDFTimeout: 0, # unlimited
+      navigationWaitUntil: 'load',
+      navigationWaitForSelector: '',
+      navigationWaitForXPath: '',
+      userAgent: '',
+      isHeadless: true,
+      viewPort: '',
+      httpAuthenticationCredentials: '',
+      isAutoHeight: false,
+      chromeOptions: []
+    }.freeze
+    DEFAULT_PDF_OPTIONS = {
+      scale: 1,
+      displayHeaderFooter: false,
+      headerTemplate: '',
+      footerTemplate: '',
+      headerTemplateFile: '',
+      footerTemplateFile: '',
+      printBackground: true,
+      landscape: false,
+      pageRanges: '',
+      format: 'A4',
+      width: '',
+      height: '',
+      margin: { top: 36, right: 36, bottom: 20, left: 36 },
+      preferCSSPageSize: true,
+      omitBackground: false
+    }.freeze
+    DEFAULT_SCREENSHOT_OPTIONS = {
+      fullPage: true,
+      clip: nil,
+      omitBackground: false
+    }.freeze
+    DEFAULT_JPEG_OPTIONS = {
+      quality: 100
+    }.freeze
+
+    private_constant :NODE_MODULES_PATH
+    private_constant :USER_OPTIONS
+    private_constant :DEFAULT_PDF_OPTIONS
+    private_constant :DEFAULT_SCREENSHOT_OPTIONS
+    private_constant :DEFAULT_JPEG_OPTIONS
+
+    private attr_accessor :page_url
+    private attr_accessor :temp_file_path
+    private attr_accessor :temp_file_extension
+    private attr_accessor :user_options
+    private attr_accessor :pdf_options
+    private attr_accessor :screenshot_options
+    private attr_accessor :jpeg_options
+
+    # @param [Hash]   custom_options        Changes that override default.
+    # @param [String] page_url              Url for Puppeteer to visit.
+    # @param [String] temp_file_path        Absolute path of temp file to use for writing script results.
+    #                                       Can be nil for scripts using stdout.
+    # @param [String] temp_file_extension   Extension of temp file. Can be nil for scripts using stdout.
+    def initialize(custom_options, page_url, temp_file_path = nil, temp_file_extension = nil)
+      self.page_url = page_url
+      self.temp_file_path = temp_file_path
+      self.temp_file_extension = temp_file_extension
+      self.user_options = USER_OPTIONS.map { |key, default_value| [key, custom_options.fetch(key, default_value)] }
+      self.pdf_options = DEFAULT_PDF_OPTIONS.map { |key, default_value| [key, custom_options.fetch(key, default_value)] }
+      self.screenshot_options = DEFAULT_SCREENSHOT_OPTIONS.map { |key, default_value| [key, custom_options.fetch(key, default_value)] }
+      self.jpeg_options = DEFAULT_JPEG_OPTIONS.map { |key, default_value| [key, custom_options.fetch(key, default_value)] }
+    end
+
+    # Returns configuration as JSON string.
+    def json
+      return {
+        webPageUrl: page_url,
+        tempFilePath: temp_file_path,
+        puppeteerPath: NODE_MODULES_PATH,
+        imageType: temp_file_extension,
+        userOptions: user_options.to_h,
+        pdfOptions: pdf_options.to_h,
+        screenshotOptions: screenshot_options.to_h,
+        jpegOptions: jpeg_options.to_h
+      }.to_json
+    end
+  end
+end

data/lib/Dhalang/node_script_invoker.rb

@@ -0,0 +1,29 @@
+module Dhalang
+  class NodeScriptInvoker
+
+    # Executes JS script under given script_path by launching a new Node process.
+    #
+    # @param [String] script_path           Absolute path of JS script to execute.
+    # @param [Configuration] configuration  Configuration to use.
+    def self.execute_script(script_path, configuration)
+      command = create_node_command(script_path, configuration)
+      Open3.popen2e(command) do |_stdin, stdouterr, wait|
+        return nil if wait.value.success?
+
+        output = stdouterr.read.strip
+        output = nil if output == ''
+        message = output || "Exited with status #{wait.value.exitstatus}"
+        raise DhalangError, message
+      end
+    end
+
+
+    # Returns a [String] with node command that invokes the provided script with the configuration.
+    #
+    # @param [String] script_path           Absolute path of JS script to invoke.
+    # @param [Configuration] configuration  Configuration to use.
+    private_class_method def self.create_node_command(script_path, configuration)
+      "node #{script_path} #{Shellwords.escape(configuration.json)}"
+    end
+  end
+end

data/lib/Dhalang/version.rb

@@ -1,3 +1,3 @@
 module Dhalang
-  VERSION = "0.6.5"
+  VERSION = "0.7.0"
 end

data/lib/Dhalang.rb

@@ -1,11 +1,13 @@
 module Dhalang
   require_relative 'PDF'
   require_relative 'Screenshot'
+  require_relative 'Scraper'
   require_relative 'Dhalang/version'
   require_relative 'Dhalang/url_utils'
   require_relative 'Dhalang/file_utils'
   require_relative 'Dhalang/error'
-  require_relative 'Dhalang/puppeteer'
+  require_relative 'Dhalang/configuration'
+  require_relative 'Dhalang/node_script_invoker'
   require 'uri'
   require 'tempfile'
   require 'shellwords'

data/lib/PDF.rb

@@ -1,8 +1,8 @@
 module Dhalang
   # Allows consumers of this library to create PDFs with Puppeteer. 
   class PDF
-    PUPPETEER_SCRIPT_PATH = File.expand_path('../js/pdf-generator.js', __FILE__).freeze
-    private_constant :PUPPETEER_SCRIPT_PATH
+    SCRIPT_PATH = File.expand_path('../js/pdf-generator.js', __FILE__).freeze
+    private_constant :SCRIPT_PATH
     
     # Captures the full webpage under the given url as PDF.
     #
@@ -43,7 +43,8 @@ module Dhalang
     private_class_method def self.get(url, options)
       temp_file = FileUtils.create_temp_file("pdf")
       begin
-        Puppeteer.visit(url, PUPPETEER_SCRIPT_PATH, temp_file.path, "pdf", options)
+        configuration = Configuration.new(options, url, temp_file.path, "pdf")
+        NodeScriptInvoker.execute_script(SCRIPT_PATH, configuration)
         binary_pdf_content = FileUtils.read_binary(temp_file.path)
       ensure
         FileUtils.delete(temp_file)

data/lib/Scraper.rb

@@ -0,0 +1,26 @@
+module Dhalang
+  # Provides functionality for scraping webpages.
+  class Scraper
+    SCRIPT_PATH = File.expand_path('../js/html-scraper.js', __FILE__).freeze
+    private_constant :SCRIPT_PATH
+    
+    # Scrapes full HTML content under given url.
+    #
+    # @param  [String] url      Url to scrape.
+    # @param  [Hash]   options  User configurable options.
+    #
+    # @return [String] Scraped HTML content.
+    def self.html(url, options = {})
+      UrlUtils.validate(url)
+      temp_file = FileUtils.create_temp_file("html")
+      begin
+        configuration = Configuration.new(options, url, temp_file.path, "html")
+        NodeScriptInvoker.execute_script(SCRIPT_PATH, configuration)
+        html = IO.read(temp_file.path)
+      ensure
+        FileUtils.delete(temp_file)
+      end
+      return html
+    end
+  end
+end

data/lib/Screenshot.rb

@@ -1,9 +1,9 @@
 module Dhalang
   # Allows consumers of this library to take screenshots with Puppeteer. 
   class Screenshot
-    PUPPETEER_SCRIPT_PATH = File.expand_path('../js/screenshot-generator.js', __FILE__).freeze
+    SCRIPT_PATH = File.expand_path('../js/screenshot-generator.js', __FILE__).freeze
     IMAGE_TYPES = [:jpeg, :png, :webp].freeze
-    private_constant :PUPPETEER_SCRIPT_PATH
+    private_constant :SCRIPT_PATH
     private_constant :IMAGE_TYPES
     
     # <b>DEPRECATED:</b> Please use `get_from_url(url, :jpeg)` instead.
@@ -44,7 +44,8 @@ module Dhalang
 
       temp_file = FileUtils.create_temp_file(image_type)
       begin
-        Puppeteer.visit(url, PUPPETEER_SCRIPT_PATH, temp_file.path, image_type, options)
+        configuration = Configuration.new(options, url, temp_file.path, image_type)
+        NodeScriptInvoker.execute_script(SCRIPT_PATH, configuration)
         binary_image_content = FileUtils.read_binary(temp_file.path)
       ensure
         FileUtils.delete(temp_file)

data/lib/js/dhalang.js

@@ -96,7 +96,7 @@ exports.navigate = async function (page, configuration) {
     } else if (navigationWaitForXPath !== "") {
         await page.waitForXPath(navigationWaitForXPath, this.getWaitingParameters(configuration));
     } else {
-        await page.waitForTimeout(250);
+        await new Promise(r => setTimeout(r, 250));
     }
 }
 

data/lib/js/html-scraper.js

@@ -0,0 +1,26 @@
+'use strict';
+const dhalang = require('./dhalang');
+const fs = require('node:fs');
+
+const scrapeHtml = async () => {
+    const configuration = dhalang.getConfiguration();
+
+    let browser;
+    try {
+        browser = await dhalang.launchPuppeteer(configuration);
+        const page = await browser.newPage();
+        await dhalang.configure(page, configuration.userOptions);
+        await dhalang.navigate(page, configuration);
+        const html = await page.content();
+        fs.writeFileSync(configuration.tempFilePath, html);
+    } catch (error) {
+        console.error(error.message);
+        process.exit(1);
+    } finally {
+        if (browser) {
+            browser.close();
+        }
+        process.exit(0);
+    }
+};
+scrapeHtml();