grover 0.13.2 → 1.0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ce6455d4d8ede0f5b91972977f8e5d17d8fab13e60096625cd9eb05e759ee8db
-  data.tar.gz: 12484117509a4f5e06bcce9cd39b95f16f105b662ff204f0a0f45b5a22455b51
+  metadata.gz: f2409cbc42fc0fcb80876e1d4238f08cfbeae1d7fc7dada4cb958e8178bafc95
+  data.tar.gz: 8189bc362f4979028340f17095c0e9aade0d7950f605fb2062740452316e2b60
 SHA512:
-  metadata.gz: b488843267799c52530b27b7a3b8ac8cc76d8767f76062aca8cd32e7560c169d5e8e4d3f4a3498041cd180c4b0487337a51039487846a8a317fb2ecc4499a24a
-  data.tar.gz: fe4681c363d61d1a50fdedd0bbc0d76efa52b5d7990443e0b5fd7f978639a9ee3907b453a54afc3d595f27e85ca0a8d07ea3c91b52784f3f33914c6e1e31b6bc
+  metadata.gz: 0ed3057d89a650c43dbd5c606edcef96e31702cde46a18896af692aaa0b000c2824df2cc90d33e2a8767486940cc4063a2c28bdd609b459579d698fb282465b4
+  data.tar.gz: 23ede80fa749b4095cf94733690d398fd5027c33a1d11adccc63fa3300dbc2cf89e466c6ff8e8b63be6f67a3a00eff91d9c3c9124ea98d37ef4ab86df439fb3d

data/lib/grover.rb

@@ -33,8 +33,8 @@ class Grover
   #   see https://github.com/GoogleChrome/puppeteer/blob/master/docs/api.md#pagepdfoptions
   #
   def initialize(url, options = {})
-    @url = url
-    @options = OptionsBuilder.new(options, url)
+    @url = url.to_s
+    @options = OptionsBuilder.new(options, @url)
     @root_path = @options.delete 'root_path'
     @front_cover_path = @options.delete 'front_cover_path'
     @back_cover_path = @options.delete 'back_cover_path'
@@ -50,6 +50,15 @@ class Grover
     processor.convert :pdf, @url, normalized_options(path: path)
   end
 
+  #
+  # Request URL with provided options and render HTML
+  #
+  # @return [String] The resulting HTML string
+  #
+  def to_html
+    processor.convert :content, @url, normalized_options(path: nil)
+  end
+
   #
   # Request URL with provided options and create screenshot
   #
@@ -135,7 +144,7 @@ class Grover
   end
 
   def normalized_options(path:)
-    normalized_options = Utils.normalize_object @options
+    normalized_options = Utils.normalize_object @options, excluding: ['extraHTTPHeaders']
     normalized_options['path'] = path if path.is_a? ::String
     normalized_options
   end

data/lib/grover/js/processor.js

@@ -11,7 +11,7 @@ try {
 process.stdout.write("[\"ok\"]\n");
 
 const _processPage = (async (convertAction, urlOrHtml, options) => {
-  let browser;
+  let browser, errors = [];
   try {
     const launchParams = {
       args: process.env.GROVER_NO_SANDBOX === 'true' ? ['--no-sandbox', '--disable-setuid-sandbox'] : []
@@ -66,12 +66,76 @@ const _processPage = (async (convertAction, urlOrHtml, options) => {
       requestOptions.timeout = timeout;
     }
 
+    // Setup user agent (if provided)
+    const userAgent = options.userAgent; delete options.userAgent;
+    if (userAgent !== undefined) {
+      await page.setUserAgent(userAgent);
+    }
+
     // Setup viewport options (if provided)
     const viewport = options.viewport; delete options.viewport;
     if (viewport !== undefined) {
       await page.setViewport(viewport);
     }
 
+    // If specified, emulate the media type
+    const emulateMedia = options.emulateMedia; delete options.emulateMedia;
+    if (emulateMedia !== undefined) {
+      if (typeof page.emulateMediaType === 'function') {
+        await page.emulateMediaType(emulateMedia);
+      } else {
+        await page.emulateMedia(emulateMedia);
+      }
+    }
+
+    // Emulate the media features, if specified
+    const mediaFeatures = options.mediaFeatures; delete options.mediaFeatures;
+    if (Array.isArray(mediaFeatures)) {
+      page.emulateMediaFeatures(mediaFeatures);
+    }
+
+    // Emulate timezone (if provided)
+    const timezone = options.timezone; delete options.timezone;
+    if (timezone !== undefined) {
+      await page.emulateTimezone(timezone);
+    }
+
+    // Emulate vision deficiency (if provided)
+    const visionDeficiency = options.visionDeficiency; delete options.visionDeficiency;
+    if (visionDeficiency !== undefined) {
+      page.emulateVisionDeficiency(type);
+    }
+
+    // Bypass CSP (content security policy), if provided
+    const bypassCSP = options.bypassCSP; delete options.bypassCSP;
+    if (bypassCSP !== undefined) {
+      page.setBypassCSP(bypassCSP);
+    }
+
+    // Add extra HTTP headers (if provided)
+    const extraHTTPHeaders = options.extraHTTPHeaders; delete options.extraHTTPHeaders;
+    if (extraHTTPHeaders !== undefined) {
+      page.setExtraHTTPHeaders(extraHTTPHeaders);
+    }
+
+    // Set geolocation (if provided)
+    const geolocation = options.geolocation; delete options.geolocation;
+    if (geolocation !== undefined) {
+      page.setGeolocation(geolocation);
+    }
+
+    const raiseOnRequestFailure = options.raiseOnRequestFailure; delete options.raiseOnRequestFailure;
+    if (raiseOnRequestFailure) {
+      page.on('requestfinished', (request) => {
+        if (request.response() && !request.response().ok() && !request.redirectChain().length > 0) {
+          errors.push(request);
+        }
+      });
+      page.on('requestfailed', (request) => {
+        errors.push(request);
+      });
+    }
+
     const waitUntil = options.waitUntil; delete options.waitUntil;
     if (urlOrHtml.match(/^http/i)) {
       // Request is for a URL, so request it
@@ -82,7 +146,7 @@ const _processPage = (async (convertAction, urlOrHtml, options) => {
       requestOptions.waitUntil = waitUntil || 'networkidle0';
       await page.setRequestInterception(true);
       page.once('request', request => {
-        request.respond({ body: urlOrHtml });
+        request.respond({ body: urlOrHtml === '' ? ' ' : urlOrHtml });
         // Reset the request interception
         // (we only want to intercept the first request - ie our HTML)
         page.on('request', request => request.continue());
@@ -91,13 +155,19 @@ const _processPage = (async (convertAction, urlOrHtml, options) => {
       await page.goto(displayUrl || 'http://example.com', requestOptions);
     }
 
-    // If specified, emulate the media type
-    const emulateMedia = options.emulateMedia; delete options.emulateMedia;
-    if (emulateMedia !== undefined) {
-      if (typeof page.emulateMediaType == 'function') {
-        await page.emulateMediaType(emulateMedia);
-      } else {
-        await page.emulateMedia(emulateMedia);
+    // add styles (if provided)
+    const styleTagOptions = options.styleTagOptions; delete options.styleTagOptions;
+    if (Array.isArray(styleTagOptions)) {
+      for (const styleTagOption of styleTagOptions) {
+        await page.addStyleTag(styleTagOption);
+      }
+    }
+
+    // add scripts (if provided)
+    const scriptTagOptions = options.scriptTagOptions; delete options.scriptTagOptions;
+    if (Array.isArray(scriptTagOptions)) {
+      for (const scriptTagOption of scriptTagOptions) {
+        await page.addScriptTag(scriptTagOption);
       }
     }
 
@@ -111,7 +181,42 @@ const _processPage = (async (convertAction, urlOrHtml, options) => {
     const waitForSelector = options.waitForSelector; delete options.waitForSelector;
     const waitForSelectorOptions = options.waitForSelectorOptions; delete options.waitForSelectorOptions;
     if (waitForSelector !== undefined) {
-      await page.waitForSelector(waitForSelector, waitForSelectorOptions)
+      await page.waitForSelector(waitForSelector, waitForSelectorOptions);
+    }
+
+    // If specified, wait for timeout
+    const waitForTimeout = options.waitForTimeout; delete options.waitForTimeout;
+    if (waitForTimeout !== undefined) {
+      await page.waitForTimeout(waitForTimeout);
+    }
+
+    // If specified, focus on the specified selector
+    const focusSelector = options.focus; delete options.focus;
+    if (focusSelector !== undefined) {
+      await page.focus(focusSelector);
+    }
+
+    // If specified, hover on the specified selector
+    const hoverSelector = options.hover; delete options.hover;
+    if (hoverSelector !== undefined) {
+      await page.hover(hoverSelector);
+    }
+
+    if (errors.length > 0) {
+      function RequestFailedError(errors) {
+        this.name = "RequestFailedError";
+        this.message = errors.map(e => {
+          if (e.failure()) {
+            return e.failure().errorText + " at " + e.url();
+          } else if (e.response() && e.response().status()) {
+            return e.response().status() + " " + e.url();
+          } else {
+            return "UnknownError " + e.url()
+          }
+        }).join("\n");
+      }
+      RequestFailedError.prototype = Error.prototype;
+      throw new RequestFailedError(errors);
     }
 
     // If we're running puppeteer in headless mode, return the converted PDF

data/lib/grover/options_fixer.rb

@@ -34,13 +34,13 @@ class Grover
     def fix_boolean_options!
       fix_options!(
         'display_header_footer', 'full_page', 'landscape', 'omit_background', 'prefer_css_page_size',
-        'print_background', 'viewport.has_touch', 'viewport.is_landscape', 'viewport.is_mobile'
+        'print_background', 'viewport.has_touch', 'viewport.is_landscape', 'viewport.is_mobile', 'bypass_csp'
       ) { |value| !FALSE_VALUES.include?(value) }
     end
 
     def fix_integer_options!
       fix_options!(
-        'viewport.height', 'viewport.width',
+        'viewport.height', 'viewport.width', 'wait_for_timeout',
         &:to_i
       )
     end
@@ -48,6 +48,7 @@ class Grover
     def fix_float_options!
       fix_options!(
         'clip.height', 'clip.width', 'clip.x', 'clip.y', 'quality', 'scale', 'viewport.device_scale_factor',
+        'geolocation.latitude', 'geolocation.longitude',
         &:to_f
       )
     end

data/lib/grover/processor.rb

@@ -19,6 +19,7 @@ class Grover
       ensure_packages_are_initiated
       result = call_js_method method, url_or_html, options
       return unless result
+      return result if result.is_a?(String)
 
       result['data'].pack('C*')
     ensure

data/lib/grover/utils.rb

@@ -6,7 +6,9 @@ class Grover
   #
   class Utils
     ACRONYMS = {
-      'css' => 'CSS'
+      'css' => 'CSS',
+      'csp' => 'CSP',
+      'http' => 'HTTP'
     }.freeze
     private_constant :ACRONYMS
 
@@ -41,11 +43,12 @@ class Grover
     # Copied from active support
     # @see active_support/core_ext/hash/keys.rb
     #
-    def self.deep_transform_keys_in_object(object, &block)
+    def self.deep_transform_keys_in_object(object, excluding: [], &block) # rubocop:disable Metrics/MethodLength
       case object
       when Hash
         object.each_with_object({}) do |(key, value), result|
-          result[yield(key)] = deep_transform_keys_in_object(value, &block)
+          new_key = yield(key)
+          result[new_key] = excluding.include?(new_key) ? value : deep_transform_keys_in_object(value, &block)
         end
       when Array
         object.map { |e| deep_transform_keys_in_object(e, &block) }
@@ -80,8 +83,8 @@ class Grover
     #
     # Recursively normalizes hash objects with camelized string keys
     #
-    def self.normalize_object(object)
-      deep_transform_keys_in_object(object) { |k| normalize_key(k) }
+    def self.normalize_object(object, excluding: [])
+      deep_transform_keys_in_object(object, excluding: excluding) { |k| normalize_key(k) }
     end
 
     #

data/lib/grover/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 class Grover
-  VERSION = '0.13.2'
+  VERSION = '1.0.1'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: grover
 version: !ruby/object:Gem::Version
-  version: 0.13.2
+  version: 1.0.1
 platform: ruby
 authors:
 - Andrew Bromwich
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-12-05 00:00:00.000000000 Z
+date: 2021-06-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: combine_pdf
@@ -192,7 +192,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: 2.5.0
   - - "<"
     - !ruby/object:Gem::Version
-      version: 2.8.0
+      version: 3.1.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="