grover 0.14.1 → 1.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b599a5462dc8008f92d9ea95c949d8de0cfd0786792b303c1e78f22a104a503d
-  data.tar.gz: bb4dbd2d6d336e88f71668853b2d9819e314a87b3b55c7c4f211f3d94ddcfe25
+  metadata.gz: afca67d672e63071ee66a65dc2d2a65e728eb06715b2c59b43cf44dce587ab1c
+  data.tar.gz: 50313e3201d5f1bbbf8e6d9c92147c3ddc373fbdd51145ee3315a616b69b303c
 SHA512:
-  metadata.gz: cc644ef00aa06d6ebb2e146165fe300f94c73749cffaa6e4e409d78e6ed5d376421a4163137557f551ea90ef984a52bda42f5db01bad1f81d03c5fb209ed5863
-  data.tar.gz: 872dd3bde76e74f201d0dc52f46614434bdb52ea7ef43deb205783241c54a4c0f107f69812a9fa70f3af1e25a46edb3ac6e13e7d17bc51ba6f3b2223c0f69f92
+  metadata.gz: f8aa164f8b52ef008c70fe7ff4918194859db0be10cee102556cc67d49fe9cf5035ee181f9dac64dce87dfa7b7295effe090b851504dd9774e33a23ecab7e15e
+  data.tar.gz: cd5b46accc2bbbdc5f3d149d62099c4d47c36744aa877876966680e687bfa0fc69f9ad6045c0a4b9a1e3f135c90279ffd595d526f373daddda02a5fb42db754d

data/lib/grover.rb

@@ -50,6 +50,15 @@ class Grover
     processor.convert :pdf, @url, normalized_options(path: path)
   end
 
+  #
+  # Request URL with provided options and render HTML
+  #
+  # @return [String] The resulting HTML string
+  #
+  def to_html
+    processor.convert :content, @url, normalized_options(path: nil)
+  end
+
   #
   # Request URL with provided options and create screenshot
   #
@@ -135,7 +144,7 @@ class Grover
   end
 
   def normalized_options(path:)
-    normalized_options = Utils.normalize_object @options
+    normalized_options = Utils.normalize_object @options, excluding: ['extraHTTPHeaders']
     normalized_options['path'] = path if path.is_a? ::String
     normalized_options
   end

data/lib/grover/js/processor.js

@@ -11,7 +11,7 @@ try {
 process.stdout.write("[\"ok\"]\n");
 
 const _processPage = (async (convertAction, urlOrHtml, options) => {
-  let browser;
+  let browser, errors = [];
   try {
     const launchParams = {
       args: process.env.GROVER_NO_SANDBOX === 'true' ? ['--no-sandbox', '--disable-setuid-sandbox'] : []
@@ -66,12 +66,76 @@ const _processPage = (async (convertAction, urlOrHtml, options) => {
       requestOptions.timeout = timeout;
     }
 
+    // Setup user agent (if provided)
+    const userAgent = options.userAgent; delete options.userAgent;
+    if (userAgent !== undefined) {
+      await page.setUserAgent(userAgent);
+    }
+
     // Setup viewport options (if provided)
     const viewport = options.viewport; delete options.viewport;
     if (viewport !== undefined) {
       await page.setViewport(viewport);
     }
 
+    // If specified, emulate the media type
+    const emulateMedia = options.emulateMedia; delete options.emulateMedia;
+    if (emulateMedia !== undefined) {
+      if (typeof page.emulateMediaType === 'function') {
+        await page.emulateMediaType(emulateMedia);
+      } else {
+        await page.emulateMedia(emulateMedia);
+      }
+    }
+
+    // Emulate the media features, if specified
+    const mediaFeatures = options.mediaFeatures; delete options.mediaFeatures;
+    if (Array.isArray(mediaFeatures)) {
+      page.emulateMediaFeatures(mediaFeatures);
+    }
+
+    // Emulate timezone (if provided)
+    const timezone = options.timezone; delete options.timezone;
+    if (timezone !== undefined) {
+      await page.emulateTimezone(timezone);
+    }
+
+    // Emulate vision deficiency (if provided)
+    const visionDeficiency = options.visionDeficiency; delete options.visionDeficiency;
+    if (visionDeficiency !== undefined) {
+      page.emulateVisionDeficiency(type);
+    }
+
+    // Bypass CSP (content security policy), if provided
+    const bypassCSP = options.bypassCSP; delete options.bypassCSP;
+    if (bypassCSP !== undefined) {
+      page.setBypassCSP(bypassCSP);
+    }
+
+    // Add extra HTTP headers (if provided)
+    const extraHTTPHeaders = options.extraHTTPHeaders; delete options.extraHTTPHeaders;
+    if (extraHTTPHeaders !== undefined) {
+      page.setExtraHTTPHeaders(extraHTTPHeaders);
+    }
+
+    // Set geolocation (if provided)
+    const geolocation = options.geolocation; delete options.geolocation;
+    if (geolocation !== undefined) {
+      page.setGeolocation(geolocation);
+    }
+
+    const raiseOnRequestFailure = options.raiseOnRequestFailure; delete options.raiseOnRequestFailure;
+    if (raiseOnRequestFailure) {
+      page.on('requestfinished', (request) => {
+        if (request.response() && !request.response().ok() && !request.redirectChain().length > 0) {
+          errors.push(request);
+        }
+      });
+      page.on('requestfailed', (request) => {
+        errors.push(request);
+      });
+    }
+
     const waitUntil = options.waitUntil; delete options.waitUntil;
     if (urlOrHtml.match(/^http/i)) {
       // Request is for a URL, so request it
@@ -91,13 +155,19 @@ const _processPage = (async (convertAction, urlOrHtml, options) => {
       await page.goto(displayUrl || 'http://example.com', requestOptions);
     }
 
-    // If specified, emulate the media type
-    const emulateMedia = options.emulateMedia; delete options.emulateMedia;
-    if (emulateMedia !== undefined) {
-      if (typeof page.emulateMediaType == 'function') {
-        await page.emulateMediaType(emulateMedia);
-      } else {
-        await page.emulateMedia(emulateMedia);
+    // add styles (if provided)
+    const styleTagOptions = options.styleTagOptions; delete options.styleTagOptions;
+    if (Array.isArray(styleTagOptions)) {
+      for (const styleTagOption of styleTagOptions) {
+        await page.addStyleTag(styleTagOption);
+      }
+    }
+
+    // add scripts (if provided)
+    const scriptTagOptions = options.scriptTagOptions; delete options.scriptTagOptions;
+    if (Array.isArray(scriptTagOptions)) {
+      for (const scriptTagOption of scriptTagOptions) {
+        await page.addScriptTag(scriptTagOption);
       }
     }
 
@@ -111,7 +181,49 @@ const _processPage = (async (convertAction, urlOrHtml, options) => {
     const waitForSelector = options.waitForSelector; delete options.waitForSelector;
     const waitForSelectorOptions = options.waitForSelectorOptions; delete options.waitForSelectorOptions;
     if (waitForSelector !== undefined) {
-      await page.waitForSelector(waitForSelector, waitForSelectorOptions)
+      await page.waitForSelector(waitForSelector, waitForSelectorOptions);
+    }
+
+    // If specified, wait for function
+    const waitForFunction = options.waitForFunction; delete options.waitForFunction;
+    const waitForFunctionOptions = options.waitForFunctionOptions; delete options.waitForFunctionOptions;
+    if (waitForFunction !== undefined) {
+      await page.waitForFunction(waitForFunction, waitForFunctionOptions);
+    }
+
+    // If specified, wait for timeout
+    const waitForTimeout = options.waitForTimeout; delete options.waitForTimeout;
+    if (waitForTimeout !== undefined) {
+      await page.waitForTimeout(waitForTimeout);
+    }
+
+    // If specified, focus on the specified selector
+    const focusSelector = options.focus; delete options.focus;
+    if (focusSelector !== undefined) {
+      await page.focus(focusSelector);
+    }
+
+    // If specified, hover on the specified selector
+    const hoverSelector = options.hover; delete options.hover;
+    if (hoverSelector !== undefined) {
+      await page.hover(hoverSelector);
+    }
+
+    if (errors.length > 0) {
+      function RequestFailedError(errors) {
+        this.name = "RequestFailedError";
+        this.message = errors.map(e => {
+          if (e.failure()) {
+            return e.failure().errorText + " at " + e.url();
+          } else if (e.response() && e.response().status()) {
+            return e.response().status() + " " + e.url();
+          } else {
+            return "UnknownError " + e.url()
+          }
+        }).join("\n");
+      }
+      RequestFailedError.prototype = Error.prototype;
+      throw new RequestFailedError(errors);
     }
 
     // If we're running puppeteer in headless mode, return the converted PDF

data/lib/grover/options_fixer.rb

@@ -34,13 +34,13 @@ class Grover
     def fix_boolean_options!
       fix_options!(
         'display_header_footer', 'full_page', 'landscape', 'omit_background', 'prefer_css_page_size',
-        'print_background', 'viewport.has_touch', 'viewport.is_landscape', 'viewport.is_mobile'
+        'print_background', 'viewport.has_touch', 'viewport.is_landscape', 'viewport.is_mobile', 'bypass_csp'
       ) { |value| !FALSE_VALUES.include?(value) }
     end
 
     def fix_integer_options!
       fix_options!(
-        'viewport.height', 'viewport.width',
+        'viewport.height', 'viewport.width', 'wait_for_timeout',
         &:to_i
       )
     end
@@ -48,6 +48,7 @@ class Grover
     def fix_float_options!
       fix_options!(
         'clip.height', 'clip.width', 'clip.x', 'clip.y', 'quality', 'scale', 'viewport.device_scale_factor',
+        'geolocation.latitude', 'geolocation.longitude',
         &:to_f
       )
     end

data/lib/grover/processor.rb

@@ -19,6 +19,7 @@ class Grover
       ensure_packages_are_initiated
       result = call_js_method method, url_or_html, options
       return unless result
+      return result if result.is_a?(String)
 
       result['data'].pack('C*')
     ensure

data/lib/grover/utils.rb

@@ -6,7 +6,9 @@ class Grover
   #
   class Utils
     ACRONYMS = {
-      'css' => 'CSS'
+      'css' => 'CSS',
+      'csp' => 'CSP',
+      'http' => 'HTTP'
     }.freeze
     private_constant :ACRONYMS
 
@@ -41,11 +43,12 @@ class Grover
     # Copied from active support
     # @see active_support/core_ext/hash/keys.rb
     #
-    def self.deep_transform_keys_in_object(object, &block)
+    def self.deep_transform_keys_in_object(object, excluding: [], &block) # rubocop:disable Metrics/MethodLength
       case object
       when Hash
         object.each_with_object({}) do |(key, value), result|
-          result[yield(key)] = deep_transform_keys_in_object(value, &block)
+          new_key = yield(key)
+          result[new_key] = excluding.include?(new_key) ? value : deep_transform_keys_in_object(value, &block)
         end
       when Array
         object.map { |e| deep_transform_keys_in_object(e, &block) }
@@ -80,8 +83,8 @@ class Grover
     #
     # Recursively normalizes hash objects with camelized string keys
     #
-    def self.normalize_object(object)
-      deep_transform_keys_in_object(object) { |k| normalize_key(k) }
+    def self.normalize_object(object, excluding: [])
+      deep_transform_keys_in_object(object, excluding: excluding) { |k| normalize_key(k) }
     end
 
     #

data/lib/grover/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 class Grover
-  VERSION = '0.14.1'
+  VERSION = '1.0.2'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: grover
 version: !ruby/object:Gem::Version
-  version: 0.14.1
+  version: 1.0.2
 platform: ruby
 authors:
 - Andrew Bromwich
-autorequire:
+autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-01-16 00:00:00.000000000 Z
+date: 2021-07-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: combine_pdf
@@ -181,7 +181,7 @@ homepage: https://github.com/Studiosity/grover
 licenses:
 - MIT
 metadata: {}
-post_install_message:
+post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
@@ -199,8 +199,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.3
-signing_key:
+rubygems_version: 3.0.6
+signing_key: 
 specification_version: 4
 summary: A Ruby gem to transform HTML into PDF, PNG or JPEG by wrapping the NodeJS
   Google Puppeteer driver for Chromium