fetch_util 0.6.2 → 0.7.0

data/lib/fetch_util/browser/navigation/navigator_patch.rb

@@ -17,6 +17,8 @@ module FetchUtil
             Object.defineProperty(navigator, "languages", { get: () => #{languages_json} });
             Object.defineProperty(navigator, "platform", { get: () => "Linux x86_64" });
 
+            #{shadow_root_patch}
+
             Object.defineProperty(navigator, "plugins", {
               get: () => {
                 const p = { 0: { name: "PDF Viewer", filename: "internal-pdf-viewer", description: "Portable Document Format" },

data/lib/fetch_util/browser/navigation/shadow_root_patch.rb

@@ -0,0 +1,85 @@
+# frozen_string_literal: true
+
+module FetchUtil
+  class Browser
+    module Navigation
+      module ShadowRootPatch
+        private
+
+        def shadow_root_patch
+          token = JSON.generate(Browser::SHADOW_ROOT_ACCESS_TOKEN)
+          property = JSON.generate(Browser::SHADOW_ROOT_READER_PROPERTY)
+          <<~JS
+            {
+              const shadowRootReaderProperty = #{property};
+              const shadowRootInstallProperty = shadowRootReaderProperty + ":installed";
+              const shadowRootToken = #{token};
+              const weakMapGet = WeakMap.prototype.get;
+              const weakMapSet = WeakMap.prototype.set;
+              const reflectApply = Reflect.apply;
+              let readShadowRoot = null;
+
+              try {
+                readShadowRoot = window.top[shadowRootReaderProperty];
+              } catch (_error) {
+              }
+
+              if (typeof readShadowRoot !== "function") {
+                const shadowRoots = new WeakMap();
+                readShadowRoot = function(candidateToken, operation, host, root) {
+                  if (candidateToken !== shadowRootToken) return null;
+                  try {
+                    if (operation === "set") {
+                      reflectApply(weakMapSet, shadowRoots, [host, root]);
+                      return true;
+                    }
+                    return operation === "get" ? reflectApply(weakMapGet, shadowRoots, [host]) || null : null;
+                  } catch (_error) {
+                    return null;
+                  }
+                };
+              }
+
+              try {
+                if (!Object.prototype.hasOwnProperty.call(window, shadowRootReaderProperty)) {
+                  Object.defineProperty(window, shadowRootReaderProperty, {
+                    value: readShadowRoot,
+                    configurable: false,
+                    enumerable: false,
+                    writable: false
+                  });
+                }
+              } catch (_error) {
+              }
+
+              const installed = window[shadowRootInstallProperty] === true;
+              const descriptor = Object.getOwnPropertyDescriptor(Element.prototype, "attachShadow");
+              const attachShadow = descriptor && descriptor.value;
+              if (attachShadow && !installed) {
+                const trackedAttachShadow = new Proxy(attachShadow, {
+                  apply: function(target, thisArg, args) {
+                    const root = reflectApply(target, thisArg, args);
+                    try {
+                      readShadowRoot(shadowRootToken, "set", thisArg, root);
+                    } catch (_error) {
+                    }
+                    return root;
+                  }
+                });
+                Object.defineProperty(Element.prototype, "attachShadow", Object.assign({}, descriptor, {
+                  value: trackedAttachShadow
+                }));
+                Object.defineProperty(window, shadowRootInstallProperty, {
+                  value: true,
+                  configurable: false,
+                  enumerable: false,
+                  writable: false
+                });
+              }
+            }
+          JS
+        end
+      end
+    end
+  end
+end

data/lib/fetch_util/browser/navigation.rb

@@ -5,8 +5,10 @@ module FetchUtil
     module Navigation
       autoload :HeadersAndReadiness, "fetch_util/browser/navigation/headers_and_readiness"
       autoload :NavigatorPatch, "fetch_util/browser/navigation/navigator_patch"
+      autoload :ShadowRootPatch, "fetch_util/browser/navigation/shadow_root_patch"
 
       include HeadersAndReadiness
+      include ShadowRootPatch
       include NavigatorPatch
     end
   end

data/lib/fetch_util/browser.rb

@@ -2,6 +2,7 @@
 
 require "ferrum"
 require "json"
+require "securerandom"
 require "uri"
 
 module FetchUtil
@@ -31,6 +32,8 @@ module FetchUtil
     DEFAULT_USER_AGENT = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 " \
                          "(KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36"
     DEFAULT_ACCEPT_LANGUAGE = "en-US,en;q=0.9"
+    SHADOW_ROOT_ACCESS_TOKEN = SecureRandom.hex(32).freeze
+    SHADOW_ROOT_READER_PROPERTY = "__fetchUtilClosedShadowRootReaderV1_#{SecureRandom.hex(16)}".freeze
     SOCIAL_LOGIN_PHASE_WAIT = 0.3
     POST_CONSENT_IDLE_TIMEOUT = 3.0
     CONTENT_READY_MIN_LENGTH = 200

data/lib/fetch_util/extractor.rb

@@ -5,6 +5,9 @@ require "json"
 module FetchUtil
   class Extractor
     INLINE_ASSET_PATHS = %w[vendor/readability.js vendor/turndown.js extract.js].freeze
+    VENDOR_ASSET_PATHS = INLINE_ASSET_PATHS.first(2).freeze
+    VENDOR_GLOBAL_NAMES = %w[Readability TurndownService].freeze
+    PRIVATE_DELIVERY_SENTINEL = '"fetch-util:standalone-api:v1"'
     @asset_cache = {}
     @cache_mutex = Mutex.new
 
@@ -22,6 +25,7 @@ module FetchUtil
       @reader_mode = reader_mode
       @asset_root = (asset_root || File.join(__dir__, "assets")).dup.freeze
       @extraction_call = nil
+      @private_asset_script = nil
     end
 
     def extract(page)
@@ -36,14 +40,26 @@ module FetchUtil
     private
 
     def inject_assets(page)
-      page.add_script_tag(path: asset_path("vendor/readability.js"))
-      page.add_script_tag(path: asset_path("vendor/turndown.js"))
-      page.add_script_tag(path: asset_path("extract.js"))
+      INLINE_ASSET_PATHS.each do |relative_path|
+        page.add_script_tag(path: asset_path(relative_path))
+      end
+    end
+
+    def inject_vendor_assets(page)
+      VENDOR_ASSET_PATHS.each do |relative_path|
+        page.add_script_tag(path: asset_path(relative_path))
+      end
     end
 
-    def inject_assets_inline(page)
-      inline_asset_scripts.each do |script|
-        page.evaluate("#{script}\ntrue")
+    def inject_vendor_assets_inline(page)
+      inline_asset_scripts.first(2).zip(VENDOR_GLOBAL_NAMES).each do |script, global_name|
+        page.evaluate(<<~JAVASCRIPT)
+          (() => {
+            #{script}
+            window.#{global_name} = #{global_name};
+            return true;
+          })()
+        JAVASCRIPT
       end
     end
 
@@ -52,14 +68,15 @@ module FetchUtil
       original_timeout = page.timeout if timeout_supported
       page.timeout = [original_timeout.to_f, 60].max if timeout_supported
 
-      inject_assets(page)
+      register_cdp_closed_shadow_roots(page)
+      inject_vendor_assets(page)
       page.evaluate(extraction_call)
     rescue Ferrum::TimeoutError
       begin
         page.evaluate("window.stop && window.stop()")
       rescue Ferrum::Error
       end
-      inject_assets_inline(page)
+      inject_vendor_assets_inline(page)
       page.evaluate(extraction_call)
     ensure
       restore_page_timeout(page, original_timeout) if timeout_supported
@@ -71,8 +88,97 @@ module FetchUtil
       nil
     end
 
+    def register_cdp_closed_shadow_roots(page)
+      return unless page.respond_to?(:command)
+
+      document = page.command("DOM.getDocument", depth: -1, pierce: true).fetch("root")
+      shadow_root_pairs(document).each do |host, root|
+        register_cdp_closed_shadow_root(page, host, root)
+      end
+    rescue Ferrum::Error
+      nil
+    end
+
+    def shadow_root_pairs(document)
+      pairs = []
+      pending = [document]
+      until pending.empty?
+        node = pending.pop
+        shadows = Array(node["shadowRoots"])
+        shadows.each { |root| pairs << [node, root] if root["shadowRootType"] == "closed" }
+        pending.concat(Array(node["children"]), shadows)
+      end
+      pairs
+    end
+
+    def register_cdp_closed_shadow_root(page, host, root)
+      object_ids = []
+      [host, root].each do |node|
+        object_ids << page.command(
+          "DOM.resolveNode",
+          backendNodeId: node.fetch("backendNodeId")
+        ).dig("object", "objectId")
+      end
+      return unless object_ids.all?
+
+      page.command(
+        "Runtime.callFunctionOn",
+        objectId: object_ids.first,
+        functionDeclaration: closed_shadow_registration_function,
+        arguments: [
+          { objectId: object_ids.last },
+          { value: Browser::SHADOW_ROOT_READER_PROPERTY },
+          { value: Browser::SHADOW_ROOT_ACCESS_TOKEN }
+        ],
+        returnByValue: true
+      )
+    rescue Ferrum::Error
+      nil
+    ensure
+      Array(object_ids).compact.each do |object_id|
+        page.command("Runtime.releaseObject", objectId: object_id)
+      rescue Ferrum::Error
+        nil
+      end
+    end
+
+    def closed_shadow_registration_function
+      <<~JAVASCRIPT
+        function(root, property, token) {
+          const reader = window[property];
+          return typeof reader === "function" && reader(token, "set", this, root) === true;
+        }
+      JAVASCRIPT
+    end
+
     def extraction_call
-      @extraction_call ||= "window.FetchUtilExtract.extract(#{JSON.generate(reader_mode: @reader_mode)})"
+      @extraction_call ||= begin
+        options = JSON.generate(
+          reader_mode: @reader_mode,
+          shadow_root_token: Browser::SHADOW_ROOT_ACCESS_TOKEN,
+          shadow_root_reader_property: Browser::SHADOW_ROOT_READER_PROPERTY
+        )
+        <<~JAVASCRIPT
+          (() => {
+            let __fetchUtilPrivateApi = null;
+            const __fetchUtilDeliverExtractApi = api => { __fetchUtilPrivateApi = api; };
+            #{private_asset_script}
+            if (!__fetchUtilPrivateApi || typeof __fetchUtilPrivateApi.extract !== "function") return null;
+            return __fetchUtilPrivateApi.extract(#{options});
+          })()
+        JAVASCRIPT
+      end
+    end
+
+    def private_asset_script
+      @private_asset_script ||= begin
+        source = inline_asset_scripts.last
+        marker_index = source.index(PRIVATE_DELIVERY_SENTINEL)
+        duplicate_index = source.index(PRIVATE_DELIVERY_SENTINEL, marker_index.to_i + PRIVATE_DELIVERY_SENTINEL.length)
+        raise ExtractionError, "Private extraction delivery sentinel is missing or duplicated" if marker_index.nil? || duplicate_index
+
+        source.sub(PRIVATE_DELIVERY_SENTINEL, "__fetchUtilDeliverExtractApi").freeze
+      end
     end
 
     def inline_asset_scripts

data/lib/fetch_util/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module FetchUtil
-  VERSION = "0.6.2"
+  VERSION = "0.7.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fetch_util
 version: !ruby/object:Gem::Version
-  version: 0.6.2
+  version: 0.7.0
 platform: ruby
 authors:
 - hmdne
@@ -100,6 +100,7 @@ files:
 - lib/fetch_util/browser/navigation.rb
 - lib/fetch_util/browser/navigation/headers_and_readiness.rb
 - lib/fetch_util/browser/navigation/navigator_patch.rb
+- lib/fetch_util/browser/navigation/shadow_root_patch.rb
 - lib/fetch_util/browser/site_stabilization.rb
 - lib/fetch_util/browser/site_stabilization/azure_devops_pr_commit_details_script.rb
 - lib/fetch_util/browser/site_stabilization/azure_devops_pr_product_state_script.rb