fetch_util 0.3.0 → 0.3.2

data/lib/fetch_util/browser/site_stabilization.rb

@@ -4,9 +4,11 @@ module FetchUtil
   class Browser
     module SiteStabilization
       autoload :CommunityAndMarketplace, "fetch_util/browser/site_stabilization/community_and_marketplace"
+      autoload :GitlabRepo, "fetch_util/browser/site_stabilization/gitlab_repo"
       autoload :SocialPlatforms, "fetch_util/browser/site_stabilization/social_platforms"
 
       include CommunityAndMarketplace
+      include GitlabRepo
       include SocialPlatforms
     end
   end

data/lib/fetch_util/browser/stabilization/page_flow.rb

@@ -2,33 +2,72 @@
 
 module FetchUtil
   class Browser
+    # rubocop:disable Metrics/ModuleLength -- shared stabilization module is intentionally centralized.
     module Stabilization
       module PageFlow
+        PAGE_FLOW_STABILIZATION_PROFILES = [
+          SiteStabilization::CommunityAndMarketplace::COMMUNITY_MARKETPLACE_STABILIZATION_PROFILES[:stabilize_reddit],
+          SiteStabilization::SocialPlatforms::SOCIAL_PLATFORM_STABILIZATION_PROFILES[:stabilize_instagram],
+          SiteStabilization::SocialPlatforms::SOCIAL_PLATFORM_STABILIZATION_PROFILES[:stabilize_facebook],
+          SiteStabilization::CommunityAndMarketplace::COMMUNITY_MARKETPLACE_STABILIZATION_PROFILES[:stabilize_ebay_search],
+          { host: "gitlab.com", path_query: ->(uri) { uri.path.split("/").reject(&:empty?).length == 2 },
+            strategy: :stabilize_gitlab_repo, notes: "Wait for repository README content on GitLab project roots.",
+            tests: "spec/fetch_util/browser_stabilization_spec.rb" }
+        ].freeze
+        POST_GENERIC_STABILIZATION_PROFILES = [
+          { host: ["wyborcza.pl", "gazeta.pl"], path_query: ->(uri) { uri.path.match?(%r{/(?:\d+,){2}\d+,|/7,}) },
+            strategy: :wait_for_agora_article,
+            notes: "After generic consent/idle handling, wait briefly for delayed Agora article bodies.",
+            tests: "spec/fetch_util/browser_stabilization_spec.rb" },
+          { host: "france24.com", path_query: ->(uri) { uri.path.match?(%r{/\d{8}-}) },
+            strategy: :wait_for_france24_article,
+            notes: "After generic consent/idle handling, wait briefly for delayed France24 article bodies.",
+            tests: "spec/fetch_util/browser_stabilization_spec.rb" }
+        ].freeze
+
         private
 
         def stabilize_page(page, url)
-          return stabilize_reddit(page) if reddit_url?(url)
-          return stabilize_instagram(page) if instagram_url?(url)
-          return stabilize_facebook(page) if facebook_url?(url)
-          return stabilize_ebay_search(page) if ebay_search_url?(url)
+          if (profile = matching_stabilization_profile(url, PAGE_FLOW_STABILIZATION_PROFILES))
+            return send(profile.fetch(:strategy), page)
+          end
 
           reached_idle = !@wait_for_idle || wait_for_idle_or_content(page)
           preserve_consent = preserve_consent_wall?(page, url)
           accepted_cookies = preserve_consent ? false : accept_cookie_consent(page)
           accepted_cookies = (!preserve_consent && dismiss_privacy_preference_overlay(page)) || accepted_cookies
-          sleep @wait if @wait.positive?
-          accepted_cookies = (!preserve_consent && accept_cookie_consent(page)) || accepted_cookies
-          accepted_cookies = (!preserve_consent && dismiss_privacy_preference_overlay(page)) || accepted_cookies
+          if @wait.positive? && (!@wait_for_idle || accepted_cookies)
+            sleep @wait
+            accepted_cookies = (!preserve_consent && accept_cookie_consent(page)) || accepted_cookies
+            accepted_cookies = (!preserve_consent && dismiss_privacy_preference_overlay(page)) || accepted_cookies
+          end
 
           wait_for_spa_hydration(page) if @wait_for_idle && reached_idle
-          accepted_cookies = (!preserve_consent && accept_cookie_consent(page)) || accepted_cookies
-          accepted_cookies = (!preserve_consent && dismiss_privacy_preference_overlay(page)) || accepted_cookies
+          if accepted_cookies
+            accepted_cookies = (!preserve_consent && accept_cookie_consent(page)) || accepted_cookies
+            accepted_cookies = (!preserve_consent && dismiss_privacy_preference_overlay(page)) || accepted_cookies
+          end
+          if (profile = matching_stabilization_profile(url, POST_GENERIC_STABILIZATION_PROFILES))
+            send(profile.fetch(:strategy), page, url)
+          end
 
           return unless accepted_cookies && @wait_for_idle && reached_idle
 
-          page.network.wait_for_idle(duration: @idle_duration, timeout: POST_CONSENT_IDLE_TIMEOUT)
+          wait_for_network_idle(page)
         end
 
+        def matching_stabilization_profile(url, profiles)
+          uri = URI.parse(url)
+          host = FetchUtil.strip_www_host(url)
+          profiles.find { |profile| profile_match?(profile, uri, host) }
+        rescue URI::InvalidURIError
+          nil
+        end
+
+        def profile_match?(profile, uri, host) = stabilization_host_matches?(profile.fetch(:host), host) && profile.fetch(:path_query, ->(_) { true }).call(uri)
+
+        def stabilization_host_matches?(matcher, host) = Array(matcher).any? { |candidate| host == candidate || host.end_with?(".#{candidate}") }
+
         def wait_for_idle_or_content(page)
           content_seen_at = nil
 
@@ -45,6 +84,17 @@ module FetchUtil
           false
         end
 
+        def wait_for_network_idle(page)
+          page.network.wait_for_idle(duration: @idle_duration, timeout: POST_CONSENT_IDLE_TIMEOUT)
+          true
+        rescue Ferrum::TimeoutError
+          false
+        rescue Ferrum::Error => e
+          raise if retryable_pending_connections_error?(e)
+
+          false
+        end
+
         def page_has_content?(page)
           page.evaluate(<<~JS)
             (() => {
@@ -60,6 +110,7 @@ module FetchUtil
 
         def preserve_consent_wall?(page, url)
           host = FetchUtil.strip_www_host(url)
+          return true if host == "france24.com" || host.end_with?(".france24.com")
           return false unless host == "youtube.com" || host.end_with?(".youtube.com") || host.match?(/\Agoogle\.[a-z.]+\z/)
 
           state = page.evaluate(<<~JS)
@@ -74,7 +125,39 @@ module FetchUtil
         rescue URI::InvalidURIError, Ferrum::JavaScriptError, Ferrum::TimeoutError
           false
         end
+
+        def wait_for_agora_article(page, _url)
+          retry_until_timeout(7.0, interval: 0.25) do
+            page.evaluate(<<~JS)
+              (() => {
+                const selectors = ['.mrf-article-body', '.article_body', 'div.articleBody', '.art_content', '.article-inner', 'section.article', '[itemprop=articleBody]'];
+                if (selectors.some((selector) => document.querySelector(selector))) return true;
+                const text = document.body ? (document.body.innerText || '') : '';
+                return !(new RegExp('Wyłącz AdBlocka/uBlocka|Nieznany błąd', 'i')).test(text) && text.length > 1200;
+              })()
+            JS
+          end
+        rescue Ferrum::JavaScriptError, Ferrum::TimeoutError
+          false
+        end
+
+        def wait_for_france24_article(page, _url)
+          retry_until_timeout(15.0, interval: 0.25) do
+            page.evaluate(<<~JS)
+              (() => {
+                const body = document.querySelector('.t-content__body') || document.querySelector('.t-content--article');
+                if (!body) return false;
+                const text = (body.innerText || '').replace(/\s+/g, ' ').trim();
+                return body.querySelectorAll('p').length >= 3 && text.length > 1000;
+              })()
+            JS
+          end
+        rescue Ferrum::JavaScriptError, Ferrum::TimeoutError
+          false
+        end
       end
     end
   end
 end
+
+# rubocop:enable Metrics/ModuleLength

data/lib/fetch_util/browser.rb

@@ -36,7 +36,10 @@ module FetchUtil
     CONTENT_READY_MIN_LENGTH = 200
     SPA_HYDRATION_TIMEOUT = 2.0
     SPA_HYDRATION_POLL = 0.15
+    PRE_EXTRACTION_SETTLE_WAIT = 0.25
+    HEAVY_SCRIPT_COUNT_THRESHOLD = 20
     NAVIGATION_MAX_RETRIES = 2
+    NAVIGATION_RETRY_WAIT = 2.0
 
     def initialize(timeout: 20, wait: 0.75, wait_for_idle: true, idle_duration: 0.35,
                    viewport: DEFAULT_VIEWPORT, user_agent: DEFAULT_USER_AGENT,
@@ -49,6 +52,10 @@ module FetchUtil
       @viewport = DEFAULT_VIEWPORT.merge(symbolize_hash(viewport || {}))
       @user_agent = user_agent
       @accept_language = accept_language
+      @default_headers = {
+        "User-Agent" => @user_agent,
+        "Accept-Language" => @accept_language
+      }.freeze
       @browser_path = browser_path || ENV["BROWSER_PATH"] || BROWSER_CANDIDATES.find { |path| File.executable?(path) }
       @full_browser = @browser_path && !@browser_path.include?("headless_shell")
       default_opts = { "no-sandbox": nil }
@@ -61,6 +68,7 @@ module FetchUtil
         default_opts["enable-automation"] = false # override Ferrum default
       end
       @browser_options = default_opts.merge(browser_options || {})
+      @navigator_patch = build_navigator_patch
       @ferrum = nil
       @mutex = Mutex.new
     end
@@ -72,22 +80,9 @@ module FetchUtil
     def with_page(url)
       raise BrowserError, "No Chromium browser found. Set BROWSER_PATH or install Chromium." unless @browser_path
 
-      ferrum = ensure_browser
-      page = ferrum.create_page
-      page.headers.set(default_headers)
-      page.bypass_csp
-      retries = 0
-      begin
-        page.go_to(url)
-      rescue Ferrum::PendingConnectionsError, Ferrum::TimeoutError
-        unless page_loaded_enough?(page)
-          raise if retries >= NAVIGATION_MAX_RETRIES
-
-          retries += 1
-          retry
-        end
-      end
-      stabilize_page(page, url)
+      normalized_url = FetchUtil.normalize_url(url)
+      page = load_page_with_retry(ensure_browser, normalized_url)
+      sleep PRE_EXTRACTION_SETTLE_WAIT if heavy_script_page?(page, normalized_url)
       yield page
     rescue Ferrum::Error => e
       raise BrowserError, e.message
@@ -131,5 +126,49 @@ module FetchUtil
       normalized_host = FetchUtil.strip_www_host(url)
       normalized_host == host || normalized_host.end_with?(".#{host}")
     end
+
+    def retryable_pending_connections_error?(error)
+      error.is_a?(Ferrum::PendingConnectionsError) || error.message.to_s.match?(/pending connections/i)
+    end
+
+    def retryable_navigation_error?(error)
+      error.is_a?(Ferrum::PendingConnectionsError) || error.is_a?(Ferrum::TimeoutError) ||
+        error.message.to_s.match?(/pending connections/i)
+    end
+
+    def heavy_script_page?(page, url)
+      return false if host_matches?(url, "google.com") || host_matches?(url, "youtube.com")
+
+      script_count = page.evaluate("document.scripts ? document.scripts.length : 0")
+      script_count = script_count.is_a?(Numeric) ? script_count.to_i : 0
+      script_count >= HEAVY_SCRIPT_COUNT_THRESHOLD
+    rescue Ferrum::Error
+      false
+    end
+
+    def load_page_with_retry(ferrum, url)
+      retries = 0
+
+      begin
+        page = ferrum.create_page
+        page.headers.set(@default_headers)
+        page.bypass_csp
+        begin
+          page.go_to(url)
+        rescue Ferrum::TimeoutError
+          raise unless page_loaded_enough?(page)
+        end
+        stabilize_page(page, url)
+        page
+      rescue Ferrum::PendingConnectionsError, Ferrum::TimeoutError, Ferrum::Error => e
+        page&.close
+        raise unless retryable_navigation_error?(e)
+        raise if retries >= NAVIGATION_MAX_RETRIES
+
+        retries += 1
+        sleep NAVIGATION_RETRY_WAIT * (2**(retries - 1))
+        retry
+      end
+    end
   end
 end

data/lib/fetch_util/cli.rb

@@ -17,6 +17,7 @@ module FetchUtil
       content_type
       suspect
       warnings
+      error_message
     ].freeze
 
     class_option :log_path, type: :string, desc: "Append-only request log path"

data/lib/fetch_util/extractor.rb

@@ -4,9 +4,24 @@ require "json"
 
 module FetchUtil
   class Extractor
+    INLINE_ASSET_PATHS = %w[vendor/readability.js vendor/turndown.js extract.js].freeze
+    @asset_cache = {}
+    @cache_mutex = Mutex.new
+
+    class << self
+      def inline_asset_scripts(asset_root)
+        @cache_mutex.synchronize do
+          @asset_cache[asset_root] ||= INLINE_ASSET_PATHS.map do |relative_path|
+            File.read(File.join(asset_root, relative_path), encoding: "UTF-8").freeze
+          end.freeze
+        end
+      end
+    end
+
     def initialize(reader_mode: true, asset_root: nil)
       @reader_mode = reader_mode
       @asset_root = asset_root || File.join(__dir__, "assets")
+      @extraction_call = nil
     end
 
     def extract(page)
@@ -27,13 +42,16 @@ module FetchUtil
     end
 
     def inject_assets_inline(page)
-      %w[vendor/readability.js vendor/turndown.js extract.js].each do |relative_path|
-        script = File.read(asset_path(relative_path), encoding: "UTF-8")
+      inline_asset_scripts.each do |script|
         page.evaluate("#{script}\ntrue")
       end
     end
 
     def extract_payload(page)
+      timeout_supported = page.respond_to?(:timeout) && page.respond_to?(:timeout=)
+      original_timeout = page.timeout if timeout_supported
+      page.timeout = [original_timeout.to_f, 60].max if timeout_supported
+
       inject_assets(page)
       page.evaluate(extraction_call)
     rescue Ferrum::TimeoutError
@@ -43,10 +61,16 @@ module FetchUtil
       end
       inject_assets_inline(page)
       page.evaluate(extraction_call)
+    ensure
+      page.timeout = original_timeout if timeout_supported && original_timeout
     end
 
     def extraction_call
-      "window.FetchUtilExtract.extract(#{JSON.generate(reader_mode: @reader_mode)})"
+      @extraction_call ||= "window.FetchUtilExtract.extract(#{JSON.generate(reader_mode: @reader_mode)})"
+    end
+
+    def inline_asset_scripts
+      self.class.inline_asset_scripts(@asset_root)
     end
 
     def asset_path(relative_path)