fetch_util 0.3.1 → 0.4.0

data/lib/fetch_util/browser/site_stabilization/travel_and_lodging.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module FetchUtil
+  class Browser
+    module SiteStabilization
+      module TravelAndLodging
+        TRAVEL_LODGING_STABILIZATION_PROFILES = {
+          stabilize_lodging_detail: {
+            host: %w[airbnb.com booking.com],
+            path_query: ->(uri) { uri.path.match?(%r{/(?:rooms|hotel)/}) },
+            strategy: :stabilize_lodging_detail,
+            notes: "Wait for client-rendered lodging detail cues without bypassing access controls.",
+            tests: "spec/fetch_util/browser_stabilization_spec.rb"
+          }
+        }.freeze
+
+        private
+
+        def stabilize_lodging_detail(page)
+          accepted_cookies = dismiss_cookie_overlays(page)
+
+          retry_until_timeout(capped_timeout(8.0), interval: 0.25) do
+            safe_evaluate(page, <<~JS, default: false)
+              (() => {
+                const bodyText = document.body ? (document.body.innerText || '') : '';
+                const structuredLodging = Array.from(document.querySelectorAll('script[type="application/ld+json"]')).some((script) => /LodgingBusiness|LodgingReservation|Hotel|VacationRental|Accommodation/i.test(script.textContent || ''));
+                if (structuredLodging) return true;
+                if (document.querySelector('[data-testid*="amenity" i], [data-testid*="facility" i], [data-testid*="review-score" i], [data-testid="property-description"], [itemprop="address"]')) return true;
+                return /amenities|facilities|guest reviews|property highlights|hosted by|check-in|check-out/i.test(bodyText) && bodyText.length > 900;
+              })()
+            JS
+          end
+
+          settle_after_stabilization(0.25) if accepted_cookies
+        end
+      end
+    end
+  end
+end

data/lib/fetch_util/browser/site_stabilization.rb

@@ -6,10 +6,12 @@ module FetchUtil
       autoload :CommunityAndMarketplace, "fetch_util/browser/site_stabilization/community_and_marketplace"
       autoload :GitlabRepo, "fetch_util/browser/site_stabilization/gitlab_repo"
       autoload :SocialPlatforms, "fetch_util/browser/site_stabilization/social_platforms"
+      autoload :TravelAndLodging, "fetch_util/browser/site_stabilization/travel_and_lodging"
 
       include CommunityAndMarketplace
       include GitlabRepo
       include SocialPlatforms
+      include TravelAndLodging
     end
   end
 end

data/lib/fetch_util/browser/stabilization/page_flow.rb

@@ -2,34 +2,79 @@
 
 module FetchUtil
   class Browser
+    # rubocop:disable Metrics/ModuleLength -- shared stabilization module is intentionally centralized.
     module Stabilization
       module PageFlow
+        PAGE_FLOW_STABILIZATION_PROFILES = [
+          SiteStabilization::CommunityAndMarketplace::COMMUNITY_MARKETPLACE_STABILIZATION_PROFILES[:stabilize_reddit],
+          SiteStabilization::SocialPlatforms::SOCIAL_PLATFORM_STABILIZATION_PROFILES[:stabilize_instagram],
+          SiteStabilization::SocialPlatforms::SOCIAL_PLATFORM_STABILIZATION_PROFILES[:stabilize_facebook],
+          SiteStabilization::CommunityAndMarketplace::COMMUNITY_MARKETPLACE_STABILIZATION_PROFILES[:stabilize_ebay_search],
+          SiteStabilization::TravelAndLodging::TRAVEL_LODGING_STABILIZATION_PROFILES[:stabilize_lodging_detail],
+          { host: "t.me", path_query: ->(uri) { uri.path.match?(%r{\A/s/[^/]+/\d+/?\z}) },
+            strategy: :wait_for_telegram_message, notes: "Wait for the requested public Telegram preview message.",
+            tests: "spec/fetch_util/browser_stabilization_spec.rb" },
+          { host: "gitlab.com", path_query: ->(uri) { uri.path.split("/").reject(&:empty?).length == 2 },
+            strategy: :stabilize_gitlab_repo, notes: "Wait for repository README content on GitLab project roots.",
+            tests: "spec/fetch_util/browser_stabilization_spec.rb" },
+          { host: "onet.pl", path_query: ->(uri) { uri.path == "/" },
+            strategy: :wait_for_onet_homepage, notes: "Wait for Onet's hydrated feed regions and cards before extraction.",
+            tests: "spec/fetch_util/browser_stabilization_spec.rb" },
+          { host: "wp.pl", path_query: ->(uri) { uri.path == "/" },
+            strategy: :wait_for_wp_homepage, notes: "Wait for WP's materialized section grids before extraction.",
+            tests: "spec/fetch_util/browser_stabilization_spec.rb" }
+        ].freeze
+        POST_GENERIC_STABILIZATION_PROFILES = [
+          { host: ["wyborcza.pl", "gazeta.pl"], path_query: ->(uri) { uri.path.match?(%r{/(?:\d+,){2}\d+,|/7,}) },
+            strategy: :wait_for_agora_article,
+            notes: "After generic consent/idle handling, wait briefly for delayed Agora article bodies.",
+            tests: "spec/fetch_util/browser_stabilization_spec.rb" },
+          { host: "france24.com", path_query: ->(uri) { uri.path.match?(%r{/\d{8}-}) },
+            strategy: :wait_for_france24_article,
+            notes: "After generic consent/idle handling, wait briefly for delayed France24 article bodies.",
+            tests: "spec/fetch_util/browser_stabilization_spec.rb" }
+        ].freeze
+
         private
 
         def stabilize_page(page, url)
-          return stabilize_reddit(page) if reddit_url?(url)
-          return stabilize_instagram(page) if instagram_url?(url)
-          return stabilize_facebook(page) if facebook_url?(url)
-          return stabilize_ebay_search(page) if ebay_search_url?(url)
-          return stabilize_gitlab_repo(page) if gitlab_repo_url?(url)
+          if (profile = matching_stabilization_profile(url, PAGE_FLOW_STABILIZATION_PROFILES))
+            return send(profile.fetch(:strategy), page)
+          end
 
           reached_idle = !@wait_for_idle || wait_for_idle_or_content(page)
           preserve_consent = preserve_consent_wall?(page, url)
-          accepted_cookies = preserve_consent ? false : accept_cookie_consent(page)
-          accepted_cookies = (!preserve_consent && dismiss_privacy_preference_overlay(page)) || accepted_cookies
-          sleep @wait if @wait.positive?
-          accepted_cookies = (!preserve_consent && accept_cookie_consent(page)) || accepted_cookies
-          accepted_cookies = (!preserve_consent && dismiss_privacy_preference_overlay(page)) || accepted_cookies
+          accepted_cookies = preserve_consent ? false : dismiss_cookie_overlays(page)
+          if @wait.positive? && (!@wait_for_idle || accepted_cookies)
+            sleep @wait
+            accepted_cookies = dismiss_cookie_overlays(page) || accepted_cookies unless preserve_consent
+          end
 
           wait_for_spa_hydration(page) if @wait_for_idle && reached_idle
-          accepted_cookies = (!preserve_consent && accept_cookie_consent(page)) || accepted_cookies
-          accepted_cookies = (!preserve_consent && dismiss_privacy_preference_overlay(page)) || accepted_cookies
+          if accepted_cookies
+            accepted_cookies = dismiss_cookie_overlays(page) || accepted_cookies
+          end
+          if (profile = matching_stabilization_profile(url, POST_GENERIC_STABILIZATION_PROFILES))
+            send(profile.fetch(:strategy), page, url)
+          end
 
           return unless accepted_cookies && @wait_for_idle && reached_idle
 
-          page.network.wait_for_idle(duration: @idle_duration, timeout: POST_CONSENT_IDLE_TIMEOUT)
+          wait_for_network_idle(page)
         end
 
+        def matching_stabilization_profile(url, profiles)
+          uri = URI.parse(url)
+          host = FetchUtil.strip_www_host(url)
+          profiles.find { |profile| profile_match?(profile, uri, host) }
+        rescue URI::InvalidURIError
+          nil
+        end
+
+        def profile_match?(profile, uri, host) = stabilization_host_matches?(profile.fetch(:host), host) && profile.fetch(:path_query, ->(_) { true }).call(uri)
+
+        def stabilization_host_matches?(matcher, host) = Array(matcher).any? { |candidate| host == candidate || host.end_with?(".#{candidate}") }
+
         def wait_for_idle_or_content(page)
           content_seen_at = nil
 
@@ -46,6 +91,17 @@ module FetchUtil
           false
         end
 
+        def wait_for_network_idle(page)
+          page.network.wait_for_idle(duration: @idle_duration, timeout: POST_CONSENT_IDLE_TIMEOUT)
+          true
+        rescue Ferrum::TimeoutError
+          false
+        rescue Ferrum::Error => e
+          raise if retryable_pending_connections_error?(e)
+
+          false
+        end
+
         def page_has_content?(page)
           page.evaluate(<<~JS)
             (() => {
@@ -61,6 +117,7 @@ module FetchUtil
 
         def preserve_consent_wall?(page, url)
           host = FetchUtil.strip_www_host(url)
+          return true if host == "france24.com" || host.end_with?(".france24.com")
           return false unless host == "youtube.com" || host.end_with?(".youtube.com") || host.match?(/\Agoogle\.[a-z.]+\z/)
 
           state = page.evaluate(<<~JS)
@@ -75,7 +132,65 @@ module FetchUtil
         rescue URI::InvalidURIError, Ferrum::JavaScriptError, Ferrum::TimeoutError
           false
         end
+
+        def wait_for_agora_article(page, _url)
+          retry_until_timeout(7.0, interval: 0.25) do
+            page.evaluate(<<~JS)
+              (() => {
+                const selectors = ['.mrf-article-body', '.article_body', 'div.articleBody', '.art_content', '.article-inner', 'section.article', '[itemprop=articleBody]'];
+                if (selectors.some((selector) => document.querySelector(selector))) return true;
+                const text = document.body ? (document.body.innerText || '') : '';
+                return !(new RegExp('Wyłącz AdBlocka/uBlocka|Nieznany błąd', 'i')).test(text) && text.length > 1200;
+              })()
+            JS
+          end
+        rescue Ferrum::JavaScriptError, Ferrum::TimeoutError
+          false
+        end
+
+        def wait_for_france24_article(page, _url)
+          retry_until_timeout(15.0, interval: 0.25) do
+            page.evaluate(<<~JS)
+              (() => {
+                const body = document.querySelector('.t-content__body') || document.querySelector('.t-content--article');
+                if (!body) return false;
+                const text = (body.innerText || '').replace(/\s+/g, ' ').trim();
+                return body.querySelectorAll('p').length >= 3 && text.length > 1000;
+              })()
+            JS
+          end
+        rescue Ferrum::JavaScriptError, Ferrum::TimeoutError
+          false
+        end
+
+        def wait_for_onet_homepage(page)
+          wait_for_structural_readiness(page, "section[class*='Feed_']", "article[class*='Card_']")
+        end
+
+        def wait_for_wp_homepage(page)
+          wait_for_structural_readiness(page, ".wp-section-grid", ".wp-teaser-tile, .wp-teaser-regular")
+        end
+
+        def wait_for_telegram_message(page)
+          target = URI.parse(page.current_url).path.delete_prefix("/s/")
+
+          ready = retry_until_timeout(capped_timeout(5.0), interval: 0.25) do
+            page.evaluate(<<~JS)
+              (() => {
+                const card = document.querySelector('.tgme_widget_message[data-post="#{target}"]');
+                const text = card && card.querySelector('.tgme_widget_message_text, .js-message_text');
+                return !!(text && (text.innerText || '').trim());
+              })()
+            JS
+          end
+          sleep PRE_EXTRACTION_SETTLE_WAIT if ready
+          ready
+        rescue URI::InvalidURIError, Ferrum::JavaScriptError, Ferrum::TimeoutError
+          false
+        end
       end
     end
   end
 end
+
+# rubocop:enable Metrics/ModuleLength

data/lib/fetch_util/browser/stabilization/spa_hydration.rb

@@ -19,6 +19,23 @@ module FetchUtil
         rescue Ferrum::JavaScriptError, Ferrum::TimeoutError
         end
 
+        def wait_for_structural_readiness(page, region_selector, card_selector)
+          previous_count = nil
+          retry_until_timeout(SPA_HYDRATION_TIMEOUT, interval: SPA_HYDRATION_POLL) do
+            counts = page.evaluate(<<~JS)
+              (() => ({
+                regions: document.querySelectorAll(#{region_selector.to_json}).length,
+                cards: document.querySelectorAll(#{card_selector.to_json}).length
+              }))()
+            JS
+            current_count = [counts["regions"], counts["cards"]]
+            ready = current_count == previous_count && current_count[0].positive? && current_count[1] >= 3
+            previous_count = current_count
+            ready
+          end
+        rescue Ferrum::JavaScriptError, Ferrum::TimeoutError
+        end
+
         def detect_spa_framework(page)
           result = page.evaluate(spa_framework_detection_script)
 

data/lib/fetch_util/browser.rb

@@ -36,7 +36,10 @@ module FetchUtil
     CONTENT_READY_MIN_LENGTH = 200
     SPA_HYDRATION_TIMEOUT = 2.0
     SPA_HYDRATION_POLL = 0.15
+    PRE_EXTRACTION_SETTLE_WAIT = 0.25
+    HEAVY_SCRIPT_COUNT_THRESHOLD = 20
     NAVIGATION_MAX_RETRIES = 2
+    NAVIGATION_RETRY_WAIT = 2.0
 
     def initialize(timeout: 20, wait: 0.75, wait_for_idle: true, idle_duration: 0.35,
                    viewport: DEFAULT_VIEWPORT, user_agent: DEFAULT_USER_AGENT,
@@ -49,6 +52,10 @@ module FetchUtil
       @viewport = DEFAULT_VIEWPORT.merge(symbolize_hash(viewport || {}))
       @user_agent = user_agent
       @accept_language = accept_language
+      @default_headers = {
+        "User-Agent" => @user_agent,
+        "Accept-Language" => @accept_language
+      }.freeze
       @browser_path = browser_path || ENV["BROWSER_PATH"] || BROWSER_CANDIDATES.find { |path| File.executable?(path) }
       @full_browser = @browser_path && !@browser_path.include?("headless_shell")
       default_opts = { "no-sandbox": nil }
@@ -61,6 +68,7 @@ module FetchUtil
         default_opts["enable-automation"] = false # override Ferrum default
       end
       @browser_options = default_opts.merge(browser_options || {})
+      @navigator_patch = build_navigator_patch
       @ferrum = nil
       @mutex = Mutex.new
     end
@@ -72,22 +80,9 @@ module FetchUtil
     def with_page(url)
       raise BrowserError, "No Chromium browser found. Set BROWSER_PATH or install Chromium." unless @browser_path
 
-      ferrum = ensure_browser
-      page = ferrum.create_page
-      page.headers.set(default_headers)
-      page.bypass_csp
-      retries = 0
-      begin
-        page.go_to(url)
-      rescue Ferrum::PendingConnectionsError, Ferrum::TimeoutError
-        unless page_loaded_enough?(page)
-          raise if retries >= NAVIGATION_MAX_RETRIES
-
-          retries += 1
-          retry
-        end
-      end
-      stabilize_page(page, url)
+      normalized_url = FetchUtil.normalize_url(url)
+      page = load_page_with_retry(ensure_browser, normalized_url)
+      sleep PRE_EXTRACTION_SETTLE_WAIT if heavy_script_page?(page, normalized_url)
       yield page
     rescue Ferrum::Error => e
       raise BrowserError, e.message
@@ -131,5 +126,49 @@ module FetchUtil
       normalized_host = FetchUtil.strip_www_host(url)
       normalized_host == host || normalized_host.end_with?(".#{host}")
     end
+
+    def retryable_pending_connections_error?(error)
+      error.is_a?(Ferrum::PendingConnectionsError) || error.message.to_s.match?(/pending connections/i)
+    end
+
+    def retryable_navigation_error?(error)
+      error.is_a?(Ferrum::PendingConnectionsError) || error.is_a?(Ferrum::TimeoutError) ||
+        error.message.to_s.match?(/pending connections/i)
+    end
+
+    def heavy_script_page?(page, url)
+      return false if host_matches?(url, "google.com") || host_matches?(url, "youtube.com")
+
+      script_count = page.evaluate("document.scripts ? document.scripts.length : 0")
+      script_count = script_count.is_a?(Numeric) ? script_count.to_i : 0
+      script_count >= HEAVY_SCRIPT_COUNT_THRESHOLD
+    rescue Ferrum::Error
+      false
+    end
+
+    def load_page_with_retry(ferrum, url)
+      retries = 0
+
+      begin
+        page = ferrum.create_page
+        page.headers.set(@default_headers)
+        page.bypass_csp
+        begin
+          page.go_to(url)
+        rescue Ferrum::TimeoutError
+          raise unless page_loaded_enough?(page)
+        end
+        stabilize_page(page, url)
+        page
+      rescue Ferrum::PendingConnectionsError, Ferrum::TimeoutError, Ferrum::Error => e
+        page&.close
+        raise unless retryable_navigation_error?(e)
+        raise if retries >= NAVIGATION_MAX_RETRIES
+
+        retries += 1
+        sleep NAVIGATION_RETRY_WAIT * (2**(retries - 1))
+        retry
+      end
+    end
   end
 end

data/lib/fetch_util/cli.rb

@@ -1,24 +1,42 @@
 # frozen_string_literal: true
 
 require "json"
+require "yaml"
 require "thor"
 
 module FetchUtil
   class CLI < Thor
     DEFAULT_FETCH_FIELDS = %i[
-      url
-      final_url
-      canonical_url
       title
       byline
       site_name
       published_time
+      language
+      name
+      company
+      location
+      description
+      ingredients
+      instructions
+      bedrooms
+      bathrooms
+      area_sqft
       markdown
       content_type
+      price
+      rating
+      address
+      social_kind
+      platform
+      handle
+      reply_count
+      community
+      score
       suspect
       warnings
       error_message
     ].freeze
+    FETCH_URL_FIELDS = %i[url final_url canonical_url].freeze
 
     class_option :log_path, type: :string, desc: "Append-only request log path"
     class_option :format, type: :string, default: "markdown", enum: %w[markdown json jsonl], desc: "Output format"
@@ -28,6 +46,7 @@ module FetchUtil
     class_option :reader_mode, type: :boolean, default: true
     class_option :wait_for_idle, type: :boolean, default: true
     class_option :include_html, type: :boolean, default: false, desc: "Include raw html in fetch output"
+    class_option :include_urls, type: :boolean, default: false, desc: "Include URL fields in fetch output"
 
     desc "version", "Display fetch_util version"
     def version
@@ -45,10 +64,7 @@ module FetchUtil
                 end
 
       if options[:format] == "markdown"
-        results.each_with_index do |result, index|
-          puts "\n---\n\n" if index > 0
-          puts result.markdown
-        end
+        puts results.map { |result| front_matter_document(result) }.join("\n\n")
       else
         emit(urls.length == 1 && options[:format] == "json" ? result_payload(results.first) : results.map { |result| result_payload(result) })
       end
@@ -56,7 +72,7 @@ module FetchUtil
 
     desc "search QUERY", "Search across configured engines and aggregate results"
     option :source, type: :array, default: FetchUtil::Searcher::DEFAULT_SOURCES, desc: "Search sources"
-    option :limit, type: :numeric, default: 10
+    option :limit, type: :numeric, default: nil, desc: "Maximum results; omit to return every result in the fetched response"
     option :verbose_search, type: :boolean, default: false, desc: "Include per-result search provenance"
     def search(*terms)
       query = terms.join(" ").strip
@@ -107,10 +123,20 @@ module FetchUtil
 
       def result_payload(result)
         payload = result.to_h
-        payload = payload.select { |key, _value| DEFAULT_FETCH_FIELDS.include?(key) }
+        fields = DEFAULT_FETCH_FIELDS
+        fields = FETCH_URL_FIELDS + fields if options[:include_urls]
+        payload = payload.select { |key, _value| fields.include?(key) }
         payload[:html] = result.html if options[:include_html]
 
-        payload.reject { |_key, value| value.nil? || value == "" }
+        payload.reject { |key, value| key != :language && (value.nil? || value == "") }
+      end
+
+      def front_matter_document(result)
+        payload = result_payload(result)
+        markdown = result.markdown
+        payload = payload.reject { |key, _value| key == :markdown }
+        yaml = YAML.dump(payload.transform_keys(&:to_s)).sub(/\A---\n/, "")
+        "---\n#{yaml}---\n#{markdown}"
       end
 
       def emit(payload)

data/lib/fetch_util/extractor.rb

@@ -4,9 +4,24 @@ require "json"
 
 module FetchUtil
   class Extractor
+    INLINE_ASSET_PATHS = %w[vendor/readability.js vendor/turndown.js extract.js].freeze
+    @asset_cache = {}
+    @cache_mutex = Mutex.new
+
+    class << self
+      def inline_asset_scripts(asset_root)
+        @cache_mutex.synchronize do
+          @asset_cache[asset_root] ||= INLINE_ASSET_PATHS.map do |relative_path|
+            File.read(File.join(asset_root, relative_path), encoding: "UTF-8").freeze
+          end.freeze
+        end
+      end
+    end
+
     def initialize(reader_mode: true, asset_root: nil)
       @reader_mode = reader_mode
       @asset_root = asset_root || File.join(__dir__, "assets")
+      @extraction_call = nil
     end
 
     def extract(page)
@@ -27,13 +42,16 @@ module FetchUtil
     end
 
     def inject_assets_inline(page)
-      %w[vendor/readability.js vendor/turndown.js extract.js].each do |relative_path|
-        script = File.read(asset_path(relative_path), encoding: "UTF-8")
+      inline_asset_scripts.each do |script|
         page.evaluate("#{script}\ntrue")
       end
     end
 
     def extract_payload(page)
+      timeout_supported = page.respond_to?(:timeout) && page.respond_to?(:timeout=)
+      original_timeout = page.timeout if timeout_supported
+      page.timeout = [original_timeout.to_f, 60].max if timeout_supported
+
       inject_assets(page)
       page.evaluate(extraction_call)
     rescue Ferrum::TimeoutError
@@ -43,10 +61,16 @@ module FetchUtil
       end
       inject_assets_inline(page)
       page.evaluate(extraction_call)
+    ensure
+      page.timeout = original_timeout if timeout_supported && original_timeout
     end
 
     def extraction_call
-      "window.FetchUtilExtract.extract(#{JSON.generate(reader_mode: @reader_mode)})"
+      @extraction_call ||= "window.FetchUtilExtract.extract(#{JSON.generate(reader_mode: @reader_mode)})"
+    end
+
+    def inline_asset_scripts
+      self.class.inline_asset_scripts(@asset_root)
     end
 
     def asset_path(relative_path)