fetch_util 0.3.2 → 0.4.0

data/lib/fetch_util/browser/interaction_helpers/consent_helpers.rb

@@ -289,6 +289,11 @@ module FetchUtil
           )
         end
 
+        def dismiss_cookie_overlays(page)
+          accepted_cookies = accept_cookie_consent(page)
+          dismiss_privacy_preference_overlay(page) || accepted_cookies
+        end
+
         def consent_quick_indicator_selector_js
           consent_selector_js(DEFAULT_CONSENT_CONFIG.quick_indicator_selectors)
         end

data/lib/fetch_util/browser/site_stabilization/travel_and_lodging.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module FetchUtil
+  class Browser
+    module SiteStabilization
+      module TravelAndLodging
+        TRAVEL_LODGING_STABILIZATION_PROFILES = {
+          stabilize_lodging_detail: {
+            host: %w[airbnb.com booking.com],
+            path_query: ->(uri) { uri.path.match?(%r{/(?:rooms|hotel)/}) },
+            strategy: :stabilize_lodging_detail,
+            notes: "Wait for client-rendered lodging detail cues without bypassing access controls.",
+            tests: "spec/fetch_util/browser_stabilization_spec.rb"
+          }
+        }.freeze
+
+        private
+
+        def stabilize_lodging_detail(page)
+          accepted_cookies = dismiss_cookie_overlays(page)
+
+          retry_until_timeout(capped_timeout(8.0), interval: 0.25) do
+            safe_evaluate(page, <<~JS, default: false)
+              (() => {
+                const bodyText = document.body ? (document.body.innerText || '') : '';
+                const structuredLodging = Array.from(document.querySelectorAll('script[type="application/ld+json"]')).some((script) => /LodgingBusiness|LodgingReservation|Hotel|VacationRental|Accommodation/i.test(script.textContent || ''));
+                if (structuredLodging) return true;
+                if (document.querySelector('[data-testid*="amenity" i], [data-testid*="facility" i], [data-testid*="review-score" i], [data-testid="property-description"], [itemprop="address"]')) return true;
+                return /amenities|facilities|guest reviews|property highlights|hosted by|check-in|check-out/i.test(bodyText) && bodyText.length > 900;
+              })()
+            JS
+          end
+
+          settle_after_stabilization(0.25) if accepted_cookies
+        end
+      end
+    end
+  end
+end

data/lib/fetch_util/browser/site_stabilization.rb

@@ -6,10 +6,12 @@ module FetchUtil
       autoload :CommunityAndMarketplace, "fetch_util/browser/site_stabilization/community_and_marketplace"
       autoload :GitlabRepo, "fetch_util/browser/site_stabilization/gitlab_repo"
       autoload :SocialPlatforms, "fetch_util/browser/site_stabilization/social_platforms"
+      autoload :TravelAndLodging, "fetch_util/browser/site_stabilization/travel_and_lodging"
 
       include CommunityAndMarketplace
       include GitlabRepo
       include SocialPlatforms
+      include TravelAndLodging
     end
   end
 end

data/lib/fetch_util/browser/stabilization/page_flow.rb

@@ -10,8 +10,18 @@ module FetchUtil
           SiteStabilization::SocialPlatforms::SOCIAL_PLATFORM_STABILIZATION_PROFILES[:stabilize_instagram],
           SiteStabilization::SocialPlatforms::SOCIAL_PLATFORM_STABILIZATION_PROFILES[:stabilize_facebook],
           SiteStabilization::CommunityAndMarketplace::COMMUNITY_MARKETPLACE_STABILIZATION_PROFILES[:stabilize_ebay_search],
+          SiteStabilization::TravelAndLodging::TRAVEL_LODGING_STABILIZATION_PROFILES[:stabilize_lodging_detail],
+          { host: "t.me", path_query: ->(uri) { uri.path.match?(%r{\A/s/[^/]+/\d+/?\z}) },
+            strategy: :wait_for_telegram_message, notes: "Wait for the requested public Telegram preview message.",
+            tests: "spec/fetch_util/browser_stabilization_spec.rb" },
           { host: "gitlab.com", path_query: ->(uri) { uri.path.split("/").reject(&:empty?).length == 2 },
             strategy: :stabilize_gitlab_repo, notes: "Wait for repository README content on GitLab project roots.",
+            tests: "spec/fetch_util/browser_stabilization_spec.rb" },
+          { host: "onet.pl", path_query: ->(uri) { uri.path == "/" },
+            strategy: :wait_for_onet_homepage, notes: "Wait for Onet's hydrated feed regions and cards before extraction.",
+            tests: "spec/fetch_util/browser_stabilization_spec.rb" },
+          { host: "wp.pl", path_query: ->(uri) { uri.path == "/" },
+            strategy: :wait_for_wp_homepage, notes: "Wait for WP's materialized section grids before extraction.",
             tests: "spec/fetch_util/browser_stabilization_spec.rb" }
         ].freeze
         POST_GENERIC_STABILIZATION_PROFILES = [
@@ -34,18 +44,15 @@ module FetchUtil
 
           reached_idle = !@wait_for_idle || wait_for_idle_or_content(page)
           preserve_consent = preserve_consent_wall?(page, url)
-          accepted_cookies = preserve_consent ? false : accept_cookie_consent(page)
-          accepted_cookies = (!preserve_consent && dismiss_privacy_preference_overlay(page)) || accepted_cookies
+          accepted_cookies = preserve_consent ? false : dismiss_cookie_overlays(page)
           if @wait.positive? && (!@wait_for_idle || accepted_cookies)
             sleep @wait
-            accepted_cookies = (!preserve_consent && accept_cookie_consent(page)) || accepted_cookies
-            accepted_cookies = (!preserve_consent && dismiss_privacy_preference_overlay(page)) || accepted_cookies
+            accepted_cookies = dismiss_cookie_overlays(page) || accepted_cookies unless preserve_consent
           end
 
           wait_for_spa_hydration(page) if @wait_for_idle && reached_idle
           if accepted_cookies
-            accepted_cookies = (!preserve_consent && accept_cookie_consent(page)) || accepted_cookies
-            accepted_cookies = (!preserve_consent && dismiss_privacy_preference_overlay(page)) || accepted_cookies
+            accepted_cookies = dismiss_cookie_overlays(page) || accepted_cookies
           end
           if (profile = matching_stabilization_profile(url, POST_GENERIC_STABILIZATION_PROFILES))
             send(profile.fetch(:strategy), page, url)
@@ -155,6 +162,32 @@ module FetchUtil
         rescue Ferrum::JavaScriptError, Ferrum::TimeoutError
           false
         end
+
+        def wait_for_onet_homepage(page)
+          wait_for_structural_readiness(page, "section[class*='Feed_']", "article[class*='Card_']")
+        end
+
+        def wait_for_wp_homepage(page)
+          wait_for_structural_readiness(page, ".wp-section-grid", ".wp-teaser-tile, .wp-teaser-regular")
+        end
+
+        def wait_for_telegram_message(page)
+          target = URI.parse(page.current_url).path.delete_prefix("/s/")
+
+          ready = retry_until_timeout(capped_timeout(5.0), interval: 0.25) do
+            page.evaluate(<<~JS)
+              (() => {
+                const card = document.querySelector('.tgme_widget_message[data-post="#{target}"]');
+                const text = card && card.querySelector('.tgme_widget_message_text, .js-message_text');
+                return !!(text && (text.innerText || '').trim());
+              })()
+            JS
+          end
+          sleep PRE_EXTRACTION_SETTLE_WAIT if ready
+          ready
+        rescue URI::InvalidURIError, Ferrum::JavaScriptError, Ferrum::TimeoutError
+          false
+        end
       end
     end
   end

data/lib/fetch_util/browser/stabilization/spa_hydration.rb

@@ -19,6 +19,23 @@ module FetchUtil
         rescue Ferrum::JavaScriptError, Ferrum::TimeoutError
         end
 
+        def wait_for_structural_readiness(page, region_selector, card_selector)
+          previous_count = nil
+          retry_until_timeout(SPA_HYDRATION_TIMEOUT, interval: SPA_HYDRATION_POLL) do
+            counts = page.evaluate(<<~JS)
+              (() => ({
+                regions: document.querySelectorAll(#{region_selector.to_json}).length,
+                cards: document.querySelectorAll(#{card_selector.to_json}).length
+              }))()
+            JS
+            current_count = [counts["regions"], counts["cards"]]
+            ready = current_count == previous_count && current_count[0].positive? && current_count[1] >= 3
+            previous_count = current_count
+            ready
+          end
+        rescue Ferrum::JavaScriptError, Ferrum::TimeoutError
+        end
+
         def detect_spa_framework(page)
           result = page.evaluate(spa_framework_detection_script)
 

data/lib/fetch_util/cli.rb

@@ -1,24 +1,42 @@
 # frozen_string_literal: true
 
 require "json"
+require "yaml"
 require "thor"
 
 module FetchUtil
   class CLI < Thor
     DEFAULT_FETCH_FIELDS = %i[
-      url
-      final_url
-      canonical_url
       title
       byline
       site_name
       published_time
+      language
+      name
+      company
+      location
+      description
+      ingredients
+      instructions
+      bedrooms
+      bathrooms
+      area_sqft
       markdown
       content_type
+      price
+      rating
+      address
+      social_kind
+      platform
+      handle
+      reply_count
+      community
+      score
       suspect
       warnings
       error_message
     ].freeze
+    FETCH_URL_FIELDS = %i[url final_url canonical_url].freeze
 
     class_option :log_path, type: :string, desc: "Append-only request log path"
     class_option :format, type: :string, default: "markdown", enum: %w[markdown json jsonl], desc: "Output format"
@@ -28,6 +46,7 @@ module FetchUtil
     class_option :reader_mode, type: :boolean, default: true
     class_option :wait_for_idle, type: :boolean, default: true
     class_option :include_html, type: :boolean, default: false, desc: "Include raw html in fetch output"
+    class_option :include_urls, type: :boolean, default: false, desc: "Include URL fields in fetch output"
 
     desc "version", "Display fetch_util version"
     def version
@@ -45,10 +64,7 @@ module FetchUtil
                 end
 
       if options[:format] == "markdown"
-        results.each_with_index do |result, index|
-          puts "\n---\n\n" if index > 0
-          puts result.markdown
-        end
+        puts results.map { |result| front_matter_document(result) }.join("\n\n")
       else
         emit(urls.length == 1 && options[:format] == "json" ? result_payload(results.first) : results.map { |result| result_payload(result) })
       end
@@ -56,7 +72,7 @@ module FetchUtil
 
     desc "search QUERY", "Search across configured engines and aggregate results"
     option :source, type: :array, default: FetchUtil::Searcher::DEFAULT_SOURCES, desc: "Search sources"
-    option :limit, type: :numeric, default: 10
+    option :limit, type: :numeric, default: nil, desc: "Maximum results; omit to return every result in the fetched response"
     option :verbose_search, type: :boolean, default: false, desc: "Include per-result search provenance"
     def search(*terms)
       query = terms.join(" ").strip
@@ -107,10 +123,20 @@ module FetchUtil
 
       def result_payload(result)
         payload = result.to_h
-        payload = payload.select { |key, _value| DEFAULT_FETCH_FIELDS.include?(key) }
+        fields = DEFAULT_FETCH_FIELDS
+        fields = FETCH_URL_FIELDS + fields if options[:include_urls]
+        payload = payload.select { |key, _value| fields.include?(key) }
         payload[:html] = result.html if options[:include_html]
 
-        payload.reject { |_key, value| value.nil? || value == "" }
+        payload.reject { |key, value| key != :language && (value.nil? || value == "") }
+      end
+
+      def front_matter_document(result)
+        payload = result_payload(result)
+        markdown = result.markdown
+        payload = payload.reject { |key, _value| key == :markdown }
+        yaml = YAML.dump(payload.transform_keys(&:to_s)).sub(/\A---\n/, "")
+        "---\n#{yaml}---\n#{markdown}"
       end
 
       def emit(payload)

data/lib/fetch_util/fetcher.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require "uri"
+require "net/http"
 
 module FetchUtil
   class Fetcher
@@ -36,6 +37,8 @@ module FetchUtil
     LINKED_MARKDOWN_ITEM_PATTERN = /(?:^|\s)(?:\d+\.|[-*])\s+\[[^\]]{8,220}\]\(/
     INDEX_QUERY_PATTERN = /(?:^|[&?])(?:q|query|search|searchtext|keyword|k)=/i
     PDF_PATH_PATTERN = %r{(?:\.pdf\z|/pdf(?:/|\z)|[?&](?:format|download)=pdf\b)}i
+    PDF_CONTENT_TYPE_PATTERN = %r{\Aapplication/(?:pdf|x-pdf)\b}i
+    PDF_REDIRECT_LIMIT = 3
     LEGAL_STATUTE_TITLE_PATTERN = Regexp.new(
       "constitution|constitutional|constitui[cç]?[aã]o|c[oó]digo|codigo|code|statute|act|law|" \
       "regulation|ordinance|decree|treaty|convention",
@@ -160,6 +163,7 @@ module FetchUtil
       @extractor = extractor || Extractor.new(reader_mode: options.fetch(:reader_mode, true))
       @raw_docs_fallback = options[:raw_docs_fallback] || RawDocsFallback.new(timeout: @timeout)
       @request_log = options[:request_log]
+      @pdf_header_probe = options[:pdf_header_probe]
     end
 
     def quit
@@ -171,8 +175,17 @@ module FetchUtil
       pending_connection_retries = 0
 
       begin
+        if (pdf_result = direct_pdf_result(url))
+          log_request(url, t0)
+          return pdf_result
+        end
+
         result = @browser.with_page(url) do |page|
           payload = @extractor.extract(page)
+          if telegram_focal_preview?(url) && payload["contentType"] == "article"
+            sleep Browser::PRE_EXTRACTION_SETTLE_WAIT
+            payload = @extractor.extract(page)
+          end
           build_result(url, page.current_url, payload)
         end
         fallback = seznam_cmp_redirect_fallback_candidate?(url, result) ? @raw_docs_fallback.fetch(url) : nil
@@ -204,6 +217,14 @@ module FetchUtil
 
     private
 
+    def telegram_focal_preview?(url)
+      uri = URI.parse(url)
+      host = FetchUtil.strip_www_host(url)
+      host == "t.me" && uri.path.match?(%r{\A/s/[^/]+/\d+/?\z})
+    rescue URI::InvalidURIError
+      false
+    end
+
     def build_result(url, final_url, payload)
       raw_final_url = final_url
       raw_canonical_url = payload["canonicalUrl"]
@@ -236,6 +257,7 @@ module FetchUtil
       final_url = snapshot.final_url
       content_type = payload["contentType"] || "article"
       return "article" if content_type == "list" && scholarly_article_markdown?(final_url, snapshot)
+      return "article" if content_type == "list" && credible_article_route?(final_url, snapshot)
 
       return content_type unless content_type == "article"
       return content_type if payload["legalProvision"]
@@ -244,6 +266,7 @@ module FetchUtil
       return content_type if legal_judgment_markdown?(snapshot) || legal_statute_markdown?(snapshot)
       return content_type if scholarly_article_markdown?(final_url, snapshot)
       return content_type if reference_table_article_markdown?(snapshot)
+      return content_type if credible_article_route?(final_url, snapshot)
       return "list" if government_service_portal?(final_url, snapshot)
       return "list" if homepage_like && homepage_index_markdown?(snapshot)
       return "list" if index_list_markdown?(final_url, snapshot)
@@ -267,9 +290,11 @@ module FetchUtil
                                            snapshot.raw_canonical_url)
         warnings.delete("url_content_mismatch")
       end
+      credible_homepage_list = credible_homepage_list_evidence?(content_type, homepage_like, snapshot, warnings)
+      warnings.delete("url_content_mismatch") if credible_homepage_list
       if content_type == "list" && homepage_like && !payload["statusPage"] &&
-         !substantial_homepage_landing?(snapshot) && !government_service_portal?(final_url, snapshot) &&
-         !research_database_landing?(snapshot)
+         !credible_homepage_list && !substantial_homepage_landing?(snapshot) &&
+         !government_service_portal?(final_url, snapshot) && !research_database_landing?(snapshot)
         warnings.delete("url_content_mismatch")
         warnings << "homepage_index_page"
       end
@@ -285,6 +310,18 @@ module FetchUtil
       warnings.uniq
     end
 
+    def credible_homepage_list_evidence?(content_type, homepage_like, snapshot, warnings)
+      return false unless content_type == "list" && homepage_like
+      return false if snapshot.payload["statusPage"]
+      return false if warnings.any? { |warning| warning.match?(/(?:access|auth|bot|challenge|consent|empty|error|interstitial|not_found|paywall|short|wall)/i) }
+      return false if generic_redirect_not_found?(snapshot) || auth_redirect_interstitial?(snapshot)
+      return false if snapshot.normalized_markdown.empty?
+
+      evidence = snapshot.payload["portalRootEvidence"]
+      evidence.is_a?(Hash) && evidence["namedSectionCount"].to_i >= 2 &&
+        evidence["canonicalCardCount"].to_i >= 4
+    end
+
     def homepage_like?(url)
       path = URI.parse(url).path
       path.nil? || path.empty? || path == "/"
@@ -384,6 +421,22 @@ module FetchUtil
       linked_headlines + linked_items >= 4
     end
 
+    def credible_article_route?(url, snapshot)
+      path = URI.parse(url).path.to_s
+      a_detail = path.match?(%r{(?:^|/)a-[a-z0-9]+(?:/|$)}i)
+      bbc_detail = path.match?(%r{/(?:articles?)/[a-z0-9]+(?:/|$)}i)
+      detail_path = a_detail || bbc_detail
+      return false unless detail_path
+      return false unless snapshot.heading_count >= 1
+      return false unless snapshot.normalized_markdown.length >= 420
+      return false unless snapshot.long_prose_line_count_one_twenty >= 2
+      return false if a_detail && snapshot.payload["byline"].to_s.strip.empty? && snapshot.payload["publishedTime"].to_s.strip.empty?
+
+      true
+    rescue URI::InvalidURIError
+      false
+    end
+
     def scholarly_article_markdown?(url, snapshot)
       payload = snapshot.payload
       normalized = snapshot.normalized_markdown
@@ -750,6 +803,100 @@ module FetchUtil
       Result.error(url: url, warning: warning, message: message)
     end
 
+    def direct_pdf_result(url)
+      pdf_info = if pdf_url?(url)
+                   { final_url: url, headers: {} }
+                 else
+                   pdf_header_info(url)
+                 end
+      return nil unless pdf_info
+
+      final_url = normalized_result_url(pdf_info.fetch(:final_url, url))
+      headers = pdf_info.fetch(:headers, {})
+      Result.from_payload(
+        url: url,
+        final_url: final_url,
+        canonical_url: nil,
+        payload: {
+          "title" => pdf_title(final_url, headers),
+          "markdown" => "",
+          "contentType" => "pdf_document",
+          "warnings" => ["pdf_document"]
+        },
+        content_type: "pdf_document",
+        suspect: true,
+        warnings: ["pdf_document"]
+      )
+    end
+
+    def pdf_header_info(url)
+      info = @pdf_header_probe ? @pdf_header_probe.call(url) : probe_pdf_headers(url)
+      return nil unless pdf_content_type?(info[:headers])
+
+      info
+    rescue URI::InvalidURIError, ArgumentError, FetchUtil::Error, IOError, SocketError, SystemCallError, Timeout::Error
+      nil
+    end
+
+    def probe_pdf_headers(url, limit = PDF_REDIRECT_LIMIT)
+      uri = parse_http_uri(url)
+      response = request_head(uri)
+      if response.is_a?(Net::HTTPRedirection) && limit.positive? && response["location"].to_s.strip != ""
+        return probe_pdf_headers(uri.merge(response["location"]).to_s, limit - 1)
+      end
+
+      { final_url: uri.to_s, headers: response.to_hash.transform_keys(&:downcase) }
+    end
+
+    def request_head(uri)
+      Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == "https", open_timeout: @timeout, read_timeout: @timeout) do |http|
+        request = Net::HTTP::Head.new(uri.request_uri.empty? ? "/" : uri.request_uri)
+        http.request(request)
+      end
+    end
+
+    def parse_http_uri(url)
+      uri = URI.parse(url.to_s)
+      raise URI::InvalidURIError, "unsupported url: #{url}" unless uri.is_a?(URI::HTTP) && uri.host
+
+      uri
+    end
+
+    def pdf_url?(url)
+      URI.parse(url.to_s).then do |uri|
+        [uri.path, uri.query && "?#{uri.query}"].compact.join.match?(PDF_PATH_PATTERN)
+      end
+    rescue URI::InvalidURIError
+      false
+    end
+
+    def pdf_content_type?(headers)
+      Array(headers&.fetch("content-type", nil)).any? { |value| value.to_s.match?(PDF_CONTENT_TYPE_PATTERN) }
+    end
+
+    def pdf_title(url, headers)
+      title_from_content_disposition(headers) || title_from_url(url)
+    end
+
+    def title_from_content_disposition(headers)
+      value = Array(headers["content-disposition"]).first.to_s
+      filename = value[/filename\*=UTF-8''([^;]+)/i, 1] || value[/filename="?([^";]+)"?/i, 1]
+      return nil if filename.to_s.strip.empty?
+
+      URI.decode_www_form_component(filename).strip
+    rescue ArgumentError
+      filename.to_s.strip
+    end
+
+    def title_from_url(url)
+      path = URI.parse(url.to_s).path.to_s
+      basename = File.basename(path)
+      title = basename.empty? || basename == "/" ? URI.parse(url.to_s).host.to_s : basename
+      URI.decode_www_form_component(title.tr("+", " ")).strip
+    rescue URI::InvalidURIError, ArgumentError
+      url.to_s
+    end
+
     def network_error?(error)
       error.message.to_s.match?(NETWORK_ERROR_PATTERN)
     end