fetch_util 0.5.0 → 0.5.2

data/lib/fetch_util/browser/stabilization/anubis.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+module FetchUtil
+  class Browser
+    module Stabilization
+      module Anubis
+        ANUBIS_POLL = 0.25
+
+        private
+
+        def wait_for_anubis_challenge(page)
+          deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + @timeout
+          state = anubis_page_state(page)
+          if !valid_anubis_state?(state) && Process.clock_gettime(Process::CLOCK_MONOTONIC) < deadline
+            state = anubis_page_state(page)
+          end
+          return false unless valid_anubis_state?(state) && state["challenge"] == true
+
+          remaining = deadline - Process.clock_gettime(Process::CLOCK_MONOTONIC)
+          return false unless remaining.positive?
+
+          retry_until_timeout(remaining, interval: ANUBIS_POLL) do
+            state = anubis_page_state(page)
+            next false unless valid_anubis_state?(state)
+            next false unless state["challenge"] == false
+
+            state["document_ready"] == true && state["body_present"] == true && state["body_text_present"] == true
+          end
+        end
+
+        def valid_anubis_state?(state)
+          state.is_a?(Hash) &&
+            [true, false].include?(state["challenge"]) &&
+            [true, false].include?(state["document_ready"]) &&
+            [true, false].include?(state["body_present"]) &&
+            [true, false].include?(state["body_text_present"]) &&
+            state["url"].is_a?(String) && !state["url"].empty?
+        end
+
+        def anubis_page_state(page)
+          safe_evaluate(page, <<~JS, default: {})
+            (() => {
+              const body = document.body;
+              const bodyText = body ? (body.innerText || '').replace(/\s+/g, ' ').trim() : '';
+              const anubisRoot = document.querySelector('#anubis_challenge');
+              const anubisScript = document.querySelector('script[src*="/.within.website/x/cmd/anubis/"]');
+              const visibleRoot = anubisRoot && (() => {
+                const style = window.getComputedStyle(anubisRoot);
+                const rect = anubisRoot.getBoundingClientRect();
+                return style.display !== 'none' && style.visibility !== 'hidden' && rect.width > 0 && rect.height > 0;
+              })();
+              const makingSure = /making sure you're not a bot/i.test((document.title || '') + ' ' + bodyText);
+              const protectedChallenge = /protected by anubis/i.test(bodyText) &&
+                /(enable javascript to get past this challenge|please wait a moment while we ensure the security of your connection|loading|calculating|challenge:\s*anubis)/i.test(bodyText);
+              const challenge = Boolean((visibleRoot || anubisScript) && (makingSure || protectedChallenge));
+              return {
+                challenge,
+                document_ready: document.readyState === 'complete',
+                body_present: Boolean(body),
+                body_text_present: bodyText.length > 0,
+                url: location.href
+              };
+            })()
+          JS
+        end
+      end
+    end
+  end
+end

data/lib/fetch_util/browser/stabilization/page_flow.rb

@@ -38,6 +38,8 @@ module FetchUtil
         private
 
         def stabilize_page(page, url)
+          wait_for_anubis_challenge(page)
+
           if (profile = matching_stabilization_profile(url, PAGE_FLOW_STABILIZATION_PROFILES))
             return send(profile.fetch(:strategy), page)
           end

data/lib/fetch_util/browser/stabilization.rb

@@ -5,9 +5,11 @@ module FetchUtil
     module Stabilization
       autoload :PageFlow, "fetch_util/browser/stabilization/page_flow"
       autoload :SpaHydration, "fetch_util/browser/stabilization/spa_hydration"
+      autoload :Anubis, "fetch_util/browser/stabilization/anubis"
 
       include PageFlow
       include SpaHydration
+      include Anubis
     end
   end
 end

data/lib/fetch_util/cli.rb

@@ -71,7 +71,7 @@ module FetchUtil
     end
 
     desc "search QUERY", "Search across configured engines and aggregate results"
-    option :source, type: :array, default: FetchUtil::Searcher::DEFAULT_SOURCES, desc: "Search sources"
+    option :source, type: :array, default: nil, desc: "Search sources"
     option :limit, type: :numeric, default: nil, desc: "Maximum results; omit to return every result in the fetched response"
     option :verbose_search, type: :boolean, default: false, desc: "Include per-result search provenance"
     def search(*terms)

data/lib/fetch_util/search_transport.rb

@@ -34,19 +34,24 @@ module FetchUtil
       "bing" => { url: "https://www.bing.com/search?q=%{query}&setlang=en-US&cc=US", hosts: %w[www.bing.com cn.bing.com] },
       "duckduckgo" => { url: "https://html.duckduckgo.com/html/?q=%{query}", hosts: %w[html.duckduckgo.com] },
       "google" => { url: "https://www.google.com/search?q=%{query}", hosts: %w[www.google.com] },
-      "ecosia" => { url: "https://www.ecosia.org/search?q=%{query}", hosts: %w[www.ecosia.org] }
+      "ecosia" => { url: "https://www.ecosia.org/search?q=%{query}", hosts: %w[www.ecosia.org] },
+      "yahoo" => { url: "https://search.yahoo.com/search?p=%{query}", hosts: %w[search.yahoo.com] }
     }.freeze
     DEFAULT_TIMEOUT = 10.0
+    YAHOO_RECOVERY_ATTEMPTS = 2
     FAILURE_REASONS = %w[challenge failed host http_status parse query_mismatch redirect size timeout].freeze
     RELEVANCE_HEALTH_CANDIDATE_COUNT = 3
     QUERY_FUNCTION_WORDS = %w[
       a an and are at be been being but by can could did do does for from had has have how i if in is it me my no not of on or our shall
       should that the their them then there these they this to was we were what when where which who why will with would you your
     ].freeze
+    SCOPED_QUERY_TERM = /(?:\A|\s)[+-]?[[:alpha:]][[:alnum:]_-]*:(?:"[^"]*"|'[^']*'|\S+)/
+    NEGATED_QUERY_TERM = /(?:\A|\s)-\S+/
     WRAPPER_HOSTS = {
       "bing" => %w[bing.com www.bing.com cn.bing.com],
       "duckduckgo" => %w[duckduckgo.com www.duckduckgo.com html.duckduckgo.com],
-      "google" => %w[google.com www.google.com]
+      "google" => %w[google.com www.google.com],
+      "yahoo" => %w[r.search.yahoo.com]
     }.freeze
 
     def initialize(sources: SOURCES.keys, timeout: DEFAULT_TIMEOUT, clock: nil, http_client: nil, html_parser: nil)
@@ -62,11 +67,28 @@ module FetchUtil
       @html_parser = html_parser || ->(body) { Nokogiri::HTML(body) }
     end
 
-    def search(query)
+    def self.candidates_match_query?(query, candidates)
+      candidates.first(RELEVANCE_HEALTH_CANDIDATE_COUNT).any? do |candidate|
+        candidate_matches_query?(query, candidate)
+      end
+    end
+
+    def self.candidate_matches_query?(query, candidate)
+      query_terms = meaningful_query_terms(normalized_query_text(query))
+      return true if query_terms.length < 3
+
+      candidate_terms = normalized_terms("#{candidate.title} #{candidate.snippet}")
+      (query_terms & candidate_terms).length >= 2
+    end
+
+    def search(query, timeout: @timeout)
       query = query.to_s.strip
       raise ArgumentError, "query must not be empty" if query.empty?
 
-      deadline = clock.call + timeout
+      request_timeout = Float(timeout)
+      raise ArgumentError, "timeout must be positive" unless request_timeout.positive?
+
+      deadline = clock.call + request_timeout
       responses = Array.new(sources.length)
       threads = sources.each_with_index.map do |source, index|
         Thread.new { responses[index] = search_source(source, query, deadline) }
@@ -82,8 +104,10 @@ module FetchUtil
     def search_source(source, query, deadline)
       started_at = clock.call
       url = build_url(source, query)
-      result = http_client.get(url, deadline: deadline, allowed_hosts: SOURCES.fetch(source).fetch(:hosts))
-      return failure(source, result.reason, elapsed_since(started_at), result.final_url) if result.is_a?(HttpFailure)
+      result = yahoo_response(source, url, deadline)
+      if result.is_a?(HttpFailure)
+        return failure(source, result.reason, elapsed_since(started_at), result.final_url)
+      end
       outcome = within_deadline(deadline) do
         document = html_parser.call(result.body)
         next [:failed, "challenge"] if challenge?(source, result, document)
@@ -92,13 +116,23 @@ module FetchUtil
 
         candidates = parse_candidates(source, document)
         next [:ok, candidates] if candidates.empty?
-        next [:failed, "query_mismatch"] unless query_matches_candidates?(query, candidates)
+        next [:suspect, candidates] unless self.class.candidates_match_query?(query, candidates)
 
         [:ok, candidates]
       end
       elapsed_ms = elapsed_since(started_at)
-      return failure(source, outcome.last, elapsed_ms, result.final_url) if outcome.first == :failed
+      return failure(source, outcome[1], elapsed_ms, result.final_url, candidates: outcome[2] || []) if outcome.first == :failed
       return empty(source, elapsed_ms, result.final_url) if outcome.first == :empty
+      if outcome.first == :suspect
+        return SourceResponse.new(
+          source: source,
+          status: "ok",
+          candidates: outcome.last,
+          elapsed_ms: elapsed_ms,
+          final_url: result.final_url,
+          reason: "query_mismatch"
+        )
+      end
 
       candidates = outcome.last
       return failure(source, "parse", elapsed_ms, result.final_url) if candidates.empty?
@@ -113,13 +147,31 @@ module FetchUtil
       failure(source, "failed", elapsed_since(started_at), url)
     end
 
+    def yahoo_response(source, url, deadline)
+      return http_client.get(url, deadline: deadline, allowed_hosts: SOURCES.fetch(source).fetch(:hosts)) unless source == "yahoo"
+
+      result = http_client.get(url, deadline: deadline, allowed_hosts: SOURCES.fetch(source).fetch(:hosts))
+      YAHOO_RECOVERY_ATTEMPTS.times do
+        break unless yahoo_retryable?(result) && deadline > clock.call
+
+        result = http_client.get(url, deadline: deadline, allowed_hosts: SOURCES.fetch(source).fetch(:hosts))
+      end
+      result
+    end
+
+    def yahoo_retryable?(result)
+      return result.reason == "failed" if result.is_a?(HttpFailure)
+
+      result.is_a?(HttpResponse) && (result.status == 429 || result.status.between?(500, 599))
+    end
+
     def build_url(source, query)
       SOURCES.fetch(source).fetch(:url) % { query: URI.encode_www_form_component(query) }
     end
 
     def parse_candidates(source, document)
       result_nodes(source, document).filter_map do |node|
-        next if excluded_node?(node)
+        next if excluded_node?(source, node)
 
         anchor = result_anchor(source, node)
         next unless anchor
@@ -135,37 +187,27 @@ module FetchUtil
       end
     end
 
-    def query_matches_candidates?(query, candidates)
-      normalized_query = normalized_query_text(query)
-      query_terms = meaningful_query_terms(normalized_query)
-      return true unless relevance_gate_applies?(normalized_query, query_terms)
-
-      candidates.first(RELEVANCE_HEALTH_CANDIDATE_COUNT).any? do |candidate|
-        candidate_terms = normalized_terms("#{candidate.title} #{candidate.snippet}")
-        (query_terms & candidate_terms).length >= 2
-      end
+    def self.meaningful_query_terms(query)
+      semantic_query = query.gsub(SCOPED_QUERY_TERM, " ").gsub(NEGATED_QUERY_TERM, " ")
+      normalized_terms(semantic_query).reject { |term| QUERY_FUNCTION_WORDS.include?(term) }
     end
 
-    def relevance_gate_applies?(query, query_terms)
-      return false if query_terms.length < 3
-
-      !query.match?(%r{["']|(?:\A|\s)[+-]?[[:alpha:]][[:alnum:]_-]*:|[+#/]|(?:\b[[:alpha:]]\.){2,}|\d|\b[[:upper:]]{2,}\b})
-    end
-
-    def meaningful_query_terms(query)
-      normalized_terms(query).reject { |term| QUERY_FUNCTION_WORDS.include?(term) }
+    def self.normalized_terms(text)
+      normalized = normalized_query_text(text)
+      normalized = normalized.gsub(/([[:upper:]]+)([[:upper:]][[:lower:]])/, '\1 \2')
+      normalized = normalized.gsub(/([[:lower:]\d])([[:upper:]])/, '\1 \2')
+      normalized.downcase.scan(/[[:alnum:]]+/).uniq
     end
 
-    def normalized_terms(text)
-      normalized_query_text(text).downcase.scan(/[[:alnum:]]+/).uniq
-    end
-
-    def normalized_query_text(text)
+    def self.normalized_query_text(text)
       text.to_s.encode("UTF-8", invalid: :replace, undef: :replace, replace: " ").unicode_normalize(:nfc)
     end
 
+    private_class_method :meaningful_query_terms, :normalized_terms, :normalized_query_text
+
     def result_nodes(source, document)
       return brave_result_nodes(document) if source == "brave"
+      return yahoo_result_nodes(document) if source == "yahoo"
 
       selector = {
         "bing" => "#b_results li.b_algo",
@@ -183,6 +225,11 @@ module FetchUtil
       (current.to_a + legacy).uniq
     end
 
+    def yahoo_result_nodes(document)
+      nodes = document.css("#web .algo, #web .algo-sr, #web .dd.algo").uniq
+      nodes.reject { |node| nodes.any? { |other| other != node && other.ancestors.include?(node) } }
+    end
+
     def legacy_brave_result?(node)
       node.at_css("h2, h3, .title.search-snippet-title") && result_anchor("brave", node)
     end
@@ -197,6 +244,7 @@ module FetchUtil
       when "bing" then node.at_css("h2 a[href]")
       when "duckduckgo" then node.at_css("a.result__a[href], h2 a[href]")
       when "google" then node.at_css("a[href]:has(h3)") || node.at_css("h3")&.ancestors("a[href]")&.first
+      when "yahoo" then node.at_css("h3 a[href], .compTitle a[href]")
       else node.at_css("h2 a[href], h3 a[href], a[href]:has(h3)")
       end
     end
@@ -213,21 +261,31 @@ module FetchUtil
         "bing" => ".b_caption p, .b_paractl",
         "duckduckgo" => ".result__snippet",
         "google" => ".VwiC3b, .aCOpRe, [data-sncf]",
-        "ecosia" => ".result-snippet, .result__description"
+        "ecosia" => ".result-snippet, .result__description",
+        "yahoo" => ".compText, .compText p"
       }.fetch(source)
     end
 
-    def excluded_node?(node)
+    def excluded_node?(source, node)
+      return yahoo_excluded_node?(node) if source == "yahoo"
+
       node.xpath("ancestor-or-self::*").any? do |ancestor|
         value = [ancestor["class"], ancestor["id"], ancestor["data-testid"]].compact.join(" ").downcase
         value.match?(/\b(ad|ads|advert|enrichment|knowledge|llm|nav|pagination|related|answer)\b/)
       end
     end
 
+    def yahoo_excluded_node?(node)
+      node.xpath("ancestor-or-self::*").any? do |ancestor|
+        value = [ancestor["class"], ancestor["id"], ancestor["data-testid"]].compact.join(" ").downcase
+        value.match?(/\b(ad|ads|advert|control|controls|enrichment|knowledge|llm|lookalike|nav|navigation|pagination|related|right[-_ ]?rail|assist)\b/)
+      end
+    end
+
     def destination(source, href)
       value = href.to_s.strip
       value = decode_bing(value) if source == "bing"
-      value = decode_wrapper(source, value) if %w[google duckduckgo].include?(source)
+      value = decode_wrapper(source, value) if %w[google duckduckgo yahoo].include?(source)
       uri = URI.parse(value)
       return unless uri.is_a?(URI::HTTP) && uri.host
 
@@ -252,6 +310,8 @@ module FetchUtil
 
     def decode_wrapper(source, value)
       uri = URI.parse(value)
+      return decode_yahoo_wrapper(uri, value) if source == "yahoo"
+
       key = source == "google" ? %w[q url] : %w[uddg]
       path = wrapper_path(source)
       return value unless uri.path == path && (uri.host.nil? || wrapper_host?(source, uri.host))
@@ -261,6 +321,21 @@ module FetchUtil
       value
     end
 
+    def decode_yahoo_wrapper(uri, value)
+      return value unless wrapper_host?("yahoo", uri.host)
+
+      segment = uri.path.split("/").find { |part| part.start_with?("RU=") }
+      return value unless segment
+
+      decoded = URI::DEFAULT_PARSER.unescape(segment.delete_prefix("RU="))
+      destination = URI.parse(decoded)
+      return nil unless destination.is_a?(URI::HTTP) && destination.host
+
+      destination.to_s
+    rescue URI::InvalidURIError
+      nil
+    end
+
     def engine_url?(source, value)
       host = URI.parse(value).host
       same_engine_host?(source, host) || wrapper_host?(source, host)
@@ -302,7 +377,8 @@ module FetchUtil
         "bing" => /there are no results|no results found/,
         "duckduckgo" => /no results|no more results/,
         "google" => /did not match any documents|no results found/,
-        "ecosia" => /no results found|we couldn't find/
+        "ecosia" => /no results found|we couldn't find/,
+        "yahoo" => /no results found|we couldn't find|did not match any results/
       }
       text.match?(patterns.fetch(source))
     end
@@ -313,10 +389,17 @@ module FetchUtil
       normalized_text(visible).downcase
     end
 
-    def failure(source, reason, elapsed_ms, final_url)
+    def failure(source, reason, elapsed_ms, final_url, candidates: [])
       reason = reason.to_s
       reason = "failed" unless FAILURE_REASONS.include?(reason)
-      SourceResponse.new(source: source, status: "failed", elapsed_ms: elapsed_ms, final_url: final_url, reason: reason)
+      SourceResponse.new(
+        source: source,
+        status: "failed",
+        candidates: candidates,
+        elapsed_ms: elapsed_ms,
+        final_url: final_url,
+        reason: reason
+      )
     end
 
     def empty(source, elapsed_ms, final_url)

data/lib/fetch_util/searcher.rb

@@ -5,8 +5,9 @@ require "uri"
 
 module FetchUtil
   class Searcher
-    DEFAULT_SOURCES = %w[brave bing].freeze
-
+    DEFAULT_SOURCES = %w[brave bing yahoo].freeze
+    STRUCTURED_QUERY_AUTHORITY = "yahoo"
+    STRUCTURED_QUERY = /"[^"]+"|'[^']+'|\(|\)|\[|\]|\{|\}|(?:\A|\s)[+-]?[[:alpha:]][[:alnum:]_-]*:/
     autoload :ResultFiltering, "fetch_util/searcher/result_filtering"
     include ResultFiltering
     private_constant :ResultFiltering
@@ -14,6 +15,7 @@ module FetchUtil
     def initialize(transport: nil, request_log: RequestLog.new, sources: nil, limit: nil, verbose: false,
                    timeout: SearchTransport::DEFAULT_TIMEOUT)
       @request_log = request_log
+      @sources_explicit = !sources.nil?
       @sources = Array(sources || DEFAULT_SOURCES).map(&:to_s).uniq
       unknown = @sources - SearchTransport::SOURCES.keys
       raise ArgumentError, "unsupported search source: #{unknown.first}" if unknown.any?
@@ -33,7 +35,7 @@ module FetchUtil
 
       payload = {
         query: encoded_query,
-        results: formatted_results(apply_limit(aggregate(responses)))
+        results: formatted_results(apply_limit(aggregate(responses, encoded_query)))
       }
       payload[:diagnostics] = diagnostics(responses) if @verbose
       payload
@@ -58,17 +60,18 @@ module FetchUtil
       "search://#{@sources.join(",")}?q=#{CGI.escape(query)}"
     end
 
-    def aggregate(responses)
+    def aggregate(responses, query)
       parsed = {}
-      max_size = 0
+      structured_query = query.match?(STRUCTURED_QUERY)
 
       @sources.each do |source|
         response = responses.find { |item| item.source == source }
-        items = response ? response.candidates.filter_map { |candidate| normalized_candidate(candidate) } : []
-        parsed[source] = items
-        max_size = [max_size, items.length].max
+        parsed[source] = response ? response.candidates.filter_map { |candidate| normalized_candidate(candidate) } : []
       end
 
+      apply_structured_query_authority!(parsed, responses) if structured_query
+      max_size = parsed.values.map(&:length).max || 0
+
       items = []
       seen = {}
 
@@ -92,6 +95,23 @@ module FetchUtil
       items
     end
 
+    def apply_structured_query_authority!(parsed, responses)
+      return if @sources_explicit
+      return unless @sources.include?(STRUCTURED_QUERY_AUTHORITY)
+
+      authority = responses.find { |response| response.source == STRUCTURED_QUERY_AUTHORITY }
+      return unless authority&.status == "ok" && authority.reason.nil?
+
+      authority_urls = parsed.fetch(STRUCTURED_QUERY_AUTHORITY).map { |item| item[:url] }
+      return if authority_urls.empty?
+
+      @sources.each do |source|
+        next if source == STRUCTURED_QUERY_AUTHORITY
+
+        parsed[source].select! { |item| authority_urls.include?(item[:url]) }
+      end
+    end
+
     def build_result(source, item)
       result = {
         title: item[:title],

data/lib/fetch_util/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module FetchUtil
-  VERSION = "0.5.0"
+  VERSION = "0.5.2"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fetch_util
 version: !ruby/object:Gem::Version
-  version: 0.5.0
+  version: 0.5.2
 platform: ruby
 authors:
 - hmdne
@@ -86,6 +86,7 @@ files:
 - lib/fetch_util/browser/site_stabilization/social_platforms.rb
 - lib/fetch_util/browser/site_stabilization/travel_and_lodging.rb
 - lib/fetch_util/browser/stabilization.rb
+- lib/fetch_util/browser/stabilization/anubis.rb
 - lib/fetch_util/browser/stabilization/page_flow.rb
 - lib/fetch_util/browser/stabilization/spa_hydration.rb
 - lib/fetch_util/cli.rb