fetch_util 0.4.0 → 0.5.1

data/lib/fetch_util/searcher.rb

@@ -5,46 +5,40 @@ require "uri"
 
 module FetchUtil
   class Searcher
-    SOURCES = {
-      "duckduckgo" => "https://duckduckgo.com/?q=%<query>s&ia=web&kl=us-en",
-      "google" => "https://www.google.com/search?hl=en&q=%<query>s",
-      "bing" => "https://www.bing.com/search?setlang=en-US&q=%<query>s",
-      "ecosia" => "https://www.ecosia.org/search?q=%<query>s",
-      "brave" => "https://search.brave.com/search?q=%<query>s"
-    }.freeze
-
-    DEFAULT_SOURCES = %w[duckduckgo google].freeze
-
+    DEFAULT_SOURCES = %w[brave bing yahoo].freeze
+    STRUCTURED_QUERY_AUTHORITY = "yahoo"
+    STRUCTURED_QUERY = /"[^"]+"|'[^']+'|\(|\)|\[|\]|\{|\}|(?:\A|\s)[+-]?[[:alpha:]][[:alnum:]_-]*:/
     autoload :ResultFiltering, "fetch_util/searcher/result_filtering"
     include ResultFiltering
     private_constant :ResultFiltering
 
-    def initialize(fetcher: nil, request_log: RequestLog.new, sources: nil, limit: nil, concurrency: 2, verbose: false, **fetch_options)
+    def initialize(transport: nil, request_log: RequestLog.new, sources: nil, limit: nil, verbose: false,
+                   timeout: SearchTransport::DEFAULT_TIMEOUT)
       @request_log = request_log
-      @sources = Array(sources || DEFAULT_SOURCES).map(&:to_s)
+      @sources_explicit = !sources.nil?
+      @sources = Array(sources || DEFAULT_SOURCES).map(&:to_s).uniq
+      unknown = @sources - SearchTransport::SOURCES.keys
+      raise ArgumentError, "unsupported search source: #{unknown.first}" if unknown.any?
+
+      validate_limit!(limit)
       @limit = limit
       @verbose = verbose
-      @fetcher = fetcher || ParallelFetcher.new(concurrency: concurrency, request_log: request_log, **fetch_options)
+      @transport = transport || SearchTransport.new(sources: @sources, timeout: timeout)
     end
 
     def search(query)
       encoded_query = query.to_s.strip
       raise ArgumentError, "query must not be empty" if encoded_query.empty?
 
-      urls = search_urls(encoded_query)
       @request_log.append(search_request_uri(encoded_query))
-      fetched = begin
-        @fetcher.fetch(urls.values)
-      rescue ParallelFetcher::ParallelFetchError => e
-        raise unless e.results&.compact&.any?
-
-        e.results
-      end
+      responses = @transport.search(encoded_query)
 
-      {
+      payload = {
         query: encoded_query,
-        results: formatted_results(apply_limit(aggregate(urls.keys, fetched)))
+        results: formatted_results(apply_limit(aggregate(responses, encoded_query)))
       }
+      payload[:diagnostics] = diagnostics(responses) if @verbose
+      payload
     end
 
     private
@@ -55,43 +49,37 @@ module FetchUtil
       limit.nil? ? results : results.first(limit)
     end
 
-    def search_urls(query)
-      urls = {}
-
-      @sources.each do |source|
-        template = SOURCES.fetch(source) do
-          raise ArgumentError, "unsupported search source: #{source}"
-        end
-        urls[source] = format(template, query: CGI.escape(query))
-      end
+    def validate_limit!(value)
+      return if value.nil?
+      return if value.is_a?(Integer) && value >= 0
 
-      urls
+      raise ArgumentError, "limit must be a nonnegative integer"
     end
 
     def search_request_uri(query)
       "search://#{@sources.join(",")}?q=#{CGI.escape(query)}"
     end
 
-    def aggregate(sources, fetched)
+    def aggregate(responses, query)
       parsed = {}
-      max_size = 0
+      structured_query = query.match?(STRUCTURED_QUERY)
 
-      sources.zip(fetched).each do |source, result|
-        items = parse_markdown(result.markdown)
-        parsed[source] = items
-        max_size = [max_size, items.length].max
+      @sources.each do |source|
+        response = responses.find { |item| item.source == source }
+        parsed[source] = response ? response.candidates.filter_map { |candidate| normalized_candidate(candidate) } : []
       end
 
+      apply_structured_query_authority!(parsed, responses) if structured_query
+      max_size = parsed.values.map(&:length).max || 0
+
       items = []
       seen = {}
 
       max_size.times do |index|
-        sources.each do |source|
+        @sources.each do |source|
           item = parsed.fetch(source)[index]
           next unless item
 
-          item = item.merge(rank: index + 1)
-
           existing = seen[item[:url]]
           if existing
             merge_result!(existing, source, item)
@@ -107,6 +95,23 @@ module FetchUtil
       items
     end
 
+    def apply_structured_query_authority!(parsed, responses)
+      return if @sources_explicit
+      return unless @sources.include?(STRUCTURED_QUERY_AUTHORITY)
+
+      authority = responses.find { |response| response.source == STRUCTURED_QUERY_AUTHORITY }
+      return unless authority&.status == "ok" && authority.reason.nil?
+
+      authority_urls = parsed.fetch(STRUCTURED_QUERY_AUTHORITY).map { |item| item[:url] }
+      return if authority_urls.empty?
+
+      @sources.each do |source|
+        next if source == STRUCTURED_QUERY_AUTHORITY
+
+        parsed[source].select! { |item| authority_urls.include?(item[:url]) }
+      end
+    end
+
     def build_result(source, item)
       result = {
         title: item[:title],
@@ -121,7 +126,7 @@ module FetchUtil
     def merge_result!(result, source, item)
       result[:sources] << source unless result[:sources].include?(source)
       result[:ranks][source] ||= item[:rank]
-      return if !item[:snippet] || (result[:snippet] && result[:snippet].length >= item[:snippet].length)
+      return if result[:snippet] || !item[:snippet]
 
       result[:snippet] = item[:snippet]
     end
@@ -141,65 +146,39 @@ module FetchUtil
       end
     end
 
-    def parse_markdown(markdown)
-      markdown.to_s.lines.filter_map do |line|
-        parsed = parse_markdown_line(line)
-        next unless parsed
-
-        normalized_item(parsed[:title], parsed[:url], parsed[:snippet])
-      end
-    end
-
-    def parse_markdown_line(line)
-      stripped = line.to_s.strip
-      return nil unless stripped.start_with?("- [")
-
-      title_end = stripped.index("](")
-      return nil unless title_end
-
-      url_start = title_end + 2
-      url_end = markdown_url_end_index(stripped, url_start)
-      return nil unless url_end
-
-      title = stripped[3...title_end]
-      url = stripped[url_start...url_end]
-      remainder = stripped[(url_end + 1)..].to_s
-      snippet = remainder.start_with?(" - ") ? remainder[3..] : nil
-
-      { title: title, url: url, snippet: snippet }
-    end
-
-    def markdown_url_end_index(line, url_start)
-      depth = 0
-
-      url_start.upto(line.length - 1) do |index|
-        char = line[index]
-        if char == "("
-          depth += 1
-        elsif char == ")"
-          return index if depth.zero?
-
-          depth -= 1
-        end
+    def diagnostics(responses)
+      @sources.filter_map do |source|
+        response = responses.find { |item| item.source == source }
+        next unless response
+
+        diagnostic = {
+          source: response.source,
+          transport: response.transport,
+          status: response.status,
+          result_count: response.candidates.length,
+          elapsed_ms: response.elapsed_ms
+        }
+        diagnostic[:final_url] = response.final_url if response.final_url
+        diagnostic[:reason] = response.reason if response.reason
+        diagnostic
       end
-
-      nil
     end
 
-    def normalized_item(title, url, snippet)
-      normalized_url = normalize_url(url)
+    def normalized_candidate(candidate)
+      normalized_url = normalize_url(candidate.url)
       return nil unless normalized_url
 
-      normalized_title = normalize_title(title, normalized_url)
+      normalized_title = normalize_title(candidate.title, normalized_url)
       return nil if normalized_title.empty? || generic_title?(normalized_title, normalized_url)
 
-      normalized_snippet = normalize_snippet(snippet, normalized_title, normalized_url)
+      normalized_snippet = normalize_snippet(candidate.snippet, normalized_title, normalized_url)
       return nil if search_engine_self_link?(normalized_title, normalized_url, normalized_snippet)
       return nil if low_value_result?(normalized_title, normalized_url, normalized_snippet)
 
       item = {
         title: normalized_title,
-        url: normalized_url
+        url: normalized_url,
+        rank: candidate.source_rank
       }
 
       item[:snippet] = normalized_snippet if normalized_snippet

data/lib/fetch_util/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module FetchUtil
-  VERSION = "0.4.0"
+  VERSION = "0.5.1"
 end

data/lib/fetch_util.rb

@@ -64,6 +64,7 @@ module FetchUtil
   autoload :RequestLog, "fetch_util/request_log"
   autoload :Result, "fetch_util/result"
   autoload :Searcher, "fetch_util/searcher"
+  autoload :SearchTransport, "fetch_util/search_transport"
 
   module_function
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fetch_util
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.5.1
 platform: ruby
 authors:
 - hmdne
@@ -86,6 +86,7 @@ files:
 - lib/fetch_util/browser/site_stabilization/social_platforms.rb
 - lib/fetch_util/browser/site_stabilization/travel_and_lodging.rb
 - lib/fetch_util/browser/stabilization.rb
+- lib/fetch_util/browser/stabilization/anubis.rb
 - lib/fetch_util/browser/stabilization/page_flow.rb
 - lib/fetch_util/browser/stabilization/spa_hydration.rb
 - lib/fetch_util/cli.rb
@@ -114,6 +115,7 @@ files:
 - lib/fetch_util/regulatory/usage_preferences.rb
 - lib/fetch_util/request_log.rb
 - lib/fetch_util/result.rb
+- lib/fetch_util/search_transport.rb
 - lib/fetch_util/searcher.rb
 - lib/fetch_util/searcher/result_filtering.rb
 - lib/fetch_util/version.rb