fetch_util 0.3.0

data/lib/fetch_util/searcher.rb

@@ -0,0 +1,332 @@
+# frozen_string_literal: true
+
+require "cgi"
+require "uri"
+
+module FetchUtil
+  class Searcher
+    MAX_SNIPPET_LENGTH = 180
+
+    SOURCES = {
+      "duckduckgo" => "https://duckduckgo.com/?q=%<query>s&ia=web&kl=us-en",
+      "google" => "https://www.google.com/search?hl=en&q=%<query>s",
+      "bing" => "https://www.bing.com/search?setlang=en-US&q=%<query>s",
+      "ecosia" => "https://www.ecosia.org/search?q=%<query>s",
+      "brave" => "https://search.brave.com/search?q=%<query>s"
+    }.freeze
+
+    DEFAULT_SOURCES = %w[duckduckgo google].freeze
+
+    autoload :ResultFiltering, "fetch_util/searcher/result_filtering"
+    include ResultFiltering
+    private_constant :ResultFiltering
+
+    def initialize(fetcher: nil, request_log: RequestLog.new, sources: nil, limit: 10, concurrency: 2, verbose: false, **fetch_options)
+      @request_log = request_log
+      @sources = Array(sources || DEFAULT_SOURCES).map(&:to_s)
+      @limit = limit.to_i
+      @verbose = verbose
+      @fetcher = fetcher || ParallelFetcher.new(concurrency: concurrency, request_log: request_log, **fetch_options)
+    end
+
+    def search(query)
+      encoded_query = query.to_s.strip
+      raise ArgumentError, "query must not be empty" if encoded_query.empty?
+
+      urls = search_urls(encoded_query)
+      @request_log.append(search_request_uri(encoded_query))
+      fetched = begin
+        @fetcher.fetch(urls.values)
+      rescue ParallelFetcher::ParallelFetchError => e
+        raise unless e.results&.compact&.any?
+
+        e.results
+      end
+
+      {
+        query: encoded_query,
+        results: formatted_results(aggregate(urls.keys, fetched).first(limit))
+      }
+    end
+
+    private
+
+    attr_reader :limit
+
+    def search_urls(query)
+      urls = {}
+
+      @sources.each do |source|
+        template = SOURCES.fetch(source) do
+          raise ArgumentError, "unsupported search source: #{source}"
+        end
+        urls[source] = format(template, query: CGI.escape(query))
+      end
+
+      urls
+    end
+
+    def search_request_uri(query)
+      "search://#{@sources.join(",")}?q=#{CGI.escape(query)}"
+    end
+
+    def aggregate(sources, fetched)
+      parsed = {}
+      max_size = 0
+
+      sources.zip(fetched).each do |source, result|
+        items = parse_markdown(result.markdown)
+        parsed[source] = items
+        max_size = [max_size, items.length].max
+      end
+
+      items = []
+      seen = {}
+
+      max_size.times do |index|
+        sources.each do |source|
+          item = parsed.fetch(source)[index]
+          next unless item
+
+          item = item.merge(rank: index + 1)
+
+          existing = seen[item[:url]]
+          if existing
+            merge_result!(existing, source, item)
+            next
+          end
+
+          result = build_result(source, item)
+          seen[item[:url]] = result
+          items << result
+        end
+      end
+
+      items
+    end
+
+    def build_result(source, item)
+      result = {
+        title: item[:title],
+        url: item[:url],
+        sources: [source],
+        ranks: { source => item[:rank] }
+      }
+      result[:snippet] = item[:snippet] if item[:snippet]
+      result
+    end
+
+    def merge_result!(result, source, item)
+      result[:sources] << source unless result[:sources].include?(source)
+      result[:ranks][source] ||= item[:rank]
+      return if !item[:snippet] || (result[:snippet] && result[:snippet].length >= item[:snippet].length)
+
+      result[:snippet] = item[:snippet]
+    end
+
+    def formatted_results(results)
+      results.map do |result|
+        item = {
+          title: result[:title],
+          url: result[:url]
+        }
+        item[:snippet] = result[:snippet] if result[:snippet]
+        if verbose?
+          item[:sources] = result[:sources]
+          item[:ranks] = result[:ranks]
+        end
+        item
+      end
+    end
+
+    def parse_markdown(markdown)
+      markdown.to_s.lines.filter_map do |line|
+        parsed = parse_markdown_line(line)
+        next unless parsed
+
+        normalized_item(parsed[:title], parsed[:url], parsed[:snippet])
+      end
+    end
+
+    def parse_markdown_line(line)
+      stripped = line.to_s.strip
+      return nil unless stripped.start_with?("- [")
+
+      title_end = stripped.index("](")
+      return nil unless title_end
+
+      url_start = title_end + 2
+      url_end = markdown_url_end_index(stripped, url_start)
+      return nil unless url_end
+
+      title = stripped[3...title_end]
+      url = stripped[url_start...url_end]
+      remainder = stripped[(url_end + 1)..].to_s
+      snippet = remainder.start_with?(" - ") ? remainder[3..] : nil
+
+      { title: title, url: url, snippet: snippet }
+    end
+
+    def markdown_url_end_index(line, url_start)
+      depth = 0
+
+      url_start.upto(line.length - 1) do |index|
+        char = line[index]
+        if char == "("
+          depth += 1
+        elsif char == ")"
+          return index if depth.zero?
+
+          depth -= 1
+        end
+      end
+
+      nil
+    end
+
+    def normalized_item(title, url, snippet)
+      normalized_url = normalize_url(url)
+      return nil unless normalized_url
+
+      normalized_title = normalize_title(title, normalized_url)
+      return nil if normalized_title.empty? || generic_title?(normalized_title, normalized_url)
+
+      normalized_snippet = normalize_snippet(snippet, normalized_title, normalized_url)
+      return nil if search_engine_self_link?(normalized_title, normalized_url, normalized_snippet)
+      return nil if low_value_result?(normalized_title, normalized_url, normalized_snippet)
+
+      item = {
+        title: normalized_title,
+        url: normalized_url
+      }
+
+      item[:snippet] = normalized_snippet if normalized_snippet
+      item
+    end
+
+    def verbose?
+      @verbose
+    end
+
+    def compact_text(value)
+      FetchUtil.normalize_whitespace(value)
+    end
+
+    def normalize_url(url)
+      parsed = URI.parse(url.to_s.strip)
+      return nil unless parsed.is_a?(URI::HTTP) && parsed.host
+
+      parsed.host = parsed.host.downcase
+      parsed.path = "/" if parsed.path.to_s.empty?
+      parsed.path = parsed.path.sub(%r{/$}, "") unless parsed.path == "/"
+      parsed.fragment = nil unless keep_fragment?(parsed)
+      parsed.to_s
+    rescue URI::InvalidURIError
+      nil
+    end
+
+    def host_for(url)
+      FetchUtil.strip_www_host(url)
+    rescue URI::InvalidURIError
+      nil
+    end
+
+    def path_for(url)
+      URI.parse(url).path.to_s
+    rescue URI::InvalidURIError
+      ""
+    end
+
+    def generic_title?(title, url)
+      return true if title.start_with?("More on ")
+
+      host = host_for(url)
+      return false if host.nil? || host.empty?
+
+      title.casecmp?(host)
+    end
+
+    def normalize_title(title, url)
+      text = compact_text(title)
+      host = host_for(url)
+
+      if host && !host.empty?
+        trimmed = text.sub(/\A#{Regexp.escape(host)}\s+/i, "")
+        text = trimmed if trimmed.length >= 8
+      end
+
+      text = text.sub(/\A(?:[[:alnum:].-]+\s+[>›]\s+)+/, "")
+      text = strip_slug_prefix(text)
+      compact_text(text)
+    end
+
+    def keep_fragment?(uri)
+      fragment = compact_text(uri.fragment)
+      return false if fragment.empty?
+      return false if noise_fragment?(fragment)
+
+      FetchUtil.docs_like_url?(uri)
+    end
+
+    def noise_fragment?(fragment)
+      fragment.match?(/\A(?:top|contents?|content|main|main-content|skip(?:-to)?-(?:content|main)|toc)\z/i)
+    end
+
+    def strip_slug_prefix(text)
+      match = text.match(/\A([a-z0-9-]{4,})\s+(?=[A-Z])/)
+      return text unless match
+
+      prefix = match[1].downcase
+      return text unless prefix.match?(/[-\d]/) || %w[api blog doc docs guide guides help kb learn manual reference tutorial wiki].include?(prefix)
+
+      text.sub(/\A#{Regexp.escape(match[1])}\s+/, "")
+    end
+
+    def normalize_snippet(snippet, title, url)
+      text = compact_text(snippet)
+      text = text.sub(/\A#{Regexp.escape(title)}\s*/i, "")
+      text = text.gsub(%r{https?://\S+}, " ")
+      text = text.sub(/\A[[:word:].-]+\s*(?:[>›]\s*[[:word:]_.()%-]+\s*)+/, "")
+      text = compact_text(text)
+      return nil if text.empty? || text.casecmp?(title)
+
+      host = host_for(url)
+      return nil if domain_only?(text, host)
+      return nil if breadcrumb_text?(text)
+      return nil if metadata_only_snippet?(text)
+      return nil if jammed_navigation_text?(text)
+
+      truncate(text, MAX_SNIPPET_LENGTH)
+    end
+
+    def truncate(text, max_length)
+      return text if text.length <= max_length
+
+      "#{text[0, max_length - 3].rstrip}..."
+    end
+
+    def domain_only?(text, host)
+      return true if text.match?(/\A[a-z0-9.-]+\.[a-z]{2,}\z/i)
+      return false if host.nil? || host.empty?
+
+      text.casecmp?(host)
+    end
+
+    def jammed_navigation_text?(text)
+      text.length > 20 && !text.include?(" ") && text.scan(/[A-Z]/).length >= 3
+    end
+
+    def breadcrumb_text?(text)
+      text.include?("›") || text.match?(/(?:\A|\s)>|>\s/)
+    end
+
+    def metadata_only_snippet?(text)
+      normalized = text.gsub(/([[:alpha:]])(\d)/, '\1 \2').gsub(/(\d)([[:alpha:]])/, '\1 \2')
+      site = normalized.match?(/\A(?:Reddit|Stack Overflow|Medium)\b/i)
+      counters = normalized.match?(/\b\d+\+?\s*(?:comments?|answers?|likes?)\b/i)
+      age = normalized.match?(/\b\d+\s*(?:years?|months?|days?|hours?|minutes?)\s+ago\b/i)
+
+      site && (counters || age)
+    end
+  end
+end

data/lib/fetch_util/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module FetchUtil
+  VERSION = "0.3.0"
+end

data/lib/fetch_util.rb

@@ -0,0 +1,115 @@
+# frozen_string_literal: true
+
+require "uri"
+
+require_relative "fetch_util/version"
+
+module FetchUtil
+  class Error < StandardError; end
+  class BrowserError < Error; end
+  class ExtractionError < Error; end
+
+  DOCS_LIKE_EXACT_HOSTS = %w[
+    developer.mozilla.org
+    doc.rust-lang.org
+    docs.rs
+    fastapi.tiangolo.com
+    learn.microsoft.com
+    ncbi.nlm.nih.gov
+    nextjs.org
+    pkg.go.dev
+    platform.claude.com
+    react.dev
+    rubydoc.info
+    rubyapi.org
+  ].freeze
+  DOCS_LIKE_PATH_KEYWORDS = %w[
+    api
+    book
+    books
+    concept
+    concepts
+    definition
+    definitions
+    dictionary
+    doc
+    docs
+    guide
+    guides
+    howto
+    library
+    libraries
+    manual
+    reference
+    sdk
+    tutorial
+  ].freeze
+  DOCS_LIKE_PATH_PATTERN = %r{/
+    (?:
+      docs?|reference|api(?:/reference)?|tutorial|guide|guides|library|libraries|
+      book|books|dictionary|definition|definitions|concept|concepts|
+      get(?:ting)?-started|quick-start|how-to|howto|manual|sdk|learn
+    )
+    (?:/|\b)
+  }x
+
+  autoload :Browser, "fetch_util/browser"
+  autoload :CLI, "fetch_util/cli"
+  autoload :Extractor, "fetch_util/extractor"
+  autoload :Fetcher, "fetch_util/fetcher"
+  autoload :ParallelFetcher, "fetch_util/parallel_fetcher"
+  autoload :Regulatory, "fetch_util/regulatory"
+  autoload :RawDocsFallback, "fetch_util/raw_docs_fallback"
+  autoload :RequestLog, "fetch_util/request_log"
+  autoload :Result, "fetch_util/result"
+  autoload :Searcher, "fetch_util/searcher"
+
+  module_function
+
+  def fetch(url, **options)
+    Fetcher.new(**options).fetch(url)
+  end
+
+  def fetch_many(urls, **options)
+    ParallelFetcher.new(**options).fetch(urls)
+  end
+
+  def search(query, **options)
+    Searcher.new(**options).search(query)
+  end
+
+  def regulatory(url, **options)
+    Regulatory.new(**options).call(url)
+  end
+
+  def normalize_whitespace(value)
+    text = value.to_s
+    text = text.encode("UTF-8", invalid: :replace, undef: :replace, replace: " ") unless text.encoding == Encoding::UTF_8 && text.valid_encoding?
+    text.gsub(/\u00A0/, " ").gsub(/\s+/, " ").strip
+  end
+
+  def strip_www_host(url)
+    URI.parse(url.to_s).host.to_s.downcase.sub(/\Awww\./, "")
+  end
+
+  def docs_like_url?(value)
+    uri = value.is_a?(URI::Generic) ? value : URI.parse(value.to_s.strip)
+    return false unless uri.is_a?(URI::HTTP) && uri.host
+
+    host = strip_www_host(uri)
+    path = uri.path.to_s.downcase
+    path_terms = path.split(/[^a-z0-9]+/)
+
+    return true if DOCS_LIKE_EXACT_HOSTS.include?(host)
+    return true if host.end_with?(".readthedocs.io")
+    return true if host.start_with?("docs.") || host.start_with?("developer.") || host.start_with?("developers.") || host.start_with?("api.")
+    return true if host.match?(/\b(?:dictionary|merriam-webster|thefreedictionary|wiktionary|collinsdictionary|reverso)\b/)
+    return true if host == "go.dev" && path.match?(%r{\A/ref(?:/|\b)})
+    return true if path.match?(DOCS_LIKE_PATH_PATTERN)
+    return true if (path_terms & DOCS_LIKE_PATH_KEYWORDS).any?
+
+    false
+  rescue URI::InvalidURIError
+    false
+  end
+end

metadata

@@ -0,0 +1,145 @@
+--- !ruby/object:Gem::Specification
+name: fetch_util
+version: !ruby/object:Gem::Version
+  version: 0.3.0
+platform: ruby
+authors:
+- hmdne
+bindir: exe
+cert_chain: []
+date: 1980-01-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: ferrum
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.17'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.17'
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.19'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.19'
+- !ruby/object:Gem::Dependency
+  name: thor
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+description: An intelligent web-fetch engine for Ruby that renders live pages, recognizes
+  what they are, and turns them into clean, usable markdown.
+email:
+- 54514036+hmdne@users.noreply.github.com
+executables:
+- fetch_util
+extensions: []
+extra_rdoc_files: []
+files:
+- ".rspec"
+- ".rubocop.yml"
+- CHANGELOG.md
+- LICENSE.txt
+- README.md
+- Rakefile
+- SKILL.md
+- exe/fetch_util
+- lib/fetch_util.rb
+- lib/fetch_util/assets/extract.js
+- lib/fetch_util/assets/vendor/readability.js
+- lib/fetch_util/assets/vendor/turndown.js
+- lib/fetch_util/browser.rb
+- lib/fetch_util/browser/interaction_helpers.rb
+- lib/fetch_util/browser/interaction_helpers/consent_helpers.rb
+- lib/fetch_util/browser/interaction_helpers/dom_interaction.rb
+- lib/fetch_util/browser/interaction_helpers/timing_helpers.rb
+- lib/fetch_util/browser/navigation.rb
+- lib/fetch_util/browser/navigation/headers_and_readiness.rb
+- lib/fetch_util/browser/navigation/navigator_patch.rb
+- lib/fetch_util/browser/site_stabilization.rb
+- lib/fetch_util/browser/site_stabilization/community_and_marketplace.rb
+- lib/fetch_util/browser/site_stabilization/social_platforms.rb
+- lib/fetch_util/browser/stabilization.rb
+- lib/fetch_util/browser/stabilization/page_flow.rb
+- lib/fetch_util/browser/stabilization/spa_hydration.rb
+- lib/fetch_util/cli.rb
+- lib/fetch_util/extractor.rb
+- lib/fetch_util/fetcher.rb
+- lib/fetch_util/parallel_fetcher.rb
+- lib/fetch_util/raw_docs_fallback.rb
+- lib/fetch_util/regulatory.rb
+- lib/fetch_util/regulatory/cache_store.rb
+- lib/fetch_util/regulatory/directives.rb
+- lib/fetch_util/regulatory/fetch_records.rb
+- lib/fetch_util/regulatory/headers.rb
+- lib/fetch_util/regulatory/http_client.rb
+- lib/fetch_util/regulatory/human.rb
+- lib/fetch_util/regulatory/orchestration.rb
+- lib/fetch_util/regulatory/page.rb
+- lib/fetch_util/regulatory/robot_globs.rb
+- lib/fetch_util/regulatory/robots.rb
+- lib/fetch_util/regulatory/signals.rb
+- lib/fetch_util/regulatory/source_selection.rb
+- lib/fetch_util/regulatory/tdm_page.rb
+- lib/fetch_util/regulatory/tdm_policy.rb
+- lib/fetch_util/regulatory/tdm_rep.rb
+- lib/fetch_util/regulatory/tdm_support.rb
+- lib/fetch_util/regulatory/trust_txt.rb
+- lib/fetch_util/regulatory/usage_preferences.rb
+- lib/fetch_util/request_log.rb
+- lib/fetch_util/result.rb
+- lib/fetch_util/searcher.rb
+- lib/fetch_util/searcher/result_filtering.rb
+- lib/fetch_util/version.rb
+homepage: https://github.com/rbutils/fetch_util
+licenses:
+- MIT
+metadata:
+  allowed_push_host: https://rubygems.org
+  source_code_uri: https://github.com/rbutils/fetch_util
+  changelog_uri: https://github.com/rbutils/fetch_util/blob/master/CHANGELOG.md
+  documentation_uri: https://github.com/rbutils/fetch_util#readme
+  bug_tracker_uri: https://github.com/rbutils/fetch_util/issues
+  rubygems_mfa_required: 'true'
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 3.2.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 4.0.10
+specification_version: 4
+summary: AI for fetching in Ruby
+test_files: []