pikuri-core 0.0.3

data/lib/pikuri/tool/scraper/simple.rb

@@ -0,0 +1,183 @@
+# frozen_string_literal: true
+
+require 'faraday'
+require 'uri'
+
+module Pikuri
+  class Tool
+    # Namespace for the URL-to-Markdown scraping stack used by
+    # {Tool::WEB_SCRAPE} and {Tool::FETCH}: a content-type-dispatching
+    # fetcher ({Simple}), pure content extractors ({HTML}, {PDF}), and a
+    # shared error type ({FetchError}). Nothing here knows about the LLM
+    # — the tools that wrap these layers turn rendered Markdown (or
+    # +FetchError+) into the next observation.
+    module Scraper
+      # Plain HTTP scraper: GET the URL with a real-browser User-Agent,
+      # follow redirects, and dispatch the response body to the parser
+      # matching its +Content-Type+. HTML and XHTML route to
+      # {HTML.extract}; +application/pdf+ routes to {PDF.extract}; any
+      # other +text/*+ type (plain text, Markdown, source files, …) is
+      # passed through verbatim since the LLM can already read it; the
+      # remaining types raise {FetchError} so the LLM observes the
+      # failure instead of receiving an empty rendering.
+      #
+      # Split into a thin HTTP fetch ({.fetch}) and a content-type
+      # dispatcher ({.visit}) so tests can drive each piece in isolation.
+      # "Simple" because everything happens in one Faraday GET — no
+      # headless browser, no JS execution.
+      module Simple
+        # @return [String] User-Agent sent with each request; many sites
+        #   reject requests with no UA or an obvious bot UA
+        USER_AGENT = 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 ' \
+                     '(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
+        # @return [String] +Accept+ header sent with each request. Lists
+        #   every content-type the dispatcher in {.visit} knows how to
+        #   render, so servers that content-negotiate hand back something
+        #   we can use. The trailing +text/*;q=0.8+ covers the verbatim
+        #   pass-through arm (plain text, Markdown, source files, …) at a
+        #   lower preference than rendered HTML/PDF.
+        ACCEPT = 'text/html,application/xhtml+xml,application/pdf,text/*;q=0.8'
+        # @return [Integer] maximum number of HTTP redirects to follow
+        #   before giving up
+        MAX_REDIRECTS = 5
+        # @return [Integer] connect timeout in seconds for the underlying
+        #   Faraday request
+        OPEN_TIMEOUT = 10
+        # @return [Integer] read timeout in seconds for the underlying
+        #   Faraday request
+        READ_TIMEOUT = 20
+
+        # @return [Integer] maximum number of characters of an error
+        #   response body to include in a {FetchError} message. The body is
+        #   often a multi-kilobyte HTML challenge page (Cloudflare, WAF
+        #   interstitial, etc.); a short excerpt tells the LLM what kind of
+        #   page came back without flooding the next observation.
+        ERROR_BODY_EXCERPT = 200
+
+        # Result of a successful {Simple.fetch}: the response body, the
+        # normalized content-type (lower-cased, with any +; charset=...+
+        # parameters stripped), and the final URL after redirects. The
+        # final URL is kept so future scrapers can resolve relative links
+        # against the actual landing page rather than the originally
+        # requested one.
+        Fetched = Data.define(:body, :content_type, :url)
+
+        # Fetch +url+ and render its main content as Markdown.
+        #
+        # No caching here — every call hits the network. Callers that want
+        # to memoize results should wrap this method themselves (see
+        # {Tool::WebScrape.visit}, which does exactly that).
+        #
+        # The dispatcher's output is +String#strip+'d so the LLM never
+        # sees a body that opens or closes with blank lines — common with
+        # +pdf-reader+'s page-feed whitespace and with text bodies that
+        # carry a trailing newline. Interior whitespace is preserved
+        # because Markdown paragraph breaks and source-code indentation
+        # are load-bearing.
+        #
+        # @param url [String] absolute HTTP(S) URL of the page to download
+        # @return [String] full Markdown representation of the page with
+        #   leading/trailing whitespace trimmed, uncapped otherwise —
+        #   caller is responsible for any size limiting before feeding
+        #   the result back to the LLM
+        # @raise [FetchError] on HTTP non-2xx, network failure, redirect
+        #   loop, a 3xx without a +Location+ header, or a response whose
+        #   content-type the dispatcher does not recognize
+        def self.visit(url)
+          dispatch(fetch(url)).strip
+        end
+
+        # Download the body of +url+, manually following up to
+        # {MAX_REDIRECTS} redirects. Faraday is configured with no
+        # middleware so behavior here mirrors the rest of the codebase
+        # (see +Tool::Search::DuckDuckGo.search+).
+        #
+        # All recoverable failures — HTTP 4xx/5xx, +Faraday::Error+ network
+        # blips, exhausted redirect budget, 3xx without a +Location+ —
+        # surface as {FetchError} so the caller has a single exception type
+        # to rescue. Error bodies are trimmed to {ERROR_BODY_EXCERPT}
+        # characters with whitespace collapsed, so a Cloudflare-challenge
+        # response doesn't dump kilobytes of inline HTML into the next LLM
+        # observation.
+        #
+        # @param url [String] absolute HTTP(S) URL to fetch
+        # @param limit [Integer] redirects remaining; recurses with
+        #   +limit - 1+ on each 3xx
+        # @return [Fetched] body, normalized content-type, and final URL
+        #   after redirects
+        # @raise [FetchError] on non-2xx/3xx responses, network errors,
+        #   redirect-loop exhaustion, or 3xx without a +Location+ header
+        def self.fetch(url, limit: MAX_REDIRECTS)
+          raise FetchError, "too many redirects fetching #{url}" if limit.zero?
+
+          response = begin
+            Faraday.new(request: { open_timeout: OPEN_TIMEOUT, timeout: READ_TIMEOUT }).get(url) do |req|
+              req.headers['User-Agent'] = USER_AGENT
+              req.headers['Accept']     = ACCEPT
+            end
+          rescue Faraday::Error => e
+            raise FetchError, "#{e.class.name.split('::').last} fetching #{url}: #{e.message}"
+          end
+
+          case response.status
+          when 200..299
+            Fetched.new(body: response.body, content_type: normalize_content_type(response.headers['content-type']), url: url)
+          when 300..399
+            location = response.headers['location']
+            raise FetchError, "HTTP #{response.status} from #{url} with no Location header" if location.nil? || location.empty?
+
+            fetch(URI.join(url, location).to_s, limit: limit - 1)
+          else
+            raise FetchError, "HTTP #{response.status} fetching #{url}: #{excerpt(response.body)}"
+          end
+        end
+
+        # Route a {Fetched} response to the parser that matches its
+        # content-type. Unknown types raise {FetchError} so the LLM gets a
+        # legible observation instead of an empty string.
+        #
+        # @param fetched [Fetched]
+        # @return [String] Markdown representation produced by the matched
+        #   parser
+        # @raise [FetchError] when no parser matches the response's
+        #   content-type
+        def self.dispatch(fetched)
+          case fetched.content_type
+          when 'text/html', 'application/xhtml+xml'
+            HTML.extract(fetched.body)
+          when 'application/pdf'
+            PDF.extract(fetched.body)
+          when %r{\Atext/}
+            fetched.body
+          else
+            raise FetchError, "unsupported content-type #{fetched.content_type.inspect} for #{fetched.url}"
+          end
+        end
+
+        # Lower-case +raw+ and strip any +; charset=...+ parameters so the
+        # dispatcher can match on a canonical token.
+        #
+        # @param raw [String, nil] raw +Content-Type+ header value
+        # @return [String] normalized content-type, or +""+ when the
+        #   header was missing
+        def self.normalize_content_type(raw)
+          raw.to_s.split(';').first.to_s.strip.downcase
+        end
+        private_class_method :normalize_content_type
+
+        # Whitespace-collapse +body+ and clip to {ERROR_BODY_EXCERPT}
+        # characters, so the {FetchError} message stays a single readable
+        # line even when the server returned a multi-KB HTML challenge
+        # page.
+        #
+        # @param body [String, nil]
+        # @return [String]
+        def self.excerpt(body)
+          text = body.to_s.gsub(/\s+/, ' ').strip
+          text.length > ERROR_BODY_EXCERPT ? "#{text[0, ERROR_BODY_EXCERPT]}..." : text
+        end
+        private_class_method :excerpt
+      end
+    end
+  end
+end

data/lib/pikuri/tool/search/brave.rb

@@ -0,0 +1,184 @@
+# frozen_string_literal: true
+
+require 'faraday'
+require 'json'
+require 'nokogiri'
+
+module Pikuri
+  class Tool
+    module Search
+      # Performs a Brave Search via the official Web Search API and returns
+      # the hits as a list of {Result} rows. Split into a thin HTTP fetch
+      # (#search) and a pure parser (#parse) so tests can exercise the
+      # parser against fixture JSON without hitting the network. The
+      # cascade in {Engines.search} owns the final Markdown rendering.
+      #
+      # Requires a Brave Search API key. Get one at
+      # https://api-dashboard.search.brave.com — the free "Data for Search"
+      # tier allows 1 query/sec and ~2k queries/month.
+      #
+      # == Privacy posture
+      #
+      # Brave's API Privacy Notice retains Search Query Logs for 90 days
+      # (billing / troubleshooting) and states +Brave does not collect any
+      # identifiers that can link a search query to an individual or their
+      # devices+. Brave publicly commits that the Search API does not use
+      # query data to train its own models, and offers Zero Data Retention
+      # — but only on the Enterprise plan, not on the free "Data for
+      # Search" tier pikuri defaults to.
+      #
+      # Bottom line: of pikuri's three providers Brave has the cleanest
+      # API-level posture — no training-on-queries, no IP linkage, capped
+      # 90-day retention by default, real ZDR if you pay for it. Still a
+      # logged 90-day window on the cheap tier, so not a substitute for
+      # ZDR for genuinely sensitive queries.
+      module Brave
+        # @return [String] Web Search endpoint
+        ENDPOINT = 'https://api.search.brave.com/res/v1/web/search'
+        # @return [Integer] default number of results returned, matching
+        #   {DuckDuckGo::DEFAULT_MAX_RESULTS}
+        DEFAULT_MAX_RESULTS = 10
+        # @return [String] env var holding the API key; +X-Subscription-Token+
+        ENV_KEY = 'BRAVE_SEARCH_API_KEY'
+        # @return [RateLimiter] free-tier Brave caps at 1 req/sec; the
+        #   5-minute cooldown protects the limited monthly quota from
+        #   being burned on doomed retries when a 429 hits.
+        LIMITER = RateLimiter.new(min_interval: 1.0, cooldown: 300.0)
+
+        # Fetch results for +query+ and return them as an +Array<Result>+.
+        # Calls are throttled to one per second and circuit-broken for 5
+        # minutes on rate-limit / quota-exhausted responses; see {LIMITER}.
+        # The caller (typically {Engines.search}) is expected to have
+        # already normalized the query and to wrap this in a result cache.
+        #
+        # @param query [String] search query (already normalized)
+        # @param max_results [Integer] maximum number of result entries;
+        #   passed through as Brave's +count+ (1..20)
+        # @param api_key [String] Brave Search subscription token; defaults to
+        #   the {ENV_KEY} environment variable
+        # @return [Array<Result>] hits, possibly empty when Brave ran the
+        #   query and matched nothing
+        # @raise [ArgumentError] if no API key is available
+        # @raise [Engines::Unavailable] when Brave returns HTTP 429
+        #   (rate limit / quota exhausted) or 5xx — "try again later"
+        #   responses the cascade in {Engines.search} can fall back
+        #   from. Also raised immediately if {LIMITER} is in cooldown.
+        #   Other non-2xx (e.g. 401/403 from a bad API key) bubble up as
+        #   +RuntimeError+ so config problems stay visible.
+        # @raise [RuntimeError] for non-rate-limit HTTP failures or when the
+        #   response shape contains no results.
+        def self.search(query, max_results: DEFAULT_MAX_RESULTS, api_key: ENV.fetch(ENV_KEY, nil))
+          raise ArgumentError, "Brave Search API key not set (#{ENV_KEY})" if api_key.to_s.strip.empty?
+
+          LIMITER.call do
+            response = Faraday.get(
+              ENDPOINT,
+              { q: query, count: max_results },
+              { 'X-Subscription-Token' => api_key, 'Accept' => 'application/json' }
+            )
+            unless response.success?
+              if response.status == 429 || response.status >= 500
+                raise Engines::Unavailable, "HTTP #{response.status}"
+              end
+
+              raise "Brave Search request failed: #{response.status} #{response.body}"
+            end
+
+            parse(response.body, max_results: max_results)
+          end
+        end
+
+        # Parse a Brave Web Search JSON response into a list of {Result} rows.
+        # HTML highlight tags (+<strong>+) inside +title+ and +description+
+        # are stripped via Nokogiri so the output is plain text.
+        #
+        # When the response yields zero result nodes, two cases are
+        # distinguished: a genuine "no results" payload (recognized search
+        # shape with empty +mixed.main+/+top+/+side+ — typically a too-narrow
+        # query Brave couldn't match) returns an empty array instead of
+        # raising, so {Engines.search} can render its standard no-results
+        # stub. Anything else (unknown layout, structured error) raises
+        # with a diagnostic so the failure surfaces.
+        #
+        # @param json [String] response body from {ENDPOINT}
+        # @param max_results [Integer] maximum number of result entries
+        # @return [Array<Result>] hits, possibly empty on a recognized
+        #   empty-results payload
+        # @raise [RuntimeError] when the response yields no result entries and
+        #   is not recognized as a genuine empty-results payload
+        def self.parse(json, max_results: DEFAULT_MAX_RESULTS)
+          data = JSON.parse(json)
+          results = Array(data.dig('web', 'results')).take(max_results).filter_map do |r|
+            href = r['url'].to_s
+            next nil if href.empty?
+
+            Result.new(
+              url: href,
+              title: strip_html(r['title']),
+              body: strip_html(r['description'])
+            )
+          end
+
+          if results.empty?
+            return [] if genuine_no_results?(data)
+
+            raise diagnose_empty(data, json)
+          end
+
+          results
+        end
+
+        # Strip HTML markup (notably +<strong>+ highlights Brave wraps around
+        # query terms in titles and descriptions) and collapse whitespace.
+        #
+        # @param html [String, nil] raw text from a Brave result field
+        # @return [String] plain text with tags removed; empty string for nil
+        def self.strip_html(html)
+          return '' if html.nil?
+
+          Nokogiri::HTML.fragment(html).text.gsub(/\s+/, ' ').strip
+        end
+        private_class_method :strip_html
+
+        # True when a parsed response with zero +web.results+ entries looks
+        # like Brave's own "search ran, nothing matched" payload (typically a
+        # too-narrow query) rather than a malformed or error response. The
+        # markers are the recognized +type: "search"+ envelope and an empty
+        # +mixed+ block — Brave populates +mixed.main/top/side+ with whichever
+        # verticals matched, so all three being empty arrays is the canonical
+        # signal that the search itself succeeded but found nothing.
+        #
+        # @param data [Hash, Object] parsed response
+        # @return [Boolean]
+        def self.genuine_no_results?(data)
+          return false unless data.is_a?(Hash) && data['type'] == 'search'
+
+          mixed = data['mixed']
+          return false unless mixed.is_a?(Hash)
+
+          %w[main top side].all? { |k| Array(mixed[k]).empty? }
+        end
+        private_class_method :genuine_no_results?
+
+        # Build an error message for a parsed response that yielded zero
+        # results. Quotes Brave's +error.detail+ if present, otherwise
+        # truncates the raw body so the caller can see the actual payload.
+        #
+        # @param data [Hash, Object] parsed response
+        # @param raw [String] raw response body
+        # @return [String] human-readable diagnostic to feed to +raise+
+        def self.diagnose_empty(data, raw)
+          if data.is_a?(Hash) && data['error'].is_a?(Hash)
+            err = data['error']
+            return "Brave Search returned an error: #{[err['code'], err['detail'] || err['meta']].compact.join(' — ')}"
+          end
+
+          snippet = raw.to_s[0, 800]
+          snippet += '…' if raw.to_s.length > 800
+          "Brave Search returned no results. Body: #{snippet}"
+        end
+        private_class_method :diagnose_empty
+      end
+    end
+  end
+end

data/lib/pikuri/tool/search/duckduckgo.rb

@@ -0,0 +1,196 @@
+# frozen_string_literal: true
+
+require 'faraday'
+require 'nokogiri'
+require 'uri'
+
+module Pikuri
+  class Tool
+    module Search
+      # Performs a DuckDuckGo search by scraping +html.duckduckgo.com+ and
+      # returns the hits as a list of {Result} rows. Split into a thin HTTP
+      # fetch (#search) and a pure parser (#parse) so tests can exercise
+      # the parser against fixture HTML without hitting the network. The
+      # cascade in {Engines.search} owns the final Markdown rendering.
+      #
+      # == Privacy posture
+      #
+      # DuckDuckGo's privacy policy states +We don't save your IP address
+      # or any unique identifiers alongside your searches+ and +We have
+      # never sold any personal information+, and they proxy requests on
+      # the user's behalf so downstream content providers can't build a
+      # per-user search history. That part is real — but DDG is mainly a
+      # relay over Bing for web results, so the *query content* still
+      # reaches Microsoft for fulfillment even though DDG strips
+      # identifying info on the way out.
+      #
+      # Bottom line: DDG is a genuine privacy improvement over hitting
+      # Bing directly (your IP isn't tied to the query, no per-user
+      # profile is built on DDG's side), but query content still lands at
+      # Microsoft, who has no comparable no-training pledge. Better than
+      # Exa for sensitive queries, worse than Brave; for anything
+      # genuinely embarrassing, don't search the web at all.
+      module DuckDuckGo
+        # @return [String] HTML search endpoint
+        ENDPOINT = 'https://html.duckduckgo.com/html/'
+        # @return [String] User-Agent sent with each request; DDG often rejects
+        #   requests with no UA or an obvious bot UA
+        USER_AGENT = 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 ' \
+                     '(KHTML, like Gecko) Chrome/120.0 Safari/537.36'
+        # @return [Integer] default number of results returned, matching smolagents
+        DEFAULT_MAX_RESULTS = 10
+        # @return [RateLimiter] paces calls (DDG bans IPs that hammer the HTML
+        #   endpoint) and circuit-breaks on {Engines::Unavailable} so a
+        #   soft-block response doesn't get retried for the next 5 minutes
+        LIMITER = RateLimiter.new(min_interval: 2.0, cooldown: 300.0)
+
+        # Fetch results for +query+ and return them as an +Array<Result>+.
+        # Calls are throttled to one every 2s and circuit-broken for 5 minutes
+        # after a soft-block; see {LIMITER}. The caller (typically
+        # {Engines.search}) is expected to have already normalized the
+        # query and to wrap this in a result cache.
+        #
+        # @param query [String] search query (already normalized)
+        # @param max_results [Integer] maximum number of result entries
+        # @return [Array<Result>] hits, possibly empty when DDG ran the
+        #   query and matched nothing
+        # @raise [Engines::Unavailable] when DDG soft-blocks the IP
+        #   (anomaly/CAPTCHA page) or returns HTTP 429/5xx — i.e. "try again
+        #   later" responses the cascade in {Engines.search} can fall
+        #   back from. Also raised immediately if {LIMITER} is in cooldown.
+        # @raise [RuntimeError] if the HTTP call fails for other reasons or
+        #   the empty-results page is in an unrecognized layout. A genuine
+        #   empty-results page is *not* an error; see {.parse}.
+        def self.search(query, max_results: DEFAULT_MAX_RESULTS)
+          LIMITER.call do
+            response = Faraday.get(ENDPOINT, { q: query }, { 'User-Agent' => USER_AGENT })
+            unless response.success?
+              if response.status == 429 || response.status >= 500
+                raise Engines::Unavailable, "HTTP #{response.status}"
+              end
+
+              raise "DuckDuckGo request failed: #{response.status} #{response.body}"
+            end
+
+            parse(response.body, max_results: max_results)
+          end
+        end
+
+        # Parse a +html.duckduckgo.com+ result page into a list of {Result}
+        # rows. +<b>+ highlights inside snippets are stripped.
+        #
+        # When the page has zero result nodes, two cases are distinguished:
+        # a genuine "no results" page (narrow query, DDG's own "No results
+        # found" indicator) returns an empty array instead of raising, so
+        # {Engines.search} can render its standard no-results stub.
+        # Anything else (anomaly modal, CAPTCHA, service-unavailable page,
+        # unknown layout) raises with the diagnostic text extracted from
+        # the body, so an IP soft-block is surfaced rather than silently
+        # masquerading as an empty search.
+        #
+        # @param html [String] HTML document body from html.duckduckgo.com
+        # @param max_results [Integer] maximum number of result entries
+        # @return [Array<Result>] hits, possibly empty on a genuine
+        #   no-results page
+        # @raise [Engines::Unavailable] when the page is the DDG
+        #   anomaly/CAPTCHA modal (IP soft-block) — a "try again later" the
+        #   cascade can fall back from.
+        # @raise [RuntimeError] when the page contains no result nodes and is
+        #   not recognized as either a genuine no-results page or the
+        #   anomaly modal (likely a layout change worth surfacing loudly).
+        def self.parse(html, max_results: DEFAULT_MAX_RESULTS)
+          doc = Nokogiri::HTML(html)
+          results = doc.css('div.result.web-result').take(max_results).filter_map do |node|
+            title_link = node.at_css('a.result__a')
+            next nil if title_link.nil?
+
+            snippet = node.at_css('a.result__snippet')
+            Result.new(
+              url: extract_url(title_link['href']),
+              title: title_link.text.strip,
+              body: snippet&.text&.strip.to_s
+            )
+          end
+
+          if results.empty?
+            return [] if genuine_no_results?(doc)
+
+            message = diagnose_empty(doc)
+            raise(anomaly_modal?(doc) ? Engines::Unavailable : RuntimeError, message)
+          end
+
+          results
+        end
+
+        # True when the page contains DDG's anomaly/CAPTCHA modal — i.e. the
+        # IP has been soft-blocked. Used by {.parse} to pick between
+        # {Engines::Unavailable} (recoverable, fall back to another
+        # provider) and {RuntimeError} (unknown layout, surface loudly).
+        #
+        # @param doc [Nokogiri::HTML::Document] parsed result page
+        # @return [Boolean]
+        def self.anomaly_modal?(doc)
+          !!(doc.at_css('.anomaly-modal__title') || doc.at_css('.anomaly-modal__description'))
+        end
+        private_class_method :anomaly_modal?
+
+        # True when a results page with zero result nodes looks like DDG's own
+        # "no results found" page (narrow query) rather than an anomaly/CAPTCHA
+        # or other non-results layout. Anomaly modal wins: if the modal divs
+        # are present we never treat the page as a genuine empty result, even
+        # if the surrounding text happens to mention "No results".
+        #
+        # @param doc [Nokogiri::HTML::Document] parsed result page
+        # @return [Boolean]
+        def self.genuine_no_results?(doc)
+          return false if doc.at_css('.anomaly-modal__title') || doc.at_css('.anomaly-modal__description')
+          return true if doc.at_css('div.no-results')
+
+          doc.text.include?('No results found')
+        end
+        private_class_method :genuine_no_results?
+
+        # Build an error message for a results page that yielded zero matches.
+        # Recognizes the DDG anomaly modal explicitly and quotes its title and
+        # description; otherwise extracts visible text from the body (with
+        # +<script>+/+<style>+/+<noscript>+ stripped and whitespace collapsed)
+        # and includes a truncated copy so the caller can see what came back.
+        #
+        # @param doc [Nokogiri::HTML::Document] parsed result page
+        # @return [String] human-readable diagnostic to feed to +raise+
+        def self.diagnose_empty(doc)
+          title = doc.at_css('.anomaly-modal__title')&.text&.strip
+          desc  = doc.at_css('.anomaly-modal__description')&.text&.strip
+          if title || desc
+            return "DuckDuckGo anomaly check (likely IP soft-block): #{[title, desc].compact.reject(&:empty?).join(' — ')}"
+          end
+
+          doc.css('script, style, noscript').remove
+          text = doc.text.gsub(/\s+/, ' ').strip
+          snippet = text.empty? ? '<empty body>' : text[0, 1500]
+          snippet += '…' if text.length > 1500
+          "DuckDuckGo returned no results. Page text: #{snippet}"
+        end
+        private_class_method :diagnose_empty
+
+        # Decode DuckDuckGo's +//duckduckgo.com/l/?uddg=<encoded>+ redirect wrapper
+        # back to the real target URL.
+        #
+        # @param href [String, nil] href as found on the search-result page
+        # @return [String] the decoded target URL, or +href+ unchanged when it is
+        #   not a recognized DDG redirect or cannot be parsed
+        def self.extract_url(href)
+          return href if href.nil? || href.empty?
+
+          uri = URI.parse(href.start_with?('//') ? "https:#{href}" : href)
+          return href unless uri.host&.end_with?('duckduckgo.com') && uri.path == '/l/'
+
+          params = URI.decode_www_form(uri.query.to_s).to_h
+          params['uddg'] || href
+        rescue URI::InvalidURIError
+          href
+        end
+      end
+    end
+  end
+end