ask-web-fetch 0.3.1 → 0.4.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9286a544b4924f7dcde61b273e367240a81f8702c62510228e2d97f330c3617e
-  data.tar.gz: '08f55182a88cbc0c62e77f2acbc0570dc09cd28476f15caa78868b4e6519afd8'
+  metadata.gz: 318ff9c594a7d5c52a710507efd3da2917bbe3319471a3e0bac8bce3a01ed30e
+  data.tar.gz: f5ae0d23b351044ecc21f4e2c4d2ec66a13d8258cd9a86d6947f0c7698c6e703
 SHA512:
-  metadata.gz: c868cc6a828fea7d421064199991e054c57fb85d02a2d7713ea043d567960060da687a998095ee64e6ae0e6e1424a2219e7046b193077dfac7128e99256c485b
-  data.tar.gz: b071c880ba9594cd21778e4debff5894a36eb8d2fbddd7aabbf88efcf0195c922d7a1aa84b957ddcecdc4364aafa6c87328166ebe7e1d533f5b3d4e3dfacbd4d
+  metadata.gz: 775322ca3144aa69da6d27e20664c5a18aa836baaf10d5431acc5cbc7dd893aff5e09c2c40751d3f19b6ea86410ad92cf992ad35b2283554b31196ba52fdea29
+  data.tar.gz: ae5f935d3b48522330b5a01f9319813bd84d157076f65ee2a17cdbd82fb3fb65916f5312ea03fefdc51e8b7a84792c9558ede959e195e386f2713fb1c31e437a

data/lib/ask/web_fetch/backend.rb

@@ -8,13 +8,22 @@ module Ask
     # next backend in the chain.
     class Error < StandardError; end
 
-    # A backend that failed to fetch (network error, non-2xx, challenge
-    # page, non-HTML response).
+    # A backend that failed to fetch because the URL itself is bad — 4xx,
+    # challenge page, non-HTML response, redirect loop. Deterministic:
+    # retrying won't change the outcome.
     class FetchError < Error; end
 
     # A backend that fetched the page but found nothing usable in it.
     class EmptyContentError < Error; end
 
+    # Network-level failure — timeout, connection refused/reset, bad
+    # socket. Transient: the same URL may succeed on retry.
+    class TimeoutError < Error; end
+
+    # The service or the target server answered 5xx/429. Transient:
+    # retrying after backoff may succeed.
+    class ServerError < Error; end
+
     # Base class for fetch backends, plus the errors they raise.
     #
     # A backend turns a URL into LLM-ready markdown. To add a new backend:

data/lib/ask/web_fetch/backends/attached_browser.rb

@@ -0,0 +1,148 @@
+# frozen_string_literal: true
+
+require 'ferrum'
+
+module Ask
+  module WebFetch
+    module Backends
+      # Drives an already-running Chrome — one started with
+      # `--remote-debugging-port=9222` — over the Chrome DevTools Protocol,
+      # reusing Ferrum's battle-tested CDP WebSocket client.
+      #
+      # This is the mode the Browser backend uses when ASK_WEB_FETCH_CDP_URL
+      # is set (e.g. http://127.0.0.1:9222). It is the *trusted context* a
+      # freshly launched automation browser can never be: a long-lived Chrome
+      # with a mature profile and any cookies (like cf_clearance) it has
+      # already earned, so sites whose invisible challenges soft-block fresh
+      # profiles load normally here.
+      #
+      # Each page is its own CDP target, created and closed by us — existing
+      # tabs are never touched.
+      class AttachedBrowser
+        def initialize(ws_url, timeout:, client: nil)
+          @ws_url = ws_url
+          @timeout = timeout
+          @client = client
+        end
+
+        # Creates a page as a fresh tab in the attached browser.
+        def create_page
+          Page.new(client, timeout: @timeout)
+        end
+
+        private
+
+        def client
+          @client ||= Ferrum::Client.new(
+            Addressable::URI.parse(@ws_url),
+            Ferrum::Browser::Options.new(timeout: @timeout, ws_max_receive_size: 20 * 1024 * 1024)
+          )
+        end
+
+        # One tab in the attached browser, closed with #close. Speaks just
+        # enough CDP for the Browser backend: navigate, read title/URL/HTML,
+        # and report the main-document HTTP status.
+        class Page
+          def initialize(client, timeout:)
+            @client = client
+            @timeout = timeout
+            @target_id = @client.command('Target.createTarget', url: 'about:blank')['targetId']
+            @session = @client.session(
+              @client.command('Target.attachToTarget', targetId: @target_id, flatten: true)['sessionId']
+            )
+          end
+
+          # Navigates and does not return until the document has actually
+          # loaded — Page.navigate only *starts* the navigation, and reading
+          # the page before it commits yields an empty document (the very
+          # failure mode this class exists to avoid).
+          def go_to(url)
+            result = @session.command('Page.navigate', url: url)
+            error = result['errorText']
+            if error && error != 'net::ERR_ABORTED'
+              raise Ferrum::StatusError, "Request to #{url} failed (#{error})"
+            end
+
+            wait_for_load
+          end
+
+          def title
+            evaluate('document.title')
+          end
+
+          def url
+            evaluate('location.href')
+          end
+
+          def body
+            evaluate('document.documentElement.outerHTML') || ''
+          end
+
+          # Minimal network facade for the Browser backend. The status comes
+          # from the Navigation Timing API (the main document's HTTP status);
+          # attached pages need no idle wait — they only exist once loaded.
+          def network
+            @network ||= Network.new(self)
+          end
+
+          def close
+            @client.command('Target.closeTarget', targetId: @target_id)
+          rescue Ferrum::Error
+            nil
+          end
+
+          def evaluate(expression)
+            response = @session.command('Runtime.evaluate', expression: expression, returnByValue: true)
+            # Ferrum's command() unwraps CDP's outer "result", leaving
+            # { "result" => RemoteObject, "exceptionDetails" => ... }.
+            response.dig('result', 'value')
+          end
+
+          private
+
+          # Wait for the DOM to be parsed, not for the load event: many
+          # pages (lazy images, analytics, keep-alive connections) never
+          # reach readyState "complete", but the content is fully readable
+          # at "interactive".
+          def wait_for_load
+            deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + @timeout
+            while Process.clock_gettime(Process::CLOCK_MONOTONIC) < deadline
+              state = ready_state
+              return if state && state != 'loading'
+
+              sleep 0.1
+            end
+            raise Ferrum::TimeoutError, "page did not finish loading within #{@timeout}s"
+          end
+
+          def ready_state
+            evaluate('document.readyState')
+          rescue Ferrum::Error
+            # Mid-navigation: the execution context is gone, keep waiting.
+            nil
+          end
+
+          # Status + no-op idle for the Browser backend's fetch flow.
+          class Network
+            def initialize(page)
+              @page = page
+            end
+
+            def status
+              @page.evaluate(
+                "performance.getEntriesByType('navigation')[0] ? " \
+                "performance.getEntriesByType('navigation')[0].responseStatus : 0"
+              ) || 0
+            rescue Ferrum::Error
+              0
+            end
+
+            def wait_for_idle(*)
+              nil
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ask/web_fetch/backends/browser.rb

@@ -0,0 +1,216 @@
+# frozen_string_literal: true
+
+require 'ferrum'
+require 'json'
+require 'net/http'
+require 'uri'
+require_relative '../backend'
+require_relative '../content_filter'
+require_relative '../markdown'
+require_relative 'attached_browser'
+
+module Ask
+  module WebFetch
+    module Backends
+      # Last-resort backend: a real Chrome driven over CDP via Ferrum.
+      # Renders JavaScript, so it reads SPAs and client-side pages that
+      # Local's plain HTTP cannot, and it lets Cloudflare-style *managed*
+      # challenges that auto-solve for real browsers complete themselves —
+      # neither Local (no JS engine) nor Jina (known datacenter renderer)
+      # can do either.
+      #
+      # Two modes:
+      #
+      # * Launched — a fresh Chrome the backend starts itself (default when
+      #   a binary is found). Honest limitation, measured in the wild: sites
+      #   running aggressive bot protection (patronview.com, npmjs.com,
+      #   stackoverflow.com all soft-block a freshly launched browser from a
+      #   datacenter IP) never clear their invisible challenge for a fresh
+      #   automation profile, however real the Chrome.
+      # * Attached — connects to an already-running Chrome via CDP
+      #   (ASK_WEB_FETCH_CDP_URL, e.g. http://127.0.0.1:9222). That browser
+      #   is a *trusted context*: long-lived, mature profile, any cookies it
+      #   has already earned (cf_clearance). Sites that soft-block fresh
+      #   profiles load normally there. See AttachedBrowser.
+      #
+      # Opt-in, like Crawl4AI: joins the chain only when a Chrome/Chromium
+      # binary is found or ASK_WEB_FETCH_CDP_URL is set. Configure the
+      # binary with ASK_WEB_FETCH_CHROME_PATH and a persistent profile
+      # directory with ASK_WEB_FETCH_PROFILE (a profile keeps solved cookies
+      # across restarts; within one process the browser is reused anyway).
+      #
+      # HTML is converted through the same Markdown pipeline as Local, with
+      # the same default adaptive ContentFilter.
+      class Browser < Backend
+        # Seconds to let a Cloudflare-style challenge auto-solve before
+        # giving up (tunable via Browser.challenge_timeout).
+        CHALLENGE_TIMEOUT = 30
+
+        # Seconds to wait for the network to go quiet after the page loads,
+        # so lazy-loaded content is in before we read the DOM.
+        IDLE_TIMEOUT = 5
+
+        # How often to poll for the challenge to clear.
+        POLL_INTERVAL = 0.5
+
+        DEFAULT_PATHS = [
+          '/Applications/Google Chrome.app/Contents/MacOS/Google Chrome',
+          '/Applications/Chromium.app/Contents/MacOS/Chromium',
+          '/usr/bin/google-chrome',
+          '/usr/bin/google-chrome-stable',
+          '/usr/bin/chromium',
+          '/usr/bin/chromium-browser',
+          '/opt/google/chrome/chrome'
+        ].freeze
+
+        class << self
+          # The shared browser (or a test double). Reuse keeps the solved
+          # challenge cookie warm across fetches within one process.
+          attr_writer :browser
+
+          # The ContentFilter applied to every page by default.
+          attr_writer :content_filter
+
+          # Absolute path to a Chrome/Chromium binary; nil when not found.
+          attr_writer :path
+
+          # CDP endpoint of an already-running Chrome ("http://host:port",
+          # "…/json/version", or a ws:// browser URL) to attach to instead.
+          attr_writer :cdp_url
+
+          attr_writer :challenge_timeout, :poll_interval
+
+          # A shared browser, built once and reused. Ferrum is loaded lazily
+          # so consumers who never hit this backend pay nothing for it.
+          def browser
+            return @browser if @browser
+
+            browser_mutex.synchronize { @browser ||= build_browser }
+          end
+
+          def content_filter
+            @content_filter ||= ContentFilter.default
+          end
+
+          def path
+            @path || ENV['ASK_WEB_FETCH_CHROME_PATH'] || DEFAULT_PATHS.find { |p| File.exist?(p) }
+          end
+
+          def cdp_url
+            @cdp_url || ENV['ASK_WEB_FETCH_CDP_URL']
+          end
+
+          def challenge_timeout
+            @challenge_timeout || CHALLENGE_TIMEOUT
+          end
+
+          def poll_interval
+            @poll_interval || POLL_INTERVAL
+          end
+
+          def configured?
+            !path.to_s.empty? || !cdp_url.to_s.empty?
+          end
+
+          # Turns an ASK_WEB_FETCH_CDP_URL into the browser-level WebSocket
+          # URL. Accepts a ws:// URL as-is, or an HTTP endpoint
+          # ("http://127.0.0.1:9222" or "…/json/version") which is probed
+          # for its webSocketDebuggerUrl — the same discovery puppeteer's
+          # connect does.
+          def ws_url_for(cdp_url)
+            return cdp_url if cdp_url.start_with?('ws://', 'wss://')
+
+            version_url = cdp_url.end_with?('/json/version') ? cdp_url : "#{cdp_url.chomp('/')}/json/version"
+            body = Net::HTTP.get(URI(version_url))
+            JSON.parse(body)['webSocketDebuggerUrl']
+          rescue Errno::ECONNREFUSED, SocketError => e
+            raise FetchError, "cannot reach CDP endpoint #{version_url}: #{e.message}"
+          rescue JSON::ParserError => e
+            raise FetchError, "bad CDP version response from #{version_url}: #{e.message}"
+          end
+
+          private
+
+          def browser_mutex
+            @browser_mutex ||= Mutex.new
+          end
+
+          def build_browser
+            return AttachedBrowser.new(ws_url_for(cdp_url), timeout: CHALLENGE_TIMEOUT + IDLE_TIMEOUT) if cdp_url
+
+            Ferrum::Browser.new(
+              browser_path: path,
+              headless: true,
+              user_data_dir: ENV['ASK_WEB_FETCH_PROFILE'],
+              timeout: CHALLENGE_TIMEOUT + IDLE_TIMEOUT
+            )
+          end
+        end
+
+        def fetch(url)
+          page = nil
+          raise FetchError, 'no Chrome/Chromium found and no CDP endpoint set' unless self.class.configured?
+          raise FetchError, 'ferrum gem unavailable' unless self.class.browser
+
+          page = self.class.browser.create_page
+          page.go_to(url)
+          wait_for_challenge(page)
+          wait_for_idle(page)
+
+          status = page.network.status
+          raise FetchError, "got #{status} at #{url}" if status && status >= 400
+
+          body = page.body
+          raise FetchError, "challenge page at #{url}" if challenge_page?(body)
+
+          result = Markdown.generate(body, base_url: url, filter: self.class.content_filter)
+          raise EmptyContentError, "no readable content at #{url}" unless usable_content?(result[:content])
+
+          result
+        rescue Ferrum::TimeoutError, Ferrum::ProcessTimeoutError, Ferrum::DeadBrowserError => e
+          raise TimeoutError, "#{e.class}: #{e.message}"
+        rescue Ferrum::StatusError => e
+          raise FetchError, "browser could not load #{url}: #{e.message}"
+        rescue Ferrum::Error => e
+          raise ServerError, "browser #{e.class}: #{e.message}"
+        rescue Errno::ECONNREFUSED, SocketError => e
+          raise TimeoutError, "browser connection #{e.class}: #{e.message}"
+        ensure
+          page&.close
+        end
+
+        private
+
+        # Cloudflare's managed challenge solves itself in a real browser (JS
+        # proof-of-work -> cf_clearance cookie -> reload). We just wait for
+        # the interstitial's title to leave the page. If it never does, the
+        # challenge is one we can't pass — deterministic, so FetchError.
+        def wait_for_challenge(page)
+          deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + self.class.challenge_timeout
+          while Process.clock_gettime(Process::CLOCK_MONOTONIC) < deadline
+            return unless challenge_active?(page)
+
+            sleep self.class.poll_interval
+          end
+          raise FetchError, "challenge did not auto-solve for #{page.url}"
+        end
+
+        def challenge_active?(page)
+          challenge_page?(page.title.to_s)
+        rescue Ferrum::Error
+          # Mid-navigation (the post-solve reload): the JS context is gone,
+          # keep waiting.
+          true
+        end
+
+        # Give lazy-loading pages a moment to settle; never fail the fetch
+        # on it — some pages stream or poll forever and never go quiet.
+        def wait_for_idle(page)
+          page.network.wait_for_idle(duration: 0.5, timeout: IDLE_TIMEOUT)
+        rescue Ferrum::Error
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/ask/web_fetch/backends/crawl4ai.rb

@@ -59,7 +59,7 @@ module Ask
           page
         rescue Net::OpenTimeout, Net::ReadTimeout, Errno::ECONNREFUSED,
                Errno::ECONNRESET, SocketError, URI::InvalidURIError => e
-          raise FetchError, "Crawl4AI #{e.class}: #{e.message}"
+          raise TimeoutError, "Crawl4AI #{e.class}: #{e.message}"
         end
 
         private
@@ -88,7 +88,9 @@ module Ask
           when '401', '403'
             raise FetchError, "Crawl4AI auth error (#{res.code})"
           else
-            raise FetchError, "Crawl4AI returned #{res.code}"
+            # The /crawl service itself answering 5xx (or 4xx beyond auth)
+            # is a service-side problem — transient, retrying may succeed.
+            raise ServerError, "Crawl4AI returned #{res.code}"
           end
         end
 
@@ -102,12 +104,41 @@ module Ask
             raise FetchError, "Crawl4AI crawl failed: #{result['error_message'] || 'unknown error'}"
           end
 
+          # A rendered page is only content when it actually loaded — a 4xx
+          # error page renders fine but is not the page. Crawl4AI reports
+          # the target's status on the result; without this check a 404
+          # shell passes as a successful fetch.
+          status = result['status_code'].to_i
+          if status >= 400
+            raise(status == 429 || status >= 500 ? ServerError : FetchError, "Crawl4AI got #{status} at #{url}")
+          end
+
           markdown = result.dig('markdown', 'fit_markdown').to_s
           markdown = result.dig('markdown', 'raw_markdown').to_s if markdown.strip.empty?
-          { title: result.dig('metadata', 'title'), content: markdown }
+          {
+            title: result.dig('metadata', 'title'),
+            description: result.dig('metadata', 'description') ||
+              result.dig('metadata', 'og_description'),
+            content: markdown,
+            # The page's own redirect, if the crawl followed one — lets
+            # consumers record permanent redirects on their ledger instead
+            # of silently indexing under the original URL.
+            redirected: redirect_of(result)
+          }
         rescue JSON::ParserError => e
           raise FetchError, "Crawl4AI bad JSON response: #{e.message}"
         end
+
+        # A redirect the browser followed: redirected_status_code is the
+        # status that started the chain (301/302/308...), redirected_url
+        # the final destination.
+        def redirect_of(result)
+          status = result['redirected_status_code'].to_i
+          url = result['redirected_url'].to_s
+          return nil if status.zero? || url.empty?
+
+          {status: status, url: url}
+        end
       end
     end
   end

data/lib/ask/web_fetch/backends/jina.rb

@@ -35,17 +35,18 @@ module Ask
             raise FetchError, 'challenge page from Jina' if challenge_page?(body)
             raise EmptyContentError, 'empty response from Jina' unless usable_content?(body)
 
-            { title: nil, content: body.strip }
+            { title: nil, description: nil, content: body.strip }
           when '429'
-            raise FetchError, 'rate limited by Jina (429)'
+            raise ServerError, 'rate limited by Jina (429)'
           when '401', '403'
             raise FetchError, "Jina access error (#{res.code})"
           else
-            raise FetchError, "Jina returned #{res.code}"
+            # 5xx = Jina-side blip (transient); other 4xx = the URL is dead.
+            raise(res.code.to_i >= 500 ? ServerError : FetchError, "Jina returned #{res.code}")
           end
         rescue Net::OpenTimeout, Net::ReadTimeout, Errno::ECONNREFUSED,
                Errno::ECONNRESET, SocketError, URI::InvalidURIError => e
-          raise FetchError, "Jina #{e.class}: #{e.message}"
+          raise TimeoutError, "Jina #{e.class}: #{e.message}"
         end
       end
     end

data/lib/ask/web_fetch/backends/local.rb

@@ -2,9 +2,9 @@
 
 require 'net/http'
 require 'uri'
-require 'nokogiri'
-require 'reverse_markdown'
 require_relative '../backend'
+require_relative '../content_filter'
+require_relative '../markdown'
 
 module Ask
   module WebFetch
@@ -12,50 +12,58 @@ module Ask
       # Default backend: pure Ruby Net::HTTP + Nokogiri + reverse_markdown.
       # No external service or API key; mirrors ask-web-search's self-hosted
       # SearXNG approach.
+      #
+      # HTML is converted through Ask::WebFetch::Markdown with a default
+      # ContentFilter, so the "fit" content — pruned by text density rather
+      # than by keyword — is what comes back.
       class Local < Backend
         MAX_REDIRECTS = 5
         OPEN_TIMEOUT = 5
         READ_TIMEOUT = 15
 
-        # Class/id fragments that mark navigation chrome worth dropping, e.g.
-        # "vector-page-toolbar", "sidebar", "toc".
-        NAV_CHROME_RE = /
-          (^|[\s_-])(nav|menu|toolbar|breadcrumb|sidebar|toc|footer|header|
-          banner|pagination|search|cookie|modal|popup)([\s_-]|$)
-        /ix
+        class << self
+          # The ContentFilter applied to every page by default. Set to nil to
+          # convert the article/main region without pruning.
+          attr_writer :content_filter
+
+          def content_filter
+            @content_filter ||= ContentFilter.default
+          end
+        end
 
         def fetch(url)
-          body, content_type = fetch_html(url)
+          body, content_type, redirect = fetch_html(url)
           raise FetchError, "expected HTML from #{url}, got #{content_type}" unless content_type.include?('html')
           raise FetchError, "challenge page at #{url}" if challenge_page?(body)
 
           page = to_markdown(body, url)
+          page[:redirected] = redirect
           raise EmptyContentError, "no readable content at #{url}" unless usable_content?(page[:content])
 
           page
         rescue Net::OpenTimeout, Net::ReadTimeout, Errno::ECONNREFUSED,
                Errno::ECONNRESET, SocketError, URI::InvalidURIError => e
-          raise FetchError, "#{e.class}: #{e.message}"
+          raise TimeoutError, "#{e.class}: #{e.message}"
         end
 
-        # Parses +html+ and returns { title:, content: } where content is
-        # clean markdown.
-        def to_markdown(html, _url)
-          doc = Nokogiri::HTML(html)
-          candidate = extract_main(doc)
-          scrub(candidate)
-          markdown = ReverseMarkdown.convert(candidate.to_html, unknown_tags: :bypass, github_flavored: true)
-          markdown = clean(markdown)
-          title = doc.at('title')&.text&.strip
-          { title: title, content: markdown }
+        # Parses +html+ and returns { title:, description:, content: } where
+        # content is clean markdown and description is the page's own meta
+        # description (meta name=description, then og:description) — what the
+        # site says about itself, free and authoritative.
+        def to_markdown(html, url)
+          Markdown.generate(html, base_url: url, filter: self.class.content_filter)
         end
 
         private
 
-        # GET with redirect following (max MAX_REDIRECTS hops).
+        # GET with redirect following (max MAX_REDIRECTS hops). Returns
+        # [body, content_type, redirect] where redirect is nil when the
+        # URL answered directly, else {status: first hop's status,
+        # url: final destination} — the chain the crawler followed.
         def fetch_html(url)
           uri = URI(url)
           hops = 0
+          first_hop_status = nil
           loop do
             http = Net::HTTP.new(uri.host, uri.port)
             http.open_timeout = OPEN_TIMEOUT
@@ -65,35 +73,22 @@ module Ask
             req['User-Agent'] = USER_AGENT
             req['Accept'] = 'text/html,application/xhtml+xml'
             res = http.request(req)
-            return [res.body, res['content-type'].to_s] if res.code.start_with?('2')
+            return [res.body, res['content-type'].to_s, redirect_info(first_hop_status, uri)] if res.code.start_with?('2')
 
-            raise FetchError, "got #{res.code} from #{url}" unless res.code.start_with?('3') && res['location']
+            unless res.code.start_with?('3') && res['location']
+              # 4xx (other than 429) = the URL is dead; 429/5xx = transient.
+              code = res.code.to_i
+              raise(code == 429 || code >= 500 ? ServerError : FetchError, "got #{res.code} from #{url}")
+            end
             raise FetchError, "hit a redirect loop at #{url}" if (hops += 1) > MAX_REDIRECTS
 
+            first_hop_status ||= res.code.to_i
             uri = URI.join(uri, res['location'])
           end
         end
 
-        def extract_main(doc)
-          doc.at('article') || doc.at('main') || doc.at('[role="main"]') || doc.at('body') || doc
-        end
-
-        # Remove chrome INSIDE the candidate only, never ancestors.
-        def scrub(candidate)
-          candidate.css('script, style, noscript, nav, footer, header, iframe, form, svg, aside').each(&:remove)
-          candidate.css('*[id], *[class]').each do |el|
-            next if el.equal?(candidate)
-
-            id_cls = [el['id'], el['class']].compact.join(' ')
-            el.remove if id_cls.match?(NAV_CHROME_RE)
-          end
-          candidate
-        end
-
-        def clean(markdown)
-          markdown.gsub(/[ \t]+\n/, "\n")
-                  .gsub(/\n{3,}/, "\n\n")
-                  .strip
+        def redirect_info(status, uri)
+          status && {status: status, url: uri.to_s}
         end
       end
     end

data/lib/ask/web_fetch/content_filter.rb

@@ -0,0 +1,183 @@
+# frozen_string_literal: true
+
+require 'nokogiri'
+
+module Ask
+  module WebFetch
+    # Density-based content pruning, ported from crawl4ai's
+    # PruningContentFilter (Apache-2.0, https://github.com/unclecode/crawl4ai).
+    #
+    # Scores every element in the page by how much of it is real text versus
+    # markup and links, then removes elements scoring below a threshold. Where
+    # a keyword scraper fails ("remove anything with 'nav' in the class"),
+    # this keeps content that scores well regardless of its label and drops
+    # link-farms that happen to dodge the keywords.
+    #
+    # Two deliberate deviations from crawl4ai, both adding tags crawl4ai's own
+    # included_tags list marks as content but whose tag_weights map left out
+    # (they defaulted to 0.5 and could be pruned despite being article
+    # content): <main> joins <article>/<section> at the top tier, and the
+    # table/pre/code family gets content-worthy weights. Third: the class/id
+    # chrome penalty actually subtracts from the score — crawl4ai floors it at
+    # zero (max(0, ...)), which makes its negative-patterns metric inert.
+    # Fourth: <svg> is excluded outright — chart text would otherwise leak
+    # into the markdown as concatenated axis labels ("01M2M3M", "10Apr15Apr").
+    class ContentFilter
+      # Structural boilerplate, removed before scoring ever runs. The
+      # preserve_* whitelist cannot save these.
+      EXCLUDED_TAGS = %w[nav footer header aside script style form iframe noscript svg].freeze
+
+      # Class/id fragments that mark non-content chrome; matching one knocks
+      # 0.5 off the node's score.
+      NEGATIVE_PATTERNS = /nav|footer|header|sidebar|ads|comment|promo|advert|social|share/i.freeze
+
+      # Semantic tag weights: how likely a tag is to carry article content.
+      TAG_WEIGHTS = {
+        'div' => 0.5, 'p' => 1.0, 'article' => 1.5, 'section' => 1.0, 'main' => 1.4,
+        'span' => 0.3, 'li' => 0.5, 'ul' => 0.5, 'ol' => 0.5,
+        'h1' => 1.2, 'h2' => 1.1, 'h3' => 1.0, 'h4' => 0.9, 'h5' => 0.8, 'h6' => 0.7,
+        'table' => 1.0, 'tr' => 0.8, 'td' => 0.8, 'th' => 0.8,
+        'pre' => 1.0, 'code' => 0.9, 'blockquote' => 1.0
+      }.freeze
+
+      # Used only by the dynamic threshold, which loosens the bar for tags
+      # that usually carry content.
+      TAG_IMPORTANCE = {
+        'article' => 1.5, 'main' => 1.4, 'section' => 1.3, 'p' => 1.2,
+        'h1' => 1.4, 'h2' => 1.3, 'h3' => 1.2, 'div' => 0.7, 'span' => 0.6
+      }.freeze
+
+      METRIC_WEIGHTS = {
+        text_density: 0.4, link_density: 0.2, tag_weight: 0.2,
+        class_id_weight: 0.1, text_length: 0.1
+      }.freeze
+
+      DEFAULT_THRESHOLD = 0.48
+
+      # The filter the backends use by default: the adaptive threshold, which
+      # loosens the bar for content-carrying tags and text-heavy nodes and
+      # tightens it for link-heavy ones — catching the classic
+      # sidebar-of-links that the fixed bar lets through.
+      def self.default
+        new(threshold_type: :dynamic)
+      end
+
+      # threshold::   score below this removes the element
+      # threshold_type:: :fixed or :dynamic — dynamic loosens the bar for
+      #                important tags and text-heavy, link-light nodes
+      # min_word_threshold:: elements with fewer words are removed outright
+      # preserve_classes:: class names never pruned, regardless of score
+      # preserve_tags::   tag names never pruned, regardless of score
+      def initialize(threshold: DEFAULT_THRESHOLD, threshold_type: :fixed,
+                     min_word_threshold: nil, preserve_classes: [], preserve_tags: [])
+        @threshold = threshold
+        @threshold_type = threshold_type.to_sym
+        @min_word_threshold = min_word_threshold
+        @preserve_classes = preserve_classes
+        @preserve_tags = preserve_tags
+      end
+
+      # html -> [String] the surviving top-level blocks, as HTML fragments.
+      # Empty input yields an empty array; malformed HTML is parsed with
+      # Nokogiri's recover mode and never raises.
+      def filter_content(html)
+        return [] if html.nil? || html.empty?
+
+        doc = Nokogiri::HTML(html)
+        body = doc.at_css('body') || doc.at_css('html')
+        return [] unless body
+
+        remove_comments(doc)
+        body.css(EXCLUDED_TAGS.join(',')).each(&:remove)
+        body.element_children.each { |child| prune(child) }
+
+        body.element_children.select { |el| el.text.strip.length.positive? }.map(&:to_html)
+      end
+
+      # html -> String, the surviving blocks wrapped in <div>s, ready for
+      # markdown conversion. Mirrors crawl4ai's generator, which wraps the
+      # filtered blocks before converting them.
+      def fit_html(html)
+        filter_content(html).map { |block| "<div>#{block}</div>" }.join
+      end
+
+      private
+
+      def prune(node)
+        if preserved?(node)
+          # A whitelisted node survives whole — no scoring, no child pruning.
+          return
+        end
+
+        text_len = node.text.gsub(/\s+/, '').length
+        tag_len = node.inner_html.length
+        link_text_len = node.element_children
+                             .select { |child| child.name == 'a' }
+                             .sum { |a| a.text.strip.length }
+
+        if should_remove?(node, text_len, tag_len, link_text_len)
+          node.remove
+        else
+          node.element_children.each { |child| prune(child) }
+        end
+      end
+
+      def preserved?(node)
+        return true if @preserve_tags.include?(node.name)
+
+        classes = node['class']
+        classes && @preserve_classes.any? { |c| classes.split.include?(c) }
+      end
+
+      def should_remove?(node, text_len, tag_len, link_text_len)
+        if @threshold_type == :dynamic
+          threshold = @threshold
+          importance = TAG_IMPORTANCE.fetch(node.name, 0.7)
+          text_ratio = tag_len.positive? ? text_len.to_f / tag_len : 0.0
+          link_ratio = text_len.positive? ? link_text_len.to_f / text_len : 1.0
+          threshold *= 0.8 if importance > 1.0
+          threshold *= 0.9 if text_ratio > 0.4
+          threshold *= 1.2 if link_ratio > 0.6
+          score(node, text_len, tag_len, link_text_len) < threshold
+        else
+          score(node, text_len, tag_len, link_text_len) < @threshold
+        end
+      end
+
+      # Weighted composite in [0, ~1.4]: text density, link density, semantic
+      # tag weight, class/id chrome penalty, and log-scaled text length.
+      #
+      # text_len is measured with all whitespace removed, matching
+      # BeautifulSoup's get_text(strip=True) that crawl4ai uses — stripping
+      # only the ends would let inter-tag whitespace inflate the density and
+      # length metrics and keep link-farms alive.
+      def score(node, text_len, tag_len, link_text_len)
+        return -1.0 if @min_word_threshold && node.text.split.size < @min_word_threshold
+
+        density = tag_len.positive? ? text_len.to_f / tag_len : 0.0
+        link_density = text_len.positive? ? 1.0 - (link_text_len.to_f / text_len) : 0.0
+        tag_score = TAG_WEIGHTS.fetch(node.name, 0.5)
+        class_score = class_id_score(node)
+
+        weighted = 0.0
+        weighted += METRIC_WEIGHTS[:text_density] * density
+        weighted += METRIC_WEIGHTS[:link_density] * link_density
+        weighted += METRIC_WEIGHTS[:tag_weight] * tag_score
+        weighted += METRIC_WEIGHTS[:class_id_weight] * class_score
+        weighted += METRIC_WEIGHTS[:text_length] * Math.log(text_len + 1)
+        weighted / METRIC_WEIGHTS.values.sum
+      end
+
+      def class_id_score(node)
+        score = 0.0
+        score -= 0.5 if node['class'].to_s.match?(NEGATIVE_PATTERNS)
+        score -= 0.5 if node['id'].to_s.match?(NEGATIVE_PATTERNS)
+        score
+      end
+
+      def remove_comments(doc)
+        doc.xpath('//comment()').each(&:remove)
+      end
+    end
+  end
+end

data/lib/ask/web_fetch/markdown.rb

@@ -0,0 +1,159 @@
+# frozen_string_literal: true
+
+require 'nokogiri'
+require 'reverse_markdown'
+require 'uri'
+
+module Ask
+  module WebFetch
+    # HTML -> clean markdown, shared by the backends. Owns the conversion
+    # pipeline that used to live inside Local:
+    #
+    #   1. pick the content region — either a ContentFilter prunes the whole
+    #      page by text density, or the article/main/body region is taken
+    #   2. drop chrome (scripts, nav, forms, keyword-matched junk)
+    #   3. convert with reverse_markdown
+    #   4. optionally rewrite inline links as numbered citations (ported from
+    #      crawl4ai's DefaultMarkdownGenerator, Apache-2.0)
+    module Markdown
+      # Class/id fragments that mark navigation chrome worth dropping, e.g.
+      # "vector-page-toolbar", "sidebar", "toc".
+      NAV_CHROME_RE = /
+        (^|[\s_-])(nav|menu|toolbar|breadcrumb|sidebar|toc|footer|header|
+        banner|pagination|search|cookie|modal|popup)([\s_-]|$)
+      /ix
+
+      # Tags that never carry article content, dropped before conversion.
+      EXCLUDED_TAGS = %w[script style noscript nav footer header aside form iframe svg].freeze
+
+      # Matches markdown links, optionally with a title, for citation
+      # rewriting. Ported from crawl4ai's LINK_PATTERN.
+      LINK_PATTERN = /
+        !?\[((?:[^\[\]]|\[(?:[^\[\]]|\[[^\]]*\])*\])*)\]
+        \(((?:[^()\s]|\([^()]*\))*)(?:\s+"([^"]*)")?\)
+      /x
+
+      module_function
+
+      # html -> { title:, description:, content: }.
+      #
+      # When +filter+ is given (a ContentFilter), content is the pruned "fit"
+      # version. If pruning empties the page the article/main/body region is
+      # used instead, so a sparse page never degrades to empty content. When
+      # +citations+ is true, inline links become numbered citations with a
+      # trailing References section.
+      def generate(html, base_url: '', filter: nil, citations: false)
+        html = html.to_s
+        doc = Nokogiri::HTML(html)
+        input = filtered(html, doc, filter)
+        markdown = ReverseMarkdown.convert(input, unknown_tags: :bypass, github_flavored: true)
+        markdown = clean(markdown)
+        markdown = cite(markdown, base_url) if citations
+
+        {
+          title: doc.at('title')&.text&.strip,
+          description: meta_description(doc),
+          content: markdown
+        }
+      end
+
+      # Picks the article/main/body region and returns it scrubbed, as HTML.
+      def cleaned_html(doc)
+        candidate = doc.at('article') || doc.at('main') || doc.at('[role="main"]') || doc.at('body') || doc
+        scrub(candidate)
+        candidate.to_html
+      end
+
+      # Remove chrome INSIDE the candidate only, never ancestors.
+      def scrub(candidate)
+        candidate.css(EXCLUDED_TAGS.join(',')).each(&:remove)
+        candidate.css('*[id], *[class]').each do |el|
+          next if el.equal?(candidate)
+
+          id_cls = [el['id'], el['class']].compact.join(' ')
+          el.remove if id_cls.match?(NAV_CHROME_RE)
+        end
+        candidate
+      end
+
+      def meta_description(doc)
+        desc = doc.at('meta[name="description"]')&.[]('content')&.strip
+        desc = doc.at('meta[property="og:description"]')&.[]('content')&.strip if desc.to_s.empty?
+        desc
+      end
+
+      # Converts inline links to numbered citations and returns
+      # [converted, references] where references is a "## References" block
+      # listing each unique URL once — empty when the markdown had no links.
+      # Ported from crawl4ai.
+      def convert_links_to_citations(markdown, base_url = '')
+        link_map = {} # url => [number, description]
+        parts = []
+        last_end = 0
+        counter = 1
+
+        pos = 0
+        while (match = LINK_PATTERN.match(markdown, pos))
+          parts << markdown[last_end...match.begin(0)]
+          text, url, title = match.captures
+
+          absolute = url.start_with?('http://', 'https://', 'mailto:')
+          url = fast_urljoin(base_url, url) if !base_url.to_s.empty? && !absolute
+
+          unless link_map.key?(url)
+            description = +''
+            description << title.to_s if title
+            if text && text != title
+              description << (description.empty? ? text.to_s : " - #{text}")
+            end
+            link_map[url] = [counter, description.empty? ? '' : ": #{description}"]
+            counter += 1
+          end
+
+          number = link_map[url][0]
+          parts << (match[0].start_with?('!') ? "![#{text}⟨#{number}⟩]" : "#{text}⟨#{number}⟩")
+          last_end = match.end(0)
+          pos = match.end(0)
+        end
+
+        return [markdown, ''] if link_map.empty?
+
+        parts << markdown[last_end..].to_s
+        references = ["\n\n## References\n\n"]
+        link_map.sort_by { |_, (number, _)| number }.each do |url, (number, description)|
+          references << "⟨#{number}⟩ #{url}#{description}\n"
+        end
+        [parts.join, references.join]
+      end
+
+      def clean(markdown)
+        markdown.gsub(/[ \t]+\n/, "\n")
+                .gsub(/\n{3,}/, "\n\n")
+                .strip
+      end
+
+      # Resolves +url+ against +base+ without re-parsing absolute URLs.
+      # crawl4ai's version of this concatenates root-absolute paths onto the
+      # base ("/stats" against "https://x.com/news/2026" becomes
+      # "…/news/2026/stats"), which mangles root-relative links; URI.join has
+      # correct URL semantics and the passthrough covers the hot path.
+      def fast_urljoin(base, url)
+        return url if url.start_with?('http://', 'https://', 'mailto:', '//')
+
+        URI.join(base, url).to_s
+      end
+
+      def filtered(html, doc, filter)
+        return cleaned_html(doc) unless filter
+
+        fit = filter.fit_html(html)
+        fit.empty? ? cleaned_html(doc) : fit
+      end
+
+      def cite(markdown, base_url)
+        converted, references = convert_links_to_citations(markdown, base_url)
+        "#{converted}#{references}"
+      end
+    end
+  end
+end

data/lib/ask/web_fetch/tool.rb

@@ -14,21 +14,25 @@ module Ask
     #
     # Chain: Crawl4AI first (self-hosted headless-Chromium renderer; used
     # when the CRAWL4AI_URL service is present, fails fast when it isn't),
-    # then the local fetcher, with Jina Reader as the last resort.
+    # then the local fetcher, Jina Reader as the last resort, and a real
+    # Chrome (via Ferrum) at the very end for pages whose Cloudflare-style
+    # challenges the others cannot pass — appended only when a browser
+    # binary is present.
     class WebFetch < Ask::Tool
       DEFAULT_MAX_CHARS = 20_000
 
       # Backend chain, tried in order. Crawl4AI leads when configured
       # (CRAWL4AI_URL), so a present self-hosted renderer is preferred;
-      # otherwise Local, with Jina as the last resort. Swap or extend for
-      # future backends; each must subclass Ask::WebFetch::Backend and
-      # implement #fetch(url).
+      # otherwise Local, with Jina as the last resort, and Browser appended
+      # when Chrome is available. Swap or extend for future backends; each
+      # must subclass Ask::WebFetch::Backend and implement #fetch(url).
       def self.backends
         @backends ||= begin
           chain = [Ask::WebFetch::Backends::Local, Ask::WebFetch::Backends::Jina]
           if Ask::WebFetch::Backends::Crawl4Ai.configured?
             chain.unshift(Ask::WebFetch::Backends::Crawl4Ai)
           end
+          chain << Ask::WebFetch::Backends::Browser if Ask::WebFetch::Backends::Browser.configured?
           chain
         end
       end

data/lib/ask/web_fetch/version.rb

@@ -2,6 +2,6 @@
 
 module Ask
   module WebFetch
-    VERSION = '0.3.1'
+    VERSION = '0.4.1'
   end
 end

data/lib/ask/web_fetch.rb

@@ -2,7 +2,10 @@
 
 require_relative 'web_fetch/version'
 require_relative 'web_fetch/backend'
+require_relative 'web_fetch/content_filter'
+require_relative 'web_fetch/markdown'
 require_relative 'web_fetch/backends/local'
 require_relative 'web_fetch/backends/crawl4ai'
 require_relative 'web_fetch/backends/jina'
+require_relative 'web_fetch/backends/browser'
 require_relative 'web_fetch/tool'

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ask-web-fetch
 version: !ruby/object:Gem::Version
-  version: 0.3.1
+  version: 0.4.1
 platform: ruby
 authors:
 - Kaka Ruto
@@ -23,6 +23,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0.1'
+- !ruby/object:Gem::Dependency
+  name: ferrum
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0.14'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0.14'
 - !ruby/object:Gem::Dependency
   name: nokogiri
   requirement: !ruby/object:Gem::Requirement
@@ -109,8 +123,10 @@ dependencies:
         version: '3.26'
 description: Provides Ask::Tools::WebFetch, a tool that fetches a URL and converts
   its content to clean markdown for LLM consumption. Defaults to a pure Ruby backend
-  (Net::HTTP + Nokogiri + reverse_markdown) with an automatic Jina Reader fallback
-  for JS-rendered or blocked pages. Works with any ask-rb chat or agent.
+  (Net::HTTP + Nokogiri + reverse_markdown) with a Jina Reader fallback for JS-rendered
+  or blocked pages, and a real-Chrome fallback (Ferrum) that renders JavaScript and
+  lets auto-solving Cloudflare challenges complete. Works with any ask-rb chat or
+  agent.
 email:
 - kaka@myrrlabs.com
 executables: []
@@ -122,9 +138,13 @@ files:
 - lib/ask-web-fetch.rb
 - lib/ask/web_fetch.rb
 - lib/ask/web_fetch/backend.rb
+- lib/ask/web_fetch/backends/attached_browser.rb
+- lib/ask/web_fetch/backends/browser.rb
 - lib/ask/web_fetch/backends/crawl4ai.rb
 - lib/ask/web_fetch/backends/jina.rb
 - lib/ask/web_fetch/backends/local.rb
+- lib/ask/web_fetch/content_filter.rb
+- lib/ask/web_fetch/markdown.rb
 - lib/ask/web_fetch/tool.rb
 - lib/ask/web_fetch/version.rb
 homepage: https://github.com/ask-rb/ask-web-fetch