ask-web-fetch 0.7.2 → 0.7.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3ad48d09aa8869b4193a703ad6a760753c2767c9e26a6919b1405a5fb186c7a4
-  data.tar.gz: 52960016a3d4740f4913d4879cbf80edacf95e4ffc07601e6dde0d80515ffe81
+  metadata.gz: 412118d0b9d8d15d1677f877dcdd06cc3ca3d64f6afb79be80ae7fb404bdc8c5
+  data.tar.gz: 18a022bec912f32ecd7f1afca82c11e7e5f016199779b92fec86ec5deb1428a7
 SHA512:
-  metadata.gz: fa6d778abc6918bb7ae839d4f9c96dafe215014a8b2f4efc8027ab1e49d287e3ca57204f54f013531e38dff7393ac23f6da9619b3afd48efae8b6f7e42dfa0d9
-  data.tar.gz: 7d4e5fe0e70b28cc3349ec96d071d4e1cc7a46c2c21ca8f41c35968d8b15221854c2418fc72ae2a8e56e97c07930592b2ee5b79da921c4a4ccb4fe6990c7e938
+  metadata.gz: b12c4fbcc8e2f76d1b76b07836e1c7aeaa735e22c75b085571a1a8f710425637b461a97f2ec5c073d6e88b1d843aeca9da50d2ecba36d72ad21fcd4da7926852
+  data.tar.gz: 054750f93da3d1af328fb312867d26cbf4f7631133d9e79f76eb759816fd376e30b1a8df1b1b3f52d5752f8eac15aa38143c1c9484222543a20301c947ea2b4d

data/lib/ask/web_fetch/backend.rb

@@ -8,10 +8,23 @@ module Ask
     # next backend in the chain.
     class Error < StandardError; end
 
-    # A backend that failed to fetch because the URL itself is bad — 4xx,
+    # A backend that failed to fetch because the URL itself is bad —
     # challenge page, non-HTML response, redirect loop. Deterministic:
     # retrying won't change the outcome.
-    class FetchError < Error; end
+    class FetchError < Error
+      attr_reader :status
+
+      def initialize(message = nil, status: nil)
+        @status = status
+        msg = status ? "[#{status}] #{message}" : message
+        super(msg)
+      end
+    end
+
+    # HTTP 404 — the URL does not exist. Some 404 pages still carry
+    # usable content (custom error pages with navigation, suggestions);
+    # the backend tries to extract it before giving up.
+    class NotFoundError < FetchError; end
 
     # A backend that fetched the page but found nothing usable in it.
     class EmptyContentError < Error; end
@@ -29,7 +42,15 @@ module Ask
 
     # The service or the target server answered 5xx/429. Transient:
     # retrying after backoff may succeed.
-    class ServerError < Error; end
+    class ServerError < Error
+      attr_reader :status
+
+      def initialize(message = nil, status: nil)
+        @status = status
+        msg = status ? "[#{status}] #{message}" : message
+        super(msg)
+      end
+    end
 
     # Base class for fetch backends, plus the errors they raise.
     #
@@ -66,7 +87,14 @@ module Ask
       # challenge/interstitial pages carry these markers, while legitimate
       # pages can contain the word "captcha" in unrelated config/JS (e.g.
       # Wikipedia embeds an hcaptcha edit-config flag on every page).
-      CHALLENGE_RE = /just a moment|checking your browser|cf-chl/i
+      # Cf-chl: a Turnstile form widget uses `cf-chl-widget-*` + `cf-turnstile-response`
+      # (a per-form CAPTCHA, not the `cf-chl` managed challenge that gates
+      # the whole page), so matching bare `cf-chl` on the body misclassifies
+      # every Turnstile form (openai.com/form/codex-for-oss) as a challenge.
+      # Accept both the classic managed-challenge markers (`challenge-platform`,
+      # `_cf_chl_opt`) and the bare `cf-chl` id, but the plain widget id is
+      # explicitly NOT a challenge — see challenge_page? below.
+      CHALLENGE_RE = /just a moment|checking your browser|cf-chl|challenge-platform|_cf_chl_opt/i
 
       # Registrar parking-page markers: the page is an ad for a parked
       # (for-sale) domain, not the site's content. A content company must
@@ -159,7 +187,17 @@ module Ask
       private
 
       def challenge_page?(body)
-        body.to_s.match?(CHALLENGE_RE)
+        text = body.to_s
+        return false unless text.match?(CHALLENGE_RE)
+        # Bare `cf-chl-widget-*` is a Turnstile form widget, not the
+        # Cloudflare managed challenge interstitial. The interstitial's
+        # `cf-chl` comes with `challenge-platform` / `_cf_chl_opt` /
+        # "just a moment" next to it; a page whose only hit is the widget
+        # id (openai.com form pages) is NOT a challenge.
+        return false if text.include?('cf-chl-widget') &&
+                        !text.match?(/challenge-platform|_cf_chl_opt|just a moment|checking your browser/i)
+
+        true
       end
 
       def parked_domain?(body)

data/lib/ask/web_fetch/backends/browser.rb

@@ -206,7 +206,10 @@ module Ask
           wait_for_idle(page)
 
           status = page.network.status
-          raise FetchError, "got #{status} at #{url}" if status && status >= 400
+          if status && status >= 400
+            error_class = status == 404 ? NotFoundError : FetchError
+            raise error_class.new("got #{status} at #{url}", status: status)
+          end
 
           body = page.body
           if challenge_page?(body)

data/lib/ask/web_fetch/backends/jina.rb

@@ -49,12 +49,14 @@ module Ask
 
             { title: nil, description: nil, content: content, outlinks: markdown_outlinks(body, url) }
           when '429'
-            raise ServerError, 'rate limited by Jina (429)'
+            raise ServerError.new('rate limited by Jina', status: 429)
           when '401', '403'
-            raise FetchError, "Jina access error (#{res.code})"
+            raise FetchError.new("Jina access error", status: res.code.to_i)
           else
             # 5xx = Jina-side blip (transient); other 4xx = the URL is dead.
-            raise(res.code.to_i >= 500 ? ServerError : FetchError, "Jina returned #{res.code}")
+            code = res.code.to_i
+            error_class = code == 404 ? NotFoundError : (code >= 500 ? ServerError : FetchError)
+            raise error_class.new("Jina returned #{res.code}", status: code)
           end
         rescue Net::OpenTimeout, Net::ReadTimeout, Errno::ECONNREFUSED,
                Errno::ECONNRESET, SocketError, URI::InvalidURIError => e

data/lib/ask/web_fetch/backends/local.rb

@@ -38,13 +38,42 @@ module Ask
           end
         end
 
+        # Agent-first content negotiation: many sites now serve clean
+        # markdown when asked via Accept: text/markdown, a .md URL twin,
+        # or an /llms.txt manifest. We probe these low-cost paths before
+        # falling back to full HTML scrape + DOM conversion.
+        #
+        # Order: (1) Accept: text/markdown on the original URL — the
+        # cheapest probe, one extra GET; (2) the .md twin — Mintlify-
+        # style sites redirect .md with content-type text/plain; (3) the
+        # full HTML scrape. llms.txt manifests are upstream of individual
+        # pages (they index the site) and are tried by the MCP tool
+        # layer, not per-URL — that avoids duplicate fetches when the
+        # same manifest covers multiple URLs.
         def fetch(url)
-          body, content_type, redirect = fetch_html(url)
-          raise FetchError, "expected HTML from #{url}, got #{content_type}" unless content_type.include?('html')
-          raise FetchError, "challenge page at #{url}" if challenge_page?(body)
+          # Probe 1: server content negotiation
+          md_body, md_ct, md_redirect = fetch_markdown(url)
+          if md_body && !md_body.empty?
+            return assemble_page(md_body, url, md_redirect, source: :accept_header)
+          end
+
+          # Probe 2: .md URL twin (Mintlify, Docusaurus, some Hugo sites)
+          twin = "#{url.chomp('/')}.md"
+          md_body, md_ct, md_redirect = fetch_markdown(twin)
+          if md_body && !md_body.empty?
+            return assemble_page(md_body, url, md_redirect, source: :md_twin, twin_url: twin)
+          end
+
+          # Probe 3: full HTML scrape (legacy path)
+          body, content_type, redirect, status = fetch_html(url)
+          unless content_type.include?('html')
+            raise FetchError.new("expected HTML from #{url}, got #{content_type}", status: status)
+          end
+          raise FetchError.new("challenge page at #{url}", status: status) if challenge_page?(body)
 
           page = to_markdown(body, url)
           page[:redirected] = redirect
+          page[:status] = status
           # Parked-domain pages are not content: the domain owner parked it
           # with a registrar and the page is an ad for buying the domain
           # (GoDaddy/Namecheap/Sedo parking). A content company must never
@@ -53,6 +82,14 @@ module Ask
           # server-rendered — the HTML-only markers live in scripts and
           # assets), then the content minimum, then the JS-shell
           # completeness signal below.
+          # 404 pages sometimes carry usable content (custom error pages
+          # with navigation, suggestions). Try to extract it before giving
+          # up — only raise NotFoundError if the content is genuinely empty.
+          if status == 404
+            return page if usable_content?(page[:content])
+            raise NotFoundError.new("not found at #{url}", status: 404)
+          end
+
           guard_page!(url, page[:content], raw_body: body)
           # The completeness signal: a JS-app shell whose server HTML
           # renders little is a TRUNCATED page, not a complete one — the
@@ -157,8 +194,8 @@ module Ask
         end
 
         # GET with redirect following (max MAX_REDIRECTS hops). Returns
-        # [body, content_type, redirect] where redirect is nil when the
-        # URL answered directly, else {status: first hop's status,
+        # [body, content_type, redirect, status] where redirect is nil
+        # when the URL answered directly, else {status: first hop's status,
         # url: final destination} — the chain the crawler followed.
         def fetch_html(url)
           uri = URI(url)
@@ -166,7 +203,9 @@ module Ask
           first_hop_status = nil
           loop do
             response = self.class.http.get(uri.to_s, headers: { 'accept' => 'text/html,application/xhtml+xml' })
-            return [response.body, response.content_type, redirect_info(first_hop_status, uri)] if (200..299).cover?(response.status)
+            if (200..299).cover?(response.status)
+              return [response.body, response.content_type, redirect_info(first_hop_status, uri), response.status]
+            end
 
             unless (300..399).cover?(response.status) && !response.location.empty?
               # 4xx (other than 429) = the URL is dead; 429/5xx = transient.
@@ -183,6 +222,60 @@ module Ask
         def redirect_info(status, uri)
           status && { status: status, url: uri.to_s }
         end
+
+        # Probes +url+ with Accept: text/markdown. Returns
+        # [body, content_type, redirect] on a text/markdown response, or
+        # nils when the server returned HTML (or anything else the caller
+        # shouldn't treat as agent-native). Follows one redirect hop —
+        # enough for Mintlify's 307 → .md twin.
+        def fetch_markdown(url)
+          uri = URI(url)
+          response = self.class.http.get(
+            uri.to_s,
+            headers: { 'accept' => 'text/markdown' }
+          )
+          return [nil, nil, nil] unless response
+          return [nil, nil, nil] if response.status >= 400
+
+          redirect_info = nil
+
+          # Follow a single redirect (Mintlify 307 → .md twin)
+          if (300..399).cover?(response.status) && !response.location.empty?
+            redirect_uri = URI.join(uri, response.location)
+            redirect_info = { status: response.status, url: redirect_uri.to_s }
+            response = self.class.http.get(
+              redirect_uri.to_s,
+              headers: { 'accept' => 'text/markdown' }
+            )
+            return [nil, nil, nil] unless response && response.status == 200
+          end
+
+          ct = response.content_type.to_s.downcase
+          return [nil, nil, nil] unless ct.include?('text/markdown') && response.status == 200
+
+          [response.body, ct, redirect_info]
+        rescue StandardError
+          [nil, nil, nil]
+        end
+
+        # Assembles a page hash from agent-native markdown (Accept or .md
+        # twin), skipping the HTML→markdown conversion pipeline. Runs the
+        # shared guards (parked domain, minimum content) so downstream
+        # behavior is identical regardless of source.
+        def assemble_page(markdown, url, redirect, source:, twin_url: nil)
+          source_url = twin_url || url
+          page = {
+            title: nil,
+            description: nil,
+            content: Markdown.clean(markdown),
+            redirected: redirect,
+            licenses: [],
+            outlinks: markdown_outlinks(markdown, source_url)
+          }
+          guard_page!(url, page[:content])
+
+          page
+        end
       end
     end
   end

data/lib/ask/web_fetch/version.rb

@@ -2,6 +2,6 @@
 
 module Ask
   module WebFetch
-    VERSION = '0.7.2'
+    VERSION = '0.7.4'
   end
 end

data/lib/ask/web_fetch.rb

@@ -37,7 +37,7 @@ module Ask
     # re-raises as FetchError so callers can fail fast; any transient
     # failure in the mix (timeout, 5xx, empty render) keeps the base
     # Error, which recovers on retry.
-    DETERMINISTIC = [Ask::WebFetch::FetchError, Ask::WebFetch::EmptyContentError].freeze
+    DETERMINISTIC = [Ask::WebFetch::FetchError, Ask::WebFetch::NotFoundError, Ask::WebFetch::EmptyContentError].freeze
 
     # Backend chain, tried in order. Crawl4AI leads when configured
     # (CRAWL4AI_URL), so a present self-hosted renderer is preferred;
@@ -83,8 +83,17 @@ module Ask
     # ParkedDomainError / EmptyContentError / FetchError are terminal —
     # retrying never changes the answer; Error may recover on retry.
     def self.collapse(failures, url)
-      detail = failures.map { |backend, e| "#{backend.backend_name}: #{e.message}" }.join('; ')
-      message = "all web fetch backends failed for #{url} (#{detail})"
+      # When all backends agree on the same root cause (same HTTP status),
+      # say so cleanly instead of listing every backend's echo of the same
+      # problem. Status is now on the error object itself (FetchError#status,
+      # NotFoundError#status), so we don't parse messages.
+      statuses = failures.filter_map { |_, e| e.respond_to?(:status) && e.status }
+      detail = if statuses.uniq.size == 1 && statuses.size == failures.size
+                 "[#{statuses.first}]"
+               else
+                 failures.map { |backend, e| "#{backend.backend_name}: #{e.message}" }.join('; ')
+               end
+      message = "#{detail} #{url}"
 
       classes = failures.map { |_, e| e.class }
       if classes.any? { |k| k <= Ask::WebFetch::ParkedDomainError }
@@ -93,6 +102,9 @@ module Ask
       if classes.any? { |k| k <= Ask::WebFetch::EmptyContentError }
         raise Ask::WebFetch::EmptyContentError, message
       end
+      if classes.any? { |k| k == Ask::WebFetch::NotFoundError }
+        raise Ask::WebFetch::NotFoundError, message
+      end
 
       deterministic = failures.all? { |_, e| DETERMINISTIC.any? { |klass| e.is_a?(klass) } }
       raise(deterministic ? Ask::WebFetch::FetchError : Ask::WebFetch::Error, message)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ask-web-fetch
 version: !ruby/object:Gem::Version
-  version: 0.7.2
+  version: 0.7.4
 platform: ruby
 authors:
 - Kaka Ruto