ask-web-fetch 0.7.2 → 0.7.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3ad48d09aa8869b4193a703ad6a760753c2767c9e26a6919b1405a5fb186c7a4
-  data.tar.gz: 52960016a3d4740f4913d4879cbf80edacf95e4ffc07601e6dde0d80515ffe81
+  metadata.gz: b2dd2d3ee4ae5d53f7e1bf441d3b598be5195d3aafd58119213b7b739ad900db
+  data.tar.gz: 49d8aa803e8f4871e278d6fbe7ad71b6a6826c5afe62c8dc8f38abc2b8b4cb8e
 SHA512:
-  metadata.gz: fa6d778abc6918bb7ae839d4f9c96dafe215014a8b2f4efc8027ab1e49d287e3ca57204f54f013531e38dff7393ac23f6da9619b3afd48efae8b6f7e42dfa0d9
-  data.tar.gz: 7d4e5fe0e70b28cc3349ec96d071d4e1cc7a46c2c21ca8f41c35968d8b15221854c2418fc72ae2a8e56e97c07930592b2ee5b79da921c4a4ccb4fe6990c7e938
+  metadata.gz: 95f40993df27de4c059f085ece2d71b5ea8c1e72c2d3d3dc42e96d25a1d5d87a400f5905cbaf6685675eb064ddebe95fa110609f16b93f586a5a31e02e17917c
+  data.tar.gz: a5b8513eef5cad48fbed247a2578e662c42a401b7b8e16028e5b6fbfa90d6b28da9e3db5786c20d43182ececf5a490ff98338a4fbf3ba286b0e4f9123210dea2

data/lib/ask/web_fetch/backend.rb

@@ -66,7 +66,14 @@ module Ask
       # challenge/interstitial pages carry these markers, while legitimate
       # pages can contain the word "captcha" in unrelated config/JS (e.g.
       # Wikipedia embeds an hcaptcha edit-config flag on every page).
-      CHALLENGE_RE = /just a moment|checking your browser|cf-chl/i
+      # Cf-chl: a Turnstile form widget uses `cf-chl-widget-*` + `cf-turnstile-response`
+      # (a per-form CAPTCHA, not the `cf-chl` managed challenge that gates
+      # the whole page), so matching bare `cf-chl` on the body misclassifies
+      # every Turnstile form (openai.com/form/codex-for-oss) as a challenge.
+      # Accept both the classic managed-challenge markers (`challenge-platform`,
+      # `_cf_chl_opt`) and the bare `cf-chl` id, but the plain widget id is
+      # explicitly NOT a challenge — see challenge_page? below.
+      CHALLENGE_RE = /just a moment|checking your browser|cf-chl|challenge-platform|_cf_chl_opt/i
 
       # Registrar parking-page markers: the page is an ad for a parked
       # (for-sale) domain, not the site's content. A content company must
@@ -159,7 +166,17 @@ module Ask
       private
 
       def challenge_page?(body)
-        body.to_s.match?(CHALLENGE_RE)
+        text = body.to_s
+        return false unless text.match?(CHALLENGE_RE)
+        # Bare `cf-chl-widget-*` is a Turnstile form widget, not the
+        # Cloudflare managed challenge interstitial. The interstitial's
+        # `cf-chl` comes with `challenge-platform` / `_cf_chl_opt` /
+        # "just a moment" next to it; a page whose only hit is the widget
+        # id (openai.com form pages) is NOT a challenge.
+        return false if text.include?('cf-chl-widget') &&
+                        !text.match?(/challenge-platform|_cf_chl_opt|just a moment|checking your browser/i)
+
+        true
       end
 
       def parked_domain?(body)

data/lib/ask/web_fetch/backends/local.rb

@@ -38,7 +38,33 @@ module Ask
           end
         end
 
+        # Agent-first content negotiation: many sites now serve clean
+        # markdown when asked via Accept: text/markdown, a .md URL twin,
+        # or an /llms.txt manifest. We probe these low-cost paths before
+        # falling back to full HTML scrape + DOM conversion.
+        #
+        # Order: (1) Accept: text/markdown on the original URL — the
+        # cheapest probe, one extra GET; (2) the .md twin — Mintlify-
+        # style sites redirect .md with content-type text/plain; (3) the
+        # full HTML scrape. llms.txt manifests are upstream of individual
+        # pages (they index the site) and are tried by the MCP tool
+        # layer, not per-URL — that avoids duplicate fetches when the
+        # same manifest covers multiple URLs.
         def fetch(url)
+          # Probe 1: server content negotiation
+          md_body, md_ct, md_redirect = fetch_markdown(url)
+          if md_body && !md_body.empty?
+            return assemble_page(md_body, url, md_redirect, source: :accept_header)
+          end
+
+          # Probe 2: .md URL twin (Mintlify, Docusaurus, some Hugo sites)
+          twin = "#{url.chomp('/')}.md"
+          md_body, md_ct, md_redirect = fetch_markdown(twin)
+          if md_body && !md_body.empty?
+            return assemble_page(md_body, url, md_redirect, source: :md_twin, twin_url: twin)
+          end
+
+          # Probe 3: full HTML scrape (legacy path)
           body, content_type, redirect = fetch_html(url)
           raise FetchError, "expected HTML from #{url}, got #{content_type}" unless content_type.include?('html')
           raise FetchError, "challenge page at #{url}" if challenge_page?(body)
@@ -183,6 +209,60 @@ module Ask
         def redirect_info(status, uri)
           status && { status: status, url: uri.to_s }
         end
+
+        # Probes +url+ with Accept: text/markdown. Returns
+        # [body, content_type, redirect] on a text/markdown response, or
+        # nils when the server returned HTML (or anything else the caller
+        # shouldn't treat as agent-native). Follows one redirect hop —
+        # enough for Mintlify's 307 → .md twin.
+        def fetch_markdown(url)
+          uri = URI(url)
+          response = self.class.http.get(
+            uri.to_s,
+            headers: { 'accept' => 'text/markdown' }
+          )
+          return [nil, nil, nil] unless response
+          return [nil, nil, nil] if response.status >= 400
+
+          redirect_info = nil
+
+          # Follow a single redirect (Mintlify 307 → .md twin)
+          if (300..399).cover?(response.status) && !response.location.empty?
+            redirect_uri = URI.join(uri, response.location)
+            redirect_info = { status: response.status, url: redirect_uri.to_s }
+            response = self.class.http.get(
+              redirect_uri.to_s,
+              headers: { 'accept' => 'text/markdown' }
+            )
+            return [nil, nil, nil] unless response && response.status == 200
+          end
+
+          ct = response.content_type.to_s.downcase
+          return [nil, nil, nil] unless ct.include?('text/markdown') && response.status == 200
+
+          [response.body, ct, redirect_info]
+        rescue StandardError
+          [nil, nil, nil]
+        end
+
+        # Assembles a page hash from agent-native markdown (Accept or .md
+        # twin), skipping the HTML→markdown conversion pipeline. Runs the
+        # shared guards (parked domain, minimum content) so downstream
+        # behavior is identical regardless of source.
+        def assemble_page(markdown, url, redirect, source:, twin_url: nil)
+          source_url = twin_url || url
+          page = {
+            title: nil,
+            description: nil,
+            content: Markdown.clean(markdown),
+            redirected: redirect,
+            licenses: [],
+            outlinks: markdown_outlinks(markdown, source_url)
+          }
+          guard_page!(url, page[:content])
+
+          page
+        end
       end
     end
   end

data/lib/ask/web_fetch/version.rb

@@ -2,6 +2,6 @@
 
 module Ask
   module WebFetch
-    VERSION = '0.7.2'
+    VERSION = '0.7.3'
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ask-web-fetch
 version: !ruby/object:Gem::Version
-  version: 0.7.2
+  version: 0.7.3
 platform: ruby
 authors:
 - Kaka Ruto