ask-web-fetch 0.7.1 → 0.7.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: efa999aac8d4321c2e8e6d463ace6c3f5ec5e95a62973ad06593c4714e1ffab1
-  data.tar.gz: 9a9e59e8da81bb4238bf1a08b0ba84634553dfd3caac99763a7bb52a54d9ac6e
+  metadata.gz: b2dd2d3ee4ae5d53f7e1bf441d3b598be5195d3aafd58119213b7b739ad900db
+  data.tar.gz: 49d8aa803e8f4871e278d6fbe7ad71b6a6826c5afe62c8dc8f38abc2b8b4cb8e
 SHA512:
-  metadata.gz: af1e312f91b9e5ea72679b7e877232cc3e4e9639b3fdf3e98291810377a450a5ce180bb4664ca651c8ccd2084220632890da92fe4eaf8c1ca472e6ba464d60ff
-  data.tar.gz: 3951989986d476f6bc008712e6451ccb148c9fbe33537544061486d15c79443e0368dd668ef40b35287173ad41f569fc519130a9cb5368e78da16eafbcdf2c91
+  metadata.gz: 95f40993df27de4c059f085ece2d71b5ea8c1e72c2d3d3dc42e96d25a1d5d87a400f5905cbaf6685675eb064ddebe95fa110609f16b93f586a5a31e02e17917c
+  data.tar.gz: a5b8513eef5cad48fbed247a2578e662c42a401b7b8e16028e5b6fbfa90d6b28da9e3db5786c20d43182ececf5a490ff98338a4fbf3ba286b0e4f9123210dea2

data/lib/ask/web_fetch/backend.rb

@@ -66,7 +66,14 @@ module Ask
       # challenge/interstitial pages carry these markers, while legitimate
       # pages can contain the word "captcha" in unrelated config/JS (e.g.
       # Wikipedia embeds an hcaptcha edit-config flag on every page).
-      CHALLENGE_RE = /just a moment|checking your browser|cf-chl/i
+      # Cf-chl: a Turnstile form widget uses `cf-chl-widget-*` + `cf-turnstile-response`
+      # (a per-form CAPTCHA, not the `cf-chl` managed challenge that gates
+      # the whole page), so matching bare `cf-chl` on the body misclassifies
+      # every Turnstile form (openai.com/form/codex-for-oss) as a challenge.
+      # Accept both the classic managed-challenge markers (`challenge-platform`,
+      # `_cf_chl_opt`) and the bare `cf-chl` id, but the plain widget id is
+      # explicitly NOT a challenge — see challenge_page? below.
+      CHALLENGE_RE = /just a moment|checking your browser|cf-chl|challenge-platform|_cf_chl_opt/i
 
       # Registrar parking-page markers: the page is an ad for a parked
       # (for-sale) domain, not the site's content. A content company must
@@ -159,7 +166,17 @@ module Ask
       private
 
       def challenge_page?(body)
-        body.to_s.match?(CHALLENGE_RE)
+        text = body.to_s
+        return false unless text.match?(CHALLENGE_RE)
+        # Bare `cf-chl-widget-*` is a Turnstile form widget, not the
+        # Cloudflare managed challenge interstitial. The interstitial's
+        # `cf-chl` comes with `challenge-platform` / `_cf_chl_opt` /
+        # "just a moment" next to it; a page whose only hit is the widget
+        # id (openai.com form pages) is NOT a challenge.
+        return false if text.include?('cf-chl-widget') &&
+                        !text.match?(/challenge-platform|_cf_chl_opt|just a moment|checking your browser/i)
+
+        true
       end
 
       def parked_domain?(body)

data/lib/ask/web_fetch/backends/browser.rb

@@ -96,6 +96,12 @@ module Ask
             browser_mutex.synchronize { @browser ||= build_browser }
           end
 
+          # Drops the shared browser so the next use builds a fresh
+          # session — called when the current one died mid-fetch.
+          def reset_browser
+            browser_mutex.synchronize { @browser = nil }
+          end
+
           def content_filter
             @content_filter ||= ContentFilter.default
           end
@@ -162,7 +168,17 @@ module Ask
 
           page = self.class.browser.create_page
           fetch_attempt(page, url)
-        rescue Ferrum::TimeoutError, Ferrum::ProcessTimeoutError, Ferrum::DeadBrowserError => e
+        rescue Ferrum::DeadBrowserError => e
+          # The shared browser died mid-fetch (the browserless server
+          # killed the session's browser). A dead browser is transient —
+          # the next session starts a fresh one — so reconnect and retry
+          # ONCE instead of failing the page (observed 2026-08-14: the
+          # server's session limits killed browsers and pages were
+          # silently dropped).
+          self.class.reset_browser
+          page = self.class.browser.create_page
+          fetch_attempt(page, url)
+        rescue Ferrum::TimeoutError, Ferrum::ProcessTimeoutError => e
           raise TimeoutError, "#{e.class}: #{e.message}"
         rescue Ferrum::StatusError => e
           raise FetchError, "browser could not load #{url}: #{e.message}"

data/lib/ask/web_fetch/backends/local.rb

@@ -38,7 +38,33 @@ module Ask
           end
         end
 
+        # Agent-first content negotiation: many sites now serve clean
+        # markdown when asked via Accept: text/markdown, a .md URL twin,
+        # or an /llms.txt manifest. We probe these low-cost paths before
+        # falling back to full HTML scrape + DOM conversion.
+        #
+        # Order: (1) Accept: text/markdown on the original URL — the
+        # cheapest probe, one extra GET; (2) the .md twin — Mintlify-
+        # style sites redirect .md with content-type text/plain; (3) the
+        # full HTML scrape. llms.txt manifests are upstream of individual
+        # pages (they index the site) and are tried by the MCP tool
+        # layer, not per-URL — that avoids duplicate fetches when the
+        # same manifest covers multiple URLs.
         def fetch(url)
+          # Probe 1: server content negotiation
+          md_body, md_ct, md_redirect = fetch_markdown(url)
+          if md_body && !md_body.empty?
+            return assemble_page(md_body, url, md_redirect, source: :accept_header)
+          end
+
+          # Probe 2: .md URL twin (Mintlify, Docusaurus, some Hugo sites)
+          twin = "#{url.chomp('/')}.md"
+          md_body, md_ct, md_redirect = fetch_markdown(twin)
+          if md_body && !md_body.empty?
+            return assemble_page(md_body, url, md_redirect, source: :md_twin, twin_url: twin)
+          end
+
+          # Probe 3: full HTML scrape (legacy path)
           body, content_type, redirect = fetch_html(url)
           raise FetchError, "expected HTML from #{url}, got #{content_type}" unless content_type.include?('html')
           raise FetchError, "challenge page at #{url}" if challenge_page?(body)
@@ -63,7 +89,7 @@ module Ask
           # and keeps a partial page from ever being stored as the real
           # thing. Two detectors: known framework markers, or a large
           # HTML page with almost no server-rendered text.
-          if js_app_shell?(body) && page[:content].length < SHELL_CONTENT_THRESHOLD
+          if js_app_shell?(body) && page[:content].length < SHELL_DEFER_THRESHOLD
             raise EmptyContentError,
               "JS-app shell at #{url} — server HTML renders only #{page[:content].length} chars; a rendering backend is required"
           end
@@ -82,10 +108,19 @@ module Ask
         # specific footprints of React/Vue/Next/Nuxt app shells.
         JS_APP_SHELL_MARKERS = /id=["'](?:root|app|__next|site-content)["']|__NEXT_DATA__|window\.__NUXT__|ng-app|data-reactroot/
 
-        # Markdown below this from a JS-app shell is "server sent a shell",
-        # not "page is genuinely short" — a real page (even a short one)
-        # is usually server-rendered above this. Tunable; the chain turns
-        # the signal into "prefer Browser for this URL".
+        # A framework marker (React/Vue/Next) alone is NOT emptiness: a
+        # marked page can server-render real content (careers.abb job
+        # pages: 2,162 chars of job description) and must be stored as-is
+        # when the rendering backends cannot do better — dropping it lost
+        # real pages (2026-08-14). The shell deferral fires only when the
+        # extraction is genuinely little: below this, the page is a true
+        # shell (airbnb: 613KB HTML -> 143 chars). Tunable; the chain
+        # turns the signal into "prefer Browser for this URL".
+        SHELL_DEFER_THRESHOLD = 500
+
+        # The ratio detector's own bar, kept for compatibility with the
+        # comment below (large HTML + near-empty text is a shell whatever
+        # the framework).
         SHELL_CONTENT_THRESHOLD = 4_000
 
         # A page whose server HTML is large but yields almost no text is a
@@ -174,6 +209,60 @@ module Ask
         def redirect_info(status, uri)
           status && { status: status, url: uri.to_s }
         end
+
+        # Probes +url+ with Accept: text/markdown. Returns
+        # [body, content_type, redirect] on a text/markdown response, or
+        # nils when the server returned HTML (or anything else the caller
+        # shouldn't treat as agent-native). Follows one redirect hop —
+        # enough for Mintlify's 307 → .md twin.
+        def fetch_markdown(url)
+          uri = URI(url)
+          response = self.class.http.get(
+            uri.to_s,
+            headers: { 'accept' => 'text/markdown' }
+          )
+          return [nil, nil, nil] unless response
+          return [nil, nil, nil] if response.status >= 400
+
+          redirect_info = nil
+
+          # Follow a single redirect (Mintlify 307 → .md twin)
+          if (300..399).cover?(response.status) && !response.location.empty?
+            redirect_uri = URI.join(uri, response.location)
+            redirect_info = { status: response.status, url: redirect_uri.to_s }
+            response = self.class.http.get(
+              redirect_uri.to_s,
+              headers: { 'accept' => 'text/markdown' }
+            )
+            return [nil, nil, nil] unless response && response.status == 200
+          end
+
+          ct = response.content_type.to_s.downcase
+          return [nil, nil, nil] unless ct.include?('text/markdown') && response.status == 200
+
+          [response.body, ct, redirect_info]
+        rescue StandardError
+          [nil, nil, nil]
+        end
+
+        # Assembles a page hash from agent-native markdown (Accept or .md
+        # twin), skipping the HTML→markdown conversion pipeline. Runs the
+        # shared guards (parked domain, minimum content) so downstream
+        # behavior is identical regardless of source.
+        def assemble_page(markdown, url, redirect, source:, twin_url: nil)
+          source_url = twin_url || url
+          page = {
+            title: nil,
+            description: nil,
+            content: Markdown.clean(markdown),
+            redirected: redirect,
+            licenses: [],
+            outlinks: markdown_outlinks(markdown, source_url)
+          }
+          guard_page!(url, page[:content])
+
+          page
+        end
       end
     end
   end

data/lib/ask/web_fetch/version.rb

@@ -2,6 +2,6 @@
 
 module Ask
   module WebFetch
-    VERSION = '0.7.1'
+    VERSION = '0.7.3'
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ask-web-fetch
 version: !ruby/object:Gem::Version
-  version: 0.7.1
+  version: 0.7.3
 platform: ruby
 authors:
 - Kaka Ruto
@@ -171,7 +171,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 4.0.3
+rubygems_version: 4.0.18
 specification_version: 4
 summary: Web fetch library for the ask-rb ecosystem
 test_files: []