ask-web-fetch 0.5.8 → 0.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d3e0494d6f9fd54b7b80b667b97d57f17d1b0e871d44c26fada9ad91669a391d
-  data.tar.gz: e06918a4ca162ee2111715530f67ab68c8feae529c27c180cebb0cf462f13867
+  metadata.gz: ac8355feef3bd62bc048223f0390f79d95506f084817853a1a572be5eff4a78a
+  data.tar.gz: ccee31246284b6236732984c5f1d32fcf54d6ffa9203fa1f51150300b4b50702
 SHA512:
-  metadata.gz: 4a045bb9226bb302cd60bda942acfa05a00e1f3d1aced99edc3bbd77a061c7650580b40447c385a4777b7156fa81aa0fcc57267caa4385b478e4d005c015f41f
-  data.tar.gz: ffab19b64968c909c67dd631902b447738248c7e6a0d3da5db8851cc2ff615ec627bc8a89407b40dbeabbebb591d2cd6726c21747de34962cc9ff77c98ce16e8
+  metadata.gz: 181316376680bbb70d193b46dd898f48547b1846726e2d5f3d0596080ed4b41ca37ae818e3bc2963e0e2e700b47ff8a2957420222c0a737e45acedd00318e68c
+  data.tar.gz: 0b039ee976010c65274c956a3f5e3182164128b6a096eae1161ea42f266004fdd878dde12378899d68a29f26b56c2f454344e32abbd65d0444367f8948e330de

data/lib/ask/web_fetch/backend.rb

@@ -16,6 +16,13 @@ module Ask
     # A backend that fetched the page but found nothing usable in it.
     class EmptyContentError < Error; end
 
+    # A backend that fetched a REGISTRAR PARKING PAGE — an ad for a
+    # parked (for-sale) domain, not the site's content. Deterministic and
+    # terminal: retrying will never turn a parking ad into content, so
+    # the pipeline must classify (not retry) it. A subclass of
+    # EmptyContentError so existing empty-content handling still applies.
+    class ParkedDomainError < EmptyContentError; end
+
     # Network-level failure — timeout, connection refused/reset, bad
     # socket. Transient: the same URL may succeed on retry.
     class TimeoutError < Error; end

data/lib/ask/web_fetch/backends/browser.rb

@@ -201,8 +201,9 @@ module Ask
           end
           # Browser renders the parked page a JS redirect lands on (the
           # server shell hands /lander to JS) — Local never sees it. The
-          # shared parked-domain detector catches it here.
-          raise EmptyContentError, "parked domain at #{url} — registrar parking page, not site content" if parked_domain?(body)
+          # shared parked-domain detector catches it here; the distinct
+          # ParkedDomainError lets the pipeline classify (never retry) it.
+          raise ParkedDomainError, "parked domain at #{url} — registrar parking page, not site content" if parked_domain?(body)
 
           result = Markdown.generate(body, base_url: url, filter: self.class.content_filter)
           result[:outlinks] = outlink_urls(body, url)

data/lib/ask/web_fetch/backends/local.rb

@@ -55,7 +55,7 @@ module Ask
           # parked pages are fully server-rendered, so both Local and
           # Browser see the same ad.
           if parked_domain?(body)
-            raise EmptyContentError, "parked domain at #{url} — registrar parking page, not site content"
+            raise ParkedDomainError, "parked domain at #{url} — registrar parking page, not site content"
           end
           raise EmptyContentError, "no readable content at #{url}" unless usable_content?(page[:content])
           # The completeness signal: a JS-app shell whose server HTML

data/lib/ask/web_fetch/tool.rb

@@ -21,6 +21,13 @@ module Ask
     class WebFetch < Ask::Tool
       DEFAULT_MAX_CHARS = 20_000
 
+      # Errors that mean "the URL is dead" — no amount of retrying changes
+      # the answer. When EVERY backend failed this way, the aggregate
+      # re-raises as FetchError so callers can fail fast; any transient
+      # failure in the mix (timeout, 5xx, empty render) keeps the base
+      # Error, which recovers on retry.
+      DETERMINISTIC = [Ask::WebFetch::FetchError, Ask::WebFetch::EmptyContentError].freeze
+
       # Backend chain, tried in order. Crawl4AI leads when configured
       # (CRAWL4AI_URL), so a present self-hosted renderer is preferred;
       # otherwise Local, with Jina as the last resort, and Browser appended
@@ -37,9 +44,34 @@ module Ask
         end
       end
 
-      class << self
-        attr_writer :backends
+    class << self
+      attr_writer :backends
+
+      # Collapses every backend's failure into ONE error whose class
+      # carries the best explanation. Precedence, most definitive first: a
+      # parked domain beats an empty shell (the shell IS the parking ad's
+      # shell — Local sees the JS redirect stub, Browser the lander),
+      # empty beats a dead 4xx (the page existed, it just had no content),
+      # and any deterministic explanation beats a transient one (transient
+      # keeps the retryable base Error). Clients read the class:
+      # ParkedDomainError / EmptyContentError / FetchError are terminal —
+      # retrying never changes the answer; Error may recover on retry.
+      def collapse(failures, url)
+        detail = failures.map { |backend, e| "#{backend.backend_name}: #{e.message}" }.join('; ')
+        message = "all web fetch backends failed for #{url} (#{detail})"
+
+        classes = failures.map { |_, e| e.class }
+        if classes.any? { |k| k <= Ask::WebFetch::ParkedDomainError }
+          raise Ask::WebFetch::ParkedDomainError, message
+        end
+        if classes.any? { |k| k <= Ask::WebFetch::EmptyContentError }
+          raise Ask::WebFetch::EmptyContentError, message
+        end
+
+        deterministic = failures.all? { |_, e| DETERMINISTIC.any? { |klass| e.is_a?(klass) } }
+        raise(deterministic ? Ask::WebFetch::FetchError : Ask::WebFetch::Error, message)
       end
+    end
 
       description 'Fetch a URL and return its content as clean markdown for LLM consumption. ' \
                   'Use this to read web pages, articles, and documentation.'
@@ -68,10 +100,9 @@ module Ask
         self.class.backends.each do |backend_class|
           return backend_class.new.fetch(url)
         rescue Ask::WebFetch::Error => e
-          failures << "#{backend_class.backend_name}: #{e.message}"
+          failures << [backend_class, e]
         end
-        raise Ask::WebFetch::Error,
-              "all web fetch backends failed for #{url} (#{failures.join('; ')})"
+        self.class.collapse(failures, url)
       end
 
       def format(page, url)

data/lib/ask/web_fetch/version.rb

@@ -2,6 +2,6 @@
 
 module Ask
   module WebFetch
-    VERSION = '0.5.8'
+    VERSION = '0.6.0'
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ask-web-fetch
 version: !ruby/object:Gem::Version
-  version: 0.5.8
+  version: 0.6.0
 platform: ruby
 authors:
 - Kaka Ruto