ask-web-fetch 0.5.9 → 0.6.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d8f3dc79c573c21dc567f86edd621ad84da98437ef512faea0e13b36219670d2
-  data.tar.gz: 694519194d63c9f4066f943ed2fe472f029ea10afd6e423a41440ff7d8a04964
+  metadata.gz: 15d4465ae7ca7805003043af5b97e96ddc8c2ba19618daab313377f4a529bd54
+  data.tar.gz: 9f83f8c2258dc973457158e7d7e2288afc43403221453545a4d19c97440b85ee
 SHA512:
-  metadata.gz: 270301527344d9b49c982e1d7c27dd83117f23c56de3e321f67e15aae7696998d423bd7cec72852a188412ee54b4183736917a5ec6c7030eab888b621af611f1
-  data.tar.gz: 66933232dbb0f106362e2a9552f19fc152022c5f9ed9b7ca4bda6fe1d2b3935ce3a3b7288bc2d0130aa8d9bddd6a4e5861ff5ed675e11b763b0e0d171e5749ef
+  metadata.gz: b34dfd5f3d45b827cf8882d3c9c3b47e05ea8c16622849b2a41fc51b8c5f005f0b149ef83a87db57f4167470e85911ec58f72830afded9a84ab707ea08dcbc68
+  data.tar.gz: c8245c574d480179cd3b8ad67f1049b096fdf298f0886a790cb557f698c11e8dbaed9bb4336de4ccf84882e03eb8cf716830db069988f310c35cf4202cf01d60

data/lib/ask/web_fetch/backends/crawl4ai.rb

@@ -55,6 +55,12 @@ module Ask
           raise FetchError, "challenge page at #{url}" if challenge_page?(body)
 
           page = to_page(body, url)
+          # A registrar parking page renders fine in headless Chrome too —
+          # the shared detector (0.5.7) catches the text markers that
+          # survive conversion, so the ad is rejected, not returned as
+          # the site's content. Crawl4AI leads the default chain, so this
+          # check is what keeps parked domains out of every result.
+          raise ParkedDomainError, "parked domain at #{url} — registrar parking page, not site content" if parked_domain?(page[:content])
           raise EmptyContentError, "no readable content at #{url}" unless usable_content?(page[:content])
 
           page

data/lib/ask/web_fetch/backends/jina.rb

@@ -39,8 +39,13 @@ module Ask
             # links, resolved against the requested URL. Content runs
             # through the same Markdown.clean as the converting backends,
             # so decorative symbol noise is stripped here too; a page
-            # whose only "content" was noise falls through as empty.
+            # whose only "content" was noise falls through as empty. A
+            # registrar parking page renders fine through Jina — the
+            # shared detector (0.5.7) catches the text markers that
+            # survive conversion, so the ad is rejected, not returned as
+            # the site's content.
             content = Markdown.clean(body)
+            raise ParkedDomainError, "parked domain at #{url} — registrar parking page, not site content" if parked_domain?(content)
             raise EmptyContentError, 'empty response from Jina' unless usable_content?(content)
 
             { title: nil, description: nil, content: content, outlinks: markdown_outlinks(body, url) }

data/lib/ask/web_fetch/tool.rb

@@ -21,6 +21,13 @@ module Ask
     class WebFetch < Ask::Tool
       DEFAULT_MAX_CHARS = 20_000
 
+      # Errors that mean "the URL is dead" — no amount of retrying changes
+      # the answer. When EVERY backend failed this way, the aggregate
+      # re-raises as FetchError so callers can fail fast; any transient
+      # failure in the mix (timeout, 5xx, empty render) keeps the base
+      # Error, which recovers on retry.
+      DETERMINISTIC = [Ask::WebFetch::FetchError, Ask::WebFetch::EmptyContentError].freeze
+
       # Backend chain, tried in order. Crawl4AI leads when configured
       # (CRAWL4AI_URL), so a present self-hosted renderer is preferred;
       # otherwise Local, with Jina as the last resort, and Browser appended
@@ -37,9 +44,34 @@ module Ask
         end
       end
 
-      class << self
-        attr_writer :backends
+    class << self
+      attr_writer :backends
+
+      # Collapses every backend's failure into ONE error whose class
+      # carries the best explanation. Precedence, most definitive first: a
+      # parked domain beats an empty shell (the shell IS the parking ad's
+      # shell — Local sees the JS redirect stub, Browser the lander),
+      # empty beats a dead 4xx (the page existed, it just had no content),
+      # and any deterministic explanation beats a transient one (transient
+      # keeps the retryable base Error). Clients read the class:
+      # ParkedDomainError / EmptyContentError / FetchError are terminal —
+      # retrying never changes the answer; Error may recover on retry.
+      def collapse(failures, url)
+        detail = failures.map { |backend, e| "#{backend.backend_name}: #{e.message}" }.join('; ')
+        message = "all web fetch backends failed for #{url} (#{detail})"
+
+        classes = failures.map { |_, e| e.class }
+        if classes.any? { |k| k <= Ask::WebFetch::ParkedDomainError }
+          raise Ask::WebFetch::ParkedDomainError, message
+        end
+        if classes.any? { |k| k <= Ask::WebFetch::EmptyContentError }
+          raise Ask::WebFetch::EmptyContentError, message
+        end
+
+        deterministic = failures.all? { |_, e| DETERMINISTIC.any? { |klass| e.is_a?(klass) } }
+        raise(deterministic ? Ask::WebFetch::FetchError : Ask::WebFetch::Error, message)
       end
+    end
 
       description 'Fetch a URL and return its content as clean markdown for LLM consumption. ' \
                   'Use this to read web pages, articles, and documentation.'
@@ -68,10 +100,9 @@ module Ask
         self.class.backends.each do |backend_class|
           return backend_class.new.fetch(url)
         rescue Ask::WebFetch::Error => e
-          failures << "#{backend_class.backend_name}: #{e.message}"
+          failures << [backend_class, e]
         end
-        raise Ask::WebFetch::Error,
-              "all web fetch backends failed for #{url} (#{failures.join('; ')})"
+        self.class.collapse(failures, url)
       end
 
       def format(page, url)

data/lib/ask/web_fetch/version.rb

@@ -2,6 +2,6 @@
 
 module Ask
   module WebFetch
-    VERSION = '0.5.9'
+    VERSION = '0.6.1'
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ask-web-fetch
 version: !ruby/object:Gem::Version
-  version: 0.5.9
+  version: 0.6.1
 platform: ruby
 authors:
 - Kaka Ruto