ask-web-fetch 0.7.3 → 0.7.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b2dd2d3ee4ae5d53f7e1bf441d3b598be5195d3aafd58119213b7b739ad900db
-  data.tar.gz: 49d8aa803e8f4871e278d6fbe7ad71b6a6826c5afe62c8dc8f38abc2b8b4cb8e
+  metadata.gz: 412118d0b9d8d15d1677f877dcdd06cc3ca3d64f6afb79be80ae7fb404bdc8c5
+  data.tar.gz: 18a022bec912f32ecd7f1afca82c11e7e5f016199779b92fec86ec5deb1428a7
 SHA512:
-  metadata.gz: 95f40993df27de4c059f085ece2d71b5ea8c1e72c2d3d3dc42e96d25a1d5d87a400f5905cbaf6685675eb064ddebe95fa110609f16b93f586a5a31e02e17917c
-  data.tar.gz: a5b8513eef5cad48fbed247a2578e662c42a401b7b8e16028e5b6fbfa90d6b28da9e3db5786c20d43182ececf5a490ff98338a4fbf3ba286b0e4f9123210dea2
+  metadata.gz: b12c4fbcc8e2f76d1b76b07836e1c7aeaa735e22c75b085571a1a8f710425637b461a97f2ec5c073d6e88b1d843aeca9da50d2ecba36d72ad21fcd4da7926852
+  data.tar.gz: 054750f93da3d1af328fb312867d26cbf4f7631133d9e79f76eb759816fd376e30b1a8df1b1b3f52d5752f8eac15aa38143c1c9484222543a20301c947ea2b4d

data/lib/ask/web_fetch/backend.rb

@@ -8,10 +8,23 @@ module Ask
     # next backend in the chain.
     class Error < StandardError; end
 
-    # A backend that failed to fetch because the URL itself is bad — 4xx,
+    # A backend that failed to fetch because the URL itself is bad —
     # challenge page, non-HTML response, redirect loop. Deterministic:
     # retrying won't change the outcome.
-    class FetchError < Error; end
+    class FetchError < Error
+      attr_reader :status
+
+      def initialize(message = nil, status: nil)
+        @status = status
+        msg = status ? "[#{status}] #{message}" : message
+        super(msg)
+      end
+    end
+
+    # HTTP 404 — the URL does not exist. Some 404 pages still carry
+    # usable content (custom error pages with navigation, suggestions);
+    # the backend tries to extract it before giving up.
+    class NotFoundError < FetchError; end
 
     # A backend that fetched the page but found nothing usable in it.
     class EmptyContentError < Error; end
@@ -29,7 +42,15 @@ module Ask
 
     # The service or the target server answered 5xx/429. Transient:
     # retrying after backoff may succeed.
-    class ServerError < Error; end
+    class ServerError < Error
+      attr_reader :status
+
+      def initialize(message = nil, status: nil)
+        @status = status
+        msg = status ? "[#{status}] #{message}" : message
+        super(msg)
+      end
+    end
 
     # Base class for fetch backends, plus the errors they raise.
     #

data/lib/ask/web_fetch/backends/browser.rb

@@ -206,7 +206,10 @@ module Ask
           wait_for_idle(page)
 
           status = page.network.status
-          raise FetchError, "got #{status} at #{url}" if status && status >= 400
+          if status && status >= 400
+            error_class = status == 404 ? NotFoundError : FetchError
+            raise error_class.new("got #{status} at #{url}", status: status)
+          end
 
           body = page.body
           if challenge_page?(body)

data/lib/ask/web_fetch/backends/jina.rb

@@ -49,12 +49,14 @@ module Ask
 
             { title: nil, description: nil, content: content, outlinks: markdown_outlinks(body, url) }
           when '429'
-            raise ServerError, 'rate limited by Jina (429)'
+            raise ServerError.new('rate limited by Jina', status: 429)
           when '401', '403'
-            raise FetchError, "Jina access error (#{res.code})"
+            raise FetchError.new("Jina access error", status: res.code.to_i)
           else
             # 5xx = Jina-side blip (transient); other 4xx = the URL is dead.
-            raise(res.code.to_i >= 500 ? ServerError : FetchError, "Jina returned #{res.code}")
+            code = res.code.to_i
+            error_class = code == 404 ? NotFoundError : (code >= 500 ? ServerError : FetchError)
+            raise error_class.new("Jina returned #{res.code}", status: code)
           end
         rescue Net::OpenTimeout, Net::ReadTimeout, Errno::ECONNREFUSED,
                Errno::ECONNRESET, SocketError, URI::InvalidURIError => e

data/lib/ask/web_fetch/backends/local.rb

@@ -65,12 +65,15 @@ module Ask
           end
 
           # Probe 3: full HTML scrape (legacy path)
-          body, content_type, redirect = fetch_html(url)
-          raise FetchError, "expected HTML from #{url}, got #{content_type}" unless content_type.include?('html')
-          raise FetchError, "challenge page at #{url}" if challenge_page?(body)
+          body, content_type, redirect, status = fetch_html(url)
+          unless content_type.include?('html')
+            raise FetchError.new("expected HTML from #{url}, got #{content_type}", status: status)
+          end
+          raise FetchError.new("challenge page at #{url}", status: status) if challenge_page?(body)
 
           page = to_markdown(body, url)
           page[:redirected] = redirect
+          page[:status] = status
           # Parked-domain pages are not content: the domain owner parked it
           # with a registrar and the page is an ad for buying the domain
           # (GoDaddy/Namecheap/Sedo parking). A content company must never
@@ -79,6 +82,14 @@ module Ask
           # server-rendered — the HTML-only markers live in scripts and
           # assets), then the content minimum, then the JS-shell
           # completeness signal below.
+          # 404 pages sometimes carry usable content (custom error pages
+          # with navigation, suggestions). Try to extract it before giving
+          # up — only raise NotFoundError if the content is genuinely empty.
+          if status == 404
+            return page if usable_content?(page[:content])
+            raise NotFoundError.new("not found at #{url}", status: 404)
+          end
+
           guard_page!(url, page[:content], raw_body: body)
           # The completeness signal: a JS-app shell whose server HTML
           # renders little is a TRUNCATED page, not a complete one — the
@@ -183,8 +194,8 @@ module Ask
         end
 
         # GET with redirect following (max MAX_REDIRECTS hops). Returns
-        # [body, content_type, redirect] where redirect is nil when the
-        # URL answered directly, else {status: first hop's status,
+        # [body, content_type, redirect, status] where redirect is nil
+        # when the URL answered directly, else {status: first hop's status,
         # url: final destination} — the chain the crawler followed.
         def fetch_html(url)
           uri = URI(url)
@@ -192,7 +203,9 @@ module Ask
           first_hop_status = nil
           loop do
             response = self.class.http.get(uri.to_s, headers: { 'accept' => 'text/html,application/xhtml+xml' })
-            return [response.body, response.content_type, redirect_info(first_hop_status, uri)] if (200..299).cover?(response.status)
+            if (200..299).cover?(response.status)
+              return [response.body, response.content_type, redirect_info(first_hop_status, uri), response.status]
+            end
 
             unless (300..399).cover?(response.status) && !response.location.empty?
               # 4xx (other than 429) = the URL is dead; 429/5xx = transient.

data/lib/ask/web_fetch/version.rb

@@ -2,6 +2,6 @@
 
 module Ask
   module WebFetch
-    VERSION = '0.7.3'
+    VERSION = '0.7.4'
   end
 end

data/lib/ask/web_fetch.rb

@@ -37,7 +37,7 @@ module Ask
     # re-raises as FetchError so callers can fail fast; any transient
     # failure in the mix (timeout, 5xx, empty render) keeps the base
     # Error, which recovers on retry.
-    DETERMINISTIC = [Ask::WebFetch::FetchError, Ask::WebFetch::EmptyContentError].freeze
+    DETERMINISTIC = [Ask::WebFetch::FetchError, Ask::WebFetch::NotFoundError, Ask::WebFetch::EmptyContentError].freeze
 
     # Backend chain, tried in order. Crawl4AI leads when configured
     # (CRAWL4AI_URL), so a present self-hosted renderer is preferred;
@@ -83,8 +83,17 @@ module Ask
     # ParkedDomainError / EmptyContentError / FetchError are terminal —
     # retrying never changes the answer; Error may recover on retry.
     def self.collapse(failures, url)
-      detail = failures.map { |backend, e| "#{backend.backend_name}: #{e.message}" }.join('; ')
-      message = "all web fetch backends failed for #{url} (#{detail})"
+      # When all backends agree on the same root cause (same HTTP status),
+      # say so cleanly instead of listing every backend's echo of the same
+      # problem. Status is now on the error object itself (FetchError#status,
+      # NotFoundError#status), so we don't parse messages.
+      statuses = failures.filter_map { |_, e| e.respond_to?(:status) && e.status }
+      detail = if statuses.uniq.size == 1 && statuses.size == failures.size
+                 "[#{statuses.first}]"
+               else
+                 failures.map { |backend, e| "#{backend.backend_name}: #{e.message}" }.join('; ')
+               end
+      message = "#{detail} #{url}"
 
       classes = failures.map { |_, e| e.class }
       if classes.any? { |k| k <= Ask::WebFetch::ParkedDomainError }
@@ -93,6 +102,9 @@ module Ask
       if classes.any? { |k| k <= Ask::WebFetch::EmptyContentError }
         raise Ask::WebFetch::EmptyContentError, message
       end
+      if classes.any? { |k| k == Ask::WebFetch::NotFoundError }
+        raise Ask::WebFetch::NotFoundError, message
+      end
 
       deterministic = failures.all? { |_, e| DETERMINISTIC.any? { |klass| e.is_a?(klass) } }
       raise(deterministic ? Ask::WebFetch::FetchError : Ask::WebFetch::Error, message)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ask-web-fetch
 version: !ruby/object:Gem::Version
-  version: 0.7.3
+  version: 0.7.4
 platform: ruby
 authors:
 - Kaka Ruto