ask-web-fetch 0.7.5 → 0.7.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7dfd9899023ac83785a8751d2256396b0d3493a1b745a895ed96badc0f8efb5d
-  data.tar.gz: 5448f692e7a10164b783108ce7c5e99cde376902b379e57890cca908024d59ce
+  metadata.gz: 3e1fdcc511917b85bfe1ab456733d89e09a24eb4dd4bd085d61ebcf5d52ff4d0
+  data.tar.gz: 59365989bd571bd06454513328a5f27c64116f6ced3669bcebb753c94fddfcee
 SHA512:
-  metadata.gz: 598e9c3c242acfe3cc405b9ed568dbd997a9e43489248d50651b79f8f2c68e8ebfb55ba3fdf4acaf8b823e2cbcce99958894f43d296cceb8a6ca37bbf5e5943a
-  data.tar.gz: f166296ca3c09ccf4870c7a68b16e8a82537adbc065e5190aba980590b9951b532c7570c37617bda868d3ac87d772a8ae0d0bf9c42926922772810a0644701c4
+  metadata.gz: 5ab5dfc8d25e5c9550a08981382a041e148291b6d83ab40c9f375404c7efd35f08d08b6444b10fabe4a8035773ea49606dc96a6c73211aeb7cd4f89e529ca359
+  data.tar.gz: 742d576e4a4f0a71ab89c47d177107f991c384795eef0775fc8579c96d6cbb41688cb30cabfb13d4d861bf6e665c54ee662998e44a66da64904955406562818f

data/lib/ask/web_fetch/backend.rb

@@ -6,7 +6,14 @@ module Ask
   module WebFetch
     # Raised by backends on any failure; the tool catches it and tries the
     # next backend in the chain.
-    class Error < StandardError; end
+    class Error < StandardError
+      attr_reader :hint
+
+      def initialize(message = nil, hint: nil)
+        @hint = hint
+        super(message)
+      end
+    end
 
     # A backend that failed to fetch because the URL itself is bad —
     # challenge page, non-HTML response, redirect loop. Deterministic:
@@ -14,17 +21,21 @@ module Ask
     class FetchError < Error
       attr_reader :status
 
-      def initialize(message = nil, status: nil)
+      def initialize(message = nil, status: nil, hint: nil)
         @status = status
         msg = status ? "[#{status}] #{message}" : message
-        super(msg)
+        super(msg, hint: hint)
       end
     end
 
     # HTTP 404 — the URL does not exist. Some 404 pages still carry
     # usable content (custom error pages with navigation, suggestions);
     # the backend tries to extract it before giving up.
-    class NotFoundError < FetchError; end
+    class NotFoundError < FetchError
+      def initialize(message = nil, status: 404, hint: nil)
+        super(message, status: status, hint: hint)
+      end
+    end
 
     # A backend that fetched the page but found nothing usable in it.
     class EmptyContentError < Error; end
@@ -38,17 +49,21 @@ module Ask
 
     # Network-level failure — timeout, connection refused/reset, bad
     # socket. Transient: the same URL may succeed on retry.
-    class TimeoutError < Error; end
+    class TimeoutError < Error
+      def initialize(message = nil, hint: nil)
+        super(message, hint: hint)
+      end
+    end
 
     # The service or the target server answered 5xx/429. Transient:
     # retrying after backoff may succeed.
     class ServerError < Error
       attr_reader :status
 
-      def initialize(message = nil, status: nil)
+      def initialize(message = nil, status: nil, hint: nil)
         @status = status
         msg = status ? "[#{status}] #{message}" : message
-        super(msg)
+        super(msg, hint: hint)
       end
     end
 

data/lib/ask/web_fetch/backends/browser.rb

@@ -45,7 +45,7 @@ module Ask
       class Browser < Backend
         # Seconds to let a Cloudflare-style challenge auto-solve before
         # giving up (tunable via Browser.challenge_timeout).
-        CHALLENGE_TIMEOUT = 30
+        CHALLENGE_TIMEOUT = 10
 
         # Seconds to wait for the network to go quiet after the page loads,
         # so lazy-loaded content is in before we read the DOM.
@@ -179,13 +179,17 @@ module Ask
           page = self.class.browser.create_page
           fetch_attempt(page, url)
         rescue Ferrum::TimeoutError, Ferrum::ProcessTimeoutError => e
-          raise TimeoutError, "#{e.class}: #{e.message}"
+          raise TimeoutError.new("#{e.class}: #{e.message}",
+                                 hint: "page load timed out — site may be slow or blocking automation")
         rescue Ferrum::StatusError => e
-          raise FetchError, "browser could not load #{url}: #{e.message}"
+          raise FetchError.new("browser could not load #{url}: #{e.message}",
+                               hint: "navigation failed — URL may be invalid or unreachable")
         rescue Ferrum::Error => e
-          raise ServerError, "browser #{e.class}: #{e.message}"
+          raise ServerError.new("browser #{e.class}: #{e.message}",
+                                hint: "browser error — check Chrome/CDP connection")
         rescue Errno::ECONNREFUSED, SocketError => e
-          raise TimeoutError, "browser connection #{e.class}: #{e.message}"
+          raise TimeoutError.new("browser connection #{e.class}: #{e.message}",
+                                 hint: "CDP endpoint unreachable — is Chrome running with --remote-debugging-port?")
         ensure
           page&.close
         end

data/lib/ask/web_fetch/backends/jina.rb

@@ -15,8 +15,8 @@ module Ask
       # pages that the Local backend cannot.
       class Jina < Backend
         BASE_URL = 'https://r.jina.ai'
-        OPEN_TIMEOUT = 5
-        READ_TIMEOUT = 30
+        OPEN_TIMEOUT = 3
+        READ_TIMEOUT = 10
 
         def fetch(url)
           uri = URI("#{BASE_URL}/#{url}")
@@ -49,18 +49,22 @@ module Ask
 
             { title: nil, description: nil, content: content, outlinks: markdown_outlinks(body, url) }
           when '429'
-            raise ServerError.new('rate limited by Jina', status: 429)
+            raise ServerError.new('rate limited by Jina', status: 429,
+                                  hint: "Jina free tier limit hit — wait or set JINA_API_KEY")
           when '401', '403'
-            raise FetchError.new("Jina access error", status: res.code.to_i)
+            raise FetchError.new("Jina access error", status: res.code.to_i,
+                                 hint: "site blocked automated requests or Jina denied access")
           else
             # 5xx = Jina-side blip (transient); other 4xx = the URL is dead.
             code = res.code.to_i
             error_class = code == 404 ? NotFoundError : (code >= 500 ? ServerError : FetchError)
-            raise error_class.new("Jina returned #{res.code}", status: code)
+            raise error_class.new("Jina returned #{res.code}", status: code,
+                                  hint: code >= 500 ? "Jina-side error — try again later" : nil)
           end
         rescue Net::OpenTimeout, Net::ReadTimeout, Errno::ECONNREFUSED,
                Errno::ECONNRESET, SocketError, URI::InvalidURIError => e
-          raise TimeoutError, "Jina #{e.class}: #{e.message}"
+          raise TimeoutError.new("Jina #{e.class}: #{e.message}",
+                                 hint: "Jina service unreachable — try another backend")
         end
       end
     end

data/lib/ask/web_fetch/backends/local.rb

@@ -67,9 +67,11 @@ module Ask
           # Probe 3: full HTML scrape (legacy path)
           body, content_type, redirect, status = fetch_html(url)
           unless content_type.include?('html')
-            raise FetchError.new("expected HTML from #{url}, got #{content_type}", status: status)
+            raise FetchError.new("expected HTML from #{url}, got #{content_type}", status: status,
+                                 hint: "server returned #{content_type} instead of HTML")
           end
-          raise FetchError.new("challenge page at #{url}", status: status) if challenge_page?(body)
+          raise FetchError.new("challenge page at #{url}", status: status,
+                               hint: "anti-bot challenge detected — try with CDP-attached browser") if challenge_page?(body)
 
           page = to_markdown(body, url)
           page[:redirected] = redirect
@@ -87,7 +89,7 @@ module Ask
           # up — only raise NotFoundError if the content is genuinely empty.
           if status == 404
             return page if usable_content?(page[:content])
-            raise NotFoundError.new("not found at #{url}", status: 404)
+            raise NotFoundError.new("not found at #{url}", hint: "page does not exist")
           end
 
           guard_page!(url, page[:content], raw_body: body)

data/lib/ask/web_fetch/http.rb

@@ -18,12 +18,12 @@ module Ask
     # TLS) all surface as Ask::WebFetch::TimeoutError, the transient
     # bucket, whatever their underlying class.
     class Http
-      CONNECT_TIMEOUT = 5
-      READ_TIMEOUT = 15
-      WRITE_TIMEOUT = 15
-      # Whole-request cap. A page that can't be read in a minute is a
+      CONNECT_TIMEOUT = 3
+      READ_TIMEOUT = 8
+      WRITE_TIMEOUT = 8
+      # Whole-request cap. A page that can't be read in 20s is a
       # problem page, not a stall worth a crawl worker.
-      OPERATION_TIMEOUT = 60
+      OPERATION_TIMEOUT = 20
       # Retries per request, on top of the crawl ledger's own auto-heal
       # rounds. GETs are idempotent; a couple of cheap retries beat a full
       # ledger round-trip for transient flakiness.

data/lib/ask/web_fetch/version.rb

@@ -2,6 +2,6 @@
 
 module Ask
   module WebFetch
-    VERSION = '0.7.5'
+    VERSION = '0.7.7'
   end
 end

data/lib/ask/web_fetch.rb

@@ -95,19 +95,25 @@ module Ask
                end
       message = "#{detail} #{url}"
 
+      # Attach the most informative hint — prefer the first backend's hint
+      # when all agree, otherwise use the first non-nil hint.
+      hints = failures.filter_map { |_, e| e.respond_to?(:hint) && e.hint }
+      hint = hints.uniq.size == 1 ? hints.first : hints.first
+
       classes = failures.map { |_, e| e.class }
       if classes.any? { |k| k <= Ask::WebFetch::ParkedDomainError }
-        raise Ask::WebFetch::ParkedDomainError, message
+        raise Ask::WebFetch::ParkedDomainError.new(message, hint: hint)
       end
       if classes.any? { |k| k <= Ask::WebFetch::EmptyContentError }
-        raise Ask::WebFetch::EmptyContentError, message
+        raise Ask::WebFetch::EmptyContentError.new(message, hint: hint)
       end
       if classes.any? { |k| k == Ask::WebFetch::NotFoundError }
-        raise Ask::WebFetch::NotFoundError, message
+        raise Ask::WebFetch::NotFoundError.new(message, hint: hint)
       end
 
       deterministic = failures.all? { |_, e| DETERMINISTIC.any? { |klass| e.is_a?(klass) } }
-      raise(deterministic ? Ask::WebFetch::FetchError : Ask::WebFetch::Error, message)
+      error_class = deterministic ? Ask::WebFetch::FetchError : Ask::WebFetch::Error
+      raise error_class.new(message, hint: hint)
     end
 
     # Fetches +url+ and returns LLM-ready markdown — "# Title\n\nSource:

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ask-web-fetch
 version: !ruby/object:Gem::Version
-  version: 0.7.5
+  version: 0.7.7
 platform: ruby
 authors:
 - Kaka Ruto