ask-web-fetch 0.7.4 → 0.7.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 412118d0b9d8d15d1677f877dcdd06cc3ca3d64f6afb79be80ae7fb404bdc8c5
-  data.tar.gz: 18a022bec912f32ecd7f1afca82c11e7e5f016199779b92fec86ec5deb1428a7
+  metadata.gz: ae762437d9eb577d28a7f88d7cc82e5cb872e29a572c8bf3ec977f2602e2d398
+  data.tar.gz: 4c52bfb0d6a69d5184eea6ad81281dbea87f3bdfcd2a72219de3f95bbcd4b524
 SHA512:
-  metadata.gz: b12c4fbcc8e2f76d1b76b07836e1c7aeaa735e22c75b085571a1a8f710425637b461a97f2ec5c073d6e88b1d843aeca9da50d2ecba36d72ad21fcd4da7926852
-  data.tar.gz: 054750f93da3d1af328fb312867d26cbf4f7631133d9e79f76eb759816fd376e30b1a8df1b1b3f52d5752f8eac15aa38143c1c9484222543a20301c947ea2b4d
+  metadata.gz: 4be64a96e76ff2335e5d4e9ced1b6fc918eff6a8e5520b7f87660b554989e9ac6beb0841fcd12b218ac33c8776f9f9d3703c95da072b7f66e3ea3095916f66a6
+  data.tar.gz: 9741eeb18451170efedf920716dd22f1db61c57b71b43e76556d3e18f9303645671e6ee93867b78faefd8e026c2e272891856fc49e4ca95b91b3e791810dbac0

data/lib/ask/web_fetch/backend.rb

@@ -6,7 +6,14 @@ module Ask
   module WebFetch
     # Raised by backends on any failure; the tool catches it and tries the
     # next backend in the chain.
-    class Error < StandardError; end
+    class Error < StandardError
+      attr_reader :hint
+
+      def initialize(message = nil, hint: nil)
+        @hint = hint
+        super(message)
+      end
+    end
 
     # A backend that failed to fetch because the URL itself is bad —
     # challenge page, non-HTML response, redirect loop. Deterministic:
@@ -14,17 +21,21 @@ module Ask
     class FetchError < Error
       attr_reader :status
 
-      def initialize(message = nil, status: nil)
+      def initialize(message = nil, status: nil, hint: nil)
         @status = status
         msg = status ? "[#{status}] #{message}" : message
-        super(msg)
+        super(msg, hint: hint)
       end
     end
 
     # HTTP 404 — the URL does not exist. Some 404 pages still carry
     # usable content (custom error pages with navigation, suggestions);
     # the backend tries to extract it before giving up.
-    class NotFoundError < FetchError; end
+    class NotFoundError < FetchError
+      def initialize(message = nil, status: 404, hint: nil)
+        super(message, status: status, hint: hint)
+      end
+    end
 
     # A backend that fetched the page but found nothing usable in it.
     class EmptyContentError < Error; end
@@ -38,17 +49,21 @@ module Ask
 
     # Network-level failure — timeout, connection refused/reset, bad
     # socket. Transient: the same URL may succeed on retry.
-    class TimeoutError < Error; end
+    class TimeoutError < Error
+      def initialize(message = nil, hint: nil)
+        super(message, hint: hint)
+      end
+    end
 
     # The service or the target server answered 5xx/429. Transient:
     # retrying after backoff may succeed.
     class ServerError < Error
       attr_reader :status
 
-      def initialize(message = nil, status: nil)
+      def initialize(message = nil, status: nil, hint: nil)
         @status = status
         msg = status ? "[#{status}] #{message}" : message
-        super(msg)
+        super(msg, hint: hint)
       end
     end
 

data/lib/ask/web_fetch/backends/browser.rb

@@ -179,13 +179,17 @@ module Ask
           page = self.class.browser.create_page
           fetch_attempt(page, url)
         rescue Ferrum::TimeoutError, Ferrum::ProcessTimeoutError => e
-          raise TimeoutError, "#{e.class}: #{e.message}"
+          raise TimeoutError.new("#{e.class}: #{e.message}",
+                                 hint: "page load timed out — site may be slow or blocking automation")
         rescue Ferrum::StatusError => e
-          raise FetchError, "browser could not load #{url}: #{e.message}"
+          raise FetchError.new("browser could not load #{url}: #{e.message}",
+                               hint: "navigation failed — URL may be invalid or unreachable")
         rescue Ferrum::Error => e
-          raise ServerError, "browser #{e.class}: #{e.message}"
+          raise ServerError.new("browser #{e.class}: #{e.message}",
+                                hint: "browser error — check Chrome/CDP connection")
         rescue Errno::ECONNREFUSED, SocketError => e
-          raise TimeoutError, "browser connection #{e.class}: #{e.message}"
+          raise TimeoutError.new("browser connection #{e.class}: #{e.message}",
+                                 hint: "CDP endpoint unreachable — is Chrome running with --remote-debugging-port?")
         ensure
           page&.close
         end

data/lib/ask/web_fetch/backends/jina.rb

@@ -49,18 +49,22 @@ module Ask
 
             { title: nil, description: nil, content: content, outlinks: markdown_outlinks(body, url) }
           when '429'
-            raise ServerError.new('rate limited by Jina', status: 429)
+            raise ServerError.new('rate limited by Jina', status: 429,
+                                  hint: "Jina free tier limit hit — wait or set JINA_API_KEY")
           when '401', '403'
-            raise FetchError.new("Jina access error", status: res.code.to_i)
+            raise FetchError.new("Jina access error", status: res.code.to_i,
+                                 hint: "site blocked automated requests or Jina denied access")
           else
             # 5xx = Jina-side blip (transient); other 4xx = the URL is dead.
             code = res.code.to_i
             error_class = code == 404 ? NotFoundError : (code >= 500 ? ServerError : FetchError)
-            raise error_class.new("Jina returned #{res.code}", status: code)
+            raise error_class.new("Jina returned #{res.code}", status: code,
+                                  hint: code >= 500 ? "Jina-side error — try again later" : nil)
           end
         rescue Net::OpenTimeout, Net::ReadTimeout, Errno::ECONNREFUSED,
                Errno::ECONNRESET, SocketError, URI::InvalidURIError => e
-          raise TimeoutError, "Jina #{e.class}: #{e.message}"
+          raise TimeoutError.new("Jina #{e.class}: #{e.message}",
+                                 hint: "Jina service unreachable — try another backend")
         end
       end
     end

data/lib/ask/web_fetch/backends/local.rb

@@ -67,9 +67,11 @@ module Ask
           # Probe 3: full HTML scrape (legacy path)
           body, content_type, redirect, status = fetch_html(url)
           unless content_type.include?('html')
-            raise FetchError.new("expected HTML from #{url}, got #{content_type}", status: status)
+            raise FetchError.new("expected HTML from #{url}, got #{content_type}", status: status,
+                                 hint: "server returned #{content_type} instead of HTML")
           end
-          raise FetchError.new("challenge page at #{url}", status: status) if challenge_page?(body)
+          raise FetchError.new("challenge page at #{url}", status: status,
+                               hint: "anti-bot challenge detected — try with CDP-attached browser") if challenge_page?(body)
 
           page = to_markdown(body, url)
           page[:redirected] = redirect
@@ -87,7 +89,7 @@ module Ask
           # up — only raise NotFoundError if the content is genuinely empty.
           if status == 404
             return page if usable_content?(page[:content])
-            raise NotFoundError.new("not found at #{url}", status: 404)
+            raise NotFoundError.new("not found at #{url}", hint: "page does not exist")
           end
 
           guard_page!(url, page[:content], raw_body: body)
@@ -208,8 +210,14 @@ module Ask
             end
 
             unless (300..399).cover?(response.status) && !response.location.empty?
-              # 4xx (other than 429) = the URL is dead; 429/5xx = transient.
               code = response.status
+              # 404: return the body so fetch() can try to extract content
+              # from custom error pages (some 404 pages carry navigation,
+              # search suggestions, or useful content).
+              if code == 404
+                return [response.body, response.content_type, redirect_info(first_hop_status, uri), code]
+              end
+              # 429/5xx = transient; other 4xx = the URL is dead.
               raise(code == 429 || code >= 500 ? ServerError : FetchError, "got #{code} from #{url}")
             end
             raise FetchError, "hit a redirect loop at #{url}" if (hops += 1) > MAX_REDIRECTS

data/lib/ask/web_fetch/version.rb

@@ -2,6 +2,6 @@
 
 module Ask
   module WebFetch
-    VERSION = '0.7.4'
+    VERSION = '0.7.6'
   end
 end

data/lib/ask/web_fetch.rb

@@ -95,19 +95,25 @@ module Ask
                end
       message = "#{detail} #{url}"
 
+      # Attach the most informative hint — prefer the first backend's hint
+      # when all agree, otherwise use the first non-nil hint.
+      hints = failures.filter_map { |_, e| e.respond_to?(:hint) && e.hint }
+      hint = hints.uniq.size == 1 ? hints.first : hints.first
+
       classes = failures.map { |_, e| e.class }
       if classes.any? { |k| k <= Ask::WebFetch::ParkedDomainError }
-        raise Ask::WebFetch::ParkedDomainError, message
+        raise Ask::WebFetch::ParkedDomainError.new(message, hint: hint)
       end
       if classes.any? { |k| k <= Ask::WebFetch::EmptyContentError }
-        raise Ask::WebFetch::EmptyContentError, message
+        raise Ask::WebFetch::EmptyContentError.new(message, hint: hint)
       end
       if classes.any? { |k| k == Ask::WebFetch::NotFoundError }
-        raise Ask::WebFetch::NotFoundError, message
+        raise Ask::WebFetch::NotFoundError.new(message, hint: hint)
       end
 
       deterministic = failures.all? { |_, e| DETERMINISTIC.any? { |klass| e.is_a?(klass) } }
-      raise(deterministic ? Ask::WebFetch::FetchError : Ask::WebFetch::Error, message)
+      error_class = deterministic ? Ask::WebFetch::FetchError : Ask::WebFetch::Error
+      raise error_class.new(message, hint: hint)
     end
 
     # Fetches +url+ and returns LLM-ready markdown — "# Title\n\nSource:

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ask-web-fetch
 version: !ruby/object:Gem::Version
-  version: 0.7.4
+  version: 0.7.6
 platform: ruby
 authors:
 - Kaka Ruto