ask-web-fetch 0.7.0 → 0.7.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6f73de0a29e2d37986a2b05a930640143f5566011782b5d68eed318bd82db746
-  data.tar.gz: 86ee4e5a800bf7fa67df0a652f40f0058ba5e1c1fa00e77a63b737017e61fbf0
+  metadata.gz: 3ad48d09aa8869b4193a703ad6a760753c2767c9e26a6919b1405a5fb186c7a4
+  data.tar.gz: 52960016a3d4740f4913d4879cbf80edacf95e4ffc07601e6dde0d80515ffe81
 SHA512:
-  metadata.gz: 9b3fe951669c892269d18304d4242786637df6b4f1ddd39aa4bfbc70d2241f90193701665f9141f5c7e816ba12e1a7bd5c48b98d2f0748ce1e093cfc70c5cd16
-  data.tar.gz: f372adee62b1ab72695509aa6bac61d105a9dc64e5209d368273c8667fc62145075b978bd572b426bd54a42770a31f0ff192245f9f37b8f9c1c9e121a3fa0360
+  metadata.gz: fa6d778abc6918bb7ae839d4f9c96dafe215014a8b2f4efc8027ab1e49d287e3ca57204f54f013531e38dff7393ac23f6da9619b3afd48efae8b6f7e42dfa0d9
+  data.tar.gz: 7d4e5fe0e70b28cc3349ec96d071d4e1cc7a46c2c21ca8f41c35968d8b15221854c2418fc72ae2a8e56e97c07930592b2ee5b79da921c4a4ccb4fe6990c7e938

data/lib/ask/web_fetch/backends/browser.rb

@@ -96,6 +96,12 @@ module Ask
             browser_mutex.synchronize { @browser ||= build_browser }
           end
 
+          # Drops the shared browser so the next use builds a fresh
+          # session — called when the current one died mid-fetch.
+          def reset_browser
+            browser_mutex.synchronize { @browser = nil }
+          end
+
           def content_filter
             @content_filter ||= ContentFilter.default
           end
@@ -162,7 +168,17 @@ module Ask
 
           page = self.class.browser.create_page
           fetch_attempt(page, url)
-        rescue Ferrum::TimeoutError, Ferrum::ProcessTimeoutError, Ferrum::DeadBrowserError => e
+        rescue Ferrum::DeadBrowserError => e
+          # The shared browser died mid-fetch (the browserless server
+          # killed the session's browser). A dead browser is transient —
+          # the next session starts a fresh one — so reconnect and retry
+          # ONCE instead of failing the page (observed 2026-08-14: the
+          # server's session limits killed browsers and pages were
+          # silently dropped).
+          self.class.reset_browser
+          page = self.class.browser.create_page
+          fetch_attempt(page, url)
+        rescue Ferrum::TimeoutError, Ferrum::ProcessTimeoutError => e
           raise TimeoutError, "#{e.class}: #{e.message}"
         rescue Ferrum::StatusError => e
           raise FetchError, "browser could not load #{url}: #{e.message}"

data/lib/ask/web_fetch/backends/local.rb

@@ -63,7 +63,7 @@ module Ask
           # and keeps a partial page from ever being stored as the real
           # thing. Two detectors: known framework markers, or a large
           # HTML page with almost no server-rendered text.
-          if js_app_shell?(body) && page[:content].length < SHELL_CONTENT_THRESHOLD
+          if js_app_shell?(body) && page[:content].length < SHELL_DEFER_THRESHOLD
             raise EmptyContentError,
               "JS-app shell at #{url} — server HTML renders only #{page[:content].length} chars; a rendering backend is required"
           end
@@ -82,10 +82,19 @@ module Ask
         # specific footprints of React/Vue/Next/Nuxt app shells.
         JS_APP_SHELL_MARKERS = /id=["'](?:root|app|__next|site-content)["']|__NEXT_DATA__|window\.__NUXT__|ng-app|data-reactroot/
 
-        # Markdown below this from a JS-app shell is "server sent a shell",
-        # not "page is genuinely short" — a real page (even a short one)
-        # is usually server-rendered above this. Tunable; the chain turns
-        # the signal into "prefer Browser for this URL".
+        # A framework marker (React/Vue/Next) alone is NOT emptiness: a
+        # marked page can server-render real content (careers.abb job
+        # pages: 2,162 chars of job description) and must be stored as-is
+        # when the rendering backends cannot do better — dropping it lost
+        # real pages (2026-08-14). The shell deferral fires only when the
+        # extraction is genuinely little: below this, the page is a true
+        # shell (airbnb: 613KB HTML -> 143 chars). Tunable; the chain
+        # turns the signal into "prefer Browser for this URL".
+        SHELL_DEFER_THRESHOLD = 500
+
+        # The ratio detector's own bar, kept for compatibility with the
+        # comment below (large HTML + near-empty text is a shell whatever
+        # the framework).
         SHELL_CONTENT_THRESHOLD = 4_000
 
         # A page whose server HTML is large but yields almost no text is a

data/lib/ask/web_fetch/tool.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+require 'ask-tools'
+
+module Ask
+  module Tools
+    # The native agent tool for web fetch — the Ask::Tool framing of the
+    # library. Consumed WITHOUT MCP by agent frameworks (ask-agent's
+    # `tool: :web_fetch`, ask-app-server, llm-proxy) that resolve tools
+    # from the Ask::Tools registry. It is a pure adapter: the capability
+    # (chain, collapse, format) lives in Ask::WebFetch, and this file is
+    # loaded only when ask-tools is present (see lib/ask/web_fetch.rb), so
+    # the library itself never depends on it.
+    class WebFetch < Ask::Tool
+      # Chain configuration delegates to the library — swap or extend
+      # backends here or via Ask::WebFetch.backends.
+      def self.backends
+        Ask::WebFetch.backends
+      end
+
+      def self.backends=(chain)
+        Ask::WebFetch.backends = chain
+      end
+
+      description 'Fetch a URL and return its content as clean markdown for LLM consumption. ' \
+                  'Use this to read web pages, articles, and documentation.'
+
+      params(
+        type: 'object',
+        properties: {
+          url: { type: 'string', description: 'The URL to fetch' },
+          max_chars: { type: 'integer', description: 'Maximum number of characters to return (default 20000)' }
+        },
+        required: ['url']
+      )
+
+      def execute(url:, max_chars: Ask::WebFetch::DEFAULT_MAX_CHARS)
+        Ask::Result.ok(data: Ask::WebFetch.fetch(url, max_chars: max_chars))
+      end
+    end
+  end
+end
+
+Ask::Tools.register(Ask::Tools::WebFetch)

data/lib/ask/web_fetch/version.rb

@@ -2,6 +2,6 @@
 
 module Ask
   module WebFetch
-    VERSION = '0.7.0'
+    VERSION = '0.7.2'
   end
 end

data/lib/ask/web_fetch.rb

@@ -10,6 +10,19 @@ require_relative 'web_fetch/backends/crawl4ai'
 require_relative 'web_fetch/backends/jina'
 require_relative 'web_fetch/backends/browser'
 
+# The native agent tool (Ask::Tools::WebFetch) is an OPTIONAL integration:
+# it loads and registers only when ask-tools is present. The library works
+# standalone — backend-only consumers (crawlers, pipelines) pay nothing —
+# while agent frameworks (ask-agent, ask-app-server, llm-proxy) all ship
+# ask-tools and get the registry tool with no extra step. Only the
+# ask-tools miss is swallowed; any other LoadError is real.
+begin
+  require 'ask-tools'
+  require_relative 'web_fetch/tool'
+rescue LoadError => e
+  raise unless e.path == 'ask-tools'
+end
+
 module Ask
   # Fetches a URL and returns its content as clean markdown for LLM
   # consumption. The capability layer: a pluggable backend chain, a

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ask-web-fetch
 version: !ruby/object:Gem::Version
-  version: 0.7.0
+  version: 0.7.2
 platform: ruby
 authors:
 - Kaka Ruto
@@ -107,6 +107,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '3.26'
+- !ruby/object:Gem::Dependency
+  name: ask-tools
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0.1'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0.1'
 description: 'Fetches a URL and converts its content to clean markdown for LLM consumption.
   A pluggable backend chain: pure Ruby httpx + Nokogiri + reverse_markdown by default,
   a Jina Reader fallback for JS-rendered or blocked pages, and a real-Chrome fallback
@@ -133,6 +147,7 @@ files:
 - lib/ask/web_fetch/http.rb
 - lib/ask/web_fetch/markdown.rb
 - lib/ask/web_fetch/noise_filter.rb
+- lib/ask/web_fetch/tool.rb
 - lib/ask/web_fetch/version.rb
 homepage: https://github.com/ask-rb/ask-web-fetch
 licenses:
@@ -156,7 +171,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 4.0.3
+rubygems_version: 4.0.18
 specification_version: 4
 summary: Web fetch library for the ask-rb ecosystem
 test_files: []