ask-web-fetch 0.6.2 → 0.7.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5e7b287b5b62f821cb0b3085082b385583d3f1a7ef2dfc0816eb311931d1e218
-  data.tar.gz: 534bce458033fb046914c169b7bb7bad982f8fcd3ffb999d6feb2b495953d90c
+  metadata.gz: efa999aac8d4321c2e8e6d463ace6c3f5ec5e95a62973ad06593c4714e1ffab1
+  data.tar.gz: 9a9e59e8da81bb4238bf1a08b0ba84634553dfd3caac99763a7bb52a54d9ac6e
 SHA512:
-  metadata.gz: 48e47dd345f498f1a50b6dc3ddb09a5f55078691ee34a617b02936fb9b7d639c2ff55c309f6fc2ecec2a4d067c487edd0975c983b4ed301471d50536e73e6bd0
-  data.tar.gz: 7bf1c58c8fb17a9c5575b41f8b2d44c6deb1c01a8b82b56b3d73f5d1df4c5d72fadf09ca60de1fc3995b8dc62581611f4aa50bcb4c47ceb9894f5bb6749490fc
+  metadata.gz: af1e312f91b9e5ea72679b7e877232cc3e4e9639b3fdf3e98291810377a450a5ce180bb4664ca651c8ccd2084220632890da92fe4eaf8c1ca472e6ba464d60ff
+  data.tar.gz: 3951989986d476f6bc008712e6451ccb148c9fbe33537544061486d15c79443e0368dd668ef40b35287173ad41f569fc519130a9cb5368e78da16eafbcdf2c91

data/lib/ask/web_fetch/backend.rb

@@ -46,7 +46,7 @@ module Ask
     #      and whitespace normalization apply everywhere
     #   5. run the extracted page through #guard_page! — the parked-domain
     #      and empty-content verdicts are identical in every backend
-    #   6. register the class in Ask::Tools::WebFetch.backends
+    #   6. register the class in Ask::WebFetch.backends
     #
     # The tool tries each backend in order and returns the first success.
     class Backend

data/lib/ask/web_fetch/tool.rb

@@ -1,77 +1,26 @@
 # frozen_string_literal: true
 
 require 'ask-tools'
-require_relative '../web_fetch/backend'
-require_relative '../web_fetch/backends/local'
-require_relative '../web_fetch/backends/crawl4ai'
-require_relative '../web_fetch/backends/jina'
 
 module Ask
   module Tools
-    # Fetches a URL and returns its content as clean markdown for LLM
-    # consumption. Tries each configured backend in order and returns the
-    # first success.
-    #
-    # Chain: Crawl4AI first (self-hosted headless-Chromium renderer; used
-    # when the CRAWL4AI_URL service is present, fails fast when it isn't),
-    # then the local fetcher, Jina Reader as the last resort, and a real
-    # Chrome (via Ferrum) at the very end for pages whose Cloudflare-style
-    # challenges the others cannot pass — appended only when a browser
-    # binary is present.
+    # The native agent tool for web fetch — the Ask::Tool framing of the
+    # library. Consumed WITHOUT MCP by agent frameworks (ask-agent's
+    # `tool: :web_fetch`, ask-app-server, llm-proxy) that resolve tools
+    # from the Ask::Tools registry. It is a pure adapter: the capability
+    # (chain, collapse, format) lives in Ask::WebFetch, and this file is
+    # loaded only when ask-tools is present (see lib/ask/web_fetch.rb), so
+    # the library itself never depends on it.
     class WebFetch < Ask::Tool
-      DEFAULT_MAX_CHARS = 20_000
-
-      # Errors that mean "the URL is dead" — no amount of retrying changes
-      # the answer. When EVERY backend failed this way, the aggregate
-      # re-raises as FetchError so callers can fail fast; any transient
-      # failure in the mix (timeout, 5xx, empty render) keeps the base
-      # Error, which recovers on retry.
-      DETERMINISTIC = [Ask::WebFetch::FetchError, Ask::WebFetch::EmptyContentError].freeze
-
-      # Backend chain, tried in order. Crawl4AI leads when configured
-      # (CRAWL4AI_URL), so a present self-hosted renderer is preferred;
-      # otherwise Local, with Jina as the last resort, and Browser appended
-      # when Chrome is available. Swap or extend for future backends; each
-      # must subclass Ask::WebFetch::Backend and implement #fetch(url).
+      # Chain configuration delegates to the library — swap or extend
+      # backends here or via Ask::WebFetch.backends.
       def self.backends
-        @backends ||= begin
-          chain = [Ask::WebFetch::Backends::Local, Ask::WebFetch::Backends::Jina]
-          if Ask::WebFetch::Backends::Crawl4Ai.configured?
-            chain.unshift(Ask::WebFetch::Backends::Crawl4Ai)
-          end
-          chain << Ask::WebFetch::Backends::Browser if Ask::WebFetch::Backends::Browser.configured?
-          chain
-        end
+        Ask::WebFetch.backends
       end
 
-    class << self
-      attr_writer :backends
-
-      # Collapses every backend's failure into ONE error whose class
-      # carries the best explanation. Precedence, most definitive first: a
-      # parked domain beats an empty shell (the shell IS the parking ad's
-      # shell — Local sees the JS redirect stub, Browser the lander),
-      # empty beats a dead 4xx (the page existed, it just had no content),
-      # and any deterministic explanation beats a transient one (transient
-      # keeps the retryable base Error). Clients read the class:
-      # ParkedDomainError / EmptyContentError / FetchError are terminal —
-      # retrying never changes the answer; Error may recover on retry.
-      def collapse(failures, url)
-        detail = failures.map { |backend, e| "#{backend.backend_name}: #{e.message}" }.join('; ')
-        message = "all web fetch backends failed for #{url} (#{detail})"
-
-        classes = failures.map { |_, e| e.class }
-        if classes.any? { |k| k <= Ask::WebFetch::ParkedDomainError }
-          raise Ask::WebFetch::ParkedDomainError, message
-        end
-        if classes.any? { |k| k <= Ask::WebFetch::EmptyContentError }
-          raise Ask::WebFetch::EmptyContentError, message
-        end
-
-        deterministic = failures.all? { |_, e| DETERMINISTIC.any? { |klass| e.is_a?(klass) } }
-        raise(deterministic ? Ask::WebFetch::FetchError : Ask::WebFetch::Error, message)
+      def self.backends=(chain)
+        Ask::WebFetch.backends = chain
       end
-    end
 
       description 'Fetch a URL and return its content as clean markdown for LLM consumption. ' \
                   'Use this to read web pages, articles, and documentation.'
@@ -85,41 +34,11 @@ module Ask
         required: ['url']
       )
 
-      def execute(url:, max_chars: DEFAULT_MAX_CHARS)
-        page = fetch_page(url)
-        markdown = format(page, url)
-        markdown = truncate(markdown, max_chars) if max_chars&.positive?
-        Ask::Result.ok(data: markdown)
-      end
-
-      private
-
-      # Try each configured backend in order; return the first success.
-      def fetch_page(url)
-        failures = []
-        self.class.backends.each do |backend_class|
-          return backend_class.new.fetch(url)
-        rescue Ask::WebFetch::Error => e
-          failures << [backend_class, e]
-        end
-        self.class.collapse(failures, url)
-      end
-
-      def format(page, url)
-        header = +''
-        title = page[:title]
-        header << "# #{title}\n\n" unless title.to_s.empty?
-        header << "Source: #{url}\n\n"
-        header + page[:content]
-      end
-
-      def truncate(markdown, max_chars)
-        return markdown if markdown.length <= max_chars
-
-        "#{markdown[0, max_chars].rstrip}\n\n…(truncated)"
+      def execute(url:, max_chars: Ask::WebFetch::DEFAULT_MAX_CHARS)
+        Ask::Result.ok(data: Ask::WebFetch.fetch(url, max_chars: max_chars))
       end
     end
   end
 end
 
-Ask::Tools.register(Ask::Tools::WebFetch) if defined?(Ask::Tools)
+Ask::Tools.register(Ask::Tools::WebFetch)

data/lib/ask/web_fetch/version.rb

@@ -2,6 +2,6 @@
 
 module Ask
   module WebFetch
-    VERSION = '0.6.2'
+    VERSION = '0.7.1'
   end
 end

data/lib/ask/web_fetch.rb

@@ -9,4 +9,120 @@ require_relative 'web_fetch/backends/local'
 require_relative 'web_fetch/backends/crawl4ai'
 require_relative 'web_fetch/backends/jina'
 require_relative 'web_fetch/backends/browser'
-require_relative 'web_fetch/tool'
+
+# The native agent tool (Ask::Tools::WebFetch) is an OPTIONAL integration:
+# it loads and registers only when ask-tools is present. The library works
+# standalone — backend-only consumers (crawlers, pipelines) pay nothing —
+# while agent frameworks (ask-agent, ask-app-server, llm-proxy) all ship
+# ask-tools and get the registry tool with no extra step. Only the
+# ask-tools miss is swallowed; any other LoadError is real.
+begin
+  require 'ask-tools'
+  require_relative 'web_fetch/tool'
+rescue LoadError => e
+  raise unless e.path == 'ask-tools'
+end
+
+module Ask
+  # Fetches a URL and returns its content as clean markdown for LLM
+  # consumption. The capability layer: a pluggable backend chain, a
+  # failure collapse, and one entry point. Tool framing — name,
+  # parameter schema, result wrapping — lives with the consumers (the
+  # MCP servers, the agents) that call this library, not here.
+  module WebFetch
+    DEFAULT_MAX_CHARS = 20_000
+
+    # Errors that mean "the URL is dead" — no amount of retrying changes
+    # the answer. When EVERY backend failed this way, the aggregate
+    # re-raises as FetchError so callers can fail fast; any transient
+    # failure in the mix (timeout, 5xx, empty render) keeps the base
+    # Error, which recovers on retry.
+    DETERMINISTIC = [Ask::WebFetch::FetchError, Ask::WebFetch::EmptyContentError].freeze
+
+    # Backend chain, tried in order. Crawl4AI leads when configured
+    # (CRAWL4AI_URL), so a present self-hosted renderer is preferred;
+    # otherwise Local, with Jina as the last resort, and Browser appended
+    # when Chrome is available. Swap or extend for future backends; each
+    # must subclass Ask::WebFetch::Backend and implement #fetch(url).
+    def self.backends
+      @backends ||= begin
+        chain = [Ask::WebFetch::Backends::Local, Ask::WebFetch::Backends::Jina]
+        if Ask::WebFetch::Backends::Crawl4Ai.configured?
+          chain.unshift(Ask::WebFetch::Backends::Crawl4Ai)
+        end
+        chain << Ask::WebFetch::Backends::Browser if Ask::WebFetch::Backends::Browser.configured?
+        chain
+      end
+    end
+
+    class << self
+      attr_writer :backends
+    end
+
+    # Fetches +url+ through the chain and returns the first success as
+    # { title:, description:, content:, outlinks:, redirected: }. Raises
+    # when every backend fails; the raised class carries the verdict (see
+    # #collapse) and the message lists every backend and what it said.
+    def self.fetch_page(url)
+      failures = []
+      backends.each do |backend_class|
+        return backend_class.new.fetch(url)
+      rescue Ask::WebFetch::Error => e
+        failures << [backend_class, e]
+      end
+      collapse(failures, url)
+    end
+
+    # Collapses every backend's failure into ONE error whose class
+    # carries the best explanation. Precedence, most definitive first: a
+    # parked domain beats an empty shell (the shell IS the parking ad's
+    # shell — Local sees the JS redirect stub, Browser the lander),
+    # empty beats a dead 4xx (the page existed, it just had no content),
+    # and any deterministic explanation beats a transient one (transient
+    # keeps the retryable base Error). Clients read the class:
+    # ParkedDomainError / EmptyContentError / FetchError are terminal —
+    # retrying never changes the answer; Error may recover on retry.
+    def self.collapse(failures, url)
+      detail = failures.map { |backend, e| "#{backend.backend_name}: #{e.message}" }.join('; ')
+      message = "all web fetch backends failed for #{url} (#{detail})"
+
+      classes = failures.map { |_, e| e.class }
+      if classes.any? { |k| k <= Ask::WebFetch::ParkedDomainError }
+        raise Ask::WebFetch::ParkedDomainError, message
+      end
+      if classes.any? { |k| k <= Ask::WebFetch::EmptyContentError }
+        raise Ask::WebFetch::EmptyContentError, message
+      end
+
+      deterministic = failures.all? { |_, e| DETERMINISTIC.any? { |klass| e.is_a?(klass) } }
+      raise(deterministic ? Ask::WebFetch::FetchError : Ask::WebFetch::Error, message)
+    end
+
+    # Fetches +url+ and returns LLM-ready markdown — "# Title\n\nSource:
+    # url\n\ncontent" — capped at +max_chars+ (default 20000; pass nil to
+    # skip the cap). The single entry point for "give me this page as
+    # markdown"; the raw page hash is #fetch_page.
+    def self.fetch(url, max_chars: DEFAULT_MAX_CHARS)
+      page = fetch_page(url)
+      markdown = format(page, url)
+      markdown = truncate(markdown, max_chars) if max_chars&.positive?
+      markdown
+    end
+
+    def self.format(page, url)
+      header = +''
+      title = page[:title]
+      header << "# #{title}\n\n" unless title.to_s.empty?
+      header << "Source: #{url}\n\n"
+      header + page[:content]
+    end
+    private_class_method :format
+
+    def self.truncate(markdown, max_chars)
+      return markdown if markdown.length <= max_chars
+
+      "#{markdown[0, max_chars].rstrip}\n\n…(truncated)"
+    end
+    private_class_method :truncate
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ask-web-fetch
 version: !ruby/object:Gem::Version
-  version: 0.6.2
+  version: 0.7.1
 platform: ruby
 authors:
 - Kaka Ruto
@@ -9,20 +9,6 @@ bindir: bin
 cert_chain: []
 date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
-- !ruby/object:Gem::Dependency
-  name: ask-tools
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0.1'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0.1'
 - !ruby/object:Gem::Dependency
   name: ferrum
   requirement: !ruby/object:Gem::Requirement
@@ -121,12 +107,26 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '3.26'
-description: Provides Ask::Tools::WebFetch, a tool that fetches a URL and converts
-  its content to clean markdown for LLM consumption. Defaults to a pure Ruby backend
-  (httpx + Nokogiri + reverse_markdown) with a Jina Reader fallback for JS-rendered
-  or blocked pages, and a real-Chrome fallback (Ferrum) that renders JavaScript and
-  lets auto-solving Cloudflare challenges complete. Works with any ask-rb chat or
-  agent.
+- !ruby/object:Gem::Dependency
+  name: ask-tools
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0.1'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0.1'
+description: 'Fetches a URL and converts its content to clean markdown for LLM consumption.
+  A pluggable backend chain: pure Ruby httpx + Nokogiri + reverse_markdown by default,
+  a Jina Reader fallback for JS-rendered or blocked pages, and a real-Chrome fallback
+  (Ferrum) that renders JavaScript and lets auto-solving Cloudflare challenges complete.
+  The capability layer only — tool framing (MCP servers, agent tools) is provided
+  by the consumers.'
 email:
 - kaka@myrrlabs.com
 executables: []
@@ -173,5 +173,5 @@ required_rubygems_version: !ruby/object:Gem::Requirement
 requirements: []
 rubygems_version: 4.0.3
 specification_version: 4
-summary: Web fetch tool for the ask-rb ecosystem
+summary: Web fetch library for the ask-rb ecosystem
 test_files: []