ask-web-fetch 0.6.2 → 0.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5e7b287b5b62f821cb0b3085082b385583d3f1a7ef2dfc0816eb311931d1e218
-  data.tar.gz: 534bce458033fb046914c169b7bb7bad982f8fcd3ffb999d6feb2b495953d90c
+  metadata.gz: 6f73de0a29e2d37986a2b05a930640143f5566011782b5d68eed318bd82db746
+  data.tar.gz: 86ee4e5a800bf7fa67df0a652f40f0058ba5e1c1fa00e77a63b737017e61fbf0
 SHA512:
-  metadata.gz: 48e47dd345f498f1a50b6dc3ddb09a5f55078691ee34a617b02936fb9b7d639c2ff55c309f6fc2ecec2a4d067c487edd0975c983b4ed301471d50536e73e6bd0
-  data.tar.gz: 7bf1c58c8fb17a9c5575b41f8b2d44c6deb1c01a8b82b56b3d73f5d1df4c5d72fadf09ca60de1fc3995b8dc62581611f4aa50bcb4c47ceb9894f5bb6749490fc
+  metadata.gz: 9b3fe951669c892269d18304d4242786637df6b4f1ddd39aa4bfbc70d2241f90193701665f9141f5c7e816ba12e1a7bd5c48b98d2f0748ce1e093cfc70c5cd16
+  data.tar.gz: f372adee62b1ab72695509aa6bac61d105a9dc64e5209d368273c8667fc62145075b978bd572b426bd54a42770a31f0ff192245f9f37b8f9c1c9e121a3fa0360

data/lib/ask/web_fetch/backend.rb

@@ -46,7 +46,7 @@ module Ask
     #      and whitespace normalization apply everywhere
     #   5. run the extracted page through #guard_page! — the parked-domain
     #      and empty-content verdicts are identical in every backend
-    #   6. register the class in Ask::Tools::WebFetch.backends
+    #   6. register the class in Ask::WebFetch.backends
     #
     # The tool tries each backend in order and returns the first success.
     class Backend

data/lib/ask/web_fetch/version.rb

@@ -2,6 +2,6 @@
 
 module Ask
   module WebFetch
-    VERSION = '0.6.2'
+    VERSION = '0.7.0'
   end
 end

data/lib/ask/web_fetch.rb

@@ -9,4 +9,107 @@ require_relative 'web_fetch/backends/local'
 require_relative 'web_fetch/backends/crawl4ai'
 require_relative 'web_fetch/backends/jina'
 require_relative 'web_fetch/backends/browser'
-require_relative 'web_fetch/tool'
+
+module Ask
+  # Fetches a URL and returns its content as clean markdown for LLM
+  # consumption. The capability layer: a pluggable backend chain, a
+  # failure collapse, and one entry point. Tool framing — name,
+  # parameter schema, result wrapping — lives with the consumers (the
+  # MCP servers, the agents) that call this library, not here.
+  module WebFetch
+    DEFAULT_MAX_CHARS = 20_000
+
+    # Errors that mean "the URL is dead" — no amount of retrying changes
+    # the answer. When EVERY backend failed this way, the aggregate
+    # re-raises as FetchError so callers can fail fast; any transient
+    # failure in the mix (timeout, 5xx, empty render) keeps the base
+    # Error, which recovers on retry.
+    DETERMINISTIC = [Ask::WebFetch::FetchError, Ask::WebFetch::EmptyContentError].freeze
+
+    # Backend chain, tried in order. Crawl4AI leads when configured
+    # (CRAWL4AI_URL), so a present self-hosted renderer is preferred;
+    # otherwise Local, with Jina as the last resort, and Browser appended
+    # when Chrome is available. Swap or extend for future backends; each
+    # must subclass Ask::WebFetch::Backend and implement #fetch(url).
+    def self.backends
+      @backends ||= begin
+        chain = [Ask::WebFetch::Backends::Local, Ask::WebFetch::Backends::Jina]
+        if Ask::WebFetch::Backends::Crawl4Ai.configured?
+          chain.unshift(Ask::WebFetch::Backends::Crawl4Ai)
+        end
+        chain << Ask::WebFetch::Backends::Browser if Ask::WebFetch::Backends::Browser.configured?
+        chain
+      end
+    end
+
+    class << self
+      attr_writer :backends
+    end
+
+    # Fetches +url+ through the chain and returns the first success as
+    # { title:, description:, content:, outlinks:, redirected: }. Raises
+    # when every backend fails; the raised class carries the verdict (see
+    # #collapse) and the message lists every backend and what it said.
+    def self.fetch_page(url)
+      failures = []
+      backends.each do |backend_class|
+        return backend_class.new.fetch(url)
+      rescue Ask::WebFetch::Error => e
+        failures << [backend_class, e]
+      end
+      collapse(failures, url)
+    end
+
+    # Collapses every backend's failure into ONE error whose class
+    # carries the best explanation. Precedence, most definitive first: a
+    # parked domain beats an empty shell (the shell IS the parking ad's
+    # shell — Local sees the JS redirect stub, Browser the lander),
+    # empty beats a dead 4xx (the page existed, it just had no content),
+    # and any deterministic explanation beats a transient one (transient
+    # keeps the retryable base Error). Clients read the class:
+    # ParkedDomainError / EmptyContentError / FetchError are terminal —
+    # retrying never changes the answer; Error may recover on retry.
+    def self.collapse(failures, url)
+      detail = failures.map { |backend, e| "#{backend.backend_name}: #{e.message}" }.join('; ')
+      message = "all web fetch backends failed for #{url} (#{detail})"
+
+      classes = failures.map { |_, e| e.class }
+      if classes.any? { |k| k <= Ask::WebFetch::ParkedDomainError }
+        raise Ask::WebFetch::ParkedDomainError, message
+      end
+      if classes.any? { |k| k <= Ask::WebFetch::EmptyContentError }
+        raise Ask::WebFetch::EmptyContentError, message
+      end
+
+      deterministic = failures.all? { |_, e| DETERMINISTIC.any? { |klass| e.is_a?(klass) } }
+      raise(deterministic ? Ask::WebFetch::FetchError : Ask::WebFetch::Error, message)
+    end
+
+    # Fetches +url+ and returns LLM-ready markdown — "# Title\n\nSource:
+    # url\n\ncontent" — capped at +max_chars+ (default 20000; pass nil to
+    # skip the cap). The single entry point for "give me this page as
+    # markdown"; the raw page hash is #fetch_page.
+    def self.fetch(url, max_chars: DEFAULT_MAX_CHARS)
+      page = fetch_page(url)
+      markdown = format(page, url)
+      markdown = truncate(markdown, max_chars) if max_chars&.positive?
+      markdown
+    end
+
+    def self.format(page, url)
+      header = +''
+      title = page[:title]
+      header << "# #{title}\n\n" unless title.to_s.empty?
+      header << "Source: #{url}\n\n"
+      header + page[:content]
+    end
+    private_class_method :format
+
+    def self.truncate(markdown, max_chars)
+      return markdown if markdown.length <= max_chars
+
+      "#{markdown[0, max_chars].rstrip}\n\n…(truncated)"
+    end
+    private_class_method :truncate
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ask-web-fetch
 version: !ruby/object:Gem::Version
-  version: 0.6.2
+  version: 0.7.0
 platform: ruby
 authors:
 - Kaka Ruto
@@ -9,20 +9,6 @@ bindir: bin
 cert_chain: []
 date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
-- !ruby/object:Gem::Dependency
-  name: ask-tools
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0.1'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0.1'
 - !ruby/object:Gem::Dependency
   name: ferrum
   requirement: !ruby/object:Gem::Requirement
@@ -121,12 +107,12 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '3.26'
-description: Provides Ask::Tools::WebFetch, a tool that fetches a URL and converts
-  its content to clean markdown for LLM consumption. Defaults to a pure Ruby backend
-  (httpx + Nokogiri + reverse_markdown) with a Jina Reader fallback for JS-rendered
-  or blocked pages, and a real-Chrome fallback (Ferrum) that renders JavaScript and
-  lets auto-solving Cloudflare challenges complete. Works with any ask-rb chat or
-  agent.
+description: 'Fetches a URL and converts its content to clean markdown for LLM consumption.
+  A pluggable backend chain: pure Ruby httpx + Nokogiri + reverse_markdown by default,
+  a Jina Reader fallback for JS-rendered or blocked pages, and a real-Chrome fallback
+  (Ferrum) that renders JavaScript and lets auto-solving Cloudflare challenges complete.
+  The capability layer only — tool framing (MCP servers, agent tools) is provided
+  by the consumers.'
 email:
 - kaka@myrrlabs.com
 executables: []
@@ -147,7 +133,6 @@ files:
 - lib/ask/web_fetch/http.rb
 - lib/ask/web_fetch/markdown.rb
 - lib/ask/web_fetch/noise_filter.rb
-- lib/ask/web_fetch/tool.rb
 - lib/ask/web_fetch/version.rb
 homepage: https://github.com/ask-rb/ask-web-fetch
 licenses:
@@ -173,5 +158,5 @@ required_rubygems_version: !ruby/object:Gem::Requirement
 requirements: []
 rubygems_version: 4.0.3
 specification_version: 4
-summary: Web fetch tool for the ask-rb ecosystem
+summary: Web fetch library for the ask-rb ecosystem
 test_files: []

data/lib/ask/web_fetch/tool.rb

@@ -1,125 +0,0 @@
-# frozen_string_literal: true
-
-require 'ask-tools'
-require_relative '../web_fetch/backend'
-require_relative '../web_fetch/backends/local'
-require_relative '../web_fetch/backends/crawl4ai'
-require_relative '../web_fetch/backends/jina'
-
-module Ask
-  module Tools
-    # Fetches a URL and returns its content as clean markdown for LLM
-    # consumption. Tries each configured backend in order and returns the
-    # first success.
-    #
-    # Chain: Crawl4AI first (self-hosted headless-Chromium renderer; used
-    # when the CRAWL4AI_URL service is present, fails fast when it isn't),
-    # then the local fetcher, Jina Reader as the last resort, and a real
-    # Chrome (via Ferrum) at the very end for pages whose Cloudflare-style
-    # challenges the others cannot pass — appended only when a browser
-    # binary is present.
-    class WebFetch < Ask::Tool
-      DEFAULT_MAX_CHARS = 20_000
-
-      # Errors that mean "the URL is dead" — no amount of retrying changes
-      # the answer. When EVERY backend failed this way, the aggregate
-      # re-raises as FetchError so callers can fail fast; any transient
-      # failure in the mix (timeout, 5xx, empty render) keeps the base
-      # Error, which recovers on retry.
-      DETERMINISTIC = [Ask::WebFetch::FetchError, Ask::WebFetch::EmptyContentError].freeze
-
-      # Backend chain, tried in order. Crawl4AI leads when configured
-      # (CRAWL4AI_URL), so a present self-hosted renderer is preferred;
-      # otherwise Local, with Jina as the last resort, and Browser appended
-      # when Chrome is available. Swap or extend for future backends; each
-      # must subclass Ask::WebFetch::Backend and implement #fetch(url).
-      def self.backends
-        @backends ||= begin
-          chain = [Ask::WebFetch::Backends::Local, Ask::WebFetch::Backends::Jina]
-          if Ask::WebFetch::Backends::Crawl4Ai.configured?
-            chain.unshift(Ask::WebFetch::Backends::Crawl4Ai)
-          end
-          chain << Ask::WebFetch::Backends::Browser if Ask::WebFetch::Backends::Browser.configured?
-          chain
-        end
-      end
-
-    class << self
-      attr_writer :backends
-
-      # Collapses every backend's failure into ONE error whose class
-      # carries the best explanation. Precedence, most definitive first: a
-      # parked domain beats an empty shell (the shell IS the parking ad's
-      # shell — Local sees the JS redirect stub, Browser the lander),
-      # empty beats a dead 4xx (the page existed, it just had no content),
-      # and any deterministic explanation beats a transient one (transient
-      # keeps the retryable base Error). Clients read the class:
-      # ParkedDomainError / EmptyContentError / FetchError are terminal —
-      # retrying never changes the answer; Error may recover on retry.
-      def collapse(failures, url)
-        detail = failures.map { |backend, e| "#{backend.backend_name}: #{e.message}" }.join('; ')
-        message = "all web fetch backends failed for #{url} (#{detail})"
-
-        classes = failures.map { |_, e| e.class }
-        if classes.any? { |k| k <= Ask::WebFetch::ParkedDomainError }
-          raise Ask::WebFetch::ParkedDomainError, message
-        end
-        if classes.any? { |k| k <= Ask::WebFetch::EmptyContentError }
-          raise Ask::WebFetch::EmptyContentError, message
-        end
-
-        deterministic = failures.all? { |_, e| DETERMINISTIC.any? { |klass| e.is_a?(klass) } }
-        raise(deterministic ? Ask::WebFetch::FetchError : Ask::WebFetch::Error, message)
-      end
-    end
-
-      description 'Fetch a URL and return its content as clean markdown for LLM consumption. ' \
-                  'Use this to read web pages, articles, and documentation.'
-
-      params(
-        type: 'object',
-        properties: {
-          url: { type: 'string', description: 'The URL to fetch' },
-          max_chars: { type: 'integer', description: 'Maximum number of characters to return (default 20000)' }
-        },
-        required: ['url']
-      )
-
-      def execute(url:, max_chars: DEFAULT_MAX_CHARS)
-        page = fetch_page(url)
-        markdown = format(page, url)
-        markdown = truncate(markdown, max_chars) if max_chars&.positive?
-        Ask::Result.ok(data: markdown)
-      end
-
-      private
-
-      # Try each configured backend in order; return the first success.
-      def fetch_page(url)
-        failures = []
-        self.class.backends.each do |backend_class|
-          return backend_class.new.fetch(url)
-        rescue Ask::WebFetch::Error => e
-          failures << [backend_class, e]
-        end
-        self.class.collapse(failures, url)
-      end
-
-      def format(page, url)
-        header = +''
-        title = page[:title]
-        header << "# #{title}\n\n" unless title.to_s.empty?
-        header << "Source: #{url}\n\n"
-        header + page[:content]
-      end
-
-      def truncate(markdown, max_chars)
-        return markdown if markdown.length <= max_chars
-
-        "#{markdown[0, max_chars].rstrip}\n\n…(truncated)"
-      end
-    end
-  end
-end
-
-Ask::Tools.register(Ask::Tools::WebFetch) if defined?(Ask::Tools)