ask-web-fetch 0.6.1 → 0.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 15d4465ae7ca7805003043af5b97e96ddc8c2ba19618daab313377f4a529bd54
-  data.tar.gz: 9f83f8c2258dc973457158e7d7e2288afc43403221453545a4d19c97440b85ee
+  metadata.gz: 6f73de0a29e2d37986a2b05a930640143f5566011782b5d68eed318bd82db746
+  data.tar.gz: 86ee4e5a800bf7fa67df0a652f40f0058ba5e1c1fa00e77a63b737017e61fbf0
 SHA512:
-  metadata.gz: b34dfd5f3d45b827cf8882d3c9c3b47e05ea8c16622849b2a41fc51b8c5f005f0b149ef83a87db57f4167470e85911ec58f72830afded9a84ab707ea08dcbc68
-  data.tar.gz: c8245c574d480179cd3b8ad67f1049b096fdf298f0886a790cb557f698c11e8dbaed9bb4336de4ccf84882e03eb8cf716830db069988f310c35cf4202cf01d60
+  metadata.gz: 9b3fe951669c892269d18304d4242786637df6b4f1ddd39aa4bfbc70d2241f90193701665f9141f5c7e816ba12e1a7bd5c48b98d2f0748ce1e093cfc70c5cd16
+  data.tar.gz: f372adee62b1ab72695509aa6bac61d105a9dc64e5209d368273c8667fc62145075b978bd572b426bd54a42770a31f0ff192245f9f37b8f9c1c9e121a3fa0360

data/lib/ask/web_fetch/backend.rb

@@ -44,7 +44,9 @@ module Ask
     #      Markdown.generate, backends fed pre-converted markdown (Jina,
     #      Crawl4AI) must call it explicitly so the shared noise removal
     #      and whitespace normalization apply everywhere
-    #   5. register the class in Ask::Tools::WebFetch.backends
+    #   5. run the extracted page through #guard_page! — the parked-domain
+    #      and empty-content verdicts are identical in every backend
+    #   6. register the class in Ask::WebFetch.backends
     #
     # The tool tries each backend in order and returns the first success.
     class Backend
@@ -92,6 +94,29 @@ module Ask
         raise NotImplementedError, "#{self.class} must implement #fetch(url)"
       end
 
+      # The shared page guard, run by EVERY backend at the same point in
+      # its flow — after extraction, before returning: a registrar parking
+      # page raises ParkedDomainError, content below the minimum raises
+      # EmptyContentError. Same verdicts, same messages, everywhere; a
+      # backend's only job is to pass the strings it has.
+      #
+      # raw_body: the raw HTML the backend saw, where it saw it (Local,
+      # Browser) — the HTML-only markers (ap:"parking", parking-lander,
+      # LANDER_SYSTEM="PW") live in scripts and assets that never survive
+      # conversion to markdown. content: what the backend would return
+      # (all four) — the prose markers survive conversion, so a backend
+      # that only ever sees rendered text (Jina, Crawl4AI) still rejects
+      # the ad.
+      #
+      # Parked is checked BEFORE the content minimum on purpose: a parking
+      # page can render above it (puncta.ai: 395c of Namecheap auction
+      # ads) and must still be rejected.
+      def guard_page!(url, content, raw_body: nil)
+        raise ParkedDomainError,
+              "parked domain at #{url} — registrar parking page, not site content" if parked_domain?(raw_body) || parked_domain?(content)
+        raise EmptyContentError, "no readable content at #{url}" unless usable_content?(content)
+      end
+
       # --- outlinks (crawler discovery) ---
 
       # The page's raw outlinks from HTML: every <a href> resolved against

data/lib/ask/web_fetch/backends/browser.rb

@@ -201,13 +201,12 @@ module Ask
           end
           # Browser renders the parked page a JS redirect lands on (the
           # server shell hands /lander to JS) — Local never sees it. The
-          # shared parked-domain detector catches it here; the distinct
-          # ParkedDomainError lets the pipeline classify (never retry) it.
-          raise ParkedDomainError, "parked domain at #{url} — registrar parking page, not site content" if parked_domain?(body)
-
+          # shared guard catches it on the rendered HTML (raw_body) and
+          # the converted content; the distinct ParkedDomainError lets the
+          # pipeline classify (never retry) it.
           result = Markdown.generate(body, base_url: url, filter: self.class.content_filter)
           result[:outlinks] = outlink_urls(body, url)
-          raise EmptyContentError, "no readable content at #{url}" unless usable_content?(result[:content])
+          guard_page!(url, result[:content], raw_body: body)
 
           result
         end

data/lib/ask/web_fetch/backends/crawl4ai.rb

@@ -56,12 +56,12 @@ module Ask
 
           page = to_page(body, url)
           # A registrar parking page renders fine in headless Chrome too —
-          # the shared detector (0.5.7) catches the text markers that
-          # survive conversion, so the ad is rejected, not returned as
-          # the site's content. Crawl4AI leads the default chain, so this
-          # check is what keeps parked domains out of every result.
-          raise ParkedDomainError, "parked domain at #{url} — registrar parking page, not site content" if parked_domain?(page[:content])
-          raise EmptyContentError, "no readable content at #{url}" unless usable_content?(page[:content])
+          # the shared guard's prose markers catch it (the HTML-only
+          # markers never reach a markdown-only backend), so the ad is
+          # rejected, not returned as the site's content. Crawl4AI leads
+          # the default chain, so this guard is what keeps parked domains
+          # out of every result.
+          guard_page!(url, page[:content])
 
           page
         rescue Net::OpenTimeout, Net::ReadTimeout, Errno::ECONNREFUSED,

data/lib/ask/web_fetch/backends/jina.rb

@@ -35,18 +35,17 @@ module Ask
             body = res.body.to_s
             raise FetchError, 'challenge page from Jina' if challenge_page?(body)
 
+            content = Markdown.clean(body)
             # Jina only sees rendered markdown — outlinks come from its
             # links, resolved against the requested URL. Content runs
             # through the same Markdown.clean as the converting backends,
             # so decorative symbol noise is stripped here too; a page
             # whose only "content" was noise falls through as empty. A
             # registrar parking page renders fine through Jina — the
-            # shared detector (0.5.7) catches the text markers that
-            # survive conversion, so the ad is rejected, not returned as
-            # the site's content.
-            content = Markdown.clean(body)
-            raise ParkedDomainError, "parked domain at #{url} — registrar parking page, not site content" if parked_domain?(content)
-            raise EmptyContentError, 'empty response from Jina' unless usable_content?(content)
+            # shared guard's prose markers catch it (the HTML-only
+            # markers never reach a markdown-only backend), so the ad is
+            # rejected, not returned as the site's content.
+            guard_page!(url, content)
 
             { title: nil, description: nil, content: content, outlinks: markdown_outlinks(body, url) }
           when '429'

data/lib/ask/web_fetch/backends/local.rb

@@ -48,16 +48,12 @@ module Ask
           # Parked-domain pages are not content: the domain owner parked it
           # with a registrar and the page is an ad for buying the domain
           # (GoDaddy/Namecheap/Sedo parking). A content company must never
-          # store these as if they were the site. Checked BEFORE the
-          # content-minimum — a parking page can render as "content" above
-          # the minimum (puncta.ai: 395c of Namecheap auction ads), and
-          # must still be rejected. Detectable from the server HTML —
-          # parked pages are fully server-rendered, so both Local and
-          # Browser see the same ad.
-          if parked_domain?(body)
-            raise ParkedDomainError, "parked domain at #{url} — registrar parking page, not site content"
-          end
-          raise EmptyContentError, "no readable content at #{url}" unless usable_content?(page[:content])
+          # store these as if they were the site. The shared guard checks
+          # the raw server HTML first (parked pages are fully
+          # server-rendered — the HTML-only markers live in scripts and
+          # assets), then the content minimum, then the JS-shell
+          # completeness signal below.
+          guard_page!(url, page[:content], raw_body: body)
           # The completeness signal: a JS-app shell whose server HTML
           # renders little is a TRUNCATED page, not a complete one — the
           # real content awaits client-side JS that Local cannot run.

data/lib/ask/web_fetch/version.rb

@@ -2,6 +2,6 @@
 
 module Ask
   module WebFetch
-    VERSION = '0.6.1'
+    VERSION = '0.7.0'
   end
 end

data/lib/ask/web_fetch.rb

@@ -9,4 +9,107 @@ require_relative 'web_fetch/backends/local'
 require_relative 'web_fetch/backends/crawl4ai'
 require_relative 'web_fetch/backends/jina'
 require_relative 'web_fetch/backends/browser'
-require_relative 'web_fetch/tool'
+
+module Ask
+  # Fetches a URL and returns its content as clean markdown for LLM
+  # consumption. The capability layer: a pluggable backend chain, a
+  # failure collapse, and one entry point. Tool framing — name,
+  # parameter schema, result wrapping — lives with the consumers (the
+  # MCP servers, the agents) that call this library, not here.
+  module WebFetch
+    DEFAULT_MAX_CHARS = 20_000
+
+    # Errors that mean "the URL is dead" — no amount of retrying changes
+    # the answer. When EVERY backend failed this way, the aggregate
+    # re-raises as FetchError so callers can fail fast; any transient
+    # failure in the mix (timeout, 5xx, empty render) keeps the base
+    # Error, which recovers on retry.
+    DETERMINISTIC = [Ask::WebFetch::FetchError, Ask::WebFetch::EmptyContentError].freeze
+
+    # Backend chain, tried in order. Crawl4AI leads when configured
+    # (CRAWL4AI_URL), so a present self-hosted renderer is preferred;
+    # otherwise Local, with Jina as the last resort, and Browser appended
+    # when Chrome is available. Swap or extend for future backends; each
+    # must subclass Ask::WebFetch::Backend and implement #fetch(url).
+    def self.backends
+      @backends ||= begin
+        chain = [Ask::WebFetch::Backends::Local, Ask::WebFetch::Backends::Jina]
+        if Ask::WebFetch::Backends::Crawl4Ai.configured?
+          chain.unshift(Ask::WebFetch::Backends::Crawl4Ai)
+        end
+        chain << Ask::WebFetch::Backends::Browser if Ask::WebFetch::Backends::Browser.configured?
+        chain
+      end
+    end
+
+    class << self
+      attr_writer :backends
+    end
+
+    # Fetches +url+ through the chain and returns the first success as
+    # { title:, description:, content:, outlinks:, redirected: }. Raises
+    # when every backend fails; the raised class carries the verdict (see
+    # #collapse) and the message lists every backend and what it said.
+    def self.fetch_page(url)
+      failures = []
+      backends.each do |backend_class|
+        return backend_class.new.fetch(url)
+      rescue Ask::WebFetch::Error => e
+        failures << [backend_class, e]
+      end
+      collapse(failures, url)
+    end
+
+    # Collapses every backend's failure into ONE error whose class
+    # carries the best explanation. Precedence, most definitive first: a
+    # parked domain beats an empty shell (the shell IS the parking ad's
+    # shell — Local sees the JS redirect stub, Browser the lander),
+    # empty beats a dead 4xx (the page existed, it just had no content),
+    # and any deterministic explanation beats a transient one (transient
+    # keeps the retryable base Error). Clients read the class:
+    # ParkedDomainError / EmptyContentError / FetchError are terminal —
+    # retrying never changes the answer; Error may recover on retry.
+    def self.collapse(failures, url)
+      detail = failures.map { |backend, e| "#{backend.backend_name}: #{e.message}" }.join('; ')
+      message = "all web fetch backends failed for #{url} (#{detail})"
+
+      classes = failures.map { |_, e| e.class }
+      if classes.any? { |k| k <= Ask::WebFetch::ParkedDomainError }
+        raise Ask::WebFetch::ParkedDomainError, message
+      end
+      if classes.any? { |k| k <= Ask::WebFetch::EmptyContentError }
+        raise Ask::WebFetch::EmptyContentError, message
+      end
+
+      deterministic = failures.all? { |_, e| DETERMINISTIC.any? { |klass| e.is_a?(klass) } }
+      raise(deterministic ? Ask::WebFetch::FetchError : Ask::WebFetch::Error, message)
+    end
+
+    # Fetches +url+ and returns LLM-ready markdown — "# Title\n\nSource:
+    # url\n\ncontent" — capped at +max_chars+ (default 20000; pass nil to
+    # skip the cap). The single entry point for "give me this page as
+    # markdown"; the raw page hash is #fetch_page.
+    def self.fetch(url, max_chars: DEFAULT_MAX_CHARS)
+      page = fetch_page(url)
+      markdown = format(page, url)
+      markdown = truncate(markdown, max_chars) if max_chars&.positive?
+      markdown
+    end
+
+    def self.format(page, url)
+      header = +''
+      title = page[:title]
+      header << "# #{title}\n\n" unless title.to_s.empty?
+      header << "Source: #{url}\n\n"
+      header + page[:content]
+    end
+    private_class_method :format
+
+    def self.truncate(markdown, max_chars)
+      return markdown if markdown.length <= max_chars
+
+      "#{markdown[0, max_chars].rstrip}\n\n…(truncated)"
+    end
+    private_class_method :truncate
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ask-web-fetch
 version: !ruby/object:Gem::Version
-  version: 0.6.1
+  version: 0.7.0
 platform: ruby
 authors:
 - Kaka Ruto
@@ -9,20 +9,6 @@ bindir: bin
 cert_chain: []
 date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
-- !ruby/object:Gem::Dependency
-  name: ask-tools
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0.1'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0.1'
 - !ruby/object:Gem::Dependency
   name: ferrum
   requirement: !ruby/object:Gem::Requirement
@@ -121,12 +107,12 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '3.26'
-description: Provides Ask::Tools::WebFetch, a tool that fetches a URL and converts
-  its content to clean markdown for LLM consumption. Defaults to a pure Ruby backend
-  (httpx + Nokogiri + reverse_markdown) with a Jina Reader fallback for JS-rendered
-  or blocked pages, and a real-Chrome fallback (Ferrum) that renders JavaScript and
-  lets auto-solving Cloudflare challenges complete. Works with any ask-rb chat or
-  agent.
+description: 'Fetches a URL and converts its content to clean markdown for LLM consumption.
+  A pluggable backend chain: pure Ruby httpx + Nokogiri + reverse_markdown by default,
+  a Jina Reader fallback for JS-rendered or blocked pages, and a real-Chrome fallback
+  (Ferrum) that renders JavaScript and lets auto-solving Cloudflare challenges complete.
+  The capability layer only — tool framing (MCP servers, agent tools) is provided
+  by the consumers.'
 email:
 - kaka@myrrlabs.com
 executables: []
@@ -147,7 +133,6 @@ files:
 - lib/ask/web_fetch/http.rb
 - lib/ask/web_fetch/markdown.rb
 - lib/ask/web_fetch/noise_filter.rb
-- lib/ask/web_fetch/tool.rb
 - lib/ask/web_fetch/version.rb
 homepage: https://github.com/ask-rb/ask-web-fetch
 licenses:
@@ -173,5 +158,5 @@ required_rubygems_version: !ruby/object:Gem::Requirement
 requirements: []
 rubygems_version: 4.0.3
 specification_version: 4
-summary: Web fetch tool for the ask-rb ecosystem
+summary: Web fetch library for the ask-rb ecosystem
 test_files: []

data/lib/ask/web_fetch/tool.rb

@@ -1,125 +0,0 @@
-# frozen_string_literal: true
-
-require 'ask-tools'
-require_relative '../web_fetch/backend'
-require_relative '../web_fetch/backends/local'
-require_relative '../web_fetch/backends/crawl4ai'
-require_relative '../web_fetch/backends/jina'
-
-module Ask
-  module Tools
-    # Fetches a URL and returns its content as clean markdown for LLM
-    # consumption. Tries each configured backend in order and returns the
-    # first success.
-    #
-    # Chain: Crawl4AI first (self-hosted headless-Chromium renderer; used
-    # when the CRAWL4AI_URL service is present, fails fast when it isn't),
-    # then the local fetcher, Jina Reader as the last resort, and a real
-    # Chrome (via Ferrum) at the very end for pages whose Cloudflare-style
-    # challenges the others cannot pass — appended only when a browser
-    # binary is present.
-    class WebFetch < Ask::Tool
-      DEFAULT_MAX_CHARS = 20_000
-
-      # Errors that mean "the URL is dead" — no amount of retrying changes
-      # the answer. When EVERY backend failed this way, the aggregate
-      # re-raises as FetchError so callers can fail fast; any transient
-      # failure in the mix (timeout, 5xx, empty render) keeps the base
-      # Error, which recovers on retry.
-      DETERMINISTIC = [Ask::WebFetch::FetchError, Ask::WebFetch::EmptyContentError].freeze
-
-      # Backend chain, tried in order. Crawl4AI leads when configured
-      # (CRAWL4AI_URL), so a present self-hosted renderer is preferred;
-      # otherwise Local, with Jina as the last resort, and Browser appended
-      # when Chrome is available. Swap or extend for future backends; each
-      # must subclass Ask::WebFetch::Backend and implement #fetch(url).
-      def self.backends
-        @backends ||= begin
-          chain = [Ask::WebFetch::Backends::Local, Ask::WebFetch::Backends::Jina]
-          if Ask::WebFetch::Backends::Crawl4Ai.configured?
-            chain.unshift(Ask::WebFetch::Backends::Crawl4Ai)
-          end
-          chain << Ask::WebFetch::Backends::Browser if Ask::WebFetch::Backends::Browser.configured?
-          chain
-        end
-      end
-
-    class << self
-      attr_writer :backends
-
-      # Collapses every backend's failure into ONE error whose class
-      # carries the best explanation. Precedence, most definitive first: a
-      # parked domain beats an empty shell (the shell IS the parking ad's
-      # shell — Local sees the JS redirect stub, Browser the lander),
-      # empty beats a dead 4xx (the page existed, it just had no content),
-      # and any deterministic explanation beats a transient one (transient
-      # keeps the retryable base Error). Clients read the class:
-      # ParkedDomainError / EmptyContentError / FetchError are terminal —
-      # retrying never changes the answer; Error may recover on retry.
-      def collapse(failures, url)
-        detail = failures.map { |backend, e| "#{backend.backend_name}: #{e.message}" }.join('; ')
-        message = "all web fetch backends failed for #{url} (#{detail})"
-
-        classes = failures.map { |_, e| e.class }
-        if classes.any? { |k| k <= Ask::WebFetch::ParkedDomainError }
-          raise Ask::WebFetch::ParkedDomainError, message
-        end
-        if classes.any? { |k| k <= Ask::WebFetch::EmptyContentError }
-          raise Ask::WebFetch::EmptyContentError, message
-        end
-
-        deterministic = failures.all? { |_, e| DETERMINISTIC.any? { |klass| e.is_a?(klass) } }
-        raise(deterministic ? Ask::WebFetch::FetchError : Ask::WebFetch::Error, message)
-      end
-    end
-
-      description 'Fetch a URL and return its content as clean markdown for LLM consumption. ' \
-                  'Use this to read web pages, articles, and documentation.'
-
-      params(
-        type: 'object',
-        properties: {
-          url: { type: 'string', description: 'The URL to fetch' },
-          max_chars: { type: 'integer', description: 'Maximum number of characters to return (default 20000)' }
-        },
-        required: ['url']
-      )
-
-      def execute(url:, max_chars: DEFAULT_MAX_CHARS)
-        page = fetch_page(url)
-        markdown = format(page, url)
-        markdown = truncate(markdown, max_chars) if max_chars&.positive?
-        Ask::Result.ok(data: markdown)
-      end
-
-      private
-
-      # Try each configured backend in order; return the first success.
-      def fetch_page(url)
-        failures = []
-        self.class.backends.each do |backend_class|
-          return backend_class.new.fetch(url)
-        rescue Ask::WebFetch::Error => e
-          failures << [backend_class, e]
-        end
-        self.class.collapse(failures, url)
-      end
-
-      def format(page, url)
-        header = +''
-        title = page[:title]
-        header << "# #{title}\n\n" unless title.to_s.empty?
-        header << "Source: #{url}\n\n"
-        header + page[:content]
-      end
-
-      def truncate(markdown, max_chars)
-        return markdown if markdown.length <= max_chars
-
-        "#{markdown[0, max_chars].rstrip}\n\n…(truncated)"
-      end
-    end
-  end
-end
-
-Ask::Tools.register(Ask::Tools::WebFetch) if defined?(Ask::Tools)