ask-web-fetch 0.4.1 → 0.5.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 318ff9c594a7d5c52a710507efd3da2917bbe3319471a3e0bac8bce3a01ed30e
-  data.tar.gz: f5ae0d23b351044ecc21f4e2c4d2ec66a13d8258cd9a86d6947f0c7698c6e703
+  metadata.gz: 0f49a52ecb57c57ee29ce475ad791f70d30fe52946c94c6bd6842dcccc477392
+  data.tar.gz: b67e97adb54db5b6a775d1ece43df51c0e5443a6ef9e2cd3a17d547af5f7d1dc
 SHA512:
-  metadata.gz: 775322ca3144aa69da6d27e20664c5a18aa836baaf10d5431acc5cbc7dd893aff5e09c2c40751d3f19b6ea86410ad92cf992ad35b2283554b31196ba52fdea29
-  data.tar.gz: ae5f935d3b48522330b5a01f9319813bd84d157076f65ee2a17cdbd82fb3fb65916f5312ea03fefdc51e8b7a84792c9558ede959e195e386f2713fb1c31e437a
+  metadata.gz: 28dd03ea3e7aae930925ea2d582ff074beba7d3030d762830024e41506a23fd671bb166fa5809417a824e8d2c1729679d00f3d710fa82de05b2ca41038dd39de
+  data.tar.gz: 84815a2d6504c81df9d81aab2fd679d5bb334f64893b05264cc4185be0e7d7439f357461494c65811cbdb16e0e4548b0b2db290f52b747412678eb953c5f40b3

data/README.md

@@ -26,6 +26,14 @@ first success:
    headless Chromium, so it renders JS pages the Local backend can't. Free
    without a key (~20 req/min per IP); set `JINA_API_KEY` for higher limits.
 
+Every backend's markdown runs through a shared cleanup (`Markdown.clean`):
+decorative symbol noise — the long, letter-free, repetitive character
+streams some pages render as animated backgrounds or section dividers — is
+stripped, and whitespace is normalized. The filter is conservative: code
+blocks, tables, headings, blockquotes, inline code, and short ASCII-art
+fragments always survive. Tunable via `Ask::WebFetch::NoiseFilter.filter(
+markdown, min_length:, max_entropy:)`.
+
 The tool falls back automatically: if Crawl4AI is absent or fails, Local is
 tried (blocked, timeout, non-HTML, anti-bot challenge, or a JS page with no
 server-side content), then Jina. If every backend fails (rate limit, access
@@ -68,9 +76,13 @@ Ask::Tools::WebFetch.backends = [MyBackend, Ask::WebFetch::Backends::Local]
 
 `#fetch` must return `{ title: String|nil, content: String }` and raise
 `Ask::WebFetch::FetchError` (hard failure) or `EmptyContentError` (page
-fetched but nothing usable). The chain then handles ordering and fallback
-for you. For tests, `Ask::Tools::WebFetch.backends = [...]` can be swapped
-and restored.
+fetched but nothing usable). Run the returned markdown through
+`Ask::WebFetch::Markdown.clean` (backends that convert HTML get this from
+`Markdown.generate`; backends fed pre-converted markdown must call it
+explicitly) so the shared noise removal and whitespace normalization apply
+everywhere. The chain then handles ordering and fallback for you. For
+tests, `Ask::Tools::WebFetch.backends = [...]` can be swapped and
+restored.
 
 ## Installation
 
@@ -125,6 +137,9 @@ No configuration required for the default chain. Optional knobs:
   no content unless Crawl4AI is configured — set `CRAWL4AI_URL` to handle
   them with a self-hosted renderer.
 - Some sites block non-browser requests regardless of User-Agent.
+- Symbol streams that a converter merges *into* a content line (rather
+  than leaving them as their own lines) are out of scope for the
+  markdown-level NoiseFilter — that would need a DOM-level pass.
 
 ## Full documentation
 

data/lib/ask/web_fetch/backend.rb

@@ -32,7 +32,12 @@ module Ask
     #   2. #fetch must return { title: String|nil, content: String }
     #   3. #fetch must raise FetchError (hard failure) or
     #      EmptyContentError (page fetched but nothing usable) on failure
-    #   4. register the class in Ask::Tools::WebFetch.backends
+    #   4. run the page's markdown through Ask::WebFetch::Markdown.clean
+    #      before returning it — backends that hold HTML get this from
+    #      Markdown.generate, backends fed pre-converted markdown (Jina,
+    #      Crawl4AI) must call it explicitly so the shared noise removal
+    #      and whitespace normalization apply everywhere
+    #   5. register the class in Ask::Tools::WebFetch.backends
     #
     # The tool tries each backend in order and returns the first success.
     class Backend
@@ -42,8 +47,11 @@ module Ask
                    "ask-web-fetch/#{Ask::WebFetch::VERSION}".freeze
 
       # Content shorter than this is treated as a page with no usable
-      # content (e.g. a JS-rendered shell with nothing server-side).
-      MIN_CONTENT_LENGTH = 100
+      # content (e.g. a JS-rendered shell with nothing server-side). Low on
+      # purpose: the content hub keeps everything that is a real page, and
+      # the crawler's soft-404 detection (not length) is what separates
+      # pages from error shells.
+      MIN_CONTENT_LENGTH = 40
 
       # Cloudflare-style anti-bot signatures. Deliberately narrow:
       # challenge/interstitial pages carry these markers, while legitimate
@@ -55,12 +63,55 @@ module Ask
         name.split('::').last
       end
 
-      # Fetches +url+ and returns { title: String|nil, content: String }.
+      # Fetches +url+ and returns { title:, description:, content:,
+      # redirected:, licenses:, outlinks: } — licenses being the page's
+      # declared license signals (hrefs/values) and outlinks the page's
+      # raw link set ([] when the backend can't see any), both consumed by
+      # the crawler's classification/discovery layers.
       # Raises FetchError or EmptyContentError on failure.
       def fetch(url)
         raise NotImplementedError, "#{self.class} must implement #fetch(url)"
       end
 
+      # --- outlinks (crawler discovery) ---
+
+      # The page's raw outlinks from HTML: every <a href> resolved against
+      # the base URL and scheme-filtered to absolute http(s). Nav and
+      # footer are included — a crawler's discovery reads the full link set
+      # even when the stored content is pruned by the ContentFilter. Shared
+      # by every backend that holds the page's HTML.
+      def outlink_urls(html, base_url)
+        Nokogiri::HTML(html).css('a[href]').filter_map do |anchor|
+          href = anchor['href'].to_s.strip
+          next if href.empty? || href.start_with?('javascript:', 'mailto:', 'tel:', '#', 'data:')
+
+          uri = URI.join(base_url, href)
+          next unless %w[http https].include?(uri.scheme)
+
+          uri.to_s
+        rescue URI::InvalidURIError
+          next
+        end.uniq
+      end
+
+      # Fallback for backends that only see rendered markdown (Jina,
+      # Crawl4AI's markdown output): the markdown's [text](url) links,
+      # resolved and scheme-filtered. Same shape as #outlink_urls, one
+      # implementation for every backend that lacks the raw HTML.
+      def markdown_outlinks(content, base_url)
+        content.to_s.scan(/\]\(([^)\s]+)\)/).filter_map do |match|
+          dest = match[0]
+          next if dest.start_with?('javascript:', 'mailto:', 'tel:', '#', 'data:')
+
+          uri = URI.join(base_url, dest)
+          next unless %w[http https].include?(uri.scheme)
+
+          uri.to_s
+        rescue URI::InvalidURIError
+          next
+        end.uniq
+      end
+
       private
 
       def challenge_page?(body)

data/lib/ask/web_fetch/backends/browser.rb

@@ -164,6 +164,7 @@ module Ask
           raise FetchError, "challenge page at #{url}" if challenge_page?(body)
 
           result = Markdown.generate(body, base_url: url, filter: self.class.content_filter)
+          result[:outlinks] = outlink_urls(body, url)
           raise EmptyContentError, "no readable content at #{url}" unless usable_content?(result[:content])
 
           result

data/lib/ask/web_fetch/backends/crawl4ai.rb

@@ -4,6 +4,7 @@ require 'net/http'
 require 'uri'
 require 'json'
 require_relative '../backend'
+require_relative '../markdown'
 
 module Ask
   module WebFetch
@@ -115,11 +116,18 @@ module Ask
 
           markdown = result.dig('markdown', 'fit_markdown').to_s
           markdown = result.dig('markdown', 'raw_markdown').to_s if markdown.strip.empty?
+          # The service's markdown bypasses Markdown.generate, so it runs
+          # through the same shared clean (noise removal + whitespace) as
+          # the converting backends.
+          markdown = Markdown.clean(markdown)
           {
             title: result.dig('metadata', 'title'),
             description: result.dig('metadata', 'description') ||
               result.dig('metadata', 'og_description'),
             content: markdown,
+            # Raw markdown keeps the nav links the ContentFilter prunes —
+            # discovery reads these even when the stored content is lean.
+            outlinks: markdown_outlinks(markdown, url),
             # The page's own redirect, if the crawl followed one — lets
             # consumers record permanent redirects on their ledger instead
             # of silently indexing under the original URL.

data/lib/ask/web_fetch/backends/jina.rb

@@ -3,6 +3,7 @@
 require 'net/http'
 require 'uri'
 require_relative '../backend'
+require_relative '../markdown'
 
 module Ask
   module WebFetch
@@ -33,9 +34,16 @@ module Ask
           when '200'
             body = res.body.to_s
             raise FetchError, 'challenge page from Jina' if challenge_page?(body)
-            raise EmptyContentError, 'empty response from Jina' unless usable_content?(body)
 
-            { title: nil, description: nil, content: body.strip }
+            # Jina only sees rendered markdown — outlinks come from its
+            # links, resolved against the requested URL. Content runs
+            # through the same Markdown.clean as the converting backends,
+            # so decorative symbol noise is stripped here too; a page
+            # whose only "content" was noise falls through as empty.
+            content = Markdown.clean(body)
+            raise EmptyContentError, 'empty response from Jina' unless usable_content?(content)
+
+            { title: nil, description: nil, content: content, outlinks: markdown_outlinks(body, url) }
           when '429'
             raise ServerError, 'rate limited by Jina (429)'
           when '401', '403'

data/lib/ask/web_fetch/backends/local.rb

@@ -1,15 +1,15 @@
 # frozen_string_literal: true
 
-require 'net/http'
 require 'uri'
 require_relative '../backend'
 require_relative '../content_filter'
 require_relative '../markdown'
+require_relative '../http'
 
 module Ask
   module WebFetch
     module Backends
-      # Default backend: pure Ruby Net::HTTP + Nokogiri + reverse_markdown.
+      # Default backend: pure Ruby httpx + Nokogiri + reverse_markdown.
       # No external service or API key; mirrors ask-web-search's self-hosted
       # SearXNG approach.
       #
@@ -18,8 +18,6 @@ module Ask
       # than by keyword — is what comes back.
       class Local < Backend
         MAX_REDIRECTS = 5
-        OPEN_TIMEOUT = 5
-        READ_TIMEOUT = 15
 
         class << self
           # The ContentFilter applied to every page by default. Set to nil to
@@ -29,6 +27,15 @@ module Ask
           def content_filter
             @content_filter ||= ContentFilter.default
           end
+
+          # The single-hop HTTP transport, Ask::WebFetch::Http by default:
+          # a pooled keep-alive httpx session. Swappable in tests — and the
+          # seam any future transport swap goes through.
+          attr_writer :http
+
+          def http
+            @http ||= Http
+          end
         end
 
         def fetch(url)
@@ -41,21 +48,50 @@ module Ask
           raise EmptyContentError, "no readable content at #{url}" unless usable_content?(page[:content])
 
           page
-        rescue Net::OpenTimeout, Net::ReadTimeout, Errno::ECONNREFUSED,
-               Errno::ECONNRESET, SocketError, URI::InvalidURIError => e
+        rescue Errno::ECONNREFUSED, Errno::ECONNRESET, SocketError, URI::InvalidURIError => e
+          # The transport maps its own failures to TimeoutError; this guard
+          # keeps the "only Ask::WebFetch errors escape" invariant even if
+          # a transport bug lets a raw socket error through.
           raise TimeoutError, "#{e.class}: #{e.message}"
         end
 
-        # Parses +html+ and returns { title:, description:, content: } where
-        # content is clean markdown and description is the page's own meta
-        # description (meta name=description, then og:description) — what the
-        # site says about itself, free and authoritative.
+        # Parses +html+ and returns { title:, description:, content:,
+        # licenses:, outlinks: } where content is clean markdown (pruned by
+        # the ContentFilter), licenses are the page's declared license
+        # signals ([] when it declares none), and outlinks are the page's
+        # RAW hrefs, resolved and scheme-filtered — nav and footer included,
+        # because a crawler's discovery layer reads these even when the
+        # stored content is pruned.
         def to_markdown(html, url)
           Markdown.generate(html, base_url: url, filter: self.class.content_filter)
+            .merge(licenses: license_signals(html), outlinks: outlink_urls(html, url))
         end
 
         private
 
+        # The page's declared license signals: <link rel="license">, the
+        # license meta tags (meta name/property license, cc:license,
+        # dc.rights), and schema.org [itemprop=license] — hrefs first, then
+        # contents. Best-effort and conservative: nothing here decides what
+        # the page IS licensed under, it only reports what the page says
+        # about itself. The consumer (the crawler's license classifier)
+        # maps known markers; unknown signals are ignored there, never here.
+        def license_signals(html)
+          doc = Nokogiri::HTML(html)
+          selectors = [
+            'link[rel~="license"]',
+            'meta[name="license"], meta[property="license"]',
+            'meta[name="cc:license"], meta[property="cc:license"]',
+            'meta[name="dc.rights"], meta[property="dc.rights"]',
+            '[itemprop="license"]'
+          ]
+          doc.css(selectors.join(',')).filter_map do |node|
+            value = node['href'] || node['content'] || node.text
+            value = value.to_s.strip
+            value unless value.empty?
+          end.uniq
+        end
+
         # GET with redirect following (max MAX_REDIRECTS hops). Returns
         # [body, content_type, redirect] where redirect is nil when the
         # URL answered directly, else {status: first hop's status,
@@ -65,30 +101,23 @@ module Ask
           hops = 0
           first_hop_status = nil
           loop do
-            http = Net::HTTP.new(uri.host, uri.port)
-            http.open_timeout = OPEN_TIMEOUT
-            http.read_timeout = READ_TIMEOUT
-            http.use_ssl = uri.scheme == 'https'
-            req = Net::HTTP::Get.new(uri)
-            req['User-Agent'] = USER_AGENT
-            req['Accept'] = 'text/html,application/xhtml+xml'
-            res = http.request(req)
-            return [res.body, res['content-type'].to_s, redirect_info(first_hop_status, uri)] if res.code.start_with?('2')
-
-            unless res.code.start_with?('3') && res['location']
+            response = self.class.http.get(uri.to_s, headers: { 'accept' => 'text/html,application/xhtml+xml' })
+            return [response.body, response.content_type, redirect_info(first_hop_status, uri)] if (200..299).cover?(response.status)
+
+            unless (300..399).cover?(response.status) && !response.location.empty?
               # 4xx (other than 429) = the URL is dead; 429/5xx = transient.
-              code = res.code.to_i
-              raise(code == 429 || code >= 500 ? ServerError : FetchError, "got #{res.code} from #{url}")
+              code = response.status
+              raise(code == 429 || code >= 500 ? ServerError : FetchError, "got #{code} from #{url}")
             end
             raise FetchError, "hit a redirect loop at #{url}" if (hops += 1) > MAX_REDIRECTS
 
-            first_hop_status ||= res.code.to_i
-            uri = URI.join(uri, res['location'])
+            first_hop_status ||= response.status
+            uri = URI.join(uri, response.location)
           end
         end
 
         def redirect_info(status, uri)
-          status && {status: status, url: uri.to_s}
+          status && { status: status, url: uri.to_s }
         end
       end
     end

data/lib/ask/web_fetch/http.rb

@@ -0,0 +1,85 @@
+# frozen_string_literal: true
+
+require 'httpx'
+
+module Ask
+  module WebFetch
+    # Minimal pooled HTTP client for the Local backend. Wraps a per-thread
+    # httpx session so every page fetch in a thread reuses its keep-alive
+    # connection to the host — no fresh TCP+TLS handshake per page (the
+    # thing that made the Net::HTTP crawler cost ~1.25s per page) — with
+    # HTTP/2 when the server negotiates it, retries with backoff on
+    # transient failures, and automatic gzip/deflate decoding.
+    #
+    # Deliberately single-hop: the backend follows redirects itself, so the
+    # hop-by-hop chain it reports is exactly what happened. Error responses
+    # (4xx/5xx) are not errors to the transport — the backend decides what
+    # they mean. Transport-level failures (timeout, refused, reset, DNS,
+    # TLS) all surface as Ask::WebFetch::TimeoutError, the transient
+    # bucket, whatever their underlying class.
+    class Http
+      CONNECT_TIMEOUT = 5
+      READ_TIMEOUT = 15
+      WRITE_TIMEOUT = 15
+      # Whole-request cap. A page that can't be read in a minute is a
+      # problem page, not a stall worth a crawl worker.
+      OPERATION_TIMEOUT = 60
+      # Retries per request, on top of the crawl ledger's own auto-heal
+      # rounds. GETs are idempotent; a couple of cheap retries beat a full
+      # ledger round-trip for transient flakiness.
+      MAX_RETRIES = 2
+      RETRY_ON_STATUS = [429, 500, 502, 503, 504].freeze
+
+      # One hop's answer. Redirects stay the backend's job, so `location`
+      # rides along for the backend to resolve.
+      Response = Data.define(:status, :body, :content_type, :location)
+
+      def self.get(url, headers: {})
+        response = session.get(url, headers: headers)
+        return raise_timeout(response) if response.is_a?(HTTPX::ErrorResponse)
+
+        Response.new(
+          status: response.status,
+          body: response.body.to_s,
+          content_type: response.headers['content-type'].to_s,
+          location: response.headers['location'].to_s
+        )
+      end
+
+      # One pooled session per thread — httpx sessions are not thread-safe,
+      # and a crawl worker thread reusing its session across every page it
+      # fetches is what makes the pooling pay off. Sessions idle-close
+      # themselves after keep-alive timeout, so nothing to reap.
+      def self.session
+        Thread.current[SESSION_KEY] ||= build_session
+      end
+
+      def self.build_session
+        # :persistent is what makes the pooling real — without it httpx
+        # opens a fresh connection per request. It loads fiber_concurrency
+        # and the retries plugin internally.
+        HTTPX.plugin(:persistent).with(
+          timeout: {
+            connect_timeout: CONNECT_TIMEOUT,
+            read_timeout: READ_TIMEOUT,
+            write_timeout: WRITE_TIMEOUT,
+            operation_timeout: OPERATION_TIMEOUT
+          },
+          headers: { 'user-agent' => Backend::USER_AGENT },
+          max_retries: MAX_RETRIES,
+          retry_on: ->(response) { response.respond_to?(:status) && RETRY_ON_STATUS.include?(response.status) }
+        )
+      end
+      private_class_method :build_session
+
+      def self.raise_timeout(response)
+        error = response.error
+        raise TimeoutError, "#{error.class}: #{error.message}"
+      end
+      private_class_method :raise_timeout
+
+      SESSION_KEY = :ask_web_fetch_http_session
+      private_constant :SESSION_KEY
+    end
+  end
+end

data/lib/ask/web_fetch/markdown.rb

@@ -3,6 +3,7 @@
 require 'nokogiri'
 require 'reverse_markdown'
 require 'uri'
+require_relative 'noise_filter'
 
 module Ask
   module WebFetch
@@ -126,7 +127,12 @@ module Ask
         [parts.join, references.join]
       end
 
+      # Shared post-conversion cleanup: drop decorative symbol noise
+      # (NoiseFilter), then normalize whitespace. Every backend's content
+      # runs through this — Local and Browser via generate, Jina and
+      # Crawl4AI explicitly — so the same noise rules apply to all.
       def clean(markdown)
+        markdown = NoiseFilter.filter(markdown)
         markdown.gsub(/[ \t]+\n/, "\n")
                 .gsub(/\n{3,}/, "\n\n")
                 .strip

data/lib/ask/web_fetch/noise_filter.rb

@@ -0,0 +1,143 @@
+# frozen_string_literal: true
+
+module Ask
+  module WebFetch
+    # Strips decorative symbol noise from markdown: the long, letter-free,
+    # repetitive character streams pages render as animated backgrounds,
+    # marquees and section dividers — e.g. Hugging Face's storage page
+    # ships a "+ = · ( ~ @ # % & * ? / : ; < > [ ] { } | ^ $ !" stream as
+    # its page background. Runs on CONVERTED markdown, so every backend
+    # benefits: Local and Browser already convert through Markdown, and
+    # Jina and Crawl4AI hand the gem pre-converted markdown — the
+    # DOM-level ContentFilter never sees either case.
+    #
+    # Conservative by design. A line is dropped only when ALL hold:
+    #
+    #   * it is long enough to matter (>= min_length characters)
+    #   * it contains no letters or digits at all
+    #   * it is repetitive — at least two distinct characters, with a
+    #     distinct/length ratio below max_entropy (a repeated stream,
+    #     not prose punctuation)
+    #   * it is not markdown structure: fenced or indented code, table
+    #     rows, headings, blockquotes, inline code, raw HTML, math
+    #
+    # Short decorative fragments (an ASCII-art header like "*****"),
+    # single-character runs ("-----" dividers), and everything containing
+    # words survive. Lines whose only content is invisible characters
+    # (zero-width spaces, combining marks) are always dropped — they carry
+    # nothing. Blank lines are untouched.
+    class NoiseFilter
+      # Longest line that is never touched, whatever its contents.
+      DEFAULT_MIN_LENGTH = 32
+
+      # Highest distinct-chars/length ratio a line may have and still
+      # count as repetitive. Below this the line reads as a repeated
+      # stream; above it, as prose punctuation (kept).
+      DEFAULT_MAX_ENTROPY = 0.3
+
+      # Characters that never render: zero-width space/joiner and bidi
+      # controls, the BOM, and combining marks.
+      INVISIBLE_RE = /[\u200B-\u200F\uFEFF\u2060\u00AD\p{Mn}]/.freeze
+
+      # A line starting with one of these is structure, not noise:
+      # headings, blockquotes, inline code, raw HTML, math, table rows.
+      STRUCTURE_PREFIX_RE = /\A[#>`<$|]/.freeze
+
+      # GFM table separator rows — "| --- | --- |" or the pipe-only
+      # "--- | ---" variant — are dashes, pipes, colons and spaces only.
+      # Kept as structure; the noise streams this filter targets always
+      # mix in other symbol types (+ = · ~ @ # % …), which this narrow
+      # pattern cannot match, so it is safe to exempt the whole class.
+      TABLE_SEPARATOR_RE = /\A\|?[\s\-:|]+\|?\z/.freeze
+
+      # Fenced code opener/closer: three or more backticks or tildes,
+      # optionally with an info string.
+      FENCE_RE = /\A(?:`{3,}|~{3,})/.freeze
+
+      class << self
+        # Returns +markdown+ with decorative noise lines removed. The
+        # options override the conservative defaults.
+        def filter(markdown, min_length: DEFAULT_MIN_LENGTH, max_entropy: DEFAULT_MAX_ENTROPY)
+          new(min_length: min_length, max_entropy: max_entropy).filter(markdown)
+        end
+      end
+
+      def initialize(min_length: DEFAULT_MIN_LENGTH, max_entropy: DEFAULT_MAX_ENTROPY)
+        @min_length = min_length
+        @max_entropy = max_entropy
+      end
+
+      def filter(markdown)
+        out = +''
+        in_fence = false
+        in_indented_code = false
+        prev_blank = false
+
+        markdown.each_line do |line|
+          stripped = line.strip
+
+          # Fenced code: flip on any fence opener/closer, then pass the
+          # whole block through untouched — code may legitimately be
+          # nothing but symbols.
+          if stripped.match?(FENCE_RE)
+            in_fence = !in_fence
+            out << line
+            next
+          end
+          if in_fence
+            out << line
+            next
+          end
+
+          # Indented code (GFM-ish: 4+ leading spaces, ends at a blank
+          # line). Passed through untouched for the same reason.
+          if in_indented_code && stripped.empty?
+            in_indented_code = false
+            out << line
+            next
+          end
+          indented = line.start_with?('    ', "\t")
+          if indented && (in_indented_code || prev_blank)
+            in_indented_code = true
+            out << line
+            next
+          end
+
+          prev_blank = stripped.empty?
+          out << line unless noise_line?(stripped)
+        end
+        out
+      end
+
+      private
+
+      def noise_line?(stripped)
+        # Blank lines are structure — never touched.
+        return false if stripped.empty?
+
+        # Nothing but invisible characters renders as blank: pure waste.
+        return true if stripped.gsub(INVISIBLE_RE, '').empty?
+
+        # Words and numbers are content, whatever surrounds them.
+        return false if stripped.match?(/[A-Za-z0-9]/)
+
+        # Structural markers and anything too short to matter survive
+        # even when symbol-only.
+        return false if stripped.match?(STRUCTURE_PREFIX_RE)
+        return false if stripped.length < @min_length
+
+        # Table separator rows ("--- | ---") are structure even without a
+        # leading pipe — dash/pipe/colon-only lines are dividers either
+        # way, and real noise streams never match their narrow alphabet.
+        return false if stripped.match?(TABLE_SEPARATOR_RE)
+
+        chars = stripped.gsub(/\s/, '')
+        distinct = chars.chars.uniq.length
+        return false if distinct < 2
+        return false if distinct.fdiv(chars.length) >= @max_entropy
+
+        true
+      end
+    end
+  end
+end

data/lib/ask/web_fetch/version.rb

@@ -2,6 +2,6 @@
 
 module Ask
   module WebFetch
-    VERSION = '0.4.1'
+    VERSION = '0.5.1'
   end
 end

data/lib/ask/web_fetch.rb

@@ -4,6 +4,7 @@ require_relative 'web_fetch/version'
 require_relative 'web_fetch/backend'
 require_relative 'web_fetch/content_filter'
 require_relative 'web_fetch/markdown'
+require_relative 'web_fetch/http'
 require_relative 'web_fetch/backends/local'
 require_relative 'web_fetch/backends/crawl4ai'
 require_relative 'web_fetch/backends/jina'

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ask-web-fetch
 version: !ruby/object:Gem::Version
-  version: 0.4.1
+  version: 0.5.1
 platform: ruby
 authors:
 - Kaka Ruto
@@ -37,6 +37,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0.14'
+- !ruby/object:Gem::Dependency
+  name: httpx
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.0'
 - !ruby/object:Gem::Dependency
   name: nokogiri
   requirement: !ruby/object:Gem::Requirement
@@ -93,20 +107,6 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '13.0'
-- !ruby/object:Gem::Dependency
-  name: vcr
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '6.0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '6.0'
 - !ruby/object:Gem::Dependency
   name: webmock
   requirement: !ruby/object:Gem::Requirement
@@ -123,7 +123,7 @@ dependencies:
         version: '3.26'
 description: Provides Ask::Tools::WebFetch, a tool that fetches a URL and converts
   its content to clean markdown for LLM consumption. Defaults to a pure Ruby backend
-  (Net::HTTP + Nokogiri + reverse_markdown) with a Jina Reader fallback for JS-rendered
+  (httpx + Nokogiri + reverse_markdown) with a Jina Reader fallback for JS-rendered
   or blocked pages, and a real-Chrome fallback (Ferrum) that renders JavaScript and
   lets auto-solving Cloudflare challenges complete. Works with any ask-rb chat or
   agent.
@@ -144,7 +144,9 @@ files:
 - lib/ask/web_fetch/backends/jina.rb
 - lib/ask/web_fetch/backends/local.rb
 - lib/ask/web_fetch/content_filter.rb
+- lib/ask/web_fetch/http.rb
 - lib/ask/web_fetch/markdown.rb
+- lib/ask/web_fetch/noise_filter.rb
 - lib/ask/web_fetch/tool.rb
 - lib/ask/web_fetch/version.rb
 homepage: https://github.com/ask-rb/ask-web-fetch