ask-web-fetch 0.5.0 → 0.5.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 706e625574f2001a28ca673c198c2f6b58db55e5558a1afdb94dc1ae36090db0
-  data.tar.gz: 559ea380079f5ab52022da161159b69b3396bc962cd9b3a96e7c3505d8307bf3
+  metadata.gz: 0f49a52ecb57c57ee29ce475ad791f70d30fe52946c94c6bd6842dcccc477392
+  data.tar.gz: b67e97adb54db5b6a775d1ece43df51c0e5443a6ef9e2cd3a17d547af5f7d1dc
 SHA512:
-  metadata.gz: 7babb287ffae5ee1f93a8ecda397b15edd161c1919baa7509904baf67016210eb8456fe23af6e98667f9da6111a03e12a2faf5f835ab2751921f31ab366da8af
-  data.tar.gz: 25e75756da81358ab07e505ce6ec611d85ae7d14d9a73ea16deee421cd3e7f102d19bcfe7a02bd633299fc6f341f4e8badd1413fbd388d323d92d5ff7539a6d4
+  metadata.gz: 28dd03ea3e7aae930925ea2d582ff074beba7d3030d762830024e41506a23fd671bb166fa5809417a824e8d2c1729679d00f3d710fa82de05b2ca41038dd39de
+  data.tar.gz: 84815a2d6504c81df9d81aab2fd679d5bb334f64893b05264cc4185be0e7d7439f357461494c65811cbdb16e0e4548b0b2db290f52b747412678eb953c5f40b3

data/README.md

@@ -26,6 +26,14 @@ first success:
    headless Chromium, so it renders JS pages the Local backend can't. Free
    without a key (~20 req/min per IP); set `JINA_API_KEY` for higher limits.
 
+Every backend's markdown runs through a shared cleanup (`Markdown.clean`):
+decorative symbol noise — the long, letter-free, repetitive character
+streams some pages render as animated backgrounds or section dividers — is
+stripped, and whitespace is normalized. The filter is conservative: code
+blocks, tables, headings, blockquotes, inline code, and short ASCII-art
+fragments always survive. Tunable via `Ask::WebFetch::NoiseFilter.filter(
+markdown, min_length:, max_entropy:)`.
+
 The tool falls back automatically: if Crawl4AI is absent or fails, Local is
 tried (blocked, timeout, non-HTML, anti-bot challenge, or a JS page with no
 server-side content), then Jina. If every backend fails (rate limit, access
@@ -68,9 +76,13 @@ Ask::Tools::WebFetch.backends = [MyBackend, Ask::WebFetch::Backends::Local]
 
 `#fetch` must return `{ title: String|nil, content: String }` and raise
 `Ask::WebFetch::FetchError` (hard failure) or `EmptyContentError` (page
-fetched but nothing usable). The chain then handles ordering and fallback
-for you. For tests, `Ask::Tools::WebFetch.backends = [...]` can be swapped
-and restored.
+fetched but nothing usable). Run the returned markdown through
+`Ask::WebFetch::Markdown.clean` (backends that convert HTML get this from
+`Markdown.generate`; backends fed pre-converted markdown must call it
+explicitly) so the shared noise removal and whitespace normalization apply
+everywhere. The chain then handles ordering and fallback for you. For
+tests, `Ask::Tools::WebFetch.backends = [...]` can be swapped and
+restored.
 
 ## Installation
 
@@ -125,6 +137,9 @@ No configuration required for the default chain. Optional knobs:
   no content unless Crawl4AI is configured — set `CRAWL4AI_URL` to handle
   them with a self-hosted renderer.
 - Some sites block non-browser requests regardless of User-Agent.
+- Symbol streams that a converter merges *into* a content line (rather
+  than leaving them as their own lines) are out of scope for the
+  markdown-level NoiseFilter — that would need a DOM-level pass.
 
 ## Full documentation
 

data/lib/ask/web_fetch/backend.rb

@@ -32,7 +32,12 @@ module Ask
     #   2. #fetch must return { title: String|nil, content: String }
     #   3. #fetch must raise FetchError (hard failure) or
     #      EmptyContentError (page fetched but nothing usable) on failure
-    #   4. register the class in Ask::Tools::WebFetch.backends
+    #   4. run the page's markdown through Ask::WebFetch::Markdown.clean
+    #      before returning it — backends that hold HTML get this from
+    #      Markdown.generate, backends fed pre-converted markdown (Jina,
+    #      Crawl4AI) must call it explicitly so the shared noise removal
+    #      and whitespace normalization apply everywhere
+    #   5. register the class in Ask::Tools::WebFetch.backends
     #
     # The tool tries each backend in order and returns the first success.
     class Backend

data/lib/ask/web_fetch/backends/crawl4ai.rb

@@ -4,6 +4,7 @@ require 'net/http'
 require 'uri'
 require 'json'
 require_relative '../backend'
+require_relative '../markdown'
 
 module Ask
   module WebFetch
@@ -115,6 +116,10 @@ module Ask
 
           markdown = result.dig('markdown', 'fit_markdown').to_s
           markdown = result.dig('markdown', 'raw_markdown').to_s if markdown.strip.empty?
+          # The service's markdown bypasses Markdown.generate, so it runs
+          # through the same shared clean (noise removal + whitespace) as
+          # the converting backends.
+          markdown = Markdown.clean(markdown)
           {
             title: result.dig('metadata', 'title'),
             description: result.dig('metadata', 'description') ||

data/lib/ask/web_fetch/backends/jina.rb

@@ -3,6 +3,7 @@
 require 'net/http'
 require 'uri'
 require_relative '../backend'
+require_relative '../markdown'
 
 module Ask
   module WebFetch
@@ -33,11 +34,16 @@ module Ask
           when '200'
             body = res.body.to_s
             raise FetchError, 'challenge page from Jina' if challenge_page?(body)
-            raise EmptyContentError, 'empty response from Jina' unless usable_content?(body)
 
             # Jina only sees rendered markdown — outlinks come from its
-            # links, resolved against the requested URL.
-            { title: nil, description: nil, content: body.strip, outlinks: markdown_outlinks(body, url) }
+            # links, resolved against the requested URL. Content runs
+            # through the same Markdown.clean as the converting backends,
+            # so decorative symbol noise is stripped here too; a page
+            # whose only "content" was noise falls through as empty.
+            content = Markdown.clean(body)
+            raise EmptyContentError, 'empty response from Jina' unless usable_content?(content)
+
+            { title: nil, description: nil, content: content, outlinks: markdown_outlinks(body, url) }
           when '429'
             raise ServerError, 'rate limited by Jina (429)'
           when '401', '403'

data/lib/ask/web_fetch/markdown.rb

@@ -3,6 +3,7 @@
 require 'nokogiri'
 require 'reverse_markdown'
 require 'uri'
+require_relative 'noise_filter'
 
 module Ask
   module WebFetch
@@ -126,7 +127,12 @@ module Ask
         [parts.join, references.join]
       end
 
+      # Shared post-conversion cleanup: drop decorative symbol noise
+      # (NoiseFilter), then normalize whitespace. Every backend's content
+      # runs through this — Local and Browser via generate, Jina and
+      # Crawl4AI explicitly — so the same noise rules apply to all.
       def clean(markdown)
+        markdown = NoiseFilter.filter(markdown)
         markdown.gsub(/[ \t]+\n/, "\n")
                 .gsub(/\n{3,}/, "\n\n")
                 .strip

data/lib/ask/web_fetch/noise_filter.rb

@@ -0,0 +1,143 @@
+# frozen_string_literal: true
+
+module Ask
+  module WebFetch
+    # Strips decorative symbol noise from markdown: the long, letter-free,
+    # repetitive character streams pages render as animated backgrounds,
+    # marquees and section dividers — e.g. Hugging Face's storage page
+    # ships a "+ = · ( ~ @ # % & * ? / : ; < > [ ] { } | ^ $ !" stream as
+    # its page background. Runs on CONVERTED markdown, so every backend
+    # benefits: Local and Browser already convert through Markdown, and
+    # Jina and Crawl4AI hand the gem pre-converted markdown — the
+    # DOM-level ContentFilter never sees either case.
+    #
+    # Conservative by design. A line is dropped only when ALL hold:
+    #
+    #   * it is long enough to matter (>= min_length characters)
+    #   * it contains no letters or digits at all
+    #   * it is repetitive — at least two distinct characters, with a
+    #     distinct/length ratio below max_entropy (a repeated stream,
+    #     not prose punctuation)
+    #   * it is not markdown structure: fenced or indented code, table
+    #     rows, headings, blockquotes, inline code, raw HTML, math
+    #
+    # Short decorative fragments (an ASCII-art header like "*****"),
+    # single-character runs ("-----" dividers), and everything containing
+    # words survive. Lines whose only content is invisible characters
+    # (zero-width spaces, combining marks) are always dropped — they carry
+    # nothing. Blank lines are untouched.
+    class NoiseFilter
+      # Longest line that is never touched, whatever its contents.
+      DEFAULT_MIN_LENGTH = 32
+
+      # Highest distinct-chars/length ratio a line may have and still
+      # count as repetitive. Below this the line reads as a repeated
+      # stream; above it, as prose punctuation (kept).
+      DEFAULT_MAX_ENTROPY = 0.3
+
+      # Characters that never render: zero-width space/joiner and bidi
+      # controls, the BOM, and combining marks.
+      INVISIBLE_RE = /[\u200B-\u200F\uFEFF\u2060\u00AD\p{Mn}]/.freeze
+
+      # A line starting with one of these is structure, not noise:
+      # headings, blockquotes, inline code, raw HTML, math, table rows.
+      STRUCTURE_PREFIX_RE = /\A[#>`<$|]/.freeze
+
+      # GFM table separator rows — "| --- | --- |" or the pipe-only
+      # "--- | ---" variant — are dashes, pipes, colons and spaces only.
+      # Kept as structure; the noise streams this filter targets always
+      # mix in other symbol types (+ = · ~ @ # % …), which this narrow
+      # pattern cannot match, so it is safe to exempt the whole class.
+      TABLE_SEPARATOR_RE = /\A\|?[\s\-:|]+\|?\z/.freeze
+
+      # Fenced code opener/closer: three or more backticks or tildes,
+      # optionally with an info string.
+      FENCE_RE = /\A(?:`{3,}|~{3,})/.freeze
+
+      class << self
+        # Returns +markdown+ with decorative noise lines removed. The
+        # options override the conservative defaults.
+        def filter(markdown, min_length: DEFAULT_MIN_LENGTH, max_entropy: DEFAULT_MAX_ENTROPY)
+          new(min_length: min_length, max_entropy: max_entropy).filter(markdown)
+        end
+      end
+
+      def initialize(min_length: DEFAULT_MIN_LENGTH, max_entropy: DEFAULT_MAX_ENTROPY)
+        @min_length = min_length
+        @max_entropy = max_entropy
+      end
+
+      def filter(markdown)
+        out = +''
+        in_fence = false
+        in_indented_code = false
+        prev_blank = false
+
+        markdown.each_line do |line|
+          stripped = line.strip
+
+          # Fenced code: flip on any fence opener/closer, then pass the
+          # whole block through untouched — code may legitimately be
+          # nothing but symbols.
+          if stripped.match?(FENCE_RE)
+            in_fence = !in_fence
+            out << line
+            next
+          end
+          if in_fence
+            out << line
+            next
+          end
+
+          # Indented code (GFM-ish: 4+ leading spaces, ends at a blank
+          # line). Passed through untouched for the same reason.
+          if in_indented_code && stripped.empty?
+            in_indented_code = false
+            out << line
+            next
+          end
+          indented = line.start_with?('    ', "\t")
+          if indented && (in_indented_code || prev_blank)
+            in_indented_code = true
+            out << line
+            next
+          end
+
+          prev_blank = stripped.empty?
+          out << line unless noise_line?(stripped)
+        end
+        out
+      end
+
+      private
+
+      def noise_line?(stripped)
+        # Blank lines are structure — never touched.
+        return false if stripped.empty?
+
+        # Nothing but invisible characters renders as blank: pure waste.
+        return true if stripped.gsub(INVISIBLE_RE, '').empty?
+
+        # Words and numbers are content, whatever surrounds them.
+        return false if stripped.match?(/[A-Za-z0-9]/)
+
+        # Structural markers and anything too short to matter survive
+        # even when symbol-only.
+        return false if stripped.match?(STRUCTURE_PREFIX_RE)
+        return false if stripped.length < @min_length
+
+        # Table separator rows ("--- | ---") are structure even without a
+        # leading pipe — dash/pipe/colon-only lines are dividers either
+        # way, and real noise streams never match their narrow alphabet.
+        return false if stripped.match?(TABLE_SEPARATOR_RE)
+
+        chars = stripped.gsub(/\s/, '')
+        distinct = chars.chars.uniq.length
+        return false if distinct < 2
+        return false if distinct.fdiv(chars.length) >= @max_entropy
+
+        true
+      end
+    end
+  end
+end

data/lib/ask/web_fetch/version.rb

@@ -2,6 +2,6 @@
 
 module Ask
   module WebFetch
-    VERSION = '0.5.0'
+    VERSION = '0.5.1'
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ask-web-fetch
 version: !ruby/object:Gem::Version
-  version: 0.5.0
+  version: 0.5.1
 platform: ruby
 authors:
 - Kaka Ruto
@@ -146,6 +146,7 @@ files:
 - lib/ask/web_fetch/content_filter.rb
 - lib/ask/web_fetch/http.rb
 - lib/ask/web_fetch/markdown.rb
+- lib/ask/web_fetch/noise_filter.rb
 - lib/ask/web_fetch/tool.rb
 - lib/ask/web_fetch/version.rb
 homepage: https://github.com/ask-rb/ask-web-fetch