ask-web-fetch 0.4.1 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 318ff9c594a7d5c52a710507efd3da2917bbe3319471a3e0bac8bce3a01ed30e
-  data.tar.gz: f5ae0d23b351044ecc21f4e2c4d2ec66a13d8258cd9a86d6947f0c7698c6e703
+  metadata.gz: 706e625574f2001a28ca673c198c2f6b58db55e5558a1afdb94dc1ae36090db0
+  data.tar.gz: 559ea380079f5ab52022da161159b69b3396bc962cd9b3a96e7c3505d8307bf3
 SHA512:
-  metadata.gz: 775322ca3144aa69da6d27e20664c5a18aa836baaf10d5431acc5cbc7dd893aff5e09c2c40751d3f19b6ea86410ad92cf992ad35b2283554b31196ba52fdea29
-  data.tar.gz: ae5f935d3b48522330b5a01f9319813bd84d157076f65ee2a17cdbd82fb3fb65916f5312ea03fefdc51e8b7a84792c9558ede959e195e386f2713fb1c31e437a
+  metadata.gz: 7babb287ffae5ee1f93a8ecda397b15edd161c1919baa7509904baf67016210eb8456fe23af6e98667f9da6111a03e12a2faf5f835ab2751921f31ab366da8af
+  data.tar.gz: 25e75756da81358ab07e505ce6ec611d85ae7d14d9a73ea16deee421cd3e7f102d19bcfe7a02bd633299fc6f341f4e8badd1413fbd388d323d92d5ff7539a6d4

data/lib/ask/web_fetch/backend.rb

@@ -42,8 +42,11 @@ module Ask
                    "ask-web-fetch/#{Ask::WebFetch::VERSION}".freeze
 
       # Content shorter than this is treated as a page with no usable
-      # content (e.g. a JS-rendered shell with nothing server-side).
-      MIN_CONTENT_LENGTH = 100
+      # content (e.g. a JS-rendered shell with nothing server-side). Low on
+      # purpose: the content hub keeps everything that is a real page, and
+      # the crawler's soft-404 detection (not length) is what separates
+      # pages from error shells.
+      MIN_CONTENT_LENGTH = 40
 
       # Cloudflare-style anti-bot signatures. Deliberately narrow:
       # challenge/interstitial pages carry these markers, while legitimate
@@ -55,12 +58,55 @@ module Ask
         name.split('::').last
       end
 
-      # Fetches +url+ and returns { title: String|nil, content: String }.
+      # Fetches +url+ and returns { title:, description:, content:,
+      # redirected:, licenses:, outlinks: } — licenses being the page's
+      # declared license signals (hrefs/values) and outlinks the page's
+      # raw link set ([] when the backend can't see any), both consumed by
+      # the crawler's classification/discovery layers.
       # Raises FetchError or EmptyContentError on failure.
       def fetch(url)
         raise NotImplementedError, "#{self.class} must implement #fetch(url)"
       end
 
+      # --- outlinks (crawler discovery) ---
+
+      # The page's raw outlinks from HTML: every <a href> resolved against
+      # the base URL and scheme-filtered to absolute http(s). Nav and
+      # footer are included — a crawler's discovery reads the full link set
+      # even when the stored content is pruned by the ContentFilter. Shared
+      # by every backend that holds the page's HTML.
+      def outlink_urls(html, base_url)
+        Nokogiri::HTML(html).css('a[href]').filter_map do |anchor|
+          href = anchor['href'].to_s.strip
+          next if href.empty? || href.start_with?('javascript:', 'mailto:', 'tel:', '#', 'data:')
+
+          uri = URI.join(base_url, href)
+          next unless %w[http https].include?(uri.scheme)
+
+          uri.to_s
+        rescue URI::InvalidURIError
+          next
+        end.uniq
+      end
+
+      # Fallback for backends that only see rendered markdown (Jina,
+      # Crawl4AI's markdown output): the markdown's [text](url) links,
+      # resolved and scheme-filtered. Same shape as #outlink_urls, one
+      # implementation for every backend that lacks the raw HTML.
+      def markdown_outlinks(content, base_url)
+        content.to_s.scan(/\]\(([^)\s]+)\)/).filter_map do |match|
+          dest = match[0]
+          next if dest.start_with?('javascript:', 'mailto:', 'tel:', '#', 'data:')
+
+          uri = URI.join(base_url, dest)
+          next unless %w[http https].include?(uri.scheme)
+
+          uri.to_s
+        rescue URI::InvalidURIError
+          next
+        end.uniq
+      end
+
       private
 
       def challenge_page?(body)

data/lib/ask/web_fetch/backends/browser.rb

@@ -164,6 +164,7 @@ module Ask
           raise FetchError, "challenge page at #{url}" if challenge_page?(body)
 
           result = Markdown.generate(body, base_url: url, filter: self.class.content_filter)
+          result[:outlinks] = outlink_urls(body, url)
           raise EmptyContentError, "no readable content at #{url}" unless usable_content?(result[:content])
 
           result

data/lib/ask/web_fetch/backends/crawl4ai.rb

@@ -120,6 +120,9 @@ module Ask
             description: result.dig('metadata', 'description') ||
               result.dig('metadata', 'og_description'),
             content: markdown,
+            # Raw markdown keeps the nav links the ContentFilter prunes —
+            # discovery reads these even when the stored content is lean.
+            outlinks: markdown_outlinks(markdown, url),
             # The page's own redirect, if the crawl followed one — lets
             # consumers record permanent redirects on their ledger instead
             # of silently indexing under the original URL.

data/lib/ask/web_fetch/backends/jina.rb

@@ -35,7 +35,9 @@ module Ask
             raise FetchError, 'challenge page from Jina' if challenge_page?(body)
             raise EmptyContentError, 'empty response from Jina' unless usable_content?(body)
 
-            { title: nil, description: nil, content: body.strip }
+            # Jina only sees rendered markdown — outlinks come from its
+            # links, resolved against the requested URL.
+            { title: nil, description: nil, content: body.strip, outlinks: markdown_outlinks(body, url) }
           when '429'
             raise ServerError, 'rate limited by Jina (429)'
           when '401', '403'

data/lib/ask/web_fetch/backends/local.rb

@@ -1,15 +1,15 @@
 # frozen_string_literal: true
 
-require 'net/http'
 require 'uri'
 require_relative '../backend'
 require_relative '../content_filter'
 require_relative '../markdown'
+require_relative '../http'
 
 module Ask
   module WebFetch
     module Backends
-      # Default backend: pure Ruby Net::HTTP + Nokogiri + reverse_markdown.
+      # Default backend: pure Ruby httpx + Nokogiri + reverse_markdown.
       # No external service or API key; mirrors ask-web-search's self-hosted
       # SearXNG approach.
       #
@@ -18,8 +18,6 @@ module Ask
       # than by keyword — is what comes back.
       class Local < Backend
         MAX_REDIRECTS = 5
-        OPEN_TIMEOUT = 5
-        READ_TIMEOUT = 15
 
         class << self
           # The ContentFilter applied to every page by default. Set to nil to
@@ -29,6 +27,15 @@ module Ask
           def content_filter
             @content_filter ||= ContentFilter.default
           end
+
+          # The single-hop HTTP transport, Ask::WebFetch::Http by default:
+          # a pooled keep-alive httpx session. Swappable in tests — and the
+          # seam any future transport swap goes through.
+          attr_writer :http
+
+          def http
+            @http ||= Http
+          end
         end
 
         def fetch(url)
@@ -41,21 +48,50 @@ module Ask
           raise EmptyContentError, "no readable content at #{url}" unless usable_content?(page[:content])
 
           page
-        rescue Net::OpenTimeout, Net::ReadTimeout, Errno::ECONNREFUSED,
-               Errno::ECONNRESET, SocketError, URI::InvalidURIError => e
+        rescue Errno::ECONNREFUSED, Errno::ECONNRESET, SocketError, URI::InvalidURIError => e
+          # The transport maps its own failures to TimeoutError; this guard
+          # keeps the "only Ask::WebFetch errors escape" invariant even if
+          # a transport bug lets a raw socket error through.
           raise TimeoutError, "#{e.class}: #{e.message}"
         end
 
-        # Parses +html+ and returns { title:, description:, content: } where
-        # content is clean markdown and description is the page's own meta
-        # description (meta name=description, then og:description) — what the
-        # site says about itself, free and authoritative.
+        # Parses +html+ and returns { title:, description:, content:,
+        # licenses:, outlinks: } where content is clean markdown (pruned by
+        # the ContentFilter), licenses are the page's declared license
+        # signals ([] when it declares none), and outlinks are the page's
+        # RAW hrefs, resolved and scheme-filtered — nav and footer included,
+        # because a crawler's discovery layer reads these even when the
+        # stored content is pruned.
         def to_markdown(html, url)
           Markdown.generate(html, base_url: url, filter: self.class.content_filter)
+            .merge(licenses: license_signals(html), outlinks: outlink_urls(html, url))
         end
 
         private
 
+        # The page's declared license signals: <link rel="license">, the
+        # license meta tags (meta name/property license, cc:license,
+        # dc.rights), and schema.org [itemprop=license] — hrefs first, then
+        # contents. Best-effort and conservative: nothing here decides what
+        # the page IS licensed under, it only reports what the page says
+        # about itself. The consumer (the crawler's license classifier)
+        # maps known markers; unknown signals are ignored there, never here.
+        def license_signals(html)
+          doc = Nokogiri::HTML(html)
+          selectors = [
+            'link[rel~="license"]',
+            'meta[name="license"], meta[property="license"]',
+            'meta[name="cc:license"], meta[property="cc:license"]',
+            'meta[name="dc.rights"], meta[property="dc.rights"]',
+            '[itemprop="license"]'
+          ]
+          doc.css(selectors.join(',')).filter_map do |node|
+            value = node['href'] || node['content'] || node.text
+            value = value.to_s.strip
+            value unless value.empty?
+          end.uniq
+        end
+
         # GET with redirect following (max MAX_REDIRECTS hops). Returns
         # [body, content_type, redirect] where redirect is nil when the
         # URL answered directly, else {status: first hop's status,
@@ -65,30 +101,23 @@ module Ask
           hops = 0
           first_hop_status = nil
           loop do
-            http = Net::HTTP.new(uri.host, uri.port)
-            http.open_timeout = OPEN_TIMEOUT
-            http.read_timeout = READ_TIMEOUT
-            http.use_ssl = uri.scheme == 'https'
-            req = Net::HTTP::Get.new(uri)
-            req['User-Agent'] = USER_AGENT
-            req['Accept'] = 'text/html,application/xhtml+xml'
-            res = http.request(req)
-            return [res.body, res['content-type'].to_s, redirect_info(first_hop_status, uri)] if res.code.start_with?('2')
-
-            unless res.code.start_with?('3') && res['location']
+            response = self.class.http.get(uri.to_s, headers: { 'accept' => 'text/html,application/xhtml+xml' })
+            return [response.body, response.content_type, redirect_info(first_hop_status, uri)] if (200..299).cover?(response.status)
+
+            unless (300..399).cover?(response.status) && !response.location.empty?
               # 4xx (other than 429) = the URL is dead; 429/5xx = transient.
-              code = res.code.to_i
-              raise(code == 429 || code >= 500 ? ServerError : FetchError, "got #{res.code} from #{url}")
+              code = response.status
+              raise(code == 429 || code >= 500 ? ServerError : FetchError, "got #{code} from #{url}")
             end
             raise FetchError, "hit a redirect loop at #{url}" if (hops += 1) > MAX_REDIRECTS
 
-            first_hop_status ||= res.code.to_i
-            uri = URI.join(uri, res['location'])
+            first_hop_status ||= response.status
+            uri = URI.join(uri, response.location)
           end
         end
 
         def redirect_info(status, uri)
-          status && {status: status, url: uri.to_s}
+          status && { status: status, url: uri.to_s }
         end
       end
     end

data/lib/ask/web_fetch/http.rb

@@ -0,0 +1,85 @@
+# frozen_string_literal: true
+
+require 'httpx'
+
+module Ask
+  module WebFetch
+    # Minimal pooled HTTP client for the Local backend. Wraps a per-thread
+    # httpx session so every page fetch in a thread reuses its keep-alive
+    # connection to the host — no fresh TCP+TLS handshake per page (the
+    # thing that made the Net::HTTP crawler cost ~1.25s per page) — with
+    # HTTP/2 when the server negotiates it, retries with backoff on
+    # transient failures, and automatic gzip/deflate decoding.
+    #
+    # Deliberately single-hop: the backend follows redirects itself, so the
+    # hop-by-hop chain it reports is exactly what happened. Error responses
+    # (4xx/5xx) are not errors to the transport — the backend decides what
+    # they mean. Transport-level failures (timeout, refused, reset, DNS,
+    # TLS) all surface as Ask::WebFetch::TimeoutError, the transient
+    # bucket, whatever their underlying class.
+    class Http
+      CONNECT_TIMEOUT = 5
+      READ_TIMEOUT = 15
+      WRITE_TIMEOUT = 15
+      # Whole-request cap. A page that can't be read in a minute is a
+      # problem page, not a stall worth a crawl worker.
+      OPERATION_TIMEOUT = 60
+      # Retries per request, on top of the crawl ledger's own auto-heal
+      # rounds. GETs are idempotent; a couple of cheap retries beat a full
+      # ledger round-trip for transient flakiness.
+      MAX_RETRIES = 2
+      RETRY_ON_STATUS = [429, 500, 502, 503, 504].freeze
+
+      # One hop's answer. Redirects stay the backend's job, so `location`
+      # rides along for the backend to resolve.
+      Response = Data.define(:status, :body, :content_type, :location)
+
+      def self.get(url, headers: {})
+        response = session.get(url, headers: headers)
+        return raise_timeout(response) if response.is_a?(HTTPX::ErrorResponse)
+
+        Response.new(
+          status: response.status,
+          body: response.body.to_s,
+          content_type: response.headers['content-type'].to_s,
+          location: response.headers['location'].to_s
+        )
+      end
+
+      # One pooled session per thread — httpx sessions are not thread-safe,
+      # and a crawl worker thread reusing its session across every page it
+      # fetches is what makes the pooling pay off. Sessions idle-close
+      # themselves after keep-alive timeout, so nothing to reap.
+      def self.session
+        Thread.current[SESSION_KEY] ||= build_session
+      end
+
+      def self.build_session
+        # :persistent is what makes the pooling real — without it httpx
+        # opens a fresh connection per request. It loads fiber_concurrency
+        # and the retries plugin internally.
+        HTTPX.plugin(:persistent).with(
+          timeout: {
+            connect_timeout: CONNECT_TIMEOUT,
+            read_timeout: READ_TIMEOUT,
+            write_timeout: WRITE_TIMEOUT,
+            operation_timeout: OPERATION_TIMEOUT
+          },
+          headers: { 'user-agent' => Backend::USER_AGENT },
+          max_retries: MAX_RETRIES,
+          retry_on: ->(response) { response.respond_to?(:status) && RETRY_ON_STATUS.include?(response.status) }
+        )
+      end
+      private_class_method :build_session
+
+      def self.raise_timeout(response)
+        error = response.error
+        raise TimeoutError, "#{error.class}: #{error.message}"
+      end
+      private_class_method :raise_timeout
+
+      SESSION_KEY = :ask_web_fetch_http_session
+      private_constant :SESSION_KEY
+    end
+  end
+end

data/lib/ask/web_fetch/version.rb

@@ -2,6 +2,6 @@
 
 module Ask
   module WebFetch
-    VERSION = '0.4.1'
+    VERSION = '0.5.0'
   end
 end

data/lib/ask/web_fetch.rb

@@ -4,6 +4,7 @@ require_relative 'web_fetch/version'
 require_relative 'web_fetch/backend'
 require_relative 'web_fetch/content_filter'
 require_relative 'web_fetch/markdown'
+require_relative 'web_fetch/http'
 require_relative 'web_fetch/backends/local'
 require_relative 'web_fetch/backends/crawl4ai'
 require_relative 'web_fetch/backends/jina'

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ask-web-fetch
 version: !ruby/object:Gem::Version
-  version: 0.4.1
+  version: 0.5.0
 platform: ruby
 authors:
 - Kaka Ruto
@@ -37,6 +37,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0.14'
+- !ruby/object:Gem::Dependency
+  name: httpx
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.0'
 - !ruby/object:Gem::Dependency
   name: nokogiri
   requirement: !ruby/object:Gem::Requirement
@@ -93,20 +107,6 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '13.0'
-- !ruby/object:Gem::Dependency
-  name: vcr
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '6.0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '6.0'
 - !ruby/object:Gem::Dependency
   name: webmock
   requirement: !ruby/object:Gem::Requirement
@@ -123,7 +123,7 @@ dependencies:
         version: '3.26'
 description: Provides Ask::Tools::WebFetch, a tool that fetches a URL and converts
   its content to clean markdown for LLM consumption. Defaults to a pure Ruby backend
-  (Net::HTTP + Nokogiri + reverse_markdown) with a Jina Reader fallback for JS-rendered
+  (httpx + Nokogiri + reverse_markdown) with a Jina Reader fallback for JS-rendered
   or blocked pages, and a real-Chrome fallback (Ferrum) that renders JavaScript and
   lets auto-solving Cloudflare challenges complete. Works with any ask-rb chat or
   agent.
@@ -144,6 +144,7 @@ files:
 - lib/ask/web_fetch/backends/jina.rb
 - lib/ask/web_fetch/backends/local.rb
 - lib/ask/web_fetch/content_filter.rb
+- lib/ask/web_fetch/http.rb
 - lib/ask/web_fetch/markdown.rb
 - lib/ask/web_fetch/tool.rb
 - lib/ask/web_fetch/version.rb