ask-web-fetch 0.2.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 33eb9a14775223cfaf64bab5b01544c8eb79bb9f4b28c2c8a414bd579116e1e7
-  data.tar.gz: b75238a2a93d2f04c8d7327a993c2a097ab09df3169ce005e67c32b4d25ba00f
+  metadata.gz: efd68d22906c21155fe37405f68ef05185732ff6f1ce39767e50c87e4d535994
+  data.tar.gz: 7d06e49acbc527f06badb034933d792203910637c0e2671f133ec2fc159be41d
 SHA512:
-  metadata.gz: 380b32b742ef62153c8def11dc619e133d1230d7ec39c73b7787448550d5d3f8b37ee9fbe7b5e09a886ba29e7cbfefbdbe0f73d4999255bd4cd19202c03f12a8
-  data.tar.gz: a0f520160d28b978adae2a52414b2625cb88f57bec7a1cab5139277db7f5d639410889915e7a2fbfebf803082cdb5470a48b145ace49776237a91ab8be302e4d
+  metadata.gz: 7736807de8ff9c7b1bad298af12abf67af77f254090532f39f080d0dd98c2bb3a7fed60aac5cf8bbeaeb977b03d3d38900525a4a85b012dc94ae171f18f38b3e
+  data.tar.gz: 2edae586d7baeb9114e29d23540ccbaa331e920a8fe83b7acea12c612b8491116fc635c948d437a157c3193b90d63b97e84857b7714f73ff5f6d19d6c907472b

data/lib/ask/web_fetch/backends/crawl4ai.rb

@@ -0,0 +1,111 @@
+# frozen_string_literal: true
+
+require 'net/http'
+require 'uri'
+require 'json'
+require_relative '../backend'
+
+module Ask
+  module WebFetch
+    module Backends
+      # Self-hosted Crawl4AI (https://docs.crawl4ai.com) — a headless
+      # Chromium crawler that renders JavaScript and returns clean markdown.
+      # Runs as its own Docker service (default http://localhost:11235), the
+      # same self-hosted pattern as ask-web-search's SearXNG. No API key;
+      # configure via CRAWL4AI_URL (and CRAWL4AI_TOKEN for 0.9+ JWT-protected
+      # servers).
+      #
+      # Kept FIRST in the default chain: when the service is present it
+      # handles the JS-rendered pages the Local backend can't. When it isn't
+      # configured — or is unreachable — it fails fast and the chain falls
+      # through to Local, with Jina as the last resort.
+      class Crawl4Ai < Backend
+        DEFAULT_URL = 'http://localhost:11235'
+        OPEN_TIMEOUT = 5
+        READ_TIMEOUT = 30
+        CRAWL_TIMEOUT = 60
+
+        class << self
+          attr_writer :url, :token
+
+          def url
+            @url || ENV['CRAWL4AI_URL']
+          end
+
+          def token
+            @token || ENV['CRAWL4AI_TOKEN']
+          end
+
+          # Presence = configuration. The tool's default chain only includes
+          # this backend when CRAWL4AI_URL is set, so consumers without a
+          # Crawl4AI service see zero behavior change (Local -> Jina).
+          def configured?
+            !url.to_s.empty?
+          end
+        end
+
+        def fetch(url)
+          raise FetchError, 'Crawl4AI not configured (set CRAWL4AI_URL)' if self.class.url.to_s.empty?
+
+          body = crawl(url)
+          raise FetchError, "challenge page at #{url}" if challenge_page?(body)
+
+          page = to_page(body, url)
+          raise EmptyContentError, "no readable content at #{url}" unless usable_content?(page[:content])
+
+          page
+        rescue Net::OpenTimeout, Net::ReadTimeout, Errno::ECONNREFUSED,
+               Errno::ECONNRESET, SocketError, URI::InvalidURIError => e
+          raise FetchError, "Crawl4AI #{e.class}: #{e.message}"
+        end
+
+        private
+
+        def crawl(url)
+          uri = URI("#{self.class.url.chomp('/')}/crawl")
+          http = Net::HTTP.new(uri.host, uri.port)
+          http.use_ssl = uri.scheme == 'https'
+          http.open_timeout = OPEN_TIMEOUT
+          http.read_timeout = READ_TIMEOUT
+
+          req = Net::HTTP::Post.new(uri)
+          req['Content-Type'] = 'application/json'
+          req['Accept'] = 'application/json'
+          req['User-Agent'] = USER_AGENT
+          req['Authorization'] = "Bearer #{self.class.token}" if self.class.token
+          req.body = JSON.generate(
+            urls: [url],
+            crawler_config: { cache_mode: 'bypass', timeout: CRAWL_TIMEOUT }
+          )
+
+          res = http.request(req)
+          case res.code
+          when '200'
+            res.body.to_s
+          when '401', '403'
+            raise FetchError, "Crawl4AI auth error (#{res.code})"
+          else
+            raise FetchError, "Crawl4AI returned #{res.code}"
+          end
+        end
+
+        # The /crawl response is {success:, results: [CrawlResult...]} where
+        # each result carries markdown (fit_markdown preferred, raw_markdown
+        # fallback) and metadata (title, description, ...).
+        def to_page(body, url)
+          data = JSON.parse(body)
+          result = Array(data['results']).first || {}
+          if result['success'] == false
+            raise FetchError, "Crawl4AI crawl failed: #{result['error_message'] || 'unknown error'}"
+          end
+
+          markdown = result.dig('markdown', 'fit_markdown').to_s
+          markdown = result.dig('markdown', 'raw_markdown').to_s if markdown.strip.empty?
+          { title: result.dig('metadata', 'title'), content: markdown }
+        rescue JSON::ParserError => e
+          raise FetchError, "Crawl4AI bad JSON response: #{e.message}"
+        end
+      end
+    end
+  end
+end

data/lib/ask/web_fetch/tool.rb

@@ -3,22 +3,34 @@
 require 'ask-tools'
 require_relative '../web_fetch/backend'
 require_relative '../web_fetch/backends/local'
+require_relative '../web_fetch/backends/crawl4ai'
 require_relative '../web_fetch/backends/jina'
 
 module Ask
   module Tools
     # Fetches a URL and returns its content as clean markdown for LLM
     # consumption. Tries each configured backend in order and returns the
-    # first success, so a blocked or JS-rendered page falls through from the
-    # local fetcher to Jina Reader.
+    # first success.
+    #
+    # Chain: Crawl4AI first (self-hosted headless-Chromium renderer; used
+    # when the CRAWL4AI_URL service is present, fails fast when it isn't),
+    # then the local fetcher, with Jina Reader as the last resort.
     class WebFetch < Ask::Tool
       DEFAULT_MAX_CHARS = 20_000
 
-      # Backend chain, tried in order. Swap or extend for future backends
-      # (e.g. a self-hosted crawler); each must subclass
-      # Ask::WebFetch::Backend and implement #fetch(url).
+      # Backend chain, tried in order. Crawl4AI leads when configured
+      # (CRAWL4AI_URL), so a present self-hosted renderer is preferred;
+      # otherwise Local, with Jina as the last resort. Swap or extend for
+      # future backends; each must subclass Ask::WebFetch::Backend and
+      # implement #fetch(url).
       def self.backends
-        @backends ||= [Ask::WebFetch::Backends::Local, Ask::WebFetch::Backends::Jina]
+        @backends ||= begin
+          chain = [Ask::WebFetch::Backends::Local, Ask::WebFetch::Backends::Jina]
+          if Ask::WebFetch::Backends::Crawl4Ai.configured?
+            chain.unshift(Ask::WebFetch::Backends::Crawl4Ai)
+          end
+          chain
+        end
       end
 
       class << self

data/lib/ask/web_fetch/version.rb

@@ -2,6 +2,6 @@
 
 module Ask
   module WebFetch
-    VERSION = '0.2.0'
+    VERSION = '0.3.0'
   end
 end

data/lib/ask/web_fetch.rb

@@ -3,5 +3,6 @@
 require_relative 'web_fetch/version'
 require_relative 'web_fetch/backend'
 require_relative 'web_fetch/backends/local'
+require_relative 'web_fetch/backends/crawl4ai'
 require_relative 'web_fetch/backends/jina'
 require_relative 'web_fetch/tool'

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ask-web-fetch
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.0
 platform: ruby
 authors:
 - Kaka Ruto
@@ -122,6 +122,7 @@ files:
 - lib/ask-web-fetch.rb
 - lib/ask/web_fetch.rb
 - lib/ask/web_fetch/backend.rb
+- lib/ask/web_fetch/backends/crawl4ai.rb
 - lib/ask/web_fetch/backends/jina.rb
 - lib/ask/web_fetch/backends/local.rb
 - lib/ask/web_fetch/tool.rb