html2rss 0.29.1 → 0.30.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 32c090eccd93210c9631778ff50ec7d32d36c7eccaa711cdd1a0dd5c3ea53616
-  data.tar.gz: 93e4d2f253a019d7de585c8acaaa8d06143accb4f3a43a4f8d13f9c96d838390
+  metadata.gz: 2d839914f2d4d26e86998293a431a76168033523857ec8bc65f3f447441cb07e
+  data.tar.gz: fd0008a8d7a527b0a2128df3a5603a06c154d61b522f6a14decded9889615b78
 SHA512:
-  metadata.gz: 93956c4dc3d35e06c81dff939d4ea5184da93834a0fb48f1b3dbd4dea21d615177ad45fa34e4f16abf396a184cc18acbe31c354df6764ba858ec340c78a3b08c
-  data.tar.gz: b906784ed2ae53b80aa2fc00fe813550befcfac723d97346ffb87cff7f01ca6b0f5db0a6e4a2430dd10485c20f6d4b251a54a1b9e0462998f2ab628c0123a809
+  metadata.gz: 418b9fd5973f19985d7e80abb7734530689bff0753ec948813907f35fd1d2a139ec9fd26353c8c4c87ffc456f446be4a9cdfee9c406022fdfb6c6f0077f886fa
+  data.tar.gz: 1a470470f9bdaea585fdccd56e231c3308624e745a98b830bfd1f0861dad4ce22a98c4a2216306cfd893d6d271efc1d68c6efe15ea6e015d43776086db3094ad

data/README.md

@@ -13,7 +13,7 @@ Most people looking for a first working feed should start with `html2rss-web`, r
 Detailed usage guides, reference docs, and the feed directory live on the project website:
 
 - [Ruby gem documentation](https://html2rss.github.io/ruby-gem)
-- [Request strategies](https://html2rss.github.io/ruby-gem/reference/strategy) (`auto` = `faraday` → `botasaurus`; or pin concrete strategies)
+- [Request strategies](https://html2rss.github.io/ruby-gem/reference/strategy) (`auto` = `default` → `botasaurus`; or pin concrete strategies)
 - [Selectors & pagination](https://html2rss.github.io/ruby-gem/reference/selectors#paginated-feeds)
 - [Web application](https://html2rss.github.io/web-application)
 - [Feed directory](https://html2rss.github.io/feed-directory)
@@ -26,7 +26,7 @@ Cloud development: [Open in GitHub Codespaces](https://github.com/codespaces/new
 ## Architecture
 
 1. **Config** — loads and validates configuration (YAML/hash); schema via `html2rss schema` / `schema/html2rss-config.schema.json`
-2. **RequestService** — fetches pages (`faraday`, `botasaurus`, or `local_file`)
+2. **RequestService** — fetches pages (`default` (HTTPX), `botasaurus`, or `local_file`)
 3. **Selectors** — extracts content via CSS selectors with extractors/post-processors
 4. **AutoSource** — auto-detects content (Schema.org, JSON state, semantic HTML, structural patterns)
 5. **FeedBuilder** — assembles Article objects and renders feeds (RSS 2.0 / JSON Feed 1.1)
@@ -157,7 +157,7 @@ Module guide: [`lib/html2rss/mcp/README.md`](lib/html2rss/mcp/README.md).
 | ----------------------- | --------------------------------------------------------------- |
 | `html2rss://schema`     | Full JSON Schema for feed configurations                        |
 | `html2rss://extractors` | Registered extractor **names** (options live in schema `$defs`) |
-| `html2rss://strategies` | Published MCP strategies (`auto`, `faraday`, `botasaurus`)      |
+| `html2rss://strategies` | Published MCP strategies (`auto`, `default`, `httpx`, `botasaurus`)      |
 | `html2rss://runtime`    | `version`, `mcp_contract_version`, `catalog_fingerprint`, `tools`, `botasaurus_configured` (never the scraper URL) |
 
 ### Prompts
@@ -183,11 +183,11 @@ Set `BOTASAURUS_SCRAPER_URL` to `http://127.0.0.1:4010` and use strategy `botasa
 
 | Strategy     | Description                                                                 |
 | ------------ | --------------------------------------------------------------------------- |
-| `auto`       | Tries `faraday`, falls back to `botasaurus` (default in gem/CLI/MCP scrape) |
-| `faraday`    | Plain HTTP requests via Faraday                                             |
+| `auto`       | Tries `default`, falls back to `botasaurus` (default in gem/CLI/MCP scrape) |
+| `default`    | Plain HTTP requests via HTTPX (alias: `httpx`; legacy: `faraday`)           |
 | `botasaurus` | Puppeteer-backed scraping for JavaScript pages                              |
 
-`inspect` keeps Faraday when `auto` for cheap diagnostics. Elsewhere, strategy can be set via CLI (`--strategy`), gem API keyword argument, or feed config `request.strategy`. See the [request strategies docs](https://html2rss.github.io/ruby-gem/reference/strategy) for more details.
+`inspect` keeps `default` when `auto` for cheap diagnostics. Elsewhere, strategy can be set via CLI (`--strategy`), gem API keyword argument, or feed config `strategy`. See the [request strategies docs](https://html2rss.github.io/ruby-gem/reference/strategy) for more details.
 
 ## License
 

data/html2rss.gemspec

@@ -35,9 +35,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency 'addressable', '~> 2.7'
   spec.add_dependency 'brotli'
   spec.add_dependency 'dry-validation'
-  spec.add_dependency 'faraday', '> 2.0.1', '< 3.0'
-  spec.add_dependency 'faraday-follow_redirects'
-  spec.add_dependency 'faraday-gzip', '~> 3'
+  spec.add_dependency 'httpx', '~> 1.8'
   spec.add_dependency 'kramdown'
   spec.add_dependency 'mcp', '~> 1.2'
   spec.add_dependency 'mime-types', '> 3.0'

data/lib/html2rss/batch.rb

@@ -77,7 +77,7 @@ module Html2rss
       # Scrapes multiple URLs in parallel with per-URL error isolation.
       #
       # @param urls [Enumerable<String>] list of URLs to scrape
-      # @param strategy [Symbol, String] request strategy (+:auto+, +:faraday+, +:botasaurus+)
+      # @param strategy [Symbol, String] request strategy (+:auto+, +:default+, +:botasaurus+)
       # @param limit [Integer] max articles to extract per URL
       # @param concurrency [Integer] number of worker threads (default 5, max 10)
       # @return [BatchResult]
@@ -89,7 +89,7 @@ module Html2rss
       # Inspects multiple URLs in parallel with per-URL error isolation.
       #
       # @param urls [Enumerable<String>] list of URLs to inspect
-      # @param strategy [Symbol, String] request strategy (+:auto+, +:faraday+, +:botasaurus+)
+      # @param strategy [Symbol, String] request strategy (+:auto+, +:default+, +:botasaurus+)
       # @param concurrency [Integer] number of worker threads (default 5, max 10)
       # @return [BatchResult]
       def batch_inspect(urls:, strategy: :auto, concurrency: DEFAULT_CONCURRENCY)
@@ -104,7 +104,7 @@ module Html2rss
       # Runs recon across multiple URLs in parallel with per-URL error isolation.
       #
       # @param urls [Enumerable<String>] list of URLs to recon
-      # @param strategy [Symbol, String] request strategy (+:auto+, +:faraday+, +:botasaurus+)
+      # @param strategy [Symbol, String] request strategy (+:auto+, +:default+, +:botasaurus+)
       # @param concurrency [Integer] number of worker threads (default 5, max 10)
       # @option options [String, nil] :cache_dir optional HTML cache directory
       # @return [BatchResult]

data/lib/html2rss/capture/README.md

@@ -40,7 +40,7 @@ Html2rss.capture('https://example.com', strategy: :local_file, local_file_path:
 Html2rss.capture('https://example.com', max_redirects: 8, max_requests: 4)
 ```
 
-`strategy: :auto` uses the same AutoFallback chain as scrape (`faraday` → `botasaurus`). When AutoFallback selects a concrete strategy (or you pin one), Capture **stamps** `strategy:` into the emitted config so later `Html2rss.apply(config)` replays the same transport.
+`strategy: :auto` uses the same AutoFallback chain as scrape (`default` → `botasaurus`). When AutoFallback selects a concrete strategy (or you pin one), Capture **stamps** `strategy:` into the emitted config so later `Html2rss.apply(config)` replays the same transport.
 
 ## CLI
 

data/lib/html2rss/capture.rb

@@ -75,7 +75,7 @@ module Html2rss
       # Analyzes a URL and builds a reusable feed config.
       #
       # @param url [String] source page URL
-      # @param strategy [Symbol] request strategy (+:auto+, +:faraday+, +:botasaurus+)
+      # @param strategy [Symbol] request strategy (+:auto+, +:default+, +:botasaurus+)
       # @option options [String, nil] :items_selector optional selector hint
       # @option options [Array<String>, nil] :topics optional directory topics override
       # @option options [String, nil] :title optional title override

data/lib/html2rss/config/request_headers.rb

@@ -24,7 +24,6 @@ module Html2rss
       DEFAULT_HEADERS = {
         'Accept' => DEFAULT_ACCEPT,
         'Cache-Control' => 'max-age=0',
-        'Connection' => 'keep-alive',
         'Sec-Fetch-Dest' => 'document',
         'Sec-Fetch-Mode' => 'navigate',
         'Sec-Fetch-Site' => 'none',
@@ -33,6 +32,9 @@ module Html2rss
         'User-Agent' => DEFAULT_USER_AGENT
       }.freeze
 
+      # Hop-by-hop headers forbidden in HTTP/2 requests (RFC 7540 §8.1.2.2 / RFC 9113 §8.2.1).
+      FORBIDDEN_H2_HEADERS = %w[connection keep-alive proxy-connection transfer-encoding upgrade].to_set.freeze
+
       class << self
         ##
         # :reek:ManualDispatch
@@ -75,7 +77,7 @@ module Html2rss
       ##
       # @return [Hash{String => String}] normalized HTTP headers
       def to_h
-        defaults = DEFAULT_HEADERS.dup
+        defaults = self.class.browser_defaults
         normalized = normalize_custom_headers(headers)
 
         accept_override = normalized.delete('Accept')
@@ -84,6 +86,7 @@ module Html2rss
         defaults['Accept'] = normalize_accept(accept_override)
         defaults['Accept-Language'] = build_accept_language
 
+        defaults.reject! { |key, _| FORBIDDEN_H2_HEADERS.include?(key.downcase) }
         defaults.compact
       end
 

data/lib/html2rss/config.rb

@@ -93,6 +93,18 @@ module Html2rss
         Validator.new.call(prepared_config)
       end
 
+      ##
+      # Loads and validates a YAML configuration file.
+      #
+      # @param file [String] the YAML file to load
+      # @param feed_name [String, nil] optional feed name for multi-feed files
+      # @param multiple_feeds_key [Symbol] key under which multiple feeds are defined
+      # @param params [Hash{Symbol => Object, Hash{String => Object, nil}}] dynamic parameters for string formatting
+      # @return [Dry::Validation::Result, Html2rss::Config::ValidationResult]
+      def validate_yaml(file, feed_name = nil, multiple_feeds_key: MultipleFeedsConfig::CONFIG_KEY_FEEDS, params: UNSET)
+        validate(load_yaml(file, feed_name, multiple_feeds_key:), params:)
+      end
+
       ##
       # Returns the packaged JSON Schema file path.
       #

data/lib/html2rss/doctor/botasaurus.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require 'faraday'
+require 'httpx'
 require 'json'
 
 module Html2rss
@@ -68,15 +68,23 @@ module Html2rss
       # @param base_url [String]
       # @return [Hash{Symbol => Object}]
       def health_check(base_url)
-        uri = Url.for_channel(base_url)
-        client = Faraday.new(url: uri.to_s.chomp('/'), request: { timeout: 5 })
-        response = client.get('/health')
-        body = JSON.parse(response.body)
+        endpoint = "#{Url.for_channel(base_url).to_s.chomp('/')}/health"
+        response = health_session.get(endpoint)
+        raise response.error if response.is_a?(HTTPX::ErrorResponse)
+
+        body = JSON.parse(response.body.to_s)
         { ok: response.status == 200, status: response.status, version: body['version'] }
       rescue StandardError => error
         { ok: false, error: "#{error.class}: #{error.message}" }
       end
 
+      def health_session
+        HTTPX.with(timeout: { operation_timeout: 5, request_timeout: 5, total_request_timeout: 5 },
+                   resolver_class: :system)
+      end
+      module_function :health_session
+      private_class_method :health_session
+
       ##
       # @param url [String]
       # @return [Hash{Symbol => Object}]

data/lib/html2rss/feed_pipeline/README.md

@@ -2,25 +2,25 @@
 
 `:auto` is the default request plan for feed builds (`auto_source`, `auto_json_feed`, Capture, and MCP `scrape` / `capture`). `FeedPipeline::StrategyPlan` resolves it; `FeedPipeline::AutoFallback` executes `AutoFallback::CHAIN`.
 
-Use `:auto` when you want Faraday first and a browser-backed hop only if that fetch fails or yields zero items. Pin a concrete strategy (`faraday`, `botasaurus`, `local_file`) when you need a single transport.
+Use `:auto` when you want `default` (HTTPX) first and a browser-backed hop only if that fetch fails or yields zero items. Pin a concrete strategy (`default`, `botasaurus`, `local_file`) when you need a single transport.
 
 ## Chain
 
 `AutoFallback::CHAIN` is:
 
-1. **Faraday** — plain HTTP (faster, cheaper).
-2. **Botasaurus** — attempted when Faraday raises a fallback-eligible error (for example `BlockedSurfaceDetected` or timeout) or extracts zero feed items.
+1. **Default (HTTPX)** — plain HTTP (faster, cheaper).
+2. **Botasaurus** — attempted when default raises a fallback-eligible error (for example `BlockedSurfaceDetected` or timeout) or extracts zero feed items.
 
-Before escalating Faraday → Botasaurus on a **weak** auto-source extract (empty / below floor / high-entropy / app-shell / unsupported — not blocked), `FeedResolution` may probe up to five same-origin listing or native-feed candidates and rewrite the effective scrape URL via {ScrapeTarget}. Botasaurus then uses the resolved URL. Direct RSS/Atom entry URLs are parsed via `Syndication::Parser` without HTML AutoSource.
+Before escalating default → Botasaurus on a **weak** auto-source extract (empty / below floor / high-entropy / app-shell / unsupported — not blocked), `FeedResolution` may probe up to five same-origin listing or native-feed candidates and rewrite the effective scrape URL via {ScrapeTarget}. Botasaurus then uses the resolved URL. Direct RSS/Atom entry URLs are parsed via `Syndication::Parser` without HTML AutoSource.
 
-There is no Browserless / Puppeteer-in-gem tier. Pin `botasaurus` when you want browser rendering without Faraday first. Botasaurus needs `BOTASAURUS_SCRAPER_URL`.
+There is no Browserless / Puppeteer-in-gem tier. Pin `botasaurus` when you want browser rendering without default first. Botasaurus needs `BOTASAURUS_SCRAPER_URL`.
 
 ## Surfaces
 
 | Surface                                  | `:auto` behavior                                                                                                                   |
 | ---------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------- |
-| Gem / CLI `apply`, MCP `scrape`, Capture | Full AutoFallback chain (`faraday` → `botasaurus`)                                                                                 |
-| MCP `inspect`                            | Cheap diagnostic: `StrategyPlan.concrete_for_diagnostic` maps `auto` to Faraday (pin `botasaurus` when you need browser rendering) |
+| Gem / CLI `apply`, MCP `scrape`, Capture | Full AutoFallback chain (`default` → `botasaurus`)                                                                                 |
+| MCP `inspect`                            | Cheap diagnostic: `StrategyPlan.concrete_for_diagnostic` maps `auto` to default (pin `botasaurus` when you need browser rendering) |
 
 ## Fallback vs abort
 
@@ -28,7 +28,7 @@ These typically hop to the next chain member (among other `StandardError`s `Auto
 
 - `Html2rss::RequestService::BlockedSurfaceDetected`
 - `Html2rss::RequestService::RequestTimedOut`
-- Faraday connection / timeout errors
+- HTTPX connection / timeout errors
 - Empty extraction results when a later chain member may succeed
 
 These abort immediately (`AutoFallback::NON_FALLBACK_ERRORS`): unknown strategy, invalid URL, unsupported scheme, budget exceeded, private network denied, cross-origin follow-up denied, response too large.

data/lib/html2rss/feed_pipeline/auto_fallback.rb

@@ -8,7 +8,7 @@ module Html2rss
     # Hosted by the pipeline instance: session + extract call back into FeedPipeline.
     class AutoFallback # rubocop:disable Metrics/ClassLength -- attempt state + logging stay co-located
       # Ordered list of concrete request strategies attempted by the :auto plan.
-      CHAIN = %i[faraday botasaurus].freeze
+      CHAIN = %i[default botasaurus].freeze
 
       # Deterministic HTTP statuses that indicate permanent failure rather than anti-bot challenges.
       DETERMINISTIC_HTTP_STATUSES = Set[400, 404, 410, 422, 451].freeze

data/lib/html2rss/feed_pipeline/strategy_plan.rb

@@ -42,7 +42,7 @@ module Html2rss
         end
 
         ##
-        # Cheap single-request diagnostic default: +:auto+ → +:faraday+.
+        # Cheap single-request diagnostic default: +:auto+ → +:default+.
         # Does not run {AutoFallback}. Prefer {resolve} for scrape/capture feeds.
         #
         # @param name [Symbol, String, nil] plan name (+nil+ → +:auto+)

data/lib/html2rss/mcp/README.md

@@ -20,7 +20,7 @@ Batch: `batch_inspect`, `batch_recon`, `batch_scrape`.
 
 ## Decision tree
 
-1. **Articles now (no saved config)?** → `scrape` (or `batch_scrape` for multiple URLs). `strategy: "auto"` runs Faraday → Botasaurus; do not retry with explicit `faraday` after `auto`.
+1. **Articles now (no saved config)?** → `scrape` (or `batch_scrape` for multiple URLs). `strategy: "auto"` runs default (HTTPX) → Botasaurus; do not retry with explicit `default` after `auto`.
 2. **Reusable feed YAML?** → `capture` → `test` → `apply`. `capture` returns YAML in `payload.yaml`. Strive `enhance: true` (false only when chrome leaks). `test` runs schema + live extraction; optional `compare_enhance` compares enhance off vs on. `apply` is the ship gate (`isError` on zero items). Both `test` and `apply` may include `quality_report.enhance_gains` when `selectors.items.enhance` is true.
 3. **Weak scrape/capture or recon?** → `inspect` (or `batch_inspect`). When alternates warrant it, follow `next_step` to `recon`.
 4. **Config already in hand?** → `validate` (schema only) → `test` → `apply`.
@@ -61,7 +61,7 @@ Do not duplicate playbook prose in `server.rb`.
 | ----------------------- | ---------------------------------------------------------- |
 | `html2rss://schema`     | Full JSON Schema for feed configurations                   |
 | `html2rss://extractors` | Registered extractor names                                 |
-| `html2rss://strategies` | Published MCP strategies (`auto`, `faraday`, `botasaurus`) |
+| `html2rss://strategies` | Published MCP strategies (`auto`, `default`, `httpx`, `botasaurus`) |
 | `html2rss://runtime`    | `version`, `mcp_contract_version`, `catalog_fingerprint`, `tools`, `botasaurus_configured` (never the scraper URL). Fingerprint covers tool names, required keys, and `oneOf` branches; bump `mcp_contract_version` for envelope semantics. |
 
 ## Prompts
@@ -73,7 +73,7 @@ Do not duplicate playbook prose in `server.rb`.
 
 ## Strategy note
 
-`scrape` / `capture` with `strategy: "auto"` run the full AutoFallback chain. `inspect` maps `auto` to Faraday for cheap diagnostics; pin `botasaurus` when you need browser rendering for inspect.
+`scrape` / `capture` with `strategy: "auto"` run the full AutoFallback chain. `inspect` maps `auto` to default (HTTPX) for cheap diagnostics; pin `botasaurus` when you need browser rendering for inspect.
 
 ## Inspect redirects
 

data/lib/html2rss/mcp/contract.rb

@@ -9,7 +9,11 @@ module Html2rss
     # annotations, and the single compact JSON envelope response.
     module Contract # rubocop:disable Metrics/ModuleLength -- published listing constants stay co-located
       # Published MCP request strategies (excludes +local_file+).
-      STRATEGIES = %w[auto faraday botasaurus].freeze
+      STRATEGIES = %w[auto default httpx botasaurus].freeze
+      # Accepted migration strategies retained for backwards compatibility.
+      MIGRATION_STRATEGIES = %w[faraday].freeze
+      # Complete set of strategies accepted by {.assert_published_request!}.
+      ALL_ACCEPTED_STRATEGIES = (STRATEGIES + MIGRATION_STRATEGIES).freeze
 
       # Bump when tool names, required inputs, or envelope semantics change (independent of gem +VERSION+).
       MCP_CONTRACT_VERSION = 2
@@ -30,12 +34,12 @@ module Html2rss
         type: 'string',
         enum: STRATEGIES,
         default: 'auto',
-        description: 'Request strategy (auto runs faraday → botasaurus fallback chain)'
+        description: 'Request strategy (auto runs default → botasaurus fallback chain)'
       }.freeze
 
-      # JSON Schema property for inspect +strategy+ (auto stays on Faraday).
+      # JSON Schema property for inspect +strategy+ (auto stays on default).
       INSPECT_STRATEGY_PROPERTY = STRATEGY_PROPERTY.merge(
-        description: 'Request strategy (auto uses Faraday for cheap diagnostics; pin botasaurus when needed)'
+        description: 'Request strategy (auto uses default for cheap diagnostics; pin botasaurus when needed)'
       ).freeze
 
       # JSON Schema +oneOf+ requiring exactly one of +config+ or +yaml+.
@@ -281,7 +285,7 @@ module Html2rss
         #   or +request.local_file_path+ is present
         def assert_published_request!(config)
           strategy = config[:strategy]
-          unless strategy.nil? || STRATEGIES.include?(strategy.to_s)
+          unless strategy.nil? || ALL_ACCEPTED_STRATEGIES.include?(strategy.to_s)
             raise UnpublishedRequestError,
                   "MCP does not accept strategy #{strategy} (published: #{STRATEGIES.join(', ')})"
           end

data/lib/html2rss/mcp/outcome/playbook.rb

@@ -36,7 +36,7 @@ module Html2rss
               html2rss MCP — decide which tool to call:
 
               1. Need articles now (no saved config)? → scrape (or batch_scrape for multiple)
-                 - strategy "auto" runs Faraday → Botasaurus AutoFallback. Do not retry with explicit faraday after auto.
+                 - strategy "auto" runs default (HTTPX) → Botasaurus AutoFallback. Do not retry with explicit default after auto.
                  - Empty scrape is still success (articles-now). Follow next_step / guidance (read_runtime if Botasaurus unset).
               2. Need a reusable feed YAML? → capture → test → apply
                  - capture returns YAML inside payload.yaml. Draft only: if destination is html2rss-configs, rewrite for directory.topics and explicit channel title/url. Default enhance follows capture evidence (false when admission_drops show chrome); override only when needed.
@@ -58,9 +58,9 @@ module Html2rss
           # @return [String]
           def scrape_webpage_prompt(url)
             <<~MSG.strip
-              Scrape #{url} with scrape (strategy auto). One call is enough — auto already runs Faraday then Botasaurus.
+              Scrape #{url} with scrape (strategy auto). One call is enough — auto already runs default then Botasaurus.
               Follow envelope next_step and guidance. Call inspect only if articles are empty/weak or you need diagnostics (final_url, status, scheme_downgrade, alternate_feeds). When inspect finds alternates, follow next_step to recon.
-              Do not retry scrape with explicit faraday after auto. Read html2rss://runtime if next_step is read_runtime.
+              Do not retry scrape with explicit default after auto. Read html2rss://runtime if next_step is read_runtime.
               Return payload.items (not a raw JSON array).
             MSG
           end
@@ -72,7 +72,7 @@ module Html2rss
             <<~MSG.strip
               Build a reusable html2rss feed config for #{url}:
               1) capture — YAML is payload.yaml. Check payload.articles_count, payload.has_selectors, and payload.suggested_channel_url. enhance defaults from admission evidence (false when chrome drops are high). When payload.native_feed is set, follow next_step (done — use the native feed).
-              2) Follow next_step. If weak or you need recon, inspect then recon when alternates warrant it. Auto already hops to Botasaurus; do not retry capture with botasaurus unless Faraday was blocked.
+              2) Follow next_step. If weak or you need recon, inspect then recon when alternates warrant it. Auto already hops to Botasaurus; do not retry capture with botasaurus unless default was blocked.
               3) test with yaml (or config hash) — schema + live extraction. On :schema failure, validate; on :execution/:min_items, recapture. Read payload.quality_report.enhance_gains when enhance is on; optional compare_enhance compares enhance off vs on without changing shipped RSS.
               4) apply — isError if zero items. Confirm payload.item_count and payload.quality_report (including enhance_gains) before shipping.
               If the destination is html2rss-configs, rewrite the draft for directory.topics and explicit channel title/url. Return YAML.
@@ -95,7 +95,7 @@ module Html2rss
             if data[:blocked_surface] || data[:surface_category].to_s == 'blocked_surface'
               return 'Blocked or anti-bot interstitial likely. Retry scrape with strategy botasaurus once ' \
                      '(or CLI inspect --deep when BOTASAURUS_SCRAPER_URL is set). ' \
-                     'Do not retry explicit faraday after auto.'
+                     'Do not retry explicit default after auto.'
             end
             if data[:likely_js_shell]
               return 'JS-rendered shell likely (html_present, zero articles). Use strategy auto or botasaurus; ' \

data/lib/html2rss/mcp/server/tools.rb

@@ -13,7 +13,7 @@ module Html2rss
             kind: :url,
             description: 'One-shot article extraction as JSON Feed items. ' \
                          'Use when you need articles now without a saved config. ' \
-                         'strategy "auto" triggers fallback chain (faraday → botasaurus) for JS-rendered sites.',
+                         'strategy "auto" triggers fallback chain (default → botasaurus) for JS-rendered sites.',
             input_schema: Contract::SCRAPE_INPUT_SCHEMA,
             handler: :scrape_outcome
           },

data/lib/html2rss/mcp/server.rb

@@ -10,7 +10,7 @@ module Html2rss
     # This module maps MCP kwargs to those APIs, then {Outcome} + {Contract} shape the envelope.
     #
     # Strategy note: MCP +auto+ passes through to FeedPipeline AutoFallback
-    # (faraday → botasaurus). Concrete strategies are used as-is.
+    # (default → botasaurus). Concrete strategies are used as-is.
     # Botasaurus requires +BOTASAURUS_SCRAPER_URL+.
     module Server # rubocop:disable Metrics/ModuleLength
       # MCP server display name.

data/lib/html2rss/recon.rb

@@ -111,7 +111,7 @@ module Html2rss
     # Runs reconnaissance on a single URL.
     #
     # @param url [String, Html2rss::Url] source page URL
-    # @param strategy [Symbol] request strategy (:auto, :faraday, :botasaurus)
+    # @param strategy [Symbol] request strategy (:auto, :default, :botasaurus)
     # @param cache_dir [String, nil] optional directory to cache raw HTML bodies
     # @param cache_mutex [Mutex, nil] optional mutex serializing cache writes
     # @option options [Integer, nil] :max_redirects optional maximum redirects

data/lib/html2rss/request_service/botasaurus_strategy.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require 'faraday'
+require 'httpx'
 require 'json'
 require 'securerandom'
 
@@ -11,7 +11,7 @@ module Html2rss
     class BotasaurusStrategy < Strategy
       # Content-Type negotiation for the scrape API transport hop.
       TRANSPORT_ACCEPT = 'application/json'
-      # Disable compressed bodies so Faraday returns raw JSON without implicit decoding surprises.
+      # Disable compressed bodies so HTTPX returns raw JSON without implicit decoding surprises.
       TRANSPORT_ENCODING = 'identity'
       # Correlates each POST /scrape with botasaurus-scrape-api request logs.
       REQUEST_ID_HEADER = 'X-Request-Id'
@@ -28,10 +28,15 @@ module Html2rss
       def post_scrape_request
         request_id = SecureRandom.uuid
         Log.debug("#{self.class}: POST /scrape #{REQUEST_ID_HEADER}=#{request_id}")
-        transport_response = client.post('/scrape', JSON.generate(contract.request_payload), post_headers(request_id))
-        contract.parse_response(transport_response)
+        payload = JSON.generate(contract.request_payload)
+        response = client.post(scrape_endpoint, body: payload, headers: post_headers(request_id))
+        raise response.error if response.is_a?(HTTPX::ErrorResponse)
+
+        contract.parse_response(response)
       end
 
+      def scrape_endpoint = "#{scraper_base_url.to_s.chomp('/')}/scrape"
+
       def build_response(parsed_response)
         Response.new(
           body: parsed_response.html,
@@ -44,22 +49,14 @@ module Html2rss
       end
 
       def raise_from_error!(error)
-        raise_if_challenge_blocked!(error)
-        raise_if_timed_out!(error)
-        raise BotasaurusServiceError, error.failure_message
-      end
-
-      def raise_if_challenge_blocked!(error)
-        return unless error.challenge_block?
+        raise BlockedSurfaceDetected, "Blocked surface detected: #{error.challenge_message}" if error.challenge_block?
 
-        raise BlockedSurfaceDetected, "Blocked surface detected: #{error.challenge_message}"
-      end
-
-      def raise_if_timed_out!(error)
-        return unless error.timeout?
+        if error.timeout?
+          log_timeout!(reason: 'botasaurus_upstream')
+          raise RequestTimedOut.new(error.failure_message, timeout_phase: error.timeout_phase)
+        end
 
-        log_timeout!(reason: 'botasaurus_upstream')
-        raise RequestTimedOut.new(error.failure_message, timeout_phase: error.timeout_phase)
+        raise BotasaurusServiceError, error.failure_message
       end
 
       def response_url(final_url)
@@ -79,8 +76,18 @@ module Html2rss
       end
 
       def client
-        # No :gzip middleware on this client — compression is for remote target fetches only.
-        @client ||= Faraday.new(url: scraper_base_url.to_s, headers: client_headers, request: request_options)
+        timeout = attempt_timeout_seconds.to_f
+        @client ||= HTTPX.with(headers: client_headers, timeout: client_timeouts(timeout), resolver_class: :system)
+      end
+
+      def client_timeouts(timeout)
+        {
+          operation_timeout: timeout,
+          connect_timeout: [ctx.policy.connect_timeout_seconds, timeout].min,
+          read_timeout: [ctx.policy.read_timeout_seconds, timeout].min,
+          request_timeout: timeout,
+          total_request_timeout: timeout
+        }
       end
 
       def client_headers
@@ -92,14 +99,7 @@ module Html2rss
       end
 
       def post_headers(request_id)
-        {
-          'Content-Type' => 'application/json',
-          REQUEST_ID_HEADER => request_id
-        }
-      end
-
-      def request_options
-        { timeout: attempt_timeout_seconds.to_i }
+        { 'Content-Type' => 'application/json', REQUEST_ID_HEADER => request_id }
       end
 
       def attempt_timeout_seconds
@@ -123,6 +123,10 @@ module Html2rss
       def translate_connection_error(error)
         raise BotasaurusConnectionFailed, "Botasaurus connection failed: #{error.message}"
       end
+
+      def timeout_error?(error) = error.is_a?(HTTPX::TimeoutError) || super
+
+      def connection_error?(error) = error.is_a?(HTTPX::ConnectionError) || error.is_a?(HTTPX::TLSError) || super
     end
   end
 end

data/lib/html2rss/request_service/compressed_body.rb

@@ -45,9 +45,12 @@ module Html2rss
 
       def inflate(raw, encoding)
         case encoding
-        when 'gzip' then uncompress_gzip(raw)
-        when 'deflate' then inflate_deflate(raw)
+        when 'gzip'
+          return raw unless raw.start_with?(GZIP_MAGIC)
+
+          uncompress_gzip(raw)
         when 'br' then try_brotli(raw, require_html: false)
+        when 'deflate' then inflate_deflate(raw)
         end
       rescue Zlib::Error, Brotli::Error, ArgumentError
         nil
@@ -55,12 +58,6 @@ module Html2rss
       module_function :inflate
       private_class_method :inflate
 
-      def uncompress_gzip(raw)
-        Zlib::GzipReader.wrap(StringIO.new(raw), encoding: 'ASCII-8BIT', &:read)
-      end
-      module_function :uncompress_gzip
-      private_class_method :uncompress_gzip
-
       def inflate_deflate(raw)
         inflater = nil
         Zlib::Inflate.inflate(raw)
@@ -73,6 +70,14 @@ module Html2rss
       module_function :inflate_deflate
       private_class_method :inflate_deflate
 
+      def uncompress_gzip(raw)
+        Zlib::GzipReader.wrap(StringIO.new(raw), encoding: 'ASCII-8BIT', &:read)
+      rescue Zlib::Error
+        nil
+      end
+      module_function :uncompress_gzip
+      private_class_method :uncompress_gzip
+
       def try_brotli(raw, require_html: true)
         inflated = Brotli.inflate(raw)
         return inflated unless require_html