html2rss 0.32.4 → 0.34.0

data/lib/html2rss/option.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+module Html2rss
+  ##
+  # Edge validation for caller-supplied option values (public API, CLI, MCP).
+  # Everything past these checks may trust its types.
+  module Option
+    class << self
+      ##
+      # @param name [Symbol] option name used in the error message
+      # @param value [Object] candidate value
+      # @param min [Integer] inclusive lower bound
+      # @param max [Integer, nil] inclusive upper bound
+      # @param optional [Boolean] whether +nil+ passes through
+      # @return [Integer, nil] +value+ when valid
+      # @raise [InvalidOption] when +value+ is not an Integer within bounds
+      def integer(name, value, min:, max: nil, optional: false)
+        return value if value.nil? && optional
+        return value if value.is_a?(Integer) && (min..max).cover?(value)
+
+        raise InvalidOption, "#{name} must be an Integer in #{min}..#{max}, got #{value.inspect}"
+      end
+
+      ##
+      # @param value [Symbol, String, nil] strategy plan name (+nil+ means +:auto+)
+      # @return [Symbol] the plan name
+      # @raise [InvalidOption] when +value+ is not a registered plan
+      def strategy(value)
+        name = value.nil? ? FeedPipeline::StrategyPlan::AUTO_NAME : value
+        return name.to_sym if FeedPipeline::StrategyPlan.valid?(name)
+
+        accepted = FeedPipeline::StrategyPlan.accepted_names.join(', ')
+        raise InvalidOption, "strategy must be one of #{accepted}, got #{value.inspect}"
+      end
+
+      ##
+      # @param name [Symbol] option name used in the error message
+      # @param value [Object] candidate list
+      # @return [Array] +value+ when it is a non-empty Array
+      # @raise [InvalidOption] otherwise
+      def list(name, value)
+        return value if value.is_a?(Array) && value.any?
+
+        raise InvalidOption, "#{name} must be a non-empty Array, got #{value.inspect}"
+      end
+    end
+  end
+end

data/lib/html2rss/page_recon/README.md

@@ -10,7 +10,7 @@
 | `final_url`        | URL after redirects (may differ from requested)                             |
 | `status`           | HTTP status of the **final** response                                       |
 | `scheme_downgrade` | `true` when HTTPS entry landed on HTTP                                      |
-| `alternate_feeds`  | `rel=alternate` RSS/Atom links found in HTML                                |
+| `alternate_feeds`  | `rel=alternate` RSS/Atom/RDF links found in HTML                                |
 | `surface_category` | AutoSource surface class (`high_entropy_surface`, `unsupported_surface`, …) |
 | `articles_count`   | Cheap AutoSource extract (limit 10) — diagnostic only, not ship quality     |
 | `strategy`         | Concrete transport used (`inspect` maps `auto` → default (HTTPX))           |

data/lib/html2rss/page_recon/diagnostics.rb

@@ -34,14 +34,12 @@ module Html2rss
       JS_SHELL_MIN_BODY_BYTES = 8_192
       private_constant :JS_SHELL_MIN_BODY_BYTES
 
-      module_function
-
       ##
       # @param url [String]
       # @param strategy [String, Symbol]
       # @param deep [Boolean] when true and strategy is auto, one Botasaurus hop if configured
       # @return [Report]
-      def call(url:, strategy: :auto, deep: false)
+      def self.call(url:, strategy: :auto, deep: false)
         probe = PageRecon.probe(url, strategy: resolve_inspect_strategy(strategy, deep:))
         recon = probe.result
         response = probe.response
@@ -56,10 +54,12 @@ module Html2rss
       # @param strategy [Symbol, String]
       # @param concurrency [Integer]
       # @return [Array<Report>]
-      def batch(urls:, strategy: :auto, concurrency: Batch::DEFAULT_CONCURRENCY)
+      def self.batch(urls:, strategy: :auto, concurrency: Batch::DEFAULT_CONCURRENCY)
         Batch.map(Array(urls), concurrency:) do |url|
           call(url:, strategy:)
-        rescue StandardError => error
+        rescue Html2rss::ContractError
+          raise
+        rescue Html2rss::Error => error
           error_report(url, error)
         end
       end
@@ -67,7 +67,7 @@ module Html2rss
       ##
       # @param response [Html2rss::RequestService::Response]
       # @return [Hash] redacted XHR capture diagnostics (no query strings)
-      def xhr_capture_info(response)
+      def self.xhr_capture_info(response)
         captured = response.captured_responses
         {
           count: captured.size,
@@ -79,7 +79,7 @@ module Html2rss
       ##
       # @param entry [Hash] captured response hash
       # @return [String, nil] scheme+host+path only
-      def redacted_endpoint(entry)
+      def self.redacted_endpoint(entry)
         raw = entry['url'] || entry[:url]
         return unless raw
 
@@ -94,7 +94,7 @@ module Html2rss
       ##
       # @param entry [Hash] captured response hash
       # @return [Boolean]
-      def xhr_candidate_articles?(entry)
+      def self.xhr_candidate_articles?(entry)
         body = entry['body'] || entry[:body]
         return false unless body.is_a?(String)
 
@@ -107,7 +107,7 @@ module Html2rss
       ##
       # @param parsed [Object] parsed response body
       # @return [Array<String>, Hash]
-      def scraper_info(parsed)
+      def self.scraper_info(parsed)
         return { error: 'Response is not HTML' } unless parsed.is_a?(Nokogiri::HTML::Document)
 
         begin

data/lib/html2rss/page_recon.rb

@@ -158,8 +158,6 @@ module Html2rss
     def self.discover_segments(sst, url)
       link_resolver = Scoring::LinkResolver.new(url)
       AutoSource::Segmenter.call(sst, base_url: url, strategy: :list, link_resolver:)
-    rescue StandardError
-      []
     end
 
     ##
@@ -268,8 +266,6 @@ module Html2rss
 
     def sst_document
       Html2rss::SST::Normalizer.call(response.body)
-    rescue ArgumentError
-      nil
     end
 
     def segment_stats(sst, page_url)

data/lib/html2rss/recon.rb

@@ -7,10 +7,8 @@ require 'fileutils'
 module Html2rss
   ##
   # Service that runs reconnaissance on a URL to discover redirect chains,
-  # native RSS/Atom feeds, surface categorization, and emit an actionable verdict.
+  # native RSS/Atom/RDF feeds, surface categorization, and emit an actionable verdict.
   module Recon # rubocop:disable Metrics/ModuleLength
-    module_function
-
     Verdict = Data.define(:name)
 
     ##
@@ -23,7 +21,7 @@ module Html2rss
       ##
       # @param name [Symbol, String]
       # @return [Verdict]
-      def self.[](name) = ALL.fetch(name.to_sym) { raise ArgumentError, "unknown verdict: #{name.inspect}" }
+      def self.[](name) = ALL.fetch(name.to_sym) { raise InvalidOption, "unknown verdict: #{name.inspect}" }
 
       ##
       # @return [Boolean]
@@ -106,19 +104,21 @@ module Html2rss
     # @option options [Integer, nil] :max_redirects optional maximum redirects
     # @option options [Integer, nil] :max_requests optional request budget
     # @return [Html2rss::Recon::Result]
-    def call(url, strategy: :auto, cache_dir: nil, cache_mutex: nil, **) # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
+    def self.call(url, strategy: :auto, cache_dir: nil, cache_mutex: nil, **) # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
       url_obj = Url.from_absolute(url)
 
       begin
         probe = PageRecon.probe(url_obj, strategy:, **)
-      rescue StandardError => error
+      rescue Html2rss::ContractError
+        raise
+      rescue Html2rss::Error => error
         return error_result(url_obj, error)
       end
 
       cache_html_body(probe.response.body, url_obj, cache_dir, cache_mutex) if cache_dir && probe.response.body
 
       recon = probe.result
-      native_feed, discovery_note = find_native_feed(url_obj, probe.session, probe.response)
+      native_feed, discovery_note = find_native_feed(probe.session, probe.response)
       notes = build_notes(recon, native_feed, probe.response)
       notes << discovery_note if discovery_note
       verdict = determine_verdict(recon, native_feed)
@@ -149,7 +149,7 @@ module Html2rss
     # @option options [Integer, nil] :max_redirects optional maximum redirects
     # @option options [Integer, nil] :max_requests optional request budget
     # @return [Array<Html2rss::Recon::Result>]
-    def batch(urls, strategy: :auto, cache_dir: nil, max_threads: 5, **options)
+    def self.batch(urls, strategy: :auto, cache_dir: nil, max_threads: 5, **options)
       url_list = urls.map { |u| u.to_s.strip }.reject(&:empty?)
       return [] if url_list.empty?
 
@@ -162,24 +162,20 @@ module Html2rss
     end
 
     ##
-    # @param url_obj [Html2rss::Url]
     # @param session [Html2rss::RequestSession]
-    # @param response [Html2rss::RequestService::Response]
+    # @param response [Html2rss::RequestService::Response] the fetched page, where any redirect landed
     # @return [Array(Html2rss::Url, nil), String, nil)] feed URL and optional discovery error note
-    def find_native_feed(url_obj, session, response)
-      feed = Syndication::Discovery.best_feed_url(
-        page_url: url_obj,
-        request_session: session,
-        parsed_body: (response.parsed_body if response.html_response?),
-        html: response.body
-      )
+    def self.find_native_feed(session, response)
+      feed = Syndication::Discovery.native_feed_url(response:, request_session: session)
       [feed, nil]
-    rescue StandardError => error
+    rescue Html2rss::ContractError
+      raise
+    rescue Html2rss::Error => error
       [nil, "discovery_error=#{error.class}: #{error.message}"]
     end
     private_class_method :find_native_feed
 
-    def build_notes(recon, native_feed, response)
+    def self.build_notes(recon, native_feed, response)
       notes = []
       notes << "native_rss=#{native_feed}" if native_feed
       if recon.scheme_downgrade
@@ -192,7 +188,7 @@ module Html2rss
     end
     private_class_method :build_notes
 
-    def determine_verdict(recon, native_feed)
+    def self.determine_verdict(recon, native_feed)
       return Verdict[:drop] if recon.status.nil? || recon.status >= 400 || recon.scheme_downgrade
       return Verdict[:defer] if native_feed
 
@@ -203,7 +199,7 @@ module Html2rss
     end
     private_class_method :determine_verdict
 
-    def error_result(url_obj, error) # rubocop:disable Metrics/MethodLength
+    def self.error_result(url_obj, error) # rubocop:disable Metrics/MethodLength
       Result.new(
         requested_url: url_obj,
         final_url: url_obj,
@@ -226,14 +222,14 @@ module Html2rss
     # @param cache_dir [String]
     # @param cache_mutex [Mutex, nil]
     # @return [void]
-    def cache_html_body(body, url_obj, cache_dir, cache_mutex)
+    def self.cache_html_body(body, url_obj, cache_dir, cache_mutex)
       host = url_obj.host.to_s.delete_prefix('www.')
       host = 'snapshot' if host.empty?
       digest = Digest::SHA256.hexdigest(url_obj.to_s)[0, 12]
       file_path = File.join(cache_dir, "#{host}-#{digest}.html")
       writer = -> { File.write(file_path, body) }
       cache_mutex ? cache_mutex.synchronize(&writer) : writer.call
-    rescue StandardError => error
+    rescue SystemCallError, IOError => error
       Log.debug("Recon cache failed: #{error.message}")
     end
     private_class_method :cache_html_body

data/lib/html2rss/request_service/blocked_surface.rb

@@ -34,7 +34,8 @@ module Html2rss
             /interstitial/i
           ],
           message: 'Blocked surface detected: DataDome anti-bot challenge page. ' \
-                   'Target a direct listing URL, or scrape via a session with resolved DataDome cookies.'
+                   'Target a direct listing URL. Clearance cookies from a successful Botasaurus scrape ' \
+                   'are reused automatically; to supply your own, set request.botasaurus.cookies.'
         },
         {
           key: :vercel_security_checkpoint,
@@ -79,10 +80,10 @@ module Html2rss
       end
 
       def self.interstitial_signature_match?(body, signature)
-        min_matches = signature.fetch(:min_matches, 1)
+        min_matches = signature[:min_matches]
         matches = 0
 
-        signature.fetch(:patterns).each do |pattern|
+        signature[:patterns].each do |pattern|
           matches += 1 if pattern.match?(body)
           return true if matches >= min_matches
         end
@@ -91,11 +92,7 @@ module Html2rss
       end
       private_class_method :interstitial_signature_match?
 
-      def self.normalize_body(body)
-        body.to_s.encode(Encoding::UTF_8, invalid: :replace, undef: :replace, replace: '')
-      rescue Encoding::CompatibilityError, Encoding::UndefinedConversionError, Encoding::InvalidByteSequenceError
-        body.to_s.force_encoding(Encoding::UTF_8).scrub
-      end
+      def self.normalize_body(body) = body.to_s.scrub
       private_class_method :normalize_body
     end
   end

data/lib/html2rss/request_service/botasaurus_contract.rb

@@ -11,7 +11,7 @@ module Html2rss
       # Closed set from OpenAPI ExecutionMode.
       EXECUTION_MODES = Set['auto', 'request', 'browser'].freeze
       # Closed set from OpenAPI NavigationMode.
-      NAVIGATION_MODES = Set['auto', 'get', 'google_get', 'google_get_bypass', 'organic_get'].freeze
+      NAVIGATION_MODES = Set['auto', 'get', 'google_get', 'google_get_bypass'].freeze
       # Closed set from OpenAPI ErrorCategory.
       ERROR_CATEGORIES = Set['timeout', 'challenge_block', 'navigation_error', 'metadata_error', 'validation'].freeze
 
@@ -84,6 +84,15 @@ module Html2rss
         # @return [String, nil] final URL reported by upstream
         def final_url = payload['final_url']
 
+        # Session credentials: keep out of logs, transport_meta and feed output.
+        # @return [Hash{String => String}] cookies earned for the target host (absent or malformed → {})
+        def cookies
+          raw = payload['cookies']
+          return {} unless raw.is_a?(Hash)
+
+          raw.select { |name, value| name.is_a?(String) && value.is_a?(String) }
+        end
+
         # @return [Hash{String => Object}] diagnostics plus success-only metadata_error (frozen)
         def transport_meta
           meta = diagnostics.dup
@@ -258,6 +267,7 @@ module Html2rss
       # @param url [Html2rss::Url] canonical URL to scrape
       # @param headers [Hash] request headers from context
       # @param options [Hash] validated request.botasaurus options
+      # @param stored_cookies [Hash{String => String}] cookies replayed from the cookie store
       # @option options [String] :execution_mode
       # @option options [String] :navigation_mode
       # @option options [Integer] :max_retries
@@ -275,18 +285,23 @@ module Html2rss
       # @option options [String] :lang
       # @option options [Hash] :cookies
       # @option options [Hash] :headers
-      def initialize(url:, headers: {}, options: {})
+      def initialize(url:, headers: {}, options: {}, stored_cookies: {})
         @url = url
         @headers = headers
         @options = options
+        @stored_cookies = stored_cookies
       end
 
+      # @return [String] omits options and stored cookies (session credentials)
+      def inspect = "#<#{self.class} url=#{url}>"
+
       # @return [Hash] payload for POST /scrape (explicit options plus wait cap)
       def request_payload
         payload = { url: url.to_s }.merge(filtered_options)
         forwarded_headers = merged_headers
         payload[:headers] = forwarded_headers if forwarded_headers&.any?
         cap_wait_timeout!(payload)
+        merge_cookies!(payload)
         payload
       end
 
@@ -302,7 +317,7 @@ module Html2rss
 
       private
 
-      attr_reader :url, :headers, :options
+      attr_reader :url, :headers, :options, :stored_cookies
 
       def json_object(body)
         payload = JSON.parse(body)
@@ -326,6 +341,13 @@ module Html2rss
         end
       end
 
+      # Stored cookies are replayed in every execution_mode; cookies from the feed config win.
+      def merge_cookies!(payload)
+        configured = (options[:cookies] || {}).transform_keys(&:to_s)
+        merged = stored_cookies.merge(configured)
+        merged.any? ? payload[:cookies] = merged : payload.delete(:cookies)
+      end
+
       def cap_wait_timeout!(payload)
         wait = payload[:wait_timeout_seconds]
         return unless wait

data/lib/html2rss/request_service/botasaurus_strategy.rb

@@ -8,7 +8,7 @@ module Html2rss
   class RequestService
     ##
     # Strategy to delegate fetching to a Botasaurus scrape API.
-    class BotasaurusStrategy < Strategy
+    class BotasaurusStrategy < Strategy # rubocop:disable Metrics/ClassLength -- transport + cookie-jar retry stay co-located
       # Content-Type negotiation for the scrape API transport hop.
       TRANSPORT_ACCEPT = 'application/json'
       # Disable compressed bodies so HTTPX returns raw JSON without implicit decoding surprises.
@@ -18,16 +18,49 @@ module Html2rss
 
       private
 
+      # With a stored jar, tries the cheap HTTP tier first; cookies never reach the browser tier,
+      # so a block falls back to a plain scrape without them.
       def fetch
-        parsed_response = post_scrape_request
+        stored = jar.stored
+        return scrape({}) unless replay?(stored)
+
+        replay(stored)
+      end
+
+      def replay?(stored) = botasaurus_options[:execution_mode] != 'browser' && jar.replayable?(stored)
+
+      def replay(stored)
+        response = scrape(stored, execution_mode: 'request')
+        jar.replayed(:pass, stored)
+        response
+      rescue BlockedSurfaceDetected
+        jar.replayed(:block, stored)
+        scrape({})
+      end
+
+      def scrape(stored, overrides = {})
+        parsed_response = post_scrape_request(stored, overrides)
         raise_from_error!(parsed_response) if parsed_response.is_a?(BotasaurusContract::Error)
 
-        build_response(parsed_response)
+        response = build_response(parsed_response)
+        ResponseGuard.new(policy: ctx.policy).inspect_body!(response.body)
+        jar.refresh(stored, response.cookies)
+        response
       end
 
-      def post_scrape_request
+      def jar
+        @jar ||= CookieJar.new(
+          host: ctx.url.host, scraper_base_url: scraper_base_url.to_s, proxy: botasaurus_options[:proxy],
+          user_agent: botasaurus_options[:user_agent] || ctx.headers['User-Agent']
+        )
+      end
+
+      def botasaurus_options = ctx.request.fetch(:botasaurus, {})
+
+      def post_scrape_request(stored, overrides)
         request_id = SecureRandom.uuid
         Log.debug("#{self.class}: POST /scrape #{REQUEST_ID_HEADER}=#{request_id}")
+        contract = contract(stored, overrides)
         payload = JSON.generate(contract.request_payload)
         response = client.post(scrape_endpoint, body: payload, headers: post_headers(request_id))
         raise response.error if response.is_a?(HTTPX::ErrorResponse)
@@ -44,7 +77,8 @@ module Html2rss
           url: response_url(parsed_response.final_url),
           status: parsed_response.status,
           transport_meta: parsed_response.transport_meta,
-          captured_responses: parsed_response.xhr_responses
+          captured_responses: parsed_response.xhr_responses,
+          cookies: parsed_response.cookies
         )
       end
 
@@ -65,15 +99,13 @@ module Html2rss
         return ctx.url if final_url.nil?
 
         Html2rss::Url.from_absolute(final_url)
-      rescue ArgumentError
+      rescue Html2rss::Url::InvalidUrlError
         ctx.url
       end
 
-      def contract
-        @contract ||= BotasaurusContract.new(
-          url: ctx.url,
-          headers: ctx.headers,
-          options: ctx.request.fetch(:botasaurus, {})
+      def contract(stored, overrides)
+        BotasaurusContract.new(
+          url: ctx.url, headers: ctx.headers, options: botasaurus_options.merge(overrides), stored_cookies: stored
         )
       end
 
@@ -117,7 +149,7 @@ module Html2rss
             raise BotasaurusConfigurationError, 'BOTASAURUS_SCRAPER_URL is required for strategy=botasaurus.'
           end
           Html2rss::Url.for_channel(configured)
-        rescue ArgumentError => error
+        rescue Html2rss::Url::InvalidUrlError => error
           raise BotasaurusConfigurationError, "BOTASAURUS_SCRAPER_URL is invalid: #{error.message}"
         end
       end

data/lib/html2rss/request_service/budget.rb

@@ -11,7 +11,7 @@ module Html2rss
       # @param max_requests [Integer] maximum HTTP request slots
       # @param total_timeout_seconds [Integer, nil] wall-clock timeout for the feed build
       def initialize(max_requests:, total_timeout_seconds: nil)
-        validate_slot_limits!(max_requests:)
+        Option.integer(:max_requests, max_requests, min: 1)
 
         @remaining_requests = max_requests
         @start_time = Process.clock_gettime(Process::CLOCK_MONOTONIC)
@@ -25,33 +25,20 @@ module Html2rss
       # @return [Integer] remaining request slots after consumption
       # @raise [RequestBudgetExceeded] if no request slots remain
       def consume!
-        consume_request!
-      end
-
-      ##
-      # Consumes one HTTP request slot.
-      #
-      # @return [Integer] remaining request slots after consumption
-      # @raise [RequestBudgetExceeded] if no request slots remain
-      def consume_request!
         @mutex.synchronize do
           raise RequestBudgetExceeded, 'Request budget exhausted' if @remaining_requests.zero?
 
           @remaining_requests -= 1
         end
       end
-
-      ##
-      # @return [Integer] HTTP request slots still available
-      def remaining
-        remaining_requests
-      end
+      alias consume_request! consume!
 
       ##
       # @return [Integer] HTTP request slots still available
       def remaining_requests
         @mutex.synchronize { @remaining_requests }
       end
+      alias remaining remaining_requests
 
       ##
       # @return [Float, nil] the remaining timeout in seconds, or nil if not tracked
@@ -91,14 +78,6 @@ module Html2rss
       def effective_timeout_ms(fallback:)
         (effective_timeout_seconds(fallback:) * 1000).to_i
       end
-
-      private
-
-      def validate_slot_limits!(max_requests:)
-        return if max_requests.is_a?(Integer) && max_requests.positive?
-
-        raise ArgumentError, 'max_requests must be positive'
-      end
     end
   end
 end