html2rss 0.30.0 → 0.31.0

data/lib/html2rss/page_recon.rb

@@ -8,12 +8,14 @@ module Html2rss
   # article count. Diagnostic fetch for Inspect/Recon lives on {.probe} — not twin
   # fetch helpers in those callers.
   class PageRecon # rubocop:disable Metrics/ClassLength -- recon bag stays co-located
+    Assessment = Data.define(:surface_category, :articles_count, :admission_drops, :html_response)
+
     ##
     # Cheap surface + admission facts shared by AutoFallback gates and FeedResolution probes.
-    Assessment = Data.define(:surface_category, :articles_count, :admission_drops, :html_response) do
+    class Assessment
       ##
       # @return [Html2rss::SurfaceCategory]
-      def category = SurfaceCategory.coerce(surface_category)
+      def category = SurfaceCategory[surface_category]
 
       ##
       # @return [Boolean]
@@ -27,8 +29,6 @@ module Html2rss
       # @return [Boolean]
       def listing_bonus? = category.listing_bonus?
     end
-    ##
-    # Recon facts used by Inspect and FeedResolution.
     Result = Data.define(
       :requested_url,
       :final_url,
@@ -43,7 +43,11 @@ module Html2rss
       :content_type,
       :blocked_surface,
       :sst
-    ) do
+    )
+
+    ##
+    # Recon facts used by Inspect and FeedResolution.
+    class Result
       ##
       # @return [Hash{Symbol => Object}]
       def to_h # rubocop:disable Metrics/AbcSize, Metrics/MethodLength -- omit-empty optional keys

data/lib/html2rss/recon.rb

@@ -16,27 +16,14 @@ module Html2rss
     ##
     # Closed curation verdict (:build / :defer / :drop).
     class Verdict
-      # Closed set of curation verdict wire names.
-      NAMES = Set[:build, :defer, :drop].freeze
-
-      class << self
-        ##
-        # @param value [Verdict, Symbol, String]
-        # @return [Verdict]
-        def coerce(value)
-          return value if value.is_a?(self)
-
-          new(name: value.to_sym)
-        end
-      end
+      # Frozen name → instance table.
+      ALL = %i[build defer drop].to_h { |name| [name, new(name:)] }.freeze
+      private_class_method :new
 
       ##
-      # @param name [Symbol]
-      def initialize(name:)
-        raise ArgumentError, "unknown verdict: #{name.inspect}" unless NAMES.include?(name)
-
-        super
-      end
+      # @param name [Symbol, String]
+      # @return [Verdict]
+      def self.[](name) = ALL.fetch(name.to_sym) { raise ArgumentError, "unknown verdict: #{name.inspect}" }
 
       ##
       # @return [Boolean]
@@ -59,8 +46,6 @@ module Html2rss
       def to_s = name.to_s
     end
 
-    ##
-    # Immutable outcome of a reconnaissance operation.
     Result = Data.define(
       :requested_url,
       :final_url,
@@ -72,7 +57,11 @@ module Html2rss
       :scheme_downgrade,
       :notes,
       :html_bytesize
-    ) do
+    )
+
+    ##
+    # Immutable outcome of a reconnaissance operation.
+    class Result
       ##
       # @return [Boolean]
       def build? = verdict.build?
@@ -140,7 +129,7 @@ module Html2rss
         status: probe.response.status,
         verdict:,
         native_feed:,
-        surface_category: SurfaceCategory.coerce(recon.surface_category),
+        surface_category: SurfaceCategory[recon.surface_category],
         articles_count: recon.articles_count,
         scheme_downgrade: recon.scheme_downgrade,
         notes:,
@@ -204,13 +193,13 @@ module Html2rss
     private_class_method :build_notes
 
     def determine_verdict(recon, native_feed)
-      return Verdict.coerce(:drop) if recon.status.nil? || recon.status >= 400 || recon.scheme_downgrade
-      return Verdict.coerce(:defer) if native_feed
+      return Verdict[:drop] if recon.status.nil? || recon.status >= 400 || recon.scheme_downgrade
+      return Verdict[:defer] if native_feed
 
-      category = SurfaceCategory.coerce(recon.surface_category)
-      return Verdict.coerce(:drop) if category.blocked?
+      category = SurfaceCategory[recon.surface_category]
+      return Verdict[:drop] if category.blocked?
 
-      Verdict.coerce(:build)
+      Verdict[:build]
     end
     private_class_method :determine_verdict
 
@@ -219,9 +208,9 @@ module Html2rss
         requested_url: url_obj,
         final_url: url_obj,
         status: nil,
-        verdict: Verdict.coerce(:drop),
+        verdict: Verdict[:drop],
         native_feed: nil,
-        surface_category: SurfaceCategory.coerce(:unsupported_surface),
+        surface_category: SurfaceCategory[:unsupported_surface],
         articles_count: 0,
         scheme_downgrade: false,
         notes: ["error: #{error.class} - #{error.message}"],

data/lib/html2rss/request_service/botasaurus_contract.rb

@@ -9,11 +9,11 @@ module Html2rss
     # +ScrapeRequest+ / +ScrapeSuccess+ / +ScrapeError+ (sibling +openapi.yaml+).
     class BotasaurusContract
       # Closed set from OpenAPI ExecutionMode.
-      EXECUTION_MODES = %w[auto request browser].freeze
+      EXECUTION_MODES = Set['auto', 'request', 'browser'].freeze
       # Closed set from OpenAPI NavigationMode.
-      NAVIGATION_MODES = %w[auto get google_get google_get_bypass organic_get].freeze
+      NAVIGATION_MODES = Set['auto', 'get', 'google_get', 'google_get_bypass', 'organic_get'].freeze
       # Closed set from OpenAPI ErrorCategory.
-      ERROR_CATEGORIES = %w[timeout challenge_block navigation_error metadata_error validation].freeze
+      ERROR_CATEGORIES = Set['timeout', 'challenge_block', 'navigation_error', 'metadata_error', 'validation'].freeze
 
       # ScrapeRequest properties except +url+ (html2rss supplies the target URL).
       REQUEST_OPTION_KEYS = %i[
@@ -25,7 +25,7 @@ module Html2rss
       # OpenAPI WindowSize required keys (positive integers).
       WINDOW_SIZE_PROPERTIES = %i[width height].freeze
 
-      # Faraday POST /scrape cap mirroring botasaurus-scrape-api total scrape wall (`SCRAPE_TIMEOUT_SECONDS`).
+      # HTTPX POST /scrape cap mirroring botasaurus-scrape-api total scrape wall (`SCRAPE_TIMEOUT_SECONDS`).
       SCRAPE_TIMEOUT_SECONDS = Integer(ENV.fetch('BOTASAURUS_SCRAPE_TIMEOUT_SECONDS', 45))
       # Post-boot navigate/wait budget mirrored from scrape-api (`SCRAPE_WORK_TIMEOUT_SECONDS`).
       SCRAPE_WORK_TIMEOUT_SECONDS = Integer(ENV.fetch('BOTASAURUS_SCRAPE_WORK_TIMEOUT_SECONDS', 30))
@@ -72,10 +72,7 @@ module Html2rss
 
         # @return [Hash{String => String}] normalized response headers (null → {})
         def headers
-          raw_headers = payload['headers']
-          return {} unless raw_headers.is_a?(Hash) && raw_headers.any?
-
-          raw_headers.to_h { |key, value| [key.to_s, value.to_s] }
+          stringify_headers(payload['headers'])
         end
 
         # @return [Integer, nil] document status_code when present
@@ -147,10 +144,12 @@ module Html2rss
           entry[key.to_s] || entry[key]
         end
 
-        def xhr_headers(raw)
-          return {} unless raw.is_a?(Hash)
+        def xhr_headers(raw) = stringify_headers(raw)
+
+        def stringify_headers(raw)
+          return {} unless raw.is_a?(Hash) && raw.any?
 
-          raw.to_h { |key, value| [key.to_s, value.to_s] }
+          raw.transform_keys(&:to_s).transform_values(&:to_s)
         end
 
         def xhr_status_code(value)
@@ -286,7 +285,7 @@ module Html2rss
         payload
       end
 
-      # @param transport_response [Faraday::Response] upstream HTTP response
+      # @param transport_response [HTTPX::Response] upstream HTTP response
       # @return [Success, Error]
       # @raise [BotasaurusServiceError] when payload is not a scrape envelope
       def parse_response(transport_response)

data/lib/html2rss/request_service/botasaurus_strategy.rb

@@ -84,7 +84,7 @@ module Html2rss
         {
           operation_timeout: timeout,
           connect_timeout: [ctx.policy.connect_timeout_seconds, timeout].min,
-          read_timeout: [ctx.policy.read_timeout_seconds, timeout].min,
+          read_timeout: timeout,
           request_timeout: timeout,
           total_request_timeout: timeout
         }

data/lib/html2rss/request_service/compressed_body.rb

@@ -91,7 +91,10 @@ module Html2rss
       private_class_method :try_brotli
 
       def content_encodings(headers)
-        header(headers, 'content-encoding').to_s.split(',').map { |value| value.strip.downcase }.reject(&:empty?)
+        header(headers, 'content-encoding')
+          .to_s
+          .split(',')
+          .filter_map { |val| (s = val.strip.downcase).empty? ? nil : s }
       end
       module_function :content_encodings
       private_class_method :content_encodings

data/lib/html2rss/request_service/httpx_strategy.rb

@@ -5,7 +5,6 @@ require 'httpx/plugins/follow_redirects'
 require 'httpx/plugins/callbacks'
 require 'httpx/plugins/ssrf_filter'
 require 'httpx/plugins/brotli'
-require 'httpx/plugins/fiber_concurrency'
 require 'httpx/plugins/retries'
 
 module Html2rss
@@ -16,39 +15,46 @@ module Html2rss
     # and redirect handling without monkey-patching.
     # rubocop:disable-next Metrics/ClassLength -- terminal redirect retry colocated with HTTPX transport
     class HttpxStrategy < Strategy
+      # HTTPX default; kept explicit so ping reconnects cannot omit a bound.
+      MAX_RETRIES = 3
+      # Forces the selector timer retry path. Unset retry_after recurses in fetch_response;
+      # combined with ping! on a reused closed selector that never decrements retries,
+      # that overflows the stack and kills Falcon workers.
+      RETRY_AFTER_SECONDS = 0.25
+      # Bounds passed to the retries plugin and every +session.with+.
+      # HTTPX retries stay inside one {RequestService::Budget} slot and the shared
+      # wall-clock deadline; do not consume a slot per attempt.
+      # Do not reload +:fiber_concurrency+: a reused closed selector made +ping!+
+      # skip decrementing retries and recurse until +SystemStackError+.
+      #
+      # @return [Hash{Symbol => Integer, Float}]
+      RETRY_OPTIONS = { max_retries: MAX_RETRIES, retry_after: RETRY_AFTER_SECONDS }.freeze
+
       class << self
         # rubocop:disable ThreadSafety/ClassInstanceVariable
         # @return [HTTPX::Session]
         def base_session
-          @base_sessions ||= {}
-          @base_sessions[HTTPX::Session] ||= HTTPX
-                                             .plugin(:follow_redirects)
-                                             .plugin(:callbacks)
-                                             .plugin(:brotli)
-                                             .plugin(:fiber_concurrency)
-                                             .plugin(:retries)
+          @base_session ||= HTTPX
+                            .plugin(:follow_redirects)
+                            .plugin(:callbacks)
+                            .plugin(:brotli)
+                            .plugin(:retries, **RETRY_OPTIONS)
         end
 
         # @return [HTTPX::Session]
         def base_ssrf_session
-          @base_ssrf_sessions ||= {}
-          @base_ssrf_sessions[HTTPX::Session] ||= base_session.plugin(:ssrf_filter)
+          @base_ssrf_session ||= base_session.plugin(:ssrf_filter)
         end
         # rubocop:enable ThreadSafety/ClassInstanceVariable
       end
 
-      ##
-      # @return [ResponseGuard]
-      attr_reader :response_guard
-
       # Executes the request with runtime policy enforcement, returning the normalized response.
       #
       # @return [Response] normalized response
       def perform_execute
         deadline = request_deadline
-        @response_guard = ResponseGuard.new(policy: ctx.policy)
         reset_redirect_tracking!
-        raw_response = request_with_terminal_redirect_retry(response_guard, deadline:)
+        raw_response = request_with_terminal_redirect_retry(deadline:)
         build_response(raw_response)
       end
 
@@ -65,15 +71,15 @@ module Html2rss
         @current_url = request_url
       end
 
-      def request_with_terminal_redirect_retry(response_guard, deadline:)
-        raw_response = execute_http_request(response_guard, deadline:)
+      def request_with_terminal_redirect_retry(deadline:)
+        raw_response = execute_http_request(deadline:)
         return raw_response unless redirect_limit_reached?(raw_response)
 
         unless terminal_redirect_retryable?(raw_response)
           raise RedirectLimitReached, "Too many redirects (status #{raw_response.status})"
         end
 
-        retry_from_terminal_redirect!(raw_response, response_guard, deadline:)
+        retry_from_terminal_redirect!(raw_response, deadline:)
       end
 
       def redirect_limit_reached?(response)
@@ -82,7 +88,7 @@ module Html2rss
         return false unless (300..399).cover?(response.status)
 
         location = response.headers['location']
-        !location.nil? && !location.strip.empty?
+        !location.to_s.strip.empty?
       end
 
       def terminal_redirect_url(raw_response)
@@ -102,12 +108,12 @@ module Html2rss
         target && target.to_s != request_url.to_s
       end
 
-      def retry_from_terminal_redirect!(raw_response, response_guard, deadline:)
+      def retry_from_terminal_redirect!(raw_response, deadline:)
         terminal_url = terminal_redirect_url(raw_response)
         @terminal_redirect_retried = true
         Log.debug("#{self.class}: redirect limit reached; retrying once from #{terminal_url}")
         begin_terminal_url_request!(terminal_url)
-        new_response = execute_http_request(response_guard, deadline:, consume_budget: false)
+        new_response = execute_http_request(deadline:, consume_budget: false)
         if redirect_limit_reached?(new_response)
           raise RedirectLimitReached, "Too many redirects (status #{new_response.status})"
         end
@@ -126,17 +132,21 @@ module Html2rss
         @request_url_override || ctx.url
       end
 
-      def execute_http_request(response_guard, deadline:, consume_budget: true)
+      def execute_http_request(deadline:, consume_budget: true)
         preflight!(consume_budget:)
-        session = build_session(response_guard, deadline:)
-        response = session.get(request_url.to_s, headers: ctx.headers)
+        session = session_client(deadline)
+        response = http_get(session)
         raise response.error if response.is_a?(HTTPX::ErrorResponse)
 
         response
       end
 
-      def build_session(_response_guard, deadline:)
-        session_client(deadline)
+      # GET only. Overflow here is HTTPX retry recursion; the same error from
+      # redirect handling or body decode must stay loud.
+      def http_get(session)
+        session.get(request_url.to_s, headers: ctx.headers)
+      rescue SystemStackError
+        raise HTTPX::ConnectionError, 'HTTPX retry recursion exhausted'
       end
 
       def session_client(deadline)
@@ -152,7 +162,8 @@ module Html2rss
           follow_insecure_redirects: false,
           max_response_body_size: ctx.policy.max_response_bytes,
           resolver_class: :system,
-          redirect_on: redirect_callback
+          redirect_on: redirect_callback,
+          **RETRY_OPTIONS
         }
       end
 

data/lib/html2rss/request_service/network_guard.rb

@@ -13,7 +13,7 @@ module Html2rss
     # {Policy} so SSRF/origin rules stay single-homed on the public façade.
     class NetworkGuard
       # Hostnames treated as local/private surfaces.
-      LOCAL_HOSTS = %w[localhost localhost.localdomain metadata.google.internal].to_set.freeze
+      LOCAL_HOSTS = Set['localhost', 'localhost.localdomain', 'metadata.google.internal'].freeze
       # IP ranges blocked when private networks are disabled.
       BLOCKED_IP_RANGES = [
         IPAddr.new('0.0.0.0/8'),
@@ -32,7 +32,9 @@ module Html2rss
 
       ##
       # @param allow_private_networks [Boolean] whether private network targets are allowed
-      # @param resolver [#each_address, #getaddrinfo] DNS resolver used for hostname classification
+      # @param resolver [Socket, Resolv] DNS resolver used for hostname classification.
+      #   +Socket+ (the default) uses +getaddrinfo+; any other resolver is Resolv-shaped
+      #   and must implement +each_address+.
       def initialize(allow_private_networks:, resolver: Socket)
         @allow_private_networks = allow_private_networks ? true : false
         @resolver = resolver
@@ -89,10 +91,10 @@ module Html2rss
         literal = parse_ip(host)
         return [literal] if literal
 
-        if resolver.respond_to?(:each_address)
-          addresses_from_each_address(host)
-        else
+        if resolver == Socket
           addresses_from_getaddrinfo(host)
+        else
+          addresses_from_each_address(host)
         end
       rescue Resolv::ResolvError, SocketError, SystemCallError
         []

data/lib/html2rss/request_service/response.rb

@@ -36,8 +36,7 @@ module Html2rss
                      captured_responses: EMPTY_CAPTURED_RESPONSES)
         @body = body
 
-        headers = headers.dup
-        headers.transform_keys!(&:to_s)
+        headers = (headers || {}).transform_keys(&:to_s)
         HashUtil.assert_string_keys!(headers, context: 'response headers', deep: false)
 
         @headers = headers

data/lib/html2rss/request_service/strategy.rb

@@ -22,7 +22,7 @@ module Html2rss
       def execute
         check_timeout!
         response = perform_execute
-        postflight!(response, response_guard:)
+        postflight!(response)
         response
       rescue StandardError => error
         handle_error(error)
@@ -67,13 +67,11 @@ module Html2rss
       # Enforces response-size and blocked-surface checks after transport work.
       #
       # @param response [Response, nil] adapter response
-      # @param response_guard [ResponseGuard, nil] existing guard to reuse (e.g. streaming)
       # @return [void]
-      def postflight!(response, response_guard: nil)
+      def postflight!(response)
         return if skip_postflight? || response.nil?
 
-        guard = response_guard || ResponseGuard.new(policy: ctx.policy)
-        guard.inspect_body!(response.body)
+        ResponseGuard.new(policy: ctx.policy).inspect_body!(response.body)
       end
 
       ##
@@ -91,9 +89,6 @@ module Html2rss
         raise
       end
 
-      # @return [ResponseGuard, nil]
-      def response_guard = nil
-
       # @param error [StandardError]
       # @return [void]
       # @raise [StandardError]

data/lib/html2rss/scoring/container_assessor.rb

@@ -70,7 +70,9 @@ module Html2rss
       end
 
       def word_count(text)
-        text.to_s.scan(/\p{Alnum}+/).size
+        count = 0
+        text.to_s.scan(/\p{Alnum}+/) { count += 1 }
+        count
       end
     end
   end

data/lib/html2rss/scoring/engine.rb

@@ -10,14 +10,14 @@ module Html2rss
       # Minimum composite score retained by {#rank_top} (precision floor).
       SCORE_FLOOR = 0.0
 
-      FEATURE_IDS = %i[
-        title_word_count_ge3 title_word_count_ge7 path_length_gt6 content_path
-        publish_marker descriptive_context article_container content_tokens
-        non_content_utility_path utility_prefix_title_short shallow weak_container
-        recommended_title_non_content high_confidence_junk_path junk_tokens
-        heading_anchor heading_text_match meaningful_text content_like_destination
-        cluster_size cluster_avg_words cluster_heading cluster_time cluster_date
-      ].to_set.freeze
+      FEATURE_IDS = Set[
+        :title_word_count_ge3, :title_word_count_ge7, :path_length_gt6, :content_path,
+        :publish_marker, :descriptive_context, :article_container, :content_tokens,
+        :non_content_utility_path, :utility_prefix_title_short, :shallow, :weak_container,
+        :recommended_title_non_content, :high_confidence_junk_path, :junk_tokens,
+        :heading_anchor, :heading_text_match, :meaningful_text, :content_like_destination,
+        :cluster_size, :cluster_avg_words, :cluster_heading, :cluster_time, :cluster_date
+      ].freeze
       private_constant :FEATURE_IDS
 
       # Quality feature triples: [feature_id, predicate, weight].

data/lib/html2rss/scoring/observation.rb

@@ -2,8 +2,6 @@
 
 module Html2rss
   module Scoring
-    ##
-    # Typed container signals produced by {ContainerAssessor} for {Engine} rules.
     Observation = Data.define(
       :title_word_count,
       :path_length,
@@ -21,7 +19,11 @@ module Html2rss
       :high_confidence_junk_path,
       :high_confidence_utility_destination,
       :selected_anchor_present
-    ) do
+    )
+
+    ##
+    # Typed container signals produced by {ContainerAssessor} for {Engine} rules.
+    class Observation
       ##
       # @return [Boolean] true when fewer than two positive content signals are present
       def weak_signals?

data/lib/html2rss/scoring/ranked_segment.rb

@@ -2,9 +2,11 @@
 
 module Html2rss
   module Scoring
+    RankedSegment = Data.define(:segment, :score)
+
     ##
     # A segment paired with its composite score.
-    RankedSegment = Data.define(:segment, :score) do
+    class RankedSegment
       ##
       # @return [SST::Node]
       def root_node = segment.root_node

data/lib/html2rss/scrape_target.rb

@@ -1,11 +1,13 @@
 # frozen_string_literal: true
 
 module Html2rss
+  ScrapeTarget = Data.define(:entry_url, :effective_url)
+
   ##
   # Immutable entry vs effective scrape URLs for one pipeline run.
   #
   # Replaces mutating +Config#scrape_url=+ after {FeedResolution} rewrites the fetch URL.
-  ScrapeTarget = Data.define(:entry_url, :effective_url) do
+  class ScrapeTarget
     ##
     # @param config [Html2rss::Config]
     # @return [ScrapeTarget]