html2rss 0.29.1 → 0.31.0

data/lib/html2rss/request_service/httpx_strategy.rb

@@ -0,0 +1,239 @@
+# frozen_string_literal: true
+
+require 'httpx'
+require 'httpx/plugins/follow_redirects'
+require 'httpx/plugins/callbacks'
+require 'httpx/plugins/ssrf_filter'
+require 'httpx/plugins/brotli'
+require 'httpx/plugins/retries'
+
+module Html2rss
+  class RequestService
+    ##
+    # Strategy to use HTTPX for HTTP requests.
+    # Provides native HTTP/2 with ALPN, built-in SSRF protection, streaming byte limits,
+    # and redirect handling without monkey-patching.
+    # rubocop:disable-next Metrics/ClassLength -- terminal redirect retry colocated with HTTPX transport
+    class HttpxStrategy < Strategy
+      # HTTPX default; kept explicit so ping reconnects cannot omit a bound.
+      MAX_RETRIES = 3
+      # Forces the selector timer retry path. Unset retry_after recurses in fetch_response;
+      # combined with ping! on a reused closed selector that never decrements retries,
+      # that overflows the stack and kills Falcon workers.
+      RETRY_AFTER_SECONDS = 0.25
+      # Bounds passed to the retries plugin and every +session.with+.
+      # HTTPX retries stay inside one {RequestService::Budget} slot and the shared
+      # wall-clock deadline; do not consume a slot per attempt.
+      # Do not reload +:fiber_concurrency+: a reused closed selector made +ping!+
+      # skip decrementing retries and recurse until +SystemStackError+.
+      #
+      # @return [Hash{Symbol => Integer, Float}]
+      RETRY_OPTIONS = { max_retries: MAX_RETRIES, retry_after: RETRY_AFTER_SECONDS }.freeze
+
+      class << self
+        # rubocop:disable ThreadSafety/ClassInstanceVariable
+        # @return [HTTPX::Session]
+        def base_session
+          @base_session ||= HTTPX
+                            .plugin(:follow_redirects)
+                            .plugin(:callbacks)
+                            .plugin(:brotli)
+                            .plugin(:retries, **RETRY_OPTIONS)
+        end
+
+        # @return [HTTPX::Session]
+        def base_ssrf_session
+          @base_ssrf_session ||= base_session.plugin(:ssrf_filter)
+        end
+        # rubocop:enable ThreadSafety/ClassInstanceVariable
+      end
+
+      # Executes the request with runtime policy enforcement, returning the normalized response.
+      #
+      # @return [Response] normalized response
+      def perform_execute
+        deadline = request_deadline
+        reset_redirect_tracking!
+        raw_response = request_with_terminal_redirect_retry(deadline:)
+        build_response(raw_response)
+      end
+
+      private
+
+      def request_deadline
+        monotonic_now + ctx.budget.effective_timeout_seconds(fallback: ctx.policy.total_timeout_seconds)
+      end
+
+      def reset_redirect_tracking!
+        @last_redirect_to = nil
+        @terminal_redirect_retried = false
+        @request_url_override = nil
+        @current_url = request_url
+      end
+
+      def request_with_terminal_redirect_retry(deadline:)
+        raw_response = execute_http_request(deadline:)
+        return raw_response unless redirect_limit_reached?(raw_response)
+
+        unless terminal_redirect_retryable?(raw_response)
+          raise RedirectLimitReached, "Too many redirects (status #{raw_response.status})"
+        end
+
+        retry_from_terminal_redirect!(raw_response, deadline:)
+      end
+
+      def redirect_limit_reached?(response)
+        return false unless response.is_a?(HTTPX::Response)
+        return false if response.status == 304
+        return false unless (300..399).cover?(response.status)
+
+        location = response.headers['location']
+        !location.to_s.strip.empty?
+      end
+
+      def terminal_redirect_url(raw_response)
+        location = raw_response.headers['location']
+        if location && !location.empty?
+          base = raw_response.uri || request_url
+          return normalize_url(URI.join(base.to_s, location))
+        end
+
+        @last_redirect_to
+      end
+
+      def terminal_redirect_retryable?(raw_response)
+        return false if @terminal_redirect_retried
+
+        target = terminal_redirect_url(raw_response)
+        target && target.to_s != request_url.to_s
+      end
+
+      def retry_from_terminal_redirect!(raw_response, deadline:)
+        terminal_url = terminal_redirect_url(raw_response)
+        @terminal_redirect_retried = true
+        Log.debug("#{self.class}: redirect limit reached; retrying once from #{terminal_url}")
+        begin_terminal_url_request!(terminal_url)
+        new_response = execute_http_request(deadline:, consume_budget: false)
+        if redirect_limit_reached?(new_response)
+          raise RedirectLimitReached, "Too many redirects (status #{new_response.status})"
+        end
+
+        new_response
+      end
+
+      def begin_terminal_url_request!(terminal_url)
+        ctx.policy.validate_request!(url: terminal_url, origin_url: ctx.origin_url, relation: ctx.relation)
+        @request_url_override = terminal_url
+        @last_redirect_to = nil
+        @current_url = terminal_url
+      end
+
+      def request_url
+        @request_url_override || ctx.url
+      end
+
+      def execute_http_request(deadline:, consume_budget: true)
+        preflight!(consume_budget:)
+        session = session_client(deadline)
+        response = http_get(session)
+        raise response.error if response.is_a?(HTTPX::ErrorResponse)
+
+        response
+      end
+
+      # GET only. Overflow here is HTTPX retry recursion; the same error from
+      # redirect handling or body decode must stay loud.
+      def http_get(session)
+        session.get(request_url.to_s, headers: ctx.headers)
+      rescue SystemStackError
+        raise HTTPX::ConnectionError, 'HTTPX retry recursion exhausted'
+      end
+
+      def session_client(deadline)
+        session = ctx.policy.allow_private_networks? ? self.class.base_session : self.class.base_ssrf_session
+        session.with(session_options(deadline))
+      end
+
+      def session_options(deadline)
+        remaining = remaining_timeout_seconds(deadline)
+        {
+          timeout: session_timeouts(remaining),
+          max_redirects: ctx.policy.max_redirects,
+          follow_insecure_redirects: false,
+          max_response_body_size: ctx.policy.max_response_bytes,
+          resolver_class: :system,
+          redirect_on: redirect_callback,
+          **RETRY_OPTIONS
+        }
+      end
+
+      def session_timeouts(remaining)
+        {
+          connect_timeout: [ctx.policy.connect_timeout_seconds, remaining].min,
+          read_timeout: [ctx.policy.read_timeout_seconds, remaining].min,
+          operation_timeout: remaining,
+          request_timeout: remaining,
+          total_request_timeout: remaining
+        }
+      end
+
+      def redirect_callback
+        lambda do |redirect_uri|
+          to_url = normalize_url(redirect_uri)
+          from_url = @current_url
+          @last_redirect_to = to_url
+          ctx.policy.validate_redirect!(from_url:, to_url:, origin_url: ctx.origin_url, relation: ctx.relation)
+          @current_url = to_url
+          true
+        end
+      end
+
+      def build_response(response)
+        headers = response.headers.to_h
+        Response.new(
+          body: CompressedBody.decode(response.body.to_s, headers:),
+          headers:,
+          url: response_url(response),
+          status: response.status
+        )
+      end
+
+      def response_url(response)
+        return ctx.url unless (uri = response.uri)
+
+        Html2rss::Url.from_absolute(uri.to_s)
+      end
+
+      def normalize_url(url)
+        Html2rss::Url.from_absolute(url.to_s)
+      end
+
+      def remaining_timeout_seconds(deadline)
+        remaining = deadline - monotonic_now
+        raise RequestTimedOut, 'Request timed out' if remaining <= 0
+
+        remaining
+      end
+
+      def monotonic_now
+        Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      end
+
+      def timeout_error?(error)
+        error.is_a?(HTTPX::TimeoutError) || super
+      end
+
+      def ssrf_error?(error)
+        error.is_a?(HTTPX::ServerSideRequestForgeryError) || super
+      end
+
+      def response_too_large_error?(error)
+        (error.is_a?(HTTPX::Error) && error.message.include?('maximum response body size exceeded')) || super
+      end
+
+      def connection_error?(error)
+        error.is_a?(HTTPX::ConnectionError) || error.is_a?(HTTPX::TLSError) || super
+      end
+    end
+  end
+end

data/lib/html2rss/request_service/network_guard.rb

@@ -13,7 +13,7 @@ module Html2rss
     # {Policy} so SSRF/origin rules stay single-homed on the public façade.
     class NetworkGuard
       # Hostnames treated as local/private surfaces.
-      LOCAL_HOSTS = %w[localhost localhost.localdomain metadata.google.internal].to_set.freeze
+      LOCAL_HOSTS = Set['localhost', 'localhost.localdomain', 'metadata.google.internal'].freeze
       # IP ranges blocked when private networks are disabled.
       BLOCKED_IP_RANGES = [
         IPAddr.new('0.0.0.0/8'),
@@ -32,7 +32,9 @@ module Html2rss
 
       ##
       # @param allow_private_networks [Boolean] whether private network targets are allowed
-      # @param resolver [#each_address, #getaddrinfo] DNS resolver used for hostname classification
+      # @param resolver [Socket, Resolv] DNS resolver used for hostname classification.
+      #   +Socket+ (the default) uses +getaddrinfo+; any other resolver is Resolv-shaped
+      #   and must implement +each_address+.
       def initialize(allow_private_networks:, resolver: Socket)
         @allow_private_networks = allow_private_networks ? true : false
         @resolver = resolver
@@ -89,10 +91,10 @@ module Html2rss
         literal = parse_ip(host)
         return [literal] if literal
 
-        if resolver.respond_to?(:each_address)
-          addresses_from_each_address(host)
-        else
+        if resolver == Socket
           addresses_from_getaddrinfo(host)
+        else
+          addresses_from_each_address(host)
         end
       rescue Resolv::ResolvError, SocketError, SystemCallError
         []

data/lib/html2rss/request_service/response.rb

@@ -36,8 +36,7 @@ module Html2rss
                      captured_responses: EMPTY_CAPTURED_RESPONSES)
         @body = body
 
-        headers = headers.dup
-        headers.transform_keys!(&:to_s)
+        headers = (headers || {}).transform_keys(&:to_s)
         HashUtil.assert_string_keys!(headers, context: 'response headers', deep: false)
 
         @headers = headers

data/lib/html2rss/request_service/response_guard.rb

@@ -9,22 +9,6 @@ module Html2rss
       # @param policy [Policy] request policy that defines byte ceilings
       def initialize(policy:)
         @policy = policy
-        @streamed_bytes = 0
-      end
-
-      ##
-      # Validates response headers and streamed byte count.
-      #
-      # @param total_bytes [Integer] cumulative byte count received so far
-      # @param headers [Hash, nil] response headers if known
-      # @return [void]
-      # @raise [ResponseTooLarge] if the response exceeds configured limits
-      def inspect_chunk!(total_bytes:, headers: nil)
-        header_length = headers&.fetch('content-length', headers&.fetch('Content-Length', nil))
-        raise_if_too_large!(header_length.to_i, policy.max_response_bytes) if header_length
-
-        @streamed_bytes = total_bytes
-        raise_if_too_large!(@streamed_bytes, policy.max_response_bytes)
       end
 
       ##

data/lib/html2rss/request_service/strategy.rb

@@ -22,7 +22,7 @@ module Html2rss
       def execute
         check_timeout!
         response = perform_execute
-        postflight!(response, response_guard:)
+        postflight!(response)
         response
       rescue StandardError => error
         handle_error(error)
@@ -67,13 +67,11 @@ module Html2rss
       # Enforces response-size and blocked-surface checks after transport work.
       #
       # @param response [Response, nil] adapter response
-      # @param response_guard [ResponseGuard, nil] existing guard to reuse (e.g. streaming)
       # @return [void]
-      def postflight!(response, response_guard: nil)
+      def postflight!(response)
         return if skip_postflight? || response.nil?
 
-        guard = response_guard || ResponseGuard.new(policy: ctx.policy)
-        guard.inspect_body!(response.body)
+        ResponseGuard.new(policy: ctx.policy).inspect_body!(response.body)
       end
 
       ##
@@ -91,17 +89,19 @@ module Html2rss
         raise
       end
 
-      # @return [ResponseGuard, nil]
-      def response_guard = nil
-
       # @param error [StandardError]
       # @return [void]
       # @raise [StandardError]
+      # rubocop:disable-next Metrics/AbcSize, Metrics/MethodLength -- error translation dispatch
       def handle_error(error)
         if timeout_error?(error)
           log_timeout!(reason: 'transport')
           Log.debug("#{self.class}: transport timeout message=#{error.message}")
           raise RequestTimedOut, error.message
+        elsif ssrf_error?(error)
+          raise PrivateNetworkDenied, error.message
+        elsif response_too_large_error?(error)
+          raise ResponseTooLarge, "Response exceeded #{ctx.policy.max_response_bytes} bytes"
         elsif connection_error?(error)
           translate_connection_error(error)
         else
@@ -135,14 +135,25 @@ module Html2rss
       # @param error [StandardError]
       # @return [Boolean]
       def timeout_error?(error)
-        error.is_a?(Faraday::TimeoutError) ||
-          error.is_a?(Timeout::Error)
+        error.is_a?(Timeout::Error)
+      end
+
+      # @param _error [StandardError]
+      # @return [Boolean]
+      def ssrf_error?(_error)
+        false
+      end
+
+      # @param _error [StandardError]
+      # @return [Boolean]
+      def response_too_large_error?(_error)
+        false
       end
 
       # @param error [StandardError]
       # @return [Boolean]
       def connection_error?(error)
-        error.is_a?(Faraday::ConnectionFailed) || error.is_a?(Faraday::SSLError)
+        error.is_a?(SocketError) || error.is_a?(SystemCallError)
       end
     end
   end

data/lib/html2rss/request_service.rb

@@ -1,18 +1,13 @@
 # frozen_string_literal: true
 
-require 'singleton'
-require 'forwardable'
-
 module Html2rss
   ##
   # Requests website URLs to retrieve their HTML for further processing.
-  # Provides concrete transport strategies (e.g. Faraday, Botasaurus).
+  # Provides concrete transport strategies (e.g. HttpxStrategy, BotasaurusStrategy).
   #
   # Feed-level +:auto+ is not registered here — {FeedPipeline::StrategyPlan} resolves
   # it to a concrete strategy (or {FeedPipeline::AutoFallback} chain) before execute.
   class RequestService
-    include Singleton
-
     # Raised when an unknown request strategy is requested.
     class UnknownStrategy < Html2rss::Error; end
     # Raised when a URL cannot be parsed or validated.
@@ -59,94 +54,60 @@ module Html2rss
     # Raised when Botasaurus responds but the scrape fails (upstream error, bad payload).
     class BotasaurusServiceError < Html2rss::Error; end
 
-    class << self
-      extend Forwardable
-
-      %i[default_strategy_name
-         default_strategy_name=
-         strategy_names
-         register_strategy
-         unregister_strategy
-         strategy_registered?
-         execute].each do |method|
-        def_delegator :instance, method
-      end
-    end
-
-    def initialize
-      @strategies = {
-        faraday: FaradayStrategy,
-        botasaurus: BotasaurusStrategy,
-        local_file: LocalFileStrategy
-      }
-      @default_strategy_name = :faraday
-    end
+    # Map of supported strategy names to their implementation classes.
+    # @return [Hash{Symbol => Class<Strategy>}]
+    STRATEGIES = {
+      default: HttpxStrategy,
+      httpx: HttpxStrategy,
+      faraday: HttpxStrategy,
+      botasaurus: BotasaurusStrategy,
+      local_file: LocalFileStrategy
+    }.freeze
 
-    # @return [Symbol] the default strategy name
-    attr_reader :default_strategy_name
+    # Canonical default strategy symbol.
+    # @return [Symbol]
+    DEFAULT_STRATEGY_NAME = :default
 
-    ##
-    # Sets the default strategy.
-    # @param strategy [Symbol] the name of the strategy
-    # @return [Symbol] the selected default strategy name
-    # @raise [UnknownStrategy] if the strategy is not registered
-    def default_strategy_name=(strategy)
-      raise UnknownStrategy unless strategy_registered?(strategy)
-
-      @default_strategy_name = strategy.to_sym
-    end
+    class << self
+      # @return [Symbol] the default strategy name
+      def default_strategy_name = DEFAULT_STRATEGY_NAME
 
-    # @return [Array<String>] the names of the registered strategies
-    def strategy_names = @strategies.keys.map(&:to_s)
+      # @return [Array<String>] the names of the registered strategies
+      def strategy_names = STRATEGIES.keys.map(&:to_s)
 
-    ##
-    # Registers a new strategy.
-    # @param name [Symbol] the name of the strategy
-    # @param strategy_class [Class] the class implementing the strategy
-    # @return [Class] the registered strategy class
-    # @raise [ArgumentError] if strategy_class is not a Class
-    def register_strategy(name, strategy_class)
-      unless strategy_class.is_a?(Class)
-        raise ArgumentError, "Expected a Class for strategy, got #{strategy_class.class}"
+      ##
+      # Checks if a strategy is registered.
+      # @param name [Symbol, String] the name of the strategy
+      # @return [Boolean] true if the strategy is registered, false otherwise.
+      def strategy_registered?(name)
+        STRATEGIES.key?(name.to_sym)
       end
 
-      @strategies[name.to_sym] = strategy_class
-    end
-
-    ##
-    # Checks if a strategy is registered.
-    # @param name [Symbol] the name of the strategy
-    # @return [Boolean] true if the strategy is registered, false otherwise.
-    def strategy_registered?(name)
-      @strategies.key?(name.to_sym)
-    end
-
-    ##
-    # Unregisters a strategy.
-    # @param name [Symbol] the name of the strategy
-    # @return [Boolean] true if the strategy was unregistered, false otherwise.
-    # @raise [ArgumentError] if attempting to unregister the default strategy.
-    def unregister_strategy(name) # rubocop:disable Naming/PredicateMethod
-      name_sym = name.to_sym
-      raise ArgumentError, 'Cannot unregister the default strategy.' if name_sym == @default_strategy_name
+      ##
+      # Executes the request using the specified strategy.
+      # @param ctx [Context] the context for the request.
+      # @param strategy [Symbol, String] the strategy to use (defaults to the default strategy).
+      # @return [Response] the response from the executed strategy.
+      # @raise [UnknownStrategy] if the strategy is not registered.
+      def execute(ctx, strategy: default_strategy_name)
+        strategy_sym = strategy.to_sym
+        strategy_class = STRATEGIES.fetch(strategy_sym) do
+          raise UnknownStrategy,
+                "The strategy '#{strategy}' is not known. Available strategies: #{strategy_names.join(', ')}"
+        end
+
+        warn_migration_strategy(strategy_sym) if strategy_sym == :faraday
+        strategy_class.new(ctx).execute
+      end
 
-      !!@strategies.delete(name_sym)
-    end
+      private
 
-    ##
-    # Executes the request using the specified strategy.
-    # @param ctx [Context] the context for the request.
-    # @param strategy [Symbol] the strategy to use (defaults to the default strategy).
-    # @return [Response] the response from the executed strategy.
-    # @raise [ArgumentError] if the context is nil.
-    # @raise [UnknownStrategy] if the strategy is not registered.
-    def execute(ctx, strategy: default_strategy_name)
-      strategy_class = @strategies.fetch(strategy.to_sym) do
-        raise UnknownStrategy,
-              "The strategy '#{strategy}' is not known. Available strategies: #{strategy_names.join(', ')}"
+      def warn_migration_strategy(strategy)
+        message = "RequestService: strategy ':#{strategy}' is deprecated for migration and will be removed " \
+                  "in a future release. Use ':default' instead."
+        warn(message, category: :deprecated)
+        Log.warn(message)
       end
-
-      strategy_class.new(ctx).execute
     end
   end
 end

data/lib/html2rss/scoring/container_assessor.rb

@@ -70,7 +70,9 @@ module Html2rss
       end
 
       def word_count(text)
-        text.to_s.scan(/\p{Alnum}+/).size
+        count = 0
+        text.to_s.scan(/\p{Alnum}+/) { count += 1 }
+        count
       end
     end
   end

data/lib/html2rss/scoring/engine.rb

@@ -10,14 +10,14 @@ module Html2rss
       # Minimum composite score retained by {#rank_top} (precision floor).
       SCORE_FLOOR = 0.0
 
-      FEATURE_IDS = %i[
-        title_word_count_ge3 title_word_count_ge7 path_length_gt6 content_path
-        publish_marker descriptive_context article_container content_tokens
-        non_content_utility_path utility_prefix_title_short shallow weak_container
-        recommended_title_non_content high_confidence_junk_path junk_tokens
-        heading_anchor heading_text_match meaningful_text content_like_destination
-        cluster_size cluster_avg_words cluster_heading cluster_time cluster_date
-      ].to_set.freeze
+      FEATURE_IDS = Set[
+        :title_word_count_ge3, :title_word_count_ge7, :path_length_gt6, :content_path,
+        :publish_marker, :descriptive_context, :article_container, :content_tokens,
+        :non_content_utility_path, :utility_prefix_title_short, :shallow, :weak_container,
+        :recommended_title_non_content, :high_confidence_junk_path, :junk_tokens,
+        :heading_anchor, :heading_text_match, :meaningful_text, :content_like_destination,
+        :cluster_size, :cluster_avg_words, :cluster_heading, :cluster_time, :cluster_date
+      ].freeze
       private_constant :FEATURE_IDS
 
       # Quality feature triples: [feature_id, predicate, weight].

data/lib/html2rss/scoring/observation.rb

@@ -2,8 +2,6 @@
 
 module Html2rss
   module Scoring
-    ##
-    # Typed container signals produced by {ContainerAssessor} for {Engine} rules.
     Observation = Data.define(
       :title_word_count,
       :path_length,
@@ -21,7 +19,11 @@ module Html2rss
       :high_confidence_junk_path,
       :high_confidence_utility_destination,
       :selected_anchor_present
-    ) do
+    )
+
+    ##
+    # Typed container signals produced by {ContainerAssessor} for {Engine} rules.
+    class Observation
       ##
       # @return [Boolean] true when fewer than two positive content signals are present
       def weak_signals?

data/lib/html2rss/scoring/ranked_segment.rb

@@ -2,9 +2,11 @@
 
 module Html2rss
   module Scoring
+    RankedSegment = Data.define(:segment, :score)
+
     ##
     # A segment paired with its composite score.
-    RankedSegment = Data.define(:segment, :score) do
+    class RankedSegment
       ##
       # @return [SST::Node]
       def root_node = segment.root_node

data/lib/html2rss/scrape_target.rb

@@ -1,11 +1,13 @@
 # frozen_string_literal: true
 
 module Html2rss
+  ScrapeTarget = Data.define(:entry_url, :effective_url)
+
   ##
   # Immutable entry vs effective scrape URLs for one pipeline run.
   #
   # Replaces mutating +Config#scrape_url=+ after {FeedResolution} rewrites the fetch URL.
-  ScrapeTarget = Data.define(:entry_url, :effective_url) do
+  class ScrapeTarget
     ##
     # @param config [Html2rss::Config]
     # @return [ScrapeTarget]