html2rss 0.32.2 → 0.32.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e095abe52bc258d262e3a24909e0feb480b2eb1d537e57580fc271203b0c6dbc
-  data.tar.gz: 8fdabe7ed5afee08c46ab94c2b6f80c1f8c66df9ffec9439e40e80fa897bfa96
+  metadata.gz: 8d37540b2f300af8a1db631e16cf6cabf4cfa76069000a71c7723737225bd6c3
+  data.tar.gz: bd8f656ae629587336257d790da183c759b9c709c1ba411e5c0041156d35dd6b
 SHA512:
-  metadata.gz: fdcce48a2a4fb3a8c3ea02fd8f4bb799bd8ed7acb4e66e74da3cde693f6a51cc3aa72d363670b13f077d1521d97321e0676d14b59c6b61f1c685fdc4db64cb67
-  data.tar.gz: ce1c1cac4c38f4660e61cb7c9506991d8c0d847c8bdfbb583c678680854b7327a16b9aa637af641aaae42bed9f15b9330cc344ee922252da16863a517569fad4
+  metadata.gz: 802b88abad9bbadfa6753a5c62694a66c6eec3774e84d85191beabdfdd2bc8df3e37fdf325bc14c1d6fda0b90528aca2484df15d0d985b6e328c07f4c27a49b9
+  data.tar.gz: 827d398c34732c10600ced1002525702392448fcb4e8b585f7c27048c8fae38e9c2652307705c8a91e82fba295dbbfa8bfb0cc4dc3a71c4d70cdf9773722a4cf

data/lib/html2rss/feed_pipeline/README.md

@@ -35,6 +35,10 @@ These abort immediately (`AutoFallback::NON_FALLBACK_ERRORS`): unknown strategy,
 
 Retries share the feed's request/session policy. Pin a concrete strategy when you need a single-hop budget profile.
 
+### Terminal fallback outcome
+
+A hop that returns zero items does **not** erase an earlier terminal transport failure (`RequestTimedOut` or `BlockedSurfaceDetected`). `AutoFallback` remembers that error across later hops and, when no later hop yields items, re-raises it with `attempts` attached. A later hop that extracts items still succeeds. Two zero-item hops with no terminal transport error still raise `NoFeedItemsExtracted`. `html2rss-web` `ErrorClassifier` maps the raised error to HTTP/cache policy; it does not invent a second cache rule for empty extraction.
+
 ## Success signals
 
 - `Html2rss::RequestService::Response` includes transport metadata for the strategy that produced it.

data/lib/html2rss/feed_pipeline/auto_fallback.rb

@@ -27,16 +27,26 @@ module Html2rss
         RequestService::RedirectLimitReached
       ].freeze
 
+      # Transport failures preserved across a later hop that extracts zero items.
+      # Re-raised from finalize_failure so web maps them to uncacheable gateway/blocked
+      # decisions instead of EXTRACTION_EMPTY.
+      TERMINAL_TRANSPORT_ERRORS = [
+        RequestService::RequestTimedOut,
+        RequestService::BlockedSurfaceDetected
+      ].freeze
+
       ##
       # Mutable run state for one auto-fallback chain: attempts plus selected result.
       class AttemptState
-        attr_reader :attempts, :result, :last_response, :last_error, :entry_resolution
+        attr_reader :attempts, :result, :last_response, :last_error, :entry_resolution,
+                    :terminal_transport_error
 
         def initialize
           @attempts = []
           @result = nil
           @last_response = nil
           @last_error = nil
+          @terminal_transport_error = nil
           @entry_resolution = nil
           @resolution_tried = false
           @aborted = false
@@ -76,6 +86,9 @@ module Html2rss
           attempt[:transport_meta] = meta if meta&.any?
           @attempts << attempt
           @last_error = error
+          return unless TERMINAL_TRANSPORT_ERRORS.any? { |klass| error.is_a?(klass) }
+
+          @terminal_transport_error = error
         end
 
         # @param strategy [Symbol] strategy that returned a response
@@ -85,7 +98,7 @@ module Html2rss
         def record_items(strategy:, items_count:, transport_meta: nil)
           @last_error = nil
           attempt = { strategy:, items_count:, error_class: nil }
-          attempt[:transport_meta] = transport_meta if transport_meta && !transport_meta.empty?
+          attempt[:transport_meta] = transport_meta if transport_meta&.any?
           @attempts << attempt
         end
 
@@ -183,7 +196,9 @@ module Html2rss
 
       # rubocop:disable-next Metrics/MethodLength, Metrics/AbcSize -- extract + resolution delegate
       def process_response(response:, strategy:, next_strategy:, request_session:, state:)
-        articles, dedup_dropped, admission_drops = articles_for(response:, request_session:)
+        articles, dedup_dropped, admission_drops = pipeline.deduplicated_articles(
+          config:, response:, request_session:
+        )
         state.remember_response(response)
         items_count = articles.size
         state.record_items(strategy:, items_count:, transport_meta: response.transport_meta)
@@ -204,7 +219,7 @@ module Html2rss
                                 state:)
         end
 
-        if deterministic_status?(response)
+        if DETERMINISTIC_HTTP_STATUSES.member?(response&.status)
           state.abort!
           Log.debug("#{self.class}: deterministic HTTP #{response.status}; aborting auto fallback")
           return
@@ -213,14 +228,6 @@ module Html2rss
         log_info_fallback_zero_items(strategy:, next_strategy:, response:) if next_strategy
       end
 
-      def deterministic_status?(response)
-        DETERMINISTIC_HTTP_STATUSES.member?(response&.status)
-      end
-
-      def articles_for(response:, request_session:)
-        pipeline.deduplicated_articles(config:, response:, request_session:)
-      end
-
       # rubocop:disable-next Metrics/ParameterLists, Metrics/MethodLength -- success kwargs + structured selected log
       def record_success(response:, strategy:, articles:, dedup_dropped:, admission_drops:, state:)
         attempt_count = state.attempts.size
@@ -243,57 +250,46 @@ module Html2rss
       end
 
       def finalize_failure(state)
-        if raise_last_error?(state)
-          error = state.last_error
+        error = failure_error(state)
+        if error
           error.attempts = state.attempts if error.is_a?(Html2rss::Error)
           raise error
         end
 
+        response = state.last_response
         raise NoFeedItemsExtracted.new(
           attempts: state.attempts,
-          surface_category: surface_category_for(state.last_response)
+          surface_category: (PageRecon.surface_category_for(response:, url: response.url) if response)
         )
       end
 
-      def raise_last_error?(state)
-        error = state.last_error
-        return false if error.nil? || error.is_a?(RequestService::BotasaurusConfigurationError)
+      # Prefer a remembered timeout/blocked-surface over a later zero-item hop so
+      # callers do not cache EXTRACTION_EMPTY for a transport failure.
+      def failure_error(state)
+        return state.terminal_transport_error if state.terminal_transport_error
 
-        true
-      end
-
-      def surface_category_for(response)
-        return unless response
+        error = state.last_error
+        return if error.nil? || error.is_a?(RequestService::BotasaurusConfigurationError)
 
-        PageRecon.surface_category_for(response:, url: response.url)
+        error
       end
 
-      # rubocop:disable-next Metrics/AbcSize, Metrics/MethodLength
+      # rubocop:disable-next Metrics/AbcSize, Metrics/MethodLength -- shared payload then info/warn dispatch
       def log_fallback_error(strategy:, next_strategy:, error:, request_session:)
         host = request_session.url.host
         detail = "host=#{host} elapsed=#{format('%.3f', budget.elapsed_seconds)}s " \
                  "budget_remaining=#{budget_remaining_label}"
-        if error.is_a?(RequestService::RequestTimedOut)
-          Log.info({
-            event_name: 'auto_fallback.timeout',
-            host:,
-            strategy: strategy.to_s,
-            next_strategy: next_strategy.to_s,
-            error_class: error.class.name,
-            message: "#{self.class}: auto fallback #{strategy} -> #{next_strategy} " \
-                     "after timeout=#{error.class} #{detail}"
-          }.to_json)
-        else
-          Log.warn({
-            event_name: 'auto_fallback.error',
-            host:,
-            strategy: strategy.to_s,
-            next_strategy: next_strategy.to_s,
-            error_class: error.class.name,
-            message: "#{self.class}: auto fallback #{strategy} -> #{next_strategy} " \
-                     "after error=#{error.class} #{detail}"
-          }.to_json)
-        end
+        timed_out = error.is_a?(RequestService::RequestTimedOut)
+        payload = {
+          event_name: timed_out ? 'auto_fallback.timeout' : 'auto_fallback.error',
+          host:,
+          strategy: strategy.to_s,
+          next_strategy: next_strategy.to_s,
+          error_class: error.class.name,
+          message: "#{self.class}: auto fallback #{strategy} -> #{next_strategy} " \
+                   "after #{timed_out ? "timeout=#{error.class}" : "error=#{error.class}"} #{detail}"
+        }.to_json
+        timed_out ? Log.info(payload) : Log.warn(payload)
         Log.debug("#{self.class}: strategy=#{strategy} error=#{error.class}: #{error.message} #{detail}")
       end
 

data/lib/html2rss/request_service/httpx_strategy.rb

@@ -21,9 +21,9 @@ module Html2rss
       # combined with ping! on a reused closed selector that never decrements retries,
       # that overflows the stack and kills Falcon workers.
       RETRY_AFTER_SECONDS = 0.25
-      # Bounds passed to the retries plugin and every +session.with+.
-      # HTTPX retries stay inside one {RequestService::Budget} slot and the shared
-      # wall-clock deadline; do not consume a slot per attempt.
+      # Base-session retries plugin defaults (stack-overflow guard for ping reconnects).
+      # Per-request +session.with+ sets +max_retries: 0+; Ruby owns TimeoutError retries
+      # against the monotonic deadline so HTTPX cannot stack read timeouts past the cap.
       # Do not reload +:fiber_concurrency+: a reused closed selector made +ping!+
       # skip decrementing retries and recurse until +SystemStackError+.
       #
@@ -54,19 +54,19 @@ module Html2rss
       def perform_execute
         deadline = request_deadline
         reset_redirect_tracking!
-        raw_response = request_with_terminal_redirect_retry(deadline:)
+        raw_response = request_with_deadline_retries(deadline:)
         build_response(raw_response)
       end
 
       private
 
-      # Caps one HTTPX execute (including internal MAX_RETRIES) so a hanging
-      # :default tier cannot consume the full shared auto-fallback budget.
+      # Caps one strategy execute (Ruby retries included) so a hanging :default tier
+      # cannot consume the full shared auto-fallback budget.
       #
       # @return [Float] monotonic deadline
       def request_deadline
         budget_timeout = ctx.budget.effective_timeout_seconds(fallback: ctx.policy.total_timeout_seconds)
-        monotonic_now + [budget_timeout, attempt_timeout_ceiling].min
+        Process.clock_gettime(Process::CLOCK_MONOTONIC) + [budget_timeout, attempt_timeout_ceiling].min
       end
 
       # @return [Float] connect + read ceiling for a single attempt
@@ -81,8 +81,30 @@ module Html2rss
         @current_url = request_url
       end
 
-      def request_with_terminal_redirect_retry(deadline:)
-        raw_response = execute_http_request(deadline:)
+      # Retries HTTPX timeouts in Ruby while the shared monotonic deadline has time left.
+      # First try consumes the budget slot; retries do not.
+      def request_with_deadline_retries(deadline:)
+        state = { retries_remaining: MAX_RETRIES, consume_budget: true }
+
+        begin
+          request_with_terminal_redirect_retry(deadline:, consume_budget: state[:consume_budget])
+        rescue HTTPX::TimeoutError => error
+          prepare_deadline_retry!(deadline:, state:, error:)
+          retry
+        end
+      end
+
+      def prepare_deadline_retry!(deadline:, state:, error:)
+        raise error if state[:retries_remaining].zero?
+
+        remaining_timeout_seconds(deadline)
+        state[:retries_remaining] -= 1
+        state[:consume_budget] = false
+        reset_redirect_tracking!
+      end
+
+      def request_with_terminal_redirect_retry(deadline:, consume_budget: true)
+        raw_response = execute_http_request(deadline:, consume_budget:)
         return raw_response unless redirect_limit_reached?(raw_response)
 
         unless terminal_redirect_retryable?(raw_response)
@@ -164,6 +186,7 @@ module Html2rss
         session.with(session_options(deadline))
       end
 
+      # Plugin retries stay off per request; Ruby owns deadline-bounded TimeoutError retries.
       def session_options(deadline)
         remaining = remaining_timeout_seconds(deadline)
         {
@@ -173,7 +196,7 @@ module Html2rss
           max_response_body_size: ctx.policy.max_response_bytes,
           resolver_class: :system,
           redirect_on: redirect_callback,
-          **RETRY_OPTIONS
+          **RETRY_OPTIONS.merge(max_retries: 0)
         }
       end
 
@@ -219,16 +242,12 @@ module Html2rss
       end
 
       def remaining_timeout_seconds(deadline)
-        remaining = deadline - monotonic_now
+        remaining = deadline - Process.clock_gettime(Process::CLOCK_MONOTONIC)
         raise RequestTimedOut, 'Request timed out' if remaining <= 0
 
         remaining
       end
 
-      def monotonic_now
-        Process.clock_gettime(Process::CLOCK_MONOTONIC)
-      end
-
       def timeout_error?(error)
         error.is_a?(HTTPX::TimeoutError) || super
       end

data/lib/html2rss/version.rb

@@ -2,6 +2,6 @@
 
 module Html2rss
   # Current application version.
-  VERSION = '0.32.2'
+  VERSION = '0.32.3'
   public_constant :VERSION
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: html2rss
 version: !ruby/object:Gem::Version
-  version: 0.32.2
+  version: 0.32.3
 platform: ruby
 authors:
 - Gil Desmarais
@@ -478,7 +478,7 @@ licenses:
 - MIT
 metadata:
   allowed_push_host: https://rubygems.org
-  changelog_uri: https://github.com/html2rss/html2rss/releases/tag/v0.32.2
+  changelog_uri: https://github.com/html2rss/html2rss/releases/tag/v0.32.3
   rubygems_mfa_required: 'true'
 rdoc_options: []
 require_paths: