html2rss 0.32.3 → 0.32.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8d37540b2f300af8a1db631e16cf6cabf4cfa76069000a71c7723737225bd6c3
-  data.tar.gz: bd8f656ae629587336257d790da183c759b9c709c1ba411e5c0041156d35dd6b
+  metadata.gz: a53d8374b6b4ce7551c9f1bffd0e14960995f7df7380554f1292e0a70317cf58
+  data.tar.gz: 90e91d4275c03c120924cca66230a7aecb3606f2f50c6224d019c138cae244e2
 SHA512:
-  metadata.gz: 802b88abad9bbadfa6753a5c62694a66c6eec3774e84d85191beabdfdd2bc8df3e37fdf325bc14c1d6fda0b90528aca2484df15d0d985b6e328c07f4c27a49b9
-  data.tar.gz: 827d398c34732c10600ced1002525702392448fcb4e8b585f7c27048c8fae38e9c2652307705c8a91e82fba295dbbfa8bfb0cc4dc3a71c4d70cdf9773722a4cf
+  metadata.gz: d2543d4431b2790c89c19ab207d0cdbbc7416493ea1184347746bc26e21068415d68f025e0cc253082224ca5191a3984163e3e48ef3e727cb3b78c19cddd3d64
+  data.tar.gz: ac588fece229b58ab2ea4de91bb467fdbfaf199f43febfd8be6b39d817fc9b9b2867c21cb472949e886ad4dd50b436d9c5c149c0976a87b8cd67d09a6a62f9f6

data/lib/html2rss/error.rb

@@ -3,11 +3,60 @@
 module Html2rss
   # The Html2rss::Error base class.
   class Error < StandardError
+    # Upstream transport_meta keys promoted into error diagnostics.
+    TRANSPORT_META_KEYS = %i[request_id strategy_used render_ms error_category timeout_phase].freeze
+
+    class << self
+      ##
+      # Builds a sparse diagnostics hash from attempt hashes and optional error metadata.
+      #
+      # @param attempts [Array<Hash>, nil] tier attempt diagnostics
+      # @param transport_meta [Hash, nil] direct error transport metadata
+      # @param timeout_phase [String, nil] timeout stage when known
+      # @param surface_category [Symbol, nil] page surface classification when known
+      # @return [Hash{Symbol => Object}]
+      def diagnostics_from_attempts(attempts, transport_meta: nil, timeout_phase: nil, surface_category: nil)
+        list = Array(attempts)
+        meta = list.filter_map { |item| item[:transport_meta] || item['transport_meta'] }.last || transport_meta
+        fields = extract_transport_fields(meta)
+        fields[:timeout_phase] ||= timeout_phase
+        fields[:surface_category] = surface_category
+        { strategy_attempts: (list if list.any?), **fields }.compact
+      end
+
+      private
+
+      def extract_transport_fields(meta)
+        return {} unless meta
+
+        TRANSPORT_META_KEYS.to_h { |key| [key, meta[key] || meta[key.to_s]] }
+      end
+    end
+
     # @return [Array<Hash{Symbol => Object}>, nil] tier attempt diagnostics when set by AutoFallback
     attr_accessor :attempts
 
     # @return [Hash{String => Object}, nil] allowlisted upstream transport telemetry when attached
     attr_accessor :transport_meta
+
+    # @return [String, nil] scrape-api timeout stage when known (overridden by RequestTimedOut)
+    def timeout_phase = nil
+
+    # @return [Symbol, nil] surface classification when known (overridden by NoFeedItemsExtracted)
+    def surface_category = nil
+
+    ##
+    # Sparse telemetry hash for observability consumers.
+    #
+    # @return [Hash{Symbol => Object}]
+    def diagnostics
+      self.class.diagnostics_from_attempts(
+        attempts,
+        transport_meta:,
+        timeout_phase:,
+        surface_category:
+      )
+    end
   end
 
   # Raised when auto fallback exhausts all concrete tiers and extractors find no feed items.

data/lib/html2rss/feed_pipeline/README.md

@@ -37,7 +37,7 @@ Retries share the feed's request/session policy. Pin a concrete strategy when yo
 
 ### Terminal fallback outcome
 
-A hop that returns zero items does **not** erase an earlier terminal transport failure (`RequestTimedOut` or `BlockedSurfaceDetected`). `AutoFallback` remembers that error across later hops and, when no later hop yields items, re-raises it with `attempts` attached. A later hop that extracts items still succeeds. Two zero-item hops with no terminal transport error still raise `NoFeedItemsExtracted`. `html2rss-web` `ErrorClassifier` maps the raised error to HTTP/cache policy; it does not invent a second cache rule for empty extraction.
+A hop that returns zero items does **not** erase an earlier terminal transport failure (`RequestTimedOut`, `BlockedSurfaceDetected`, or `ConnectionFailed`). `AutoFallback` remembers that error across later hops and, when no later hop yields items, re-raises it with `attempts` attached. A later hop that extracts items still succeeds. Two zero-item hops with no terminal transport error still raise `NoFeedItemsExtracted`. True empty stays the web policy already in force: listing `EXTRACTION_EMPTY` is cacheable; `app_shell` and `high_entropy_surface` are not. `html2rss-web` `ErrorClassifier` maps the raised error to HTTP/cache policy; it does not invent a second cache rule for empty extraction.
 
 ## Success signals
 

data/lib/html2rss/feed_pipeline/auto_fallback.rb

@@ -32,7 +32,8 @@ module Html2rss
       # decisions instead of EXTRACTION_EMPTY.
       TERMINAL_TRANSPORT_ERRORS = [
         RequestService::RequestTimedOut,
-        RequestService::BlockedSurfaceDetected
+        RequestService::BlockedSurfaceDetected,
+        RequestService::ConnectionFailed
       ].freeze
 
       ##
@@ -79,12 +80,11 @@ module Html2rss
 
         # @param strategy [Symbol] strategy that raised
         # @param error [Exception] caught error
+        # @param elapsed_ms [Integer, nil] monotonic hop duration in milliseconds
         # @return [void]
-        def record_error(strategy:, error:)
-          attempt = { strategy:, items_count: nil, error_class: error.class.name }
+        def record_error(strategy:, error:, elapsed_ms: nil)
           meta = error.is_a?(Html2rss::Error) ? error.transport_meta : nil
-          attempt[:transport_meta] = meta if meta&.any?
-          @attempts << attempt
+          append_attempt(strategy:, error_class: error.class.name, transport_meta: meta, elapsed_ms:)
           @last_error = error
           return unless TERMINAL_TRANSPORT_ERRORS.any? { |klass| error.is_a?(klass) }
 
@@ -94,12 +94,11 @@ module Html2rss
         # @param strategy [Symbol] strategy that returned a response
         # @param items_count [Integer] extracted article count
         # @param transport_meta [Hash, nil] optional allowlisted upstream telemetry
+        # @param elapsed_ms [Integer, nil] monotonic hop duration in milliseconds
         # @return [void]
-        def record_items(strategy:, items_count:, transport_meta: nil)
+        def record_items(strategy:, items_count:, transport_meta: nil, elapsed_ms: nil)
           @last_error = nil
-          attempt = { strategy:, items_count:, error_class: nil }
-          attempt[:transport_meta] = transport_meta if transport_meta&.any?
-          @attempts << attempt
+          append_attempt(strategy:, items_count:, transport_meta:, elapsed_ms:)
         end
 
         # @param response [RequestService::Response] last response that reached extraction
@@ -131,6 +130,37 @@ module Html2rss
             entry_resolution:
           )
         end
+
+        # Raises the terminal transport error, last error, or NoFeedItemsExtracted with attempts attached.
+        #
+        # @return [void] always raises
+        # @raise [Html2rss::Error, StandardError]
+        def raise_failure!
+          if (error = failure_error)
+            error.attempts = attempts if error.is_a?(Html2rss::Error)
+            raise error
+          end
+
+          response = @last_response
+          surface_category = PageRecon.surface_category_for(response:, url: response.url) if response
+          raise NoFeedItemsExtracted.new(attempts:, surface_category:)
+        end
+
+        private
+
+        def append_attempt(strategy:, items_count: nil, error_class: nil, transport_meta: nil, elapsed_ms: nil)
+          attempt = { strategy:, items_count:, error_class: }
+          attempt[:elapsed_ms] = elapsed_ms unless elapsed_ms.nil?
+          attempt[:transport_meta] = transport_meta if transport_meta&.any?
+          @attempts << attempt
+        end
+
+        def failure_error
+          return @terminal_transport_error if @terminal_transport_error
+          return if @last_error.nil? || @last_error.is_a?(RequestService::BotasaurusConfigurationError)
+
+          @last_error
+        end
       end
 
       ##
@@ -155,7 +185,7 @@ module Html2rss
         state = run_attempts
         return state.result if state.succeeded?
 
-        finalize_failure(state)
+        state.raise_failure!
       end
 
       private
@@ -171,37 +201,33 @@ module Html2rss
         end
       end
 
+      # rubocop:disable-next Metrics/MethodLength, Metrics/AbcSize -- hop timing + NON_FALLBACK rescue stay co-located
       def attempt(strategy:, next_strategy:, state:)
+        started_ms = monotonic_ms
         request_session = pipeline.request_session_for(
           config, strategy:, resources:, scrape_url: scrape_target.effective_url
         )
-        response = fetch_response(request_session:, strategy:, next_strategy:, state:)
-        return unless response
-
-        process_response(response:, strategy:, next_strategy:, request_session:, state:)
-      rescue RequestService::UnsupportedResponseContentType => error
-        state.record_error(strategy:, error:)
-        log_fallback_error(strategy:, next_strategy:, error:, request_session:) if next_strategy
-      end
-
-      def fetch_response(request_session:, strategy:, next_strategy:, state:)
-        request_session.fetch_initial_response
-      rescue *NON_FALLBACK_ERRORS
-        raise
+        response = request_session.fetch_initial_response
+        process_response(
+          response:, strategy:, next_strategy:, request_session:, state:,
+          elapsed_ms: monotonic_ms - started_ms
+        )
+      rescue *NON_FALLBACK_ERRORS => error
+        state.record_error(strategy:, error:, elapsed_ms: monotonic_ms - started_ms)
+        state.abort!
       rescue StandardError => error
-        state.record_error(strategy:, error:)
-        log_fallback_error(strategy:, next_strategy:, error:, request_session:) if next_strategy
-        nil
+        state.record_error(strategy:, error:, elapsed_ms: monotonic_ms - started_ms)
+        log_fallback_error(strategy:, next_strategy:, error:, request_session:) if next_strategy && request_session
       end
 
-      # rubocop:disable-next Metrics/MethodLength, Metrics/AbcSize -- extract + resolution delegate
-      def process_response(response:, strategy:, next_strategy:, request_session:, state:)
+      # rubocop:disable-next Metrics/MethodLength, Metrics/AbcSize, Metrics/ParameterLists -- extract + resolution delegate
+      def process_response(response:, strategy:, next_strategy:, request_session:, state:, elapsed_ms: nil)
         articles, dedup_dropped, admission_drops = pipeline.deduplicated_articles(
           config:, response:, request_session:
         )
         state.remember_response(response)
         items_count = articles.size
-        state.record_items(strategy:, items_count:, transport_meta: response.transport_meta)
+        state.record_items(strategy:, items_count:, transport_meta: response.transport_meta, elapsed_ms:)
         Log.debug("#{self.class}: strategy=#{strategy} items=#{items_count} " \
                   "host=#{response.url.host} elapsed=#{format('%.3f', budget.elapsed_seconds)}s " \
                   "budget_remaining=#{budget_remaining_label}")
@@ -249,30 +275,7 @@ module Html2rss
         }.to_json)
       end
 
-      def finalize_failure(state)
-        error = failure_error(state)
-        if error
-          error.attempts = state.attempts if error.is_a?(Html2rss::Error)
-          raise error
-        end
-
-        response = state.last_response
-        raise NoFeedItemsExtracted.new(
-          attempts: state.attempts,
-          surface_category: (PageRecon.surface_category_for(response:, url: response.url) if response)
-        )
-      end
-
-      # Prefer a remembered timeout/blocked-surface over a later zero-item hop so
-      # callers do not cache EXTRACTION_EMPTY for a transport failure.
-      def failure_error(state)
-        return state.terminal_transport_error if state.terminal_transport_error
-
-        error = state.last_error
-        return if error.nil? || error.is_a?(RequestService::BotasaurusConfigurationError)
-
-        error
-      end
+      def monotonic_ms = Process.clock_gettime(Process::CLOCK_MONOTONIC, :millisecond)
 
       # rubocop:disable-next Metrics/AbcSize, Metrics/MethodLength -- shared payload then info/warn dispatch
       def log_fallback_error(strategy:, next_strategy:, error:, request_session:)

data/lib/html2rss/request_service/local_file_strategy.rb

@@ -15,6 +15,9 @@ module Html2rss
 
       def skip_postflight? = true
 
+      # Local files never open a socket; keep filesystem errors as wire classes.
+      def connection_error?(_error) = false
+
       ##
       # @return [Response] the mock response wrapped around the file contents
       # @raise [ArgumentError] if the local file path is missing

data/lib/html2rss/request_service/strategy.rb

@@ -133,9 +133,9 @@ module Html2rss
 
       # @param error [StandardError]
       # @return [void]
-      # @raise [StandardError]
+      # @raise [ConnectionFailed]
       def translate_connection_error(error)
-        raise error
+        raise ConnectionFailed, error.message
       end
 
       # @param error [StandardError]

data/lib/html2rss/request_service.rb

@@ -28,6 +28,8 @@ module Html2rss
     class RedirectLimitReached < Html2rss::Error; end
     # Raised when blocked content surfaces are detected.
     class BlockedSurfaceDetected < Html2rss::Error; end
+    # Raised when the target connection fails before a response document exists.
+    class ConnectionFailed < Html2rss::Error; end
 
     # Raised when a request times out.
     class RequestTimedOut < Html2rss::Error
@@ -76,6 +78,12 @@ module Html2rss
     # @return [Symbol]
     DEFAULT_STRATEGY_NAME = :default
 
+    # Maximum distinct [strategy, host] deprecation warnings tracked per process.
+    MAX_DEPRECATION_WARNINGS = 256
+    # Mutex guarding process-local deprecation warning deduplication.
+    DEPRECATION_MUTEX = Mutex.new
+    private_constant :DEPRECATION_MUTEX
+
     class << self
       # @return [Symbol] the default strategy name
       def default_strategy_name = DEFAULT_STRATEGY_NAME
@@ -107,6 +115,13 @@ module Html2rss
         strategy_class.new(ctx).execute
       end
 
+      ##
+      # Clears process-local deprecation warning state (used by specs).
+      # @return [void]
+      def reset_deprecation_warnings!
+        DEPRECATION_MUTEX.synchronize { @warned_deprecations = Set.new }
+      end
+
       private
 
       ##
@@ -125,14 +140,15 @@ module Html2rss
       end
 
       ##
-      # Emits a structured +strategy_deprecated+ warning once per session (+ :initial + only).
+      # Emits a structured +strategy_deprecated+ warning once per +(strategy, host)+ per process
+      # on initial session requests (+relation == :initial+). Multi-worker servers may emit once per worker.
       # @param strategy [Symbol] deprecated strategy alias
       # @param replacement [Symbol] preferred strategy name
       # @param relation [Symbol] request relation
       # @param host [String, nil] request host
       # @return [void]
       def warn_migration_strategy(strategy, replacement, relation:, host:)
-        return unless relation == :initial
+        return unless relation == :initial && claim_deprecation_warning?(strategy, host)
 
         Log.warn({
           event_name: 'strategy_deprecated',
@@ -142,6 +158,14 @@ module Html2rss
           message: "#{name}: strategy=#{strategy.inspect} is deprecated; using #{replacement.inspect} instead."
         }.to_json)
       end
+
+      def claim_deprecation_warning?(strategy, host)
+        DEPRECATION_MUTEX.synchronize do
+          @warned_deprecations ||= Set.new
+          @warned_deprecations.clear if @warned_deprecations.size >= MAX_DEPRECATION_WARNINGS
+          !@warned_deprecations.add?([strategy, host]).nil?
+        end
+      end
     end
   end
 end

data/lib/html2rss/version.rb

@@ -2,6 +2,6 @@
 
 module Html2rss
   # Current application version.
-  VERSION = '0.32.3'
+  VERSION = '0.32.4'
   public_constant :VERSION
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: html2rss
 version: !ruby/object:Gem::Version
-  version: 0.32.3
+  version: 0.32.4
 platform: ruby
 authors:
 - Gil Desmarais
@@ -478,7 +478,7 @@ licenses:
 - MIT
 metadata:
   allowed_push_host: https://rubygems.org
-  changelog_uri: https://github.com/html2rss/html2rss/releases/tag/v0.32.3
+  changelog_uri: https://github.com/html2rss/html2rss/releases/tag/v0.32.4
   rubygems_mfa_required: 'true'
 rdoc_options: []
 require_paths: