html2rss 0.32.3 → 0.33.0

data/lib/html2rss/error.rb

@@ -3,13 +3,65 @@
 module Html2rss
   # The Html2rss::Error base class.
   class Error < StandardError
+    # Upstream transport_meta keys promoted into error diagnostics.
+    TRANSPORT_META_KEYS = %i[request_id strategy_used render_ms error_category timeout_phase].freeze
+
+    class << self
+      ##
+      # Builds a sparse diagnostics hash from attempt hashes and optional error metadata.
+      #
+      # @param attempts [Array<Hash>, nil] tier attempt diagnostics
+      # @param transport_meta [Hash, nil] direct error transport metadata
+      # @param timeout_phase [String, nil] timeout stage when known
+      # @param surface_category [Symbol, nil] page surface classification when known
+      # @return [Hash{Symbol => Object}]
+      def diagnostics_from_attempts(attempts, transport_meta: nil, timeout_phase: nil, surface_category: nil)
+        list = Array(attempts)
+        meta = list.filter_map { |item| item[:transport_meta] || item['transport_meta'] }.last || transport_meta
+        fields = extract_transport_fields(meta)
+        fields[:timeout_phase] ||= timeout_phase
+        fields[:surface_category] = surface_category
+        { strategy_attempts: (list if list.any?), **fields }.compact
+      end
+
+      private
+
+      def extract_transport_fields(meta)
+        return {} unless meta
+
+        TRANSPORT_META_KEYS.to_h { |key| [key, meta[key] || meta[key.to_s]] }
+      end
+    end
+
     # @return [Array<Hash{Symbol => Object}>, nil] tier attempt diagnostics when set by AutoFallback
     attr_accessor :attempts
 
     # @return [Hash{String => Object}, nil] allowlisted upstream transport telemetry when attached
     attr_accessor :transport_meta
+
+    # @return [String, nil] scrape-api timeout stage when known (overridden by RequestTimedOut)
+    def timeout_phase; end
+
+    # @return [Symbol, nil] surface classification when known (overridden by NoFeedItemsExtracted)
+    def surface_category; end
+
+    ##
+    # Sparse telemetry hash for observability consumers.
+    #
+    # @return [Hash{Symbol => Object}]
+    def diagnostics
+      self.class.diagnostics_from_attempts(
+        attempts,
+        transport_meta:,
+        timeout_phase:,
+        surface_category:
+      )
+    end
   end
 
+  # Raised when a caller-supplied option value is malformed or out of range.
+  class InvalidOption < Error; end
+
   # Raised when auto fallback exhausts all concrete tiers and extractors find no feed items.
   class NoFeedItemsExtracted < Error
     # Categories that append shared surface guidance to the empty-feed message.
@@ -38,7 +90,8 @@ module Html2rss
 
     def base_message
       summaries = attempts.map do |attempt|
-        details = attempt[:items_count].nil? ? "#{attempt[:error_class]} error" : "#{attempt[:items_count]} items"
+        count = attempt[:items_count]
+        details = count.nil? ? "#{attempt[:error_class]} error" : "#{count} items"
         "#{attempt[:strategy]} (#{details})"
       end.join(', ')
 

data/lib/html2rss/feed_builder/rss.rb

@@ -37,7 +37,7 @@ module Html2rss
           item_maker.title = article.title
           item_maker.author = article.author
           description = ItemPresentation.description_for(article)
-          item_maker.description = description if description && !description.empty?
+          item_maker.description = description if description
           item_maker.link = article.url.to_s if article.url
           item_maker.pubDate = article.published_at&.rfc2822
         end

data/lib/html2rss/feed_pipeline/README.md

@@ -37,7 +37,7 @@ Retries share the feed's request/session policy. Pin a concrete strategy when yo
 
 ### Terminal fallback outcome
 
-A hop that returns zero items does **not** erase an earlier terminal transport failure (`RequestTimedOut` or `BlockedSurfaceDetected`). `AutoFallback` remembers that error across later hops and, when no later hop yields items, re-raises it with `attempts` attached. A later hop that extracts items still succeeds. Two zero-item hops with no terminal transport error still raise `NoFeedItemsExtracted`. `html2rss-web` `ErrorClassifier` maps the raised error to HTTP/cache policy; it does not invent a second cache rule for empty extraction.
+A hop that returns zero items does **not** erase an earlier terminal transport failure (`RequestTimedOut`, `BlockedSurfaceDetected`, or `ConnectionFailed`). `AutoFallback` remembers that error across later hops and, when no later hop yields items, re-raises it with `attempts` attached. A later hop that extracts items still succeeds. Two zero-item hops with no terminal transport error still raise `NoFeedItemsExtracted`. True empty stays the web policy already in force: listing `EXTRACTION_EMPTY` is cacheable; `app_shell` and `high_entropy_surface` are not. `html2rss-web` `ErrorClassifier` maps the raised error to HTTP/cache policy; it does not invent a second cache rule for empty extraction.
 
 ## Success signals
 

data/lib/html2rss/feed_pipeline/auto_fallback.rb

@@ -32,7 +32,8 @@ module Html2rss
       # decisions instead of EXTRACTION_EMPTY.
       TERMINAL_TRANSPORT_ERRORS = [
         RequestService::RequestTimedOut,
-        RequestService::BlockedSurfaceDetected
+        RequestService::BlockedSurfaceDetected,
+        RequestService::ConnectionFailed
       ].freeze
 
       ##
@@ -79,12 +80,11 @@ module Html2rss
 
         # @param strategy [Symbol] strategy that raised
         # @param error [Exception] caught error
+        # @param elapsed_ms [Integer, nil] monotonic hop duration in milliseconds
         # @return [void]
-        def record_error(strategy:, error:)
-          attempt = { strategy:, items_count: nil, error_class: error.class.name }
-          meta = error.is_a?(Html2rss::Error) ? error.transport_meta : nil
-          attempt[:transport_meta] = meta if meta&.any?
-          @attempts << attempt
+        def record_error(strategy:, error:, elapsed_ms: nil)
+          meta = error.transport_meta if error.is_a?(Error)
+          append_attempt(strategy:, error_class: error.class.name, transport_meta: meta, elapsed_ms:)
           @last_error = error
           return unless TERMINAL_TRANSPORT_ERRORS.any? { |klass| error.is_a?(klass) }
 
@@ -94,12 +94,11 @@ module Html2rss
         # @param strategy [Symbol] strategy that returned a response
         # @param items_count [Integer] extracted article count
         # @param transport_meta [Hash, nil] optional allowlisted upstream telemetry
+        # @param elapsed_ms [Integer, nil] monotonic hop duration in milliseconds
         # @return [void]
-        def record_items(strategy:, items_count:, transport_meta: nil)
+        def record_items(strategy:, items_count:, transport_meta: nil, elapsed_ms: nil)
           @last_error = nil
-          attempt = { strategy:, items_count:, error_class: nil }
-          attempt[:transport_meta] = transport_meta if transport_meta&.any?
-          @attempts << attempt
+          append_attempt(strategy:, items_count:, transport_meta:, elapsed_ms:)
         end
 
         # @param response [RequestService::Response] last response that reached extraction
@@ -131,6 +130,37 @@ module Html2rss
             entry_resolution:
           )
         end
+
+        # Raises the terminal transport error, last error, or NoFeedItemsExtracted with attempts attached.
+        #
+        # @return [void] always raises
+        # @raise [Html2rss::Error, StandardError]
+        def raise_failure!
+          if (error = failure_error)
+            error.attempts = attempts if error.is_a?(Error)
+            raise error
+          end
+
+          response = @last_response
+          surface_category = PageRecon.surface_category_for(response:, url: response.url) if response
+          raise NoFeedItemsExtracted.new(attempts:, surface_category:)
+        end
+
+        private
+
+        def append_attempt(strategy:, items_count: nil, error_class: nil, transport_meta: nil, elapsed_ms: nil)
+          attempt = { strategy:, items_count:, error_class: }
+          attempt[:elapsed_ms] = elapsed_ms unless elapsed_ms.nil?
+          attempt[:transport_meta] = transport_meta if transport_meta&.any?
+          @attempts << attempt
+        end
+
+        def failure_error
+          return @terminal_transport_error if @terminal_transport_error
+          return if @last_error.nil? || @last_error.is_a?(RequestService::BotasaurusConfigurationError)
+
+          @last_error
+        end
       end
 
       ##
@@ -155,7 +185,7 @@ module Html2rss
         state = run_attempts
         return state.result if state.succeeded?
 
-        finalize_failure(state)
+        state.raise_failure!
       end
 
       private
@@ -171,37 +201,33 @@ module Html2rss
         end
       end
 
+      # rubocop:disable-next Metrics/MethodLength, Metrics/AbcSize -- hop timing + NON_FALLBACK rescue stay co-located
       def attempt(strategy:, next_strategy:, state:)
+        started_ms = monotonic_ms
         request_session = pipeline.request_session_for(
           config, strategy:, resources:, scrape_url: scrape_target.effective_url
         )
-        response = fetch_response(request_session:, strategy:, next_strategy:, state:)
-        return unless response
-
-        process_response(response:, strategy:, next_strategy:, request_session:, state:)
-      rescue RequestService::UnsupportedResponseContentType => error
-        state.record_error(strategy:, error:)
-        log_fallback_error(strategy:, next_strategy:, error:, request_session:) if next_strategy
-      end
-
-      def fetch_response(request_session:, strategy:, next_strategy:, state:)
-        request_session.fetch_initial_response
-      rescue *NON_FALLBACK_ERRORS
-        raise
-      rescue StandardError => error
-        state.record_error(strategy:, error:)
-        log_fallback_error(strategy:, next_strategy:, error:, request_session:) if next_strategy
-        nil
+        response = request_session.fetch_initial_response
+        process_response(
+          response:, strategy:, next_strategy:, request_session:, state:,
+          elapsed_ms: monotonic_ms - started_ms
+        )
+      rescue *NON_FALLBACK_ERRORS => error
+        state.record_error(strategy:, error:, elapsed_ms: monotonic_ms - started_ms)
+        state.abort!
+      rescue Html2rss::Error => error
+        state.record_error(strategy:, error:, elapsed_ms: monotonic_ms - started_ms)
+        log_fallback_error(strategy:, next_strategy:, error:, request_session:) if next_strategy && request_session
       end
 
-      # rubocop:disable-next Metrics/MethodLength, Metrics/AbcSize -- extract + resolution delegate
-      def process_response(response:, strategy:, next_strategy:, request_session:, state:)
+      # rubocop:disable-next Metrics/MethodLength, Metrics/AbcSize, Metrics/ParameterLists -- extract + resolution delegate
+      def process_response(response:, strategy:, next_strategy:, request_session:, state:, elapsed_ms: nil)
         articles, dedup_dropped, admission_drops = pipeline.deduplicated_articles(
           config:, response:, request_session:
         )
         state.remember_response(response)
         items_count = articles.size
-        state.record_items(strategy:, items_count:, transport_meta: response.transport_meta)
+        state.record_items(strategy:, items_count:, transport_meta: response.transport_meta, elapsed_ms:)
         Log.debug("#{self.class}: strategy=#{strategy} items=#{items_count} " \
                   "host=#{response.url.host} elapsed=#{format('%.3f', budget.elapsed_seconds)}s " \
                   "budget_remaining=#{budget_remaining_label}")
@@ -249,30 +275,7 @@ module Html2rss
         }.to_json)
       end
 
-      def finalize_failure(state)
-        error = failure_error(state)
-        if error
-          error.attempts = state.attempts if error.is_a?(Html2rss::Error)
-          raise error
-        end
-
-        response = state.last_response
-        raise NoFeedItemsExtracted.new(
-          attempts: state.attempts,
-          surface_category: (PageRecon.surface_category_for(response:, url: response.url) if response)
-        )
-      end
-
-      # Prefer a remembered timeout/blocked-surface over a later zero-item hop so
-      # callers do not cache EXTRACTION_EMPTY for a transport failure.
-      def failure_error(state)
-        return state.terminal_transport_error if state.terminal_transport_error
-
-        error = state.last_error
-        return if error.nil? || error.is_a?(RequestService::BotasaurusConfigurationError)
-
-        error
-      end
+      def monotonic_ms = Process.clock_gettime(Process::CLOCK_MONOTONIC, :millisecond)
 
       # rubocop:disable-next Metrics/AbcSize, Metrics/MethodLength -- shared payload then info/warn dispatch
       def log_fallback_error(strategy:, next_strategy:, error:, request_session:)

data/lib/html2rss/feed_resolution/candidate_generator.rb

@@ -50,7 +50,6 @@ module Html2rss
         return unless parsed_html
 
         url = Syndication::Discovery.candidate_urls(page_url: entry_url, parsed_body: parsed_html).first
-        return unless url
         return unless eligible_url?(url)
 
         url
@@ -86,6 +85,8 @@ module Html2rss
         return [] unless parsed_html
 
         sst = SST::Normalizer.call(parsed_html)
+        return [] unless sst
+
         link_resolver = Scoring::LinkResolver.new(entry_url)
 
         SEGMENT_STRATEGIES.each do |strategy|
@@ -93,8 +94,6 @@ module Html2rss
           return urls if urls.size >= 2
         end
         []
-      rescue StandardError
-        []
       end
 
       def segment_urls_for(sst, strategy, link_resolver)
@@ -120,7 +119,7 @@ module Html2rss
         return if href.nil? || href.empty? || href.start_with?('#', 'mailto:', 'javascript:')
 
         Html2rss::Url.from_relative(href, entry_url)
-      rescue ArgumentError
+      rescue Html2rss::Url::InvalidUrlError
         nil
       end
 

data/lib/html2rss/feed_resolution/options.rb

@@ -8,15 +8,11 @@ module Html2rss
     # Typed +auto_source.entry_resolution+ options (one expansion from config Hash).
     class Options
       ##
-      # @param auto_source [Hash, nil]
+      # @param auto_source [Hash, nil] validated +auto_source+ config
       # @return [Options]
       def self.from_auto_source(auto_source)
-        raw = auto_source.is_a?(Hash) ? auto_source[:entry_resolution] : nil
-        raw = {} unless raw.is_a?(Hash)
-        new(
-          enabled: raw.fetch(:enabled, FeedResolution::DEFAULT_CONFIG[:enabled]) != false,
-          max_probes: Integer(raw.fetch(:max_probes, FeedResolution::DEFAULT_CONFIG[:max_probes]))
-        )
+        raw = auto_source&.dig(:entry_resolution) || {}
+        new(enabled: raw.fetch(:enabled, true), max_probes: raw.fetch(:max_probes, DEFAULT_CONFIG[:max_probes]))
       end
 
       ##

data/lib/html2rss/feed_resolution/policy.rb

@@ -8,40 +8,36 @@ module Html2rss
       # Minimum admitted articles before the entry URL is considered strong enough.
       ARTICLE_FLOOR = 3
 
-      module_function
-
-      ##
-      # @param config [Html2rss::Config]
-      # @param articles [Array<Html2rss::Article>] typed extract results (count derived)
-      # @param surface_category [Html2rss::SurfaceCategory, Symbol]
-      # @return [Boolean]
-      def resolve?(config:, articles:, surface_category:)
-        raise ArgumentError, 'articles must be an Array' unless articles.is_a?(Array)
-        return false unless eligible_config?(config)
-
-        category = SurfaceCategory[surface_category]
-        return false if category.blocked?
-
-        articles.size < ARTICLE_FLOOR || category.weak? || native_feed_majority?(articles)
-      end
-
-      def eligible_config?(config)
-        return false unless config.auto_source
-        return false if config.selectors
-
-        Options.from_auto_source(config.auto_source).enabled?
-      end
-      module_function :eligible_config?
-      private_class_method :eligible_config?
-
-      def native_feed_majority?(articles)
-        return false if articles.empty?
-
-        native = articles.count { |article| article.scraper == AutoSource::Scraper::NativeFeed }
-        native * 2 >= articles.size
+      class << self
+        ##
+        # @param config [Html2rss::Config]
+        # @param articles [Array<Html2rss::Article>] typed extract results (count derived)
+        # @param surface_category [Html2rss::SurfaceCategory, Symbol]
+        # @return [Boolean]
+        def resolve?(config:, articles:, surface_category:)
+          raise ArgumentError, 'articles must be an Array' unless articles.is_a?(Array)
+          return false unless eligible_config?(config)
+
+          category = SurfaceCategory[surface_category]
+          return false if category.blocked?
+
+          articles.size < ARTICLE_FLOOR || category.weak? || native_feed_majority?(articles)
+        end
+
+        private
+
+        def eligible_config?(config)
+          return false unless config.auto_source
+          return false if config.selectors
+
+          Options.from_auto_source(config.auto_source).enabled?
+        end
+
+        def native_feed_majority?(articles)
+          native = articles.count { |article| article.scraper == AutoSource::Scraper::NativeFeed }
+          native * 2 >= articles.size
+        end
       end
-      module_function :native_feed_majority?
-      private_class_method :native_feed_majority?
     end
   end
 end

data/lib/html2rss/feed_resolution/probe.rb

@@ -16,7 +16,7 @@ module Html2rss
       # @param origin_url [String, Html2rss::Url]
       def initialize(request_session:, origin_url:)
         @request_session = request_session
-        @origin_url = Html2rss::Url.from_absolute(origin_url)
+        @origin_url = Url.from_absolute(origin_url)
       end
 
       ##
@@ -28,7 +28,7 @@ module Html2rss
           relation: :entry_resolution,
           origin_url:
         )
-        score_response(url, response)
+        score_response(response)
       rescue Html2rss::Error
         nil
       end
@@ -37,7 +37,7 @@ module Html2rss
 
       attr_reader :request_session, :origin_url
 
-      def score_response(_url, response) # rubocop:disable Metrics/MethodLength -- feed vs HTML branches
+      def score_response(response) # rubocop:disable Metrics/MethodLength -- feed vs HTML branches
         if response.feed_response?
           count = Syndication::Parser.parse_response(response).size
           return Scored.new(

data/lib/html2rss/feed_resolution/scorer.rb

@@ -11,38 +11,38 @@ module Html2rss
       # Bonus when the surface looks like a listing rather than a hub/shell.
       LISTING_SURFACE_BONUS = 5
 
-      module_function
+      class << self
+        ##
+        # @param articles_count [Integer]
+        # @return [Integer]
+        def score_feed(articles_count:)
+          articles_count * ARTICLE_WEIGHT
+        end
 
-      ##
-      # @param articles_count [Integer]
-      # @return [Integer]
-      def score_feed(articles_count:)
-        articles_count.to_i * ARTICLE_WEIGHT
-      end
-
-      ##
-      # @param assessment [Html2rss::PageRecon::Assessment]
-      # @return [Integer]
-      def score_assessment(assessment)
-        score = assessment.articles_count * ARTICLE_WEIGHT
-        score += LISTING_SURFACE_BONUS if assessment.listing_bonus?
-        drops = assessment.admission_drops.values.sum
-        total = assessment.articles_count + drops
-        score -= ((drops.to_f / total) * ARTICLE_WEIGHT).round if total.positive?
-        score
-      end
+        ##
+        # @param assessment [Html2rss::PageRecon::Assessment]
+        # @return [Integer]
+        def score_assessment(assessment)
+          score = assessment.articles_count * ARTICLE_WEIGHT
+          score += LISTING_SURFACE_BONUS if assessment.listing_bonus?
+          drops = assessment.admission_drops.values.sum
+          total = assessment.articles_count + drops
+          score -= ((drops.to_f / total) * ARTICLE_WEIGHT).round if total.positive?
+          score
+        end
 
-      ##
-      # @param scored [Array<Probe::Scored>]
-      # @param entry_articles_count [Integer]
-      # @return [Probe::Scored, nil]
-      def pick_winner(scored:, entry_articles_count:)
-        winner = scored.max_by(&:score)
-        return unless winner
-        return if winner.articles_count <= entry_articles_count.to_i
-        return if winner.score <= 0
+        ##
+        # @param scored [Array<Probe::Scored>]
+        # @param entry_articles_count [Integer]
+        # @return [Probe::Scored, nil]
+        def pick_winner(scored:, entry_articles_count:)
+          winner = scored.max_by(&:score)
+          return unless winner
+          return if winner.articles_count <= entry_articles_count.to_i
+          return if winner.score <= 0
 
-        winner
+          winner
+        end
       end
     end
   end

data/lib/html2rss/feed_resolution.rb

@@ -116,17 +116,13 @@ module Html2rss
         return unless resolution.applied
 
         effective = scrape_target.with_effective(resolution.scrape_url)
-        return ApplyOutcome.new(scrape_target: effective, status: :succeeded) if retry_with(
-          resolution, effective:
-        )
-
-        nil
+        ApplyOutcome.new(scrape_target: effective, status: :succeeded) if retry_with(effective:)
       rescue RequestService::RequestBudgetExceeded
         log_budget_exhausted
         nil
       rescue *NON_FALLBACK_ERRORS
         raise
-      rescue StandardError => error
+      rescue Html2rss::Error => error
         log_retry_failed(error)
         nil
       end
@@ -158,7 +154,7 @@ module Html2rss
       # @return [String, nil]
       def channel_host
         Url.from_absolute(config.url.to_s).host
-      rescue ArgumentError
+      rescue Html2rss::Url::InvalidUrlError
         nil
       end
 
@@ -184,7 +180,7 @@ module Html2rss
         )
       end
 
-      def retry_with(_resolution, effective:) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength -- retry extract path
+      def retry_with(effective:) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength -- retry extract path
         retry_session = pipeline.request_session_for(
           config, strategy:, resources:, scrape_url: effective.effective_url
         )
@@ -220,7 +216,7 @@ module Html2rss
       def initialize(entry_url:, response:, session:, config:, articles:, surface_category: nil)
         raise ArgumentError, 'articles must be an Array' unless articles.is_a?(Array)
 
-        @entry_url = Html2rss::Url.from_absolute(entry_url)
+        @entry_url = Url.from_absolute(entry_url)
         @response = response
         @session = session
         @config = config

data/lib/html2rss/feed_result.rb

@@ -23,8 +23,7 @@ module Html2rss
     def initialize(channel:, articles:, status:, stylesheets: [])
       raise ArgumentError, 'channel must be a Html2rss::Channel' unless channel.is_a?(Channel)
       raise ArgumentError, 'status must be a Html2rss::Status' unless status.is_a?(Status)
-
-      articles = Array(articles)
+      raise ArgumentError, 'articles must be an Array' unless articles.is_a?(Array)
       raise ArgumentError, 'articles must all be Html2rss::Article' unless articles.all?(Article)
 
       @channel = channel
@@ -82,7 +81,9 @@ module Html2rss
     end
 
     def freeze_stylesheets(stylesheets)
-      Array(stylesheets).map do |sheet|
+      raise ArgumentError, 'stylesheets must be an Array' unless stylesheets.is_a?(Array)
+
+      stylesheets.map do |sheet|
         sheet.to_h.transform_keys(&:to_sym).transform_values do |value|
           value.is_a?(String) ? value.dup.freeze : value
         end.freeze