html2rss 0.32.4 → 0.34.0

data/lib/html2rss/test.rb

@@ -5,8 +5,6 @@ module Html2rss
   # Service that runs schema validation and live feed extraction on a configuration,
   # enforcing minimum item thresholds and capturing sample output.
   module Test # rubocop:disable Metrics/ModuleLength -- Result + FailureKind nest with owner
-    module_function
-
     FailureKind = Data.define(:name)
 
     ##
@@ -19,7 +17,7 @@ module Html2rss
       ##
       # @param name [Symbol, String]
       # @return [FailureKind]
-      def self.[](name) = ALL.fetch(name.to_sym) { raise ArgumentError, "unknown failure kind: #{name.inspect}" }
+      def self.[](name) = ALL.fetch(name.to_sym) { raise InvalidOption, "unknown failure kind: #{name.inspect}" }
 
       ##
       # @return [Boolean]
@@ -80,6 +78,7 @@ module Html2rss
       :success,
       :item_count,
       :sample_items,
+      :items,
       :channel_title,
       :channel_url,
       :strategy_used,
@@ -102,6 +101,8 @@ module Html2rss
       # @param success [Boolean]
       # @param item_count [Integer]
       # @param sample_items [Array<Hash>]
+      # @param items [Array<Hash>] up to +sample_limit+ items (title, url, published_at, description, image,
+      #   categories) built from extracted articles regardless of the quality gate
       # @param channel_title [String, nil]
       # @param channel_url [String, nil]
       # @param strategy_used [Symbol, nil]
@@ -115,7 +116,7 @@ module Html2rss
       # @param response_body [String, nil] fetched page HTML copied from the pipeline response
       def initialize(success:, item_count:, sample_items:, channel_title:, channel_url:, # rubocop:disable Metrics/ParameterLists
                      strategy_used:, duration_seconds:, validation_issues:, error_message:,
-                     failure_kind:, rss:, quality_report: nil, enhance_compare: nil, response_body: nil)
+                     failure_kind:, rss:, items: [], quality_report: nil, enhance_compare: nil, response_body: nil)
         super
       end
 
@@ -138,6 +139,7 @@ module Html2rss
           success:,
           item_count:,
           sample_items:,
+          items:,
           channel_title:,
           channel_url:,
           strategy_used:,
@@ -162,28 +164,31 @@ module Html2rss
     # @param strategy [Symbol, nil] optional strategy override
     # @param strict_quality [Boolean] when true, fail on ship-quality audit thresholds
     # @param compare_enhance [Boolean] diagnostic enhance off vs on comparison on cached HTML
+    # @param sample_limit [Integer] maximum number of {Result#items} (default 3)
     # @return [Html2rss::Test::Result]
-    def call(config_input, feed_name = nil, min_items: 1, params: {}, strategy: nil, # rubocop:disable Metrics/ParameterLists
-             strict_quality: false, compare_enhance: false)
+    def self.call(config_input, feed_name = nil, min_items: 1, params: {}, strategy: nil, # rubocop:disable Metrics/ParameterLists
+                  strict_quality: false, compare_enhance: false, sample_limit: 3)
       raw_config, validation = Config.resolve_and_validate(config_input, feed_name:, params:)
       return validation_failure_result(validation, raw_config) unless validation.success?
 
       config = build_test_config(raw_config, strategy, params)
       started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
-      execute_timed_pipeline(raw_config, config, started, min_items:, strict_quality:, compare_enhance:)
-    rescue StandardError => error
+      execute_timed_pipeline(raw_config, config, started, min_items:, strict_quality:, compare_enhance:, sample_limit:)
+    rescue Html2rss::ContractError
+      raise
+    rescue Html2rss::Error => error
       duration = Process.clock_gettime(Process::CLOCK_MONOTONIC) - started
       execution_failure_result(error, raw_config, duration)
     end
 
-    def build_test_config(raw_config, strategy, params)
+    def self.build_test_config(raw_config, strategy, params)
       raw_config[:strategy] = strategy.to_sym if strategy
       raw_config[:params] = params if params&.any?
       Config.from_hash(raw_config)
     end
     private_class_method :build_test_config
 
-    def compile_quality_report(raw_config, config, feed_result, pipeline_outcome)
+    def self.compile_quality_report(raw_config, config, feed_result, pipeline_outcome)
       quality_report_for(
         feed_result.to_rss.items,
         channel_url: raw_config.dig(:channel, :url).to_s,
@@ -194,20 +199,21 @@ module Html2rss
     end
     private_class_method :compile_quality_report
 
-    def execute_timed_pipeline(raw_config, config, started, min_items:, strict_quality:, # rubocop:disable Metrics/ParameterLists
-                               compare_enhance:)
+    def self.execute_timed_pipeline(raw_config, config, started, min_items:, strict_quality:, # rubocop:disable Metrics/ParameterLists
+                                    compare_enhance:, sample_limit:)
       feed_result, pipeline_outcome = extract_feed(raw_config)
       duration = Process.clock_gettime(Process::CLOCK_MONOTONIC) - started
       quality_report = compile_quality_report(raw_config, config, feed_result, pipeline_outcome)
       enhance_compare = build_enhance_compare(config, pipeline_outcome) if compare_enhance
 
       build_test_result(raw_config, feed_result, duration, quality_report, enhance_compare,
-                        min_items:, strict_quality:, response_body: pipeline_outcome&.response&.body)
+                        min_items:, strict_quality:, response_body: pipeline_outcome.response.body,
+                        items: ItemSample.call(pipeline_outcome.articles, limit: sample_limit))
     end
     private_class_method :execute_timed_pipeline
 
-    def build_test_result(raw_config, feed_result, duration, quality_report, enhance_compare, # rubocop:disable Metrics/AbcSize, Metrics/MethodLength, Metrics/ParameterLists
-                          min_items:, strict_quality:, response_body:)
+    def self.build_test_result(raw_config, feed_result, duration, quality_report, enhance_compare, # rubocop:disable Metrics/AbcSize, Metrics/MethodLength, Metrics/ParameterLists
+                               min_items:, strict_quality:, response_body:, items:)
       rss_doc = feed_result.to_rss
       outcome = Policy.evaluate(item_count: rss_doc.items.size, min_items:, strict_quality:, quality_report:)
       strategy_used = feed_result.status.selected_strategy || raw_config[:strategy] ||
@@ -217,6 +223,7 @@ module Html2rss
         success: outcome.passed,
         item_count: rss_doc.items.size,
         sample_items: extract_samples(rss_doc.items),
+        items:,
         channel_title: feed_result.channel_title,
         channel_url: raw_config.dig(:channel, :url).to_s,
         strategy_used:,
@@ -232,27 +239,14 @@ module Html2rss
     end
     private_class_method :build_test_result
 
-    def extract_samples(items, limit: 3)
+    def self.extract_samples(items, limit: 3)
       items.first(limit).map do |item|
-        article = feed_item_as_article(item)
-        {
-          title: article.title.to_s.strip,
-          url: article.url.to_s,
-          published_at: article.published_at
-        }
+        article = Article.new(title: item.title, url: item.link, published_at: item.pubDate)
+        { title: article.title.to_s.strip, url: article.url.to_s, published_at: article.published_at }
       end
     end
     private_class_method :extract_samples
 
-    def feed_item_as_article(item)
-      case item
-      when Article then item
-      else
-        Article.new(title: item.title, url: item.link, published_at: item.pubDate)
-      end
-    end
-    private_class_method :feed_item_as_article
-
     ##
     # Builds a ship-quality report for RSS items (shared by test and MCP apply).
     #
@@ -261,32 +255,33 @@ module Html2rss
     # @param config [Html2rss::Config]
     # @param feed_result [Html2rss::FeedResult]
     # @param pipeline_outcome [Html2rss::FeedPipeline::PipelineOutcome]
-    # @param probe_native_feed [Boolean] when false, skip syndication discovery (apply ship gate)
+    # @param probe_native_feed [Boolean] when false, skip syndication discovery (apply ship gate);
+    #   when true, discovery reads the page in +pipeline_outcome.response+ (no extra page fetch)
     # @return [QualityReport]
-    def quality_report_for(items, channel_url:, config:, feed_result:, pipeline_outcome:, # rubocop:disable Metrics/ParameterLists
-                           probe_native_feed: true)
+    def self.quality_report_for(items, channel_url:, config:, feed_result:, pipeline_outcome:, # rubocop:disable Metrics/ParameterLists
+                                probe_native_feed: true)
       audit = AutoSource::Cleanup.audit_feed_items(items)
-      native_feed = probe_native_feed ? probe_native_feed_url(channel_url, config)&.to_s : nil
+      native_feed = probe_native_feed_url(channel_url, config, pipeline_outcome.response)&.to_s if probe_native_feed
       report = QualityReport.from_audit(audit, native_feed:)
       report = append_url_mismatch_warning(report, channel_url, feed_result)
+      report = append_items_without_url_warning(report, pipeline_outcome)
       merge_enhance_audit(report, config, pipeline_outcome)
     end
 
-    def extract_feed(raw_config)
+    def self.extract_feed(raw_config)
       outcome, feed_result = FeedPipeline.new(raw_config).to_outcome_and_result
       [feed_result, outcome]
     end
     private_class_method :extract_feed
 
-    def enhance_enabled?(config)
+    def self.enhance_enabled?(config)
       return false unless config.selectors
 
       !!config.selectors.dig(:items, :enhance)
     end
     private_class_method :enhance_enabled?
 
-    def build_enhance_compare(config, pipeline_outcome)
-      return nil unless pipeline_outcome
+    def self.build_enhance_compare(config, pipeline_outcome)
       return nil unless config.selectors
 
       EnhanceAudit.compare(
@@ -297,9 +292,8 @@ module Html2rss
     end
     private_class_method :build_enhance_compare
 
-    def merge_enhance_audit(report, config, pipeline_outcome) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength -- warning merge + metrics
-      return report unless pipeline_outcome && enhance_enabled?(config)
-      return report unless config.selectors
+    def self.merge_enhance_audit(report, config, pipeline_outcome) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength -- warning merge + metrics
+      return report unless enhance_enabled?(config)
 
       slice = EnhanceAudit.probe(
         response: pipeline_outcome.response,
@@ -320,7 +314,7 @@ module Html2rss
     end
     private_class_method :merge_enhance_audit
 
-    def enhance_gains_to_h(enhance_gains)
+    def self.enhance_gains_to_h(enhance_gains)
       {
         items_probed: enhance_gains.items_probed,
         keys_added: enhance_gains.keys_added,
@@ -330,7 +324,22 @@ module Html2rss
     end
     private_class_method :enhance_gains_to_h
 
-    def append_url_mismatch_warning(report, channel_url, feed_result)
+    # Selector items dropped for having no URL ({Selectors#admission_drops}) leave nothing for the audit to
+    # see, so the warning names the +url+ selector as the place to look.
+    def self.append_items_without_url_warning(report, pipeline_outcome)
+      dropped = pipeline_outcome.admission_drops.fetch(Selectors::MISSING_URL_REASON, 0)
+      return report unless dropped.positive?
+
+      QualityReport.new(
+        warnings: (report.warnings + [:items_without_url]).freeze,
+        metrics: report.metrics.merge(items_without_url: dropped),
+        native_feed: report.native_feed,
+        defer_reason: report.defer_reason
+      )
+    end
+    private_class_method :append_items_without_url_warning
+
+    def self.append_url_mismatch_warning(report, channel_url, feed_result)
       return report unless url_mismatch?(channel_url, feed_result)
 
       warnings = report.warnings.dup
@@ -344,36 +353,36 @@ module Html2rss
     end
     private_class_method :append_url_mismatch_warning
 
-    def url_mismatch?(channel_url, feed_result)
-      status = feed_result&.status
-      return false unless status
-
+    def self.url_mismatch?(channel_url, feed_result)
+      status = feed_result.status
       configured = channel_url.to_s
       final = status.scrape_url.to_s
       final = status.entry_url.to_s if final.empty?
       return false if configured.empty? || final.empty?
 
       !Url.from_absolute(configured).same_document?(Url.from_absolute(final))
-    rescue ArgumentError
+    rescue Html2rss::Url::InvalidUrlError
       false
     end
     private_class_method :url_mismatch?
 
-    def probe_native_feed_url(channel_url, config)
+    # The page the pipeline fetched (where any redirect landed) is the page probed, as in Recon and Capture: no
+    # extra page fetch.
+    def self.probe_native_feed_url(channel_url, config, response)
       return nil if channel_url.to_s.empty?
 
       resources = FeedPipeline::RuntimePolicy.resources_for(config)
       strategy = FeedPipeline::StrategyPlan.concrete_for_diagnostic(config.strategy)
-      session = RequestSession.build(
-        config:, strategy:, budget: resources.budget, policy: resources.policy
-      )
-      Syndication::Discovery.best_feed_url(page_url: channel_url, request_session: session)
-    rescue StandardError
+      session = RequestSession.build(config:, strategy:, budget: resources.budget, policy: resources.policy)
+      Syndication::Discovery.native_feed_url(response:, request_session: session)
+    rescue Html2rss::ContractError
+      raise
+    rescue Html2rss::Error
       nil
     end
     private_class_method :probe_native_feed_url
 
-    def validation_failure_result(report, raw_config) # rubocop:disable Metrics/MethodLength
+    def self.validation_failure_result(report, raw_config) # rubocop:disable Metrics/MethodLength
       Result.new(
         success: false,
         item_count: 0,
@@ -391,7 +400,7 @@ module Html2rss
     end
     private_class_method :validation_failure_result
 
-    def execution_failure_result(error, raw_config, duration) # rubocop:disable Metrics/MethodLength
+    def self.execution_failure_result(error, raw_config, duration) # rubocop:disable Metrics/MethodLength
       Result.new(
         success: false,
         item_count: 0,