html2rss 0.30.0 → 0.31.0

data/lib/html2rss/sst/normalizer.rb

@@ -35,6 +35,7 @@ module Html2rss
           style |
           datetime |
           itemprop |
+          role |
           type |
           category |
           categories |
@@ -54,7 +55,7 @@ module Html2rss
       /xi
 
       # Typed Attrs fields excluded from the leftover raw hash.
-      TYPED_ATTR_NAMES = %w[href src id class datetime itemprop style srcset type].to_set.freeze
+      TYPED_ATTR_NAMES = Set['href', 'src', 'id', 'class', 'datetime', 'itemprop', 'style', 'srcset', 'type'].freeze
 
       # String form of Tags::IGNORED_CONTAINER_NAMES for chrome checks without to_sym.
       IGNORED_CONTAINER_TAGS = Tags::IGNORED_CONTAINER_NAMES.to_set(&:to_s).freeze
@@ -113,23 +114,19 @@ module Html2rss
       def resolve_root_nk
         parsed = @parsed_body
 
-        if parsed.respond_to?(:at_css)
-          html = parsed.at_css('html')
-          return html if html
+        html = parsed.at_css('html')
+        return html if html
 
-          body = parsed.at_css('body')
-          if body
-            Html2rss::Log.warn('sst.normalizer root fallback: body')
-            return body
-          end
+        body = parsed.at_css('body')
+        if body
+          Html2rss::Log.warn('sst.normalizer root fallback: body')
+          return body
         end
 
-        if parsed.respond_to?(:element_children)
-          first = parsed.element_children.find { |child| element_root?(child) }
-          if first
-            Html2rss::Log.warn('sst.normalizer root fallback: first element child')
-            return first
-          end
+        first = parsed.element_children.find { |child| element_root?(child) }
+        if first
+          Html2rss::Log.warn('sst.normalizer root fallback: first element child')
+          return first
         end
 
         Html2rss::Log.warn('sst.normalizer root fallback: self')
@@ -142,7 +139,6 @@ module Html2rss
 
       # rubocop:disable-next Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
       def normalize_element(nk_node, parent:, depth:, path:, chrome:)
-        return unless nk_node.respond_to?(:name)
         return unless nk_node.element?
 
         tag = Html2rss::Html::Probe.tag(nk_node)

data/lib/html2rss/sst/tags.rb

@@ -6,21 +6,21 @@ module Html2rss
     # Shared tag-name and href constants for SST nodes.
     module Tags
       # Heading element names.
-      HEADING_NAMES = %i[h1 h2 h3 h4 h5 h6].to_set.freeze
+      HEADING_NAMES = Set[:h1, :h2, :h3, :h4, :h5, :h6].freeze
       # Block-level names that contribute structural newlines in text.
-      BLOCK_NAMES = %i[p div li ul ol h1 h2 h3 h4 h5 h6 tr br].to_set.freeze
+      BLOCK_NAMES = Set[:p, :div, :li, :ul, :ol, :h1, :h2, :h3, :h4, :h5, :h6, :tr, :br].freeze
       # Names skipped when collecting visible text.
-      INVISIBLE_NAMES = %i[svg script noscript style template].to_set.freeze
+      INVISIBLE_NAMES = Set[:svg, :script, :noscript, :style, :template].freeze
       # Landmark names treated as utility chrome.
-      UTILITY_LANDMARK_NAMES = %i[nav aside footer menu].to_set.freeze
+      UTILITY_LANDMARK_NAMES = Set[:nav, :aside, :footer, :menu].freeze
       # Container names ignored as article roots.
-      IGNORED_CONTAINER_NAMES = %i[nav footer header svg script style].to_set.freeze
+      IGNORED_CONTAINER_NAMES = Set[:nav, :footer, :header, :svg, :script, :style].freeze
       # Names excluded from class/structure clustering.
-      CLUSTER_EXCLUDED_NAMES = %i[html body nav footer header svg script style].to_set.freeze
+      CLUSTER_EXCLUDED_NAMES = Set[:html, :body, :nav, :footer, :header, :svg, :script, :style].freeze
       # Href prefixes that never count as content destinations.
       SKIP_HREF_PREFIXES = ['#', 'javascript:', 'mailto:', 'tel:', 'file://', 'sms:', 'data:'].freeze
       # Layout-ish names used when resolving nested wrapper groups.
-      LAYOUT_NAMES = %i[div section article li ul ol].to_set.freeze
+      LAYOUT_NAMES = Set[:div, :section, :article, :li, :ul, :ol].freeze
     end
   end
 end

data/lib/html2rss/status.rb

@@ -1,16 +1,18 @@
 # frozen_string_literal: true
 
-module Html2rss # rubocop:disable Metrics/ModuleLength -- Status Data.define + marshal stay co-located
+module Html2rss
+  Status = Data.define(
+    :version, :scraper_tallies, :dedup_dropped, :selected_strategy, :attempt_count,
+    :strategy_attempts, :admission_drops, :entry_url, :scrape_url, :entry_resolution
+  )
+
   ##
   # Shared RSS +generator+ / JSON Feed +user_comment+ formatter string.
   #
   # Exposed publicly via {FeedResult#status}. Safe to log without reading articles.
   # Stable telemetry payload for cross-repo consumers (e.g. html2rss-web observability).
   # Tallies and counters are validated and frozen at construction (including Marshal load).
-  Status = Data.define(
-    :version, :scraper_tallies, :dedup_dropped, :selected_strategy, :attempt_count,
-    :strategy_attempts, :admission_drops, :entry_url, :scrape_url, :entry_resolution
-  ) do
+  class Status
     class << self
       ##
       # Builds status from extracted articles and scrape telemetry.
@@ -63,7 +65,7 @@ module Html2rss # rubocop:disable Metrics/ModuleLength -- Status Data.define + m
     # @param admission_drops [Hash{String => Integer}]
     # @param entry_url [String, nil]
     # @param scrape_url [String, nil]
-    # @param entry_resolution [Hash, nil]
+    # @param entry_resolution [Html2rss::FeedResolution::Diag, Hash, nil]
     # rubocop:disable-next Metrics/ParameterLists, Metrics/MethodLength -- Status Data.define members
     def initialize(
       version:, scraper_tallies:, dedup_dropped:, selected_strategy: nil, attempt_count: 0,
@@ -152,8 +154,11 @@ module Html2rss # rubocop:disable Metrics/ModuleLength -- Status Data.define + m
     def freeze_entry_resolution(value)
       return if value.nil?
 
-      hash = value.respond_to?(:to_h) ? value.to_h : value
-      hash.to_h.transform_keys(&:to_sym).freeze
+      case value
+      when Hash, FeedResolution::Diag then value.to_h
+      else
+        raise ArgumentError, "entry_resolution must be a Hash or FeedResolution::Diag, got #{value.class}"
+      end.transform_keys(&:to_sym).freeze
     end
 
     def validate_counters!(dedup:, attempts:, selected_strategy:)

data/lib/html2rss/surface_category.rb

@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
 module Html2rss
+  SurfaceCategory = Data.define(:name)
+
   ##
   # Closed surface class for no-scraper / page-assessment gates.
   #
@@ -10,27 +12,15 @@ module Html2rss
     # Surfaces that warrant listing/feed resolution (hubs / shells — not blocked).
     WEAK = Set[:high_entropy_surface, :app_shell, :unsupported_surface].freeze
 
-    class << self
-      ##
-      # @param value [SurfaceCategory, Symbol, String, nil]
-      # @return [SurfaceCategory]
-      def coerce(value)
-        return value if value.is_a?(self)
-        return new(name: nil) if value.nil?
-
-        new(name: value.to_sym)
-      end
-    end
-
-    ##
-    # @return [Symbol, nil]
-    attr_reader :name
+    # Frozen name → instance table.
+    ALL = %i[listing blocked_surface app_shell high_entropy_surface unsupported_surface]
+          .to_h { |name| [name, new(name:)] }.freeze
+    private_class_method :new
 
     ##
-    # @param name [Symbol, nil]
-    def initialize(name:)
-      @name = name
-    end
+    # @param name [Symbol, String]
+    # @return [SurfaceCategory]
+    def self.[](name) = ALL.fetch(name.to_sym) { raise ArgumentError, "unknown surface category: #{name.inspect}" }
 
     ##
     # @return [Boolean]
@@ -41,23 +31,15 @@ module Html2rss
     def blocked? = name == :blocked_surface
 
     ##
-    # @return [Boolean] non-weak, non-blocked surface eligible for listing bonus
-    def listing_bonus? = !name.nil? && !weak? && !blocked?
+    # @return [Boolean] listing surface eligible for listing bonus
+    def listing_bonus? = name == :listing
 
     ##
-    # @return [Symbol, nil]
+    # @return [Symbol]
     def to_sym = name
 
     ##
-    # @param other [Object]
-    # @return [Boolean]
-    def ==(other)
-      other.is_a?(self.class) && name == other.name
-    end
-    alias eql? ==
-
-    ##
-    # @return [Integer]
-    def hash = [self.class, name].hash
+    # @return [String]
+    def to_s = name.to_s
   end
 end

data/lib/html2rss/syndication/parser.rb

@@ -95,9 +95,9 @@ module Html2rss
       private_class_method :atom_link
 
       def atom_text(node)
-        return if node.nil?
+        return unless node.is_a?(RSS::Atom::TextConstruct) || node.is_a?(RSS::Atom::Feed::Entry::Content)
 
-        node.respond_to?(:content) ? node.content : node.to_s
+        node.content
       end
       module_function :atom_text
       private_class_method :atom_text

data/lib/html2rss/test/enhance_audit.rb

@@ -40,9 +40,7 @@ module Html2rss
         }
         warnings = []
 
-        response.parsed_body.css(selector_engine.items_selector).each do |item|
-          baseline = selector_engine.extract_article(item, response)
-          enhanced = selector_engine.enhance_article_hash(baseline.dup, item, response.url)
+        selector_engine.each_enhance_pair do |baseline, enhanced, item|
           item_keys = keys_added(baseline, enhanced)
           merge_keys_added!(gains, item_keys)
           append_item_warnings!(warnings, item:, enhanced:, item_keys:)

data/lib/html2rss/test/policy.rb

@@ -0,0 +1,106 @@
+# frozen_string_literal: true
+
+module Html2rss
+  module Test
+    ##
+    # Pass/fail classification for a configuration test after live extraction.
+    # Owns min_items vs quality thresholds; {Test.call} orchestrates only.
+    module Policy
+      module_function
+
+      Outcome = Data.define(:passed, :failure_kind, :error_message)
+
+      ##
+      # Outcome of evaluating test thresholds.
+      class Outcome
+        # @return [Boolean]
+        def success? = passed
+
+        # @return [Boolean]
+        def failure? = !passed
+      end
+
+      ##
+      # Evaluates whether the test passed thresholds and determines any failure kind and error message.
+      #
+      # @param item_count [Integer]
+      # @param min_items [Integer]
+      # @param strict_quality [Boolean]
+      # @param quality_report [QualityReport]
+      # @return [Outcome]
+      def evaluate(item_count:, min_items:, strict_quality:, quality_report:)
+        min_items_passed = item_count >= min_items
+        quality_failed = strict_quality && min_items_passed && quality_failure?(quality_report)
+        passed = min_items_passed && !quality_failed
+        failure_kind, error_message = outcome_failure(
+          min_items_passed:, quality_failed:, item_count:, min_items:, quality_report:
+        )
+        log_strict_quality_failure(failure_kind, item_count) if quality_failed
+        Outcome.new(passed:, failure_kind:, error_message:)
+      end
+
+      def log_strict_quality_failure(failure_kind, item_count)
+        Log.info("Test strict quality: failure_kind=#{failure_kind.to_sym} item_count=#{item_count}")
+      end
+      private_class_method :log_strict_quality_failure
+
+      ##
+      # @param quality_report [QualityReport]
+      # @return [Boolean]
+      def quality_failure?(quality_report)
+        metrics = quality_report.metrics
+        item_count = metrics[:item_count]
+        return false if item_count.zero?
+
+        duplicate_urls_failure?(metrics) || junk_title_ratio_failure?(metrics) || short_title_failure?(metrics)
+      end
+
+      ##
+      # @param min_items_passed [Boolean]
+      # @param quality_failed [Boolean]
+      # @param item_count [Integer]
+      # @param min_items [Integer]
+      # @param quality_report [QualityReport]
+      # @return [Array(FailureKind, nil), Array(String, nil)]
+      def outcome_failure(min_items_passed:, quality_failed:, item_count:, min_items:, quality_report:)
+        return [nil, nil] if min_items_passed && !quality_failed
+        unless min_items_passed
+          return [FailureKind[:min_items],
+                  "Extracted #{item_count} items (minimum required: #{min_items})"]
+        end
+
+        [FailureKind[:quality], quality_failure_message(quality_report)]
+      end
+
+      def quality_failure_message(quality_report)
+        reasons = quality_failure_reasons(quality_report.metrics)
+        "Feed quality check failed (#{reasons.join(', ')})"
+      end
+      private_class_method :quality_failure_message
+
+      def quality_failure_reasons(metrics)
+        reasons = []
+        reasons << 'duplicate_urls' if duplicate_urls_failure?(metrics)
+        reasons << 'generic_titles' if junk_title_ratio_failure?(metrics)
+        reasons << 'short_titles' if short_title_failure?(metrics)
+        reasons
+      end
+      private_class_method :quality_failure_reasons
+
+      def duplicate_urls_failure?(metrics)
+        metrics[:item_count] >= 2 && metrics[:unique_url_count] < 2
+      end
+      private_class_method :duplicate_urls_failure?
+
+      def junk_title_ratio_failure?(metrics)
+        metrics[:junk_title_count] > (metrics[:item_count] / 2.0)
+      end
+      private_class_method :junk_title_ratio_failure?
+
+      def short_title_failure?(metrics)
+        metrics[:short_title_count].positive?
+      end
+      private_class_method :short_title_failure?
+    end
+  end
+end