html2rss 0.23.0 → 0.25.0

data/lib/html2rss/scoring/link_resolver.rb

@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+module Html2rss
+  module Scoring
+    ##
+    # Resolves href strings into memoized LinkDestination::DestinationFacts for one page base URL.
+    class LinkResolver
+      # Captures the href portion before a fragment for memoization keys.
+      HREF_BASE_PATTERN = /\A([^#]*)/
+
+      # @param base_url [String, Html2rss::Url]
+      def initialize(base_url)
+        @base_url = base_url
+        @by_href = {}
+        @by_node = {}.compare_by_identity
+        @text_classifier = LinkDestination::TextClassifier.new
+      end
+
+      # @return [LinkDestination::TextClassifier]
+      attr_reader :text_classifier
+
+      ##
+      # @param node_or_href [SST::Node, String, #to_s]
+      # @return [LinkDestination::DestinationFacts, nil]
+      def destination_facts(node_or_href)
+        return @by_node[node_or_href] if node_or_href.is_a?(SST::Node) && @by_node.key?(node_or_href)
+
+        href = extract_href(node_or_href)
+        return unless href
+
+        facts = (@by_href[href] ||= build_facts(href))
+        @by_node[node_or_href] = facts if node_or_href.is_a?(SST::Node)
+        facts
+      rescue ArgumentError
+        nil
+      end
+
+      ##
+      # @param text [String, #to_s]
+      # @return [Boolean]
+      def utility_text?(text) = @text_classifier.utility?(text)
+
+      ##
+      # @param text [String, #to_s]
+      # @return [Boolean]
+      def utility_prefix_text?(text) = @text_classifier.utility_prefix?(text)
+
+      ##
+      # @param text [String, #to_s]
+      # @return [Boolean]
+      def recommended_text?(text) = @text_classifier.recommended?(text)
+
+      private
+
+      def extract_href(node_or_href)
+        raw = case node_or_href
+              when SST::Node then node_or_href.attrs.href
+              else node_or_href
+              end
+        return unless raw
+
+        base = raw.to_s[HREF_BASE_PATTERN, 1].to_s.strip
+        base unless base.empty?
+      end
+
+      def build_facts(href)
+        url = Html2rss::Url.from_relative(href, @base_url)
+        LinkDestination::DestinationFacts.build(url)
+      end
+    end
+  end
+end

data/lib/html2rss/scoring/observation.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+module Html2rss
+  module Scoring
+    ##
+    # Typed container signals produced by {ContainerAssessor} for {Engine} rules.
+    Observation = Data.define(
+      :title_word_count,
+      :path_length,
+      :content_path,
+      :publish_marker,
+      :descriptive_context,
+      :article_container,
+      :content_tokens,
+      :junk_tokens,
+      :utility_prefix_title,
+      :recommended_title,
+      :utility_path,
+      :strong_post_suffix,
+      :shallow,
+      :high_confidence_junk_path,
+      :high_confidence_utility_destination,
+      :selected_anchor_present
+    ) do
+      ##
+      # @return [Boolean] true when fewer than two positive content signals are present
+      def weak_signals?
+        [article_container, publish_marker, descriptive_context, content_path].count(&:itself) < 2
+      end
+
+      ##
+      # @param mode [Symbol] +:strict+ for rank/rank_top; +:lenient+ for select_eligible
+      # @return [Boolean] true when the observation should be dropped before ranking
+      def hard_junk?(mode: :strict)
+        case mode
+        when :strict
+          high_confidence_junk_path || chrome_title_junk?
+        when :lenient
+          (high_confidence_junk_path && weak_signals?) || chrome_title_junk?
+        else
+          raise ArgumentError, "unknown hard_junk mode: #{mode.inspect}"
+        end
+      end
+
+      private
+
+      def chrome_title_junk? # rubocop:disable Metrics/CyclomaticComplexity
+        (selected_anchor_present && recommended_title && shallow && weak_signals?) ||
+          (selected_anchor_present && utility_prefix_title && high_confidence_utility_destination && weak_signals?)
+      end
+    end
+  end
+end

data/lib/html2rss/scoring/ranked_segment.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+module Html2rss
+  module Scoring
+    ##
+    # A segment paired with its composite score.
+    RankedSegment = Data.define(:segment, :score) do
+      ##
+      # @param segment [Html2rss::AutoSource::Segment]
+      # @param score [Score]
+      # @return [RankedSegment]
+      # @raise [ArgumentError] on invalid types
+      def self.build(segment:, score:)
+        raise ArgumentError, 'segment must be AutoSource::Segment' unless segment.is_a?(Html2rss::AutoSource::Segment)
+        raise ArgumentError, 'score must be Scoring::Score' unless score.is_a?(Score)
+
+        new(segment:, score:)
+      end
+
+      ##
+      # @return [SST::Node]
+      def root_node = segment.root_node
+
+      ##
+      # @return [SST::Node, nil]
+      def primary_link = segment.primary_link
+
+      ##
+      # @return [Float]
+      def final_score = score.composite
+
+      ##
+      # @return [Float]
+      def quality_score = score.quality
+
+      ##
+      # @return [Float]
+      def junk_score = score.junk
+
+      ##
+      # @return [Integer]
+      def position = segment.position
+    end
+  end
+end

data/lib/html2rss/scoring/score.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module Html2rss
+  module Scoring
+    ##
+    # Composite score with quality/junk split and optional typed feature breakdown.
+    Score = Data.define(:composite, :quality, :junk, :breakdown) do
+      ##
+      # @param composite [Numeric]
+      # @param quality [Numeric, nil]
+      # @param junk [Numeric, nil]
+      # @param breakdown [Hash{Symbol => Numeric}, nil]
+      # @return [Score]
+      # @raise [ArgumentError] when composite is not numeric
+      def self.build(composite:, quality: nil, junk: nil, breakdown: nil)
+        raise ArgumentError, 'composite must be Numeric' unless composite.is_a?(Numeric)
+
+        parts = breakdown.nil? ? Score::EMPTY_BREAKDOWN : breakdown.transform_keys { FeatureId.assert!(_1) }.freeze
+        new(
+          composite: composite.to_f,
+          quality: (quality.nil? ? composite : quality).to_f,
+          junk: (junk || 0).to_f,
+          breakdown: parts
+        )
+      end
+    end
+    # Shared empty feature breakdown for scores without per-feature tallies.
+    Score::EMPTY_BREAKDOWN = {}.freeze
+  end
+end

data/lib/html2rss/scoring.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module Html2rss
+  ##
+  # Declarative feature scoring for AutoSource SST segments.
+  module Scoring
+    ##
+    # Closed set of feature identifiers used by the scoring registry.
+    module FeatureId
+      # Closed set of allowed feature identifier symbols.
+      IDS = %i[
+        title_word_count_ge3 title_word_count_ge7 path_length_gt6 content_path
+        publish_marker descriptive_context article_container content_tokens
+        non_content_utility_path utility_prefix_title_short shallow weak_container
+        recommended_title_non_content high_confidence_junk_path junk_tokens
+        heading_anchor heading_text_match meaningful_text content_like_destination
+        cluster_size cluster_avg_words cluster_heading cluster_time cluster_date
+      ].to_set.freeze
+
+      module_function
+
+      ##
+      # @param id [Symbol]
+      # @return [Symbol]
+      # @raise [ArgumentError] when id is not in the closed set
+      def assert!(id)
+        raise ArgumentError, "unknown FeatureId: #{id.inspect}" unless IDS.include?(id)
+
+        id
+      end
+    end
+  end
+end

data/lib/html2rss/selectors/post_processors/sanitize_html.rb

@@ -92,12 +92,19 @@ module Html2rss
         # @param html [String]
         # @param url [String, Html2rss::Url]
         # @return [String, nil]
+        # rubocop:disable ThreadSafety/ClassInstanceVariable
         def self.get(html, url)
           return nil if String(html).empty?
 
+          @fragment_cache ||= {}
+          key = [html, url.to_s]
+          return @fragment_cache[key] if @fragment_cache.key?(key)
+
+          @fragment_cache.clear if @fragment_cache.size > 256
           context = Selectors::Context.new(config: { channel: { url: } }, options: {})
-          new(html, context).get
+          @fragment_cache[key] = new(html, context).get
         end
+        # rubocop:enable ThreadSafety/ClassInstanceVariable
 
         ##
         # @param channel_url [String, Html2rss::Url]
@@ -117,6 +124,8 @@ module Html2rss
                 ->(env) { HtmlTransformers::WrapImgInA.new.call(**env) }
               ]
             )
+            config[:elements].delete('style')
+            config.delete(:css)
             config[:elements].push('audio', 'video', 'source')
             config.freeze
           end

data/lib/html2rss/selectors.rb

@@ -31,6 +31,11 @@ module Html2rss
     ITEM_TAGS = %i[title url description author comments published_at guid enclosure categories].freeze
     # Item attributes that require dedicated extraction logic.
     SPECIAL_ATTRIBUTES = Set[:guid, :enclosure, :categories].freeze
+    # Config selector keys that map onto a different {Article} attribute.
+    # +:enclosure+ stays singular in YAML; Article stores +:enclosures+.
+    SELECTOR_TO_ARTICLE_KEY = { enclosure: :enclosures }.freeze
+    # Selector keys that may be copied onto an Article (PROVIDED_KEYS + mapped aliases).
+    SELECTABLE_SELECTOR_KEYS = (Html2rss::Article::PROVIDED_KEYS + SELECTOR_TO_ARTICLE_KEY.keys).to_set.freeze
 
     ##
     # Initializes a new Selectors instance.
@@ -43,9 +48,7 @@ module Html2rss
       @url = response.url
       @selectors = selectors
       @time_zone = time_zone
-
-      prepare_selectors!
-      @rss_item_attributes = @selectors.keys & Html2rss::Article::PROVIDED_KEYS
+      @rss_item_attributes = @selectors.keys.select { |key| SELECTABLE_SELECTOR_KEYS.include?(key) }
     end
 
     ##
@@ -92,7 +95,13 @@ module Html2rss
     # @return [Hash] Hash of attributes for the article.
     def extract_article(item, page_response = response)
       scope = item_scope_for(item, page_response.url)
-      @rss_item_attributes.to_h { |key| [key, scope.select(key)] }.compact
+      @rss_item_attributes.each_with_object({}) do |selector_key, hash|
+        value = scope.select(selector_key)
+        next if value.nil?
+
+        article_key = SELECTOR_TO_ARTICLE_KEY.fetch(selector_key, selector_key)
+        hash[article_key] = article_key == :enclosures ? wrap_enclosure_value(value) : value
+      end
     end
 
     ##
@@ -169,24 +178,6 @@ module Html2rss
       )
     end
 
-    def prepare_selectors!
-      validate_url_and_link_exclusivity!
-      fix_url_and_link!
-    end
-
-    def validate_url_and_link_exclusivity!
-      return unless @selectors.key?(:url) && @selectors.key?(:link)
-
-      raise InvalidSelectorName, 'You must either use "url" or "link" your selectors. Using both is not supported.'
-    end
-
-    def fix_url_and_link!
-      return if @selectors[:url] || !@selectors.key?(:link)
-
-      @selectors = @selectors.dup
-      @selectors[:url] = @selectors[:link]
-    end
-
     def parsed_body
       parsed_body_for(response)
     end
@@ -308,9 +299,20 @@ module Html2rss
       @channel_contexts[base_url] ||= { url: base_url, time_zone: @time_zone }.freeze
     end
 
-    # @return [Hash] enclosure details.
+    # Keep a single enclosure Hash as one list entry; +Array(hash)+ would split pairs.
+    #
+    # @param value [Hash, Array] enclosure hash or list of hashes
+    # @return [Array]
+    def wrap_enclosure_value(value)
+      value.is_a?(Array) ? value : [value]
+    end
+
+    # @return [Hash, nil] enclosure details, or nil when the selector yields nothing.
     def enclosure(scope:, config:)
-      url = Url.from_relative(select_regular(:enclosure, scope:, config:), scope.base_url)
+      selected = select_regular(:enclosure, scope:, config:)
+      return if selected.nil? || selected.to_s.strip.empty?
+
+      url = Url.from_relative(selected, scope.base_url)
 
       { url:, type: config[:content_type] }
     end

data/lib/html2rss/sst/attrs.rb

@@ -0,0 +1,91 @@
+# frozen_string_literal: true
+
+module Html2rss
+  module SST
+    ##
+    # Shared empty defaults for {Attrs} (kept outside Data.define for constant visibility).
+    module AttrDefaults
+      # Shared empty class_names array for Attrs defaults.
+      EMPTY_CLASS_NAMES = [].freeze
+      # Shared empty raw attribute hash for Attrs defaults.
+      EMPTY_RAW = {}.freeze
+    end
+
+    ##
+    # Typed HTML attributes for an SST node. Absent fields are +nil+.
+    Attrs = Data.define(
+      :href, :src, :id, :class_names, :datetime, :itemprop, :style, :srcset, :type, :raw
+    ) do
+      class << self
+        ##
+        # @param href [String, nil]
+        # @param src [String, nil]
+        # @param id [String, nil]
+        # @param class_names [Array<String>, nil]
+        # @param datetime [String, nil]
+        # @param itemprop [String, nil]
+        # @param style [String, nil]
+        # @param srcset [String, nil]
+        # @param type [String, nil]
+        # @param raw [Hash{String => String}, nil]
+        # @return [Attrs]
+        # rubocop:disable Metrics/MethodLength, Metrics/ParameterLists
+        def build(href: nil, src: nil, id: nil, class_names: nil, datetime: nil, itemprop: nil,
+                  style: nil, srcset: nil, type: nil, raw: nil)
+          new(
+            href: blank_to_nil(href),
+            src: blank_to_nil(src),
+            id: blank_to_nil(id),
+            class_names: normalize_class_names(class_names),
+            datetime: blank_to_nil(datetime),
+            itemprop: blank_to_nil(itemprop),
+            style: blank_to_nil(style),
+            srcset: blank_to_nil(srcset),
+            type: blank_to_nil(type),
+            raw: normalize_raw(raw)
+          )
+        end
+        # rubocop:enable Metrics/MethodLength, Metrics/ParameterLists
+
+        ##
+        # @return [Attrs]
+        def empty = EMPTY_ATTRS
+
+        private
+
+        def blank_to_nil(value)
+          str = value&.to_s
+          return if str.nil? || str.strip.empty?
+
+          str.strip.freeze
+        end
+
+        def normalize_class_names(class_names)
+          case class_names
+          when nil then AttrDefaults::EMPTY_CLASS_NAMES
+          when Array then class_names.map { |token| token.to_s.freeze }.freeze
+          else raise ArgumentError, "class_names must be an Array, got #{class_names.class}"
+          end
+        end
+
+        def normalize_raw(raw)
+          case raw
+          when nil then AttrDefaults::EMPTY_RAW
+          when Hash then raw.transform_values { |v| v.to_s.freeze }.freeze
+          else raise ArgumentError, "raw must be a Hash, got #{raw.class}"
+          end
+        end
+      end
+
+      ##
+      # @return [String]
+      def class_attr = class_names.join(' ')
+    end
+
+    # Shared empty Attrs instance for nodes without attributes.
+    EMPTY_ATTRS = Attrs.new(
+      href: nil, src: nil, id: nil, class_names: AttrDefaults::EMPTY_CLASS_NAMES, datetime: nil,
+      itemprop: nil, style: nil, srcset: nil, type: nil, raw: AttrDefaults::EMPTY_RAW
+    ).freeze
+  end
+end

data/lib/html2rss/sst/document.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Html2rss
+  module SST
+    ##
+    # Immutable SST document: root node plus relationship index.
+    Document = Data.define(:root, :index, :degraded, :node_count) do
+      ##
+      # @param root [Node]
+      # @param index [Index]
+      # @param degraded [Boolean] true when MAX_NODES forced semantic-tag-only mode
+      # @param node_count [Integer]
+      # @return [Document]
+      # @raise [ArgumentError] when root/index missing
+      def self.build(root:, index:, degraded: false, node_count: 0)
+        raise ArgumentError, 'root is required' unless root.is_a?(Node)
+        raise ArgumentError, 'index is required' unless index.is_a?(Index)
+
+        new(root:, index:, degraded: !!degraded, node_count: Integer(node_count))
+      end
+    end
+  end
+end

data/lib/html2rss/sst/index.rb

@@ -0,0 +1,112 @@
+# frozen_string_literal: true
+
+module Html2rss
+  module SST
+    ##
+    # Parent/depth indices for an SST tree. Built once by the Normalizer so
+    # Scoring/Segmenter never need Nokogiri ancestor walks.
+    class Index
+      # Weak node→index bindings so +Node#visible_text+ can memoize without call-site churn.
+      NODE_BINDINGS = ObjectSpace::WeakMap.new
+
+      ##
+      # @param node [Node]
+      # @return [Index, nil]
+      def self.for_node(node) = NODE_BINDINGS[node]
+
+      ##
+      # @param root [Node]
+      # @param parents [Hash{Node => Node, nil}]
+      # @param depths [Hash{Node => Integer}]
+      # @param ignored_chrome [Hash{Node => Boolean}]
+      def initialize(root:, parents:, depths:, ignored_chrome:)
+        @root = root
+        @parents = parents
+        @depths = depths
+        @ignored_chrome = ignored_chrome
+        @visible_text = {}.compare_by_identity
+        @word_count = {}.compare_by_identity
+        bind_tree(root)
+      end
+
+      attr_reader :root
+
+      ##
+      # @param node [Node]
+      # @return [Node, nil]
+      def parent_of(node) = @parents[node]
+
+      ##
+      # @param node [Node]
+      # @return [Integer]
+      def depth_of(node) = @depths.fetch(node, 0)
+
+      ##
+      # @param node [Node]
+      # @return [Boolean]
+      def ignored_chrome?(node) = @ignored_chrome.fetch(node, false)
+
+      ##
+      # Memoized default visible text (separator space, no excludes).
+      #
+      # @param node [Node]
+      # @return [String, nil]
+      def memo_visible_text(node)
+        return @visible_text[node] if @visible_text.key?(node)
+
+        @visible_text[node] = Text.extract(node)
+      end
+
+      ##
+      # @param node [Node]
+      # @return [Integer]
+      def memo_word_count(node)
+        return @word_count[node] if @word_count.key?(node)
+
+        @word_count[node] = memo_visible_text(node).to_s.scan(/\p{Alnum}+/).size
+      end
+
+      ##
+      # @param child [Node]
+      # @param ancestor [Node]
+      # @return [Boolean]
+      def descendant_of?(child, ancestor)
+        curr = parent_of(child)
+        while curr
+          return true if curr.equal?(ancestor)
+
+          curr = parent_of(curr)
+        end
+        false
+      end
+
+      ##
+      # @param node [Node]
+      # @param condition [#call]
+      # @return [Node, nil]
+      def parent_until(node, condition)
+        curr = node
+        while curr && curr.name != :html
+          return curr if condition.call(curr)
+
+          curr = parent_of(curr)
+        end
+        nil
+      end
+
+      ##
+      # @yieldparam node [Node]
+      # @return [Enumerator]
+      def each_node(&)
+        root.each_node(&)
+      end
+
+      private
+
+      def bind_tree(node)
+        NODE_BINDINGS[node] = self
+        node.children.each { bind_tree(_1) }
+      end
+    end
+  end
+end