html2rss 0.24.0 → 0.25.0

data/lib/html2rss/scoring/link_resolver.rb

@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+module Html2rss
+  module Scoring
+    ##
+    # Resolves href strings into memoized LinkDestination::DestinationFacts for one page base URL.
+    class LinkResolver
+      # Captures the href portion before a fragment for memoization keys.
+      HREF_BASE_PATTERN = /\A([^#]*)/
+
+      # @param base_url [String, Html2rss::Url]
+      def initialize(base_url)
+        @base_url = base_url
+        @by_href = {}
+        @by_node = {}.compare_by_identity
+        @text_classifier = LinkDestination::TextClassifier.new
+      end
+
+      # @return [LinkDestination::TextClassifier]
+      attr_reader :text_classifier
+
+      ##
+      # @param node_or_href [SST::Node, String, #to_s]
+      # @return [LinkDestination::DestinationFacts, nil]
+      def destination_facts(node_or_href)
+        return @by_node[node_or_href] if node_or_href.is_a?(SST::Node) && @by_node.key?(node_or_href)
+
+        href = extract_href(node_or_href)
+        return unless href
+
+        facts = (@by_href[href] ||= build_facts(href))
+        @by_node[node_or_href] = facts if node_or_href.is_a?(SST::Node)
+        facts
+      rescue ArgumentError
+        nil
+      end
+
+      ##
+      # @param text [String, #to_s]
+      # @return [Boolean]
+      def utility_text?(text) = @text_classifier.utility?(text)
+
+      ##
+      # @param text [String, #to_s]
+      # @return [Boolean]
+      def utility_prefix_text?(text) = @text_classifier.utility_prefix?(text)
+
+      ##
+      # @param text [String, #to_s]
+      # @return [Boolean]
+      def recommended_text?(text) = @text_classifier.recommended?(text)
+
+      private
+
+      def extract_href(node_or_href)
+        raw = case node_or_href
+              when SST::Node then node_or_href.attrs.href
+              else node_or_href
+              end
+        return unless raw
+
+        base = raw.to_s[HREF_BASE_PATTERN, 1].to_s.strip
+        base unless base.empty?
+      end
+
+      def build_facts(href)
+        url = Html2rss::Url.from_relative(href, @base_url)
+        LinkDestination::DestinationFacts.build(url)
+      end
+    end
+  end
+end

data/lib/html2rss/scoring/observation.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+module Html2rss
+  module Scoring
+    ##
+    # Typed container signals produced by {ContainerAssessor} for {Engine} rules.
+    Observation = Data.define(
+      :title_word_count,
+      :path_length,
+      :content_path,
+      :publish_marker,
+      :descriptive_context,
+      :article_container,
+      :content_tokens,
+      :junk_tokens,
+      :utility_prefix_title,
+      :recommended_title,
+      :utility_path,
+      :strong_post_suffix,
+      :shallow,
+      :high_confidence_junk_path,
+      :high_confidence_utility_destination,
+      :selected_anchor_present
+    ) do
+      ##
+      # @return [Boolean] true when fewer than two positive content signals are present
+      def weak_signals?
+        [article_container, publish_marker, descriptive_context, content_path].count(&:itself) < 2
+      end
+
+      ##
+      # @param mode [Symbol] +:strict+ for rank/rank_top; +:lenient+ for select_eligible
+      # @return [Boolean] true when the observation should be dropped before ranking
+      def hard_junk?(mode: :strict)
+        case mode
+        when :strict
+          high_confidence_junk_path || chrome_title_junk?
+        when :lenient
+          (high_confidence_junk_path && weak_signals?) || chrome_title_junk?
+        else
+          raise ArgumentError, "unknown hard_junk mode: #{mode.inspect}"
+        end
+      end
+
+      private
+
+      def chrome_title_junk? # rubocop:disable Metrics/CyclomaticComplexity
+        (selected_anchor_present && recommended_title && shallow && weak_signals?) ||
+          (selected_anchor_present && utility_prefix_title && high_confidence_utility_destination && weak_signals?)
+      end
+    end
+  end
+end

data/lib/html2rss/scoring/ranked_segment.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+module Html2rss
+  module Scoring
+    ##
+    # A segment paired with its composite score.
+    RankedSegment = Data.define(:segment, :score) do
+      ##
+      # @param segment [Html2rss::AutoSource::Segment]
+      # @param score [Score]
+      # @return [RankedSegment]
+      # @raise [ArgumentError] on invalid types
+      def self.build(segment:, score:)
+        raise ArgumentError, 'segment must be AutoSource::Segment' unless segment.is_a?(Html2rss::AutoSource::Segment)
+        raise ArgumentError, 'score must be Scoring::Score' unless score.is_a?(Score)
+
+        new(segment:, score:)
+      end
+
+      ##
+      # @return [SST::Node]
+      def root_node = segment.root_node
+
+      ##
+      # @return [SST::Node, nil]
+      def primary_link = segment.primary_link
+
+      ##
+      # @return [Float]
+      def final_score = score.composite
+
+      ##
+      # @return [Float]
+      def quality_score = score.quality
+
+      ##
+      # @return [Float]
+      def junk_score = score.junk
+
+      ##
+      # @return [Integer]
+      def position = segment.position
+    end
+  end
+end

data/lib/html2rss/scoring/score.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module Html2rss
+  module Scoring
+    ##
+    # Composite score with quality/junk split and optional typed feature breakdown.
+    Score = Data.define(:composite, :quality, :junk, :breakdown) do
+      ##
+      # @param composite [Numeric]
+      # @param quality [Numeric, nil]
+      # @param junk [Numeric, nil]
+      # @param breakdown [Hash{Symbol => Numeric}, nil]
+      # @return [Score]
+      # @raise [ArgumentError] when composite is not numeric
+      def self.build(composite:, quality: nil, junk: nil, breakdown: nil)
+        raise ArgumentError, 'composite must be Numeric' unless composite.is_a?(Numeric)
+
+        parts = breakdown.nil? ? Score::EMPTY_BREAKDOWN : breakdown.transform_keys { FeatureId.assert!(_1) }.freeze
+        new(
+          composite: composite.to_f,
+          quality: (quality.nil? ? composite : quality).to_f,
+          junk: (junk || 0).to_f,
+          breakdown: parts
+        )
+      end
+    end
+    # Shared empty feature breakdown for scores without per-feature tallies.
+    Score::EMPTY_BREAKDOWN = {}.freeze
+  end
+end

data/lib/html2rss/scoring.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module Html2rss
+  ##
+  # Declarative feature scoring for AutoSource SST segments.
+  module Scoring
+    ##
+    # Closed set of feature identifiers used by the scoring registry.
+    module FeatureId
+      # Closed set of allowed feature identifier symbols.
+      IDS = %i[
+        title_word_count_ge3 title_word_count_ge7 path_length_gt6 content_path
+        publish_marker descriptive_context article_container content_tokens
+        non_content_utility_path utility_prefix_title_short shallow weak_container
+        recommended_title_non_content high_confidence_junk_path junk_tokens
+        heading_anchor heading_text_match meaningful_text content_like_destination
+        cluster_size cluster_avg_words cluster_heading cluster_time cluster_date
+      ].to_set.freeze
+
+      module_function
+
+      ##
+      # @param id [Symbol]
+      # @return [Symbol]
+      # @raise [ArgumentError] when id is not in the closed set
+      def assert!(id)
+        raise ArgumentError, "unknown FeatureId: #{id.inspect}" unless IDS.include?(id)
+
+        id
+      end
+    end
+  end
+end

data/lib/html2rss/selectors/post_processors/sanitize_html.rb

@@ -92,12 +92,19 @@ module Html2rss
         # @param html [String]
         # @param url [String, Html2rss::Url]
         # @return [String, nil]
+        # rubocop:disable ThreadSafety/ClassInstanceVariable
         def self.get(html, url)
           return nil if String(html).empty?
 
+          @fragment_cache ||= {}
+          key = [html, url.to_s]
+          return @fragment_cache[key] if @fragment_cache.key?(key)
+
+          @fragment_cache.clear if @fragment_cache.size > 256
           context = Selectors::Context.new(config: { channel: { url: } }, options: {})
-          new(html, context).get
+          @fragment_cache[key] = new(html, context).get
         end
+        # rubocop:enable ThreadSafety/ClassInstanceVariable
 
         ##
         # @param channel_url [String, Html2rss::Url]
@@ -117,6 +124,8 @@ module Html2rss
                 ->(env) { HtmlTransformers::WrapImgInA.new.call(**env) }
               ]
             )
+            config[:elements].delete('style')
+            config.delete(:css)
             config[:elements].push('audio', 'video', 'source')
             config.freeze
           end

data/lib/html2rss/selectors.rb

@@ -48,8 +48,6 @@ module Html2rss
       @url = response.url
       @selectors = selectors
       @time_zone = time_zone
-
-      prepare_selectors!
       @rss_item_attributes = @selectors.keys.select { |key| SELECTABLE_SELECTOR_KEYS.include?(key) }
     end
 
@@ -180,24 +178,6 @@ module Html2rss
       )
     end
 
-    def prepare_selectors!
-      validate_url_and_link_exclusivity!
-      fix_url_and_link!
-    end
-
-    def validate_url_and_link_exclusivity!
-      return unless @selectors.key?(:url) && @selectors.key?(:link)
-
-      raise InvalidSelectorName, 'You must either use "url" or "link" your selectors. Using both is not supported.'
-    end
-
-    def fix_url_and_link!
-      return if @selectors[:url] || !@selectors.key?(:link)
-
-      @selectors = @selectors.dup
-      @selectors[:url] = @selectors[:link]
-    end
-
     def parsed_body
       parsed_body_for(response)
     end

data/lib/html2rss/sst/attrs.rb

@@ -0,0 +1,91 @@
+# frozen_string_literal: true
+
+module Html2rss
+  module SST
+    ##
+    # Shared empty defaults for {Attrs} (kept outside Data.define for constant visibility).
+    module AttrDefaults
+      # Shared empty class_names array for Attrs defaults.
+      EMPTY_CLASS_NAMES = [].freeze
+      # Shared empty raw attribute hash for Attrs defaults.
+      EMPTY_RAW = {}.freeze
+    end
+
+    ##
+    # Typed HTML attributes for an SST node. Absent fields are +nil+.
+    Attrs = Data.define(
+      :href, :src, :id, :class_names, :datetime, :itemprop, :style, :srcset, :type, :raw
+    ) do
+      class << self
+        ##
+        # @param href [String, nil]
+        # @param src [String, nil]
+        # @param id [String, nil]
+        # @param class_names [Array<String>, nil]
+        # @param datetime [String, nil]
+        # @param itemprop [String, nil]
+        # @param style [String, nil]
+        # @param srcset [String, nil]
+        # @param type [String, nil]
+        # @param raw [Hash{String => String}, nil]
+        # @return [Attrs]
+        # rubocop:disable Metrics/MethodLength, Metrics/ParameterLists
+        def build(href: nil, src: nil, id: nil, class_names: nil, datetime: nil, itemprop: nil,
+                  style: nil, srcset: nil, type: nil, raw: nil)
+          new(
+            href: blank_to_nil(href),
+            src: blank_to_nil(src),
+            id: blank_to_nil(id),
+            class_names: normalize_class_names(class_names),
+            datetime: blank_to_nil(datetime),
+            itemprop: blank_to_nil(itemprop),
+            style: blank_to_nil(style),
+            srcset: blank_to_nil(srcset),
+            type: blank_to_nil(type),
+            raw: normalize_raw(raw)
+          )
+        end
+        # rubocop:enable Metrics/MethodLength, Metrics/ParameterLists
+
+        ##
+        # @return [Attrs]
+        def empty = EMPTY_ATTRS
+
+        private
+
+        def blank_to_nil(value)
+          str = value&.to_s
+          return if str.nil? || str.strip.empty?
+
+          str.strip.freeze
+        end
+
+        def normalize_class_names(class_names)
+          case class_names
+          when nil then AttrDefaults::EMPTY_CLASS_NAMES
+          when Array then class_names.map { |token| token.to_s.freeze }.freeze
+          else raise ArgumentError, "class_names must be an Array, got #{class_names.class}"
+          end
+        end
+
+        def normalize_raw(raw)
+          case raw
+          when nil then AttrDefaults::EMPTY_RAW
+          when Hash then raw.transform_values { |v| v.to_s.freeze }.freeze
+          else raise ArgumentError, "raw must be a Hash, got #{raw.class}"
+          end
+        end
+      end
+
+      ##
+      # @return [String]
+      def class_attr = class_names.join(' ')
+    end
+
+    # Shared empty Attrs instance for nodes without attributes.
+    EMPTY_ATTRS = Attrs.new(
+      href: nil, src: nil, id: nil, class_names: AttrDefaults::EMPTY_CLASS_NAMES, datetime: nil,
+      itemprop: nil, style: nil, srcset: nil, type: nil, raw: AttrDefaults::EMPTY_RAW
+    ).freeze
+  end
+end

data/lib/html2rss/sst/document.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Html2rss
+  module SST
+    ##
+    # Immutable SST document: root node plus relationship index.
+    Document = Data.define(:root, :index, :degraded, :node_count) do
+      ##
+      # @param root [Node]
+      # @param index [Index]
+      # @param degraded [Boolean] true when MAX_NODES forced semantic-tag-only mode
+      # @param node_count [Integer]
+      # @return [Document]
+      # @raise [ArgumentError] when root/index missing
+      def self.build(root:, index:, degraded: false, node_count: 0)
+        raise ArgumentError, 'root is required' unless root.is_a?(Node)
+        raise ArgumentError, 'index is required' unless index.is_a?(Index)
+
+        new(root:, index:, degraded: !!degraded, node_count: Integer(node_count))
+      end
+    end
+  end
+end

data/lib/html2rss/sst/index.rb

@@ -0,0 +1,112 @@
+# frozen_string_literal: true
+
+module Html2rss
+  module SST
+    ##
+    # Parent/depth indices for an SST tree. Built once by the Normalizer so
+    # Scoring/Segmenter never need Nokogiri ancestor walks.
+    class Index
+      # Weak node→index bindings so +Node#visible_text+ can memoize without call-site churn.
+      NODE_BINDINGS = ObjectSpace::WeakMap.new
+
+      ##
+      # @param node [Node]
+      # @return [Index, nil]
+      def self.for_node(node) = NODE_BINDINGS[node]
+
+      ##
+      # @param root [Node]
+      # @param parents [Hash{Node => Node, nil}]
+      # @param depths [Hash{Node => Integer}]
+      # @param ignored_chrome [Hash{Node => Boolean}]
+      def initialize(root:, parents:, depths:, ignored_chrome:)
+        @root = root
+        @parents = parents
+        @depths = depths
+        @ignored_chrome = ignored_chrome
+        @visible_text = {}.compare_by_identity
+        @word_count = {}.compare_by_identity
+        bind_tree(root)
+      end
+
+      attr_reader :root
+
+      ##
+      # @param node [Node]
+      # @return [Node, nil]
+      def parent_of(node) = @parents[node]
+
+      ##
+      # @param node [Node]
+      # @return [Integer]
+      def depth_of(node) = @depths.fetch(node, 0)
+
+      ##
+      # @param node [Node]
+      # @return [Boolean]
+      def ignored_chrome?(node) = @ignored_chrome.fetch(node, false)
+
+      ##
+      # Memoized default visible text (separator space, no excludes).
+      #
+      # @param node [Node]
+      # @return [String, nil]
+      def memo_visible_text(node)
+        return @visible_text[node] if @visible_text.key?(node)
+
+        @visible_text[node] = Text.extract(node)
+      end
+
+      ##
+      # @param node [Node]
+      # @return [Integer]
+      def memo_word_count(node)
+        return @word_count[node] if @word_count.key?(node)
+
+        @word_count[node] = memo_visible_text(node).to_s.scan(/\p{Alnum}+/).size
+      end
+
+      ##
+      # @param child [Node]
+      # @param ancestor [Node]
+      # @return [Boolean]
+      def descendant_of?(child, ancestor)
+        curr = parent_of(child)
+        while curr
+          return true if curr.equal?(ancestor)
+
+          curr = parent_of(curr)
+        end
+        false
+      end
+
+      ##
+      # @param node [Node]
+      # @param condition [#call]
+      # @return [Node, nil]
+      def parent_until(node, condition)
+        curr = node
+        while curr && curr.name != :html
+          return curr if condition.call(curr)
+
+          curr = parent_of(curr)
+        end
+        nil
+      end
+
+      ##
+      # @yieldparam node [Node]
+      # @return [Enumerator]
+      def each_node(&)
+        root.each_node(&)
+      end
+
+      private
+
+      def bind_tree(node)
+        NODE_BINDINGS[node] = self
+        node.children.each { bind_tree(_1) }
+      end
+    end
+  end
+end

data/lib/html2rss/sst/node.rb

@@ -0,0 +1,147 @@
+# frozen_string_literal: true
+
+module Html2rss
+  module SST
+    ##
+    # Immutable simplified semantic tree node. Predicates and traversal live on
+    # the type so Scoring/Segmenter never touch Nokogiri.
+    Node = Data.define(:name, :attrs, :own_text, :children, :tag_path) do
+      ##
+      # @param name [Symbol, String]
+      # @param attrs [Attrs, nil]
+      # @param own_text [String, nil]
+      # @param children [Array<Node>, nil]
+      # @param tag_path [String, nil] index-free ancestry path for list clustering
+      # @return [Node]
+      # @raise [ArgumentError] on invalid name/attrs/children
+      def self.build(name:, attrs: nil, own_text: nil, children: nil, tag_path: nil)
+        attrs_value = coerce_attrs(attrs)
+        kids = coerce_children(children)
+
+        new(
+          name: coerce_name(name),
+          attrs: attrs_value,
+          own_text: own_text.to_s.freeze,
+          children: kids,
+          tag_path: tag_path.to_s.freeze
+        )
+      end
+
+      def self.coerce_name(name)
+        raise ArgumentError, 'name is required' if name.nil? || name.to_s.strip.empty?
+
+        name.to_sym
+      end
+      private_class_method :coerce_name
+
+      def self.coerce_attrs(attrs)
+        attrs_value = attrs.nil? ? Attrs.empty : attrs
+        raise ArgumentError, 'attrs must be SST::Attrs' unless attrs_value.is_a?(Attrs)
+
+        attrs_value
+      end
+      private_class_method :coerce_attrs
+
+      def self.coerce_children(children)
+        kids = Array(children).freeze
+        raise ArgumentError, 'children must be SST::Node instances' unless kids.all?(Node)
+
+        kids
+      end
+      private_class_method :coerce_children
+
+      ##
+      # @return [Boolean]
+      def link?
+        name == :a && eligible_href?
+      end
+
+      ##
+      # @return [Boolean]
+      def image? = name == :img
+
+      ##
+      # @return [Boolean]
+      def heading? = Tags::HEADING_NAMES.include?(name)
+
+      ##
+      # @return [Boolean]
+      def utility_landmark? = Tags::UTILITY_LANDMARK_NAMES.include?(name)
+
+      ##
+      # @return [Boolean]
+      def ignored_container_name? = Tags::IGNORED_CONTAINER_NAMES.include?(name)
+
+      ##
+      # @return [Boolean]
+      def eligible_href?
+        href = attrs.href
+        return false if href.nil? || href.empty?
+
+        Tags::SKIP_HREF_PREFIXES.none? { |prefix| href.start_with?(prefix) }
+      end
+
+      ##
+      # Depth-first enumeration of self and descendants.
+      #
+      # @yieldparam node [Node]
+      # @return [Enumerator, Array<Node>]
+      def each_node(&block)
+        return enum_for(:each_node) unless block
+
+        yield self
+        children.each { |child| child.each_node(&block) }
+      end
+
+      ##
+      # @return [Array<Node>] all descendants excluding self
+      def descendants
+        children.flat_map { |child| [child, *child.descendants] }
+      end
+
+      ##
+      # @return [Array<Node>]
+      def find_all(&predicate)
+        each_node.select(&predicate)
+      end
+
+      ##
+      # @return [Node, nil]
+      def find(&predicate)
+        each_node.find(&predicate)
+      end
+
+      ##
+      # Visible text with block newlines, excluding optional subtrees.
+      #
+      # @param separator [String]
+      # @param exclude [Array<Node>, nil]
+      # @return [String, nil]
+      def visible_text(separator: ' ', exclude: nil)
+        if exclude.nil? && separator == ' ' && (index = Index.for_node(self))
+          return index.memo_visible_text(self)
+        end
+
+        Text.extract(self, separator:, exclude:)
+      end
+
+      ##
+      # @return [Integer] alphanumeric word count of visible text
+      def word_count
+        if (index = Index.for_node(self))
+          return index.memo_word_count(self)
+        end
+
+        visible_text.to_s.scan(/\p{Alnum}+/).size
+      end
+
+      ##
+      # @return [Float] words per descendant link
+      def text_density
+        words = word_count
+        links = descendants.count(&:link?)
+        links.zero? ? words.to_f : words.to_f / links
+      end
+    end
+  end
+end