html2rss 0.24.0 → 0.26.0

data/lib/html2rss/scoring/container_assessor.rb

@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+module Html2rss
+  module Scoring
+    ##
+    # Observes an SST container and builds typed {Observation} score inputs.
+    class ContainerAssessor
+      # Microdata itemprop values treated as publish/update markers.
+      PUBLISH_ITEMPROPS = %w[datePublished dateModified].freeze
+
+      # @param text_classifier [LinkDestination::TextClassifier]
+      # @param content_token_regexp [Regexp]
+      # @param junk_token_regexp [Regexp]
+      def initialize(text_classifier: LinkDestination::TextClassifier.new,
+                     content_token_regexp: LinkResolver::CONTENT_TOKEN_REGEXP,
+                     junk_token_regexp: LinkResolver::JUNK_TOKEN_REGEXP)
+        @text_classifier = text_classifier
+        @content_token_regexp = content_token_regexp
+        @junk_token_regexp = junk_token_regexp
+      end
+
+      ##
+      # @param container [SST::Node]
+      # @param selected_anchor [SST::Node, nil]
+      # @param destination_facts [LinkDestination::DestinationFacts, nil]
+      # @return [Observation]
+      def call(container, selected_anchor, destination_facts:) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
+        title = entry_title(container, selected_anchor)
+        tokens = "#{container.attrs.class_attr} #{container.attrs.id}"
+
+        Observation.new(
+          title_word_count: word_count(title),
+          path_length: destination_facts&.url&.path.to_s.length,
+          content_path: destination_facts&.content_path,
+          publish_marker: publish_marker?(container),
+          descriptive_context: descriptive_context?(container.visible_text, title),
+          article_container: container.name == :article,
+          content_tokens: tokens.match?(@content_token_regexp),
+          junk_tokens: tokens.match?(@junk_token_regexp),
+          utility_prefix_title: @text_classifier.utility_prefix?(title),
+          recommended_title: @text_classifier.recommended?(title),
+          utility_path: destination_facts&.utility_path,
+          strong_post_suffix: destination_facts&.strong_post_suffix,
+          shallow: destination_facts&.shallow,
+          high_confidence_junk_path: destination_facts&.high_confidence_junk_path,
+          high_confidence_utility_destination: destination_facts&.high_confidence_utility_destination,
+          selected_anchor_present: !selected_anchor.nil?
+        )
+      end
+
+      private
+
+      def publish_marker?(container)
+        !!container.find do |n|
+          n.name == :time ||
+            n.attrs.datetime ||
+            PUBLISH_ITEMPROPS.include?(n.attrs.itemprop.to_s)
+        end
+      end
+
+      def descriptive_context?(container_text, title)
+        snippet = container_text.to_s.sub(/\A#{Regexp.escape(title.to_s)}/i, '')
+        snippet.length > 30 && word_count(snippet) >= 8
+      end
+
+      def entry_title(container, selected_anchor)
+        heading = container.find(&:heading?)
+        source = heading || selected_anchor
+        source ? source.visible_text.to_s.strip : ''
+      end
+
+      def word_count(text)
+        text.to_s.scan(/\p{Alnum}+/).size
+      end
+    end
+  end
+end

data/lib/html2rss/scoring/engine.rb

@@ -0,0 +1,145 @@
+# frozen_string_literal: true
+
+module Html2rss
+  module Scoring
+    ##
+    # Ranks AutoSource::Segment instances with an explicit feature registry (no send).
+    class Engine
+      # Soft cap after score-floor filtering (lazy extraction guard; breadth > blind top-50).
+      TOP_K = 99
+      # Minimum composite score retained by {#rank_top} (precision floor).
+      SCORE_FLOOR = 0.0
+
+      FEATURE_IDS = %i[
+        title_word_count_ge3 title_word_count_ge7 path_length_gt6 content_path
+        publish_marker descriptive_context article_container content_tokens
+        non_content_utility_path utility_prefix_title_short shallow weak_container
+        recommended_title_non_content high_confidence_junk_path junk_tokens
+        heading_anchor heading_text_match meaningful_text content_like_destination
+        cluster_size cluster_avg_words cluster_heading cluster_time cluster_date
+      ].to_set.freeze
+      private_constant :FEATURE_IDS
+
+      # Quality feature triples: [feature_id, predicate, weight].
+      QUALITY_RULES = [
+        [:title_word_count_ge3, ->(o) { o.title_word_count >= 3 }, 40],
+        [:title_word_count_ge7, ->(o) { o.title_word_count >= 7 }, 15],
+        [:path_length_gt6, ->(o) { o.path_length > 6 }, 20],
+        [:content_path, lambda(&:content_path), 15],
+        [:publish_marker, lambda(&:publish_marker), 15],
+        [:descriptive_context, lambda(&:descriptive_context), 10],
+        [:article_container, lambda(&:article_container), 10],
+        [:content_tokens, lambda(&:content_tokens), 10]
+      ].freeze
+
+      # Junk feature triples: [feature_id, predicate, weight].
+      JUNK_RULES = [
+        [:non_content_utility_path, ->(o) { o.utility_path && !o.content_path && !o.strong_post_suffix }, 25],
+        [:utility_prefix_title_short, ->(o) { o.utility_prefix_title && o.title_word_count <= 6 }, 15],
+        [:shallow, lambda(&:shallow), 10],
+        [:weak_container, ->(o) { !o.publish_marker && !o.descriptive_context }, 10],
+        [:recommended_title_non_content, ->(o) { o.recommended_title && !o.content_path }, 10],
+        [:high_confidence_junk_path, lambda(&:high_confidence_junk_path), 5],
+        [:junk_tokens, lambda(&:junk_tokens), 15]
+      ].freeze
+
+      ##
+      # @param id [Symbol]
+      # @return [Symbol]
+      # @raise [ArgumentError] when id is not in the closed set
+      def self.assert_feature_id!(id)
+        raise ArgumentError, "unknown feature: #{id.inspect}" unless FEATURE_IDS.include?(id)
+
+        id
+      end
+
+      (QUALITY_RULES + JUNK_RULES).each { |feature_id, _, _| assert_feature_id!(feature_id) }
+
+      # @param link_resolver [LinkResolver]
+      def initialize(link_resolver:)
+        @link_resolver = link_resolver
+        @assessor = ContainerAssessor.new(text_classifier: link_resolver.text_classifier)
+      end
+
+      ##
+      # @param segments [Array<Html2rss::AutoSource::Segment>]
+      # @return [Array<RankedSegment>] sorted by composite desc, position asc; hard-junk dropped
+      def rank(segments)
+        segments.filter_map { |segment| rank_one(segment) }
+                .sort_by { |ranked| [-ranked.final_score, ranked.position] }
+      end
+
+      ##
+      # Filters hard-junk but preserves discovery order (list/cluster scrapers).
+      #
+      # @param segments [Array<Html2rss::AutoSource::Segment>]
+      # @param limit [Integer]
+      # @return [Array<RankedSegment>]
+      def select_eligible(segments, limit: TOP_K)
+        segments.filter_map { |segment| rank_one(segment, hard_junk_mode: :lenient) }
+                .sort_by(&:position)
+                .first(limit)
+      end
+
+      ##
+      # @param segments [Array<Html2rss::AutoSource::Segment>]
+      # @param limit [Integer]
+      # @return [Array<RankedSegment>] score-floor filtered, then capped
+      def rank_top(segments, limit: TOP_K)
+        rank(segments).select { |ranked| ranked.final_score >= SCORE_FLOOR }.first(limit)
+      end
+
+      private
+
+      def rank_one(segment, hard_junk_mode: :strict) # rubocop:disable Metrics/MethodLength
+        facts = segment.primary_link ? @link_resolver.destination_facts(segment.primary_link) : nil
+        obs = @assessor.call(segment.root_node, segment.primary_link, destination_facts: facts)
+        return if obs.hard_junk?(mode: hard_junk_mode)
+
+        quality, quality_parts = apply_rules(QUALITY_RULES, obs)
+        junk, junk_parts = apply_rules(JUNK_RULES, obs)
+        ranked(
+          segment:,
+          score: build_score(
+            composite: quality - junk,
+            quality:,
+            junk:,
+            breakdown: quality_parts.merge(junk_parts)
+          )
+        )
+      end
+
+      def apply_rules(rules, obs)
+        parts = {}
+        rules.each do |feature_id, predicate, weight|
+          parts[feature_id] = weight if predicate.call(obs)
+        end
+        [parts.values.sum, parts]
+      end
+
+      def build_score(composite:, quality: nil, junk: nil, breakdown: nil)
+        raise ArgumentError, 'composite must be Numeric' unless composite.is_a?(Numeric)
+
+        Score.new(
+          composite: composite.to_f,
+          quality: (quality.nil? ? composite : quality).to_f,
+          junk: (junk || 0).to_f,
+          breakdown: normalize_breakdown(breakdown)
+        )
+      end
+
+      def normalize_breakdown(breakdown)
+        return Score::EMPTY_BREAKDOWN if breakdown.nil?
+
+        breakdown.transform_keys { |id| self.class.assert_feature_id!(id) }.freeze
+      end
+
+      def ranked(segment:, score:)
+        raise ArgumentError, 'segment must be AutoSource::Segment' unless segment.is_a?(Html2rss::AutoSource::Segment)
+        raise ArgumentError, 'score must be Scoring::Score' unless score.is_a?(Score)
+
+        RankedSegment.new(segment:, score:)
+      end
+    end
+  end
+end

data/lib/html2rss/scoring/link_resolver.rb

@@ -0,0 +1,84 @@
+# frozen_string_literal: true
+
+module Html2rss
+  module Scoring
+    ##
+    # Resolves href strings into memoized LinkDestination::DestinationFacts for one page base URL.
+    class LinkResolver
+      # Captures the href portion before a fragment for memoization keys.
+      HREF_BASE_PATTERN = /\A([^#]*)/
+
+      # Matches content-like tokens in class/id strings (from PathClassifier vocabulary).
+      CONTENT_TOKEN_REGEXP = begin
+        words = LinkDestination::PathClassifier::SEGMENT_SETS.fetch(:content)
+        /(?:^|\s|[-_])(#{Regexp.union(words.to_a).source})(?:\s|[-_]|$)/i
+      end.freeze
+
+      # Matches utility/junk tokens in class/id strings (from PathClassifier vocabulary).
+      JUNK_TOKEN_REGEXP = begin
+        words = LinkDestination::PathClassifier::SEGMENT_SETS.fetch(:utility)
+        /(?:^|\s|[-_])(#{Regexp.union(words.to_a).source})(?:\s|[-_]|$)/i
+      end.freeze
+
+      # @param base_url [String, Html2rss::Url]
+      def initialize(base_url)
+        @base_url = base_url
+        @by_href = {}
+        @by_node = {}.compare_by_identity
+        @text_classifier = LinkDestination::TextClassifier.new
+      end
+
+      # @return [LinkDestination::TextClassifier]
+      attr_reader :text_classifier
+
+      ##
+      # @param node_or_href [SST::Node, String, #to_s]
+      # @return [LinkDestination::DestinationFacts, nil]
+      def destination_facts(node_or_href)
+        return @by_node[node_or_href] if node_or_href.is_a?(SST::Node) && @by_node.key?(node_or_href)
+
+        href = extract_href(node_or_href)
+        return unless href
+
+        facts = (@by_href[href] ||= build_facts(href))
+        @by_node[node_or_href] = facts if node_or_href.is_a?(SST::Node)
+        facts
+      rescue ArgumentError
+        nil
+      end
+
+      ##
+      # @param text [String, #to_s]
+      # @return [Boolean]
+      def utility_text?(text) = @text_classifier.utility?(text)
+
+      ##
+      # @param text [String, #to_s]
+      # @return [Boolean]
+      def utility_prefix_text?(text) = @text_classifier.utility_prefix?(text)
+
+      ##
+      # @param text [String, #to_s]
+      # @return [Boolean]
+      def recommended_text?(text) = @text_classifier.recommended?(text)
+
+      private
+
+      def extract_href(node_or_href)
+        raw = case node_or_href
+              when SST::Node then node_or_href.attrs.href
+              else node_or_href
+              end
+        return unless raw
+
+        base = raw.to_s[HREF_BASE_PATTERN, 1].to_s.strip
+        base unless base.empty?
+      end
+
+      def build_facts(href)
+        url = Html2rss::Url.from_relative(href, @base_url)
+        LinkDestination::DestinationFacts.build(url)
+      end
+    end
+  end
+end

data/lib/html2rss/scoring/observation.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+module Html2rss
+  module Scoring
+    ##
+    # Typed container signals produced by {ContainerAssessor} for {Engine} rules.
+    Observation = Data.define(
+      :title_word_count,
+      :path_length,
+      :content_path,
+      :publish_marker,
+      :descriptive_context,
+      :article_container,
+      :content_tokens,
+      :junk_tokens,
+      :utility_prefix_title,
+      :recommended_title,
+      :utility_path,
+      :strong_post_suffix,
+      :shallow,
+      :high_confidence_junk_path,
+      :high_confidence_utility_destination,
+      :selected_anchor_present
+    ) do
+      ##
+      # @return [Boolean] true when fewer than two positive content signals are present
+      def weak_signals?
+        [article_container, publish_marker, descriptive_context, content_path].count(&:itself) < 2
+      end
+
+      ##
+      # @param mode [Symbol] +:strict+ for rank/rank_top; +:lenient+ for select_eligible
+      # @return [Boolean] true when the observation should be dropped before ranking
+      def hard_junk?(mode: :strict)
+        case mode
+        when :strict
+          high_confidence_junk_path || chrome_title_junk?
+        when :lenient
+          (high_confidence_junk_path && weak_signals?) || chrome_title_junk?
+        else
+          raise ArgumentError, "unknown hard_junk mode: #{mode.inspect}"
+        end
+      end
+
+      private
+
+      def chrome_title_junk? # rubocop:disable Metrics/CyclomaticComplexity
+        (selected_anchor_present && recommended_title && shallow && weak_signals?) ||
+          (selected_anchor_present && utility_prefix_title && high_confidence_utility_destination && weak_signals?)
+      end
+    end
+  end
+end

data/lib/html2rss/scoring/ranked_segment.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module Html2rss
+  module Scoring
+    ##
+    # A segment paired with its composite score.
+    RankedSegment = Data.define(:segment, :score) do
+      ##
+      # @return [SST::Node]
+      def root_node = segment.root_node
+
+      ##
+      # @return [SST::Node, nil]
+      def primary_link = segment.primary_link
+
+      ##
+      # @return [Float]
+      def final_score = score.composite
+
+      ##
+      # @return [Float]
+      def quality_score = score.quality
+
+      ##
+      # @return [Float]
+      def junk_score = score.junk
+
+      ##
+      # @return [Integer]
+      def position = segment.position
+    end
+  end
+end

data/lib/html2rss/scoring/score.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module Html2rss
+  module Scoring
+    ##
+    # Composite score with quality/junk split and optional typed feature breakdown.
+    Score = Data.define(:composite, :quality, :junk, :breakdown)
+    # Shared empty feature breakdown for scores without per-feature tallies.
+    Score::EMPTY_BREAKDOWN = {}.freeze
+  end
+end

data/lib/html2rss/scoring.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+module Html2rss
+  ##
+  # Declarative feature scoring for AutoSource SST segments.
+  module Scoring
+  end
+end

data/lib/html2rss/selectors/extractors/attribute.rb

@@ -24,9 +24,24 @@ module Html2rss
       # In case you're extracting a date or a time, consider parsing it
       # during post processing with {PostProcessors::ParseTime}.
       class Attribute
+        # Config-facing option types (excluding shared selector fields supplied at runtime).
+        OPTION_TYPES = { attribute: String }.freeze
+
         # The available options for the attribute extractor.
         Options = Struct.new('AttributeOptions', :selector, :attribute, keyword_init: true)
 
+        # JSON Schema description exported via +schema_doc+.
+        DESCRIPTION = 'Return the value of an HTML attribute on the selected element. ' \
+                      'Requires sibling selector option `attribute` (attribute name).'
+
+        # Example extractor name values for JSON Schema +examples+.
+        EXAMPLES = [
+          'attribute'
+        ].freeze
+
+        # @return [Hash{Symbol => Object}] JSON Schema fragment for this extractor name
+        def self.schema_doc = SchemaDoc.for_extractor(name: :attribute, klass: self)
+
         ##
         # Initializes the Attribute extractor.
         #

data/lib/html2rss/selectors/extractors/href.rb

@@ -27,6 +27,18 @@ module Html2rss
         # The available options for the href (attribute) extractor.
         Options = Struct.new('HrefOptions', :selector, :channel, keyword_init: true)
 
+        # JSON Schema description exported via +schema_doc+.
+        DESCRIPTION = 'Return the absolute URL from the selected element\'s `href` attribute ' \
+                      '(relative hrefs are resolved against the channel URL).'
+
+        # Example extractor name values for JSON Schema +examples+.
+        EXAMPLES = [
+          'href'
+        ].freeze
+
+        # @return [Hash{Symbol => Object}] JSON Schema fragment for this extractor name
+        def self.schema_doc = SchemaDoc.for_extractor(name: :href, klass: self)
+
         ##
         # Initializes the Href extractor.
         #

data/lib/html2rss/selectors/extractors/html.rb

@@ -26,6 +26,18 @@ module Html2rss
         # The available options for the html extractor.
         Options = Struct.new('HtmlOptions', :selector, keyword_init: true)
 
+        # JSON Schema description exported via +schema_doc+.
+        DESCRIPTION = 'Return the outer HTML of the selected element. ' \
+                      'Sanitize during post-processing (e.g. `sanitize_html`).'
+
+        # Example extractor name values for JSON Schema +examples+.
+        EXAMPLES = [
+          'html'
+        ].freeze
+
+        # @return [Hash{Symbol => Object}] JSON Schema fragment for this extractor name
+        def self.schema_doc = SchemaDoc.for_extractor(name: :html, klass: self)
+
         ##
         # Initializes the Html extractor.
         #

data/lib/html2rss/selectors/extractors/static.rb

@@ -16,9 +16,23 @@ module Html2rss
       # Would return:
       #    'Foobar'
       class Static
+        # Config-facing option types (excluding shared selector fields supplied at runtime).
+        OPTION_TYPES = { static: String }.freeze
+
         # The available option for the static extractor.
         Options = Struct.new('StaticOptions', :static, keyword_init: true)
 
+        # JSON Schema description exported via +schema_doc+.
+        DESCRIPTION = 'Return a fixed value from sibling selector option `static` (no DOM read).'
+
+        # Example extractor name values for JSON Schema +examples+.
+        EXAMPLES = [
+          'static'
+        ].freeze
+
+        # @return [Hash{Symbol => Object}] JSON Schema fragment for this extractor name
+        def self.schema_doc = SchemaDoc.for_extractor(name: :static, klass: self)
+
         ##
         # Initializes the Static extractor.
         #

data/lib/html2rss/selectors/extractors/text.rb

@@ -24,6 +24,17 @@ module Html2rss
         # The available options for the text extractor.
         Options = Struct.new('TextOptions', :selector, keyword_init: true)
 
+        # JSON Schema description exported via +schema_doc+.
+        DESCRIPTION = 'Return collapsed visible text of the selected element (default extractor).'
+
+        # Example extractor name values for JSON Schema +examples+.
+        EXAMPLES = [
+          'text'
+        ].freeze
+
+        # @return [Hash{Symbol => Object}] JSON Schema fragment for this extractor name
+        def self.schema_doc = SchemaDoc.for_extractor(name: :text, klass: self)
+
         ##
         # Initializes the Text extractor.
         #

data/lib/html2rss/selectors/post_processors/gsub.rb

@@ -29,6 +29,24 @@ module Html2rss
       #
       # See the doc on [String#gsub](https://ruby-doc.org/core/String.html#method-i-gsub) for more info.
       class Gsub < Base
+        # Required config field types (validator introspection via +Options+).
+        OPTION_TYPES = { pattern: String, replacement: String }.freeze
+
+        # Config fields required by this post-processor (validator / schema introspection).
+        Options = Struct.new(*OPTION_TYPES.keys, keyword_init: true)
+
+        # JSON Schema description exported via +schema_doc+.
+        DESCRIPTION = 'Replace matches of `pattern` in the extracted string with `replacement` ' \
+                      '(Ruby String#gsub; pattern may be a regexp-like string).'
+
+        # Example post-process objects for JSON Schema +examples+.
+        EXAMPLES = [
+          { 'name' => 'gsub', 'pattern' => 'boo', 'replacement' => 'baz' }
+        ].freeze
+
+        # @return [Hash{Symbol => Object}] JSON Schema fragment for this post-processor
+        def self.schema_doc = SchemaDoc.for_post_processor(name: :gsub, klass: self)
+
         # @param value [String] extracted selector value
         # @param context [Selectors::Context] post-processor context
         # @return [void]

data/lib/html2rss/selectors/post_processors/html_to_markdown.rb

@@ -28,6 +28,17 @@ module Html2rss
       # Would return:
       #    'Lorem **ipsum** dolor'
       class HtmlToMarkdown < Base
+        # JSON Schema description exported via +schema_doc+.
+        DESCRIPTION = 'Sanitize HTML then convert it to Markdown (via ReverseMarkdown).'
+
+        # Example post-process objects for JSON Schema +examples+.
+        EXAMPLES = [
+          { 'name' => 'html_to_markdown' }
+        ].freeze
+
+        # @return [Hash{Symbol => Object}] JSON Schema fragment for this post-processor
+        def self.schema_doc = SchemaDoc.for_post_processor(name: :html_to_markdown, klass: self)
+
         # @param value [String] extracted selector value
         # @param context [Selectors::Context] post-processor context
         # @return [void]

data/lib/html2rss/selectors/post_processors/markdown_to_html.rb

@@ -33,6 +33,18 @@ module Html2rss
       #
       #    <p>Price: 12.34</p>
       class MarkdownToHtml < Base
+        # JSON Schema description exported via +schema_doc+.
+        DESCRIPTION = 'Convert Markdown to HTML (Kramdown) and sanitize the result. ' \
+                      'Often chained after `template`.'
+
+        # Example post-process objects for JSON Schema +examples+.
+        EXAMPLES = [
+          { 'name' => 'markdown_to_html' }
+        ].freeze
+
+        # @return [Hash{Symbol => Object}] JSON Schema fragment for this post-processor
+        def self.schema_doc = SchemaDoc.for_post_processor(name: :markdown_to_html, klass: self)
+
         # @param value [String] extracted selector value
         # @param context [Selectors::Context] post-processor context
         # @return [void]

data/lib/html2rss/selectors/post_processors/parse_time.rb

@@ -27,6 +27,17 @@ module Html2rss
       #
       # It uses `Time.parse`.
       class ParseTime < Base
+        # JSON Schema description exported via +schema_doc+.
+        DESCRIPTION = 'Parse a time string with Time.parse and return RFC822, using the channel `time_zone`.'
+
+        # Example post-process objects for JSON Schema +examples+.
+        EXAMPLES = [
+          { 'name' => 'parse_time' }
+        ].freeze
+
+        # @return [Hash{Symbol => Object}] JSON Schema fragment for this post-processor
+        def self.schema_doc = SchemaDoc.for_post_processor(name: :parse_time, klass: self)
+
         # @param value [String] extracted selector value
         # @param context [Selectors::Context] post-processor context
         # @return [void]

data/lib/html2rss/selectors/post_processors/parse_uri.rb

@@ -23,6 +23,17 @@ module Html2rss
       # Would return:
       #    'http://why-not-use-a-link.uh'
       class ParseUri < Base
+        # JSON Schema description exported via +schema_doc+.
+        DESCRIPTION = 'Normalize a URL string; resolve relative URLs against the channel URL.'
+
+        # Example post-process objects for JSON Schema +examples+.
+        EXAMPLES = [
+          { 'name' => 'parse_uri' }
+        ].freeze
+
+        # @return [Hash{Symbol => Object}] JSON Schema fragment for this post-processor
+        def self.schema_doc = SchemaDoc.for_post_processor(name: :parse_uri, klass: self)
+
         # @param value [String] extracted selector value
         # @param _context [Selectors::Context] post-processor context
         # @return [void]