html2rss 0.25.0 → 0.26.0

data/lib/html2rss/scoring/ranked_segment.rb

@@ -5,18 +5,6 @@ module Html2rss
     ##
     # A segment paired with its composite score.
     RankedSegment = Data.define(:segment, :score) do
-      ##
-      # @param segment [Html2rss::AutoSource::Segment]
-      # @param score [Score]
-      # @return [RankedSegment]
-      # @raise [ArgumentError] on invalid types
-      def self.build(segment:, score:)
-        raise ArgumentError, 'segment must be AutoSource::Segment' unless segment.is_a?(Html2rss::AutoSource::Segment)
-        raise ArgumentError, 'score must be Scoring::Score' unless score.is_a?(Score)
-
-        new(segment:, score:)
-      end
-
       ##
       # @return [SST::Node]
       def root_node = segment.root_node

data/lib/html2rss/scoring/score.rb

@@ -4,26 +4,7 @@ module Html2rss
   module Scoring
     ##
     # Composite score with quality/junk split and optional typed feature breakdown.
-    Score = Data.define(:composite, :quality, :junk, :breakdown) do
-      ##
-      # @param composite [Numeric]
-      # @param quality [Numeric, nil]
-      # @param junk [Numeric, nil]
-      # @param breakdown [Hash{Symbol => Numeric}, nil]
-      # @return [Score]
-      # @raise [ArgumentError] when composite is not numeric
-      def self.build(composite:, quality: nil, junk: nil, breakdown: nil)
-        raise ArgumentError, 'composite must be Numeric' unless composite.is_a?(Numeric)
-
-        parts = breakdown.nil? ? Score::EMPTY_BREAKDOWN : breakdown.transform_keys { FeatureId.assert!(_1) }.freeze
-        new(
-          composite: composite.to_f,
-          quality: (quality.nil? ? composite : quality).to_f,
-          junk: (junk || 0).to_f,
-          breakdown: parts
-        )
-      end
-    end
+    Score = Data.define(:composite, :quality, :junk, :breakdown)
     # Shared empty feature breakdown for scores without per-feature tallies.
     Score::EMPTY_BREAKDOWN = {}.freeze
   end

data/lib/html2rss/scoring.rb

@@ -4,30 +4,5 @@ module Html2rss
   ##
   # Declarative feature scoring for AutoSource SST segments.
   module Scoring
-    ##
-    # Closed set of feature identifiers used by the scoring registry.
-    module FeatureId
-      # Closed set of allowed feature identifier symbols.
-      IDS = %i[
-        title_word_count_ge3 title_word_count_ge7 path_length_gt6 content_path
-        publish_marker descriptive_context article_container content_tokens
-        non_content_utility_path utility_prefix_title_short shallow weak_container
-        recommended_title_non_content high_confidence_junk_path junk_tokens
-        heading_anchor heading_text_match meaningful_text content_like_destination
-        cluster_size cluster_avg_words cluster_heading cluster_time cluster_date
-      ].to_set.freeze
-
-      module_function
-
-      ##
-      # @param id [Symbol]
-      # @return [Symbol]
-      # @raise [ArgumentError] when id is not in the closed set
-      def assert!(id)
-        raise ArgumentError, "unknown FeatureId: #{id.inspect}" unless IDS.include?(id)
-
-        id
-      end
-    end
   end
 end

data/lib/html2rss/selectors/extractors/attribute.rb

@@ -24,9 +24,24 @@ module Html2rss
       # In case you're extracting a date or a time, consider parsing it
       # during post processing with {PostProcessors::ParseTime}.
       class Attribute
+        # Config-facing option types (excluding shared selector fields supplied at runtime).
+        OPTION_TYPES = { attribute: String }.freeze
+
         # The available options for the attribute extractor.
         Options = Struct.new('AttributeOptions', :selector, :attribute, keyword_init: true)
 
+        # JSON Schema description exported via +schema_doc+.
+        DESCRIPTION = 'Return the value of an HTML attribute on the selected element. ' \
+                      'Requires sibling selector option `attribute` (attribute name).'
+
+        # Example extractor name values for JSON Schema +examples+.
+        EXAMPLES = [
+          'attribute'
+        ].freeze
+
+        # @return [Hash{Symbol => Object}] JSON Schema fragment for this extractor name
+        def self.schema_doc = SchemaDoc.for_extractor(name: :attribute, klass: self)
+
         ##
         # Initializes the Attribute extractor.
         #

data/lib/html2rss/selectors/extractors/href.rb

@@ -27,6 +27,18 @@ module Html2rss
         # The available options for the href (attribute) extractor.
         Options = Struct.new('HrefOptions', :selector, :channel, keyword_init: true)
 
+        # JSON Schema description exported via +schema_doc+.
+        DESCRIPTION = 'Return the absolute URL from the selected element\'s `href` attribute ' \
+                      '(relative hrefs are resolved against the channel URL).'
+
+        # Example extractor name values for JSON Schema +examples+.
+        EXAMPLES = [
+          'href'
+        ].freeze
+
+        # @return [Hash{Symbol => Object}] JSON Schema fragment for this extractor name
+        def self.schema_doc = SchemaDoc.for_extractor(name: :href, klass: self)
+
         ##
         # Initializes the Href extractor.
         #

data/lib/html2rss/selectors/extractors/html.rb

@@ -26,6 +26,18 @@ module Html2rss
         # The available options for the html extractor.
         Options = Struct.new('HtmlOptions', :selector, keyword_init: true)
 
+        # JSON Schema description exported via +schema_doc+.
+        DESCRIPTION = 'Return the outer HTML of the selected element. ' \
+                      'Sanitize during post-processing (e.g. `sanitize_html`).'
+
+        # Example extractor name values for JSON Schema +examples+.
+        EXAMPLES = [
+          'html'
+        ].freeze
+
+        # @return [Hash{Symbol => Object}] JSON Schema fragment for this extractor name
+        def self.schema_doc = SchemaDoc.for_extractor(name: :html, klass: self)
+
         ##
         # Initializes the Html extractor.
         #

data/lib/html2rss/selectors/extractors/static.rb

@@ -16,9 +16,23 @@ module Html2rss
       # Would return:
       #    'Foobar'
       class Static
+        # Config-facing option types (excluding shared selector fields supplied at runtime).
+        OPTION_TYPES = { static: String }.freeze
+
         # The available option for the static extractor.
         Options = Struct.new('StaticOptions', :static, keyword_init: true)
 
+        # JSON Schema description exported via +schema_doc+.
+        DESCRIPTION = 'Return a fixed value from sibling selector option `static` (no DOM read).'
+
+        # Example extractor name values for JSON Schema +examples+.
+        EXAMPLES = [
+          'static'
+        ].freeze
+
+        # @return [Hash{Symbol => Object}] JSON Schema fragment for this extractor name
+        def self.schema_doc = SchemaDoc.for_extractor(name: :static, klass: self)
+
         ##
         # Initializes the Static extractor.
         #

data/lib/html2rss/selectors/extractors/text.rb

@@ -24,6 +24,17 @@ module Html2rss
         # The available options for the text extractor.
         Options = Struct.new('TextOptions', :selector, keyword_init: true)
 
+        # JSON Schema description exported via +schema_doc+.
+        DESCRIPTION = 'Return collapsed visible text of the selected element (default extractor).'
+
+        # Example extractor name values for JSON Schema +examples+.
+        EXAMPLES = [
+          'text'
+        ].freeze
+
+        # @return [Hash{Symbol => Object}] JSON Schema fragment for this extractor name
+        def self.schema_doc = SchemaDoc.for_extractor(name: :text, klass: self)
+
         ##
         # Initializes the Text extractor.
         #

data/lib/html2rss/selectors/post_processors/gsub.rb

@@ -29,6 +29,24 @@ module Html2rss
       #
       # See the doc on [String#gsub](https://ruby-doc.org/core/String.html#method-i-gsub) for more info.
       class Gsub < Base
+        # Required config field types (validator introspection via +Options+).
+        OPTION_TYPES = { pattern: String, replacement: String }.freeze
+
+        # Config fields required by this post-processor (validator / schema introspection).
+        Options = Struct.new(*OPTION_TYPES.keys, keyword_init: true)
+
+        # JSON Schema description exported via +schema_doc+.
+        DESCRIPTION = 'Replace matches of `pattern` in the extracted string with `replacement` ' \
+                      '(Ruby String#gsub; pattern may be a regexp-like string).'
+
+        # Example post-process objects for JSON Schema +examples+.
+        EXAMPLES = [
+          { 'name' => 'gsub', 'pattern' => 'boo', 'replacement' => 'baz' }
+        ].freeze
+
+        # @return [Hash{Symbol => Object}] JSON Schema fragment for this post-processor
+        def self.schema_doc = SchemaDoc.for_post_processor(name: :gsub, klass: self)
+
         # @param value [String] extracted selector value
         # @param context [Selectors::Context] post-processor context
         # @return [void]

data/lib/html2rss/selectors/post_processors/html_to_markdown.rb

@@ -28,6 +28,17 @@ module Html2rss
       # Would return:
       #    'Lorem **ipsum** dolor'
       class HtmlToMarkdown < Base
+        # JSON Schema description exported via +schema_doc+.
+        DESCRIPTION = 'Sanitize HTML then convert it to Markdown (via ReverseMarkdown).'
+
+        # Example post-process objects for JSON Schema +examples+.
+        EXAMPLES = [
+          { 'name' => 'html_to_markdown' }
+        ].freeze
+
+        # @return [Hash{Symbol => Object}] JSON Schema fragment for this post-processor
+        def self.schema_doc = SchemaDoc.for_post_processor(name: :html_to_markdown, klass: self)
+
         # @param value [String] extracted selector value
         # @param context [Selectors::Context] post-processor context
         # @return [void]

data/lib/html2rss/selectors/post_processors/markdown_to_html.rb

@@ -33,6 +33,18 @@ module Html2rss
       #
       #    <p>Price: 12.34</p>
       class MarkdownToHtml < Base
+        # JSON Schema description exported via +schema_doc+.
+        DESCRIPTION = 'Convert Markdown to HTML (Kramdown) and sanitize the result. ' \
+                      'Often chained after `template`.'
+
+        # Example post-process objects for JSON Schema +examples+.
+        EXAMPLES = [
+          { 'name' => 'markdown_to_html' }
+        ].freeze
+
+        # @return [Hash{Symbol => Object}] JSON Schema fragment for this post-processor
+        def self.schema_doc = SchemaDoc.for_post_processor(name: :markdown_to_html, klass: self)
+
         # @param value [String] extracted selector value
         # @param context [Selectors::Context] post-processor context
         # @return [void]

data/lib/html2rss/selectors/post_processors/parse_time.rb

@@ -27,6 +27,17 @@ module Html2rss
       #
       # It uses `Time.parse`.
       class ParseTime < Base
+        # JSON Schema description exported via +schema_doc+.
+        DESCRIPTION = 'Parse a time string with Time.parse and return RFC822, using the channel `time_zone`.'
+
+        # Example post-process objects for JSON Schema +examples+.
+        EXAMPLES = [
+          { 'name' => 'parse_time' }
+        ].freeze
+
+        # @return [Hash{Symbol => Object}] JSON Schema fragment for this post-processor
+        def self.schema_doc = SchemaDoc.for_post_processor(name: :parse_time, klass: self)
+
         # @param value [String] extracted selector value
         # @param context [Selectors::Context] post-processor context
         # @return [void]

data/lib/html2rss/selectors/post_processors/parse_uri.rb

@@ -23,6 +23,17 @@ module Html2rss
       # Would return:
       #    'http://why-not-use-a-link.uh'
       class ParseUri < Base
+        # JSON Schema description exported via +schema_doc+.
+        DESCRIPTION = 'Normalize a URL string; resolve relative URLs against the channel URL.'
+
+        # Example post-process objects for JSON Schema +examples+.
+        EXAMPLES = [
+          { 'name' => 'parse_uri' }
+        ].freeze
+
+        # @return [Hash{Symbol => Object}] JSON Schema fragment for this post-processor
+        def self.schema_doc = SchemaDoc.for_post_processor(name: :parse_uri, klass: self)
+
         # @param value [String] extracted selector value
         # @param _context [Selectors::Context] post-processor context
         # @return [void]

data/lib/html2rss/selectors/post_processors/sanitize_html.rb

@@ -38,6 +38,15 @@ module Html2rss
       # Would return:
       #    '<p>Lorem <b>ipsum</b> dolor ...</p>'
       class SanitizeHtml < Base
+        # JSON Schema description exported via +schema_doc+.
+        DESCRIPTION = 'Sanitize HTML (sanitize gem RELAXED plus html2rss defaults: absolute URLs, ' \
+                      'safe link/img attributes, wrap lone images in anchors).'
+
+        # Example post-process objects for JSON Schema +examples+.
+        EXAMPLES = [
+          { 'name' => 'sanitize_html' }
+        ].freeze
+
         # @see https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Referrer-Policy
         TAG_ATTRIBUTES = {
           'a' => {
@@ -81,6 +90,10 @@ module Html2rss
             'preload' => 'none'
           }
         }.freeze
+
+        # @return [Hash{Symbol => Object}] JSON Schema fragment for this post-processor
+        def self.schema_doc = SchemaDoc.for_post_processor(name: :sanitize_html, klass: self)
+
         # @param value [String] extracted selector value
         # @param context [Selectors::Context] post-processor context
         # @return [void]

data/lib/html2rss/selectors/post_processors/substring.rb

@@ -30,6 +30,27 @@ module Html2rss
       # Would return:
       #    'bar'
       class Substring < Base
+        # Required config field types (validator introspection via +Options+).
+        OPTION_TYPES = { start: Integer }.freeze
+
+        # Optional config field types (validated when the key is present and non-nil).
+        OPTIONAL_OPTION_TYPES = { end: Integer }.freeze
+
+        # Config fields required by this post-processor (validator / schema introspection).
+        Options = Struct.new(*OPTION_TYPES.keys, keyword_init: true)
+
+        # JSON Schema description exported via +schema_doc+.
+        DESCRIPTION = 'Return a slice of the extracted string using Integer `start` and optional `end` ' \
+                      '(Ruby String#[] range semantics; end may be omitted).'
+
+        # Example post-process objects for JSON Schema +examples+.
+        EXAMPLES = [
+          { 'name' => 'substring', 'start' => 4, 'end' => 6 }
+        ].freeze
+
+        # @return [Hash{Symbol => Object}] JSON Schema fragment for this post-processor
+        def self.schema_doc = SchemaDoc.for_post_processor(name: :substring, klass: self)
+
         # @param value [String] extracted selector value
         # @param context [Selectors::Context] post-processor context
         # @return [void]

data/lib/html2rss/selectors/post_processors/template.rb

@@ -34,6 +34,26 @@ module Html2rss
       # Would return:
       #    'Product (23,42€)'
       class Template < Base
+        # Required config field types (validator introspection via +Options+).
+        OPTION_TYPES = { string: String }.freeze
+
+        # Config fields required by this post-processor (validator / schema introspection).
+        Options = Struct.new(*OPTION_TYPES.keys, keyword_init: true)
+
+        # JSON Schema description exported via +schema_doc+.
+        # rubocop:disable Style/FormatStringToken -- documents Kernel#format `%{key}` placeholders
+        DESCRIPTION = 'Format a string with Kernel#format-style placeholders (`%{key}` / `%<key>s`). ' \
+                      '`%{self}` is the current selector value; other keys resolve sibling selectors.'
+
+        # Example post-process objects for JSON Schema +examples+.
+        EXAMPLES = [
+          { 'name' => 'template', 'string' => '`%{self}` (`%{price}`)' }
+        ].freeze
+        # rubocop:enable Style/FormatStringToken
+
+        # @return [Hash{Symbol => Object}] JSON Schema fragment for this post-processor
+        def self.schema_doc = SchemaDoc.for_post_processor(name: :template, klass: self)
+
         # @param value [String] extracted selector value
         # @param context [Selectors::Context] post-processor context
         # @return [void]

data/lib/html2rss/selectors/schema_doc.rb

@@ -0,0 +1,99 @@
+# frozen_string_literal: true
+
+module Html2rss
+  class Selectors
+    ##
+    # Builds JSON Schema fragments from extractor / post-processor class constants.
+    #
+    # Each registry class owns +DESCRIPTION+, +EXAMPLES+, and optional +OPTION_TYPES+;
+    # this module is the export adapter only.
+    module SchemaDoc
+      # Maps Ruby option types to JSON Schema +type+ strings.
+      RUBY_TO_JSON_TYPE = {
+        String => 'string',
+        Integer => 'integer',
+        Hash => 'object',
+        Array => 'array'
+      }.freeze
+
+      module_function
+
+      ##
+      # @param name [Symbol, String] registry key
+      # @param klass [Class] post-processor class
+      # @return [Hash{Symbol => Object}] JSON Schema object for one post-processor
+      def for_post_processor(name:, klass:)
+        name = name.to_s
+        {
+          type: 'object',
+          title: name,
+          description: klass::DESCRIPTION,
+          examples: klass::EXAMPLES,
+          properties: post_processor_properties(name, klass),
+          required: post_processor_required(klass),
+          additionalProperties: true
+        }
+      end
+
+      ##
+      # @param name [Symbol, String] registry key
+      # @param klass [Class] extractor class
+      # @return [Hash{Symbol => Object}] JSON Schema for one extractor name
+      def for_extractor(name:, klass:)
+        name = name.to_s
+        {
+          type: 'string',
+          const: name,
+          title: name,
+          description: klass::DESCRIPTION,
+          examples: klass::EXAMPLES
+        }
+      end
+
+      ##
+      # @param name [String] registry key
+      # @param klass [Class]
+      # @return [Hash{Symbol => Hash}]
+      def post_processor_properties(name, klass)
+        properties = { name: { type: 'string', const: name } }
+        option_types_for(klass).each do |field, ruby_type|
+          properties[field] = { type: json_type_for(ruby_type) }
+        end
+        properties
+      end
+      module_function :post_processor_properties
+
+      ##
+      # @param klass [Class]
+      # @return [Array<String>]
+      def post_processor_required(klass)
+        required = ['name']
+        return required unless klass.const_defined?(:OPTION_TYPES)
+
+        required + klass::OPTION_TYPES.keys.map(&:to_s)
+      end
+      module_function :post_processor_required
+
+      ##
+      # @param klass [Class]
+      # @return [Hash{Symbol => Class}]
+      def option_types_for(klass)
+        types = {}
+        types.merge!(klass::OPTION_TYPES) if klass.const_defined?(:OPTION_TYPES)
+        types.merge!(klass::OPTIONAL_OPTION_TYPES) if klass.const_defined?(:OPTIONAL_OPTION_TYPES)
+        types
+      end
+      module_function :option_types_for
+
+      ##
+      # @param ruby_type [Class]
+      # @return [String]
+      def json_type_for(ruby_type)
+        RUBY_TO_JSON_TYPE.fetch(ruby_type) do
+          raise ArgumentError, "unsupported OPTION_TYPES mapping for #{ruby_type}"
+        end
+      end
+      module_function :json_type_for
+    end
+  end
+end

data/lib/html2rss/url.rb

@@ -21,6 +21,7 @@ module Html2rss
   #   url = Url.from_relative('/foo-bar/baz.txt', 'https://example.com')
   #   url.titleized # => "Foo Bar Baz"
   #   url.channel_titleized # => "example.com: Foo Bar Baz"
+  # rubocop:disable Metrics/ClassLength -- value object owns resolve/sanitize/titleize/identity
   class Url
     include Comparable
 
@@ -189,6 +190,14 @@ module Html2rss
       self.class.from_absolute(uri.normalize.to_s)
     end
 
+    ##
+    # Returns a copy of the URL with the fragment removed (dedup / self-link identity).
+    #
+    # @return [Url] this URL when already fragment-free; otherwise a new URL
+    def without_fragment
+      @uri.fragment ? self.class.from_absolute(@uri.omit(:fragment).normalize.to_s) : self
+    end
+
     ##
     # Returns a copy of the URL with the provided query values.
     #
@@ -259,13 +268,7 @@ module Html2rss
     # @param other [Object] the other object to compare with
     # @return [Boolean] true if the URLs are equal
     def ==(other) = other.is_a?(Url) && to_s == other.to_s
-
-    ##
-    # Supports hash-based comparisons by ensuring equality semantics match `hash`.
-    #
-    # @param other [Object] the other object to compare with
-    # @return [Boolean] true if the URLs are considered equal
-    def eql?(other) = other.is_a?(Url) && to_s == other.to_s
+    alias eql? ==
 
     ##
     # Returns the hash code for this URL.
@@ -279,4 +282,5 @@ module Html2rss
     # @return [String] the debug representation
     def inspect = "#<#{self.class}:#{object_id} @uri=#{@uri.inspect}>"
   end
+  # rubocop:enable Metrics/ClassLength
 end

data/lib/html2rss/version.rb

@@ -2,6 +2,6 @@
 
 module Html2rss
   # Current application version.
-  VERSION = '0.25.0'
+  VERSION = '0.26.0'
   public_constant :VERSION
 end

data/lib/html2rss.rb

@@ -5,7 +5,8 @@ require 'zeitwerk'
 loader = Zeitwerk::Loader.for_gem
 loader.inflector.inflect(
   'cli' => 'CLI',
-  'sst' => 'SST'
+  'sst' => 'SST',
+  'mcp' => 'MCP'
 )
 loader.setup
 
@@ -118,6 +119,36 @@ module Html2rss
                                        local_file_path:, limit:))
   end
 
+  ##
+  # Analyzes a URL and produces a reusable YAML-ready feed config hash.
+  #
+  # Uses auto-source discovery to extract articles, then derives CSS selectors
+  # from the structural analysis.
+  #
+  # @param url [String] source page URL
+  # @param strategy [Symbol] request strategy (+:auto+, +:faraday+, +:botasaurus+)
+  # @param items_selector [String, nil] optional CSS selector hint for items
+  # @param max_redirects [Integer, nil] optional redirect limit override
+  # @param max_requests [Integer, nil] optional request budget override
+  # @param limit [Integer, nil] max articles to keep
+  # @param local_file_path [String, nil] optional local HTML file path
+  # @return [Hash] feed config hash with +:channel+ and +:selectors+
+  def self.capture(url,
+                   strategy: :auto,
+                   items_selector: nil,
+                   max_redirects: nil,
+                   max_requests: nil,
+                   limit: nil,
+                   local_file_path: nil)
+    Capture.build(url,
+                  strategy:,
+                  items_selector:,
+                  max_redirects:,
+                  max_requests:,
+                  limit:,
+                  local_file_path:).config
+  end
+
   # rubocop:enable Metrics/ParameterLists
 
   # rubocop:disable ThreadSafety/ClassInstanceVariable