html2rss 0.25.0 → 0.27.0

data/lib/html2rss/selectors/post_processors/gsub.rb

@@ -29,6 +29,24 @@ module Html2rss
       #
       # See the doc on [String#gsub](https://ruby-doc.org/core/String.html#method-i-gsub) for more info.
       class Gsub < Base
+        # Required config field types (validator introspection via +Options+).
+        OPTION_TYPES = { pattern: String, replacement: String }.freeze
+
+        # Config fields required by this post-processor (validator / schema introspection).
+        Options = Struct.new(*OPTION_TYPES.keys, keyword_init: true)
+
+        # JSON Schema description exported via +schema_doc+.
+        DESCRIPTION = 'Replace matches of `pattern` in the extracted string with `replacement` ' \
+                      '(Ruby String#gsub; pattern may be a regexp-like string).'
+
+        # Example post-process objects for JSON Schema +examples+.
+        EXAMPLES = [
+          { 'name' => 'gsub', 'pattern' => 'boo', 'replacement' => 'baz' }
+        ].freeze
+
+        # @return [Hash{Symbol => Object}] JSON Schema fragment for this post-processor
+        def self.schema_doc = SchemaDoc.for_post_processor(name: :gsub, klass: self)
+
         # @param value [String] extracted selector value
         # @param context [Selectors::Context] post-processor context
         # @return [void]

data/lib/html2rss/selectors/post_processors/html_to_markdown.rb

@@ -28,6 +28,17 @@ module Html2rss
       # Would return:
       #    'Lorem **ipsum** dolor'
       class HtmlToMarkdown < Base
+        # JSON Schema description exported via +schema_doc+.
+        DESCRIPTION = 'Sanitize HTML then convert it to Markdown (via ReverseMarkdown).'
+
+        # Example post-process objects for JSON Schema +examples+.
+        EXAMPLES = [
+          { 'name' => 'html_to_markdown' }
+        ].freeze
+
+        # @return [Hash{Symbol => Object}] JSON Schema fragment for this post-processor
+        def self.schema_doc = SchemaDoc.for_post_processor(name: :html_to_markdown, klass: self)
+
         # @param value [String] extracted selector value
         # @param context [Selectors::Context] post-processor context
         # @return [void]

data/lib/html2rss/selectors/post_processors/markdown_to_html.rb

@@ -33,6 +33,18 @@ module Html2rss
       #
       #    <p>Price: 12.34</p>
       class MarkdownToHtml < Base
+        # JSON Schema description exported via +schema_doc+.
+        DESCRIPTION = 'Convert Markdown to HTML (Kramdown) and sanitize the result. ' \
+                      'Often chained after `template`.'
+
+        # Example post-process objects for JSON Schema +examples+.
+        EXAMPLES = [
+          { 'name' => 'markdown_to_html' }
+        ].freeze
+
+        # @return [Hash{Symbol => Object}] JSON Schema fragment for this post-processor
+        def self.schema_doc = SchemaDoc.for_post_processor(name: :markdown_to_html, klass: self)
+
         # @param value [String] extracted selector value
         # @param context [Selectors::Context] post-processor context
         # @return [void]

data/lib/html2rss/selectors/post_processors/parse_time.rb

@@ -27,6 +27,17 @@ module Html2rss
       #
       # It uses `Time.parse`.
       class ParseTime < Base
+        # JSON Schema description exported via +schema_doc+.
+        DESCRIPTION = 'Parse a time string with Time.parse and return RFC822, using the channel `time_zone`.'
+
+        # Example post-process objects for JSON Schema +examples+.
+        EXAMPLES = [
+          { 'name' => 'parse_time' }
+        ].freeze
+
+        # @return [Hash{Symbol => Object}] JSON Schema fragment for this post-processor
+        def self.schema_doc = SchemaDoc.for_post_processor(name: :parse_time, klass: self)
+
         # @param value [String] extracted selector value
         # @param context [Selectors::Context] post-processor context
         # @return [void]

data/lib/html2rss/selectors/post_processors/parse_uri.rb

@@ -23,6 +23,17 @@ module Html2rss
       # Would return:
       #    'http://why-not-use-a-link.uh'
       class ParseUri < Base
+        # JSON Schema description exported via +schema_doc+.
+        DESCRIPTION = 'Normalize a URL string; resolve relative URLs against the channel URL.'
+
+        # Example post-process objects for JSON Schema +examples+.
+        EXAMPLES = [
+          { 'name' => 'parse_uri' }
+        ].freeze
+
+        # @return [Hash{Symbol => Object}] JSON Schema fragment for this post-processor
+        def self.schema_doc = SchemaDoc.for_post_processor(name: :parse_uri, klass: self)
+
         # @param value [String] extracted selector value
         # @param _context [Selectors::Context] post-processor context
         # @return [void]

data/lib/html2rss/selectors/post_processors/sanitize_html.rb

@@ -38,6 +38,15 @@ module Html2rss
       # Would return:
       #    '<p>Lorem <b>ipsum</b> dolor ...</p>'
       class SanitizeHtml < Base
+        # JSON Schema description exported via +schema_doc+.
+        DESCRIPTION = 'Sanitize HTML (sanitize gem RELAXED plus html2rss defaults: absolute URLs, ' \
+                      'safe link/img attributes, wrap lone images in anchors).'
+
+        # Example post-process objects for JSON Schema +examples+.
+        EXAMPLES = [
+          { 'name' => 'sanitize_html' }
+        ].freeze
+
         # @see https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Referrer-Policy
         TAG_ATTRIBUTES = {
           'a' => {
@@ -81,6 +90,10 @@ module Html2rss
             'preload' => 'none'
           }
         }.freeze
+
+        # @return [Hash{Symbol => Object}] JSON Schema fragment for this post-processor
+        def self.schema_doc = SchemaDoc.for_post_processor(name: :sanitize_html, klass: self)
+
         # @param value [String] extracted selector value
         # @param context [Selectors::Context] post-processor context
         # @return [void]

data/lib/html2rss/selectors/post_processors/substring.rb

@@ -30,6 +30,27 @@ module Html2rss
       # Would return:
       #    'bar'
       class Substring < Base
+        # Required config field types (validator introspection via +Options+).
+        OPTION_TYPES = { start: Integer }.freeze
+
+        # Optional config field types (validated when the key is present and non-nil).
+        OPTIONAL_OPTION_TYPES = { end: Integer }.freeze
+
+        # Config fields required by this post-processor (validator / schema introspection).
+        Options = Struct.new(*OPTION_TYPES.keys, keyword_init: true)
+
+        # JSON Schema description exported via +schema_doc+.
+        DESCRIPTION = 'Return a slice of the extracted string using Integer `start` and optional `end` ' \
+                      '(Ruby String#[] range semantics; end may be omitted).'
+
+        # Example post-process objects for JSON Schema +examples+.
+        EXAMPLES = [
+          { 'name' => 'substring', 'start' => 4, 'end' => 6 }
+        ].freeze
+
+        # @return [Hash{Symbol => Object}] JSON Schema fragment for this post-processor
+        def self.schema_doc = SchemaDoc.for_post_processor(name: :substring, klass: self)
+
         # @param value [String] extracted selector value
         # @param context [Selectors::Context] post-processor context
         # @return [void]

data/lib/html2rss/selectors/post_processors/template.rb

@@ -34,6 +34,26 @@ module Html2rss
       # Would return:
       #    'Product (23,42€)'
       class Template < Base
+        # Required config field types (validator introspection via +Options+).
+        OPTION_TYPES = { string: String }.freeze
+
+        # Config fields required by this post-processor (validator / schema introspection).
+        Options = Struct.new(*OPTION_TYPES.keys, keyword_init: true)
+
+        # JSON Schema description exported via +schema_doc+.
+        # rubocop:disable Style/FormatStringToken -- documents Kernel#format `%{key}` placeholders
+        DESCRIPTION = 'Format a string with Kernel#format-style placeholders (`%{key}` / `%<key>s`). ' \
+                      '`%{self}` is the current selector value; other keys resolve sibling selectors.'
+
+        # Example post-process objects for JSON Schema +examples+.
+        EXAMPLES = [
+          { 'name' => 'template', 'string' => '`%{self}` (`%{price}`)' }
+        ].freeze
+        # rubocop:enable Style/FormatStringToken
+
+        # @return [Hash{Symbol => Object}] JSON Schema fragment for this post-processor
+        def self.schema_doc = SchemaDoc.for_post_processor(name: :template, klass: self)
+
         # @param value [String] extracted selector value
         # @param context [Selectors::Context] post-processor context
         # @return [void]

data/lib/html2rss/selectors/schema_doc.rb

@@ -0,0 +1,99 @@
+# frozen_string_literal: true
+
+module Html2rss
+  class Selectors
+    ##
+    # Builds JSON Schema fragments from extractor / post-processor class constants.
+    #
+    # Each registry class owns +DESCRIPTION+, +EXAMPLES+, and optional +OPTION_TYPES+;
+    # this module is the export adapter only.
+    module SchemaDoc
+      # Maps Ruby option types to JSON Schema +type+ strings.
+      RUBY_TO_JSON_TYPE = {
+        String => 'string',
+        Integer => 'integer',
+        Hash => 'object',
+        Array => 'array'
+      }.freeze
+
+      module_function
+
+      ##
+      # @param name [Symbol, String] registry key
+      # @param klass [Class] post-processor class
+      # @return [Hash{Symbol => Object}] JSON Schema object for one post-processor
+      def for_post_processor(name:, klass:)
+        name = name.to_s
+        {
+          type: 'object',
+          title: name,
+          description: klass::DESCRIPTION,
+          examples: klass::EXAMPLES,
+          properties: post_processor_properties(name, klass),
+          required: post_processor_required(klass),
+          additionalProperties: true
+        }
+      end
+
+      ##
+      # @param name [Symbol, String] registry key
+      # @param klass [Class] extractor class
+      # @return [Hash{Symbol => Object}] JSON Schema for one extractor name
+      def for_extractor(name:, klass:)
+        name = name.to_s
+        {
+          type: 'string',
+          const: name,
+          title: name,
+          description: klass::DESCRIPTION,
+          examples: klass::EXAMPLES
+        }
+      end
+
+      ##
+      # @param name [String] registry key
+      # @param klass [Class]
+      # @return [Hash{Symbol => Hash}]
+      def post_processor_properties(name, klass)
+        properties = { name: { type: 'string', const: name } }
+        option_types_for(klass).each do |field, ruby_type|
+          properties[field] = { type: json_type_for(ruby_type) }
+        end
+        properties
+      end
+      module_function :post_processor_properties
+
+      ##
+      # @param klass [Class]
+      # @return [Array<String>]
+      def post_processor_required(klass)
+        required = ['name']
+        return required unless klass.const_defined?(:OPTION_TYPES)
+
+        required + klass::OPTION_TYPES.keys.map(&:to_s)
+      end
+      module_function :post_processor_required
+
+      ##
+      # @param klass [Class]
+      # @return [Hash{Symbol => Class}]
+      def option_types_for(klass)
+        types = {}
+        types.merge!(klass::OPTION_TYPES) if klass.const_defined?(:OPTION_TYPES)
+        types.merge!(klass::OPTIONAL_OPTION_TYPES) if klass.const_defined?(:OPTIONAL_OPTION_TYPES)
+        types
+      end
+      module_function :option_types_for
+
+      ##
+      # @param ruby_type [Class]
+      # @return [String]
+      def json_type_for(ruby_type)
+        RUBY_TO_JSON_TYPE.fetch(ruby_type) do
+          raise ArgumentError, "unsupported OPTION_TYPES mapping for #{ruby_type}"
+        end
+      end
+      module_function :json_type_for
+    end
+  end
+end

data/lib/html2rss/selectors.rb

@@ -119,7 +119,8 @@ module Html2rss
         article_tag,
         base_url:,
         selected_anchor:,
-        fallback_anchorless: true
+        fallback_anchorless: true,
+        time_zone: @time_zone
       )
       return article_hash unless extracted
 

data/lib/html2rss/status.rb

@@ -8,7 +8,8 @@ module Html2rss
   # Stable telemetry payload for cross-repo consumers (e.g. html2rss-web observability).
   # Tallies and counters are validated and frozen at construction (including Marshal load).
   Status = Data.define(
-    :version, :scraper_tallies, :dedup_dropped, :selected_strategy, :attempt_count, :strategy_attempts
+    :version, :scraper_tallies, :dedup_dropped, :selected_strategy, :attempt_count,
+    :strategy_attempts, :admission_drops
   ) do
     class << self
       ##
@@ -19,8 +20,11 @@ module Html2rss
       # @param selected_strategy [Symbol, nil] concrete strategy that succeeded under +:auto+ (else +nil+)
       # @param attempt_count [Integer] auto-fallback attempt count (0 when not under +:auto+)
       # @param strategy_attempts [Array<Hash>] auto-fallback attempt hashes (empty outside +:auto+)
+      # @param admission_drops [Hash{String => Integer}] Cleanup reason → count (empty when unused)
       # @return [Html2rss::Status]
-      def build(articles:, dedup_dropped: 0, selected_strategy: nil, attempt_count: 0, strategy_attempts: [])
+      # rubocop:disable Metrics/ParameterLists -- Status kwargs stay co-located
+      def build(articles:, dedup_dropped: 0, selected_strategy: nil, attempt_count: 0,
+                strategy_attempts: [], admission_drops: {})
         tallies = articles.filter_map(&:scraper).tally.transform_keys { |klass| scraper_name(klass) }
         new(
           version: Html2rss::VERSION,
@@ -28,9 +32,11 @@ module Html2rss
           dedup_dropped:,
           selected_strategy:,
           attempt_count:,
-          strategy_attempts:
+          strategy_attempts:,
+          admission_drops:
         )
       end
+      # rubocop:enable Metrics/ParameterLists
 
       ##
       # @param klass [Class, #to_s] scraper class
@@ -47,9 +53,11 @@ module Html2rss
     # @param selected_strategy [Symbol, nil]
     # @param attempt_count [Integer]
     # @param strategy_attempts [Array<Hash>]
+    # @param admission_drops [Hash{String => Integer}]
     # rubocop:disable Metrics/ParameterLists, Metrics/MethodLength -- Status Data.define members
     def initialize(
-      version:, scraper_tallies:, dedup_dropped:, selected_strategy: nil, attempt_count: 0, strategy_attempts: []
+      version:, scraper_tallies:, dedup_dropped:, selected_strategy: nil, attempt_count: 0,
+      strategy_attempts: [], admission_drops: {}
     )
       dedup = Integer(dedup_dropped)
       attempts = Integer(attempt_count)
@@ -61,7 +69,8 @@ module Html2rss
         dedup_dropped: dedup,
         selected_strategy:,
         attempt_count: attempts,
-        strategy_attempts: freeze_attempts(strategy_attempts)
+        strategy_attempts: freeze_attempts(strategy_attempts),
+        admission_drops: freeze_tallies(admission_drops)
       )
     end
     # rubocop:enable Metrics/ParameterLists, Metrics/MethodLength
@@ -69,11 +78,13 @@ module Html2rss
     ##
     # Observability hash for web (+scraper_status+). Omits empty/absent optional keys:
     # +:scraper_tallies+ when empty, +:selected_strategy+ when +nil+, +:attempt_count+ when zero,
-    # +:strategy_attempts+ when empty.
+    # +:strategy_attempts+ / +:admission_drops+ when empty.
     # Data members remain available via readers even when omitted here.
     #
     # @return [Hash{Symbol => Object}] always +:version+ (String), +:dedup_dropped+ (Integer);
-    #   optionally +:scraper_tallies+, +:selected_strategy+, +:attempt_count+, +:strategy_attempts+
+    #   optionally +:scraper_tallies+, +:selected_strategy+, +:attempt_count+,
+    #   +:strategy_attempts+, +:admission_drops+
+    # rubocop:disable Metrics/AbcSize -- omit-empty optional keys stay explicit
     def to_h
       {
         version:,
@@ -81,9 +92,11 @@ module Html2rss
         **(scraper_tallies.any? ? { scraper_tallies: } : {}),
         **(selected_strategy.nil? ? {} : { selected_strategy: }),
         **(attempt_count.positive? ? { attempt_count: } : {}),
-        **(strategy_attempts.any? ? { strategy_attempts: } : {})
+        **(strategy_attempts.any? ? { strategy_attempts: } : {}),
+        **(admission_drops.any? ? { admission_drops: } : {})
       }
     end
+    # rubocop:enable Metrics/AbcSize
 
     ##
     # Formats the RSS +generator+ string and JSON Feed +user_comment+.
@@ -101,11 +114,14 @@ module Html2rss
     private
 
     def marshal_dump
-      [version, scraper_tallies, dedup_dropped, selected_strategy, attempt_count, strategy_attempts]
+      [version, scraper_tallies, dedup_dropped, selected_strategy, attempt_count, strategy_attempts,
+       admission_drops]
     end
 
-    def marshal_load((version, scraper_tallies, dedup_dropped, selected_strategy, attempt_count, strategy_attempts))
-      initialize(version:, scraper_tallies:, dedup_dropped:, selected_strategy:, attempt_count:, strategy_attempts:)
+    def marshal_load((version, scraper_tallies, dedup_dropped, selected_strategy, attempt_count,
+                      strategy_attempts, admission_drops))
+      initialize(version:, scraper_tallies:, dedup_dropped:, selected_strategy:, attempt_count:,
+                 strategy_attempts:, admission_drops: admission_drops || {})
     end
 
     def freeze_tallies(tallies)

data/lib/html2rss/url.rb

@@ -2,6 +2,7 @@
 
 require 'addressable/uri'
 require 'cgi'
+require 'ipaddr'
 require 'nokogiri'
 
 module Html2rss
@@ -21,6 +22,7 @@ module Html2rss
   #   url = Url.from_relative('/foo-bar/baz.txt', 'https://example.com')
   #   url.titleized # => "Foo Bar Baz"
   #   url.channel_titleized # => "example.com: Foo Bar Baz"
+  # rubocop:disable Metrics/ClassLength -- value object owns resolve/sanitize/titleize/identity
   class Url
     include Comparable
 
@@ -151,6 +153,16 @@ module Html2rss
     # @return [String, nil] URI host component
     def host = @uri.host
 
+    # Registrable domain (eTLD+1). Same-site hosts like www.wp.pl and wiadomosci.wp.pl
+    # share `wp.pl`. IP literals stay the host so 1.2.3.4 is not treated as `3.4`.
+    #
+    # @return [String, nil]
+    def domain
+      return host if ip_address_host?
+
+      @uri.domain || host
+    end
+
     # @return [Integer, nil] URI port component
     def port = @uri.port
 
@@ -189,6 +201,14 @@ module Html2rss
       self.class.from_absolute(uri.normalize.to_s)
     end
 
+    ##
+    # Returns a copy of the URL with the fragment removed (dedup / self-link identity).
+    #
+    # @return [Url] this URL when already fragment-free; otherwise a new URL
+    def without_fragment
+      @uri.fragment ? self.class.from_absolute(@uri.omit(:fragment).normalize.to_s) : self
+    end
+
     ##
     # Returns a copy of the URL with the provided query values.
     #
@@ -259,13 +279,7 @@ module Html2rss
     # @param other [Object] the other object to compare with
     # @return [Boolean] true if the URLs are equal
     def ==(other) = other.is_a?(Url) && to_s == other.to_s
-
-    ##
-    # Supports hash-based comparisons by ensuring equality semantics match `hash`.
-    #
-    # @param other [Object] the other object to compare with
-    # @return [Boolean] true if the URLs are considered equal
-    def eql?(other) = other.is_a?(Url) && to_s == other.to_s
+    alias eql? ==
 
     ##
     # Returns the hash code for this URL.
@@ -278,5 +292,15 @@ module Html2rss
     #
     # @return [String] the debug representation
     def inspect = "#<#{self.class}:#{object_id} @uri=#{@uri.inspect}>"
+
+    private
+
+    def ip_address_host?
+      IPAddr.new(host.to_s)
+      true
+    rescue IPAddr::Error
+      false
+    end
   end
+  # rubocop:enable Metrics/ClassLength
 end

data/lib/html2rss/version.rb

@@ -2,6 +2,6 @@
 
 module Html2rss
   # Current application version.
-  VERSION = '0.25.0'
+  VERSION = '0.27.0'
   public_constant :VERSION
 end

data/lib/html2rss.rb

@@ -5,7 +5,8 @@ require 'zeitwerk'
 loader = Zeitwerk::Loader.for_gem
 loader.inflector.inflect(
   'cli' => 'CLI',
-  'sst' => 'SST'
+  'sst' => 'SST',
+  'mcp' => 'MCP'
 )
 loader.setup
 
@@ -69,11 +70,35 @@ module Html2rss
 
   # rubocop:disable Metrics/ParameterLists
 
+  ##
+  # Scrapes the provided URL without hand-written selectors and returns {FeedResult}.
+  #
+  # Prefer this when callers need {FeedResult#status} (MCP envelope payload, CLI +--explain+).
+  #
+  # @param url [String] source page URL
+  # @param strategy [Symbol] request strategy to use
+  # @param items_selector [String, nil] optional selector hint for item extraction
+  # @param max_redirects [Integer, nil] optional redirect limit override
+  # @param max_requests [Integer] optional request budget override (default: 4 for sitemap sub-fetches)
+  # @param local_file_path [String, nil] optional local HTML file path
+  # @param limit [Integer, nil] max articles to keep (default: {AutoSource::DEFAULT_LIMIT})
+  # @return [Html2rss::FeedResult]
+  def self.auto_feed_result(url,
+                            strategy: :auto,
+                            items_selector: nil,
+                            max_redirects: nil,
+                            max_requests: 4,
+                            local_file_path: nil,
+                            limit: nil)
+    feed_result(build_auto_source_config(url:, strategy:, items_selector:, max_redirects:, max_requests:,
+                                         local_file_path:, limit:))
+  end
+
   ##
   # Scrapes the provided URL without hand-written selectors and returns an RSS object.
   #
   # Builds an auto_source config, then FeedPipeline runs structured scrapers and
-  # (when needed) the SST heuristic path — see docs/auto_source.md.
+  # (when needed) the SST heuristic path — see lib/html2rss/auto_source/README.md.
   #
   # @param url [String] source page URL
   # @param strategy [Symbol] request strategy to use
@@ -90,14 +115,14 @@ module Html2rss
                        max_requests: 4,
                        local_file_path: nil,
                        limit: nil)
-    feed(build_auto_source_config(url:, strategy:, items_selector:, max_redirects:, max_requests:,
-                                  local_file_path:, limit:))
+    auto_feed_result(url, strategy:, items_selector:, max_redirects:, max_requests:,
+                          local_file_path:, limit:).to_rss
   end
 
   ##
   # Scrapes the provided URL without hand-written selectors and returns a JSONFeed 1.1 hash.
   #
-  # Same auto_source pipeline as {.auto_source}; see docs/auto_source.md.
+  # Same auto_source pipeline as {.auto_source}; see lib/html2rss/auto_source/README.md.
   #
   # @param url [String] source page URL
   # @param strategy [Symbol] request strategy to use
@@ -114,8 +139,38 @@ module Html2rss
                           max_requests: 4,
                           local_file_path: nil,
                           limit: nil)
-    json_feed(build_auto_source_config(url:, strategy:, items_selector:, max_redirects:, max_requests:,
-                                       local_file_path:, limit:))
+    auto_feed_result(url, strategy:, items_selector:, max_redirects:, max_requests:,
+                          local_file_path:, limit:).to_json_feed
+  end
+
+  ##
+  # Analyzes a URL and produces a reusable YAML-ready feed config hash.
+  #
+  # Uses auto-source discovery to extract articles, then derives CSS selectors
+  # from the structural analysis.
+  #
+  # @param url [String] source page URL
+  # @param strategy [Symbol] request strategy (+:auto+, +:faraday+, +:botasaurus+)
+  # @param items_selector [String, nil] optional CSS selector hint for items
+  # @param max_redirects [Integer, nil] optional redirect limit override
+  # @param max_requests [Integer, nil] optional request budget override
+  # @param limit [Integer, nil] max articles to keep
+  # @param local_file_path [String, nil] optional local HTML file path
+  # @return [Hash] feed config hash with +:channel+ and +:selectors+
+  def self.capture(url,
+                   strategy: :auto,
+                   items_selector: nil,
+                   max_redirects: nil,
+                   max_requests: nil,
+                   limit: nil,
+                   local_file_path: nil)
+    Capture.build(url,
+                  strategy:,
+                  items_selector:,
+                  max_redirects:,
+                  max_requests:,
+                  limit:,
+                  local_file_path:).config
   end
 
   # rubocop:enable Metrics/ParameterLists