html2rss 0.32.4 → 0.34.0

data/lib/html2rss/cli.rb

@@ -114,27 +114,29 @@ module Html2rss
     method_option :explain, type: :boolean, desc: 'Print capture diagnostics to stderr', default: false
     # @param target [String, nil] URL or local HTML file
     # @return [void]
-    def capture(target = nil) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength, Metrics/CyclomaticComplexity
+    def capture(target = nil) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
       strategy, local_file_path, url = prepare_auto_inputs(target, options[:input])
       topics = options[:topics]&.split(',')&.map(&:strip)
 
-      result = Html2rss::Capture.build(
-        url,
-        strategy:,
-        items_selector: options[:items_selector],
-        topics:,
-        title: options[:title],
-        summary: options[:summary],
-        force: options[:force],
-        enhance: options[:enhance],
-        limit: options[:limit]&.to_i,
-        max_redirects: options[:max_redirects],
-        max_requests: options[:max_requests],
-        local_file_path:
-      )
+      result = execute_feed do
+        Html2rss::Capture.build(
+          url,
+          strategy:,
+          items_selector: options[:items_selector],
+          topics:,
+          title: options[:title],
+          summary: options[:summary],
+          force: options[:force],
+          enhance: options[:enhance],
+          limit: options[:limit],
+          max_redirects: options[:max_redirects],
+          max_requests: options[:max_requests],
+          local_file_path:
+        )
+      end
 
       if result.native_feed? && !options[:force]
-        $stderr.puts "First-party RSS/Atom feed detected at #{result.native_feed}. Use --force to capture anyway." # rubocop:disable Style/StderrPuts
+        $stderr.puts "First-party RSS/Atom/RDF feed detected at #{result.native_feed}. Use --force to capture anyway." # rubocop:disable Style/StderrPuts
         exit(3)
       end
 
@@ -158,15 +160,17 @@ module Html2rss
     # @return [void]
     def test(config_input = nil, feed_name = nil) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
       raw_content, input_source = read_config_input(config_input)
-      result = Html2rss.test(
-        raw_content,
-        feed_name,
-        min_items: options.fetch(:min_items, 1).to_i,
-        params: options[:params] || {},
-        strategy: options[:strategy],
-        strict_quality: options.fetch(:strict_quality, false),
-        compare_enhance: options.fetch(:compare_enhance, false)
-      )
+      result = execute_feed do
+        Html2rss.test(
+          raw_content,
+          feed_name,
+          min_items: options.fetch(:min_items, 1),
+          params: options[:params] || {},
+          strategy: options[:strategy],
+          strict_quality: options.fetch(:strict_quality, false),
+          compare_enhance: options.fetch(:compare_enhance, false)
+        )
+      end
 
       if options[:json]
         puts JSON.pretty_generate(result.to_h)
@@ -194,15 +198,10 @@ module Html2rss
     # @param source [String, nil] YAML file path or '-'
     # @param feed_name [String, nil] optional named feed
     # @return [void]
-    def apply(source = nil, feed_name = nil) # rubocop:disable Metrics/AbcSize
-      config = if File.file?(source.to_s)
-                 Html2rss.config_from_yaml_file(source.to_s, feed_name)
-               else
-                 raw_content, = read_config_input(source)
-                 Config.from_yaml(raw_content)
-               end
+    def apply(source = nil, feed_name = nil)
+      config = execute_feed { load_apply_config(source, feed_name) }
       config[:params] = options[:params] || {}
-      apply_runtime_request_overrides!(config)
+      execute_feed { apply_runtime_request_overrides!(config) }
       apply_local_file_input!(config, options[:input]) if options[:input]
 
       run_feed_command { Html2rss.apply(config) }
@@ -216,6 +215,8 @@ module Html2rss
     method_option :keep_different_domain, type: :boolean, default: false,
                                           desc: 'Allow published items to link to any domain ' \
                                                 'supplied by the source (no affiliation/safety check)'
+    method_option :keep_sponsored, type: :boolean, default: false,
+                                   desc: 'Keep items the page marks as sponsored (rel="sponsored" or ad labels)'
     # @param url [String, nil]
     # @return [void]
     def scrape(url = nil)
@@ -253,6 +254,8 @@ module Html2rss
     # @return [void]
     def mcp
       Html2rss::MCP.start(transport: options[:transport].to_sym, port: options[:port])
+    rescue Html2rss::MCP::Server::HttpTransportUnavailable => error
+      raise Thor::Error, error.message
     end
 
     desc 'doctor [TYPE]', 'Runtime preflight checks (botasaurus)'
@@ -305,7 +308,7 @@ module Html2rss
 
       expected = Recon::Verdict[verdict_filter.downcase]
       results.select { |r| r.verdict == expected }
-    rescue ArgumentError => error
+    rescue InvalidOption => error
       raise Thor::Error, error.message
     end
 
@@ -328,7 +331,7 @@ module Html2rss
 
     def handle_capture_output(result, url) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
       if options[:output_dir]
-        dir = File.join(options[:output_dir], URI.parse(url).host.to_s.delete_prefix('www.'))
+        dir = File.join(options[:output_dir], Html2rss::Url.from_absolute(url).host.to_s.delete_prefix('www.'))
         FileUtils.mkdir_p(dir)
         file_path = File.join(dir, 'index.yml')
         File.write(file_path, result.yaml)
@@ -408,6 +411,13 @@ module Html2rss
       options[:strategy]&.to_sym || :auto
     end
 
+    def load_apply_config(source, feed_name)
+      return Html2rss.config_from_yaml_file(source.to_s, feed_name) if File.file?(source.to_s)
+
+      raw_content, = read_config_input(source)
+      Config.from_yaml(raw_content)
+    end
+
     def execute_feed # rubocop:disable Metrics/MethodLength
       yield
     rescue Html2rss::RequestService::RedirectLimitReached => error
@@ -421,7 +431,10 @@ module Html2rss
            Html2rss::RequestService::BotasaurusConnectionFailed,
            Html2rss::RequestService::BotasaurusServiceError,
            Html2rss::RequestService::BlockedSurfaceDetected,
-           Html2rss::NoFeedItemsExtracted => error
+           Html2rss::NoFeedItemsExtracted,
+           Html2rss::InvalidOption,
+           Html2rss::Config::InvalidConfig,
+           Html2rss::Url::InvalidUrlError => error
       raise Thor::Error, error.message
     end
 
@@ -433,11 +446,15 @@ module Html2rss
         max_redirects: options[:max_redirects],
         max_requests: options[:max_requests],
         local_file_path:,
-        limit: options[:limit]&.to_i,
-        keep_different_domain: options[:keep_different_domain]
+        limit: options[:limit],
+        **cleanup_overrides
       )
     end
 
+    def cleanup_overrides
+      { keep_different_domain: options[:keep_different_domain], keep_sponsored: options[:keep_sponsored] }
+    end
+
     def explain_json!(payload)
       $stderr.puts JSON.pretty_generate(payload) # rubocop:disable Style/StderrPuts
     end

data/lib/html2rss/config/auto_source_contract.rb

@@ -4,61 +4,68 @@ require 'dry-validation'
 
 module Html2rss
   class Config
+    # Scraper options are splatted into scraper constructors, so a key no scraper accepts is a
+    # validation error. Key validation applies only when a schema is called directly, so the
+    # auto-source contract keeps +scraper+ an open hash and {Validator} applies this one to it.
+    # The raw values reach the scrapers, so this schema checks types without coercing them.
+    AutoSourceScraperContract = Dry::Schema.JSON do # rubocop:disable Metrics/BlockLength
+      config.validate_keys = true
+
+      optional(:native_feed).hash do
+        optional(:enabled).filled(:bool)
+      end
+      optional(:wordpress_api).hash do
+        optional(:enabled).filled(:bool)
+      end
+      optional(:sitemap).hash do
+        optional(:enabled).filled(:bool)
+        optional(:min_priority).filled { int? | float? }
+        optional(:max_age_days).filled(:integer, gt?: 0)
+      end
+      optional(:schema).hash do
+        optional(:enabled).filled(:bool)
+      end
+      optional(:microdata).hash do
+        optional(:enabled).filled(:bool)
+      end
+      optional(:microformats2).hash do
+        optional(:enabled).filled(:bool)
+      end
+      optional(:json_state).hash do
+        optional(:enabled).filled(:bool)
+      end
+      optional(:json_document).hash do
+        optional(:enabled).filled(:bool)
+      end
+      optional(:xhr_articles).hash do
+        optional(:enabled).filled(:bool)
+      end
+      optional(:meta_oembed).hash do
+        optional(:enabled).filled(:bool)
+      end
+      optional(:semantic_html).hash do
+        optional(:enabled).filled(:bool)
+      end
+      optional(:html).hash do
+        optional(:enabled).filled(:bool)
+        optional(:minimum_selector_frequency).filled(:integer, gt?: 0)
+        optional(:use_top_selectors).filled(:integer, gt?: 0)
+      end
+    end
+
     # Runtime source of truth for validating auto-source config values.
-    AutoSourceContract = Dry::Schema.Params do # rubocop:disable Metrics/BlockLength
+    AutoSourceContract = Dry::Schema.Params do
       optional(:limit).filled(:integer, gt?: 0)
       optional(:entry_resolution).hash do
         optional(:enabled).filled(:bool)
         optional(:max_probes).filled(:integer, gt?: 0)
       end
 
-      optional(:scraper).hash do # rubocop:disable Metrics/BlockLength
-        optional(:native_feed).hash do
-          optional(:enabled).filled(:bool)
-        end
-        optional(:wordpress_api).hash do
-          optional(:enabled).filled(:bool)
-        end
-        optional(:sitemap).hash do
-          optional(:enabled).filled(:bool)
-          optional(:min_priority).filled(:float)
-          optional(:max_age_days).filled(:integer, gt?: 0)
-        end
-        optional(:schema).hash do
-          optional(:enabled).filled(:bool)
-        end
-        optional(:microdata).hash do
-          optional(:enabled).filled(:bool)
-        end
-        optional(:microformats2).hash do
-          optional(:enabled).filled(:bool)
-        end
-        optional(:json_state).hash do
-          optional(:enabled).filled(:bool)
-        end
-        optional(:json_document).hash do
-          optional(:enabled).filled(:bool)
-        end
-        optional(:xhr_articles).hash do
-          optional(:enabled).filled(:bool)
-        end
-        optional(:meta_oembed).hash do
-          optional(:enabled).filled(:bool)
-        end
-        optional(:semantic_html).hash do
-          optional(:enabled).filled(:bool)
-          optional(:fallback_anchorless).filled(:bool)
-        end
-        optional(:html).hash do
-          optional(:enabled).filled(:bool)
-          optional(:minimum_selector_frequency).filled(:integer, gt?: 0)
-          optional(:use_top_selectors).filled(:integer, gt?: 0)
-          optional(:fallback_anchorless).filled(:bool)
-        end
-      end
+      optional(:scraper).hash
 
       optional(:cleanup).hash do
         optional(:keep_different_domain).filled(:bool)
+        optional(:keep_sponsored).filled(:bool)
       end
     end
   end

data/lib/html2rss/config/issue_mapper.rb

@@ -6,6 +6,9 @@ module Html2rss
     # Maps internal Dry::Validation messages to {ValidationIssue} (predicate → code).
     # Not a public API — Dry stays inside Validator / this mapper.
     module IssueMapper
+      # Raised when the schema yields a Dry predicate this mapper has no code for (a mapping-table gap).
+      class InvalidError < Html2rss::ContractError; end
+
       # Dry schema predicates → closed {ValidationIssue::CODES}.
       PREDICATE_CODES = {
         key?: :missing_key,
@@ -65,7 +68,7 @@ module Html2rss
           return :invalid_value if predicate.nil?
 
           PREDICATE_CODES.fetch(predicate) do
-            raise ArgumentError, "unmapped Dry validation predicate: #{predicate.inspect}"
+            raise InvalidError, "unmapped Dry validation predicate: #{predicate.inspect}"
           end
         end
 

data/lib/html2rss/config/request_controls.rb

@@ -50,13 +50,18 @@ module Html2rss
       # @param strategy [Symbol, nil]
       # @param max_redirects [Integer, nil]
       # @param max_requests [Integer, nil]
+      # @param total_timeout_seconds [Integer, nil] wall-clock bound for the whole request chain
       # @return [RequestControls]
-      def self.from_shortcut(strategy:, max_redirects: nil, max_requests: nil)
+      def self.from_shortcut(strategy:, max_redirects: nil, max_requests: nil, total_timeout_seconds: nil)
+        strategy = Option.strategy(strategy) unless strategy.nil?
+        check_budgets!(max_redirects, max_requests)
+        Option.integer(:total_timeout_seconds, total_timeout_seconds, min: 1, optional: true)
         keys = []
         keys << :strategy unless strategy.nil? || strategy == Config.default_strategy_name
         keys << :max_redirects unless max_redirects.nil?
         keys << :max_requests unless max_requests.nil?
-        new(strategy:, max_redirects:, max_requests:, explicit_keys: keys)
+        keys << :total_timeout_seconds unless total_timeout_seconds.nil?
+        new(strategy:, max_redirects:, max_requests:, total_timeout_seconds:, explicit_keys: keys)
       end
 
       ##
@@ -68,7 +73,8 @@ module Html2rss
       # @param max_requests [Integer, nil]
       # @return [RequestControls]
       def self.from_cli_options(strategy:, max_redirects: nil, max_requests: nil)
-        strategy_sym = strategy&.to_sym
+        strategy_sym = Option.strategy(strategy) if strategy
+        check_budgets!(max_redirects, max_requests)
         keys = []
         keys << :strategy if strategy
         keys << :max_redirects unless max_redirects.nil?
@@ -76,7 +82,12 @@ module Html2rss
         new(strategy: strategy_sym, max_redirects:, max_requests:, explicit_keys: keys)
       end
 
-      private_class_method :explicit_keys_for, :request_value_for, :request_key?
+      def self.check_budgets!(max_redirects, max_requests)
+        Option.integer(:max_redirects, max_redirects, min: 0, optional: true)
+        Option.integer(:max_requests, max_requests, min: 1, optional: true)
+      end
+
+      private_class_method :explicit_keys_for, :request_value_for, :request_key?, :check_budgets!
 
       ##
       # @param strategy [Symbol, nil] effective request strategy
@@ -162,7 +173,7 @@ module Html2rss
         return config[:request] = {} if request_config.nil?
         return if request_config.is_a?(Hash)
 
-        raise ArgumentError, 'request config must be a hash'
+        raise InvalidOption, 'request config must be a hash'
       end
     end
   end

data/lib/html2rss/config/schema.rb

@@ -7,8 +7,6 @@ module Html2rss
     ##
     # JSON export adapter from the runtime validators to the packaged schema artifact.
     module Schema
-      module_function
-
       # Canonical filename for the exported config JSON schema artifact.
       SCHEMA_FILENAME = 'html2rss-config.schema.json'
 
@@ -16,7 +14,7 @@ module Html2rss
       # Returns the exported configuration JSON Schema.
       #
       # @return [Hash{String => Object}] JSON Schema represented as a Ruby hash
-      def json_schema
+      def self.json_schema
         load_json_schema_extension!
         Builder.call
       end
@@ -26,7 +24,7 @@ module Html2rss
       #
       # @param pretty [Boolean] whether to pretty-print the JSON output
       # @return [String] serialized JSON Schema
-      def json_schema_json(pretty: true)
+      def self.json_schema_json(pretty: true)
         pretty ? JSON.pretty_generate(json_schema) : JSON.generate(json_schema)
       end
 
@@ -34,7 +32,7 @@ module Html2rss
       # Resolves the packaged schema path used by downstream tools.
       #
       # @return [String] absolute path to the packaged JSON schema file
-      def path
+      def self.path
         search_path = File.expand_path(__dir__)
 
         loop do
@@ -51,7 +49,7 @@ module Html2rss
       end
 
       # @return [void]
-      def load_json_schema_extension!
+      def self.load_json_schema_extension!
         require 'dry/schema/extensions/json_schema'
         Dry::Schema.load_extensions(:json_schema)
       end
@@ -107,6 +105,17 @@ module Html2rss
           request[:botasaurus] = exported
         end
 
+        # Scraper options are splatted into scraper constructors, so the scraper map and each
+        # scraper object reject keys the contract does not name.
+        def auto_source_schema
+          schema = Html2rss::Config::AutoSourceContract.json_schema(loose: true)
+          scraper = Html2rss::Config::AutoSourceScraperContract.json_schema(loose: true).except(:$schema)
+          scraper[:additionalProperties] = false
+          scraper.fetch(:properties).each_value { |options| options[:additionalProperties] = false }
+          schema.fetch(:properties)[:scraper] = scraper
+          schema.merge(default: HashUtil.deep_stringify_keys(Html2rss::AutoSource::DEFAULT_CONFIG))
+        end
+
         # @return [Hash{Symbol => Hash}] catalog under $defs.post_processors / $defs.extractors
         def registry_catalog_defs
           {
@@ -129,8 +138,9 @@ module Html2rss
             default: Html2rss::Selectors::DEFAULT_ITEMS_SELECTOR
           )
           items_schema[:properties][:enhance] = items_schema.fetch(:properties).fetch(:enhance).merge(
-            description: 'List-card enrichment: run Html::ArticleExtractor on each matched item node ' \
-                         'to fill missing fields from the card HTML.'
+            description: 'Fills missing title, link, description, date, image and categories from each ' \
+                         "item's own markup via Html::ArticleExtractor. Explicit selectors win; no extra " \
+                         'requests are made.'
           )
           items_schema[:properties][:pagination] = {
             description: 'Pagination configuration or maximum page count integer.',
@@ -177,9 +187,7 @@ module Html2rss
               description: 'Collection of stylesheets to attach to the RSS feed.',
               items: Html2rss::Config::Validator::StylesheetConfig.json_schema(loose: true)
             },
-            auto_source: Html2rss::Config::AutoSourceContract.json_schema(loose: true).merge(
-              default: HashUtil.deep_stringify_keys(Html2rss::AutoSource::DEFAULT_CONFIG)
-            ),
+            auto_source: auto_source_schema,
             selectors: {
               type: 'object',
               description: 'Selectors used to extract article attributes.',
@@ -221,7 +229,7 @@ module Html2rss
       #
       # @param schema [Hash] dry-schema JSON schema fragment for a selector
       # @return [Hash] same schema with registry $ref wiring applied
-      def apply_selector_registry_refs!(schema)
+      def self.apply_selector_registry_refs!(schema)
         properties = schema.fetch(:properties)
         properties[:extractor] = extractor_wire_schema
         properties[:post_process] = post_process_wire_schema
@@ -229,7 +237,7 @@ module Html2rss
       end
 
       # @return [Hash{Symbol => Object}]
-      def extractor_wire_schema
+      def self.extractor_wire_schema
         {
           description: 'Extractor used to pull a value from the selected element.',
           oneOf: registry_ref_list('extractors', Selectors::Extractors::NAME_TO_CLASS)
@@ -237,7 +245,7 @@ module Html2rss
       end
 
       # @return [Hash{Symbol => Object}]
-      def post_process_wire_schema
+      def self.post_process_wire_schema
         {
           type: 'array',
           description: 'Ordered transforms applied to the extracted value.',
@@ -251,7 +259,7 @@ module Html2rss
       # @param catalog [String] $defs catalog name (`extractors` or `post_processors`)
       # @param registry [Hash{Symbol => Class}]
       # @return [Array<Hash>] oneOf entries of `{ '$ref' => ... }`
-      def registry_ref_list(catalog, registry)
+      def self.registry_ref_list(catalog, registry)
         registry.keys.sort.map do |name|
           { '$ref' => "#/$defs/#{catalog}/#{name}" }
         end

data/lib/html2rss/config/selectors_validator.rb

@@ -142,7 +142,7 @@ module Html2rss
           end
         end
 
-        # Same compile as Gsub#call. ArgumentError is the bound; Gsub maps
+        # Same compile as Gsub#call. InvalidOption is the bound; Gsub maps
         # parser errors onto that path so admit and execute still share it.
         #
         # @param klass [Class]
@@ -156,7 +156,7 @@ module Html2rss
 
           Selectors::PostProcessors::Gsub.compiled_pattern(pattern)
           nil
-        rescue ArgumentError => error
+        rescue InvalidOption => error
           error.message
         end
       end
@@ -216,6 +216,7 @@ module Html2rss
         def array_selector_shape_error(key, selector)
           return "`#{key}` must be an array" unless selector.is_a?(Array)
           return "`#{key}` must contain at least one element" if selector.empty?
+          return "`#{key}` must contain only selector names" unless selector.all? { _1 in String | Symbol }
 
           nil
         end

data/lib/html2rss/config/validation_issue.rb

@@ -10,6 +10,9 @@ module Html2rss
     # +path+ is JSON-pointer-friendly segments. +code+ is closed ({CODES}).
     # +expected+ / +actual+ are JSON-ish or +nil+ (never invented).
     class ValidationIssue
+      # Raised when an issue is built with a code outside {CODES}.
+      class InvalidError < Html2rss::ContractError; end
+
       # Closed set of issue codes (mapper owns Dry predicate → code).
       CODES = Set[
         :missing_key,
@@ -27,7 +30,7 @@ module Html2rss
       # @param expected [Hash, String, Numeric, true, false, nil]
       # @param actual [Hash, String, Numeric, true, false, nil]
       def initialize(path:, code:, message:, expected: nil, actual: nil)
-        raise ArgumentError, "unknown ValidationIssue code: #{code.inspect}" unless CODES.include?(code)
+        raise InvalidError, "unknown ValidationIssue code: #{code.inspect}" unless CODES.include?(code)
 
         super(
           path: Array(path).freeze,

data/lib/html2rss/config/validator.rb

@@ -153,20 +153,31 @@ module Html2rss
         end
       end
 
+      rule(auto_source: :scraper) do
+        next unless value
+
+        result = Config::AutoSourceScraperContract.call(value)
+        next if result.success?
+
+        result.errors.each do |error|
+          key([:auto_source, :scraper, *error.path]).failure(error.text)
+        end
+      end
+
       # URL validation delegated to Url class
       rule(:channel) do
         if (url_string = values.dig(:channel, :url)) && !url_string.empty?
           begin
             Html2rss::Url.for_channel(url_string)
-          rescue ArgumentError => error
+          rescue Html2rss::Url::InvalidUrlError => error
             key(%i[channel url]).failure(error.message)
           end
         end
 
         if (image_string = values.dig(:channel, :image)) && !image_string.empty?
           begin
-            Html2rss::Url.from_absolute(image_string)
-          rescue ArgumentError => error
+            Html2rss::Url.from_absolute_web(image_string)
+          rescue Html2rss::Url::InvalidUrlError => error
             key(%i[channel image]).failure(error.message)
           end
         end