html2rss 0.30.0 → 0.32.0

data/lib/html2rss/capture.rb

@@ -25,12 +25,15 @@ module Html2rss
     CHROME_DROP_THRESHOLD = 3
     private_constant :CHROME_DROP_THRESHOLD
 
-    ##
-    # Result of a capture operation (config plus quality meta).
     CaptureResult = Data.define(
       :config, :yaml, :articles_count, :channel_title, :has_selectors, :segment_strategy,
-      :admission_drops, :selected_strategy, :inferred_topics, :native_feed, :suggested_channel_url
-    ) do
+      :admission_drops, :selected_strategy, :inferred_topics, :native_feed, :suggested_channel_url,
+      :candidates
+    )
+
+    ##
+    # Result of a capture operation (config plus quality meta).
+    class CaptureResult
       # rubocop:disable Metrics/ParameterLists
       ##
       # @param config [Hash]
@@ -44,9 +47,10 @@ module Html2rss
       # @param inferred_topics [Array<String>]
       # @param native_feed [String, nil]
       # @param suggested_channel_url [String, nil]
+      # @param candidates [Hash{Symbol=>Array}, nil] ranked selector buckets from {SelectorCandidates}
       def initialize(config:, articles_count:, channel_title:, has_selectors:, segment_strategy:, # rubocop:disable Metrics/MethodLength
                      yaml: nil, admission_drops: {}, selected_strategy: nil, inferred_topics: [], native_feed: nil,
-                     suggested_channel_url: nil)
+                     suggested_channel_url: nil, candidates: nil)
         super(
           config:,
           yaml: yaml || "#{SCHEMA_MODELINE}\n#{Config.to_yaml(config)}",
@@ -58,7 +62,8 @@ module Html2rss
           selected_strategy:,
           inferred_topics:,
           native_feed:,
-          suggested_channel_url:
+          suggested_channel_url:,
+          candidates: candidates || SelectorCandidates.empty
         )
       end
       # rubocop:enable Metrics/ParameterLists
@@ -127,7 +132,7 @@ module Html2rss
     def build # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
       outcome = FeedPipeline.new(raw_config).to_outcome
       @admission_drops = outcome.admission_drops
-      selectors, segment_strategy = derive_selectors(outcome.response, outcome.articles)
+      selectors, segment_strategy, candidates = derive_selectors(outcome.response, outcome.articles)
       ch_title = @title || channel_title_from(outcome.response)
       topics = @topics || infer_topics("#{@url} #{ch_title}")
       native_feed = probe_native_feed(outcome.response) unless @force
@@ -151,7 +156,8 @@ module Html2rss
         selected_strategy: outcome.selected_strategy,
         inferred_topics: topics,
         native_feed:,
-        suggested_channel_url: suggested_channel_url(outcome)
+        suggested_channel_url: suggested_channel_url(outcome),
+        candidates:
       )
     end
 
@@ -254,40 +260,68 @@ module Html2rss
       { strategy: :local_file, request: { local_file_path: @local_file_path } }
     end
 
-    # @return [Array(Hash, Symbol, nil)] selectors hash and winning segment strategy
+    # @return [Array(Hash, Symbol, nil, Hash)] selectors, segment strategy, candidate buckets
     def derive_selectors(response, articles)
-      return hint_selectors if @items_selector_hint
-      return [{}, nil] if articles.empty? || !response.html_response?
+      return [*hint_selectors, SelectorCandidates.empty] if @items_selector_hint
+      return [{}, nil, SelectorCandidates.empty] unless response.html_response?
+      return [*default_selectors, SelectorCandidates.empty] if articles.empty?
 
       sst = SST::Normalizer.call(response.body)
-      return [{}, nil] unless sst
+      return [*default_selectors, SelectorCandidates.empty] unless sst
 
       select_enhance_selectors(sst, articles)
     rescue ArgumentError => error
       Log.warn("Capture selector derivation failed: #{error.message}")
-      [{}, nil]
+      [*default_selectors, SelectorCandidates.empty]
     end
 
     def hint_selectors
       [{ items: { selector: @items_selector_hint, enhance: resolve_enhance } }, :hint]
     end
 
-    def select_enhance_selectors(sst, articles) # rubocop:disable Metrics/MethodLength -- strategy loop + gate
+    def default_selectors
+      [{ items: { selector: Selectors::DEFAULT_ITEMS_SELECTOR, enhance: resolve_enhance } }, :default]
+    end
+
+    def select_enhance_selectors(sst, articles)
+      evidence = collect_items_evidence(sst, articles)
+      candidates = SelectorCandidates.call(items_evidence: evidence, min_matches: MIN_SELECTOR_MATCHES)
+      top = candidates[:items].first
+      return [*default_selectors, candidates] unless top
+
+      strategy = evidence.find { |entry| entry[:selector] == top[:selector] }&.fetch(:strategy)
+      items = { selector: top[:selector], enhance: top[:enhance] }
+      [{ items: }, strategy, candidates]
+    end
+
+    def collect_items_evidence(sst, articles)
       link_resolver = Scoring::LinkResolver.new(@url)
       enhance = resolve_enhance
 
-      SEGMENT_STRATEGIES.each do |strategy|
+      SEGMENT_STRATEGIES.flat_map do |strategy|
         segments = AutoSource::Segmenter.call(
           sst, base_url: @url, strategy:, permit_unanchored: false, link_resolver:
         )
         matched = match_segments_to_articles(segments, articles)
-        items_sel = items_selector(matched)
-        next unless items_sel && matched.size >= MIN_SELECTOR_MATCHES
+        next [] if matched.size < MIN_SELECTOR_MATCHES
+
+        items_evidence_for(matched, strategy:, enhance:)
+      end
+    end
 
-        return [{ items: { selector: items_sel, enhance: } }, strategy]
+    def items_evidence_for(matched, strategy:, enhance:)
+      roots = lift_heading_link_roots(matched.map { |pair| pair[:segment].root_node })
+      selector_variants_for(roots).map do |kind, selector|
+        { selector:, enhance:, strategy:, kind:, match_count: matched.size, roots: }
       end
+    end
 
-      [{}, nil]
+    def selector_variants_for(roots)
+      [
+        [:shared_class, shared_class_items_selector(roots)],
+        [:unique_tag, unique_tag_items_selector(roots)],
+        [:path, path_items_selector(roots)]
+      ].filter_map { |kind, selector| [kind, selector] if selector }
     end
 
     def match_segments_to_articles(segments, articles)
@@ -322,21 +356,6 @@ module Html2rss
       segment.root_node.visible_text.to_s.strip
     end
 
-    def items_selector(matched)
-      return nil if matched.empty?
-
-      roots = lift_heading_link_roots(matched.map { |m| m[:segment].root_node })
-      shared = shared_class_items_selector(roots)
-      return shared if shared
-
-      paths = roots.map(&:tag_path)
-      common = common_path_prefix(paths)
-      tag_path = common.empty? ? paths.first.to_s : common
-      return nil if tag_path.empty?
-
-      css_from_trimmed_tag_path(tag_path)
-    end
-
     def lift_heading_link_roots(roots)
       roots.map { |root| lift_heading_link_root(root, roots) }
     end
@@ -371,7 +390,6 @@ module Html2rss
 
     def wrapping_anchor_root?(node)
       return false unless node.name.to_s == 'a'
-      return false unless node.respond_to?(:find)
 
       tags = Html::Navigator::WRAPPING_ANCHOR_CHILD_TAGS
       node.find { |child| !child.equal?(node) && tags.include?(child.name.to_s) }
@@ -396,6 +414,20 @@ module Html2rss
       "#{roots.first.name}.#{shared.min}"
     end
 
+    def unique_tag_items_selector(roots)
+      names = roots.map { |root| root.name.to_s }.uniq
+      names.join(', ') if names.size > 1
+    end
+
+    def path_items_selector(roots)
+      paths = roots.map(&:tag_path)
+      common = common_path_prefix(paths)
+      tag_path = common.empty? ? paths.first.to_s : common
+      return if tag_path.empty?
+
+      css_from_trimmed_tag_path(tag_path)
+    end
+
     def css_from_trimmed_tag_path(tag_path)
       segments = tag_path.split('/').reject(&:empty?)
       trimmed = trim_leading_items_segments(segments)

data/lib/html2rss/channel.rb

@@ -2,8 +2,9 @@
 
 require 'time'
 
-# rubocop:disable Metrics/ModuleLength -- Channel value object owns extract + freeze + Marshal
 module Html2rss
+  Channel = Data.define(:title, :url, :description, :language, :ttl, :last_build_date, :image, :author)
+
   ##
   # Materialized channel metadata for feed builders and {FeedResult}.
   #
@@ -27,7 +28,7 @@ module Html2rss
   #   @return [Html2rss::Url, nil]
   # @!attribute [r] author
   #   @return [String, nil]
-  Channel = Data.define(:title, :url, :description, :language, :ttl, :last_build_date, :image, :author) do
+  class Channel # rubocop:disable Metrics/ClassLength -- Channel value object owns extract + freeze + Marshal
     class << self
       ##
       # Materializes channel attributes from an HTTP response and optional overrides.
@@ -193,5 +194,4 @@ module Html2rss
       Time.httpdate(value)
     end
   end
-  # rubocop:enable Metrics/ModuleLength
 end

data/lib/html2rss/cli/probe_view.rb

@@ -2,8 +2,6 @@
 
 module Html2rss
   class CLI
-    ##
-    # Display-only probe facts for inspect and recon text cards.
     ProbeView = Data.define(
       :requested,
       :final,
@@ -15,7 +13,11 @@ module Html2rss
       :alternate_feeds,
       :notes,
       :strategy
-    ) do
+    )
+
+    ##
+    # Display-only probe facts for inspect and recon text cards.
+    class ProbeView
       class << self
         ##
         # @param data [Hash] inspect wire payload

data/lib/html2rss/cli/render.rb

@@ -93,9 +93,11 @@ module Html2rss
             end
           end
         else
-          warn "\e[31m✗ Test failed\e[0m (#{source})"
-          warn "  Error: #{result.error_message}" if result.error_message
-          result.validation_errors&.each { |k, v| warn "  Schema error [#{k}]: #{Array(v).join(', ')}" }
+          $stderr.puts "\e[31m✗ Test failed\e[0m (#{source})" # rubocop:disable Style/StderrPuts
+          $stderr.puts "  Error: #{result.error_message}" if result.error_message # rubocop:disable Style/StderrPuts
+          result.validation_issues&.each do |issue|
+            $stderr.puts "  Schema error #{issue}" # rubocop:disable Style/StderrPuts
+          end
         end
       end
 
@@ -115,8 +117,7 @@ module Html2rss
         if view.final && view.final != view.requested
           puts "        Final:    #{view.final} (HTTP #{view.status || 'ERR'})"
         end
-        surface = view.surface.respond_to?(:to_s) ? view.surface.to_s : view.surface
-        puts "        Surface:  #{surface} (#{view.articles_count} articles)"
+        puts "        Surface:  #{view.surface} (#{view.articles_count} articles)"
       end
       private_class_method :probe_lines
     end

data/lib/html2rss/cli/validate.rb

@@ -29,11 +29,10 @@ module Html2rss
           if result.success?
             puts(target_files.size == 1 ? 'Configuration is valid' : "ok   #{file}") unless quiet
           else
-            error_details = result.errors.to_h
-            raise Thor::Error, "Invalid configuration: #{error_details}" if target_files.size == 1
+            raise Thor::Error, format_failure(result) if target_files.size == 1
 
             warn "FAIL #{file}"
-            error_details.each { |key, msg| warn "       #{key}: #{Array(msg).join(', ')}" }
+            result.issues.each { |issue| warn "       #{issue}" }
             failed << file
           end
         end
@@ -57,35 +56,37 @@ module Html2rss
         resolved.uniq
       end
 
-      def run_named_feed(file, feed_name, params:, quiet:)
-        result = Html2rss.validate(file, feed_name, params:)
-        raise Thor::Error, "Invalid configuration: #{result.errors.to_h}" unless result.success?
+      class << self
+        private
 
-        puts 'Configuration is valid' unless quiet
-      end
-      module_function :run_named_feed
-      private_class_method :run_named_feed
-
-      def validate_file(file, params:)
-        if file == '-'
-          Html2rss.validate($stdin.read, params:)
-        else
-          Html2rss.validate(file, params:)
+        def run_named_feed(file, feed_name, params:, quiet:)
+          result = Html2rss.validate(file, feed_name, params:)
+          raise Thor::Error, format_failure(result) unless result.success?
+
+          puts 'Configuration is valid' unless quiet
+        end
+
+        def validate_file(file, params:)
+          if file == '-'
+            Html2rss.validate($stdin.read, params:)
+          else
+            Html2rss.validate(file, params:)
+          end
         end
-      end
-      module_function :validate_file
-      private_class_method :validate_file
 
-      def path_like_config?(arg)
-        path = arg.to_s
-        return true if path.start_with?('-')
-        return true if path.end_with?('.yml', '.yaml')
-        return true if path.include?('/') || path.include?('*')
+        def format_failure(report)
+          "Invalid configuration: #{report}"
+        end
 
-        false
+        def path_like_config?(arg)
+          path = arg.to_s
+          return true if path.start_with?('-')
+          return true if path.end_with?('.yml', '.yaml')
+          return true if path.include?('/') || path.include?('*')
+
+          false
+        end
       end
-      module_function :path_like_config?
-      private_class_method :path_like_config?
     end
   end
 end

data/lib/html2rss/cli.rb

@@ -213,6 +213,9 @@ module Html2rss
     desc 'scrape [URL]', 'Automatically source an RSS feed from a URL (one-shot auto-source)'
     method_option :items_selector, type: :string, desc: 'CSS selector hint for items'
     method_option :limit, type: :numeric, desc: 'Max articles (auto-source)'
+    method_option :keep_different_domain, type: :boolean, default: false,
+                                          desc: 'Allow published items to link to any domain ' \
+                                                'supplied by the source (no affiliation/safety check)'
     # @param url [String, nil]
     # @return [void]
     def scrape(url = nil)
@@ -300,7 +303,7 @@ module Html2rss
     def filter_recon_results(results, verdict_filter)
       return results unless verdict_filter
 
-      expected = Recon::Verdict.coerce(verdict_filter.downcase)
+      expected = Recon::Verdict[verdict_filter.downcase]
       results.select { |r| r.verdict == expected }
     rescue ArgumentError => error
       raise Thor::Error, error.message
@@ -430,7 +433,8 @@ module Html2rss
         max_redirects: options[:max_redirects],
         max_requests: options[:max_requests],
         local_file_path:,
-        limit: options[:limit]&.to_i
+        limit: options[:limit]&.to_i,
+        keep_different_domain: options[:keep_different_domain]
       )
     end
 

data/lib/html2rss/config/auto_source_contract.rb

@@ -36,6 +36,9 @@ module Html2rss
         optional(:json_state).hash do
           optional(:enabled).filled(:bool)
         end
+        optional(:json_document).hash do
+          optional(:enabled).filled(:bool)
+        end
         optional(:xhr_articles).hash do
           optional(:enabled).filled(:bool)
         end

data/lib/html2rss/config/issue_mapper.rb

@@ -0,0 +1,108 @@
+# frozen_string_literal: true
+
+module Html2rss
+  class Config
+    ##
+    # Maps internal Dry::Validation messages to {ValidationIssue} (predicate → code).
+    # Not a public API — Dry stays inside Validator / this mapper.
+    module IssueMapper
+      # Dry schema predicates → closed {ValidationIssue::CODES}.
+      PREDICATE_CODES = {
+        key?: :missing_key,
+        filled?: :missing_key,
+        str?: :type_mismatch,
+        int?: :type_mismatch,
+        bool?: :type_mismatch,
+        hash?: :type_mismatch,
+        array?: :type_mismatch,
+        type?: :type_mismatch,
+        format?: :invalid_value,
+        included_in?: :unknown_value,
+        gt?: :constraint,
+        gteq?: :constraint,
+        lt?: :constraint,
+        lteq?: :constraint,
+        max_size?: :constraint,
+        min_size?: :constraint
+      }.freeze
+
+      class << self
+        ##
+        # @param dry_result [Dry::Validation::Result]
+        # @param values [Hash, nil] config under validation (for actual/expected dig)
+        # @return [Html2rss::Config::ValidationReport]
+        def from(dry_result, values: nil)
+          return ValidationReport.ok if dry_result.success?
+
+          ValidationReport.failure(dry_result.errors.map { |message| issue_for(message, values:) })
+        end
+
+        ##
+        # @param message [String]
+        # @return [Html2rss::Config::ValidationReport]
+        def parse_failure(message)
+          ValidationReport.failure(
+            [
+              ValidationIssue.new(path: %i[parse], code: :parse, message:)
+            ]
+          )
+        end
+
+        private
+
+        def issue_for(message, values:)
+          path = Array(message.path).compact
+          ValidationIssue.new(
+            path:,
+            code: code_for(message.predicate),
+            message: message.text,
+            expected: expected_for(path, values),
+            actual: actual_for(path, values, message)
+          )
+        end
+
+        def code_for(predicate)
+          return :invalid_value if predicate.nil?
+
+          PREDICATE_CODES.fetch(predicate) do
+            raise ArgumentError, "unmapped Dry validation predicate: #{predicate.inspect}"
+          end
+        end
+
+        def actual_for(path, values, message)
+          dug = dig_path(values, path)
+          return dug unless dug.nil?
+          return message.input if message.is_a?(Dry::Schema::Message) && !message.input.nil?
+
+          nil
+        end
+
+        def expected_for(path, values)
+          return if values.nil? || path.empty?
+          return unless path.last.is_a?(Symbol)
+
+          parent = dig_path(values, path[0...-1])
+          return unless parent.is_a?(Hash)
+
+          option = Selectors::OptionSpec.option_for(parent, path.last)
+          return unless option
+
+          {
+            type: option.json_type,
+            required: option.required
+          }
+        end
+
+        def dig_path(values, path)
+          return if values.nil? || path.empty?
+
+          path.reduce(values) do |cursor, segment|
+            break unless cursor.is_a?(Hash) || (cursor.is_a?(Array) && segment.is_a?(Integer))
+
+            cursor[segment]
+          end
+        end
+      end
+    end
+  end
+end

data/lib/html2rss/config/request_headers.rb

@@ -33,18 +33,17 @@ module Html2rss
       }.freeze
 
       # Hop-by-hop headers forbidden in HTTP/2 requests (RFC 7540 §8.1.2.2 / RFC 9113 §8.2.1).
-      FORBIDDEN_H2_HEADERS = %w[connection keep-alive proxy-connection transfer-encoding upgrade].to_set.freeze
+      FORBIDDEN_H2_HEADERS = Set['connection', 'keep-alive', 'proxy-connection', 'transfer-encoding', 'upgrade'].freeze
 
       class << self
         ##
-        # :reek:ManualDispatch
         # :reek:TooManyStatements
         #
         # @return [Hash{String => String}] the default header set merged with global defaults
         def browser_defaults
           defaults = DEFAULT_HEADERS.dup
           global_headers = Html2rss.defaults.headers
-          global_headers = global_headers.call if global_headers.respond_to?(:call)
+          global_headers = global_headers.call if global_headers.is_a?(Proc)
 
           if global_headers.is_a?(Hash)
             global_headers.each do |key, value|

data/lib/html2rss/config/schema.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require 'json'
+
 module Html2rss
   class Config
     ##
@@ -19,6 +21,15 @@ module Html2rss
         Builder.call
       end
 
+      ##
+      # Returns the exported JSON Schema as JSON.
+      #
+      # @param pretty [Boolean] whether to pretty-print the JSON output
+      # @return [String] serialized JSON Schema
+      def json_schema_json(pretty: true)
+        pretty ? JSON.pretty_generate(json_schema) : JSON.generate(json_schema)
+      end
+
       ##
       # Resolves the packaged schema path used by downstream tools.
       #
@@ -107,12 +118,16 @@ module Html2rss
         # @param registry [Hash{Symbol => Class}]
         # @return [Hash{String => Hash}]
         def catalog_from_registry(registry)
-          registry.keys.sort.to_h { |name| [name.to_s, registry.fetch(name).schema_doc] }
+          registry.keys.sort.to_h { |name| [name.to_s, registry.fetch(name).schema_export] }
         end
 
         # rubocop:disable-next Metrics/MethodLength, Metrics/AbcSize, Layout/LineLength
         def overlay
           items_schema = Html2rss::Config::SelectorsValidator::Items.new.schema.json_schema(loose: true)
+          items_schema[:required] = Array(items_schema[:required]).reject { |key| key.to_sym == :selector }
+          items_schema[:properties][:selector] = items_schema.fetch(:properties).fetch(:selector).merge(
+            default: Html2rss::Selectors::DEFAULT_ITEMS_SELECTOR
+          )
           items_schema[:properties][:enhance] = items_schema.fetch(:properties).fetch(:enhance).merge(
             description: 'List-card enrichment: run Html::ArticleExtractor on each matched item node ' \
                          'to fill missing fields from the card HTML.'