html2rss 0.27.2 → 0.29.1

data/lib/html2rss/capture.rb

@@ -2,28 +2,73 @@
 
 module Html2rss
   ##
-  # Analyzes a URL and produces a durable feed config: items selector + +enhance: true+.
+  # Analyzes a URL and produces a durable feed config: items selector + metadata.
   #
   # Fetches via {FeedPipeline} (including AutoFallback for +:auto+), extracts articles,
-  # then derives a reusable items CSS selector from SST segments (list → cluster → semantic).
+  # derives reusable items CSS selectors from SST segments, and infers directory catalog metadata.
   #
   # {include:file:lib/html2rss/capture/README.md}
   class Capture # rubocop:disable Metrics/ClassLength -- segment strategies + CSS trim stay co-located
     LEADING_TRIM_TAGS = %w[html body].freeze
     private_constant :LEADING_TRIM_TAGS
 
+    # Packaged YAML language server modeline.
+    SCHEMA_MODELINE = '# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json'
+
     # Ordered Segmenter strategies tried until the items selector quality gate passes.
     SEGMENT_STRATEGIES = %i[list cluster semantic].freeze
 
     # Minimum matched segment/article pairs required to emit an items selector.
     MIN_SELECTOR_MATCHES = 2
 
+    # Admission drops at or above this sum default enhance to false (chrome-heavy listing).
+    CHROME_DROP_THRESHOLD = 3
+    private_constant :CHROME_DROP_THRESHOLD
+
     ##
     # Result of a capture operation (config plus quality meta).
     CaptureResult = Data.define(
-      :config, :articles_count, :channel_title, :has_selectors, :segment_strategy,
-      :admission_drops, :selected_strategy
-    )
+      :config, :yaml, :articles_count, :channel_title, :has_selectors, :segment_strategy,
+      :admission_drops, :selected_strategy, :inferred_topics, :native_feed, :suggested_channel_url
+    ) do
+      # rubocop:disable Metrics/ParameterLists
+      ##
+      # @param config [Hash]
+      # @param articles_count [Integer]
+      # @param channel_title [String]
+      # @param has_selectors [Boolean]
+      # @param segment_strategy [Symbol]
+      # @param yaml [String, nil]
+      # @param admission_drops [Hash]
+      # @param selected_strategy [Symbol, nil]
+      # @param inferred_topics [Array<String>]
+      # @param native_feed [String, nil]
+      # @param suggested_channel_url [String, nil]
+      def initialize(config:, articles_count:, channel_title:, has_selectors:, segment_strategy:, # rubocop:disable Metrics/MethodLength
+                     yaml: nil, admission_drops: {}, selected_strategy: nil, inferred_topics: [], native_feed: nil,
+                     suggested_channel_url: nil)
+        super(
+          config:,
+          yaml: yaml || "#{SCHEMA_MODELINE}\n#{Config.to_yaml(config)}",
+          articles_count:,
+          channel_title:,
+          has_selectors:,
+          segment_strategy:,
+          admission_drops:,
+          selected_strategy:,
+          inferred_topics:,
+          native_feed:,
+          suggested_channel_url:
+        )
+      end
+      # rubocop:enable Metrics/ParameterLists
+
+      ##
+      # @return [Boolean] whether first-party RSS was detected
+      def native_feed?
+        !native_feed.nil?
+      end
+    end
 
     class << self
       ##
@@ -32,6 +77,11 @@ module Html2rss
       # @param url [String] source page URL
       # @param strategy [Symbol] request strategy (+:auto+, +:faraday+, +:botasaurus+)
       # @option options [String, nil] :items_selector optional selector hint
+      # @option options [Array<String>, nil] :topics optional directory topics override
+      # @option options [String, nil] :title optional title override
+      # @option options [String, nil] :summary optional summary override
+      # @option options [Boolean] :force whether to ignore native feed detection
+      # @option options [Boolean, nil] :enhance whether to force enhance: true
       # @option options [Integer, nil] :max_redirects optional redirect limit override
       # @option options [Integer, nil] :max_requests optional request budget override
       # @option options [Integer, nil] :limit max articles to keep
@@ -45,16 +95,25 @@ module Html2rss
     ##
     # @param url [String] source page URL
     # @param strategy [Symbol] request strategy
-    # @param options [Hash] additional options
     # @option options [String, nil] :items_selector optional selector hint
+    # @option options [Array<String>, nil] :topics optional directory topics override
+    # @option options [String, nil] :title optional title override
+    # @option options [String, nil] :summary optional summary override
+    # @option options [Boolean] :force whether to ignore native feed detection
+    # @option options [Boolean, nil] :enhance whether to force enhance: true
     # @option options [Integer, nil] :max_redirects optional redirect limit override
     # @option options [Integer, nil] :max_requests optional request budget override
     # @option options [Integer, nil] :limit max articles to keep
     # @option options [String, nil] :local_file_path optional local HTML file path
-    def initialize(url, strategy: :auto, **options)
+    def initialize(url, strategy: :auto, **options) # rubocop:disable Metrics/MethodLength
       @url = url
       @strategy = strategy
       @items_selector_hint = options.delete(:items_selector)
+      @topics = options.delete(:topics)
+      @title = options.delete(:title)
+      @summary = options.delete(:summary)
+      @force = options.delete(:force) || false
+      @enhance_option = options.delete(:enhance)
       @max_redirects = options.delete(:max_redirects)
       @max_requests = options.delete(:max_requests)
       @limit = options.delete(:limit)
@@ -65,12 +124,17 @@ module Html2rss
     # Runs the capture pipeline.
     #
     # @return [CaptureResult]
-    def build # rubocop:disable Metrics/AbcSize, Metrics/MethodLength -- outcome + selector + result assembly
+    def build # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
       outcome = FeedPipeline.new(raw_config).to_outcome
+      @admission_drops = outcome.admission_drops
       selectors, segment_strategy = derive_selectors(outcome.response, outcome.articles)
+      ch_title = @title || channel_title_from(outcome.response)
+      topics = @topics || infer_topics("#{@url} #{ch_title}")
+      native_feed = probe_native_feed(outcome.response) unless @force
 
       config = {
-        channel: build_channel(outcome.response),
+        directory: build_directory(ch_title, topics),
+        channel: build_channel(outcome.response, ch_title),
         selectors: selectors.empty? ? nil : selectors,
         **strategy_stamp(outcome),
         **local_file_request_overlay
@@ -78,17 +142,75 @@ module Html2rss
 
       CaptureResult.new(
         config:,
+        yaml: "#{SCHEMA_MODELINE}\n#{Config.to_yaml(config)}",
         articles_count: outcome.articles.size,
-        channel_title: channel_title_from(outcome.response),
+        channel_title: ch_title,
         has_selectors: !selectors.empty?,
         segment_strategy:,
         admission_drops: outcome.admission_drops,
-        selected_strategy: outcome.selected_strategy
+        selected_strategy: outcome.selected_strategy,
+        inferred_topics: topics,
+        native_feed:,
+        suggested_channel_url: suggested_channel_url(outcome)
       )
     end
 
     private
 
+    def build_directory(title, topics)
+      summary_text = @summary || "#{title} updates and announcements."
+      {
+        topics: topics.take(2),
+        title:,
+        summary: summary_text[0, 160]
+      }.compact
+    end
+
+    def probe_native_feed(response)
+      return nil unless response && @url
+
+      Syndication::Discovery.best_feed_url(
+        page_url: @url,
+        request_session: discovery_session,
+        parsed_body: (response.parsed_body if response.html_response?),
+        html: response.body
+      )&.to_s
+    rescue StandardError
+      nil
+    end
+
+    def discovery_session
+      config = Config.from_hash(raw_config)
+      resources = FeedPipeline::RuntimePolicy.resources_for(config)
+      strategy = FeedPipeline::StrategyPlan.concrete_for_diagnostic(@strategy)
+      RequestSession.build(
+        config:,
+        strategy:,
+        budget: resources.budget,
+        policy: resources.policy
+      )
+    end
+
+    def infer_topics(text) # rubocop:disable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity, Metrics/AbcSize, Metrics/MethodLength
+      t = text.to_s.downcase
+      inferred = []
+      inferred << 'tech' if /tech|developer|software|cloud|api|compute|ai|data|model|hardware|cyber/.match?(t)
+      inferred << 'security' if /security|vulnerab|advis|threat|breach|cve|cert|patch/.match?(t)
+      inferred << 'science' if /science|space|astron|physics|bio|research|institut|discover|nature/.match?(t)
+      inferred << 'energy' if /energy|wind|solar|power|grid|renewab|oil|gas|hydro/.match?(t)
+      inferred << 'finance' if /financ|bank|invest|monetary|market|regulat|treasury|econom/.match?(t)
+      inferred << 'civic' if /govern|policy|council|parliament|court|treaty|public/.match?(t)
+      inferred << 'environment' if /climat|environ|planet|sustainab|carbon|ecolog|earth/.match?(t)
+      inferred << 'consumer' if /consum|recall|safety|rating|test|product/.match?(t)
+      inferred << 'travel' if /travel|touris|destination|trip|flight|hotel|transit|rail/.match?(t)
+      inferred << 'sports' if /sport|olympic|racing|football|soccer|tennis|game/.match?(t)
+      inferred << 'health' if /health|medic|pharma|hospital|disease|patient/.match?(t)
+      inferred << 'education' if /educat|universit|school|student|academi|learn/.match?(t)
+      inferred << 'transport' if /transport|traffic|automotive|vehicle|train|bus/.match?(t)
+      valid = inferred.select { |topic| Config::Validator::DIRECTORY_TOPICS.include?(topic) }
+      valid.empty? ? ['news'] : valid.first(2)
+    end
+
     def strategy_stamp(outcome)
       concrete = outcome.selected_strategy || concrete_request_strategy
       return {} if concrete.nil? || concrete == :auto
@@ -147,11 +269,12 @@ module Html2rss
     end
 
     def hint_selectors
-      [{ items: { selector: @items_selector_hint, enhance: true } }, :hint]
+      [{ items: { selector: @items_selector_hint, enhance: resolve_enhance } }, :hint]
     end
 
     def select_enhance_selectors(sst, articles) # rubocop:disable Metrics/MethodLength -- strategy loop + gate
       link_resolver = Scoring::LinkResolver.new(@url)
+      enhance = resolve_enhance
 
       SEGMENT_STRATEGIES.each do |strategy|
         segments = AutoSource::Segmenter.call(
@@ -161,7 +284,7 @@ module Html2rss
         items_sel = items_selector(matched)
         next unless items_sel && matched.size >= MIN_SELECTOR_MATCHES
 
-        return [{ items: { selector: items_sel, enhance: true } }, strategy]
+        return [{ items: { selector: items_sel, enhance: } }, strategy]
       end
 
       [{}, nil]
@@ -301,10 +424,10 @@ module Html2rss
       "/#{prefix.join('/')}"
     end
 
-    def build_channel(response)
+    def build_channel(response, title = nil)
       {
         url: @url,
-        title: channel_title_from(response),
+        title: title || channel_title_from(response),
         time_zone: 'UTC'
       }.compact
     end
@@ -314,5 +437,25 @@ module Html2rss
     rescue StandardError
       nil
     end
+
+    def resolve_enhance
+      return @enhance_option unless @enhance_option.nil?
+
+      drops = (@admission_drops || {}).values.sum
+      drops < CHROME_DROP_THRESHOLD
+    end
+
+    def suggested_channel_url(outcome)
+      entry = outcome.scrape_target.entry_url.to_s
+      effective = outcome.scrape_target.effective_url.to_s
+      return effective unless Url.from_absolute(entry).same_document?(Url.from_absolute(effective))
+
+      final = outcome.response.url.to_s
+      return final unless Url.from_absolute(@url).same_document?(Url.from_absolute(final))
+
+      nil
+    rescue ArgumentError
+      nil
+    end
   end
 end

data/lib/html2rss/channel.rb

@@ -155,7 +155,7 @@ module Html2rss
     # @param last_build_date [Time, String]
     # @param image [Html2rss::Url, String, nil]
     # @param author [String, nil]
-    # rubocop:disable Metrics/ParameterLists -- Data.define members are the channel contract
+    # rubocop:disable-next Metrics/ParameterLists -- Data.define members are the channel contract
     def initialize(title:, url:, description:, language:, ttl:, last_build_date:, image:, author:)
       super(
         title: freeze_string(title),
@@ -168,7 +168,6 @@ module Html2rss
         author: author && freeze_string(author)
       )
     end
-    # rubocop:enable Metrics/ParameterLists
 
     private
 

data/lib/html2rss/cli/probe_view.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+module Html2rss
+  class CLI
+    ##
+    # Display-only probe facts for inspect and recon text cards.
+    ProbeView = Data.define(
+      :requested,
+      :final,
+      :status,
+      :surface,
+      :articles_count,
+      :verdict,
+      :native_feed,
+      :alternate_feeds,
+      :notes,
+      :strategy
+    ) do
+      class << self
+        ##
+        # @param data [Hash] inspect wire payload
+        # @return [ProbeView]
+        def from_wire(data) # rubocop:disable Metrics/MethodLength
+          new(
+            requested: wire_val(data, :requested_url),
+            final: wire_val(data, :final_url),
+            status: wire_val(data, :status),
+            surface: wire_val(data, :surface_category),
+            articles_count: wire_val(data, :articles_count),
+            verdict: nil,
+            native_feed: nil,
+            alternate_feeds: wire_val(data, :alternate_feeds) || [],
+            notes: [],
+            strategy: wire_val(data, :strategy)
+          )
+        end
+
+        ##
+        # @param result [Html2rss::Recon::Result]
+        # @return [ProbeView]
+        def from_recon(result) # rubocop:disable Metrics/MethodLength
+          new(
+            requested: result.requested_url.to_s,
+            final: result.final_url.to_s,
+            status: result.status,
+            surface: result.surface_category,
+            articles_count: result.articles_count,
+            verdict: result.verdict,
+            native_feed: result.native_feed&.to_s,
+            alternate_feeds: [],
+            notes: result.notes,
+            strategy: nil
+          )
+        end
+
+        ##
+        # @param data [Hash]
+        # @param key [Symbol]
+        # @return [Object, nil]
+        def wire_val(data, key)
+          data[key] || data[key.to_s]
+        end
+      end
+    end
+  end
+end

data/lib/html2rss/cli/render.rb

@@ -0,0 +1,124 @@
+# frozen_string_literal: true
+
+require 'json'
+
+module Html2rss
+  class CLI
+    ##
+    # Text and JSON rendering for CLI command output (display only; no policy).
+    module Render
+      # ANSI colors for recon verdict labels in text output.
+      VERDICT_COLORS = { build: "\e[32m", defer: "\e[33m" }.freeze
+
+      module_function
+
+      ##
+      # @param results [Array<Hash>]
+      # @param format [String]
+      # @param batch_mode [Boolean]
+      # @return [void]
+      def inspect_output(results, format:, batch_mode:)
+        if format == 'json'
+          json(results, batch_mode)
+        else
+          results.each { |data| probe_card(ProbeView.from_wire(data)) }
+        end
+      end
+
+      ##
+      # @param results [Array<Html2rss::Recon::Result>]
+      # @param format [String]
+      # @param batch_mode [Boolean]
+      # @param url_only [Boolean]
+      # @return [void]
+      def recon_output(results, format:, batch_mode:, url_only: false) # rubocop:disable Metrics/MethodLength, Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
+        if url_only
+          results.each { |r| puts r.requested_url }
+        elsif format == 'json'
+          json(results.map(&:to_h), batch_mode)
+        elsif format == 'tsv'
+          puts %w[verdict status requested_url final_url native_feed notes].join("\t")
+          results.each do |r|
+            row = [
+              r.verdict.to_s.upcase,
+              r.status || '-',
+              r.requested_url,
+              r.final_url,
+              r.native_feed || '-',
+              r.notes.join('; ')
+            ]
+            puts row.join("\t")
+          end
+        else
+          results.each { |r| probe_card(ProbeView.from_recon(r)) }
+        end
+      end
+
+      ##
+      # @param view [ProbeView]
+      # @return [void]
+      def probe_card(view) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
+        if view.verdict
+          color = VERDICT_COLORS.fetch(view.verdict.to_sym, "\e[31m")
+          puts "#{color}[#{view.verdict.to_s.upcase}]\e[0m #{view.requested}"
+        else
+          puts view.requested
+        end
+        probe_lines(view)
+        if view.alternate_feeds.any?
+          hrefs = view.alternate_feeds.filter_map { |f| ProbeView.wire_val(f, :href) }
+          puts "        Feeds:    #{hrefs.join(', ')}" if hrefs.any?
+        end
+        puts "        Feed:     #{view.native_feed}" if view.native_feed
+        puts "        Notes:    #{view.notes.join(', ')}" if view.notes.any?
+        puts "        Strategy: #{view.strategy}" if view.strategy
+        puts ''
+      end
+
+      ##
+      # @param result [Html2rss::Test::Result]
+      # @param source [String]
+      # @return [void]
+      def test_card(result, source) # rubocop:disable Metrics/MethodLength, Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
+        if result.success
+          puts "\e[32m✓ Schema valid\e[0m (#{source})"
+          dur = "#{result.duration_seconds}s"
+          puts "\e[32m✓ Extracted #{result.item_count} items in #{dur}\e[0m (strategy: #{result.strategy_used})"
+          puts "\nChannel: #{result.channel_title} (#{result.channel_url})"
+          if result.sample_items.any?
+            puts 'Sample items:'
+            result.sample_items.each_with_index do |item, i|
+              puts "  #{i + 1}. #{"#{item[:published_at]} | " if item[:published_at]}#{item[:title]}"
+              puts "     #{item[:url]}"
+            end
+          end
+        else
+          warn "\e[31m✗ Test failed\e[0m (#{source})"
+          warn "  Error: #{result.error_message}" if result.error_message
+          result.validation_errors&.each { |k, v| warn "  Schema error [#{k}]: #{Array(v).join(', ')}" }
+        end
+      end
+
+      ##
+      # @param data [Object]
+      # @param batch_mode [Boolean]
+      # @return [void]
+      def json(data, batch_mode)
+        payload = batch_mode ? data : data.first
+        puts JSON.pretty_generate(payload)
+      end
+
+      ##
+      # @param view [ProbeView]
+      # @return [void]
+      def probe_lines(view)
+        if view.final && view.final != view.requested
+          puts "        Final:    #{view.final} (HTTP #{view.status || 'ERR'})"
+        end
+        surface = view.surface.respond_to?(:to_s) ? view.surface.to_s : view.surface
+        puts "        Surface:  #{surface} (#{view.articles_count} articles)"
+      end
+      private_class_method :probe_lines
+    end
+  end
+end

data/lib/html2rss/cli/validate.rb

@@ -0,0 +1,91 @@
+# frozen_string_literal: true
+
+module Html2rss
+  class CLI
+    ##
+    # Validate command: file/glob/stdin branching and multi-file reporting.
+    module Validate
+      module_function
+
+      ##
+      # @param files [Array<String>]
+      # @param params [Hash]
+      # @param quiet [Boolean]
+      # @return [void]
+      # @raise [Thor::Error]
+      def run(files, params:, quiet:) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
+        if files.size == 2 && File.file?(files[0].to_s) && !File.exist?(files[1].to_s)
+          raise Thor::Error, "No such file: #{files[1]}" if path_like_config?(files[1])
+
+          return run_named_feed(files[0], files[1], params:, quiet:)
+        end
+
+        target_files = resolve_files(files)
+        failed = []
+
+        target_files.each do |file|
+          result = validate_file(file, params:)
+
+          if result.success?
+            puts(target_files.size == 1 ? 'Configuration is valid' : "ok   #{file}") unless quiet
+          else
+            error_details = result.errors.to_h
+            raise Thor::Error, "Invalid configuration: #{error_details}" if target_files.size == 1
+
+            warn "FAIL #{file}"
+            error_details.each { |key, msg| warn "       #{key}: #{Array(msg).join(', ')}" }
+            failed << file
+          end
+        end
+
+        return if failed.empty?
+
+        raise Thor::Error, "#{failed.size}/#{target_files.size} configurations failed validation."
+      end
+
+      ##
+      # @param files [Array<String>]
+      # @return [Array<String>]
+      def resolve_files(files)
+        return ['-'] if files.empty? || files == ['-']
+
+        resolved = []
+        files.each do |f|
+          matched = Dir.glob(f)
+          resolved.concat(matched.empty? ? [f] : matched)
+        end
+        resolved.uniq
+      end
+
+      def run_named_feed(file, feed_name, params:, quiet:)
+        result = Html2rss.validate(file, feed_name, params:)
+        raise Thor::Error, "Invalid configuration: #{result.errors.to_h}" unless result.success?
+
+        puts 'Configuration is valid' unless quiet
+      end
+      module_function :run_named_feed
+      private_class_method :run_named_feed
+
+      def validate_file(file, params:)
+        if file == '-'
+          Html2rss.validate($stdin.read, params:)
+        else
+          Html2rss.validate(file, params:)
+        end
+      end
+      module_function :validate_file
+      private_class_method :validate_file
+
+      def path_like_config?(arg)
+        path = arg.to_s
+        return true if path.start_with?('-')
+        return true if path.end_with?('.yml', '.yaml')
+        return true if path.include?('/') || path.include?('*')
+
+        false
+      end
+      module_function :path_like_config?
+      private_class_method :path_like_config?
+    end
+  end
+end