html2rss 0.32.4 → 0.34.0

data/lib/html2rss/capture.rb

@@ -89,6 +89,8 @@ module Html2rss
       # @option options [Boolean, nil] :enhance whether to force enhance: true
       # @option options [Integer, nil] :max_redirects optional redirect limit override
       # @option options [Integer, nil] :max_requests optional request budget override
+      # @option options [Integer, nil] :total_timeout_seconds optional wall-clock bound (seconds) for the whole
+      #   request chain, including every +:auto+ fallback hop (shared budget, not per hop)
       # @option options [Integer, nil] :limit max articles to keep
       # @option options [String, nil] :local_file_path optional local HTML file path
       # @return [CaptureResult]
@@ -108,6 +110,8 @@ module Html2rss
     # @option options [Boolean, nil] :enhance whether to force enhance: true
     # @option options [Integer, nil] :max_redirects optional redirect limit override
     # @option options [Integer, nil] :max_requests optional request budget override
+    # @option options [Integer, nil] :total_timeout_seconds optional wall-clock bound (seconds) for the whole
+    #   request chain, including every +:auto+ fallback hop (shared budget, not per hop)
     # @option options [Integer, nil] :limit max articles to keep
     # @option options [String, nil] :local_file_path optional local HTML file path
     def initialize(url, strategy: :auto, **options) # rubocop:disable Metrics/MethodLength
@@ -119,8 +123,7 @@ module Html2rss
       @summary = options.delete(:summary)
       @force = options.delete(:force) || false
       @enhance_option = options.delete(:enhance)
-      @max_redirects = options.delete(:max_redirects)
-      @max_requests = options.delete(:max_requests)
+      @request_options = options.slice(:max_redirects, :max_requests, :total_timeout_seconds)
       @limit = options.delete(:limit)
       @local_file_path = options.delete(:local_file_path)
     end
@@ -147,7 +150,6 @@ module Html2rss
 
       CaptureResult.new(
         config:,
-        yaml: "#{SCHEMA_MODELINE}\n#{Config.to_yaml(config)}",
         articles_count: outcome.articles.size,
         channel_title: ch_title,
         has_selectors: !selectors.empty?,
@@ -173,15 +175,10 @@ module Html2rss
     end
 
     def probe_native_feed(response)
-      return nil unless response && @url
-
-      Syndication::Discovery.best_feed_url(
-        page_url: @url,
-        request_session: discovery_session,
-        parsed_body: (response.parsed_body if response.html_response?),
-        html: response.body
-      )&.to_s
-    rescue StandardError
+      Syndication::Discovery.native_feed_url(response:, request_session: discovery_session)&.to_s
+    rescue Html2rss::ContractError
+      raise
+    rescue Html2rss::Error
       nil
     end
 
@@ -237,11 +234,7 @@ module Html2rss
       Config.auto_source_config(
         url: @url,
         items_selector: @items_selector_hint,
-        request_controls: Config::RequestControls.from_shortcut(
-          strategy: @strategy,
-          max_redirects: @max_redirects,
-          max_requests: @max_requests
-        ),
+        request_controls: Config::RequestControls.from_shortcut(strategy: @strategy, **@request_options),
         limit: @limit
       ).tap { |config| apply_local_file_path!(config) }
     end
@@ -270,9 +263,6 @@ module Html2rss
       return [*default_selectors, SelectorCandidates.empty] unless sst
 
       select_enhance_selectors(sst, articles)
-    rescue ArgumentError => error
-      Log.warn("Capture selector derivation failed: #{error.message}")
-      [*default_selectors, SelectorCandidates.empty]
     end
 
     def hint_selectors
@@ -300,7 +290,7 @@ module Html2rss
 
       SEGMENT_STRATEGIES.flat_map do |strategy|
         segments = AutoSource::Segmenter.call(
-          sst, base_url: @url, strategy:, permit_unanchored: false, link_resolver:
+          sst, base_url: @url, strategy:, link_resolver:
         )
         matched = match_segments_to_articles(segments, articles)
         next [] if matched.size < MIN_SELECTOR_MATCHES
@@ -338,42 +328,34 @@ module Html2rss
       end
     end
 
-    def find_matching_article(segment, articles_by_url) # rubocop:disable Metrics/AbcSize
-      href = segment.primary_link.attrs.href.to_s
-      return articles_by_url.values.find { |a| a.title == title_from_segment(segment) } if href.empty?
-
-      resolved = Url.from_relative(href, @url)
+    def find_matching_article(segment, articles_by_url)
+      resolved = Url.from_relative(segment.primary_link.attrs.href.to_s, @url)
       articles_by_url[resolved.to_s] || articles_by_url.values.find { |a| fuzzy_url_match?(a.url, resolved) }
     end
 
     def fuzzy_url_match?(article_url, resolved)
       article_url.to_s.end_with?(resolved.path.to_s)
-    rescue StandardError
-      false
-    end
-
-    def title_from_segment(segment)
-      segment.root_node.visible_text.to_s.strip
     end
 
     def lift_heading_link_roots(roots)
-      roots.map { |root| lift_heading_link_root(root, roots) }
+      enclosed = enclosed_root_counts(roots)
+      roots.map { |root| lift_heading_link_root(root, enclosed) }
     end
 
-    def lift_heading_link_root(root, all_roots)
+    def lift_heading_link_root(root, enclosed)
       return root unless heading_or_inner_title_link?(root)
 
       index = SST::Index.for_node(root)
       return root unless index
 
-      walk_usable_card(root, index, all_roots)
+      walk_usable_card(root, index, enclosed)
     end
 
-    def walk_usable_card(root, index, all_roots)
+    def walk_usable_card(root, index, enclosed)
       candidate = root
       parent = index.parent_of(root)
-      while parent && Html::Navigator.usable_card_parent?(parent)
-        break if contains_other_root?(parent, root, all_roots)
+      while Html::Navigator.usable_card_parent?(parent)
+        break if contains_other_root?(parent, enclosed)
 
         candidate = parent
         parent = index.parent_of(parent)
@@ -381,6 +363,28 @@ module Html2rss
       candidate
     end
 
+    # Identity-keyed tally of how many distinct roots each node is, or strictly encloses, so the
+    # card walk answers "does this ancestor hold another root?" without rescanning all roots.
+    def enclosed_root_counts(roots)
+      tally = Hash.new(0).compare_by_identity
+      roots.uniq(&:object_id).each do |root|
+        tally[root] += 1
+        count_enclosing_ancestors(root, tally)
+      end
+      tally
+    end
+
+    def count_enclosing_ancestors(root, tally)
+      index = SST::Index.for_node(root)
+      return unless index
+
+      ancestor = index.parent_of(root)
+      while ancestor
+        tally[ancestor] += 1
+        ancestor = index.parent_of(ancestor)
+      end
+    end
+
     def heading_or_inner_title_link?(node)
       name = node.name.to_s
       return false if wrapping_anchor_root?(node)
@@ -395,16 +399,9 @@ module Html2rss
       node.find { |child| !child.equal?(node) && tags.include?(child.name.to_s) }
     end
 
-    def contains_other_root?(parent, root, all_roots)
-      all_roots.any? do |other|
-        next if other.equal?(root)
-
-        other.equal?(parent) || sst_descendant?(other, parent)
-      end
-    end
-
-    def sst_descendant?(child, ancestor)
-      SST::Index.for_node(child)&.descendant_of?(child, ancestor)
+    # The walked-from root is itself tallied under every ancestor, so more than one means another root.
+    def contains_other_root?(parent, enclosed)
+      enclosed[parent] > 1
     end
 
     def shared_class_items_selector(roots)
@@ -420,12 +417,7 @@ module Html2rss
     end
 
     def path_items_selector(roots)
-      paths = roots.map(&:tag_path)
-      common = common_path_prefix(paths)
-      tag_path = common.empty? ? paths.first.to_s : common
-      return if tag_path.empty?
-
-      css_from_trimmed_tag_path(tag_path)
+      css_from_trimmed_tag_path(common_path_prefix(roots.map(&:tag_path)))
     end
 
     def css_from_trimmed_tag_path(tag_path)
@@ -443,9 +435,7 @@ module Html2rss
       trimmed.empty? ? [segments.last] : trimmed
     end
 
-    def common_path_prefix(paths) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity
-      return '' if paths.empty?
-
+    def common_path_prefix(paths)
       segments = paths.map { |p| p.split('/').reject(&:empty?) }
       prefix = segments.first
       segments.each do |seg|
@@ -466,7 +456,9 @@ module Html2rss
 
     def channel_title_from(response)
       Channel.from_response(response).title
-    rescue StandardError
+    rescue Html2rss::ContractError
+      raise
+    rescue Html2rss::Error
       nil
     end
 
@@ -486,7 +478,7 @@ module Html2rss
       return final unless Url.from_absolute(@url).same_document?(Url.from_absolute(final))
 
       nil
-    rescue ArgumentError
+    rescue Html2rss::Url::InvalidUrlError
       nil
     end
   end

data/lib/html2rss/channel.rb

@@ -3,7 +3,9 @@
 require 'time'
 
 module Html2rss
-  Channel = Data.define(:title, :url, :description, :language, :ttl, :last_build_date, :image, :author)
+  Channel = Data.define(
+    :title, :url, :description, :language, :ttl, :last_build_date, :image, :author, :categories, :explicit
+  )
 
   ##
   # Materialized channel metadata for feed builders and {FeedResult}.
@@ -12,6 +14,10 @@ module Html2rss
   # do not own channel extraction. Safe to Marshal (no Response / Nokogiri retained).
   # Strings and image URL are normalized and frozen at construction (including Marshal load).
   #
+  # The author is the config +channel.author+, else the followed native feed's author
+  # ({Syndication::ChannelFacts}), else the page's +<meta name="author">+. Categories and the explicit
+  # flag come only from that feed, when it states them.
+  #
   # @!attribute [r] title
   #   @return [String]
   # @!attribute [r] url
@@ -28,6 +34,10 @@ module Html2rss
   #   @return [Html2rss::Url, nil]
   # @!attribute [r] author
   #   @return [String, nil]
+  # @!attribute [r] categories
+  #   @return [Array<Array<String>>] each category followed by its subcategories (empty when unstated)
+  # @!attribute [r] explicit
+  #   @return [Boolean, nil] whether the content is declared explicit; nil when unstated
   class Channel # rubocop:disable Metrics/ClassLength -- Channel value object owns extract + freeze + Marshal
     class << self
       ##
@@ -35,8 +45,9 @@ module Html2rss
       #
       # @param response [Html2rss::RequestService::Response]
       # @param overrides [Hash{Symbol => Object}] optional overrides for channel attributes
+      # @param feed [Html2rss::Syndication::ChannelFacts, nil] what the followed native feed states
       # @return [Html2rss::Channel]
-      def from_response(response, overrides: {})
+      def from_response(response, overrides: {}, feed: nil) # rubocop:disable Metrics/MethodLength -- one line per member
         new(
           title: title_for(response, overrides),
           url: url_for(response),
@@ -45,7 +56,9 @@ module Html2rss
           ttl: ttl_for(response, overrides),
           last_build_date: last_build_date_for(response),
           image: image_for(response, overrides),
-          author: author_for(response, overrides)
+          author: overrides[:author] || feed&.author || meta_author(response),
+          categories: feed&.categories || [],
+          explicit: feed&.explicit
         )
       end
 
@@ -117,8 +130,7 @@ module Html2rss
         parsed_body(response)['lang'] || parsed_body(response).at_css('[lang]')&.[]('lang')
       end
 
-      def author_for(response, overrides)
-        return overrides[:author] if overrides[:author]
+      def meta_author(response)
         return unless html_response?(response)
 
         parsed_body(response).at_css('meta[name="author"]')&.[]('content')
@@ -156,8 +168,11 @@ module Html2rss
     # @param last_build_date [Time, String]
     # @param image [Html2rss::Url, String, nil]
     # @param author [String, nil]
-    # rubocop:disable-next Metrics/ParameterLists -- Data.define members are the channel contract
-    def initialize(title:, url:, description:, language:, ttl:, last_build_date:, image:, author:)
+    # @param categories [Array<Array<String>>] each category followed by its subcategories
+    # @param explicit [Boolean, nil]
+    # rubocop:disable-next Metrics/ParameterLists, Metrics/MethodLength -- Data.define members are the channel contract
+    def initialize(title:, url:, description:, language:, ttl:, last_build_date:, image:, author:,
+                   categories: [], explicit: nil)
       super(
         title: freeze_string(title),
         url: url.is_a?(Url) ? url : Url.from_absolute(url),
@@ -166,16 +181,19 @@ module Html2rss
         ttl: Integer(ttl),
         last_build_date: normalize_last_build_date(last_build_date),
         image: normalize_image(image),
-        author: author && freeze_string(author)
+        author: author && freeze_string(author),
+        categories: categories.map { |path| path.map { freeze_string(_1) }.freeze }.freeze,
+        explicit:
       )
     end
 
     private
 
-    def marshal_dump = [title, url, description, language, ttl, last_build_date, image, author]
+    def marshal_dump = [title, url, description, language, ttl, last_build_date, image, author, categories, explicit]
 
-    def marshal_load((title, url, description, language, ttl, last_build_date, image, author))
-      initialize(title:, url:, description:, language:, ttl:, last_build_date:, image:, author:)
+    def marshal_load((title, url, description, language, ttl, last_build_date, image, author, categories, explicit))
+      initialize(title:, url:, description:, language:, ttl:, last_build_date:, image:, author:, categories:,
+                 explicit:)
     end
 
     def freeze_string(value) = value.to_s.dup.freeze
@@ -189,7 +207,7 @@ module Html2rss
 
     def normalize_last_build_date(value)
       return value if value.is_a?(Time)
-      raise ArgumentError, 'last_build_date must be a Time or HTTP-date String' unless value.is_a?(String)
+      raise InvalidOption, 'last_build_date must be a Time or HTTP-date String' unless value.is_a?(String)
 
       Time.httpdate(value)
     end

data/lib/html2rss/cli/render.rb

@@ -10,116 +10,117 @@ module Html2rss
       # ANSI colors for recon verdict labels in text output.
       VERDICT_COLORS = { build: "\e[32m", defer: "\e[33m" }.freeze
 
-      module_function
-
-      ##
-      # @param results [Array<Hash>]
-      # @param format [String]
-      # @param batch_mode [Boolean]
-      # @return [void]
-      def inspect_output(results, format:, batch_mode:)
-        if format == 'json'
-          json(results, batch_mode)
-        else
-          results.each { |data| probe_card(ProbeView.from_wire(data)) }
+      class << self
+        ##
+        # @param results [Array<Hash>]
+        # @param format [String]
+        # @param batch_mode [Boolean]
+        # @return [void]
+        def inspect_output(results, format:, batch_mode:)
+          if format == 'json'
+            json(results, batch_mode)
+          else
+            results.each { |data| probe_card(ProbeView.from_wire(data)) }
+          end
         end
-      end
 
-      ##
-      # @param results [Array<Html2rss::Recon::Result>]
-      # @param format [String]
-      # @param batch_mode [Boolean]
-      # @param url_only [Boolean]
-      # @return [void]
-      def recon_output(results, format:, batch_mode:, url_only: false) # rubocop:disable Metrics/MethodLength, Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
-        if url_only
-          results.each { |r| puts r.requested_url }
-        elsif format == 'json'
-          json(results.map(&:to_h), batch_mode)
-        elsif format == 'tsv'
-          puts %w[verdict status requested_url final_url native_feed notes].join("\t")
-          results.each do |r|
-            row = [
-              r.verdict.to_s.upcase,
-              r.status || '-',
-              r.requested_url,
-              r.final_url,
-              r.native_feed || '-',
-              r.notes.join('; ')
-            ]
-            puts row.join("\t")
+        ##
+        # @param results [Array<Html2rss::Recon::Result>]
+        # @param format [String]
+        # @param batch_mode [Boolean]
+        # @param url_only [Boolean]
+        # @return [void]
+        def recon_output(results, format:, batch_mode:, url_only: false) # rubocop:disable Metrics/MethodLength, Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
+          if url_only
+            results.each { |r| puts r.requested_url }
+          elsif format == 'json'
+            json(results.map(&:to_h), batch_mode)
+          elsif format == 'tsv'
+            puts %w[verdict status requested_url final_url native_feed notes].join("\t")
+            results.each do |r|
+              row = [
+                r.verdict.to_s.upcase,
+                r.status || '-',
+                r.requested_url,
+                r.final_url,
+                r.native_feed || '-',
+                r.notes.join('; ')
+              ]
+              puts row.join("\t")
+            end
+          else
+            results.each { |r| probe_card(ProbeView.from_recon(r)) }
           end
-        else
-          results.each { |r| probe_card(ProbeView.from_recon(r)) }
         end
-      end
 
-      ##
-      # @param view [ProbeView]
-      # @return [void]
-      def probe_card(view) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
-        if view.verdict
-          color = VERDICT_COLORS.fetch(view.verdict.to_sym, "\e[31m")
-          puts "#{color}[#{view.verdict.to_s.upcase}]\e[0m #{view.requested}"
-        else
-          puts view.requested
-        end
-        probe_lines(view)
-        if view.alternate_feeds.any?
-          hrefs = view.alternate_feeds.filter_map { |f| ProbeView.wire_val(f, :href) }
-          puts "        Feeds:    #{hrefs.join(', ')}" if hrefs.any?
+        ##
+        # @param view [ProbeView]
+        # @return [void]
+        def probe_card(view) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
+          if view.verdict
+            color = VERDICT_COLORS.fetch(view.verdict.to_sym, "\e[31m")
+            puts "#{color}[#{view.verdict.to_s.upcase}]\e[0m #{view.requested}"
+          else
+            puts view.requested
+          end
+          probe_lines(view)
+          if view.alternate_feeds.any?
+            hrefs = view.alternate_feeds.filter_map { |f| ProbeView.wire_val(f, :href) }
+            puts "        Feeds:    #{hrefs.join(', ')}" if hrefs.any?
+          end
+          puts "        Feed:     #{view.native_feed}" if view.native_feed
+          puts "        Notes:    #{view.notes.join(', ')}" if view.notes.any?
+          puts "        Strategy: #{view.strategy}" if view.strategy
+          puts ''
         end
-        puts "        Feed:     #{view.native_feed}" if view.native_feed
-        puts "        Notes:    #{view.notes.join(', ')}" if view.notes.any?
-        puts "        Strategy: #{view.strategy}" if view.strategy
-        puts ''
-      end
 
-      ##
-      # @param result [Html2rss::Test::Result]
-      # @param source [String]
-      # @return [void]
-      def test_card(result, source) # rubocop:disable Metrics/MethodLength, Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
-        if result.success
-          puts "\e[32m✓ Schema valid\e[0m (#{source})"
-          dur = "#{result.duration_seconds}s"
-          puts "\e[32m✓ Extracted #{result.item_count} items in #{dur}\e[0m (strategy: #{result.strategy_used})"
-          puts "\nChannel: #{result.channel_title} (#{result.channel_url})"
-          if result.sample_items.any?
-            puts 'Sample items:'
-            result.sample_items.each_with_index do |item, i|
-              puts "  #{i + 1}. #{"#{item[:published_at]} | " if item[:published_at]}#{item[:title]}"
-              puts "     #{item[:url]}"
+        ##
+        # @param result [Html2rss::Test::Result]
+        # @param source [String]
+        # @return [void]
+        def test_card(result, source) # rubocop:disable Metrics/MethodLength, Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
+          if result.success
+            puts "\e[32m✓ Schema valid\e[0m (#{source})"
+            dur = "#{result.duration_seconds}s"
+            puts "\e[32m✓ Extracted #{result.item_count} items in #{dur}\e[0m (strategy: #{result.strategy_used})"
+            puts "\nChannel: #{result.channel_title} (#{result.channel_url})"
+            if result.sample_items.any?
+              puts 'Sample items:'
+              result.sample_items.each_with_index do |item, i|
+                puts "  #{i + 1}. #{"#{item[:published_at]} | " if item[:published_at]}#{item[:title]}"
+                puts "     #{item[:url]}"
+              end
+            end
+          else
+            $stderr.puts "\e[31m✗ Test failed\e[0m (#{source})" # rubocop:disable Style/StderrPuts
+            $stderr.puts "  Error: #{result.error_message}" if result.error_message # rubocop:disable Style/StderrPuts
+            result.validation_issues&.each do |issue|
+              $stderr.puts "  Schema error #{issue}" # rubocop:disable Style/StderrPuts
             end
-          end
-        else
-          $stderr.puts "\e[31m✗ Test failed\e[0m (#{source})" # rubocop:disable Style/StderrPuts
-          $stderr.puts "  Error: #{result.error_message}" if result.error_message # rubocop:disable Style/StderrPuts
-          result.validation_issues&.each do |issue|
-            $stderr.puts "  Schema error #{issue}" # rubocop:disable Style/StderrPuts
           end
         end
-      end
 
-      ##
-      # @param data [Object]
-      # @param batch_mode [Boolean]
-      # @return [void]
-      def json(data, batch_mode)
-        payload = batch_mode ? data : data.first
-        puts JSON.pretty_generate(payload)
-      end
+        ##
+        # @param data [Object]
+        # @param batch_mode [Boolean]
+        # @return [void]
+        def json(data, batch_mode)
+          payload = batch_mode ? data : data.first
+          puts JSON.pretty_generate(payload)
+        end
 
-      ##
-      # @param view [ProbeView]
-      # @return [void]
-      def probe_lines(view)
-        if view.final && view.final != view.requested
-          puts "        Final:    #{view.final} (HTTP #{view.status || 'ERR'})"
+        private
+
+        ##
+        # @param view [ProbeView]
+        # @return [void]
+        def probe_lines(view)
+          if view.final && view.final != view.requested
+            puts "        Final:    #{view.final} (HTTP #{view.status || 'ERR'})"
+          end
+          puts "        Surface:  #{view.surface} (#{view.articles_count} articles)"
         end
-        puts "        Surface:  #{view.surface} (#{view.articles_count} articles)"
       end
-      private_class_method :probe_lines
     end
   end
 end

data/lib/html2rss/cli/validate.rb

@@ -5,15 +5,13 @@ module Html2rss
     ##
     # Validate command: file/glob/stdin branching and multi-file reporting.
     module Validate
-      module_function
-
       ##
       # @param files [Array<String>]
       # @param params [Hash]
       # @param quiet [Boolean]
       # @return [void]
       # @raise [Thor::Error]
-      def run(files, params:, quiet:) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
+      def self.run(files, params:, quiet:) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
         if files.size == 2 && File.file?(files[0].to_s) && !File.exist?(files[1].to_s)
           raise Thor::Error, "No such file: #{files[1]}" if path_like_config?(files[1])
 
@@ -45,7 +43,7 @@ module Html2rss
       ##
       # @param files [Array<String>]
       # @return [Array<String>]
-      def resolve_files(files)
+      def self.resolve_files(files)
         return ['-'] if files.empty? || files == ['-']
 
         resolved = []