html2rss 0.32.3 → 0.33.0

data/lib/html2rss/hash_util.rb

@@ -3,116 +3,117 @@
 module Html2rss
   # Shared helpers for hash normalization and structural operations.
   module HashUtil
-    module_function
+    class << self
+      # Deeply duplicates nested arrays and hashes.
+      #
+      # @param object [Object] nested value from configuration or runtime state
+      # @return [Object] deep duplicated object
+      def deep_dup(object)
+        case object
+        in Hash
+          object.transform_values { deep_dup(_1) }
+        in Array
+          object.map { deep_dup(_1) }
+        else
+          object.dup
+        end
+      end
 
-    # Deeply duplicates nested arrays and hashes.
-    #
-    # @param object [Object] nested value from configuration or runtime state
-    # @return [Object] deep duplicated object
-    def deep_dup(object)
-      case object
-      in Hash
-        object.transform_values { deep_dup(_1) }
-      in Array
-        object.map { deep_dup(_1) }
-      else
-        object.dup rescue StandardError # rubocop:disable Style/RescueModifier
+      # Deeply merges nested hashes while replacing non-hash values from override.
+      #
+      # @param base [Hash] base hash
+      # @param override [Hash] override hash
+      # @return [Hash] merged hash
+      def deep_merge(base, override)
+        base.merge(override) do |_key, old_val, new_val|
+          case [old_val, new_val]
+          in [Hash, Hash]
+            deep_merge(old_val, new_val)
+          else
+            new_val
+          end
+        end
       end
-    end
 
-    # Deeply merges nested hashes while replacing non-hash values from override.
-    #
-    # @param base [Hash] base hash
-    # @param override [Hash] override hash
-    # @return [Hash] merged hash
-    def deep_merge(base, override)
-      base.merge(override) do |_key, old_val, new_val|
-        case [old_val, new_val]
-        in [Hash, Hash]
-          deep_merge(old_val, new_val)
+      # Converts string-keyed hashes to symbol-keyed hashes recursively.
+      #
+      # @param object [Object] value to normalize
+      # @param context [String] error context
+      # @return [Object] normalized value
+      def deep_symbolize_keys(object, context: 'hash')
+        case object
+        in Hash
+          object.each_with_object({}) do |(k, v), memo|
+            memo[symbol_key(k, context:)] = deep_symbolize_keys(v, context:)
+          end
+        in Array
+          object.map { deep_symbolize_keys(_1, context:) }
         else
-          new_val
+          object
         end
       end
-    end
 
-    # Converts string-keyed hashes to symbol-keyed hashes recursively.
-    #
-    # @param object [Object] value to normalize
-    # @param context [String] error context
-    # @return [Object] normalized value
-    def deep_symbolize_keys(object, context: 'hash')
-      case object
-      in Hash
-        object.each_with_object({}) do |(k, v), memo|
-          memo[symbol_key(k, context:)] = deep_symbolize_keys(v, context:)
+      # Converts hash keys (and Symbol values) to strings recursively.
+      #
+      # @param object [Object] value to normalize
+      # @return [Object] normalized value with string keys
+      def deep_stringify_keys(object)
+        case object
+        when Hash
+          object.to_h { |key, value| [key.to_s, deep_stringify_keys(value)] }
+        when Array
+          object.map { deep_stringify_keys(_1) }
+        when Symbol
+          object.to_s
+        else
+          object
         end
-      in Array
-        object.map { deep_symbolize_keys(_1, context:) }
-      else
-        object
       end
-    end
 
-    # Converts hash keys (and Symbol values) to strings recursively.
-    #
-    # @param object [Object] value to normalize
-    # @return [Object] normalized value with string keys
-    def deep_stringify_keys(object)
-      case object
-      when Hash
-        object.to_h { |key, value| [key.to_s, deep_stringify_keys(value)] }
-      when Array
-        object.map { deep_stringify_keys(_1) }
-      when Symbol
-        object.to_s
-      else
-        object
-      end
-    end
+      # Validates that hash keys are symbols.
+      #
+      # @param value [Object] candidate hash container whose keys must be symbols
+      # @param context [String] error context
+      # @param deep [Boolean] whether nested hashes should also be validated
+      # @return [void]
+      def assert_symbol_keys!(value, context: 'hash', deep: true)
+        return unless value in Hash
 
-    # Validates that hash keys are symbols.
-    #
-    # @param value [Object] candidate hash container whose keys must be symbols
-    # @param context [String] error context
-    # @param deep [Boolean] whether nested hashes should also be validated
-    # @return [void]
-    def assert_symbol_keys!(value, context: 'hash', deep: true)
-      return unless value in Hash
+        unless value.each_key.all?(Symbol)
+          invalid_key = value.keys.find { _1.class != Symbol }
+          raise ArgumentError, "#{context} must use symbol keys (found #{invalid_key.inspect})"
+        end
 
-      unless value.each_key.all?(Symbol)
-        invalid_key = value.keys.find { _1.class != Symbol }
-        raise ArgumentError, "#{context} must use symbol keys (found #{invalid_key.inspect})"
+        value.each_value { assert_symbol_keys!(_1, context:, deep:) } if deep
       end
 
-      value.each_value { assert_symbol_keys!(_1, context:, deep:) } if deep
-    end
+      # Validates that hash keys are strings.
+      #
+      # @param value [Object] candidate hash container whose keys must be strings
+      # @param context [String] error context
+      # @param deep [Boolean] whether nested hashes should also be validated
+      # @return [void]
+      def assert_string_keys!(value, context: 'hash', deep: true)
+        return unless value in Hash
 
-    # Validates that hash keys are strings.
-    #
-    # @param value [Object] candidate hash container whose keys must be strings
-    # @param context [String] error context
-    # @param deep [Boolean] whether nested hashes should also be validated
-    # @return [void]
-    def assert_string_keys!(value, context: 'hash', deep: true)
-      return unless value in Hash
+        unless value.each_key.all?(String)
+          invalid_key = value.keys.find { _1.class != String }
+          raise ArgumentError, "#{context} must use string keys (found #{invalid_key.inspect})"
+        end
 
-      unless value.each_key.all?(String)
-        invalid_key = value.keys.find { _1.class != String }
-        raise ArgumentError, "#{context} must use string keys (found #{invalid_key.inspect})"
+        value.each_value { assert_string_keys!(_1, context:, deep:) } if deep
       end
 
-      value.each_value { assert_string_keys!(_1, context:, deep:) } if deep
-    end
+      private
 
-    def symbol_key(key, context:)
-      case key
-      in Symbol then key
-      in String then key.to_sym
-      else
-        raise ArgumentError, "#{context} must use string or symbol keys (found #{key.inspect})"
+      def symbol_key(key, context:)
+        case key
+        in Symbol then key
+        in String then key.to_sym
+        else
+          raise ArgumentError, "#{context} must use string or symbol keys (found #{key.inspect})"
+        end
       end
     end
-    private_class_method :symbol_key
   end
 end

data/lib/html2rss/html/article_extractor/category_extractor.rb

@@ -28,23 +28,20 @@ module Html2rss
         def self.call(article_tag, title: nil)
           return [] unless article_tag
 
-          extract_all_categories(article_tag, title:)
-            .map(&:strip)
-            .reject(&:empty?)
+          extract_all_categories(article_tag, title:).to_a
         end
 
         ##
         # Optimized single DOM traversal that extracts all category types.
+        # `Node#css` only yields descendants, so the container itself is never visited.
         #
         # @param article_tag [Nokogiri::XML::Element] The article element
         # @param title [String, nil]
         # @return [Set<String>] Set of category strings
-        def self.extract_all_categories(article_tag, title: nil)
+        def self.extract_all_categories(article_tag, title:)
           Set.new.tap do |categories|
             extract_element_data_categories!(categories, article_tag, title:)
             article_tag.css(CATEGORY_SELECTORS.join(',')).each do |element|
-              next if element == article_tag
-
               if ArticleRules::Category.class_match?(element['class'])
                 extract_text_categories!(categories, element, title:)
               end
@@ -60,7 +57,7 @@ module Html2rss
         # @param element [Nokogiri::XML::Element] metadata element that may contain category links
         # @param title [String, nil]
         # @return [void]
-        def self.extract_element_data_categories!(categories, element, title: nil)
+        def self.extract_element_data_categories!(categories, element, title:)
           element.attributes.each_value do |attr|
             next unless ArticleRules::Category.attr_name_match?(attr.name)
 
@@ -75,7 +72,7 @@ module Html2rss
         # @param element [Nokogiri::XML::Element] metadata element whose text may contain delimiters
         # @param title [String, nil]
         # @return [void]
-        def self.extract_text_categories!(categories, element, title: nil)
+        def self.extract_text_categories!(categories, element, title:)
           return if category_container?(element)
 
           anchors = element.name == 'a' ? [element] : element.css('a').to_a
@@ -96,7 +93,7 @@ module Html2rss
         # @param element [Nokogiri::XML::Element] The element to extract text from
         # @param title [String, nil]
         # @return [void]
-        def self.add_text_to_categories!(categories, element, title: nil)
+        def self.add_text_to_categories!(categories, element, title:)
           ArticleRules::Category.add_text!(categories, Navigator.extract_visible_text(element), title:)
         end
 

data/lib/html2rss/html/article_extractor/date_extractor.rb

@@ -9,7 +9,7 @@ module Html2rss
         # @param leftover_lines [Array<String>] leftover visible lines (already split)
         # @param time_zone [String] channel time zone for naive values
         # @return [DateTime, nil]
-        def self.call(article_tag, leftover_lines: [], time_zone: 'UTC')
+        def self.call(article_tag, time_zone:, leftover_lines: [])
           datetimes = article_tag.css('[datetime]').map { |tag| tag['datetime'] }
           ArticleRules::Date.earliest(datetimes, leftover_lines:, time_zone:)
         end

data/lib/html2rss/html/article_extractor/enclosure_extractor.rb

@@ -23,8 +23,6 @@ module Html2rss
         # @param base_url [String, Html2rss::Url] base URL for relative enclosure links
         # @return [Array<Hash{Symbol => Object}>] normalized enclosure hashes
         def self.call(article_tag, base_url)
-          return [] unless article_tag
-
           article_tag.css(SELECTOR).filter_map do |element|
             extract_from_element(element, base_url)
           end
@@ -34,7 +32,7 @@ module Html2rss
           case element.name
           when 'img'
             ArticleRules::Enclosure.from_image(element['src'], base_url)
-          when 'video', 'audio', 'source'
+          when 'audio', 'source'
             ArticleRules::Enclosure.from_media(element['src'], element['type'], base_url)
           when 'iframe'
             ArticleRules::Enclosure.from_iframe(element['src'], base_url)

data/lib/html2rss/html/article_extractor/id_generator.rb

@@ -32,7 +32,7 @@ module Html2rss
 
           def parse_id_from_dom(article_tag, url, selected_anchor)
             candidates = [article_tag['id'], article_tag.at_css('[id]')&.attr('id')]
-            candidates += [url&.path, url&.query] if selected_anchor
+            candidates += [url&.path] if selected_anchor
             candidates.compact.reject(&:empty?).first
           end
 

data/lib/html2rss/html/probe.rb

@@ -16,66 +16,66 @@ module Html2rss
       # @see #mime_match?
       APPLICATION_ATOM_XML = 'application/atom+xml'
 
-      module_function
+      class << self
+        ##
+        # @param string [String, Symbol, #to_s] wire value
+        # @return [String] folded comparison key
+        def fold(string)
+          string.to_s.downcase
+        end
 
-      ##
-      # @param string [String, Symbol, #to_s] wire value
-      # @return [String] folded comparison key
-      def fold(string)
-        string.to_s.downcase
-      end
-
-      ##
-      # @param node [Nokogiri::XML::Node]
-      # @return [String] folded element name
-      def tag(node)
-        name = node.name
-        return fold(name) unless name.is_a?(String)
-        return name if name.ascii_only? && !name.match?(/[A-Z]/)
+        ##
+        # @param node [Nokogiri::XML::Node]
+        # @return [String] folded element name
+        def tag(node)
+          name = node.name
+          return fold(name) unless name.is_a?(String)
+          return name if name.ascii_only? && !name.match?(/[A-Z]/)
 
-        fold(name)
-      end
+          fold(name)
+        end
 
-      ##
-      # @param type [String, Symbol, #to_s, nil] MIME type attribute value
-      # @return [String] media type without parameters, folded
-      def mime_base(type)
-        raw = type.to_s
-        return fold(raw) unless raw.include?(';') || raw.match?(/\A\s|\s\z/)
+        ##
+        # @param type [String, Symbol, #to_s, nil] MIME type attribute value
+        # @return [String] media type without parameters, folded
+        def mime_base(type)
+          raw = type.to_s
+          return fold(raw) unless raw.include?(';') || raw.match?(/\A\s|\s\z/)
 
-        fold(raw.split(';', 2).first.to_s.strip)
-      end
+          fold(raw.split(';', 2).first.to_s.strip)
+        end
 
-      ##
-      # @param actual [String, Symbol, #to_s, nil] observed MIME type
-      # @param expected [Array<String>] canonical MIME types to match
-      # @return [Boolean]
-      def mime_match?(actual, *expected)
-        base = mime_base(actual)
-        expected.any? { |candidate| base == candidate || base == mime_base(candidate) }
-      end
+        ##
+        # @param actual [String, Symbol, #to_s, nil] observed MIME type
+        # @param expected [Array<String>] canonical MIME types to match
+        # @return [Boolean]
+        def mime_match?(actual, *expected)
+          base = mime_base(actual)
+          expected.any? { |candidate| base == candidate || base == mime_base(candidate) }
+        end
 
-      ##
-      # @param doc [Nokogiri::XML::Node]
-      # @param mime_types [Array<String>] optional MIME filters (folded once per call)
-      # @return [Array<Nokogiri::XML::Element>]
-      def scripts(doc, *mime_types)
-        return doc.css('script') if mime_types.empty?
+        ##
+        # @param doc [Nokogiri::XML::Node]
+        # @param mime_types [Array<String>] optional MIME filters (folded once per call)
+        # @return [Array<Nokogiri::XML::Element>]
+        def scripts(doc, *mime_types)
+          return doc.css('script') if mime_types.empty?
 
-        allowed = mime_types.to_set { |mime| mime_base(mime) }
-        doc.css('script').select { |script| allowed.include?(mime_base(script['type'])) }
-      end
+          allowed = mime_types.to_set { |mime| mime_base(mime) }
+          doc.css('script').select { |script| allowed.include?(mime_base(script['type'])) }
+        end
 
-      ##
-      # @param doc [Nokogiri::XML::Node]
-      # @param rel [String] link rel token
-      # @param mime [String, nil] optional MIME filter
-      # @return [Array<Nokogiri::XML::Element>]
-      def alternate_links(doc, rel:, mime: nil)
-        nodes = doc.css(%(link[rel~="#{rel}"][href]))
-        return nodes if mime.nil?
+        ##
+        # @param doc [Nokogiri::XML::Node]
+        # @param rel [String] link rel token
+        # @param mime [String, nil] optional MIME filter
+        # @return [Array<Nokogiri::XML::Element>]
+        def alternate_links(doc, rel:, mime: nil)
+          nodes = doc.css(%(link[rel~="#{rel}"][href]))
+          return nodes if mime.nil?
 
-        nodes.select { |node| mime_match?(node['type'], mime) }
+          nodes.select { |node| mime_match?(node['type'], mime) }
+        end
       end
     end
   end

data/lib/html2rss/link_destination/path_classifier.rb

@@ -106,13 +106,13 @@ module Html2rss
       def shallow?
         segment_count = segments.size
 
-        segment_count <= 1 || (segment_count == 2 && high_confidence_junk_at?(segment_count - 1))
+        segment_count <= 1 || (segment_count == 2 && high_confidence_junk_at?(1))
       end
 
       # @return [Boolean] true when the final path segment looks like a post slug
       def strong_post_suffix?
         @strong_post_suffix ||= segments.any? &&
-                                included_last_segment? &&
+                                slug_last_segment? &&
                                 trusted_post_context?(segments.size - 1)
       end
 
@@ -121,13 +121,9 @@ module Html2rss
         (0...segments.size).all? { |i| high_confidence_junk_at?(i) }
       end
 
-      # @return [Boolean] true when the route is shallow and contains high-confidence noise
-      def shallow_high_confidence_route?
-        vanity_segments = SEGMENT_SETS.fetch(:vanity)
-
-        shallow? && (0...segments.size).any? do |i|
-          high_confidence_junk_at?(i) || vanity_segments.include?(@lexicon[i])
-        end
+      # @return [Boolean] true when the route is shallow and names a soft vanity segment
+      def shallow_vanity_route?
+        shallow? && @lexicon.any? { |folded| SEGMENT_SETS.fetch(:vanity).include?(folded) }
       end
 
       # @return [Boolean] true when the leading segments are all utility chrome
@@ -139,11 +135,7 @@ module Html2rss
       def junk_path?
         return false if excluded_content_route?
 
-        any_high_confidence_junk_segment? ||
-          taxonomy_path? ||
-          utility_only_route? ||
-          deep_utility_context_route? ||
-          shallow_high_confidence_route?
+        any_high_confidence_junk_segment? || shallow_vanity_route?
       end
 
       # @return [Boolean] true when the route points at conversion or account chrome
@@ -164,8 +156,7 @@ module Html2rss
       end
 
       def yearish_content_context?
-        segments.any? { |segment| segment.match?(YEARISH_SEGMENT) } &&
-          (strong_post_suffix? || trusted_post_context?(segments.size - 1))
+        segments.any? { |segment| segment.match?(YEARISH_SEGMENT) } && trusted_post_context?(segments.size - 1)
       end
 
       def excluded_content_route?
@@ -199,28 +190,15 @@ module Html2rss
       end
 
       def trusted_post_context?(limit)
-        return false if limit <= 0
         return false if leading_high_confidence_junk?
 
-        content_segments = SEGMENT_SETS.fetch(:content)
-        context_segments = SEGMENT_SETS.fetch(:deep_post_context)
-
-        (0...limit).any? do |i|
-          content_segments.include?(@lexicon[i]) ||
-            segments[i].match?(PathClassifier::YEARISH_SEGMENT) ||
-            context_segments.include?(@lexicon[i])
+        content = SEGMENT_SETS.fetch(:content)
+        context = SEGMENT_SETS.fetch(:deep_post_context)
+        segments.zip(@lexicon).first(limit.clamp(0..)).any? do |segment, folded|
+          content.include?(folded) || segment.match?(YEARISH_SEGMENT) || context.include?(folded)
         end
       end
 
-      def included_last_segment?
-        !excluded_last_segment? && slug_last_segment?
-      end
-
-      def excluded_last_segment?
-        last_i = segments.size - 1
-        high_confidence_junk_at?(last_i) || SEGMENT_SETS[:vanity].include?(@lexicon[last_i])
-      end
-
       def slug_last_segment?
         last = segments.last
         last.match?(YEARISH_SEGMENT) || last.match?(POST_SLUG_SEGMENT)

data/lib/html2rss/mcp/config_argument.rb

@@ -14,14 +14,15 @@ module Html2rss
         # @param yaml [String, nil] feed configuration YAML
         # @return [ConfigArgument]
         # @raise [ArgumentError] unless exactly one of +config+ or +yaml+ is present
+        # @raise [Html2rss::InvalidOption] when +config+ is not a Hash or +yaml+ is not a String
         # @raise [Contract::UnpublishedRequestError] when the parsed config uses
         #   an unpublished MCP strategy or +request.local_file_path+
         def parse(config: nil, yaml: nil)
           parsed = case [present?(config), present?(yaml)]
                    in [true, false]
-                     HashUtil.deep_symbolize_keys(config, context: 'config')
+                     HashUtil.deep_symbolize_keys(require_type!(config, Hash, 'config'), context: 'config')
                    in [false, true]
-                     Config.from_yaml(yaml)
+                     Config.from_yaml(require_type!(yaml, String, 'yaml'))
                    else
                      raise ArgumentError, 'Provide exactly one of config or yaml'
                    end
@@ -31,6 +32,12 @@ module Html2rss
 
         private
 
+        def require_type!(value, type, name)
+          return value if value.is_a?(type)
+
+          raise Html2rss::InvalidOption, "#{name} must be a #{type}, got #{value.class}"
+        end
+
         def present?(value)
           case value
           in nil then false

data/lib/html2rss/mcp/contract.rb

@@ -103,7 +103,7 @@ module Html2rss
         properties: {
           url: URL_PROPERTY,
           strategy: STRATEGY_PROPERTY,
-          limit: { type: 'integer', description: 'Max articles to keep (default 25)', default: 25 },
+          limit: { type: 'integer', minimum: 1, description: 'Max articles to keep (default 25)', default: 25 },
           items_selector: { type: 'string', description: 'Optional CSS selector hint for items' }
         }.freeze,
         required: %w[url]
@@ -135,9 +135,9 @@ module Html2rss
           title: { type: 'string', description: 'Channel title override' },
           summary: { type: 'string', description: 'Directory summary override' },
           enhance: { type: 'boolean', description: 'Force enhance on or off' },
-          limit: { type: 'integer', description: 'Max articles to keep' },
-          max_redirects: { type: 'integer', description: 'Optional redirect limit override' },
-          max_requests: { type: 'integer', description: 'Optional request budget override' }
+          limit: { type: 'integer', minimum: 1, description: 'Max articles to keep' },
+          max_redirects: { type: 'integer', minimum: 0, description: 'Optional redirect limit override' },
+          max_requests: { type: 'integer', minimum: 1, description: 'Optional request budget override' }
         }.freeze,
         required: %w[url]
       }.freeze
@@ -163,6 +163,8 @@ module Html2rss
             strategy: strategy_property,
             concurrency: {
               type: 'integer',
+              minimum: 1,
+              maximum: 10,
               description: 'Max parallel worker threads (1..10, default: 5)',
               default: 5
             },
@@ -177,7 +179,7 @@ module Html2rss
         urls_description: 'List of page URLs to scrape (1..25)',
         strategy_property: STRATEGY_PROPERTY,
         extra_properties: {
-          limit: { type: 'integer', description: 'Max articles per URL to keep (default 10)', default: 10 }
+          limit: { type: 'integer', minimum: 1, description: 'Max articles per URL to keep (default 10)', default: 10 }
         }
       ).freeze
 

data/lib/html2rss/mcp/outcome.rb

@@ -60,7 +60,7 @@ module Html2rss
         ##
         # @param report [PageRecon::Diagnostics::Report]
         # @return [Outcome]
-        def inspect(report:)
+        def inspect_report(report:)
           next_step = inspect_next_step(report)
           guidance = Playbook.inspect_guidance(report)
           new(ok: true, next_step:, guidance:, payload: report.to_wire_h)
@@ -194,7 +194,7 @@ module Html2rss
           NextStep[:scrape]
         end
 
-        def capture_next_step(articles_count:, has_selectors:, native_feed: nil)
+        def capture_next_step(articles_count:, has_selectors:, native_feed:)
           return NextStep[:done] if native_feed
 
           articles_count.positive? && has_selectors ? NextStep[:test] : NextStep[:inspect]
@@ -205,7 +205,6 @@ module Html2rss
 
           kind = test_result.failure_kind
           return NextStep[:validate] if kind&.schema?
-          return NextStep[:capture] if kind&.execution? || kind&.min_items? || kind&.quality?
 
           NextStep[:capture]
         end
@@ -223,8 +222,8 @@ module Html2rss
         end
 
         def capture_payload(yaml:, articles_count:, has_selectors:, channel_title:, requested_strategy:, # rubocop:disable Metrics/ParameterLists
-                            segment_strategy:, selected_strategy:, admission_drops:, native_feed: nil,
-                            suggested_channel_url: nil)
+                            segment_strategy:, selected_strategy:, admission_drops:, native_feed:,
+                            suggested_channel_url:)
           {
             yaml:, articles_count:, has_selectors:, channel_title:, requested_strategy: requested_strategy.to_s,
             **(native_feed ? { native_feed: native_feed.to_s } : {}),