html2rss 0.32.4 → 0.33.0

data/lib/html2rss/html/article_extractor/id_generator.rb

@@ -32,7 +32,7 @@ module Html2rss
 
           def parse_id_from_dom(article_tag, url, selected_anchor)
             candidates = [article_tag['id'], article_tag.at_css('[id]')&.attr('id')]
-            candidates += [url&.path, url&.query] if selected_anchor
+            candidates += [url&.path] if selected_anchor
             candidates.compact.reject(&:empty?).first
           end
 

data/lib/html2rss/html/probe.rb

@@ -16,66 +16,66 @@ module Html2rss
       # @see #mime_match?
       APPLICATION_ATOM_XML = 'application/atom+xml'
 
-      module_function
+      class << self
+        ##
+        # @param string [String, Symbol, #to_s] wire value
+        # @return [String] folded comparison key
+        def fold(string)
+          string.to_s.downcase
+        end
 
-      ##
-      # @param string [String, Symbol, #to_s] wire value
-      # @return [String] folded comparison key
-      def fold(string)
-        string.to_s.downcase
-      end
-
-      ##
-      # @param node [Nokogiri::XML::Node]
-      # @return [String] folded element name
-      def tag(node)
-        name = node.name
-        return fold(name) unless name.is_a?(String)
-        return name if name.ascii_only? && !name.match?(/[A-Z]/)
+        ##
+        # @param node [Nokogiri::XML::Node]
+        # @return [String] folded element name
+        def tag(node)
+          name = node.name
+          return fold(name) unless name.is_a?(String)
+          return name if name.ascii_only? && !name.match?(/[A-Z]/)
 
-        fold(name)
-      end
+          fold(name)
+        end
 
-      ##
-      # @param type [String, Symbol, #to_s, nil] MIME type attribute value
-      # @return [String] media type without parameters, folded
-      def mime_base(type)
-        raw = type.to_s
-        return fold(raw) unless raw.include?(';') || raw.match?(/\A\s|\s\z/)
+        ##
+        # @param type [String, Symbol, #to_s, nil] MIME type attribute value
+        # @return [String] media type without parameters, folded
+        def mime_base(type)
+          raw = type.to_s
+          return fold(raw) unless raw.include?(';') || raw.match?(/\A\s|\s\z/)
 
-        fold(raw.split(';', 2).first.to_s.strip)
-      end
+          fold(raw.split(';', 2).first.to_s.strip)
+        end
 
-      ##
-      # @param actual [String, Symbol, #to_s, nil] observed MIME type
-      # @param expected [Array<String>] canonical MIME types to match
-      # @return [Boolean]
-      def mime_match?(actual, *expected)
-        base = mime_base(actual)
-        expected.any? { |candidate| base == candidate || base == mime_base(candidate) }
-      end
+        ##
+        # @param actual [String, Symbol, #to_s, nil] observed MIME type
+        # @param expected [Array<String>] canonical MIME types to match
+        # @return [Boolean]
+        def mime_match?(actual, *expected)
+          base = mime_base(actual)
+          expected.any? { |candidate| base == candidate || base == mime_base(candidate) }
+        end
 
-      ##
-      # @param doc [Nokogiri::XML::Node]
-      # @param mime_types [Array<String>] optional MIME filters (folded once per call)
-      # @return [Array<Nokogiri::XML::Element>]
-      def scripts(doc, *mime_types)
-        return doc.css('script') if mime_types.empty?
+        ##
+        # @param doc [Nokogiri::XML::Node]
+        # @param mime_types [Array<String>] optional MIME filters (folded once per call)
+        # @return [Array<Nokogiri::XML::Element>]
+        def scripts(doc, *mime_types)
+          return doc.css('script') if mime_types.empty?
 
-        allowed = mime_types.to_set { |mime| mime_base(mime) }
-        doc.css('script').select { |script| allowed.include?(mime_base(script['type'])) }
-      end
+          allowed = mime_types.to_set { |mime| mime_base(mime) }
+          doc.css('script').select { |script| allowed.include?(mime_base(script['type'])) }
+        end
 
-      ##
-      # @param doc [Nokogiri::XML::Node]
-      # @param rel [String] link rel token
-      # @param mime [String, nil] optional MIME filter
-      # @return [Array<Nokogiri::XML::Element>]
-      def alternate_links(doc, rel:, mime: nil)
-        nodes = doc.css(%(link[rel~="#{rel}"][href]))
-        return nodes if mime.nil?
+        ##
+        # @param doc [Nokogiri::XML::Node]
+        # @param rel [String] link rel token
+        # @param mime [String, nil] optional MIME filter
+        # @return [Array<Nokogiri::XML::Element>]
+        def alternate_links(doc, rel:, mime: nil)
+          nodes = doc.css(%(link[rel~="#{rel}"][href]))
+          return nodes if mime.nil?
 
-        nodes.select { |node| mime_match?(node['type'], mime) }
+          nodes.select { |node| mime_match?(node['type'], mime) }
+        end
       end
     end
   end

data/lib/html2rss/link_destination/path_classifier.rb

@@ -106,13 +106,13 @@ module Html2rss
       def shallow?
         segment_count = segments.size
 
-        segment_count <= 1 || (segment_count == 2 && high_confidence_junk_at?(segment_count - 1))
+        segment_count <= 1 || (segment_count == 2 && high_confidence_junk_at?(1))
       end
 
       # @return [Boolean] true when the final path segment looks like a post slug
       def strong_post_suffix?
         @strong_post_suffix ||= segments.any? &&
-                                included_last_segment? &&
+                                slug_last_segment? &&
                                 trusted_post_context?(segments.size - 1)
       end
 
@@ -121,13 +121,9 @@ module Html2rss
         (0...segments.size).all? { |i| high_confidence_junk_at?(i) }
       end
 
-      # @return [Boolean] true when the route is shallow and contains high-confidence noise
-      def shallow_high_confidence_route?
-        vanity_segments = SEGMENT_SETS.fetch(:vanity)
-
-        shallow? && (0...segments.size).any? do |i|
-          high_confidence_junk_at?(i) || vanity_segments.include?(@lexicon[i])
-        end
+      # @return [Boolean] true when the route is shallow and names a soft vanity segment
+      def shallow_vanity_route?
+        shallow? && @lexicon.any? { |folded| SEGMENT_SETS.fetch(:vanity).include?(folded) }
       end
 
       # @return [Boolean] true when the leading segments are all utility chrome
@@ -139,11 +135,7 @@ module Html2rss
       def junk_path?
         return false if excluded_content_route?
 
-        any_high_confidence_junk_segment? ||
-          taxonomy_path? ||
-          utility_only_route? ||
-          deep_utility_context_route? ||
-          shallow_high_confidence_route?
+        any_high_confidence_junk_segment? || shallow_vanity_route?
       end
 
       # @return [Boolean] true when the route points at conversion or account chrome
@@ -164,8 +156,7 @@ module Html2rss
       end
 
       def yearish_content_context?
-        segments.any? { |segment| segment.match?(YEARISH_SEGMENT) } &&
-          (strong_post_suffix? || trusted_post_context?(segments.size - 1))
+        segments.any? { |segment| segment.match?(YEARISH_SEGMENT) } && trusted_post_context?(segments.size - 1)
       end
 
       def excluded_content_route?
@@ -199,28 +190,15 @@ module Html2rss
       end
 
       def trusted_post_context?(limit)
-        return false if limit <= 0
         return false if leading_high_confidence_junk?
 
-        content_segments = SEGMENT_SETS.fetch(:content)
-        context_segments = SEGMENT_SETS.fetch(:deep_post_context)
-
-        (0...limit).any? do |i|
-          content_segments.include?(@lexicon[i]) ||
-            segments[i].match?(PathClassifier::YEARISH_SEGMENT) ||
-            context_segments.include?(@lexicon[i])
+        content = SEGMENT_SETS.fetch(:content)
+        context = SEGMENT_SETS.fetch(:deep_post_context)
+        segments.zip(@lexicon).first(limit.clamp(0..)).any? do |segment, folded|
+          content.include?(folded) || segment.match?(YEARISH_SEGMENT) || context.include?(folded)
         end
       end
 
-      def included_last_segment?
-        !excluded_last_segment? && slug_last_segment?
-      end
-
-      def excluded_last_segment?
-        last_i = segments.size - 1
-        high_confidence_junk_at?(last_i) || SEGMENT_SETS[:vanity].include?(@lexicon[last_i])
-      end
-
       def slug_last_segment?
         last = segments.last
         last.match?(YEARISH_SEGMENT) || last.match?(POST_SLUG_SEGMENT)

data/lib/html2rss/mcp/config_argument.rb

@@ -14,14 +14,15 @@ module Html2rss
         # @param yaml [String, nil] feed configuration YAML
         # @return [ConfigArgument]
         # @raise [ArgumentError] unless exactly one of +config+ or +yaml+ is present
+        # @raise [Html2rss::InvalidOption] when +config+ is not a Hash or +yaml+ is not a String
         # @raise [Contract::UnpublishedRequestError] when the parsed config uses
         #   an unpublished MCP strategy or +request.local_file_path+
         def parse(config: nil, yaml: nil)
           parsed = case [present?(config), present?(yaml)]
                    in [true, false]
-                     HashUtil.deep_symbolize_keys(config, context: 'config')
+                     HashUtil.deep_symbolize_keys(require_type!(config, Hash, 'config'), context: 'config')
                    in [false, true]
-                     Config.from_yaml(yaml)
+                     Config.from_yaml(require_type!(yaml, String, 'yaml'))
                    else
                      raise ArgumentError, 'Provide exactly one of config or yaml'
                    end
@@ -31,6 +32,12 @@ module Html2rss
 
         private
 
+        def require_type!(value, type, name)
+          return value if value.is_a?(type)
+
+          raise Html2rss::InvalidOption, "#{name} must be a #{type}, got #{value.class}"
+        end
+
         def present?(value)
           case value
           in nil then false

data/lib/html2rss/mcp/contract.rb

@@ -103,7 +103,7 @@ module Html2rss
         properties: {
           url: URL_PROPERTY,
           strategy: STRATEGY_PROPERTY,
-          limit: { type: 'integer', description: 'Max articles to keep (default 25)', default: 25 },
+          limit: { type: 'integer', minimum: 1, description: 'Max articles to keep (default 25)', default: 25 },
           items_selector: { type: 'string', description: 'Optional CSS selector hint for items' }
         }.freeze,
         required: %w[url]
@@ -135,9 +135,9 @@ module Html2rss
           title: { type: 'string', description: 'Channel title override' },
           summary: { type: 'string', description: 'Directory summary override' },
           enhance: { type: 'boolean', description: 'Force enhance on or off' },
-          limit: { type: 'integer', description: 'Max articles to keep' },
-          max_redirects: { type: 'integer', description: 'Optional redirect limit override' },
-          max_requests: { type: 'integer', description: 'Optional request budget override' }
+          limit: { type: 'integer', minimum: 1, description: 'Max articles to keep' },
+          max_redirects: { type: 'integer', minimum: 0, description: 'Optional redirect limit override' },
+          max_requests: { type: 'integer', minimum: 1, description: 'Optional request budget override' }
         }.freeze,
         required: %w[url]
       }.freeze
@@ -163,6 +163,8 @@ module Html2rss
             strategy: strategy_property,
             concurrency: {
               type: 'integer',
+              minimum: 1,
+              maximum: 10,
               description: 'Max parallel worker threads (1..10, default: 5)',
               default: 5
             },
@@ -177,7 +179,7 @@ module Html2rss
         urls_description: 'List of page URLs to scrape (1..25)',
         strategy_property: STRATEGY_PROPERTY,
         extra_properties: {
-          limit: { type: 'integer', description: 'Max articles per URL to keep (default 10)', default: 10 }
+          limit: { type: 'integer', minimum: 1, description: 'Max articles per URL to keep (default 10)', default: 10 }
         }
       ).freeze
 

data/lib/html2rss/mcp/outcome.rb

@@ -60,7 +60,7 @@ module Html2rss
         ##
         # @param report [PageRecon::Diagnostics::Report]
         # @return [Outcome]
-        def inspect(report:)
+        def inspect_report(report:)
           next_step = inspect_next_step(report)
           guidance = Playbook.inspect_guidance(report)
           new(ok: true, next_step:, guidance:, payload: report.to_wire_h)
@@ -194,7 +194,7 @@ module Html2rss
           NextStep[:scrape]
         end
 
-        def capture_next_step(articles_count:, has_selectors:, native_feed: nil)
+        def capture_next_step(articles_count:, has_selectors:, native_feed:)
           return NextStep[:done] if native_feed
 
           articles_count.positive? && has_selectors ? NextStep[:test] : NextStep[:inspect]
@@ -205,7 +205,6 @@ module Html2rss
 
           kind = test_result.failure_kind
           return NextStep[:validate] if kind&.schema?
-          return NextStep[:capture] if kind&.execution? || kind&.min_items? || kind&.quality?
 
           NextStep[:capture]
         end
@@ -223,8 +222,8 @@ module Html2rss
         end
 
         def capture_payload(yaml:, articles_count:, has_selectors:, channel_title:, requested_strategy:, # rubocop:disable Metrics/ParameterLists
-                            segment_strategy:, selected_strategy:, admission_drops:, native_feed: nil,
-                            suggested_channel_url: nil)
+                            segment_strategy:, selected_strategy:, admission_drops:, native_feed:,
+                            suggested_channel_url:)
           {
             yaml:, articles_count:, has_selectors:, channel_title:, requested_strategy: requested_strategy.to_s,
             **(native_feed ? { native_feed: native_feed.to_s } : {}),

data/lib/html2rss/mcp/runtime.rb

@@ -14,31 +14,24 @@ module Html2rss
         :botasaurus_configured
       )
 
-      module_function
+      class << self
+        ##
+        # @return [Boolean] whether Botasaurus transport is configured in this process
+        def botasaurus_configured?
+          !ENV['BOTASAURUS_SCRAPER_URL'].to_s.strip.empty?
+        end
 
-      ##
-      # @return [Boolean] whether Botasaurus transport is configured in this process
-      def botasaurus_configured?
-        !ENV['BOTASAURUS_SCRAPER_URL'].to_s.strip.empty?
-      end
-
-      ##
-      # @return [Snapshot] runtime capabilities and catalog identity for MCP clients
-      def snapshot
-        Snapshot.new(
-          version: Html2rss::VERSION,
-          mcp_contract_version: Contract::MCP_CONTRACT_VERSION,
-          catalog_fingerprint: Contract.catalog_fingerprint,
-          tools: Contract.catalog_tools,
-          botasaurus_configured: botasaurus_configured?
-        )
-      end
-
-      ##
-      # @param strategy [String, Symbol, nil]
-      # @return [Symbol]
-      def coerce_strategy(strategy)
-        (strategy || :auto).to_sym
+        ##
+        # @return [Snapshot] runtime capabilities and catalog identity for MCP clients
+        def snapshot
+          Snapshot.new(
+            version: Html2rss::VERSION,
+            mcp_contract_version: Contract::MCP_CONTRACT_VERSION,
+            catalog_fingerprint: Contract.catalog_fingerprint,
+            tools: Contract.catalog_tools,
+            botasaurus_configured: botasaurus_configured?
+          )
+        end
       end
     end
   end

data/lib/html2rss/mcp/server/tools.rb

@@ -36,9 +36,9 @@ module Html2rss
             input_schema: Contract::INSPECT_INPUT_SCHEMA,
             annotations: Contract::ANNOTATIONS_OPEN_WORLD,
             call: lambda { |url:, strategy: 'auto', **|
-              Outcome.inspect(
+              Outcome.inspect_report(
                 report: PageRecon::Diagnostics.call(
-                  url:, strategy: Runtime.coerce_strategy(strategy), deep: false
+                  url:, strategy: Option.strategy(strategy), deep: false
                 )
               )
             }
@@ -51,7 +51,7 @@ module Html2rss
             input_schema: Contract::RECON_INPUT_SCHEMA,
             annotations: Contract::ANNOTATIONS_OPEN_WORLD,
             call: lambda { |url:, strategy: 'auto', **|
-              Outcome.recon(result: Html2rss.recon(url, strategy: Runtime.coerce_strategy(strategy)))
+              Outcome.recon(result: Html2rss.recon(url, strategy: Option.strategy(strategy)))
             }
           ),
           Tool.new(
@@ -63,7 +63,7 @@ module Html2rss
             annotations: Contract::ANNOTATIONS_OPEN_WORLD,
             call: lambda { |urls:, strategy: 'auto', concurrency: Batch::DEFAULT_CONCURRENCY, limit: 10, **|
               Outcome.batch_scrape(
-                Batch.batch_scrape(urls:, strategy: Runtime.coerce_strategy(strategy), concurrency:, limit:)
+                Batch.batch_scrape(urls:, strategy: Option.strategy(strategy), concurrency:, limit:)
               )
             }
           ),
@@ -76,7 +76,7 @@ module Html2rss
             annotations: Contract::ANNOTATIONS_OPEN_WORLD,
             call: lambda { |urls:, strategy: 'auto', concurrency: Batch::DEFAULT_CONCURRENCY, **|
               Outcome.batch_inspect(
-                Batch.batch_inspect(urls:, strategy: Runtime.coerce_strategy(strategy), concurrency:)
+                Batch.batch_inspect(urls:, strategy: Option.strategy(strategy), concurrency:)
               )
             }
           ),
@@ -89,7 +89,7 @@ module Html2rss
             annotations: Contract::ANNOTATIONS_OPEN_WORLD,
             call: lambda { |urls:, strategy: 'auto', concurrency: Batch::DEFAULT_CONCURRENCY, **|
               Outcome.batch_recon(
-                Batch.batch_recon(urls:, strategy: Runtime.coerce_strategy(strategy), concurrency:)
+                Batch.batch_recon(urls:, strategy: Option.strategy(strategy), concurrency:)
               )
             }
           ),
@@ -136,7 +136,7 @@ module Html2rss
                               compare_enhance: false, **kwargs|
               feed_config = ConfigArgument.parse(config:, yaml:).config
               test_args = { min_items:, strict_quality:, compare_enhance: }
-              test_args[:strategy] = Runtime.coerce_strategy(kwargs[:strategy]) if kwargs.key?(:strategy)
+              test_args[:strategy] = Option.strategy(kwargs[:strategy]) if kwargs.key?(:strategy)
               Outcome.test(Html2rss.test(feed_config, **test_args))
             }
           ),
@@ -163,7 +163,7 @@ module Html2rss
           # @param items_selector [String, nil]
           # @return [Outcome]
           def scrape_outcome(url:, strategy: 'auto', limit: 25, items_selector: nil)
-            wire = Batch.scrape_wire(url:, strategy: Runtime.coerce_strategy(strategy), limit:, items_selector:)
+            wire = Batch.scrape_wire(url:, strategy: Option.strategy(strategy), limit:, items_selector:)
             Outcome.scrape(
               items: wire[:items],
               requested_strategy: wire[:strategy],
@@ -189,7 +189,7 @@ module Html2rss
           # @return [Outcome]
           def capture_outcome(url:, strategy: 'auto', items_selector: nil, force: false, topics: nil, title: nil, # rubocop:disable Metrics/MethodLength, Metrics/ParameterLists
                               summary: nil, enhance: nil, limit: nil, max_redirects: nil, max_requests: nil)
-            plan = Runtime.coerce_strategy(strategy)
+            plan = Option.strategy(strategy)
             result = Html2rss::Capture.build(
               url,
               strategy: plan,

data/lib/html2rss/option.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+module Html2rss
+  ##
+  # Edge validation for caller-supplied option values (public API, CLI, MCP).
+  # Everything past these checks may trust its types.
+  module Option
+    class << self
+      ##
+      # @param name [Symbol] option name used in the error message
+      # @param value [Object] candidate value
+      # @param min [Integer] inclusive lower bound
+      # @param max [Integer, nil] inclusive upper bound
+      # @param optional [Boolean] whether +nil+ passes through
+      # @return [Integer, nil] +value+ when valid
+      # @raise [InvalidOption] when +value+ is not an Integer within bounds
+      def integer(name, value, min:, max: nil, optional: false)
+        return value if value.nil? && optional
+        return value if value.is_a?(Integer) && (min..max).cover?(value)
+
+        raise InvalidOption, "#{name} must be an Integer in #{min}..#{max}, got #{value.inspect}"
+      end
+
+      ##
+      # @param value [Symbol, String, nil] strategy plan name (+nil+ means +:auto+)
+      # @return [Symbol] the plan name
+      # @raise [InvalidOption] when +value+ is not a registered plan
+      def strategy(value)
+        name = value.nil? ? FeedPipeline::StrategyPlan::AUTO_NAME : value
+        return name.to_sym if FeedPipeline::StrategyPlan.valid?(name)
+
+        accepted = FeedPipeline::StrategyPlan.accepted_names.join(', ')
+        raise InvalidOption, "strategy must be one of #{accepted}, got #{value.inspect}"
+      end
+
+      ##
+      # @param name [Symbol] option name used in the error message
+      # @param value [Object] candidate list
+      # @return [Array] +value+ when it is a non-empty Array
+      # @raise [InvalidOption] otherwise
+      def list(name, value)
+        return value if value.is_a?(Array) && value.any?
+
+        raise InvalidOption, "#{name} must be a non-empty Array, got #{value.inspect}"
+      end
+    end
+  end
+end

data/lib/html2rss/page_recon/diagnostics.rb

@@ -34,14 +34,12 @@ module Html2rss
       JS_SHELL_MIN_BODY_BYTES = 8_192
       private_constant :JS_SHELL_MIN_BODY_BYTES
 
-      module_function
-
       ##
       # @param url [String]
       # @param strategy [String, Symbol]
       # @param deep [Boolean] when true and strategy is auto, one Botasaurus hop if configured
       # @return [Report]
-      def call(url:, strategy: :auto, deep: false)
+      def self.call(url:, strategy: :auto, deep: false)
         probe = PageRecon.probe(url, strategy: resolve_inspect_strategy(strategy, deep:))
         recon = probe.result
         response = probe.response
@@ -56,10 +54,10 @@ module Html2rss
       # @param strategy [Symbol, String]
       # @param concurrency [Integer]
       # @return [Array<Report>]
-      def batch(urls:, strategy: :auto, concurrency: Batch::DEFAULT_CONCURRENCY)
+      def self.batch(urls:, strategy: :auto, concurrency: Batch::DEFAULT_CONCURRENCY)
         Batch.map(Array(urls), concurrency:) do |url|
           call(url:, strategy:)
-        rescue StandardError => error
+        rescue Html2rss::Error => error
           error_report(url, error)
         end
       end
@@ -67,7 +65,7 @@ module Html2rss
       ##
       # @param response [Html2rss::RequestService::Response]
       # @return [Hash] redacted XHR capture diagnostics (no query strings)
-      def xhr_capture_info(response)
+      def self.xhr_capture_info(response)
         captured = response.captured_responses
         {
           count: captured.size,
@@ -79,7 +77,7 @@ module Html2rss
       ##
       # @param entry [Hash] captured response hash
       # @return [String, nil] scheme+host+path only
-      def redacted_endpoint(entry)
+      def self.redacted_endpoint(entry)
         raw = entry['url'] || entry[:url]
         return unless raw
 
@@ -94,7 +92,7 @@ module Html2rss
       ##
       # @param entry [Hash] captured response hash
       # @return [Boolean]
-      def xhr_candidate_articles?(entry)
+      def self.xhr_candidate_articles?(entry)
         body = entry['body'] || entry[:body]
         return false unless body.is_a?(String)
 
@@ -107,7 +105,7 @@ module Html2rss
       ##
       # @param parsed [Object] parsed response body
       # @return [Array<String>, Hash]
-      def scraper_info(parsed)
+      def self.scraper_info(parsed)
         return { error: 'Response is not HTML' } unless parsed.is_a?(Nokogiri::HTML::Document)
 
         begin

data/lib/html2rss/page_recon.rb

@@ -158,8 +158,6 @@ module Html2rss
     def self.discover_segments(sst, url)
       link_resolver = Scoring::LinkResolver.new(url)
       AutoSource::Segmenter.call(sst, base_url: url, strategy: :list, link_resolver:)
-    rescue StandardError
-      []
     end
 
     ##
@@ -268,8 +266,6 @@ module Html2rss
 
     def sst_document
       Html2rss::SST::Normalizer.call(response.body)
-    rescue ArgumentError
-      nil
     end
 
     def segment_stats(sst, page_url)