html2rss 0.27.2 → 0.29.1

data/lib/html2rss/mcp/outcome.rb

@@ -5,10 +5,10 @@ module Html2rss
     Outcome = Data.define(:ok, :next_step, :guidance, :payload)
 
     ##
-    # Typed MCP tool result. Owns next-step policy and guidance copy so the
-    # protocol adapter does not branch on quality heuristics.
-    class Outcome
-      # Matches {ConfigArgument} XOR {ArgumentError} messages.
+    # Typed MCP tool result. Owns next-step policy; guidance copy lives in
+    # {Playbook}.
+    class Outcome # rubocop:disable Metrics/ClassLength -- next-step policy + factories stay co-located
+      # Matches {ConfigArgument} XOR +ArgumentError+ messages.
       XOR_ERROR = /exactly one of config or yaml/
       NextStep = Data.define(:name, :guidance)
 
@@ -16,19 +16,7 @@ module Html2rss
       # Closed set of agent next actions. Invalid names cannot be constructed.
       class NextStep
         # Wire names for +next_step+.
-        NAMES = %i[done inspect_url validate_config apply_config scrape_url capture_config read_runtime].freeze
-        # Default guidance copy keyed by {NAMES}.
-        GUIDANCE = {
-          done: 'Done. Read payload for the result.',
-          inspect_url: 'Call inspect_url next. Read payload for recon (final_url, status, ' \
-                       'scheme_downgrade, alternate_feeds).',
-          validate_config: 'Call validate_config with payload.yaml or a config hash (XOR, not both).',
-          apply_config: 'Call apply_config next. Confirm payload.item_count before shipping.',
-          scrape_url: 'Call scrape_url for articles now. strategy auto already runs Faraday then Botasaurus.',
-          capture_config: 'Call capture_config for a reusable YAML draft, then follow next_step.',
-          read_runtime: 'Read html2rss://runtime. Set BOTASAURUS_SCRAPER_URL on the MCP process ' \
-                        'if botasaurus_configured is false.'
-        }.freeze
+        NAMES = %i[done inspect recon validate apply scrape capture read_runtime test].freeze
 
         ##
         # @param name [Symbol, String]
@@ -37,7 +25,7 @@ module Html2rss
           step = name.to_sym
           raise ArgumentError, "unknown next_step: #{name.inspect}" unless NAMES.include?(step)
 
-          super(name: step, guidance: (guidance || GUIDANCE.fetch(step)).freeze)
+          super(name: step, guidance: (guidance || Playbook::GUIDANCE.fetch(step)).freeze)
         end
 
         class << self
@@ -78,11 +66,21 @@ module Html2rss
         end
 
         ##
-        # @param payload [Hash] inspect recon Hash
+        # @param report [PageRecon::Diagnostics::Report]
         # @return [Outcome]
-        def inspect(payload:)
-          next_step = inspect_next_step(payload)
-          new(ok: true, next_step:, guidance: next_step.guidance, payload:)
+        def inspect(report:)
+          next_step = inspect_next_step(report)
+          guidance = Playbook.inspect_guidance(report)
+          new(ok: true, next_step:, guidance:, payload: report.to_wire_h)
+        end
+
+        ##
+        # @param result [Html2rss::Recon::Result]
+        # @return [Outcome]
+        def recon(result:)
+          next_step = recon_next_step(result)
+          guidance = Playbook.recon_guidance(result, next_step)
+          new(ok: true, next_step:, guidance:, payload: result.to_h)
         end
 
         ##
@@ -94,13 +92,17 @@ module Html2rss
         # @param segment_strategy [Symbol, String, nil]
         # @param selected_strategy [Symbol, String, nil]
         # @param admission_drops [Hash]
+        # @param native_feed [String, nil]
+        # @param suggested_channel_url [String, nil]
         # @return [Outcome]
         def capture(yaml:, articles_count:, has_selectors:, channel_title:, requested_strategy:, # rubocop:disable Metrics/ParameterLists
-                    segment_strategy: nil, selected_strategy: nil, admission_drops: {})
-          next_step = capture_next_step(articles_count:, has_selectors:)
+                    segment_strategy: nil, selected_strategy: nil, admission_drops: {}, native_feed: nil,
+                    suggested_channel_url: nil)
+          next_step = capture_next_step(articles_count:, has_selectors:, native_feed:)
           new(ok: true, next_step:, guidance: next_step.guidance, payload: capture_payload(
             yaml:, articles_count:, has_selectors:, channel_title:, requested_strategy:,
-            segment_strategy:, selected_strategy:, admission_drops:
+            segment_strategy:, selected_strategy:, admission_drops:, native_feed:,
+            suggested_channel_url:
           ))
         end
 
@@ -109,19 +111,50 @@ module Html2rss
         # @return [Outcome]
         def validate(errors:)
           ok = errors.nil?
-          next_step = ok ? NextStep.apply_config : NextStep.validate_config
+          next_step = ok ? NextStep.test : NextStep.validate
           new(ok:, next_step:, guidance: next_step.guidance, payload: ok ? {} : { errors: })
         end
 
+        ##
+        # @param test_result [Html2rss::Test::Result]
+        # @return [Outcome]
+        def test(test_result)
+          next_step = test_next_step(test_result)
+          new(
+            ok: test_result.success,
+            next_step:,
+            guidance: test_guidance(test_result, next_step),
+            payload: test_result.to_h
+          )
+        end
+
+        ##
+        # @param batch_result [Html2rss::Batch::BatchResult]
+        # @return [Outcome]
+        def batch_scrape(batch_result) = batch(batch_result, NextStep.scrape)
+
+        ##
+        # @param batch_result [Html2rss::Batch::BatchResult]
+        # @return [Outcome]
+        def batch_inspect(batch_result) = batch(batch_result, NextStep.inspect)
+
+        ##
+        # @param batch_result [Html2rss::Batch::BatchResult]
+        # @return [Outcome]
+        def batch_recon(batch_result) = batch(batch_result, NextStep.recon)
+
         ##
         # @param rss [String]
         # @param item_count [Integer]
         # @param empty [Boolean] {FeedResult#empty?} (ship gate); defaults to zero items
+        # @param quality_report [Hash, nil] optional ship-quality audit summary
         # @return [Outcome]
-        def apply(rss:, item_count:, empty: item_count.zero?)
+        def apply(rss:, item_count:, empty: item_count.zero?, quality_report: nil)
           ok = !empty
-          next_step = ok ? NextStep.done : NextStep.inspect_url
-          new(ok:, next_step:, guidance: next_step.guidance, payload: { rss:, item_count: })
+          next_step = ok ? NextStep.done : NextStep.inspect
+          payload = { rss:, item_count: }
+          payload[:quality_report] = quality_report if quality_report
+          new(ok:, next_step:, guidance: next_step.guidance, payload:)
         end
 
         ##
@@ -135,11 +168,16 @@ module Html2rss
 
         private
 
+        def batch(batch_result, failure_step)
+          step = batch_result.successful.positive? ? NextStep.done : failure_step
+          new(ok: true, next_step: step, guidance: step.guidance, payload: batch_result.to_h)
+        end
+
         def scrape_next_step(empty, botasaurus_configured:)
           return NextStep.done unless empty
           return NextStep.read_runtime unless botasaurus_configured
 
-          NextStep.inspect_url
+          NextStep.inspect
         end
 
         def scrape_payload(items:, requested_strategy:, channel_title:, admission_drops:)
@@ -149,21 +187,55 @@ module Html2rss
           }
         end
 
-        def inspect_next_step(payload)
-          return NextStep.done if Array(payload[:alternate_feeds]).any?
-          return NextStep.capture_config if payload[:articles_count].to_i.positive?
+        def inspect_next_step(report)
+          return NextStep.recon if report.alternate_feeds?
+          return NextStep.capture if report.articles_count.positive?
 
-          NextStep.scrape_url
+          NextStep.scrape
         end
 
-        def capture_next_step(articles_count:, has_selectors:)
-          articles_count.positive? && has_selectors ? NextStep.validate_config : NextStep.inspect_url
+        def recon_next_step(result)
+          return NextStep.done if result.defer?
+          return NextStep.capture if result.build?
+
+          NextStep.scrape
+        end
+
+        def capture_next_step(articles_count:, has_selectors:, native_feed: nil)
+          return NextStep.done if native_feed
+
+          articles_count.positive? && has_selectors ? NextStep.test : NextStep.inspect
+        end
+
+        def test_next_step(test_result)
+          return NextStep.apply if test_result.success
+
+          kind = test_result.failure_kind
+          return NextStep.validate if kind&.schema?
+          return NextStep.capture if kind&.execution? || kind&.min_items? || kind&.quality?
+
+          NextStep.capture
+        end
+
+        def test_guidance(test_result, next_step)
+          base = if test_result.success
+                   next_step.guidance
+                 else
+                   test_result.error_message || next_step.guidance
+                 end
+          warnings = test_result.quality_report&.warnings
+          return base if warnings.nil? || warnings.empty?
+
+          "#{base} Review payload.quality_report warnings: #{warnings.join(', ')}."
         end
 
         def capture_payload(yaml:, articles_count:, has_selectors:, channel_title:, requested_strategy:, # rubocop:disable Metrics/ParameterLists
-                            segment_strategy:, selected_strategy:, admission_drops:)
+                            segment_strategy:, selected_strategy:, admission_drops:, native_feed: nil,
+                            suggested_channel_url: nil)
           {
             yaml:, articles_count:, has_selectors:, channel_title:, requested_strategy: requested_strategy.to_s,
+            **(native_feed ? { native_feed: native_feed.to_s } : {}),
+            **(suggested_channel_url ? { suggested_channel_url: suggested_channel_url.to_s } : {}),
             **(segment_strategy ? { segment_strategy: segment_strategy.to_s } : {}),
             **(selected_strategy ? { selected_strategy: selected_strategy.to_s } : {}),
             **(admission_drops.any? ? { admission_drops: } : {})
@@ -173,14 +245,14 @@ module Html2rss
         def next_step_for_error(error)
           case error
           when RequestService::BotasaurusConfigurationError then NextStep.read_runtime
-          when Contract::UnpublishedRequestError then NextStep.validate_config
+          when Contract::UnpublishedRequestError then NextStep.validate
           when ArgumentError then argument_error_next_step(error)
-          else NextStep.inspect_url
+          else NextStep.inspect
           end
         end
 
         def argument_error_next_step(error)
-          XOR_ERROR.match?(error.message) ? NextStep.validate_config : NextStep.inspect_url
+          XOR_ERROR.match?(error.message) ? NextStep.validate : NextStep.inspect
         end
       end
     end

data/lib/html2rss/mcp/runtime.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+module Html2rss
+  module MCP
+    ##
+    # Runtime facts for the MCP process (env configuration, wire coercion).
+    module Runtime
+      # Published +html2rss://runtime+ resource (never leaks secrets).
+      Snapshot = Data.define(
+        :version,
+        :mcp_contract_version,
+        :catalog_fingerprint,
+        :tools,
+        :botasaurus_configured
+      )
+
+      module_function
+
+      ##
+      # @return [Boolean] whether Botasaurus transport is configured in this process
+      def botasaurus_configured?
+        !ENV['BOTASAURUS_SCRAPER_URL'].to_s.strip.empty?
+      end
+
+      ##
+      # @return [Snapshot] runtime capabilities and catalog identity for MCP clients
+      def snapshot
+        Snapshot.new(
+          version: Html2rss::VERSION,
+          mcp_contract_version: Contract::MCP_CONTRACT_VERSION,
+          catalog_fingerprint: Contract.catalog_fingerprint,
+          tools: Contract.catalog_tools,
+          botasaurus_configured: botasaurus_configured?
+        )
+      end
+
+      ##
+      # @param strategy [String, Symbol, nil]
+      # @return [Symbol]
+      def coerce_strategy(strategy)
+        (strategy || :auto).to_sym
+      end
+    end
+  end
+end

data/lib/html2rss/mcp/server/tools.rb

@@ -0,0 +1,277 @@
+# frozen_string_literal: true
+
+module Html2rss
+  module MCP
+    module Server
+      ##
+      # MCP tool registration and outcome mapping over public html2rss APIs.
+      module Tools # rubocop:disable Metrics/ModuleLength -- declarative registry + substantive handlers
+        # Declarative MCP tool registrations consumed by {register_all}.
+        TOOLS = [
+          {
+            name: 'scrape',
+            kind: :url,
+            description: 'One-shot article extraction as JSON Feed items. ' \
+                         'Use when you need articles now without a saved config. ' \
+                         'strategy "auto" triggers fallback chain (faraday → botasaurus) for JS-rendered sites.',
+            input_schema: Contract::SCRAPE_INPUT_SCHEMA,
+            handler: :scrape_outcome
+          },
+          {
+            name: 'inspect',
+            kind: :url,
+            description: 'Diagnostic page analysis (scrapers, SST, segments, final URL, status, ' \
+                         'rel=alternate feeds). Use recon for BUILD/DEFER/DROP verdict and native_feed preference.',
+            input_schema: Contract::INSPECT_INPUT_SCHEMA,
+            call: lambda { |url:, strategy: 'auto', **|
+              Outcome.inspect(
+                report: PageRecon::Diagnostics.call(
+                  url:, strategy: Runtime.coerce_strategy(strategy), deep: false
+                )
+              )
+            }
+          },
+          {
+            name: 'recon',
+            kind: :url,
+            description: 'Curation verdict and native_feed preference for a URL. ' \
+                         'Use after inspect when alternates warrant deeper recon, or when you need BUILD/DEFER/DROP.',
+            input_schema: Contract::RECON_INPUT_SCHEMA,
+            call: lambda { |url:, strategy: 'auto', **|
+              Outcome.recon(result: Html2rss.recon(url, strategy: Runtime.coerce_strategy(strategy)))
+            }
+          },
+          {
+            name: 'batch_scrape',
+            kind: :batch,
+            batch_method: :batch_scrape,
+            limit_default: 10,
+            description: 'Scrape multiple URLs in parallel with per-URL error isolation. ' \
+                         'Returns structured JSON Feed items and extraction counts.',
+            input_schema: Contract::BATCH_SCRAPE_INPUT_SCHEMA
+          },
+          {
+            name: 'batch_inspect',
+            kind: :batch,
+            batch_method: :batch_inspect,
+            description: 'Inspect multiple URLs in parallel with per-URL error isolation. ' \
+                         'Returns final redirected URLs, status codes, and rel="alternate" feeds.',
+            input_schema: Contract::BATCH_INSPECT_INPUT_SCHEMA
+          },
+          {
+            name: 'batch_recon',
+            kind: :batch,
+            batch_method: :batch_recon,
+            description: 'Run recon across multiple URLs in parallel with per-URL error isolation. ' \
+                         'Returns verdict, native_feed, and surface classification per URL.',
+            input_schema: Contract::BATCH_RECON_INPUT_SCHEMA
+          },
+          {
+            name: 'capture',
+            kind: :capture,
+            description: 'Derive a reusable html2rss feed config from a URL. ' \
+                         'Use when the goal is a durable YAML (then test → apply). ' \
+                         'Returns YAML inside payload.yaml (same serializer as CLI capture). ' \
+                         'Draft only — catalog feeds still need directory.topics and title/url; ' \
+                         'enhance defaults from admission evidence (false when chrome drops are high). ' \
+                         'Full schema options live in resource html2rss://schema.',
+            input_schema: Contract::CAPTURE_INPUT_SCHEMA,
+            handler: :capture_outcome
+          },
+          {
+            name: 'validate',
+            kind: :config_xor,
+            description: 'Validate a feed config hash XOR yaml string against the html2rss JSON schema. ' \
+                         'Call before test. Failures return isError with payload.errors. ' \
+                         'Full schema lives in resource html2rss://schema.',
+            input_schema: Contract::CONFIG_XOR_SCHEMA,
+            annotations: Contract::ANNOTATIONS_VALIDATE,
+            call: lambda { |config: nil, yaml: nil, **|
+              validation = Html2rss::Config.validate(ConfigArgument.parse(config:, yaml:).config)
+              Outcome.validate(errors: validation.success? ? nil : validation.errors.to_h)
+            }
+          },
+          {
+            name: 'test',
+            kind: :config_xor,
+            description: 'Validate schema and execute live extraction (asserting >= min_items items). ' \
+                         'Call after capture or validate; on success next_step is apply. ' \
+                         'Returns test summary in payload with sample items, timing, failure_kind, ' \
+                         'and quality_report (warnings for duplicate URLs, junk titles, native feed). ' \
+                         'Set strict_quality to fail on duplicate URLs, >50% junk titles, or short titles.',
+            input_schema: Contract::TEST_INPUT_SCHEMA,
+            call: lambda { |config: nil, yaml: nil, min_items: 1, strict_quality: false,
+                              compare_enhance: false, **kwargs|
+              feed_config = ConfigArgument.parse(config:, yaml:).config
+              test_args = { min_items:, strict_quality:, compare_enhance: }
+              test_args[:strategy] = Runtime.coerce_strategy(kwargs[:strategy]) if kwargs.key?(:strategy)
+              test_result = Html2rss.test(feed_config, **test_args)
+              Outcome.test(test_result)
+            }
+          },
+          {
+            name: 'apply',
+            kind: :config_xor,
+            description: 'Apply a validated feed config (hash XOR yaml) and return RSS XML in payload.rss. ' \
+                         'isError when the feed has zero items (ship gate). payload.item_count is RSS item count. ' \
+                         'Use after test succeeds.',
+            input_schema: Contract::APPLY_INPUT_SCHEMA,
+            handler: :apply_outcome
+          }
+        ].freeze
+
+        class << self # rubocop:disable Metrics/ClassLength -- registration engine + handlers
+          ##
+          # Registers all MCP tools on +server+ via +registrar+ (Server.define_envelope_tool).
+          #
+          # @param server [::MCP::Server]
+          # @param registrar [Proc]
+          # @return [void]
+          def register_all(server, registrar:)
+            TOOLS.each { |entry| register_tool(server, registrar, entry) }
+          end
+
+          private
+
+          def register_tool(server, registrar, entry)
+            case entry[:kind]
+            when :url then register_url_tool(server, registrar, entry)
+            when :batch then register_batch_tool(server, registrar, entry)
+            when :config_xor then register_config_xor_tool(server, registrar, entry)
+            when :capture then register_capture_tool(server, registrar, entry)
+            else raise ArgumentError, "unknown tool kind: #{entry[:kind].inspect}"
+            end
+          end
+
+          def register_url_tool(server, registrar, entry)
+            handler = entry[:handler] ? method(entry[:handler]) : entry[:call]
+            registrar.call(
+              server,
+              name: entry[:name],
+              description: entry[:description],
+              input_schema: entry[:input_schema],
+              annotations: entry.fetch(:annotations, Contract::ANNOTATIONS_OPEN_WORLD)
+            ) do |**kwargs|
+              handler.call(**tool_kwargs(kwargs))
+            end
+          end
+
+          def register_batch_tool(server, registrar, entry) # rubocop:disable Metrics/MethodLength
+            name = entry[:name]
+            batch_method = entry[:batch_method]
+            limit_default = entry[:limit_default]
+            registrar.call(
+              server,
+              name:,
+              description: entry[:description],
+              input_schema: entry[:input_schema]
+            ) do |urls:, strategy: 'auto', **kwargs|
+              concurrency = kwargs.fetch(:concurrency, Batch::DEFAULT_CONCURRENCY)
+              batch_args = { urls:, strategy: Runtime.coerce_strategy(strategy), concurrency: }
+              batch_args[:limit] = kwargs.fetch(:limit, limit_default) unless limit_default.nil?
+              Outcome.public_send(name, Batch.public_send(batch_method, **batch_args))
+            end
+          end
+
+          def register_config_xor_tool(server, registrar, entry)
+            handler = entry[:handler] ? method(entry[:handler]) : entry[:call]
+            registrar.call(
+              server,
+              name: entry[:name],
+              description: entry[:description],
+              input_schema: entry[:input_schema],
+              annotations: entry.fetch(:annotations, Contract::ANNOTATIONS_OPEN_WORLD)
+            ) do |**kwargs|
+              handler.call(**tool_kwargs(kwargs))
+            end
+          end
+
+          def register_capture_tool(server, registrar, entry)
+            handler = method(entry[:handler])
+            registrar.call(
+              server,
+              name: entry[:name],
+              description: entry[:description],
+              input_schema: entry[:input_schema]
+            ) do |**kwargs|
+              handler.call(**tool_kwargs(kwargs))
+            end
+          end
+
+          def tool_kwargs(inputs)
+            args = inputs.dup
+            args.delete(:server_context)
+            args
+          end
+
+          def scrape_outcome(url:, strategy: 'auto', limit: 25, items_selector: nil)
+            wire = Batch.scrape_wire(url:, strategy: Runtime.coerce_strategy(strategy), limit:, items_selector:)
+            Outcome.scrape(
+              items: wire[:items],
+              requested_strategy: wire[:strategy],
+              channel_title: wire[:channel_title],
+              admission_drops: wire[:admission_drops],
+              botasaurus_configured: Runtime.botasaurus_configured?
+            )
+          end
+
+          def capture_outcome(url:, strategy: 'auto', items_selector: nil, force: false, topics: nil, title: nil, # rubocop:disable Metrics/MethodLength, Metrics/ParameterLists
+                              summary: nil, enhance: nil, limit: nil, max_redirects: nil, max_requests: nil)
+            plan = Runtime.coerce_strategy(strategy)
+            result = Html2rss::Capture.build(
+              url,
+              strategy: plan,
+              items_selector:,
+              force:,
+              topics:,
+              title:,
+              summary:,
+              enhance:,
+              limit:,
+              max_redirects:,
+              max_requests:
+            )
+            Outcome.capture(
+              yaml: result.yaml,
+              articles_count: result.articles_count,
+              has_selectors: result.has_selectors,
+              channel_title: result.channel_title,
+              requested_strategy: plan,
+              segment_strategy: result.segment_strategy,
+              selected_strategy: result.selected_strategy,
+              admission_drops: result.admission_drops,
+              native_feed: result.native_feed,
+              suggested_channel_url: result.suggested_channel_url
+            )
+          end
+
+          def apply_outcome(url:, config: nil, yaml: nil)
+            feed_config = HashUtil.deep_dup(ConfigArgument.parse(config:, yaml:).config)
+            feed_config[:channel] ||= {}
+            feed_config[:channel][:url] ||= url
+            outcome, feed_result = FeedPipeline.new(feed_config).to_outcome_and_result
+            apply_feed_outcome(feed_config, feed_result, outcome)
+          end
+
+          def apply_feed_outcome(feed_config, feed_result, outcome) # rubocop:disable Metrics/MethodLength -- quality_report + RSS payload
+            rss = feed_result.to_rss
+            quality_report = Html2rss::Test.quality_report_for(
+              rss.items,
+              channel_url: feed_config.dig(:channel, :url).to_s,
+              raw_config: feed_config,
+              feed_result:,
+              pipeline_outcome: outcome,
+              probe_native_feed: false
+            )
+            Outcome.apply(
+              rss: rss.to_s,
+              item_count: rss.items.size,
+              empty: feed_result.empty?,
+              quality_report: quality_report.to_h
+            )
+          end
+        end
+      end
+    end
+  end
+end