html2rss 0.25.0 → 0.27.0

data/lib/html2rss/mcp/outcome.rb

@@ -0,0 +1,188 @@
+# frozen_string_literal: true
+
+module Html2rss
+  module MCP
+    Outcome = Data.define(:ok, :next_step, :guidance, :payload)
+
+    ##
+    # Typed MCP tool result. Owns next-step policy and guidance copy so the
+    # protocol adapter does not branch on quality heuristics.
+    class Outcome
+      # Matches {ConfigArgument} XOR {ArgumentError} messages.
+      XOR_ERROR = /exactly one of config or yaml/
+      NextStep = Data.define(:name, :guidance)
+
+      ##
+      # Closed set of agent next actions. Invalid names cannot be constructed.
+      class NextStep
+        # Wire names for +next_step+.
+        NAMES = %i[done inspect_url validate_config apply_config scrape_url capture_config read_runtime].freeze
+        # Default guidance copy keyed by {NAMES}.
+        GUIDANCE = {
+          done: 'Done. Read payload for the result.',
+          inspect_url: 'Call inspect_url next. Read payload for recon (final_url, status, ' \
+                       'scheme_downgrade, alternate_feeds).',
+          validate_config: 'Call validate_config with payload.yaml or a config hash (XOR, not both).',
+          apply_config: 'Call apply_config next. Confirm payload.item_count before shipping.',
+          scrape_url: 'Call scrape_url for articles now. strategy auto already runs Faraday then Botasaurus.',
+          capture_config: 'Call capture_config for a reusable YAML draft, then follow next_step.',
+          read_runtime: 'Read html2rss://runtime. Set BOTASAURUS_SCRAPER_URL on the MCP process ' \
+                        'if botasaurus_configured is false.'
+        }.freeze
+
+        ##
+        # @param name [Symbol, String]
+        # @param guidance [String, nil]
+        def initialize(name:, guidance: nil)
+          step = name.to_sym
+          raise ArgumentError, "unknown next_step: #{name.inspect}" unless NAMES.include?(step)
+
+          super(name: step, guidance: (guidance || GUIDANCE.fetch(step)).freeze)
+        end
+
+        class << self
+          NAMES.each { |step| define_method(step) { new(name: step) } }
+        end
+      end
+
+      ##
+      # @param ok [Boolean]
+      # @param next_step [NextStep]
+      # @param guidance [String]
+      # @param payload [Hash]
+      def initialize(ok:, next_step:, guidance:, payload:) # rubocop:disable Naming/MethodParameterName -- +ok+ is the envelope field
+        raise ArgumentError, 'next_step must be a NextStep' unless next_step.is_a?(NextStep)
+        raise ArgumentError, 'payload must be a Hash' unless payload.is_a?(Hash)
+
+        super(ok: !!ok, next_step:, guidance: guidance.to_s.freeze, payload: payload.dup.freeze)
+      end
+
+      ##
+      # @return [Hash{Symbol => Object}] envelope for {Contract.response}
+      def to_h
+        { ok:, next_step: next_step.name.to_s, guidance:, payload: }
+      end
+
+      class << self
+        ##
+        # @param items [Array]
+        # @param requested_strategy [String, Symbol]
+        # @param channel_title [String, nil]
+        # @param admission_drops [Hash]
+        # @param botasaurus_configured [Boolean]
+        # @return [Outcome]
+        def scrape(items:, requested_strategy:, channel_title:, botasaurus_configured:, admission_drops: {})
+          next_step = scrape_next_step(items.empty?, botasaurus_configured:)
+          new(ok: true, next_step:, guidance: next_step.guidance,
+              payload: scrape_payload(items:, requested_strategy:, channel_title:, admission_drops:))
+        end
+
+        ##
+        # @param payload [Hash] inspect recon Hash
+        # @return [Outcome]
+        def inspect(payload:)
+          next_step = inspect_next_step(payload)
+          new(ok: true, next_step:, guidance: next_step.guidance, payload:)
+        end
+
+        ##
+        # @param yaml [String]
+        # @param articles_count [Integer]
+        # @param has_selectors [Boolean]
+        # @param channel_title [String, nil]
+        # @param requested_strategy [String, Symbol]
+        # @param segment_strategy [Symbol, String, nil]
+        # @param selected_strategy [Symbol, String, nil]
+        # @param admission_drops [Hash]
+        # @return [Outcome]
+        def capture(yaml:, articles_count:, has_selectors:, channel_title:, requested_strategy:, # rubocop:disable Metrics/ParameterLists
+                    segment_strategy: nil, selected_strategy: nil, admission_drops: {})
+          next_step = capture_next_step(articles_count:, has_selectors:)
+          new(ok: true, next_step:, guidance: next_step.guidance, payload: capture_payload(
+            yaml:, articles_count:, has_selectors:, channel_title:, requested_strategy:,
+            segment_strategy:, selected_strategy:, admission_drops:
+          ))
+        end
+
+        ##
+        # @param errors [Hash, nil] schema errors; +nil+ means success
+        # @return [Outcome]
+        def validate(errors:)
+          ok = errors.nil?
+          next_step = ok ? NextStep.apply_config : NextStep.validate_config
+          new(ok:, next_step:, guidance: next_step.guidance, payload: ok ? {} : { errors: })
+        end
+
+        ##
+        # @param rss [String]
+        # @param item_count [Integer]
+        # @param empty [Boolean] {FeedResult#empty?} (ship gate); defaults to zero items
+        # @return [Outcome]
+        def apply(rss:, item_count:, empty: item_count.zero?)
+          ok = !empty
+          next_step = ok ? NextStep.done : NextStep.inspect_url
+          new(ok:, next_step:, guidance: next_step.guidance, payload: { rss:, item_count: })
+        end
+
+        ##
+        # @param error [Exception]
+        # @return [Outcome]
+        def from_error(error)
+          next_step = next_step_for_error(error)
+          new(ok: false, next_step:, guidance: next_step.guidance,
+              payload: { class: error.class.name, message: error.message })
+        end
+
+        private
+
+        def scrape_next_step(empty, botasaurus_configured:)
+          return NextStep.done unless empty
+          return NextStep.read_runtime unless botasaurus_configured
+
+          NextStep.inspect_url
+        end
+
+        def scrape_payload(items:, requested_strategy:, channel_title:, admission_drops:)
+          {
+            items:, total: items.size, requested_strategy: requested_strategy.to_s, channel_title:,
+            **(admission_drops.any? ? { admission_drops: } : {})
+          }
+        end
+
+        def inspect_next_step(payload)
+          return NextStep.done if Array(payload[:alternate_feeds]).any?
+          return NextStep.capture_config if payload[:articles_count].to_i.positive?
+
+          NextStep.scrape_url
+        end
+
+        def capture_next_step(articles_count:, has_selectors:)
+          articles_count.positive? && has_selectors ? NextStep.validate_config : NextStep.inspect_url
+        end
+
+        def capture_payload(yaml:, articles_count:, has_selectors:, channel_title:, requested_strategy:, # rubocop:disable Metrics/ParameterLists
+                            segment_strategy:, selected_strategy:, admission_drops:)
+          {
+            yaml:, articles_count:, has_selectors:, channel_title:, requested_strategy: requested_strategy.to_s,
+            **(segment_strategy ? { segment_strategy: segment_strategy.to_s } : {}),
+            **(selected_strategy ? { selected_strategy: selected_strategy.to_s } : {}),
+            **(admission_drops.any? ? { admission_drops: } : {})
+          }
+        end
+
+        def next_step_for_error(error)
+          case error
+          when RequestService::BotasaurusConfigurationError then NextStep.read_runtime
+          when Contract::UnpublishedRequestError then NextStep.validate_config
+          when ArgumentError then argument_error_next_step(error)
+          else NextStep.inspect_url
+          end
+        end
+
+        def argument_error_next_step(error)
+          XOR_ERROR.match?(error.message) ? NextStep.validate_config : NextStep.inspect_url
+        end
+      end
+    end
+  end
+end

data/lib/html2rss/mcp/server.rb

@@ -0,0 +1,421 @@
+# frozen_string_literal: true
+
+module Html2rss
+  module MCP
+    ##
+    # Thin MCP wire adapter over public html2rss APIs.
+    #
+    # Ownership: scraping/capture/validate/feed stay on gem entrypoints
+    # ({Html2rss.auto_feed_result}, {Capture.build}, {Config.validate}, {Html2rss.feed_result}).
+    # This module maps MCP kwargs to those APIs, then {Outcome} + {Contract} shape the envelope.
+    #
+    # Strategy note: MCP +auto+ passes through to FeedPipeline AutoFallback
+    # (faraday → botasaurus). Concrete strategies are used as-is.
+    # Botasaurus requires +BOTASAURUS_SCRAPER_URL+.
+    module Server # rubocop:disable Metrics/ModuleLength
+      # MCP server display name.
+      SERVER_NAME = 'html2rss'
+      # MCP server version (mirrors the gem version).
+      SERVER_VERSION = Html2rss::VERSION
+      # Loopback bind for HTTP transport (local use only).
+      HTTP_BIND_HOST = '127.0.0.1'
+
+      class << self # rubocop:disable Metrics/ClassLength
+        ##
+        # Starts the MCP server with the given transport.
+        #
+        # Points {Html2rss.logger} at +$stderr+ so stdio JSON-RPC on stdout stays
+        # intact, and raises the process log level to +info+ unless +LOG_LEVEL+ is set.
+        # A foreground watcher then sees the start banner, tool calls, and pipeline warns.
+        #
+        # @param transport [Symbol] +:stdio+ or +:http+
+        # @param port [Integer] port for HTTP transport
+        def start(transport: :stdio, port: 8080)
+          raise ArgumentError, "Unknown transport: #{transport.inspect}" unless %i[stdio http].include?(transport)
+
+          configure_daemon_logging!
+          app = build
+          Log.info(start_banner(transport:, port:))
+          return start_http(app, port:) if transport == :http
+
+          ::MCP::Server::Transports::StdioTransport.new(app).open
+        end
+
+        ##
+        # Builds the configured MCP protocol server (tools/resources/prompts).
+        #
+        # @return [::MCP::Server]
+        def build # rubocop:disable Metrics/MethodLength -- protocol server construction
+          ::MCP::Server.new(
+            name: SERVER_NAME,
+            title: SERVER_NAME,
+            version: SERVER_VERSION,
+            instructions: instructions_text,
+            configuration: protocol_configuration
+          ).tap do |server|
+            register_tools(server)
+            register_resources(server)
+            register_prompts(server)
+          end
+        end
+
+        private
+
+        def configure_daemon_logging!
+          Html2rss.configure do |config|
+            config.logger = Logger.new($stderr)
+            config.log_level = ENV.fetch('LOG_LEVEL', :info)
+          end
+        end
+
+        def start_banner(transport:, port:)
+          bind = transport == :http ? " bind=#{HTTP_BIND_HOST}:#{port}" : ''
+          "html2rss MCP #{SERVER_VERSION} starting transport=#{transport}#{bind}"
+        end
+
+        def protocol_configuration
+          ::MCP::Configuration.new.tap do |config|
+            config.exception_reporter = method(:report_protocol_exception)
+            config.around_request = method(:around_protocol_request)
+            config.validate_tool_call_results = true
+          end
+        end
+
+        def report_protocol_exception(error, server_context)
+          detail = server_context.is_a?(Hash) && server_context[:error]
+          suffix = detail ? " (#{detail})" : ''
+          Log.error("#{error.class}: #{error.message}#{suffix}")
+        end
+
+        def around_protocol_request(data)
+          return yield unless log_protocol_request?(data)
+
+          started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+          begin
+            Log.info(protocol_request_line('start', data))
+            yield
+          ensure
+            duration = Process.clock_gettime(Process::CLOCK_MONOTONIC) - started
+            Log.info(protocol_request_line('done', data, duration:))
+          end
+        end
+
+        def log_protocol_request?(data)
+          method = data[:method]
+          method.is_a?(String) &&
+            method != ::MCP::Methods::PING &&
+            !::MCP::Methods.notification?(method)
+        end
+
+        def protocol_request_line(phase, data, duration: nil)
+          parts = ['mcp', phase, data[:method], *protocol_request_labels(data)]
+          parts << format('%.2fs', duration) if duration
+          parts << "error=#{data[:error]}" if data[:error]
+          parts.join(' ')
+        end
+
+        def protocol_request_labels(data)
+          [
+            data[:tool_name] && "tool=#{data[:tool_name]}",
+            data[:prompt_name] && "prompt=#{data[:prompt_name]}",
+            data[:resource_uri] && "uri=#{data[:resource_uri]}"
+          ].compact
+        end
+
+        def start_http(app, port:) # rubocop:disable Metrics/MethodLength -- require + bind + LoadError message
+          require 'rackup'
+          require 'rackup/handler/webrick'
+          require 'webrick'
+
+          handler = ::MCP::Server::Transports::StreamableHTTPTransport.new(app, stateless: true)
+          Rackup::Handler::WEBrick.run(
+            handler,
+            Host: HTTP_BIND_HOST,
+            Port: port,
+            Logger: Html2rss.logger
+          )
+        rescue LoadError => error
+          raise LoadError,
+                'HTTP transport requires the rackup and webrick gems ' \
+                "(#{error.message}). Install them or use --transport stdio."
+        end
+
+        def instructions_text # rubocop:disable Metrics/MethodLength -- agent decision tree is the published contract
+          <<~TEXT.strip
+            html2rss MCP — decide which tool to call:
+
+            1. Need articles now (no saved config)? → scrape_url (1 call)
+               - strategy "auto" runs Faraday → Botasaurus AutoFallback. Do not retry with explicit faraday after auto.
+               - Empty scrape is still success (articles-now). Follow next_step / guidance (read_runtime if Botasaurus unset).
+            2. Need a reusable feed YAML? → capture_config → validate_config → apply_config
+               - capture_config returns YAML inside payload.yaml. Draft only: if destination is html2rss-configs, rewrite for directory.topics and explicit channel title/url. Strive enhance: true (false only when chrome leaks).
+               - validate_config / apply_config accept config hash XOR yaml string.
+               - apply_config isError when zero RSS items (ship gate). Confirm payload.item_count.
+            3. Weak scrape/capture or recon (final URL, status, https→http, rel=alternate feeds)? → inspect_url only if weak or recon.
+            4. Have a config already? → validate_config (must succeed) → apply_config
+            5. Schema / extractors / strategies / runtime → resources html2rss://schema|extractors|strategies|runtime
+
+            Prefer capture_config for durable config; scrape_url for one-shot extraction.
+            Follow envelope next_step and guidance. Botasaurus needs BOTASAURUS_SCRAPER_URL in this process env (boolean at html2rss://runtime; the URL is never returned).
+          TEXT
+        end
+
+        def botasaurus_configured?
+          !ENV['BOTASAURUS_SCRAPER_URL'].to_s.strip.empty?
+        end
+
+        def tool_error_response(error)
+          Log.error("mcp error #{error.class}: #{error.message}")
+          Contract.response(Outcome.from_error(error))
+        end
+
+        def handle_tool_call
+          Contract.response(yield)
+        rescue StandardError => error
+          tool_error_response(error)
+        end
+
+        # rubocop:disable Metrics/MethodLength -- listing fields stay together
+        def define_envelope_tool(server, name:, description:, input_schema:,
+                                 annotations: Contract::ANNOTATIONS_OPEN_WORLD)
+          run = method(:handle_tool_call)
+          server.define_tool(
+            name:,
+            title: Contract::TITLES.fetch(name.to_sym),
+            description:,
+            annotations:,
+            input_schema:,
+            output_schema: Contract.output_schema
+          ) do |**kwargs|
+            run.call { yield(**kwargs) }
+          end
+        end
+        # rubocop:enable Metrics/MethodLength
+
+        def register_tools(server)
+          register_scrape_url(server)
+          register_inspect_url(server)
+          register_capture_config(server)
+          register_validate_config(server)
+          register_apply_config(server)
+        end
+
+        def register_scrape_url(server)
+          define_envelope_tool(
+            server,
+            name: 'scrape_url',
+            description: 'One-shot article extraction as JSON Feed items. ' \
+                         'Use when you need articles now without a saved config. ' \
+                         'strategy "auto" triggers fallback chain (faraday → botasaurus) for JS-rendered sites.',
+            input_schema: Contract::SCRAPE_INPUT_SCHEMA
+          ) do |server_context:, url:, strategy: 'auto', limit: 25, items_selector: nil| # rubocop:disable Lint/UnusedBlockArgument
+            scrape_outcome(url:, strategy:, limit:, items_selector:)
+          end
+        end
+
+        def scrape_outcome(url:, strategy:, limit:, items_selector:)
+          plan = (strategy || :auto).to_sym
+          feed_result = Html2rss.auto_feed_result(url, strategy: plan, limit:, items_selector:)
+          feed = feed_result.to_json_feed
+          Outcome.scrape(
+            items: feed[:items] || [],
+            requested_strategy: plan,
+            channel_title: feed[:title],
+            admission_drops: feed_result.status.admission_drops,
+            botasaurus_configured: botasaurus_configured?
+          )
+        end
+
+        def register_inspect_url(server)
+          define_envelope_tool(
+            server,
+            name: 'inspect_url',
+            description: 'Diagnostic page analysis (scrapers, SST, segments) plus recon: ' \
+                         'final_url, status, scheme_downgrade, rel=alternate RSS/Atom feeds. ' \
+                         'Use when scrape/capture is weak or you need those recon facts.',
+            input_schema: Contract::INSPECT_INPUT_SCHEMA
+          ) do |server_context:, url:, strategy: 'auto'| # rubocop:disable Lint/UnusedBlockArgument
+            Outcome.inspect(payload: Inspect.call(url:, strategy:))
+          end
+        end
+
+        def register_capture_config(server) # rubocop:disable Metrics/MethodLength
+          define_envelope_tool(
+            server,
+            name: 'capture_config',
+            description: 'Derive a reusable html2rss feed config from a URL. ' \
+                         'Use when the goal is a durable YAML (then validate_config). ' \
+                         'Returns YAML inside payload.yaml (same serializer as CLI capture). ' \
+                         'Draft only — catalog feeds still need directory.topics and title/url; ' \
+                         'strive enhance: true. Full schema options live in resource html2rss://schema.',
+            input_schema: Contract::CAPTURE_INPUT_SCHEMA
+          ) do |server_context:, url:, strategy: 'auto', items_selector: nil| # rubocop:disable Lint/UnusedBlockArgument
+            capture_outcome(url:, strategy:, items_selector:)
+          end
+        end
+
+        def capture_outcome(url:, strategy:, items_selector:) # rubocop:disable Metrics/MethodLength -- CaptureResult maps 1:1 onto Outcome
+          plan = (strategy || :auto).to_sym
+          result = Html2rss::Capture.build(url, strategy: plan, items_selector:)
+          Outcome.capture(
+            yaml: Config.to_yaml(result.config),
+            articles_count: result.articles_count,
+            has_selectors: result.has_selectors,
+            channel_title: result.channel_title,
+            requested_strategy: plan,
+            segment_strategy: result.segment_strategy,
+            selected_strategy: result.selected_strategy,
+            admission_drops: result.admission_drops
+          )
+        end
+
+        def register_validate_config(server) # rubocop:disable Metrics/MethodLength -- description is the published contract
+          define_envelope_tool(
+            server,
+            name: 'validate_config',
+            description: 'Validate a feed config hash XOR yaml string against the html2rss JSON schema. ' \
+                         'Call before apply_config. Failures return isError with payload.errors. ' \
+                         'Full schema lives in resource html2rss://schema.',
+            input_schema: Contract::CONFIG_XOR_SCHEMA,
+            annotations: Contract::ANNOTATIONS_VALIDATE
+          ) do |server_context:, config: nil, yaml: nil| # rubocop:disable Lint/UnusedBlockArgument
+            validate_outcome(config:, yaml:)
+          end
+        end
+
+        def validate_outcome(config:, yaml:)
+          validation = Html2rss::Config.validate(ConfigArgument.parse(config:, yaml:).config)
+          Outcome.validate(errors: validation.success? ? nil : validation.errors.to_h)
+        end
+
+        def register_apply_config(server)
+          define_envelope_tool(
+            server,
+            name: 'apply_config',
+            description: 'Apply a validated feed config (hash XOR yaml) and return RSS XML in payload.rss. ' \
+                         'isError when the feed has zero items (ship gate). payload.item_count is RSS item count. ' \
+                         'Use after validate_config succeeds.',
+            input_schema: Contract::APPLY_INPUT_SCHEMA
+          ) do |server_context:, url:, config: nil, yaml: nil| # rubocop:disable Lint/UnusedBlockArgument
+            apply_outcome(url:, config:, yaml:)
+          end
+        end
+
+        def apply_outcome(url:, config:, yaml:)
+          feed_config = ConfigArgument.parse(config:, yaml:).config
+          feed_config[:channel] ||= {}
+          feed_config[:channel][:url] ||= url
+          feed_result = Html2rss.feed_result(feed_config)
+          rss = feed_result.to_rss
+          Outcome.apply(rss: rss.to_s, item_count: rss.items.size, empty: feed_result.empty?)
+        end
+
+        def register_resources(server) # rubocop:disable Metrics/MethodLength
+          configured = method(:botasaurus_configured?)
+          server.define_resource(
+            uri: 'html2rss://schema',
+            name: 'Configuration JSON Schema',
+            description: 'Full JSON Schema for html2rss feed configurations',
+            mime_type: 'application/json'
+          ) do |server_context: nil| # rubocop:disable Lint/UnusedBlockArgument
+            schema = Html2rss::Config.json_schema_json(pretty: true)
+            [{ uri: 'html2rss://schema', mimeType: 'application/json', text: schema }]
+          end
+
+          server.define_resource(
+            uri: 'html2rss://extractors',
+            name: 'Available Extractors',
+            description: 'Registered extractor names for selector configs ' \
+                         '(full option docs live in html2rss://schema $defs)',
+            mime_type: 'application/json'
+          ) do |server_context: nil| # rubocop:disable Lint/UnusedBlockArgument
+            extractors = Html2rss::Selectors::Extractors::NAME_TO_CLASS.keys.map(&:to_s).sort
+            [{ uri: 'html2rss://extractors', mimeType: 'application/json',
+               text: JSON.pretty_generate(extractors) }]
+          end
+
+          server.define_resource(
+            uri: 'html2rss://strategies',
+            name: 'Available Strategies',
+            description: 'Published MCP request strategy names',
+            mime_type: 'application/json'
+          ) do |server_context: nil| # rubocop:disable Lint/UnusedBlockArgument
+            [{ uri: 'html2rss://strategies', mimeType: 'application/json',
+               text: JSON.generate(Contract::STRATEGIES) }]
+          end
+
+          server.define_resource(
+            uri: 'html2rss://runtime',
+            name: 'Runtime capabilities',
+            description: 'Whether optional transports are configured in this MCP process (never leaks secrets)',
+            mime_type: 'application/json'
+          ) do |server_context: nil| # rubocop:disable Lint/UnusedBlockArgument
+            [{ uri: 'html2rss://runtime', mimeType: 'application/json',
+               text: JSON.pretty_generate(botasaurus_configured: configured.call) }]
+          end
+        end
+
+        def register_prompts(server)
+          register_scrape_webpage_prompt(server)
+          register_capture_feed_config_prompt(server)
+        end
+
+        def register_scrape_webpage_prompt(server) # rubocop:disable Metrics/MethodLength -- SDK prompt types stay together
+          to_result = method(:prompt_result)
+          text_for = method(:scrape_webpage_text)
+          server.define_prompt(
+            name: 'scrape-webpage',
+            description: 'Guided one-shot scrape: one scrape_url call (auto already falls back)',
+            arguments: [
+              ::MCP::Prompt::Argument.new(name: 'url', description: 'URL to scrape', required: true)
+            ]
+          ) do |args, server_context:| # rubocop:disable Lint/UnusedBlockArgument
+            to_result.call(text_for.call(args.fetch(:url)))
+          end
+        end
+
+        def register_capture_feed_config_prompt(server) # rubocop:disable Metrics/MethodLength -- SDK prompt types stay together
+          to_result = method(:prompt_result)
+          text_for = method(:capture_feed_config_text)
+          server.define_prompt(
+            name: 'capture-feed-config',
+            description: 'Guided capture → validate → apply; YAML draft plus catalog rewrite',
+            arguments: [
+              ::MCP::Prompt::Argument.new(name: 'url', description: 'URL to analyze', required: true)
+            ]
+          ) do |args, server_context:| # rubocop:disable Lint/UnusedBlockArgument
+            to_result.call(text_for.call(args.fetch(:url)))
+          end
+        end
+
+        def prompt_result(text)
+          ::MCP::Prompt::Result.new(
+            messages: [
+              ::MCP::Prompt::Message.new(role: 'user', content: ::MCP::Content::Text.new(text))
+            ]
+          )
+        end
+
+        def scrape_webpage_text(url)
+          <<~MSG.strip
+            Scrape #{url} with scrape_url (strategy auto). One call is enough — auto already runs Faraday then Botasaurus.
+            Follow envelope next_step and guidance. Call inspect_url only if articles are empty/weak or you need recon (final_url, status, scheme_downgrade, alternate_feeds).
+            Do not retry scrape_url with explicit faraday after auto. Read html2rss://runtime if next_step is read_runtime.
+            Return payload.items (not a raw JSON array).
+          MSG
+        end
+
+        def capture_feed_config_text(url)
+          <<~MSG.strip
+            Build a reusable html2rss feed config for #{url}:
+            1) capture_config — YAML is payload.yaml. Check payload.articles_count and payload.has_selectors. Strive to keep enhance: true (false only when chrome leaks into items).
+            2) Follow next_step. If weak or you need recon, inspect_url. Auto already hops to Botasaurus; do not retry capture with botasaurus unless Faraday was blocked.
+            3) validate_config with yaml (or config hash) — must not be isError
+            4) apply_config — isError if zero items. Confirm payload.item_count before shipping.
+            If the destination is html2rss-configs, rewrite the draft for directory.topics and explicit channel title/url. Return YAML.
+          MSG
+        end
+      end
+    end
+  end
+end

data/lib/html2rss/mcp.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module Html2rss
+  ##
+  # MCP server for AI client consumption.
+  # Lazy-loads the mcp gem; no cost when the server is not started.
+  module MCP
+    class << self
+      ##
+      # Starts the MCP server using the given transport.
+      #
+      # @param transport [Symbol] +:stdio+ or +:http+
+      # @param port [Integer] port for HTTP transport (bound to 127.0.0.1)
+      def start(transport: :stdio, port: 8080)
+        require 'mcp'
+        require_relative 'mcp/server'
+        Server.start(transport:, port:)
+      end
+    end
+  end
+end

data/lib/html2rss/request_service/blocked_surface.rb

@@ -21,8 +21,31 @@ module Html2rss
             /cloudflare ray id/i
           ],
           message: 'Blocked surface detected: Cloudflare anti-bot interstitial page. ' \
-                   'Retry with --strategy browserless, try a more specific public listing URL, ' \
+                   'Target a direct listing URL, configure BOTASAURUS_SCRAPER_URL, ' \
                    'or run from an environment that can complete anti-bot checks.'
+        },
+        {
+          key: :datadome_interstitial,
+          min_matches: 2,
+          patterns: [
+            /captcha-delivery\.com/,
+            /DataDome/i,
+            /interstitial/i
+          ],
+          message: 'Blocked surface detected: DataDome anti-bot challenge page. ' \
+                   'Target a direct listing URL, or scrape via a session with resolved DataDome cookies.'
+        },
+        {
+          key: :vercel_security_checkpoint,
+          min_matches: 1,
+          patterns: [
+            /Vercel Security Checkpoint/i,
+            %r{vercel\.com/security}i,
+            /checking the security/i
+          ],
+          message: 'Blocked surface detected: Vercel Security Checkpoint. ' \
+                   'This site is a JS-rendered SPA behind Vercel edge protection. ' \
+                   'Configure BOTASAURUS_SCRAPER_URL or target a direct listing URL.'
         }
       ].freeze