openclacky 1.1.6 → 1.2.0

data/lib/clacky/mcp/http_transport.rb

@@ -0,0 +1,155 @@
+# frozen_string_literal: true
+
+require "json"
+require "monitor"
+require "net/http"
+require "uri"
+require "securerandom"
+
+require_relative "transport"
+
+module Clacky
+  module Mcp
+    # MCP streamable-http transport (spec 2025-03-26).
+    #
+    # One endpoint URL handles both client→server (POST) and server→client (SSE).
+    # We POST every JSON-RPC message; the server may respond with either:
+    #   - application/json   → single response, deliver immediately
+    #   - text/event-stream  → one or more "data:" SSE events, each a JSON-RPC msg
+    #
+    # Session tracking: the server returns Mcp-Session-Id on the initialize
+    # response; we echo it on every subsequent request.
+    class HttpTransport < Transport
+      DEFAULT_OPEN_TIMEOUT = 10
+      DEFAULT_READ_TIMEOUT = 120
+
+      def initialize(name:, url:, headers: {}, open_timeout: DEFAULT_OPEN_TIMEOUT, read_timeout: DEFAULT_READ_TIMEOUT)
+        @name = name
+        @uri  = URI.parse(url)
+        raise TransportError, "MCP server '#{name}' url is not http(s): #{url}" unless %w[http https].include?(@uri.scheme)
+
+        @extra_headers = (headers || {}).transform_keys(&:to_s).transform_values(&:to_s)
+        @open_timeout  = open_timeout
+        @read_timeout  = read_timeout
+
+        @session_id = nil
+        @on_message = nil
+        @lock = Monitor.new
+        @alive = false
+        @last_error = nil
+      end
+
+      def start
+        @alive = true
+        self
+      end
+
+      def stop
+        @alive = false
+      end
+
+      def alive?
+        @alive
+      end
+
+      def send_message(payload)
+        raise TransportError, "transport stopped" unless @alive
+
+        body = JSON.generate(payload)
+        is_request = payload.is_a?(Hash) && payload.key?(:id) || (payload.is_a?(Hash) && payload.key?("id"))
+
+        Thread.new do
+          begin
+            dispatch_post(body, is_request: is_request)
+          rescue StandardError => e
+            @last_error = e
+            @on_message&.call({
+              "id"    => payload[:id] || payload["id"],
+              "error" => { "code" => -32000, "message" => "HTTP transport error: #{e.message}" }
+            })
+          end
+        end
+      end
+
+      def on_message(&blk)
+        @on_message = blk
+      end
+
+      def stderr_tail(bytes: 4096)
+        @last_error ? "last error: #{@last_error.class}: #{@last_error.message}" : ""
+      end
+
+      private def dispatch_post(body, is_request:)
+        http = Net::HTTP.new(@uri.host, @uri.port)
+        http.use_ssl = (@uri.scheme == "https")
+        http.open_timeout = @open_timeout
+        http.read_timeout = @read_timeout
+
+        req = Net::HTTP::Post.new(@uri.request_uri)
+        req["Content-Type"] = "application/json"
+        req["Accept"]       = "application/json, text/event-stream"
+        req["MCP-Protocol-Version"] = Client::PROTOCOL_VERSION if defined?(Client::PROTOCOL_VERSION)
+        @lock.synchronize { req["Mcp-Session-Id"] = @session_id if @session_id }
+        @extra_headers.each { |k, v| req[k] = v }
+        req.body = body
+
+        http.request(req) do |res|
+          if (sid = res["Mcp-Session-Id"])
+            @lock.synchronize { @session_id = sid }
+          end
+
+          status = res.code.to_i
+          if status == 202
+            return
+          end
+          if status >= 400
+            text = res.read_body.to_s
+            raise TransportError, "HTTP #{status} from MCP server '#{@name}': #{text[0, 500]}"
+          end
+
+          ctype = (res["Content-Type"] || "").downcase
+          if ctype.include?("text/event-stream")
+            consume_sse(res)
+          else
+            text = res.read_body.to_s
+            return if text.strip.empty?
+            begin
+              msg = JSON.parse(text)
+            rescue JSON::ParserError => e
+              raise TransportError, "invalid JSON from MCP server '#{@name}': #{e.message}"
+            end
+            deliver(msg)
+          end
+        end
+      end
+
+      private def consume_sse(res)
+        buffer = String.new
+        res.read_body do |chunk|
+          buffer << chunk
+          while (idx = buffer.index("\n\n"))
+            event = buffer.slice!(0, idx + 2)
+            data_lines = event.each_line.map(&:chomp).select { |l| l.start_with?("data:") }
+            next if data_lines.empty?
+            payload = data_lines.map { |l| l.sub(/\Adata:\s?/, "") }.join("\n")
+            next if payload.empty?
+            begin
+              msg = JSON.parse(payload)
+            rescue JSON::ParserError
+              next
+            end
+            deliver(msg)
+          end
+        end
+      end
+
+      private def deliver(msg)
+        if msg.is_a?(Array)
+          msg.each { |m| @on_message&.call(m) }
+        else
+          @on_message&.call(msg)
+        end
+      end
+    end
+  end
+end

data/lib/clacky/mcp/registry.rb

@@ -0,0 +1,229 @@
+# frozen_string_literal: true
+
+require "json"
+require "pathname"
+require "monitor"
+
+require_relative "client"
+require_relative "virtual_skill"
+
+module Clacky
+  module Mcp
+    # Central registry for MCP servers configured by the user.
+    #
+    # Responsibilities:
+    #   - Load ~/.clacky/mcp.json (or project .clacky/mcp.json) on demand.
+    #   - For each declared server, expose a VirtualSkill so the main agent
+    #     sees it as a one-line capability in the AVAILABLE SKILLS section.
+    #     No tool schemas leak into the main context.
+    #   - Lazily spawn the server process the first time invoke_skill('mcp:xxx')
+    #     happens, cache the connection, and reap idle servers after a timeout.
+    #   - Provide a single call_tool entry point for Tools::McpCall to dispatch
+    #     into.
+    class Registry
+      # User-facing config files, in priority order (later wins).
+      # Resolved at load time (not at constant evaluation) so HOME changes — e.g.
+      # in tests using stub_const or Dir.mktmpdir + ENV — are honored.
+      private def global_config_paths
+        [File.join(Dir.home, ".clacky", "mcp.json")]
+      end
+
+      # How long an MCP server may sit idle before we reap it. Vital for the
+      # "no gateway" promise: we never keep stale processes around.
+      DEFAULT_IDLE_TIMEOUT = 300  # 5 min
+
+      # How long to wait for tools/list during cold metadata collection.
+      DESCRIPTION_FETCH_TIMEOUT = 20
+
+      attr_reader :servers
+
+      def initialize(working_dir: nil, idle_timeout: DEFAULT_IDLE_TIMEOUT)
+        @working_dir   = working_dir
+        @idle_timeout  = idle_timeout
+        @servers       = {}        # name => spec hash
+        @clients       = {}        # name => Client (only when started)
+        @virtual_skills_cache = nil
+        @lock          = Monitor.new
+        @reaper_thread = nil
+
+        load_config
+        start_reaper
+      end
+
+      # Reload mcp.json (e.g. user added a server) and invalidate caches.
+      # Existing live clients survive; only stopped/removed servers get cleaned.
+      def reload
+        @lock.synchronize do
+          old_names = @servers.keys
+          @servers = {}
+          load_config
+          @virtual_skills_cache = nil
+
+          (old_names - @servers.keys).each do |gone|
+            @clients.delete(gone)&.stop
+          end
+        end
+      end
+
+      # Map of server name -> VirtualSkill. Cached because rebuilding it triggers
+      # tools/list against every cold server, which we want to do at most once
+      # per process.
+      #
+      # Implementation note: we do NOT pre-spawn servers here. We need their
+      # tool list to populate the VirtualSkill body, but we only fetch it the
+      # first time the *subagent* actually fires up. For the system-prompt
+      # description we use the user-provided "description" field from mcp.json,
+      # falling back to a placeholder. This keeps app startup zero-cost.
+      def virtual_skills
+        @lock.synchronize do
+          return @virtual_skills_cache.values if @virtual_skills_cache
+
+          @virtual_skills_cache = {}
+          @servers.each do |name, spec|
+            @virtual_skills_cache[name] = VirtualSkill.new(
+              server_name: name,
+              description: spec["description"] || default_description_for(name)
+            )
+          end
+          @virtual_skills_cache.values
+        end
+      end
+
+      # Return a fresh VirtualSkill for a server. The HttpServer's MCP tools
+      # endpoint uses this to satisfy probe requests; tool schemas are no
+      # longer attached to the skill itself — clients fetch them separately
+      # via /api/mcp/:name/tools.
+      # @param server_name [String]
+      # @return [VirtualSkill, nil]
+      def virtual_skill_for(server_name)
+        return nil unless @servers.key?(server_name)
+
+        spec = @servers[server_name]
+        VirtualSkill.new(
+          server_name: server_name,
+          description: spec["description"] || default_description_for(server_name)
+        )
+      end
+
+      # Fetch the live tool list (and lazily cold-start the server). Used by
+      # the HttpServer for /api/mcp/:name/tools and /api/mcp/:name/probe.
+      def tool_definitions(server_name)
+        client = ensure_started(server_name)
+        return [] unless client
+
+        client.tool_definitions
+      end
+
+      # Execute a tool call against an MCP server. Used by Tools::McpCall.
+      # @return [Hash] MCP `tools/call` result
+      def call_tool(server_name, tool_name, arguments)
+        client = ensure_started(server_name)
+        raise Mcp::Client::TransportError, "MCP server '#{server_name}' is not configured" unless client
+
+        client.call_tool(tool_name, arguments)
+      end
+
+      # Has the user configured any MCP servers?
+      def any?
+        !@servers.empty?
+      end
+
+      def configured?(server_name)
+        @servers.key?(server_name)
+      end
+
+      # Stop all live MCP server processes. Safe to call from at_exit hooks
+      # and on agent shutdown.
+      def shutdown
+        @lock.synchronize do
+          @reaper_thread&.kill rescue nil
+          @reaper_thread = nil
+          @clients.each_value { |c| c.stop rescue nil }
+          @clients.clear
+        end
+      end
+
+      # Spawn (if needed) and return the client for a server. Returns nil if the
+      # server is not configured. Raises Mcp::Client::TransportError if the
+      # process refuses to start.
+      private def ensure_started(server_name)
+        spec = @servers[server_name]
+        return nil unless spec
+
+        @lock.synchronize do
+          existing = @clients[server_name]
+          return existing if existing&.started?
+
+          client = Client.from_spec(server_name, spec)
+          client.start
+          @clients[server_name] = client
+          client
+        end
+      end
+
+      private def load_config
+        paths = global_config_paths
+        if @working_dir
+          paths = paths + [File.join(@working_dir, ".clacky", "mcp.json")]
+        end
+
+        paths.each do |path|
+          next unless File.exist?(path)
+
+          begin
+            data = JSON.parse(File.read(path))
+          rescue JSON::ParserError => e
+            Clacky::Logger.warn("Skipping malformed MCP config #{path}: #{e.message}") if defined?(Clacky::Logger)
+            next
+          end
+
+          servers = data["mcpServers"] || data["servers"] || {}
+          servers.each do |name, spec|
+            next unless spec.is_a?(Hash)
+            next if spec["disabled"] == true
+
+            type = (spec["type"] || (spec["url"] ? "http" : "stdio")).to_s
+            case type
+            when "stdio"
+              next unless spec["command"]
+            when "http", "streamable-http"
+              next unless spec["url"]
+            else
+              next
+            end
+
+            @servers[name.to_s] = spec
+          end
+        end
+      end
+
+      private def default_description_for(name)
+        "MCP server '#{name}'. Use this skill to delegate any task that this server " \
+          "can handle. The subagent will see the server's full tool list at fork time."
+      end
+
+      private def start_reaper
+        return if @idle_timeout.nil? || @idle_timeout <= 0
+
+        @reaper_thread = Thread.new do
+          loop do
+            sleep [@idle_timeout / 5, 30].min
+            now = Time.now
+            @lock.synchronize do
+              @clients.each do |name, client|
+                next unless client.last_used_at
+                if now - client.last_used_at > @idle_timeout
+                  client.stop
+                  @clients.delete(name)
+                end
+              end
+            end
+          end
+        rescue StandardError
+          # Reaper thread must never crash the main agent. Best-effort.
+        end
+        @reaper_thread.name = "mcp-reaper" if @reaper_thread.respond_to?(:name=)
+      end
+    end
+  end
+end

data/lib/clacky/mcp/skill_provider.rb

@@ -0,0 +1,75 @@
+# frozen_string_literal: true
+
+require "json"
+
+require_relative "virtual_skill"
+
+module Clacky
+  module Mcp
+    # Static, read-only provider that translates ~/.clacky/mcp.json (and the
+    # project-level override) into VirtualSkill instances for the SkillLoader.
+    #
+    # Unlike Mcp::Registry, this class never spawns server processes, never
+    # talks JSON-RPC, and holds no mutable state. All actual MCP traffic flows
+    # through the local Clacky HTTP API (/api/mcp/:server/tools and /call),
+    # which subagents reach via curl. This keeps agents process-light and
+    # decouples skill discovery from server lifecycle.
+    class SkillProvider
+      def initialize(working_dir: nil)
+        @working_dir = working_dir
+      end
+
+      def virtual_skills
+        load_servers.map do |name, spec|
+          VirtualSkill.new(
+            server_name: name,
+            description: spec["description"] || default_description_for(name)
+          )
+        end
+      end
+
+      private def load_servers
+        servers = {}
+        config_paths.each do |path|
+          next unless File.exist?(path)
+
+          begin
+            data = JSON.parse(File.read(path))
+          rescue JSON::ParserError => e
+            Clacky::Logger.warn("Skipping malformed MCP config #{path}: #{e.message}") if defined?(Clacky::Logger)
+            next
+          end
+
+          (data["mcpServers"] || data["servers"] || {}).each do |name, spec|
+            next unless spec.is_a?(Hash)
+            next if spec["disabled"] == true
+
+            type = (spec["type"] || (spec["url"] ? "http" : "stdio")).to_s
+            case type
+            when "stdio"
+              next unless spec["command"]
+            when "http", "streamable-http"
+              next unless spec["url"]
+            else
+              next
+            end
+
+            servers[name.to_s] = spec
+          end
+        end
+        servers
+      end
+
+      private def config_paths
+        paths = [File.join(Dir.home, ".clacky", "mcp.json")]
+        paths << File.join(@working_dir, ".clacky", "mcp.json") if @working_dir
+        paths
+      end
+
+      private def default_description_for(name)
+        "MCP server '#{name}'. Required entry point for any operation against " \
+          "this server — invoke this skill so a subagent runs the calls."
+      end
+    end
+  end
+end

data/lib/clacky/mcp/stdio_transport.rb

@@ -0,0 +1,112 @@
+# frozen_string_literal: true
+
+require "json"
+require "open3"
+require "monitor"
+
+require_relative "transport"
+
+module Clacky
+  module Mcp
+    class StdioTransport < Transport
+      def initialize(name:, command:, args: [], env: {}, cwd: nil)
+        @name    = name
+        @command = command
+        @args    = Array(args)
+        @env     = env || {}
+        @cwd     = cwd
+
+        @stdin = @stdout = @stderr = nil
+        @wait_thr = nil
+        @reader_thr = nil
+        @on_message = nil
+        @lock = Monitor.new
+        @stderr_buf = String.new
+      end
+
+      def start
+        full_env = ENV.to_h.merge(@env.transform_keys(&:to_s).transform_values(&:to_s))
+        opts = { unsetenv_others: false }
+        opts[:chdir] = @cwd if @cwd && File.directory?(@cwd)
+
+        @stdin, @stdout, @stderr, @wait_thr = Open3.popen3(full_env, @command, *@args, opts)
+        @stdin.sync = true
+
+        Thread.new do
+          @stderr.each_line do |line|
+            @lock.synchronize do
+              @stderr_buf << line
+              @stderr_buf.replace(@stderr_buf[-32_768, 32_768] || @stderr_buf) if @stderr_buf.bytesize > 65_536
+            end
+          end
+        rescue IOError
+        end
+
+        start_reader
+        self
+      rescue Errno::ENOENT => e
+        raise TransportError, "MCP server '#{@name}' command not found: #{@command} (#{e.message})"
+      end
+
+      def stop
+        @lock.synchronize do
+          return unless @wait_thr&.alive?
+          begin
+            Process.kill("TERM", @wait_thr.pid)
+          rescue Errno::ESRCH, Errno::EPERM
+          end
+          deadline = Time.now + 2
+          sleep 0.05 while @wait_thr.alive? && Time.now < deadline
+          if @wait_thr.alive?
+            begin
+              Process.kill("KILL", @wait_thr.pid)
+            rescue Errno::ESRCH, Errno::EPERM
+            end
+          end
+        ensure
+          [@stdin, @stdout, @stderr].each { |io| io&.close rescue nil }
+          @reader_thr&.kill rescue nil
+        end
+      end
+
+      def alive?
+        !!(@wait_thr && @wait_thr.alive?)
+      end
+
+      def send_message(payload)
+        line = JSON.generate(payload) + "\n"
+        @lock.synchronize do
+          raise TransportError, "MCP server '#{@name}' stdin closed" if @stdin.nil? || @stdin.closed?
+          @stdin.write(line)
+        end
+      rescue Errno::EPIPE => e
+        raise TransportError, "MCP server '#{@name}' stdin pipe broken: #{e.message}"
+      end
+
+      def on_message(&blk)
+        @on_message = blk
+      end
+
+      def stderr_tail(bytes: 4096)
+        @lock.synchronize { @stderr_buf[-bytes, bytes] || @stderr_buf.dup }
+      end
+
+      private def start_reader
+        @reader_thr = Thread.new do
+          @stdout.each_line do |line|
+            line = line.strip
+            next if line.empty?
+            begin
+              msg = JSON.parse(line)
+            rescue JSON::ParserError
+              next
+            end
+            @on_message&.call(msg)
+          end
+        rescue IOError
+          @on_message&.call({ "__transport_closed__" => true })
+        end
+      end
+    end
+  end
+end

data/lib/clacky/mcp/transport.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Clacky
+  module Mcp
+    # Abstract transport. Concrete transports must implement:
+    #   #start            -> open the channel (spawn process / open connection)
+    #   #stop             -> close everything; must be idempotent
+    #   #alive?           -> whether the channel is healthy
+    #   #send_message(h)  -> serialize and write one JSON-RPC message
+    #   #on_message(&b)   -> register callback invoked with each parsed inbound JSON message
+    #   #stderr_tail(bytes:) -> recent diagnostic text (may be empty)
+    class Transport
+      class TransportError < StandardError; end
+
+      def start;             raise NotImplementedError; end
+      def stop;              raise NotImplementedError; end
+      def alive?;            raise NotImplementedError; end
+      def send_message(_);   raise NotImplementedError; end
+      def on_message(&_blk); raise NotImplementedError; end
+      def stderr_tail(bytes: 4096); ""; end
+    end
+  end
+end