swarm_sdk 2.7.14 → 3.0.0.alpha2

data/lib/swarm_sdk/v3/mcp/ssl_http_transport.rb

@@ -0,0 +1,103 @@
+# frozen_string_literal: true
+
+module SwarmSDK
+  module V3
+    module MCP
+      # HTTP transport for MCP with configurable SSL verification.
+      #
+      # Uses Net::HTTP directly instead of Faraday to avoid the async-http
+      # adapter that Faraday selects when running inside Async. The async-http
+      # adapter ignores Faraday's SSL settings and uses IO::Endpoint::SSLEndpoint
+      # which enforces CRL checking on OpenSSL 3.6+, breaking most HTTPS MCP
+      # connections with "certificate verify failed (unable to get certificate CRL)".
+      #
+      # Conforms to the MCP transport interface: responds to +send_request(request:)+
+      # returning a parsed Hash.
+      #
+      # @example Default (SSL peer verification, no CRL checking)
+      #   SslHttpTransport.new(url: "https://api.example.com/mcp")
+      #
+      # @example Disable SSL verification entirely
+      #   SslHttpTransport.new(url: "https://localhost/mcp", ssl_verify: false)
+      class SslHttpTransport
+        # @return [String] HTTP endpoint URL
+        attr_reader :url
+
+        # @param url [String] HTTP endpoint URL
+        # @param headers [Hash] HTTP headers
+        # @param ssl_verify [Boolean] Whether to verify SSL certificates (default: true)
+        def initialize(url:, headers: {}, ssl_verify: true)
+          @url = url
+          @headers = headers
+          @ssl_verify = ssl_verify
+          @uri = URI.parse(url)
+        end
+
+        # Send a JSON-RPC request to the MCP server.
+        #
+        # @param request [Hash] JSON-RPC request body
+        # @return [Hash] Parsed JSON response
+        # @raise [MCP::Client::RequestHandlerError] On HTTP or connection errors
+        def send_request(request:)
+          http = build_http
+          post = build_post(request)
+
+          response = http.request(post)
+          JSON.parse(response.body)
+        rescue OpenSSL::SSL::SSLError => e
+          raise ::MCP::Client::RequestHandlerError.new(
+            "SSL error connecting to MCP server #{@uri.host}: #{e.message}",
+            extract_method_params(request),
+            error_type: :internal_error,
+            original_error: e,
+          )
+        rescue Net::HTTPError, Net::OpenTimeout, Net::ReadTimeout, SocketError, Errno::ECONNREFUSED => e
+          raise ::MCP::Client::RequestHandlerError.new(
+            "Connection error to MCP server #{@uri.host}: #{e.message}",
+            extract_method_params(request),
+            error_type: :internal_error,
+            original_error: e,
+          )
+        end
+
+        private
+
+        # Build a Net::HTTP client with SSL configuration.
+        #
+        # @return [Net::HTTP]
+        def build_http
+          http = Net::HTTP.new(@uri.host, @uri.port)
+
+          if @uri.scheme == "https"
+            http.use_ssl = true
+            http.verify_mode = @ssl_verify ? OpenSSL::SSL::VERIFY_PEER : OpenSSL::SSL::VERIFY_NONE
+          end
+
+          http
+        end
+
+        # Build the POST request with JSON body and headers.
+        #
+        # @param body [Hash] Request body
+        # @return [Net::HTTP::Post]
+        def build_post(body)
+          post = Net::HTTP::Post.new(@uri.request_uri)
+          post["Content-Type"] = "application/json"
+          post["Accept"] = "application/json"
+
+          @headers.each { |k, v| post[k] = v }
+          post.body = JSON.generate(body)
+          post
+        end
+
+        # Extract method and params from request for error reporting.
+        #
+        # @param request [Hash]
+        # @return [Hash]
+        def extract_method_params(request)
+          { method: request[:method] || request["method"], params: request[:params] || request["params"] }
+        end
+      end
+    end
+  end
+end

data/lib/swarm_sdk/v3/mcp/stdio_transport.rb

@@ -0,0 +1,135 @@
+# frozen_string_literal: true
+
+module SwarmSDK
+  module V3
+    module MCP
+      # Subprocess-based JSON-RPC transport for MCP stdio servers
+      #
+      # Spawns a child process and communicates via JSON-RPC over stdin/stdout.
+      # Performs the MCP protocol initialization handshake automatically.
+      # Implements the same duck type as `MCP::Client::HTTP` (`send_request(request:)`).
+      #
+      # @example
+      #   transport = StdioTransport.new(
+      #     command: "npx",
+      #     args: ["-y", "@modelcontextprotocol/server-filesystem", "/tmp"],
+      #   )
+      #   response = transport.send_request(request: { jsonrpc: "2.0", id: "1", method: "tools/list" })
+      #   transport.close
+      class StdioTransport
+        PROTOCOL_VERSION = "2024-11-05"
+
+        # Create a new stdio transport, spawn the subprocess, and initialize the MCP session
+        #
+        # @param command [String] Command to execute
+        # @param args [Array<String>] Command arguments
+        # @param env [Hash<String, String>] Environment variables
+        #
+        # @raise [McpError] If the subprocess fails to start or initialization fails
+        #
+        # @example
+        #   StdioTransport.new(command: "node", args: ["server.js"])
+        def initialize(command:, args: [], env: {})
+          @stdin, @stdout, @wait_thread = Open3.popen2(
+            env.transform_keys(&:to_s), command, *args
+          )
+          initialize_session
+        end
+
+        # Send a JSON-RPC request and return the response
+        #
+        # Writes the request as a single JSON line to the subprocess stdin,
+        # then reads lines from stdout until a response with an "id" field
+        # is received (skipping server-initiated notifications).
+        #
+        # @param request [Hash] Complete JSON-RPC 2.0 request object
+        # @return [Hash] Parsed JSON-RPC response
+        # @raise [McpError] If the subprocess exits unexpectedly
+        #
+        # @example
+        #   response = transport.send_request(
+        #     request: { jsonrpc: "2.0", id: "1", method: "tools/list" }
+        #   )
+        def send_request(request:)
+          @stdin.puts(JSON.generate(request))
+          @stdin.flush
+          read_response
+        end
+
+        # Gracefully shut down the subprocess
+        #
+        # Closes stdin to signal the subprocess to exit, then waits
+        # briefly before sending SIGTERM if still alive.
+        #
+        # @return [void]
+        def close
+          @stdin&.close unless @stdin&.closed?
+          return unless @wait_thread
+
+          # Give the process a moment to exit gracefully
+          return if @wait_thread.join(2)
+
+          # Force terminate if still running
+          Process.kill("TERM", @wait_thread.pid) if @wait_thread.alive?
+          @wait_thread.join(2)
+        rescue Errno::ESRCH, Errno::ECHILD
+          # Process already exited
+        end
+
+        # Whether the subprocess is still running
+        #
+        # @return [Boolean]
+        def alive?
+          @wait_thread&.alive? || false
+        end
+
+        private
+
+        # Perform the MCP protocol initialization handshake
+        #
+        # Sends `initialize` request followed by `notifications/initialized`
+        # notification, as required by the MCP specification for stdio servers.
+        #
+        # @return [void]
+        # @raise [McpError] If the server returns an error during initialization
+        def initialize_session
+          response = send_request(request: {
+            jsonrpc: "2.0",
+            id: SecureRandom.uuid,
+            method: "initialize",
+            params: {
+              protocolVersion: PROTOCOL_VERSION,
+              capabilities: {},
+              clientInfo: { name: "swarm_sdk", version: "1.0" },
+            },
+          })
+
+          if response.key?("error")
+            raise McpError, "MCP initialization failed: #{response["error"]["message"]}"
+          end
+
+          # Send initialized notification (no id — it's a notification, not a request)
+          @stdin.puts(JSON.generate({ jsonrpc: "2.0", method: "notifications/initialized" }))
+          @stdin.flush
+        end
+
+        # Read the next JSON-RPC response from the subprocess
+        #
+        # Skips server-initiated notifications (messages without an "id" field)
+        # and returns the first response that has one.
+        #
+        # @return [Hash] Parsed JSON-RPC response
+        # @raise [McpError] If the subprocess exits (stdout returns nil)
+        def read_response
+          loop do
+            line = @stdout.gets
+            raise McpError, "MCP server process exited unexpectedly" if line.nil?
+
+            parsed = JSON.parse(line)
+            return parsed if parsed.key?("id")
+          end
+        end
+      end
+    end
+  end
+end

data/lib/swarm_sdk/v3/mcp/tool_proxy.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+module SwarmSDK
+  module V3
+    module MCP
+      # Bridges MCP tools into RubyLLM::Tool instances
+      #
+      # Creates anonymous RubyLLM::Tool subclasses that delegate execution
+      # to an MCP connector. Each MCP tool gets its own Tool subclass with
+      # the correct name, description, and JSON Schema parameters.
+      #
+      # @example
+      #   mcp_tool = connector.available_tools.first
+      #   ruby_llm_tool = ToolProxy.create(mcp_tool, connector)
+      #   ruby_llm_tool.name        #=> "echo"
+      #   ruby_llm_tool.call(message: "hi")  #=> "hi"
+      module ToolProxy
+        extend self
+
+        # Create a RubyLLM::Tool instance from an MCP tool
+        #
+        # Builds an anonymous subclass of RubyLLM::Tool with the MCP tool's
+        # name, description, and input schema. The `execute` method delegates
+        # to the connector's `call_tool`.
+        #
+        # @param mcp_tool [MCP::Client::Tool] MCP tool descriptor
+        # @param connector [Connector] MCP connector for tool execution
+        # @return [RubyLLM::Tool] Instantiated tool ready for use with RubyLLM::Chat
+        #
+        # @example
+        #   tool = ToolProxy.create(mcp_tool, connector)
+        #   chat.with_tools(tool)
+        def create(mcp_tool, connector)
+          tool_name = mcp_tool.name
+          tool_desc = mcp_tool.description || "MCP tool: #{tool_name}"
+          schema = mcp_tool.input_schema
+
+          klass = Class.new(RubyLLM::Tool) do
+            description tool_desc
+            params schema if schema
+          end
+
+          klass.define_method(:name) { tool_name }
+          klass.define_method(:execute) do |**args|
+            connector.call_tool(tool_name, **args)
+          end
+
+          klass.new
+        end
+      end
+    end
+  end
+end

data/lib/swarm_sdk/v3/memory/adapters/base.rb

@@ -0,0 +1,297 @@
+# frozen_string_literal: true
+
+module SwarmSDK
+  module V3
+    module Memory
+      module Adapters
+        # Abstract adapter interface for memory storage
+        #
+        # Defines the contract that all storage backends must satisfy.
+        # SDK users can implement custom adapters (e.g., Postgres + pgvector)
+        # by subclassing this and implementing all methods.
+        #
+        # ## Why adapters own vector operations
+        #
+        # The SDK code (ContextBuilder, IngestionPipeline, Retriever, etc.)
+        # **never** computes similarity directly. All vector math goes through
+        # the adapter so that each backend can use its native capabilities:
+        #
+        # - **FilesystemAdapter**: FAISS for indexed search, Ruby cosine
+        #   similarity (via {VectorUtils}) for pairwise comparison
+        # - **PgvectorAdapter**: `<=>` operator for both indexed and pairwise
+        # - **Qdrant/Pinecone**: Client API for all similarity operations
+        #
+        # There are two distinct vector operations:
+        #
+        # ### `vector_search(embedding, top_k:, threshold:)`
+        # Searches an **index** of card embeddings. Used by the Retriever
+        # for semantic search and by the Consolidator for dedup/conflict
+        # detection. The filesystem adapter uses FAISS for this. A pgvector
+        # adapter would use `ORDER BY embedding <=> query LIMIT k`.
+        #
+        # ### `similarity(embedding_a, embedding_b)`
+        # Computes pairwise similarity between **two arbitrary vectors**.
+        # This is needed when comparing vectors that aren't both in the
+        # card index — for example:
+        # - Card embedding vs. cluster centroid (clusters aren't indexed)
+        # - Two cards already in hand during dedup in the ContextBuilder
+        # - Query embedding vs. candidate cards in the exploration sprinkle
+        #
+        # FAISS cannot do this — it only supports `index.search([query], k)`,
+        # which searches vectors that have been `add`'d to the index.
+        #
+        # For in-memory adapters, include {VectorUtils} to get a default
+        # Ruby implementation. For database adapters, implement it using
+        # your backend's native operator.
+        #
+        # ## Implementing a custom adapter
+        #
+        # Subclass {Base} and implement every method. Methods that raise
+        # {NotImplementedError} are required. For a starting point, see
+        # {FilesystemAdapter} which implements the full interface.
+        #
+        # If your adapter doesn't have native vector math, include
+        # {VectorUtils} to get a default {#similarity} implementation:
+        #
+        #   class MyAdapter < Base
+        #     include VectorUtils  # provides similarity()
+        #     # ... implement the rest
+        #   end
+        #
+        # @example Postgres + pgvector adapter
+        #   class PgvectorAdapter < SwarmSDK::V3::Memory::Adapters::Base
+        #     def initialize(connection_string)
+        #       @db = PG.connect(connection_string)
+        #     end
+        #
+        #     def write_card(card)
+        #       @db.exec_params(
+        #         "INSERT INTO cards (id, text, type, embedding, ...) VALUES ($1, $2, $3, $4, ...)",
+        #         [card.id, card.text, card.type.to_s, card.embedding.to_s]
+        #       )
+        #     end
+        #
+        #     def similarity(embedding_a, embedding_b)
+        #       result = @db.exec_params(
+        #         "SELECT 1 - ($1::vector <=> $2::vector) AS sim",
+        #         [embedding_a.to_s, embedding_b.to_s]
+        #       )
+        #       result[0]["sim"].to_f
+        #     end
+        #
+        #     def vector_search(embedding, top_k:, threshold: 0.0)
+        #       rows = @db.exec_params(
+        #         "SELECT id, 1 - (embedding <=> $1::vector) AS similarity " \
+        #         "FROM cards WHERE 1 - (embedding <=> $1::vector) >= $3 " \
+        #         "ORDER BY embedding <=> $1::vector LIMIT $2",
+        #         [embedding.to_s, top_k, threshold]
+        #       )
+        #       rows.map { |r| { id: r["id"], similarity: r["similarity"].to_f } }
+        #     end
+        #
+        #     # ... implement remaining methods
+        #   end
+        class Base
+          # --- Card CRUD ---
+
+          # Write a card to storage (insert or update)
+          #
+          # @param card [Card] Card to store
+          # @return [void]
+          def write_card(card)
+            raise NotImplementedError, "#{self.class}#write_card not implemented"
+          end
+
+          # Read a card by ID
+          #
+          # @param id [String] Card ID
+          # @return [Card, nil] Card or nil if not found
+          def read_card(id)
+            raise NotImplementedError, "#{self.class}#read_card not implemented"
+          end
+
+          # Delete a card by ID
+          #
+          # @param id [String] Card ID
+          # @return [void]
+          def delete_card(id)
+            raise NotImplementedError, "#{self.class}#delete_card not implemented"
+          end
+
+          # List cards, optionally filtered by ID prefix
+          #
+          # @param prefix [String, nil] ID prefix filter
+          # @return [Array<Card>]
+          def list_cards(prefix: nil)
+            raise NotImplementedError, "#{self.class}#list_cards not implemented"
+          end
+
+          # List cards eligible for compression
+          #
+          # Default implementation loads all cards and filters in Ruby.
+          # Database adapters should override with a server-side query.
+          #
+          # @param max_level [Integer] Maximum compression level to include (default: 3)
+          # @return [Array<Card>] Cards eligible for compression
+          #
+          # @example Override in a Postgres adapter
+          #   def list_cards_for_compression(max_level: 3)
+          #     @db.exec_params("SELECT * FROM cards WHERE compression_level <= $1", [max_level])
+          #       .map { |row| card_from_row(row) }
+          #   end
+          def list_cards_for_compression(max_level: 3)
+            list_cards.select { |c| c.compression_level <= max_level }
+          end
+
+          # --- Edge CRUD ---
+
+          # Write an edge to storage
+          #
+          # @param edge [Edge] Edge to store
+          # @return [void]
+          def write_edge(edge)
+            raise NotImplementedError, "#{self.class}#write_edge not implemented"
+          end
+
+          # Get edges for a card (as source or target)
+          #
+          # @param card_id [String] Card ID
+          # @param type [Symbol, nil] Filter by edge type
+          # @return [Array<Edge>]
+          def edges_for(card_id, type: nil)
+            raise NotImplementedError, "#{self.class}#edges_for not implemented"
+          end
+
+          # Delete all edges involving a card (as source or target)
+          #
+          # @param card_id [String] Card ID
+          # @return [void]
+          def delete_edges_for(card_id)
+            raise NotImplementedError, "#{self.class}#delete_edges_for not implemented"
+          end
+
+          # --- Cluster CRUD ---
+
+          # Write a cluster to storage (insert or update)
+          #
+          # @param cluster [Cluster] Cluster to store
+          # @return [void]
+          def write_cluster(cluster)
+            raise NotImplementedError, "#{self.class}#write_cluster not implemented"
+          end
+
+          # Read a cluster by ID
+          #
+          # @param id [String] Cluster ID
+          # @return [Cluster, nil]
+          def read_cluster(id)
+            raise NotImplementedError, "#{self.class}#read_cluster not implemented"
+          end
+
+          # List all clusters
+          #
+          # @return [Array<Cluster>]
+          def list_clusters
+            raise NotImplementedError, "#{self.class}#list_clusters not implemented"
+          end
+
+          # --- Vector Operations ---
+          #
+          # These two methods are the adapter's responsibility because
+          # different backends compute similarity differently:
+          #
+          # - FAISS: L2-normalized inner product (IndexFlatIP)
+          # - pgvector: `<=>` cosine distance operator
+          # - Qdrant: client.search() with scoring
+          #
+          # The SDK never computes cosine similarity directly. It always
+          # calls adapter.similarity() or adapter.vector_search().
+
+          # Compute similarity between two arbitrary embedding vectors
+          #
+          # Used by the SDK for pairwise comparisons where an indexed
+          # search doesn't apply:
+          # - Card vs. cluster centroid (cluster assignment in IngestionPipeline)
+          # - Card vs. card (deduplication in ContextBuilder)
+          # - Query vs. card (exploration sprinkle in ContextBuilder)
+          #
+          # For in-memory adapters, include {VectorUtils} to get a default
+          # cosine similarity implementation. For database adapters,
+          # implement using your backend's native operator.
+          #
+          # @param embedding_a [Array<Float>] First embedding vector
+          # @param embedding_b [Array<Float>] Second embedding vector
+          # @return [Float] Similarity score (-1.0 to 1.0 for cosine)
+          #
+          # @see VectorUtils Default in-memory implementation
+          def similarity(embedding_a, embedding_b)
+            raise NotImplementedError, "#{self.class}#similarity not implemented"
+          end
+
+          # Search the card embedding index for similar vectors
+          #
+          # Used by the SDK for top-k retrieval:
+          # - Retriever: semantic search for relevant cards
+          # - Consolidator: finding near-duplicates and conflicts
+          #
+          # This searches an **index** of card embeddings. The adapter
+          # decides how to implement the index (FAISS, pgvector, etc.).
+          #
+          # @param embedding [Array<Float>] Query embedding
+          # @param top_k [Integer] Maximum number of results
+          # @param threshold [Float] Minimum similarity to include
+          # @return [Array<Hash>] Array of `{ id: String, similarity: Float }`
+          def vector_search(embedding, top_k:, threshold: 0.0)
+            raise NotImplementedError, "#{self.class}#vector_search not implemented"
+          end
+
+          # Rebuild the vector index from all stored cards
+          #
+          # Called when the index needs to be reconstructed, e.g., after
+          # bulk imports or if the index file is missing/corrupted.
+          #
+          # @return [void]
+          def rebuild_index
+            raise NotImplementedError, "#{self.class}#rebuild_index not implemented"
+          end
+
+          # --- Transactions ---
+
+          # Execute a block within a transaction
+          #
+          # The default implementation is a no-op pass-through. Adapters
+          # with transactional backends (e.g., SQLite, Postgres) should
+          # override this to wrap the block in a real transaction.
+          #
+          # @yield Block to execute within the transaction
+          # @return [Object] Return value of the block
+          #
+          # @example
+          #   adapter.transaction do
+          #     adapter.write_card(card)
+          #     adapter.write_edge(edge)
+          #   end
+          def transaction
+            yield
+          end
+
+          # --- Persistence ---
+
+          # Save all in-memory state to durable storage
+          #
+          # @return [void]
+          def save
+            raise NotImplementedError, "#{self.class}#save not implemented"
+          end
+
+          # Load state from durable storage into memory
+          #
+          # @return [void]
+          def load
+            raise NotImplementedError, "#{self.class}#load not implemented"
+          end
+        end
+      end
+    end
+  end
+end