swarm_sdk 2.7.14 → 3.0.0.alpha2

data/lib/swarm_sdk/v3/memory/adapters/sqlite_adapter.rb

@@ -0,0 +1,507 @@
+# frozen_string_literal: true
+
+module SwarmSDK
+  module V3
+    module Memory
+      module Adapters
+        # SQLite-based storage adapter with sqlite-vec vector index
+        #
+        # Stores cards, edges, clusters, and vector embeddings in a single
+        # SQLite database for durable, transactional, multi-process-safe storage.
+        # Vector search uses sqlite-vec's vec0 virtual table with cosine
+        # distance, eliminating external FAISS files and ID mapping tables.
+        #
+        # Database structure:
+        #   .swarm/memory/
+        #   └── memory.db        (SQLite database with vec0 virtual table)
+        #
+        # ## Why UPSERT instead of INSERT OR REPLACE
+        #
+        # `INSERT OR REPLACE` deletes the old row then inserts a new one.
+        # With `ON DELETE CASCADE` on edges, this silently destroys all
+        # edges for the card on every update (including `record_access!`).
+        # `ON CONFLICT DO UPDATE` updates in-place without triggering CASCADE.
+        #
+        # @example
+        #   adapter = SqliteAdapter.new("/path/to/.swarm/memory")
+        #   adapter.load
+        #   adapter.write_card(card)
+        #   adapter.save
+        class SqliteAdapter < Base
+          include VectorUtils
+
+          # Embedding dimensions for MiniLM-L6-v2 model
+          EMBEDDING_DIMENSIONS = 384
+
+          # @return [String] Root directory for storage
+          attr_reader :directory
+
+          # SQL schema for the memory database
+          SCHEMA_SQL = <<~SQL
+            CREATE TABLE IF NOT EXISTS cards (
+              id TEXT PRIMARY KEY,
+              text TEXT NOT NULL,
+              type TEXT NOT NULL,
+              entities TEXT NOT NULL DEFAULT '[]',
+              source_turn_ids TEXT NOT NULL DEFAULT '[]',
+              embedding TEXT,
+              importance REAL NOT NULL DEFAULT 0.5,
+              confidence REAL NOT NULL DEFAULT 1.0,
+              access_count INTEGER NOT NULL DEFAULT 0,
+              last_accessed TEXT,
+              dwell REAL NOT NULL DEFAULT 0.0,
+              compression_level INTEGER NOT NULL DEFAULT 0,
+              canonical_id TEXT,
+              created_at TEXT NOT NULL,
+              updated_at TEXT NOT NULL
+            );
+
+            CREATE TABLE IF NOT EXISTS edges (
+              id INTEGER PRIMARY KEY AUTOINCREMENT,
+              from_id TEXT NOT NULL REFERENCES cards(id) ON DELETE CASCADE,
+              to_id TEXT NOT NULL REFERENCES cards(id) ON DELETE CASCADE,
+              type TEXT NOT NULL,
+              weight REAL NOT NULL DEFAULT 1.0,
+              created_at TEXT NOT NULL
+            );
+
+            CREATE INDEX IF NOT EXISTS idx_edges_from_id ON edges(from_id);
+            CREATE INDEX IF NOT EXISTS idx_edges_to_id ON edges(to_id);
+
+            CREATE TABLE IF NOT EXISTS clusters (
+              id TEXT PRIMARY KEY,
+              title TEXT NOT NULL,
+              embedding TEXT,
+              rolling_summary TEXT NOT NULL DEFAULT '',
+              decision_log TEXT NOT NULL DEFAULT '[]',
+              key_entities TEXT NOT NULL DEFAULT '[]',
+              card_ids TEXT NOT NULL DEFAULT '[]',
+              created_at TEXT NOT NULL,
+              updated_at TEXT NOT NULL
+            );
+          SQL
+
+          # @param directory [String] Root directory for memory storage
+          def initialize(directory)
+            super()
+            @directory = File.expand_path(directory)
+            @db = nil
+          end
+
+          # --- Card CRUD ---
+
+          # @param card [Card] Card to write (insert or update)
+          # @return [void]
+          def write_card(card)
+            @db.execute(<<~SQL, card_params(card))
+              INSERT INTO cards (id, text, type, entities, source_turn_ids, embedding,
+                                 importance, confidence, access_count, last_accessed,
+                                 dwell, compression_level, canonical_id, created_at, updated_at)
+              VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+              ON CONFLICT(id) DO UPDATE SET
+                text = excluded.text,
+                type = excluded.type,
+                entities = excluded.entities,
+                source_turn_ids = excluded.source_turn_ids,
+                embedding = excluded.embedding,
+                importance = excluded.importance,
+                confidence = excluded.confidence,
+                access_count = excluded.access_count,
+                last_accessed = excluded.last_accessed,
+                dwell = excluded.dwell,
+                compression_level = excluded.compression_level,
+                canonical_id = excluded.canonical_id,
+                updated_at = excluded.updated_at
+            SQL
+            upsert_vec_embedding(card.id, card.embedding) if card.embedding
+          end
+
+          # @param id [String] Card ID
+          # @return [Card, nil]
+          def read_card(id)
+            row = @db.execute("SELECT * FROM cards WHERE id = ?", [id]).first
+            return unless row
+
+            card_from_row(row)
+          end
+
+          # @param id [String] Card ID
+          # @return [void]
+          def delete_card(id)
+            @db.execute("DELETE FROM cards WHERE id = ?", [id])
+            @db.execute("DELETE FROM vec_cards WHERE card_id = ?", [id])
+          end
+
+          # @param prefix [String, nil] ID prefix filter
+          # @return [Array<Card>]
+          def list_cards(prefix: nil)
+            rows = if prefix
+              @db.execute("SELECT * FROM cards WHERE id LIKE ?", ["#{prefix}%"])
+            else
+              @db.execute("SELECT * FROM cards")
+            end
+            rows.map { |row| card_from_row(row) }
+          end
+
+          # @param max_level [Integer] Maximum compression level to include
+          # @return [Array<Card>] Cards eligible for compression
+          def list_cards_for_compression(max_level: 3)
+            rows = @db.execute("SELECT * FROM cards WHERE compression_level <= ?", [max_level])
+            rows.map { |row| card_from_row(row) }
+          end
+
+          # --- Edge CRUD ---
+
+          # @param edge [Edge] Edge to write
+          # @return [void]
+          def write_edge(edge)
+            @db.execute(
+              "INSERT INTO edges (from_id, to_id, type, weight, created_at) VALUES (?, ?, ?, ?, ?)",
+              [edge.from_id, edge.to_id, edge.type.to_s, edge.weight, edge.created_at.iso8601],
+            )
+          end
+
+          # @param card_id [String] Card ID
+          # @param type [Symbol, nil] Filter by edge type
+          # @return [Array<Edge>]
+          def edges_for(card_id, type: nil)
+            rows = if type
+              @db.execute(
+                "SELECT * FROM edges WHERE (from_id = ? OR to_id = ?) AND type = ?",
+                [card_id, card_id, type.to_s],
+              )
+            else
+              @db.execute(
+                "SELECT * FROM edges WHERE from_id = ? OR to_id = ?",
+                [card_id, card_id],
+              )
+            end
+            rows.map { |row| edge_from_row(row) }
+          end
+
+          # @param card_id [String] Card ID
+          # @return [void]
+          def delete_edges_for(card_id)
+            @db.execute("DELETE FROM edges WHERE from_id = ? OR to_id = ?", [card_id, card_id])
+          end
+
+          # --- Cluster CRUD ---
+
+          # @param cluster [Cluster] Cluster to write (insert or update)
+          # @return [void]
+          def write_cluster(cluster)
+            @db.execute(<<~SQL, cluster_params(cluster))
+              INSERT INTO clusters (id, title, embedding, rolling_summary, decision_log,
+                                    key_entities, card_ids, created_at, updated_at)
+              VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
+              ON CONFLICT(id) DO UPDATE SET
+                title = excluded.title,
+                embedding = excluded.embedding,
+                rolling_summary = excluded.rolling_summary,
+                decision_log = excluded.decision_log,
+                key_entities = excluded.key_entities,
+                card_ids = excluded.card_ids,
+                updated_at = excluded.updated_at
+            SQL
+          end
+
+          # @param id [String] Cluster ID
+          # @return [Cluster, nil]
+          def read_cluster(id)
+            row = @db.execute("SELECT * FROM clusters WHERE id = ?", [id]).first
+            return unless row
+
+            cluster_from_row(row)
+          end
+
+          # @return [Array<Cluster>]
+          def list_clusters
+            rows = @db.execute("SELECT * FROM clusters")
+            rows.map { |row| cluster_from_row(row) }
+          end
+
+          # --- Vector Search ---
+
+          # Search the vec0 virtual table for similar vectors
+          #
+          # sqlite-vec's cosine distance returns values where 0 = identical
+          # and 2 = opposite. We convert to similarity: `1.0 - distance`.
+          #
+          # @param embedding [Array<Float>] Query embedding
+          # @param top_k [Integer] Maximum number of results
+          # @param threshold [Float] Minimum cosine similarity to include
+          # @return [Array<Hash>] Array of `{ id: String, similarity: Float }`
+          #
+          # @example
+          #   results = adapter.vector_search(query_embedding, top_k: 5, threshold: 0.7)
+          #   results.each { |r| puts "#{r[:id]}: #{r[:similarity]}" }
+          def vector_search(embedding, top_k:, threshold: 0.0)
+            rows = @db.execute(
+              "SELECT card_id, distance FROM vec_cards WHERE embedding MATCH ? AND k = ?",
+              [embedding.pack("f*"), top_k],
+            )
+
+            results = []
+            rows.each do |row|
+              sim = 1.0 - row["distance"]
+              next if sim < threshold
+
+              results << { id: row["card_id"], similarity: sim }
+            end
+            results
+          end
+
+          # Rebuild the vec0 index from all stored card embeddings
+          #
+          # Drops and recreates the vec_cards virtual table, then re-inserts
+          # all card embeddings from the cards table.
+          #
+          # @return [void]
+          def rebuild_index
+            @db.execute("DROP TABLE IF EXISTS vec_cards")
+            create_vec_table!
+            @db.execute("SELECT id, embedding FROM cards WHERE embedding IS NOT NULL").each do |row|
+              embedding = JSON.parse(row["embedding"])
+              @db.execute(
+                "INSERT INTO vec_cards(card_id, embedding) VALUES (?, ?)",
+                [row["id"], embedding.pack("f*")],
+              )
+            end
+          end
+
+          # --- Transactions ---
+
+          # Execute a block within a SQLite transaction
+          #
+          # Uses IMMEDIATE mode to acquire a write lock at the start,
+          # preventing deadlocks in WAL mode.
+          #
+          # @yield Block to execute within the transaction
+          # @return [Object] Return value of the block
+          def transaction(&block)
+            @db.transaction(:immediate, &block)
+          end
+
+          # --- Persistence ---
+
+          # Ensure the storage directory exists
+          #
+          # SQLite with WAL mode is already durable — no external
+          # index files need saving.
+          #
+          # @return [void]
+          def save
+            FileUtils.mkdir_p(@directory)
+          end
+
+          # Load state from SQLite and initialize vec0 index
+          #
+          # Opens the database, creates schema if needed, loads the
+          # sqlite-vec extension, and creates the vec0 virtual table.
+          # If vec_cards is empty but cards have embeddings, auto-rebuilds
+          # the index (handles FAISS migration and first-run).
+          #
+          # @return [void]
+          def load
+            load_sqlite!
+            auto_rebuild_vec_if_needed!
+          end
+
+          private
+
+          # Open SQLite database, initialize schema, and load sqlite-vec
+          #
+          # @return [void]
+          def load_sqlite!
+            begin
+              require "sqlite3"
+            rescue LoadError
+              raise LoadError,
+                "sqlite3 gem is required for SqliteAdapter. Add `gem 'sqlite3'` to your Gemfile."
+            end
+
+            begin
+              require "sqlite_vec"
+            rescue LoadError
+              raise LoadError,
+                "sqlite_vec gem is required for SqliteAdapter. Add `gem 'sqlite_vec'` to your Gemfile."
+            end
+
+            FileUtils.mkdir_p(@directory)
+            db_path = File.join(@directory, "memory.db")
+            @db = SQLite3::Database.new(db_path)
+            @db.results_as_hash = true
+            @db.execute("PRAGMA journal_mode=WAL")
+            @db.execute("PRAGMA foreign_keys=ON")
+            @db.execute("PRAGMA busy_timeout=5000")
+            @db.execute_batch(SCHEMA_SQL)
+
+            @db.enable_load_extension(true)
+            SqliteVec.load(@db)
+            @db.enable_load_extension(false)
+
+            create_vec_table!
+          end
+
+          # Create the vec0 virtual table for vector search
+          #
+          # @return [void]
+          def create_vec_table!
+            @db.execute(<<~SQL)
+              CREATE VIRTUAL TABLE IF NOT EXISTS vec_cards USING vec0(
+                card_id TEXT PRIMARY KEY,
+                embedding FLOAT[#{EMBEDDING_DIMENSIONS}] DISTANCE_METRIC=cosine
+              )
+            SQL
+          end
+
+          # Upsert a card's embedding into the vec0 virtual table
+          #
+          # vec0 does not support ON CONFLICT, so we DELETE then INSERT.
+          #
+          # @param card_id [String] Card ID
+          # @param embedding [Array<Float>] Embedding vector
+          # @return [void]
+          def upsert_vec_embedding(card_id, embedding)
+            @db.execute("DELETE FROM vec_cards WHERE card_id = ?", [card_id])
+            @db.execute(
+              "INSERT INTO vec_cards(card_id, embedding) VALUES (?, ?)",
+              [card_id, embedding.pack("f*")],
+            )
+          end
+
+          # Auto-rebuild vec0 index if cards have embeddings but vec_cards is empty
+          #
+          # Handles migration from FAISS and first-run scenarios.
+          #
+          # @return [void]
+          def auto_rebuild_vec_if_needed!
+            vec_count = @db.execute("SELECT COUNT(*) AS cnt FROM vec_cards").first["cnt"]
+            return if vec_count > 0
+
+            has_embeddings = @db.execute(
+              "SELECT COUNT(*) AS cnt FROM cards WHERE embedding IS NOT NULL",
+            ).first["cnt"] > 0
+            rebuild_index if has_embeddings
+          end
+
+          # --- Card Serialization ---
+
+          # Convert a Card to bind parameter array
+          #
+          # @param card [Card] Card to serialize
+          # @return [Array] Bind values for INSERT/UPSERT
+          def card_params(card)
+            [
+              card.id,
+              card.text,
+              card.type.to_s,
+              JSON.generate(card.entities),
+              JSON.generate(card.source_turn_ids),
+              card.embedding ? JSON.generate(card.embedding) : nil,
+              card.importance,
+              card.confidence,
+              card.access_count,
+              card.last_accessed&.iso8601,
+              card.dwell,
+              card.compression_level,
+              card.canonical_id,
+              card.created_at.iso8601,
+              card.updated_at.iso8601,
+            ]
+          end
+
+          # Convert a SQLite row hash to a Card
+          #
+          # @param row [Hash] Row from SQLite (results_as_hash mode)
+          # @return [Card]
+          def card_from_row(row)
+            Card.new(
+              id: row["id"],
+              text: row["text"],
+              type: row["type"]&.to_sym || :fact,
+              entities: parse_json_array(row["entities"]),
+              source_turn_ids: parse_json_array(row["source_turn_ids"]),
+              embedding: row["embedding"] ? JSON.parse(row["embedding"]) : nil,
+              importance: row["importance"] || 0.5,
+              confidence: row["confidence"] || 1.0,
+              access_count: row["access_count"] || 0,
+              last_accessed: row["last_accessed"] ? Time.parse(row["last_accessed"]) : nil,
+              dwell: row["dwell"] || 0.0,
+              compression_level: row["compression_level"] || 0,
+              canonical_id: row["canonical_id"],
+              created_at: row["created_at"] ? Time.parse(row["created_at"]) : nil,
+              updated_at: row["updated_at"] ? Time.parse(row["updated_at"]) : nil,
+            )
+          end
+
+          # --- Edge Serialization ---
+
+          # Convert a SQLite row hash to an Edge
+          #
+          # @param row [Hash] Row from SQLite
+          # @return [Edge]
+          def edge_from_row(row)
+            Edge.new(
+              from_id: row["from_id"],
+              to_id: row["to_id"],
+              type: row["type"]&.to_sym,
+              weight: row["weight"] || 1.0,
+              created_at: row["created_at"] ? Time.parse(row["created_at"]) : nil,
+            )
+          end
+
+          # --- Cluster Serialization ---
+
+          # Convert a Cluster to bind parameter array
+          #
+          # @param cluster [Cluster] Cluster to serialize
+          # @return [Array] Bind values for INSERT/UPSERT
+          def cluster_params(cluster)
+            [
+              cluster.id,
+              cluster.title,
+              cluster.embedding ? JSON.generate(cluster.embedding) : nil,
+              cluster.rolling_summary,
+              JSON.generate(cluster.decision_log),
+              JSON.generate(cluster.key_entities),
+              JSON.generate(cluster.card_ids),
+              cluster.created_at.iso8601,
+              cluster.updated_at.iso8601,
+            ]
+          end
+
+          # Convert a SQLite row hash to a Cluster
+          #
+          # @param row [Hash] Row from SQLite
+          # @return [Cluster]
+          def cluster_from_row(row)
+            Cluster.new(
+              id: row["id"],
+              title: row["title"],
+              embedding: row["embedding"] ? JSON.parse(row["embedding"]) : nil,
+              rolling_summary: row["rolling_summary"] || "",
+              decision_log: parse_json_array(row["decision_log"]),
+              key_entities: parse_json_array(row["key_entities"]),
+              card_ids: parse_json_array(row["card_ids"]),
+              created_at: row["created_at"] ? Time.parse(row["created_at"]) : nil,
+              updated_at: row["updated_at"] ? Time.parse(row["updated_at"]) : nil,
+            )
+          end
+
+          # Parse a JSON array string, returning empty array on nil/error
+          #
+          # @param json_str [String, nil] JSON string
+          # @return [Array]
+          def parse_json_array(json_str)
+            return [] if json_str.nil? || json_str.empty?
+
+            JSON.parse(json_str)
+          rescue JSON::ParserError
+            []
+          end
+        end
+      end
+    end
+  end
+end

data/lib/swarm_sdk/v3/memory/adapters/vector_utils.rb

@@ -0,0 +1,88 @@
+# frozen_string_literal: true
+
+module SwarmSDK
+  module V3
+    module Memory
+      module Adapters
+        # Default in-memory vector math for adapters
+        #
+        # Provides a pure-Ruby cosine similarity implementation that adapters
+        # can include when they don't have a native vector engine. The
+        # {FilesystemAdapter} includes this module because its FAISS index
+        # only handles indexed top-k search — it cannot compute pairwise
+        # similarity between two arbitrary vectors.
+        #
+        # **When to use this module:**
+        # Include it if your adapter stores embeddings as Ruby arrays and
+        # doesn't have a native way to compute pairwise similarity. This is
+        # the common case for file-based, SQLite, or simple database adapters.
+        #
+        # **When NOT to use this module:**
+        # If your storage backend has native vector operations, implement
+        # {Base#similarity} directly using the backend's operator. Examples:
+        #
+        # - **pgvector**: `SELECT 1 - (a <=> b)` (cosine distance operator)
+        # - **Qdrant/Pinecone**: Use the client's similarity API
+        # - **Redis with RediSearch**: `FT.SEARCH` with vector scoring
+        #
+        # In those cases you skip this module entirely and write your own
+        # `similarity` method that delegates to the backend.
+        #
+        # @example Including in a custom adapter
+        #   class SqliteAdapter < SwarmSDK::V3::Memory::Adapters::Base
+        #     include SwarmSDK::V3::Memory::Adapters::VectorUtils
+        #
+        #     # similarity() is now available via VectorUtils
+        #     # Override vector_search with your own indexed search
+        #   end
+        #
+        # @example NOT including — pgvector adapter
+        #   class PgvectorAdapter < SwarmSDK::V3::Memory::Adapters::Base
+        #     def similarity(embedding_a, embedding_b)
+        #       # Compute server-side using pgvector's cosine distance
+        #       result = @db.exec_params(
+        #         "SELECT 1 - ($1::vector <=> $2::vector) AS sim",
+        #         [embedding_a.to_s, embedding_b.to_s]
+        #       )
+        #       result[0]["sim"].to_f
+        #     end
+        #   end
+        module VectorUtils
+          # Compute cosine similarity between two embedding vectors
+          #
+          # This is a pure-Ruby implementation suitable for in-memory use.
+          # It computes: dot(a, b) / (||a|| * ||b||)
+          #
+          # For production adapters with native vector engines, override
+          # this method to delegate to the backend instead.
+          #
+          # @param embedding_a [Array<Float>] First embedding vector
+          # @param embedding_b [Array<Float>] Second embedding vector
+          # @return [Float] Cosine similarity (-1.0 to 1.0)
+          #
+          # @example
+          #   sim = adapter.similarity(card_a.embedding, card_b.embedding)
+          #   puts "Cards are duplicates" if sim > 0.92
+          def similarity(embedding_a, embedding_b)
+            dot = 0.0
+            mag_a = 0.0
+            mag_b = 0.0
+
+            embedding_a.each_with_index do |va, i|
+              vb = embedding_b[i]
+              dot += va * vb
+              mag_a += va * va
+              mag_b += vb * vb
+            end
+
+            mag_a = Math.sqrt(mag_a)
+            mag_b = Math.sqrt(mag_b)
+            return 0.0 if mag_a.zero? || mag_b.zero?
+
+            dot / (mag_a * mag_b)
+          end
+        end
+      end
+    end
+  end
+end