swarm_sdk 2.7.14 → 3.0.0.alpha2

data/lib/swarm_sdk/v3/memory/adapters/faiss_support.rb

@@ -0,0 +1,194 @@
+# frozen_string_literal: true
+
+module SwarmSDK
+  module V3
+    module Memory
+      module Adapters
+        # Shared FAISS vector index support for adapters
+        #
+        # Provides FAISS-based vector search and index management that can
+        # be included by any adapter that stores embeddings as Ruby arrays.
+        # The FAISS index (IndexFlatIP with L2-normalized vectors) handles
+        # efficient top-k nearest neighbor search.
+        #
+        # ## Requirements for including classes
+        #
+        # The including class must:
+        # - Call {#initialize_faiss!} from its constructor
+        # - Define a `faiss_directory` method returning the path for FAISS files
+        # - Implement `list_cards` (used by {#rebuild_index})
+        #
+        # @example Including in a custom adapter
+        #   class MyAdapter < Base
+        #     include VectorUtils    # pairwise similarity
+        #     include FaissSupport   # FAISS vector index
+        #
+        #     def initialize(directory)
+        #       super()
+        #       @directory = directory
+        #       initialize_faiss!
+        #     end
+        #
+        #     private
+        #
+        #     def faiss_directory
+        #       @directory
+        #     end
+        #   end
+        module FaissSupport
+          # Embedding dimensions for MiniLM-L6-v2 model
+          EMBEDDING_DIMENSIONS = 384
+
+          # Search the FAISS index for similar vectors
+          #
+          # @param embedding [Array<Float>] Query embedding
+          # @param top_k [Integer] Maximum number of results
+          # @param threshold [Float] Minimum cosine similarity to include
+          # @return [Array<Hash>] Array of `{ id: String, similarity: Float }`
+          def vector_search(embedding, top_k:, threshold: 0.0)
+            return [] unless @faiss_index && @faiss_index.ntotal > 0
+
+            normalized = normalize_vector(embedding)
+            effective_k = [top_k, @faiss_index.ntotal].min
+            # Explicit Numo conversion for safe FAISS C extension boundary crossing
+            query_vector = Numo::SFloat.cast([normalized])
+            distances, ids = @faiss_index.search(query_vector, effective_k)
+
+            index_to_id = @id_to_index.invert
+            # FAISS returns Numo::NArray — convert to Ruby arrays
+            id_row = ids.to_a.flatten
+            dist_row = distances.to_a.flatten
+            results = []
+            id_row.each_with_index do |idx, i|
+              next if idx < 0
+
+              card_id = index_to_id[idx]
+              next unless card_id
+
+              similarity = dist_row[i]
+              next if similarity < threshold
+
+              results << { id: card_id, similarity: similarity }
+            end
+
+            results
+          end
+
+          # Rebuild the FAISS index from all stored cards
+          #
+          # @return [void]
+          def rebuild_index
+            @faiss_index = create_faiss_index
+            @id_to_index = {}
+            @next_index_id = 0
+
+            list_cards.each do |card|
+              add_to_vector_index(card) if card.embedding
+            end
+          end
+
+          private
+
+          # Initialize FAISS instance variables
+          #
+          # Must be called from the including class's constructor.
+          #
+          # @return [void]
+          def initialize_faiss!
+            @id_to_index = {}
+            @next_index_id = 0
+            @faiss_index = nil
+          end
+
+          # Create a new FAISS IndexIDMap backed by IndexFlatIP
+          #
+          # @return [Faiss::IndexIDMap] New FAISS index
+          def create_faiss_index
+            require "faiss"
+            # Keep a reference to the inner index to prevent GC from collecting
+            # it while IndexIDMap holds a raw C pointer to it. Without this,
+            # Ruby's GC can free the inner index, leaving IndexIDMap with a
+            # dangling pointer that causes segfaults on add/search.
+            @faiss_inner_index = Faiss::IndexFlatIP.new(EMBEDDING_DIMENSIONS)
+            Faiss::IndexIDMap.new(@faiss_inner_index)
+          end
+
+          # Add a card's embedding to the FAISS index
+          #
+          # @param card [Card] Card with embedding
+          # @return [void]
+          def add_to_vector_index(card)
+            @faiss_index ||= create_faiss_index
+            normalized = normalize_vector(card.embedding)
+            index_id = @next_index_id
+            @next_index_id += 1
+            # Explicit Numo conversion avoids segfaults from implicit
+            # Ruby Array → Numo cast at the FAISS C extension boundary.
+            vectors = Numo::SFloat.cast([normalized])
+            ids = Numo::Int64.cast([index_id])
+            @faiss_index.add_with_ids(vectors, ids)
+            @id_to_index[card.id] = index_id
+          end
+
+          # Remove a card from the vector index
+          #
+          # Removes the ID mapping. Full FAISS rebuild needed for actual removal.
+          #
+          # @param card_id [String] Card ID
+          # @return [void]
+          def remove_from_vector_index(card_id)
+            @id_to_index.delete(card_id)
+          end
+
+          # L2-normalize a vector for cosine similarity via inner product
+          #
+          # @param vec [Array<Float>] Input vector
+          # @return [Array<Float>] Normalized vector
+          def normalize_vector(vec)
+            magnitude = Math.sqrt(vec.sum { |v| v * v })
+            return vec if magnitude.zero?
+
+            vec.map { |v| v / magnitude }
+          end
+
+          # Save the FAISS index and ID mapping to disk
+          #
+          # @return [void]
+          def save_faiss_index
+            return unless @faiss_index
+
+            path = File.join(faiss_directory, "index.faiss")
+            @faiss_index.save(path)
+
+            # Save ID mapping separately
+            mapping_path = File.join(faiss_directory, "index_mapping.json")
+            File.write(mapping_path, JSON.generate(
+              id_to_index: @id_to_index,
+              next_index_id: @next_index_id,
+            ))
+          end
+
+          # Load the FAISS index and ID mapping from disk
+          #
+          # @return [void]
+          def load_faiss_index
+            path = File.join(faiss_directory, "index.faiss")
+            mapping_path = File.join(faiss_directory, "index_mapping.json")
+
+            if File.exist?(path) && File.exist?(mapping_path)
+              require "faiss"
+              @faiss_index = Faiss::Index.load(path)
+              mapping = JSON.parse(File.read(mapping_path))
+              @id_to_index = mapping["id_to_index"] || {}
+              @next_index_id = mapping["next_index_id"] || 0
+            else
+              @faiss_index = nil
+              @id_to_index = {}
+              @next_index_id = 0
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/swarm_sdk/v3/memory/adapters/filesystem_adapter.rb

@@ -0,0 +1,212 @@
+# frozen_string_literal: true
+
+module SwarmSDK
+  module V3
+    module Memory
+      module Adapters
+        # Filesystem-based storage adapter with FAISS vector index
+        #
+        # Stores cards as individual JSON files, edges and clusters in
+        # aggregate JSON files, and vectors in a FAISS IndexFlatIP index
+        # (cosine similarity via L2-normalized vectors + inner product).
+        #
+        # Directory structure:
+        #   .swarm/memory/
+        #   ├── cards/
+        #   │   ├── card_a1b2c3.json
+        #   │   └── ...
+        #   ├── edges.json
+        #   ├── clusters.json
+        #   └── index.faiss
+        #
+        # @example
+        #   adapter = FilesystemAdapter.new("/path/to/.swarm/memory")
+        #   adapter.load
+        #   adapter.write_card(card)
+        #   adapter.save
+        class FilesystemAdapter < Base
+          include VectorUtils
+          include FaissSupport
+
+          # @return [String] Root directory for storage
+          attr_reader :directory
+
+          # @param directory [String] Root directory for memory storage
+          def initialize(directory)
+            super()
+            @directory = File.expand_path(directory)
+            @edges = []
+            @clusters = []
+            initialize_faiss!
+          end
+
+          # --- Card CRUD ---
+
+          # @param card [Card] Card to write
+          # @return [void]
+          def write_card(card)
+            ensure_directories!
+            path = card_path(card.id)
+            File.write(path, JSON.pretty_generate(card.to_h))
+            add_to_vector_index(card) if card.embedding
+          end
+
+          # @param id [String] Card ID
+          # @return [Card, nil]
+          def read_card(id)
+            path = card_path(id)
+            return unless File.exist?(path)
+
+            data = JSON.parse(File.read(path))
+            Card.from_h(data)
+          end
+
+          # @param id [String] Card ID
+          # @return [void]
+          def delete_card(id)
+            path = card_path(id)
+            File.delete(path) if File.exist?(path)
+            remove_from_vector_index(id)
+          end
+
+          # @param prefix [String, nil] ID prefix filter
+          # @return [Array<Card>]
+          def list_cards(prefix: nil)
+            ensure_directories!
+            pattern = prefix ? "#{prefix}*.json" : "*.json"
+            Dir.glob(File.join(cards_dir, pattern)).map do |path|
+              data = JSON.parse(File.read(path))
+              Card.from_h(data)
+            end
+          end
+
+          # --- Edge CRUD ---
+
+          # @param edge [Edge] Edge to write
+          # @return [void]
+          def write_edge(edge)
+            @edges << edge
+          end
+
+          # @param card_id [String] Card ID
+          # @param type [Symbol, nil] Filter by edge type
+          # @return [Array<Edge>]
+          def edges_for(card_id, type: nil)
+            results = @edges.select { |e| e.from_id == card_id || e.to_id == card_id }
+            results = results.select { |e| e.type == type } if type
+            results
+          end
+
+          # @param card_id [String] Card ID
+          # @return [void]
+          def delete_edges_for(card_id)
+            @edges.reject! { |e| e.from_id == card_id || e.to_id == card_id }
+          end
+
+          # --- Cluster CRUD ---
+
+          # @param cluster [Cluster] Cluster to write
+          # @return [void]
+          def write_cluster(cluster)
+            idx = @clusters.index { |c| c.id == cluster.id }
+            if idx
+              @clusters[idx] = cluster
+            else
+              @clusters << cluster
+            end
+          end
+
+          # @param id [String] Cluster ID
+          # @return [Cluster, nil]
+          def read_cluster(id)
+            @clusters.find { |c| c.id == id }
+          end
+
+          # @return [Array<Cluster>]
+          def list_clusters
+            @clusters.dup
+          end
+
+          # --- Persistence ---
+
+          # Save edges, clusters, and FAISS index to disk
+          #
+          # @return [void]
+          def save
+            ensure_directories!
+            save_edges
+            save_clusters
+            save_faiss_index
+          end
+
+          # Load edges, clusters, and FAISS index from disk
+          #
+          # @return [void]
+          def load
+            ensure_directories!
+            load_edges
+            load_clusters
+            load_faiss_index
+          end
+
+          private
+
+          # @return [String] Path to the FAISS index directory
+          def faiss_directory
+            @directory
+          end
+
+          # @return [String] Path to cards directory
+          def cards_dir
+            File.join(@directory, "cards")
+          end
+
+          # @param id [String] Card ID
+          # @return [String] Path to card JSON file
+          def card_path(id)
+            File.join(cards_dir, "#{id}.json")
+          end
+
+          # Ensure required directories exist
+          #
+          # @return [void]
+          def ensure_directories!
+            FileUtils.mkdir_p(cards_dir)
+          end
+
+          # --- Serialization ---
+
+          # @return [void]
+          def save_edges
+            path = File.join(@directory, "edges.json")
+            File.write(path, JSON.pretty_generate(@edges.map(&:to_h)))
+          end
+
+          # @return [void]
+          def load_edges
+            path = File.join(@directory, "edges.json")
+            return unless File.exist?(path)
+
+            data = JSON.parse(File.read(path))
+            @edges = data.map { |h| Edge.from_h(h) }
+          end
+
+          # @return [void]
+          def save_clusters
+            path = File.join(@directory, "clusters.json")
+            File.write(path, JSON.pretty_generate(@clusters.map(&:to_h)))
+          end
+
+          # @return [void]
+          def load_clusters
+            path = File.join(@directory, "clusters.json")
+            return unless File.exist?(path)
+
+            data = JSON.parse(File.read(path))
+            @clusters = data.map { |h| Cluster.from_h(h) }
+          end
+        end
+      end
+    end
+  end
+end