swarm_sdk 2.7.14 → 3.0.0.alpha2

data/lib/swarm_sdk/v3/memory/context_builder.rb

@@ -0,0 +1,339 @@
+# frozen_string_literal: true
+
+module SwarmSDK
+  module V3
+    module Memory
+      # Assembles working context from memory tiers
+      #
+      # Combines retrieved memory cards, recent turns (STM buffer),
+      # cluster summaries, active constraints, and an exploration sample
+      # into a coherent working context for the LLM.
+      #
+      # Includes an "exploration sprinkle" — 1-2 low-exposure cards that are
+      # loosely relevant to the query. This prevents permanent forgetting of
+      # rarely-accessed memories by giving them occasional exposure.
+      #
+      # ## Associative Memory
+      #
+      # When `associative_memory: true`, exploration cards are labeled under a
+      # distinct "YOU ALSO REMEMBER:" section, and a brief guidance is appended
+      # after the memory context encouraging the LLM to naturally bring up
+      # these tangential memories when the conversation allows — like a person
+      # who says "btw, how was Porto?" after discussing Portugal.
+      #
+      # When `associative_memory: false` (default), exploration cards are
+      # formatted identically to retrieved cards. They still serve their
+      # anti-forgetting purpose (exposure bumping via `record_access!`),
+      # but the LLM is not encouraged to surface them conversationally.
+      #
+      # @example
+      #   builder = ContextBuilder.new(retriever: retriever, adapter: adapter)
+      #   context = builder.build(query: "How does auth work?", recent_turns: messages)
+      class ContextBuilder
+        # @param retriever [Retriever] Hybrid search retriever
+        # @param adapter [Adapters::Base] Storage adapter
+        # @param retrieval_top_k [Integer] Cards to retrieve per query
+        # @param embedder [Embedder, nil] Embedder for exploration similarity
+        # @param associative_memory [Boolean] Whether to label exploration cards distinctly
+        def initialize(retriever:, adapter:, retrieval_top_k: 15, embedder: nil, associative_memory: false)
+          @retriever = retriever
+          @adapter = adapter
+          @retrieval_top_k = retrieval_top_k
+          @embedder = embedder
+          @associative_memory = associative_memory
+          @config = Configuration.instance
+        end
+
+        # Build working context for a query
+        #
+        # @param query [String] Current user query
+        # @param recent_turns [Array<Hash>] Recent conversation turns (STM)
+        # @param system_prompt [String, nil] Agent's system prompt
+        # @param read_only [Boolean] When true, skips recording access on retrieved cards
+        # @return [Array<Hash>] Messages array ready for LLM
+        #
+        # @example
+        #   messages = builder.build(
+        #     query: "What auth system are we using?",
+        #     recent_turns: last_8_turns,
+        #     system_prompt: "You are a backend developer.",
+        #   )
+        #
+        # @example Read-only mode (for subtasks)
+        #   messages = builder.build(
+        #     query: "Check auth approach",
+        #     read_only: true,
+        #   )
+        def build(query:, recent_turns: [], system_prompt: nil, read_only: false)
+          DebugLog.log("context_builder", "build: query=#{query[0..60].inspect}")
+
+          retrieved_cards = DebugLog.time("context_builder", "retriever.search(top_k=#{@retrieval_top_k})") do
+            @retriever.search(query, top_k: @retrieval_top_k)
+          end
+
+          exploration_cards = DebugLog.time("context_builder", "find_exploration_cards") do
+            find_exploration_cards(query, retrieved_cards)
+          end
+          all_cards = retrieved_cards + exploration_cards
+
+          relevant_clusters = find_relevant_clusters(all_cards)
+          active_constraints = find_active_constraints(all_cards)
+
+          all_cards = DebugLog.time("context_builder", "deduplicate_cards(#{all_cards.size})") do
+            deduplicate_cards(all_cards)
+          end
+
+          DebugLog.log("context_builder", "retrieved=#{retrieved_cards.size} exploration=#{exploration_cards.size} deduped=#{all_cards.size} clusters=#{relevant_clusters.size} constraints=#{active_constraints.size}")
+
+          # Record access on all cards included in context (skip in read-only mode)
+          unless read_only
+            all_cards.each do |card|
+              card.record_access!
+              @adapter.write_card(card)
+            end
+          end
+
+          messages = []
+
+          # Build memory context from all tiers
+          memory_context = format_memory_context(
+            retrieved_cards, exploration_cards, relevant_clusters, active_constraints
+          )
+
+          # System prompt with memory context
+          if system_prompt
+            system_content = system_prompt
+            system_content = "#{system_content}\n\n#{memory_context}" unless memory_context.empty?
+            messages << { role: "system", content: system_content }
+          elsif !memory_context.empty?
+            messages << { role: "system", content: memory_context }
+          end
+
+          # Recent turns (STM buffer)
+          messages.concat(recent_turns)
+
+          messages
+        end
+
+        private
+
+        # Deduplicate cards for working context
+        #
+        # Removes near-duplicate cards by:
+        # 1. Preferring canonical cards over merged duplicates
+        # 2. Using the adapter's similarity method to detect near-duplicates (>0.92)
+        #
+        # All similarity computation is delegated to the adapter so that
+        # storage backends like pgvector can compute it server-side.
+        #
+        # @param cards [Array<Card>] Cards to deduplicate
+        # @return [Array<Card>] Deduplicated cards
+        def deduplicate_cards(cards)
+          return cards if cards.size <= 1
+
+          # Prefer canonical cards — skip cards that have been merged into another
+          cards = cards.reject { |c| c.canonical_id && cards.any? { |other| other.id == c.canonical_id } }
+
+          # Remove near-duplicates by embedding similarity (via adapter)
+          kept = []
+          cards.each do |card|
+            duplicate = kept.any? do |existing|
+              next false unless card.embedding && existing.embedding
+
+              @adapter.similarity(card.embedding, existing.embedding) > @config.dedup_similarity_threshold
+            end
+            kept << card unless duplicate
+          end
+
+          kept
+        end
+
+        # Find low-exposure cards loosely relevant to the query
+        #
+        # Prevents permanent forgetting by occasionally surfacing rarely-accessed
+        # memories that have some relevance to the current query.
+        #
+        # Uses the adapter's vector_search to find loosely relevant cards among
+        # the low-exposure pool, keeping similarity computation in the adapter.
+        #
+        # @param query [String] Current query
+        # @param retrieved_cards [Array<Card>] Already-retrieved cards to exclude
+        # @return [Array<Card>] Exploration cards (0-2)
+        def find_exploration_cards(query, retrieved_cards)
+          return [] unless @embedder
+
+          retrieved_ids = Set.new(retrieved_cards.map(&:id))
+          all_cards = @adapter.list_cards.reject { |c| retrieved_ids.include?(c.id) || c.embedding.nil? }
+          return [] if all_cards.empty?
+
+          # Sort by exposure (ascending) — least-exposed first
+          tracker = ExposureTracker.new(@adapter)
+          scored = all_cards.map { |c| { card: c, exposure: tracker.exposure_score(c) } }
+          scored.sort_by! { |s| s[:exposure] }
+
+          # From the least-exposed cards, pick ones with minimum similarity to query
+          query_embedding = @embedder.embed(query)
+          candidates = scored.take([scored.size, 20].min).filter_map do |entry|
+            sim = @adapter.similarity(query_embedding, entry[:card].embedding)
+            next if sim < @config.exploration_min_similarity
+
+            { card: entry[:card], similarity: sim }
+          end
+
+          candidates.sort_by { |c| -c[:similarity] }
+            .take(@config.exploration_sample_size)
+            .map { |c| c[:card] }
+        end
+
+        # Rehydrate a compressed card (L3/L4) with neighbor and cluster context
+        #
+        # Pulls 1-hop neighbors via the adapter and includes relevant cluster
+        # summaries to reconstruct richer context from compressed traces.
+        #
+        # @param card [Card] Compressed card (compression_level >= 3)
+        # @param clusters [Array<Cluster>] Relevant clusters already found
+        # @return [String] Rehydrated context string
+        def rehydrate_card(card, clusters)
+          parts = ["[Rehydrated] [#{card.type.to_s.capitalize}] #{card.text}"]
+
+          # Pull 1-hop neighbors for additional context
+          neighbors = collect_neighbors(card.id)
+          unless neighbors.empty?
+            neighbor_texts = neighbors.take(3).map { |n| "  - #{n.text[0..150]}" }
+            parts << "  Related context:"
+            parts.concat(neighbor_texts)
+          end
+
+          # Include cluster summary if available
+          card_cluster = clusters.find { |c| c.card_ids.include?(card.id) }
+          if card_cluster && !card_cluster.rolling_summary.empty?
+            parts << "  Topic: #{card_cluster.title} — #{card_cluster.rolling_summary[0..200]}"
+          end
+
+          parts.join("\n")
+        end
+
+        # Collect 1-hop neighbor cards via edges
+        #
+        # @param card_id [String] Card ID to find neighbors for
+        # @return [Array<Card>] Neighbor cards (excluding compressed L4 cards)
+        def collect_neighbors(card_id)
+          edges = @adapter.edges_for(card_id)
+          neighbor_ids = edges.map do |edge|
+            edge.from_id == card_id ? edge.to_id : edge.from_id
+          end.uniq
+
+          neighbor_ids.filter_map do |nid|
+            card = @adapter.read_card(nid)
+            card if card && card.compression_level < 4
+          end
+        end
+
+        # Find clusters relevant to the retrieved cards
+        #
+        # @param cards [Array<Card>] Retrieved cards
+        # @return [Array<Cluster>] Relevant clusters
+        def find_relevant_clusters(cards)
+          return [] if cards.empty?
+
+          card_ids = Set.new(cards.map(&:id))
+          @adapter.list_clusters
+            .select { |c| c.card_ids.any? { |id| card_ids.include?(id) } }
+            .reject { |c| c.rolling_summary.empty? }
+        end
+
+        # Find active constraints not already in the retrieved cards
+        #
+        # Constraints (type: :constraint) with high importance (>=0.7) should
+        # appear in every context build to prevent accidental violation.
+        # Only high-importance constraints are always surfaced; lower-importance
+        # ones must be found through retrieval.
+        #
+        # @param already_retrieved [Array<Card>] Cards already in context
+        # @return [Array<Card>] Active constraint cards
+        def find_active_constraints(already_retrieved)
+          retrieved_ids = Set.new(already_retrieved.map(&:id))
+
+          @adapter.list_cards
+            .select { |c| c.type == :constraint && c.importance >= 0.7 && !retrieved_ids.include?(c.id) }
+            .sort_by { |c| -c.importance }
+            .take(3)
+        end
+
+        # Format all memory tiers into a single context string
+        #
+        # Includes: active constraints, retrieved cards, cluster summaries,
+        # and exploration sprinkle — in that priority order.
+        #
+        # @param retrieved [Array<Card>] Retrieved cards
+        # @param exploration [Array<Card>] Exploration sprinkle cards
+        # @param clusters [Array<Cluster>] Relevant clusters
+        # @param constraints [Array<Card>] Active constraint cards
+        # @return [String] Formatted memory context
+        def format_memory_context(retrieved, exploration = [], clusters = [], constraints = [])
+          has_content = !retrieved.empty? || !exploration.empty? || !clusters.empty? || !constraints.empty?
+          return "" unless has_content
+
+          sections = []
+
+          # Active constraints first (highest priority)
+          unless constraints.empty?
+            sections << "ACTIVE CONSTRAINTS:"
+            constraints.each { |c| sections << "- #{c.text}" }
+            sections << ""
+          end
+
+          # Retrieved cards (rehydrate compressed ones)
+          retrieved.each do |card|
+            sections << if card.compression_level >= 3
+              rehydrate_card(card, clusters)
+            else
+              "[#{card.type.to_s.capitalize}] #{card.text}"
+            end
+          end
+
+          # Exploration sprinkle
+          unless exploration.empty?
+            if @associative_memory
+              sections << ""
+              sections << "YOU ALSO REMEMBER:"
+            end
+            exploration.each do |card|
+              sections << "[#{card.type.to_s.capitalize}] #{card.text}"
+            end
+          end
+
+          # Cluster summaries
+          unless clusters.empty?
+            sections << ""
+            sections << "TOPIC SUMMARIES:"
+            clusters.each do |cluster|
+              sections << "#{cluster.title}: #{cluster.rolling_summary}"
+              next if cluster.decision_log.empty?
+
+              sections << "Key decisions: #{cluster.decision_log.join("; ")}"
+            end
+          end
+
+          context = <<~CONTEXT.strip
+            <memory-context>
+            The following information was retrieved from your memory:
+
+            #{sections.join("\n")}
+            </memory-context>
+          CONTEXT
+
+          if @associative_memory && !exploration.empty?
+            context = "#{context}\n\n" \
+              "The \"YOU ALSO REMEMBER\" section contains tangential memories. " \
+              "Only bring these up when the conversation is open-ended or exploratory. " \
+              "Never mention them during task-focused work like writing code, debugging, " \
+              "or following specific instructions."
+          end
+
+          context
+        end
+      end
+    end
+  end
+end

data/lib/swarm_sdk/v3/memory/edge.rb

@@ -0,0 +1,105 @@
+# frozen_string_literal: true
+
+module SwarmSDK
+  module V3
+    module Memory
+      # A typed relationship between two memory cards
+      #
+      # Edges form a knowledge graph connecting related cards.
+      # They enable graph-based retrieval — when a card is relevant,
+      # its neighbors can be pulled in for richer context.
+      #
+      # @example Create a dependency edge
+      #   edge = Edge.new(
+      #     from_id: "card_a1b2c3",
+      #     to_id: "card_d4e5f6",
+      #     type: :depends_on,
+      #     weight: 0.8,
+      #   )
+      class Edge
+        TYPES = [
+          :elaborates,
+          :depends_on,
+          :supports,
+          :contradicts,
+          :same_entity,
+          :same_episode,
+          :decision_reason,
+        ].freeze
+
+        # @return [String] Source card ID
+        attr_reader :from_id
+
+        # @return [String] Target card ID
+        attr_reader :to_id
+
+        # @return [Symbol] Relationship type
+        attr_reader :type
+
+        # @return [Float] Edge weight (0.0-1.0)
+        attr_reader :weight
+
+        # @return [Time] Creation timestamp
+        attr_reader :created_at
+
+        # Create a new edge
+        #
+        # @param from_id [String] Source card ID
+        # @param to_id [String] Target card ID
+        # @param type [Symbol] Relationship type
+        # @param weight [Float] Edge weight (0.0-1.0)
+        # @param created_at [Time, nil] Creation time
+        #
+        # @raise [ArgumentError] If type is invalid
+        def initialize(from_id:, to_id:, type:, weight: 1.0, created_at: nil)
+          @from_id = from_id
+          @to_id = to_id
+          @type = type.to_sym
+          @weight = weight.to_f.clamp(0.0, 1.0)
+          @created_at = created_at || Time.now
+
+          validate!
+        end
+
+        # Serialize to a hash
+        #
+        # @return [Hash]
+        def to_h
+          {
+            from_id: @from_id,
+            to_id: @to_id,
+            type: @type.to_s,
+            weight: @weight,
+            created_at: @created_at.iso8601,
+          }
+        end
+
+        class << self
+          # Deserialize from a hash
+          #
+          # @param hash [Hash] Serialized edge data
+          # @return [Edge]
+          def from_h(hash)
+            hash = hash.transform_keys(&:to_sym)
+            new(
+              from_id: hash[:from_id],
+              to_id: hash[:to_id],
+              type: hash[:type]&.to_sym,
+              weight: hash[:weight] || 1.0,
+              created_at: hash[:created_at] ? Time.parse(hash[:created_at]) : nil,
+            )
+          end
+        end
+
+        private
+
+        # @raise [ArgumentError] If validation fails
+        def validate!
+          raise ArgumentError, "from_id is required" if @from_id.nil? || @from_id.strip.empty?
+          raise ArgumentError, "to_id is required" if @to_id.nil? || @to_id.strip.empty?
+          raise ArgumentError, "Invalid edge type: #{@type}. Must be one of: #{TYPES.join(", ")}" unless TYPES.include?(@type)
+        end
+      end
+    end
+  end
+end

data/lib/swarm_sdk/v3/memory/embedder.rb

@@ -0,0 +1,185 @@
+# frozen_string_literal: true
+
+module SwarmSDK
+  module V3
+    module Memory
+      # ONNX-based text embedder using Informers gem
+      #
+      # Uses the `Informers.pipeline("embedding", ...)` API to generate
+      # sentence embeddings via ONNX inference. Lazy-loads the model on
+      # first use to avoid startup cost.
+      #
+      # Model name and cache directory are read from {Configuration},
+      # with environment variable overrides.
+      #
+      # The model is automatically downloaded on first use if not cached.
+      # Use {#preload!} to trigger an eager download before first embedding.
+      #
+      # @example Basic usage
+      #   embedder = Embedder.new
+      #   vector = embedder.embed("The API uses JWT tokens")
+      #   vector.length  #=> 384
+      #
+      # @example Eager download
+      #   embedder = Embedder.new
+      #   embedder.preload!  # Downloads model if not cached
+      #
+      # @example Custom model via environment variable
+      #   ENV["SWARM_EMBEDDING_MODEL"] = "sentence-transformers/all-MiniLM-L6-v2"
+      #   embedder = Embedder.new
+      class Embedder
+        DIMENSIONS = 384
+
+        # Default embedding model (QA-optimized, 384 dimensions)
+        DEFAULT_MODEL = "sentence-transformers/multi-qa-MiniLM-L6-cos-v1"
+
+        # Environment variable for model override
+        MODEL_ENV_VAR = "SWARM_EMBEDDING_MODEL"
+
+        # Number of dimensions in the embedding vector
+        #
+        # @return [Integer]
+        def dimensions
+          DIMENSIONS
+        end
+
+        # The model name used for embeddings
+        #
+        # Resolution order:
+        # 1. SWARM_EMBEDDING_MODEL environment variable
+        # 2. Configuration.instance.embedding_model
+        # 3. DEFAULT_MODEL constant
+        #
+        # @return [String] Sentence-transformer model identifier
+        def model_name
+          ENV[MODEL_ENV_VAR] || Configuration.instance.embedding_model || DEFAULT_MODEL
+        end
+
+        # Generate an embedding vector for text
+        #
+        # Lazy-loads the model on first call. The model will be downloaded
+        # automatically if not cached locally.
+        #
+        # @param text [String] Text to embed
+        # @return [Array<Float>] Embedding vector (384 dimensions)
+        # @raise [MemoryError] If model loading or embedding fails
+        #
+        # @example
+        #   vector = embedder.embed("Hello world")
+        #   vector.length  #=> 384
+        def embed(text)
+          pipeline.call(text)
+        rescue StandardError => e
+          raise MemoryError, "Embedding failed for text (#{text.length} chars): #{e.message}"
+        end
+
+        # Generate embeddings for multiple texts
+        #
+        # More efficient than calling embed() in a loop because the
+        # pipeline can batch internally.
+        #
+        # @param texts [Array<String>] Texts to embed
+        # @return [Array<Array<Float>>] Embedding vectors
+        # @raise [MemoryError] If embedding fails
+        def embed_batch(texts)
+          return [] if texts.empty?
+
+          pipeline.call(texts)
+        rescue StandardError => e
+          raise MemoryError, "Batch embedding failed (#{texts.size} texts): #{e.message}"
+        end
+
+        # Eagerly download and load the embedding model
+        #
+        # Triggers model download if not already cached. Useful for
+        # ensuring the model is available before first use. This is a
+        # one-time download of ~90MB.
+        #
+        # @return [void]
+        # @raise [MemoryError] If model download or loading fails
+        #
+        # @example
+        #   embedder = Embedder.new
+        #   embedder.preload!  # Downloads model files (~90MB)
+        def preload!
+          pipeline
+          nil
+        rescue StandardError => e
+          raise MemoryError, "Failed to preload embedding model '#{model_name}': #{e.message}"
+        end
+
+        # Check if the embedding model is cached locally
+        #
+        # Verifies that the model directory exists and contains the
+        # required tokenizer and ONNX model files.
+        #
+        # @return [Boolean] true if model files exist on disk
+        #
+        # @example
+        #   embedder = Embedder.new
+        #   embedder.cached?  #=> true (if already downloaded)
+        def cached?
+          cache_dir = resolve_cache_dir
+          model_dir = File.join(cache_dir, model_name)
+          return false unless File.directory?(model_dir)
+
+          # Check for required model files
+          tokenizer = File.join(model_dir, "tokenizer.json")
+          onnx_model = File.join(model_dir, "onnx", "model.onnx")
+          File.exist?(tokenizer) && File.exist?(onnx_model)
+        end
+
+        private
+
+        # Lazy-load the Informers embedding pipeline
+        #
+        # Uses `Informers.pipeline("embedding", model_name)` — the current
+        # recommended API. The pipeline handles model downloading, tokenization,
+        # and ONNX inference internally.
+        #
+        # @return [Informers::EmbeddingPipeline] Lazy-loaded pipeline
+        def pipeline
+          @pipeline ||= begin
+            require "informers"
+            configure_cache_dir!
+            Informers.pipeline("embedding", model_name)
+          rescue LoadError
+            raise MemoryError,
+              "Informers gem is not available. Install with: gem install informers"
+          rescue StandardError => e
+            raise MemoryError,
+              "Failed to load embedding model '#{model_name}': #{e.message}"
+          end
+        end
+
+        # Configure Informers cache directory if set in configuration
+        #
+        # Sets the global Informers.cache_dir which affects all
+        # subsequent model loading.
+        #
+        # @return [void]
+        def configure_cache_dir!
+          cache_dir = Configuration.instance.embedding_cache_dir
+          return unless cache_dir
+
+          FileUtils.mkdir_p(cache_dir) unless File.directory?(cache_dir)
+          Informers.cache_dir = cache_dir
+        end
+
+        # Resolve the cache directory for checking model presence
+        #
+        # Uses the configured cache dir, falls back to the XDG default
+        # that Informers uses internally.
+        #
+        # @return [String] Path to cache directory
+        def resolve_cache_dir
+          Configuration.instance.embedding_cache_dir ||
+            File.join(
+              ENV.fetch("XDG_CACHE_HOME", File.join(Dir.home, ".cache")),
+              "informers",
+            )
+        end
+      end
+    end
+  end
+end