claude_memory 0.8.0 → 0.9.0

data/lib/claude_memory/embeddings/fastembed_adapter.rb

@@ -2,37 +2,50 @@
 
 module ClaudeMemory
   module Embeddings
-    # Adapter wrapping fastembed-rb for high-quality local embeddings
-    # Uses BAAI/bge-small-en-v1.5 by default (384-dim, ~67MB ONNX model)
+    # Adapter wrapping fastembed-rb for high-quality local embeddings.
+    # Supports any model available in fastembed-rb's SUPPORTED_MODELS.
     #
-    # Implements the same generate(text) interface as Generator for DI compatibility.
-    # Supports asymmetric query/passage encoding for better retrieval accuracy.
+    # Model selection (in priority order):
+    #   1. Explicit model_name parameter
+    #   2. CLAUDE_MEMORY_EMBEDDING_MODEL env var
+    #   3. Default: BAAI/bge-small-en-v1.5 (384-dim, ~67MB ONNX)
+    #
+    # Dimensions are resolved from the ModelRegistry for known models,
+    # or probed from fastembed's ModelInfo for unknown models.
     #
     # Usage:
     #   adapter = FastembedAdapter.new
     #   query_vec = adapter.generate("What database?")         # query encoding
     #   passage_vec = adapter.generate_passage("Uses PostgreSQL") # passage encoding
     #
+    #   # Use a larger model:
+    #   adapter = FastembedAdapter.new(model_name: "BAAI/bge-base-en-v1.5")
+    #   adapter.dimensions  # => 768
+    #
     class FastembedAdapter
-      EMBEDDING_DIM = 384
       DEFAULT_MODEL = "BAAI/bge-small-en-v1.5"
 
+      attr_reader :model_name, :dimensions
+
       def name = "fastembed"
 
-      def dimensions = EMBEDDING_DIM
+      def initialize(model_name: nil, env: ENV)
+        @model_name = model_name || env["CLAUDE_MEMORY_EMBEDDING_MODEL"] || DEFAULT_MODEL
+        @dimensions = resolve_dimensions(@model_name)
 
-      def initialize(model_name: DEFAULT_MODEL)
         require "fastembed"
-        @model = Fastembed::TextEmbedding.new(model_name: model_name)
+        @model = Fastembed::TextEmbedding.new(model_name: @model_name)
+
+        # If dimensions weren't known from registry, probe from fastembed
+        @dimensions ||= probe_dimensions_from_fastembed
       rescue LoadError
         raise LoadError,
           "fastembed gem is required for FastembedAdapter. Add `gem 'fastembed'` to your Gemfile."
       end
 
       # Generate query embedding (optimized for search queries)
-      # Compatible with Recall's embedding_generator interface
       # @param text [String] query text to embed
-      # @return [Array<Float>] normalized 384-dimensional vector
+      # @return [Array<Float>] normalized embedding vector
       def generate(text)
         return zero_vector if text.nil? || text.empty?
 
@@ -40,9 +53,8 @@ module ClaudeMemory
       end
 
       # Generate passage embedding (optimized for document/fact indexing)
-      # Use this when storing embeddings for facts
       # @param text [String] passage text to embed
-      # @return [Array<Float>] normalized 384-dimensional vector
+      # @return [Array<Float>] normalized embedding vector
       def generate_passage(text)
         return zero_vector if text.nil? || text.empty?
 
@@ -51,8 +63,26 @@ module ClaudeMemory
 
       private
 
+      # Resolve dimensions from the model registry (fast, no I/O).
+      # Returns nil if the model isn't in the registry.
+      def resolve_dimensions(model)
+        ModelRegistry.dimensions_for(model)
+      end
+
+      # Fallback: probe fastembed's SUPPORTED_MODELS for dimension info.
+      # This handles models added to fastembed-rb but not yet in our registry.
+      def probe_dimensions_from_fastembed
+        if defined?(Fastembed::SUPPORTED_MODELS)
+          info = Fastembed::SUPPORTED_MODELS[@model_name]
+          return info.dim if info
+        end
+
+        # Last resort: generate a test embedding and measure its size
+        @model.query_embed("dimension probe").first.size
+      end
+
       def zero_vector
-        Array.new(EMBEDDING_DIM, 0.0)
+        Array.new(@dimensions, 0.0)
       end
     end
   end

data/lib/claude_memory/embeddings/inspector.rb

@@ -0,0 +1,91 @@
+# frozen_string_literal: true
+
+module ClaudeMemory
+  module Embeddings
+    # Reads embedding metadata from global and project databases.
+    # Returns structured data — no I/O formatting or stdout output.
+    #
+    # Used by EmbeddingsCommand to separate DB concerns from presentation.
+    class Inspector
+      DatabaseState = Data.define(:label, :provider, :dimensions)
+      DimensionResult = Data.define(:label, :status, :stored_dims, :stored_provider, :current_dims)
+
+      def database_states
+        results = []
+
+        with_each_store do |label, store|
+          provider = store.get_meta("embedding_provider")
+          dims = store.get_meta("embedding_dimensions")
+
+          next unless provider || dims
+
+          results << DatabaseState.new(label: label, provider: provider, dimensions: dims)
+        end
+
+        results
+      end
+
+      def dimension_checks(provider_name, model_name)
+        results = []
+
+        with_each_store do |label, store|
+          stored_dims = store.get_meta("embedding_dimensions")&.to_i
+          stored_provider = store.get_meta("embedding_provider")
+
+          if stored_dims
+            current_dims = resolve_current_dimensions(provider_name, model_name)
+
+            status = if current_dims && current_dims != stored_dims
+              :mismatch
+            else
+              :match
+            end
+
+            results << DimensionResult.new(
+              label: label,
+              status: status,
+              stored_dims: stored_dims,
+              stored_provider: stored_provider,
+              current_dims: current_dims
+            )
+          else
+            results << DimensionResult.new(
+              label: label,
+              status: :fresh,
+              stored_dims: nil,
+              stored_provider: nil,
+              current_dims: nil
+            )
+          end
+        end
+
+        results
+      end
+
+      private
+
+      def resolve_current_dimensions(provider_name, model_name)
+        if model_name
+          ModelRegistry.dimensions_for(model_name)
+        else
+          ModelRegistry.default_for_provider(provider_name)&.dimensions
+        end
+      end
+
+      def with_each_store
+        config = Configuration.new
+
+        [["global", config.global_db_path], ["project", config.project_db_path]].each do |label, path|
+          next unless File.exist?(path)
+
+          store = Store::SQLiteStore.new(path)
+          begin
+            yield label, store
+          ensure
+            store.close
+          end
+        end
+      end
+    end
+  end
+end

data/lib/claude_memory/embeddings/model_registry.rb

@@ -0,0 +1,210 @@
+# frozen_string_literal: true
+
+module ClaudeMemory
+  module Embeddings
+    # Registry of known embedding models with their properties.
+    # Enables model validation, dimension lookup, and discoverability.
+    #
+    # Models are registered by canonical name (e.g., "BAAI/bge-small-en-v1.5")
+    # with provider type, dimensions, and description.
+    #
+    # Usage:
+    #   ModelRegistry.find("BAAI/bge-small-en-v1.5")
+    #   # => {provider: "fastembed", dimensions: 384, description: "...", ...}
+    #
+    #   ModelRegistry.models_for_provider("fastembed")
+    #   # => [...]
+    #
+    class ModelRegistry
+      ModelInfo = Data.define(:name, :provider, :dimensions, :description, :size_mb, :max_tokens)
+
+      # Known models with validated dimensions.
+      # Fastembed models sourced from fastembed-rb SUPPORTED_MODELS.
+      # API models sourced from provider documentation.
+      MODELS = [
+        # --- fastembed: local ONNX models (no API key needed) ---
+        ModelInfo.new(
+          name: "BAAI/bge-small-en-v1.5",
+          provider: "fastembed",
+          dimensions: 384,
+          description: "Fast English embedding (default)",
+          size_mb: 67,
+          max_tokens: 512
+        ),
+        ModelInfo.new(
+          name: "BAAI/bge-base-en-v1.5",
+          provider: "fastembed",
+          dimensions: 768,
+          description: "Balanced English embedding, higher accuracy",
+          size_mb: 210,
+          max_tokens: 512
+        ),
+        ModelInfo.new(
+          name: "BAAI/bge-large-en-v1.5",
+          provider: "fastembed",
+          dimensions: 1024,
+          description: "High accuracy English embedding",
+          size_mb: 1200,
+          max_tokens: 512
+        ),
+        ModelInfo.new(
+          name: "sentence-transformers/all-MiniLM-L6-v2",
+          provider: "fastembed",
+          dimensions: 384,
+          description: "Lightweight general-purpose sentence embedding",
+          size_mb: 90,
+          max_tokens: 512
+        ),
+        ModelInfo.new(
+          name: "intfloat/multilingual-e5-small",
+          provider: "fastembed",
+          dimensions: 384,
+          description: "Multilingual embedding, 100+ languages",
+          size_mb: 450,
+          max_tokens: 512
+        ),
+        ModelInfo.new(
+          name: "intfloat/multilingual-e5-base",
+          provider: "fastembed",
+          dimensions: 768,
+          description: "Larger multilingual embedding",
+          size_mb: 1110,
+          max_tokens: 512
+        ),
+        ModelInfo.new(
+          name: "nomic-ai/nomic-embed-text-v1.5",
+          provider: "fastembed",
+          dimensions: 768,
+          description: "Long context (8192 tokens) with Matryoshka support",
+          size_mb: 520,
+          max_tokens: 8192
+        ),
+        ModelInfo.new(
+          name: "jinaai/jina-embeddings-v2-small-en",
+          provider: "fastembed",
+          dimensions: 512,
+          description: "Small English embedding, 8192 token context",
+          size_mb: 60,
+          max_tokens: 8192
+        ),
+        ModelInfo.new(
+          name: "jinaai/jina-embeddings-v2-base-en",
+          provider: "fastembed",
+          dimensions: 768,
+          description: "Base English embedding, 8192 token context",
+          size_mb: 520,
+          max_tokens: 8192
+        ),
+
+        # --- api: OpenAI-compatible endpoints ---
+        ModelInfo.new(
+          name: "text-embedding-3-small",
+          provider: "api",
+          dimensions: 1536,
+          description: "OpenAI small embedding (default API model)",
+          size_mb: nil,
+          max_tokens: 8191
+        ),
+        ModelInfo.new(
+          name: "text-embedding-3-large",
+          provider: "api",
+          dimensions: 3072,
+          description: "OpenAI large embedding, highest accuracy",
+          size_mb: nil,
+          max_tokens: 8191
+        ),
+        ModelInfo.new(
+          name: "text-embedding-ada-002",
+          provider: "api",
+          dimensions: 1536,
+          description: "OpenAI legacy embedding",
+          size_mb: nil,
+          max_tokens: 8191
+        ),
+        ModelInfo.new(
+          name: "voyage-3",
+          provider: "api",
+          dimensions: 1024,
+          description: "Voyage AI general-purpose embedding",
+          size_mb: nil,
+          max_tokens: 32000
+        ),
+        ModelInfo.new(
+          name: "voyage-3-lite",
+          provider: "api",
+          dimensions: 512,
+          description: "Voyage AI lightweight embedding",
+          size_mb: nil,
+          max_tokens: 32000
+        ),
+        ModelInfo.new(
+          name: "voyage-code-3",
+          provider: "api",
+          dimensions: 1024,
+          description: "Voyage AI code-optimized embedding",
+          size_mb: nil,
+          max_tokens: 32000
+        ),
+
+        # --- tfidf: built-in, no dependencies ---
+        ModelInfo.new(
+          name: "tfidf",
+          provider: "tfidf",
+          dimensions: 384,
+          description: "Built-in TF-IDF embedding (no dependencies)",
+          size_mb: 0,
+          max_tokens: nil
+        )
+      ].freeze
+
+      MODELS_BY_NAME = MODELS.each_with_object({}) { |m, h| h[m.name] = m }.freeze
+
+      DEFAULTS = {
+        "fastembed" => "BAAI/bge-small-en-v1.5",
+        "api" => "text-embedding-3-small",
+        "tfidf" => "tfidf"
+      }.freeze
+
+      # Find a model by name.
+      # @param name [String] model name (e.g., "BAAI/bge-small-en-v1.5")
+      # @return [ModelInfo, nil]
+      def self.find(name)
+        MODELS_BY_NAME[name]
+      end
+
+      # List all models for a given provider.
+      # @param provider [String] "fastembed", "api", or "tfidf"
+      # @return [Array<ModelInfo>]
+      def self.models_for_provider(provider)
+        MODELS.select { |m| m.provider == provider }
+      end
+
+      # All known model names.
+      # @return [Array<String>]
+      def self.model_names
+        MODELS.map(&:name)
+      end
+
+      # All provider names.
+      # @return [Array<String>]
+      def self.providers
+        MODELS.map(&:provider).uniq
+      end
+
+      # Look up dimensions for a model name. Returns nil if unknown.
+      # @param name [String] model name
+      # @return [Integer, nil]
+      def self.dimensions_for(name)
+        find(name)&.dimensions
+      end
+
+      # Return the default ModelInfo for a provider.
+      # @param provider [String] "fastembed", "api", or "tfidf"
+      # @return [ModelInfo, nil]
+      def self.default_for_provider(provider)
+        default_name = DEFAULTS[provider]
+        find(default_name) if default_name
+      end
+    end
+  end
+end

data/lib/claude_memory/embeddings/resolver.rb

@@ -2,17 +2,43 @@
 
 module ClaudeMemory
   module Embeddings
-    # Resolves an embedding provider by name or ENV.
-    # Three providers: tfidf (default), fastembed, api.
-    def self.resolve(name = nil, env: ENV)
-      provider = name || env["CLAUDE_MEMORY_EMBEDDING_PROVIDER"] || "tfidf"
+    # Resolves an embedding provider by name, model, or ENV.
+    #
+    # Provider selection (in priority order):
+    #   1. Explicit name parameter
+    #   2. CLAUDE_MEMORY_EMBEDDING_PROVIDER env var
+    #   3. Default: "tfidf"
+    #
+    # Model selection is forwarded to the provider via CLAUDE_MEMORY_EMBEDDING_MODEL
+    # or the model parameter. The model can also imply the provider:
+    #   - "BAAI/bge-small-en-v1.5" → fastembed
+    #   - "text-embedding-3-small" → api
+    #
+    # Examples:
+    #   Embeddings.resolve                                    # tfidf default
+    #   Embeddings.resolve("fastembed")                       # fastembed with default model
+    #   Embeddings.resolve("fastembed", model: "BAAI/bge-base-en-v1.5")
+    #   Embeddings.resolve(model: "text-embedding-3-small")   # auto-detects api provider
+    #
+    def self.resolve(name = nil, model: nil, env: ENV)
+      model ||= env["CLAUDE_MEMORY_EMBEDDING_MODEL"]
+      provider = name || env["CLAUDE_MEMORY_EMBEDDING_PROVIDER"] || infer_provider(model) || "tfidf"
 
       case provider
       when "tfidf" then Generator.new
-      when "fastembed" then FastembedAdapter.new
-      when "api" then ApiAdapter.new(env: env)
+      when "fastembed" then FastembedAdapter.new(model_name: model, env: env)
+      when "api" then ApiAdapter.new(model: model, env: env)
       else raise ArgumentError, "Unknown embedding provider: #{provider}. Available: tfidf, fastembed, api"
       end
     end
+
+    # Infer provider from a model name using the registry.
+    # Returns nil if the model is unknown.
+    def self.infer_provider(model)
+      return nil unless model
+
+      ModelRegistry.find(model)&.provider
+    end
+    private_class_method :infer_provider
   end
 end

data/lib/claude_memory/ingest/ingester.rb

@@ -4,7 +4,17 @@ require "digest"
 
 module ClaudeMemory
   module Ingest
+    # Delta-based transcript ingestion with cursor tracking.
+    # Reads new content from transcripts, extracts metadata and tool calls,
+    # sanitizes private tags, and persists to the content_items table with FTS indexing.
     class Ingester
+      # @param store [Store::SQLiteStore] database store for persistence
+      # @param fts [Index::LexicalFTS, nil] full-text search index (default: new from store)
+      # @param env [Hash] environment variables
+      # @param metadata_extractor [MetadataExtractor, nil] extracts git branch, cwd, etc.
+      # @param tool_extractor [ToolExtractor, nil] extracts tool calls from transcript text
+      # @param tool_filter [ToolFilter, nil] filters irrelevant tool calls
+      # @param observation_compressor [ObservationCompressor, nil] compresses tool observations
       def initialize(store, fts: nil, env: ENV, metadata_extractor: nil, tool_extractor: nil, tool_filter: nil, observation_compressor: nil)
         @store = store
         @fts = fts || Index::LexicalFTS.new(store)
@@ -15,6 +25,13 @@ module ClaudeMemory
         @observation_compressor = observation_compressor || ObservationCompressor.new
       end
 
+      # Ingest new content from a transcript file
+      # @param source [String] content source identifier (e.g., "hook", "cli")
+      # @param session_id [String] Claude session ID
+      # @param transcript_path [String] path to the transcript file
+      # @param project_path [String, nil] project root (defaults to detected path)
+      # @return [Hash] result with :status (:ingested, :skipped, or :no_change),
+      #   :content_id, :bytes_read, and optional :reason
       def ingest(source:, session_id:, transcript_path:, project_path: nil)
         unless should_ingest?(transcript_path)
           ClaudeMemory.logger.debug("ingest", message: "Skipped unchanged file", transcript_path: transcript_path)

data/lib/claude_memory/mcp/handlers/management_handlers.rb

@@ -66,6 +66,30 @@ module ClaudeMemory
           end
         end
 
+        def reject_fact(args)
+          scope = args["scope"] || "project"
+          store = get_store_for_scope(scope)
+          return {error: "Database not available"} unless store
+
+          fact_id = args["fact_id"]
+          if fact_id.nil? && args["docid"]
+            row = store.find_fact_by_docid(args["docid"])
+            fact_id = row && row[:id]
+          end
+          return {error: "fact_id or docid required"} if fact_id.nil?
+
+          result = store.reject_fact(fact_id, reason: args["reason"])
+          return {error: "Fact #{fact_id} not found in #{scope} database"} if result.nil?
+
+          {
+            success: true,
+            scope: scope,
+            fact_id: fact_id,
+            conflicts_resolved: result[:conflicts_resolved],
+            message: "Fact rejected"
+          }
+        end
+
         def sweep_now(args)
           scope = args["scope"] || "project"
           store = get_store_for_scope(scope)

data/lib/claude_memory/mcp/handlers/stats_handlers.rb

@@ -129,13 +129,16 @@ module ClaudeMemory
           }
 
           if active_facts > 0
-            stats[:top_predicates] = store.db[:facts]
+            all_predicates = store.db[:facts]
               .where(status: "active")
               .group_and_count(:predicate)
               .order(Sequel.desc(:count))
-              .limit(10)
               .all
               .map { |row| {predicate: row[:predicate], count: row[:count]} }
+
+            stats[:top_predicates] = all_predicates.first(10)
+            stats[:predicates_known], stats[:predicates_novel] =
+              all_predicates.partition { |row| Resolve::PredicatePolicy.known_predicates.include?(row[:predicate]) }
           end
 
           stats

data/lib/claude_memory/mcp/instructions_builder.rb

@@ -108,9 +108,26 @@ module ClaudeMemory
 
         escalation = vec ? "recall_semantic, explain, or fact_graph" : "explain or fact_graph"
         lines << "Start with fast tools (recall, decisions, conventions) before escalating to #{escalation}."
+
+        lines << proactive_recall_guidance
         lines.join("\n")
       end
 
+      # Directive guidance for when Claude should proactively consult memory.
+      # Validated by A/B testing: without these directives, Claude writes code
+      # using known-dangerous patterns (e.g. Sequel.sqlite) and hallucinates
+      # file paths instead of consulting memory for the correct structure.
+      def proactive_recall_guidance
+        <<~GUIDANCE.strip
+          IMPORTANT — check memory proactively in these situations:
+          - Before writing code: call memory.conventions to verify project patterns and avoid known gotchas
+          - Before explaining architecture: call memory.architecture for structural knowledge without file traversal
+          - Before refactoring: call memory.decisions to understand why past choices were made
+          - When asked about preferences: global facts store user environment and style preferences across all projects
+          - When adding to the codebase: recall which files and patterns to follow (memory knows correct paths and relationships)
+        GUIDANCE
+      end
+
       def count_by_predicates(store, predicates)
         store.facts
           .where(status: "active")

data/lib/claude_memory/mcp/server.rb

@@ -5,20 +5,30 @@ require_relative "instructions_builder"
 require_relative "query_guide"
 require_relative "text_summary"
 require_relative "error_classifier"
+require_relative "telemetry"
 
 module ClaudeMemory
   module MCP
+    # MCP JSON-RPC server over stdio.
+    # Reads newline-delimited JSON requests from input, dispatches to Tools,
+    # and writes JSON responses to output.
     class Server
       PROTOCOL_VERSION = "2024-11-05"
 
+      # @param store_or_manager [Store::SQLiteStore, Store::StoreManager] database backend
+      # @param input [IO] input stream for JSON-RPC requests (default: $stdin)
+      # @param output [IO] output stream for JSON-RPC responses (default: $stdout)
       def initialize(store_or_manager, input: $stdin, output: $stdout)
         @store_or_manager = store_or_manager
         @tools = Tools.new(store_or_manager)
+        @telemetry = Telemetry.new(store_or_manager)
         @input = input
         @output = output
         @running = false
       end
 
+      # Start the read loop, blocking until input is exhausted or stop is called.
+      # @return [void]
       def run
         @running = true
         while @running
@@ -29,12 +39,15 @@ module ClaudeMemory
         end
       end
 
+      # Signal the read loop to exit after the current message.
+      # @return [void]
       def stop
         @running = false
       end
 
       private
 
+      # @return [void]
       def handle_message(line)
         return if line.empty?
 
@@ -51,6 +64,7 @@ module ClaudeMemory
         end
       end
 
+      # @return [Hash, nil] JSON-RPC response hash, or nil for notifications
       def process_request(request)
         id = request["id"]
         method = request["method"]
@@ -74,6 +88,7 @@ module ClaudeMemory
         end
       end
 
+      # @return [Hash] initialize response with capabilities and server info
       def handle_initialize(id, _params)
         {
           jsonrpc: "2.0",
@@ -93,6 +108,7 @@ module ClaudeMemory
         }
       end
 
+      # @return [Hash] list of available tool definitions
       def handle_tools_list(id)
         {
           jsonrpc: "2.0",
@@ -103,11 +119,14 @@ module ClaudeMemory
         }
       end
 
+      # @return [Hash] tool result with dual content/structuredContent
       def handle_tools_call(id, params)
         name = params["name"]
         arguments = params["arguments"] || {}
 
-        result = @tools.call(name, arguments)
+        result = @telemetry.record(name, arguments) do
+          @tools.call(name, arguments)
+        end
 
         # Release database connections after each tool call
         # This prevents lock contention with hook commands
@@ -128,6 +147,7 @@ module ClaudeMemory
         }
       end
 
+      # @return [Hash] list of available prompt definitions
       def handle_prompts_list(id)
         {
           jsonrpc: "2.0",
@@ -138,6 +158,7 @@ module ClaudeMemory
         }
       end
 
+      # @return [Hash] prompt content or error if unknown
       def handle_prompts_get(id, params)
         name = params&.dig("name")