claude_memory 0.7.1 → 0.9.0

data/lib/claude_memory/embeddings/inspector.rb

@@ -0,0 +1,91 @@
+# frozen_string_literal: true
+
+module ClaudeMemory
+  module Embeddings
+    # Reads embedding metadata from global and project databases.
+    # Returns structured data — no I/O formatting or stdout output.
+    #
+    # Used by EmbeddingsCommand to separate DB concerns from presentation.
+    class Inspector
+      DatabaseState = Data.define(:label, :provider, :dimensions)
+      DimensionResult = Data.define(:label, :status, :stored_dims, :stored_provider, :current_dims)
+
+      def database_states
+        results = []
+
+        with_each_store do |label, store|
+          provider = store.get_meta("embedding_provider")
+          dims = store.get_meta("embedding_dimensions")
+
+          next unless provider || dims
+
+          results << DatabaseState.new(label: label, provider: provider, dimensions: dims)
+        end
+
+        results
+      end
+
+      def dimension_checks(provider_name, model_name)
+        results = []
+
+        with_each_store do |label, store|
+          stored_dims = store.get_meta("embedding_dimensions")&.to_i
+          stored_provider = store.get_meta("embedding_provider")
+
+          if stored_dims
+            current_dims = resolve_current_dimensions(provider_name, model_name)
+
+            status = if current_dims && current_dims != stored_dims
+              :mismatch
+            else
+              :match
+            end
+
+            results << DimensionResult.new(
+              label: label,
+              status: status,
+              stored_dims: stored_dims,
+              stored_provider: stored_provider,
+              current_dims: current_dims
+            )
+          else
+            results << DimensionResult.new(
+              label: label,
+              status: :fresh,
+              stored_dims: nil,
+              stored_provider: nil,
+              current_dims: nil
+            )
+          end
+        end
+
+        results
+      end
+
+      private
+
+      def resolve_current_dimensions(provider_name, model_name)
+        if model_name
+          ModelRegistry.dimensions_for(model_name)
+        else
+          ModelRegistry.default_for_provider(provider_name)&.dimensions
+        end
+      end
+
+      def with_each_store
+        config = Configuration.new
+
+        [["global", config.global_db_path], ["project", config.project_db_path]].each do |label, path|
+          next unless File.exist?(path)
+
+          store = Store::SQLiteStore.new(path)
+          begin
+            yield label, store
+          ensure
+            store.close
+          end
+        end
+      end
+    end
+  end
+end

data/lib/claude_memory/embeddings/model_registry.rb

@@ -0,0 +1,210 @@
+# frozen_string_literal: true
+
+module ClaudeMemory
+  module Embeddings
+    # Registry of known embedding models with their properties.
+    # Enables model validation, dimension lookup, and discoverability.
+    #
+    # Models are registered by canonical name (e.g., "BAAI/bge-small-en-v1.5")
+    # with provider type, dimensions, and description.
+    #
+    # Usage:
+    #   ModelRegistry.find("BAAI/bge-small-en-v1.5")
+    #   # => {provider: "fastembed", dimensions: 384, description: "...", ...}
+    #
+    #   ModelRegistry.models_for_provider("fastembed")
+    #   # => [...]
+    #
+    class ModelRegistry
+      ModelInfo = Data.define(:name, :provider, :dimensions, :description, :size_mb, :max_tokens)
+
+      # Known models with validated dimensions.
+      # Fastembed models sourced from fastembed-rb SUPPORTED_MODELS.
+      # API models sourced from provider documentation.
+      MODELS = [
+        # --- fastembed: local ONNX models (no API key needed) ---
+        ModelInfo.new(
+          name: "BAAI/bge-small-en-v1.5",
+          provider: "fastembed",
+          dimensions: 384,
+          description: "Fast English embedding (default)",
+          size_mb: 67,
+          max_tokens: 512
+        ),
+        ModelInfo.new(
+          name: "BAAI/bge-base-en-v1.5",
+          provider: "fastembed",
+          dimensions: 768,
+          description: "Balanced English embedding, higher accuracy",
+          size_mb: 210,
+          max_tokens: 512
+        ),
+        ModelInfo.new(
+          name: "BAAI/bge-large-en-v1.5",
+          provider: "fastembed",
+          dimensions: 1024,
+          description: "High accuracy English embedding",
+          size_mb: 1200,
+          max_tokens: 512
+        ),
+        ModelInfo.new(
+          name: "sentence-transformers/all-MiniLM-L6-v2",
+          provider: "fastembed",
+          dimensions: 384,
+          description: "Lightweight general-purpose sentence embedding",
+          size_mb: 90,
+          max_tokens: 512
+        ),
+        ModelInfo.new(
+          name: "intfloat/multilingual-e5-small",
+          provider: "fastembed",
+          dimensions: 384,
+          description: "Multilingual embedding, 100+ languages",
+          size_mb: 450,
+          max_tokens: 512
+        ),
+        ModelInfo.new(
+          name: "intfloat/multilingual-e5-base",
+          provider: "fastembed",
+          dimensions: 768,
+          description: "Larger multilingual embedding",
+          size_mb: 1110,
+          max_tokens: 512
+        ),
+        ModelInfo.new(
+          name: "nomic-ai/nomic-embed-text-v1.5",
+          provider: "fastembed",
+          dimensions: 768,
+          description: "Long context (8192 tokens) with Matryoshka support",
+          size_mb: 520,
+          max_tokens: 8192
+        ),
+        ModelInfo.new(
+          name: "jinaai/jina-embeddings-v2-small-en",
+          provider: "fastembed",
+          dimensions: 512,
+          description: "Small English embedding, 8192 token context",
+          size_mb: 60,
+          max_tokens: 8192
+        ),
+        ModelInfo.new(
+          name: "jinaai/jina-embeddings-v2-base-en",
+          provider: "fastembed",
+          dimensions: 768,
+          description: "Base English embedding, 8192 token context",
+          size_mb: 520,
+          max_tokens: 8192
+        ),
+
+        # --- api: OpenAI-compatible endpoints ---
+        ModelInfo.new(
+          name: "text-embedding-3-small",
+          provider: "api",
+          dimensions: 1536,
+          description: "OpenAI small embedding (default API model)",
+          size_mb: nil,
+          max_tokens: 8191
+        ),
+        ModelInfo.new(
+          name: "text-embedding-3-large",
+          provider: "api",
+          dimensions: 3072,
+          description: "OpenAI large embedding, highest accuracy",
+          size_mb: nil,
+          max_tokens: 8191
+        ),
+        ModelInfo.new(
+          name: "text-embedding-ada-002",
+          provider: "api",
+          dimensions: 1536,
+          description: "OpenAI legacy embedding",
+          size_mb: nil,
+          max_tokens: 8191
+        ),
+        ModelInfo.new(
+          name: "voyage-3",
+          provider: "api",
+          dimensions: 1024,
+          description: "Voyage AI general-purpose embedding",
+          size_mb: nil,
+          max_tokens: 32000
+        ),
+        ModelInfo.new(
+          name: "voyage-3-lite",
+          provider: "api",
+          dimensions: 512,
+          description: "Voyage AI lightweight embedding",
+          size_mb: nil,
+          max_tokens: 32000
+        ),
+        ModelInfo.new(
+          name: "voyage-code-3",
+          provider: "api",
+          dimensions: 1024,
+          description: "Voyage AI code-optimized embedding",
+          size_mb: nil,
+          max_tokens: 32000
+        ),
+
+        # --- tfidf: built-in, no dependencies ---
+        ModelInfo.new(
+          name: "tfidf",
+          provider: "tfidf",
+          dimensions: 384,
+          description: "Built-in TF-IDF embedding (no dependencies)",
+          size_mb: 0,
+          max_tokens: nil
+        )
+      ].freeze
+
+      MODELS_BY_NAME = MODELS.each_with_object({}) { |m, h| h[m.name] = m }.freeze
+
+      DEFAULTS = {
+        "fastembed" => "BAAI/bge-small-en-v1.5",
+        "api" => "text-embedding-3-small",
+        "tfidf" => "tfidf"
+      }.freeze
+
+      # Find a model by name.
+      # @param name [String] model name (e.g., "BAAI/bge-small-en-v1.5")
+      # @return [ModelInfo, nil]
+      def self.find(name)
+        MODELS_BY_NAME[name]
+      end
+
+      # List all models for a given provider.
+      # @param provider [String] "fastembed", "api", or "tfidf"
+      # @return [Array<ModelInfo>]
+      def self.models_for_provider(provider)
+        MODELS.select { |m| m.provider == provider }
+      end
+
+      # All known model names.
+      # @return [Array<String>]
+      def self.model_names
+        MODELS.map(&:name)
+      end
+
+      # All provider names.
+      # @return [Array<String>]
+      def self.providers
+        MODELS.map(&:provider).uniq
+      end
+
+      # Look up dimensions for a model name. Returns nil if unknown.
+      # @param name [String] model name
+      # @return [Integer, nil]
+      def self.dimensions_for(name)
+        find(name)&.dimensions
+      end
+
+      # Return the default ModelInfo for a provider.
+      # @param provider [String] "fastembed", "api", or "tfidf"
+      # @return [ModelInfo, nil]
+      def self.default_for_provider(provider)
+        default_name = DEFAULTS[provider]
+        find(default_name) if default_name
+      end
+    end
+  end
+end

data/lib/claude_memory/embeddings/resolver.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+module ClaudeMemory
+  module Embeddings
+    # Resolves an embedding provider by name, model, or ENV.
+    #
+    # Provider selection (in priority order):
+    #   1. Explicit name parameter
+    #   2. CLAUDE_MEMORY_EMBEDDING_PROVIDER env var
+    #   3. Default: "tfidf"
+    #
+    # Model selection is forwarded to the provider via CLAUDE_MEMORY_EMBEDDING_MODEL
+    # or the model parameter. The model can also imply the provider:
+    #   - "BAAI/bge-small-en-v1.5" → fastembed
+    #   - "text-embedding-3-small" → api
+    #
+    # Examples:
+    #   Embeddings.resolve                                    # tfidf default
+    #   Embeddings.resolve("fastembed")                       # fastembed with default model
+    #   Embeddings.resolve("fastembed", model: "BAAI/bge-base-en-v1.5")
+    #   Embeddings.resolve(model: "text-embedding-3-small")   # auto-detects api provider
+    #
+    def self.resolve(name = nil, model: nil, env: ENV)
+      model ||= env["CLAUDE_MEMORY_EMBEDDING_MODEL"]
+      provider = name || env["CLAUDE_MEMORY_EMBEDDING_PROVIDER"] || infer_provider(model) || "tfidf"
+
+      case provider
+      when "tfidf" then Generator.new
+      when "fastembed" then FastembedAdapter.new(model_name: model, env: env)
+      when "api" then ApiAdapter.new(model: model, env: env)
+      else raise ArgumentError, "Unknown embedding provider: #{provider}. Available: tfidf, fastembed, api"
+      end
+    end
+
+    # Infer provider from a model name using the registry.
+    # Returns nil if the model is unknown.
+    def self.infer_provider(model)
+      return nil unless model
+
+      ModelRegistry.find(model)&.provider
+    end
+    private_class_method :infer_provider
+  end
+end

data/lib/claude_memory/hook/context_injector.rb

@@ -9,6 +9,10 @@ module ClaudeMemory
       MAX_DECISIONS = 5
       MAX_CONVENTIONS = 5
       MAX_ARCHITECTURE = 5
+      MAX_UNDISTILLED = 3
+      MAX_TEXT_PER_ITEM = 1500
+
+      FRESH_SESSION_SOURCES = %w[startup resume clear].freeze
 
       QUERIES = {
         decisions: {query: "decision constraint rule requirement", scope: "all"},
@@ -16,8 +20,9 @@ module ClaudeMemory
         architecture: {query: "uses framework implements architecture pattern", scope: "all"}
       }.freeze
 
-      def initialize(manager)
+      def initialize(manager, source: nil)
         @manager = manager
+        @source = source
         @recall = Recall.new(manager)
       end
 
@@ -33,6 +38,11 @@ module ClaudeMemory
         architecture = fetch(:architecture, MAX_ARCHITECTURE)
         sections << format_section("Architecture", architecture) if architecture.any?
 
+        if fresh_session?
+          undistilled = fetch_undistilled(MAX_UNDISTILLED)
+          sections << format_distillation_prompt(undistilled) if undistilled.any?
+        end
+
         return nil if sections.empty?
 
         sections.join("\n")
@@ -40,11 +50,16 @@ module ClaudeMemory
 
       private
 
+      def fresh_session?
+        @source.nil? || FRESH_SESSION_SOURCES.include?(@source)
+      end
+
       def fetch(category, limit)
         config = QUERIES.fetch(category)
         results = @recall.query(config[:query], limit: limit, scope: config[:scope])
         results.map { |r| format_fact(r[:fact]) }
-      rescue => _e
+      rescue => e
+        ClaudeMemory.logger.debug("ContextInjector#fetch(#{category}) failed: #{e.message}")
         []
       end
 
@@ -62,6 +77,47 @@ module ClaudeMemory
         end
       end
 
+      def fetch_undistilled(limit)
+        stores = []
+        stores << @manager.project_store if @manager.project_store
+        stores << @manager.global_store if @manager.global_store
+
+        items = stores.flat_map { |s|
+          s.undistilled_content_items(limit: limit, min_length: 200)
+        }
+
+        items
+          .sort_by { |i| i[:occurred_at] || "" }
+          .reverse
+          .first(limit)
+      rescue => e
+        ClaudeMemory.logger.warn("ContextInjector#fetch_undistilled failed: #{e.message}")
+        []
+      end
+
+      def format_distillation_prompt(items)
+        lines = [
+          "## Pending Knowledge Extraction",
+          "",
+          "The following transcript segments haven't been deeply analyzed yet.",
+          "Extract facts, entities, and decisions, then call `memory.store_extraction`",
+          "followed by `memory.mark_distilled` for each item.",
+          "",
+          "**What to extract:** technology decisions, conventions, preferences, architecture",
+          "**What to skip:** debugging steps, code output, transient errors"
+        ]
+
+        items.each do |item|
+          ago = Core::RelativeTime.format(item[:occurred_at]) || "unknown"
+          truncated = Core::TextBuilder.truncate(item[:raw_text], MAX_TEXT_PER_ITEM)
+          lines << ""
+          lines << "### Content Item #{item[:id]} (#{ago})"
+          lines << truncated
+        end
+
+        lines.join("\n")
+      end
+
       def format_section(title, items)
         items = items.compact.uniq
         return nil if items.empty?

data/lib/claude_memory/hook/distillation_runner.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+module ClaudeMemory
+  module Hook
+    class DistillationRunner
+      MIN_TEXT_LENGTH = 200
+
+      def initialize(store, distiller: Distill::NullDistiller.new)
+        @store = store
+        @distiller = distiller
+      end
+
+      def distill_item(content_id, project_path:, scope: "project")
+        item = @store.get_content_item(content_id)
+        return unless item
+
+        raw_text = item[:raw_text]
+        return unless raw_text && raw_text.length >= MIN_TEXT_LENGTH
+
+        extraction = @distiller.distill(raw_text, content_item_id: content_id)
+        return if extraction.empty?
+
+        resolver = Resolve::Resolver.new(@store)
+        @store.db.transaction do
+          resolve_result = resolver.apply(
+            extraction, content_item_id: content_id,
+            project_path: project_path, scope: scope
+          )
+          @store.record_ingestion_metrics(
+            content_item_id: content_id, input_tokens: 0,
+            output_tokens: 0, facts_extracted: resolve_result[:facts_created]
+          )
+        end
+      rescue => e
+        ClaudeMemory.logger.warn("DistillationRunner#distill_item(#{content_id}) failed: #{e.class} - #{e.message}")
+        ClaudeMemory.logger.warn(e.backtrace.first(5).join("\n"))
+      end
+
+      def distill_batch(project_path:, limit: 5)
+        items = @store.undistilled_content_items(limit: limit, min_length: MIN_TEXT_LENGTH)
+        items.each { |item| distill_item(item[:id], project_path: project_path) }
+        items.size
+      end
+    end
+  end
+end

data/lib/claude_memory/hook/handler.rb

@@ -23,12 +23,20 @@ module ClaudeMemory
         raise PayloadError, "Missing required field: transcript_path" if transcript_path.nil? || transcript_path.empty?
 
         ingester = Ingest::Ingester.new(@store, env: @env)
-        ingester.ingest(
+        result = ingester.ingest(
           source: "claude_code",
           session_id: session_id,
           transcript_path: transcript_path,
           project_path: project_path
         )
+
+        if result[:status] == :ingested && result[:content_id]
+          DistillationRunner.new(@store).distill_item(
+            result[:content_id], project_path: project_path
+          )
+        end
+
+        result
       rescue Ingest::TranscriptReader::FileNotFoundError => e
         # Transcript file doesn't exist (e.g., headless Claude session)
         # This is expected, not an error - return success with no-op status
@@ -56,7 +64,8 @@ module ClaudeMemory
         manager = @manager || build_manager(payload)
         manager.ensure_both!
 
-        injector = ContextInjector.new(manager)
+        source = payload["source"]
+        injector = ContextInjector.new(manager, source: source)
         context_text = injector.generate_context
 
         {status: :ok, context: context_text}

data/lib/claude_memory/index/vector_index.rb

@@ -6,13 +6,16 @@ module ClaudeMemory
     # Follows the same lazy-init pattern as LexicalFTS:
     # the extension and virtual table are created on first use.
     class VectorIndex
-      EMBEDDING_DIMENSIONS = 384
+      DEFAULT_DIMENSIONS = 384
+
+      attr_reader :dimensions
 
       def initialize(store)
         @store = store
         @db = store.db
         @available = nil
         @vec_table_ensured = false
+        @dimensions = store.get_meta("embedding_dimensions")&.to_i || DEFAULT_DIMENSIONS
       end
 
       # Is the sqlite-vec extension loadable?
@@ -121,6 +124,16 @@ module ClaudeMemory
         indexed_ids.size
       end
 
+      # Delete all entries from the vec0 virtual table.
+      # Used when clearing stale embeddings after a dimension change.
+      def clear!
+        return false unless available?
+
+        ensure_vec_table!
+        @db.run("DELETE FROM facts_vec")
+        true
+      end
+
       # Number of entries in the vec0 virtual table
       def count
         return 0 unless available?
@@ -162,7 +175,7 @@ module ClaudeMemory
 
         @db.run(<<~SQL)
           CREATE VIRTUAL TABLE IF NOT EXISTS facts_vec
-          USING vec0(fact_id INTEGER PRIMARY KEY, embedding float[#{EMBEDDING_DIMENSIONS}] distance_metric=cosine)
+          USING vec0(fact_id INTEGER PRIMARY KEY, embedding float[#{@dimensions}] distance_metric=cosine)
         SQL
         @vec_table_ensured = true
       end

data/lib/claude_memory/infrastructure/schema_validator.rb

@@ -166,7 +166,7 @@ module ClaudeMemory
       end
 
       def check_embedding_dimensions(issues)
-        # Check that all embeddings have correct dimensions (384)
+        expected = @store.get_meta("embedding_dimensions")&.to_i || 384
         facts_with_embeddings = @store.facts
           .where(Sequel.~(embedding_json: nil))
           .select(:id, :embedding_json)
@@ -174,8 +174,8 @@ module ClaudeMemory
 
         facts_with_embeddings.each do |fact|
           embedding = JSON.parse(fact[:embedding_json])
-          if embedding.size != 384
-            issues << {severity: "error", message: "Fact #{fact[:id]} has embedding with incorrect dimensions (#{embedding.size}, expected 384)"}
+          if embedding.size != expected
+            issues << {severity: "error", message: "Fact #{fact[:id]} has embedding with incorrect dimensions (#{embedding.size}, expected #{expected})"}
             break  # Only report first occurrence
           end
         end

data/lib/claude_memory/ingest/ingester.rb

@@ -4,7 +4,17 @@ require "digest"
 
 module ClaudeMemory
   module Ingest
+    # Delta-based transcript ingestion with cursor tracking.
+    # Reads new content from transcripts, extracts metadata and tool calls,
+    # sanitizes private tags, and persists to the content_items table with FTS indexing.
     class Ingester
+      # @param store [Store::SQLiteStore] database store for persistence
+      # @param fts [Index::LexicalFTS, nil] full-text search index (default: new from store)
+      # @param env [Hash] environment variables
+      # @param metadata_extractor [MetadataExtractor, nil] extracts git branch, cwd, etc.
+      # @param tool_extractor [ToolExtractor, nil] extracts tool calls from transcript text
+      # @param tool_filter [ToolFilter, nil] filters irrelevant tool calls
+      # @param observation_compressor [ObservationCompressor, nil] compresses tool observations
       def initialize(store, fts: nil, env: ENV, metadata_extractor: nil, tool_extractor: nil, tool_filter: nil, observation_compressor: nil)
         @store = store
         @fts = fts || Index::LexicalFTS.new(store)
@@ -15,6 +25,13 @@ module ClaudeMemory
         @observation_compressor = observation_compressor || ObservationCompressor.new
       end
 
+      # Ingest new content from a transcript file
+      # @param source [String] content source identifier (e.g., "hook", "cli")
+      # @param session_id [String] Claude session ID
+      # @param transcript_path [String] path to the transcript file
+      # @param project_path [String, nil] project root (defaults to detected path)
+      # @return [Hash] result with :status (:ingested, :skipped, or :no_change),
+      #   :content_id, :bytes_read, and optional :reason
       def ingest(source:, session_id:, transcript_path:, project_path: nil)
         unless should_ingest?(transcript_path)
           ClaudeMemory.logger.debug("ingest", message: "Skipped unchanged file", transcript_path: transcript_path)

data/lib/claude_memory/mcp/handlers/context_handlers.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+module ClaudeMemory
+  module MCP
+    module Handlers
+      # Context-aware query handlers (facts by tool, branch, directory)
+      module ContextHandlers
+        def facts_by_tool(args)
+          tool_name = args["tool_name"]
+          scope = extract_scope(args)
+          limit = extract_limit(args, default: 20)
+
+          results = @recall.facts_by_tool(tool_name, limit: limit, scope: scope)
+          ResponseFormatter.format_tool_facts(tool_name, scope, results)
+        end
+
+        def facts_by_context(args)
+          scope = extract_scope(args)
+          limit = extract_limit(args, default: 20)
+
+          if args["git_branch"]
+            results = @recall.facts_by_branch(args["git_branch"], limit: limit, scope: scope)
+            context_type = "git_branch"
+            context_value = args["git_branch"]
+          elsif args["cwd"]
+            results = @recall.facts_by_directory(args["cwd"], limit: limit, scope: scope)
+            context_type = "cwd"
+            context_value = args["cwd"]
+          else
+            return {error: "Must provide either git_branch or cwd parameter"}
+          end
+
+          ResponseFormatter.format_context_facts(context_type, context_value, scope, results)
+        end
+      end
+    end
+  end
+end