claude_memory 0.7.1 → 0.8.0

data/lib/claude_memory/embeddings/resolver.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module ClaudeMemory
+  module Embeddings
+    # Resolves an embedding provider by name or ENV.
+    # Three providers: tfidf (default), fastembed, api.
+    def self.resolve(name = nil, env: ENV)
+      provider = name || env["CLAUDE_MEMORY_EMBEDDING_PROVIDER"] || "tfidf"
+
+      case provider
+      when "tfidf" then Generator.new
+      when "fastembed" then FastembedAdapter.new
+      when "api" then ApiAdapter.new(env: env)
+      else raise ArgumentError, "Unknown embedding provider: #{provider}. Available: tfidf, fastembed, api"
+      end
+    end
+  end
+end

data/lib/claude_memory/hook/context_injector.rb

@@ -9,6 +9,10 @@ module ClaudeMemory
       MAX_DECISIONS = 5
       MAX_CONVENTIONS = 5
       MAX_ARCHITECTURE = 5
+      MAX_UNDISTILLED = 3
+      MAX_TEXT_PER_ITEM = 1500
+
+      FRESH_SESSION_SOURCES = %w[startup resume clear].freeze
 
       QUERIES = {
         decisions: {query: "decision constraint rule requirement", scope: "all"},
@@ -16,8 +20,9 @@ module ClaudeMemory
         architecture: {query: "uses framework implements architecture pattern", scope: "all"}
       }.freeze
 
-      def initialize(manager)
+      def initialize(manager, source: nil)
         @manager = manager
+        @source = source
         @recall = Recall.new(manager)
       end
 
@@ -33,6 +38,11 @@ module ClaudeMemory
         architecture = fetch(:architecture, MAX_ARCHITECTURE)
         sections << format_section("Architecture", architecture) if architecture.any?
 
+        if fresh_session?
+          undistilled = fetch_undistilled(MAX_UNDISTILLED)
+          sections << format_distillation_prompt(undistilled) if undistilled.any?
+        end
+
         return nil if sections.empty?
 
         sections.join("\n")
@@ -40,11 +50,16 @@ module ClaudeMemory
 
       private
 
+      def fresh_session?
+        @source.nil? || FRESH_SESSION_SOURCES.include?(@source)
+      end
+
       def fetch(category, limit)
         config = QUERIES.fetch(category)
         results = @recall.query(config[:query], limit: limit, scope: config[:scope])
         results.map { |r| format_fact(r[:fact]) }
-      rescue => _e
+      rescue => e
+        ClaudeMemory.logger.debug("ContextInjector#fetch(#{category}) failed: #{e.message}")
         []
       end
 
@@ -62,6 +77,47 @@ module ClaudeMemory
         end
       end
 
+      def fetch_undistilled(limit)
+        stores = []
+        stores << @manager.project_store if @manager.project_store
+        stores << @manager.global_store if @manager.global_store
+
+        items = stores.flat_map { |s|
+          s.undistilled_content_items(limit: limit, min_length: 200)
+        }
+
+        items
+          .sort_by { |i| i[:occurred_at] || "" }
+          .reverse
+          .first(limit)
+      rescue => e
+        ClaudeMemory.logger.warn("ContextInjector#fetch_undistilled failed: #{e.message}")
+        []
+      end
+
+      def format_distillation_prompt(items)
+        lines = [
+          "## Pending Knowledge Extraction",
+          "",
+          "The following transcript segments haven't been deeply analyzed yet.",
+          "Extract facts, entities, and decisions, then call `memory.store_extraction`",
+          "followed by `memory.mark_distilled` for each item.",
+          "",
+          "**What to extract:** technology decisions, conventions, preferences, architecture",
+          "**What to skip:** debugging steps, code output, transient errors"
+        ]
+
+        items.each do |item|
+          ago = Core::RelativeTime.format(item[:occurred_at]) || "unknown"
+          truncated = Core::TextBuilder.truncate(item[:raw_text], MAX_TEXT_PER_ITEM)
+          lines << ""
+          lines << "### Content Item #{item[:id]} (#{ago})"
+          lines << truncated
+        end
+
+        lines.join("\n")
+      end
+
       def format_section(title, items)
         items = items.compact.uniq
         return nil if items.empty?

data/lib/claude_memory/hook/distillation_runner.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+module ClaudeMemory
+  module Hook
+    class DistillationRunner
+      MIN_TEXT_LENGTH = 200
+
+      def initialize(store, distiller: Distill::NullDistiller.new)
+        @store = store
+        @distiller = distiller
+      end
+
+      def distill_item(content_id, project_path:, scope: "project")
+        item = @store.get_content_item(content_id)
+        return unless item
+
+        raw_text = item[:raw_text]
+        return unless raw_text && raw_text.length >= MIN_TEXT_LENGTH
+
+        extraction = @distiller.distill(raw_text, content_item_id: content_id)
+        return if extraction.empty?
+
+        resolver = Resolve::Resolver.new(@store)
+        @store.db.transaction do
+          resolve_result = resolver.apply(
+            extraction, content_item_id: content_id,
+            project_path: project_path, scope: scope
+          )
+          @store.record_ingestion_metrics(
+            content_item_id: content_id, input_tokens: 0,
+            output_tokens: 0, facts_extracted: resolve_result[:facts_created]
+          )
+        end
+      rescue => e
+        ClaudeMemory.logger.warn("DistillationRunner#distill_item(#{content_id}) failed: #{e.class} - #{e.message}")
+        ClaudeMemory.logger.warn(e.backtrace.first(5).join("\n"))
+      end
+
+      def distill_batch(project_path:, limit: 5)
+        items = @store.undistilled_content_items(limit: limit, min_length: MIN_TEXT_LENGTH)
+        items.each { |item| distill_item(item[:id], project_path: project_path) }
+        items.size
+      end
+    end
+  end
+end

data/lib/claude_memory/hook/handler.rb

@@ -23,12 +23,20 @@ module ClaudeMemory
         raise PayloadError, "Missing required field: transcript_path" if transcript_path.nil? || transcript_path.empty?
 
         ingester = Ingest::Ingester.new(@store, env: @env)
-        ingester.ingest(
+        result = ingester.ingest(
           source: "claude_code",
           session_id: session_id,
           transcript_path: transcript_path,
           project_path: project_path
         )
+
+        if result[:status] == :ingested && result[:content_id]
+          DistillationRunner.new(@store).distill_item(
+            result[:content_id], project_path: project_path
+          )
+        end
+
+        result
       rescue Ingest::TranscriptReader::FileNotFoundError => e
         # Transcript file doesn't exist (e.g., headless Claude session)
         # This is expected, not an error - return success with no-op status
@@ -56,7 +64,8 @@ module ClaudeMemory
         manager = @manager || build_manager(payload)
         manager.ensure_both!
 
-        injector = ContextInjector.new(manager)
+        source = payload["source"]
+        injector = ContextInjector.new(manager, source: source)
         context_text = injector.generate_context
 
         {status: :ok, context: context_text}

data/lib/claude_memory/index/vector_index.rb

@@ -6,13 +6,16 @@ module ClaudeMemory
     # Follows the same lazy-init pattern as LexicalFTS:
     # the extension and virtual table are created on first use.
     class VectorIndex
-      EMBEDDING_DIMENSIONS = 384
+      DEFAULT_DIMENSIONS = 384
+
+      attr_reader :dimensions
 
       def initialize(store)
         @store = store
         @db = store.db
         @available = nil
         @vec_table_ensured = false
+        @dimensions = store.get_meta("embedding_dimensions")&.to_i || DEFAULT_DIMENSIONS
       end
 
       # Is the sqlite-vec extension loadable?
@@ -121,6 +124,16 @@ module ClaudeMemory
         indexed_ids.size
       end
 
+      # Delete all entries from the vec0 virtual table.
+      # Used when clearing stale embeddings after a dimension change.
+      def clear!
+        return false unless available?
+
+        ensure_vec_table!
+        @db.run("DELETE FROM facts_vec")
+        true
+      end
+
       # Number of entries in the vec0 virtual table
       def count
         return 0 unless available?
@@ -162,7 +175,7 @@ module ClaudeMemory
 
         @db.run(<<~SQL)
           CREATE VIRTUAL TABLE IF NOT EXISTS facts_vec
-          USING vec0(fact_id INTEGER PRIMARY KEY, embedding float[#{EMBEDDING_DIMENSIONS}] distance_metric=cosine)
+          USING vec0(fact_id INTEGER PRIMARY KEY, embedding float[#{@dimensions}] distance_metric=cosine)
         SQL
         @vec_table_ensured = true
       end

data/lib/claude_memory/infrastructure/schema_validator.rb

@@ -166,7 +166,7 @@ module ClaudeMemory
       end
 
       def check_embedding_dimensions(issues)
-        # Check that all embeddings have correct dimensions (384)
+        expected = @store.get_meta("embedding_dimensions")&.to_i || 384
         facts_with_embeddings = @store.facts
           .where(Sequel.~(embedding_json: nil))
           .select(:id, :embedding_json)
@@ -174,8 +174,8 @@ module ClaudeMemory
 
         facts_with_embeddings.each do |fact|
           embedding = JSON.parse(fact[:embedding_json])
-          if embedding.size != 384
-            issues << {severity: "error", message: "Fact #{fact[:id]} has embedding with incorrect dimensions (#{embedding.size}, expected 384)"}
+          if embedding.size != expected
+            issues << {severity: "error", message: "Fact #{fact[:id]} has embedding with incorrect dimensions (#{embedding.size}, expected #{expected})"}
             break  # Only report first occurrence
           end
         end

data/lib/claude_memory/mcp/handlers/context_handlers.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+module ClaudeMemory
+  module MCP
+    module Handlers
+      # Context-aware query handlers (facts by tool, branch, directory)
+      module ContextHandlers
+        def facts_by_tool(args)
+          tool_name = args["tool_name"]
+          scope = extract_scope(args)
+          limit = extract_limit(args, default: 20)
+
+          results = @recall.facts_by_tool(tool_name, limit: limit, scope: scope)
+          ResponseFormatter.format_tool_facts(tool_name, scope, results)
+        end
+
+        def facts_by_context(args)
+          scope = extract_scope(args)
+          limit = extract_limit(args, default: 20)
+
+          if args["git_branch"]
+            results = @recall.facts_by_branch(args["git_branch"], limit: limit, scope: scope)
+            context_type = "git_branch"
+            context_value = args["git_branch"]
+          elsif args["cwd"]
+            results = @recall.facts_by_directory(args["cwd"], limit: limit, scope: scope)
+            context_type = "cwd"
+            context_value = args["cwd"]
+          else
+            return {error: "Must provide either git_branch or cwd parameter"}
+          end
+
+          ResponseFormatter.format_context_facts(context_type, context_value, scope, results)
+        end
+      end
+    end
+  end
+end

data/lib/claude_memory/mcp/handlers/management_handlers.rb

@@ -0,0 +1,145 @@
+# frozen_string_literal: true
+
+module ClaudeMemory
+  module MCP
+    module Handlers
+      # Management tool handlers (store_extraction, promote, sweep, changes, conflicts)
+      module ManagementHandlers
+        def store_extraction(args)
+          scope = args["scope"] || "project"
+          store = get_store_for_scope(scope)
+          return {error: "Database not available"} unless store
+
+          entities = (args["entities"] || []).map { |e| symbolize_keys(e) }
+          facts = (args["facts"] || []).map { |f| symbolize_keys(f) }
+          decisions = (args["decisions"] || []).map { |d| symbolize_keys(d) }
+
+          config = Configuration.new
+          project_path = config.project_dir
+          occurred_at = Time.now.utc.iso8601
+
+          searchable_text = Core::TextBuilder.build_searchable_text(entities, facts, decisions)
+          content_item_id = create_synthetic_content_item(store, searchable_text, project_path, occurred_at)
+          index_content_item(store, content_item_id, searchable_text)
+
+          extraction = Distill::Extraction.new(
+            entities: entities,
+            facts: facts,
+            decisions: decisions,
+            signals: []
+          )
+
+          resolver = Resolve::Resolver.new(store)
+          result = resolver.apply(
+            extraction,
+            content_item_id: content_item_id,
+            occurred_at: occurred_at,
+            project_path: project_path,
+            scope: scope
+          )
+
+          {
+            success: true,
+            scope: scope,
+            entities_created: result[:entities_created],
+            facts_created: result[:facts_created],
+            facts_superseded: result[:facts_superseded],
+            conflicts_created: result[:conflicts_created]
+          }
+        end
+
+        def promote(args)
+          return {error: "Promote requires StoreManager"} unless @manager
+
+          fact_id = args["fact_id"]
+          global_fact_id = @manager.promote_fact(fact_id)
+
+          if global_fact_id
+            {
+              success: true,
+              project_fact_id: fact_id,
+              global_fact_id: global_fact_id,
+              message: "Fact promoted to global memory"
+            }
+          else
+            {error: "Fact #{fact_id} not found in project database"}
+          end
+        end
+
+        def sweep_now(args)
+          scope = args["scope"] || "project"
+          store = get_store_for_scope(scope)
+          return {error: "Database not available"} unless store
+
+          sweeper = Sweep::Sweeper.new(store)
+          budget = args["budget_seconds"] || 5
+          stats = if args["escalate"]
+            sweeper.run_with_escalation!(budget_seconds: budget)
+          else
+            sweeper.run!(budget_seconds: budget)
+          end
+          ResponseFormatter.format_sweep_stats(scope, stats)
+        end
+
+        def changes(args)
+          since = args["since"] || (Time.now - 86400 * 7).utc.iso8601
+          scope = args["scope"] || "all"
+          list = @recall.changes(since: since, limit: args["limit"] || 20, scope: scope)
+          ResponseFormatter.format_changes(since, list)
+        end
+
+        def conflicts(args)
+          scope = args["scope"] || "all"
+          list = @recall.conflicts(scope: scope)
+          ResponseFormatter.format_conflicts(list)
+        end
+
+        def mark_distilled(args)
+          content_item_id = args["content_item_id"]
+          facts_extracted = args["facts_extracted"] || 0
+
+          store = find_store_for_content_item(content_item_id)
+          return {error: "Content item #{content_item_id} not found"} unless store
+
+          store.record_ingestion_metrics(
+            content_item_id: content_item_id,
+            input_tokens: 0,
+            output_tokens: 0,
+            facts_extracted: facts_extracted
+          )
+
+          {
+            success: true,
+            content_item_id: content_item_id,
+            facts_extracted: facts_extracted
+          }
+        end
+
+        private
+
+        def create_synthetic_content_item(store, text, project_path, occurred_at)
+          text_hash = Digest::SHA256.hexdigest(text)
+          store.upsert_content_item(
+            source: "mcp_extraction",
+            session_id: "mcp-#{Time.now.to_i}",
+            transcript_path: nil,
+            project_path: project_path,
+            text_hash: text_hash,
+            byte_len: text.bytesize,
+            raw_text: text,
+            occurred_at: occurred_at
+          )
+        end
+
+        def index_content_item(store, content_item_id, text)
+          fts = Index::LexicalFTS.new(store)
+          fts.index_content_item(content_item_id, text)
+        end
+
+        def symbolize_keys(hash)
+          Core::TextBuilder.symbolize_keys(hash)
+        end
+      end
+    end
+  end
+end

data/lib/claude_memory/mcp/handlers/query_handlers.rb

@@ -0,0 +1,115 @@
+# frozen_string_literal: true
+
+module ClaudeMemory
+  module MCP
+    module Handlers
+      # Query and recall tool handlers
+      module QueryHandlers
+        def recall(args)
+          return database_not_found_error unless databases_exist?
+
+          scope = extract_scope(args)
+          limit = extract_limit(args)
+          compact = args["compact"] == true
+          query = args["query"]
+          intent = extract_intent(args)
+          results = @recall.query(query, limit: limit, scope: scope, include_raw_text: !compact, intent: intent)
+          ResponseFormatter.format_recall_results(results, compact: compact, query: query)
+        rescue Sequel::DatabaseError, Sequel::DatabaseConnectionError, Errno::ENOENT => e
+          classified_error(e, tool_name: "memory.recall")
+        end
+
+        def recall_index(args)
+          scope = extract_scope(args)
+          limit = extract_limit(args, default: 20)
+          intent = extract_intent(args)
+          results = @recall.query_index(args["query"], limit: limit, scope: scope, intent: intent)
+          ResponseFormatter.format_index_results(args["query"], scope, results)
+        end
+
+        def recall_details(args)
+          fact_ids = args["fact_ids"]
+          scope = args["scope"] || "project"
+
+          explanations = fact_ids.map do |fact_id|
+            explanation = @recall.explain(fact_id, scope: scope)
+            next nil if explanation.is_a?(Core::NullExplanation)
+
+            ResponseFormatter.format_detailed_explanation(explanation)
+          end.compact
+
+          {
+            fact_count: explanations.size,
+            facts: explanations
+          }
+        end
+
+        def explain(args)
+          scope = args["scope"] || "project"
+          explanation = @recall.explain(args["fact_id"], scope: scope)
+          return {error: "Fact not found in #{scope} database"} if explanation.is_a?(Core::NullExplanation)
+
+          ResponseFormatter.format_explanation(explanation, scope)
+        end
+
+        def recall_semantic(args)
+          query = args["query"]
+          mode = (args["mode"] || "both").to_sym
+          scope = extract_scope(args)
+          limit = extract_limit(args)
+          compact = args["compact"] == true
+          explain = args["explain"] == true
+          intent = extract_intent(args)
+
+          results = @recall.query_semantic(query, limit: limit, scope: scope, mode: mode, explain: explain, intent: intent)
+          ResponseFormatter.format_semantic_results(query, mode.to_s, scope, results, compact: compact)
+        end
+
+        def search_concepts(args)
+          concepts = args["concepts"]
+          scope = extract_scope(args)
+          limit = extract_limit(args)
+          compact = args["compact"] == true
+
+          return {error: "Must provide 2-5 concepts"} unless (2..5).cover?(concepts.size)
+
+          results = @recall.query_concepts(concepts, limit: limit, scope: scope)
+          ResponseFormatter.format_concept_results(concepts, scope, results, compact: compact)
+        end
+
+        def fact_graph(args)
+          fact_id = args["fact_id"]
+          depth = args["depth"] || 2
+          scope = args["scope"] || "project"
+
+          graph = @recall.fact_graph(fact_id, depth: depth, scope: scope)
+
+          return {error: "Fact #{fact_id} not found in #{scope} database"} if graph[:node_count] == 0
+
+          graph
+        end
+
+        def undistilled(args)
+          limit = args["limit"] || 3
+          min_length = args["min_length"] || 200
+          max_text = 2000
+
+          items = collect_undistilled_items(limit: limit, min_length: min_length)
+
+          {
+            count: items.size,
+            items: items.map { |item|
+              {
+                content_item_id: item[:id],
+                occurred_at: item[:occurred_at],
+                occurred_ago: Core::RelativeTime.format(item[:occurred_at]),
+                project_path: item[:project_path],
+                raw_text: Core::TextBuilder.truncate(item[:raw_text], max_text)
+              }
+            }
+          }
+        end
+      end
+    end
+  end
+end