claude_memory 0.5.1 → 0.7.0

data/lib/claude_memory/mcp/tool_definitions.rb

@@ -5,27 +5,37 @@ module ClaudeMemory
     # MCP tool definitions for Claude Memory
     # Pure data structure - no logic, just tool schemas
     module ToolDefinitions
+      # Annotations for read-only query tools (safe to call anytime)
+      READ_ONLY = {readOnlyHint: true, idempotentHint: true, destructiveHint: false}.freeze
+
+      # Annotations for state-changing but non-destructive tools
+      WRITE = {readOnlyHint: false, idempotentHint: false, destructiveHint: false}.freeze
+
+      # Annotations for idempotent writes (safe to retry)
+      WRITE_IDEMPOTENT = {readOnlyHint: false, idempotentHint: true, destructiveHint: false}.freeze
+
       # Returns array of tool definitions for MCP protocol
       # @return [Array<Hash>] Tool definitions with name, description, and inputSchema
       def self.all
         [
           {
             name: "memory.recall",
-            description: "Search facts matching a query from both global and project memory databases.",
+            description: "Search facts matching a query from both global and project memory databases. Returns full facts with provenance (~800 tokens/result, ~300 with compact: true). For token-efficient browsing, use memory.recall_index first (~200 tokens/result), then memory.recall_details for selected facts.",
             inputSchema: {
               type: "object",
               properties: {
                 query: {type: "string", description: "Search query for existing knowledge (e.g., 'authentication flow', 'error handling', 'database setup')"},
                 limit: {type: "integer", description: "Max results", default: 10},
                 scope: {type: "string", enum: ["all", "global", "project"], description: "Filter by scope: 'all' (default), 'global', or 'project'", default: "all"},
-                compact: {type: "boolean", description: "Omit provenance receipts for ~60% smaller responses", default: false}
+                compact: {type: "boolean", description: "Omit provenance receipts for ~60% smaller responses (~800 → ~300 tokens/result)", default: false}
               },
               required: ["query"]
-            }
+            },
+            annotations: READ_ONLY
           },
           {
             name: "memory.recall_index",
-            description: "Lightweight search returning fact previews, IDs, and token costs. Follow up with memory.recall_details for full information.",
+            description: "Lightweight search returning fact previews, IDs, and token costs (~200 tokens/result). Step 1 of progressive disclosure: browse results here, then call memory.recall_details with selected fact IDs for full information (~500 tokens/fact). Saves ~60% tokens vs memory.recall when you only need a few facts.",
             inputSchema: {
               type: "object",
               properties: {
@@ -34,11 +44,12 @@ module ClaudeMemory
                 scope: {type: "string", enum: ["all", "global", "project"], description: "Scope: 'all' (both), 'global' (user-wide), 'project' (current only)", default: "all"}
               },
               required: ["query"]
-            }
+            },
+            annotations: READ_ONLY
           },
           {
             name: "memory.recall_details",
-            description: "Fetch full details for specific fact IDs. Use after memory.recall_index.",
+            description: "Fetch full details for specific fact IDs (~500 tokens/fact). Step 2 of progressive disclosure: use after memory.recall_index to get provenance and metadata for selected facts only.",
             inputSchema: {
               type: "object",
               properties: {
@@ -46,7 +57,8 @@ module ClaudeMemory
                 scope: {type: "string", enum: ["project", "global"], description: "Database to query", default: "project"}
               },
               required: ["fact_ids"]
-            }
+            },
+            annotations: READ_ONLY
           },
           {
             name: "memory.explain",
@@ -58,7 +70,8 @@ module ClaudeMemory
                 scope: {type: "string", enum: ["global", "project"], description: "Which database to look in", default: "project"}
               },
               required: ["fact_id"]
-            }
+            },
+            annotations: READ_ONLY
           },
           {
             name: "memory.changes",
@@ -70,7 +83,8 @@ module ClaudeMemory
                 limit: {type: "integer", default: 20},
                 scope: {type: "string", enum: ["all", "global", "project"], default: "all"}
               }
-            }
+            },
+            annotations: READ_ONLY
           },
           {
             name: "memory.conflicts",
@@ -80,7 +94,8 @@ module ClaudeMemory
               properties: {
                 scope: {type: "string", enum: ["all", "global", "project"], default: "all"}
               }
-            }
+            },
+            annotations: READ_ONLY
           },
           {
             name: "memory.sweep_now",
@@ -91,7 +106,8 @@ module ClaudeMemory
                 budget_seconds: {type: "integer", default: 5},
                 scope: {type: "string", enum: ["global", "project"], default: "project"}
               }
-            }
+            },
+            annotations: WRITE
           },
           {
             name: "memory.status",
@@ -99,7 +115,8 @@ module ClaudeMemory
             inputSchema: {
               type: "object",
               properties: {}
-            }
+            },
+            annotations: READ_ONLY
           },
           {
             name: "memory.stats",
@@ -109,7 +126,8 @@ module ClaudeMemory
               properties: {
                 scope: {type: "string", enum: ["all", "global", "project"], description: "Show stats for: all (default), global, or project", default: "all"}
               }
-            }
+            },
+            annotations: READ_ONLY
           },
           {
             name: "memory.promote",
@@ -120,7 +138,8 @@ module ClaudeMemory
                 fact_id: {type: "integer", description: "Project fact ID to promote to global"}
               },
               required: ["fact_id"]
-            }
+            },
+            annotations: WRITE_IDEMPOTENT
           },
           {
             name: "memory.store_extraction",
@@ -174,7 +193,8 @@ module ClaudeMemory
                 scope: {type: "string", enum: ["global", "project"], description: "Default scope for facts", default: "project"}
               },
               required: ["facts"]
-            }
+            },
+            annotations: WRITE
           },
           {
             name: "memory.decisions",
@@ -184,7 +204,8 @@ module ClaudeMemory
               properties: {
                 limit: {type: "integer", default: 10, description: "Maximum results to return"}
               }
-            }
+            },
+            annotations: READ_ONLY
           },
           {
             name: "memory.conventions",
@@ -194,7 +215,8 @@ module ClaudeMemory
               properties: {
                 limit: {type: "integer", default: 20, description: "Maximum results to return"}
               }
-            }
+            },
+            annotations: READ_ONLY
           },
           {
             name: "memory.architecture",
@@ -204,7 +226,8 @@ module ClaudeMemory
               properties: {
                 limit: {type: "integer", default: 10, description: "Maximum results to return"}
               }
-            }
+            },
+            annotations: READ_ONLY
           },
           {
             name: "memory.facts_by_tool",
@@ -217,7 +240,8 @@ module ClaudeMemory
                 scope: {type: "string", enum: ["all", "global", "project"], default: "all", description: "Filter by scope"}
               },
               required: ["tool_name"]
-            }
+            },
+            annotations: READ_ONLY
           },
           {
             name: "memory.facts_by_context",
@@ -230,11 +254,12 @@ module ClaudeMemory
                 limit: {type: "integer", default: 20, description: "Maximum results to return"},
                 scope: {type: "string", enum: ["all", "global", "project"], default: "all", description: "Filter by scope"}
               }
-            }
+            },
+            annotations: READ_ONLY
           },
           {
             name: "memory.recall_semantic",
-            description: "Search facts using semantic similarity (finds conceptually related facts using vector embeddings)",
+            description: "Search facts using semantic similarity (finds conceptually related facts using vector embeddings). ~800 tokens/result, ~300 with compact: true.",
             inputSchema: {
               type: "object",
               properties: {
@@ -242,10 +267,11 @@ module ClaudeMemory
                 mode: {type: "string", enum: ["vector", "text", "both"], default: "both", description: "Search mode: vector (embeddings), text (FTS), or both (hybrid)"},
                 limit: {type: "integer", default: 10, description: "Maximum results to return"},
                 scope: {type: "string", enum: ["all", "global", "project"], default: "all", description: "Filter by scope"},
-                compact: {type: "boolean", description: "Omit provenance receipts for ~60% smaller responses", default: false}
+                compact: {type: "boolean", description: "Omit provenance receipts for ~60% smaller responses (~800 → ~300 tokens/result)", default: false}
               },
               required: ["query"]
-            }
+            },
+            annotations: READ_ONLY
           },
           {
             name: "memory.search_concepts",
@@ -262,10 +288,11 @@ module ClaudeMemory
                 },
                 limit: {type: "integer", default: 10, description: "Maximum results to return"},
                 scope: {type: "string", enum: ["all", "global", "project"], default: "all", description: "Filter by scope"},
-                compact: {type: "boolean", description: "Omit provenance receipts for ~60% smaller responses", default: false}
+                compact: {type: "boolean", description: "Omit provenance receipts for ~60% smaller responses (~800 → ~300 tokens/result)", default: false}
               },
               required: ["concepts"]
-            }
+            },
+            annotations: READ_ONLY
           },
           {
             name: "memory.fact_graph",
@@ -278,7 +305,8 @@ module ClaudeMemory
                 scope: {type: "string", enum: ["global", "project"], description: "Which database to search", default: "project"}
               },
               required: ["fact_id"]
-            }
+            },
+            annotations: READ_ONLY
           },
           {
             name: "memory.check_setup",
@@ -286,7 +314,17 @@ module ClaudeMemory
             inputSchema: {
               type: "object",
               properties: {}
-            }
+            },
+            annotations: READ_ONLY
+          },
+          {
+            name: "memory.list_projects",
+            description: "List all known memory databases with fact counts and status. Shows global database, current project, and other projects discovered from promoted facts. Helps discover available search scopes before querying.",
+            inputSchema: {
+              type: "object",
+              properties: {}
+            },
+            annotations: READ_ONLY
           }
         ]
       end

data/lib/claude_memory/mcp/tools.rb

@@ -68,6 +68,8 @@ module ClaudeMemory
           fact_graph(arguments)
         when "memory.check_setup"
           check_setup
+        when "memory.list_projects"
+          list_projects
         else
           {error: "Unknown tool: #{name}"}
         end
@@ -507,8 +509,112 @@ module ClaudeMemory
         }
       end
 
+      def list_projects
+        result = {global: nil, current_project: nil, other_projects: []}
+
+        if @manager
+          result[:global] = list_global_database
+          result[:current_project] = list_current_project
+          result[:other_projects] = discover_other_projects
+        elsif @legacy_store
+          result[:global] = {
+            exists: true,
+            path: @legacy_store.db.opts[:database],
+            facts_active: @legacy_store.facts.where(status: "active").count,
+            entities: @legacy_store.entities.count
+          }
+        end
+
+        result[:project_count] = 1 + result[:other_projects].size
+        result
+      end
+
+      def list_global_database
+        if @manager.global_exists?
+          @manager.ensure_global!
+          store = @manager.global_store
+          {
+            exists: true,
+            path: @manager.global_db_path,
+            facts_active: store.facts.where(status: "active").count,
+            facts_total: store.facts.count,
+            entities: store.entities.count
+          }
+        else
+          {exists: false, path: @manager.global_db_path}
+        end
+      end
+
+      def list_current_project
+        if @manager.project_exists?
+          @manager.ensure_project!
+          store = @manager.project_store
+          {
+            exists: true,
+            path: @manager.project_path,
+            db_path: @manager.project_db_path,
+            facts_active: store.facts.where(status: "active").count,
+            facts_total: store.facts.count,
+            entities: store.entities.count
+          }
+        else
+          {exists: false, path: @manager.project_path, db_path: @manager.project_db_path}
+        end
+      end
+
+      def discover_other_projects
+        return [] unless @manager.global_exists?
+
+        @manager.ensure_global!
+        global = @manager.global_store
+
+        # Find project paths from promoted facts
+        promoted_paths = global.facts
+          .where(Sequel.like(:created_from, "promoted:%"))
+          .select(:created_from)
+          .distinct
+          .all
+          .filter_map { |f|
+            match = f[:created_from]&.match(/\Apromoted:(.+):\d+\z/)
+            match[1] if match
+          }
+          .uniq
+
+        # Also check for project_path values on facts
+        fact_paths = global.facts
+          .exclude(project_path: nil)
+          .select(:project_path)
+          .distinct
+          .all
+          .map { |f| f[:project_path] }
+
+        all_paths = (promoted_paths + fact_paths).uniq
+        current = @manager.project_path
+
+        all_paths.filter_map { |path|
+          next if path == current
+
+          db_path = File.join(path, ".claude", "memory.sqlite3")
+          entry = {path: path, db_path: db_path, exists: File.exist?(db_path)}
+
+          if entry[:exists]
+            begin
+              temp_store = Store::SQLiteStore.new(db_path)
+              entry[:facts_active] = temp_store.facts.where(status: "active").count
+              entry[:facts_total] = temp_store.facts.count
+              entry[:entities] = temp_store.entities.count
+              temp_store.close
+            rescue Sequel::DatabaseError, Extralite::Error, IOError => _e
+              entry[:error] = "Could not read database"
+            end
+          end
+
+          entry
+        }
+      end
+
       def db_stats(store)
-        {
+        stats = {
           exists: true,
           facts_total: store.facts.count,
           facts_active: store.facts.where(status: "active").count,
@@ -516,12 +622,30 @@ module ClaudeMemory
           open_conflicts: store.conflicts.where(status: "open").count,
           schema_version: store.schema_version
         }
+
+        vec_index = store.vector_index
+        stats[:vec_available] = vec_index.available?
+        stats[:vec_indexed] = vec_index.coverage_stats[:vec_indexed] if vec_index.available?
+
+        if fts_legacy?(store)
+          stats[:fts_legacy] = true
+          stats[:optimization_hint] = "Run 'claude-memory compact' to reduce database size by ~40%"
+        end
+
+        stats
+      end
+
+      def fts_legacy?(store)
+        row = store.db.fetch("SELECT sql FROM sqlite_master WHERE name = 'content_fts' AND type = 'table'").first
+        row && !row[:sql].to_s.include?("content=''")
+      rescue
+        false
       end
 
       def detailed_stats(store)
         active_facts = store.facts.where(status: "active").count
 
-        {
+        stats = {
           exists: true,
           facts: fact_stats(store, active_facts),
           entities: entity_stats(store),
@@ -530,6 +654,10 @@ module ClaudeMemory
           conflicts: conflict_stats(store),
           schema_version: store.schema_version
         }
+
+        stats[:vec] = vec_stats(store, active_facts)
+
+        stats
       end
 
       def fact_stats(store, active_facts)
@@ -594,6 +722,13 @@ module ClaudeMemory
         }
       end
 
+      def vec_stats(store, _active_facts)
+        vec_index = store.vector_index
+        result = {available: vec_index.available?}
+        result.merge!(vec_index.coverage_stats) if vec_index.available?
+        result
+      end
+
       def conflict_stats(store)
         open = store.conflicts.where(status: "open").count
         resolved = store.conflicts.where(status: "resolved").count

data/lib/claude_memory/publish.rb

@@ -14,19 +14,8 @@ module ClaudeMemory
     end
 
     def generate_snapshot(since: nil)
-      facts = fetch_active_facts
-      conflicts = @store.open_conflicts
-      recent_supersessions = fetch_recent_supersessions(since)
-
-      sections = []
-      sections << generate_decisions_section(facts)
-      sections << generate_conventions_section(facts)
-      sections << generate_constraints_section(facts)
-      sections << generate_conflicts_section(conflicts) if conflicts.any?
-      sections << generate_supersessions_section(recent_supersessions) if recent_supersessions.any?
-
       header = <<~HEADER
-        <!-- 
+        <!--
           This file is auto-generated by claude-memory.
           Do not edit manually - changes will be overwritten.
           Generated: #{Time.now.utc.iso8601}
@@ -36,14 +25,15 @@ module ClaudeMemory
 
       HEADER
 
-      header + sections.compact.reject(&:empty?).join("\n")
+      header + generate_body(since: since)
     end
 
     def publish!(mode: :shared, granularity: :repo, since: nil, rules_dir: nil)
-      content = generate_snapshot(since: since)
       path = output_path(mode, rules_dir: rules_dir)
+      body = generate_body(since: since)
 
-      if should_write?(path, content)
+      if should_write?(path, body)
+        content = generate_snapshot(since: since)
         @fs.write(path, content)
         ensure_import_exists(mode, path, rules_dir: rules_dir)
         {status: :updated, path: path}
@@ -163,22 +153,33 @@ module ClaudeMemory
       lines.join("\n") + "\n"
     end
 
-    def should_write?(path, content)
-      return true unless @fs.exist?(path)
+    def generate_body(since: nil)
+      facts = fetch_active_facts
+      conflicts = @store.open_conflicts
+      recent_supersessions = fetch_recent_supersessions(since)
+
+      sections = []
+      sections << generate_decisions_section(facts)
+      sections << generate_conventions_section(facts)
+      sections << generate_constraints_section(facts)
+      sections << generate_conflicts_section(conflicts) if conflicts.any?
+      sections << generate_supersessions_section(recent_supersessions) if recent_supersessions.any?
 
-      # Compare content without timestamp to avoid unnecessary rewrites
-      existing_content = @fs.read(path)
-      existing_normalized = normalize_for_comparison(existing_content)
-      new_normalized = normalize_for_comparison(content)
+      sections.compact.reject(&:empty?).join("\n")
+    end
+
+    def should_write?(path, new_body)
+      return true unless @fs.exist?(path)
 
-      existing_hash = Digest::SHA256.hexdigest(existing_normalized)
-      new_hash = Digest::SHA256.hexdigest(new_normalized)
-      existing_hash != new_hash
+      existing_body = extract_body(@fs.read(path))
+      existing_body != new_body
     end
 
-    def normalize_for_comparison(content)
-      # Remove timestamp line for comparison to prevent churn on timestamp-only changes
-      content.gsub(/^  Generated: .+$/, "")
+    def extract_body(content)
+      # Strip the HTML comment header and "# Project Memory" heading
+      content
+        .sub(/\A<!--.*?-->\s*/m, "")
+        .sub(/\A# Project Memory\s*/m, "")
     end
 
     def ensure_import_exists(mode, path, rules_dir: nil)

data/lib/claude_memory/recall/dual_query_template.rb

@@ -44,20 +44,9 @@ module ClaudeMemory
       end
 
       def query_store(source_label, &operation)
-        store = (source_label == :project) ? @manager.project_store : @manager.global_store
-        return [] unless store
-
-        ensure_store!(source_label)
+        store = @manager.store_for_scope(source_label.to_s)
         operation.call(store, source_label)
       end
-
-      def ensure_store!(source_label)
-        if source_label == :project
-          @manager.ensure_project!
-        else
-          @manager.ensure_global!
-        end
-      end
     end
   end
 end

data/lib/claude_memory/recall.rb

@@ -499,7 +499,8 @@ module ClaudeMemory
       results = template.execute(scope: scope, limit: limit) do |store, source|
         query_semantic_single(store, text, limit: limit * 3, mode: mode, source: source)
       end
-      dedupe_and_sort(results, limit)
+      # Use similarity-preserving dedupe (not source/time sort) to keep RRF ordering
+      Core::FactRanker.dedupe_by_fact_id(results, limit)
     end
 
     def query_semantic_legacy(text, limit:, scope:, mode:)
@@ -529,6 +530,33 @@ module ClaudeMemory
       # Generate query embedding
       query_embedding = @embedding_generator.generate(query_text)
 
+      # Fast path: use sqlite-vec KNN when available
+      vec_index = store.vector_index
+      if vec_index.available?
+        return search_by_vector_native(store, vec_index, query_embedding, limit, source)
+      end
+
+      # Fallback: JSON + Ruby cosine similarity
+      search_by_vector_fallback(store, query_embedding, limit, source)
+    end
+
+    def search_by_vector_native(store, vec_index, query_embedding, limit, source)
+      matches = vec_index.search(query_embedding, k: limit)
+      return [] if matches.empty?
+
+      fact_ids = matches.map { |m| m[:fact_id] }
+      facts_by_id = batch_find_facts(store, fact_ids)
+      receipts_by_fact_id = batch_find_receipts(store, fact_ids)
+
+      Core::ResultBuilder.build_results_with_scores(
+        matches,
+        facts_by_id: facts_by_id,
+        receipts_by_fact_id: receipts_by_fact_id,
+        source: source
+      )
+    end
+
+    def search_by_vector_fallback(store, query_embedding, limit, source)
       # Load facts with embeddings
       facts_data = store.facts_with_embeddings(limit: 5000)
       return [] if facts_data.empty?
@@ -556,33 +584,59 @@ module ClaudeMemory
     end
 
     def search_by_fts(store, query_text, limit, source)
-      # Use existing FTS search infrastructure
       fts = Index::LexicalFTS.new(store)
-      content_ids = fts.search(query_text, limit: limit * 2)
+      ranked_results = fts.search_with_ranks(query_text, limit: limit * 2)
 
-      return [] if content_ids.empty?
+      return [] if ranked_results.empty?
 
-      # Find facts from content items
-      fact_ids = store.provenance
+      content_ids = ranked_results.map { |r| r[:content_item_id] }
+
+      # Map content_item_ids to fact_ids, preserving FTS rank order
+      provenance_rows = store.provenance
         .where(content_item_id: content_ids)
-        .select(:fact_id)
-        .distinct
-        .map { |row| row[:fact_id] }
+        .select(:fact_id, :content_item_id)
+        .all
 
-      return [] if fact_ids.empty?
+      content_to_facts = provenance_rows.group_by { |r| r[:content_item_id] }
+
+      # Build ordered fact list with normalized BM25 scores
+      # FTS5 rank values are negative (more negative = better match)
+      ranks = ranked_results.map { |r| r[:rank] }
+      min_rank = ranks.min # Most negative = best
+      max_rank = ranks.max # Least negative = worst
+      range = (max_rank - min_rank).abs
+
+      seen_fact_ids = Set.new
+      scored_matches = []
+
+      ranked_results.each do |r|
+        similarity = if range > 0
+          # Normalize: best rank → 1.0, worst rank → 0.1
+          0.1 + 0.9 * ((max_rank - r[:rank]).abs / range)
+        else
+          0.8 # Single result gets a reasonable score
+        end
 
-      # Batch fetch facts
+        fact_ids = content_to_facts[r[:content_item_id]]&.map { |p| p[:fact_id] } || []
+        fact_ids.each do |fid|
+          next if seen_fact_ids.include?(fid)
+          seen_fact_ids.add(fid)
+          scored_matches << {fact_id: fid, similarity: similarity}
+        end
+      end
+
+      return [] if scored_matches.empty?
+
+      fact_ids = scored_matches.map { |m| m[:fact_id] }
       facts_by_id = batch_find_facts(store, fact_ids)
       receipts_by_fact_id = batch_find_receipts(store, fact_ids)
 
-      results = Core::ResultBuilder.build_results(
-        fact_ids,
+      Core::ResultBuilder.build_results_with_scores(
+        scored_matches,
         facts_by_id: facts_by_id,
         receipts_by_fact_id: receipts_by_fact_id,
-        source: source,
-        similarity: 0.5  # Default score for FTS results
-      )
-      results.take(limit)
+        source: source
+      ).take(limit)
     end
 
     def merge_search_results(vector_results, text_results, limit)