claude_memory 0.7.0 → 0.8.0

data/lib/claude_memory/commands/checks/vec_check.rb

@@ -52,7 +52,8 @@ module ClaudeMemory
               stats = store.vector_index.coverage_stats
               totals[:with_embedding] += stats[:with_embedding]
               totals[:vec_indexed] += stats[:vec_indexed]
-            rescue => _e
+            rescue => e
+              ClaudeMemory.logger.debug("VecCheck failed for #{db_path}: #{e.message}")
               next
             ensure
               store&.close

data/lib/claude_memory/commands/completion_command.rb

@@ -0,0 +1,179 @@
+# frozen_string_literal: true
+
+module ClaudeMemory
+  module Commands
+    # Generates shell completion scripts for bash and zsh.
+    # Outputs completion script to stdout for eval or redirection.
+    class CompletionCommand < BaseCommand
+      def call(args)
+        opts = parse_options(args, {shell: detect_shell}) do |o|
+          OptionParser.new do |parser|
+            parser.banner = "Usage: claude-memory completion [options]"
+            parser.on("--shell SHELL", %w[bash zsh], "Shell type: bash or zsh (auto-detected)") { |v| o[:shell] = v }
+          end
+        end
+        return 1 if opts.nil?
+
+        case opts[:shell]
+        when "zsh"
+          stdout.puts zsh_completion
+        when "bash"
+          stdout.puts bash_completion
+        else
+          return failure("Unknown shell: #{opts[:shell]}. Use --shell bash or --shell zsh")
+        end
+        0
+      end
+
+      private
+
+      def detect_shell
+        shell = ENV.fetch("SHELL", "/bin/bash")
+        File.basename(shell)
+      end
+
+      def command_names
+        Registry.all_commands.sort
+      end
+
+      def zsh_completion
+        commands_with_desc = command_descriptions.map { |name, desc|
+          "    '#{name}:#{desc}'"
+        }.join("\n")
+
+        <<~ZSH
+          #compdef claude-memory
+
+          _claude_memory() {
+            local -a commands
+            commands=(
+          #{commands_with_desc}
+            )
+
+            _arguments -C \\
+              '1:command:->command' \\
+              '*::arg:->args'
+
+            case $state in
+              command)
+                _describe 'command' commands
+                ;;
+              args)
+                case $words[1] in
+                  recall|search|explain)
+                    _arguments '*:query:'
+                    ;;
+                  promote)
+                    _arguments '*:fact_id:'
+                    ;;
+                  hook)
+                    local -a subcommands
+                    subcommands=('ingest:Ingest transcript' 'sweep:Run maintenance' 'publish:Publish snapshot' 'context:Inject context')
+                    _describe 'subcommand' subcommands
+                    ;;
+                  compact|export|changes|stats|sweep|conflicts)
+                    _arguments '--scope[Scope]:scope:(all global project)'
+                    ;;
+                  index)
+                    _arguments '--vec[Build vector index]' '--rebuild[Rebuild from scratch]'
+                    ;;
+                  completion)
+                    _arguments '--shell[Shell type]:shell:(bash zsh)'
+                    ;;
+                  install-skill)
+                    local -a skills
+                    skills=(#{skill_names.map { |s| "'#{s}'" }.join(" ")})
+                    _arguments '--list[List available skills]' '--force[Overwrite existing]' '1:skill:($skills)'
+                    ;;
+                esac
+                ;;
+            esac
+          }
+
+          _claude_memory "$@"
+        ZSH
+      end
+
+      def bash_completion
+        <<~BASH
+          # bash completion for claude-memory
+
+          _claude_memory() {
+            local cur prev commands
+            COMPREPLY=()
+            cur="${COMP_WORDS[COMP_CWORD]}"
+            prev="${COMP_WORDS[COMP_CWORD-1]}"
+            commands="#{command_names.join(" ")}"
+
+            if [[ ${COMP_CWORD} -eq 1 ]]; then
+              COMPREPLY=( $(compgen -W "${commands}" -- "${cur}") )
+              return 0
+            fi
+
+            case "${COMP_WORDS[1]}" in
+              hook)
+                COMPREPLY=( $(compgen -W "ingest sweep publish context" -- "${cur}") )
+                ;;
+              compact|export|changes|stats|sweep|conflicts)
+                if [[ "${prev}" == "--scope" ]]; then
+                  COMPREPLY=( $(compgen -W "all global project" -- "${cur}") )
+                else
+                  COMPREPLY=( $(compgen -W "--scope" -- "${cur}") )
+                fi
+                ;;
+              install-skill)
+                if [[ "${prev}" == "install-skill" ]]; then
+                  COMPREPLY=( $(compgen -W "#{skill_names.join(" ")} --list --force" -- "${cur}") )
+                fi
+                ;;
+              completion)
+                if [[ "${prev}" == "--shell" ]]; then
+                  COMPREPLY=( $(compgen -W "bash zsh" -- "${cur}") )
+                else
+                  COMPREPLY=( $(compgen -W "--shell" -- "${cur}") )
+                fi
+                ;;
+            esac
+            return 0
+          }
+
+          complete -F _claude_memory claude-memory
+        BASH
+      end
+
+      def command_descriptions
+        {
+          "changes" => "Show recent fact changes",
+          "compact" => "Compact databases",
+          "completion" => "Generate shell completions",
+          "conflicts" => "Show open conflicts",
+          "db:init" => "Initialize database",
+          "doctor" => "Check system health",
+          "explain" => "Explain a fact with receipts",
+          "export" => "Export facts to JSON",
+          "git-lfs" => "Git LFS integration",
+          "help" => "Show help message",
+          "hook" => "Run hook entrypoints",
+          "index" => "Index content",
+          "ingest" => "Ingest transcript delta",
+          "init" => "Initialize ClaudeMemory",
+          "install-skill" => "Install agent skills",
+          "promote" => "Promote fact to global",
+          "publish" => "Publish snapshot",
+          "recall" => "Recall facts matching query",
+          "recover" => "Recover database",
+          "search" => "Search indexed content",
+          "serve-mcp" => "Start MCP server",
+          "stats" => "Show statistics",
+          "sweep" => "Run maintenance",
+          "uninstall" => "Remove configuration",
+          "version" => "Show version"
+        }
+      end
+
+      def skill_names
+        InstallSkillCommand::AVAILABLE_SKILLS.keys
+      end
+    end
+  end
+end

data/lib/claude_memory/commands/doctor_command.rb

@@ -20,6 +20,8 @@ module ClaudeMemory
         checks = [
           Checks::DatabaseCheck.new(manager.global_db_path, "global"),
           Checks::DatabaseCheck.new(manager.project_db_path, "project"),
+          Checks::DistillCheck.new(manager.global_db_path, "global"),
+          Checks::DistillCheck.new(manager.project_db_path, "project"),
           Checks::VecCheck.new,
           Checks::SnapshotCheck.new,
           Checks::ClaudeMdCheck.new,

data/lib/claude_memory/commands/help_command.rb

@@ -31,6 +31,10 @@ module ClaudeMemory
             uninstall  Remove ClaudeMemory configuration
             version    Show version number
 
+          Utilities:
+            completion     Generate shell completions (bash/zsh)
+            install-skill  Install agent skills to ~/.claude/commands/
+
           Run 'claude-memory <command> --help' for more information on a command.
         HELP
         0

data/lib/claude_memory/commands/hook_command.rb

@@ -171,13 +171,14 @@ module ClaudeMemory
 
       def hook_context(payload, db_path)
         project_path = payload["project_path"] || payload["cwd"]
+        source = payload["source"]
         manager = ClaudeMemory::Store::StoreManager.new(
           project_db_path: db_path,
           project_path: project_path
         )
         manager.ensure_both!
 
-        injector = ClaudeMemory::Hook::ContextInjector.new(manager)
+        injector = ClaudeMemory::Hook::ContextInjector.new(manager, source: source)
         context_text = injector.generate_context
 
         if context_text

data/lib/claude_memory/commands/index_command.rb

@@ -9,13 +9,14 @@ module ClaudeMemory
       SCOPE_PROJECT = "project"
 
       def call(args)
-        opts = parse_options(args, {scope: SCOPE_ALL, batch_size: 100, force: false, vec: false}) do |o|
+        opts = parse_options(args, {scope: SCOPE_ALL, batch_size: 100, force: false, vec: false, provider: nil}) do |o|
           OptionParser.new do |parser|
             parser.banner = "Usage: claude-memory index [options]"
             parser.on("--scope SCOPE", "Scope: global, project, or all (default: all)") { |v| o[:scope] = v }
             parser.on("--batch-size SIZE", Integer, "Batch size (default: 100)") { |v| o[:batch_size] = v }
             parser.on("--force", "Re-index facts that already have embeddings") { o[:force] = true }
             parser.on("--vec", "Backfill vec0 index from existing embeddings (no regeneration)") { o[:vec] = true }
+            parser.on("--provider NAME", "Embedding provider: tfidf, fastembed, api") { |v| o[:provider] = v }
           end
         end
         return 1 if opts.nil?
@@ -30,7 +31,7 @@ module ClaudeMemory
           return vec_backfill(opts)
         end
 
-        generator = Embeddings::Generator.new
+        generator = Embeddings.resolve(opts[:provider])
 
         scopes_for(opts[:scope]).each do |label, db_path|
           index_database(label, db_path, generator, opts)
@@ -42,9 +43,10 @@ module ClaudeMemory
       private
 
       def scopes_for(scope)
+        config = Configuration.new
         pairs = []
-        pairs << ["global", Configuration.global_db_path] if scope == SCOPE_ALL || scope == SCOPE_GLOBAL
-        pairs << ["project", Configuration.project_db_path] if scope == SCOPE_ALL || scope == SCOPE_PROJECT
+        pairs << ["global", config.global_db_path] if scope == SCOPE_ALL || scope == SCOPE_GLOBAL
+        pairs << ["project", config.project_db_path] if scope == SCOPE_ALL || scope == SCOPE_PROJECT
         pairs
       end
 
@@ -55,96 +57,85 @@ module ClaudeMemory
         end
 
         store = Store::SQLiteStore.new(db_path)
+        handle_dimension_mismatch(store, generator, label)
         tracker = Infrastructure::OperationTracker.new(store)
 
-        # Check for existing progress (resumption support)
+        facts, checkpoint = find_facts_to_index(store, tracker, label, opts)
+        unless facts
+          store.close
+          return
+        end
+
+        operation_id = checkpoint ? checkpoint[:operation_id] : tracker.start_operation(
+          operation_type: "index_embeddings",
+          scope: label,
+          total_items: facts.size,
+          checkpoint_data: {last_fact_id: nil}
+        )
+
+        stdout.puts "#{label.capitalize} database: Indexing #{facts.size} facts..."
+        run_indexing(store, facts, generator, tracker, operation_id, checkpoint, opts)
+      end
+
+      def handle_dimension_mismatch(store, generator, label)
+        check = Embeddings::DimensionCheck.call(store, generator)
+        return unless check.status == :mismatch
+
+        stdout.puts "#{label.capitalize}: Embedding dimensions changed (#{check.stored} → #{check.current}), clearing stale embeddings..."
+        clear_stale_embeddings(store)
+      end
+
+      def find_facts_to_index(store, tracker, label, opts)
         checkpoint = tracker.get_checkpoint(operation_type: "index_embeddings", scope: label)
+
         if checkpoint && !opts[:force]
           stdout.puts "#{label.capitalize} database: Resuming from previous run (processed #{checkpoint[:processed_items]} facts)..."
           resume_from_fact_id = checkpoint[:checkpoint_data][:last_fact_id]
-        else
-          resume_from_fact_id = nil
-        end
-
-        # Find facts to index
-        facts_dataset = if opts[:force]
-          store.facts
-        else
-          store.facts.where(embedding_json: nil)
-        end
-
-        # If resuming, skip facts we've already processed
-        if resume_from_fact_id
-          facts_dataset = facts_dataset.where(Sequel.lit("id > ?", resume_from_fact_id))
         end
 
+        facts_dataset = opts[:force] ? store.facts : store.facts.where(embedding_json: nil)
+        facts_dataset = facts_dataset.where(Sequel.lit("id > ?", resume_from_fact_id)) if resume_from_fact_id
         facts = facts_dataset.order(:id).all
 
         if facts.empty? && !checkpoint
           stdout.puts "#{label.capitalize} database: All facts already indexed"
-          store.close
-          return
+          return nil
         elsif facts.empty? && checkpoint
-          # Resume found nothing left to do - mark as completed
           tracker.complete_operation(checkpoint[:operation_id])
           stdout.puts "#{label.capitalize} database: Resumed operation completed (nothing left to index)"
-          store.close
-          return
+          return nil
         end
 
-        # Start or continue operation tracking
-        operation_id = checkpoint ? checkpoint[:operation_id] : tracker.start_operation(
-          operation_type: "index_embeddings",
-          scope: label,
-          total_items: facts.size,
-          checkpoint_data: {last_fact_id: nil}
-        )
-
-        stdout.puts "#{label.capitalize} database: Indexing #{facts.size} facts..."
+        [facts, checkpoint]
+      end
 
+      def run_indexing(store, facts, generator, tracker, operation_id, checkpoint, opts)
         vec_index = store.vector_index
-        if vec_index.available?
-          stdout.puts "  sqlite-vec available, dual-writing to vec0 index"
-        end
+        stdout.puts "  sqlite-vec available, dual-writing to vec0 index" if vec_index.available?
 
+        embedding_cache = build_embedding_cache(store)
+        cache_hits = 0
         processed = checkpoint ? checkpoint[:processed_items] : 0
+
         begin
           facts.each_slice(opts[:batch_size]) do |batch|
-            # Wrap batch processing in transaction for atomicity
-            store.db.transaction do
-              batch.each do |fact|
-                # Generate text representation
-                text = build_fact_text(fact, store)
-
-                # Generate embedding
-                embedding = generator.generate(text)
-
-                # Store embedding (JSON column)
-                store.update_fact_embedding(fact[:id], embedding)
-
-                # Dual-write to vec0 if available (insert_embedding manages vec_indexed_at)
-                vec_index.insert_embedding(fact[:id], embedding) if vec_index.available?
-
-                processed += 1
-              end
-
-              # Update checkpoint after batch commits
-              last_fact_id = batch.last[:id]
-              tracker.update_progress(
-                operation_id,
-                processed_items: processed,
-                checkpoint_data: {last_fact_id: last_fact_id}
-              )
-            end
-
+            cache_hits += process_batch(store, batch, generator, vec_index, embedding_cache)
+            processed += batch.size
+
+            tracker.update_progress(
+              operation_id,
+              processed_items: processed,
+              checkpoint_data: {last_fact_id: batch.last[:id]}
+            )
             stdout.puts "  Processed #{processed} facts..."
           end
 
-          # Mark operation as completed
+          report_dedup_stats(processed, cache_hits)
+          store.set_meta("embedding_dimensions", generator.dimensions.to_s)
+          store.set_meta("embedding_provider", generator.name)
           tracker.complete_operation(operation_id)
           stdout.puts "  Done!"
         rescue => e
-          # Mark operation as failed
           tracker.fail_operation(operation_id, e.message)
           stderr.puts "  Failed: #{e.message}"
           raise
@@ -153,6 +144,32 @@ module ClaudeMemory
         end
       end
 
+      def process_batch(store, batch, generator, vec_index, embedding_cache)
+        cache_hits = 0
+        store.db.transaction do
+          batch.each do |fact|
+            text = build_fact_text(fact, store)
+            embedding = embedding_cache[text]
+            if embedding
+              cache_hits += 1
+            else
+              embedding = generator.generate(text)
+              embedding_cache[text] = embedding
+            end
+            store.update_fact_embedding(fact[:id], embedding)
+            vec_index.insert_embedding(fact[:id], embedding) if vec_index.available?
+          end
+        end
+        cache_hits
+      end
+
+      def report_dedup_stats(processed, cache_hits)
+        return unless processed > 0
+
+        pct = (cache_hits > 0) ? "#{(cache_hits * 100.0 / processed).round(1)}%" : "0%"
+        stdout.puts "  Cache hits: #{cache_hits}/#{processed} (#{pct} dedup)"
+      end
+
       def vec_backfill(opts)
         scopes_for(opts[:scope]).each do |label, db_path|
           unless File.exist?(db_path)
@@ -192,6 +209,19 @@ module ClaudeMemory
         0
       end
 
+      def build_embedding_cache(store)
+        cache = {}
+        store.facts
+          .where(status: "active")
+          .where(Sequel.~(embedding_json: nil))
+          .select(:id, :subject_entity_id, :predicate, :object_entity_id, :object_literal, :embedding_json)
+          .each do |fact|
+            text = build_fact_text(fact, store)
+            cache[text] ||= JSON.parse(fact[:embedding_json])
+          end
+        cache
+      end
+
       def build_fact_text(fact, store)
         # Build rich text representation for embedding
         parts = []
@@ -216,6 +246,11 @@ module ClaudeMemory
         parts.join(" ")
       end
 
+      def clear_stale_embeddings(store)
+        store.facts.where(Sequel.~(embedding_json: nil)).update(embedding_json: nil, vec_indexed_at: nil)
+        store.vector_index.clear!
+      end
+
       def valid_scope?(scope)
         [SCOPE_ALL, SCOPE_GLOBAL, SCOPE_PROJECT].include?(scope)
       end

data/lib/claude_memory/commands/initializers/database_ensurer.rb

@@ -15,6 +15,10 @@ module ClaudeMemory
           @stdout.puts "✓ Global database: #{manager.global_db_path}"
           manager.ensure_project!
           @stdout.puts "✓ Project database: #{manager.project_db_path}"
+
+          backfill_distillation_metrics(manager.global_store, "global")
+          backfill_distillation_metrics(manager.project_store, "project")
+
           manager.close
         end
 
@@ -22,8 +26,20 @@ module ClaudeMemory
           manager = ClaudeMemory::Store::StoreManager.new
           manager.ensure_global!
           @stdout.puts "✓ Created global database: #{manager.global_db_path}"
+
+          backfill_distillation_metrics(manager.global_store, "global")
+
           manager.close
         end
+
+        private
+
+        def backfill_distillation_metrics(store, label)
+          backfilled = store.backfill_distillation_metrics!
+          if backfilled > 0
+            @stdout.puts "✓ Marked #{backfilled} pre-existing content items as distilled (#{label})"
+          end
+        end
       end
     end
   end

data/lib/claude_memory/commands/initializers/global_initializer.rb

@@ -65,6 +65,7 @@ module ClaudeMemory
             @skip_hooks = true
           else
             @stdout.puts "\nUpdating hooks..."
+            @replace_hooks = true
           end
         end
 
@@ -73,7 +74,7 @@ module ClaudeMemory
         end
 
         def configure_hooks
-          HooksConfigurator.new(@stdout).configure_global_hooks
+          HooksConfigurator.new(@stdout).configure_global_hooks(replace: @replace_hooks || false)
         end
 
         def configure_mcp

data/lib/claude_memory/commands/initializers/hooks_configurator.rb

@@ -12,7 +12,7 @@ module ClaudeMemory
           @stdout = stdout
         end
 
-        def configure_project_hooks
+        def configure_project_hooks(replace: false)
           settings_path = ".claude/settings.json"
           FileUtils.mkdir_p(File.dirname(settings_path))
 
@@ -24,13 +24,13 @@ module ClaudeMemory
 
           existing = load_json_file(settings_path)
           existing["hooks"] ||= {}
-          merge_hooks!(existing["hooks"], hooks_config["hooks"])
+          merge_hooks!(existing["hooks"], hooks_config["hooks"], replace: replace)
 
           File.write(settings_path, JSON.pretty_generate(existing))
           @stdout.puts "✓ Configured hooks in #{settings_path}"
         end
 
-        def configure_global_hooks
+        def configure_global_hooks(replace: false)
           settings_path = File.join(Dir.home, ".claude", "settings.json")
           FileUtils.mkdir_p(File.dirname(settings_path))
 
@@ -42,7 +42,7 @@ module ClaudeMemory
 
           existing = load_json_file(settings_path)
           existing["hooks"] ||= {}
-          merge_hooks!(existing["hooks"], hooks_config["hooks"])
+          merge_hooks!(existing["hooks"], hooks_config["hooks"], replace: replace)
 
           File.write(settings_path, JSON.pretty_generate(existing))
           @stdout.puts "✓ Configured hooks in #{settings_path}"
@@ -97,28 +97,61 @@ module ClaudeMemory
         private
 
         def build_hooks_config(ingest_cmd, sweep_cmd)
+          context_cmd = "claude-memory hook context"
+
           {
             "hooks" => {
               "Stop" => [{
                 "hooks" => [
-                  {"type" => "command", "command" => ingest_cmd, "timeout" => 10}
+                  {"type" => "command", "command" => ingest_cmd, "timeout" => 5,
+                   "statusMessage" => "Saving memory..."}
+                ]
+              }],
+              "StopFailure" => [{
+                "hooks" => [
+                  {"type" => "command", "command" => ingest_cmd, "timeout" => 5,
+                   "statusMessage" => "Saving memory..."}
                 ]
               }],
               "SessionStart" => [{
                 "hooks" => [
-                  {"type" => "command", "command" => ingest_cmd, "timeout" => 10}
+                  {"type" => "command", "command" => context_cmd, "timeout" => 5,
+                   "statusMessage" => "Loading memory..."}
                 ]
               }],
               "PreCompact" => [{
                 "hooks" => [
-                  {"type" => "command", "command" => ingest_cmd, "timeout" => 30},
-                  {"type" => "command", "command" => sweep_cmd, "timeout" => 30}
+                  {"type" => "command", "command" => ingest_cmd, "timeout" => 30,
+                   "statusMessage" => "Saving memory..."},
+                  {"type" => "command", "command" => sweep_cmd, "timeout" => 30,
+                   "statusMessage" => "Sweeping memory..."}
                 ]
               }],
               "SessionEnd" => [{
                 "hooks" => [
-                  {"type" => "command", "command" => ingest_cmd, "timeout" => 30},
-                  {"type" => "command", "command" => sweep_cmd, "timeout" => 30}
+                  {"type" => "command", "command" => ingest_cmd, "timeout" => 30,
+                   "statusMessage" => "Saving memory..."},
+                  {"type" => "command", "command" => sweep_cmd, "timeout" => 30,
+                   "statusMessage" => "Sweeping memory..."}
+                ]
+              }],
+              "TaskCompleted" => [{
+                "hooks" => [
+                  {"type" => "command", "command" => ingest_cmd, "timeout" => 10,
+                   "statusMessage" => "Saving memory..."}
+                ]
+              }],
+              "TeammateIdle" => [{
+                "hooks" => [
+                  {"type" => "command", "command" => ingest_cmd, "timeout" => 15,
+                   "statusMessage" => "Saving memory..."}
+                ]
+              }],
+              "Notification" => [{
+                "matcher" => "idle_prompt",
+                "hooks" => [
+                  {"type" => "command", "command" => sweep_cmd, "timeout" => 10,
+                   "statusMessage" => "Sweeping memory..."}
                 ]
               }]
             }
@@ -132,7 +165,18 @@ module ClaudeMemory
           {}
         end
 
-        def merge_hooks!(existing_hooks, new_hooks)
+        def merge_hooks!(existing_hooks, new_hooks, replace: false)
+          if replace
+            # Remove all claude-memory hooks first, then add fresh ones
+            existing_hooks.each do |event, hook_arrays|
+              next unless hook_arrays.is_a?(Array)
+              hook_arrays.reject! do |ha|
+                ha.is_a?(Hash) && ha["hooks"]&.any? { |h| h["command"]&.include?("claude-memory") }
+              end
+            end
+            existing_hooks.delete_if { |_, v| v.is_a?(Array) && v.empty? }
+          end
+
           new_hooks.each do |event, hook_arrays|
             existing_hooks[event] ||= []
 

data/lib/claude_memory/commands/initializers/project_initializer.rb

@@ -68,6 +68,7 @@ module ClaudeMemory
             @skip_hooks = true
           else
             @stdout.puts "\nUpdating hooks..."
+            @replace_hooks = true
           end
         end
 
@@ -81,7 +82,7 @@ module ClaudeMemory
         end
 
         def configure_hooks
-          HooksConfigurator.new(@stdout).configure_project_hooks
+          HooksConfigurator.new(@stdout).configure_project_hooks(replace: @replace_hooks || false)
         end
 
         def configure_mcp