claude_memory 0.11.0 → 0.12.1

data/lib/claude_memory/commands/checks/embeddings_check.rb

@@ -0,0 +1,97 @@
+# frozen_string_literal: true
+
+module ClaudeMemory
+  module Commands
+    module Checks
+      # Surfaces the active embedding provider, model, and dimension
+      # alignment between provider and stored vectors.
+      #
+      # Doctor previously had VecCheck (sqlite-vec extension + index
+      # coverage) but no signal about which provider was actually in use —
+      # so a user could see "sqlite-vec available ✓" while silently
+      # running on tfidf default when fastembed was loadable. This check
+      # closes that visibility gap and points users at
+      # `claude-memory setup-vectors` to opt into fastembed.
+      class EmbeddingsCheck
+        FASTEMBED_HINT = "Set CLAUDE_MEMORY_EMBEDDING_PROVIDER=fastembed for higher-quality semantic recall (fastembed is loadable on this system). " \
+          "Run 'claude-memory setup-vectors' to configure."
+
+        FASTEMBED_INSTALL_HINT = "fastembed is not installed; semantic recall is using tfidf (lower quality). " \
+          "Run 'claude-memory setup-vectors' to install fastembed and switch."
+
+        def call
+          provider = Embeddings.resolve
+          provider_name = provider.name
+          warnings = []
+
+          # Hint when user is on default tfidf — different message
+          # depending on whether fastembed is even loadable.
+          if provider_name == "tfidf"
+            warnings << (fastembed_loadable? ? FASTEMBED_HINT : FASTEMBED_INSTALL_HINT)
+          end
+
+          dim_mismatches = check_dimension_alignment(provider)
+          warnings.concat(dim_mismatches)
+
+          {
+            status: warnings.any? ? :warning : :ok,
+            label: "embeddings",
+            message: "Embedding provider: #{provider_name}, dimensions: #{provider.dimensions}",
+            details: {
+              provider: provider_name,
+              dimensions: provider.dimensions,
+              fastembed_loadable: fastembed_loadable?
+            },
+            warnings: warnings
+          }
+        rescue => e
+          {
+            status: :warning,
+            label: "embeddings",
+            message: "Embedding provider check failed: #{e.message}",
+            details: {},
+            warnings: []
+          }
+        end
+
+        private
+
+        def fastembed_loadable?
+          return @fastembed_loadable if defined?(@fastembed_loadable)
+          @fastembed_loadable = begin
+            require "fastembed"
+            true
+          rescue LoadError
+            false
+          end
+        end
+
+        def check_dimension_alignment(provider)
+          config = Configuration.new
+          mismatches = []
+
+          [config.global_db_path, config.project_db_path].each do |db_path|
+            next unless File.exist?(db_path)
+
+            store = nil
+            begin
+              store = Store::SQLiteStore.new(db_path)
+              result = Embeddings::DimensionCheck.call(store, provider)
+              next unless result.status == :mismatch
+
+              mismatches << "Dimension mismatch in #{File.basename(File.dirname(db_path))} DB: " \
+                "stored=#{result.stored} but current provider produces #{result.current}. " \
+                "Run 'claude-memory index --force' to re-embed under the current provider."
+            rescue => e
+              ClaudeMemory.logger.debug("EmbeddingsCheck dimension check failed for #{db_path}: #{e.message}")
+            ensure
+              store&.close
+            end
+          end
+
+          mismatches
+        end
+      end
+    end
+  end
+end

data/lib/claude_memory/commands/dashboard_command.rb

@@ -36,7 +36,8 @@ module ClaudeMemory
         manager.ensure_global! if manager.global_exists?
         manager.ensure_project! if manager.project_exists?
 
-        stdout.puts "Starting ClaudeMemory dashboard on http://localhost:#{opts[:port]}"
+        stdout.puts "Starting ClaudeMemory dashboard on http://127.0.0.1:#{opts[:port]}"
+        stdout.puts "OTel receiver listening at http://127.0.0.1:#{opts[:port]}/v1/{metrics,logs,traces}"
         stdout.puts "Press Ctrl+C to stop."
 
         server = Dashboard::Server.new(

data/lib/claude_memory/commands/doctor_command.rb

@@ -23,6 +23,7 @@ module ClaudeMemory
           Checks::DistillCheck.new(manager.global_db_path, "global"),
           Checks::DistillCheck.new(manager.project_db_path, "project"),
           Checks::VecCheck.new,
+          Checks::EmbeddingsCheck.new,
           Checks::SnapshotCheck.new,
           Checks::ClaudeMdCheck.new,
           Checks::HooksCheck.new

data/lib/claude_memory/commands/import_auto_memory_command.rb

@@ -0,0 +1,180 @@
+# frozen_string_literal: true
+
+require "digest"
+require "optparse"
+
+module ClaudeMemory
+  module Commands
+    # Imports Claude Code auto-memory files (~/.claude/projects/<slug>/memory/*.md)
+    # into the project database as durable facts. Before this command, those
+    # markdown files were only surfaced transiently via `Hook::AutoMemoryMirror`
+    # at SessionStart — they were invisible to `memory.recall` and the
+    # shortcut tools. Importing them as facts (predicate=convention for
+    # gotcha/feedback/project files, predicate=reference for reference
+    # type) makes that knowledge first-class queryable knowledge.
+    #
+    # Idempotent on object_literal prefix: re-running skips files whose
+    # body text is already present.
+    class ImportAutoMemoryCommand < BaseCommand
+      def call(args)
+        opts = parse_opts(args)
+        return 1 if opts.nil?
+
+        auto_dir = resolve_auto_dir
+        files = list_files(auto_dir)
+        if files.empty?
+          stdout.puts "No auto-memory files found in #{auto_dir}"
+          return 0
+        end
+
+        manager = Store::StoreManager.new
+        manager.ensure_project!
+        store = manager.project_store
+
+        imported = 0
+        skipped = 0
+        files.each do |path|
+          fact_data = parse_file(path)
+          next if fact_data.nil?
+
+          if already_imported?(store, fact_data[:object_literal])
+            skipped += 1
+            next
+          end
+
+          if opts[:dry_run]
+            stdout.puts "[DRY] #{File.basename(path)} → #{fact_data[:predicate]}"
+          else
+            insert(store, fact_data, path)
+            stdout.puts "Imported: #{File.basename(path)} → #{fact_data[:predicate]}"
+          end
+          imported += 1
+        end
+
+        stdout.puts ""
+        verb = opts[:dry_run] ? "Would import" : "Imported"
+        stdout.puts "#{verb}: #{imported}  Skipped (already present): #{skipped}"
+        manager.close
+        0
+      end
+
+      private
+
+      def parse_opts(args)
+        options = {dry_run: false}
+        parser = OptionParser.new do |o|
+          o.banner = "Usage: claude-memory import-auto-memory [--dry-run]"
+          o.on("--dry-run", "Show what would be imported without writing") { options[:dry_run] = true }
+        end
+        parser.parse!(args.dup)
+        options
+      rescue OptionParser::InvalidOption => e
+        stderr.puts e.message
+        nil
+      end
+
+      def resolve_auto_dir
+        config = Configuration.new
+        Hook::AutoMemoryMirror.default_dir(config.project_dir, config.claude_config_dir)
+      end
+
+      def list_files(dir)
+        return [] unless Dir.exist?(dir)
+        Dir.glob(File.join(dir, "*.md")).reject { |f| File.basename(f) == "MEMORY.md" }.sort
+      end
+
+      def parse_file(path)
+        text = File.read(path)
+        return nil unless (match = text.match(/\A---\n(.*?)\n---\n(.*)\z/m))
+
+        frontmatter = match[1]
+        body = match[2].strip
+
+        name = frontmatter[/^name:\s*(.+)/, 1]&.strip
+        type = frontmatter[/^type:\s*(.+)/, 1]&.strip
+        description = frontmatter[/^description:\s*(.+)/, 1]&.strip
+        return nil if name.nil? || type.nil?
+
+        predicate, subject, scope = map_type(type)
+        object = build_object(name, description, body)
+
+        {
+          subject: subject,
+          predicate: predicate,
+          object_literal: object,
+          scope: scope
+        }
+      end
+
+      def map_type(type)
+        case type
+        when "feedback", "user"
+          ["convention", "user", "global"]
+        when "reference"
+          ["reference", "repo", "project"]
+        else
+          # gotcha, project, anything else
+          ["convention", "repo", "project"]
+        end
+      end
+
+      # Build an object string that carries a reason clause so
+      # BareConclusionDetector does not flag the fact as bare. Auto-memory
+      # files conventionally include a **Why:** section; we surface the first
+      # 400 chars of the body alongside the name as the object text.
+      def build_object(name, description, body)
+        first_para = body.split("\n\n").first.to_s.strip
+        first_para = first_para[0..400] + "..." if first_para.length > 400
+
+        parts = [name]
+        parts << description if description && !description.empty? && description != name
+        parts << first_para unless first_para.empty?
+        text = parts.join(" — ").gsub(/\s+/, " ").strip
+
+        # If no reason clause is present, attach a stable suffix so the fact
+        # is structurally distinguishable from bare conclusions.
+        text += " (imported from project auto-memory; see source file for full reasoning)" unless reason_present?(text)
+        text
+      end
+
+      def reason_present?(text)
+        text.match?(/\b(because|so that|so the|so we|in order to|to avoid|to ensure|to prevent|prevents|otherwise|caused by|breaks when)\b/i)
+      end
+
+      def already_imported?(store, object_text)
+        needle = object_text[0..80].gsub(/[%_]/) { |c| "\\#{c}" }
+        !store.facts.where(Sequel.like(:object_literal, "#{needle}%")).limit(1).all.empty?
+      end
+
+      def insert(store, fact_data, path)
+        store.db.transaction do
+          subject_type = (fact_data[:subject] == "user") ? "person" : "repo"
+          subject_id = store.find_or_create_entity(type: subject_type, name: fact_data[:subject])
+
+          fact_id = store.insert_fact(
+            subject_entity_id: subject_id,
+            predicate: fact_data[:predicate],
+            object_literal: fact_data[:object_literal],
+            scope: fact_data[:scope],
+            project_path: (fact_data[:scope] == "global") ? nil : Configuration.new.project_dir
+          )
+
+          content_id = store.upsert_content_item(
+            source: "auto_memory_import",
+            session_id: nil,
+            text_hash: Digest::SHA256.hexdigest(path + fact_data[:object_literal]),
+            byte_len: fact_data[:object_literal].bytesize,
+            raw_text: fact_data[:object_literal]
+          )
+
+          store.insert_provenance(
+            fact_id: fact_id,
+            content_item_id: content_id,
+            quote: fact_data[:object_literal][0..200],
+            strength: "stated"
+          )
+        end
+      end
+    end
+  end
+end

data/lib/claude_memory/commands/otel_command.rb

@@ -0,0 +1,240 @@
+# frozen_string_literal: true
+
+require "optparse"
+require "net/http"
+require "uri"
+require "json"
+
+module ClaudeMemory
+  module Commands
+    # CLI shell for the OTel ingestion feature. Subcommands flip flags in
+    # `.claude/settings.json` (delegated to OTel::SettingsWriter) or report
+    # status (delegated to OTel::Status). The command itself contains no
+    # domain logic.
+    #
+    #   claude-memory otel                       # default --status
+    #   claude-memory otel --status
+    #   claude-memory otel --enable
+    #   claude-memory otel --disable
+    #   claude-memory otel --enable-traces
+    #   claude-memory otel --disable-traces
+    #   claude-memory otel --capture-prompts    # opt-in: OTEL_LOG_USER_PROMPTS=1
+    #   claude-memory otel --no-capture-prompts
+    #   claude-memory otel --verify             # POST a fixture and confirm round-trip
+    class OtelCommand < BaseCommand
+      def call(args)
+        opts = parse_options(args, default_options) do |o|
+          OptionParser.new do |parser|
+            parser.banner = "Usage: claude-memory otel [options]"
+            parser.on("--status", "Show ingestion status (default)") { o[:action] = :status }
+            parser.on("--enable", "Configure Claude Code to export telemetry") { o[:action] = :enable }
+            parser.on("--disable", "Remove telemetry env from settings.json") { o[:action] = :disable }
+            parser.on("--enable-traces", "Opt in to trace ingestion") { o[:action] = :enable_traces }
+            parser.on("--disable-traces", "Tell Claude Code to skip trace export") { o[:action] = :disable_traces }
+            parser.on("--capture-prompts", "Opt in to OTEL_LOG_USER_PROMPTS=1") { o[:action] = :capture_prompts }
+            parser.on("--no-capture-prompts", "Stop capturing prompt content") { o[:action] = :disable_capture_prompts }
+            parser.on("--verify", "Send a sample envelope and confirm it persisted") { o[:action] = :verify }
+            parser.on("--backfill", "Tag historical activity_events from prior OTel events") { o[:action] = :backfill }
+            parser.on("--port PORT", Integer, "Receiver port for the dashboard (default 3377)") { |v| o[:port] = v }
+            parser.on("--batch-size N", Integer, "Backfill batch size (default 500)") { |v| o[:batch_size] = v }
+          end
+        end
+        return 1 if opts.nil?
+
+        case opts[:action]
+        when :enable then run_enable(opts)
+        when :disable then run_disable(opts)
+        when :enable_traces then run_settings_change(opts) { |w| w.enable_traces! }
+        when :disable_traces then run_settings_change(opts) { |w| w.disable_traces! }
+        when :capture_prompts then run_capture_prompts(opts)
+        when :disable_capture_prompts then run_settings_change(opts) { |w| w.disable_capture_prompts! }
+        when :verify then run_verify(opts)
+        when :backfill then run_backfill(opts)
+        else run_status(opts)
+        end
+      end
+
+      private
+
+      DEFAULT_BACKFILL_BATCH_SIZE = 500
+
+      def default_options
+        {action: :status, port: OTel::SettingsWriter::DEFAULT_PORT, batch_size: DEFAULT_BACKFILL_BATCH_SIZE}
+      end
+
+      def writer(opts)
+        OTel::SettingsWriter.new(claude_dir, port: opts[:port])
+      end
+
+      def claude_dir
+        File.join(ClaudeMemory::Configuration.new.project_dir, ".claude")
+      end
+
+      def run_status(opts)
+        manager = Store::StoreManager.new
+        manager.ensure_global! if manager.global_exists?
+        store = manager.global_store
+        status = OTel::Status.new(store, settings_writer: writer(opts)).snapshot
+
+        stdout.puts "OTel telemetry status:"
+        stdout.puts "  metrics ingested:    #{status[:metric_count]}"
+        stdout.puts "  events ingested:     #{status[:event_count]}"
+        stdout.puts "  traces ingested:     #{status[:trace_count]}  (enabled: #{status[:traces_enabled]})"
+        stdout.puts "  last metric:         #{status[:last_metric_at] || "never"}"
+        stdout.puts "  configured endpoint: #{status[:endpoint] || "(not configured — run --enable)"}"
+        manager.close
+        0
+      end
+
+      def run_enable(opts)
+        result = writer(opts).enable!
+        return failure("could not enable telemetry: #{result.error}") if result.failure?
+        stdout.puts "Enabled OTel telemetry. Restart any active claude sessions for changes to take effect."
+        stdout.puts "  endpoint: http://127.0.0.1:#{opts[:port]}"
+        stdout.puts "  protocol: http/json"
+        stdout.puts ""
+        stdout.puts "Traces are off by default. Opt in with `claude-memory otel --enable-traces`."
+        0
+      end
+
+      def run_disable(opts)
+        result = writer(opts).disable!
+        return failure("could not disable telemetry: #{result.error}") if result.failure?
+        stdout.puts "Disabled OTel telemetry. Removed env keys from settings.json."
+        0
+      end
+
+      def run_settings_change(opts)
+        result = yield(writer(opts))
+        return failure("settings update failed: #{result.error}") if result.failure?
+        stdout.puts "Settings updated:"
+        result.value.each { |key, value| stdout.puts "  #{key}=#{value}" }
+        0
+      end
+
+      def run_capture_prompts(opts)
+        stdout.puts "WARNING: enabling OTEL_LOG_USER_PROMPTS=1 will cause Claude Code"
+        stdout.puts "to send your verbatim prompts (and the conversation history) to"
+        stdout.puts "this dashboard's local SQLite. Type 'yes' to continue."
+        stdout.print "> "
+        answer = stdin.gets&.strip
+        unless answer == "yes"
+          stdout.puts "Aborted. No changes made."
+          return 0
+        end
+        run_settings_change(opts) { |w| w.capture_prompts! }
+      end
+
+      # One-time pass: stream every otel_events row with a prompt_id from
+      # the global store and run it through PromptScope so historical
+      # activity_events get tagged. Subsequent runs are cheap because
+      # PromptScope is idempotent (already-tagged rows are excluded by
+      # the `prompt_id: nil` filter).
+      def run_backfill(opts)
+        manager = Store::StoreManager.new
+        unless manager.global_exists?
+          return failure("Global memory DB not found — nothing to backfill against.")
+        end
+        manager.ensure_global!
+        manager.ensure_project! if manager.project_exists?
+
+        store = manager.global_store
+        unless store.db.table_exists?(:otel_events)
+          manager.close
+          return failure("otel_events table missing — run `claude-memory dashboard` first to migrate.")
+        end
+
+        total_events = store.otel_events.exclude(prompt_id: nil).count
+        if total_events.zero?
+          stdout.puts "No OTel events with prompt_id to backfill from."
+          manager.close
+          return 0
+        end
+
+        stdout.puts "Backfilling #{total_events} OTel event(s) into activity_events…"
+        scope = OTel::PromptScope.new(manager)
+        tagged = groups = batches = 0
+        offset = 0
+        batch_size = opts[:batch_size]
+
+        loop do
+          rows = store.otel_events
+            .exclude(prompt_id: nil)
+            .order(:occurred_at, :id)
+            .limit(batch_size)
+            .offset(offset)
+            .select(:event_name, :session_id, :prompt_id, :occurred_at)
+            .all
+          break if rows.empty?
+
+          events = rows.map { |r|
+            {event_name: r[:event_name], session_id: r[:session_id],
+             prompt_id: r[:prompt_id], occurred_at: r[:occurred_at]}
+          }
+          result = scope.tag(events)
+          tagged += result[:tagged].to_i
+          groups += result[:groups].to_i
+          batches += 1
+          offset += rows.size
+          stdout.print "."
+          stdout.flush
+        end
+
+        stdout.puts ""
+        stdout.puts "Backfill complete: tagged #{tagged} activity_event(s) across #{groups} prompt group(s) in #{batches} batch(es)."
+        manager.close
+        0
+      end
+
+      def run_verify(opts)
+        url = URI.parse("http://127.0.0.1:#{opts[:port]}/v1/metrics")
+        body = JSON.generate(sample_metrics_envelope)
+        response = post_json(url, body)
+        if response.is_a?(Net::HTTPSuccess)
+          stdout.puts "Verify OK: dashboard accepted the sample metrics envelope."
+          0
+        else
+          failure("Verify failed: dashboard returned #{response&.code || "no response"}")
+        end
+      rescue Errno::ECONNREFUSED, SocketError => e
+        failure("Verify failed: #{e.message}. Is the dashboard running on port #{opts[:port]}?")
+      end
+
+      def post_json(url, body)
+        Net::HTTP.start(url.host, url.port, open_timeout: 2, read_timeout: 5) do |http|
+          req = Net::HTTP::Post.new(url.path, "Content-Type" => "application/json")
+          req.body = body
+          http.request(req)
+        end
+      end
+
+      # Smallest valid OTLP/HTTP/JSON metrics envelope — one counter point.
+      # Used only for the --verify subcommand so users can confirm
+      # end-to-end wiring without running a real claude session.
+      def sample_metrics_envelope
+        nano = (Time.now.to_f * 1_000_000_000).to_i.to_s
+        {
+          "resourceMetrics" => [{
+            "resource" => {"attributes" => [
+              {"key" => "service.name", "value" => {"stringValue" => "claude-memory-verify"}}
+            ]},
+            "scopeMetrics" => [{
+              "scope" => {"name" => "claude-memory.verify"},
+              "metrics" => [{
+                "name" => "claude_memory.verify",
+                "unit" => "count",
+                "sum" => {
+                  "dataPoints" => [{
+                    "asInt" => "1",
+                    "timeUnixNano" => nano,
+                    "attributes" => []
+                  }]
+                }
+              }]
+            }]
+          }]
+        }
+      end
+    end
+  end
+end

data/lib/claude_memory/commands/registry.rb

@@ -43,7 +43,11 @@ module ClaudeMemory
         "census" => {class: CensusCommand, description: "Aggregate predicate usage across project databases"},
         "dashboard" => {class: DashboardCommand, description: "Open debugging dashboard"},
         "digest" => {class: DigestCommand, description: "Render a weekly markdown digest of memory activity"},
-        "show" => {class: ShowCommand, description: "Print what memory would inject at the next SessionStart"}
+        "show" => {class: ShowCommand, description: "Print what memory would inject at the next SessionStart"},
+        "otel" => {class: OtelCommand, description: "Configure or inspect OpenTelemetry ingestion from Claude Code"},
+        "setup-vectors" => {class: SetupVectorsCommand, description: "Opt into vector recall — write provider env to .claude/settings.json + re-index"},
+        "import-auto-memory" => {class: ImportAutoMemoryCommand, description: "Import Claude Code auto-memory .md files into project DB as facts"},
+        "audit" => {class: AuditCommand, description: "Run memory health audit; report inconsistencies and optimizations"}
       }.freeze
 
       # Find a command class by name