claude_memory 0.10.0 → 0.12.0

data/lib/claude_memory/commands/otel_command.rb

@@ -0,0 +1,240 @@
+# frozen_string_literal: true
+
+require "optparse"
+require "net/http"
+require "uri"
+require "json"
+
+module ClaudeMemory
+  module Commands
+    # CLI shell for the OTel ingestion feature. Subcommands flip flags in
+    # `.claude/settings.json` (delegated to OTel::SettingsWriter) or report
+    # status (delegated to OTel::Status). The command itself contains no
+    # domain logic.
+    #
+    #   claude-memory otel                       # default --status
+    #   claude-memory otel --status
+    #   claude-memory otel --enable
+    #   claude-memory otel --disable
+    #   claude-memory otel --enable-traces
+    #   claude-memory otel --disable-traces
+    #   claude-memory otel --capture-prompts    # opt-in: OTEL_LOG_USER_PROMPTS=1
+    #   claude-memory otel --no-capture-prompts
+    #   claude-memory otel --verify             # POST a fixture and confirm round-trip
+    class OtelCommand < BaseCommand
+      def call(args)
+        opts = parse_options(args, default_options) do |o|
+          OptionParser.new do |parser|
+            parser.banner = "Usage: claude-memory otel [options]"
+            parser.on("--status", "Show ingestion status (default)") { o[:action] = :status }
+            parser.on("--enable", "Configure Claude Code to export telemetry") { o[:action] = :enable }
+            parser.on("--disable", "Remove telemetry env from settings.json") { o[:action] = :disable }
+            parser.on("--enable-traces", "Opt in to trace ingestion") { o[:action] = :enable_traces }
+            parser.on("--disable-traces", "Tell Claude Code to skip trace export") { o[:action] = :disable_traces }
+            parser.on("--capture-prompts", "Opt in to OTEL_LOG_USER_PROMPTS=1") { o[:action] = :capture_prompts }
+            parser.on("--no-capture-prompts", "Stop capturing prompt content") { o[:action] = :disable_capture_prompts }
+            parser.on("--verify", "Send a sample envelope and confirm it persisted") { o[:action] = :verify }
+            parser.on("--backfill", "Tag historical activity_events from prior OTel events") { o[:action] = :backfill }
+            parser.on("--port PORT", Integer, "Receiver port for the dashboard (default 3377)") { |v| o[:port] = v }
+            parser.on("--batch-size N", Integer, "Backfill batch size (default 500)") { |v| o[:batch_size] = v }
+          end
+        end
+        return 1 if opts.nil?
+
+        case opts[:action]
+        when :enable then run_enable(opts)
+        when :disable then run_disable(opts)
+        when :enable_traces then run_settings_change(opts) { |w| w.enable_traces! }
+        when :disable_traces then run_settings_change(opts) { |w| w.disable_traces! }
+        when :capture_prompts then run_capture_prompts(opts)
+        when :disable_capture_prompts then run_settings_change(opts) { |w| w.disable_capture_prompts! }
+        when :verify then run_verify(opts)
+        when :backfill then run_backfill(opts)
+        else run_status(opts)
+        end
+      end
+
+      private
+
+      DEFAULT_BACKFILL_BATCH_SIZE = 500
+
+      def default_options
+        {action: :status, port: OTel::SettingsWriter::DEFAULT_PORT, batch_size: DEFAULT_BACKFILL_BATCH_SIZE}
+      end
+
+      def writer(opts)
+        OTel::SettingsWriter.new(claude_dir, port: opts[:port])
+      end
+
+      def claude_dir
+        File.join(ClaudeMemory::Configuration.new.project_dir, ".claude")
+      end
+
+      def run_status(opts)
+        manager = Store::StoreManager.new
+        manager.ensure_global! if manager.global_exists?
+        store = manager.global_store
+        status = OTel::Status.new(store, settings_writer: writer(opts)).snapshot
+
+        stdout.puts "OTel telemetry status:"
+        stdout.puts "  metrics ingested:    #{status[:metric_count]}"
+        stdout.puts "  events ingested:     #{status[:event_count]}"
+        stdout.puts "  traces ingested:     #{status[:trace_count]}  (enabled: #{status[:traces_enabled]})"
+        stdout.puts "  last metric:         #{status[:last_metric_at] || "never"}"
+        stdout.puts "  configured endpoint: #{status[:endpoint] || "(not configured — run --enable)"}"
+        manager.close
+        0
+      end
+
+      def run_enable(opts)
+        result = writer(opts).enable!
+        return failure("could not enable telemetry: #{result.error}") if result.failure?
+        stdout.puts "Enabled OTel telemetry. Restart any active claude sessions for changes to take effect."
+        stdout.puts "  endpoint: http://127.0.0.1:#{opts[:port]}"
+        stdout.puts "  protocol: http/json"
+        stdout.puts ""
+        stdout.puts "Traces are off by default. Opt in with `claude-memory otel --enable-traces`."
+        0
+      end
+
+      def run_disable(opts)
+        result = writer(opts).disable!
+        return failure("could not disable telemetry: #{result.error}") if result.failure?
+        stdout.puts "Disabled OTel telemetry. Removed env keys from settings.json."
+        0
+      end
+
+      def run_settings_change(opts)
+        result = yield(writer(opts))
+        return failure("settings update failed: #{result.error}") if result.failure?
+        stdout.puts "Settings updated:"
+        result.value.each { |key, value| stdout.puts "  #{key}=#{value}" }
+        0
+      end
+
+      def run_capture_prompts(opts)
+        stdout.puts "WARNING: enabling OTEL_LOG_USER_PROMPTS=1 will cause Claude Code"
+        stdout.puts "to send your verbatim prompts (and the conversation history) to"
+        stdout.puts "this dashboard's local SQLite. Type 'yes' to continue."
+        stdout.print "> "
+        answer = stdin.gets&.strip
+        unless answer == "yes"
+          stdout.puts "Aborted. No changes made."
+          return 0
+        end
+        run_settings_change(opts) { |w| w.capture_prompts! }
+      end
+
+      # One-time pass: stream every otel_events row with a prompt_id from
+      # the global store and run it through PromptScope so historical
+      # activity_events get tagged. Subsequent runs are cheap because
+      # PromptScope is idempotent (already-tagged rows are excluded by
+      # the `prompt_id: nil` filter).
+      def run_backfill(opts)
+        manager = Store::StoreManager.new
+        unless manager.global_exists?
+          return failure("Global memory DB not found — nothing to backfill against.")
+        end
+        manager.ensure_global!
+        manager.ensure_project! if manager.project_exists?
+
+        store = manager.global_store
+        unless store.db.table_exists?(:otel_events)
+          manager.close
+          return failure("otel_events table missing — run `claude-memory dashboard` first to migrate.")
+        end
+
+        total_events = store.otel_events.exclude(prompt_id: nil).count
+        if total_events.zero?
+          stdout.puts "No OTel events with prompt_id to backfill from."
+          manager.close
+          return 0
+        end
+
+        stdout.puts "Backfilling #{total_events} OTel event(s) into activity_events…"
+        scope = OTel::PromptScope.new(manager)
+        tagged = groups = batches = 0
+        offset = 0
+        batch_size = opts[:batch_size]
+
+        loop do
+          rows = store.otel_events
+            .exclude(prompt_id: nil)
+            .order(:occurred_at, :id)
+            .limit(batch_size)
+            .offset(offset)
+            .select(:event_name, :session_id, :prompt_id, :occurred_at)
+            .all
+          break if rows.empty?
+
+          events = rows.map { |r|
+            {event_name: r[:event_name], session_id: r[:session_id],
+             prompt_id: r[:prompt_id], occurred_at: r[:occurred_at]}
+          }
+          result = scope.tag(events)
+          tagged += result[:tagged].to_i
+          groups += result[:groups].to_i
+          batches += 1
+          offset += rows.size
+          stdout.print "."
+          stdout.flush
+        end
+
+        stdout.puts ""
+        stdout.puts "Backfill complete: tagged #{tagged} activity_event(s) across #{groups} prompt group(s) in #{batches} batch(es)."
+        manager.close
+        0
+      end
+
+      def run_verify(opts)
+        url = URI.parse("http://127.0.0.1:#{opts[:port]}/v1/metrics")
+        body = JSON.generate(sample_metrics_envelope)
+        response = post_json(url, body)
+        if response.is_a?(Net::HTTPSuccess)
+          stdout.puts "Verify OK: dashboard accepted the sample metrics envelope."
+          0
+        else
+          failure("Verify failed: dashboard returned #{response&.code || "no response"}")
+        end
+      rescue Errno::ECONNREFUSED, SocketError => e
+        failure("Verify failed: #{e.message}. Is the dashboard running on port #{opts[:port]}?")
+      end
+
+      def post_json(url, body)
+        Net::HTTP.start(url.host, url.port, open_timeout: 2, read_timeout: 5) do |http|
+          req = Net::HTTP::Post.new(url.path, "Content-Type" => "application/json")
+          req.body = body
+          http.request(req)
+        end
+      end
+
+      # Smallest valid OTLP/HTTP/JSON metrics envelope — one counter point.
+      # Used only for the --verify subcommand so users can confirm
+      # end-to-end wiring without running a real claude session.
+      def sample_metrics_envelope
+        nano = (Time.now.to_f * 1_000_000_000).to_i.to_s
+        {
+          "resourceMetrics" => [{
+            "resource" => {"attributes" => [
+              {"key" => "service.name", "value" => {"stringValue" => "claude-memory-verify"}}
+            ]},
+            "scopeMetrics" => [{
+              "scope" => {"name" => "claude-memory.verify"},
+              "metrics" => [{
+                "name" => "claude_memory.verify",
+                "unit" => "count",
+                "sum" => {
+                  "dataPoints" => [{
+                    "asInt" => "1",
+                    "timeUnixNano" => nano,
+                    "attributes" => []
+                  }]
+                }
+              }]
+            }]
+          }]
+        }
+      end
+    end
+  end
+end

data/lib/claude_memory/commands/registry.rb

@@ -42,7 +42,11 @@ module ClaudeMemory
         "reclassify-references" => {class: ReclassifyReferencesCommand, description: "Retag existing convention facts that match reference-material heuristics"},
         "census" => {class: CensusCommand, description: "Aggregate predicate usage across project databases"},
         "dashboard" => {class: DashboardCommand, description: "Open debugging dashboard"},
-        "digest" => {class: DigestCommand, description: "Render a weekly markdown digest of memory activity"}
+        "digest" => {class: DigestCommand, description: "Render a weekly markdown digest of memory activity"},
+        "show" => {class: ShowCommand, description: "Print what memory would inject at the next SessionStart"},
+        "otel" => {class: OtelCommand, description: "Configure or inspect OpenTelemetry ingestion from Claude Code"},
+        "import-auto-memory" => {class: ImportAutoMemoryCommand, description: "Import Claude Code auto-memory .md files into project DB as facts"},
+        "audit" => {class: AuditCommand, description: "Run memory health audit; report inconsistencies and optimizations"}
       }.freeze
 
       # Find a command class by name

data/lib/claude_memory/commands/show_command.rb

@@ -0,0 +1,90 @@
+# frozen_string_literal: true
+
+require "optparse"
+
+module ClaudeMemory
+  module Commands
+    # Prints what memory would inject on the next SessionStart.
+    #
+    # The trust answer to "is this still worth it?" requires
+    # inspectability: a user who can't see what memory will inject can't
+    # develop confidence in it. The CLAUDE.md alternative is `cat
+    # CLAUDE.md` — instant, plain English, no tooling. This command is
+    # the same one-line inspect surface for the curated facts the
+    # injector picks each session.
+    #
+    # Runs the exact `Hook::ContextInjector` path real sessions use, so
+    # what you see here is what Claude actually receives — not a
+    # rebuilt approximation that could drift.
+    #
+    # The default suppresses the "Pending Knowledge Extraction" dump
+    # (which contains raw transcript JSON intended for LLM distillation)
+    # so the output stays human-readable. Pass `--pending` to see the
+    # full fresh-session payload, including those raw items.
+    class ShowCommand < BaseCommand
+      VALID_SOURCES = %w[startup resume clear].freeze
+
+      # Any string outside FRESH_SESSION_SOURCES skips the pending-knowledge
+      # block. "preview" reads naturally in any debug log this surfaces in.
+      NON_FRESH_SOURCE = "preview"
+
+      def call(args)
+        opts = parse_options(args, {source: nil, pending: false}) do |o|
+          OptionParser.new do |parser|
+            parser.banner = "Usage: claude-memory show [--source SOURCE] [--pending]"
+            parser.on("--source SOURCE", VALID_SOURCES,
+              "Simulate fresh-session source (#{VALID_SOURCES.join(", ")}). " \
+              "Forces inclusion of pending-knowledge and auto-memory-mirror " \
+              "sections regardless of --pending.") { |v| o[:source] = v }
+            parser.on("--pending",
+              "Include the pending-knowledge dump (raw transcript JSON " \
+              "for LLM distillation). Default suppresses it for readability.") { o[:pending] = true }
+          end
+        end
+        return 1 if opts.nil?
+
+        effective_source = opts[:source] || (opts[:pending] ? nil : NON_FRESH_SOURCE)
+
+        manager = Store::StoreManager.new
+        manager.ensure_both!
+        injector = Hook::ContextInjector.new(manager, source: effective_source)
+        context = injector.generate_context
+
+        print_header(opts[:source])
+        stdout.puts ""
+
+        if context.nil? || context.strip.empty?
+          stdout.puts "_Memory has no facts to inject yet._"
+          stdout.puts ""
+          stdout.puts "Run a few Claude Code sessions in this project, or use"
+          stdout.puts "`memory.store_extraction` from a session to seed facts."
+        else
+          stdout.puts context
+          stdout.puts ""
+          print_footer(injector, context)
+        end
+
+        manager.close
+        0
+      rescue Sequel::DatabaseError => e
+        failure("Database error: #{e.message}")
+      end
+
+      private
+
+      def print_header(source)
+        label = source ? " (source=#{source})" : ""
+        stdout.puts "## Memory snapshot — would be injected at next SessionStart#{label}"
+      end
+
+      def print_footer(injector, context)
+        tokens = Core::TokenEstimator.estimate(context)
+        fact_count = injector.emitted_fact_ids.size
+        stdout.puts "---"
+        stdout.puts "#{fact_count} fact#{"s" unless fact_count == 1} • " \
+          "~#{tokens} token#{"s" unless tokens == 1} • " \
+          "#{context.length} chars"
+      end
+    end
+  end
+end

data/lib/claude_memory/commands/stats_command.rb

@@ -13,14 +13,15 @@ module ClaudeMemory
       SCOPE_PROJECT = "project"
 
       def call(args)
-        opts = parse_options(args, {scope: SCOPE_ALL, tools: false, stale: false, since_days: nil, stale_days: nil}) do |o|
+        opts = parse_options(args, {scope: SCOPE_ALL, tools: false, tokens: false, stale: false, since_days: nil, stale_days: nil}) do |o|
           OptionParser.new do |parser|
             parser.banner = "Usage: claude-memory stats [options]"
             parser.on("--scope SCOPE", ["all", "global", "project"],
               "Show stats for: all (default), global, or project") { |v| o[:scope] = v }
             parser.on("--tools", "Show MCP tool-call usage stats") { o[:tools] = true }
+            parser.on("--tokens", "Show SessionStart context-injection token budget") { o[:tokens] = true }
             parser.on("--stale", "Show facts not recalled in CLAUDE_MEMORY_STALE_DAYS (default 14)") { o[:stale] = true }
-            parser.on("--since DAYS", Integer, "Limit --tools to last N days") { |v| o[:since_days] = v }
+            parser.on("--since DAYS", Integer, "Limit --tools/--tokens to last N days") { |v| o[:since_days] = v }
             parser.on("--stale-days N", Integer, "Override staleness threshold for --stale") { |v| o[:stale_days] = v }
           end
         end
@@ -30,6 +31,10 @@ module ClaudeMemory
           return print_mcp_tool_call_stats(opts[:since_days])
         end
 
+        if opts[:tokens]
+          return print_token_budget_stats(opts[:since_days])
+        end
+
         if opts[:stale]
           return print_stale_facts(opts[:stale_days])
         end
@@ -349,6 +354,93 @@ module ClaudeMemory
         1
       end
 
+      TOKEN_BUCKETS = [
+        ["<500", 0, 500],
+        ["500-1000", 500, 1000],
+        ["1000-2000", 1000, 2000],
+        ["2000-5000", 2000, 5000],
+        ["5000+", 5000, Float::INFINITY]
+      ].freeze
+
+      def print_token_budget_stats(since_days)
+        manager = ClaudeMemory::Store::StoreManager.new
+        db_path = manager.project_db_path
+
+        stdout.puts "SessionStart Context Token Budget"
+        stdout.puts "=" * 50
+
+        unless File.exist?(db_path)
+          stdout.puts "Project database does not exist: #{db_path}"
+          manager.close
+          return 0
+        end
+
+        db = open_readonly(db_path)
+
+        unless db.table_exists?(:activity_events)
+          stdout.puts "No activity telemetry recorded yet."
+          db.disconnect
+          manager.close
+          return 0
+        end
+
+        dataset = db[:activity_events]
+          .where(event_type: "hook_context", status: "success")
+        if since_days
+          cutoff = (Time.now - since_days * 86400).utc.iso8601
+          dataset = dataset.where { occurred_at >= cutoff }
+          stdout.puts "Window: last #{since_days} day#{"s" unless since_days == 1}"
+        else
+          stdout.puts "Window: all time"
+        end
+        stdout.puts
+
+        tokens = dataset.select_map(:detail_json).filter_map do |json|
+          next unless json
+          value = JSON.parse(json)["context_tokens"]
+          value if value.is_a?(Integer) && value > 0
+        end
+
+        if tokens.empty?
+          stdout.puts "No context injections recorded in window."
+          stdout.puts ""
+          stdout.puts "Token telemetry is recorded automatically on SessionStart hooks."
+          stdout.puts "Run a Claude Code session in this project to populate."
+          db.disconnect
+          manager.close
+          return 0
+        end
+
+        sorted = tokens.sort
+        total = sorted.size
+        stdout.puts "Sessions: #{format_number(total)}"
+        stdout.puts "p50: #{format_number(percentile(sorted, 0.50))} tokens"
+        stdout.puts "p95: #{format_number(percentile(sorted, 0.95))} tokens"
+        stdout.puts "Avg: #{format_number((sorted.sum.to_f / total).round)} tokens"
+        stdout.puts "Min: #{format_number(sorted.first)} tokens"
+        stdout.puts "Max: #{format_number(sorted.last)} tokens"
+        stdout.puts ""
+        print_token_distribution(sorted)
+
+        db.disconnect
+        manager.close
+        0
+      rescue Sequel::DatabaseError, JSON::ParserError, Extralite::Error => e
+        stderr.puts "Error reading token telemetry: #{e.message}"
+        1
+      end
+
+      def print_token_distribution(sorted)
+        total = sorted.size
+        stdout.puts "Distribution:"
+        TOKEN_BUCKETS.each do |label, low, high|
+          count = sorted.count { |t| t >= low && t < high }
+          pct = (count * 100.0 / total).round(1)
+          bar = "█" * (pct / 5).round
+          stdout.puts "  #{label.ljust(12)} #{count.to_s.rjust(5)} (#{pct.to_s.rjust(5)}%) #{bar}"
+        end
+      end
+
       def print_per_tool_breakdown(dataset)
         stdout.puts "Per-tool breakdown:"
         stdout.puts "  #{"Tool".ljust(28)} #{"Calls".rjust(7)}  #{"Avg ms".rjust(8)}  #{"P95 ms".rjust(8)}  #{"Err %".rjust(6)}"

data/lib/claude_memory/configuration.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require "open3"
+require "json"
 
 module ClaudeMemory
   # Centralized configuration and ENV access
@@ -66,6 +67,65 @@ module ClaudeMemory
       DEFAULT_STALE_DAYS
     end
 
+    # Threshold (in days) for the context-injection staleness marker. A
+    # single-value fact older than this and not recalled within it gets a
+    # "verify before relying" annotation when injected at SessionStart.
+    # Deliberately much longer than DEFAULT_STALE_DAYS (the dashboard's
+    # review-candidate window) — the injection marker should fire only on
+    # facts old enough to be genuinely risky, not merely unused for a
+    # couple weeks. Override via CLAUDE_MEMORY_INJECTION_STALE_DAYS.
+    DEFAULT_INJECTION_STALE_DAYS = 180
+
+    # @return [Integer] injection staleness threshold in days
+    def injection_stale_days
+      raw = env["CLAUDE_MEMORY_INJECTION_STALE_DAYS"]
+      return DEFAULT_INJECTION_STALE_DAYS if raw.nil? || raw.empty?
+      parsed = Integer(raw, 10)
+      (parsed > 0) ? parsed : DEFAULT_INJECTION_STALE_DAYS
+    rescue ArgumentError
+      DEFAULT_INJECTION_STALE_DAYS
+    end
+
+    # Whether OTel trace ingestion is opted in. Reads OTEL_TRACES_EXPORTER
+    # from .claude/settings.json's env block. Traces are off unless the
+    # value is present and non-empty and not "none". Set by
+    # `claude-memory otel --enable-traces`.
+    #
+    # @return [Boolean]
+    def otel_traces_enabled?
+      value = settings_env["OTEL_TRACES_EXPORTER"]
+      return false if value.nil?
+      stripped = value.to_s.strip
+      !stripped.empty? && stripped != "none"
+    end
+
+    # Read the env block from .claude/settings.json (project scope) so
+    # callers can inspect what Claude Code sees at session start. Returns
+    # an empty hash when the file is missing or unparseable — matches the
+    # tolerant behavior of Claude Code's settings loader.
+    #
+    # @return [Hash]
+    def settings_env
+      path = settings_json_path
+      return {} unless path
+      raw = File.read(path)
+      parsed = JSON.parse(raw)
+      env_block = parsed.is_a?(Hash) ? parsed["env"] : nil
+      env_block.is_a?(Hash) ? env_block : {}
+    rescue JSON::ParserError, Errno::ENOENT
+      {}
+    end
+
+    # Path to the project-scoped settings.json. nil when no project_dir
+    # exists (e.g. running outside any directory).
+    #
+    # @return [String, nil]
+    def settings_json_path
+      dir = project_dir
+      return nil unless dir
+      File.join(dir, ".claude", "settings.json")
+    end
+
     private
 
     def resolve_project_dir

data/lib/claude_memory/core/fact_query_builder.rb

@@ -141,6 +141,7 @@ module ClaudeMemory
             Sequel[:facts][:valid_from],
             Sequel[:facts][:valid_to],
             Sequel[:facts][:created_at],
+            Sequel[:facts][:last_recalled_at],
             Sequel[:entities][:canonical_name].as(:subject_name),
             Sequel[:facts][:scope],
             Sequel[:facts][:project_path]

data/lib/claude_memory/dashboard/api.rb

@@ -460,6 +460,14 @@ module ClaudeMemory
         Timeline.new(@manager).days
       end
 
+      def telemetry
+        Telemetry.new(@manager).snapshot
+      end
+
+      def prompt_journey(prompt_id)
+        PromptJourney.new(@manager).for(prompt_id.to_s)
+      end
+
       private
 
       CONTENT_ITEM_PREVIEW_BYTES = 8000