claude_memory 0.4.0 → 0.5.0

data/lib/claude_memory/infrastructure/operation_tracker.rb

@@ -107,23 +107,7 @@ module ClaudeMemory
         count = stuck.count
         return 0 if count.zero?
 
-        now = Time.now.utc.iso8601
-        error_message = "Reset by recover command - operation exceeded 24h timeout"
-
-        # Fetch each stuck operation, update checkpoint in Ruby, then save
-        stuck.all.each do |op|
-          checkpoint = op[:checkpoint_data] ? JSON.parse(op[:checkpoint_data]) : {}
-          checkpoint["error"] = error_message
-
-          @store.db[:operation_progress]
-            .where(id: op[:id])
-            .update(
-              status: "failed",
-              completed_at: now,
-              checkpoint_data: JSON.generate(checkpoint)
-            )
-        end
-
+        fail_operations(stuck, "Reset by recover command - operation exceeded 24h timeout")
         count
       end
 
@@ -132,15 +116,17 @@ module ClaudeMemory
       # Mark stale operations as failed before starting new operation
       def cleanup_stale_operations!(operation_type, scope)
         threshold_time = (Time.now.utc - STALE_THRESHOLD_SECONDS).iso8601
-        now = Time.now.utc.iso8601
-        error_message = "Automatically marked as failed - operation exceeded 24h timeout"
 
         stale = @store.db[:operation_progress]
           .where(operation_type: operation_type, scope: scope, status: "running")
           .where { started_at < threshold_time }
 
-        # Fetch each stale operation, update checkpoint in Ruby, then save
-        stale.all.each do |op|
+        fail_operations(stale, "Automatically marked as failed - operation exceeded 24h timeout")
+      end
+
+      def fail_operations(dataset, error_message)
+        now = Time.now.utc.iso8601
+        dataset.all.each do |op|
           checkpoint = op[:checkpoint_data] ? JSON.parse(op[:checkpoint_data]) : {}
           checkpoint["error"] = error_message
 

data/lib/claude_memory/infrastructure/schema_validator.rb

@@ -33,57 +33,62 @@ module ClaudeMemory
 
       def validate
         issues = []
-
-        # Check tables exist
         tables = @store.db.tables
-        missing_tables = EXPECTED_TABLES - tables
-        missing_tables.each do |table|
+
+        check_tables(tables, issues)
+        check_columns(tables, issues)
+        check_indexes(issues)
+        check_orphaned_records(issues)
+        check_enum_values(issues)
+        check_embedding_dimensions(issues)
+
+        record_health_check(issues)
+
+        {
+          valid: issues.none? { |i| i[:severity] == "error" },
+          issues: issues
+        }
+      end
+
+      private
+
+      def check_tables(tables, issues)
+        (EXPECTED_TABLES - tables).each do |table|
           issues << {severity: "error", message: "Missing table: #{table}"}
         end
+      end
 
-        # Check critical columns exist
+      def check_columns(tables, issues)
         CRITICAL_COLUMNS.each do |table, columns|
           next unless tables.include?(table)
 
           existing_columns = @store.db.schema(table).map(&:first)
-          missing_columns = columns - existing_columns
-          missing_columns.each do |column|
+          (columns - existing_columns).each do |column|
             issues << {severity: "error", message: "Missing column #{table}.#{column}"}
           end
         end
+      end
 
-        # Check critical indexes exist
+      def check_indexes(issues)
         index_names = @store.db["SELECT name FROM sqlite_master WHERE type='index'"]
           .all.map { |r| r[:name] }
-        missing_indexes = CRITICAL_INDEXES - index_names.map(&:to_sym)
-        missing_indexes.each do |index|
+        (CRITICAL_INDEXES - index_names.map(&:to_sym)).each do |index|
           issues << {severity: "warning", message: "Missing index: #{index}"}
         end
+      end
 
-        # Check for orphaned records
+      def check_orphaned_records(issues)
         check_orphaned_provenance(issues)
         check_orphaned_fact_links(issues)
         check_orphaned_tool_calls(issues)
+      end
 
-        # Check for invalid enum values
+      def check_enum_values(issues)
         check_invalid_fact_scopes(issues)
         check_invalid_fact_status(issues)
         check_invalid_operation_status(issues)
-
-        # Check embedding dimensions
-        check_embedding_dimensions(issues)
-
-        # Record validation result
-        record_health_check(issues)
-
-        {
-          valid: issues.none? { |i| i[:severity] == "error" },
-          issues: issues
-        }
       end
 
-      private
-
       def check_orphaned_provenance(issues)
         orphaned = @store.db[:provenance]
           .left_join(:facts, id: :fact_id)

data/lib/claude_memory/ingest/content_sanitizer.rb

@@ -9,7 +9,14 @@ module ClaudeMemory
     # Performance is O(n) and excellent even with 1000+ tags (~0.6ms).
     # Long Claude sessions legitimately accumulate many tags (100-200+).
     class ContentSanitizer
-      SYSTEM_TAGS = ["claude-memory-context"].freeze
+      SYSTEM_TAGS = [
+        "claude-memory-context",
+        "system-reminder",
+        "local-command-caveat",
+        "command-message",
+        "command-name",
+        "command-args"
+      ].freeze
       USER_TAGS = ["private", "no-memory", "secret"].freeze
 
       def self.strip_tags(text)

data/lib/claude_memory/ingest/ingester.rb

@@ -5,85 +5,91 @@ require "digest"
 module ClaudeMemory
   module Ingest
     class Ingester
-      def initialize(store, fts: nil, env: ENV, metadata_extractor: nil, tool_extractor: nil)
+      def initialize(store, fts: nil, env: ENV, metadata_extractor: nil, tool_extractor: nil, tool_filter: nil)
         @store = store
         @fts = fts || Index::LexicalFTS.new(store)
         @config = Configuration.new(env)
         @metadata_extractor = metadata_extractor || MetadataExtractor.new
         @tool_extractor = tool_extractor || ToolExtractor.new
+        @tool_filter = tool_filter || ToolFilter.new
       end
 
       def ingest(source:, session_id:, transcript_path:, project_path: nil)
-        # Check if file has been modified since last ingestion (incremental sync)
         unless should_ingest?(transcript_path)
+          ClaudeMemory.logger.debug("ingest", message: "Skipped unchanged file", transcript_path: transcript_path)
           return {status: :skipped, bytes_read: 0, reason: "unchanged"}
         end
 
-        current_offset = @store.get_delta_cursor(session_id, transcript_path) || 0
-        delta, new_offset = TranscriptReader.read_delta(transcript_path, current_offset)
+        prepared = prepare_delta(session_id, transcript_path, project_path)
+        return {status: :no_change, bytes_read: 0} if prepared.nil?
 
-        return {status: :no_change, bytes_read: 0} if delta.nil?
+        content_id = persist_content(source, session_id, transcript_path, prepared)
 
-        # Extract session metadata and tool calls before sanitization
-        metadata = @metadata_extractor.extract(delta)
-        tool_calls = @tool_extractor.extract(delta)
+        log_ingestion(content_id, prepared, session_id)
+        {status: :ingested, content_id: content_id, bytes_read: prepared[:delta].bytesize, project_path: prepared[:project_path]}
+      end
 
-        # Strip privacy tags before storing
-        delta = ContentSanitizer.strip_tags(delta)
+      private
 
-        resolved_project = project_path || detect_project_path
+      def prepare_delta(session_id, transcript_path, project_path)
+        current_offset = @store.get_delta_cursor(session_id, transcript_path) || 0
+        delta, new_offset = TranscriptReader.read_delta(transcript_path, current_offset)
+        return nil if delta.nil?
 
-        # Get source file mtime for incremental sync
-        source_mtime = File.exist?(transcript_path) ? File.mtime(transcript_path).utc.iso8601 : nil
+        metadata = @metadata_extractor.extract(delta)
+        tool_calls = @tool_filter.filter(@tool_extractor.extract(delta))
+        delta = ContentSanitizer.strip_tags(delta)
 
-        text_hash = Digest::SHA256.hexdigest(delta)
+        {
+          delta: delta,
+          new_offset: new_offset,
+          metadata: metadata,
+          tool_calls: tool_calls,
+          project_path: project_path || detect_project_path,
+          source_mtime: File.exist?(transcript_path) ? File.mtime(transcript_path).utc.iso8601 : nil,
+          text_hash: Digest::SHA256.hexdigest(delta)
+        }
+      end
 
-        # Wrap entire ingestion pipeline in transaction for atomicity
-        # If any step fails, cursor position is not updated, allowing retry
-        content_id = nil
-        begin
-          content_id = with_retry do
-            @store.db.transaction do
-              content_id = @store.upsert_content_item(
-                source: source,
-                session_id: session_id,
-                transcript_path: transcript_path,
-                project_path: resolved_project,
-                text_hash: text_hash,
-                byte_len: delta.bytesize,
-                raw_text: delta,
-                git_branch: metadata[:git_branch],
-                cwd: metadata[:cwd],
-                claude_version: metadata[:claude_version],
-                thinking_level: metadata[:thinking_level],
-                source_mtime: source_mtime
-              )
-
-              # Store tool calls if any were extracted
-              @store.insert_tool_calls(content_id, tool_calls) unless tool_calls.empty?
-
-              # FTS indexing (FTS5 supports transactions)
-              @fts.index_content_item(content_id, delta)
-
-              # Update cursor LAST - only after all other operations succeed
-              # This ensures that if any step fails, we can retry from the same offset
-              @store.update_delta_cursor(session_id, transcript_path, new_offset)
-
-              content_id
-            end
+      def persist_content(source, session_id, transcript_path, prepared)
+        with_retry do
+          @store.db.transaction do
+            content_id = @store.upsert_content_item(
+              source: source,
+              session_id: session_id,
+              transcript_path: transcript_path,
+              project_path: prepared[:project_path],
+              text_hash: prepared[:text_hash],
+              byte_len: prepared[:delta].bytesize,
+              raw_text: prepared[:delta],
+              git_branch: prepared[:metadata][:git_branch],
+              cwd: prepared[:metadata][:cwd],
+              claude_version: prepared[:metadata][:claude_version],
+              thinking_level: prepared[:metadata][:thinking_level],
+              source_mtime: prepared[:source_mtime]
+            )
+
+            @store.insert_tool_calls(content_id, prepared[:tool_calls]) unless prepared[:tool_calls].empty?
+            @fts.index_content_item(content_id, prepared[:delta])
+            @store.update_delta_cursor(session_id, transcript_path, prepared[:new_offset])
+
+            content_id
           end
-        rescue Extralite::BusyError => e
-          # Re-raise BusyError with context after all retries exhausted
-          raise StandardError, "Ingestion failed for session #{session_id} after retries: #{e.message}"
-        rescue => e
-          # Re-raise other errors with context for better error messages
-          raise StandardError, "Ingestion failed for session #{session_id}: #{e.message}"
         end
-
-        {status: :ingested, content_id: content_id, bytes_read: delta.bytesize, project_path: resolved_project}
+      rescue Extralite::BusyError => e
+        raise StandardError, "Ingestion failed for session #{session_id} after retries: #{e.message}"
+      rescue => e
+        raise StandardError, "Ingestion failed for session #{session_id}: #{e.message}"
       end
 
-      private
+      def log_ingestion(content_id, prepared, session_id)
+        ClaudeMemory.logger.info("ingest",
+          message: "Ingested content",
+          content_id: content_id,
+          bytes_read: prepared[:delta].bytesize,
+          session_id: session_id,
+          tool_calls: prepared[:tool_calls].size)
+      end
 
       # Retry database operations with exponential backoff + jitter
       # This handles concurrent access when MCP server and hooks both write simultaneously
@@ -105,6 +111,11 @@ module ClaudeMemory
             exponential_delay = [base_delay * (2**(attempt - 1)), max_delay].min
             jitter = rand * exponential_delay * 0.5
             total_delay = exponential_delay + jitter
+            ClaudeMemory.logger.warn("ingest",
+              message: "Database busy, retrying",
+              attempt: attempt,
+              max_attempts: max_attempts,
+              delay_seconds: total_delay.round(3))
             sleep(total_delay)
             retry
           elsif is_busy

data/lib/claude_memory/ingest/tool_extractor.rb

@@ -25,7 +25,7 @@ module ClaudeMemory
         end
 
         tools
-      rescue
+      rescue JSON::ParserError
         # If we encounter any parsing errors, return what we've collected so far
         tools
       end

data/lib/claude_memory/ingest/tool_filter.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+module ClaudeMemory
+  module Ingest
+    # Filters tool calls during ingestion to reduce noise
+    # Follows Functional Core pattern - no I/O, just predicate logic
+    #
+    # Supports two modes:
+    #   - skip_tools (blacklist): Skip listed tools, capture everything else
+    #   - capture_tools (whitelist): Only capture listed tools, skip everything else
+    #
+    # When both are empty, all tools are captured (no filtering).
+    class ToolFilter
+      # Default tools to skip - high-volume, read-only tools that add noise
+      DEFAULT_SKIP_TOOLS = %w[Read Glob Grep].freeze
+
+      attr_reader :skip_tools, :capture_tools
+
+      # @param skip_tools [Array<String>] Tool names to skip (blacklist mode)
+      # @param capture_tools [Array<String>] Tool names to capture exclusively (whitelist mode)
+      def initialize(skip_tools: nil, capture_tools: nil)
+        @skip_tools = skip_tools || DEFAULT_SKIP_TOOLS
+        @capture_tools = capture_tools
+      end
+
+      # Check if a tool call should be captured
+      # @param tool_name [String] Name of the tool
+      # @return [Boolean] true if the tool should be captured
+      def capture?(tool_name)
+        return false if tool_name.nil?
+
+        # Whitelist mode takes precedence
+        if @capture_tools && !@capture_tools.empty?
+          return @capture_tools.include?(tool_name)
+        end
+
+        # Blacklist mode
+        !@skip_tools.include?(tool_name)
+      end
+
+      # Filter an array of tool call hashes
+      # @param tool_calls [Array<Hash>] Tool calls with :tool_name key
+      # @return [Array<Hash>] Filtered tool calls
+      def filter(tool_calls)
+        tool_calls.select { |tc| capture?(tc[:tool_name]) }
+      end
+
+      # Create a filter with no filtering (captures all tools)
+      # @return [ToolFilter] Permissive filter
+      def self.allow_all
+        new(skip_tools: [], capture_tools: nil)
+      end
+    end
+  end
+end

data/lib/claude_memory/logging/logger.rb

@@ -0,0 +1,112 @@
+# frozen_string_literal: true
+
+require "json"
+
+module ClaudeMemory
+  module Logging
+    # Structured JSON logger for ClaudeMemory operations.
+    # Outputs machine-readable JSON log entries to a configurable stream.
+    #
+    # Log levels: DEBUG (0), INFO (1), WARN (2), ERROR (3)
+    # Configure via CLAUDE_MEMORY_LOG_LEVEL env var (default: WARN)
+    #
+    # @example Basic usage
+    #   logger = ClaudeMemory::Logging::Logger.new
+    #   logger.info("ingest", message: "Ingested 1024 bytes", content_id: 42)
+    #
+    # @example With custom output
+    #   logger = ClaudeMemory::Logging::Logger.new(output: StringIO.new, level: :debug)
+    #   logger.debug("recall", message: "Query executed", query: "test", results: 5)
+    class Logger
+      DEBUG = 0
+      INFO = 1
+      WARN = 2
+      ERROR = 3
+
+      LEVELS = {debug: DEBUG, info: INFO, warn: WARN, error: ERROR}.freeze
+
+      attr_reader :level
+
+      # @param output [IO] output stream for log entries (default: $stderr)
+      # @param level [Symbol, String] minimum log level (default: from env or :warn)
+      def initialize(output: $stderr, level: nil)
+        @output = output
+        @level = resolve_level(level)
+      end
+
+      def debug(component, **fields)
+        log(:debug, component, **fields)
+      end
+
+      def info(component, **fields)
+        log(:info, component, **fields)
+      end
+
+      def warn(component, **fields)
+        log(:warn, component, **fields)
+      end
+
+      def error(component, **fields)
+        log(:error, component, **fields)
+      end
+
+      def debug?
+        @level <= DEBUG
+      end
+
+      def info?
+        @level <= INFO
+      end
+
+      private
+
+      def log(level_sym, component, **fields)
+        return unless LEVELS[level_sym] >= @level
+
+        entry = {
+          timestamp: Time.now.utc.iso8601(3),
+          level: level_sym.to_s.upcase,
+          component: component
+        }.merge(fields)
+
+        @output.puts(JSON.generate(entry))
+      rescue IOError
+        # Silently ignore write failures (e.g., closed pipe)
+      end
+
+      def resolve_level(explicit_level)
+        if explicit_level
+          sym = explicit_level.to_s.downcase.to_sym
+          LEVELS.fetch(sym, WARN)
+        else
+          env_level = ENV["CLAUDE_MEMORY_LOG_LEVEL"]
+          if env_level
+            sym = env_level.downcase.to_sym
+            LEVELS.fetch(sym, WARN)
+          else
+            WARN
+          end
+        end
+      end
+    end
+
+    # Null logger that discards all output (for testing or silent operation)
+    class NullLogger
+      def debug(component, **fields) = nil
+
+      def info(component, **fields) = nil
+
+      def warn(component, **fields) = nil
+
+      def error(component, **fields) = nil
+
+      def debug? = false
+
+      def info? = false
+
+      def level
+        Logger::ERROR + 1
+      end
+    end
+  end
+end

data/lib/claude_memory/mcp/query_guide.rb

@@ -0,0 +1,96 @@
+# frozen_string_literal: true
+
+module ClaudeMemory
+  module MCP
+    # MCP prompt that teaches Claude when to use each memory tool.
+    # Registered as "memory_guide" via prompts/list.
+    module QueryGuide
+      PROMPT_NAME = "memory_guide"
+      PROMPT_DESCRIPTION = "Guide for choosing the right memory search tool"
+
+      PROMPT_TEXT = <<~GUIDE
+        # ClaudeMemory Search Strategy Guide
+
+        ## Tool Selection
+
+        **memory.recall** — Full-text keyword search (fastest)
+        - Use for: exact terms, known predicates, specific entity names
+        - Example: "PostgreSQL", "authentication", "deployment"
+        - Returns: facts with provenance receipts
+
+        **memory.recall_semantic** — Vector similarity search
+        - Use for: conceptual queries, paraphrased questions, "find things like X"
+        - Modes: `vector` (embeddings only), `text` (FTS only), `both` (hybrid, recommended)
+        - Example: "how does the app handle user sessions" (no exact keyword match needed)
+        - Returns: facts ranked by similarity score (0.0-1.0)
+
+        **memory.search_concepts** — Multi-concept AND query
+        - Use for: intersection of 2-5 concepts that must ALL be present
+        - Example: concepts=["authentication", "JWT", "middleware"]
+        - Returns: facts matching all concepts, ranked by average similarity
+
+        **memory.recall_index** → **memory.recall_details** — Progressive disclosure
+        - Use for: browsing large result sets efficiently
+        - Step 1: `recall_index` returns lightweight previews with token estimates
+        - Step 2: `recall_details` fetches full data for selected fact IDs
+        - Saves tokens when you only need a few facts from many matches
+
+        ## Shortcut Tools
+
+        **memory.decisions** — Architectural decisions and constraints
+        **memory.conventions** — Coding style preferences and rules
+        **memory.architecture** — Framework choices and patterns
+
+        ## Context-Aware Tools
+
+        **memory.facts_by_tool** — Facts discovered via specific tool (Read, Edit, Bash)
+        **memory.facts_by_context** — Facts from specific git branch or directory
+
+        ## Decision Tree
+
+        1. Know the exact keyword? → `memory.recall`
+        2. Conceptual/fuzzy question? → `memory.recall_semantic` (mode: both)
+        3. Need intersection of topics? → `memory.search_concepts`
+        4. Looking for decisions? → `memory.decisions`
+        5. Looking for conventions? → `memory.conventions`
+        6. Many results expected? → `memory.recall_index` then `memory.recall_details`
+        7. Need provenance? → `memory.explain` with fact ID
+
+        ## Score Interpretation (semantic search)
+
+        - **> 0.85**: Strong match, high confidence
+        - **0.70-0.85**: Good match, likely relevant
+        - **0.55-0.70**: Moderate match, may be tangentially related
+        - **< 0.55**: Weak match, probably not relevant
+
+        ## Scope Parameter
+
+        All query tools accept `scope`: `"all"` (default), `"global"`, or `"project"`.
+        - `global`: User-wide preferences and conventions
+        - `project`: Current project facts only
+        - `all`: Both (project facts take precedence)
+      GUIDE
+
+      def self.definition
+        {
+          name: PROMPT_NAME,
+          description: PROMPT_DESCRIPTION
+        }
+      end
+
+      def self.content
+        {
+          messages: [
+            {
+              role: "user",
+              content: {
+                type: "text",
+                text: PROMPT_TEXT
+              }
+            }
+          ]
+        }
+      end
+    end
+  end
+end