claude_memory 0.2.0 → 0.3.0

data/lib/claude_memory/ingest/ingester.rb

@@ -5,44 +5,128 @@ require "digest"
 module ClaudeMemory
   module Ingest
     class Ingester
-      def initialize(store, fts: nil, env: ENV)
+      def initialize(store, fts: nil, env: ENV, metadata_extractor: nil, tool_extractor: nil)
         @store = store
         @fts = fts || Index::LexicalFTS.new(store)
-        @env = env
+        @config = Configuration.new(env)
+        @metadata_extractor = metadata_extractor || MetadataExtractor.new
+        @tool_extractor = tool_extractor || ToolExtractor.new
       end
 
       def ingest(source:, session_id:, transcript_path:, project_path: nil)
+        # Check if file has been modified since last ingestion (incremental sync)
+        unless should_ingest?(transcript_path)
+          return {status: :skipped, bytes_read: 0, reason: "unchanged"}
+        end
+
         current_offset = @store.get_delta_cursor(session_id, transcript_path) || 0
         delta, new_offset = TranscriptReader.read_delta(transcript_path, current_offset)
 
         return {status: :no_change, bytes_read: 0} if delta.nil?
 
+        # Extract session metadata and tool calls before sanitization
+        metadata = @metadata_extractor.extract(delta)
+        tool_calls = @tool_extractor.extract(delta)
+
         # Strip privacy tags before storing
         delta = ContentSanitizer.strip_tags(delta)
 
         resolved_project = project_path || detect_project_path
 
+        # Get source file mtime for incremental sync
+        source_mtime = File.exist?(transcript_path) ? File.mtime(transcript_path).utc.iso8601 : nil
+
         text_hash = Digest::SHA256.hexdigest(delta)
-        content_id = @store.upsert_content_item(
-          source: source,
-          session_id: session_id,
-          transcript_path: transcript_path,
-          project_path: resolved_project,
-          text_hash: text_hash,
-          byte_len: delta.bytesize,
-          raw_text: delta
-        )
-
-        @fts.index_content_item(content_id, delta)
-        @store.update_delta_cursor(session_id, transcript_path, new_offset)
+
+        # Wrap entire ingestion pipeline in transaction for atomicity
+        # If any step fails, cursor position is not updated, allowing retry
+        content_id = nil
+        begin
+          content_id = with_retry do
+            @store.db.transaction do
+              content_id = @store.upsert_content_item(
+                source: source,
+                session_id: session_id,
+                transcript_path: transcript_path,
+                project_path: resolved_project,
+                text_hash: text_hash,
+                byte_len: delta.bytesize,
+                raw_text: delta,
+                git_branch: metadata[:git_branch],
+                cwd: metadata[:cwd],
+                claude_version: metadata[:claude_version],
+                thinking_level: metadata[:thinking_level],
+                source_mtime: source_mtime
+              )
+
+              # Store tool calls if any were extracted
+              @store.insert_tool_calls(content_id, tool_calls) unless tool_calls.empty?
+
+              # FTS indexing (FTS5 supports transactions)
+              @fts.index_content_item(content_id, delta)
+
+              # Update cursor LAST - only after all other operations succeed
+              # This ensures that if any step fails, we can retry from the same offset
+              @store.update_delta_cursor(session_id, transcript_path, new_offset)
+
+              content_id
+            end
+          end
+        rescue Extralite::BusyError => e
+          # Re-raise BusyError with context after all retries exhausted
+          raise StandardError, "Ingestion failed for session #{session_id} after retries: #{e.message}"
+        rescue => e
+          # Re-raise other errors with context for better error messages
+          raise StandardError, "Ingestion failed for session #{session_id}: #{e.message}"
+        end
 
         {status: :ingested, content_id: content_id, bytes_read: delta.bytesize, project_path: resolved_project}
       end
 
       private
 
+      # Retry database operations with exponential backoff + jitter
+      # This handles concurrent access when MCP server and hooks both write simultaneously
+      # With busy_timeout=30000ms, each attempt waits up to 30s before raising BusyError
+      # Total potential wait time: 30s * 10 attempts + backoff delays = ~5 minutes max
+      def with_retry(max_attempts: 10, base_delay: 0.2, max_delay: 5.0)
+        attempt = 0
+        begin
+          attempt += 1
+          yield
+        rescue Extralite::BusyError, Sequel::DatabaseError => e
+          # Handle busy errors from extralite adapter
+          is_busy = e.is_a?(Extralite::BusyError) || e.message.include?("busy")
+          if is_busy && attempt < max_attempts
+            # Exponential backoff with jitter to avoid thundering herd
+            exponential_delay = [base_delay * (2**(attempt - 1)), max_delay].min
+            jitter = rand * exponential_delay * 0.5
+            total_delay = exponential_delay + jitter
+            sleep(total_delay)
+            retry
+          elsif is_busy
+            raise
+          else
+            # Not a busy error, re-raise immediately
+            raise
+          end
+        end
+      end
+
+      def should_ingest?(transcript_path)
+        return true unless File.exist?(transcript_path)
+
+        file_mtime = File.mtime(transcript_path).utc.iso8601
+
+        # Check if we've already processed this version of the file
+        existing = @store.content_item_by_transcript_and_mtime(transcript_path, file_mtime)
+
+        # Ingest if we haven't seen this version before
+        existing.nil?
+      end
+
       def detect_project_path
-        @env["CLAUDE_PROJECT_DIR"] || Dir.pwd
+        @config.project_dir
       end
     end
   end

data/lib/claude_memory/ingest/metadata_extractor.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+require "json"
+
+module ClaudeMemory
+  module Ingest
+    # Extracts session metadata from JSONL transcript messages
+    # Captures git branch, working directory, Claude version, thinking level
+    class MetadataExtractor
+      # Extract metadata from raw transcript text
+      # @param raw_text [String] the raw JSONL transcript content
+      # @return [Hash] metadata hash with extracted values
+      def extract(raw_text)
+        return {} if raw_text.nil? || raw_text.empty?
+
+        # Parse first JSONL message for metadata
+        first_line = raw_text.lines.first
+        return {} unless first_line&.strip&.start_with?("{")
+
+        message = JSON.parse(first_line)
+        {
+          git_branch: extract_git_branch(message),
+          cwd: extract_cwd(message),
+          claude_version: extract_claude_version(message),
+          thinking_level: extract_thinking_level(message)
+        }.compact
+      rescue JSON::ParserError
+        {}
+      end
+
+      private
+
+      def extract_git_branch(message)
+        # Check for gitBranch in top-level or nested metadata
+        message["gitBranch"] || message.dig("metadata", "gitBranch")
+      end
+
+      def extract_cwd(message)
+        # Check for cwd or workingDirectory
+        message["cwd"] || message["workingDirectory"] ||
+          message.dig("metadata", "cwd") || message.dig("metadata", "workingDirectory")
+      end
+
+      def extract_claude_version(message)
+        # Check various version fields
+        message["version"] || message["claude_version"] ||
+          message.dig("metadata", "version") || message.dig("metadata", "claude_version")
+      end
+
+      def extract_thinking_level(message)
+        # Extract thinking metadata level
+        message.dig("thinkingMetadata", "level") ||
+          message.dig("metadata", "thinkingLevel")
+      end
+    end
+  end
+end

data/lib/claude_memory/ingest/tool_extractor.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+require "json"
+
+module ClaudeMemory
+  module Ingest
+    # Extracts tool usage information from JSONL transcript messages
+    # Tracks which tools were called during a session
+    class ToolExtractor
+      # Extract tool calls from raw transcript text
+      # @param raw_text [String] the raw JSONL transcript content
+      # @return [Array<Hash>] array of tool call hashes
+      def extract(raw_text)
+        return [] if raw_text.nil? || raw_text.empty?
+
+        tools = []
+
+        raw_text.lines.each do |line|
+          next unless line.strip.start_with?("{")
+
+          message = parse_message(line)
+          next unless message
+
+          extract_tools_from_message(message, tools)
+        end
+
+        tools
+      rescue
+        # If we encounter any parsing errors, return what we've collected so far
+        tools
+      end
+
+      private
+
+      def parse_message(line)
+        JSON.parse(line)
+      rescue JSON::ParserError
+        nil
+      end
+
+      def extract_tools_from_message(message, tools)
+        # Look for assistant messages with content blocks
+        return unless message["type"] == "assistant"
+
+        content = message.dig("message", "content")
+        return unless content.is_a?(Array)
+
+        timestamp = message["timestamp"] || Time.now.utc.iso8601
+
+        content.each do |block|
+          next unless block["type"] == "tool_use"
+
+          tools << {
+            tool_name: block["name"],
+            tool_input: serialize_tool_input(block["input"]),
+            timestamp: timestamp,
+            is_error: false
+          }
+        end
+      end
+
+      def serialize_tool_input(input)
+        return nil unless input
+
+        # Convert to JSON, truncating if too large
+        json = input.to_json
+        (json.length > 1000) ? json[0...1000] + "..." : json
+      end
+    end
+  end
+end

data/lib/claude_memory/mcp/response_formatter.rb

@@ -0,0 +1,331 @@
+# frozen_string_literal: true
+
+module ClaudeMemory
+  module MCP
+    # Pure logic for formatting domain objects into MCP tool responses
+    # Follows Functional Core pattern - no I/O, just transformations
+    class ResponseFormatter
+      # Format recall query results into MCP response
+      # @param results [Array<Hash>] Recall results with :fact and :receipts
+      # @return [Hash] MCP response with facts array
+      def self.format_recall_results(results)
+        {
+          facts: results.map { |r| format_recall_fact(r) }
+        }
+      end
+
+      # Format single recall fact result
+      # @param result [Hash] Single result with :fact, :receipts, :source
+      # @return [Hash] Formatted fact for MCP response
+      def self.format_recall_fact(result)
+        {
+          id: result[:fact][:id],
+          subject: result[:fact][:subject_name],
+          predicate: result[:fact][:predicate],
+          object: result[:fact][:object_literal],
+          status: result[:fact][:status],
+          source: result[:source],
+          receipts: result[:receipts].map { |p| format_receipt(p) }
+        }
+      end
+
+      # Format index query results with token estimates
+      # @param query [String] Original query
+      # @param scope [String] Scope used
+      # @param results [Array<Hash>] Index results with fact data
+      # @return [Hash] MCP response with metadata and facts
+      def self.format_index_results(query, scope, results)
+        total_tokens = results.sum { |r| r[:token_estimate] }
+
+        {
+          query: query,
+          scope: scope,
+          result_count: results.size,
+          total_estimated_tokens: total_tokens,
+          facts: results.map { |r| format_index_fact(r) }
+        }
+      end
+
+      # Format single index fact with preview
+      # @param result [Hash] Index result with fact data and token estimate
+      # @return [Hash] Formatted fact for index response
+      def self.format_index_fact(result)
+        {
+          id: result[:id],
+          subject: result[:subject],
+          predicate: result[:predicate],
+          object_preview: result[:object_preview],
+          status: result[:status],
+          scope: result[:scope],
+          confidence: result[:confidence],
+          tokens: result[:token_estimate],
+          source: result[:source]
+        }
+      end
+
+      # Format explanation with full fact details and relationships
+      # @param explanation [Hash] Explanation with :fact, :receipts, :supersedes, etc.
+      # @param scope [String] Source scope
+      # @return [Hash] MCP response with fact, receipts, and relationships
+      def self.format_explanation(explanation, scope)
+        {
+          fact: {
+            id: explanation[:fact][:id],
+            subject: explanation[:fact][:subject_name],
+            predicate: explanation[:fact][:predicate],
+            object: explanation[:fact][:object_literal],
+            status: explanation[:fact][:status],
+            valid_from: explanation[:fact][:valid_from],
+            valid_to: explanation[:fact][:valid_to]
+          },
+          source: scope,
+          receipts: explanation[:receipts].map { |p| format_receipt(p) },
+          supersedes: explanation[:supersedes],
+          superseded_by: explanation[:superseded_by],
+          conflicts: explanation[:conflicts].map { |c| c[:id] }
+        }
+      end
+
+      # Format detailed explanation for recall_details response
+      # @param explanation [Hash] Explanation with full relationships
+      # @return [Hash] Detailed fact response
+      def self.format_detailed_explanation(explanation)
+        {
+          fact: {
+            id: explanation[:fact][:id],
+            subject: explanation[:fact][:subject_name],
+            predicate: explanation[:fact][:predicate],
+            object: explanation[:fact][:object_literal],
+            status: explanation[:fact][:status],
+            confidence: explanation[:fact][:confidence],
+            scope: explanation[:fact][:scope],
+            valid_from: explanation[:fact][:valid_from],
+            valid_to: explanation[:fact][:valid_to]
+          },
+          receipts: explanation[:receipts].map { |r| format_detailed_receipt(r) },
+          relationships: {
+            supersedes: explanation[:supersedes],
+            superseded_by: explanation[:superseded_by],
+            conflicts: explanation[:conflicts].map { |c| {id: c[:id], status: c[:status]} }
+          }
+        }
+      end
+
+      # Format receipt (provenance) with minimal fields
+      # @param receipt [Hash] Receipt with :quote and :strength
+      # @return [Hash] Formatted receipt
+      def self.format_receipt(receipt)
+        {quote: receipt[:quote], strength: receipt[:strength]}
+      end
+
+      # Format detailed receipt with session and timestamp
+      # @param receipt [Hash] Receipt with full fields
+      # @return [Hash] Formatted detailed receipt
+      def self.format_detailed_receipt(receipt)
+        {
+          quote: receipt[:quote],
+          strength: receipt[:strength],
+          session_id: receipt[:session_id],
+          occurred_at: receipt[:occurred_at]
+        }
+      end
+
+      # Format changes list into MCP response
+      # @param since [String] ISO timestamp
+      # @param changes [Array<Hash>] Change records
+      # @return [Hash] MCP response with since and formatted changes
+      def self.format_changes(since, changes)
+        {
+          since: since,
+          changes: changes.map { |c| format_change(c) }
+        }
+      end
+
+      # Format single change record
+      # @param change [Hash] Change with fact fields
+      # @return [Hash] Formatted change
+      def self.format_change(change)
+        {
+          id: change[:id],
+          predicate: change[:predicate],
+          object: change[:object_literal],
+          status: change[:status],
+          created_at: change[:created_at],
+          source: change[:source]
+        }
+      end
+
+      # Format conflicts list into MCP response
+      # @param conflicts [Array<Hash>] Conflict records
+      # @return [Hash] MCP response with count and formatted conflicts
+      def self.format_conflicts(conflicts)
+        {
+          count: conflicts.size,
+          conflicts: conflicts.map { |c| format_conflict(c) }
+        }
+      end
+
+      # Format single conflict record
+      # @param conflict [Hash] Conflict with fact IDs
+      # @return [Hash] Formatted conflict
+      def self.format_conflict(conflict)
+        {
+          id: conflict[:id],
+          fact_a: conflict[:fact_a_id],
+          fact_b: conflict[:fact_b_id],
+          status: conflict[:status],
+          source: conflict[:source]
+        }
+      end
+
+      # Format sweep statistics into MCP response
+      # @param scope [String] Database scope swept
+      # @param stats [Hash] Sweeper stats
+      # @return [Hash] Formatted sweep response
+      def self.format_sweep_stats(scope, stats)
+        {
+          scope: scope,
+          proposed_expired: stats[:proposed_facts_expired],
+          disputed_expired: stats[:disputed_facts_expired],
+          orphaned_deleted: stats[:orphaned_provenance_deleted],
+          content_pruned: stats[:old_content_pruned],
+          elapsed_seconds: stats[:elapsed_seconds].round(3)
+        }
+      end
+
+      # Format semantic search results with similarity scores
+      # @param query [String] Search query
+      # @param mode [String] Search mode (vector, text, both)
+      # @param scope [String] Scope
+      # @param results [Array<Hash>] Results with similarity scores
+      # @return [Hash] Formatted semantic search response
+      def self.format_semantic_results(query, mode, scope, results)
+        {
+          query: query,
+          mode: mode,
+          scope: scope,
+          count: results.size,
+          facts: results.map { |r| format_semantic_fact(r) }
+        }
+      end
+
+      # Format single semantic search fact with similarity
+      # @param result [Hash] Result with fact, receipts, and similarity
+      # @return [Hash] Formatted fact with similarity
+      def self.format_semantic_fact(result)
+        {
+          id: result[:fact][:id],
+          subject: result[:fact][:subject_name],
+          predicate: result[:fact][:predicate],
+          object: result[:fact][:object_literal],
+          scope: result[:fact][:scope],
+          source: result[:source],
+          similarity: result[:similarity],
+          receipts: result[:receipts].map { |r| format_receipt(r) }
+        }
+      end
+
+      # Format concept search results
+      # @param concepts [Array<String>] Concepts searched
+      # @param scope [String] Scope
+      # @param results [Array<Hash>] Results with similarity scores
+      # @return [Hash] Formatted concept search response
+      def self.format_concept_results(concepts, scope, results)
+        {
+          concepts: concepts,
+          scope: scope,
+          count: results.size,
+          facts: results.map { |r| format_concept_fact(r) }
+        }
+      end
+
+      # Format single concept search fact with multi-concept similarity
+      # @param result [Hash] Result with average and per-concept similarities
+      # @return [Hash] Formatted fact with concept similarities
+      def self.format_concept_fact(result)
+        {
+          id: result[:fact][:id],
+          subject: result[:fact][:subject_name],
+          predicate: result[:fact][:predicate],
+          object: result[:fact][:object_literal],
+          scope: result[:fact][:scope],
+          source: result[:source],
+          average_similarity: result[:similarity],
+          concept_similarities: result[:concept_similarities],
+          receipts: result[:receipts].map { |r| format_receipt(r) }
+        }
+      end
+
+      # Format shortcut query results (decisions, architecture, etc.)
+      # @param category [String] Shortcut category name
+      # @param results [Array<Hash>] Query results
+      # @return [Hash] Formatted shortcut response
+      def self.format_shortcut_results(category, results)
+        {
+          category: category,
+          count: results.size,
+          facts: results.map { |r| format_shortcut_fact(r) }
+        }
+      end
+
+      # Format fact for shortcut queries (includes scope, no status)
+      # @param result [Hash] Result with fact data
+      # @return [Hash] Formatted fact
+      def self.format_shortcut_fact(result)
+        {
+          id: result[:fact][:id],
+          subject: result[:fact][:subject_name],
+          predicate: result[:fact][:predicate],
+          object: result[:fact][:object_literal],
+          scope: result[:fact][:scope],
+          source: result[:source]
+        }
+      end
+
+      # Format facts_by_tool query results
+      # @param tool_name [String] Tool name
+      # @param scope [String] Scope
+      # @param results [Array<Hash>] Query results
+      # @return [Hash] Formatted tool facts response
+      def self.format_tool_facts(tool_name, scope, results)
+        {
+          tool_name: tool_name,
+          scope: scope,
+          count: results.size,
+          facts: results.map { |r| format_generic_fact(r) }
+        }
+      end
+
+      # Format facts_by_context query results
+      # @param context_type [String] Type (git_branch, cwd)
+      # @param context_value [String] Value
+      # @param scope [String] Scope
+      # @param results [Array<Hash>] Query results
+      # @return [Hash] Formatted context facts response
+      def self.format_context_facts(context_type, context_value, scope, results)
+        {
+          context_type: context_type,
+          context_value: context_value,
+          scope: scope,
+          count: results.size,
+          facts: results.map { |r| format_generic_fact(r) }
+        }
+      end
+
+      # Format generic fact with scope and receipts
+      # @param result [Hash] Result with fact and receipts
+      # @return [Hash] Formatted fact
+      def self.format_generic_fact(result)
+        {
+          id: result[:fact][:id],
+          subject: result[:fact][:subject_name],
+          predicate: result[:fact][:predicate],
+          object: result[:fact][:object_literal],
+          scope: result[:fact][:scope],
+          source: result[:source],
+          receipts: result[:receipts].map { |r| format_receipt(r) }
+        }
+      end
+    end
+  end
+end

data/lib/claude_memory/mcp/server.rb

@@ -99,6 +99,11 @@ module ClaudeMemory
 
         result = @tools.call(name, arguments)
 
+        # Release database connections after each tool call
+        # This prevents lock contention with hook commands
+        # Connections are automatically reopened on next use
+        release_connections
+
         {
           jsonrpc: "2.0",
           id: id,
@@ -110,6 +115,20 @@ module ClaudeMemory
         }
       end
 
+      def release_connections
+        if @store_or_manager.is_a?(Store::StoreManager)
+          # Release both global and project store connections
+          @store_or_manager.global_store&.db&.disconnect
+          @store_or_manager.project_store&.db&.disconnect
+        elsif @store_or_manager.respond_to?(:db)
+          # Release single store connection (legacy)
+          @store_or_manager.db.disconnect
+        end
+      rescue
+        # Silently ignore disconnect errors
+        # Connection will be reopened automatically on next use
+      end
+
       def send_response(response)
         @output.puts(JSON.generate(response))
         @output.flush