claude-memory-layer 1.0.10 → 1.0.12

package/src/core/sqlite-wrapper.ts

@@ -4,6 +4,8 @@
  */
 
 import Database from 'better-sqlite3';
+import * as fs from 'fs';
+import * as nodePath from 'path';
 
 export type SQLiteDatabase = Database.Database;
 
@@ -16,6 +18,12 @@ export interface SQLiteOptions {
  * Creates a new SQLite database with WAL mode
  */
 export function createSQLiteDatabase(path: string, options?: SQLiteOptions): SQLiteDatabase {
+  // Ensure parent directory exists
+  const dir = nodePath.dirname(path);
+  if (!fs.existsSync(dir)) {
+    fs.mkdirSync(dir, { recursive: true });
+  }
+
   const db = new Database(path, {
     readonly: options?.readonly ?? false,
   });

package/src/core/tag-taxonomy.ts

@@ -0,0 +1,51 @@
+export const TAG_NAMESPACES = {
+  SYSTEM: 'sys:',
+  QUALITY: 'q:',
+  PROJECT: 'proj:',
+  TOPIC: 'topic:',
+  TEMPORAL: 't:',
+  USER: 'user:',
+  AGENT: 'agent:'
+} as const;
+
+export const VALID_TAG_NAMESPACES = new Set<string>(Object.values(TAG_NAMESPACES));
+
+export function parseTag(tag: string): { namespace?: string; value: string } {
+  const value = (tag || '').trim();
+  const idx = value.indexOf(':');
+  if (idx <= 0) return { value };
+
+  const namespace = `${value.slice(0, idx)}:`;
+  const tagValue = value.slice(idx + 1);
+  if (!tagValue) return { value };
+
+  return { namespace, value: tagValue };
+}
+
+export function validateTag(tag: string): boolean {
+  const normalized = (tag || '').trim();
+  if (!normalized) return false;
+
+  const { namespace } = parseTag(normalized);
+  if (!namespace) return true; // backward compatibility for legacy tags
+  return VALID_TAG_NAMESPACES.has(namespace);
+}
+
+export function withNamespace(value: string, namespace: string): string {
+  const clean = parseTag(value).value.trim();
+  return `${namespace}${clean}`;
+}
+
+export function normalizeTags(tags: unknown): string[] {
+  if (!Array.isArray(tags)) return [];
+
+  const dedup = new Set<string>();
+  for (const item of tags) {
+    if (typeof item !== 'string') continue;
+    const normalized = item.trim();
+    if (!validateTag(normalized)) continue;
+    dedup.add(normalized);
+  }
+
+  return [...dedup];
+}

package/src/core/turn-state.ts

@@ -0,0 +1,159 @@
+/**
+ * Turn State Management
+ *
+ * Manages a per-session turn_id state file that links events within a conversation turn.
+ *
+ * Flow:
+ * 1. UserPromptSubmit generates a new turn_id and writes it to a state file
+ * 2. PostToolUse reads the current turn_id to associate tool observations with the turn
+ * 3. Stop reads the turn_id to associate agent responses, then cleans up
+ *
+ * State file location: ~/.claude-code/memory/.turn-state-{session_id}.json
+ *
+ * The file is small (just a JSON with turnId + timestamp) and uses atomic writes
+ * to prevent corruption from concurrent hook execution.
+ */
+
+import * as fs from 'fs';
+import * as path from 'path';
+import * as os from 'os';
+
+const TURN_STATE_DIR = path.join(os.homedir(), '.claude-code', 'memory');
+
+interface TurnState {
+  turnId: string;
+  sessionId: string;
+  createdAt: string;
+}
+
+/**
+ * Get the state file path for a session
+ */
+function getStatePath(sessionId: string): string {
+  return path.join(TURN_STATE_DIR, `.turn-state-${sessionId}.json`);
+}
+
+/**
+ * Write a new turn state for a session.
+ * Called by UserPromptSubmit hook when a new user prompt arrives.
+ */
+export function writeTurnState(sessionId: string, turnId: string): void {
+  try {
+    // Ensure directory exists
+    if (!fs.existsSync(TURN_STATE_DIR)) {
+      fs.mkdirSync(TURN_STATE_DIR, { recursive: true });
+    }
+
+    const state: TurnState = {
+      turnId,
+      sessionId,
+      createdAt: new Date().toISOString()
+    };
+
+    const filePath = getStatePath(sessionId);
+    const tempPath = filePath + '.tmp';
+
+    // Atomic write: write to temp file then rename
+    fs.writeFileSync(tempPath, JSON.stringify(state));
+    fs.renameSync(tempPath, filePath);
+  } catch (error) {
+    // Non-critical: if we can't write turn state, events just won't be grouped
+    if (process.env.CLAUDE_MEMORY_DEBUG) {
+      console.error('Failed to write turn state:', error);
+    }
+  }
+}
+
+/**
+ * Read the current turn_id for a session.
+ * Called by PostToolUse and Stop hooks to associate events with the current turn.
+ * Returns null if no turn state exists (events won't be grouped).
+ */
+export function readTurnState(sessionId: string): string | null {
+  try {
+    const filePath = getStatePath(sessionId);
+
+    if (!fs.existsSync(filePath)) {
+      return null;
+    }
+
+    const data = fs.readFileSync(filePath, 'utf-8');
+    const state: TurnState = JSON.parse(data);
+
+    // Validate the state belongs to this session
+    if (state.sessionId !== sessionId) {
+      return null;
+    }
+
+    // Check staleness: if the turn state is older than 30 minutes, ignore it
+    const createdAt = new Date(state.createdAt).getTime();
+    const now = Date.now();
+    if (now - createdAt > 30 * 60 * 1000) {
+      // Stale turn state, clean up
+      clearTurnState(sessionId);
+      return null;
+    }
+
+    return state.turnId;
+  } catch (error) {
+    // Non-critical: return null if we can't read
+    if (process.env.CLAUDE_MEMORY_DEBUG) {
+      console.error('Failed to read turn state:', error);
+    }
+    return null;
+  }
+}
+
+/**
+ * Clear the turn state for a session.
+ * Called by Stop hook after processing agent responses.
+ */
+export function clearTurnState(sessionId: string): void {
+  try {
+    const filePath = getStatePath(sessionId);
+    if (fs.existsSync(filePath)) {
+      fs.unlinkSync(filePath);
+    }
+  } catch (error) {
+    // Non-critical
+    if (process.env.CLAUDE_MEMORY_DEBUG) {
+      console.error('Failed to clear turn state:', error);
+    }
+  }
+}
+
+/**
+ * Clean up stale turn state files (older than 1 hour).
+ * Can be called periodically to prevent file accumulation.
+ */
+export function cleanupStaleTurnStates(): number {
+  let cleaned = 0;
+
+  try {
+    if (!fs.existsSync(TURN_STATE_DIR)) return 0;
+
+    const files = fs.readdirSync(TURN_STATE_DIR);
+    const now = Date.now();
+
+    for (const file of files) {
+      if (!file.startsWith('.turn-state-') || !file.endsWith('.json')) continue;
+
+      const filePath = path.join(TURN_STATE_DIR, file);
+
+      try {
+        const stat = fs.statSync(filePath);
+        // Remove files older than 1 hour
+        if (now - stat.mtimeMs > 60 * 60 * 1000) {
+          fs.unlinkSync(filePath);
+          cleaned++;
+        }
+      } catch {
+        // Skip files we can't stat
+      }
+    }
+  } catch {
+    // Non-critical
+  }
+
+  return cleaned;
+}

package/src/core/types.ts

@@ -256,25 +256,40 @@ export interface UserPromptSubmitOutput {
   context?: string;
 }
 
+// Stop Hook Input (matches actual Claude Code hook format)
 export interface StopInput {
   session_id: string;
-  stop_reason: string;
-  messages: Array<{ role: string; content: string }>;
+  transcript_path: string;
+  cwd: string;
+  permission_mode: string;
+  hook_event_name: string;
+  stop_hook_active: boolean;
 }
 
 export interface SessionEndInput {
   session_id: string;
 }
 
-// PostToolUse Hook Input
+// PostToolUse Hook Input (matches actual Claude Code hook format)
 export interface PostToolUseInput {
+  session_id: string;
+  hook_event_name: string;
   tool_name: string;
   tool_input: Record<string, unknown>;
-  tool_output: string;
-  tool_error?: string;
-  session_id: string;
-  started_at: string;
-  ended_at: string;
+  tool_use_id: string;
+  // Claude Code sends tool_response as an object, not tool_output as string
+  tool_response: {
+    stdout?: string;
+    stderr?: string;
+    content?: string;
+    interrupted?: boolean;
+    isImage?: boolean;
+    // For non-Bash tools, response may be a plain string or other format
+    [key: string]: unknown;
+  };
+  cwd: string;
+  transcript_path: string;
+  permission_mode: string;
 }
 
 // ============================================================
@@ -803,6 +818,34 @@ export interface ConsolidatedMemoryInput {
   confidence: number;
 }
 
+// Long-term Rule (promoted from stable summaries)
+export const ConsolidationRuleSchema = z.object({
+  ruleId: z.string(),
+  rule: z.string(),
+  topics: z.array(z.string()),
+  sourceMemoryIds: z.array(z.string()),
+  sourceEvents: z.array(z.string()),
+  confidence: z.number(),
+  createdAt: z.date()
+});
+export type ConsolidationRule = z.infer<typeof ConsolidationRuleSchema>;
+
+export interface ConsolidationRuleInput {
+  rule: string;
+  topics: string[];
+  sourceMemoryIds: string[];
+  sourceEvents: string[];
+  confidence: number;
+}
+
+export interface ConsolidationCostQualityReport {
+  beforeTokenEstimate: number;
+  afterTokenEstimate: number;
+  reductionRatio: number;
+  qualityGuardPassed: boolean;
+  details: string;
+}
+
 // Event Group (for consolidation)
 export interface EventGroup {
   topics: string[];

package/src/core/vector-store.ts

@@ -65,8 +65,17 @@ export class VectorStore {
     };
 
     if (!this.table) {
-      // Create table with first record
-      this.table = await this.db.createTable(this.tableName, [data]);
+      // Create table with first record (handle race condition)
+      try {
+        this.table = await this.db.createTable(this.tableName, [data]);
+      } catch (e: any) {
+        if (e?.message?.includes('already exists')) {
+          this.table = await this.db.openTable(this.tableName);
+          await this.table.add([data]);
+        } else {
+          throw e;
+        }
+      }
     } else {
       await this.table.add([data]);
     }
@@ -96,7 +105,16 @@ export class VectorStore {
     }));
 
     if (!this.table) {
-      this.table = await this.db.createTable(this.tableName, data);
+      try {
+        this.table = await this.db.createTable(this.tableName, data);
+      } catch (e: any) {
+        if (e?.message?.includes('already exists')) {
+          this.table = await this.db.openTable(this.tableName);
+          await this.table.add(data);
+        } else {
+          throw e;
+        }
+      }
     } else {
       await this.table.add(data);
     }

package/src/hooks/post-tool-use.ts

@@ -2,14 +2,22 @@
 /**
  * PostToolUse Hook
  * Called after each tool execution - stores tool observations
+ *
+ * Actual Claude Code input format:
+ * {
+ *   session_id, tool_name, tool_input, tool_use_id,
+ *   tool_response: { stdout?, stderr?, content?, interrupted?, isImage? },
+ *   cwd, transcript_path, permission_mode, hook_event_name
+ * }
  */
 
-import { getDefaultMemoryService } from '../services/memory-service.js';
+import { getLightweightMemoryService } from '../services/memory-service.js';
 import { applyPrivacyFilter, maskSensitiveInput, truncateOutput } from '../core/privacy/index.js';
-import { extractMetadata, createToolObservationEmbedding } from '../core/metadata-extractor.js';
+import { extractMetadata } from '../core/metadata-extractor.js';
+import { readTurnState } from '../core/turn-state.js';
 import type { PostToolUseInput, ToolObservationPayload, Config } from '../core/types.js';
 
-// Default config (will be overridden by actual config when available)
+// Default config
 const DEFAULT_CONFIG: Config['toolObservation'] = {
   enabled: true,
   excludedTools: ['TodoWrite', 'TodoRead'],
@@ -30,12 +38,38 @@ const DEFAULT_PRIVACY_CONFIG: Config['privacy'] = {
 };
 
 /**
- * Calculate duration from ISO timestamps
+ * Extract text output from tool_response object
  */
-function calculateDuration(startedAt: string, endedAt: string): number {
-  const start = new Date(startedAt).getTime();
-  const end = new Date(endedAt).getTime();
-  return end - start;
+function extractToolOutput(response: PostToolUseInput['tool_response']): string {
+  if (!response) return '';
+
+  // Bash tools: stdout + stderr
+  if (response.stdout !== undefined) {
+    const parts: string[] = [];
+    if (response.stdout) parts.push(response.stdout);
+    if (response.stderr) parts.push(`[stderr] ${response.stderr}`);
+    return parts.join('\n') || '';
+  }
+
+  // Other tools may have content field
+  if (response.content !== undefined) {
+    return typeof response.content === 'string'
+      ? response.content
+      : JSON.stringify(response.content);
+  }
+
+  // Fallback: stringify the whole response
+  return JSON.stringify(response);
+}
+
+/**
+ * Determine if the tool execution was successful
+ */
+function isToolSuccess(response: PostToolUseInput['tool_response']): boolean {
+  if (!response) return false;
+  if (response.interrupted) return false;
+  // If stderr has content but stdout also has content, still consider success
+  return true;
 }
 
 async function main(): Promise<void> {
@@ -58,55 +92,66 @@ async function main(): Promise<void> {
     return;
   }
 
-  // 3. Check success filter
-  const success = !input.tool_error;
+  // 3. Extract output from tool_response object
+  const toolOutput = extractToolOutput(input.tool_response);
+  const success = isToolSuccess(input.tool_response);
+
+  // 4. Check success filter
   if (!success && config.storeOnlyOnSuccess) {
     console.log(JSON.stringify({}));
     return;
   }
 
   try {
-    const memoryService = getDefaultMemoryService();
+    const memoryService = getLightweightMemoryService(input.session_id);
 
-    // 4. Mask sensitive data in input
+    // 5. Mask sensitive data in input
     const maskedInput = maskSensitiveInput(input.tool_input);
 
-    // 5. Apply privacy filter to output
-    const filterResult = applyPrivacyFilter(input.tool_output, privacyConfig);
+    // 6. Apply privacy filter to output
+    const filterResult = applyPrivacyFilter(toolOutput, privacyConfig);
     const maskedOutput = filterResult.content;
 
-    // 6. Truncate output
+    // 7. Truncate output
     const truncatedOutput = truncateOutput(maskedOutput, {
       maxLength: config.maxOutputLength,
       maxLines: config.maxOutputLines
     });
 
-    // 7. Extract metadata
+    // 8. Extract metadata
     const metadata = extractMetadata(
       input.tool_name,
       maskedInput,
-      input.tool_output,
+      toolOutput,
       success
     );
 
-    // 8. Create payload
+    // 8.5. Read current turn_id from state file
+    const turnId = readTurnState(input.session_id);
+
+    // 9. Create payload (include turnId in metadata for grouping)
     const payload: ToolObservationPayload = {
       toolName: input.tool_name,
       toolInput: maskedInput,
       toolOutput: truncatedOutput,
-      durationMs: calculateDuration(input.started_at, input.ended_at),
+      durationMs: 0, // Claude Code doesn't provide timing info
       success,
-      errorMessage: input.tool_error,
-      metadata
+      errorMessage: input.tool_response?.stderr || undefined,
+      metadata: {
+        ...metadata,
+        ...(turnId ? { turnId } : {})
+      }
     };
 
-    // 9. Store observation
+    // 10. Store observation
     await memoryService.storeToolObservation(input.session_id, payload);
 
     // Output empty (hook doesn't return context)
     console.log(JSON.stringify({}));
   } catch (error) {
-    console.error('PostToolUse hook error:', error);
+    if (process.env.CLAUDE_MEMORY_DEBUG) {
+      console.error('PostToolUse hook error:', error);
+    }
     console.log(JSON.stringify({}));
   }
 }

package/src/hooks/session-end.ts

@@ -4,7 +4,7 @@
  * Called when session ends - generates and stores session summary
  */
 
-import { getMemoryServiceForSession } from '../services/memory-service.js';
+import { getLightweightMemoryService } from '../services/memory-service.js';
 import type { SessionEndInput } from '../core/types.js';
 
 async function main(): Promise<void> {
@@ -12,8 +12,8 @@ async function main(): Promise<void> {
   const inputData = await readStdin();
   const input: SessionEndInput = JSON.parse(inputData);
 
-  // Get project-specific memory service via session lookup
-  const memoryService = getMemoryServiceForSession(input.session_id);
+  // Use lightweight service (SQLite only, no embedder/vector - FAST!)
+  const memoryService = getLightweightMemoryService(input.session_id);
 
   try {
     // Get session history
@@ -29,6 +29,11 @@ async function main(): Promise<void> {
       // End session with summary
       await memoryService.endSession(input.session_id, summary);
 
+      // Evaluate helpfulness of memory retrievals in this session
+      try {
+        await memoryService.evaluateSessionHelpfulness(input.session_id);
+      } catch { /* non-critical */ }
+
       // Process any pending embeddings
       await memoryService.processPendingEmbeddings();
     }

package/src/hooks/stop.ts

@@ -1,11 +1,23 @@
 #!/usr/bin/env node
 /**
  * Stop Hook
- * Called when agent stops - stores the conversation messages
+ * Called when agent stops - reads transcript and stores assistant responses
+ *
+ * Actual Claude Code input format:
+ * {
+ *   session_id, transcript_path, cwd, permission_mode,
+ *   hook_event_name: "Stop", stop_hook_active
+ * }
+ *
+ * NOTE: Claude Code does NOT send messages in the Stop hook.
+ * We read them from the transcript JSONL file instead.
  */
 
-import { getMemoryServiceForSession } from '../services/memory-service.js';
+import * as fs from 'fs';
+import * as readline from 'readline';
+import { getLightweightMemoryService } from '../services/memory-service.js';
 import { applyPrivacyFilter } from '../core/privacy/index.js';
+import { readTurnState, clearTurnState } from '../core/turn-state.js';
 import type { StopInput, Config } from '../core/types.js';
 
 // Default privacy config
@@ -20,45 +32,104 @@ const DEFAULT_PRIVACY_CONFIG: Config['privacy'] = {
   }
 };
 
+/**
+ * Extract assistant text messages from transcript JSONL.
+ * Only reads the last N lines to avoid processing entire transcript.
+ */
+async function extractAssistantMessages(transcriptPath: string): Promise<string[]> {
+  if (!fs.existsSync(transcriptPath)) return [];
+
+  const messages: string[] = [];
+
+  // Read last portion of file (last ~200KB should cover recent messages)
+  const stats = fs.statSync(transcriptPath);
+  const readStart = Math.max(0, stats.size - 200 * 1024);
+
+  const stream = fs.createReadStream(transcriptPath, {
+    start: readStart,
+    encoding: 'utf8'
+  });
+
+  const rl = readline.createInterface({ input: stream, crlfDelay: Infinity });
+
+  for await (const line of rl) {
+    try {
+      const entry = JSON.parse(line);
+
+      // Only process assistant messages with text content
+      if (entry.type !== 'assistant') continue;
+
+      const content = entry.message?.content;
+      if (!Array.isArray(content)) continue;
+
+      // Extract text blocks from content array
+      const textParts = content
+        .filter((c: { type: string }) => c.type === 'text')
+        .map((c: { text: string }) => c.text)
+        .filter(Boolean);
+
+      if (textParts.length > 0) {
+        messages.push(textParts.join('\n'));
+      }
+    } catch {
+      // Skip malformed lines (e.g., partial first line from readStart offset)
+    }
+  }
+
+  return messages;
+}
+
 async function main(): Promise<void> {
   // Read input from stdin
   const inputData = await readStdin();
   const input: StopInput = JSON.parse(inputData);
 
-  // Get project-specific memory service via session lookup
-  const memoryService = getMemoryServiceForSession(input.session_id);
+  // Use lightweight service (SQLite only, no embedder/vector - FAST!)
+  const memoryService = getLightweightMemoryService(input.session_id);
 
   try {
-    // Store agent responses from the conversation
-    for (const message of input.messages) {
-      if (message.role === 'assistant' && message.content) {
-        // Apply privacy filter
-        const filterResult = applyPrivacyFilter(message.content, DEFAULT_PRIVACY_CONFIG);
-        let content = filterResult.content;
-
-        // Truncate very long responses
-        if (content.length > 5000) {
-          content = content.slice(0, 5000) + '...[truncated]';
-        }
+    // Read current turn_id from state file
+    const turnId = readTurnState(input.session_id);
+
+    // Read assistant messages from transcript
+    const assistantMessages = await extractAssistantMessages(input.transcript_path);
 
-        await memoryService.storeAgentResponse(
-          input.session_id,
-          content,
-          {
-            stopReason: input.stop_reason,
-            privacy: filterResult.metadata
-          }
-        );
+    // Store each assistant response
+    for (const text of assistantMessages) {
+      // Apply privacy filter
+      const filterResult = applyPrivacyFilter(text, DEFAULT_PRIVACY_CONFIG);
+      let content = filterResult.content;
+
+      // Truncate very long responses
+      if (content.length > 5000) {
+        content = content.slice(0, 5000) + '...[truncated]';
       }
+
+      // Skip very short responses (likely just tool calls)
+      if (content.trim().length < 10) continue;
+
+      await memoryService.storeAgentResponse(
+        input.session_id,
+        content,
+        {
+          privacy: filterResult.metadata,
+          ...(turnId ? { turnId } : {})
+        }
+      );
     }
 
-    // Process embeddings immediately
+    // Clean up turn state file after processing
+    clearTurnState(input.session_id);
+
+    // Embeddings enqueued in SQLite - will be processed by vector worker when server runs
     await memoryService.processPendingEmbeddings();
 
     // Output empty (stop hook doesn't return context)
     console.log(JSON.stringify({}));
   } catch (error) {
-    console.error('Memory hook error:', error);
+    if (process.env.CLAUDE_MEMORY_DEBUG) {
+      console.error('Stop hook error:', error);
+    }
     console.log(JSON.stringify({}));
   }
 }