claude-memory-layer 1.0.9 → 1.0.11

package/src/hooks/user-prompt-submit.ts

@@ -1,93 +1,106 @@
 #!/usr/bin/env node
 /**
  * User Prompt Submit Hook
- * Called when user submits a prompt - retrieves relevant memories
+ * Called when user submits a prompt - retrieves relevant memories using fast keyword search
+ *
+ * Uses SQLite FTS5 for fast keyword-based search (no ML model needed)
+ * Much faster than vector search (~100ms vs 3-5s)
+ *
+ * Turn Grouping: Generates a turn_id and persists it to a state file
+ * so PostToolUse and Stop hooks can associate their events with this turn.
  */
 
-import { getMemoryServiceForSession } from '../services/memory-service.js';
+import { randomUUID } from 'crypto';
+import { getLightweightMemoryService } from '../services/memory-service.js';
+import { writeTurnState } from '../core/turn-state.js';
 import type { UserPromptSubmitInput, UserPromptSubmitOutput } from '../core/types.js';
 
+// Configuration
+const MAX_MEMORIES = parseInt(process.env.CLAUDE_MEMORY_MAX_COUNT || '5');
+const MIN_SCORE = parseFloat(process.env.CLAUDE_MEMORY_MIN_SCORE || '0.3');
+const ENABLE_SEARCH = process.env.CLAUDE_MEMORY_SEARCH !== 'false';
+
+/**
+ * Determine if a prompt is worth storing as a memory.
+ * Filters slash commands, very short inputs, and trivial patterns.
+ */
+function shouldStorePrompt(prompt: string): boolean {
+  const trimmed = prompt.trim();
+  if (trimmed.startsWith('/')) return false;
+  if (trimmed.length < 15) return false;
+  if (!/[a-zA-Z가-힣]{2,}/.test(trimmed)) return false;
+  return true;
+}
+
 async function main(): Promise<void> {
   // Read input from stdin
   const inputData = await readStdin();
   const input: UserPromptSubmitInput = JSON.parse(inputData);
 
-  // Get project-specific memory service via session lookup
-  const memoryService = getMemoryServiceForSession(input.session_id);
+  // Generate a new turn_id for this user prompt
+  // This groups the prompt with subsequent tool calls and the final agent response
+  const turnId = randomUUID();
+
+  // Persist turn state so PostToolUse and Stop hooks can read it
+  writeTurnState(input.session_id, turnId);
 
-  // Configuration from environment variables
-  const config = {
-    candidateCount: parseInt(process.env.CLAUDE_MEMORY_CANDIDATES || '10'),
-    minScore: parseFloat(process.env.CLAUDE_MEMORY_MIN_SCORE || '0.7'),
-    maxMemories: parseInt(process.env.CLAUDE_MEMORY_MAX_COUNT || '5'),
-    dynamicThresholdRatio: parseFloat(process.env.CLAUDE_MEMORY_THRESHOLD_RATIO || '0.85'),
-    strictMinScore: parseFloat(process.env.CLAUDE_MEMORY_STRICT_MIN || '0.75')
-  };
+  // Use lightweight service (SQLite only, no embedder/vector - FAST!)
+  const memoryService = getLightweightMemoryService(input.session_id);
 
   try {
-    // Check if shared store is enabled
-    const includeShared = memoryService.isSharedStoreEnabled();
-
-    // Retrieve relevant memories for the prompt (including shared if enabled)
-    // First, get more candidates to filter from
-    const retrievalResult = await memoryService.retrieveMemories(input.prompt, {
-      topK: config.candidateCount,  // Get more candidates initially
-      minScore: config.minScore,
-      includeShared
-    });
+    // Store only non-trivial prompts (skip /commands, short inputs)
+    if (shouldStorePrompt(input.prompt)) {
+      await memoryService.storeUserPrompt(
+        input.session_id,
+        input.prompt,
+        { turnId }
+      );
+    }
 
-    // Store the user prompt for future retrieval
-    await memoryService.storeUserPrompt(
-      input.session_id,
-      input.prompt
-    );
-
-    // Filter memories based on relevance and confidence
-    let relevantMemories = [];
-    if (retrievalResult.memories && retrievalResult.memories.length > 0) {
-      // Dynamic threshold based on the best score
-      const bestScore = Math.max(...retrievalResult.memories.map(m => m.score));
-      const dynamicThreshold = Math.max(config.strictMinScore, bestScore * config.dynamicThresholdRatio);
-
-      // Filter memories that meet the dynamic threshold
-      relevantMemories = retrievalResult.memories.filter(m => m.score >= dynamicThreshold);
-
-      // Limit to configured max memories
-      relevantMemories = relevantMemories.slice(0, config.maxMemories);
-
-      // Check if we have enough highly relevant memories
-      if (relevantMemories.length === 0) {
-        // If no memories meet the high threshold, take top 3 with relaxed threshold
-        const fallbackCount = Math.min(3, config.maxMemories);
-        relevantMemories = retrievalResult.memories
-          .filter(m => m.score >= config.minScore)
-          .slice(0, fallbackCount);
-      }
+    let context = '';
 
-      // Log filtering statistics for debugging
-      if (process.env.CLAUDE_MEMORY_DEBUG) {
-        console.error(`Memory filtering: ${retrievalResult.memories.length} candidates -> ${relevantMemories.length} selected`);
-        console.error(`Threshold: ${dynamicThreshold.toFixed(3)}, Best score: ${bestScore.toFixed(3)}`);
-      }
+    // Fast keyword search if enabled
+    if (ENABLE_SEARCH && input.prompt.length > 10) {
+      const results = await memoryService.keywordSearch(input.prompt, {
+        topK: MAX_MEMORIES,
+        minScore: MIN_SCORE
+      });
 
-      // Increment access count only for memories that will actually be included
-      if (relevantMemories.length > 0) {
-        const eventIds = relevantMemories.map(m => m.event.id);
+      if (results.length > 0) {
+        // Increment access count for found memories
+        const eventIds = results.map(r => r.event.id);
         await memoryService.incrementMemoryAccess(eventIds);
+
+        // Record each retrieval for helpfulness tracking
+        for (const r of results) {
+          try {
+            await memoryService.recordRetrieval(
+              r.event.id,
+              input.session_id,
+              r.score,
+              input.prompt
+            );
+          } catch { /* non-critical */ }
+        }
+
+        // Format context
+        const memories = results.map(r => {
+          const preview = r.event.content.length > 300
+            ? r.event.content.substring(0, 300) + '...'
+            : r.event.content;
+          return `- [${r.event.eventType}] ${preview}`;
+        });
+
+        context = `💡 **Related memories found:**\n\n${memories.join('\n\n')}`;
       }
     }
 
-    // Format context for Claude with only the relevant memories
-    const filteredResult = {
-      ...retrievalResult,
-      memories: relevantMemories
-    };
-    const context = memoryService.formatAsContext(filteredResult);
-
     const output: UserPromptSubmitOutput = { context };
     console.log(JSON.stringify(output));
   } catch (error) {
-    console.error('Memory hook error:', error);
+    if (process.env.CLAUDE_MEMORY_DEBUG) {
+      console.error('Memory hook error:', error);
+    }
     console.log(JSON.stringify({ context: '' }));
   }
 }

package/src/server/api/chat.ts

@@ -0,0 +1,244 @@
+/**
+ * Chat API
+ * Endpoints for memory-aware chat using Claude CLI
+ */
+
+import { Hono } from 'hono';
+import { streamSSE } from 'hono/streaming';
+import { spawn } from 'child_process';
+import type { ChildProcess } from 'child_process';
+import { getServiceFromQuery } from './utils.js';
+
+export const chatRouter = new Hono();
+
+interface ChatRequest {
+  message: string;
+  history?: Array<{ role: 'user' | 'assistant'; content: string }>;
+}
+
+const CLAUDE_TIMEOUT_MS = 120_000;
+
+chatRouter.post('/', async (c) => {
+  let body: ChatRequest;
+  try {
+    body = await c.req.json<ChatRequest>();
+  } catch {
+    return c.json({ error: 'Invalid JSON body' }, 400);
+  }
+
+  if (!body.message?.trim()) {
+    return c.json({ error: 'Message is required' }, 400);
+  }
+
+  const memoryService = getServiceFromQuery(c);
+
+  try {
+    await memoryService.initialize();
+
+    // Retrieve relevant memories for context
+    let memoryContext = '';
+    let statsContext = '';
+
+    try {
+      const result = await memoryService.retrieveMemories(body.message, {
+        topK: 8,
+        minScore: 0.5
+      });
+
+      if (result.memories.length > 0) {
+        const parts: string[] = ['## Relevant Memories\n'];
+        for (const m of result.memories) {
+          const date = new Date(m.event.timestamp).toISOString().split('T')[0];
+          const content = m.event.content.slice(0, 500);
+          parts.push(`### [${m.event.eventType}] ${date} (score: ${m.score.toFixed(2)})`);
+          parts.push(content);
+          if (m.sessionContext) {
+            parts.push(`_Context: ${m.sessionContext}_`);
+          }
+          parts.push('');
+        }
+        memoryContext = parts.join('\n');
+      }
+    } catch {
+      // Continue without memory context if retrieval fails
+    }
+
+    try {
+      const stats = await memoryService.getStats();
+      const levels = stats.levelStats.map(l => `${l.level}: ${l.count}`).join(', ');
+      statsContext = [
+        '## Memory Stats',
+        `- Total events: ${stats.totalEvents}`,
+        `- Vector nodes: ${stats.vectorCount}`,
+        `- By level: ${levels}`
+      ].join('\n');
+    } catch {
+      // Continue without stats if it fails
+    }
+
+    const fullPrompt = buildPrompt(
+      statsContext,
+      memoryContext,
+      body.history || [],
+      body.message
+    );
+
+    // Stream response via SSE
+    return streamSSE(c, async (stream) => {
+      try {
+        await streamClaudeResponse(fullPrompt, stream);
+      } catch (err) {
+        await stream.writeSSE({
+          event: 'error',
+          data: JSON.stringify({ error: (err as Error).message })
+        });
+      }
+    });
+  } catch (error) {
+    return c.json({ error: (error as Error).message }, 500);
+  } finally {
+    await memoryService.shutdown();
+  }
+});
+
+function buildPrompt(
+  statsContext: string,
+  memoryContext: string,
+  history: Array<{ role: string; content: string }>,
+  currentMessage: string
+): string {
+  const parts: string[] = [];
+
+  parts.push('You are a helpful assistant that answers questions about the user\'s code memory data.');
+  parts.push('The memory system tracks coding sessions, tool usage, prompts, and responses.');
+  parts.push('Answer concisely based on the memory context below. If you don\'t have enough data, say so.');
+  parts.push('Use markdown formatting in your responses.\n');
+
+  if (statsContext) {
+    parts.push(statsContext);
+    parts.push('');
+  }
+
+  if (memoryContext) {
+    parts.push(memoryContext);
+  } else {
+    parts.push('No directly relevant memories found for this query.');
+    parts.push('Answer based on general knowledge or suggest the user rephrase.\n');
+  }
+
+  parts.push('---\n');
+
+  // Include recent history (last 10 turns)
+  const recentHistory = history.slice(-10);
+  if (recentHistory.length > 0) {
+    parts.push('## Conversation History\n');
+    for (const msg of recentHistory) {
+      const prefix = msg.role === 'user' ? 'User' : 'Assistant';
+      parts.push(`**${prefix}:** ${msg.content}\n`);
+    }
+  }
+
+  parts.push(`**User:** ${currentMessage}`);
+
+  return parts.join('\n');
+}
+
+function streamClaudeResponse(
+  prompt: string,
+  stream: { writeSSE: (msg: { event?: string; data: string }) => Promise<void> }
+): Promise<void> {
+  return new Promise((resolve, reject) => {
+    const proc: ChildProcess = spawn('claude', [
+      '-p',
+      '--output-format', 'stream-json',
+      '--verbose'
+    ], {
+      stdio: ['pipe', 'pipe', 'pipe'],
+      env: { ...process.env }
+    });
+
+    const timeout = setTimeout(() => {
+      proc.kill('SIGTERM');
+      reject(new Error('Chat response timed out after 2 minutes'));
+    }, CLAUDE_TIMEOUT_MS);
+
+    // Write prompt to stdin
+    proc.stdin!.write(prompt);
+    proc.stdin!.end();
+
+    let buffer = '';
+    let lastSentText = '';
+
+    proc.stdout!.on('data', async (chunk: Buffer) => {
+      buffer += chunk.toString();
+      const lines = buffer.split('\n');
+      buffer = lines.pop() || '';
+
+      for (const line of lines) {
+        if (!line.trim()) continue;
+        try {
+          const parsed = JSON.parse(line);
+
+          // Extract text from assistant messages
+          if (parsed.type === 'assistant' && parsed.message?.content) {
+            const textBlocks = parsed.message.content
+              .filter((b: { type: string }) => b.type === 'text')
+              .map((b: { text: string }) => b.text)
+              .join('');
+
+            if (textBlocks.length > lastSentText.length) {
+              const delta = textBlocks.slice(lastSentText.length);
+              lastSentText = textBlocks;
+              await stream.writeSSE({
+                event: 'message',
+                data: JSON.stringify({ content: delta })
+              });
+            }
+          }
+
+          // Handle completion
+          if (parsed.type === 'result') {
+            await stream.writeSSE({ event: 'done', data: '{}' });
+          }
+        } catch {
+          // Skip non-JSON lines
+        }
+      }
+    });
+
+    proc.stderr!.on('data', (chunk: Buffer) => {
+      if (process.env.CLAUDE_MEMORY_DEBUG) {
+        console.error('[chat] claude stderr:', chunk.toString());
+      }
+    });
+
+    proc.on('error', (err) => {
+      clearTimeout(timeout);
+      if ((err as NodeJS.ErrnoException).code === 'ENOENT') {
+        reject(new Error('Claude CLI not found. Install with: npm install -g @anthropic-ai/claude-code'));
+      } else {
+        reject(err);
+      }
+    });
+
+    proc.on('close', async (code) => {
+      clearTimeout(timeout);
+
+      // Flush remaining buffer
+      if (buffer.trim()) {
+        try {
+          const parsed = JSON.parse(buffer);
+          if (parsed.type === 'result') {
+            await stream.writeSSE({ event: 'done', data: '{}' });
+          }
+        } catch { /* ignore */ }
+      }
+
+      if (code !== 0 && code !== null) {
+        reject(new Error(`Claude CLI exited with code ${code}`));
+      } else {
+        resolve();
+      }
+    });
+  });
+}

package/src/server/api/citations.ts

@@ -4,7 +4,7 @@
  */
 
 import { Hono } from 'hono';
-import { getReadOnlyMemoryService } from '../../services/memory-service.js';
+import { getServiceFromQuery } from './utils.js';
 import { generateCitationId, parseCitationId } from '../../core/citation-generator.js';
 
 export const citationsRouter = new Hono();
@@ -15,7 +15,7 @@ citationsRouter.get('/:id', async (c) => {
 
   // Support both formats: "a7Bc3x" or "mem:a7Bc3x"
   const citationId = parseCitationId(id) || id;
-  const memoryService = getReadOnlyMemoryService();
+  const memoryService = getServiceFromQuery(c);
 
   try {
     await memoryService.initialize();
@@ -57,7 +57,7 @@ citationsRouter.get('/:id', async (c) => {
 citationsRouter.get('/:id/related', async (c) => {
   const { id } = c.req.param();
   const citationId = parseCitationId(id) || id;
-  const memoryService = getReadOnlyMemoryService();
+  const memoryService = getServiceFromQuery(c);
 
   try {
     await memoryService.initialize();

package/src/server/api/events.ts

@@ -4,7 +4,7 @@
  */
 
 import { Hono } from 'hono';
-import { getReadOnlyMemoryService } from '../../services/memory-service.js';
+import { getServiceFromQuery } from './utils.js';
 
 export const eventsRouter = new Hono();
 
@@ -12,14 +12,23 @@ export const eventsRouter = new Hono();
 eventsRouter.get('/', async (c) => {
   const sessionId = c.req.query('sessionId');
   const eventType = c.req.query('type');
+  const level = c.req.query('level');
+  const sort = c.req.query('sort') || 'recent'; // recent | accessed | oldest
   const limit = parseInt(c.req.query('limit') || '100', 10);
   const offset = parseInt(c.req.query('offset') || '0', 10);
-  const memoryService = getReadOnlyMemoryService();
+  const memoryService = getServiceFromQuery(c);
 
   try {
     await memoryService.initialize();
 
-    let events = await memoryService.getRecentEvents(limit + offset + 1000);
+    let events: any[];
+
+    // Filter by level using the dedicated method
+    if (level) {
+      events = await memoryService.getEventsByLevel(level, { limit: limit + offset + 1000, offset: 0 });
+    } else {
+      events = await memoryService.getRecentEvents(limit + offset + 1000);
+    }
 
     // Filter by session
     if (sessionId) {
@@ -31,6 +40,20 @@ eventsRouter.get('/', async (c) => {
       events = events.filter(e => e.eventType === eventType);
     }
 
+    // Sort
+    if (sort === 'accessed') {
+      events.sort((a: any, b: any) => {
+        const aTime = a.last_accessed_at || '';
+        const bTime = b.last_accessed_at || '';
+        return bTime.localeCompare(aTime);
+      });
+    } else if (sort === 'most-accessed') {
+      events.sort((a: any, b: any) => (b.access_count || 0) - (a.access_count || 0));
+    } else if (sort === 'oldest') {
+      events.sort((a, b) => new Date(a.timestamp).getTime() - new Date(b.timestamp).getTime());
+    }
+    // 'recent' is default (already sorted by timestamp DESC from the query)
+
     // Pagination
     const total = events.length;
     events = events.slice(offset, offset + limit);
@@ -42,7 +65,9 @@ eventsRouter.get('/', async (c) => {
         timestamp: e.timestamp,
         sessionId: e.sessionId,
         preview: e.content.slice(0, 200) + (e.content.length > 200 ? '...' : ''),
-        contentLength: e.content.length
+        contentLength: e.content.length,
+        accessCount: (e as any).access_count || 0,
+        lastAccessedAt: (e as any).last_accessed_at || null
       })),
       total,
       limit,
@@ -59,7 +84,7 @@ eventsRouter.get('/', async (c) => {
 // GET /api/events/:id - Get event details
 eventsRouter.get('/:id', async (c) => {
   const { id } = c.req.param();
-  const memoryService = getReadOnlyMemoryService();
+  const memoryService = getServiceFromQuery(c);
 
   try {
     await memoryService.initialize();

package/src/server/api/index.ts

@@ -9,10 +9,16 @@ import { eventsRouter } from './events.js';
 import { searchRouter } from './search.js';
 import { statsRouter } from './stats.js';
 import { citationsRouter } from './citations.js';
+import { turnsRouter } from './turns.js';
+import { projectsRouter } from './projects.js';
+import { chatRouter } from './chat.js';
 
 export const apiRouter = new Hono()
   .route('/sessions', sessionsRouter)
   .route('/events', eventsRouter)
   .route('/search', searchRouter)
   .route('/stats', statsRouter)
-  .route('/citations', citationsRouter);
+  .route('/citations', citationsRouter)
+  .route('/turns', turnsRouter)
+  .route('/projects', projectsRouter)
+  .route('/chat', chatRouter);

package/src/server/api/projects.ts

@@ -0,0 +1,74 @@
+/**
+ * Projects API
+ * Endpoints for listing available projects
+ */
+
+import { Hono } from 'hono';
+import * as fs from 'fs';
+import * as path from 'path';
+import * as os from 'os';
+import { loadSessionRegistry } from '../../services/memory-service.js';
+
+export const projectsRouter = new Hono();
+
+// GET /api/projects - List available projects
+projectsRouter.get('/', async (c) => {
+  try {
+    const projectsDir = path.join(os.homedir(), '.claude-code', 'memory', 'projects');
+
+    if (!fs.existsSync(projectsDir)) {
+      return c.json({ projects: [] });
+    }
+
+    // Read project directories
+    const projectHashes = fs.readdirSync(projectsDir)
+      .filter(name => {
+        const fullPath = path.join(projectsDir, name);
+        return fs.statSync(fullPath).isDirectory();
+      });
+
+    // Load session registry to map hashes to project paths
+    const registry = loadSessionRegistry();
+    const hashToPath = new Map<string, string>();
+    for (const entry of Object.values(registry.sessions)) {
+      if (!hashToPath.has(entry.projectHash)) {
+        hashToPath.set(entry.projectHash, entry.projectPath);
+      }
+    }
+
+    // Build project list
+    const projects = projectHashes.map(hash => {
+      const dirPath = path.join(projectsDir, hash);
+      const dbPath = path.join(dirPath, 'events.sqlite');
+      let dbSize = 0;
+      if (fs.existsSync(dbPath)) {
+        dbSize = fs.statSync(dbPath).size;
+      }
+
+      const projectPath = hashToPath.get(hash) || `unknown (${hash})`;
+
+      return {
+        hash,
+        projectPath,
+        projectName: path.basename(projectPath),
+        dbSize,
+        dbSizeHuman: formatBytes(dbSize)
+      };
+    });
+
+    // Sort by project name
+    projects.sort((a, b) => a.projectName.localeCompare(b.projectName));
+
+    return c.json({ projects });
+  } catch (error) {
+    return c.json({ projects: [], error: (error as Error).message }, 500);
+  }
+});
+
+function formatBytes(bytes: number): string {
+  if (bytes === 0) return '0 B';
+  const k = 1024;
+  const sizes = ['B', 'KB', 'MB', 'GB'];
+  const i = Math.floor(Math.log(bytes) / Math.log(k));
+  return parseFloat((bytes / Math.pow(k, i)).toFixed(1)) + ' ' + sizes[i];
+}

package/src/server/api/search.ts

@@ -4,7 +4,7 @@
  */
 
 import { Hono } from 'hono';
-import { getReadOnlyMemoryService } from '../../services/memory-service.js';
+import { getServiceFromQuery } from './utils.js';
 
 export const searchRouter = new Hono();
 
@@ -20,7 +20,7 @@ interface SearchRequest {
 
 // POST /api/search - Search memories
 searchRouter.post('/', async (c) => {
-  const memoryService = getReadOnlyMemoryService();
+  const memoryService = getServiceFromQuery(c);
   try {
     const body = await c.req.json<SearchRequest>();
 
@@ -74,7 +74,7 @@ searchRouter.get('/', async (c) => {
   }
 
   const topK = parseInt(c.req.query('topK') || '5', 10);
-  const memoryService = getReadOnlyMemoryService();
+  const memoryService = getServiceFromQuery(c);
 
   try {
     await memoryService.initialize();

package/src/server/api/sessions.ts

@@ -4,7 +4,7 @@
  */
 
 import { Hono } from 'hono';
-import { getReadOnlyMemoryService } from '../../services/memory-service.js';
+import { getServiceFromQuery } from './utils.js';
 
 export const sessionsRouter = new Hono();
 
@@ -12,7 +12,7 @@ export const sessionsRouter = new Hono();
 sessionsRouter.get('/', async (c) => {
   const page = parseInt(c.req.query('page') || '1', 10);
   const pageSize = parseInt(c.req.query('pageSize') || '20', 10);
-  const memoryService = getReadOnlyMemoryService();
+  const memoryService = getServiceFromQuery(c);
 
   try {
     await memoryService.initialize();
@@ -73,7 +73,7 @@ sessionsRouter.get('/', async (c) => {
 // GET /api/sessions/:id - Get session details
 sessionsRouter.get('/:id', async (c) => {
   const { id } = c.req.param();
-  const memoryService = getReadOnlyMemoryService();
+  const memoryService = getServiceFromQuery(c);
 
   try {
     await memoryService.initialize();