claude-memory-layer 1.0.9 → 1.0.11

package/src/server/api/stats.ts

@@ -4,13 +4,14 @@
  */
 
 import { Hono } from 'hono';
-import { getReadOnlyMemoryService, getMemoryServiceForProject } from '../../services/memory-service.js';
+import { getMemoryServiceForProject } from '../../services/memory-service.js';
+import { getServiceFromQuery } from './utils.js';
 
 export const statsRouter = new Hono();
 
 // GET /api/stats/shared - Get shared store statistics
 statsRouter.get('/shared', async (c) => {
-  const memoryService = getReadOnlyMemoryService();
+  const memoryService = getServiceFromQuery(c);
   try {
     await memoryService.initialize();
     const sharedStats = await memoryService.getSharedStoreStats();
@@ -74,7 +75,7 @@ statsRouter.get('/levels/:level', async (c) => {
     return c.json({ error: `Invalid level. Must be one of: ${validLevels.join(', ')}` }, 400);
   }
 
-  const memoryService = getReadOnlyMemoryService();
+  const memoryService = getServiceFromQuery(c);
   try {
     await memoryService.initialize();
     let events = await memoryService.getEventsByLevel(level, { limit: limit * 2, offset });
@@ -131,7 +132,7 @@ statsRouter.get('/levels/:level', async (c) => {
 
 // GET /api/stats - Get overall statistics
 statsRouter.get('/', async (c) => {
-  const memoryService = getReadOnlyMemoryService();
+  const memoryService = getServiceFromQuery(c);
   try {
     await memoryService.initialize();
     const stats = await memoryService.getStats();
@@ -187,7 +188,7 @@ statsRouter.get('/', async (c) => {
 statsRouter.get('/most-accessed', async (c) => {
   const limit = parseInt(c.req.query('limit') || '10', 10);
   // Use the same read-only service that other stats endpoints use
-  const memoryService = getReadOnlyMemoryService();
+  const memoryService = getServiceFromQuery(c);
 
   try {
     await memoryService.initialize();
@@ -222,7 +223,7 @@ statsRouter.get('/most-accessed', async (c) => {
 // GET /api/stats/timeline - Get activity timeline
 statsRouter.get('/timeline', async (c) => {
   const days = parseInt(c.req.query('days') || '7', 10);
-  const memoryService = getReadOnlyMemoryService();
+  const memoryService = getServiceFromQuery(c);
 
   try {
     await memoryService.initialize();
@@ -255,9 +256,44 @@ statsRouter.get('/timeline', async (c) => {
   }
 });
 
+// GET /api/stats/helpfulness - Get helpfulness statistics and top helpful memories
+statsRouter.get('/helpfulness', async (c) => {
+  const limit = parseInt(c.req.query('limit') || '10', 10);
+  const memoryService = getServiceFromQuery(c);
+
+  try {
+    await memoryService.initialize();
+    const stats = await memoryService.getHelpfulnessStats();
+    const topMemories = await memoryService.getHelpfulMemories(limit);
+
+    return c.json({
+      ...stats,
+      topMemories: topMemories.map(m => ({
+        eventId: m.eventId,
+        summary: m.summary,
+        helpfulnessScore: m.helpfulnessScore,
+        accessCount: m.accessCount,
+        evaluationCount: m.evaluationCount
+      }))
+    });
+  } catch (error) {
+    return c.json({
+      avgScore: 0,
+      totalEvaluated: 0,
+      totalRetrievals: 0,
+      helpful: 0,
+      neutral: 0,
+      unhelpful: 0,
+      topMemories: []
+    });
+  } finally {
+    await memoryService.shutdown();
+  }
+});
+
 // POST /api/stats/graduation/run - Force graduation evaluation
 statsRouter.post('/graduation/run', async (c) => {
-  const memoryService = getReadOnlyMemoryService();
+  const memoryService = getServiceFromQuery(c);
   try {
     await memoryService.initialize();
     const result = await memoryService.forceGraduation();

package/src/server/api/turns.ts

@@ -0,0 +1,143 @@
+/**
+ * Turns API
+ * Endpoints for viewing events grouped by conversation turn
+ *
+ * A "turn" groups a user_prompt with its associated tool_observations
+ * and the final agent_response into a single logical unit.
+ */
+
+import { Hono } from 'hono';
+import { getServiceFromQuery } from './utils.js';
+
+export const turnsRouter = new Hono();
+
+// GET /api/turns?sessionId=xxx - List turns for a session
+turnsRouter.get('/', async (c) => {
+  const sessionId = c.req.query('sessionId');
+  const limit = parseInt(c.req.query('limit') || '20', 10);
+  const offset = parseInt(c.req.query('offset') || '0', 10);
+
+  if (!sessionId) {
+    return c.json({ error: 'sessionId is required' }, 400);
+  }
+
+  const memoryService = getServiceFromQuery(c);
+
+  try {
+    await memoryService.initialize();
+
+    const turns = await memoryService.getSessionTurns(sessionId, { limit, offset });
+    const totalTurns = await memoryService.countSessionTurns(sessionId);
+
+    return c.json({
+      turns: turns.map(t => ({
+        turnId: t.turnId,
+        startedAt: t.startedAt.toISOString(),
+        promptPreview: t.promptPreview,
+        eventCount: t.eventCount,
+        toolCount: t.toolCount,
+        hasResponse: t.hasResponse,
+        events: t.events.map(e => ({
+          id: e.id,
+          eventType: e.eventType,
+          timestamp: e.timestamp instanceof Date ? e.timestamp.toISOString() : e.timestamp,
+          preview: e.content.slice(0, 300) + (e.content.length > 300 ? '...' : ''),
+          contentLength: e.content.length
+        }))
+      })),
+      total: totalTurns,
+      limit,
+      offset,
+      hasMore: offset + limit < totalTurns
+    });
+  } catch (error) {
+    return c.json({ error: (error as Error).message }, 500);
+  } finally {
+    await memoryService.shutdown();
+  }
+});
+
+// GET /api/turns/:turnId - Get full turn details
+turnsRouter.get('/:turnId', async (c) => {
+  const { turnId } = c.req.param();
+  const memoryService = getServiceFromQuery(c);
+
+  try {
+    await memoryService.initialize();
+
+    const events = await memoryService.getEventsByTurn(turnId);
+
+    if (events.length === 0) {
+      return c.json({ error: 'Turn not found' }, 404);
+    }
+
+    const promptEvent = events.find(e => e.eventType === 'user_prompt');
+    const toolEvents = events.filter(e => e.eventType === 'tool_observation');
+    const responseEvents = events.filter(e => e.eventType === 'agent_response');
+
+    return c.json({
+      turnId,
+      sessionId: events[0].sessionId,
+      startedAt: events[0].timestamp instanceof Date
+        ? events[0].timestamp.toISOString()
+        : events[0].timestamp,
+      prompt: promptEvent ? {
+        id: promptEvent.id,
+        content: promptEvent.content,
+        timestamp: promptEvent.timestamp instanceof Date
+          ? promptEvent.timestamp.toISOString()
+          : promptEvent.timestamp
+      } : null,
+      tools: toolEvents.map(e => {
+        let toolName = '';
+        let success = true;
+        try {
+          const parsed = JSON.parse(e.content);
+          toolName = parsed.toolName || '';
+          success = parsed.success !== false;
+        } catch { /* ignore */ }
+
+        return {
+          id: e.id,
+          toolName,
+          success,
+          timestamp: e.timestamp instanceof Date ? e.timestamp.toISOString() : e.timestamp,
+          preview: e.content.slice(0, 500) + (e.content.length > 500 ? '...' : '')
+        };
+      }),
+      responses: responseEvents.map(e => ({
+        id: e.id,
+        content: e.content,
+        timestamp: e.timestamp instanceof Date ? e.timestamp.toISOString() : e.timestamp
+      })),
+      totalEvents: events.length
+    });
+  } catch (error) {
+    return c.json({ error: (error as Error).message }, 500);
+  } finally {
+    await memoryService.shutdown();
+  }
+});
+
+// POST /api/turns/backfill - Backfill turn_ids from metadata
+turnsRouter.post('/backfill', async (c) => {
+  const memoryService = getServiceFromQuery(c);
+
+  try {
+    await memoryService.initialize();
+    const updated = await memoryService.backfillTurnIds();
+
+    return c.json({
+      success: true,
+      updated,
+      message: `Backfilled turn_id for ${updated} events`
+    });
+  } catch (error) {
+    return c.json({
+      success: false,
+      error: (error as Error).message
+    }, 500);
+  } finally {
+    await memoryService.shutdown();
+  }
+});

package/src/server/api/utils.ts

@@ -0,0 +1,46 @@
+/**
+ * API Utilities
+ * Shared helpers for API endpoints
+ */
+
+import type { Context } from 'hono';
+import * as path from 'path';
+import * as os from 'os';
+import { getReadOnlyMemoryService } from '../../services/memory-service.js';
+import { MemoryService } from '../../services/memory-service.js';
+
+/**
+ * Get the appropriate MemoryService based on the ?project= query parameter.
+ * - If ?project=<hash> is set (8 hex chars), resolves directly to project storage
+ * - If ?project=<path> is set, computes hash from path
+ * - Otherwise, returns the global read-only service
+ *
+ * Always creates read-only services for the dashboard API to avoid
+ * VectorWorker lifecycle issues with per-request services.
+ */
+export function getServiceFromQuery(c: Context): MemoryService {
+  const project = c.req.query('project');
+  if (project) {
+    // Check if it's a hash (8 hex chars) or a path
+    const isHash = /^[a-f0-9]{8}$/.test(project);
+    let storagePath: string;
+
+    if (isHash) {
+      storagePath = path.join(os.homedir(), '.claude-code', 'memory', 'projects', project);
+    } else {
+      // Import hashProjectPath dynamically to compute the hash from path
+      const crypto = require('crypto');
+      const normalized = project.replace(/\/+$/, '') || '/';
+      const hash = crypto.createHash('sha256').update(normalized).digest('hex').slice(0, 8);
+      storagePath = path.join(os.homedir(), '.claude-code', 'memory', 'projects', hash);
+    }
+
+    return new MemoryService({
+      storagePath,
+      readOnly: true,
+      analyticsEnabled: false,
+      sharedStoreConfig: { enabled: false }
+    });
+  }
+  return getReadOnlyMemoryService();
+}

package/src/services/memory-service.ts

@@ -52,6 +52,8 @@ export interface MemoryServiceConfig {
   readOnly?: boolean;
   /** Enable DuckDB analytics store (default: true for server, false for hooks) */
   analyticsEnabled?: boolean;
+  /** Lightweight mode for hooks - skip heavy initialization (default: false) */
+  lightweightMode?: boolean;
 }
 
 // ============================================================
@@ -101,18 +103,18 @@ export function getProjectStoragePath(projectPath: string): string {
 const REGISTRY_PATH = path.join(os.homedir(), '.claude-code', 'memory', 'session-registry.json');
 const SHARED_STORAGE_PATH = path.join(os.homedir(), '.claude-code', 'memory', 'shared');
 
-interface SessionRegistryEntry {
+export interface SessionRegistryEntry {
   projectPath: string;
   projectHash: string;
   registeredAt: string;
 }
 
-interface SessionRegistry {
+export interface SessionRegistry {
   version: number;
   sessions: Record<string, SessionRegistryEntry>;
 }
 
-function loadSessionRegistry(): SessionRegistry {
+export function loadSessionRegistry(): SessionRegistry {
   try {
     if (fs.existsSync(REGISTRY_PATH)) {
       const data = fs.readFileSync(REGISTRY_PATH, 'utf-8');
@@ -200,10 +202,12 @@ export class MemoryService {
   private projectHash: string | null = null;
 
   private readonly readOnly: boolean;
+  private readonly lightweightMode: boolean;
 
   constructor(config: MemoryServiceConfig & { projectHash?: string; sharedStoreConfig?: SharedStoreConfig }) {
     const storagePath = this.expandPath(config.storagePath);
     this.readOnly = config.readOnly ?? false;
+    this.lightweightMode = config.lightweightMode ?? false;
 
     // Ensure storage directory exists (only if not read-only)
     if (!this.readOnly && !fs.existsSync(storagePath)) {
@@ -272,6 +276,13 @@ export class MemoryService {
     // Initialize PRIMARY store: SQLite (always)
     await this.sqliteStore.initialize();
 
+    // Lightweight mode: only SQLite, no embedder/vector/workers
+    // Used for hooks that just need to store data quickly
+    if (this.lightweightMode) {
+      this.initialized = true;
+      return;
+    }
+
     // Initialize analytics store if available (DuckDB)
     if (this.analyticsStore) {
       try {
@@ -479,6 +490,9 @@ export class MemoryService {
     // Create content for storage (JSON stringified payload)
     const content = JSON.stringify(payload);
 
+    // Extract turnId from payload metadata if present (set by PostToolUse hook)
+    const turnId = payload.metadata?.turnId;
+
     const result = await this.sqliteStore.append({
       eventType: 'tool_observation',
       sessionId,
@@ -486,7 +500,8 @@ export class MemoryService {
       content,
       metadata: {
         toolName: payload.toolName,
-        success: payload.success
+        success: payload.success,
+        ...(turnId ? { turnId } : {})
       }
     });
 
@@ -517,10 +532,8 @@ export class MemoryService {
   ): Promise<UnifiedRetrievalResult> {
     await this.initialize();
 
-    // Process any pending embeddings first
-    if (this.vectorWorker) {
-      await this.vectorWorker.processAll();
-    }
+    // Note: Pending embeddings are processed by the background worker
+    // Don't block retrieval - search with whatever vectors are available
 
     // Use unified retrieval if shared search is requested
     if (options?.includeShared && this.sharedStore) {
@@ -534,6 +547,38 @@ export class MemoryService {
     return this.retriever.retrieve(query, options);
   }
 
+  /**
+   * Fast keyword search using SQLite FTS5
+   * Much faster than vector search - no embedding model needed
+   */
+  async keywordSearch(
+    query: string,
+    options?: { topK?: number; minScore?: number }
+  ): Promise<Array<{event: MemoryEvent; score: number}>> {
+    await this.initialize();
+
+    const results = await this.sqliteStore.keywordSearch(query, options?.topK ?? 10);
+
+    // Normalize FTS5 rank to a score (0-1 range)
+    // FTS5 rank is negative (higher is worse), so we convert it
+    const maxRank = Math.min(...results.map(r => r.rank), -0.001);
+    const minRank = Math.max(...results.map(r => r.rank), -1000);
+    const rankRange = maxRank - minRank || 1;
+
+    return results.map(r => ({
+      event: r.event,
+      score: 1 - (r.rank - minRank) / rankRange  // Normalize to 0-1
+    })).filter(r => !options?.minScore || r.score >= options.minScore);
+  }
+
+  /**
+   * Rebuild FTS index (call after database upgrade)
+   */
+  async rebuildFtsIndex(): Promise<number> {
+    await this.initialize();
+    return this.sqliteStore.rebuildFtsIndex();
+  }
+
   /**
    * Get session history
    */
@@ -811,6 +856,37 @@ export class MemoryService {
     return this.consolidatedStore.getAll({ limit });
   }
 
+  /**
+   * Extract topic keywords from event content (markdown headings and key terms)
+   */
+  private extractTopicsFromContent(content: string): string[] {
+    const topics: Set<string> = new Set();
+
+    // Extract markdown headings (## heading)
+    const headings = content.match(/^#{1,3}\s+(.+)$/gm);
+    if (headings) {
+      for (const h of headings.slice(0, 5)) {
+        const text = h.replace(/^#+\s+/, '').replace(/[*_`#]/g, '').trim();
+        if (text.length > 2 && text.length < 50) {
+          topics.add(text);
+        }
+      }
+    }
+
+    // Extract bold terms (**term**)
+    const boldTerms = content.match(/\*\*([^*]+)\*\*/g);
+    if (boldTerms) {
+      for (const b of boldTerms.slice(0, 5)) {
+        const text = b.replace(/\*\*/g, '').trim();
+        if (text.length > 2 && text.length < 30) {
+          topics.add(text);
+        }
+      }
+    }
+
+    return Array.from(topics).slice(0, 5);
+  }
+
   /**
    * Increment access count for memories that were used in prompts
    */
@@ -839,7 +915,7 @@ export class MemoryService {
       return events.map(event => ({
         memoryId: event.id,
         summary: event.content.substring(0, 200) + (event.content.length > 200 ? '...' : ''),
-        topics: [], // Could extract topics from content if needed
+        topics: this.extractTopicsFromContent(event.content),
         accessCount: (event as any).access_count || 0,
         lastAccessed: (event as any).last_accessed_at || null,
         confidence: 1.0,
@@ -864,6 +940,51 @@ export class MemoryService {
     return [];
   }
 
+  /**
+   * Record a memory retrieval for helpfulness tracking
+   */
+  async recordRetrieval(eventId: string, sessionId: string, score: number, query: string): Promise<void> {
+    await this.initialize();
+    await this.sqliteStore.recordRetrieval(eventId, sessionId, score, query);
+  }
+
+  /**
+   * Evaluate helpfulness of retrievals in a session (called at session end)
+   */
+  async evaluateSessionHelpfulness(sessionId: string): Promise<void> {
+    await this.initialize();
+    await this.sqliteStore.evaluateSessionHelpfulness(sessionId);
+  }
+
+  /**
+   * Get most helpful memories ranked by helpfulness score
+   */
+  async getHelpfulMemories(limit: number = 10): Promise<Array<{
+    eventId: string;
+    summary: string;
+    helpfulnessScore: number;
+    accessCount: number;
+    evaluationCount: number;
+  }>> {
+    await this.initialize();
+    return this.sqliteStore.getHelpfulMemories(limit);
+  }
+
+  /**
+   * Get helpfulness statistics for dashboard
+   */
+  async getHelpfulnessStats(): Promise<{
+    avgScore: number;
+    totalEvaluated: number;
+    totalRetrievals: number;
+    helpful: number;
+    neutral: number;
+    unhelpful: number;
+  }> {
+    await this.initialize();
+    return this.sqliteStore.getHelpfulnessStats();
+  }
+
   /**
    * Mark a consolidated memory as accessed
    */
@@ -938,6 +1059,58 @@ export class MemoryService {
     };
   }
 
+  // ============================================================
+  // Turn Grouping Methods
+  // ============================================================
+
+  /**
+   * Get events grouped by turn for a session
+   */
+  async getSessionTurns(sessionId: string, options?: { limit?: number; offset?: number }): Promise<Array<{
+    turnId: string;
+    events: MemoryEvent[];
+    startedAt: Date;
+    promptPreview: string;
+    eventCount: number;
+    toolCount: number;
+    hasResponse: boolean;
+  }>> {
+    await this.initialize();
+    return this.sqliteStore.getSessionTurns(sessionId, options);
+  }
+
+  /**
+   * Get all events for a specific turn
+   */
+  async getEventsByTurn(turnId: string): Promise<MemoryEvent[]> {
+    await this.initialize();
+    return this.sqliteStore.getEventsByTurn(turnId);
+  }
+
+  /**
+   * Count total turns for a session
+   */
+  async countSessionTurns(sessionId: string): Promise<number> {
+    await this.initialize();
+    return this.sqliteStore.countSessionTurns(sessionId);
+  }
+
+  /**
+   * Backfill turn_ids from metadata for events stored before the migration
+   */
+  async backfillTurnIds(): Promise<number> {
+    await this.initialize();
+    return this.sqliteStore.backfillTurnIds();
+  }
+
+  /**
+   * Delete all events for a session (for force reimport)
+   */
+  async deleteSessionEvents(sessionId: string): Promise<number> {
+    await this.initialize();
+    return this.sqliteStore.deleteSessionEvents(sessionId);
+  }
+
   /**
    * Format Endless Mode context for Claude
    */
@@ -1129,6 +1302,32 @@ export function getMemoryServiceForSession(sessionId: string): MemoryService {
   return getDefaultMemoryService();
 }
 
+/**
+ * Get a lightweight memory service for hooks
+ * Only initializes SQLite - no embedder, no vector store, no workers
+ * This is FAST (<100ms) compared to full initialization (3-5s)
+ */
+export function getLightweightMemoryService(sessionId: string): MemoryService {
+  const projectInfo = getSessionProject(sessionId);
+  const key = projectInfo ? `lightweight_${projectInfo.projectHash}` : 'lightweight_global';
+
+  if (!serviceCache.has(key)) {
+    const storagePath = projectInfo
+      ? getProjectStoragePath(projectInfo.projectPath)
+      : path.join(os.homedir(), '.claude-code', 'memory');
+
+    serviceCache.set(key, new MemoryService({
+      storagePath,
+      projectHash: projectInfo?.projectHash,
+      lightweightMode: true,  // Skip embedder/vector/workers
+      analyticsEnabled: false,
+      sharedStoreConfig: { enabled: false }
+    }));
+  }
+
+  return serviceCache.get(key)!;
+}
+
 export function createMemoryService(config: MemoryServiceConfig): MemoryService {
   return new MemoryService(config);
 }