claude-memory-layer 1.0.10 → 1.0.12

package/src/services/memory-service.ts

@@ -45,6 +45,13 @@ import { ConsolidatedStore, createConsolidatedStore } from '../core/consolidated
 import { ConsolidationWorker, createConsolidationWorker } from '../core/consolidation-worker.js';
 import { ContinuityManager, createContinuityManager } from '../core/continuity-manager.js';
 import { GraduationWorker, createGraduationWorker, GraduationRunResult } from '../core/graduation-worker.js';
+import { MarkdownMirror } from '../core/md-mirror.js';
+import {
+  IngestInterceptor,
+  IngestInterceptorRegistry,
+  mergeHierarchicalMetadata
+} from '../core/ingest-interceptor.js';
+import { normalizeTags } from '../core/tag-taxonomy.js';
 
 export interface MemoryServiceConfig {
   storagePath: string;
@@ -103,18 +110,18 @@ export function getProjectStoragePath(projectPath: string): string {
 const REGISTRY_PATH = path.join(os.homedir(), '.claude-code', 'memory', 'session-registry.json');
 const SHARED_STORAGE_PATH = path.join(os.homedir(), '.claude-code', 'memory', 'shared');
 
-interface SessionRegistryEntry {
+export interface SessionRegistryEntry {
   projectPath: string;
   projectHash: string;
   registeredAt: string;
 }
 
-interface SessionRegistry {
+export interface SessionRegistry {
   version: number;
   sessions: Record<string, SessionRegistryEntry>;
 }
 
-function loadSessionRegistry(): SessionRegistry {
+export function loadSessionRegistry(): SessionRegistry {
   try {
     if (fs.existsSync(REGISTRY_PATH)) {
       const data = fs.readFileSync(REGISTRY_PATH, 'utf-8');
@@ -185,6 +192,7 @@ export class MemoryService {
   private vectorWorker: VectorWorker | null = null;
   private graduationWorker: GraduationWorker | null = null;
   private initialized = false;
+  private readonly ingestInterceptors = new IngestInterceptorRegistry();
 
   // Endless Mode components
   private workingSetStore: WorkingSetStore | null = null;
@@ -200,14 +208,17 @@ export class MemoryService {
   private sharedPromoter: SharedPromoter | null = null;
   private sharedStoreConfig: SharedStoreConfig | null = null;
   private projectHash: string | null = null;
+  private projectPath: string | null = null;
 
   private readonly readOnly: boolean;
   private readonly lightweightMode: boolean;
+  private readonly mdMirror: MarkdownMirror;
 
-  constructor(config: MemoryServiceConfig & { projectHash?: string; sharedStoreConfig?: SharedStoreConfig }) {
+  constructor(config: MemoryServiceConfig & { projectHash?: string; projectPath?: string; sharedStoreConfig?: SharedStoreConfig }) {
     const storagePath = this.expandPath(config.storagePath);
     this.readOnly = config.readOnly ?? false;
     this.lightweightMode = config.lightweightMode ?? false;
+    this.mdMirror = new MarkdownMirror(process.cwd());
 
     // Ensure storage directory exists (only if not read-only)
     if (!this.readOnly && !fs.existsSync(storagePath)) {
@@ -216,6 +227,7 @@ export class MemoryService {
 
     // Store project hash for shared store operations
     this.projectHash = config.projectHash || null;
+    this.projectPath = config.projectPath || null;
     // Default: shared store enabled
     this.sharedStoreConfig = config.sharedStoreConfig ?? { enabled: true };
 
@@ -223,7 +235,10 @@ export class MemoryService {
     // This is always used for writes and is the source of truth
     this.sqliteStore = new SQLiteEventStore(
       path.join(storagePath, 'events.sqlite'),
-      { readonly: this.readOnly }
+      {
+        readonly: this.readOnly,
+        markdownMirrorRoot: storagePath
+      }
     );
 
     // Initialize ANALYTICS store: DuckDB (optional, for server reads)
@@ -264,6 +279,7 @@ export class MemoryService {
       this.embedder,
       this.matcher
     );
+    this.retriever.setQueryRewriter((q) => this.rewriteQueryIntent(q));
     this.graduation = createGraduationPipeline(this.sqliteStore as unknown as EventStore);
   }
 
@@ -377,6 +393,105 @@ export class MemoryService {
     this.retriever.setSharedStores(this.sharedStore, this.sharedVectorStore);
   }
 
+  registerIngestBefore(interceptor: IngestInterceptor): () => void {
+    return this.ingestInterceptors.registerBefore(interceptor);
+  }
+
+  registerIngestAfter(interceptor: IngestInterceptor): () => void {
+    return this.ingestInterceptors.registerAfter(interceptor);
+  }
+
+  registerIngestOnError(interceptor: IngestInterceptor): () => void {
+    return this.ingestInterceptors.registerOnError(interceptor);
+  }
+
+  private async ingestWithInterceptors(
+    operation: 'user_prompt' | 'agent_response' | 'session_summary' | 'tool_observation',
+    input: MemoryEventInput,
+    onSuccess?: (eventId: string) => Promise<void>
+  ): Promise<AppendResult> {
+    const normalizedInput: MemoryEventInput = {
+      ...input,
+      metadata: mergeHierarchicalMetadata(
+        {
+          ingest: {
+            operation,
+            pipeline: 'default',
+            ts: new Date().toISOString()
+          },
+          ...(this.projectHash
+            ? {
+                scope: {
+                  project: {
+                    hash: this.projectHash,
+                    ...(this.projectPath ? { path: this.projectPath } : {})
+                  }
+                },
+                tags: [`proj:${this.projectHash}`]
+              }
+            : {})
+        },
+        input.metadata
+      )
+    };
+
+    if (this.projectHash && normalizedInput.metadata) {
+      const meta = normalizedInput.metadata as Record<string, unknown>;
+      const currentTags = Array.isArray(meta.tags)
+        ? meta.tags.filter((x): x is string => typeof x === 'string')
+        : [];
+      const projectTag = `proj:${this.projectHash}`;
+      if (!currentTags.includes(projectTag)) {
+        meta.tags = [...currentTags, projectTag];
+      }
+    }
+
+    if (normalizedInput.metadata) {
+      const meta = normalizedInput.metadata as Record<string, unknown>;
+      const normalizedTags = normalizeTags(meta.tags);
+      if (normalizedTags.length > 0) {
+        meta.tags = normalizedTags;
+      }
+    }
+
+    await this.ingestInterceptors.run('before', {
+      operation,
+      sessionId: normalizedInput.sessionId,
+      event: normalizedInput
+    });
+
+    try {
+      const result = await this.sqliteStore.append(normalizedInput);
+      if (result.success && !result.isDuplicate) {
+        if (onSuccess) {
+          await onSuccess(result.eventId);
+        }
+        try {
+          await this.mdMirror.append(normalizedInput, result.eventId);
+        } catch {
+          // non-breaking markdown mirror write
+        }
+      }
+
+      await this.ingestInterceptors.run('after', {
+        operation,
+        sessionId: normalizedInput.sessionId,
+        event: normalizedInput
+      });
+
+      return result;
+    } catch (error) {
+      const normalizedError = error instanceof Error ? error : new Error(String(error));
+      await this.ingestInterceptors.run('error', {
+        operation,
+        sessionId: normalizedInput.sessionId,
+        event: normalizedInput,
+        error: normalizedError
+      });
+      throw error;
+    }
+  }
+
   /**
    * Start a new session
    */
@@ -413,20 +528,19 @@ export class MemoryService {
   ): Promise<AppendResult> {
     await this.initialize();
 
-    const result = await this.sqliteStore.append({
-      eventType: 'user_prompt',
-      sessionId,
-      timestamp: new Date(),
-      content,
-      metadata
-    });
-
-    // Enqueue for embedding if new
-    if (result.success && !result.isDuplicate) {
-      await this.sqliteStore.enqueueForEmbedding(result.eventId, content);
-    }
-
-    return result;
+    return this.ingestWithInterceptors(
+      'user_prompt',
+      {
+        eventType: 'user_prompt',
+        sessionId,
+        timestamp: new Date(),
+        content,
+        metadata
+      },
+      async (eventId) => {
+        await this.sqliteStore.enqueueForEmbedding(eventId, content);
+      }
+    );
   }
 
   /**
@@ -439,20 +553,19 @@ export class MemoryService {
   ): Promise<AppendResult> {
     await this.initialize();
 
-    const result = await this.sqliteStore.append({
-      eventType: 'agent_response',
-      sessionId,
-      timestamp: new Date(),
-      content,
-      metadata
-    });
-
-    // Enqueue for embedding if new
-    if (result.success && !result.isDuplicate) {
-      await this.sqliteStore.enqueueForEmbedding(result.eventId, content);
-    }
-
-    return result;
+    return this.ingestWithInterceptors(
+      'agent_response',
+      {
+        eventType: 'agent_response',
+        sessionId,
+        timestamp: new Date(),
+        content,
+        metadata
+      },
+      async (eventId) => {
+        await this.sqliteStore.enqueueForEmbedding(eventId, content);
+      }
+    );
   }
 
   /**
@@ -460,22 +573,24 @@ export class MemoryService {
    */
   async storeSessionSummary(
     sessionId: string,
-    summary: string
+    summary: string,
+    metadata?: Record<string, unknown>
   ): Promise<AppendResult> {
     await this.initialize();
 
-    const result = await this.sqliteStore.append({
-      eventType: 'session_summary',
-      sessionId,
-      timestamp: new Date(),
-      content: summary
-    });
-
-    if (result.success && !result.isDuplicate) {
-      await this.sqliteStore.enqueueForEmbedding(result.eventId, summary);
-    }
-
-    return result;
+    return this.ingestWithInterceptors(
+      'session_summary',
+      {
+        eventType: 'session_summary',
+        sessionId,
+        timestamp: new Date(),
+        content: summary,
+        metadata
+      },
+      async (eventId) => {
+        await this.sqliteStore.enqueueForEmbedding(eventId, summary);
+      }
+    );
   }
 
   /**
@@ -490,28 +605,31 @@ export class MemoryService {
     // Create content for storage (JSON stringified payload)
     const content = JSON.stringify(payload);
 
-    const result = await this.sqliteStore.append({
-      eventType: 'tool_observation',
-      sessionId,
-      timestamp: new Date(),
-      content,
-      metadata: {
-        toolName: payload.toolName,
-        success: payload.success
+    // Extract turnId from payload metadata if present (set by PostToolUse hook)
+    const turnId = payload.metadata?.turnId;
+
+    return this.ingestWithInterceptors(
+      'tool_observation',
+      {
+        eventType: 'tool_observation',
+        sessionId,
+        timestamp: new Date(),
+        content,
+        metadata: {
+          toolName: payload.toolName,
+          success: payload.success,
+          ...(turnId ? { turnId } : {})
+        }
+      },
+      async (eventId) => {
+        const embeddingContent = createToolObservationEmbedding(
+          payload.toolName,
+          payload.metadata || {},
+          payload.success
+        );
+        await this.sqliteStore.enqueueForEmbedding(eventId, embeddingContent);
       }
-    });
-
-    // Create embedding content (optimized for search)
-    if (result.success && !result.isDuplicate) {
-      const embeddingContent = createToolObservationEmbedding(
-        payload.toolName,
-        payload.metadata || {},
-        payload.success
-      );
-      await this.sqliteStore.enqueueForEmbedding(result.eventId, embeddingContent);
-    }
-
-    return result;
+    );
   }
 
   /**
@@ -524,6 +642,10 @@ export class MemoryService {
       minScore?: number;
       sessionId?: string;
       includeShared?: boolean;
+      adaptiveRerank?: boolean;
+      intentRewrite?: boolean;
+      projectScopeMode?: 'strict' | 'prefer' | 'global';
+      allowedProjectHashes?: string[];
     }
   ): Promise<UnifiedRetrievalResult> {
     await this.initialize();
@@ -531,16 +653,177 @@ export class MemoryService {
     // Note: Pending embeddings are processed by the background worker
     // Don't block retrieval - search with whatever vectors are available
 
+    const rerankWeights = await this.getRerankWeights(options?.adaptiveRerank === true);
+
     // Use unified retrieval if shared search is requested
+    let result: UnifiedRetrievalResult;
+
     if (options?.includeShared && this.sharedStore) {
-      return this.retriever.retrieveUnified(query, {
+      result = await this.retriever.retrieveUnified(query, {
         ...options,
+        intentRewrite: options?.intentRewrite === true,
+        rerankWeights,
         includeShared: true,
-        projectHash: this.projectHash || undefined
+        projectHash: this.projectHash || undefined,
+        projectScopeMode: options?.projectScopeMode ?? (this.projectHash ? 'strict' : 'global'),
+        allowedProjectHashes: options?.allowedProjectHashes
+      });
+    } else {
+      result = await this.retriever.retrieve(query, {
+        ...options,
+        intentRewrite: options?.intentRewrite === true,
+        rerankWeights,
+        projectHash: this.projectHash || undefined,
+        projectScopeMode: options?.projectScopeMode ?? (this.projectHash ? 'strict' : 'global'),
+        allowedProjectHashes: options?.allowedProjectHashes
+      });
+    }
+
+    try {
+      const selectedEventIds = result.memories.map((m) => m.event.id);
+      const selectedDetails = (result.selectedDebug || []).map((d) => ({
+        eventId: d.eventId,
+        score: d.score,
+        semanticScore: d.semanticScore,
+        lexicalScore: d.lexicalScore,
+        recencyScore: d.recencyScore,
+      }));
+      const candidateDetails = (result.candidateDebug || []).map((d) => ({
+        eventId: d.eventId,
+        score: d.score,
+        semanticScore: d.semanticScore,
+        lexicalScore: d.lexicalScore,
+        recencyScore: d.recencyScore,
+      }));
+      const candidateEventIds = candidateDetails.length > 0
+        ? candidateDetails.map((d) => d.eventId)
+        : selectedEventIds;
+      await this.sqliteStore.recordRetrievalTrace({
+        sessionId: options?.sessionId,
+        projectHash: this.projectHash || undefined,
+        queryText: query,
+        strategy: options?.strategy || 'auto',
+        candidateEventIds,
+        selectedEventIds,
+        candidateDetails,
+        selectedDetails,
+        confidence: result.matchResult.confidence,
+        fallbackTrace: result.fallbackTrace || []
+      });
+    } catch {
+      // non-blocking telemetry
+    }
+
+    return result;
+  }
+
+  private getConfiguredRerankWeights(): { semantic: number; lexical: number; recency: number } | undefined {
+    const semantic = Number(process.env.MEMORY_RERANK_WEIGHT_SEMANTIC ?? '');
+    const lexical = Number(process.env.MEMORY_RERANK_WEIGHT_LEXICAL ?? '');
+    const recency = Number(process.env.MEMORY_RERANK_WEIGHT_RECENCY ?? '');
+
+    const allFinite = [semantic, lexical, recency].every((v) => Number.isFinite(v));
+    if (!allFinite) return undefined;
+
+    const nonNegative = [semantic, lexical, recency].every((v) => v >= 0);
+    const total = semantic + lexical + recency;
+    if (!nonNegative || total <= 0) return undefined;
+
+    return {
+      semantic: semantic / total,
+      lexical: lexical / total,
+      recency: recency / total,
+    };
+  }
+
+  private async getRerankWeights(adaptive: boolean): Promise<{ semantic: number; lexical: number; recency: number } | undefined> {
+    const configured = this.getConfiguredRerankWeights();
+    if (configured) return configured;
+    if (adaptive) return this.getAdaptiveRerankWeights();
+    return undefined;
+  }
+
+  private async rewriteQueryIntent(query: string): Promise<string | null> {
+    if (process.env.MEMORY_INTENT_REWRITE_ENABLED !== '1') return null;
+
+    const apiUrl = process.env.COMPANY_STOCK_API_URL || process.env.COMPANY_INT_API_URL;
+    if (!apiUrl) return null;
+
+    const controller = new AbortController();
+    const timeoutMs = Number(process.env.MEMORY_INTENT_REWRITE_TIMEOUT_MS || 5000);
+    const timeout = setTimeout(() => controller.abort(), timeoutMs);
+
+    try {
+      const prompt = [
+        'Rewrite user query for memory retrieval intent expansion.',
+        'Return plain text only, one line, no markdown.',
+        `Query: ${query}`,
+      ].join('\n');
+
+      const res = await fetch(apiUrl, {
+        method: 'POST',
+        headers: {
+          'Content-Type': 'application/json',
+          Accept: '*/*',
+          Origin: process.env.COMPANY_INT_ORIGIN || 'http://company-int.aplusai.ai',
+          Referer: process.env.COMPANY_INT_REFERER || 'http://company-int.aplusai.ai/',
+        },
+        body: JSON.stringify({
+          question: prompt,
+          company_name: null,
+          conversation_id: null,
+        }),
+        signal: controller.signal,
       });
+
+      const text = (await res.text()).trim();
+      if (!text) return null;
+
+      const oneLine = text
+        .replace(/^data:\s*/gm, '')
+        .split(/\r?\n/)
+        .map((x) => x.trim())
+        .filter(Boolean)
+        .join(' ')
+        .slice(0, 240);
+
+      if (!oneLine || oneLine.toLowerCase() === query.toLowerCase()) return null;
+      return oneLine;
+    } catch {
+      return null;
+    } finally {
+      clearTimeout(timeout);
     }
+  }
 
-    return this.retriever.retrieve(query, options);
+  private async getAdaptiveRerankWeights(): Promise<{ semantic: number; lexical: number; recency: number } | undefined> {
+    try {
+      const s = await this.sqliteStore.getHelpfulnessStats();
+      if (s.totalEvaluated < 20) return undefined;
+
+      // base weights
+      let semantic = 0.7;
+      let lexical = 0.2;
+      let recency = 0.1;
+
+      if (s.avgScore < 0.45) {
+        semantic -= 0.1;
+        lexical += 0.1;
+      } else if (s.avgScore > 0.75) {
+        semantic += 0.05;
+        lexical -= 0.05;
+      }
+
+      if (s.unhelpful > s.helpful) {
+        recency += 0.05;
+        semantic -= 0.03;
+        lexical -= 0.02;
+      }
+
+      return { semantic, lexical, recency };
+    } catch {
+      return undefined;
+    }
   }
 
   /**
@@ -594,6 +877,30 @@ export class MemoryService {
   /**
    * Get memory statistics
    */
+
+  async getOutboxStats(): Promise<{
+    embedding: { pending: number; processing: number; failed: number; total: number };
+    vector: { pending: number; processing: number; failed: number; total: number };
+  }> {
+    await this.initialize();
+    return this.sqliteStore.getOutboxStats();
+  }
+
+  async getRetrievalTraceStats(): Promise<{
+    totalQueries: number;
+    avgCandidateCount: number;
+    avgSelectedCount: number;
+    selectionRate: number;
+  }> {
+    await this.initialize();
+    return this.sqliteStore.getRetrievalTraceStats();
+  }
+
+  async getRecentRetrievalTraces(limit: number = 50) {
+    await this.initialize();
+    return this.sqliteStore.getRecentRetrievalTraces(limit);
+  }
+
   async getStats(): Promise<{
     totalEvents: number;
     vectorCount: number;
@@ -852,6 +1159,37 @@ export class MemoryService {
     return this.consolidatedStore.getAll({ limit });
   }
 
+  /**
+   * Extract topic keywords from event content (markdown headings and key terms)
+   */
+  private extractTopicsFromContent(content: string): string[] {
+    const topics: Set<string> = new Set();
+
+    // Extract markdown headings (## heading)
+    const headings = content.match(/^#{1,3}\s+(.+)$/gm);
+    if (headings) {
+      for (const h of headings.slice(0, 5)) {
+        const text = h.replace(/^#+\s+/, '').replace(/[*_`#]/g, '').trim();
+        if (text.length > 2 && text.length < 50) {
+          topics.add(text);
+        }
+      }
+    }
+
+    // Extract bold terms (**term**)
+    const boldTerms = content.match(/\*\*([^*]+)\*\*/g);
+    if (boldTerms) {
+      for (const b of boldTerms.slice(0, 5)) {
+        const text = b.replace(/\*\*/g, '').trim();
+        if (text.length > 2 && text.length < 30) {
+          topics.add(text);
+        }
+      }
+    }
+
+    return Array.from(topics).slice(0, 5);
+  }
+
   /**
    * Increment access count for memories that were used in prompts
    */
@@ -880,7 +1218,7 @@ export class MemoryService {
       return events.map(event => ({
         memoryId: event.id,
         summary: event.content.substring(0, 200) + (event.content.length > 200 ? '...' : ''),
-        topics: [], // Could extract topics from content if needed
+        topics: this.extractTopicsFromContent(event.content),
         accessCount: (event as any).access_count || 0,
         lastAccessed: (event as any).last_accessed_at || null,
         confidence: 1.0,
@@ -905,6 +1243,51 @@ export class MemoryService {
     return [];
   }
 
+  /**
+   * Record a memory retrieval for helpfulness tracking
+   */
+  async recordRetrieval(eventId: string, sessionId: string, score: number, query: string): Promise<void> {
+    await this.initialize();
+    await this.sqliteStore.recordRetrieval(eventId, sessionId, score, query);
+  }
+
+  /**
+   * Evaluate helpfulness of retrievals in a session (called at session end)
+   */
+  async evaluateSessionHelpfulness(sessionId: string): Promise<void> {
+    await this.initialize();
+    await this.sqliteStore.evaluateSessionHelpfulness(sessionId);
+  }
+
+  /**
+   * Get most helpful memories ranked by helpfulness score
+   */
+  async getHelpfulMemories(limit: number = 10): Promise<Array<{
+    eventId: string;
+    summary: string;
+    helpfulnessScore: number;
+    accessCount: number;
+    evaluationCount: number;
+  }>> {
+    await this.initialize();
+    return this.sqliteStore.getHelpfulMemories(limit);
+  }
+
+  /**
+   * Get helpfulness statistics for dashboard
+   */
+  async getHelpfulnessStats(): Promise<{
+    avgScore: number;
+    totalEvaluated: number;
+    totalRetrievals: number;
+    helpful: number;
+    neutral: number;
+    unhelpful: number;
+  }> {
+    await this.initialize();
+    return this.sqliteStore.getHelpfulnessStats();
+  }
+
   /**
    * Mark a consolidated memory as accessed
    */
@@ -979,6 +1362,58 @@ export class MemoryService {
     };
   }
 
+  // ============================================================
+  // Turn Grouping Methods
+  // ============================================================
+
+  /**
+   * Get events grouped by turn for a session
+   */
+  async getSessionTurns(sessionId: string, options?: { limit?: number; offset?: number }): Promise<Array<{
+    turnId: string;
+    events: MemoryEvent[];
+    startedAt: Date;
+    promptPreview: string;
+    eventCount: number;
+    toolCount: number;
+    hasResponse: boolean;
+  }>> {
+    await this.initialize();
+    return this.sqliteStore.getSessionTurns(sessionId, options);
+  }
+
+  /**
+   * Get all events for a specific turn
+   */
+  async getEventsByTurn(turnId: string): Promise<MemoryEvent[]> {
+    await this.initialize();
+    return this.sqliteStore.getEventsByTurn(turnId);
+  }
+
+  /**
+   * Count total turns for a session
+   */
+  async countSessionTurns(sessionId: string): Promise<number> {
+    await this.initialize();
+    return this.sqliteStore.countSessionTurns(sessionId);
+  }
+
+  /**
+   * Backfill turn_ids from metadata for events stored before the migration
+   */
+  async backfillTurnIds(): Promise<number> {
+    await this.initialize();
+    return this.sqliteStore.backfillTurnIds();
+  }
+
+  /**
+   * Delete all events for a session (for force reimport)
+   */
+  async deleteSessionEvents(sessionId: string): Promise<number> {
+    await this.initialize();
+    return this.sqliteStore.deleteSessionEvents(sessionId);
+  }
+
   /**
    * Format Endless Mode context for Claude
    */
@@ -1146,6 +1581,7 @@ export function getMemoryServiceForProject(
     serviceCache.set(hash, new MemoryService({
       storagePath,
       projectHash: hash,
+      projectPath,
       // Override shared store config - hooks don't need DuckDB
       sharedStoreConfig: sharedStoreConfig ?? { enabled: false },
       analyticsEnabled: false  // Hooks don't need DuckDB
@@ -1187,6 +1623,7 @@ export function getLightweightMemoryService(sessionId: string): MemoryService {
     serviceCache.set(key, new MemoryService({
       storagePath,
       projectHash: projectInfo?.projectHash,
+      projectPath: projectInfo?.projectPath,
       lightweightMode: true,  // Skip embedder/vector/workers
       analyticsEnabled: false,
       sharedStoreConfig: { enabled: false }