claude-memory-layer 1.0.11 → 1.0.13

package/src/services/memory-service.ts

@@ -45,6 +45,13 @@ import { ConsolidatedStore, createConsolidatedStore } from '../core/consolidated
 import { ConsolidationWorker, createConsolidationWorker } from '../core/consolidation-worker.js';
 import { ContinuityManager, createContinuityManager } from '../core/continuity-manager.js';
 import { GraduationWorker, createGraduationWorker, GraduationRunResult } from '../core/graduation-worker.js';
+import { MarkdownMirror } from '../core/md-mirror.js';
+import {
+  IngestInterceptor,
+  IngestInterceptorRegistry,
+  mergeHierarchicalMetadata
+} from '../core/ingest-interceptor.js';
+import { normalizeTags } from '../core/tag-taxonomy.js';
 
 export interface MemoryServiceConfig {
   storagePath: string;
@@ -185,6 +192,7 @@ export class MemoryService {
   private vectorWorker: VectorWorker | null = null;
   private graduationWorker: GraduationWorker | null = null;
   private initialized = false;
+  private readonly ingestInterceptors = new IngestInterceptorRegistry();
 
   // Endless Mode components
   private workingSetStore: WorkingSetStore | null = null;
@@ -200,14 +208,17 @@ export class MemoryService {
   private sharedPromoter: SharedPromoter | null = null;
   private sharedStoreConfig: SharedStoreConfig | null = null;
   private projectHash: string | null = null;
+  private projectPath: string | null = null;
 
   private readonly readOnly: boolean;
   private readonly lightweightMode: boolean;
+  private readonly mdMirror: MarkdownMirror;
 
-  constructor(config: MemoryServiceConfig & { projectHash?: string; sharedStoreConfig?: SharedStoreConfig }) {
+  constructor(config: MemoryServiceConfig & { projectHash?: string; projectPath?: string; sharedStoreConfig?: SharedStoreConfig }) {
     const storagePath = this.expandPath(config.storagePath);
     this.readOnly = config.readOnly ?? false;
     this.lightweightMode = config.lightweightMode ?? false;
+    this.mdMirror = new MarkdownMirror(process.cwd());
 
     // Ensure storage directory exists (only if not read-only)
     if (!this.readOnly && !fs.existsSync(storagePath)) {
@@ -216,6 +227,7 @@ export class MemoryService {
 
     // Store project hash for shared store operations
     this.projectHash = config.projectHash || null;
+    this.projectPath = config.projectPath || null;
     // Default: shared store enabled
     this.sharedStoreConfig = config.sharedStoreConfig ?? { enabled: true };
 
@@ -223,7 +235,10 @@ export class MemoryService {
     // This is always used for writes and is the source of truth
     this.sqliteStore = new SQLiteEventStore(
       path.join(storagePath, 'events.sqlite'),
-      { readonly: this.readOnly }
+      {
+        readonly: this.readOnly,
+        markdownMirrorRoot: storagePath
+      }
     );
 
     // Initialize ANALYTICS store: DuckDB (optional, for server reads)
@@ -264,6 +279,7 @@ export class MemoryService {
       this.embedder,
       this.matcher
     );
+    this.retriever.setQueryRewriter((q) => this.rewriteQueryIntent(q));
     this.graduation = createGraduationPipeline(this.sqliteStore as unknown as EventStore);
   }
 
@@ -377,6 +393,105 @@ export class MemoryService {
     this.retriever.setSharedStores(this.sharedStore, this.sharedVectorStore);
   }
 
+  registerIngestBefore(interceptor: IngestInterceptor): () => void {
+    return this.ingestInterceptors.registerBefore(interceptor);
+  }
+
+  registerIngestAfter(interceptor: IngestInterceptor): () => void {
+    return this.ingestInterceptors.registerAfter(interceptor);
+  }
+
+  registerIngestOnError(interceptor: IngestInterceptor): () => void {
+    return this.ingestInterceptors.registerOnError(interceptor);
+  }
+
+  private async ingestWithInterceptors(
+    operation: 'user_prompt' | 'agent_response' | 'session_summary' | 'tool_observation',
+    input: MemoryEventInput,
+    onSuccess?: (eventId: string) => Promise<void>
+  ): Promise<AppendResult> {
+    const normalizedInput: MemoryEventInput = {
+      ...input,
+      metadata: mergeHierarchicalMetadata(
+        {
+          ingest: {
+            operation,
+            pipeline: 'default',
+            ts: new Date().toISOString()
+          },
+          ...(this.projectHash
+            ? {
+                scope: {
+                  project: {
+                    hash: this.projectHash,
+                    ...(this.projectPath ? { path: this.projectPath } : {})
+                  }
+                },
+                tags: [`proj:${this.projectHash}`]
+              }
+            : {})
+        },
+        input.metadata
+      )
+    };
+
+    if (this.projectHash && normalizedInput.metadata) {
+      const meta = normalizedInput.metadata as Record<string, unknown>;
+      const currentTags = Array.isArray(meta.tags)
+        ? meta.tags.filter((x): x is string => typeof x === 'string')
+        : [];
+      const projectTag = `proj:${this.projectHash}`;
+      if (!currentTags.includes(projectTag)) {
+        meta.tags = [...currentTags, projectTag];
+      }
+    }
+
+    if (normalizedInput.metadata) {
+      const meta = normalizedInput.metadata as Record<string, unknown>;
+      const normalizedTags = normalizeTags(meta.tags);
+      if (normalizedTags.length > 0) {
+        meta.tags = normalizedTags;
+      }
+    }
+
+    await this.ingestInterceptors.run('before', {
+      operation,
+      sessionId: normalizedInput.sessionId,
+      event: normalizedInput
+    });
+
+    try {
+      const result = await this.sqliteStore.append(normalizedInput);
+      if (result.success && !result.isDuplicate) {
+        if (onSuccess) {
+          await onSuccess(result.eventId);
+        }
+        try {
+          await this.mdMirror.append(normalizedInput, result.eventId);
+        } catch {
+          // non-breaking markdown mirror write
+        }
+      }
+
+      await this.ingestInterceptors.run('after', {
+        operation,
+        sessionId: normalizedInput.sessionId,
+        event: normalizedInput
+      });
+
+      return result;
+    } catch (error) {
+      const normalizedError = error instanceof Error ? error : new Error(String(error));
+      await this.ingestInterceptors.run('error', {
+        operation,
+        sessionId: normalizedInput.sessionId,
+        event: normalizedInput,
+        error: normalizedError
+      });
+      throw error;
+    }
+  }
+
   /**
    * Start a new session
    */
@@ -413,20 +528,19 @@ export class MemoryService {
   ): Promise<AppendResult> {
     await this.initialize();
 
-    const result = await this.sqliteStore.append({
-      eventType: 'user_prompt',
-      sessionId,
-      timestamp: new Date(),
-      content,
-      metadata
-    });
-
-    // Enqueue for embedding if new
-    if (result.success && !result.isDuplicate) {
-      await this.sqliteStore.enqueueForEmbedding(result.eventId, content);
-    }
-
-    return result;
+    return this.ingestWithInterceptors(
+      'user_prompt',
+      {
+        eventType: 'user_prompt',
+        sessionId,
+        timestamp: new Date(),
+        content,
+        metadata
+      },
+      async (eventId) => {
+        await this.sqliteStore.enqueueForEmbedding(eventId, content);
+      }
+    );
   }
 
   /**
@@ -439,20 +553,19 @@ export class MemoryService {
   ): Promise<AppendResult> {
     await this.initialize();
 
-    const result = await this.sqliteStore.append({
-      eventType: 'agent_response',
-      sessionId,
-      timestamp: new Date(),
-      content,
-      metadata
-    });
-
-    // Enqueue for embedding if new
-    if (result.success && !result.isDuplicate) {
-      await this.sqliteStore.enqueueForEmbedding(result.eventId, content);
-    }
-
-    return result;
+    return this.ingestWithInterceptors(
+      'agent_response',
+      {
+        eventType: 'agent_response',
+        sessionId,
+        timestamp: new Date(),
+        content,
+        metadata
+      },
+      async (eventId) => {
+        await this.sqliteStore.enqueueForEmbedding(eventId, content);
+      }
+    );
   }
 
   /**
@@ -460,22 +573,24 @@ export class MemoryService {
    */
   async storeSessionSummary(
     sessionId: string,
-    summary: string
+    summary: string,
+    metadata?: Record<string, unknown>
   ): Promise<AppendResult> {
     await this.initialize();
 
-    const result = await this.sqliteStore.append({
-      eventType: 'session_summary',
-      sessionId,
-      timestamp: new Date(),
-      content: summary
-    });
-
-    if (result.success && !result.isDuplicate) {
-      await this.sqliteStore.enqueueForEmbedding(result.eventId, summary);
-    }
-
-    return result;
+    return this.ingestWithInterceptors(
+      'session_summary',
+      {
+        eventType: 'session_summary',
+        sessionId,
+        timestamp: new Date(),
+        content: summary,
+        metadata
+      },
+      async (eventId) => {
+        await this.sqliteStore.enqueueForEmbedding(eventId, summary);
+      }
+    );
   }
 
   /**
@@ -493,29 +608,28 @@ export class MemoryService {
     // Extract turnId from payload metadata if present (set by PostToolUse hook)
     const turnId = payload.metadata?.turnId;
 
-    const result = await this.sqliteStore.append({
-      eventType: 'tool_observation',
-      sessionId,
-      timestamp: new Date(),
-      content,
-      metadata: {
-        toolName: payload.toolName,
-        success: payload.success,
-        ...(turnId ? { turnId } : {})
+    return this.ingestWithInterceptors(
+      'tool_observation',
+      {
+        eventType: 'tool_observation',
+        sessionId,
+        timestamp: new Date(),
+        content,
+        metadata: {
+          toolName: payload.toolName,
+          success: payload.success,
+          ...(turnId ? { turnId } : {})
+        }
+      },
+      async (eventId) => {
+        const embeddingContent = createToolObservationEmbedding(
+          payload.toolName,
+          payload.metadata || {},
+          payload.success
+        );
+        await this.sqliteStore.enqueueForEmbedding(eventId, embeddingContent);
       }
-    });
-
-    // Create embedding content (optimized for search)
-    if (result.success && !result.isDuplicate) {
-      const embeddingContent = createToolObservationEmbedding(
-        payload.toolName,
-        payload.metadata || {},
-        payload.success
-      );
-      await this.sqliteStore.enqueueForEmbedding(result.eventId, embeddingContent);
-    }
-
-    return result;
+    );
   }
 
   /**
@@ -528,6 +642,10 @@ export class MemoryService {
       minScore?: number;
       sessionId?: string;
       includeShared?: boolean;
+      adaptiveRerank?: boolean;
+      intentRewrite?: boolean;
+      projectScopeMode?: 'strict' | 'prefer' | 'global';
+      allowedProjectHashes?: string[];
     }
   ): Promise<UnifiedRetrievalResult> {
     await this.initialize();
@@ -535,16 +653,177 @@ export class MemoryService {
     // Note: Pending embeddings are processed by the background worker
     // Don't block retrieval - search with whatever vectors are available
 
+    const rerankWeights = await this.getRerankWeights(options?.adaptiveRerank === true);
+
     // Use unified retrieval if shared search is requested
+    let result: UnifiedRetrievalResult;
+
     if (options?.includeShared && this.sharedStore) {
-      return this.retriever.retrieveUnified(query, {
+      result = await this.retriever.retrieveUnified(query, {
         ...options,
+        intentRewrite: options?.intentRewrite === true,
+        rerankWeights,
         includeShared: true,
-        projectHash: this.projectHash || undefined
+        projectHash: this.projectHash || undefined,
+        projectScopeMode: options?.projectScopeMode ?? (this.projectHash ? 'strict' : 'global'),
+        allowedProjectHashes: options?.allowedProjectHashes
+      });
+    } else {
+      result = await this.retriever.retrieve(query, {
+        ...options,
+        intentRewrite: options?.intentRewrite === true,
+        rerankWeights,
+        projectHash: this.projectHash || undefined,
+        projectScopeMode: options?.projectScopeMode ?? (this.projectHash ? 'strict' : 'global'),
+        allowedProjectHashes: options?.allowedProjectHashes
+      });
+    }
+
+    try {
+      const selectedEventIds = result.memories.map((m) => m.event.id);
+      const selectedDetails = (result.selectedDebug || []).map((d) => ({
+        eventId: d.eventId,
+        score: d.score,
+        semanticScore: d.semanticScore,
+        lexicalScore: d.lexicalScore,
+        recencyScore: d.recencyScore,
+      }));
+      const candidateDetails = (result.candidateDebug || []).map((d) => ({
+        eventId: d.eventId,
+        score: d.score,
+        semanticScore: d.semanticScore,
+        lexicalScore: d.lexicalScore,
+        recencyScore: d.recencyScore,
+      }));
+      const candidateEventIds = candidateDetails.length > 0
+        ? candidateDetails.map((d) => d.eventId)
+        : selectedEventIds;
+      await this.sqliteStore.recordRetrievalTrace({
+        sessionId: options?.sessionId,
+        projectHash: this.projectHash || undefined,
+        queryText: query,
+        strategy: options?.strategy || 'auto',
+        candidateEventIds,
+        selectedEventIds,
+        candidateDetails,
+        selectedDetails,
+        confidence: result.matchResult.confidence,
+        fallbackTrace: result.fallbackTrace || []
+      });
+    } catch {
+      // non-blocking telemetry
+    }
+
+    return result;
+  }
+
+  private getConfiguredRerankWeights(): { semantic: number; lexical: number; recency: number } | undefined {
+    const semantic = Number(process.env.MEMORY_RERANK_WEIGHT_SEMANTIC ?? '');
+    const lexical = Number(process.env.MEMORY_RERANK_WEIGHT_LEXICAL ?? '');
+    const recency = Number(process.env.MEMORY_RERANK_WEIGHT_RECENCY ?? '');
+
+    const allFinite = [semantic, lexical, recency].every((v) => Number.isFinite(v));
+    if (!allFinite) return undefined;
+
+    const nonNegative = [semantic, lexical, recency].every((v) => v >= 0);
+    const total = semantic + lexical + recency;
+    if (!nonNegative || total <= 0) return undefined;
+
+    return {
+      semantic: semantic / total,
+      lexical: lexical / total,
+      recency: recency / total,
+    };
+  }
+
+  private async getRerankWeights(adaptive: boolean): Promise<{ semantic: number; lexical: number; recency: number } | undefined> {
+    const configured = this.getConfiguredRerankWeights();
+    if (configured) return configured;
+    if (adaptive) return this.getAdaptiveRerankWeights();
+    return undefined;
+  }
+
+  private async rewriteQueryIntent(query: string): Promise<string | null> {
+    if (process.env.MEMORY_INTENT_REWRITE_ENABLED !== '1') return null;
+
+    const apiUrl = process.env.COMPANY_STOCK_API_URL || process.env.COMPANY_INT_API_URL;
+    if (!apiUrl) return null;
+
+    const controller = new AbortController();
+    const timeoutMs = Number(process.env.MEMORY_INTENT_REWRITE_TIMEOUT_MS || 5000);
+    const timeout = setTimeout(() => controller.abort(), timeoutMs);
+
+    try {
+      const prompt = [
+        'Rewrite user query for memory retrieval intent expansion.',
+        'Return plain text only, one line, no markdown.',
+        `Query: ${query}`,
+      ].join('\n');
+
+      const res = await fetch(apiUrl, {
+        method: 'POST',
+        headers: {
+          'Content-Type': 'application/json',
+          Accept: '*/*',
+          Origin: process.env.COMPANY_INT_ORIGIN || 'http://company-int.aplusai.ai',
+          Referer: process.env.COMPANY_INT_REFERER || 'http://company-int.aplusai.ai/',
+        },
+        body: JSON.stringify({
+          question: prompt,
+          company_name: null,
+          conversation_id: null,
+        }),
+        signal: controller.signal,
       });
+
+      const text = (await res.text()).trim();
+      if (!text) return null;
+
+      const oneLine = text
+        .replace(/^data:\s*/gm, '')
+        .split(/\r?\n/)
+        .map((x) => x.trim())
+        .filter(Boolean)
+        .join(' ')
+        .slice(0, 240);
+
+      if (!oneLine || oneLine.toLowerCase() === query.toLowerCase()) return null;
+      return oneLine;
+    } catch {
+      return null;
+    } finally {
+      clearTimeout(timeout);
     }
+  }
+
+  private async getAdaptiveRerankWeights(): Promise<{ semantic: number; lexical: number; recency: number } | undefined> {
+    try {
+      const s = await this.sqliteStore.getHelpfulnessStats();
+      if (s.totalEvaluated < 20) return undefined;
+
+      // base weights
+      let semantic = 0.7;
+      let lexical = 0.2;
+      let recency = 0.1;
+
+      if (s.avgScore < 0.45) {
+        semantic -= 0.1;
+        lexical += 0.1;
+      } else if (s.avgScore > 0.75) {
+        semantic += 0.05;
+        lexical -= 0.05;
+      }
+
+      if (s.unhelpful > s.helpful) {
+        recency += 0.05;
+        semantic -= 0.03;
+        lexical -= 0.02;
+      }
 
-    return this.retriever.retrieve(query, options);
+      return { semantic, lexical, recency };
+    } catch {
+      return undefined;
+    }
   }
 
   /**
@@ -598,6 +877,30 @@ export class MemoryService {
   /**
    * Get memory statistics
    */
+
+  async getOutboxStats(): Promise<{
+    embedding: { pending: number; processing: number; failed: number; total: number };
+    vector: { pending: number; processing: number; failed: number; total: number };
+  }> {
+    await this.initialize();
+    return this.sqliteStore.getOutboxStats();
+  }
+
+  async getRetrievalTraceStats(): Promise<{
+    totalQueries: number;
+    avgCandidateCount: number;
+    avgSelectedCount: number;
+    selectionRate: number;
+  }> {
+    await this.initialize();
+    return this.sqliteStore.getRetrievalTraceStats();
+  }
+
+  async getRecentRetrievalTraces(limit: number = 50) {
+    await this.initialize();
+    return this.sqliteStore.getRecentRetrievalTraces(limit);
+  }
+
   async getStats(): Promise<{
     totalEvents: number;
     vectorCount: number;
@@ -1278,6 +1581,7 @@ export function getMemoryServiceForProject(
     serviceCache.set(hash, new MemoryService({
       storagePath,
       projectHash: hash,
+      projectPath,
       // Override shared store config - hooks don't need DuckDB
       sharedStoreConfig: sharedStoreConfig ?? { enabled: false },
       analyticsEnabled: false  // Hooks don't need DuckDB
@@ -1319,6 +1623,7 @@ export function getLightweightMemoryService(sessionId: string): MemoryService {
     serviceCache.set(key, new MemoryService({
       storagePath,
       projectHash: projectInfo?.projectHash,
+      projectPath: projectInfo?.projectPath,
       lightweightMode: true,  // Skip embedder/vector/workers
       analyticsEnabled: false,
       sharedStoreConfig: { enabled: false }

package/src/services/session-history-importer.ts

@@ -126,19 +126,31 @@ export class SessionHistoryImporter {
 
     // Find project directory
     onProgress?.({ phase: 'scan', message: 'Scanning for session files...' });
-    const projectDir = await this.findProjectDir(projectPath);
-    if (!projectDir) {
+    const projectDirs = await this.findProjectDirs(projectPath);
+    if (projectDirs.length === 0) {
       result.errors.push(`Project directory not found for: ${projectPath}`);
       return result;
     }
 
-    // Find all session files
-    const sessionFiles = await this.findSessionFiles(projectDir);
+    // Find all session files across matched directories
+    const allSessionFiles: string[] = [];
+    for (const dir of projectDirs) {
+      const files = await this.findSessionFiles(dir);
+      allSessionFiles.push(...files);
+    }
+    const sessionFiles = [...new Set(allSessionFiles)];
     result.totalSessions = sessionFiles.length;
-    onProgress?.({ phase: 'scan', message: `Found ${sessionFiles.length} sessions in ${path.basename(projectDir)}` });
+    onProgress?.({
+      phase: 'scan',
+      message: `Found ${sessionFiles.length} sessions in ${projectDirs.length} matched project folder(s)`
+    });
 
     if (options.verbose) {
-      console.log(`Found ${sessionFiles.length} session files in ${projectDir}`);
+      console.log(`Matched project folders:`);
+      for (const dir of projectDirs) {
+        console.log(`  - ${dir}`);
+      }
+      console.log(`Found ${sessionFiles.length} session files across matched folders`);
     }
 
     // Import each session
@@ -401,33 +413,52 @@ export class SessionHistoryImporter {
   }
 
   /**
-   * Find project directory from project path
+   * Find project directories from project path.
+   * Supports wrappers (e.g. happy) that append extra path segments in folder names.
    */
-  private async findProjectDir(projectPath: string): Promise<string | null> {
+  private async findProjectDirs(projectPath: string): Promise<string[]> {
     const projectsDir = path.join(this.claudeDir, 'projects');
     if (!fs.existsSync(projectsDir)) {
-      return null;
+      return [];
     }
 
-    // Claude uses a hash of the project path as directory name
-    // Try to find matching directory by checking all projects
     const projectDirs = fs.readdirSync(projectsDir)
       .map(name => path.join(projectsDir, name))
       .filter(p => fs.statSync(p).isDirectory());
 
-    // Look for directory that matches the project path pattern
-    // The directory name format is: -home-user-project-name
-    const normalizedPath = projectPath.replace(/\//g, '-').replace(/^-/, '');
+    const normalizedPath = projectPath.replace(/\/+/g, '/').replace(/\/$/, '');
+    const normalizedDashed = normalizedPath.replace(/\//g, '-').replace(/^-/, '');
+    const baseName = path.basename(normalizedPath);
 
-    for (const dir of projectDirs) {
+    const scored = projectDirs.map((dir) => {
       const dirName = path.basename(dir);
-      if (dirName.includes(normalizedPath) || normalizedPath.includes(dirName)) {
-        return dir;
-      }
-    }
+      let score = 0;
+
+      // strong matches
+      if (dirName.includes(normalizedDashed)) score += 100;
+      if (normalizedDashed.includes(dirName)) score += 80;
+
+      // basename signal (handles wrappers adding extra suffix)
+      if (baseName && dirName.includes(baseName)) score += 30;
+
+      // token overlap signal
+      const pathTokens = normalizedDashed.split('-').filter(Boolean);
+      const tokenHits = pathTokens.filter(t => t.length >= 3 && dirName.includes(t)).length;
+      score += Math.min(tokenHits, 20);
 
-    // If exact match not found, return first match or null
-    return projectDirs.length > 0 ? projectDirs[0] : null;
+      return { dir, score, dirName };
+    }).filter(x => x.score > 0)
+      .sort((a, b) => b.score - a.score);
+
+    if (scored.length === 0) return [];
+
+    // Keep close matches (same family) to include wrapper-generated variants
+    const top = scored[0].score;
+    const threshold = Math.max(30, top - 25);
+
+    return scored
+      .filter(x => x.score >= threshold)
+      .map(x => x.dir);
   }
 
   /**
@@ -489,10 +520,7 @@ export class SessionHistoryImporter {
     let projectDirs: string[] = [];
 
     if (projectPath) {
-      const projectDir = await this.findProjectDir(projectPath);
-      if (projectDir) {
-        projectDirs = [projectDir];
-      }
+      projectDirs = await this.findProjectDirs(projectPath);
     } else {
       const projectsDir = path.join(this.claudeDir, 'projects');
       if (fs.existsSync(projectsDir)) {