claude-memory-layer 1.0.27 → 1.0.29

package/src/core/engine/memory-runtime-service.ts

@@ -0,0 +1,162 @@
+/**
+ * Memory Runtime Service
+ * Owns MemoryService lifecycle concerns: initialization, background workers,
+ * lightweight/read-only modes, and orderly shutdown.
+ */
+
+import type { EventStore } from '../event-store.js';
+import type { Embedder } from '../embedder.js';
+import type { GraduationPipeline } from '../graduation.js';
+import {
+  createGraduationWorker as defaultCreateGraduationWorker,
+  type GraduationRunResult,
+  type GraduationWorker
+} from '../graduation-worker.js';
+import type { Retriever } from '../retriever.js';
+import type { VectorStore } from '../vector-store.js';
+import {
+  createVectorWorker as defaultCreateVectorWorker,
+  type VectorWorker
+} from '../vector-worker.js';
+
+export interface RuntimeSQLiteStore {
+  initialize(): Promise<void>;
+  close(): Promise<void>;
+}
+
+export interface RuntimeEndlessMemoryServices {
+  initializeFromSavedMode(): Promise<void>;
+  shutdown(): void;
+}
+
+export interface RuntimeSharedMemoryServices {
+  initialize(): Promise<void>;
+  close(): Promise<void>;
+}
+
+export interface MemoryRuntimeServicesFactories {
+  createVectorWorker?: typeof defaultCreateVectorWorker;
+  createGraduationWorker?: typeof defaultCreateGraduationWorker;
+}
+
+export interface MemoryRuntimeServicesDeps {
+  sqliteStore: RuntimeSQLiteStore;
+  eventStore: EventStore;
+  vectorStore: VectorStore;
+  embedder: Embedder;
+  retriever: Retriever;
+  graduation: GraduationPipeline;
+  endlessMemoryServices: RuntimeEndlessMemoryServices;
+  sharedMemoryServices: RuntimeSharedMemoryServices;
+  readOnly: boolean;
+  lightweightMode: boolean;
+  embeddingOnly: boolean;
+  factories?: MemoryRuntimeServicesFactories;
+}
+
+export interface MemoryRuntimeService {
+  initialize(): Promise<void>;
+  shutdown(): Promise<void>;
+  processPendingEmbeddings(): Promise<number>;
+  forceGraduation(): Promise<GraduationRunResult>;
+  recordMemoryAccess(eventId: string, sessionId: string, confidence?: number): void;
+  getVectorWorker(): VectorWorker | null;
+  isInitialized(): boolean;
+}
+
+function createEmptyGraduationResult(): GraduationRunResult {
+  return { evaluated: 0, graduated: 0, byLevel: {} };
+}
+
+export function createMemoryRuntimeService(deps: MemoryRuntimeServicesDeps): MemoryRuntimeService {
+  const createVectorWorker = deps.factories?.createVectorWorker ?? defaultCreateVectorWorker;
+  const createGraduationWorker = deps.factories?.createGraduationWorker ?? defaultCreateGraduationWorker;
+
+  let initialized = false;
+  let vectorWorker: VectorWorker | null = null;
+  let graduationWorker: GraduationWorker | null = null;
+
+  return {
+    async initialize(): Promise<void> {
+      if (initialized) return;
+
+      // Initialize PRIMARY store: SQLite (always)
+      await deps.sqliteStore.initialize();
+
+      // Lightweight mode: only SQLite, no embedder/vector/workers.
+      // Used for hooks that just need to store data quickly.
+      if (deps.lightweightMode) {
+        initialized = true;
+        return;
+      }
+
+      await deps.vectorStore.initialize();
+      await deps.embedder.initialize();
+
+      // Skip write-related workers in read-only mode.
+      if (!deps.readOnly) {
+        vectorWorker = createVectorWorker(
+          deps.eventStore,
+          deps.vectorStore,
+          deps.embedder
+        );
+        vectorWorker.start();
+
+        if (!deps.embeddingOnly) {
+          deps.retriever.setGraduationPipeline(deps.graduation);
+          graduationWorker = createGraduationWorker(
+            deps.eventStore,
+            deps.graduation
+          );
+          graduationWorker.start();
+        }
+
+        await deps.endlessMemoryServices.initializeFromSavedMode();
+        await deps.sharedMemoryServices.initialize();
+      }
+
+      initialized = true;
+    },
+
+    async shutdown(): Promise<void> {
+      if (graduationWorker) {
+        graduationWorker.stop();
+      }
+
+      deps.endlessMemoryServices.shutdown();
+
+      if (vectorWorker) {
+        vectorWorker.stop();
+      }
+
+      await deps.sharedMemoryServices.close();
+      await deps.sqliteStore.close();
+    },
+
+    async processPendingEmbeddings(): Promise<number> {
+      if (vectorWorker) {
+        return vectorWorker.processAll();
+      }
+      return 0;
+    },
+
+    async forceGraduation(): Promise<GraduationRunResult> {
+      if (!graduationWorker) {
+        return createEmptyGraduationResult();
+      }
+      return graduationWorker.forceRun();
+    },
+
+    recordMemoryAccess(eventId: string, sessionId: string, confidence: number = 1.0): void {
+      deps.graduation.recordAccess(eventId, sessionId, confidence);
+    },
+
+    getVectorWorker(): VectorWorker | null {
+      return vectorWorker;
+    },
+
+    isInitialized(): boolean {
+      return initialized;
+    }
+  };
+}

package/src/core/engine/memory-service-composition.ts

@@ -0,0 +1,231 @@
+/**
+ * Memory Service Composition
+ *
+ * Owns constructor-time service graph wiring for MemoryService so the public
+ * facade can stay focused on state assignment and method delegation.
+ */
+
+import * as os from 'os';
+import * as path from 'path';
+
+import type { EventStore } from '../event-store.js';
+import type { Embedder } from '../embedder.js';
+import type { GraduationPipeline } from '../graduation.js';
+import { createToolObservationEmbedding as defaultCreateToolObservationEmbedding } from '../metadata-extractor.js';
+import type { Retriever } from '../retriever.js';
+import type { SQLiteEventStore } from '../sqlite-event-store.js';
+import type { ToolObservationPayload, SharedStoreConfig } from '../types.js';
+import type { VectorStore } from '../vector-store.js';
+import {
+  createEmbeddingMaintenanceService as defaultCreateEmbeddingMaintenanceService,
+  type EmbeddingMaintenanceService,
+  type EmbeddingMaintenanceServiceOptions
+} from './embedding-maintenance-service.js';
+import {
+  createEndlessMemoryServices as defaultCreateEndlessMemoryServices,
+  type EndlessMemoryServices,
+  type EndlessMemoryServicesOptions
+} from './endless-memory-services.js';
+import {
+  createMemoryEngineServices as defaultCreateMemoryEngineServices,
+  type MemoryEngineServices,
+  type MemoryEngineServicesOptions
+} from './memory-engine-services.js';
+import {
+  createMemoryRuntimeService as defaultCreateMemoryRuntimeService,
+  type MemoryRuntimeService,
+  type MemoryRuntimeServicesDeps
+} from './memory-runtime-service.js';
+import type {
+  MemoryIngestService
+} from './memory-ingest-service.js';
+import type {
+  MemoryQueryService
+} from './memory-query-service.js';
+import {
+  createSharedMemoryServices as defaultCreateSharedMemoryServices,
+  type SharedMemoryServices,
+  type SharedMemoryServicesOptions
+} from './shared-memory-services.js';
+import type {
+  RetrievalAnalyticsService,
+  RetrievalDisclosureService,
+  RetrievalOrchestrator
+} from './retrieval-services.js';
+
+export interface MemoryServiceCompositionConfig {
+  storagePath: string;
+  embeddingModel?: string;
+  readOnly?: boolean;
+  lightweightMode?: boolean;
+  embeddingOnly?: boolean;
+  projectHash?: string;
+  projectPath?: string;
+  sharedStoreConfig?: SharedStoreConfig;
+}
+
+export interface MemoryServiceCompositionFactories {
+  expandPath?: (targetPath: string) => string;
+  createToolObservationEmbedding?: (
+    toolName: string,
+    metadata: Record<string, unknown>,
+    success: boolean
+  ) => string;
+  createMemoryEngineServices?: (options: MemoryEngineServicesOptions) => MemoryEngineServices;
+  createEndlessMemoryServices?: (options: EndlessMemoryServicesOptions) => EndlessMemoryServices;
+  createSharedMemoryServices?: (options: SharedMemoryServicesOptions) => SharedMemoryServices;
+  createMemoryRuntimeService?: (deps: MemoryRuntimeServicesDeps) => MemoryRuntimeService;
+  createEmbeddingMaintenanceService?: (
+    options: EmbeddingMaintenanceServiceOptions
+  ) => EmbeddingMaintenanceService;
+}
+
+export interface MemoryServiceCompositionOptions {
+  config: MemoryServiceCompositionConfig;
+  defaultSharedStoragePath: string;
+  defaultSharedStoreConfig?: SharedStoreConfig;
+  initialize: () => Promise<void>;
+  getProjectHash: () => string | null;
+  getProjectPath?: () => string | null;
+  factories?: MemoryServiceCompositionFactories;
+}
+
+export interface MemoryServiceComposition {
+  storagePath: string;
+  readOnly: boolean;
+  lightweightMode: boolean;
+  embeddingOnly: boolean;
+  sqliteStore: SQLiteEventStore;
+  vectorStore: VectorStore;
+  embedder: Embedder;
+  retriever: Retriever;
+  retrievalOrchestrator: RetrievalOrchestrator;
+  retrievalDisclosureService: RetrievalDisclosureService;
+  retrievalAnalyticsService: RetrievalAnalyticsService;
+  embeddingMaintenanceService: EmbeddingMaintenanceService;
+  runtimeService: MemoryRuntimeService;
+  graduation: GraduationPipeline;
+  endlessMemoryServices: EndlessMemoryServices;
+  sharedMemoryServices: SharedMemoryServices;
+  ingestService: MemoryIngestService;
+  queryService: MemoryQueryService;
+}
+
+export function createMemoryServiceComposition(
+  options: MemoryServiceCompositionOptions
+): MemoryServiceComposition {
+  const factories = options.factories ?? {};
+  const expandPath = factories.expandPath ?? defaultExpandPath;
+  const createToolEmbedding = factories.createToolObservationEmbedding ?? defaultCreateToolObservationEmbedding;
+
+  const storagePath = expandPath(options.config.storagePath);
+  const projectPathForMirror = options.getProjectPath?.() ?? options.config.projectPath ?? process.cwd();
+  const readOnly = options.config.readOnly ?? false;
+  const lightweightMode = options.config.lightweightMode ?? false;
+  const embeddingOnly = options.config.embeddingOnly ?? false;
+  const sharedStoreConfig = options.config.sharedStoreConfig ?? options.defaultSharedStoreConfig ?? {
+    enabled: true,
+    autoPromote: true,
+    searchShared: true,
+    minConfidenceForPromotion: 0.8,
+    sharedStoragePath: options.defaultSharedStoragePath
+  };
+
+  let sharedMemoryServices: SharedMemoryServices | null = null;
+
+  const engineServices = (factories.createMemoryEngineServices ?? defaultCreateMemoryEngineServices)({
+    storagePath,
+    readOnly,
+    embeddingModel: options.config.embeddingModel,
+    cwd: projectPathForMirror,
+    initialize: options.initialize,
+    getProjectHash: options.getProjectHash,
+    getProjectPath: options.getProjectPath,
+    hasSharedStore: () => sharedMemoryServices?.isEnabled() ?? false,
+    sharedStore: {
+      get: (entryId: string) => sharedMemoryServices?.getEntryForDisclosure(entryId) ?? Promise.resolve(null)
+    },
+    createToolObservationEmbedding: (payload: ToolObservationPayload) => createToolEmbedding(
+      payload.toolName,
+      payload.metadata || {},
+      payload.success
+    )
+  });
+
+  const endlessMemoryServices = (factories.createEndlessMemoryServices ?? defaultCreateEndlessMemoryServices)({
+    eventStore: engineServices.sqliteStore as unknown as EventStore,
+    configStore: engineServices.sqliteStore,
+    initialize: options.initialize
+  });
+
+  sharedMemoryServices = (factories.createSharedMemoryServices ?? defaultCreateSharedMemoryServices)({
+    config: sharedStoreConfig,
+    defaultSharedStoragePath: options.defaultSharedStoragePath,
+    readOnly,
+    expandPath,
+    embedder: engineServices.embedder,
+    retriever: engineServices.retriever
+  });
+
+  const runtimeService = (factories.createMemoryRuntimeService ?? defaultCreateMemoryRuntimeService)({
+    sqliteStore: engineServices.sqliteStore,
+    eventStore: engineServices.sqliteStore as unknown as EventStore,
+    vectorStore: engineServices.vectorStore,
+    embedder: engineServices.embedder,
+    retriever: engineServices.retriever,
+    graduation: engineServices.graduation,
+    endlessMemoryServices,
+    sharedMemoryServices,
+    readOnly,
+    lightweightMode,
+    embeddingOnly
+  });
+
+  const embeddingMaintenanceService = (
+    factories.createEmbeddingMaintenanceService ?? defaultCreateEmbeddingMaintenanceService
+  )({
+    storagePath,
+    initialize: options.initialize,
+    getEmbeddingModelName: () => engineServices.embedder.getModelName(),
+    vectorStore: engineServices.vectorStore,
+    eventStore: {
+      clearEmbeddingOutbox: () => engineServices.sqliteStore.clearEmbeddingOutbox(),
+      getEventsPage: async (limit, offset) => {
+        const events = await engineServices.sqliteStore.getEventsPage(limit, offset);
+        return events.map((event) => ({ id: event.id, content: event.content }));
+      },
+      enqueueForEmbedding: async (eventId, content) => {
+        await engineServices.sqliteStore.enqueueForEmbedding(eventId, content);
+      }
+    },
+    getVectorWorker: () => runtimeService.getVectorWorker()
+  });
+
+  return {
+    storagePath,
+    readOnly,
+    lightweightMode,
+    embeddingOnly,
+    sqliteStore: engineServices.sqliteStore,
+    vectorStore: engineServices.vectorStore,
+    embedder: engineServices.embedder,
+    retriever: engineServices.retriever,
+    retrievalOrchestrator: engineServices.retrievalOrchestrator,
+    retrievalDisclosureService: engineServices.retrievalDisclosureService,
+    retrievalAnalyticsService: engineServices.retrievalAnalyticsService,
+    embeddingMaintenanceService,
+    runtimeService,
+    graduation: engineServices.graduation,
+    endlessMemoryServices,
+    sharedMemoryServices,
+    ingestService: engineServices.ingestService,
+    queryService: engineServices.queryService
+  };
+}
+
+function defaultExpandPath(targetPath: string): string {
+  if (targetPath.startsWith('~')) {
+    return path.join(os.homedir(), targetPath.slice(1));
+  }
+  return targetPath;
+}

package/src/core/engine/retrieval-analytics-service.ts

@@ -0,0 +1,181 @@
+/**
+ * Retrieval Analytics Service
+ *
+ * Owns retrieval telemetry read-model and helpfulness evaluation workflows so
+ * MemoryService can remain a thin facade over focused engine services.
+ */
+
+import type { MemoryEvent } from '../types.js';
+
+export interface RetrievalTraceStats {
+  totalQueries: number;
+  avgCandidateCount: number;
+  avgSelectedCount: number;
+  selectionRate: number;
+}
+
+export interface HelpfulnessStats {
+  avgScore: number;
+  totalEvaluated: number;
+  totalRetrievals: number;
+  helpful: number;
+  neutral: number;
+  unhelpful: number;
+}
+
+export interface HelpfulMemory {
+  eventId: string;
+  summary: string;
+  helpfulnessScore: number;
+  accessCount: number;
+  evaluationCount: number;
+}
+
+export interface RetrievalTraceDetail {
+  eventId: string;
+  score: number;
+  semanticScore?: number;
+  lexicalScore?: number;
+  recencyScore?: number;
+}
+
+export interface RetrievalTrace {
+  traceId: string;
+  sessionId?: string;
+  projectHash?: string;
+  queryText: string;
+  strategy?: string;
+  candidateEventIds: string[];
+  selectedEventIds: string[];
+  candidateDetails: RetrievalTraceDetail[];
+  selectedDetails: RetrievalTraceDetail[];
+  candidateCount: number;
+  selectedCount: number;
+  confidence?: string;
+  fallbackTrace: string[];
+  createdAt: Date;
+}
+
+export interface AccessedMemory {
+  memoryId: string;
+  summary: string;
+  topics: string[];
+  accessCount: number;
+  lastAccessed: string | null;
+  confidence: number;
+  createdAt: Date;
+}
+
+type AccessedMemoryEvent = MemoryEvent & {
+  access_count?: number;
+  last_accessed_at?: string | null;
+};
+
+export interface RetrievalAnalyticsStore {
+  getRetrievalTraceStats(): Promise<RetrievalTraceStats>;
+  getRecentRetrievalTraces(limit?: number): Promise<RetrievalTrace[]>;
+  getMostAccessed(limit?: number): Promise<AccessedMemoryEvent[]>;
+  evaluateSessionHelpfulness(sessionId: string): Promise<void>;
+  getUnevaluatedSessions(currentSessionId: string, limit?: number): Promise<string[]>;
+  getHelpfulMemories(limit?: number): Promise<HelpfulMemory[]>;
+  getHelpfulnessStats(): Promise<HelpfulnessStats>;
+}
+
+export interface RetrievalAnalyticsServiceDeps {
+  initialize: () => Promise<void>;
+  retrievalStore: RetrievalAnalyticsStore;
+}
+
+export class RetrievalAnalyticsService {
+  constructor(private readonly deps: RetrievalAnalyticsServiceDeps) {}
+
+  async getRetrievalTraceStats(): Promise<RetrievalTraceStats> {
+    await this.deps.initialize();
+    return this.deps.retrievalStore.getRetrievalTraceStats();
+  }
+
+  async getRecentRetrievalTraces(limit: number = 50): Promise<RetrievalTrace[]> {
+    await this.deps.initialize();
+    return this.deps.retrievalStore.getRecentRetrievalTraces(limit);
+  }
+
+  async getMostAccessedMemories(limit: number = 10): Promise<AccessedMemory[]> {
+    // Preserve the historical lightweight path: SQLiteEventStore.getMostAccessed()
+    // initializes itself and no-ops safely in read-only scenarios, so dashboard
+    // access summaries should not trigger vector/embedder/worker initialization.
+    const events = await this.deps.retrievalStore.getMostAccessed(limit);
+
+    return events.map((event) => ({
+      memoryId: event.id,
+      summary: event.content.substring(0, 200) + (event.content.length > 200 ? '...' : ''),
+      topics: this.extractTopicsFromContent(event.content),
+      accessCount: event.access_count || 0,
+      lastAccessed: event.last_accessed_at || null,
+      confidence: 1.0,
+      createdAt: event.timestamp,
+    }));
+  }
+
+  async evaluateSessionHelpfulness(sessionId: string): Promise<void> {
+    await this.deps.initialize();
+    await this.deps.retrievalStore.evaluateSessionHelpfulness(sessionId);
+  }
+
+  async evaluatePendingSessions(currentSessionId: string, limit: number = 5): Promise<void> {
+    await this.deps.initialize();
+    const sessions = await this.deps.retrievalStore.getUnevaluatedSessions(currentSessionId, limit);
+
+    for (const sessionId of sessions) {
+      try {
+        await this.deps.retrievalStore.evaluateSessionHelpfulness(sessionId);
+      } catch {
+        // Best-effort backfill: one broken session should not block hook startup.
+      }
+    }
+  }
+
+  async getHelpfulMemories(limit: number = 10): Promise<HelpfulMemory[]> {
+    await this.deps.initialize();
+    return this.deps.retrievalStore.getHelpfulMemories(limit);
+  }
+
+  async getHelpfulnessStats(): Promise<HelpfulnessStats> {
+    await this.deps.initialize();
+    return this.deps.retrievalStore.getHelpfulnessStats();
+  }
+
+  /**
+   * Extract topic keywords from event content (markdown headings and key terms).
+   */
+  private extractTopicsFromContent(content: string): string[] {
+    const topics: Set<string> = new Set();
+
+    const headings = content.match(/^#{1,3}\s+(.+)$/gm);
+    if (headings) {
+      for (const heading of headings.slice(0, 5)) {
+        const text = heading.replace(/^#+\s+/, '').replace(/[*_`#]/g, '').trim();
+        if (text.length > 2 && text.length < 50) {
+          topics.add(text);
+        }
+      }
+    }
+
+    const boldTerms = content.match(/\*\*([^*]+)\*\*/g);
+    if (boldTerms) {
+      for (const boldTerm of boldTerms.slice(0, 5)) {
+        const text = boldTerm.replace(/\*\*/g, '').trim();
+        if (text.length > 2 && text.length < 30) {
+          topics.add(text);
+        }
+      }
+    }
+
+    return Array.from(topics).slice(0, 5);
+  }
+}
+
+export function createRetrievalAnalyticsService(
+  deps: RetrievalAnalyticsServiceDeps
+): RetrievalAnalyticsService {
+  return new RetrievalAnalyticsService(deps);
+}