claude-memory-layer 1.0.27 → 1.0.28

package/src/core/engine/memory-engine-services.ts

@@ -0,0 +1,170 @@
+/**
+ * Memory Engine Services Bundle
+ *
+ * Owns construction and wiring for storage-backed engine services so
+ * MemoryService can stay focused on public facade/lifecycle behavior.
+ */
+
+import * as fs from 'fs';
+import * as path from 'path';
+
+import { Embedder, getDefaultEmbedder } from '../embedder.js';
+import type { EventStore } from '../event-store.js';
+import { createGraduationPipeline, type GraduationPipeline } from '../graduation.js';
+import { getDefaultMatcher, type Matcher } from '../matcher.js';
+import { MarkdownMirror } from '../md-mirror.js';
+import type { Retriever } from '../retriever.js';
+import { SQLiteEventStore } from '../sqlite-event-store.js';
+import type { ToolObservationPayload } from '../types.js';
+import { VectorStore } from '../vector-store.js';
+import { MemoryIngestService } from './memory-ingest-service.js';
+import { MemoryQueryService } from './memory-query-service.js';
+import {
+  createRetrievalServices,
+  type RetrievalAnalyticsService,
+  type RetrievalDisclosureService,
+  type RetrievalDisclosureSharedStore,
+  type RetrievalEventStore,
+  type RetrievalOrchestrator,
+  type RetrievalServices,
+  type RetrievalServicesDeps
+} from './retrieval-services.js';
+
+export interface MemoryEngineServicesOptions {
+  storagePath: string;
+  readOnly: boolean;
+  embeddingModel?: string;
+  cwd?: string;
+  initialize: () => Promise<void>;
+  getProjectHash: () => string | null;
+  getProjectPath?: () => string | null;
+  hasSharedStore: () => boolean;
+  sharedStore?: RetrievalDisclosureSharedStore;
+  createToolObservationEmbedding: (payload: ToolObservationPayload) => string;
+  factories?: MemoryEngineServicesFactories;
+}
+
+export interface MemoryEngineServicesFactories {
+  createSQLiteEventStore?: (
+    dbPath: string,
+    options: { readonly: boolean; markdownMirrorRoot: string }
+  ) => SQLiteEventStore;
+  createVectorStore?: (vectorsPath: string) => VectorStore;
+  createEmbedder?: (model: string) => Embedder;
+  getDefaultEmbedder?: () => Embedder;
+  getDefaultMatcher?: () => Matcher;
+  createMarkdownMirror?: (cwd: string) => MarkdownMirror;
+  createGraduationPipeline?: (eventStore: EventStore) => GraduationPipeline;
+  createRetrievalServices?: (deps: RetrievalServicesDeps) => RetrievalServices;
+}
+
+export interface MemoryEngineServices {
+  storagePath: string;
+  sqliteStore: SQLiteEventStore;
+  vectorStore: VectorStore;
+  embedder: Embedder;
+  matcher: Matcher;
+  retriever: Retriever;
+  retrievalOrchestrator: RetrievalOrchestrator;
+  retrievalDisclosureService: RetrievalDisclosureService;
+  retrievalAnalyticsService: RetrievalAnalyticsService;
+  graduation: GraduationPipeline;
+  mdMirror: MarkdownMirror;
+  ingestService: MemoryIngestService;
+  queryService: MemoryQueryService;
+}
+
+export function createMemoryEngineServices(options: MemoryEngineServicesOptions): MemoryEngineServices {
+  const factories = options.factories ?? {};
+  const storagePath = options.storagePath;
+
+  if (!options.readOnly && !fs.existsSync(storagePath)) {
+    fs.mkdirSync(storagePath, { recursive: true });
+  }
+
+  const sqliteStore = (factories.createSQLiteEventStore ?? defaultCreateSQLiteEventStore)(
+    path.join(storagePath, 'events.sqlite'),
+    {
+      readonly: options.readOnly,
+      markdownMirrorRoot: storagePath
+    }
+  );
+  const vectorStore = (factories.createVectorStore ?? defaultCreateVectorStore)(
+    path.join(storagePath, 'vectors')
+  );
+  const embeddingModel = options.embeddingModel || process.env.CLAUDE_MEMORY_EMBEDDING_MODEL;
+  const embedder = embeddingModel
+    ? (factories.createEmbedder ?? defaultCreateEmbedder)(embeddingModel)
+    : (factories.getDefaultEmbedder ?? getDefaultEmbedder)();
+  const matcher = (factories.getDefaultMatcher ?? getDefaultMatcher)();
+  const mdMirror = (factories.createMarkdownMirror ?? defaultCreateMarkdownMirror)(
+    options.cwd ?? process.cwd()
+  );
+  const graduation = (factories.createGraduationPipeline ?? defaultCreateGraduationPipeline)(
+    sqliteStore as unknown as EventStore
+  );
+
+  const retrievalServices = (factories.createRetrievalServices ?? createRetrievalServices)({
+    initialize: options.initialize,
+    eventStore: sqliteStore as unknown as RetrievalEventStore,
+    vectorStore,
+    embedder,
+    matcher,
+    getProjectHash: options.getProjectHash,
+    hasSharedStore: options.hasSharedStore,
+    sharedStore: options.sharedStore
+  });
+
+  const ingestService = new MemoryIngestService({
+    initialize: options.initialize,
+    eventStore: sqliteStore,
+    markdownMirror: mdMirror,
+    createToolEmbedding: options.createToolObservationEmbedding,
+    getProjectHash: options.getProjectHash,
+    getProjectPath: options.getProjectPath
+  });
+  const queryService = new MemoryQueryService(
+    () => sqliteStore.initialize(),
+    sqliteStore,
+    { vectorStore, graduation }
+  );
+
+  return {
+    storagePath,
+    sqliteStore,
+    vectorStore,
+    embedder,
+    matcher,
+    retriever: retrievalServices.retriever,
+    retrievalOrchestrator: retrievalServices.retrievalOrchestrator,
+    retrievalDisclosureService: retrievalServices.retrievalDisclosureService,
+    retrievalAnalyticsService: retrievalServices.retrievalAnalyticsService,
+    graduation,
+    mdMirror,
+    ingestService,
+    queryService
+  };
+}
+
+function defaultCreateSQLiteEventStore(
+  dbPath: string,
+  options: { readonly: boolean; markdownMirrorRoot: string }
+): SQLiteEventStore {
+  return new SQLiteEventStore(dbPath, options);
+}
+
+function defaultCreateVectorStore(vectorsPath: string): VectorStore {
+  return new VectorStore(vectorsPath);
+}
+
+function defaultCreateEmbedder(model: string): Embedder {
+  return new Embedder(model);
+}
+
+function defaultCreateMarkdownMirror(cwd: string): MarkdownMirror {
+  return new MarkdownMirror(cwd);
+}
+
+function defaultCreateGraduationPipeline(eventStore: EventStore): GraduationPipeline {
+  return createGraduationPipeline(eventStore);
+}

package/src/core/engine/memory-ingest-service.ts

@@ -0,0 +1,317 @@
+import {
+  IngestInterceptor,
+  IngestInterceptorRegistry,
+  mergeHierarchicalMetadata
+} from '../ingest-interceptor.js';
+import { normalizeTags } from '../tag-taxonomy.js';
+import { createSummaryDeriver, type SummaryDeriver } from '../derive/summary-deriver.js';
+import type { AppendResult, MemoryEvent, MemoryEventInput, ToolObservationPayload } from '../types.js';
+
+interface SessionRecord {
+  id: string;
+  startedAt?: Date;
+  endedAt?: Date;
+  projectPath?: string;
+  summary?: string;
+}
+
+interface SessionUpsertStore {
+  upsertSession(session: SessionRecord): Promise<void>;
+  getSessionEvents(sessionId: string): Promise<MemoryEvent[]>;
+  getSessionsWithoutSummary(currentSessionId: string, limit?: number): Promise<string[]>;
+}
+
+interface IngestEventStore extends SessionUpsertStore {
+  append(input: MemoryEventInput): Promise<AppendResult>;
+  enqueueForEmbedding(eventId: string, content: string): Promise<unknown>;
+}
+
+interface IngestMarkdownMirror {
+  append(event: MemoryEventInput, eventId?: string): Promise<void>;
+}
+
+export type IngestOperation = 'user_prompt' | 'agent_response' | 'session_summary' | 'tool_observation';
+
+export interface MemoryIngestServiceOptions {
+  initialize: () => Promise<void>;
+  eventStore: IngestEventStore;
+  markdownMirror: IngestMarkdownMirror;
+  createToolEmbedding: (payload: ToolObservationPayload) => string;
+  getProjectHash?: () => string | null;
+  getProjectPath?: () => string | null;
+  summaryDeriver?: SummaryDeriver;
+}
+
+/**
+ * Thin-core ingest service for session lifecycle and event writes.
+ *
+ * Owns the ingest normalization/interceptor/append pipeline so the public
+ * MemoryService facade can delegate ingest behavior without coordinating
+ * storage-side effects itself.
+ */
+export class MemoryIngestService {
+  private readonly initialize: () => Promise<void>;
+  private readonly eventStore: IngestEventStore;
+  private readonly markdownMirror: IngestMarkdownMirror;
+  private readonly createToolEmbedding: (payload: ToolObservationPayload) => string;
+  private readonly getProjectHash: () => string | null;
+  private readonly getProjectPath: () => string | null;
+  private readonly summaryDeriver: SummaryDeriver;
+  private readonly ingestInterceptors = new IngestInterceptorRegistry();
+
+  constructor(options: MemoryIngestServiceOptions) {
+    this.initialize = options.initialize;
+    this.eventStore = options.eventStore;
+    this.markdownMirror = options.markdownMirror;
+    this.createToolEmbedding = options.createToolEmbedding;
+    this.getProjectHash = options.getProjectHash ?? (() => null);
+    this.getProjectPath = options.getProjectPath ?? (() => null);
+    this.summaryDeriver = options.summaryDeriver ?? createSummaryDeriver();
+  }
+
+  registerIngestBefore(interceptor: IngestInterceptor): () => void {
+    return this.ingestInterceptors.registerBefore(interceptor);
+  }
+
+  registerIngestAfter(interceptor: IngestInterceptor): () => void {
+    return this.ingestInterceptors.registerAfter(interceptor);
+  }
+
+  registerIngestOnError(interceptor: IngestInterceptor): () => void {
+    return this.ingestInterceptors.registerOnError(interceptor);
+  }
+
+  async startSession(sessionId: string, projectPath?: string): Promise<void> {
+    await this.initialize();
+
+    await this.eventStore.upsertSession({
+      id: sessionId,
+      startedAt: new Date(),
+      projectPath
+    });
+  }
+
+  async endSession(sessionId: string, summary?: string): Promise<void> {
+    await this.initialize();
+
+    await this.eventStore.upsertSession({
+      id: sessionId,
+      endedAt: new Date(),
+      summary
+    });
+  }
+
+  async storeUserPrompt(
+    sessionId: string,
+    content: string,
+    metadata?: Record<string, unknown>
+  ): Promise<AppendResult> {
+    return this.ingestEvent({
+      operation: 'user_prompt',
+      input: {
+        eventType: 'user_prompt',
+        sessionId,
+        timestamp: new Date(),
+        content,
+        metadata
+      },
+      embeddingContent: content
+    });
+  }
+
+  async storeAgentResponse(
+    sessionId: string,
+    content: string,
+    metadata?: Record<string, unknown>
+  ): Promise<AppendResult> {
+    return this.ingestEvent({
+      operation: 'agent_response',
+      input: {
+        eventType: 'agent_response',
+        sessionId,
+        timestamp: new Date(),
+        content,
+        metadata
+      },
+      embeddingContent: content
+    });
+  }
+
+  async storeSessionSummary(
+    sessionId: string,
+    summary: string,
+    metadata?: Record<string, unknown>
+  ): Promise<AppendResult> {
+    return this.ingestEvent({
+      operation: 'session_summary',
+      input: {
+        eventType: 'session_summary',
+        sessionId,
+        timestamp: new Date(),
+        content: summary,
+        metadata
+      },
+      embeddingContent: summary
+    });
+  }
+
+  /**
+   * Backfill session summaries for recent sessions that are missing them.
+   * Called from session-start hook to catch sessions that ended without Stop hook.
+   */
+  async backfillMissingSummaries(currentSessionId: string, limit = 5): Promise<void> {
+    await this.initialize();
+
+    const recentSessionIds = await this.eventStore.getSessionsWithoutSummary(currentSessionId, limit);
+    for (const sessionId of recentSessionIds) {
+      try {
+        await this.generateSessionSummary(sessionId);
+      } catch {
+        // non-critical backfill path
+      }
+    }
+  }
+
+  /**
+   * Generate a rule-based session summary from stored events.
+   * Skips short sessions and sessions that already contain a summary event.
+   */
+  async generateSessionSummary(sessionId: string): Promise<void> {
+    await this.initialize();
+
+    const events = await this.eventStore.getSessionEvents(sessionId);
+    const summary = this.summaryDeriver.deriveSessionSummary(events);
+    if (!summary) return;
+
+    await this.storeSessionSummary(sessionId, summary.text, summary.metadata);
+  }
+
+  async storeToolObservation(
+    sessionId: string,
+    payload: ToolObservationPayload
+  ): Promise<AppendResult> {
+    const content = JSON.stringify(payload);
+    const turnId = (payload.metadata as Record<string, unknown> | undefined)?.turnId;
+
+    return this.ingestEvent({
+      operation: 'tool_observation',
+      input: {
+        eventType: 'tool_observation',
+        sessionId,
+        timestamp: new Date(),
+        content,
+        metadata: {
+          toolName: payload.toolName,
+          success: payload.success,
+          ...(typeof turnId === 'string' && turnId.length > 0 ? { turnId } : {})
+        }
+      },
+      embeddingContent: this.createToolEmbedding(payload)
+    });
+  }
+
+  private async ingestEvent(options: {
+    operation: IngestOperation;
+    input: MemoryEventInput;
+    embeddingContent?: string;
+  }): Promise<AppendResult> {
+    const normalizedInput = this.normalizeInput(options.operation, options.input);
+
+    await this.ingestInterceptors.run('before', {
+      operation: options.operation,
+      sessionId: normalizedInput.sessionId,
+      event: normalizedInput
+    });
+
+    try {
+      const result = await this.eventStore.append(normalizedInput);
+      if (result.success === false) {
+        await this.ingestInterceptors.run('error', {
+          operation: options.operation,
+          sessionId: normalizedInput.sessionId,
+          event: normalizedInput,
+          error: new Error(result.error)
+        });
+        return result;
+      }
+
+      if (!result.isDuplicate) {
+        if (options.embeddingContent) {
+          await this.eventStore.enqueueForEmbedding(result.eventId, options.embeddingContent);
+        }
+        try {
+          await this.markdownMirror.append(normalizedInput, result.eventId);
+        } catch {
+          // non-breaking markdown mirror write
+        }
+      }
+
+      await this.ingestInterceptors.run('after', {
+        operation: options.operation,
+        sessionId: normalizedInput.sessionId,
+        event: normalizedInput
+      });
+
+      return result;
+    } catch (error) {
+      const normalizedError = error instanceof Error ? error : new Error(String(error));
+      await this.ingestInterceptors.run('error', {
+        operation: options.operation,
+        sessionId: normalizedInput.sessionId,
+        event: normalizedInput,
+        error: normalizedError
+      });
+      throw error;
+    }
+  }
+
+  private normalizeInput(operation: IngestOperation, input: MemoryEventInput): MemoryEventInput {
+    const projectHash = this.getProjectHash();
+    const projectPath = this.getProjectPath();
+    const normalizedInput: MemoryEventInput = {
+      ...input,
+      metadata: mergeHierarchicalMetadata(
+        {
+          ingest: {
+            operation,
+            pipeline: 'default',
+            ts: new Date().toISOString()
+          },
+          ...(projectHash
+            ? {
+                scope: {
+                  project: {
+                    hash: projectHash,
+                    ...(projectPath ? { path: projectPath } : {})
+                  }
+                },
+                tags: [`proj:${projectHash}`]
+              }
+            : {})
+        },
+        input.metadata
+      )
+    };
+
+    if (projectHash && normalizedInput.metadata) {
+      const meta = normalizedInput.metadata as Record<string, unknown>;
+      const currentTags = Array.isArray(meta.tags)
+        ? meta.tags.filter((x): x is string => typeof x === 'string')
+        : [];
+      const projectTag = `proj:${projectHash}`;
+      if (!currentTags.includes(projectTag)) {
+        meta.tags = [...currentTags, projectTag];
+      }
+    }
+
+    if (normalizedInput.metadata) {
+      const meta = normalizedInput.metadata as Record<string, unknown>;
+      const normalizedTags = normalizeTags(meta.tags);
+      if (normalizedTags.length > 0) {
+        meta.tags = normalizedTags;
+      }
+    }
+
+    return normalizedInput;
+  }
+}

package/src/core/engine/memory-query-service.ts

@@ -0,0 +1,173 @@
+import type { MemoryEvent } from '../types.js';
+
+interface RankedKeywordResult {
+  event: MemoryEvent;
+  rank: number;
+}
+
+export interface MemorySessionTurn {
+  turnId: string;
+  events: MemoryEvent[];
+  startedAt: Date;
+  promptPreview: string;
+  eventCount: number;
+  toolCount: number;
+  hasResponse: boolean;
+}
+
+export interface MemoryOutboxStats {
+  embedding: { pending: number; processing: number; failed: number; total: number };
+  vector: { pending: number; processing: number; failed: number; total: number };
+}
+
+export interface MemoryStats {
+  totalEvents: number;
+  vectorCount: number;
+  levelStats: Array<{ level: string; count: number }>;
+}
+
+interface QueryStore {
+  keywordSearch(query: string, topK: number): Promise<RankedKeywordResult[]>;
+  getSessionEvents(sessionId: string): Promise<MemoryEvent[]>;
+  getRecentEvents(limit: number): Promise<MemoryEvent[]>;
+}
+
+interface QueryMaintenanceStore extends QueryStore {
+  rebuildFtsIndex(): Promise<number>;
+  getOutboxStats(): Promise<MemoryOutboxStats>;
+  getEventsByLevel(level: string, options?: { limit?: number; offset?: number }): Promise<MemoryEvent[]>;
+  getEventLevel(eventId: string): Promise<string | null>;
+  getSessionTurns(sessionId: string, options?: { limit?: number; offset?: number }): Promise<MemorySessionTurn[]>;
+  getEventsByTurn(turnId: string): Promise<MemoryEvent[]>;
+  countSessionTurns(sessionId: string): Promise<number>;
+  backfillTurnIds(): Promise<number>;
+  deleteSessionEvents(sessionId: string): Promise<number>;
+}
+
+interface MemoryQueryServiceDeps {
+  vectorStore: { count(): Promise<number> };
+  graduation: { getStats(): Promise<Array<{ level: string; count: number }>> };
+}
+
+/**
+ * Thin-core query service for lightweight read and maintenance paths.
+ *
+ * Higher-level retrieval orchestration lives in RetrievalOrchestrator;
+ * this service keeps storage-backed read models and maintenance delegates separate.
+ */
+export class MemoryQueryService {
+  constructor(
+    private readonly initialize: () => Promise<void>,
+    private readonly queryStore: QueryStore,
+    private readonly deps?: MemoryQueryServiceDeps
+  ) {}
+
+  async keywordSearch(
+    query: string,
+    options?: { topK?: number; minScore?: number }
+  ): Promise<Array<{ event: MemoryEvent; score: number }>> {
+    await this.initialize();
+
+    const results = await this.queryStore.keywordSearch(query, options?.topK ?? 10);
+    if (results.length === 0) return [];
+
+    const maxRank = Math.min(...results.map((r) => r.rank), -0.001);
+    const minRank = Math.max(...results.map((r) => r.rank), -1000);
+    const rankRange = maxRank - minRank || 1;
+
+    return results
+      .map((r) => ({
+        event: r.event,
+        score: 1 - (r.rank - minRank) / rankRange
+      }))
+      .filter((r) => !options?.minScore || r.score >= options.minScore);
+  }
+
+  async getSessionHistory(sessionId: string): Promise<MemoryEvent[]> {
+    await this.initialize();
+    return this.queryStore.getSessionEvents(sessionId);
+  }
+
+  async getRecentEvents(limit: number = 100): Promise<MemoryEvent[]> {
+    await this.initialize();
+    return this.queryStore.getRecentEvents(limit);
+  }
+
+  async rebuildFtsIndex(): Promise<number> {
+    await this.initialize();
+    return this.getMaintenanceStore('rebuildFtsIndex').rebuildFtsIndex();
+  }
+
+  async getOutboxStats(): Promise<MemoryOutboxStats> {
+    await this.initialize();
+    return this.getMaintenanceStore('getOutboxStats').getOutboxStats();
+  }
+
+  async getStats(): Promise<MemoryStats> {
+    await this.initialize();
+
+    const deps = this.getStatsDeps();
+    const recentEvents = await this.queryStore.getRecentEvents(10000);
+    const vectorCount = await deps.vectorStore.count();
+    const levelStats = await deps.graduation.getStats();
+
+    return {
+      totalEvents: recentEvents.length,
+      vectorCount,
+      levelStats
+    };
+  }
+
+  async getEventsByLevel(level: string, options?: { limit?: number; offset?: number }): Promise<MemoryEvent[]> {
+    await this.initialize();
+    return this.getMaintenanceStore('getEventsByLevel').getEventsByLevel(level, options);
+  }
+
+  async getEventLevel(eventId: string): Promise<string | null> {
+    await this.initialize();
+    return this.getMaintenanceStore('getEventLevel').getEventLevel(eventId);
+  }
+
+  async getSessionTurns(
+    sessionId: string,
+    options?: { limit?: number; offset?: number }
+  ): Promise<MemorySessionTurn[]> {
+    await this.initialize();
+    return this.getMaintenanceStore('getSessionTurns').getSessionTurns(sessionId, options);
+  }
+
+  async getEventsByTurn(turnId: string): Promise<MemoryEvent[]> {
+    await this.initialize();
+    return this.getMaintenanceStore('getEventsByTurn').getEventsByTurn(turnId);
+  }
+
+  async countSessionTurns(sessionId: string): Promise<number> {
+    await this.initialize();
+    return this.getMaintenanceStore('countSessionTurns').countSessionTurns(sessionId);
+  }
+
+  async backfillTurnIds(): Promise<number> {
+    await this.initialize();
+    return this.getMaintenanceStore('backfillTurnIds').backfillTurnIds();
+  }
+
+  async deleteSessionEvents(sessionId: string): Promise<number> {
+    await this.initialize();
+    return this.getMaintenanceStore('deleteSessionEvents').deleteSessionEvents(sessionId);
+  }
+
+  private getMaintenanceStore(method: keyof QueryMaintenanceStore): QueryMaintenanceStore {
+    const store = this.queryStore as QueryStore & Partial<QueryMaintenanceStore>;
+    if (typeof store[method] !== 'function') {
+      throw new Error(`MemoryQueryService requires queryStore.${String(method)}() for this operation`);
+    }
+    return store as QueryMaintenanceStore;
+  }
+
+  private getStatsDeps(): MemoryQueryServiceDeps {
+    if (!this.deps) {
+      throw new Error('MemoryQueryService requires vectorStore and graduation dependencies for getStats()');
+    }
+    return this.deps;
+  }
+}