claude-memory-layer 1.0.31 → 1.0.32

package/src/core/engine/embedding-maintenance-service.ts

@@ -1,187 +0,0 @@
-import * as fs from 'fs';
-import * as path from 'path';
-
-export interface EmbeddingMaintenanceEvent {
-  id: string;
-  content: string;
-}
-
-export interface EmbeddingMaintenanceEventStore {
-  clearEmbeddingOutbox(): Promise<void>;
-  getEventsPage(limit: number, offset: number): Promise<EmbeddingMaintenanceEvent[]>;
-  enqueueForEmbedding(eventId: string, content: string): Promise<void>;
-}
-
-export interface EmbeddingMaintenanceVectorStore {
-  count(): Promise<number>;
-  clearAll(): Promise<void>;
-}
-
-export interface EmbeddingMaintenanceVectorWorker {
-  isRunning(): boolean;
-  stop(): void;
-  start(): void;
-}
-
-export interface EmbeddingMaintenanceFileSystem {
-  existsSync(targetPath: string): boolean;
-  readFileSync(targetPath: string, encoding: BufferEncoding): string;
-  writeFileSync(targetPath: string, content: string): void;
-}
-
-export interface EmbeddingModelMaintenanceOptions {
-  autoMigrate?: boolean;
-}
-
-export interface EmbeddingModelMaintenanceResult {
-  changed: boolean;
-  previousModel: string | null;
-  currentModel: string;
-  enqueued: number;
-  reason?: string;
-}
-
-export interface EmbeddingMaintenanceServiceOptions {
-  storagePath: string;
-  initialize: () => Promise<void>;
-  getEmbeddingModelName: () => string;
-  vectorStore: EmbeddingMaintenanceVectorStore;
-  eventStore: EmbeddingMaintenanceEventStore;
-  getVectorWorker: () => EmbeddingMaintenanceVectorWorker | null;
-  fileSystem?: EmbeddingMaintenanceFileSystem;
-}
-
-export interface EmbeddingMaintenanceService {
-  getEmbeddingModelName(): string;
-  ensureEmbeddingModelForImport(options?: EmbeddingModelMaintenanceOptions): Promise<EmbeddingModelMaintenanceResult>;
-}
-
-const DEFAULT_PAGE_SIZE = 1000;
-
-const defaultFileSystem: EmbeddingMaintenanceFileSystem = {
-  existsSync: fs.existsSync,
-  readFileSync: (targetPath, encoding) => fs.readFileSync(targetPath, encoding),
-  writeFileSync: (targetPath, content) => fs.writeFileSync(targetPath, content)
-};
-
-class DefaultEmbeddingMaintenanceService implements EmbeddingMaintenanceService {
-  private readonly fileSystem: EmbeddingMaintenanceFileSystem;
-
-  constructor(private readonly options: EmbeddingMaintenanceServiceOptions) {
-    this.fileSystem = options.fileSystem ?? defaultFileSystem;
-  }
-
-  getEmbeddingModelName(): string {
-    return this.options.getEmbeddingModelName();
-  }
-
-  async ensureEmbeddingModelForImport(
-    options?: EmbeddingModelMaintenanceOptions
-  ): Promise<EmbeddingModelMaintenanceResult> {
-    await this.options.initialize();
-
-    const currentModel = this.getEmbeddingModelName();
-    const metaPath = path.join(this.options.storagePath, 'embedding-meta.json');
-    const previousModel = this.readPreviousModel(metaPath);
-    const vectorCount = await this.options.vectorStore.count();
-    const hasExistingVectors = vectorCount > 0;
-
-    // First-time metadata write (no migration needed unless legacy vectors exist)
-    if (!previousModel && !hasExistingVectors) {
-      this.fileSystem.writeFileSync(
-        metaPath,
-        JSON.stringify({ model: currentModel, updatedAt: new Date().toISOString() }, null, 2)
-      );
-      return { changed: false, previousModel: null, currentModel, enqueued: 0, reason: 'initialized-meta' };
-    }
-
-    const modelChanged = previousModel !== currentModel;
-    const legacyUnknownButVectorsExist = !previousModel && hasExistingVectors;
-
-    if (!modelChanged && !legacyUnknownButVectorsExist) {
-      return { changed: false, previousModel, currentModel, enqueued: 0 };
-    }
-
-    if (options?.autoMigrate === false) {
-      return {
-        changed: true,
-        previousModel,
-        currentModel,
-        enqueued: 0,
-        reason: legacyUnknownButVectorsExist ? 'legacy-vectors-without-meta' : 'model-mismatch'
-      };
-    }
-
-    const worker = this.options.getVectorWorker();
-    const wasRunning = worker?.isRunning() || false;
-    if (wasRunning) worker?.stop();
-
-    await this.options.vectorStore.clearAll();
-    await this.options.eventStore.clearEmbeddingOutbox();
-
-    const enqueued = await this.reenqueueAllEvents();
-
-    this.fileSystem.writeFileSync(
-      metaPath,
-      JSON.stringify(
-        {
-          model: currentModel,
-          previousModel,
-          migratedAt: new Date().toISOString(),
-          enqueued
-        },
-        null,
-        2
-      )
-    );
-
-    if (wasRunning) worker?.start();
-
-    return {
-      changed: true,
-      previousModel,
-      currentModel,
-      enqueued,
-      reason: legacyUnknownButVectorsExist ? 'legacy-vectors-without-meta' : 'model-mismatch'
-    };
-  }
-
-  private readPreviousModel(metaPath: string): string | null {
-    try {
-      if (this.fileSystem.existsSync(metaPath)) {
-        const parsed = JSON.parse(this.fileSystem.readFileSync(metaPath, 'utf-8')) as { model?: string };
-        return parsed?.model || null;
-      }
-    } catch {
-      return null;
-    }
-
-    return null;
-  }
-
-  private async reenqueueAllEvents(): Promise<number> {
-    let offset = 0;
-    let enqueued = 0;
-
-    while (true) {
-      const page = await this.options.eventStore.getEventsPage(DEFAULT_PAGE_SIZE, offset);
-      if (page.length === 0) break;
-
-      for (const event of page) {
-        await this.options.eventStore.enqueueForEmbedding(event.id, event.content);
-        enqueued += 1;
-      }
-
-      offset += page.length;
-      if (page.length < DEFAULT_PAGE_SIZE) break;
-    }
-
-    return enqueued;
-  }
-}
-
-export function createEmbeddingMaintenanceService(
-  options: EmbeddingMaintenanceServiceOptions
-): EmbeddingMaintenanceService {
-  return new DefaultEmbeddingMaintenanceService(options);
-}

package/src/core/engine/endless-memory-services.ts

@@ -1,4 +0,0 @@
-// Compatibility re-export. Endless memory is an optional extension, but existing
-// engine/facade imports still resolve through this path during the strangler
-// migration.
-export * from '../../extensions/endless-memory/index.js';

package/src/core/engine/index.ts

@@ -1,19 +0,0 @@
-export * from './memory-ingest-service.js';
-export * from './memory-query-service.js';
-export * from './memory-engine-services.js';
-export * from './retrieval-orchestrator.js';
-export * from './retrieval-disclosure-service.js';
-export * from './retrieval-analytics-service.js';
-export * from './shared-memory-services.js';
-export * from './endless-memory-services.js';
-export * from './embedding-maintenance-service.js';
-export * from './memory-runtime-service.js';
-export {
-  createRetrievalServices
-} from './retrieval-services.js';
-export type {
-  CreateRetrieverFn,
-  RetrievalEventStore,
-  RetrievalServices,
-  RetrievalServicesDeps
-} from './retrieval-services.js';

package/src/core/engine/memory-engine-services.ts

@@ -1,170 +0,0 @@
-/**
- * Memory Engine Services Bundle
- *
- * Owns construction and wiring for storage-backed engine services so
- * MemoryService can stay focused on public facade/lifecycle behavior.
- */
-
-import * as fs from 'fs';
-import * as path from 'path';
-
-import { Embedder, getDefaultEmbedder } from '../embedder.js';
-import type { EventStore } from '../event-store.js';
-import { createGraduationPipeline, type GraduationPipeline } from '../graduation.js';
-import { getDefaultMatcher, type Matcher } from '../matcher.js';
-import { MarkdownMirror } from '../md-mirror.js';
-import type { Retriever } from '../retriever.js';
-import { SQLiteEventStore } from '../sqlite-event-store.js';
-import type { ToolObservationPayload } from '../types.js';
-import { VectorStore } from '../vector-store.js';
-import { MemoryIngestService } from './memory-ingest-service.js';
-import { MemoryQueryService } from './memory-query-service.js';
-import {
-  createRetrievalServices,
-  type RetrievalAnalyticsService,
-  type RetrievalDisclosureService,
-  type RetrievalDisclosureSharedStore,
-  type RetrievalEventStore,
-  type RetrievalOrchestrator,
-  type RetrievalServices,
-  type RetrievalServicesDeps
-} from './retrieval-services.js';
-
-export interface MemoryEngineServicesOptions {
-  storagePath: string;
-  readOnly: boolean;
-  embeddingModel?: string;
-  cwd?: string;
-  initialize: () => Promise<void>;
-  getProjectHash: () => string | null;
-  getProjectPath?: () => string | null;
-  hasSharedStore: () => boolean;
-  sharedStore?: RetrievalDisclosureSharedStore;
-  createToolObservationEmbedding: (payload: ToolObservationPayload) => string;
-  factories?: MemoryEngineServicesFactories;
-}
-
-export interface MemoryEngineServicesFactories {
-  createSQLiteEventStore?: (
-    dbPath: string,
-    options: { readonly: boolean; markdownMirrorRoot: string }
-  ) => SQLiteEventStore;
-  createVectorStore?: (vectorsPath: string) => VectorStore;
-  createEmbedder?: (model: string) => Embedder;
-  getDefaultEmbedder?: () => Embedder;
-  getDefaultMatcher?: () => Matcher;
-  createMarkdownMirror?: (cwd: string) => MarkdownMirror;
-  createGraduationPipeline?: (eventStore: EventStore) => GraduationPipeline;
-  createRetrievalServices?: (deps: RetrievalServicesDeps) => RetrievalServices;
-}
-
-export interface MemoryEngineServices {
-  storagePath: string;
-  sqliteStore: SQLiteEventStore;
-  vectorStore: VectorStore;
-  embedder: Embedder;
-  matcher: Matcher;
-  retriever: Retriever;
-  retrievalOrchestrator: RetrievalOrchestrator;
-  retrievalDisclosureService: RetrievalDisclosureService;
-  retrievalAnalyticsService: RetrievalAnalyticsService;
-  graduation: GraduationPipeline;
-  mdMirror: MarkdownMirror;
-  ingestService: MemoryIngestService;
-  queryService: MemoryQueryService;
-}
-
-export function createMemoryEngineServices(options: MemoryEngineServicesOptions): MemoryEngineServices {
-  const factories = options.factories ?? {};
-  const storagePath = options.storagePath;
-
-  if (!options.readOnly && !fs.existsSync(storagePath)) {
-    fs.mkdirSync(storagePath, { recursive: true });
-  }
-
-  const sqliteStore = (factories.createSQLiteEventStore ?? defaultCreateSQLiteEventStore)(
-    path.join(storagePath, 'events.sqlite'),
-    {
-      readonly: options.readOnly,
-      markdownMirrorRoot: storagePath
-    }
-  );
-  const vectorStore = (factories.createVectorStore ?? defaultCreateVectorStore)(
-    path.join(storagePath, 'vectors')
-  );
-  const embeddingModel = options.embeddingModel || process.env.CLAUDE_MEMORY_EMBEDDING_MODEL;
-  const embedder = embeddingModel
-    ? (factories.createEmbedder ?? defaultCreateEmbedder)(embeddingModel)
-    : (factories.getDefaultEmbedder ?? getDefaultEmbedder)();
-  const matcher = (factories.getDefaultMatcher ?? getDefaultMatcher)();
-  const mdMirror = (factories.createMarkdownMirror ?? defaultCreateMarkdownMirror)(
-    options.cwd ?? process.cwd()
-  );
-  const graduation = (factories.createGraduationPipeline ?? defaultCreateGraduationPipeline)(
-    sqliteStore as unknown as EventStore
-  );
-
-  const retrievalServices = (factories.createRetrievalServices ?? createRetrievalServices)({
-    initialize: options.initialize,
-    eventStore: sqliteStore as unknown as RetrievalEventStore,
-    vectorStore,
-    embedder,
-    matcher,
-    getProjectHash: options.getProjectHash,
-    hasSharedStore: options.hasSharedStore,
-    sharedStore: options.sharedStore
-  });
-
-  const ingestService = new MemoryIngestService({
-    initialize: options.initialize,
-    eventStore: sqliteStore,
-    markdownMirror: mdMirror,
-    createToolEmbedding: options.createToolObservationEmbedding,
-    getProjectHash: options.getProjectHash,
-    getProjectPath: options.getProjectPath
-  });
-  const queryService = new MemoryQueryService(
-    () => sqliteStore.initialize(),
-    sqliteStore,
-    { vectorStore, graduation }
-  );
-
-  return {
-    storagePath,
-    sqliteStore,
-    vectorStore,
-    embedder,
-    matcher,
-    retriever: retrievalServices.retriever,
-    retrievalOrchestrator: retrievalServices.retrievalOrchestrator,
-    retrievalDisclosureService: retrievalServices.retrievalDisclosureService,
-    retrievalAnalyticsService: retrievalServices.retrievalAnalyticsService,
-    graduation,
-    mdMirror,
-    ingestService,
-    queryService
-  };
-}
-
-function defaultCreateSQLiteEventStore(
-  dbPath: string,
-  options: { readonly: boolean; markdownMirrorRoot: string }
-): SQLiteEventStore {
-  return new SQLiteEventStore(dbPath, options);
-}
-
-function defaultCreateVectorStore(vectorsPath: string): VectorStore {
-  return new VectorStore(vectorsPath);
-}
-
-function defaultCreateEmbedder(model: string): Embedder {
-  return new Embedder(model);
-}
-
-function defaultCreateMarkdownMirror(cwd: string): MarkdownMirror {
-  return new MarkdownMirror(cwd);
-}
-
-function defaultCreateGraduationPipeline(eventStore: EventStore): GraduationPipeline {
-  return createGraduationPipeline(eventStore);
-}

package/src/core/engine/memory-ingest-service.ts

@@ -1,317 +0,0 @@
-import {
-  IngestInterceptor,
-  IngestInterceptorRegistry,
-  mergeHierarchicalMetadata
-} from '../ingest-interceptor.js';
-import { normalizeTags } from '../tag-taxonomy.js';
-import { createSummaryDeriver, type SummaryDeriver } from '../derive/summary-deriver.js';
-import type { AppendResult, MemoryEvent, MemoryEventInput, ToolObservationPayload } from '../types.js';
-
-interface SessionRecord {
-  id: string;
-  startedAt?: Date;
-  endedAt?: Date;
-  projectPath?: string;
-  summary?: string;
-}
-
-interface SessionUpsertStore {
-  upsertSession(session: SessionRecord): Promise<void>;
-  getSessionEvents(sessionId: string): Promise<MemoryEvent[]>;
-  getSessionsWithoutSummary(currentSessionId: string, limit?: number): Promise<string[]>;
-}
-
-interface IngestEventStore extends SessionUpsertStore {
-  append(input: MemoryEventInput): Promise<AppendResult>;
-  enqueueForEmbedding(eventId: string, content: string): Promise<unknown>;
-}
-
-interface IngestMarkdownMirror {
-  append(event: MemoryEventInput, eventId?: string): Promise<void>;
-}
-
-export type IngestOperation = 'user_prompt' | 'agent_response' | 'session_summary' | 'tool_observation';
-
-export interface MemoryIngestServiceOptions {
-  initialize: () => Promise<void>;
-  eventStore: IngestEventStore;
-  markdownMirror: IngestMarkdownMirror;
-  createToolEmbedding: (payload: ToolObservationPayload) => string;
-  getProjectHash?: () => string | null;
-  getProjectPath?: () => string | null;
-  summaryDeriver?: SummaryDeriver;
-}
-
-/**
- * Thin-core ingest service for session lifecycle and event writes.
- *
- * Owns the ingest normalization/interceptor/append pipeline so the public
- * MemoryService facade can delegate ingest behavior without coordinating
- * storage-side effects itself.
- */
-export class MemoryIngestService {
-  private readonly initialize: () => Promise<void>;
-  private readonly eventStore: IngestEventStore;
-  private readonly markdownMirror: IngestMarkdownMirror;
-  private readonly createToolEmbedding: (payload: ToolObservationPayload) => string;
-  private readonly getProjectHash: () => string | null;
-  private readonly getProjectPath: () => string | null;
-  private readonly summaryDeriver: SummaryDeriver;
-  private readonly ingestInterceptors = new IngestInterceptorRegistry();
-
-  constructor(options: MemoryIngestServiceOptions) {
-    this.initialize = options.initialize;
-    this.eventStore = options.eventStore;
-    this.markdownMirror = options.markdownMirror;
-    this.createToolEmbedding = options.createToolEmbedding;
-    this.getProjectHash = options.getProjectHash ?? (() => null);
-    this.getProjectPath = options.getProjectPath ?? (() => null);
-    this.summaryDeriver = options.summaryDeriver ?? createSummaryDeriver();
-  }
-
-  registerIngestBefore(interceptor: IngestInterceptor): () => void {
-    return this.ingestInterceptors.registerBefore(interceptor);
-  }
-
-  registerIngestAfter(interceptor: IngestInterceptor): () => void {
-    return this.ingestInterceptors.registerAfter(interceptor);
-  }
-
-  registerIngestOnError(interceptor: IngestInterceptor): () => void {
-    return this.ingestInterceptors.registerOnError(interceptor);
-  }
-
-  async startSession(sessionId: string, projectPath?: string): Promise<void> {
-    await this.initialize();
-
-    await this.eventStore.upsertSession({
-      id: sessionId,
-      startedAt: new Date(),
-      projectPath
-    });
-  }
-
-  async endSession(sessionId: string, summary?: string): Promise<void> {
-    await this.initialize();
-
-    await this.eventStore.upsertSession({
-      id: sessionId,
-      endedAt: new Date(),
-      summary
-    });
-  }
-
-  async storeUserPrompt(
-    sessionId: string,
-    content: string,
-    metadata?: Record<string, unknown>
-  ): Promise<AppendResult> {
-    return this.ingestEvent({
-      operation: 'user_prompt',
-      input: {
-        eventType: 'user_prompt',
-        sessionId,
-        timestamp: new Date(),
-        content,
-        metadata
-      },
-      embeddingContent: content
-    });
-  }
-
-  async storeAgentResponse(
-    sessionId: string,
-    content: string,
-    metadata?: Record<string, unknown>
-  ): Promise<AppendResult> {
-    return this.ingestEvent({
-      operation: 'agent_response',
-      input: {
-        eventType: 'agent_response',
-        sessionId,
-        timestamp: new Date(),
-        content,
-        metadata
-      },
-      embeddingContent: content
-    });
-  }
-
-  async storeSessionSummary(
-    sessionId: string,
-    summary: string,
-    metadata?: Record<string, unknown>
-  ): Promise<AppendResult> {
-    return this.ingestEvent({
-      operation: 'session_summary',
-      input: {
-        eventType: 'session_summary',
-        sessionId,
-        timestamp: new Date(),
-        content: summary,
-        metadata
-      },
-      embeddingContent: summary
-    });
-  }
-
-  /**
-   * Backfill session summaries for recent sessions that are missing them.
-   * Called from session-start hook to catch sessions that ended without Stop hook.
-   */
-  async backfillMissingSummaries(currentSessionId: string, limit = 5): Promise<void> {
-    await this.initialize();
-
-    const recentSessionIds = await this.eventStore.getSessionsWithoutSummary(currentSessionId, limit);
-    for (const sessionId of recentSessionIds) {
-      try {
-        await this.generateSessionSummary(sessionId);
-      } catch {
-        // non-critical backfill path
-      }
-    }
-  }
-
-  /**
-   * Generate a rule-based session summary from stored events.
-   * Skips short sessions and sessions that already contain a summary event.
-   */
-  async generateSessionSummary(sessionId: string): Promise<void> {
-    await this.initialize();
-
-    const events = await this.eventStore.getSessionEvents(sessionId);
-    const summary = this.summaryDeriver.deriveSessionSummary(events);
-    if (!summary) return;
-
-    await this.storeSessionSummary(sessionId, summary.text, summary.metadata);
-  }
-
-  async storeToolObservation(
-    sessionId: string,
-    payload: ToolObservationPayload
-  ): Promise<AppendResult> {
-    const content = JSON.stringify(payload);
-    const turnId = (payload.metadata as Record<string, unknown> | undefined)?.turnId;
-
-    return this.ingestEvent({
-      operation: 'tool_observation',
-      input: {
-        eventType: 'tool_observation',
-        sessionId,
-        timestamp: new Date(),
-        content,
-        metadata: {
-          toolName: payload.toolName,
-          success: payload.success,
-          ...(typeof turnId === 'string' && turnId.length > 0 ? { turnId } : {})
-        }
-      },
-      embeddingContent: this.createToolEmbedding(payload)
-    });
-  }
-
-  private async ingestEvent(options: {
-    operation: IngestOperation;
-    input: MemoryEventInput;
-    embeddingContent?: string;
-  }): Promise<AppendResult> {
-    const normalizedInput = this.normalizeInput(options.operation, options.input);
-
-    await this.ingestInterceptors.run('before', {
-      operation: options.operation,
-      sessionId: normalizedInput.sessionId,
-      event: normalizedInput
-    });
-
-    try {
-      const result = await this.eventStore.append(normalizedInput);
-      if (result.success === false) {
-        await this.ingestInterceptors.run('error', {
-          operation: options.operation,
-          sessionId: normalizedInput.sessionId,
-          event: normalizedInput,
-          error: new Error(result.error)
-        });
-        return result;
-      }
-
-      if (!result.isDuplicate) {
-        if (options.embeddingContent) {
-          await this.eventStore.enqueueForEmbedding(result.eventId, options.embeddingContent);
-        }
-        try {
-          await this.markdownMirror.append(normalizedInput, result.eventId);
-        } catch {
-          // non-breaking markdown mirror write
-        }
-      }
-
-      await this.ingestInterceptors.run('after', {
-        operation: options.operation,
-        sessionId: normalizedInput.sessionId,
-        event: normalizedInput
-      });
-
-      return result;
-    } catch (error) {
-      const normalizedError = error instanceof Error ? error : new Error(String(error));
-      await this.ingestInterceptors.run('error', {
-        operation: options.operation,
-        sessionId: normalizedInput.sessionId,
-        event: normalizedInput,
-        error: normalizedError
-      });
-      throw error;
-    }
-  }
-
-  private normalizeInput(operation: IngestOperation, input: MemoryEventInput): MemoryEventInput {
-    const projectHash = this.getProjectHash();
-    const projectPath = this.getProjectPath();
-    const normalizedInput: MemoryEventInput = {
-      ...input,
-      metadata: mergeHierarchicalMetadata(
-        {
-          ingest: {
-            operation,
-            pipeline: 'default',
-            ts: new Date().toISOString()
-          },
-          ...(projectHash
-            ? {
-                scope: {
-                  project: {
-                    hash: projectHash,
-                    ...(projectPath ? { path: projectPath } : {})
-                  }
-                },
-                tags: [`proj:${projectHash}`]
-              }
-            : {})
-        },
-        input.metadata
-      )
-    };
-
-    if (projectHash && normalizedInput.metadata) {
-      const meta = normalizedInput.metadata as Record<string, unknown>;
-      const currentTags = Array.isArray(meta.tags)
-        ? meta.tags.filter((x): x is string => typeof x === 'string')
-        : [];
-      const projectTag = `proj:${projectHash}`;
-      if (!currentTags.includes(projectTag)) {
-        meta.tags = [...currentTags, projectTag];
-      }
-    }
-
-    if (normalizedInput.metadata) {
-      const meta = normalizedInput.metadata as Record<string, unknown>;
-      const normalizedTags = normalizeTags(meta.tags);
-      if (normalizedTags.length > 0) {
-        meta.tags = normalizedTags;
-      }
-    }
-
-    return normalizedInput;
-  }
-}