claude-memory-layer 1.0.10 → 1.0.12

package/src/core/consolidation-worker.ts

@@ -8,7 +8,8 @@ import type {
   EndlessModeConfig,
   MemoryEvent,
   EventGroup,
-  WorkingSet
+  WorkingSet,
+  ConsolidationCostQualityReport
 } from './types.js';
 import { WorkingSetStore } from './working-set-store.js';
 import { ConsolidatedStore } from './consolidated-store.js';
@@ -62,7 +63,19 @@ export class ConsolidationWorker {
    * Force a consolidation run (manual trigger)
    */
   async forceRun(): Promise<number> {
-    return await this.consolidate();
+    const out = await this.consolidateWithReport();
+    return out.consolidatedCount;
+  }
+
+  /**
+   * Force a consolidation run and return metrics report
+   */
+  async forceRunWithReport(): Promise<{
+    consolidatedCount: number;
+    promotedRuleCount: number;
+    report: ConsolidationCostQualityReport;
+  }> {
+    return this.consolidateWithReport();
   }
 
   /**
@@ -109,15 +122,29 @@ export class ConsolidationWorker {
    * Perform consolidation
    */
   private async consolidate(): Promise<number> {
+    const out = await this.consolidateWithReport();
+    return out.consolidatedCount;
+  }
+
+  private async consolidateWithReport(): Promise<{
+    consolidatedCount: number;
+    promotedRuleCount: number;
+    report: ConsolidationCostQualityReport;
+  }> {
     const workingSet = await this.workingSetStore.get();
 
     if (workingSet.recentEvents.length < 3) {
-      return 0; // Not enough events to consolidate
+      return {
+        consolidatedCount: 0,
+        promotedRuleCount: 0,
+        report: this.buildCostQualityReport(workingSet.recentEvents, [], 0)
+      };
     }
 
     // Group events by topic
     const groups = this.groupByTopic(workingSet.recentEvents);
     let consolidatedCount = 0;
+    const createdMemoryIds: string[] = [];
 
     for (const group of groups) {
       // Require minimum 3 events per group
@@ -132,16 +159,18 @@ export class ConsolidationWorker {
       const summary = await this.summarize(group);
 
       // Create consolidated memory
-      await this.consolidatedStore.create({
+      const memoryId = await this.consolidatedStore.create({
         summary,
         topics: group.topics,
         sourceEvents: eventIds,
         confidence: this.calculateConfidence(group)
       });
-
+      createdMemoryIds.push(memoryId);
       consolidatedCount++;
     }
 
+    const promotedRuleCount = await this.promoteStableSummariesToRules(createdMemoryIds);
+
     // Prune consolidated events from working set
     if (consolidatedCount > 0) {
       const consolidatedEventIds = groups
@@ -161,7 +190,87 @@ export class ConsolidationWorker {
       }
     }
 
-    return consolidatedCount;
+    const report = this.buildCostQualityReport(workingSet.recentEvents, groups, consolidatedCount);
+    return { consolidatedCount, promotedRuleCount, report };
+  }
+
+  private async promoteStableSummariesToRules(memoryIds: string[]): Promise<number> {
+    let promoted = 0;
+
+    for (const memoryId of memoryIds) {
+      const memory = await this.consolidatedStore.get(memoryId);
+      if (!memory) continue;
+      if (memory.confidence < 0.55) continue;
+      if (memory.sourceEvents.length < 4) continue;
+
+      const exists = await this.consolidatedStore.hasRuleForSourceMemory(memoryId);
+      if (exists) continue;
+
+      const rule = this.buildRuleFromSummary(memory.summary, memory.topics);
+      if (!rule) continue;
+
+      await this.consolidatedStore.createRule({
+        rule,
+        topics: memory.topics,
+        sourceMemoryIds: [memory.memoryId],
+        sourceEvents: memory.sourceEvents,
+        confidence: Math.min(1, memory.confidence + 0.08)
+      });
+      promoted++;
+    }
+
+    return promoted;
+  }
+
+  private buildRuleFromSummary(summary: string, topics: string[]): string | null {
+    const lines = summary
+      .split(/\r?\n/)
+      .map((l) => l.trim())
+      .filter(Boolean)
+      .filter((l) => !l.toLowerCase().startsWith('topics:'));
+
+    const bullet = lines.find((l) => l.startsWith('- '))?.replace(/^-\s*/, '');
+    const seed = bullet || lines[0];
+    if (!seed || seed.length < 8) return null;
+
+    const topicPrefix = topics.length > 0 ? `[${topics.slice(0, 2).join(', ')}] ` : '';
+    return `${topicPrefix}${seed}`;
+  }
+
+  private buildCostQualityReport(
+    events: MemoryEvent[],
+    groups: EventGroup[],
+    consolidatedCount: number
+  ): ConsolidationCostQualityReport {
+    const beforeTokenEstimate = events.reduce((acc, e) => acc + this.estimateTokens(e.content), 0);
+
+    const afterSummaries = groups
+      .filter((g) => g.events.length >= 3)
+      .slice(0, Math.max(consolidatedCount, 1));
+
+    const afterTokenEstimate = afterSummaries.length > 0
+      ? afterSummaries.reduce((acc, g) => acc + this.estimateTokens(this.ruleBasedSummary(g)), 0)
+      : beforeTokenEstimate;
+
+    const reductionRatio = beforeTokenEstimate > 0
+      ? Math.max(0, (beforeTokenEstimate - afterTokenEstimate) / beforeTokenEstimate)
+      : 0;
+
+    const qualityGuardPassed = consolidatedCount === 0
+      ? true
+      : groups.filter((g) => g.events.length >= 3).every((g) => this.calculateConfidence(g) >= 0.55);
+
+    return {
+      beforeTokenEstimate,
+      afterTokenEstimate,
+      reductionRatio,
+      qualityGuardPassed,
+      details: `groups=${groups.length}, consolidated=${consolidatedCount}`
+    };
+  }
+
+  private estimateTokens(text: string): number {
+    return Math.ceil((text || '').length / 4);
   }
 
   /**

package/src/core/event-store.ts

@@ -280,6 +280,19 @@ export class EventStore {
       )
     `);
 
+    // Consolidated Rules table (long-term stable memory)
+    await dbRun(this.db, `
+      CREATE TABLE IF NOT EXISTS consolidated_rules (
+        rule_id VARCHAR PRIMARY KEY,
+        rule TEXT NOT NULL,
+        topics JSON,
+        source_memory_ids JSON,
+        source_events JSON,
+        confidence FLOAT DEFAULT 0.5,
+        created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
+      )
+    `);
+
     // Endless Mode Config table
     await dbRun(this.db, `
       CREATE TABLE IF NOT EXISTS endless_config (
@@ -314,6 +327,7 @@ export class EventStore {
     await dbRun(this.db, `CREATE INDEX IF NOT EXISTS idx_working_set_expires ON working_set(expires_at)`);
     await dbRun(this.db, `CREATE INDEX IF NOT EXISTS idx_working_set_relevance ON working_set(relevance_score DESC)`);
     await dbRun(this.db, `CREATE INDEX IF NOT EXISTS idx_consolidated_confidence ON consolidated_memories(confidence DESC)`);
+    await dbRun(this.db, `CREATE INDEX IF NOT EXISTS idx_consolidated_rules_confidence ON consolidated_rules(confidence DESC)`);
     await dbRun(this.db, `CREATE INDEX IF NOT EXISTS idx_continuity_created ON continuity_log(created_at)`);
 
     this.initialized = true;

package/src/core/index.ts

@@ -14,6 +14,7 @@ export * from './event-store.js';
 export * from './sqlite-wrapper.js';
 export * from './sqlite-event-store.js';
 export * from './sync-worker.js';
+export * from './mongo-sync-worker.js';
 export * from './entity-repo.js';
 export * from './edge-repo.js';
 

package/src/core/ingest-interceptor.ts

@@ -0,0 +1,80 @@
+import type { MemoryEventInput } from './types.js';
+
+export type IngestStage = 'before' | 'after' | 'error';
+
+export interface IngestContext {
+  stage: IngestStage;
+  operation: 'user_prompt' | 'agent_response' | 'session_summary' | 'tool_observation';
+  sessionId: string;
+  event: MemoryEventInput;
+  error?: Error;
+}
+
+export type IngestInterceptor = (context: IngestContext) => Promise<void> | void;
+
+export class IngestInterceptorRegistry {
+  private before: IngestInterceptor[] = [];
+  private after: IngestInterceptor[] = [];
+  private onError: IngestInterceptor[] = [];
+
+  registerBefore(interceptor: IngestInterceptor): () => void {
+    this.before.push(interceptor);
+    return () => {
+      this.before = this.before.filter((i) => i !== interceptor);
+    };
+  }
+
+  registerAfter(interceptor: IngestInterceptor): () => void {
+    this.after.push(interceptor);
+    return () => {
+      this.after = this.after.filter((i) => i !== interceptor);
+    };
+  }
+
+  registerOnError(interceptor: IngestInterceptor): () => void {
+    this.onError.push(interceptor);
+    return () => {
+      this.onError = this.onError.filter((i) => i !== interceptor);
+    };
+  }
+
+  async run(stage: IngestStage, context: Omit<IngestContext, 'stage'>): Promise<void> {
+    const interceptors = stage === 'before'
+      ? this.before
+      : stage === 'after'
+      ? this.after
+      : this.onError;
+
+    for (const interceptor of interceptors) {
+      await interceptor({ ...context, stage });
+    }
+  }
+}
+
+export function mergeHierarchicalMetadata(
+  base: Record<string, unknown> | undefined,
+  patch: Record<string, unknown> | undefined
+): Record<string, unknown> | undefined {
+  if (!base && !patch) return undefined;
+  if (!base) return patch;
+  if (!patch) return base;
+
+  const result: Record<string, unknown> = { ...base };
+
+  for (const [key, value] of Object.entries(patch)) {
+    const current = result[key];
+    if (
+      typeof current === 'object' && current !== null && !Array.isArray(current) &&
+      typeof value === 'object' && value !== null && !Array.isArray(value)
+    ) {
+      result[key] = mergeHierarchicalMetadata(
+        current as Record<string, unknown>,
+        value as Record<string, unknown>
+      );
+    } else {
+      result[key] = value;
+    }
+  }
+
+  return result;
+}

package/src/core/markdown-mirror.ts

@@ -0,0 +1,70 @@
+import * as fs from 'fs/promises';
+import * as path from 'path';
+import type { MemoryEvent } from './types.js';
+
+const DEFAULT_NAMESPACE = 'default';
+const DEFAULT_CATEGORY = 'uncategorized';
+
+export function sanitizeSegment(input: unknown, fallback: string): string {
+  const raw = String(input ?? '').trim().toLowerCase();
+  const safe = raw
+    .normalize('NFKD')
+    .replace(/[^a-z0-9_-]+/g, '-')
+    .replace(/^-+|-+$/g, '');
+
+  if (!safe || safe === '.' || safe === '..') return fallback;
+  return safe;
+}
+
+function getCategorySegments(metadata: Record<string, unknown> | undefined, eventType: string): string[] {
+  const raw = metadata?.categoryPath;
+  if (Array.isArray(raw) && raw.length > 0) {
+    return raw.map((s) => sanitizeSegment(s, DEFAULT_CATEGORY));
+  }
+  const single = metadata?.category;
+  if (typeof single === 'string' && single.trim()) {
+    return [sanitizeSegment(single, DEFAULT_CATEGORY)];
+  }
+  return [sanitizeSegment(eventType, DEFAULT_CATEGORY)];
+}
+
+export function buildMirrorPath(rootDir: string, event: MemoryEvent): string {
+  const metadata = event.metadata as Record<string, unknown> | undefined;
+  const namespace = sanitizeSegment(metadata?.namespace, DEFAULT_NAMESPACE);
+  const categories = getCategorySegments(metadata, event.eventType);
+
+  const d = event.timestamp;
+  const yyyy = d.getFullYear();
+  const mm = String(d.getMonth() + 1).padStart(2, '0');
+  const dd = String(d.getDate()).padStart(2, '0');
+
+  return path.join(rootDir, 'memory', namespace, ...categories, `${yyyy}-${mm}-${dd}.md`);
+}
+
+export function formatMirrorEntry(event: MemoryEvent): string {
+  const category = Array.isArray((event.metadata as any)?.categoryPath)
+    ? ((event.metadata as any).categoryPath as unknown[]).join('/')
+    : String((event.metadata as any)?.category ?? event.eventType);
+
+  return [
+    '',
+    `- ts: ${event.timestamp.toISOString()}`,
+    `  id: ${event.id}`,
+    `  type: ${event.eventType}`,
+    `  session: ${event.sessionId}`,
+    `  category: ${category}`,
+    '  content: |',
+    ...event.content.split('\n').map((line) => `    ${line}`)
+  ].join('\n') + '\n';
+}
+
+export class MarkdownMirror {
+  constructor(private readonly rootDir: string) {}
+
+  async append(event: MemoryEvent): Promise<string> {
+    const outPath = buildMirrorPath(this.rootDir, event);
+    await fs.mkdir(path.dirname(outPath), { recursive: true });
+    await fs.appendFile(outPath, formatMirrorEntry(event), 'utf8');
+    return outPath;
+  }
+}

package/src/core/md-mirror.ts

@@ -0,0 +1,92 @@
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+import type { MemoryEventInput } from './types.js';
+
+function sanitizeSegment(input: string | undefined, fallback: string): string {
+  const v = (input || '').trim().toLowerCase().replace(/[^a-z0-9._-]+/g, '-').replace(/^-+|-+$/g, '');
+  return v || fallback;
+}
+
+function getAtPath(obj: Record<string, unknown> | undefined, dotted: string): unknown {
+  if (!obj) return undefined;
+  return dotted.split('.').reduce<unknown>((acc, key) => {
+    if (!acc || typeof acc !== 'object') return undefined;
+    return (acc as Record<string, unknown>)[key];
+  }, obj);
+}
+
+export function buildMirrorPath(rootDir: string, event: MemoryEventInput): string {
+  const meta = event.metadata as Record<string, unknown> | undefined;
+
+  const namespaceRaw = getAtPath(meta, 'namespace') ?? getAtPath(meta, 'scope.namespace') ?? event.eventType;
+  const namespace = sanitizeSegment(typeof namespaceRaw === 'string' ? namespaceRaw : undefined, 'general');
+
+  const categoryRaw = getAtPath(meta, 'categoryPath') ?? getAtPath(meta, 'scope.categoryPath');
+  const categoryPath = Array.isArray(categoryRaw) && categoryRaw.length > 0
+    ? categoryRaw.map((x) => sanitizeSegment(typeof x === 'string' ? x : undefined, 'uncategorized'))
+    : ['uncategorized'];
+
+  const d = event.timestamp;
+  const yyyy = d.getFullYear();
+  const mm = String(d.getMonth() + 1).padStart(2, '0');
+  const dd = String(d.getDate()).padStart(2, '0');
+
+  return path.join(rootDir, 'memory', namespace, ...categoryPath, `${yyyy}-${mm}-${dd}.md`);
+}
+
+export class MarkdownMirror {
+  constructor(private readonly rootDir: string) {}
+
+  async append(event: MemoryEventInput, eventId?: string): Promise<void> {
+    const out = buildMirrorPath(this.rootDir, event);
+    fs.mkdirSync(path.dirname(out), { recursive: true });
+
+    const lines = [
+      '',
+      `## ${event.timestamp.toISOString()} | ${eventId ?? 'pending-id'}`,
+      `- type: ${event.eventType}`,
+      `- session: ${event.sessionId}`,
+      event.content,
+    ];
+
+    await fs.promises.appendFile(out, lines.join('\n'), 'utf8');
+    await this.refreshIndex();
+  }
+
+  private async refreshIndex(): Promise<void> {
+    const memoryRoot = path.join(this.rootDir, 'memory');
+    await fs.promises.mkdir(memoryRoot, { recursive: true });
+
+    const files: string[] = [];
+    await this.walk(memoryRoot, files);
+
+    const mdFiles = files
+      .filter((f) => f.endsWith('.md'))
+      .map((f) => path.relative(this.rootDir, f))
+      .filter((rel) => rel !== path.join('memory', '_index.md'))
+      .sort();
+
+    const index = [
+      '# Memory Index',
+      '',
+      'Generated automatically by MarkdownMirror.',
+      '',
+      ...mdFiles.map((rel) => `- ${rel}`),
+      '',
+    ].join('\n');
+
+    await fs.promises.writeFile(path.join(memoryRoot, '_index.md'), index, 'utf8');
+  }
+
+  private async walk(dir: string, out: string[]): Promise<void> {
+    const entries = await fs.promises.readdir(dir, { withFileTypes: true });
+    for (const e of entries) {
+      const full = path.join(dir, e.name);
+      if (e.isDirectory()) {
+        await this.walk(full, out);
+      } else {
+        out.push(full);
+      }
+    }
+  }
+}

package/src/core/mongo-sync-config.ts

@@ -0,0 +1,165 @@
+/**
+ * Mongo Sync Config
+ * Persistent per-project config for enabling auto-sync from hooks (e.g., SessionEnd).
+ *
+ * Stored as JSON under the project's storagePath next to events.sqlite.
+ * Note: This may include credentials in plaintext (MongoDB URI). Treat accordingly.
+ */
+
+import * as fs from 'fs';
+import * as path from 'path';
+
+import type { MongoSyncDirection } from './mongo-sync-worker.js';
+
+export type MongoSyncConfig = {
+  version: 1;
+  enabled: boolean;
+  /** MongoDB connection URI (may include credentials). */
+  uri: string;
+  /** MongoDB database name. */
+  dbName: string;
+  /** Remote project key (shared across machines). */
+  projectKey: string;
+  /** push|pull|both */
+  direction: MongoSyncDirection;
+  /** Batch size for push/pull loops. */
+  batchSize: number;
+  /** If true, hooks will run a sync at SessionEnd. */
+  autoSyncOnSessionEnd: boolean;
+  createdAt: string;
+  updatedAt: string;
+};
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return typeof value === 'object' && value !== null;
+}
+
+function asNonEmptyString(value: unknown): string | null {
+  if (typeof value !== 'string') return null;
+  const s = value.trim();
+  return s.length > 0 ? s : null;
+}
+
+function asPositiveInt(value: unknown): number | null {
+  if (typeof value === 'number' && Number.isFinite(value)) {
+    const n = Math.trunc(value);
+    return n > 0 ? n : null;
+  }
+  if (typeof value === 'string') {
+    const n = parseInt(value, 10);
+    return Number.isFinite(n) && n > 0 ? n : null;
+  }
+  return null;
+}
+
+function normalizeDirection(value: unknown): MongoSyncDirection | null {
+  if (typeof value !== 'string') return null;
+  const v = value.toLowerCase();
+  if (v === 'push' || v === 'pull' || v === 'both') return v;
+  return null;
+}
+
+export function getMongoSyncConfigPath(storagePath: string): string {
+  return path.join(storagePath, 'mongo-sync.json');
+}
+
+export function readMongoSyncConfig(storagePath: string): MongoSyncConfig | null {
+  const configPath = getMongoSyncConfigPath(storagePath);
+  if (!fs.existsSync(configPath)) return null;
+
+  try {
+    const raw = JSON.parse(fs.readFileSync(configPath, 'utf8')) as unknown;
+    if (!isRecord(raw)) return null;
+
+    const version = raw.version;
+    if (version !== 1) return null;
+
+    const enabled = raw.enabled;
+    if (typeof enabled !== 'boolean') return null;
+
+    const uri = asNonEmptyString(raw.uri);
+    const dbName = asNonEmptyString(raw.dbName);
+    const projectKey = asNonEmptyString(raw.projectKey);
+    const direction = normalizeDirection(raw.direction);
+    const batchSize = asPositiveInt(raw.batchSize) ?? 500;
+    const autoSyncOnSessionEnd = typeof raw.autoSyncOnSessionEnd === 'boolean' ? raw.autoSyncOnSessionEnd : true;
+    const createdAt = asNonEmptyString(raw.createdAt) ?? new Date(0).toISOString();
+    const updatedAt = asNonEmptyString(raw.updatedAt) ?? new Date(0).toISOString();
+
+    if (!uri || !dbName || !projectKey || !direction) return null;
+
+    return {
+      version: 1,
+      enabled,
+      uri,
+      dbName,
+      projectKey,
+      direction,
+      batchSize,
+      autoSyncOnSessionEnd,
+      createdAt,
+      updatedAt
+    };
+  } catch {
+    return null;
+  }
+}
+
+export function writeMongoSyncConfig(storagePath: string, config: Omit<MongoSyncConfig, 'version' | 'createdAt' | 'updatedAt'> & {
+  createdAt?: string;
+  updatedAt?: string;
+}): MongoSyncConfig {
+  if (!fs.existsSync(storagePath)) {
+    fs.mkdirSync(storagePath, { recursive: true });
+  }
+
+  const now = new Date().toISOString();
+  const existing = readMongoSyncConfig(storagePath);
+
+  const normalized: MongoSyncConfig = {
+    version: 1,
+    enabled: config.enabled,
+    uri: config.uri,
+    dbName: config.dbName,
+    projectKey: config.projectKey,
+    direction: config.direction,
+    batchSize: config.batchSize,
+    autoSyncOnSessionEnd: config.autoSyncOnSessionEnd,
+    createdAt: config.createdAt ?? existing?.createdAt ?? now,
+    updatedAt: config.updatedAt ?? now
+  };
+
+  const configPath = getMongoSyncConfigPath(storagePath);
+  const tmpPath = `${configPath}.tmp`;
+
+  // 0600: contains secrets (Mongo URI may embed credentials)
+  fs.writeFileSync(tmpPath, JSON.stringify(normalized, null, 2), { mode: 0o600 });
+  fs.renameSync(tmpPath, configPath);
+
+  return normalized;
+}
+
+export function removeMongoSyncConfig(storagePath: string): boolean {
+  const configPath = getMongoSyncConfigPath(storagePath);
+  if (!fs.existsSync(configPath)) return false;
+  fs.unlinkSync(configPath);
+  return true;
+}
+
+export function redactMongoUri(uri: string): string {
+  // mongodb://user:pass@host:port/  -> mongodb://user:***@host:port/
+  // mongodb+srv://user:pass@host/  -> mongodb+srv://user:***@host/
+  const schemeIdx = uri.indexOf('://');
+  if (schemeIdx === -1) return uri;
+  const atIdx = uri.indexOf('@', schemeIdx + 3);
+  if (atIdx === -1) return uri;
+
+  const creds = uri.slice(schemeIdx + 3, atIdx); // user:pass
+  const colonIdx = creds.indexOf(':');
+  if (colonIdx === -1) return uri;
+
+  const prefix = uri.slice(0, schemeIdx + 3 + colonIdx + 1);
+  const suffix = uri.slice(atIdx);
+  return `${prefix}***${suffix}`;
+}
+