claude-memory-layer 1.0.31 → 1.0.32

package/src/core/mongo-sync-config.ts

@@ -1,165 +0,0 @@
-/**
- * Mongo Sync Config
- * Persistent per-project config for enabling auto-sync from hooks (e.g., SessionEnd).
- *
- * Stored as JSON under the project's storagePath next to events.sqlite.
- * Note: This may include credentials in plaintext (MongoDB URI). Treat accordingly.
- */
-
-import * as fs from 'fs';
-import * as path from 'path';
-
-import type { MongoSyncDirection } from './mongo-sync-worker.js';
-
-export type MongoSyncConfig = {
-  version: 1;
-  enabled: boolean;
-  /** MongoDB connection URI (may include credentials). */
-  uri: string;
-  /** MongoDB database name. */
-  dbName: string;
-  /** Remote project key (shared across machines). */
-  projectKey: string;
-  /** push|pull|both */
-  direction: MongoSyncDirection;
-  /** Batch size for push/pull loops. */
-  batchSize: number;
-  /** If true, hooks will run a sync at SessionEnd. */
-  autoSyncOnSessionEnd: boolean;
-  createdAt: string;
-  updatedAt: string;
-};
-
-function isRecord(value: unknown): value is Record<string, unknown> {
-  return typeof value === 'object' && value !== null;
-}
-
-function asNonEmptyString(value: unknown): string | null {
-  if (typeof value !== 'string') return null;
-  const s = value.trim();
-  return s.length > 0 ? s : null;
-}
-
-function asPositiveInt(value: unknown): number | null {
-  if (typeof value === 'number' && Number.isFinite(value)) {
-    const n = Math.trunc(value);
-    return n > 0 ? n : null;
-  }
-  if (typeof value === 'string') {
-    const n = parseInt(value, 10);
-    return Number.isFinite(n) && n > 0 ? n : null;
-  }
-  return null;
-}
-
-function normalizeDirection(value: unknown): MongoSyncDirection | null {
-  if (typeof value !== 'string') return null;
-  const v = value.toLowerCase();
-  if (v === 'push' || v === 'pull' || v === 'both') return v;
-  return null;
-}
-
-export function getMongoSyncConfigPath(storagePath: string): string {
-  return path.join(storagePath, 'mongo-sync.json');
-}
-
-export function readMongoSyncConfig(storagePath: string): MongoSyncConfig | null {
-  const configPath = getMongoSyncConfigPath(storagePath);
-  if (!fs.existsSync(configPath)) return null;
-
-  try {
-    const raw = JSON.parse(fs.readFileSync(configPath, 'utf8')) as unknown;
-    if (!isRecord(raw)) return null;
-
-    const version = raw.version;
-    if (version !== 1) return null;
-
-    const enabled = raw.enabled;
-    if (typeof enabled !== 'boolean') return null;
-
-    const uri = asNonEmptyString(raw.uri);
-    const dbName = asNonEmptyString(raw.dbName);
-    const projectKey = asNonEmptyString(raw.projectKey);
-    const direction = normalizeDirection(raw.direction);
-    const batchSize = asPositiveInt(raw.batchSize) ?? 500;
-    const autoSyncOnSessionEnd = typeof raw.autoSyncOnSessionEnd === 'boolean' ? raw.autoSyncOnSessionEnd : true;
-    const createdAt = asNonEmptyString(raw.createdAt) ?? new Date(0).toISOString();
-    const updatedAt = asNonEmptyString(raw.updatedAt) ?? new Date(0).toISOString();
-
-    if (!uri || !dbName || !projectKey || !direction) return null;
-
-    return {
-      version: 1,
-      enabled,
-      uri,
-      dbName,
-      projectKey,
-      direction,
-      batchSize,
-      autoSyncOnSessionEnd,
-      createdAt,
-      updatedAt
-    };
-  } catch {
-    return null;
-  }
-}
-
-export function writeMongoSyncConfig(storagePath: string, config: Omit<MongoSyncConfig, 'version' | 'createdAt' | 'updatedAt'> & {
-  createdAt?: string;
-  updatedAt?: string;
-}): MongoSyncConfig {
-  if (!fs.existsSync(storagePath)) {
-    fs.mkdirSync(storagePath, { recursive: true });
-  }
-
-  const now = new Date().toISOString();
-  const existing = readMongoSyncConfig(storagePath);
-
-  const normalized: MongoSyncConfig = {
-    version: 1,
-    enabled: config.enabled,
-    uri: config.uri,
-    dbName: config.dbName,
-    projectKey: config.projectKey,
-    direction: config.direction,
-    batchSize: config.batchSize,
-    autoSyncOnSessionEnd: config.autoSyncOnSessionEnd,
-    createdAt: config.createdAt ?? existing?.createdAt ?? now,
-    updatedAt: config.updatedAt ?? now
-  };
-
-  const configPath = getMongoSyncConfigPath(storagePath);
-  const tmpPath = `${configPath}.tmp`;
-
-  // 0600: contains secrets (Mongo URI may embed credentials)
-  fs.writeFileSync(tmpPath, JSON.stringify(normalized, null, 2), { mode: 0o600 });
-  fs.renameSync(tmpPath, configPath);
-
-  return normalized;
-}
-
-export function removeMongoSyncConfig(storagePath: string): boolean {
-  const configPath = getMongoSyncConfigPath(storagePath);
-  if (!fs.existsSync(configPath)) return false;
-  fs.unlinkSync(configPath);
-  return true;
-}
-
-export function redactMongoUri(uri: string): string {
-  // mongodb://user:pass@host:port/  -> mongodb://user:***@host:port/
-  // mongodb+srv://user:pass@host/  -> mongodb+srv://user:***@host/
-  const schemeIdx = uri.indexOf('://');
-  if (schemeIdx === -1) return uri;
-  const atIdx = uri.indexOf('@', schemeIdx + 3);
-  if (atIdx === -1) return uri;
-
-  const creds = uri.slice(schemeIdx + 3, atIdx); // user:pass
-  const colonIdx = creds.indexOf(':');
-  if (colonIdx === -1) return uri;
-
-  const prefix = uri.slice(0, schemeIdx + 3 + colonIdx + 1);
-  const suffix = uri.slice(atIdx);
-  return `${prefix}***${suffix}`;
-}
-

package/src/core/mongo-sync-worker.ts

@@ -1,381 +0,0 @@
-/**
- * Mongo Sync Worker
- * Optional: sync per-project SQLite events to a shared MongoDB database.
- *
- * Design goals:
- * - Optional and decoupled (doesn't affect default local-only flow)
- * - Idempotent (safe to retry)
- * - Incremental push (SQLite rowid) and incremental pull (Mongo seq per project)
- *
- * NOTE:
- * - We only sync immutable L0 events (events table). Derived tables can be rebuilt.
- */
-
-import { randomUUID } from 'crypto';
-import * as os from 'os';
-import { MongoClient } from 'mongodb';
-import type { Collection, Db } from 'mongodb';
-
-import type { MemoryEvent } from './types.js';
-import { SQLiteEventStore } from './sqlite-event-store.js';
-
-export type MongoSyncDirection = 'push' | 'pull' | 'both';
-
-export interface MongoSyncWorkerConfig {
-  uri: string;
-  dbName: string;
-  projectKey: string;
-  direction?: MongoSyncDirection;
-  intervalMs?: number;
-  batchSize?: number;
-  instanceId?: string;
-}
-
-export interface MongoSyncStats {
-  lastSyncAt: Date | null;
-  pushedEvents: number;
-  pulledEvents: number;
-  errors: number;
-  status: 'idle' | 'syncing' | 'error' | 'stopped';
-}
-
-interface CounterDoc {
-  _id: string;
-  seq: number;
-}
-
-interface RemoteEventDoc {
-  _id: string;
-  projectKey: string;
-  seq: number;
-  eventId: string;
-  eventType: string;
-  sessionId: string;
-  timestamp: Date;
-  content: string;
-  canonicalKey: string;
-  dedupeKey: string;
-  metadata?: Record<string, unknown> | null;
-  insertedAt: Date;
-  updatedAt: Date;
-  source?: {
-    hostname?: string;
-    instanceId?: string;
-  };
-}
-
-function redactMongoUri(uri: string): string {
-  // mongodb://user:pass@host:port/  -> mongodb://user:***@host:port/
-  // mongodb+srv://user:pass@host/  -> mongodb+srv://user:***@host/
-  const schemeIdx = uri.indexOf('://');
-  if (schemeIdx === -1) return uri;
-  const atIdx = uri.indexOf('@', schemeIdx + 3);
-  if (atIdx === -1) return uri;
-
-  const creds = uri.slice(schemeIdx + 3, atIdx); // user:pass
-  const colonIdx = creds.indexOf(':');
-  if (colonIdx === -1) return uri;
-
-  const prefix = uri.slice(0, schemeIdx + 3 + colonIdx + 1);
-  const suffix = uri.slice(atIdx);
-  return `${prefix}***${suffix}`;
-}
-
-function parseIntOrZero(value: string | null | undefined): number {
-  if (!value) return 0;
-  const n = parseInt(value, 10);
-  return Number.isFinite(n) ? n : 0;
-}
-
-export class MongoSyncWorker {
-  private readonly config: Required<Omit<MongoSyncWorkerConfig, 'instanceId'>> & { instanceId: string };
-  private intervalHandle: NodeJS.Timeout | null = null;
-  private running = false;
-
-  private client: MongoClient | null = null;
-  private db: Db | null = null;
-  private counters: Collection<CounterDoc> | null = null;
-  private events: Collection<RemoteEventDoc> | null = null;
-  private indexesEnsured = false;
-
-  private stats: MongoSyncStats = {
-    lastSyncAt: null,
-    pushedEvents: 0,
-    pulledEvents: 0,
-    errors: 0,
-    status: 'idle'
-  };
-
-  constructor(
-    private readonly sqliteStore: SQLiteEventStore,
-    config: MongoSyncWorkerConfig
-  ) {
-    this.config = {
-      uri: config.uri,
-      dbName: config.dbName,
-      projectKey: config.projectKey,
-      direction: config.direction ?? 'both',
-      intervalMs: config.intervalMs ?? 30000,
-      batchSize: config.batchSize ?? 500,
-      instanceId: config.instanceId ?? randomUUID()
-    };
-  }
-
-  start(): void {
-    if (this.running) return;
-    this.running = true;
-    this.stats.status = 'idle';
-
-    // Initial sync
-    this.syncNow().catch((err) => {
-      console.error('[MongoSyncWorker] Initial sync failed:', err);
-    });
-
-    // Periodic sync
-    this.intervalHandle = setInterval(() => {
-      this.syncNow().catch((err) => {
-        console.error('[MongoSyncWorker] Periodic sync failed:', err);
-      });
-    }, this.config.intervalMs);
-  }
-
-  stop(): void {
-    this.running = false;
-    this.stats.status = 'stopped';
-
-    if (this.intervalHandle) {
-      clearInterval(this.intervalHandle);
-      this.intervalHandle = null;
-    }
-  }
-
-  async shutdown(): Promise<void> {
-    this.stop();
-    await this.disconnect();
-  }
-
-  getStats(): MongoSyncStats {
-    return { ...this.stats };
-  }
-
-  isRunning(): boolean {
-    return this.running;
-  }
-
-  async syncNow(): Promise<{ pushed: number; pulled: number }> {
-    if (this.stats.status === 'syncing') return { pushed: 0, pulled: 0 };
-
-    this.stats.status = 'syncing';
-    let pushed = 0;
-    let pulled = 0;
-
-    try {
-      await this.sqliteStore.initialize();
-      await this.ensureConnected();
-      await this.ensureIndexes();
-
-      if (this.config.direction === 'push' || this.config.direction === 'both') {
-        pushed = await this.pushEvents();
-        this.stats.pushedEvents += pushed;
-      }
-
-      if (this.config.direction === 'pull' || this.config.direction === 'both') {
-        pulled = await this.pullEvents();
-        this.stats.pulledEvents += pulled;
-      }
-
-      this.stats.lastSyncAt = new Date();
-      this.stats.status = 'idle';
-      return { pushed, pulled };
-    } catch (error) {
-      this.stats.errors++;
-      this.stats.status = 'error';
-      throw error;
-    }
-  }
-
-  private async ensureConnected(): Promise<void> {
-    if (this.client && this.db && this.counters && this.events) return;
-
-    try {
-      this.client = new MongoClient(this.config.uri, {
-        appName: 'claude-memory-layer',
-        serverSelectionTimeoutMS: 5000
-      });
-      await this.client.connect();
-      this.db = this.client.db(this.config.dbName);
-      this.counters = this.db.collection<CounterDoc>('cml_counters');
-      this.events = this.db.collection<RemoteEventDoc>('cml_events');
-    } catch (err) {
-      // Avoid leaking credentials in logs
-      const safeUri = redactMongoUri(this.config.uri);
-      throw new Error(`MongoDB connection failed (${safeUri}, db=${this.config.dbName}): ${String(err)}`);
-    }
-  }
-
-  private async disconnect(): Promise<void> {
-    try {
-      await this.client?.close();
-    } finally {
-      this.client = null;
-      this.db = null;
-      this.counters = null;
-      this.events = null;
-      this.indexesEnsured = false;
-    }
-  }
-
-  private async ensureIndexes(): Promise<void> {
-    if (this.indexesEnsured) return;
-    if (!this.events || !this.counters) throw new Error('Mongo not connected');
-
-    // Best-effort: if the user lacks index privileges, sync can still work (slower)
-    try {
-      await this.events.createIndex({ projectKey: 1, seq: 1 }, { unique: true });
-      await this.events.createIndex({ projectKey: 1, eventId: 1 }, { unique: true });
-      await this.events.createIndex({ projectKey: 1, dedupeKey: 1 });
-    } catch (err) {
-      console.warn('[MongoSyncWorker] Failed to ensure indexes (continuing):', err);
-    }
-
-    this.indexesEnsured = true;
-  }
-
-  private counterKey(kind: 'events'): string {
-    return `${kind}:${this.config.projectKey}`;
-  }
-
-  private async allocateSeqRange(kind: 'events', count: number): Promise<number> {
-    if (!this.counters) throw new Error('Mongo not connected');
-    if (count <= 0) return 1;
-
-    const key = this.counterKey(kind);
-    const doc = await this.counters.findOneAndUpdate(
-      { _id: key },
-      { $inc: { seq: count } },
-      { upsert: true, returnDocument: 'after' }
-    );
-
-    const endSeq = doc?.seq;
-    if (typeof endSeq !== 'number') {
-      throw new Error(`Failed to allocate seq range for ${key}`);
-    }
-
-    return endSeq - count + 1;
-  }
-
-  private pushTargetName(): string {
-    return `mongo_push_events_rowid:${this.config.projectKey}`;
-  }
-
-  private pullTargetName(): string {
-    return `mongo_pull_events_seq:${this.config.projectKey}`;
-  }
-
-  private async pushEvents(): Promise<number> {
-    if (!this.events) throw new Error('Mongo not connected');
-
-    const position = await this.sqliteStore.getSyncPosition(this.pushTargetName());
-    let lastRowid = parseIntOrZero(position.lastEventId);
-
-    let pushed = 0;
-
-    while (true) {
-      const batch = await this.sqliteStore.getEventsSinceRowid(lastRowid, this.config.batchSize);
-      if (batch.length === 0) break;
-
-      const startSeq = await this.allocateSeqRange('events', batch.length);
-      const now = new Date();
-      const hostname = os.hostname();
-
-      const ops = batch.map((item, idx) => {
-        const event = item.event as unknown as MemoryEvent;
-        const seq = startSeq + idx;
-        const docId = `${this.config.projectKey}:${event.id}`;
-
-        return {
-          updateOne: {
-            filter: { _id: docId },
-            update: {
-              $setOnInsert: {
-                _id: docId,
-                projectKey: this.config.projectKey,
-                seq,
-                eventId: event.id,
-                eventType: event.eventType,
-                sessionId: event.sessionId,
-                timestamp: event.timestamp,
-                content: event.content,
-                canonicalKey: event.canonicalKey,
-                dedupeKey: event.dedupeKey,
-                metadata: event.metadata ?? null,
-                insertedAt: now,
-                updatedAt: now,
-                source: { hostname, instanceId: this.config.instanceId }
-              }
-            },
-            upsert: true
-          }
-        };
-      });
-
-      await this.events.bulkWrite(ops, { ordered: false });
-
-      const last = batch[batch.length - 1];
-      lastRowid = last.rowid;
-      await this.sqliteStore.updateSyncPosition(
-        this.pushTargetName(),
-        String(lastRowid),
-        last.event.timestamp.toISOString()
-      );
-
-      pushed += batch.length;
-      if (batch.length < this.config.batchSize) break;
-    }
-
-    return pushed;
-  }
-
-  private async pullEvents(): Promise<number> {
-    if (!this.events) throw new Error('Mongo not connected');
-
-    const position = await this.sqliteStore.getSyncPosition(this.pullTargetName());
-    let lastSeq = parseIntOrZero(position.lastEventId);
-
-    let pulled = 0;
-
-    while (true) {
-      const docs = await this.events.find(
-        { projectKey: this.config.projectKey, seq: { $gt: lastSeq } },
-        { sort: { seq: 1 }, limit: this.config.batchSize }
-      ).toArray();
-
-      if (docs.length === 0) break;
-
-      const events: MemoryEvent[] = docs.map((d) => ({
-        id: d.eventId,
-        eventType: d.eventType as any,
-        sessionId: d.sessionId,
-        timestamp: d.timestamp instanceof Date ? d.timestamp : new Date(d.timestamp),
-        content: d.content,
-        canonicalKey: d.canonicalKey,
-        dedupeKey: d.dedupeKey,
-        metadata: d.metadata ?? undefined
-      }));
-
-      const result = await this.sqliteStore.importEvents(events);
-      pulled += result.inserted;
-
-      lastSeq = docs[docs.length - 1].seq;
-      await this.sqliteStore.updateSyncPosition(
-        this.pullTargetName(),
-        String(lastSeq),
-        new Date().toISOString()
-      );
-
-      if (docs.length < this.config.batchSize) break;
-    }
-
-    return pulled;
-  }
-}