claude-memory-layer 1.0.10 → 1.0.12

package/src/cli/index.ts

@@ -11,10 +11,14 @@ import * as path from 'path';
 import * as os from 'os';
 import {
   getDefaultMemoryService,
-  getMemoryServiceForProject
+  getMemoryServiceForProject,
+  getProjectStoragePath
 } from '../services/memory-service.js';
-import { createSessionHistoryImporter } from '../services/session-history-importer.js';
+import { createSessionHistoryImporter, type ProgressEvent } from '../services/session-history-importer.js';
+import { bootstrapKnowledgeBase } from '../services/bootstrap-organizer.js';
 import { startServer, stopServer, isServerRunning } from '../server/index.js';
+import { SQLiteEventStore } from '../core/sqlite-event-store.js';
+import { MongoSyncWorker } from '../core/mongo-sync-worker.js';
 
 // ============================================================
 // Hook Installation Utilities
@@ -107,7 +111,7 @@ const program = new Command();
 program
   .name('claude-memory-layer')
   .description('Claude Code Memory Plugin CLI')
-  .version('1.0.0');
+  .version(process.env.CLAUDE_MEMORY_LAYER_VERSION || '0.0.0');
 
 // ============================================================
 // Install / Uninstall Commands
@@ -427,6 +431,356 @@ program
     }
   });
 
+/**
+ * Mongo Sync command - sync local SQLite events with a shared MongoDB database (optional)
+ */
+program
+  .command('mongo-sync')
+  .description('Sync events with MongoDB for multi-server collaboration (optional)')
+  .option('-p, --project <path>', 'Project path (defaults to cwd)')
+  .option('--mongo-uri <uri>', 'MongoDB connection URI (env: CLAUDE_MEMORY_MONGO_URI)')
+  .option('--mongo-db <name>', 'MongoDB database name (env: CLAUDE_MEMORY_MONGO_DB)')
+  .option('--mongo-project <key>', 'Remote project key (env: CLAUDE_MEMORY_MONGO_PROJECT, default: basename(projectPath))')
+  .option('--direction <dir>', 'push|pull|both', 'both')
+  .option('--batch-size <n>', 'Batch size', '500')
+  .option('--interval <ms>', 'Watch interval ms', '30000')
+  .option('--watch', 'Run continuously')
+  .action(async (options) => {
+    const projectPath = options.project || process.cwd();
+    const mongoUri = options.mongoUri || process.env.CLAUDE_MEMORY_MONGO_URI;
+    const mongoDb = options.mongoDb || process.env.CLAUDE_MEMORY_MONGO_DB;
+    const projectKey = options.mongoProject || process.env.CLAUDE_MEMORY_MONGO_PROJECT || path.basename(projectPath);
+    const direction = String(options.direction || 'both').toLowerCase();
+
+    if (!mongoUri || !mongoDb) {
+      console.error('\n❌ MongoDB sync is not configured.');
+      console.error('   Set --mongo-uri/--mongo-db or env CLAUDE_MEMORY_MONGO_URI/CLAUDE_MEMORY_MONGO_DB.\n');
+      process.exit(1);
+    }
+
+    if (!['push', 'pull', 'both'].includes(direction)) {
+      console.error('\n❌ Invalid --direction. Use: push | pull | both\n');
+      process.exit(1);
+    }
+
+    const storagePath = getProjectStoragePath(projectPath);
+    if (!fs.existsSync(storagePath)) {
+      fs.mkdirSync(storagePath, { recursive: true });
+    }
+
+    const batchSizeParsed = parseInt(options.batchSize, 10);
+    const intervalParsed = parseInt(options.interval, 10);
+    const batchSize = (Number.isFinite(batchSizeParsed) && batchSizeParsed > 0) ? batchSizeParsed : 500;
+    const intervalMs = (Number.isFinite(intervalParsed) && intervalParsed > 0) ? intervalParsed : 30000;
+
+    const sqliteStore = new SQLiteEventStore(path.join(storagePath, 'events.sqlite'));
+    const worker = new MongoSyncWorker(sqliteStore, {
+      uri: mongoUri,
+      dbName: mongoDb,
+      projectKey,
+      direction,
+      batchSize,
+      intervalMs
+    });
+
+    const runOnce = async () => {
+      const { pushed, pulled } = await worker.syncNow();
+      const ts = new Date().toISOString();
+      process.stdout.write(`[mongo-sync] ${ts} project=${projectKey} pushed=${pushed} pulled=${pulled}\n`);
+    };
+
+    try {
+      if (!options.watch) {
+        await runOnce();
+        await worker.shutdown();
+        sqliteStore.close();
+        return;
+      }
+
+      console.log(`[mongo-sync] Watch mode started (interval=${intervalMs}ms, project=${projectKey})`);
+
+      const handle = setInterval(() => {
+        runOnce().catch((err) => {
+          console.error('[mongo-sync] Sync failed:', err);
+        });
+      }, intervalMs);
+
+      const shutdown = async () => {
+        clearInterval(handle);
+        console.log('\n[mongo-sync] Shutting down...');
+        try {
+          await worker.shutdown();
+        } finally {
+          sqliteStore.close();
+        }
+        process.exit(0);
+      };
+
+      process.on('SIGINT', () => { void shutdown(); });
+      process.on('SIGTERM', () => { void shutdown(); });
+
+      // Run immediately, then keep alive
+      await runOnce();
+      await new Promise(() => {});
+    } catch (error) {
+      console.error('[mongo-sync] Failed:', error);
+      process.exit(1);
+    }
+  });
+
+/**
+ * Render import progress to terminal
+ */
+function renderProgress(event: ProgressEvent): void {
+  switch (event.phase) {
+    case 'scan':
+      console.log(`  🔍 ${event.message}`);
+      break;
+    case 'session-start': {
+      const pct = Math.round(((event.sessionIndex) / event.totalSessions) * 100);
+      const sessionName = path.basename(event.filePath, '.jsonl').slice(0, 8);
+      process.stdout.write(
+        `\r  📄 [${event.sessionIndex + 1}/${event.totalSessions}] ${pct}% | Session ${sessionName}... `
+      );
+      break;
+    }
+    case 'session-progress': {
+      process.stdout.write(
+        `\r  📄 [${event.sessionIndex + 1}/...] ${event.messagesProcessed} msgs | +${event.imported} imported, ~${event.skipped} skipped `
+      );
+      break;
+    }
+    case 'session-done': {
+      const imported = event.importedPrompts + event.importedResponses;
+      if (imported > 0) {
+        process.stdout.write(
+          `\r  ✅ [${event.sessionIndex + 1}] +${event.importedPrompts} prompts, +${event.importedResponses} responses${event.skipped > 0 ? `, ~${event.skipped} skipped` : ''}     \n`
+        );
+      } else if (event.skipped > 0) {
+        process.stdout.write(
+          `\r  ⏭️  [${event.sessionIndex + 1}] All ${event.skipped} already imported                          \n`
+        );
+      } else {
+        process.stdout.write(
+          `\r  ⏭️  [${event.sessionIndex + 1}] Empty session                                              \n`
+        );
+      }
+      break;
+    }
+    case 'embedding':
+      process.stdout.write(
+        `\r  🧠 Embeddings: ${event.processed}/${event.total} processed `
+      );
+      if (event.processed >= event.total) {
+        process.stdout.write('\n');
+      }
+      break;
+    case 'done':
+      break;
+  }
+}
+
+function printImportSummary(result: import('../services/session-history-importer.js').ImportResult, embedCount: number): void {
+  console.log('\n┌─────────────────────────────────┐');
+  console.log('│       ✅ Import Complete         │');
+  console.log('├─────────────────────────────────┤');
+  console.log(`│  Sessions processed:  ${String(result.totalSessions).padStart(8)} │`);
+  console.log(`│  Total messages:      ${String(result.totalMessages).padStart(8)} │`);
+  console.log(`│  Imported prompts:    ${String(result.importedPrompts).padStart(8)} │`);
+  console.log(`│  Imported responses:  ${String(result.importedResponses).padStart(8)} │`);
+  console.log(`│  Skipped duplicates:  ${String(result.skippedDuplicates).padStart(8)} │`);
+  console.log(`│  Embeddings queued:   ${String(embedCount).padStart(8)} │`);
+  console.log('└─────────────────────────────────┘');
+
+  if (result.errors.length > 0) {
+    console.log(`\n⚠️  Errors (${result.errors.length}):`);
+    for (const error of result.errors.slice(0, 5)) {
+      console.log(`  - ${error}`);
+    }
+    if (result.errors.length > 5) {
+      console.log(`  ... and ${result.errors.length - 5} more`);
+    }
+  }
+}
+
+function sanitizeSegment(input: string | undefined, fallback: string): string {
+  const v = (input || '').trim().toLowerCase().replace(/[^a-z0-9._-]+/g, '-').replace(/^-+|-+$/g, '');
+  return v || fallback;
+}
+
+async function listMarkdownFiles(root: string): Promise<string[]> {
+  const out: string[] = [];
+  const stack = [root];
+
+  while (stack.length > 0) {
+    const dir = stack.pop()!;
+    const entries = await fs.promises.readdir(dir, { withFileTypes: true });
+    for (const e of entries) {
+      const full = path.join(dir, e.name);
+      if (e.isDirectory()) stack.push(full);
+      else if (e.isFile() && e.name.endsWith('.md') && e.name !== '_index.md') out.push(full);
+    }
+  }
+
+  return out.sort();
+}
+
+function deriveNamespaceCategory(sourceRoot: string, filePath: string): { namespace: string; categoryPath: string[] } {
+  const rel = path.relative(sourceRoot, filePath);
+  const dirSeg = path.dirname(rel).split(path.sep).filter(Boolean);
+
+  if (dirSeg.length >= 2) {
+    const namespace = sanitizeSegment(dirSeg[0], 'default');
+    const categoryPath = dirSeg.slice(1).map((s) => sanitizeSegment(s, 'uncategorized'));
+    return { namespace, categoryPath: categoryPath.length > 0 ? categoryPath : ['uncategorized'] };
+  }
+
+  return { namespace: 'default', categoryPath: ['uncategorized'] };
+}
+
+function extractImportEvidence(markdown: string): { confidence?: string; sources: string[] } {
+  const confidenceMatch = markdown.match(/^-\s*confidence:\s*([^\n]+)/m);
+  const sources = markdown
+    .split(/\r?\n/)
+    .map((line) => line.trim())
+    .filter((line) => line.startsWith('- source:'))
+    .map((line) => line.replace(/^-\s*source:\s*/i, '').trim())
+    .filter(Boolean)
+    .slice(0, 30);
+
+  return {
+    confidence: confidenceMatch ? confidenceMatch[1].trim() : undefined,
+    sources
+  };
+}
+
+/**
+ * Organize-import command - import legacy markdown memories into structured mirror
+ */
+program
+  .command('organize-import [sourceDir]')
+  .description('Import existing markdown memory files, or bootstrap knowledge docs from codebase/git when markdown is missing')
+  .option('-p, --project <path>', 'Project path (defaults to cwd)')
+  .option('--session <id>', 'Session id for imported events (default: import:organized)')
+  .option('--limit <n>', 'Limit number of files to import')
+  .option('--dry-run', 'Preview mapping without writing')
+  .option('--bootstrap', 'Force-generate structured markdown from codebase + git history before import')
+  .option('--bootstrap-if-empty', 'Auto-bootstrap when source has no markdown files (default: true)', true)
+  .option('--no-bootstrap-if-empty', 'Disable auto-bootstrap when source has no markdown files')
+  .option('--force-bootstrap', 'Run bootstrap even when markdown files exist')
+  .option('--repo <path>', 'Repository root for bootstrap analysis (default: project path)')
+  .option('--out <path>', 'Output directory for generated bootstrap markdown (default: <sourceDir>/bootstrap-kb)')
+  .option('--since <range>', 'Git history range for bootstrap (default: "180 days ago")')
+  .option('--max-commits <n>', 'Max commits to analyze for bootstrap (default: 1000)')
+  .option('--incremental', 'Use previous bootstrap manifest as baseline for incremental updates (default: true)', true)
+  .option('--no-incremental', 'Disable incremental bootstrap; regenerate full snapshot')
+  .action(async (sourceDir: string | undefined, options) => {
+    const projectPath = options.project || process.cwd();
+    const sessionId = options.session || 'import:organized';
+    const sourceRoot = path.resolve(sourceDir || options.out || projectPath);
+    const repoPath = path.resolve(options.repo || projectPath);
+
+    if (!fs.existsSync(sourceRoot)) {
+      fs.mkdirSync(sourceRoot, { recursive: true });
+    }
+
+    const service = getMemoryServiceForProject(projectPath);
+
+    try {
+      let activeSourceRoot = sourceRoot;
+      let importRoot = sourceRoot;
+      let files = await listMarkdownFiles(importRoot);
+      const hasMarkdown = files.length > 0;
+      const shouldBootstrap = Boolean(options.forceBootstrap || options.bootstrap || (!hasMarkdown && options.bootstrapIfEmpty));
+
+      if (shouldBootstrap) {
+        const outDir = path.resolve(options.out || path.join(sourceRoot, 'bootstrap-kb'));
+        const since = options.since || '180 days ago';
+        const maxCommits = options.maxCommits ? Math.max(1, parseInt(options.maxCommits, 10)) : 1000;
+
+        console.log('\n🧠 Bootstrapping markdown knowledge base...');
+        const bootstrap = await bootstrapKnowledgeBase({
+          repoPath,
+          outDir,
+          since,
+          maxCommits,
+          incremental: options.incremental
+        });
+        console.log(`  Repo: ${repoPath}`);
+        console.log(`  Output: ${bootstrap.outDir}`);
+        console.log(`  Files analyzed: ${bootstrap.fileCount}`);
+        console.log(`  Commits analyzed: ${bootstrap.commitCount}`);
+        console.log(`  Modules: ${bootstrap.moduleCount}`);
+
+        activeSourceRoot = outDir;
+        importRoot = outDir;
+        files = await listMarkdownFiles(importRoot);
+      }
+
+      if (files.length === 0) {
+        console.error('\n❌ organize-import found no markdown files to import.\n');
+        process.exit(1);
+      }
+
+      const limit = options.limit ? Math.max(1, parseInt(options.limit, 10)) : files.length;
+      const targets = files.slice(0, limit);
+
+      console.log(`\n📦 organize-import`);
+      console.log(`  Source: ${activeSourceRoot}`);
+      console.log(`  Project: ${projectPath}`);
+      console.log(`  Files: ${targets.length}${targets.length < files.length ? `/${files.length}` : ''}`);
+      console.log(`  Dry-run: ${options.dryRun ? 'yes' : 'no'}\n`);
+
+      if (!options.dryRun) {
+        await service.initialize();
+      }
+
+      let imported = 0;
+      let skipped = 0;
+
+      for (const file of targets) {
+        const text = await fs.promises.readFile(file, 'utf8');
+        if (!text.trim()) {
+          skipped += 1;
+          continue;
+        }
+
+        const { namespace, categoryPath } = deriveNamespaceCategory(activeSourceRoot, file);
+        const rel = path.relative(activeSourceRoot, file);
+        const evidence = extractImportEvidence(text);
+
+        if (options.dryRun) {
+          console.log(`- ${rel} -> namespace=${namespace} category=${categoryPath.join('/')} confidence=${evidence.confidence || 'n/a'} sources=${evidence.sources.length}`);
+          continue;
+        }
+
+        await service.storeSessionSummary(sessionId, text, {
+          namespace,
+          categoryPath,
+          confidence: evidence.confidence,
+          sources: evidence.sources,
+          import: {
+            sourceFile: rel,
+            importedAt: new Date().toISOString(),
+            bootstrap: shouldBootstrap === true
+          }
+        });
+        imported += 1;
+      }
+
+      if (!options.dryRun) {
+        const embed = await service.processPendingEmbeddings();
+        await service.shutdown();
+        console.log(`\n✅ Imported: ${imported}, skipped-empty: ${skipped}, embeddings: ${embed}\n`);
+      } else {
+        console.log(`\n✅ Dry-run complete (planned imports: ${targets.length - skipped}, skipped-empty: ${skipped})\n`);
+      }
+    } catch (error) {
+      console.error('\n❌ organize-import failed:', error);
+      process.exit(1);
+    }
+  });
+
 /**
  * Import command - import existing Claude Code sessions
  */
@@ -437,8 +791,11 @@ program
   .option('-s, --session <file>', 'Import specific session file (JSONL)')
   .option('-a, --all', 'Import all sessions from all projects')
   .option('-l, --limit <number>', 'Limit messages per session')
+  .option('-f, --force', 'Force reimport: delete existing events and reimport with turn_id grouping')
   .option('-v, --verbose', 'Show detailed progress')
   .action(async (options) => {
+    const startTime = Date.now();
+
     // Determine target project path for storage
     const targetProjectPath = options.project || process.cwd();
 
@@ -446,102 +803,76 @@ program
     const service = getMemoryServiceForProject(targetProjectPath);
     const importer = createSessionHistoryImporter(service);
 
+    const importOpts = {
+      limit: options.limit ? parseInt(options.limit) : undefined,
+      force: options.force,
+      verbose: options.verbose,
+      onProgress: renderProgress
+    };
+
     try {
+      console.log('\n⏳ Initializing memory service...');
       await service.initialize();
+      console.log('  ✅ Ready\n');
+
+      if (options.force) {
+        console.log('🔄 Force mode: existing events will be deleted and reimported with turn_id grouping\n');
+      }
 
       let result;
 
       if (options.session) {
         // Import specific session file
-        console.log(`\n📥 Importing session: ${options.session}`);
-        console.log(`   Target project: ${targetProjectPath}\n`);
+        console.log(`📥 Importing session: ${options.session}`);
+        console.log(`   Target: ${targetProjectPath}\n`);
         result = await importer.importSessionFile(options.session, {
+          ...importOpts,
           projectPath: targetProjectPath,
-          limit: options.limit ? parseInt(options.limit) : undefined,
-          verbose: options.verbose
         });
       } else if (options.project) {
         // Import all sessions from a project
-        console.log(`\n📥 Importing project: ${options.project}\n`);
-        result = await importer.importProject(options.project, {
-          limit: options.limit ? parseInt(options.limit) : undefined,
-          verbose: options.verbose
-        });
+        console.log(`📥 Importing project: ${options.project}\n`);
+        result = await importer.importProject(options.project, importOpts);
       } else if (options.all) {
         // Import all sessions from all projects
-        // Note: --all imports to global storage for backward compatibility
-        console.log('\n📥 Importing all sessions from all projects');
+        console.log('📥 Importing all sessions from all projects');
         console.log('   ⚠️  Using global storage (use -p for project-specific)\n');
         const globalService = getDefaultMemoryService();
         const globalImporter = createSessionHistoryImporter(globalService);
         await globalService.initialize();
-        result = await globalImporter.importAll({
-          limit: options.limit ? parseInt(options.limit) : undefined,
-          verbose: options.verbose
-        });
+        result = await globalImporter.importAll(importOpts);
 
         // Process embeddings
-        console.log('\n⏳ Processing embeddings...');
+        console.log('\n🧠 Processing embeddings...');
         const embedCount = await globalService.processPendingEmbeddings();
 
-        // Show results
-        console.log('\n✅ Import Complete\n');
-        console.log(`Sessions processed: ${result.totalSessions}`);
-        console.log(`Total messages: ${result.totalMessages}`);
-        console.log(`Imported prompts: ${result.importedPrompts}`);
-        console.log(`Imported responses: ${result.importedResponses}`);
-        console.log(`Skipped duplicates: ${result.skippedDuplicates}`);
-        console.log(`Embeddings processed: ${embedCount}`);
-
-        if (result.errors.length > 0) {
-          console.log(`\n⚠️  Errors (${result.errors.length}):`);
-          for (const error of result.errors.slice(0, 5)) {
-            console.log(`  - ${error}`);
-          }
-          if (result.errors.length > 5) {
-            console.log(`  ... and ${result.errors.length - 5} more`);
-          }
-        }
+        const elapsed = ((Date.now() - startTime) / 1000).toFixed(1);
+        printImportSummary(result, embedCount);
+        console.log(`\n⏱️  Completed in ${elapsed}s`);
 
         await globalService.shutdown();
         return;
       } else {
         // Default: import current project
         const cwd = process.cwd();
-        console.log(`\n📥 Importing sessions for current project: ${cwd}\n`);
+        console.log(`📥 Importing sessions for: ${cwd}\n`);
         result = await importer.importProject(cwd, {
+          ...importOpts,
           projectPath: cwd,
-          limit: options.limit ? parseInt(options.limit) : undefined,
-          verbose: options.verbose
         });
       }
 
       // Process embeddings
-      console.log('\n⏳ Processing embeddings...');
+      console.log('\n🧠 Processing embeddings...');
       const embedCount = await service.processPendingEmbeddings();
 
-      // Show results
-      console.log('\n✅ Import Complete\n');
-      console.log(`Sessions processed: ${result.totalSessions}`);
-      console.log(`Total messages: ${result.totalMessages}`);
-      console.log(`Imported prompts: ${result.importedPrompts}`);
-      console.log(`Imported responses: ${result.importedResponses}`);
-      console.log(`Skipped duplicates: ${result.skippedDuplicates}`);
-      console.log(`Embeddings processed: ${embedCount}`);
-
-      if (result.errors.length > 0) {
-        console.log(`\n⚠️  Errors (${result.errors.length}):`);
-        for (const error of result.errors.slice(0, 5)) {
-          console.log(`  - ${error}`);
-        }
-        if (result.errors.length > 5) {
-          console.log(`  ... and ${result.errors.length - 5} more`);
-        }
-      }
+      const elapsed = ((Date.now() - startTime) / 1000).toFixed(1);
+      printImportSummary(result, embedCount);
+      console.log(`\n⏱️  Completed in ${elapsed}s`);
 
       await service.shutdown();
     } catch (error) {
-      console.error('Import failed:', error);
+      console.error('\n❌ Import failed:', error);
       process.exit(1);
     }
   });

package/src/core/consolidated-store.ts

@@ -8,7 +8,9 @@ import { randomUUID } from 'crypto';
 import { dbRun, dbAll, toDate, type Database } from './db-wrapper.js';
 import type {
   ConsolidatedMemory,
-  ConsolidatedMemoryInput
+  ConsolidatedMemoryInput,
+  ConsolidationRule,
+  ConsolidationRuleInput
 } from './types.js';
 import { EventStore } from './event-store.js';
 
@@ -170,6 +172,66 @@ export class ConsolidatedStore {
     );
   }
 
+  /**
+   * Create a long-term rule promoted from stable summaries
+   */
+  async createRule(input: ConsolidationRuleInput): Promise<string> {
+    const ruleId = randomUUID();
+
+    await dbRun(
+      this.db,
+      `INSERT INTO consolidated_rules
+        (rule_id, rule, topics, source_memory_ids, source_events, confidence, created_at)
+       VALUES (?, ?, ?, ?, ?, ?, CURRENT_TIMESTAMP)`,
+      [
+        ruleId,
+        input.rule,
+        JSON.stringify(input.topics),
+        JSON.stringify(input.sourceMemoryIds),
+        JSON.stringify(input.sourceEvents),
+        input.confidence
+      ]
+    );
+
+    return ruleId;
+  }
+
+  async getRules(options?: { limit?: number }): Promise<ConsolidationRule[]> {
+    const limit = options?.limit || 100;
+    const rows = await dbAll<Record<string, unknown>>(
+      this.db,
+      `SELECT * FROM consolidated_rules ORDER BY confidence DESC, created_at DESC LIMIT ?`,
+      [limit]
+    );
+
+    return rows.map((row) => ({
+      ruleId: row.rule_id as string,
+      rule: row.rule as string,
+      topics: JSON.parse((row.topics as string) || '[]'),
+      sourceMemoryIds: JSON.parse((row.source_memory_ids as string) || '[]'),
+      sourceEvents: JSON.parse((row.source_events as string) || '[]'),
+      confidence: Number(row.confidence ?? 0.5),
+      createdAt: toDate(row.created_at) || new Date()
+    }));
+  }
+
+  async countRules(): Promise<number> {
+    const result = await dbAll<{ count: number }>(
+      this.db,
+      `SELECT COUNT(*) as count FROM consolidated_rules`
+    );
+    return result[0]?.count || 0;
+  }
+
+  async hasRuleForSourceMemory(memoryId: string): Promise<boolean> {
+    const rows = await dbAll<{ count: number }>(
+      this.db,
+      `SELECT COUNT(*) as count FROM consolidated_rules WHERE source_memory_ids LIKE ?`,
+      [`%"${memoryId}"%`]
+    );
+    return (rows[0]?.count || 0) > 0;
+  }
+
   /**
    * Get count of consolidated memories
    */