codemem 0.20.10 → 0.21.0

package/dist/commands/db.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"db.d.ts","sourceRoot":"","sources":["../../src/commands/db.ts"],"names":[],"mappings":"AAgBA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AA2BpC,eAAO,MAAM,SAAS,SAEe,CAAC"}
+{"version":3,"file":"db.d.ts","sourceRoot":"","sources":["../../src/commands/db.ts"],"names":[],"mappings":"AAmBA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AA4CpC,eAAO,MAAM,SAAS,SAEe,CAAC"}

package/dist/commands/serve.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"serve.d.ts","sourceRoot":"","sources":["../../src/commands/serve.ts"],"names":[],"mappings":"AAeA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAEpC,OAAO,EAEN,KAAK,uBAAuB,EAG5B,MAAM,uBAAuB,CAAC;AAQ/B,wBAAgB,gBAAgB,CAAC,OAAO,EAAE,OAAO,GAAG,MAAM,GAAG,IAAI,CAKhE;AAED,wBAAgB,WAAW,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CASjD;AAED,wBAAgB,qBAAqB,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAQ9D;AAED,wBAAgB,sBAAsB,CACrC,QAAQ,EAAE,MAAM,GAAG,IAAI,EACvB,WAAW,EAAE,MAAM,GAAG,IAAI,GACxB,MAAM,GAAG,IAAI,CAGf;AAqLD,wBAAgB,yBAAyB,CACxC,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,uBAAuB,EACnC,QAAQ,GAAE,MAAM,EAAqB,GACnC,MAAM,EAAE,CAgBV;AAED,wBAAgB,sBAAsB,CAAC,KAAK,EAAE,OAAO,GAAG,OAAO,CAS9D;AAED,wBAAgB,2BAA2B,CAAC,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM,EAAE,CAUpF;AAkRD,eAAO,MAAM,YAAY,SAuBtB,CAAC"}
+{"version":3,"file":"serve.d.ts","sourceRoot":"","sources":["../../src/commands/serve.ts"],"names":[],"mappings":"AAgBA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAEpC,OAAO,EAEN,KAAK,uBAAuB,EAG5B,MAAM,uBAAuB,CAAC;AAQ/B,wBAAgB,gBAAgB,CAAC,OAAO,EAAE,OAAO,GAAG,MAAM,GAAG,IAAI,CAKhE;AAED,wBAAgB,WAAW,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CASjD;AAED,wBAAgB,qBAAqB,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAQ9D;AAED,wBAAgB,sBAAsB,CACrC,QAAQ,EAAE,MAAM,GAAG,IAAI,EACvB,WAAW,EAAE,MAAM,GAAG,IAAI,GACxB,MAAM,GAAG,IAAI,CAGf;AAqLD,wBAAgB,yBAAyB,CACxC,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,uBAAuB,EACnC,QAAQ,GAAE,MAAM,EAAqB,GACnC,MAAM,EAAE,CAgBV;AAED,wBAAgB,sBAAsB,CAAC,KAAK,EAAE,OAAO,GAAG,OAAO,CAS9D;AAED,wBAAgB,2BAA2B,CAAC,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM,EAAE,CAUpF;AA6RD,eAAO,MAAM,YAAY,SAuBtB,CAAC"}

package/dist/index.js

@@ -1,5 +1,5 @@
 #!/usr/bin/env node
-import { DEFAULT_COORDINATOR_DB_PATH, MemoryStore, ObserverClient, RawEventSweeper, VERSION, backfillTagsText, backfillVectors, buildRawEventEnvelopeFromHook, connect, coordinatorCreateInviteAction, coordinatorImportInviteAction, coordinatorListJoinRequestsAction, coordinatorReviewJoinRequestAction, createCoordinatorApp, deactivateLowSignalMemories, deactivateLowSignalObservations, ensureDeviceIdentity, exportMemories, fingerprintPublicKey, getRawEventStatus, importMemories, initDatabase, isEmbeddingDisabled, loadPublicKey, loadSqliteVec, rawEventsGate, readCodememConfigFile, readCoordinatorSyncConfig, readImportPayload, resolveDbPath, resolveProject, retryRawEventFailures, runSyncDaemon, runSyncPass, schema, setPeerProjectFilter, stripJsonComments, stripPrivateObj, stripTrailingCommas, syncPassPreflight, updatePeerAddresses, vacuumDatabase, writeCodememConfigFile } from "@codemem/core";
+import { DEFAULT_COORDINATOR_DB_PATH, MemoryStore, ObserverClient, RawEventSweeper, SyncRetentionRunner, VERSION, backfillTagsText, backfillVectors, buildRawEventEnvelopeFromHook, bulkPruneReplicationOpsByAgeCutoff, connect, coordinatorCreateInviteAction, coordinatorImportInviteAction, coordinatorListJoinRequestsAction, coordinatorReviewJoinRequestAction, createCoordinatorApp, deactivateLowSignalMemories, deactivateLowSignalObservations, ensureDeviceIdentity, exportMemories, fingerprintPublicKey, getRawEventStatus, importMemories, initDatabase, isEmbeddingDisabled, loadPublicKey, loadSqliteVec, planReplicationOpsAgePrune, pruneReplicationOps, rawEventsGate, readCodememConfigFile, readCoordinatorSyncConfig, readImportPayload, resolveDbPath, resolveProject, retryRawEventFailures, runSyncDaemon, runSyncPass, schema, setPeerProjectFilter, stripJsonComments, stripPrivateObj, stripTrailingCommas, syncPassPreflight, updatePeerAddresses, vacuumDatabase, writeCodememConfigFile } from "@codemem/core";
 import { Command } from "commander";
 import omelette from "omelette";
 import { existsSync, mkdirSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs";
@@ -197,6 +197,18 @@ function parseKindsCsv(value) {
 	const kinds = value.split(",").map((kind) => kind.trim()).filter((kind) => kind.length > 0);
 	return kinds.length > 0 ? kinds : void 0;
 }
+function estimateReplicationOpsBytes(db) {
+	try {
+		const row = db.prepare(`SELECT COALESCE(SUM(pgsize), 0) AS total_bytes
+				 FROM dbstat
+				 WHERE name = 'replication_ops'
+				    OR name LIKE 'idx_replication_ops_%'
+				    OR name LIKE 'sqlite_autoindex_replication_ops_%'`).get();
+		return Number(row?.total_bytes ?? 0);
+	} catch {
+		return 0;
+	}
+}
 var dbCommand = new Command("db").configureHelp(helpStyle).description("Database maintenance");
 dbCommand.addCommand(new Command("init").configureHelp(helpStyle).description("Verify the SQLite database is present and schema-ready").option("--db <path>", "database path (default: $CODEMEM_DB or ~/.codemem/mem.sqlite)").option("--db-path <path>", "database path (default: $CODEMEM_DB or ~/.codemem/mem.sqlite)").action((opts) => {
 	const result = initDatabase(opts.db ?? opts.dbPath);
@@ -208,6 +220,67 @@ dbCommand.addCommand(new Command("init").configureHelp(helpStyle).description("V
 	p.intro("codemem db vacuum");
 	p.log.success(`Vacuumed: ${result.path}`);
 	p.outro(`Size: ${result.sizeBytes.toLocaleString()} bytes`);
+})).addCommand(new Command("prune-replication-ops").configureHelp(helpStyle).description("Prune replication op history with approximate oldest-first retention, dry-run, and progress reporting").option("--db <path>", "database path (default: $CODEMEM_DB or ~/.codemem/mem.sqlite)").option("--db-path <path>", "database path (default: $CODEMEM_DB or ~/.codemem/mem.sqlite)").option("--dry-run", "show current size/targets without deleting").option("--max-age-days <days>", "retention age threshold in days", "30").option("--max-size-mb <mb>", "target replication log budget in MB", "512").option("--batch-ops <n>", "max ops deleted per batch", "5000").option("--batch-runtime-ms <ms>", "max runtime per batch in ms", "2000").option("--vacuum", "run VACUUM explicitly after prune completes").action((opts) => {
+	const dbPath = resolveDbPath(opts.db ?? opts.dbPath);
+	const db = connect(dbPath);
+	let dbOpen = true;
+	try {
+		const maxAgeDays = Number.parseInt(opts.maxAgeDays, 10) || 30;
+		const maxSizeMb = Number.parseInt(opts.maxSizeMb, 10) || 512;
+		const batchOps = Number.parseInt(opts.batchOps, 10) || 5e3;
+		const batchRuntimeMs = Number.parseInt(opts.batchRuntimeMs, 10) || 2e3;
+		const beforeBytes = estimateReplicationOpsBytes(db);
+		p.intro("codemem db prune-replication-ops");
+		p.log.info(`Replication ops size: ${formatBytes(beforeBytes)}`);
+		p.log.info(`Policy: approximately keep <= ${maxSizeMb} MB and <= ${maxAgeDays} day old history via oldest-first chunk pruning`);
+		const agePlan = planReplicationOpsAgePrune(db, maxAgeDays, batchOps);
+		if (agePlan.candidate_ops > 0) p.log.info(`Age pass plan: ${agePlan.candidate_ops.toLocaleString()} ops, ~${formatBytes(agePlan.estimated_candidate_bytes)} in ~${agePlan.estimated_batches.toLocaleString()} batch(es), cutoff ${agePlan.cutoff_cursor}`);
+		else p.log.info("Age pass plan: no ops older than cutoff");
+		if (opts.dryRun) {
+			p.outro("Dry run only; no changes made");
+			return;
+		}
+		let totalDeleted = 0;
+		let batches = 0;
+		let lastFloor = null;
+		let afterBytes = beforeBytes;
+		const ageResult = bulkPruneReplicationOpsByAgeCutoff(db, maxAgeDays, batchOps);
+		totalDeleted += ageResult.deleted;
+		lastFloor = ageResult.retained_floor_cursor;
+		afterBytes = ageResult.estimated_bytes_after ?? beforeBytes;
+		if (ageResult.deleted > 0) {
+			batches += 1;
+			p.log.step(`Age pass batch: deleted ${ageResult.deleted.toLocaleString()} ops, remaining size ~${formatBytes(afterBytes)}`);
+		}
+		while (true) {
+			const result = pruneReplicationOps(db, {
+				maxAgeDays: 365e3,
+				maxSizeBytes: maxSizeMb * 1024 * 1024,
+				maxDeleteOps: batchOps,
+				maxRuntimeMs: batchRuntimeMs
+			});
+			batches += 1;
+			totalDeleted += result.deleted;
+			lastFloor = result.retained_floor_cursor;
+			afterBytes = result.estimated_bytes_after ?? estimateReplicationOpsBytes(db);
+			p.log.step(`Batch ${batches}: deleted ${result.deleted.toLocaleString()} ops, remaining size ~${formatBytes(afterBytes)}`);
+			if (result.deleted === 0 || !result.stopped_by_budget) break;
+		}
+		p.log.info(`Deleted ops: ${totalDeleted.toLocaleString()}`);
+		p.log.info(`Estimated replication ops size after prune: ${formatBytes(afterBytes)} (approximate)`);
+		if (lastFloor) p.log.info(`Retained floor: ${lastFloor}`);
+		if (opts.vacuum) {
+			p.log.step("Running VACUUM as requested...");
+			db.close();
+			dbOpen = false;
+			const vacuumed = vacuumDatabase(dbPath);
+			p.outro(`Done. VACUUM complete. File size is now ${formatBytes(vacuumed.sizeBytes)}.`);
+			return;
+		}
+		p.outro("Done. Retention is approximate oldest-first pruning. SQLite file size may not shrink until you run `codemem db vacuum` explicitly (or re-run this command with --vacuum).");
+	} finally {
+		if (dbOpen) db.close();
+	}
 })).addCommand(new Command("raw-events-status").configureHelp(helpStyle).description("Show pending raw-event backlog by source stream").option("--db <path>", "database path (default: $CODEMEM_DB or ~/.codemem/mem.sqlite)").option("--db-path <path>", "database path (default: $CODEMEM_DB or ~/.codemem/mem.sqlite)").option("-n, --limit <n>", "max rows to show", "25").option("--json", "output as JSON").action((opts) => {
 	const result = getRawEventStatus(opts.db ?? opts.dbPath, Number.parseInt(opts.limit, 10) || 25);
 	if (opts.json) {
@@ -1155,8 +1228,13 @@ async function startForegroundViewer(invocation) {
 	const sweeper = new RawEventSweeper(store, { observer });
 	sweeper.start();
 	const syncAbort = new AbortController();
+	const retentionAbort = new AbortController();
 	const syncConfig = readCoordinatorSyncConfig(readCodememConfigFile());
 	const syncEnabled = syncConfig.syncEnabled;
+	const retentionRunner = new SyncRetentionRunner({
+		dbPath: resolveDbPath(invocation.dbPath ?? void 0),
+		signal: retentionAbort.signal
+	});
 	const syncRuntimeStatus = {
 		phase: syncEnabled ? "starting" : "disabled",
 		detail: syncEnabled ? "Waiting for viewer startup to finish" : "Sync is disabled"
@@ -1200,6 +1278,10 @@ async function startForegroundViewer(invocation) {
 		p.log.success(`Listening on http://${info.address}:${info.port}`);
 		p.log.info(`Database: ${dbPath}`);
 		p.log.step("Raw event sweeper started");
+		if (syncConfig.syncRetentionEnabled) {
+			retentionRunner.start();
+			p.log.step("Retention maintenance runner started");
+		}
 		if (syncEnabled) {
 			const syncStartDelayMs = 3e3;
 			p.log.step(`Sync daemon will start in background (${syncStartDelayMs / 1e3}s delay)`);
@@ -1243,7 +1325,9 @@ async function startForegroundViewer(invocation) {
 	const shutdown = async () => {
 		p.outro("shutting down");
 		syncAbort.abort();
+		retentionAbort.abort();
 		await sweeper.stop();
+		await retentionRunner.stop();
 		await new Promise((resolve) => {
 			let remaining = syncServer ? 2 : 1;
 			const done = () => {