codemem 0.21.1 → 0.22.0-alpha.0

package/dist/commands/db.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"db.d.ts","sourceRoot":"","sources":["../../src/commands/db.ts"],"names":[],"mappings":"AAmBA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AA4CpC,eAAO,MAAM,SAAS,SAEe,CAAC"}
+{"version":3,"file":"db.d.ts","sourceRoot":"","sources":["../../src/commands/db.ts"],"names":[],"mappings":"AAkBA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AA4CpC,eAAO,MAAM,SAAS,SAEe,CAAC"}

package/dist/commands/sync.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"sync.d.ts","sourceRoot":"","sources":["../../src/commands/sync.ts"],"names":[],"mappings":"AAAA;;GAEG;AA8BH,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAwHpC,eAAO,MAAM,WAAW,SAE+B,CAAC"}
+{"version":3,"file":"sync.d.ts","sourceRoot":"","sources":["../../src/commands/sync.ts"],"names":[],"mappings":"AAAA;;GAEG;AAqCH,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AA0JpC,eAAO,MAAM,WAAW,SAE+B,CAAC"}

package/dist/index.js

@@ -1,5 +1,5 @@
 #!/usr/bin/env node
-import { DEFAULT_COORDINATOR_DB_PATH, MemoryStore, ObserverClient, RawEventSweeper, SyncRetentionRunner, VERSION, backfillTagsText, backfillVectors, buildRawEventEnvelopeFromHook, bulkPruneReplicationOpsByAgeCutoff, connect, coordinatorCreateInviteAction, coordinatorImportInviteAction, coordinatorListJoinRequestsAction, coordinatorReviewJoinRequestAction, createCoordinatorApp, deactivateLowSignalMemories, deactivateLowSignalObservations, ensureDeviceIdentity, exportMemories, fingerprintPublicKey, getRawEventStatus, importMemories, initDatabase, isEmbeddingDisabled, loadPublicKey, loadSqliteVec, planReplicationOpsAgePrune, pruneReplicationOps, rawEventsGate, readCodememConfigFile, readCoordinatorSyncConfig, readImportPayload, resolveDbPath, resolveProject, retryRawEventFailures, runSyncDaemon, runSyncPass, schema, setPeerProjectFilter, stripJsonComments, stripPrivateObj, stripTrailingCommas, syncPassPreflight, updatePeerAddresses, vacuumDatabase, writeCodememConfigFile } from "@codemem/core";
+import { DEFAULT_COORDINATOR_DB_PATH, MemoryStore, ObserverClient, RawEventSweeper, SyncRetentionRunner, VERSION, backfillTagsText, backfillVectors, buildRawEventEnvelopeFromHook, connect, coordinatorCreateGroupAction, coordinatorCreateInviteAction, coordinatorDisableDeviceAction, coordinatorEnrollDeviceAction, coordinatorImportInviteAction, coordinatorListDevicesAction, coordinatorListGroupsAction, coordinatorListJoinRequestsAction, coordinatorRemoveDeviceAction, coordinatorRenameDeviceAction, coordinatorReviewJoinRequestAction, createCoordinatorApp, deactivateLowSignalMemories, deactivateLowSignalObservations, ensureDeviceIdentity, exportMemories, fingerprintPublicKey, getRawEventStatus, importMemories, initDatabase, isEmbeddingDisabled, loadPublicKey, loadSqliteVec, planReplicationOpsAgePrune, pruneReplicationOpsUntilCaughtUp, rawEventsGate, readCodememConfigFile, readCoordinatorSyncConfig, readImportPayload, resolveDbPath, resolveProject, retryRawEventFailures, runSyncDaemon, runSyncPass, schema, setPeerProjectFilter, stripJsonComments, stripPrivateObj, stripTrailingCommas, syncPassPreflight, updatePeerAddresses, vacuumDatabase, writeCodememConfigFile } from "@codemem/core";
 import { Command } from "commander";
 import omelette from "omelette";
 import { existsSync, mkdirSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs";
@@ -232,7 +232,7 @@ dbCommand.addCommand(new Command("init").configureHelp(helpStyle).description("V
 		const beforeBytes = estimateReplicationOpsBytes(db);
 		p.intro("codemem db prune-replication-ops");
 		p.log.info(`Replication ops size: ${formatBytes(beforeBytes)}`);
-		p.log.info(`Policy: approximately keep <= ${maxSizeMb} MB and <= ${maxAgeDays} day old history via oldest-first chunk pruning`);
+		p.log.info(`Policy: approximately prune oldest-first toward <= ${maxSizeMb} MB while removing history older than ${maxAgeDays} day(s), subject to per-pass batch/runtime limits`);
 		const agePlan = planReplicationOpsAgePrune(db, maxAgeDays, batchOps);
 		if (agePlan.candidate_ops > 0) p.log.info(`Age pass plan: ${agePlan.candidate_ops.toLocaleString()} ops, ~${formatBytes(agePlan.estimated_candidate_bytes)} in ~${agePlan.estimated_batches.toLocaleString()} batch(es), cutoff ${agePlan.cutoff_cursor}`);
 		else p.log.info("Age pass plan: no ops older than cutoff");
@@ -240,35 +240,21 @@ dbCommand.addCommand(new Command("init").configureHelp(helpStyle).description("V
 			p.outro("Dry run only; no changes made");
 			return;
 		}
-		let totalDeleted = 0;
-		let batches = 0;
-		let lastFloor = null;
-		let afterBytes = beforeBytes;
-		const ageResult = bulkPruneReplicationOpsByAgeCutoff(db, maxAgeDays, batchOps);
-		totalDeleted += ageResult.deleted;
-		lastFloor = ageResult.retained_floor_cursor;
-		afterBytes = ageResult.estimated_bytes_after ?? beforeBytes;
-		if (ageResult.deleted > 0) {
-			batches += 1;
-			p.log.step(`Age pass batch: deleted ${ageResult.deleted.toLocaleString()} ops, remaining size ~${formatBytes(afterBytes)}`);
-		}
-		while (true) {
-			const result = pruneReplicationOps(db, {
-				maxAgeDays: 365e3,
-				maxSizeBytes: maxSizeMb * 1024 * 1024,
-				maxDeleteOps: batchOps,
-				maxRuntimeMs: batchRuntimeMs
-			});
-			batches += 1;
-			totalDeleted += result.deleted;
-			lastFloor = result.retained_floor_cursor;
-			afterBytes = result.estimated_bytes_after ?? estimateReplicationOpsBytes(db);
-			p.log.step(`Batch ${batches}: deleted ${result.deleted.toLocaleString()} ops, remaining size ~${formatBytes(afterBytes)}`);
-			if (result.deleted === 0 || !result.stopped_by_budget) break;
-		}
-		p.log.info(`Deleted ops: ${totalDeleted.toLocaleString()}`);
-		p.log.info(`Estimated replication ops size after prune: ${formatBytes(afterBytes)} (approximate)`);
-		if (lastFloor) p.log.info(`Retained floor: ${lastFloor}`);
+		const loopResult = pruneReplicationOpsUntilCaughtUp(db, {
+			maxAgeDays,
+			maxSizeBytes: maxSizeMb * 1024 * 1024,
+			maxDeleteOps: batchOps,
+			maxRuntimeMs: batchRuntimeMs,
+			onPass: (pass) => {
+				if (pass.deleted === 0 && !pass.stoppedByBudget) return;
+				const suffix = pass.stoppedByBudget ? " (budget-limited)" : "";
+				p.log.step(`Pass ${pass.passNumber}: deleted ${pass.deleted.toLocaleString()} ops, remaining size ~${formatBytes(pass.afterBytes)}${suffix}`);
+			}
+		});
+		p.log.info(`Deleted ops: ${loopResult.totalDeleted.toLocaleString()}`);
+		p.log.info(`Estimated replication ops size after prune: ${formatBytes(loopResult.afterBytes)} (approximate)`);
+		if (loopResult.lastFloor) p.log.info(`Retained floor: ${loopResult.lastFloor}`);
+		if (loopResult.stoppedByBudget) p.log.warn("Prune stopped because the current batch/runtime budget was exhausted before retention caught up. Re-run with higher --batch-ops or --batch-runtime-ms for faster catch-up.");
 		if (opts.vacuum) {
 			p.log.step("Running VACUUM as requested...");
 			db.close();
@@ -1748,6 +1734,33 @@ function parseAttemptsLimit(value) {
 	if (!/^\d+$/.test(value.trim())) throw new Error(`Invalid --limit: ${value}`);
 	return Number.parseInt(value, 10);
 }
+function resolvePeerMatch(db, peerRef) {
+	const trimmed = peerRef.trim();
+	if (!trimmed) return null;
+	const byId = db.select({
+		peer_device_id: schema.syncPeers.peer_device_id,
+		name: schema.syncPeers.name
+	}).from(schema.syncPeers).where(eq(schema.syncPeers.peer_device_id, trimmed)).get();
+	if (byId) return byId;
+	const byName = db.select({
+		peer_device_id: schema.syncPeers.peer_device_id,
+		name: schema.syncPeers.name
+	}).from(schema.syncPeers).where(eq(schema.syncPeers.name, trimmed)).all();
+	if (byName.length > 1) return "ambiguous";
+	return byName[0] ?? null;
+}
+function readCoordinatorPublicKey(opts) {
+	const inline = String(opts.publicKey ?? "").trim();
+	const filePath = String(opts.publicKeyFile ?? "").trim();
+	if (inline && filePath) throw new Error("Use only one of --public-key or --public-key-file");
+	if (filePath) {
+		const text = readFileSync(filePath, "utf8").trim();
+		if (!text) throw new Error(`Public key file is empty: ${filePath}`);
+		return text;
+	}
+	if (!inline) throw new Error("Public key required via --public-key or --public-key-file");
+	return inline;
+}
 async function portOpen(host, port) {
 	return new Promise((resolve) => {
 		const socket = net.createConnection({
@@ -2147,7 +2160,7 @@ syncCommand.addCommand(new Command("disable").configureHelp(helpStyle).descripti
 	p.intro("codemem sync disable");
 	p.outro("Sync disabled — restart `codemem serve` to take effect");
 }));
-syncCommand.addCommand(new Command("peers").configureHelp(helpStyle).description("List known sync peers").option("--db <path>", "database path").option("--db-path <path>", "database path").option("--json", "output as JSON").action((opts) => {
+var peersCommand = new Command("peers").configureHelp(helpStyle).description("List known sync peers").option("--db <path>", "database path").option("--db-path <path>", "database path").option("--json", "output as JSON").action((opts) => {
 	const store = new MemoryStore(resolveDbPath(opts.db ?? opts.dbPath));
 	try {
 		const peers = drizzle(store.db, { schema }).select({
@@ -2175,7 +2188,41 @@ syncCommand.addCommand(new Command("peers").configureHelp(helpStyle).description
 	} finally {
 		store.close();
 	}
+});
+peersCommand.addCommand(new Command("remove").configureHelp(helpStyle).description("Remove a sync peer by device id or exact name").argument("<peer>", "peer device id or exact name").option("--db <path>", "database path").option("--db-path <path>", "database path").option("--json", "output as JSON").action((peerRef, opts) => {
+	const store = new MemoryStore(resolveDbPath(opts.db ?? opts.dbPath));
+	try {
+		const d = drizzle(store.db, { schema });
+		const match = resolvePeerMatch(d, peerRef);
+		if (match === "ambiguous") {
+			p.log.error(`Peer name is ambiguous: ${peerRef.trim()}`);
+			process.exitCode = 1;
+			return;
+		}
+		if (!match) {
+			p.log.error(`Peer not found: ${peerRef.trim()}`);
+			process.exitCode = 1;
+			return;
+		}
+		d.delete(schema.replicationCursors).where(eq(schema.replicationCursors.peer_device_id, match.peer_device_id)).run();
+		d.delete(schema.syncPeers).where(eq(schema.syncPeers.peer_device_id, match.peer_device_id)).run();
+		const payload = {
+			ok: true,
+			peer_device_id: match.peer_device_id,
+			name: match.name
+		};
+		if (opts.json) {
+			console.log(JSON.stringify(payload, null, 2));
+			return;
+		}
+		p.intro("codemem sync peers remove");
+		p.log.success(`Removed peer ${match.name || match.peer_device_id}`);
+		p.outro(match.peer_device_id);
+	} finally {
+		store.close();
+	}
 }));
+syncCommand.addCommand(peersCommand);
 syncCommand.addCommand(new Command("connect").configureHelp(helpStyle).description("Configure coordinator URL for cloud sync").argument("<url>", "coordinator URL (e.g. https://coordinator.example.com)").option("--group <group>", "sync group ID").action((url, opts) => {
 	const config = readCodememConfigFile();
 	config.sync_coordinator_url = url.trim();
@@ -2187,9 +2234,152 @@ syncCommand.addCommand(new Command("connect").configureHelp(helpStyle).descripti
 	p.outro("Restart `codemem serve` to activate coordinator sync");
 }));
 var coordinatorCommand = new Command("coordinator").configureHelp(helpStyle).description("Manage coordinator invites, join requests, and relay server");
-coordinatorCommand.addCommand(new Command("serve").configureHelp(helpStyle).description("Run the coordinator relay HTTP server").option("--db <path>", "coordinator database path").option("--db-path <path>", "coordinator database path").option("--host <host>", "bind host", "127.0.0.1").option("--port <port>", "bind port", "7340").action(async (opts) => {
+coordinatorCommand.addCommand(new Command("group-create").configureHelp(helpStyle).description("Create a coordinator group in the local store").argument("<group>", "group id").option("--name <name>", "display name override").option("--db <path>", "coordinator database path").option("--db-path <path>", "coordinator database path").option("--json", "output as JSON").action(async (groupId, opts) => {
+	const group = coordinatorCreateGroupAction({
+		groupId,
+		displayName: opts.name?.trim() || null,
+		dbPath: opts.db ?? opts.dbPath ?? null
+	});
+	if (opts.json) {
+		console.log(JSON.stringify(group, null, 2));
+		return;
+	}
+	p.intro("codemem sync coordinator group-create");
+	p.log.success(`Group ready: ${groupId.trim()}`);
+	p.outro(String(group.display_name ?? group.group_id ?? groupId.trim()));
+}));
+coordinatorCommand.addCommand(new Command("list-groups").configureHelp(helpStyle).description("List coordinator groups from the local store").option("--db <path>", "coordinator database path").option("--db-path <path>", "coordinator database path").option("--json", "output as JSON").action((opts) => {
+	const groups = coordinatorListGroupsAction({ dbPath: opts.db ?? opts.dbPath ?? null });
+	if (opts.json) {
+		console.log(JSON.stringify(groups, null, 2));
+		return;
+	}
+	p.intro("codemem sync coordinator list-groups");
+	if (groups.length === 0) {
+		p.outro("No coordinator groups found");
+		return;
+	}
+	for (const group of groups) p.log.message(`- ${String(group.group_id ?? "")}${group.display_name ? ` (${String(group.display_name)})` : ""}`);
+	p.outro(`${groups.length} group(s)`);
+}));
+coordinatorCommand.addCommand(new Command("enroll-device").configureHelp(helpStyle).description("Enroll a device in a local coordinator group").argument("<group>", "group id").argument("<device-id>", "device id").option("--fingerprint <fingerprint>", "device fingerprint").option("--public-key <key>", "device public key").option("--public-key-file <path>", "path to device public key").option("--name <name>", "display name").option("--db <path>", "coordinator database path").option("--db-path <path>", "coordinator database path").option("--json", "output as JSON").action(async (groupId, deviceId, opts) => {
+	const publicKey = readCoordinatorPublicKey(opts);
+	const fingerprint = String(opts.fingerprint ?? "").trim();
+	if (!fingerprint) {
+		p.log.error("Fingerprint required via --fingerprint");
+		process.exitCode = 1;
+		return;
+	}
+	if (fingerprintPublicKey(publicKey) !== fingerprint) {
+		p.log.error("Fingerprint does not match the provided public key");
+		process.exitCode = 1;
+		return;
+	}
+	const enrollment = coordinatorEnrollDeviceAction({
+		groupId,
+		deviceId,
+		fingerprint,
+		publicKey,
+		displayName: opts.name?.trim() || null,
+		dbPath: opts.db ?? opts.dbPath ?? null
+	});
+	if (opts.json) {
+		console.log(JSON.stringify(enrollment, null, 2));
+		return;
+	}
+	p.intro("codemem sync coordinator enroll-device");
+	p.log.success(`Enrolled ${deviceId.trim()} in ${groupId.trim()}`);
+	p.outro(String(enrollment.display_name ?? enrollment.device_id ?? deviceId.trim()));
+}));
+coordinatorCommand.addCommand(new Command("list-devices").configureHelp(helpStyle).description("List enrolled devices in a local coordinator group").argument("<group>", "group id").option("--include-disabled", "include disabled devices").option("--db <path>", "coordinator database path").option("--db-path <path>", "coordinator database path").option("--json", "output as JSON").action((groupId, opts) => {
+	const rows = coordinatorListDevicesAction({
+		groupId,
+		includeDisabled: opts.includeDisabled === true,
+		dbPath: opts.db ?? opts.dbPath ?? null
+	});
+	if (opts.json) {
+		console.log(JSON.stringify(rows, null, 2));
+		return;
+	}
+	p.intro("codemem sync coordinator list-devices");
+	if (rows.length === 0) {
+		p.outro(`No enrolled devices for ${groupId.trim()}`);
+		return;
+	}
+	for (const row of rows) {
+		const label = String(row.display_name ?? row.device_id ?? "").trim() || String(row.device_id ?? "");
+		const enabled = Number(row.enabled ?? 1) === 1 ? "enabled" : "disabled";
+		p.log.message(`- ${label} (${String(row.device_id ?? "")}) ${enabled}`);
+	}
+	p.outro(`${rows.length} device(s)`);
+}));
+coordinatorCommand.addCommand(new Command("rename-device").configureHelp(helpStyle).description("Rename an enrolled device in the local coordinator store").argument("<group>", "group id").argument("<device-id>", "device id").requiredOption("--name <name>", "display name").option("--db <path>", "coordinator database path").option("--db-path <path>", "coordinator database path").option("--json", "output as JSON").action((groupId, deviceId, opts) => {
+	const result = coordinatorRenameDeviceAction({
+		groupId,
+		deviceId,
+		displayName: opts.name.trim(),
+		dbPath: opts.db ?? opts.dbPath ?? null
+	});
+	if (!result) {
+		p.log.error(`Device not found: ${deviceId.trim()}`);
+		process.exitCode = 1;
+		return;
+	}
+	if (opts.json) {
+		console.log(JSON.stringify(result, null, 2));
+		return;
+	}
+	p.intro("codemem sync coordinator rename-device");
+	p.log.success(`Renamed ${deviceId.trim()} in ${groupId.trim()}`);
+	p.outro(String(result.display_name ?? result.device_id ?? deviceId.trim()));
+}));
+coordinatorCommand.addCommand(new Command("disable-device").configureHelp(helpStyle).description("Disable an enrolled device in the local coordinator store").argument("<group>", "group id").argument("<device-id>", "device id").option("--db <path>", "coordinator database path").option("--db-path <path>", "coordinator database path").option("--json", "output as JSON").action((groupId, deviceId, opts) => {
+	if (!coordinatorDisableDeviceAction({
+		groupId,
+		deviceId,
+		dbPath: opts.db ?? opts.dbPath ?? null
+	})) {
+		p.log.error(`Device not found: ${deviceId.trim()}`);
+		process.exitCode = 1;
+		return;
+	}
+	if (opts.json) {
+		console.log(JSON.stringify({
+			ok: true,
+			group_id: groupId.trim(),
+			device_id: deviceId.trim()
+		}, null, 2));
+		return;
+	}
+	p.intro("codemem sync coordinator disable-device");
+	p.log.success(`Disabled ${deviceId.trim()} in ${groupId.trim()}`);
+	p.outro("disabled");
+}));
+coordinatorCommand.addCommand(new Command("remove-device").configureHelp(helpStyle).description("Remove an enrolled device from the local coordinator store").argument("<group>", "group id").argument("<device-id>", "device id").option("--db <path>", "coordinator database path").option("--db-path <path>", "coordinator database path").option("--json", "output as JSON").action((groupId, deviceId, opts) => {
+	if (!coordinatorRemoveDeviceAction({
+		groupId,
+		deviceId,
+		dbPath: opts.db ?? opts.dbPath ?? null
+	})) {
+		p.log.error(`Device not found: ${deviceId.trim()}`);
+		process.exitCode = 1;
+		return;
+	}
+	if (opts.json) {
+		console.log(JSON.stringify({
+			ok: true,
+			group_id: groupId.trim(),
+			device_id: deviceId.trim()
+		}, null, 2));
+		return;
+	}
+	p.intro("codemem sync coordinator remove-device");
+	p.log.success(`Removed ${deviceId.trim()} from ${groupId.trim()}`);
+	p.outro("removed");
+}));
+coordinatorCommand.addCommand(new Command("serve").configureHelp(helpStyle).description("Run the coordinator relay HTTP server").option("--db <path>", "coordinator database path").option("--db-path <path>", "coordinator database path").option("--host <host>", "bind host", "127.0.0.1").option("--port <port>", "bind port", "7347").action(async (opts) => {
 	const host = String(opts.host ?? "127.0.0.1").trim() || "127.0.0.1";
-	const port = Number.parseInt(String(opts.port ?? "7340"), 10);
+	const port = Number.parseInt(String(opts.port ?? "7347"), 10);
 	const dbPath = opts.db ?? opts.dbPath ?? DEFAULT_COORDINATOR_DB_PATH;
 	const app = createCoordinatorApp({ dbPath });
 	p.intro("codemem sync coordinator serve");
@@ -2222,6 +2412,7 @@ coordinatorCommand.addCommand(new Command("create-invite").configureHelp(helpSty
 	p.log.success(`Invite created for ${groupId}`);
 	if (typeof result.link === "string") p.log.message(`- link: ${result.link}`);
 	if (typeof result.encoded === "string") p.log.message(`- invite: ${result.encoded}`);
+	for (const warning of Array.isArray(result.warnings) ? result.warnings : []) p.log.warn(String(warning));
 	p.outro("Invite ready");
 }));
 coordinatorCommand.addCommand(new Command("import-invite").configureHelp(helpStyle).description("Import a coordinator invite").argument("<invite>", "invite value or link").option("--db <path>", "database path").option("--db-path <path>", "database path").option("--keys-dir <path>", "keys directory").option("--config <path>", "config path").option("--json", "output as JSON").action(async (invite, opts) => {