codemem 0.26.0 → 0.26.2

package/dist/index.js

@@ -1,5 +1,5 @@
 #!/usr/bin/env node
-import { DEFAULT_COORDINATOR_DB_PATH, DedupKeyBackfillRunner, MUTATING_TOOL_NAMES, MemoryStore, ObserverClient, RawEventSweeper, RefBackfillRunner, SessionContextBackfillRunner, SyncRetentionRunner, VERSION, VectorModelMigrationRunner, aiBackfillStructuredContent, applyBootstrapSnapshot, backfillMemoryDedupKeys, backfillNarrativeFromBody, backfillTagsText, backfillVectors, buildAuthHeaders, buildBaseUrl, buildRawEventEnvelopeFromHook, compareMemoryRoleReports, connect, coordinatorCreateGroupAction, coordinatorCreateInviteAction, coordinatorDisableDeviceAction, coordinatorEnrollDeviceAction, coordinatorImportInviteAction, coordinatorListBootstrapGrantsAction, coordinatorListDevicesAction, coordinatorListGroupsAction, coordinatorListJoinRequestsAction, coordinatorRemoveDeviceAction, coordinatorRenameDeviceAction, coordinatorReviewJoinRequestAction, coordinatorRevokeBootstrapGrantAction, createBetterSqliteCoordinatorApp, deactivateLowSignalMemories, deactivateLowSignalObservations, dedupNearDuplicateMemories, ensureDeviceIdentity, ensureSchemaBootstrapped, exportMemories, extractApplyPatchPaths, fetchAllSnapshotPages, fingerprintPublicKey, flushRawEvents, getExtractionBenchmarkProfile, getInjectionEvalScenarioPack, getInjectionEvalScenarioPrompts, getMemoryRoleReport, getRawEventRelinkPlan, getRawEventRelinkReport, getRawEventStatus, getSemanticIndexDiagnostics, getSessionExtractionEval, getSessionExtractionEvalScenario, getWorkspaceCodememConfigPath, hasPendingDedupKeyBackfill, hasPendingRefBackfill, hasPendingSessionContextBackfill, hasUnsyncedSharedMemoryChanges, importMemories, initDatabase, isEmbeddingDisabled, loadObserverConfig, loadPublicKey, loadSqliteVec, planReplicationOpsAgePrune, pruneReplicationOpsUntilCaughtUp, rawEventsGate, readCodememConfigFile, readCodememConfigFileAtPath, readCoordinatorSyncConfig, readImportPayload, replayBatchExtraction, replayBatchExtractionWithTierRouting, requestJson, resolveCodememConfigPath, resolveDbPath, resolveHookProject, resolveProject, retryRawEventFailures, runSyncDaemon, runSyncPass, schema, setPeerProjectFilter, stripJsonComments, stripPrivateObj, stripTrailingCommas, syncPassPreflight, updatePeerAddresses, vacuumDatabase, writeCodememConfigFile } from "@codemem/core";
+import { DEDUP_KEY_BACKFILL_JOB, DEFAULT_COORDINATOR_DB_PATH, DedupKeyBackfillRunner, MUTATING_TOOL_NAMES, MemoryStore, ObserverClient, REF_BACKFILL_JOB, RawEventSweeper, RefBackfillRunner, SESSION_CONTEXT_BACKFILL_JOB, SessionContextBackfillRunner, SyncRetentionRunner, VERSION, VectorModelMigrationRunner, aiBackfillStructuredContent, applyBootstrapSnapshot, backfillMemoryDedupKeys, backfillNarrativeFromBody, backfillTagsText, backfillVectors, buildAuthHeaders, buildBaseUrl, buildRawEventEnvelopeFromHook, compareMemoryRoleReports, connect, coordinatorCreateGroupAction, coordinatorCreateInviteAction, coordinatorDisableDeviceAction, coordinatorEnrollDeviceAction, coordinatorImportInviteAction, coordinatorListBootstrapGrantsAction, coordinatorListDevicesAction, coordinatorListGroupsAction, coordinatorListJoinRequestsAction, coordinatorRemoveDeviceAction, coordinatorRenameDeviceAction, coordinatorReviewJoinRequestAction, coordinatorRevokeBootstrapGrantAction, createBetterSqliteCoordinatorApp, deactivateLowSignalMemories, deactivateLowSignalObservations, dedupNearDuplicateMemories, ensureDeviceIdentity, ensureSchemaBootstrapped, exportMemories, extractApplyPatchPaths, fetchAllSnapshotPages, fingerprintPublicKey, flushRawEvents, getExtractionBenchmarkProfile, getInjectionEvalScenarioPack, getInjectionEvalScenarioPrompts, getMaintenanceJob, getMemoryRoleReport, getRawEventRelinkPlan, getRawEventRelinkReport, getRawEventStatus, getSemanticIndexDiagnostics, getSessionExtractionEval, getSessionExtractionEvalScenario, getWorkspaceCodememConfigPath, hasPendingDedupKeyBackfill, hasPendingRefBackfill, hasPendingSessionContextBackfill, hasUnsyncedSharedMemoryChanges, importMemories, initDatabase, isEmbeddingDisabled, loadObserverConfig, loadPublicKey, loadSqliteVec, planReplicationOpsAgePrune, pruneReplicationOpsUntilCaughtUp, rawEventsGate, readCodememConfigFile, readCodememConfigFileAtPath, readCoordinatorSyncConfig, readImportPayload, replayBatchExtraction, replayBatchExtractionWithTierRouting, requestJson, resolveCodememConfigPath, resolveDbPath, resolveHookProject, resolveProject, retryRawEventFailures, runSyncDaemon, runSyncPass, schema, setPeerProjectFilter, stripJsonComments, stripPrivateObj, stripTrailingCommas, syncPassPreflight, updatePeerAddresses, vacuumDatabase, writeCodememConfigFile } from "@codemem/core";
 import { Command, Option } from "commander";
 import omelette from "omelette";
 import { appendFileSync, existsSync, mkdirSync, readFileSync, readdirSync, renameSync, rmSync, rmdirSync, statSync, unlinkSync, writeFileSync } from "node:fs";
@@ -3564,6 +3564,19 @@ function readViewerPidRecord(dbPath) {
 	}
 	return null;
 }
+function normalizeViewerHost(host) {
+	const normalized = host.trim().toLowerCase();
+	if (normalized === "localhost" || normalized === "127.0.0.1" || normalized === "::1" || normalized === "[::1]") return "loopback";
+	return normalized;
+}
+async function findRuntimeViewerConflict(dbPath, target) {
+	const record = readViewerPidRecord(dbPath);
+	if (!record) return null;
+	if (normalizeViewerHost(record.host) === normalizeViewerHost(target.host) && record.port === target.port) return null;
+	if (!isProcessRunning(record.pid)) return null;
+	if (!await respondsLikeCodememViewer(record)) return null;
+	return record;
+}
 function isProcessRunning(pid) {
 	try {
 		process.kill(pid, 0);
@@ -3735,6 +3748,88 @@ async function startBackgroundViewer(invocation) {
 	p.intro("codemem viewer");
 	p.outro(`Viewer started in background (pid ${child.pid}) at http://${invocation.host}:${invocation.port}`);
 }
+function createSequentialBackfillCoordinator(store, jobPlans, signal) {
+	const pollIntervalMs = 1e3;
+	let activeRunner = null;
+	let activePlan = null;
+	let activePollTimer = null;
+	let nextJobIndex = 0;
+	let stopped = false;
+	const clearPollTimer = () => {
+		if (!activePollTimer) return;
+		clearTimeout(activePollTimer);
+		activePollTimer = null;
+	};
+	const schedulePoll = (fn) => {
+		clearPollTimer();
+		activePollTimer = setTimeout(fn, pollIntervalMs);
+		if (typeof activePollTimer === "object" && "unref" in activePollTimer) activePollTimer.unref();
+	};
+	const waitForCurrentJob = () => {
+		if (stopped || signal?.aborted || !activePlan || !activeRunner) return;
+		const job = getMaintenanceJob(store.db, activePlan.kind);
+		if (!activePlan.isPending(store.db)) {
+			const finishedPlan = activePlan;
+			const finishedRunner = activeRunner;
+			activePlan = null;
+			activeRunner = null;
+			finishedRunner.stop().finally(() => {
+				if (!stopped && !signal?.aborted) {
+					p.log.step(`${finishedPlan.name} backfill complete`);
+					startNextJob();
+				}
+			});
+			return;
+		}
+		if (job?.status === "failed") {
+			const failedPlan = activePlan;
+			const failedRunner = activeRunner;
+			activePlan = null;
+			activeRunner = null;
+			failedRunner.stop().finally(() => {
+				if (!stopped && !signal?.aborted) {
+					p.log.warn(`${failedPlan.name} backfill failed and will be retried on a later startup`);
+					startNextJob();
+				}
+			});
+			return;
+		}
+		schedulePoll(waitForCurrentJob);
+	};
+	const startNextJob = () => {
+		clearPollTimer();
+		if (stopped || signal?.aborted) return;
+		while (nextJobIndex < jobPlans.length) {
+			const plan = jobPlans[nextJobIndex++];
+			if (!plan) continue;
+			if (!plan.isPending(store.db)) continue;
+			activePlan = plan;
+			activeRunner = plan.createRunner();
+			p.log.step(`${plan.name} backfill started`);
+			activeRunner.start();
+			schedulePoll(waitForCurrentJob);
+			return;
+		}
+		p.log.step("All backfill jobs complete");
+	};
+	return {
+		start: () => {
+			if (stopped || signal?.aborted) return;
+			const pendingCount = jobPlans.filter((plan) => plan.isPending(store.db)).length;
+			if (pendingCount === 0) return;
+			p.log.step(`${pendingCount} backfill job(s) pending — starting sequential runners`);
+			startNextJob();
+		},
+		stop: async () => {
+			stopped = true;
+			clearPollTimer();
+			const runner = activeRunner;
+			activeRunner = null;
+			activePlan = null;
+			if (runner) await runner.stop();
+		}
+	};
+}
 async function startForegroundViewer(invocation) {
 	const { createApp, createSyncApp, closeStore, getStore } = await import("@codemem/server");
 	const { serve } = await import("@hono/node-server");
@@ -3764,16 +3859,44 @@ async function startForegroundViewer(invocation) {
 	sweeper.start();
 	const syncAbort = new AbortController();
 	const retentionAbort = new AbortController();
+	const backfillAbort = new AbortController();
 	const syncConfig = readCoordinatorSyncConfig(invocation.configPath ? readCodememConfigFileAtPath(invocation.configPath) : readCodememConfigFile());
 	const syncEnabled = syncConfig.syncEnabled;
+	const dbPath = resolveDbPath(invocation.dbPath ?? void 0);
 	const retentionRunner = new SyncRetentionRunner({
-		dbPath: resolveDbPath(invocation.dbPath ?? void 0),
+		dbPath,
 		signal: retentionAbort.signal
 	});
-	const vectorMigrationRunner = new VectorModelMigrationRunner({ dbPath: resolveDbPath(invocation.dbPath ?? void 0) });
-	const dedupKeyBackfillRunner = new DedupKeyBackfillRunner({ dbPath: resolveDbPath(invocation.dbPath ?? void 0) });
-	const sessionContextBackfillRunner = new SessionContextBackfillRunner({ dbPath: resolveDbPath(invocation.dbPath ?? void 0) });
-	const refBackfillRunner = new RefBackfillRunner({ dbPath: resolveDbPath(invocation.dbPath ?? void 0) });
+	const vectorMigrationRunner = new VectorModelMigrationRunner({ dbPath });
+	const backfillCoordinator = createSequentialBackfillCoordinator(store, [
+		{
+			name: "Dedup-key",
+			kind: DEDUP_KEY_BACKFILL_JOB,
+			isPending: hasPendingDedupKeyBackfill,
+			createRunner: () => new DedupKeyBackfillRunner({
+				dbPath,
+				signal: backfillAbort.signal
+			})
+		},
+		{
+			name: "Session-context",
+			kind: SESSION_CONTEXT_BACKFILL_JOB,
+			isPending: hasPendingSessionContextBackfill,
+			createRunner: () => new SessionContextBackfillRunner({
+				dbPath,
+				signal: backfillAbort.signal
+			})
+		},
+		{
+			name: "Ref",
+			kind: REF_BACKFILL_JOB,
+			isPending: hasPendingRefBackfill,
+			createRunner: () => new RefBackfillRunner({
+				dbPath,
+				signal: backfillAbort.signal
+			})
+		}
+	], backfillAbort.signal);
 	const syncRuntimeStatus = {
 		phase: syncEnabled ? "starting" : "disabled",
 		detail: syncEnabled ? "Waiting for viewer startup to finish" : "Sync is disabled"
@@ -3785,7 +3908,7 @@ async function startForegroundViewer(invocation) {
 		getSyncRuntimeStatus: () => syncRuntimeStatus
 	};
 	const app = createApp(appOpts);
-	const pidPath = pidFilePath(resolveDbPath(invocation.dbPath ?? void 0));
+	const pidPath = pidFilePath(dbPath);
 	let syncServer = null;
 	let syncListenerReady = false;
 	if (syncEnabled) {
@@ -3820,18 +3943,7 @@ async function startForegroundViewer(invocation) {
 			vectorMigrationRunner.start();
 			p.log.step("Vector maintenance runner started");
 		}
-		if (hasPendingDedupKeyBackfill(store.db)) {
-			dedupKeyBackfillRunner.start();
-			p.log.step("Dedup-key maintenance runner started");
-		}
-		if (hasPendingSessionContextBackfill(store.db)) {
-			sessionContextBackfillRunner.start();
-			p.log.step("Session-context backfill runner started");
-		}
-		if (hasPendingRefBackfill(store.db)) {
-			refBackfillRunner.start();
-			p.log.step("Ref backfill runner started");
-		}
+		backfillCoordinator.start();
 		if (syncConfig.syncRetentionEnabled) {
 			retentionRunner.start();
 			p.log.step("Retention maintenance runner started");
@@ -3889,12 +4001,11 @@ async function startForegroundViewer(invocation) {
 		clearInterval(walCheckpointTimer);
 		syncAbort.abort();
 		retentionAbort.abort();
+		backfillAbort.abort();
 		await sweeper.stop();
-		await sessionContextBackfillRunner.stop();
-		await refBackfillRunner.stop();
-		await dedupKeyBackfillRunner.stop();
 		await vectorMigrationRunner.stop();
 		await retentionRunner.stop();
+		await backfillCoordinator.stop();
 		await new Promise((resolve) => {
 			let remaining = syncServer ? 2 : 1;
 			const done = () => {
@@ -3929,6 +4040,17 @@ async function startForegroundViewer(invocation) {
 }
 async function runServeInvocation(invocation) {
 	const dbPath = resolveDbPath(invocation.dbPath ?? void 0);
+	const runtimeConflict = await findRuntimeViewerConflict(dbPath, {
+		host: invocation.host,
+		port: invocation.port
+	});
+	if (runtimeConflict) {
+		p.intro("codemem viewer");
+		p.log.error(`Database runtime at ${dirname(dbPath)} is already managed by viewer ${runtimeConflict.host}:${runtimeConflict.port} (pid ${runtimeConflict.pid})`);
+		p.log.info("Use the matching --host/--port, stop the existing viewer first, or use a separate db/runtime folder for another viewer.");
+		process.exitCode = 1;
+		return;
+	}
 	if (invocation.mode === "stop" || invocation.mode === "restart") {
 		const result = await stopExistingViewer(dbPath, {
 			host: invocation.host,