codemem 0.25.3 → 0.26.1

package/dist/index.js

@@ -1,5 +1,5 @@
 #!/usr/bin/env node
-import { DEFAULT_COORDINATOR_DB_PATH, DedupKeyBackfillRunner, MemoryStore, ObserverClient, RawEventSweeper, SyncRetentionRunner, VERSION, VectorModelMigrationRunner, aiBackfillStructuredContent, applyBootstrapSnapshot, backfillMemoryDedupKeys, backfillNarrativeFromBody, backfillTagsText, backfillVectors, buildAuthHeaders, buildBaseUrl, buildRawEventEnvelopeFromHook, compareMemoryRoleReports, connect, coordinatorCreateGroupAction, coordinatorCreateInviteAction, coordinatorDisableDeviceAction, coordinatorEnrollDeviceAction, coordinatorImportInviteAction, coordinatorListBootstrapGrantsAction, coordinatorListDevicesAction, coordinatorListGroupsAction, coordinatorListJoinRequestsAction, coordinatorRemoveDeviceAction, coordinatorRenameDeviceAction, coordinatorReviewJoinRequestAction, coordinatorRevokeBootstrapGrantAction, createBetterSqliteCoordinatorApp, deactivateLowSignalMemories, deactivateLowSignalObservations, dedupNearDuplicateMemories, ensureDeviceIdentity, ensureSchemaBootstrapped, exportMemories, fetchAllSnapshotPages, fingerprintPublicKey, flushRawEvents, getExtractionBenchmarkProfile, getInjectionEvalScenarioPack, getInjectionEvalScenarioPrompts, getMemoryRoleReport, getRawEventRelinkPlan, getRawEventRelinkReport, getRawEventStatus, getSessionExtractionEval, getSessionExtractionEvalScenario, getWorkspaceCodememConfigPath, hasPendingDedupKeyBackfill, hasUnsyncedSharedMemoryChanges, importMemories, initDatabase, isEmbeddingDisabled, loadObserverConfig, loadPublicKey, loadSqliteVec, planReplicationOpsAgePrune, pruneReplicationOpsUntilCaughtUp, rawEventsGate, readCodememConfigFile, readCodememConfigFileAtPath, readCoordinatorSyncConfig, readImportPayload, replayBatchExtraction, replayBatchExtractionWithTierRouting, requestJson, resolveCodememConfigPath, resolveDbPath, resolveHookProject, resolveProject, retryRawEventFailures, runSyncDaemon, runSyncPass, schema, setPeerProjectFilter, stripJsonComments, stripPrivateObj, stripTrailingCommas, syncPassPreflight, updatePeerAddresses, vacuumDatabase, writeCodememConfigFile } from "@codemem/core";
+import { DEDUP_KEY_BACKFILL_JOB, DEFAULT_COORDINATOR_DB_PATH, DedupKeyBackfillRunner, MUTATING_TOOL_NAMES, MemoryStore, ObserverClient, REF_BACKFILL_JOB, RawEventSweeper, RefBackfillRunner, SESSION_CONTEXT_BACKFILL_JOB, SessionContextBackfillRunner, SyncRetentionRunner, VERSION, VectorModelMigrationRunner, aiBackfillStructuredContent, applyBootstrapSnapshot, backfillMemoryDedupKeys, backfillNarrativeFromBody, backfillTagsText, backfillVectors, buildAuthHeaders, buildBaseUrl, buildRawEventEnvelopeFromHook, compareMemoryRoleReports, connect, coordinatorCreateGroupAction, coordinatorCreateInviteAction, coordinatorDisableDeviceAction, coordinatorEnrollDeviceAction, coordinatorImportInviteAction, coordinatorListBootstrapGrantsAction, coordinatorListDevicesAction, coordinatorListGroupsAction, coordinatorListJoinRequestsAction, coordinatorRemoveDeviceAction, coordinatorRenameDeviceAction, coordinatorReviewJoinRequestAction, coordinatorRevokeBootstrapGrantAction, createBetterSqliteCoordinatorApp, deactivateLowSignalMemories, deactivateLowSignalObservations, dedupNearDuplicateMemories, ensureDeviceIdentity, ensureSchemaBootstrapped, exportMemories, extractApplyPatchPaths, fetchAllSnapshotPages, fingerprintPublicKey, flushRawEvents, getExtractionBenchmarkProfile, getInjectionEvalScenarioPack, getInjectionEvalScenarioPrompts, getMaintenanceJob, getMemoryRoleReport, getRawEventRelinkPlan, getRawEventRelinkReport, getRawEventStatus, getSemanticIndexDiagnostics, getSessionExtractionEval, getSessionExtractionEvalScenario, getWorkspaceCodememConfigPath, hasPendingDedupKeyBackfill, hasPendingRefBackfill, hasPendingSessionContextBackfill, hasUnsyncedSharedMemoryChanges, importMemories, initDatabase, isEmbeddingDisabled, loadObserverConfig, loadPublicKey, loadSqliteVec, planReplicationOpsAgePrune, pruneReplicationOpsUntilCaughtUp, rawEventsGate, readCodememConfigFile, readCodememConfigFileAtPath, readCoordinatorSyncConfig, readImportPayload, replayBatchExtraction, replayBatchExtractionWithTierRouting, requestJson, resolveCodememConfigPath, resolveDbPath, resolveHookProject, resolveProject, retryRawEventFailures, runSyncDaemon, runSyncPass, schema, setPeerProjectFilter, stripJsonComments, stripPrivateObj, stripTrailingCommas, syncPassPreflight, updatePeerAddresses, vacuumDatabase, writeCodememConfigFile } from "@codemem/core";
 import { Command, Option } from "commander";
 import omelette from "omelette";
 import { appendFileSync, existsSync, mkdirSync, readFileSync, readdirSync, renameSync, rmSync, rmdirSync, statSync, unlinkSync, writeFileSync } from "node:fs";
@@ -536,18 +536,6 @@ var MAX_FILES_MODIFIED = 64;
 var MAX_WORKING_SET_PATHS = 8;
 var MAX_QUERY_CHARS = 500;
 var MAX_QUERY_FILE_BASENAMES = 5;
-var MUTATING_TOOL_NAMES = new Set([
-	"edit",
-	"write",
-	"multiedit",
-	"notebookedit",
-	"apply_patch"
-]);
-var APPLY_PATCH_PATH_PREFIXES = [
-	"*** Add File: ",
-	"*** Update File: ",
-	"*** Delete File: "
-];
 function defaultSessionState() {
 	return {
 		first_prompt: "",
@@ -629,14 +617,6 @@ function clearSessionState(sessionId) {
 		rmSync(path, { force: true });
 	} catch {}
 }
-function extractApplyPatchPaths(patchText) {
-	const out = [];
-	for (const line of patchText.split("\n")) for (const prefix of APPLY_PATCH_PATH_PREFIXES) if (line.startsWith(prefix)) {
-		const path = line.slice(prefix.length).trim();
-		if (path) out.push(path);
-	}
-	return out;
-}
 function extractModifiedPathsFromHook(payload) {
 	const toolName = String(payload.tool_name ?? "").trim().toLowerCase();
 	if (!MUTATING_TOOL_NAMES.has(toolName)) return [];
@@ -3584,6 +3564,19 @@ function readViewerPidRecord(dbPath) {
 	}
 	return null;
 }
+function normalizeViewerHost(host) {
+	const normalized = host.trim().toLowerCase();
+	if (normalized === "localhost" || normalized === "127.0.0.1" || normalized === "::1" || normalized === "[::1]") return "loopback";
+	return normalized;
+}
+async function findRuntimeViewerConflict(dbPath, target) {
+	const record = readViewerPidRecord(dbPath);
+	if (!record) return null;
+	if (normalizeViewerHost(record.host) === normalizeViewerHost(target.host) && record.port === target.port) return null;
+	if (!isProcessRunning(record.pid)) return null;
+	if (!await respondsLikeCodememViewer(record)) return null;
+	return record;
+}
 function isProcessRunning(pid) {
 	try {
 		process.kill(pid, 0);
@@ -3755,6 +3748,88 @@ async function startBackgroundViewer(invocation) {
 	p.intro("codemem viewer");
 	p.outro(`Viewer started in background (pid ${child.pid}) at http://${invocation.host}:${invocation.port}`);
 }
+function createSequentialBackfillCoordinator(store, jobPlans, signal) {
+	const pollIntervalMs = 1e3;
+	let activeRunner = null;
+	let activePlan = null;
+	let activePollTimer = null;
+	let nextJobIndex = 0;
+	let stopped = false;
+	const clearPollTimer = () => {
+		if (!activePollTimer) return;
+		clearTimeout(activePollTimer);
+		activePollTimer = null;
+	};
+	const schedulePoll = (fn) => {
+		clearPollTimer();
+		activePollTimer = setTimeout(fn, pollIntervalMs);
+		if (typeof activePollTimer === "object" && "unref" in activePollTimer) activePollTimer.unref();
+	};
+	const waitForCurrentJob = () => {
+		if (stopped || signal?.aborted || !activePlan || !activeRunner) return;
+		const job = getMaintenanceJob(store.db, activePlan.kind);
+		if (!activePlan.isPending(store.db)) {
+			const finishedPlan = activePlan;
+			const finishedRunner = activeRunner;
+			activePlan = null;
+			activeRunner = null;
+			finishedRunner.stop().finally(() => {
+				if (!stopped && !signal?.aborted) {
+					p.log.step(`${finishedPlan.name} backfill complete`);
+					startNextJob();
+				}
+			});
+			return;
+		}
+		if (job?.status === "failed") {
+			const failedPlan = activePlan;
+			const failedRunner = activeRunner;
+			activePlan = null;
+			activeRunner = null;
+			failedRunner.stop().finally(() => {
+				if (!stopped && !signal?.aborted) {
+					p.log.warn(`${failedPlan.name} backfill failed and will be retried on a later startup`);
+					startNextJob();
+				}
+			});
+			return;
+		}
+		schedulePoll(waitForCurrentJob);
+	};
+	const startNextJob = () => {
+		clearPollTimer();
+		if (stopped || signal?.aborted) return;
+		while (nextJobIndex < jobPlans.length) {
+			const plan = jobPlans[nextJobIndex++];
+			if (!plan) continue;
+			if (!plan.isPending(store.db)) continue;
+			activePlan = plan;
+			activeRunner = plan.createRunner();
+			p.log.step(`${plan.name} backfill started`);
+			activeRunner.start();
+			schedulePoll(waitForCurrentJob);
+			return;
+		}
+		p.log.step("All backfill jobs complete");
+	};
+	return {
+		start: () => {
+			if (stopped || signal?.aborted) return;
+			const pendingCount = jobPlans.filter((plan) => plan.isPending(store.db)).length;
+			if (pendingCount === 0) return;
+			p.log.step(`${pendingCount} backfill job(s) pending — starting sequential runners`);
+			startNextJob();
+		},
+		stop: async () => {
+			stopped = true;
+			clearPollTimer();
+			const runner = activeRunner;
+			activeRunner = null;
+			activePlan = null;
+			if (runner) await runner.stop();
+		}
+	};
+}
 async function startForegroundViewer(invocation) {
 	const { createApp, createSyncApp, closeStore, getStore } = await import("@codemem/server");
 	const { serve } = await import("@hono/node-server");
@@ -3784,14 +3859,44 @@ async function startForegroundViewer(invocation) {
 	sweeper.start();
 	const syncAbort = new AbortController();
 	const retentionAbort = new AbortController();
+	const backfillAbort = new AbortController();
 	const syncConfig = readCoordinatorSyncConfig(invocation.configPath ? readCodememConfigFileAtPath(invocation.configPath) : readCodememConfigFile());
 	const syncEnabled = syncConfig.syncEnabled;
+	const dbPath = resolveDbPath(invocation.dbPath ?? void 0);
 	const retentionRunner = new SyncRetentionRunner({
-		dbPath: resolveDbPath(invocation.dbPath ?? void 0),
+		dbPath,
 		signal: retentionAbort.signal
 	});
-	const vectorMigrationRunner = new VectorModelMigrationRunner({ dbPath: resolveDbPath(invocation.dbPath ?? void 0) });
-	const dedupKeyBackfillRunner = new DedupKeyBackfillRunner({ dbPath: resolveDbPath(invocation.dbPath ?? void 0) });
+	const vectorMigrationRunner = new VectorModelMigrationRunner({ dbPath });
+	const backfillCoordinator = createSequentialBackfillCoordinator(store, [
+		{
+			name: "Dedup-key",
+			kind: DEDUP_KEY_BACKFILL_JOB,
+			isPending: hasPendingDedupKeyBackfill,
+			createRunner: () => new DedupKeyBackfillRunner({
+				dbPath,
+				signal: backfillAbort.signal
+			})
+		},
+		{
+			name: "Session-context",
+			kind: SESSION_CONTEXT_BACKFILL_JOB,
+			isPending: hasPendingSessionContextBackfill,
+			createRunner: () => new SessionContextBackfillRunner({
+				dbPath,
+				signal: backfillAbort.signal
+			})
+		},
+		{
+			name: "Ref",
+			kind: REF_BACKFILL_JOB,
+			isPending: hasPendingRefBackfill,
+			createRunner: () => new RefBackfillRunner({
+				dbPath,
+				signal: backfillAbort.signal
+			})
+		}
+	], backfillAbort.signal);
 	const syncRuntimeStatus = {
 		phase: syncEnabled ? "starting" : "disabled",
 		detail: syncEnabled ? "Waiting for viewer startup to finish" : "Sync is disabled"
@@ -3803,7 +3908,7 @@ async function startForegroundViewer(invocation) {
 		getSyncRuntimeStatus: () => syncRuntimeStatus
 	};
 	const app = createApp(appOpts);
-	const pidPath = pidFilePath(resolveDbPath(invocation.dbPath ?? void 0));
+	const pidPath = pidFilePath(dbPath);
 	let syncServer = null;
 	let syncListenerReady = false;
 	if (syncEnabled) {
@@ -3838,10 +3943,7 @@ async function startForegroundViewer(invocation) {
 			vectorMigrationRunner.start();
 			p.log.step("Vector maintenance runner started");
 		}
-		if (hasPendingDedupKeyBackfill(store.db)) {
-			dedupKeyBackfillRunner.start();
-			p.log.step("Dedup-key maintenance runner started");
-		}
+		backfillCoordinator.start();
 		if (syncConfig.syncRetentionEnabled) {
 			retentionRunner.start();
 			p.log.step("Retention maintenance runner started");
@@ -3899,10 +4001,11 @@ async function startForegroundViewer(invocation) {
 		clearInterval(walCheckpointTimer);
 		syncAbort.abort();
 		retentionAbort.abort();
+		backfillAbort.abort();
 		await sweeper.stop();
-		await dedupKeyBackfillRunner.stop();
 		await vectorMigrationRunner.stop();
 		await retentionRunner.stop();
+		await backfillCoordinator.stop();
 		await new Promise((resolve) => {
 			let remaining = syncServer ? 2 : 1;
 			const done = () => {
@@ -3937,6 +4040,17 @@ async function startForegroundViewer(invocation) {
 }
 async function runServeInvocation(invocation) {
 	const dbPath = resolveDbPath(invocation.dbPath ?? void 0);
+	const runtimeConflict = await findRuntimeViewerConflict(dbPath, {
+		host: invocation.host,
+		port: invocation.port
+	});
+	if (runtimeConflict) {
+		p.intro("codemem viewer");
+		p.log.error(`Database runtime at ${dirname(dbPath)} is already managed by viewer ${runtimeConflict.host}:${runtimeConflict.port} (pid ${runtimeConflict.pid})`);
+		p.log.info("Use the matching --host/--port, stop the existing viewer first, or use a separate db/runtime folder for another viewer.");
+		process.exitCode = 1;
+		return;
+	}
 	if (invocation.mode === "stop" || invocation.mode === "restart") {
 		const result = await stopExistingViewer(dbPath, {
 			host: invocation.host,
@@ -4822,12 +4936,14 @@ statusCmd.action((opts) => {
 			last_sync_at: schema.syncPeers.last_sync_at,
 			last_error: schema.syncPeers.last_error
 		}).from(schema.syncPeers).all();
+		const semanticIndex = getSemanticIndexDiagnostics(store.db);
 		if (opts.json) {
 			console.log(JSON.stringify({
 				enabled: config.sync_enabled === true,
 				host: config.sync_host ?? "0.0.0.0",
 				port: config.sync_port ?? 7337,
 				interval_s: config.sync_interval_s ?? 120,
+				semantic_index: semanticIndex,
 				device_id: deviceRow?.device_id ?? null,
 				fingerprint: deviceRow?.fingerprint ?? null,
 				coordinator_url: config.sync_coordinator_url ?? null,
@@ -4855,6 +4971,13 @@ statusCmd.action((opts) => {
 			const label = peer.name || peer.peer_device_id;
 			p.log.message(`  ${label}: last_sync=${peer.last_sync_at ?? "never"}, status=${peer.last_error ?? "ok"}`);
 		}
+		p.log.info([
+			"Semantic index:",
+			`  State: ${semanticIndex.state}`,
+			`  Summary: ${semanticIndex.summary}`,
+			`  Coverage: ${semanticIndex.indexed_memory_count}/${semanticIndex.embeddable_memory_count}`,
+			`  Mode: ${semanticIndex.mode === "semantic" ? `semantic (${semanticIndex.semantic_search_model ?? semanticIndex.current_model})` : "keyword-only"}`
+		].join("\n"));
 		p.outro(`${peers.length} peer(s)`);
 	} finally {
 		store.close();