@moonshot-ai/kimi-code 0.42.0 → 0.43.0

package/dist/search-worker.mjs

@@ -3,7 +3,7 @@ import fs from "node:fs";
 import path, { join, relative } from "node:path";
 import { fileURLToPath } from "node:url";
 import { createHash, randomUUID } from "node:crypto";
-import fs$1, { open, readFile, readdir, rm, stat } from "node:fs/promises";
+import fs$1, { open, readFile, readdir, stat } from "node:fs/promises";
 import os from "node:os";
 import { AsyncLocalStorage } from "node:async_hooks";
 
@@ -1469,6 +1469,14 @@ const HEADER_SIZE = 22;
 const CRC_SIZE = 4;
 const MAX_KEY_LEN = 65535;
 const MAX_VAL_LEN = 4294967295;
+var CorruptFrameError = class extends Error {
+	offset;
+	constructor(message, offset) {
+		super(message);
+		this.name = "CorruptFrameError";
+		this.offset = offset;
+	}
+};
 const EMPTY = Buffer.alloc(0);
 /**
 * Encode one record into a single Buffer.
@@ -2493,17 +2501,23 @@ async function catchUpWalAsync(walPath, offset, anchor, apply) {
 //#endregion
 //#region ../../packages/minidb/src/rename-replace.ts
 const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
-async function renameReplace(src, dst, opts = {}) {
-	if (process.platform !== "win32") return fs$1.rename(src, dst);
+async function retryEperm(op, opts = {}) {
 	const retries = opts.retries ?? 100;
 	const base = opts.baseDelayMs ?? 20;
 	for (let attempt = 0;; attempt++) try {
-		return await fs$1.rename(src, dst);
+		return await op();
 	} catch (e) {
 		if (e.code !== "EPERM" || attempt >= retries) throw e;
 		await sleep(base + Math.floor(Math.random() * (base + 10)));
 	}
 }
+async function withWindowsEpermRetry(op, opts = {}) {
+	if (process.platform !== "win32") return op();
+	return retryEperm(op, opts);
+}
+async function renameReplace(src, dst, opts = {}) {
+	return withWindowsEpermRetry(() => fs$1.rename(src, dst), opts);
+}
 
 //#endregion
 //#region ../../packages/minidb/src/snapshot.ts
@@ -9212,6 +9226,7 @@ var GenerationBuilder = class {
 			};
 			this.deps.stats.generationBuilds++;
 			this.deps.stats.generationBuildDurationMs += performance.now() - t0;
+			this.deps.noteBuildSuccess?.();
 			const keep = new Set(prevCurrent ? [id, prevCurrent] : [id]);
 			cleanupGenerations(this.deps.dir(), keep).catch(() => {});
 			for (const name of textStates.keys()) await fs$1.rm(this.deps.textRegistry.textPostingsPath(name), { force: true }).catch(() => {});
@@ -9227,7 +9242,7 @@ var GenerationBuilder = class {
 				return;
 			}
 			this.deps.stats.generationBuildErrors++;
-			this.deps.noteBuildFailure?.();
+			this.deps.noteBuildFailure?.(e);
 			throw e;
 		} finally {
 			if (this.genBuild === gb) this.genBuild = null;
@@ -10000,6 +10015,51 @@ var WritePath = class {
 	}
 };
 
+//#endregion
+//#region ../../packages/minidb/src/wipe.ts
+async function acquireWithWait(lock, timeoutMs) {
+	const deadline = Date.now() + timeoutMs;
+	let delay = 10;
+	for (;;) {
+		if (await lock.acquire()) return true;
+		if (Date.now() + delay > deadline) return false;
+		const wait = delay + Math.floor(Math.random() * delay);
+		await new Promise((resolve) => {
+			setTimeout(resolve, wait);
+		});
+		delay = Math.min(delay * 2, 250);
+	}
+}
+async function wipeStoreDir(opts) {
+	const lockFiles = opts.lockFiles ?? ["db.lock"];
+	const timeoutMs = opts.lockAcquireTimeoutMs ?? 0;
+	const locks = lockFiles.map((file) => new LockFile(path.join(opts.dir, file)));
+	const releaseAll = () => Promise.all(locks.map((lock) => lock.release().catch(() => {})));
+	const results = await Promise.allSettled(locks.map((lock) => acquireWithWait(lock, timeoutMs)));
+	const failed = results.find((r) => r.status === "rejected");
+	if (failed !== void 0 || results.some((r) => r.status === "fulfilled" && !r.value)) {
+		await releaseAll();
+		if (failed !== void 0) throw failed.reason;
+		return "locked";
+	}
+	try {
+		const isolated = `${opts.dir}.wiping-${process.pid}-${randomUUID()}`;
+		try {
+			await withWindowsEpermRetry(() => fs$1.rename(opts.dir, isolated));
+		} catch (error) {
+			if (error.code === "ENOENT") return "wiped";
+			throw error;
+		}
+		await fs$1.rm(isolated, {
+			recursive: true,
+			force: true
+		});
+	} finally {
+		await releaseAll();
+	}
+	return "wiped";
+}
+
 //#endregion
 //#region ../../packages/minidb/src/lifecycle.ts
 /** The open() flow: configure, acquire, recover, and kick background
@@ -10266,10 +10326,7 @@ async function openOrRebuildMiniDb(opts, hooks, open) {
 			}
 			return await open(opts);
 		} catch {}
-		await fs$1.rm(opts.dir, {
-			recursive: true,
-			force: true
-		});
+		if (await wipeStoreDir({ dir: opts.dir }) === "locked") throw err;
 		return open(opts);
 	}
 }
@@ -10880,6 +10937,7 @@ var MiniDb = class MiniDb {
 	genBuildKickFailureBackoffMs = 3e5;
 	lastGenBuildKickAt = 0;
 	lastGenBuildFailureAt = 0;
+	lastGenBuildError = null;
 	/** Abort handle / mutation queue / single-flight guard / status of the
 	*  generation build all live in the GenerationBuilder facet (declared
 	*  below); these views keep the open / write / close paths' call sites
@@ -11014,8 +11072,12 @@ var MiniDb = class MiniDb {
 		ensureOpen: () => this.ensureOpen(),
 		ensureWritable: () => this.ensureWritable(),
 		boundedTextBuild: (name, ti, def, checkpoint) => this.boundedTextBuild(name, ti, def, checkpoint),
-		noteBuildFailure: () => {
+		noteBuildFailure: (err) => {
 			this.lastGenBuildFailureAt = Date.now();
+			if (err !== void 0) this.lastGenBuildError = err;
+		},
+		noteBuildSuccess: () => {
+			this.lastGenBuildError = null;
 		}
 	});
 	/** The write path facet (write-path.ts): set/del/batch/expire and their
@@ -11573,8 +11635,10 @@ var MiniDb = class MiniDb {
 		poison = wal.poison;
 		if (!poison) return;
 		try {
-			const st = await fs$1.stat(this.walPath);
-			if (poison.failedAtOffset <= st.size) await fs$1.truncate(this.walPath, poison.failedAtOffset);
+			await withWindowsEpermRetry(async () => {
+				const st = await fs$1.stat(this.walPath);
+				if (poison.failedAtOffset <= st.size) await fs$1.truncate(this.walPath, poison.failedAtOffset);
+			});
 		} catch (err) {
 			this.writeDisabled = err;
 			return;
@@ -11876,6 +11940,16 @@ var MiniDb = class MiniDb {
 	}
 };
 
+//#endregion
+//#region ../../packages/minidb/src/error-classification.ts
+function classifyStorageError(error) {
+	if (error instanceof AggregateError) return error.errors.some((inner) => classifyStorageError(inner) === "rebuild") ? "rebuild" : "transient";
+	if (error instanceof SyntaxError || error instanceof CorruptFrameError) return "rebuild";
+	const code = error?.code;
+	if (code === "WAL_WRITE_DISABLED" || code === "WAL_POISONED") return "rebuild";
+	return "transient";
+}
+
 //#endregion
 //#region ../../packages/kap-server/src/search/docs.ts
 const MAX_DOC_TEXT_CHARS = 2e4;
@@ -12344,7 +12418,15 @@ function legacyFileMetaKey(filePath) {
 }
 const WIRE_READ_CHUNK_BYTES = 1 << 20;
 const WIRE_BATCH_OPS = 1e3;
+const SYNC_ROUND_BYTE_BUDGET = 64 << 20;
+const SYNC_ROUND_TIME_BUDGET_MS = 3e4;
+const SYNC_FAILURE_ESCALATION_LIMIT = 5;
+const SESSION_SYNC_FAILURE_SKIP_LIMIT = 5;
+const SESSION_SYNC_SKIP_COOLDOWN_MS = 3e5;
 const EMPTY_BUFFER = Buffer.alloc(0);
+function syncBudgetExhausted(budget) {
+	return budget.bytesLeft <= 0 || Date.now() >= budget.deadline;
+}
 function errorMessage(error) {
 	return error instanceof Error ? error.message : String(error);
 }
@@ -12434,10 +12516,17 @@ var SearchIndexCore = class {
 	openError = null;
 	fileMetaMigrated = false;
 	lockToken;
+	lastMaintenanceDetail;
+	consecutiveSyncFailures = 0;
+	sessionSyncFailures = /* @__PURE__ */ new Map();
+	sessionSyncSkips = /* @__PURE__ */ new Map();
 	db = null;
 	openPromise = null;
 	refreshPromise = null;
 	fullSyncDone = false;
+	syncRoundBytes = SYNC_ROUND_BYTE_BUDGET;
+	syncRoundMs = SYNC_ROUND_TIME_BUDGET_MS;
+	syncSkipCooldownMs = SESSION_SYNC_SKIP_COOLDOWN_MS;
 	constructor(options) {
 		this.options = options;
 	}
@@ -12517,6 +12606,7 @@ var SearchIndexCore = class {
 		const opts = {
 			dir: this.indexDir,
 			valueCodec: "json",
+			valueMode: "disk",
 			fsyncPolicy: "everysec",
 			onLockFail: "readonly",
 			onLockAcquired: (info) => {
@@ -12527,26 +12617,12 @@ var SearchIndexCore = class {
 		try {
 			return await MiniDb.open(opts);
 		} catch (error) {
-			if (!isRebuildableCorruption(error)) throw error;
-			let probeError;
-			try {
-				await (await MiniDb.open({
-					dir: opts.dir,
-					valueCodec: opts.valueCodec
-				})).close().catch(() => {});
-				probeError = void 0;
-			} catch (error) {
-				probeError = error;
-			}
-			if (probeError instanceof LockError) throw error;
+			if (classifyStorageError(error) !== "rebuild") throw error;
+			if (await wipeStoreDir({ dir: this.indexDir }) === "locked") throw error;
 			this.log.warn("global search: search-index corruption detected; rebuilding from scratch", {
 				dir: this.indexDir,
 				error: errorMessage(error)
 			});
-			await rm(this.indexDir, {
-				recursive: true,
-				force: true
-			});
 			return MiniDb.open(opts);
 		}
 	}
@@ -12627,10 +12703,29 @@ var SearchIndexCore = class {
 		let outcome = {
 			noop: true,
 			sessions: 0,
-			documents: 0
+			documents: 0,
+			truncated: false,
+			failures: 0
 		};
 		await this.tracked(async () => {
-			outcome = await this.runSync(sessions);
+			try {
+				outcome = await this.runSync(sessions);
+				this.consecutiveSyncFailures = 0;
+			} catch (error) {
+				if (classifyStorageError(error) !== "rebuild") {
+					this.consecutiveSyncFailures += 1;
+					if (this.consecutiveSyncFailures < SYNC_FAILURE_ESCALATION_LIMIT) throw error;
+				}
+				this.consecutiveSyncFailures = 0;
+				await this.recoverByRebuild(error);
+				outcome = {
+					noop: false,
+					sessions: 0,
+					documents: 0,
+					truncated: true,
+					failures: 0
+				};
+			}
 		});
 		return {
 			...outcome,
@@ -12638,56 +12733,78 @@ var SearchIndexCore = class {
 			lifecycle: this.lifecycleState()
 		};
 	}
+	async recoverByRebuild(cause) {
+		this.log.warn("global search: search-index state is unrecoverable; wiping and rebuilding from the transcripts", {
+			dir: this.indexDir,
+			error: errorMessage(cause)
+		});
+		await this.reindex();
+	}
 	async runSync(sessions) {
-		if (this.disposed) return {
+		const noop = {
 			noop: true,
 			sessions: 0,
-			documents: 0
+			documents: 0,
+			truncated: false,
+			failures: 0
 		};
+		if (this.disposed) return noop;
 		this.syncReplaced = false;
 		await this.ensureOpen();
 		const db = this.db;
-		if (!db || db.readOnly || this.disposed) return {
-			noop: true,
-			sessions: 0,
-			documents: 0
-		};
+		if (!db || db.readOnly || this.disposed) return noop;
 		await this.migrateFileMetaKeys(db);
 		const currentIds = new Set(sessions.map((s) => s.id));
 		for (const row of db.query({
 			key: { prefix: SESSION_META_PREFIX },
 			project: []
 		})) {
-			if (this.disposed) return {
-				noop: true,
-				sessions: 0,
-				documents: 0
-			};
+			if (this.disposed) return noop;
 			const sessionId = row.key.slice(14);
 			if (!currentIds.has(sessionId)) await this.deleteSessionDocs(db, sessionId);
 		}
+		const budget = {
+			bytesLeft: this.syncRoundBytes,
+			deadline: Date.now() + this.syncRoundMs
+		};
 		let indexed = 0;
+		let truncated = false;
+		let failures = 0;
 		for (const summary of sessions) {
-			if (this.disposed) return {
-				noop: true,
-				sessions: 0,
-				documents: 0
-			};
-			try {
-				await this.syncSession(db, summary);
-				indexed++;
-			} catch (error) {
-				this.log.warn("global search: failed to index session", {
-					sessionId: summary.id,
-					error: error instanceof Error ? error.message : String(error)
-				});
+			if (this.disposed) return noop;
+			const skip = this.sessionSyncSkips.get(summary.id);
+			if (skip !== void 0 && summary.updatedAt === skip.updatedAt && Date.now() - skip.at < this.syncSkipCooldownMs) continue;
+			this.sessionSyncSkips.delete(summary.id);
+			if (syncBudgetExhausted(budget)) {
+				truncated = true;
+				break;
 			}
+			const result = await this.syncSession(db, summary, budget);
+			if (result.truncated) truncated = true;
+			if (result.failed) {
+				const count = (this.sessionSyncFailures.get(summary.id) ?? 0) + 1;
+				this.sessionSyncFailures.set(summary.id, count);
+				if (count >= SESSION_SYNC_FAILURE_SKIP_LIMIT) {
+					this.sessionSyncFailures.delete(summary.id);
+					this.sessionSyncSkips.set(summary.id, {
+						at: Date.now(),
+						updatedAt: summary.updatedAt
+					});
+					this.log.warn("global search: giving up on a session whose wire transcript stays unreadable", {
+						sessionId: summary.id,
+						error: result.error
+					});
+				} else {
+					failures += 1;
+					this.log.warn("global search: failed to index session", {
+						sessionId: summary.id,
+						error: result.error
+					});
+				}
+			} else this.sessionSyncFailures.delete(summary.id);
+			indexed++;
 		}
-		if (this.disposed) return {
-			noop: true,
-			sessions: 0,
-			documents: 0
-		};
+		if (this.disposed) return noop;
 		const metaCount = db.query({
 			key: { prefix: "\0meta\\" },
 			project: []
@@ -12699,12 +12816,14 @@ var SearchIndexCore = class {
 			lastIndexedAt: Date.now()
 		};
 		await db.set(STATS_KEY, stats);
-		this.fullSyncDone = true;
+		if (!truncated && failures === 0) this.fullSyncDone = true;
 		if (this.syncReplaced) this.generation++;
 		return {
 			noop: false,
 			sessions: indexed,
-			documents: stats.documents
+			documents: stats.documents,
+			truncated,
+			failures
 		};
 	}
 	async migrateFileMetaKeys(db) {
@@ -12741,7 +12860,7 @@ var SearchIndexCore = class {
 		})) await db.del(row.key);
 		await db.del(SESSION_META_PREFIX + sessionId);
 	}
-	async syncSession(db, summary) {
+	async syncSession(db, summary, budget) {
 		const wireFiles = await collectWireFiles(summary.dir);
 		const seenPaths = new Set(wireFiles.map((file) => file.path));
 		for (const row of db.query({ key: { prefix: fileMetaPrefixFor(summary.id) } })) {
@@ -12751,7 +12870,17 @@ var SearchIndexCore = class {
 			await this.deleteFileDocs(db, meta);
 			await db.del(row.key);
 		}
-		for (const file of wireFiles) await this.syncWireFile(db, summary, file);
+		let truncated = false;
+		let failed = false;
+		let error;
+		for (const file of wireFiles) {
+			const result = await this.syncWireFile(db, summary, file, budget);
+			if (result.truncated) truncated = true;
+			if (result.failed) {
+				failed = true;
+				error ??= result.error;
+			}
+		}
 		const title = summary.title ?? "";
 		const titleKey = `${summary.id}/$title`;
 		const existing = db.get(titleKey);
@@ -12772,6 +12901,11 @@ var SearchIndexCore = class {
 			}
 		} else if (existing !== void 0) await db.del(titleKey);
 		if (db.get(SESSION_META_PREFIX + summary.id) === void 0) await db.set(SESSION_META_PREFIX + summary.id, { kind: "sessionMeta" });
+		return {
+			truncated,
+			failed,
+			error
+		};
 	}
 	async deleteFileDocs(db, meta) {
 		const prefix = `${meta.sessionId}/${meta.agentId}/${meta.source}:`;
@@ -12780,12 +12914,15 @@ var SearchIndexCore = class {
 			project: []
 		})) await db.del(row.key);
 	}
-	async syncWireFile(db, summary, file) {
+	async syncWireFile(db, summary, file, budget) {
 		let st;
 		try {
 			st = await stat(file.path);
 		} catch {
-			return;
+			return {
+				truncated: false,
+				failed: false
+			};
 		}
 		const size = st.size;
 		const metaKey = fileMetaKey(summary.id, file.path);
@@ -12839,22 +12976,51 @@ var SearchIndexCore = class {
 				});
 				await db.batch(ops);
 			}
-			return;
+			return {
+				truncated: false,
+				failed: false
+			};
+		}
+		let handle;
+		try {
+			handle = await open(file.path, "r");
+		} catch (error) {
+			return {
+				truncated: false,
+				failed: true,
+				error: errorMessage(error)
+			};
 		}
-		const handle = await open(file.path, "r");
 		const ops = [];
 		let byteCursor = offset;
+		let position = offset;
+		let wireError;
 		try {
-			let position = offset;
 			let pending = EMPTY_BUFFER;
+			let finishing = false;
 			const chunk = Buffer.allocUnsafe(WIRE_READ_CHUNK_BYTES);
 			while (position < size) {
-				if (this.disposed) return;
-				const { bytesRead } = await handle.read(chunk, 0, Math.min(chunk.length, size - position), position);
+				if (this.disposed) return {
+					truncated: false,
+					failed: false
+				};
+				if (syncBudgetExhausted(budget)) {
+					if (pending.length === 0) break;
+					finishing = true;
+				}
+				let bytesRead;
+				try {
+					({bytesRead} = await handle.read(chunk, 0, Math.min(chunk.length, size - position), position));
+				} catch (error) {
+					wireError = error;
+					break;
+				}
 				if (bytesRead === 0) break;
+				budget.bytesLeft -= bytesRead;
 				const slice = chunk.subarray(0, bytesRead);
 				position += bytesRead;
 				let start = 0;
+				let completedRecord = false;
 				for (;;) {
 					const nl = slice.indexOf(10, start);
 					if (nl === -1) break;
@@ -12866,10 +13032,24 @@ var SearchIndexCore = class {
 						turnState,
 						stepState
 					}));
+					completedRecord = true;
 					start = nl + 1;
 				}
 				pending = pending.length > 0 ? Buffer.concat([pending, slice.subarray(start)]) : Buffer.from(slice.subarray(start));
+				if (finishing && completedRecord) break;
 				if (ops.length >= WIRE_BATCH_OPS) {
+					ops.push({
+						op: "set",
+						key: metaKey,
+						value: fileMeta(byteCursor, turnState, stepState)
+					});
+					if (legacyKey !== null) {
+						ops.push({
+							op: "del",
+							key: legacyKey
+						});
+						legacyKey = null;
+					}
 					await db.batch(ops);
 					ops.length = 0;
 				}
@@ -12877,17 +13057,24 @@ var SearchIndexCore = class {
 		} finally {
 			await handle.close();
 		}
-		if (byteCursor === offset && legacyKey === null) return;
-		ops.push({
-			op: "set",
-			key: metaKey,
-			value: fileMeta(byteCursor, turnState, stepState)
-		});
-		if (legacyKey !== null) ops.push({
-			op: "del",
-			key: legacyKey
-		});
-		await db.batch(ops);
+		const truncated = position < size && syncBudgetExhausted(budget);
+		if (byteCursor !== offset || legacyKey !== null) {
+			ops.push({
+				op: "set",
+				key: metaKey,
+				value: fileMeta(byteCursor, turnState, stepState)
+			});
+			if (legacyKey !== null) ops.push({
+				op: "del",
+				key: legacyKey
+			});
+			await db.batch(ops);
+		}
+		return {
+			truncated,
+			failed: wireError !== void 0,
+			error: wireError !== void 0 ? errorMessage(wireError) : void 0
+		};
 	}
 	collectWireLine(ops, summary, file, line, lineOffset, counters) {
 		let { turnState, stepState } = counters;
@@ -13042,16 +13229,16 @@ var SearchIndexCore = class {
 		}
 		this.openPromise = null;
 		this.fullSyncDone = false;
+		this.sessionSyncFailures.clear();
+		this.sessionSyncSkips.clear();
 		this.lockToken = void 0;
-		await rm(this.indexDir, {
-			recursive: true,
-			force: true
-		});
+		if (await wipeStoreDir({ dir: this.indexDir }) === "locked") throw new GlobalSearchError("readonly_index", "another process holds the search-index write lock; reindex from that process");
 		await this.ensureOpen();
 	}
 	lifecycleState() {
 		if (this.disposed) return { state: this.db === null ? "stopped" : "closing" };
 		const db = this.db;
+		if (db !== null) this.noteMaintenanceDetail(db);
 		if (db === null) {
 			if (this.openPromise !== null) return { state: "opening" };
 			if (this.openError !== null) return {
@@ -13063,10 +13250,23 @@ var SearchIndexCore = class {
 		if (db.textIndexBuilding(TEXT_INDEX_NAME) || db.textIndexBuilding(TRI_INDEX_NAME)) return { state: "building" };
 		return { state: "ready" };
 	}
+	maintenanceDetail(db) {
+		const failure = db.lastCompactError ?? db.lastGenBuildError;
+		return failure === null ? void 0 : errorMessage(failure);
+	}
+	noteMaintenanceDetail(db) {
+		const detail = this.maintenanceDetail(db);
+		if (detail === this.lastMaintenanceDetail) return;
+		const previous = this.lastMaintenanceDetail;
+		this.lastMaintenanceDetail = detail;
+		if (detail !== void 0) this.log.warn("global search: index maintenance is failing; the WAL keeps growing until it recovers", { error: detail });
+		else if (previous !== void 0) this.log.info("global search: index maintenance recovered");
+	}
 	async status() {
 		await this.ensureOpen();
 		if (this.db?.readOnly === true) await this.refresh();
 		const stats = this.db?.get(STATS_KEY);
+		const maintenance = this.db !== null ? this.maintenanceDetail(this.db) : void 0;
 		return {
 			sessions: stats?.kind === "stats" ? stats.sessions : 0,
 			documents: stats?.kind === "stats" ? stats.documents : 0,
@@ -13074,7 +13274,7 @@ var SearchIndexCore = class {
 			generation: this.generation,
 			readOnly: this.db?.readOnly === true,
 			lockToken: this.lockToken,
-			degraded: this.lastRefreshError?.message,
+			degraded: this.lastRefreshError?.message ?? maintenance,
 			lifecycle: this.lifecycleState()
 		};
 	}
@@ -13137,9 +13337,6 @@ async function collectWireFiles(sessionDir) {
 function docKeyPrefix(sessionId, file) {
 	return `${sessionId}/${file.agentId}/${file.source}:`;
 }
-function isRebuildableCorruption(error) {
-	return error instanceof SyntaxError || error !== null && typeof error === "object" && error.name === "CorruptFrameError";
-}
 
 //#endregion
 //#region ../../packages/kap-server/src/search/worker/entry.ts