knodin 0.5.0

package/dist/src/repair-progress.js

@@ -0,0 +1,262 @@
+const DEFAULT_PROGRESS_INTERVAL_MS = 30_000;
+const TTY_THROTTLE_MS = 100;
+const CLEAR_TTY_LINE = "\r\x1b[2K";
+const TERMINAL_PHASES = new Set(["completed", "cancelled", "failed"]);
+const RESPONSE_BUDGET_FLAGS = new Set(["--bytes", "--tokens", "--items"]);
+function parseDuration(value) {
+    const match = /^(\d+(?:\.\d+)?)(ms|s|m)$/.exec(value.trim());
+    if (!match) {
+        throw new Error("knodin repair: --progress-interval must be a positive duration such as 750ms, 30s, or 2m");
+    }
+    const amount = Number(match[1]);
+    const multiplier = match[2] === "ms" ? 1 : match[2] === "s" ? 1_000 : 60_000;
+    const milliseconds = amount * multiplier;
+    if (!Number.isFinite(milliseconds) || milliseconds <= 0) {
+        throw new Error("knodin repair: --progress-interval must be a positive duration");
+    }
+    return milliseconds;
+}
+export function parseRepairCliArgs(args) {
+    let progress = "auto";
+    let progressExplicit = false;
+    let progressIntervalMs = DEFAULT_PROGRESS_INTERVAL_MS;
+    let json = false;
+    let jsonl = false;
+    let plan = false;
+    const seen = new Set();
+    const markSeen = (flag) => {
+        if (seen.has(flag))
+            throw new Error(`knodin repair: repeated ${flag} argument`);
+        seen.add(flag);
+    };
+    const consumeValue = (flag, index) => {
+        const value = args[index + 1];
+        if (!value || value.startsWith("--"))
+            throw new Error(`knodin repair: ${flag} requires a value`);
+        return value;
+    };
+    for (let index = 0; index < args.length; index++) {
+        const argument = args[index];
+        if (argument === "--plan") {
+            markSeen(argument);
+            plan = true;
+            continue;
+        }
+        if (argument === "--json" || argument === "--jsonl") {
+            markSeen(argument);
+            if (argument === "--json")
+                json = true;
+            else
+                jsonl = true;
+            continue;
+        }
+        if (argument === "--progress" || argument.startsWith("--progress=")) {
+            markSeen("--progress");
+            const value = argument === "--progress"
+                ? consumeValue("--progress", index++)
+                : argument.slice("--progress=".length);
+            if (!value)
+                throw new Error("knodin repair: --progress requires a value");
+            if (!["auto", "tty", "plain", "none"].includes(value)) {
+                throw new Error("knodin repair: --progress must be auto, tty, plain, or none");
+            }
+            progress = value;
+            progressExplicit = true;
+            continue;
+        }
+        if (argument === "--progress-interval" || argument.startsWith("--progress-interval=")) {
+            markSeen("--progress-interval");
+            const value = argument === "--progress-interval"
+                ? consumeValue("--progress-interval", index++)
+                : argument.slice("--progress-interval=".length);
+            if (!value)
+                throw new Error("knodin repair: --progress-interval requires a value");
+            progressIntervalMs = parseDuration(value);
+            continue;
+        }
+        if (RESPONSE_BUDGET_FLAGS.has(argument)) {
+            markSeen(argument);
+            consumeValue(argument, index++);
+            continue;
+        }
+        if (argument.startsWith("--")) {
+            throw new Error(`knodin repair: unknown argument ${argument}`);
+        }
+        throw new Error(`knodin repair: unexpected positional argument ${argument}`);
+    }
+    if (json && jsonl)
+        throw new Error("knodin repair: --json and --jsonl are mutually exclusive");
+    return {
+        progress,
+        progressExplicit,
+        progressIntervalMs,
+        output: jsonl ? "jsonl" : json ? "json" : "human",
+        plan,
+    };
+}
+export function createRepairPlan(health) {
+    const staleRecords = health.missing.records.filter((record) => /^(indexed file no longer exists|indexed file is no longer eligible): /.test(record));
+    const damagedSnapshots = health.missing.records.filter((record) => record.startsWith("indexed snapshot differs from disk: "));
+    const incompletePostProcessing = health.missing.records.filter((record) => record.includes("post-processing is incomplete"));
+    const fileReconciliations = new Set([
+        ...health.missing.files,
+        ...staleRecords.map((record) => record.slice(record.indexOf(": ") + 2)),
+        ...damagedSnapshots.map((record) => record.slice(record.indexOf(": ") + 2)),
+    ]);
+    const orphanGroups = Object.values(health.orphaned).filter((count) => count > 0).length;
+    return {
+        schemaVersion: 1,
+        mutating: false,
+        healthFindings: {
+            missingFiles: health.missing.files.length,
+            damagedSnapshots: damagedSnapshots.length,
+            staleOrIneligibleRecords: staleRecords.length,
+            orphanedEmbeddings: health.orphaned.embeddings,
+            orphanedReferences: health.orphaned.references,
+            orphanedDependencies: health.orphaned.dependencies,
+            incompletePostProcessing: incompletePostProcessing.length,
+            total: health.missing.files.length +
+                health.missing.records.length +
+                Object.values(health.orphaned).reduce((sum, count) => sum + count, 0),
+        },
+        operations: {
+            fileReconciliations: fileReconciliations.size,
+            removals: staleRecords.length,
+            identityPasses: fileReconciliations.size > 0 ? 1 : 0,
+            embeddingCandidates: fileReconciliations.size,
+            orphanCleanupGroups: orphanGroups,
+            verification: 1,
+            total: fileReconciliations.size +
+                staleRecords.length +
+                (fileReconciliations.size > 0 ? 1 : 0) +
+                fileReconciliations.size +
+                orphanGroups +
+                1,
+        },
+        explanation: "Health findings count observed problems; operations count bounded repair actions, so the totals need not match one-to-one.",
+        health,
+    };
+}
+export function resolveRepairProgressMode(options, environment, stderrIsTTY) {
+    if (options.output === "jsonl")
+        return "jsonl";
+    if (options.output === "json" && !options.progressExplicit)
+        return "none";
+    if (options.progress !== "auto")
+        return options.progress;
+    const ci = environment.CI;
+    const runsInCi = ci !== undefined && ci !== "" && ci !== "0" && ci.toLowerCase() !== "false";
+    return stderrIsTTY && !runsInCi && environment.TERM?.toLowerCase() !== "dumb" ? "tty" : "plain";
+}
+export function formatRepairProgress(event) {
+    const total = event.phaseTotal === undefined
+        ? `${event.phaseCompleted}`
+        : `${event.phaseCompleted}/${event.phaseTotal}`;
+    const location = [event.currentFamily, event.currentPath].filter(Boolean).join(" ");
+    const suffix = location ? ` — ${location}` : "";
+    return `[repair:${event.phase}] ${total} ${event.message}${suffix}`;
+}
+export function createRepairProgressRenderer(options) {
+    const now = options.now ?? Date.now;
+    const schedule = options.setTimeout ??
+        ((callback, intervalMs) => setTimeout(callback, intervalMs));
+    const unschedule = options.clearTimeout ?? ((handle) => clearTimeout(handle));
+    let heartbeatHandle;
+    let started = false;
+    let stopped = false;
+    let ttyHasLine = false;
+    let lastWriteAt = Number.NEGATIVE_INFINITY;
+    let lastEvent;
+    let lastPhase;
+    let lastPercentBucket = -1;
+    const scheduleHeartbeat = () => {
+        if (options.mode !== "plain" && options.mode !== "tty")
+            return;
+        if (heartbeatHandle !== undefined)
+            unschedule(heartbeatHandle);
+        const heartbeatDelay = options.mode === "tty" ? Math.max(options.intervalMs, TTY_THROTTLE_MS) : options.intervalMs;
+        heartbeatHandle = schedule(() => {
+            heartbeatHandle = undefined;
+            if (stopped)
+                return;
+            if (lastEvent) {
+                if (options.mode === "plain")
+                    writePlain(`${formatRepairProgress(lastEvent)} (heartbeat)`);
+                else
+                    writeTty(`${formatRepairProgress(lastEvent)} (heartbeat)`);
+            }
+            scheduleHeartbeat();
+        }, heartbeatDelay);
+        if (typeof heartbeatHandle === "object" &&
+            heartbeatHandle !== null &&
+            "unref" in heartbeatHandle) {
+            heartbeatHandle.unref();
+        }
+    };
+    const writePlain = (line) => {
+        options.stderr.write(`${line}\n`);
+        lastWriteAt = now();
+        scheduleHeartbeat();
+    };
+    const writeTty = (line) => {
+        options.stderr.write(`${CLEAR_TTY_LINE}${line}`);
+        ttyHasLine = true;
+        lastWriteAt = now();
+        scheduleHeartbeat();
+    };
+    return {
+        start() {
+            if (started || stopped)
+                return;
+            started = true;
+            if (options.mode === "tty")
+                writeTty("Starting repair…");
+            else if (options.mode === "plain")
+                writePlain("[repair] Starting repair");
+        },
+        onProgress(event) {
+            if (stopped)
+                return;
+            if (!started)
+                this.start();
+            lastEvent = event;
+            if (options.mode === "none")
+                return;
+            if (options.mode === "jsonl") {
+                options.stdout.write(serializeRepairJsonlRecord("progress", event));
+                return;
+            }
+            if (options.mode === "tty") {
+                if (now() - lastWriteAt < TTY_THROTTLE_MS && !TERMINAL_PHASES.has(event.phase))
+                    return;
+                writeTty(formatRepairProgress(event));
+                return;
+            }
+            const phaseChanged = event.phase !== lastPhase;
+            const percentage = event.phaseTotal && event.phaseTotal > 0
+                ? Math.floor((event.phaseCompleted / event.phaseTotal) * 100)
+                : undefined;
+            const percentBucket = percentage === undefined ? -1 : Math.floor(percentage / 5);
+            if (phaseChanged || percentBucket > lastPercentBucket) {
+                writePlain(formatRepairProgress(event));
+                if (phaseChanged)
+                    lastPercentBucket = percentBucket;
+                else
+                    lastPercentBucket = Math.max(lastPercentBucket, percentBucket);
+                lastPhase = event.phase;
+            }
+        },
+        stop() {
+            if (stopped)
+                return;
+            stopped = true;
+            if (heartbeatHandle !== undefined)
+                unschedule(heartbeatHandle);
+            if (options.mode === "tty" && ttyHasLine)
+                options.stderr.write("\n");
+        },
+    };
+}
+export function serializeRepairJsonlRecord(type, value) {
+    return `${JSON.stringify(type === "progress" ? { type, event: value } : { type, result: value })}\n`;
+}

package/dist/src/repository-init-process.js

@@ -0,0 +1,173 @@
+import { spawn } from "node:child_process";
+import process from "node:process";
+export const DEFAULT_REPOSITORY_INIT_MEMORY_LIMIT_BYTES = 768 * 1024 * 1024;
+const MAX_DIAGNOSTIC_BYTES = 64 * 1024;
+const TERMINATION_GRACE_MS = 1_000;
+const RSS_POLL_MS = 250;
+function boundedAppend(current, chunk) {
+    if (current.length >= MAX_DIAGNOSTIC_BYTES)
+        return current;
+    return `${current}${chunk.toString("utf-8")}`.slice(0, MAX_DIAGNOSTIC_BYTES);
+}
+function terminateProcessTree(pid) {
+    if (!Number.isSafeInteger(pid) || pid <= 1)
+        return;
+    try {
+        process.kill(process.platform === "win32" ? pid : -pid, "SIGTERM");
+    }
+    catch {
+        // The worker may have exited between observation and termination.
+    }
+}
+function forceTerminateProcessTree(pid) {
+    if (!Number.isSafeInteger(pid) || pid <= 1)
+        return;
+    try {
+        process.kill(process.platform === "win32" ? pid : -pid, "SIGKILL");
+    }
+    catch {
+        // Already terminated.
+    }
+}
+function isWorkerMessage(value) {
+    if (!value || typeof value !== "object")
+        return false;
+    const record = value;
+    const result = record.result;
+    return ((record.type === "rss" &&
+        Number.isSafeInteger(record.rssBytes) &&
+        Number(record.rssBytes) >= 0) ||
+        (record.type === "result" &&
+            Boolean(result) &&
+            typeof result?.repository === "string" &&
+            typeof result.message === "string" &&
+            ["initialized", "updated", "already-current", "skipped", "failed"].includes(String(result.status))));
+}
+/**
+ * Initialize one repository in a disposable process. The parent observes RSS
+ * heartbeats, kills the whole process group on a ceiling breach, and converts
+ * every worker failure into a repository-scoped result so the portfolio can
+ * continue.
+ */
+export async function runRepositoryInitializationProcess(options) {
+    const memoryLimitBytes = options.memoryLimitBytes ?? DEFAULT_REPOSITORY_INIT_MEMORY_LIMIT_BYTES;
+    if (!Number.isSafeInteger(memoryLimitBytes) || memoryLimitBytes < 128 * 1024 * 1024)
+        throw new Error("repository init memory limit must be an integer of at least 128 MiB");
+    if (options.command.length === 0 || !options.command[0])
+        throw new Error("repository init worker command is empty");
+    return await new Promise((resolve) => {
+        let stderr = "";
+        let peakRssBytes = 0;
+        let result;
+        let exceeded = false;
+        let timedOut = false;
+        let settled = false;
+        let rssPollRunning = false;
+        const child = spawn(options.command[0], [...options.command.slice(1), "__repository-init-worker", options.repository], {
+            cwd: options.repository,
+            detached: process.platform !== "win32",
+            env: {
+                ...process.env,
+                RECKON_INTERNAL_INIT_MEMORY_LIMIT_BYTES: String(memoryLimitBytes),
+            },
+            stdio: ["ignore", "ignore", "pipe", "ipc"],
+        });
+        child.stderr?.on("data", (chunk) => {
+            stderr = boundedAppend(stderr, chunk);
+        });
+        const observeRss = (rssBytes) => {
+            peakRssBytes = Math.max(peakRssBytes, rssBytes);
+            if (rssBytes <= memoryLimitBytes || exceeded)
+                return;
+            exceeded = true;
+            const pid = child.pid ?? 0;
+            terminateProcessTree(pid);
+            setTimeout(() => forceTerminateProcessTree(pid), TERMINATION_GRACE_MS).unref();
+        };
+        child.on("message", (message) => {
+            if (!isWorkerMessage(message))
+                return;
+            if (message.type === "result") {
+                result = message.result;
+                return;
+            }
+            observeRss(message.rssBytes);
+        });
+        const rssPoll = setInterval(() => {
+            const pid = child.pid;
+            if (process.platform === "win32" || !pid || rssPollRunning || settled)
+                return;
+            rssPollRunning = true;
+            const probe = spawn("/bin/ps", ["-o", "rss=", "-p", String(pid)], {
+                stdio: ["ignore", "pipe", "ignore"],
+            });
+            let output = "";
+            probe.stdout?.on("data", (chunk) => {
+                output = boundedAppend(output, chunk);
+            });
+            probe.on("close", () => {
+                rssPollRunning = false;
+                if (settled)
+                    return;
+                const rssKiB = Number.parseInt(output.trim(), 10);
+                if (Number.isSafeInteger(rssKiB) && rssKiB >= 0)
+                    observeRss(rssKiB * 1024);
+            });
+            probe.on("error", () => {
+                rssPollRunning = false;
+            });
+        }, RSS_POLL_MS);
+        rssPoll.unref();
+        child.on("error", (error) => {
+            stderr = boundedAppend(stderr, Buffer.from(error.message));
+        });
+        const timeout = setTimeout(() => {
+            timedOut = true;
+            const pid = child.pid ?? 0;
+            terminateProcessTree(pid);
+            setTimeout(() => forceTerminateProcessTree(pid), TERMINATION_GRACE_MS).unref();
+        }, options.timeoutMs ?? 30 * 60_000);
+        timeout.unref();
+        child.on("close", (code, signal) => {
+            if (settled)
+                return;
+            settled = true;
+            clearTimeout(timeout);
+            clearInterval(rssPoll);
+            if (exceeded) {
+                resolve({
+                    repository: options.repository,
+                    status: "failed",
+                    message: `memory ceiling exceeded (${peakRssBytes} > ${memoryLimitBytes} bytes); worker terminated`,
+                    peakRssBytes,
+                    memoryLimitBytes,
+                });
+                return;
+            }
+            if (timedOut) {
+                resolve({
+                    repository: options.repository,
+                    status: "failed",
+                    message: "isolated initialization timed out; worker process tree terminated",
+                    peakRssBytes,
+                    memoryLimitBytes,
+                });
+                return;
+            }
+            if (result) {
+                resolve({ ...result, repository: options.repository, peakRssBytes, memoryLimitBytes });
+                return;
+            }
+            const diagnostic = stderr.trim().replace(/\s+/g, " ");
+            const outcome = signal ? `signal ${signal}` : `exit ${code ?? "unknown"}`;
+            const detail = diagnostic ? `: ${diagnostic}` : "";
+            resolve({
+                repository: options.repository,
+                status: "failed",
+                message: `isolated initialization failed (${outcome})${detail}`,
+                peakRssBytes,
+                memoryLimitBytes,
+            });
+        });
+    });
+}