knodin 0.7.4 → 0.7.6

package/dist/src/repair-progress.js

@@ -1,262 +0,0 @@
-const DEFAULT_PROGRESS_INTERVAL_MS = 30_000;
-const TTY_THROTTLE_MS = 100;
-const CLEAR_TTY_LINE = "\r\x1b[2K";
-const TERMINAL_PHASES = new Set(["completed", "cancelled", "failed"]);
-const RESPONSE_BUDGET_FLAGS = new Set(["--bytes", "--tokens", "--items"]);
-function parseDuration(value) {
-    const match = /^(\d+(?:\.\d+)?)(ms|s|m)$/.exec(value.trim());
-    if (!match) {
-        throw new Error("knodin repair: --progress-interval must be a positive duration such as 750ms, 30s, or 2m");
-    }
-    const amount = Number(match[1]);
-    const multiplier = match[2] === "ms" ? 1 : match[2] === "s" ? 1_000 : 60_000;
-    const milliseconds = amount * multiplier;
-    if (!Number.isFinite(milliseconds) || milliseconds <= 0) {
-        throw new Error("knodin repair: --progress-interval must be a positive duration");
-    }
-    return milliseconds;
-}
-export function parseRepairCliArgs(args) {
-    let progress = "auto";
-    let progressExplicit = false;
-    let progressIntervalMs = DEFAULT_PROGRESS_INTERVAL_MS;
-    let json = false;
-    let jsonl = false;
-    let plan = false;
-    const seen = new Set();
-    const markSeen = (flag) => {
-        if (seen.has(flag))
-            throw new Error(`knodin repair: repeated ${flag} argument`);
-        seen.add(flag);
-    };
-    const consumeValue = (flag, index) => {
-        const value = args[index + 1];
-        if (!value || value.startsWith("--"))
-            throw new Error(`knodin repair: ${flag} requires a value`);
-        return value;
-    };
-    for (let index = 0; index < args.length; index++) {
-        const argument = args[index];
-        if (argument === "--plan") {
-            markSeen(argument);
-            plan = true;
-            continue;
-        }
-        if (argument === "--json" || argument === "--jsonl") {
-            markSeen(argument);
-            if (argument === "--json")
-                json = true;
-            else
-                jsonl = true;
-            continue;
-        }
-        if (argument === "--progress" || argument.startsWith("--progress=")) {
-            markSeen("--progress");
-            const value = argument === "--progress"
-                ? consumeValue("--progress", index++)
-                : argument.slice("--progress=".length);
-            if (!value)
-                throw new Error("knodin repair: --progress requires a value");
-            if (!["auto", "tty", "plain", "none"].includes(value)) {
-                throw new Error("knodin repair: --progress must be auto, tty, plain, or none");
-            }
-            progress = value;
-            progressExplicit = true;
-            continue;
-        }
-        if (argument === "--progress-interval" || argument.startsWith("--progress-interval=")) {
-            markSeen("--progress-interval");
-            const value = argument === "--progress-interval"
-                ? consumeValue("--progress-interval", index++)
-                : argument.slice("--progress-interval=".length);
-            if (!value)
-                throw new Error("knodin repair: --progress-interval requires a value");
-            progressIntervalMs = parseDuration(value);
-            continue;
-        }
-        if (RESPONSE_BUDGET_FLAGS.has(argument)) {
-            markSeen(argument);
-            consumeValue(argument, index++);
-            continue;
-        }
-        if (argument.startsWith("--")) {
-            throw new Error(`knodin repair: unknown argument ${argument}`);
-        }
-        throw new Error(`knodin repair: unexpected positional argument ${argument}`);
-    }
-    if (json && jsonl)
-        throw new Error("knodin repair: --json and --jsonl are mutually exclusive");
-    return {
-        progress,
-        progressExplicit,
-        progressIntervalMs,
-        output: jsonl ? "jsonl" : json ? "json" : "human",
-        plan,
-    };
-}
-export function createRepairPlan(health) {
-    const staleRecords = health.missing.records.filter((record) => /^(indexed file no longer exists|indexed file is no longer eligible): /.test(record));
-    const damagedSnapshots = health.missing.records.filter((record) => record.startsWith("indexed snapshot differs from disk: "));
-    const incompletePostProcessing = health.missing.records.filter((record) => record.includes("post-processing is incomplete"));
-    const fileReconciliations = new Set([
-        ...health.missing.files,
-        ...staleRecords.map((record) => record.slice(record.indexOf(": ") + 2)),
-        ...damagedSnapshots.map((record) => record.slice(record.indexOf(": ") + 2)),
-    ]);
-    const orphanGroups = Object.values(health.orphaned).filter((count) => count > 0).length;
-    return {
-        schemaVersion: 1,
-        mutating: false,
-        healthFindings: {
-            missingFiles: health.missing.files.length,
-            damagedSnapshots: damagedSnapshots.length,
-            staleOrIneligibleRecords: staleRecords.length,
-            orphanedEmbeddings: health.orphaned.embeddings,
-            orphanedReferences: health.orphaned.references,
-            orphanedDependencies: health.orphaned.dependencies,
-            incompletePostProcessing: incompletePostProcessing.length,
-            total: health.missing.files.length +
-                health.missing.records.length +
-                Object.values(health.orphaned).reduce((sum, count) => sum + count, 0),
-        },
-        operations: {
-            fileReconciliations: fileReconciliations.size,
-            removals: staleRecords.length,
-            identityPasses: fileReconciliations.size > 0 ? 1 : 0,
-            embeddingCandidates: fileReconciliations.size,
-            orphanCleanupGroups: orphanGroups,
-            verification: 1,
-            total: fileReconciliations.size +
-                staleRecords.length +
-                (fileReconciliations.size > 0 ? 1 : 0) +
-                fileReconciliations.size +
-                orphanGroups +
-                1,
-        },
-        explanation: "Health findings count observed problems; operations count bounded repair actions, so the totals need not match one-to-one.",
-        health,
-    };
-}
-export function resolveRepairProgressMode(options, environment, stderrIsTTY) {
-    if (options.output === "jsonl")
-        return "jsonl";
-    if (options.output === "json" && !options.progressExplicit)
-        return "none";
-    if (options.progress !== "auto")
-        return options.progress;
-    const ci = environment.CI;
-    const runsInCi = ci !== undefined && ci !== "" && ci !== "0" && ci.toLowerCase() !== "false";
-    return stderrIsTTY && !runsInCi && environment.TERM?.toLowerCase() !== "dumb" ? "tty" : "plain";
-}
-export function formatRepairProgress(event) {
-    const total = event.phaseTotal === undefined
-        ? `${event.phaseCompleted}`
-        : `${event.phaseCompleted}/${event.phaseTotal}`;
-    const location = [event.currentFamily, event.currentPath].filter(Boolean).join(" ");
-    const suffix = location ? ` — ${location}` : "";
-    return `[repair:${event.phase}] ${total} ${event.message}${suffix}`;
-}
-export function createRepairProgressRenderer(options) {
-    const now = options.now ?? Date.now;
-    const schedule = options.setTimeout ??
-        ((callback, intervalMs) => setTimeout(callback, intervalMs));
-    const unschedule = options.clearTimeout ?? ((handle) => clearTimeout(handle));
-    let heartbeatHandle;
-    let started = false;
-    let stopped = false;
-    let ttyHasLine = false;
-    let lastWriteAt = Number.NEGATIVE_INFINITY;
-    let lastEvent;
-    let lastPhase;
-    let lastPercentBucket = -1;
-    const scheduleHeartbeat = () => {
-        if (options.mode !== "plain" && options.mode !== "tty")
-            return;
-        if (heartbeatHandle !== undefined)
-            unschedule(heartbeatHandle);
-        const heartbeatDelay = options.mode === "tty" ? Math.max(options.intervalMs, TTY_THROTTLE_MS) : options.intervalMs;
-        heartbeatHandle = schedule(() => {
-            heartbeatHandle = undefined;
-            if (stopped)
-                return;
-            if (lastEvent) {
-                if (options.mode === "plain")
-                    writePlain(`${formatRepairProgress(lastEvent)} (heartbeat)`);
-                else
-                    writeTty(`${formatRepairProgress(lastEvent)} (heartbeat)`);
-            }
-            scheduleHeartbeat();
-        }, heartbeatDelay);
-        if (typeof heartbeatHandle === "object" &&
-            heartbeatHandle !== null &&
-            "unref" in heartbeatHandle) {
-            heartbeatHandle.unref();
-        }
-    };
-    const writePlain = (line) => {
-        options.stderr.write(`${line}\n`);
-        lastWriteAt = now();
-        scheduleHeartbeat();
-    };
-    const writeTty = (line) => {
-        options.stderr.write(`${CLEAR_TTY_LINE}${line}`);
-        ttyHasLine = true;
-        lastWriteAt = now();
-        scheduleHeartbeat();
-    };
-    return {
-        start() {
-            if (started || stopped)
-                return;
-            started = true;
-            if (options.mode === "tty")
-                writeTty("Starting repair…");
-            else if (options.mode === "plain")
-                writePlain("[repair] Starting repair");
-        },
-        onProgress(event) {
-            if (stopped)
-                return;
-            if (!started)
-                this.start();
-            lastEvent = event;
-            if (options.mode === "none")
-                return;
-            if (options.mode === "jsonl") {
-                options.stdout.write(serializeRepairJsonlRecord("progress", event));
-                return;
-            }
-            if (options.mode === "tty") {
-                if (now() - lastWriteAt < TTY_THROTTLE_MS && !TERMINAL_PHASES.has(event.phase))
-                    return;
-                writeTty(formatRepairProgress(event));
-                return;
-            }
-            const phaseChanged = event.phase !== lastPhase;
-            const percentage = event.phaseTotal && event.phaseTotal > 0
-                ? Math.floor((event.phaseCompleted / event.phaseTotal) * 100)
-                : undefined;
-            const percentBucket = percentage === undefined ? -1 : Math.floor(percentage / 5);
-            if (phaseChanged || percentBucket > lastPercentBucket) {
-                writePlain(formatRepairProgress(event));
-                if (phaseChanged)
-                    lastPercentBucket = percentBucket;
-                else
-                    lastPercentBucket = Math.max(lastPercentBucket, percentBucket);
-                lastPhase = event.phase;
-            }
-        },
-        stop() {
-            if (stopped)
-                return;
-            stopped = true;
-            if (heartbeatHandle !== undefined)
-                unschedule(heartbeatHandle);
-            if (options.mode === "tty" && ttyHasLine)
-                options.stderr.write("\n");
-        },
-    };
-}
-export function serializeRepairJsonlRecord(type, value) {
-    return `${JSON.stringify(type === "progress" ? { type, event: value } : { type, result: value })}\n`;
-}

package/dist/src/repository-init-process.js

@@ -1,173 +0,0 @@
-import { spawn } from "node:child_process";
-import process from "node:process";
-export const DEFAULT_REPOSITORY_INIT_MEMORY_LIMIT_BYTES = 768 * 1024 * 1024;
-const MAX_DIAGNOSTIC_BYTES = 64 * 1024;
-const TERMINATION_GRACE_MS = 1_000;
-const RSS_POLL_MS = 250;
-function boundedAppend(current, chunk) {
-    if (current.length >= MAX_DIAGNOSTIC_BYTES)
-        return current;
-    return `${current}${chunk.toString("utf-8")}`.slice(0, MAX_DIAGNOSTIC_BYTES);
-}
-function terminateProcessTree(pid) {
-    if (!Number.isSafeInteger(pid) || pid <= 1)
-        return;
-    try {
-        process.kill(process.platform === "win32" ? pid : -pid, "SIGTERM");
-    }
-    catch {
-        // The worker may have exited between observation and termination.
-    }
-}
-function forceTerminateProcessTree(pid) {
-    if (!Number.isSafeInteger(pid) || pid <= 1)
-        return;
-    try {
-        process.kill(process.platform === "win32" ? pid : -pid, "SIGKILL");
-    }
-    catch {
-        // Already terminated.
-    }
-}
-function isWorkerMessage(value) {
-    if (!value || typeof value !== "object")
-        return false;
-    const record = value;
-    const result = record.result;
-    return ((record.type === "rss" &&
-        Number.isSafeInteger(record.rssBytes) &&
-        Number(record.rssBytes) >= 0) ||
-        (record.type === "result" &&
-            Boolean(result) &&
-            typeof result?.repository === "string" &&
-            typeof result.message === "string" &&
-            ["initialized", "updated", "already-current", "skipped", "failed"].includes(String(result.status))));
-}
-/**
- * Initialize one repository in a disposable process. The parent observes RSS
- * heartbeats, kills the whole process group on a ceiling breach, and converts
- * every worker failure into a repository-scoped result so the portfolio can
- * continue.
- */
-export async function runRepositoryInitializationProcess(options) {
-    const memoryLimitBytes = options.memoryLimitBytes ?? DEFAULT_REPOSITORY_INIT_MEMORY_LIMIT_BYTES;
-    if (!Number.isSafeInteger(memoryLimitBytes) || memoryLimitBytes < 128 * 1024 * 1024)
-        throw new Error("repository init memory limit must be an integer of at least 128 MiB");
-    if (options.command.length === 0 || !options.command[0])
-        throw new Error("repository init worker command is empty");
-    return await new Promise((resolve) => {
-        let stderr = "";
-        let peakRssBytes = 0;
-        let result;
-        let exceeded = false;
-        let timedOut = false;
-        let settled = false;
-        let rssPollRunning = false;
-        const child = spawn(options.command[0], [...options.command.slice(1), "__repository-init-worker", options.repository], {
-            cwd: options.repository,
-            detached: process.platform !== "win32",
-            env: {
-                ...process.env,
-                KNODIN_INTERNAL_INIT_MEMORY_LIMIT_BYTES: String(memoryLimitBytes),
-            },
-            stdio: ["ignore", "ignore", "pipe", "ipc"],
-        });
-        child.stderr?.on("data", (chunk) => {
-            stderr = boundedAppend(stderr, chunk);
-        });
-        const observeRss = (rssBytes) => {
-            peakRssBytes = Math.max(peakRssBytes, rssBytes);
-            if (rssBytes <= memoryLimitBytes || exceeded)
-                return;
-            exceeded = true;
-            const pid = child.pid ?? 0;
-            terminateProcessTree(pid);
-            setTimeout(() => forceTerminateProcessTree(pid), TERMINATION_GRACE_MS).unref();
-        };
-        child.on("message", (message) => {
-            if (!isWorkerMessage(message))
-                return;
-            if (message.type === "result") {
-                result = message.result;
-                return;
-            }
-            observeRss(message.rssBytes);
-        });
-        const rssPoll = setInterval(() => {
-            const pid = child.pid;
-            if (process.platform === "win32" || !pid || rssPollRunning || settled)
-                return;
-            rssPollRunning = true;
-            const probe = spawn("/bin/ps", ["-o", "rss=", "-p", String(pid)], {
-                stdio: ["ignore", "pipe", "ignore"],
-            });
-            let output = "";
-            probe.stdout?.on("data", (chunk) => {
-                output = boundedAppend(output, chunk);
-            });
-            probe.on("close", () => {
-                rssPollRunning = false;
-                if (settled)
-                    return;
-                const rssKiB = Number.parseInt(output.trim(), 10);
-                if (Number.isSafeInteger(rssKiB) && rssKiB >= 0)
-                    observeRss(rssKiB * 1024);
-            });
-            probe.on("error", () => {
-                rssPollRunning = false;
-            });
-        }, RSS_POLL_MS);
-        rssPoll.unref();
-        child.on("error", (error) => {
-            stderr = boundedAppend(stderr, Buffer.from(error.message));
-        });
-        const timeout = setTimeout(() => {
-            timedOut = true;
-            const pid = child.pid ?? 0;
-            terminateProcessTree(pid);
-            setTimeout(() => forceTerminateProcessTree(pid), TERMINATION_GRACE_MS).unref();
-        }, options.timeoutMs ?? 30 * 60_000);
-        timeout.unref();
-        child.on("close", (code, signal) => {
-            if (settled)
-                return;
-            settled = true;
-            clearTimeout(timeout);
-            clearInterval(rssPoll);
-            if (exceeded) {
-                resolve({
-                    repository: options.repository,
-                    status: "failed",
-                    message: `memory ceiling exceeded (${peakRssBytes} > ${memoryLimitBytes} bytes); worker terminated`,
-                    peakRssBytes,
-                    memoryLimitBytes,
-                });
-                return;
-            }
-            if (timedOut) {
-                resolve({
-                    repository: options.repository,
-                    status: "failed",
-                    message: "isolated initialization timed out; worker process tree terminated",
-                    peakRssBytes,
-                    memoryLimitBytes,
-                });
-                return;
-            }
-            if (result) {
-                resolve({ ...result, repository: options.repository, peakRssBytes, memoryLimitBytes });
-                return;
-            }
-            const diagnostic = stderr.trim().replace(/\s+/g, " ");
-            const outcome = signal ? `signal ${signal}` : `exit ${code ?? "unknown"}`;
-            const detail = diagnostic ? `: ${diagnostic}` : "";
-            resolve({
-                repository: options.repository,
-                status: "failed",
-                message: `isolated initialization failed (${outcome})${detail}`,
-                peakRssBytes,
-                memoryLimitBytes,
-            });
-        });
-    });
-}