@astrosheep/keiyaku 2.9.8 → 2.9.10

package/build/core/settlement/verification.js

@@ -2,17 +2,41 @@ import { spawn } from "node:child_process";
 import { getConfig } from "../../config/env.js";
 import { assertSettingsKnobUsable } from "../../config/settings/disease.js";
 import { loadKeiyakuSettings } from "../../config/settings/loader.js";
-import { FlowError } from "../../flow-error.js";
-const AUDIT_OUTPUT_TAIL_BYTES = 8_192;
-function tail(value, limit = AUDIT_OUTPUT_TAIL_BYTES) {
-    if (value.byteLength <= limit) {
-        return { tail: value.toString("utf8"), truncated: false, omittedBytes: 0 };
+import { PROCESS_GROUP_KILL_GRACE_MS, ProcessTreeTerminationError, ownedProcessIdentity, ownedProcessIsLive, terminateOwnedProcess, terminateProcessGroup, } from "../process-group.js";
+import { VerificationCoordination } from "./verification-coordination.js";
+import { VERIFICATION_SUPERVISOR_PROTOCOL_VERSION, supervisorMessageSchema, verificationSupervisorArguments, } from "./verification-supervisor.js";
+const OUTPUT_TAIL_BYTES = 8_192;
+const COORDINATION_POLL_MS = 50;
+const COORDINATION_RECOVERY_MARGIN_MS = 60_000;
+class OutputTailCollector {
+    #chunks = [];
+    #retainedBytes = 0;
+    #totalBytes = 0;
+    push(chunk) {
+        const value = Buffer.from(chunk);
+        this.#totalBytes += value.byteLength;
+        this.#chunks.push(value);
+        this.#retainedBytes += value.byteLength;
+        while (this.#retainedBytes > OUTPUT_TAIL_BYTES && this.#chunks.length > 0) {
+            const first = this.#chunks[0];
+            const excess = this.#retainedBytes - OUTPUT_TAIL_BYTES;
+            if (first.byteLength <= excess) {
+                this.#chunks.shift();
+                this.#retainedBytes -= first.byteLength;
+            }
+            else {
+                this.#chunks[0] = first.subarray(excess);
+                this.#retainedBytes -= excess;
+            }
+        }
+    }
+    result() {
+        return {
+            tail: Buffer.concat(this.#chunks).toString("utf8"),
+            truncated: this.#totalBytes > this.#retainedBytes,
+            omittedBytes: this.#totalBytes - this.#retainedBytes,
+        };
     }
-    return {
-        tail: value.subarray(value.byteLength - limit).toString("utf8"),
-        truncated: true,
-        omittedBytes: value.byteLength - limit,
-    };
 }
 export function verificationExecutor(command) {
     const basename = command.command.split(/[\\/]/).pop() ?? "";
@@ -26,34 +50,377 @@ export function verificationExecutor(command) {
         return { cause: "missing_command_body" };
     return { executor, body };
 }
-export async function resolveVerificationCommands(cwd) {
+/** The document schema owns parsing and persistence; this only consumes its resolved declarations. */
+export function contractVerificationDeclarations(lifecycle) {
+    const bind = lifecycle.history.find(({ entry }) => entry.kind === "bind")?.entry;
+    return bind?.kind === "bind" ? bind.data.verification : [];
+}
+export function composeVerificationPlan(input) {
+    const contract = input.contract ?? [];
+    const timeoutMs = input.timeoutMs ?? 300_000;
+    return [
+        ...contract.map((declaration, index) => ({
+            source: "contract",
+            label: `Verification ${index + 1}`,
+            declaration,
+            command: declaration,
+            args: [],
+            shell: true,
+            timeoutMs,
+        })),
+        ...input.repositoryGates.map((gate) => ({
+            source: "repository-gate",
+            label: gate.label,
+            declaration: gate.label,
+            command: gate.command,
+            args: [...gate.args],
+            shell: false,
+            timeoutMs,
+        })),
+    ];
+}
+export async function resolveVerificationPolicy(cwd) {
     const loaded = await loadKeiyakuSettings(cwd);
     assertSettingsKnobUsable(loaded, "gate");
-    return loaded.knobs?.gate?.commands ?? [];
+    return {
+        commands: loaded.knobs?.gate?.commands ?? [],
+        timeoutMs: loaded.knobs?.gate?.timeoutMs ?? 300_000,
+    };
+}
+function executorFor(entry) {
+    if (!entry.shell)
+        return entry.command;
+    return process.platform === "win32" ? (process.env.ComSpec ?? "cmd.exe") : "/bin/sh";
 }
-export async function runVerificationCommand(cwd, verification) {
+export async function runVerificationPlanEntry(cwd, entry, signal, onSupervisorSpawned) {
     const started = process.hrtime.bigint();
-    return await new Promise((resolve, reject) => {
-        const child = spawn(verification.command, verification.args, {
-            cwd,
-            stdio: ["ignore", "pipe", "pipe"],
+    return new Promise((resolve, reject) => {
+        const stdout = new OutputTailCollector();
+        const stderr = new OutputTailCollector();
+        let settled = false;
+        let timer;
+        let cancel;
+        const finish = (execution) => {
+            if (settled)
+                return;
+            settled = true;
+            if (timer)
+                clearTimeout(timer);
+            if (cancel)
+                signal?.removeEventListener("abort", cancel);
+            resolve(execution);
+        };
+        const streams = () => ({
+            durationMs: Number((process.hrtime.bigint() - started) / 1000000n),
+            stdout: stdout.result(),
+            stderr: stderr.result(),
+        });
+        let terminating;
+        let terminal;
+        const child = spawn(process.execPath, verificationSupervisorArguments(), {
+            cwd: process.cwd(),
+            shell: false,
+            stdio: ["ignore", "pipe", "pipe", "ipc"],
             env: getConfig().runtimeEnv,
+            detached: process.platform !== "win32",
+        });
+        child.stdout?.on("data", (chunk) => stdout.push(chunk));
+        child.stderr?.on("data", (chunk) => stderr.push(chunk));
+        child.once("error", (error) => {
+            finish({
+                status: "runner_failed",
+                ...streams(),
+                cause: { code: "SPAWN_FAILED", message: error.message },
+            });
+        });
+        const detachUncleanSupervisor = () => {
+            child.stdout?.destroy();
+            child.stderr?.destroy();
+            if (child.connected)
+                child.disconnect();
+            child.unref();
+        };
+        const identity = ownedProcessIdentity(child);
+        if (identity && onSupervisorSpawned) {
+            try {
+                onSupervisorSpawned(identity);
+            }
+            catch (error) {
+                void terminateProcessGroup(child).then(() => reject(error), (terminationError) => reject(new ProcessTreeTerminationError(`runner registration failed and cleanup failed: ${terminationError instanceof Error ? terminationError.message : String(terminationError)}`, { cause: error })));
+                return;
+            }
+        }
+        child.on("message", (raw) => {
+            const parsed = supervisorMessageSchema.safeParse(raw);
+            if (!parsed.success)
+                return;
+            const message = parsed.data;
+            if (message.type === "ready") {
+                if (terminating)
+                    return;
+                const configureMessage = {
+                    version: VERIFICATION_SUPERVISOR_PROTOCOL_VERSION,
+                    type: "configure",
+                    cwd,
+                    command: entry.command,
+                    args: entry.args,
+                    shell: entry.shell,
+                    timeoutMs: entry.timeoutMs,
+                };
+                child.send?.(configureMessage);
+            }
+            else if (message.type === "configured") {
+                if (terminating)
+                    return;
+                const startMessage = {
+                    version: VERIFICATION_SUPERVISOR_PROTOCOL_VERSION,
+                    type: "start",
+                };
+                child.send?.(startMessage);
+            }
+            else if (message.type === "terminal") {
+                terminal = message;
+                if (message.outcome.kind === "tree-termination-failed" && !terminating) {
+                    const supervisorFailure = message.outcome.message;
+                    terminating = "TIMED_OUT";
+                    void terminateProcessGroup(child).then(() => {
+                        finish({
+                            status: "runner_failed",
+                            ...streams(),
+                            cause: { code: "TIMED_OUT", message: `verification exceeded ${entry.timeoutMs}ms` },
+                        });
+                    }, (error) => {
+                        detachUncleanSupervisor();
+                        finish({
+                            status: "runner_failed",
+                            ...streams(),
+                            cause: {
+                                code: "TREE_TERMINATION_FAILED",
+                                message: `${supervisorFailure}; parent cleanup failed: ${error instanceof Error ? error.message : String(error)}`,
+                            },
+                        });
+                    });
+                }
+            }
         });
-        const stdout = [];
-        const stderr = [];
-        child.stdout.on("data", (chunk) => stdout.push(Buffer.from(chunk)));
-        child.stderr.on("data", (chunk) => stderr.push(Buffer.from(chunk)));
-        child.on("error", reject);
-        child.on("close", (code) => {
-            const durationMs = Number((process.hrtime.bigint() - started) / 1000000n);
-            resolve({
-                code: typeof code === "number" ? code : -1,
-                durationMs,
-                stdout: tail(Buffer.concat(stdout)),
-                stderr: tail(Buffer.concat(stderr)),
+        const terminate = (cause) => {
+            if (settled || terminating)
+                return;
+            terminating = cause;
+            void terminateProcessGroup(child).then(() => {
+                finish({
+                    status: "runner_failed",
+                    ...streams(),
+                    cause: {
+                        code: cause,
+                        message: cause === "TIMED_OUT" ? `verification exceeded ${entry.timeoutMs}ms` : "verification cancelled",
+                    },
+                });
+            }, (error) => {
+                detachUncleanSupervisor();
+                finish({
+                    status: "runner_failed",
+                    ...streams(),
+                    cause: {
+                        code: error instanceof ProcessTreeTerminationError ? error.code : "TREE_TERMINATION_FAILED",
+                        message: error instanceof Error ? error.message : String(error),
+                    },
+                });
             });
+        };
+        timer = setTimeout(() => terminate("TIMED_OUT"), entry.timeoutMs);
+        timer.unref?.();
+        cancel = () => terminate("CANCELLED");
+        signal?.addEventListener("abort", cancel, { once: true });
+        if (signal?.aborted)
+            cancel();
+        child.once("close", (code, closeSignal) => {
+            void (async () => {
+                if (timer)
+                    clearTimeout(timer);
+                if (cancel)
+                    signal?.removeEventListener("abort", cancel);
+                if (terminating)
+                    return;
+                if (identity && ownedProcessIsLive(identity)) {
+                    try {
+                        await terminateOwnedProcess(identity);
+                    }
+                    catch (error) {
+                        finish({
+                            status: "runner_failed",
+                            ...streams(),
+                            cause: {
+                                code: "TREE_TERMINATION_FAILED",
+                                message: error instanceof Error ? error.message : String(error),
+                            },
+                        });
+                        return;
+                    }
+                }
+                if (!terminal) {
+                    finish({
+                        status: "runner_failed",
+                        ...streams(),
+                        cause: {
+                            code: "PROCESS_TERMINATED",
+                            message: `verification supervisor terminated${closeSignal ? ` by ${closeSignal}` : code === null ? " without an exit status" : ` with exit ${code}`}`,
+                        },
+                    });
+                    return;
+                }
+                const observed = { durationMs: terminal.durationMs, stdout: stdout.result(), stderr: stderr.result() };
+                if (terminal.outcome.kind === "exit") {
+                    finish({ status: "executed", ...observed, code: terminal.outcome.code });
+                }
+                else if (terminal.outcome.kind === "spawn-failed") {
+                    finish({ status: "runner_failed", ...observed, cause: { code: "SPAWN_FAILED", message: terminal.outcome.message } });
+                }
+                else if (terminal.outcome.kind === "timeout") {
+                    finish({ status: "runner_failed", ...observed, cause: { code: "TIMED_OUT", message: `verification exceeded ${entry.timeoutMs}ms` } });
+                }
+                else if (terminal.outcome.kind === "signal") {
+                    finish({
+                        status: "runner_failed",
+                        ...observed,
+                        cause: {
+                            code: "PROCESS_TERMINATED",
+                            message: `executor terminated${terminal.outcome.signal ? ` by ${terminal.outcome.signal}` : " without an exit status"}`,
+                        },
+                    });
+                }
+                else {
+                    finish({
+                        status: "runner_failed",
+                        ...observed,
+                        cause: { code: "TREE_TERMINATION_FAILED", message: terminal.outcome.message },
+                    });
+                }
+            })();
         });
-    }).catch((error) => {
-        throw new FlowError("CLOSE_QUALITY_GATE_FAILED", `verification '${verification.label}' could not start`, { cause: error });
     });
 }
+export async function runVerificationPlan(input) {
+    const execute = async (supervisorLifecycle) => {
+        const results = [];
+        for (const entry of input.plan) {
+            if (input.signal?.aborted) {
+                const empty = { tail: "", truncated: false, omittedBytes: 0 };
+                results.push({
+                    ...entry,
+                    candidate: input.candidate,
+                    executor: executorFor(entry),
+                    execution: {
+                        status: "runner_failed",
+                        durationMs: 0,
+                        stdout: empty,
+                        stderr: empty,
+                        cause: { code: "CANCELLED", message: "verification cancelled before runner spawn" },
+                    },
+                });
+                break;
+            }
+            let supervisorRecorded = false;
+            const execution = await runVerificationPlanEntry(input.cwd, entry, input.signal, supervisorLifecycle
+                ? (identity) => {
+                    supervisorLifecycle.spawned(identity);
+                    supervisorRecorded = true;
+                }
+                : undefined);
+            const cleanupFailed = execution.status === "runner_failed"
+                && execution.cause.code === "TREE_TERMINATION_FAILED";
+            if (supervisorRecorded && !cleanupFailed)
+                supervisorLifecycle?.cleaned();
+            results.push({
+                ...entry,
+                candidate: input.candidate,
+                executor: executorFor(entry),
+                execution,
+            });
+            if (execution.status === "runner_failed"
+                && (execution.cause.code === "CANCELLED" || execution.cause.code === "TREE_TERMINATION_FAILED"))
+                break;
+        }
+        return results;
+    };
+    const key = input.coordinationKey ?? JSON.stringify({
+        cwd: input.cwd,
+        candidate: input.candidate,
+        plan: input.plan,
+    });
+    const memoryKey = JSON.stringify({ root: input.coordinationRoot ?? null, key });
+    const existing = verificationRuns.get(memoryKey);
+    if (existing)
+        return await existing;
+    const run = input.coordinationRoot
+        ? runCoordinatedVerification({
+            root: input.coordinationRoot,
+            key,
+            execute,
+            signal: input.signal,
+            candidate: input.candidate,
+            plan: input.plan,
+        })
+        : execute();
+    const tracked = run.finally(() => verificationRuns.delete(memoryKey));
+    verificationRuns.set(memoryKey, tracked);
+    return await tracked;
+}
+const verificationRuns = new Map();
+async function wait(ms) {
+    await new Promise((resolve) => setTimeout(resolve, ms));
+}
+function cancelledResults(candidate, plan) {
+    const entry = plan[0];
+    if (!entry)
+        return [];
+    const empty = { tail: "", truncated: false, omittedBytes: 0 };
+    return [{
+            ...entry,
+            candidate,
+            executor: executorFor(entry),
+            execution: {
+                status: "runner_failed",
+                durationMs: 0,
+                stdout: empty,
+                stderr: empty,
+                cause: { code: "CANCELLED", message: "verification cancelled while waiting for an identical active execution" },
+            },
+        }];
+}
+async function runCoordinatedVerification(input) {
+    const coordination = new VerificationCoordination(input.root);
+    const fingerprint = coordination.fingerprint(input.key);
+    const staleAfterMs = input.plan.reduce((total, entry) => total + entry.timeoutMs + PROCESS_GROUP_KILL_GRACE_MS, COORDINATION_RECOVERY_MARGIN_MS);
+    for (;;) {
+        if (input.signal?.aborted)
+            return cancelledResults(input.candidate, input.plan);
+        const claim = coordination.claim(fingerprint, staleAfterMs);
+        if (claim.status === "receipt")
+            return claim.results;
+        if (claim.status === "wait") {
+            await wait(COORDINATION_POLL_MS);
+            continue;
+        }
+        try {
+            if (claim.priorSupervisor) {
+                await terminateOwnedProcess(claim.priorSupervisor);
+                coordination.clearSupervisor(fingerprint, claim.token);
+            }
+            const lifecycle = {
+                spawned: (identity) => coordination.recordSupervisor(fingerprint, claim.token, identity),
+                cleaned: () => coordination.clearSupervisor(fingerprint, claim.token),
+            };
+            const results = await input.execute(lifecycle);
+            const completion = coordination.complete(fingerprint, claim.token, results);
+            if (completion.status === "ownership-lost") {
+                throw new Error("verification coordination ownership changed before receipt publication");
+            }
+            return results;
+        }
+        catch (error) {
+            coordination.releaseForTakeover(fingerprint, claim.token);
+            throw error;
+        }
+    }
+}