@deksden-com/dd-flow-cli 0.9.0-beta.123 → 0.9.0-beta.127

package/dist/services/code-checks.js

@@ -1,13 +1,16 @@
 import crypto from "node:crypto";
 import { isGitlessWorkspace, ownedWorkspaceHookPath, projectServicePathFilter, workspaceGitFiles } from "./workspace-files.js";
 import { spawn, spawnSync } from "node:child_process";
+import { ObservationClock, validateObservationDuration } from "../harness-runtime/lib/observation-clock.mjs";
 import fs from "node:fs";
 import path from "node:path";
 import { isDeepStrictEqual } from "node:util";
 import { parse } from "shell-quote";
 import { assertRunMutationAllowed } from "./run-recovery.js";
-import { AppError } from "../shared/errors.js";
-import { confirmManagedProcess, finishManagedProcess, heartbeatManagedProcess, managedProcessStatus, managedProcessRuntimeHome, managedOwnerIsAlive, processTreeIsAlive, registerManagedProcess, reservePorts } from "./managed-processes.js";
+import { AppError, errorRecord } from "../shared/errors.js";
+import { stopProcessGroup } from "../harness-runtime/lib/managed-daemon.mjs";
+import { createRenewalState, maintenanceRetryable, renewalConfirmed, renewalDelay, renewalFailure, renewalRemaining, RENEWAL_POLICY } from "../harness-runtime/lib/lease-renewal.mjs";
+import { confirmManagedProcess, finishManagedProcess, renewManagedProcess, managedProcessStatus, managedProcessRuntimeHome, managedOwnerIsAlive, processTreeIsAlive, registerManagedProcess, reservePorts } from "./managed-processes.js";
 import { admitRuntimeCheck, runtimeBudgetForRun } from "./runtime-budget.js";
 import { getResourceDatabase } from "../storage/database.js";
 const profileRelativePath = path.join(".memory-bank", "spec", "engineering", "code-check-profile.json");
@@ -233,6 +236,8 @@ function assertDirectCheckExecutable(workspaceRoot, command, checkId) {
     if (!program)
         return;
     const probe = spawnSync("/bin/sh", ["-c", "command -v -- \"$1\" >/dev/null", "dd-flow-check", program], { cwd: workspaceRoot, env, encoding: "utf8", timeout: 5000 });
+    if (probe.error || probe.signal)
+        throw new AppError("check_command_probe_unavailable", "Executable probe could not be observed; availability is unknown", 2, { check_id: checkId, executable: program, cause: probe.error?.message, signal: probe.signal });
     if (probe.status !== 0)
         throw new AppError("check_command_unavailable", "Available CODE check names an executable that is unavailable in this environment", 2, { check_id: checkId, command, executable: program });
 }
@@ -330,7 +335,7 @@ async function runCodeChecksOwned(context, input, workspaceGateId, generation) {
                     assertRunMutationAllowed(context, input.projectId, input.runId, generation);
                     if (budget)
                         admitRuntimeCheck(context, managed.id, managed.lease_token);
-                }, () => heartbeatManagedProcess(context, { id: managed.id, leaseToken: managed.lease_token }));
+                }, attemptMs => renewManagedProcess({ ...context, resourceDatabaseMode: "prepared_existing", databaseDeadlineAt: Date.now() + attemptMs }, { id: managed.id, leaseToken: managed.lease_token }));
             }
             finally {
                 fs.closeSync(stdout);
@@ -340,7 +345,8 @@ async function runCodeChecksOwned(context, input, workspaceGateId, generation) {
             if (processTreeIsAlive(current))
                 throw new AppError("check_in_progress", "The check shell exited but its process tree is still live; retain this receipt and its resources until settlement", 2, { process_id: managed.id, check_receipt_id: id });
             allocation.release();
-            finishManagedProcess(context, { id: managed.id, leaseToken: managed.lease_token, state: result.aborted || result.exitCode !== 0 || Boolean(result.error) ? "failed" : "stopped", reason: result.error });
+            if (!result.cleanupUnconfirmed)
+                finishManagedProcess(context, { id: managed.id, leaseToken: managed.lease_token, state: result.aborted || result.exitCode !== 0 || Boolean(result.error) ? "failed" : "stopped", reason: result.error });
         }
         catch (error) {
             if (error instanceof AppError && error.code === "check_in_progress")
@@ -421,7 +427,7 @@ export function reconcileUnfinishedChecks(context, input) {
         if (process)
             finishManagedProcess(context, { id: process.id, leaseToken: process.lease_token, state: status === "passed" ? "stopped" : "failed", reason: status === "passed" ? "reconciled_completion" : completion === null ? "completion_marker_missing" : "reconciled_check_failed" });
         if (receipt) {
-            const repaired = { ...receipt, completion_path: receipt.completion_path || path.join(path.dirname(pendingReceipt.receipt_path), "completion.json"), required_artifacts: receipt.required_artifacts ?? [], status, exit_code: exitCode, after_fingerprint: after.fingerprint, mutation_paths: workspaceUnchanged ? [] : ["<workspace changed while detached check ran>"], artifacts: artifacts.items, finished_at: finishedAt };
+            const repaired = { ...receipt, completion_path: receipt.completion_path || path.join(path.dirname(pendingReceipt.receipt_path), "completion.json"), required_artifacts: receipt.required_artifacts ?? [], ...(status === "aborted" ? { abort_reason: "completion_marker_missing" } : {}), status, exit_code: exitCode, after_fingerprint: after.fingerprint, mutation_paths: workspaceUnchanged ? [] : ["<workspace changed while detached check ran>"], artifacts: artifacts.items, finished_at: finishedAt };
             writeReceipt(pendingReceipt.receipt_path, repaired);
             reconciled.push(repaired);
         }
@@ -485,7 +491,7 @@ function reserveOrReuseReceipt(context, input) {
         return { reserved };
     }, "checks.reserveOrReuseReceipt");
 }
-function receiptFor(input) { return { id: input.id, local_id: input.localId, declaration_id: input.group.declarations[0].id, check_refs: input.group.declarations.map(checkRef), gate: input.group.gate, scope: input.scope, work_id: input.workId ?? null, command: input.group.command, input_hash: input.inputHash, verification_epoch: input.epoch, retry_of_receipt_id: input.retryOf ?? null, retry_reason: input.retryReason ?? null, status: input.status, exit_code: input.status === "running" ? null : input.result.exitCode, stdout_path: input.stdoutPath, stderr_path: input.stderrPath, receipt_path: input.receiptPath, completion_path: input.completionPath, required_artifacts: input.group.requiredArtifacts, inputs: input.group.inputs, workspace_fingerprint: input.before.fingerprint, before_fingerprint: input.before.fingerprint, after_fingerprint: input.after.fingerprint, profile_hash: input.profileHash, mutation_paths: input.mutationPaths, artifacts: input.artifacts, resources: { ports: input.resources }, started_at: input.startedAt, finished_at: input.status === "running" ? input.startedAt : new Date().toISOString() }; }
+function receiptFor(input) { return { id: input.id, local_id: input.localId, declaration_id: input.group.declarations[0].id, check_refs: input.group.declarations.map(checkRef), gate: input.group.gate, scope: input.scope, work_id: input.workId ?? null, command: input.group.command, input_hash: input.inputHash, verification_epoch: input.epoch, retry_of_receipt_id: input.retryOf ?? null, retry_reason: input.retryReason ?? null, status: input.status, ...(input.status === "aborted" ? { abort_reason: input.result.error } : {}), exit_code: input.status === "running" ? null : input.result.exitCode, stdout_path: input.stdoutPath, stderr_path: input.stderrPath, receipt_path: input.receiptPath, completion_path: input.completionPath, required_artifacts: input.group.requiredArtifacts, inputs: input.group.inputs, workspace_fingerprint: input.before.fingerprint, before_fingerprint: input.before.fingerprint, after_fingerprint: input.after.fingerprint, profile_hash: input.profileHash, mutation_paths: input.mutationPaths, artifacts: input.artifacts, resources: { ports: input.resources }, started_at: input.startedAt, finished_at: input.status === "running" ? input.startedAt : new Date().toISOString() }; }
 function writeReceipt(file, receipt) { const temporary = `${file}.tmp-${process.pid}-${crypto.randomUUID()}`; fs.writeFileSync(temporary, `${JSON.stringify(receipt, null, 2)}\n`); fs.renameSync(temporary, file); }
 function readReceipt(file) { try {
     return JSON.parse(fs.readFileSync(file, "utf8"));
@@ -818,14 +824,23 @@ function listFiles(root) {
     });
     return visit(root).filter(file => file.split(path.sep).join("/") !== ownedHook).sort();
 }
-export function runCheck(command, cwd, environment, stdout, stderr, heartbeat, onSpawn, renewLease) {
+export function runCheck(command, cwd, environment, stdout, stderr, heartbeat, onSpawn, renewLease, observation = {}) {
+    const inactivityTimeoutMs = validateObservationDuration(observation.inactivityTimeoutMs ?? 30 * 60_000);
     return new Promise((resolve) => {
         const started = Date.now();
-        const child = spawn("/bin/sh", ["-c", checkShellScript()], { cwd, env: { ...environment, DD_FLOW_CHECK_COMMAND: command }, stdio: ["pipe", stdout, stderr], detached: process.platform !== "win32" });
+        const child = spawn("/bin/sh", ["-c", checkShellScript()], { cwd, env: { ...environment, DD_FLOW_CHECK_COMMAND: command }, stdio: ["pipe", "pipe", "pipe"], detached: process.platform !== "win32" });
         let error = null;
         let stopping = false;
+        let closed = false;
         let progress;
-        let escalation;
+        let cleanup;
+        let cleanupUnconfirmed = false;
+        const renewal = createRenewalState();
+        let renewalTimer;
+        const clock = new ObservationClock({ timeoutMs: inactivityTimeoutMs });
+        let outputCursor = 0;
+        const outputs = [fs.createWriteStream("", { fd: stdout, autoClose: false }), fs.createWriteStream("", { fd: stderr, autoClose: false })];
+        const flushed = outputs.map(output => new Promise(resolve => { output.once("finish", resolve); output.once("error", () => resolve()); }));
         const stop = (cause) => {
             if (stopping)
                 return;
@@ -835,23 +850,42 @@ export function runCheck(command, cwd, environment, stdout, stderr, heartbeat, o
                 clearInterval(progress);
                 progress = undefined;
             }
-            if (child.pid) {
-                terminateProcessGroup(child.pid, "SIGTERM");
-                escalation = setTimeout(() => terminateProcessGroup(child.pid, "SIGKILL"), 5_000);
-            }
+            if (renewalTimer)
+                clearTimeout(renewalTimer);
+            if (observationTimer)
+                clearInterval(observationTimer);
+            cleanup = stopProcessGroup(child, 5_000).catch(cause => {
+                cleanupUnconfirmed = true;
+                error = `${error}; cleanup: ${JSON.stringify(errorRecord(cause))}`;
+                resolve({ exitCode: null, error, aborted: true, cleanupUnconfirmed });
+            });
         };
+        const observationTimer = setInterval(() => {
+            if (stopping || closed || observation.phase?.() === "serving")
+                return;
+            if (clock.sample(outputCursor))
+                stop(new AppError("operation_observation_lost", "Owned command has no confirmed progress within its inactivity window", 1, { phase: observation.phase?.() ?? "finite", inactivity_timeout_ms: inactivityTimeoutMs }));
+        }, Math.min(1000, inactivityTimeoutMs));
         // Register handlers before checking pid or calling persistence callbacks.
         // Failure is not completion until the spawned process has actually closed.
         child.once("error", (value) => { error = value.message; });
+        for (const [index, stream] of [child.stdout, child.stderr].entries()) {
+            outputs[index].once("error", stop);
+            stream.on("data", () => { outputCursor++; clock.sample(outputCursor); });
+            stream.pipe(outputs[index]);
+        }
         child.stdin.on("error", stop);
-        child.once("close", (exitCode, signal) => {
+        child.once("close", async (exitCode, signal) => {
+            closed = true;
             if (progress)
                 clearInterval(progress);
-            if (escalation)
-                clearTimeout(escalation);
-            if (stopping)
-                terminateProcessGroup(child.pid, "SIGKILL");
-            resolve({ exitCode, error: error ?? (signal ? `terminated by ${signal}` : null), aborted: stopping || Boolean(signal) || Boolean(error) });
+            if (renewalTimer)
+                clearTimeout(renewalTimer);
+            if (observationTimer)
+                clearInterval(observationTimer);
+            await cleanup;
+            await Promise.all(flushed);
+            resolve({ exitCode, error: error ?? (signal ? `terminated by ${signal}` : null), aborted: stopping || Boolean(signal) || Boolean(error), ...(cleanupUnconfirmed ? { cleanupUnconfirmed } : {}) });
         });
         if (!child.pid) {
             error = "check process did not provide a PID";
@@ -865,18 +899,55 @@ export function runCheck(command, cwd, environment, stdout, stderr, heartbeat, o
             return;
         }
         child.stdin.end("admitted\n");
-        progress = setInterval(() => {
+        clock.sample(outputCursor);
+        const renew = () => {
+            if (stopping || closed)
+                return;
+            if (renewalTimer) {
+                clearTimeout(renewalTimer);
+                renewalTimer = undefined;
+            }
             try {
-                if (!renewLease()) {
+                renewal.uncertainSince ??= performance.now();
+                const remaining = renewalRemaining(renewal);
+                if (!remaining) {
+                    stop("managed process renewal budget exhausted");
+                    return;
+                }
+                const receipt = renewLease(Math.min(RENEWAL_POLICY.attemptMs, remaining));
+                if (receipt === false || typeof receipt === "object" && !receipt.owned) {
                     stop("managed process lease lost");
                     return;
                 }
+                if (typeof receipt === "object" && receipt.owned) {
+                    const expiry = Date.parse(receipt.lease_expires_at);
+                    if (!Number.isFinite(expiry))
+                        throw new AppError("managed_lease_receipt_invalid", "Check renewal returned invalid expiry", 1);
+                    if (expiry <= Date.now())
+                        throw Object.assign(new Error("Check renewal receipt expired before acknowledgement"), { code: "process_maintenance_timeout" });
+                }
+                if (!renewalRemaining(renewal) || !renewalConfirmed(renewal)) {
+                    stop("managed process renewal budget exhausted");
+                    return;
+                }
+            }
+            catch (cause) {
+                if (!maintenanceRetryable(cause) || !renewalFailure(renewal)) {
+                    stop(cause);
+                    return;
+                }
+                renewalTimer = setTimeout(renew, Math.min(renewalDelay(renewal), renewalRemaining(renewal)));
+                return;
+            }
+            // A progress callback error is not a retryable renewal failure.
+            try {
                 heartbeat(Math.floor((Date.now() - started) / 1000));
             }
             catch (cause) {
                 stop(cause);
             }
-        }, 15_000);
+        };
+        progress = setInterval(renew, 15_000);
     });
 }
 function checkShellScript() {
@@ -890,8 +961,3 @@ function checkShellScript() {
         'exit "$code"'
     ].join("\n");
 }
-function terminateProcessGroup(pid, signal) { if (!pid)
-    return; try {
-    process.kill(process.platform === "win32" ? pid : -pid, signal);
-}
-catch { /* already exited */ } }