@deksden-com/dd-flow-cli 0.9.0-beta.7 → 0.9.0-beta.75

package/dist/services/run-controller-recovery.js

@@ -0,0 +1,222 @@
+import crypto from "node:crypto";
+import fs from "node:fs";
+import path from "node:path";
+import { setTimeout as delay } from "node:timers/promises";
+import { AppError } from "../shared/errors.js";
+import { resolveProjectRoot } from "../storage/paths.js";
+import { requireProjectByRoot } from "./projects.js";
+import { loadExternalStageContext } from "./stage-context.js";
+import { verifyRecoverySnapshot } from "./eval-snapshots.js";
+import { recoveryGuard, resumeRunRecovery } from "./run-recovery.js";
+import { prepareRunRecoveryDispatch } from "./run-recovery-runtime.js";
+import { runtimeHarness } from "./execution-policy.js";
+import { managedProcessRuntimeHome, managedProcessStatus, processIsAlive, processTreeIsAlive } from "./managed-processes.js";
+import { requireRun, requireController, controllerReceipt, appendEvent, saveState, digest, errorRecord } from "./run-controller-state.js";
+import { controllerAdapter, sessionArguments } from "./run-controller-adapter.js";
+import { spawnController, recordControllerLauncher } from "./run-controller-process.js";
+import { readNativeDaemonState, requireManagedDaemonBinding } from "./managed-daemon-binding.js";
+import { managedInvocationContext } from "./lifecycle-invocations.js";
+import { runControlStatus } from "./run-control.js";
+import { runtimeBudgetStatus, waitForRuntimeScopeRecovery } from "./runtime-budget.js";
+/** CLI observation is bounded; the physical controller owns the recovery work. */
+export async function resumeRunController(context, input) {
+    if (input.waitMs !== undefined && (!Number.isSafeInteger(input.waitMs) || input.waitMs < 0 || input.waitMs > 60_000))
+        throw new AppError("usage", "--wait-ms must be between 0 and 60000", 2);
+    const intent = recordControllerResume(context, input);
+    const manifest = JSON.parse(intent.controller.manifest_json);
+    const resourceContext = { ...context, env: { ...context.env, DD_FLOW_RESOURCE_HOME: manifest.resource_home } };
+    await spawnController(resourceContext, intent.controller);
+    const deadline = performance.now() + (input.waitMs ?? 0);
+    const status = () => context.db.get("SELECT status FROM run_controller_operations WHERE operation_id = ?", [intent.operationId]).status;
+    while (status() === "accepted" && performance.now() < deadline)
+        await delay(Math.min(100, Math.max(1, deadline - performance.now())));
+    const observation = runControlStatus(context, { ...input, action: "resume" });
+    return { ...observation, controller: controllerReceipt(requireController(context, intent.controller.controller_id)).controller, operation_id: intent.operationId, status: observation.resume_operation.status, ...(intent.retiredProcess ? {} : { reused: true }) };
+}
+/** Retain the logical controller and its receipts; replace only its dead owner. */
+export function recordControllerResume(context, input) {
+    if (!/^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/.test(input.requestId) || !input.controlId.trim())
+        throw new AppError("usage", "Resume requires request-id and the exact source control", 2);
+    const project = requireProjectByRoot(context, resolveProjectRoot(input.projectRoot));
+    const run = requireRun(context, project.id, input.runId);
+    if (!context.db.get("SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'run_controllers'"))
+        throw new AppError("controller_attach_required", "RUN has no retained managed controller", 1);
+    const request = { action: "control_resume", request_id: input.requestId, control_id: input.controlId };
+    const hash = digest(request);
+    const findPrior = () => {
+        const prior = context.db.get("SELECT o.controller_id, o.operation_id, o.input_hash FROM run_controller_operations o JOIN run_controllers c ON c.controller_id = o.controller_id WHERE c.project_id = ? AND c.run_id = ? AND json_extract(o.input_json, '$.action') = 'control_resume' AND json_extract(o.input_json, '$.request_id') = ?", [project.id, run.id, input.requestId]);
+        if (!prior)
+            return null;
+        if (prior.input_hash !== hash)
+            throw new AppError("runtime_request_conflict", "Resume request id has different inputs", 1);
+        return { controller: requireController(context, prior.controller_id), operationId: prior.operation_id };
+    };
+    const prior = findPrior();
+    if (prior)
+        return prior;
+    try {
+        const row = context.db.get("SELECT * FROM run_controllers WHERE project_id = ? AND run_id = ? AND status NOT IN ('completed','cancelled','superseded')", [project.id, run.id]);
+        if (!row)
+            throw new AppError("controller_attach_required", "RUN has no retained resumable controller", 1);
+        const operationId = `${row.controller_id}:resume:${input.requestId}`;
+        if (!["running", "paused", "waiting_for_user"].includes(run.status))
+            throw new AppError("invalid_run_state", "Resume cannot resurrect a terminal RUN", 1, { status: run.status });
+        const guard = recoveryGuard(context, project.id, run.id);
+        const control = context.db.get("SELECT recovery_id, status FROM run_controls WHERE control_id = ? AND project_id = ? AND run_id = ?", [input.controlId, project.id, run.id]);
+        if (!guard || guard.recovery_id !== control?.recovery_id || guard.status !== "sealed" || control.status !== "settled" || !guard.capture_path || JSON.parse(guard.settlement_json ?? "{}").settled !== true)
+            throw new AppError("run_control_not_resumable", "Resume requires the current settled control and its immutable recovery capture", 1);
+        const manifest = JSON.parse(row.manifest_json), state = JSON.parse(row.state_json);
+        if (!manifest.execution_profile?.settings.execution || !manifest.execution_profile.agent_profiles || !path.isAbsolute(manifest.resource_home) || !path.isAbsolute(manifest.flow_executable))
+            throw new AppError("controller_manifest_invalid", "Resume requires the original frozen controller manifest", 1);
+        if (state.current_session === null || !state.sessions[state.current_session]?.id)
+            throw new AppError("recovery_owner_missing", "Resume requires the retained coordinator Session", 1);
+        for (const [stage, source] of Object.entries(manifest.contexts))
+            loadExternalStageContext({ file: source.file, expectedSha256: source.sha256, stage });
+        verifyRecoverySnapshot(guard.capture_path, { projectRoot: project.root, runId: run.id, recoveryId: guard.recovery_id, workspaceRoot: run.workspace_root });
+        const resourceContext = { ...context, env: { ...context.env, DD_FLOW_RESOURCE_HOME: manifest.resource_home } };
+        const previousOwner = managedProcessStatus(resourceContext).find(record => record.id === row.process_id);
+        if (!previousOwner || previousOwner.lease_token !== row.process_lease_token || previousOwner.kind !== "run-controller" || previousOwner.owner_id !== row.controller_id || previousOwner.project_id !== project.id || previousOwner.run_id !== run.id || managedProcessRuntimeHome(previousOwner, context) !== context.ddFlowHome || !previousOwner.pid || !previousOwner.pid_started_at || processTreeIsAlive(previousOwner))
+            throw new AppError("controller_owner_unsettled", "Resume requires proof that the previous physical controller has exited", 1);
+        const token = crypto.randomUUID(), now = context.now();
+        context.db.beginWriteTransaction();
+        try {
+            const duplicate = context.db.get("SELECT input_hash, status FROM run_controller_operations WHERE operation_id = ?", [operationId]);
+            if (duplicate) {
+                if (duplicate.input_hash !== hash)
+                    throw new AppError("runtime_request_conflict", "Resume request id has different inputs", 1);
+                context.db.exec("COMMIT");
+                return { controller: requireController(context, row.controller_id), operationId };
+            }
+            const current = recoveryGuard(context, project.id, run.id);
+            if (current?.recovery_id !== guard.recovery_id || current.generation !== guard.generation || current.status !== "sealed" || current.capture_path !== guard.capture_path)
+                throw new AppError("run_control_superseded", "Control changed before resume could be accepted", 1);
+            if (!["running", "paused", "waiting_for_user"].includes(requireRun(context, project.id, run.id).status))
+                throw new AppError("invalid_run_state", "RUN became terminal before resume could be accepted", 1);
+            const changed = context.db.run("UPDATE run_controls SET status = 'resuming', updated_at = ? WHERE control_id = ? AND recovery_id = ? AND status = 'settled'", [now, input.controlId, guard.recovery_id]);
+            if (changed.changes !== 1)
+                throw new AppError("run_control_superseded", "Control changed before resume could be accepted", 1);
+            state.recovery = { control_id: input.controlId, recovery_id: guard.recovery_id, operation_id: operationId };
+            const claimed = context.db.run("UPDATE run_controllers SET generation = ?, status = 'starting', owner_token = ?, owner_pid = NULL, process_id = NULL, process_lease_token = NULL, state_json = ?, last_error_json = NULL, updated_at = ? WHERE controller_id = ? AND owner_token = ? AND state_json = ? AND manifest_json = ? AND process_id = ?", [guard.generation, token, JSON.stringify(state), now, row.controller_id, row.owner_token, row.state_json, row.manifest_json, row.process_id]);
+            if (claimed.changes !== 1)
+                throw new AppError("controller_owner_mismatch", "Controller changed before resume could claim its next physical owner", 1);
+            context.db.run("INSERT INTO run_controller_operations (operation_id, controller_id, generation, input_hash, input_json, status, created_at, updated_at) VALUES (?, ?, ?, ?, ?, 'accepted', ?, ?)", [operationId, row.controller_id, guard.generation, hash, JSON.stringify(request), now, now]);
+            recordControllerLauncher(context, requireController(context, row.controller_id), previousOwner);
+            appendEvent(context, row.controller_id, "resume_requested", { operation_id: operationId, control_id: input.controlId, recovery_id: guard.recovery_id, generation: guard.generation, retired_process_id: previousOwner.id, retained_session_id: state.sessions[state.current_session].id });
+            context.db.exec("COMMIT");
+        }
+        catch (error) {
+            context.db.exec("ROLLBACK");
+            throw error;
+        }
+        return { controller: requireController(context, row.controller_id), operationId, retiredProcess: previousOwner };
+    }
+    catch (error) {
+        // A concurrent identical client may commit between our first lookup and
+        // validation. Its durable request wins even if our preflight saw the fence.
+        const concurrent = findPrior();
+        if (concurrent)
+            return concurrent;
+        throw error;
+    }
+}
+export async function resumeControllerSession(context, row, manifest, state, assertPhysicalOwner) {
+    const recovery = state.recovery;
+    const session = state.current_session === null ? undefined : state.sessions[state.current_session];
+    if (!session?.id)
+        throw new AppError("recovery_owner_missing", "Recovery lost its retained coordinator Session", 1);
+    let executionContext = { ...context, env: { ...context.env, DD_FLOW_BIN: manifest.flow_executable, DD_FLOW_RESOURCE_HOME: manifest.resource_home } };
+    const scopeControl = manifest.budget ? runtimeBudgetStatus(executionContext, manifest.budget.scope_id).control : null;
+    const assertRecoveryOwner = () => {
+        assertPhysicalOwner();
+        const guard = recoveryGuard(context, row.project_id, row.run_id);
+        if (guard?.generation !== row.generation || guard.recovery_id !== recovery.recovery_id)
+            throw new AppError("run_recovery_generation_stale", "A newer control superseded this recovery owner", 1);
+        if (!["sealed", "resuming", "recovered"].includes(guard.status) || !context.db.get("SELECT 1 FROM run_controls WHERE control_id = ? AND recovery_id = ? AND status = 'resuming'", [recovery.control_id, recovery.recovery_id]))
+            throw new AppError("run_recovery_guarded", "Recovery no longer owns this operator intent", 1);
+    };
+    assertRecoveryOwner();
+    const readDaemon = () => readNativeDaemonState(session.state_dir);
+    const { state: retired, process: record } = requireManagedDaemonBinding(executionContext, { stateDir: session.state_dir, projectId: row.project_id, runId: row.run_id, ownerId: session.owner.owner_id });
+    const call = (args, operationId) => controllerAdapter(executionContext, row, manifest, session, args, assertRecoveryOwner, operationId);
+    if (processTreeIsAlive(record)) {
+        const stopped = await call(["daemon", "stop", "--state-dir", session.state_dir, "--json"]);
+        if (stopped.stopped !== true || stopped.clean !== true)
+            throw new AppError("recovery_native_shutdown_unproven", "Recovery requires a clean native close before replacing its daemon", 1);
+        const deadline = Date.now() + 30_000;
+        while (processTreeIsAlive(record) && Date.now() < deadline) {
+            await delay(100);
+            assertRecoveryOwner();
+        }
+    }
+    if (processTreeIsAlive(record) || readDaemon().shutdown_state !== "clean")
+        throw new AppError("recovery_native_shutdown_unproven", "The previous native daemon is not proven clean and closed", 1);
+    assertRecoveryOwner();
+    session.stopped = false;
+    saveState(context, row, state);
+    const started = await call(["daemon", "start", "--session-id", session.id, ...sessionArguments(context, manifest, session)]);
+    const daemon = readDaemon(), daemonId = daemon.daemon_id ?? daemon.config?.daemonId;
+    if (!daemonId || daemonId === (retired.daemon_id ?? retired.config?.daemonId) || started.daemon_id !== daemonId || started.already_running === true)
+        throw new AppError("recovery_daemon_stale", "Recovery requires a fresh daemon, not reuse of the retired physical owner", 1);
+    executionContext = managedInvocationContext(executionContext, { harness: runtimeHarness(session.profile), stateDir: session.state_dir, sessionId: session.id,
+        projectRoot: manifest.request.project_root, runId: row.run_id, generation: row.generation });
+    assertRecoveryOwner();
+    const scope = { projectRoot: manifest.request.project_root, runId: row.run_id, recoveryId: recovery.recovery_id };
+    resumeRunRecovery(executionContext, { ...scope, daemonId, harness: runtimeHarness(session.profile), nativeSessionId: session.id });
+    const dispatch = prepareRunRecoveryDispatch(executionContext, { ...scope, stateDir: session.state_dir, operationId: `${recovery.operation_id}:ack` });
+    const inspected = await call(["session", "resume", "--session-id", session.id, ...sessionArguments(context, manifest, session)], dispatch.resume_operation_id);
+    if (inspected.settled !== true)
+        throw new AppError("recovery_session_unsettled", "Retained native Session is not ready for same-Session recovery", 1);
+    assertRecoveryOwner();
+    const prepared = { prepared: true, recovery_id: recovery.recovery_id, session_id: session.id, daemon_id: daemonId, process_id: dispatch.process_id, resume_operation_id: dispatch.resume_operation_id };
+    context.db.beginWriteTransaction();
+    try {
+        assertRecoveryOwner();
+        const binding = requireManagedDaemonBinding(executionContext, { stateDir: session.state_dir, projectId: row.project_id, runId: row.run_id, ownerId: session.owner.owner_id });
+        if (binding.process.id !== dispatch.process_id || binding.state.daemon_id !== daemonId || binding.process.state !== "running" || !processIsAlive(binding.process))
+            throw new AppError("recovery_runtime_mismatch", "Prepared recovery lost its physical Session owner", 1);
+        const recorded = context.db.run("UPDATE run_controller_operations SET receipt_json = ?, updated_at = ? WHERE operation_id = ? AND controller_id = ? AND generation = ? AND status = 'accepted'", [JSON.stringify(prepared), context.now(), recovery.operation_id, row.controller_id, row.generation]);
+        if (recorded.changes !== 1)
+            throw new AppError("run_recovery_generation_stale", "Recovery preparation no longer owns its operation", 1);
+        appendEvent(context, row.controller_id, "recovery_prepared", prepared);
+        context.db.exec("COMMIT");
+    }
+    catch (error) {
+        context.db.exec("ROLLBACK");
+        throw error;
+    }
+    if (scopeControl && manifest.budget) {
+        await waitForRuntimeScopeRecovery(executionContext, { scopeId: manifest.budget.scope_id, generation: scopeControl.generation, controllerId: row.controller_id, operationId: recovery.operation_id }, assertRecoveryOwner);
+    }
+    const file = path.join(row.state_dir, `turn-${String(state.turns + 1).padStart(5, "0")}.md`);
+    fs.writeFileSync(file, dispatch.prompt, { flag: "wx", mode: 0o600 });
+    state.turns += 1;
+    saveState(context, row, state);
+    const receipt = await call(["session", "prompt", "--session-id", session.id, "--prompt-file", file, ...sessionArguments(context, manifest, session)], dispatch.operation_id);
+    assertRecoveryOwner();
+    if (recoveryGuard(context, row.project_id, row.run_id)?.status !== "recovered")
+        throw new AppError("trusted_recovery_acceptance_required", "Recovery Turn ended without its native acknowledgement", 1);
+    context.db.beginWriteTransaction();
+    try {
+        assertRecoveryOwner();
+        context.db.run("UPDATE run_controls SET status = 'resumed', updated_at = ? WHERE control_id = ? AND status = 'resuming'", [context.now(), recovery.control_id]);
+        context.db.run("UPDATE run_controller_operations SET status = 'completed', receipt_json = ?, updated_at = ? WHERE operation_id = ? AND status = 'accepted'", [JSON.stringify({ resumed: true, recovery_id: recovery.recovery_id, session_id: session.id, daemon_id: daemonId }), context.now(), recovery.operation_id]);
+        state.last_receipt = receipt;
+        state.last_continuation = null;
+        delete state.recovery;
+        saveState(context, row, state);
+        appendEvent(context, row.controller_id, "recovery_resumed", { control_id: recovery.control_id, recovery_id: recovery.recovery_id, session_id: session.id, daemon_id: daemonId, generation: row.generation });
+        context.db.exec("COMMIT");
+    }
+    catch (error) {
+        context.db.exec("ROLLBACK");
+        throw error;
+    }
+}
+export function failControllerRecovery(context, row, state, error) {
+    if (state.recovery) {
+        // Native ACK may already have opened this generation before its Turn lost
+        // observation. Retain the accepted evidence but close further dispatch.
+        context.db.run("UPDATE run_recovery_guards SET status = 'resuming', updated_at = ? WHERE recovery_id = ? AND status = 'recovered' AND generation = (SELECT MAX(generation) FROM run_recovery_guards WHERE project_id = ? AND run_id = ?) AND EXISTS (SELECT 1 FROM run_controls WHERE control_id = ? AND recovery_id = run_recovery_guards.recovery_id AND status = 'resuming')", [context.now(), state.recovery.recovery_id, row.project_id, row.run_id, state.recovery.control_id]);
+        context.db.run("UPDATE run_controller_operations SET status = 'failed', error_json = ?, updated_at = ? WHERE operation_id = ? AND status = 'accepted'", [JSON.stringify(errorRecord(error)), context.now(), state.recovery.operation_id]);
+    }
+}

package/dist/services/run-controller-state.js

@@ -0,0 +1,31 @@
+import crypto from "node:crypto";
+import { AppError } from "../shared/errors.js";
+import { runtimeHarness } from "./execution-policy.js";
+export function requireRun(context, projectId, runId) {
+    const run = context.db.get("SELECT id, project_id, project_root, workspace_root, run_root, status, index_json FROM runs WHERE project_id = ? AND id = ?", [projectId, runId]);
+    if (!run?.run_root)
+        throw new AppError("not_found", "Managed RUN has no materialized root", 1, { run_id: runId });
+    return run;
+}
+export function requireController(context, id) {
+    const row = context.db.get("SELECT * FROM run_controllers WHERE controller_id = ?", [id]);
+    if (!row)
+        throw new AppError("not_found", "Managed controller was not found", 1, { controller_id: id });
+    return row;
+}
+export function controllerReceipt(row) {
+    const state = JSON.parse(row.state_json);
+    const manifest = JSON.parse(row.manifest_json);
+    return { ok: true, schema_id: "dd-flow/run-controller-receipt@1", controller: { controller_id: row.controller_id, request_id: row.request_id, project_id: row.project_id, run_id: row.run_id, generation: row.generation, runtime_budget: manifest.budget ?? null, status: row.status, process_id: row.process_id, owner_pid: row.owner_pid, state_dir: row.state_dir, stage: state.stage, sessions: state.sessions.map(session => ({ session_id: session.id, harness: runtimeHarness(session.profile), profile_id: session.profile.id, state_dir: session.state_dir, journal: session.journal, stopped: session.stopped === true })), error: row.last_error_json ? JSON.parse(row.last_error_json) : null } };
+}
+export function appendEvent(context, id, type, data) { context.db.run("INSERT INTO run_controller_events (controller_id, event_type, data_json, created_at) VALUES (?, ?, ?, ?)", [id, type, JSON.stringify(data), context.now()]); }
+export function saveState(context, row, state) { context.db.run("UPDATE run_controllers SET state_json = ?, updated_at = ? WHERE controller_id = ? AND owner_token = ?", [JSON.stringify(state), context.now(), row.controller_id, row.owner_token]); }
+export function digest(value) { return digestBytes(JSON.stringify(value)); }
+export function digestBytes(value) { return crypto.createHash("sha256").update(value).digest("hex"); }
+export function errorRecord(error) {
+    return {
+        code: error instanceof AppError ? error.code : "controller_failed",
+        message: error instanceof Error ? error.message : String(error),
+        ...(error instanceof AppError && Object.keys(error.details).length ? { details: error.details } : {})
+    };
+}