@deksden-com/dd-flow-cli 0.9.0-beta.16 → 0.9.0-beta.161

package/dist/services/run-recovery-guard.js

@@ -0,0 +1,107 @@
+import crypto from "node:crypto";
+import { AppError } from "../shared/errors.js";
+import { RECOVERY_POLICY as CAPACITY_POLICY, MAX_OVERLOAD_RETRIES, recoveryExhausted as overloadExhausted } from "../harness-runtime/lib/provider-overload-policy.mjs";
+import { assertRunStartupSettled } from "./run-start-custody.js";
+export function recoveryPacketReceipt(binding) {
+    if (!binding.packet_json || !binding.packet_sha256 || crypto.createHash("sha256").update(binding.packet_json).digest("hex") !== binding.packet_sha256)
+        throw new AppError("recovery_packet_missing", "Recovery requires its original verified Work packet", 1);
+    return { packet_json: binding.packet_json, packet_sha256: binding.packet_sha256, packet: JSON.parse(binding.packet_json) };
+}
+/** A cooperative drain permits only the WorkSession present at its fence. */
+export function assertWorkSettlementAllowed(context, projectId, runId, workId) {
+    const guard = recoveryGuard(context, projectId, runId);
+    if (guard?.status === "draining" && context.db.get("SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'run_controls'")) {
+        const admission = context.db.get("SELECT a.work_session_id FROM run_control_admissions a JOIN run_controls c ON c.control_id = a.control_id JOIN work_sessions ws ON ws.id = a.work_session_id JOIN works w ON w.work_id = a.work_id WHERE c.project_id = ? AND c.run_id = ? AND c.recovery_id = ? AND c.action = 'pause' AND c.status = 'requested' AND a.work_id = ? AND ws.work_id = a.work_id AND ws.session_id = a.session_id AND ws.status = 'running' AND w.status = 'running' AND w.project_id = c.project_id AND w.run_id = c.run_id LIMIT 1", [projectId, runId, guard.recovery_id, workId]);
+        if (admission)
+            return;
+    }
+    assertRunMutationAllowed(context, projectId, runId);
+}
+/** Completion rights are pinned to the Stage attempt present at the pause fence. */
+export function assertStageSettlementAllowed(context, projectId, runId, stage, expectedGeneration) {
+    const guard = recoveryGuard(context, projectId, runId);
+    const generation = guard?.generation ?? 0;
+    if (expectedGeneration !== undefined && generation !== expectedGeneration)
+        throw new AppError("run_recovery_generation_stale", "RUN execution generation changed while the operation was in flight", 1, { run_id: runId, expected_generation: expectedGeneration, generation });
+    if (guard?.status === "draining" && context.db.get("SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'run_control_stage_admissions'")) {
+        const admitted = context.db.get(`SELECT a.work_session_id FROM run_control_stage_admissions a JOIN run_controls c ON c.control_id = a.control_id JOIN run_control_admissions w ON w.control_id = a.control_id AND w.work_session_id = a.work_session_id JOIN work_sessions ws ON ws.id = w.work_session_id JOIN runs r ON r.project_id = c.project_id AND r.id = c.run_id, json_each(r.index_json, '$.stage_runs') live
+      WHERE c.project_id = ? AND c.run_id = ? AND c.recovery_id = ? AND c.action = 'pause' AND c.status = 'requested' AND a.stage = ? AND json_extract(live.value, '$.stage') = a.stage AND COALESCE(json_extract(live.value, '$.attempt'), 'try-001') = a.attempt AND ws.work_id = w.work_id AND ws.session_id = w.session_id AND NOT EXISTS (SELECT 1 FROM work_sessions newer WHERE newer.work_id = w.work_id AND newer.status = 'running' AND newer.id <> ws.id) LIMIT 1`, [projectId, runId, guard.recovery_id, stage]);
+        if (admitted)
+            return generation;
+    }
+    return assertRunMutationAllowed(context, projectId, runId, expectedGeneration);
+}
+export function recoveryGuard(context, projectId, runId) { return context.db.get("SELECT recovery_id, project_id, run_id, generation, status, interruption_json, settlement_json, capture_path, created_at, updated_at, sealed_at, resumed_at FROM run_recovery_guards WHERE project_id = ? AND run_id = ? ORDER BY generation DESC LIMIT 1", [projectId, runId]) ?? null; }
+export function assertRunRecoveryAllowed(context, projectId, runId, expectedGeneration) {
+    const guard = recoveryGuard(context, projectId, runId);
+    const generation = guard?.generation ?? 0;
+    if (expectedGeneration !== undefined && generation !== expectedGeneration)
+        throw new AppError("run_recovery_generation_stale", "RUN execution generation changed while the operation was in flight", 1, { run_id: runId, expected_generation: expectedGeneration, generation });
+    if (guard && ["draining", "sealed", "resuming"].includes(guard.status))
+        throw new AppError("run_recovery_guarded", "RUN is retained for recovery; reconcile or resume its recovery operation before accepting a new lifecycle mutation", 1, { run_id: runId, recovery_id: guard.recovery_id, generation, status: guard.status });
+    return generation;
+}
+export function assertRunMutationAllowed(context, projectId, runId, expectedGeneration) {
+    assertRunStartupSettled(context, projectId, runId);
+    const generation = assertRunRecoveryAllowed(context, projectId, runId, expectedGeneration);
+    if (context.db.get("SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'run_controllers'")
+        && context.db.get("SELECT 1 FROM run_controllers WHERE project_id = ? AND run_id = ? AND status IN ('capturing','capture_failed') LIMIT 1", [projectId, runId]))
+        throw new AppError("run_boundary_capturing", "RUN writers are fenced while its controller captures a Stage boundary", 1, { run_id: runId });
+    return generation;
+}
+/** The caller has already checked PID/start/lease and runtime-home identity. */
+export function recoveryRuntimeAdmission(context, record, input) {
+    if (!record.project_id || !record.run_id || !input.daemonId)
+        return false;
+    const guard = recoveryGuard(context, record.project_id, record.run_id);
+    if (!guard || !["resuming", "recovered"].includes(guard.status))
+        return false;
+    const binding = context.db.get("SELECT * FROM run_recovery_bindings WHERE recovery_id = ? AND daemon_id = ?", [guard.recovery_id, input.daemonId]);
+    if (!binding?.runtime_dispatch_json)
+        return false;
+    const dispatch = JSON.parse(binding.runtime_dispatch_json);
+    if (dispatch.process_id !== record.id || dispatch.owner_id !== record.owner_id || dispatch.state_dir !== input.stateDir || dispatch.daemon_id !== input.daemonId)
+        return false;
+    const metadata = JSON.parse(record.metadata_json);
+    if (metadata.run_generation !== guard.generation)
+        return false;
+    if (guard.status === "recovered" && binding.status === "accepted")
+        return !input.operationId?.startsWith(`${dispatch.operation_id}:capacity:`);
+    if (guard.status === "resuming" && binding.status === "prepared" && input.operation === "session.resume")
+        return typeof dispatch.resume_operation_id === "string" && input.operationId === dispatch.resume_operation_id && input.nativeSessionId === binding.provider_session_id;
+    const active = dispatch.capacity_intents?.at(-1);
+    if (active) {
+        const ordinal = dispatch.capacity_intents.length;
+        const predecessor = ordinal === 1 ? dispatch.operation_id : dispatch.capacity_intents[ordinal - 2].operation_id;
+        if (ordinal > MAX_OVERLOAD_RETRIES || active.policy !== CAPACITY_POLICY || active.ordinal !== ordinal || active.root_operation_id !== dispatch.operation_id || active.session_id !== binding.provider_session_id
+            || active.predecessor !== predecessor || active.operation_id !== `${dispatch.operation_id}:capacity:${ordinal}:session.prompt`
+            || active.refusal?.policy !== CAPACITY_POLICY || active.refusal.ordinal !== ordinal - 1 || active.refusal.turn_id !== active.turn_id
+            || active.refusal.session_id !== binding.provider_session_id || active.deadline !== null || !Number.isFinite(Date.parse(active.not_before)))
+            return false;
+        const failure = dispatch.capacity_failures?.find(item => item.ordinal === ordinal - 1);
+        if (!failure || JSON.stringify(failure) !== JSON.stringify(active.refusal))
+            return false;
+        try {
+            if (overloadExhausted(dispatch.capacity_failures))
+                return false;
+        }
+        catch {
+            return false;
+        }
+        const predecessorRow = context.db.get("SELECT status,receipt_json FROM run_controller_operations WHERE operation_id = ? AND controller_id = ? AND generation = ?", [predecessor, record.owner_id, guard.generation]);
+        if (predecessorRow?.status !== "failed")
+            return false;
+        try {
+            if (JSON.stringify(JSON.parse(predecessorRow.receipt_json).capacity_intent) !== JSON.stringify(active))
+                return false;
+        }
+        catch {
+            return false;
+        }
+    }
+    if (active && active.not_before > context.now())
+        return false;
+    return guard.status === "resuming" && binding.status === "prepared" && input.operation === "session.prompt"
+        && input.operationId === (active?.operation_id ?? dispatch.operation_id) && input.nativeSessionId === binding.provider_session_id
+        && input.promptSha256 === (active?.prompt_sha256 ?? dispatch.prompt_sha256);
+}

package/dist/services/run-recovery-runtime.js

@@ -0,0 +1,112 @@
+import crypto from "node:crypto";
+import path from "node:path";
+import { AppError } from "../shared/errors.js";
+import { resolveProjectRoot } from "../storage/paths.js";
+import { requireProjectByRoot } from "./projects.js";
+import { acceptanceCommand, recoveryGuard, recoveryPacketReceipt, requireRecoveryRun } from "./run-recovery.js";
+import { managedProcessBinding, processIsAlive, resourceHome } from "./managed-processes.js";
+import { getResourceDatabase } from "../storage/database.js";
+import { requireManagedDaemonBinding } from "./managed-daemon-binding.js";
+import { lifecycleRetryInstruction } from "../harness-runtime/lib/delegation-instructions.mjs";
+import { RECOVERY_POLICY as CAPACITY_POLICY, MAX_OVERLOAD_RETRIES, recoveryExhausted as overloadExhausted } from "../harness-runtime/lib/provider-overload-policy.mjs";
+/** Pin retained Session loading and its first acknowledgement Turn, never a general fence bypass. */
+export function prepareRunRecoveryDispatch(context, input) {
+    if (!/^[A-Za-z0-9][A-Za-z0-9._:-]{0,255}$/.test(input.operationId) || !path.isAbsolute(input.stateDir))
+        throw new AppError("usage", "Recovery dispatch requires an operation id and absolute state directory", 2);
+    const project = requireProjectByRoot(context, resolveProjectRoot(input.projectRoot));
+    const run = requireRecoveryRun(context, project.id, input.runId);
+    const prepared = context.db.readSnapshot(() => {
+        const guard = recoveryGuard(context, project.id, run.id);
+        const binding = context.db.get("SELECT * FROM run_recovery_bindings WHERE recovery_id = ?", [input.recoveryId]);
+        if (!guard || guard.recovery_id !== input.recoveryId || !binding || !["resuming", "recovered"].includes(guard.status))
+            throw new AppError("recovery_state_changed", "Recovery dispatch no longer belongs to the current generation", 1);
+        recoveryPacketReceipt(binding);
+        if (binding.runtime_dispatch_json) {
+            const prior = JSON.parse(binding.runtime_dispatch_json);
+            if (prior.state_dir !== input.stateDir || prior.operation_id !== input.operationId)
+                throw new AppError("recovery_binding_mismatch", "Recovery dispatch is already pinned to another operation", 1);
+            return { dispatch: prior, guard, binding };
+        }
+        if (guard.status !== "resuming" || binding.status !== "prepared")
+            throw new AppError("recovery_state_changed", "Recovery acknowledgement is no longer awaiting dispatch", 1);
+        const { state, process: record } = requireManagedDaemonBinding(context, { stateDir: input.stateDir, projectId: project.id, runId: run.id });
+        if ((state.daemon_id ?? state.config?.daemonId) !== binding.daemon_id || record.state !== "running" || !processIsAlive(record))
+            throw new AppError("recovery_runtime_mismatch", "Recovery requires its live replacement daemon", 1);
+        const metadata = JSON.parse(record.metadata_json);
+        if (metadata.state_dir !== input.stateDir || metadata.run_generation !== guard.generation || metadata.dispatch_barrier !== "run-generation@1" || !["subject", "merge"].includes(metadata.role ?? ""))
+            throw new AppError("recovery_runtime_mismatch", "Replacement daemon lacks this recovery generation's native dispatch barrier", 1);
+        return { guard, binding, record };
+    });
+    let dispatch = prepared.dispatch;
+    if (!dispatch) {
+        const prompt = `Resume this retained RUN in its existing native Session. Your first tool call must be this exact standalone command:\n\n${acceptanceCommand(context, run.id, prepared.guard.recovery_id, run.project_root)}\n\nFollow the verified recovery packet returned by that command. Preserve completed Work and existing HITL; do not start a successor Stage yourself. Stop at the Stage or declared Work-graph boundary.\n\n${lifecycleRetryInstruction}\n`;
+        dispatch = { process_id: prepared.record.id, owner_id: prepared.record.owner_id, state_dir: input.stateDir, daemon_id: prepared.binding.daemon_id, operation_id: input.operationId, resume_operation_id: `${input.operationId}:load`, prompt_sha256: crypto.createHash("sha256").update(prompt).digest("hex"), prompt };
+    }
+    const registry = prepared.record ? getResourceDatabase(resourceHome(context)) : undefined;
+    return context.db.writeTransaction(() => {
+        const guard = recoveryGuard(context, project.id, run.id);
+        const binding = context.db.get("SELECT * FROM run_recovery_bindings WHERE recovery_id = ?", [input.recoveryId]);
+        if (JSON.stringify(guard) !== JSON.stringify(prepared.guard) || JSON.stringify(binding) !== JSON.stringify(prepared.binding))
+            throw new AppError("recovery_state_changed", "Recovery generation changed while dispatch was prepared", 1);
+        if (prepared.record) {
+            const current = registry.get("SELECT * FROM managed_processes WHERE id = ?", [prepared.record.id]);
+            if (!current || current.state !== "running" || current.lease_expires_at <= context.now() || managedProcessBinding(current, resourceHome(context)).registration_sha256 !== managedProcessBinding(prepared.record, resourceHome(context)).registration_sha256)
+                throw new AppError("recovery_runtime_mismatch", "Recovery replacement daemon lease changed before dispatch publication", 1);
+            context.db.run("UPDATE run_recovery_bindings SET runtime_dispatch_json = ? WHERE recovery_id = ? AND status = 'prepared' AND runtime_dispatch_json IS NULL", [JSON.stringify(dispatch), prepared.binding.recovery_id]);
+        }
+        return dispatch;
+    });
+}
+/** Publish only an exact successor to a failed, effect-free recovery ACK Turn. */
+export function retainRecoveryCapacityIntent(context, input) {
+    const { intent } = input;
+    if (!Number.isSafeInteger(intent.ordinal) || intent.ordinal < 1 || intent.ordinal > MAX_OVERLOAD_RETRIES || !intent.turn_id || !Number.isFinite(Date.parse(intent.not_before)) || intent.policy !== CAPACITY_POLICY
+        || !intent.refusal || intent.refusal.policy !== CAPACITY_POLICY || intent.refusal.ordinal !== intent.ordinal - 1 || intent.refusal.turn_id !== intent.turn_id
+        || !Number.isFinite(Date.parse(intent.refusal.first_observed_at)) || intent.deadline !== null)
+        throw new AppError("recovery_capacity_intent_invalid", "Recovery capacity successor is invalid", 1);
+    context.db.writeTransaction(() => {
+        const guard = recoveryGuard(context, input.projectId, input.runId);
+        const binding = context.db.get("SELECT * FROM run_recovery_bindings WHERE recovery_id = ? AND daemon_id = ? AND status = 'prepared'", [input.recoveryId, input.daemonId]);
+        if (!guard || guard.recovery_id !== input.recoveryId || guard.generation !== input.generation || guard.status !== "resuming" || !binding?.runtime_dispatch_json)
+            throw new AppError("recovery_capacity_owner_changed", "Recovery ACK no longer owns the current generation", 1);
+        const controller = context.db.get("SELECT generation, status FROM run_controllers WHERE controller_id = ? AND project_id = ? AND run_id = ?", [input.controllerId, input.projectId, input.runId]);
+        const control = context.db.get("SELECT status FROM run_controls WHERE recovery_id = ?", [input.recoveryId]);
+        if (controller?.generation !== input.generation || ["completed", "cancelled", "superseded"].includes(controller.status) || control?.status !== "resuming")
+            throw new AppError("recovery_capacity_owner_changed", "Recovery controller or control changed before successor publication", 1);
+        const dispatch = JSON.parse(binding.runtime_dispatch_json);
+        if (intent.root_operation_id !== dispatch.operation_id || intent.session_id !== binding.provider_session_id || intent.refusal.session_id !== binding.provider_session_id)
+            throw new AppError("recovery_capacity_intent_conflict", "Recovery capacity intent belongs to another root or Session", 1);
+        const currentProcess = getResourceDatabase(resourceHome(context)).get("SELECT * FROM managed_processes WHERE id = ?", [dispatch.process_id]);
+        if (!currentProcess || currentProcess.state !== "running" || currentProcess.lease_expires_at <= context.now() || currentProcess.owner_id !== dispatch.owner_id
+            || currentProcess.project_id !== input.projectId || currentProcess.run_id !== input.runId)
+            throw new AppError("recovery_capacity_owner_changed", "Replacement daemon lease changed before successor publication", 1);
+        const prior = dispatch.capacity_intents ?? [];
+        for (let index = 0; index < prior.length; index++) {
+            const retained = prior[index];
+            if (retained.policy !== CAPACITY_POLICY || retained.ordinal !== index + 1 || retained.root_operation_id !== dispatch.operation_id || retained.session_id !== binding.provider_session_id
+                || retained.predecessor !== (index === 0 ? dispatch.operation_id : prior[index - 1].operation_id)
+                || retained.operation_id !== `${dispatch.operation_id}:capacity:${index + 1}:session.prompt` || retained.refusal?.ordinal !== index
+                || retained.refusal.turn_id !== retained.turn_id || retained.refusal.session_id !== binding.provider_session_id)
+                throw new AppError("recovery_capacity_intent_conflict", "Retained recovery capacity chain is invalid", 1);
+        }
+        const refusals = dispatch.capacity_failures ?? [];
+        const observed = refusals.find(item => item.ordinal === intent.refusal.ordinal);
+        if (!observed || JSON.stringify(observed) !== JSON.stringify(intent.refusal) || overloadExhausted(refusals))
+            throw new AppError("recovery_capacity_intent_conflict", "Recovery successor lacks an immutable non-burst refusal binding", 1);
+        const predecessor = intent.ordinal === 1 ? dispatch.operation_id : prior[intent.ordinal - 2]?.operation_id;
+        if (prior.length !== intent.ordinal - 1 || predecessor !== intent.predecessor || intent.operation_id !== `${dispatch.operation_id}:capacity:${intent.ordinal}:session.prompt`)
+            throw new AppError("recovery_capacity_intent_conflict", "Recovery capacity successor does not extend the pinned ACK", 1);
+        const failed = context.db.get("SELECT status FROM run_controller_operations WHERE operation_id = ? AND controller_id = ? AND generation = ?", [intent.predecessor, input.controllerId, input.generation]);
+        if (failed?.status !== "outcome_unknown")
+            throw new AppError("recovery_capacity_intent_conflict", "Recovery predecessor is not the failed native operation", 1);
+        const next = { ...intent, prompt_sha256: crypto.createHash("sha256").update(input.prompt).digest("hex") };
+        const updated = { ...dispatch, capacity_intents: [...prior, next] };
+        const changed = context.db.run("UPDATE run_recovery_bindings SET runtime_dispatch_json = ? WHERE recovery_id = ? AND daemon_id = ? AND status = 'prepared' AND runtime_dispatch_json = ?", [JSON.stringify(updated), input.recoveryId, input.daemonId, binding.runtime_dispatch_json]);
+        if (changed.changes !== 1)
+            throw new AppError("recovery_capacity_intent_conflict", "Recovery dispatch changed before successor publication", 1);
+        const marked = context.db.run("UPDATE run_controller_operations SET status = 'failed', receipt_json = ?, updated_at = ? WHERE operation_id = ? AND controller_id = ? AND generation = ? AND status = 'outcome_unknown'", [JSON.stringify({ capacity_failure: next.refusal, capacity_intent: next }), context.now(), intent.predecessor, input.controllerId, input.generation]);
+        if (marked.changes !== 1)
+            throw new AppError("recovery_capacity_intent_conflict", "Recovery predecessor changed before successor publication", 1);
+    });
+}
+export { recoveryRuntimeAdmission } from "./run-recovery-guard.js";