@deksden-com/dd-flow-cli 0.9.0-beta.135 → 0.9.0-beta.137

package/dist/services/run-controller.js

@@ -31,6 +31,7 @@ import { requireRun, requireController, controllerReceipt, appendEvent, saveStat
 import { assertNoCorrelatedRejection } from "./continuation-outcome.js";
 import { spawnController, recordControllerLauncher, claimControllerProcess, markControllerDispatchReady, assertControllerRecoveryAllowed } from "./run-controller-process.js";
 import { controllerAdapter, controllerEnvironment, sessionArguments } from "./run-controller-adapter.js";
+import { retainedChildObserver } from "./controller-child-observer.js";
 import { capacityDelay, continueCodexCapacity, eligibleCodexChildOverload } from "./codex-capacity.js";
 import { resumeControllerSession, failControllerRecovery } from "./run-controller-recovery.js";
 import { captureControllerBoundary } from "./run-controller-capture.js";
@@ -460,6 +461,52 @@ async function executeController(context, row, manifest, state, assertOwnership,
     let fatal;
     let stopping;
     const cancellation = new AbortController();
+    const observationCancellation = new AbortController();
+    const childObservers = new Map();
+    let promptPending = false;
+    let observedPromptOperation = null;
+    const childObserver = (session) => {
+        if (runtimeHarness(session.profile) !== "zcode-acp" || !state.stage || !["plan-review", "code", "code-review"].includes(state.stage))
+            return undefined;
+        if (readNativeDaemonState(session.state_dir).capabilities?.retained_child_observation !== true)
+            return undefined;
+        const scope = { controller_id: row.controller_id, generation: row.generation, stage: state.stage, root_session_id: session.id };
+        const key = JSON.stringify(scope);
+        let retained = childObservers.get(key);
+        if (!retained) {
+            retained = retainedChildObserver({ file: path.join(row.state_dir, "native-child-observations.jsonl"), scope, assertOwner: assertOwnership,
+                gap: error => { process.stderr.write(`${JSON.stringify({ event: "native_child_observation_gap", scope, error_code: String(error?.code ?? "observation_unavailable") })}\n`); },
+                read: async () => {
+                    assertOwnership();
+                    const operation = observedPromptOperation;
+                    const binding = requireManagedDaemonBinding(executionContext, { stateDir: session.state_dir, projectId: row.project_id, runId: row.run_id });
+                    let receipt;
+                    try {
+                        receipt = await runHarnessAdapter({ executable: manifest.adapters["zcode-acp"].executable,
+                            args: ["session", "observe-children", "--session-id", session.id, ...sessionArguments(context, manifest, session)],
+                            env: controllerEnvironment(context, manifest, session), operationId: crypto.randomUUID(), harness: "zcode", timeoutMs: 30_000,
+                            signal: AbortSignal.any([cancellation.signal, observationCancellation.signal]) });
+                    }
+                    catch (error) {
+                        if (!promptPending || observedPromptOperation !== operation)
+                            return null;
+                        throw error;
+                    }
+                    if (!promptPending || observedPromptOperation !== operation || receipt.active_operation_id === null)
+                        return null;
+                    if (receipt.schema_id !== "dd-zcode/child-observation@1" || receipt.provider_session_id !== session.id || receipt.daemon_id !== binding.state.daemon_id)
+                        throw new AppError("native_child_observation_conflict", "Retained child observation has foreign daemon/root identity", 1);
+                    if (receipt.active_operation_id !== operation)
+                        throw new AppError("native_child_observation_conflict", "Retained observation has a foreign productive operation", 1);
+                    const children = controllerNativeChildren(receipt, session.id).filter(child => !(state.native_baseline ?? []).includes(child.session_id));
+                    if (children.some(child => child.status === "failed" && !child.terminal_turn_id))
+                        throw new AppError("operation_observation_lost", "Retained child terminal identity is unavailable", 1);
+                    return children;
+                } });
+            childObservers.set(key, retained);
+        }
+        return retained;
+    };
     const capacityUncertain = (error) => error instanceof AppError && ["capacity_continuation_blocked", "operation_observation_lost"].includes(error.code) && error.details.capacity_uncertainty === true || Boolean(state.pending_capacity) && (String(error?.code) === "capacity_continuation_blocked" || isObservationLoss(error));
     const stopAfterFailure = (error) => {
         // A retained successor may already be running. Fencing/recovery records
@@ -492,10 +539,15 @@ async function executeController(context, row, manifest, state, assertOwnership,
         }).finally(() => cancellation.abort());
         return stopping;
     };
-    const setStatus = (status) => {
-        const changed = context.db.run("UPDATE run_controllers SET status = ?, updated_at = ? WHERE controller_id = ? AND owner_token = ? AND status <> ?", [status, context.now(), row.controller_id, row.owner_token, status]);
-        if (changed.changes)
-            appendEvent(context, row.controller_id, "status_changed", { status, stage: state.stage });
+    const setStatus = (status, terminalData) => {
+        context.db.writeTransaction(() => {
+            const changed = context.db.run("UPDATE run_controllers SET status = ?, updated_at = ? WHERE controller_id = ? AND owner_token = ? AND status <> ?", [status, context.now(), row.controller_id, row.owner_token, status]);
+            if (changed.changes) {
+                appendEvent(context, row.controller_id, "status_changed", { status, stage: state.stage });
+                if (terminalData)
+                    appendEvent(context, row.controller_id, status, terminalData);
+            }
+        });
     };
     const call = (session, args, productive = true) => controllerAdapter(executionContext, row, manifest, session, args, productive ? assertOwner : undefined, undefined, productive ? cancellation.signal : undefined, productive ? renewLease : undefined);
     const confirmStopped = async (stateDir, receipt) => {
@@ -575,7 +627,17 @@ async function executeController(context, row, manifest, state, assertOwnership,
                         throw new AppError("capacity_continuation_conflict", "Retained capacity root has no failure receipt", 1);
                     throw JSON.parse(root.error_json);
                 }
-                receipt = await send(text, firstOperationId);
+                if (childObserver(session)?.failed().length)
+                    throw new AppError("fanout_dispatch_fenced", "A retained child failure forbids another coordinator dispatch", 1);
+                promptPending = true;
+                observedPromptOperation = firstOperationId;
+                try {
+                    receipt = await send(text, firstOperationId);
+                }
+                finally {
+                    promptPending = false;
+                    observedPromptOperation = null;
+                }
             }
             catch (error) {
                 if (!["codex-desktop", "opencode-server"].includes(runtimeHarness(session.profile)))
@@ -640,6 +702,11 @@ async function executeController(context, row, manifest, state, assertOwnership,
                 receipt = resumed;
             }
             checkOutcomes();
+            await childObserver(session)?.drain();
+            childObserver(session)?.completePostTurn(receipt);
+            const observationGap = childObserver(session)?.gap();
+            if (observationGap)
+                throw new AppError("operation_observation_lost", "In-flight child observation remained unavailable at the safe Turn boundary", 1, { observation_error_code: observationGap, provider_session_id: session.id, stage: state.stage });
             if (fatal)
                 throw fatal;
         }
@@ -661,6 +728,8 @@ async function executeController(context, row, manifest, state, assertOwnership,
             return;
         try {
             checkOutcomes();
+            if (promptPending && state.current_session !== null)
+                childObserver(state.sessions[state.current_session])?.tick();
         }
         catch (error) {
             void stopAfterFailure(error);
@@ -692,14 +761,16 @@ async function executeController(context, row, manifest, state, assertOwnership,
             const next = view.continuation;
             terminalFinalization = next.kind === "terminal";
             const completed = state.stage && view.index.stage_runs.find(stage => stage.stage === state.stage && ["done", "skipped"].includes(stage.status));
+            if (state.current_session !== null && (completed || next.stage !== state.stage) && childObserver(state.sessions[state.current_session])?.failed().length)
+                throw new AppError("fanout_dispatch_fenced", "A retained child failure forbids completing or advancing its Stage", 1);
             if (completed && manifest.request.stop_after === state.stage) {
                 // A bounded run is final evidence, not an entry fixture for its successor.
                 // Stop owned observers before capture; keep the controller lease/fence.
                 await settle();
                 if (manifest.request.capture_root)
                     await captureControllerBoundary(executionContext, row, manifest, completed, null, assertPhysicalOwner);
-                setStatus("stop_target_reached");
-                appendEvent(context, row.controller_id, "stop_target_reached", { stage: state.stage, attempt: completed.attempt ?? null });
+                setFlowRunRecoveryPaused(context, row.project_id, row.run_id, true);
+                setStatus("stop_target_reached", { stage: state.stage, attempt: completed.attempt ?? null });
                 return;
             }
             if (completed && manifest.request.capture_root && ["start_stage", "terminal"].includes(next.kind)) {
@@ -838,6 +909,8 @@ async function executeController(context, row, manifest, state, assertOwnership,
             const session = state.sessions[state.current_session];
             const baseline = new Set(state.native_baseline ?? []);
             let children = controllerNativeChildren(state.last_receipt ?? {}, session.id).filter(child => !baseline.has(child.session_id));
+            const retainedFailures = childObserver(session)?.failed() ?? [];
+            children = controllerNativeChildren({ descendants: [...children, ...retainedFailures] }, session.id);
             if (runtimeHarness(session.profile) === "codex-desktop" && children.some(child => child.status === "failed")) {
                 const inspected = await call(session, ["session", "inspect", "--session-id", session.id, ...sessionArguments(context, manifest, session)], false);
                 state.last_receipt = inspected;
@@ -940,7 +1013,7 @@ async function executeController(context, row, manifest, state, assertOwnership,
                     continue;
                 }
             }
-            const fanout = await nextControllerFanout(executionContext, { projectRoot: run.project_root, runId: run.id, stage: next.stage, harness: runtimeHarness(session.profile), sessionId: session.id, children, policy, profiles });
+            const fanout = await nextControllerFanout(executionContext, { projectRoot: run.project_root, runId: run.id, stage: next.stage, harness: runtimeHarness(session.profile), sessionId: session.id, children, policy, profiles, dispatchFence: retainedFailures.length > 0 });
             assertOwner();
             if (fanout.kind === "none") {
                 assertNoCorrelatedRejection(context, { projectId: row.project_id, runId: row.run_id, generation: row.generation, stage: next.stage, turn: state.turns });
@@ -1012,6 +1085,9 @@ async function executeController(context, row, manifest, state, assertOwnership,
     }
     finally {
         clearInterval(monitor);
+        const drainedObservations = [...childObservers.values()].map(retained => retained.close());
+        observationCancellation.abort();
+        await Promise.allSettled(drainedObservations);
         if (stopping)
             await stopping;
         observer.db.close?.();

package/dist/services/run-health.js

@@ -0,0 +1,149 @@
+import { AppError } from "../shared/errors.js";
+import { managedProcessBinding, managedProcessRuntimeHome, managedProcessStatus, managedProcessObservation, processTreeIsAlive } from "./managed-processes.js";
+import { reconcileStoredFlowRun } from "./runs.js";
+import { recoveryGuard } from "./run-recovery.js";
+import { spawnSync } from "node:child_process";
+function physicalInventory() {
+    const reply = spawnSync('ps', ['-axo', 'pid=,pgid=,stat='], { encoding: 'utf8', timeout: 5000 });
+    if (reply.status !== 0)
+        return null;
+    return reply.stdout.trim().split('\n').map(line => line.trim().split(/\s+/)).filter(parts => parts.length === 3).map(parts => ({ pid: Number(parts[0]), group: Number(parts[1]), zombie: parts[2].startsWith('Z') }));
+}
+const terminalRun = (status) => ["done", "failed", "cancelled", "discarded"].includes(status);
+const waiting = new Set(["waiting_for_user", "waiting_for_context", "waiting_for_capacity", "waiting_for_children", "waiting_for_merge", "stop_target_reached", "paused", "control_requested"]);
+const table = (context, name) => Boolean(context.db.get("SELECT 1 FROM sqlite_schema WHERE name = ?", [name]));
+const latest = (values) => values.filter((value) => typeof value === "string" && Number.isFinite(Date.parse(value))).sort((a, b) => Date.parse(b) - Date.parse(a))[0] ?? null;
+/** Evidence only: lease expiry and quiet providers never prove an exit. */
+export function assessRunHealth(input) {
+    const phase = input.phase ?? null;
+    let state = "unknown", reason = "owner_identity_missing";
+    if (terminalRun(input.status)) {
+        state = "settled";
+        reason = "saved_terminal_result";
+    }
+    else if (input.childrenAlive && input.owner === "absent") {
+        state = "owner_lost";
+        reason = "native_children_still_active";
+    }
+    else if (phase === 'recovery_required' || phase === 'capture_failed') {
+        state = input.owner === 'absent' ? 'owner_lost' : 'unknown';
+        reason = phase;
+    }
+    else if (phase && waiting.has(phase) || ["waiting_for_user", "blocked"].includes(input.status)) {
+        state = "waiting";
+        reason = phase ?? input.status;
+    }
+    else if (input.owner === "absent") {
+        state = "owner_lost";
+        reason = "physical_owner_retired";
+    }
+    else if (input.owner === "alive") {
+        const age = input.lastProgress ? Date.parse(input.now) - Date.parse(input.lastProgress) : NaN;
+        state = Number.isFinite(age) && age > input.noProgressMs ? "no_progress" : "active";
+        reason = state === "no_progress" ? "progress_overdue" : "physical_owner_observed";
+    }
+    else if (input.status === "paused") {
+        state = "waiting";
+        reason = "saved_pause";
+    }
+    return { schema_id: "dd-flow/run-health@1", state, reason, phase, owner_state: input.owner, last_progress_at: input.lastProgress ?? null, last_heartbeat_at: input.heartbeat ?? null, observed_at: input.now, controller_id: input.controllerId ?? null, generation: input.generation ?? null, no_progress_ms: input.noProgressMs };
+}
+/** No dispatch, cancellation, process signals or provider calls. All changes
+ * are fenced by the retained RUN revision, controller and process binding. */
+export function reconcileFlowRunHealth(context, input = {}) {
+    const noProgressMs = input.noProgressMs ?? 30 * 60_000;
+    if (!Number.isSafeInteger(noProgressMs) || noProgressMs < 1_000)
+        throw new AppError("usage", "no-progress-ms must be an integer of at least 1000", 2);
+    const runs = context.db.all(`SELECT * FROM runs${input.runId ? " WHERE (id = ? OR short_id = ?)" : ""}`, input.runId ? [input.runId, input.runId] : []).filter(run => !input.projectRoot || run.project_root === input.projectRoot);
+    if (input.runId && runs.length !== 1)
+        throw new AppError("not_found", "RUN scope is missing or ambiguous", 1);
+    const processes = physicalInventory();
+    const alive = (record) => {
+        if (!record.pid)
+            return !['stopped', 'failed'].includes(record.state);
+        const physical = processes?.find(item => item.pid === record.pid);
+        if (processes && (!physical || physical.zombie)) {
+            const metadata = JSON.parse(record.metadata_json);
+            // A retired group leader may still own physical children. Merely ending
+            // the controller does not settle those children or remote provider work.
+            return Boolean(metadata.process_group_id && processes.some(item => item.group === metadata.process_group_id && !item.zombie));
+        }
+        return !record.pid_started_at || processTreeIsAlive(record);
+    };
+    return { ok: true, results: runs.map(run => {
+            const index = JSON.parse(run.index_json);
+            const controllers = table(context, "run_controllers") ? context.db.all("SELECT * FROM run_controllers WHERE project_id = ? AND run_id = ? ORDER BY generation DESC, created_at DESC", [run.project_id, run.id]) : [];
+            const controller = controllers[0];
+            const manifest = controller ? JSON.parse(controller.manifest_json) : null;
+            const resourceContext = manifest?.resource_home ? { ...context, env: { ...context.env, DD_FLOW_RESOURCE_HOME: manifest.resource_home } } : context;
+            let records = [], inventoryComplete = false;
+            try {
+                records = managedProcessStatus(resourceContext).filter(record => record.project_id === run.project_id && record.run_id === run.id && managedProcessRuntimeHome(record, context) === context.ddFlowHome);
+                inventoryComplete = true;
+            }
+            catch { /* Missing registry is uncertainty, never retirement proof. */ }
+            const physical = controller && records.find(record => record.id === controller.process_id && record.lease_token === controller.process_lease_token && record.owner_id === controller.controller_id && record.kind === "run-controller");
+            const owner = physical?.pid && physical.pid_started_at ? processes && !processes.some(item => item.pid === physical.pid && !item.zombie) && !alive(physical) ? 'absent' : managedProcessObservation(physical) : "unknown";
+            const retireController = Boolean(controller && owner === 'absent' && !['completed', 'cancelled', 'superseded', 'stop_target_reached', 'recovery_required', 'capture_failed', 'waiting_for_user', 'waiting_for_context', 'control_requested', 'paused'].includes(controller.status));
+            const phase = retireController ? 'recovery_required' : controller?.status ?? null;
+            const childrenAlive = records.some(record => record.id !== physical?.id && alive(record)) || Boolean(physical && owner === 'absent' && alive(physical));
+            const progressEvent = controller && table(context, "run_controller_events") ? context.db.get("SELECT MAX(created_at) AS at FROM run_controller_events WHERE controller_id = ? AND event_type NOT IN ('health_reconciled','heartbeat')", [controller.controller_id])?.at : null;
+            const workProgress = table(context, "works") ? context.db.get("SELECT MAX(at) AS at FROM (SELECT created_at AS at FROM works WHERE project_id=? AND run_id=? UNION ALL SELECT started_at FROM works WHERE project_id=? AND run_id=? UNION ALL SELECT completed_at FROM works WHERE project_id=? AND run_id=?)", [run.project_id, run.id, run.project_id, run.id, run.project_id, run.id])?.at : null;
+            let hookProgress = null;
+            if (table(context, 'hook_events') && table(context, 'sessions'))
+                try {
+                    hookProgress = context.db.get("SELECT MAX(h.created_at) AS at FROM hook_events h JOIN sessions s ON s.session_id=h.session_id AND s.project_id=h.project_id WHERE s.project_id=? AND s.run_id=? AND h.event_name IN ('PostToolUse','UserPromptSubmit','SubagentStop','Stop','SessionStart')", [run.project_id, run.id])?.at ?? null;
+                }
+                catch { /* Legacy hooks have no certified Run membership. */ }
+            const lastProgress = latest([run.created_at, progressEvent, workProgress, hookProgress, ...index.stage_runs.flatMap(stage => [stage.started_at, stage.completed_at])]);
+            const policy = input.noProgressMs ?? (phase === 'starting' ? 120_000 : phase === 'capturing' ? 300_000 : 30 * 60_000);
+            const health = assessRunHealth({ status: run.status, phase, owner, childrenAlive, lastProgress, heartbeat: physical?.updated_at ?? null, now: context.now(), noProgressMs: policy, controllerId: controller?.controller_id ?? null, generation: controller?.generation ?? null });
+            if (terminalRun(run.status) && (childrenAlive || owner === "alive")) {
+                health.state = "unknown";
+                health.reason = "terminal_result_has_live_runtime";
+            }
+            const sessionState = controller ? JSON.parse(controller.state_json) : null;
+            const nativeSettled = Boolean(sessionState?.sessions.length && sessionState.sessions.every(session => session.stopped === true));
+            const outcomeUnknown = controller && table(context, "run_controller_operations") && context.db.get("SELECT 1 FROM run_controller_operations WHERE controller_id = ? AND generation = ? AND status IN ('accepted','outcome_unknown') LIMIT 1", [controller.controller_id, controller.generation]);
+            let transition;
+            const guard = table(context, 'run_recovery_guards') ? recoveryGuard(context, run.project_id, run.id) : null;
+            const currentGeneration = !guard || !controller || controller.generation === guard.generation;
+            if (!terminalRun(run.status) && currentGeneration) {
+                if (controller && ["recovery_required", "capture_failed"].includes(controller.status))
+                    transition = "paused";
+                else if (controller?.status === "stop_target_reached" && owner !== "alive" && nativeSettled && inventoryComplete && !childrenAlive)
+                    transition = "paused";
+                else if (health.state === "owner_lost" && owner === "absent") {
+                    transition = controller?.last_error_json && nativeSettled && inventoryComplete && !childrenAlive && !outcomeUnknown && !guard ? "failed" : "paused";
+                }
+            }
+            const nextStatus = transition ?? run.status;
+            if (transition === 'failed') {
+                health.state = 'settled';
+                health.reason = 'saved_terminal_result';
+            }
+            const unchanged = index.health && JSON.stringify({ ...index.health, observed_at: null }) === JSON.stringify({ ...health, observed_at: null }) && nextStatus === run.status;
+            let changed = false;
+            if (input.apply && !unchanged) {
+                // Verify the same physical inventory immediately before publishing. A
+                // replacement/renewal invalidates this observation, even with the same PID.
+                const binding = (items) => JSON.stringify(items.map(item => ({ ...managedProcessBinding(item, manifest?.resource_home ?? context.ddFlowHome), state: item.state, updated_at: item.updated_at, lease_expires_at: item.lease_expires_at })));
+                if (inventoryComplete) {
+                    const fresh = managedProcessStatus(resourceContext).filter(record => record.project_id === run.project_id && record.run_id === run.id && managedProcessRuntimeHome(record, context) === context.ddFlowHome);
+                    if (binding(fresh) !== binding(records) || physical && owner === "absent" && managedProcessObservation(physical) !== 'absent')
+                        throw new AppError("run_health_changed", "Runtime ownership changed during reconciliation", 1);
+                }
+                context.db.writeTransaction(() => {
+                    const current = controller && context.db.get("SELECT * FROM run_controllers WHERE controller_id = ?", [controller.controller_id]);
+                    if (controller && JSON.stringify(current) !== JSON.stringify(controller))
+                        throw new AppError("run_health_changed", "Controller changed during reconciliation", 1);
+                    reconcileStoredFlowRun(context, run, health, transition, inventoryComplete && owner !== "alive" && !childrenAlive && nativeSettled);
+                    if (controller && retireController) {
+                        context.db.run("UPDATE run_controllers SET status = 'recovery_required', updated_at = ? WHERE controller_id = ? AND owner_token = ? AND generation = ?", [context.now(), controller.controller_id, controller.owner_token, controller.generation]);
+                    }
+                    changed = true;
+                });
+            }
+            return { project_id: run.project_id, run_id: run.id, previous_status: run.status, status: nextStatus, health, changed, would_change: !unchanged };
+        }) };
+}

package/dist/services/runs.js

@@ -911,6 +911,46 @@ export function setFlowRunRecoveryPaused(context, projectId, runId, paused) {
     const project = requireProjectByRoot(context, run.project_root);
     persistRunState(context, project, run, index);
 }
+/** Reconciliation repairs projections under the same revision CAS as normal
+ * lifecycle writes. Unknown physical ownership never closes child records. */
+export function reconcileStoredFlowRun(context, expected, health, transition, childrenSettled = false) {
+    const run = resolveRun(context, expected.project_id, expected.id);
+    if (run.index_json !== expected.index_json || run.status !== expected.status)
+        throw new AppError("run_revision_changed", "RUN changed during health observation", 1);
+    const index = authoritativeIndex(run), now = context.now();
+    index.health = health;
+    if (transition && !["done", "failed", "cancelled", "discarded"].includes(run.status)) {
+        index.status = transition;
+        if (transition === "failed") {
+            index.verdict = "failed";
+            // Observation time is not the historical exit time. Without a retained
+            // finish receipt, keep timing unknown rather than manufacturing days of work.
+            delete index.completed_at;
+            delete index.finished_at;
+            index.duration_ms = null;
+            index.timing_status = 'legacy_incomplete';
+            closeOpenStagesForOverride(index, "failed", now, false);
+            closeOpenWorksForOverride(context, run.project_id, run.id, "failed", now, false);
+        }
+        else if (childrenSettled && health.state === "owner_lost") {
+            for (const stage of index.stage_runs)
+                if (stage.status === "running") {
+                    stage.status = "paused";
+                    stage.updated_at = now;
+                }
+        }
+    }
+    if (childrenSettled && ["failed", "cancelled", "discarded"].includes(index.status)) {
+        closeOpenStagesForOverride(index, index.status === "failed" ? "failed" : "cancelled", now, false);
+        closeOpenWorksForOverride(context, run.project_id, run.id, index.status === "failed" ? "failed" : "cancelled", now, false);
+        context.db.run("UPDATE sessions SET status = 'stopped', stop_reason = ?, updated_at = ? WHERE project_id = ? AND run_id = ? AND status IN ('pending','active','waiting_user','blocked','stopping')", [`run_${index.status}`, now, run.project_id, run.id]);
+        context.db.run("UPDATE check_receipts SET status = 'aborted' WHERE project_id = ? AND run_id = ? AND status = 'running'", [run.project_id, run.id]);
+    }
+    // Health observation is not semantic progress or a new completion timestamp.
+    const project = requireProjectByRoot(context, run.project_root);
+    persistRunState(context, project, run, index, { healthOnly: true });
+    appendAudit(context, { projectId: project.id, eventType: "flow_run.health_reconciled", payload: { run_id: run.id, previous_status: run.status, status: index.status, health } });
+}
 export function prepareFlowRunCompletion(context, input) {
     if (input.manualOverrideReason !== undefined) {
         requiredPlain(input.manualOverrideReason, "override reason");
@@ -1061,15 +1101,21 @@ export function prepareVnextMergeSourceRepairAttempt(context, input) {
     appendRunTimeline(context, project.id, run.id, { at: now, type: "merge_source_repair_prepared", run_id: run.id, merge_request_id: input.mergeRequestId, cycle });
     return { run_id: run.id, cycle };
 }
-function closeOpenStagesForOverride(index, status, now) {
+function closeOpenStagesForOverride(index, status, now, timingKnown = true) {
     const stageStatus = status === "cancelled" ? "skipped" : "failed";
     for (const stage of index.stage_runs) {
-        if (stage.status !== "running" && stage.status !== "pending" && stage.status !== "paused" && stage.status !== "blocked")
+        if (stage.status !== "running" && stage.status !== "paused" && stage.status !== "waiting_for_user" && stage.status !== "blocked")
             continue;
         stage.status = stageStatus;
         stage.updated_at = now;
-        stage.completed_at = now;
-        stage.duration_ms = durationMs(stage.started_at ?? null, now);
+        if (timingKnown) {
+            stage.completed_at = now;
+            stage.duration_ms = durationMs(stage.started_at ?? null, now);
+        }
+        else {
+            delete stage.completed_at;
+            stage.duration_ms = null;
+        }
     }
     index.attempts = index.stage_runs.map((stage) => ({
         stage: stage.stage,
@@ -1077,20 +1123,20 @@ function closeOpenStagesForOverride(index, status, now) {
         root: stage.dir,
         archive: null,
         status: stage.status,
-        started_at: stage.started_at ?? now,
+        started_at: stage.started_at ?? null,
         finished_at: stage.completed_at ?? null
     }));
 }
-function closeOpenWorksForOverride(context, projectId, runId, status, now) {
+function closeOpenWorksForOverride(context, projectId, runId, status, now, timingKnown = true) {
     const workStatus = status === "cancelled" ? "cancelled" : "failed";
     context.db.run(`UPDATE works
-       SET status = ?, updated_at = ?, completed_at = ?
+       SET status = ?, updated_at = ?, completed_at = CASE WHEN ? = 1 THEN ? ELSE completed_at END
      WHERE project_id = ? AND run_id = ?
-       AND status IN ('created', 'running', 'paused')`, [workStatus, now, now, projectId, runId]);
+       AND status IN ('created', 'running', 'paused')`, [workStatus, now, timingKnown ? 1 : 0, now, projectId, runId]);
     context.db.run(`UPDATE work_sessions
-       SET status = ?, updated_at = ?, completed_at = ?
+       SET status = ?, updated_at = ?, completed_at = CASE WHEN ? = 1 THEN ? ELSE completed_at END
      WHERE work_id IN (SELECT work_id FROM works WHERE project_id = ? AND run_id = ?)
-       AND status <> 'completed'`, [workStatus, now, now, projectId, runId]);
+       AND status <> 'completed'`, [workStatus, now, timingKnown ? 1 : 0, now, projectId, runId]);
 }
 function closeRunOrchestrators(context, projectId, run, status, now) {
     if (!["done", "blocked", "cancelled", "failed"].includes(status))
@@ -1327,8 +1373,10 @@ function vnextSpecifyGuidance(run, index) {
 function persistRunState(context, project, run, index, options = {}) {
     // Fenced productive admission is a logical pause, even while physical work
     // drains. Late admitted Stage results must not revive a stopped RUN.
-    if (index.status === "running" && ["draining", "sealed", "resuming"].includes(recoveryGuard(context, project.id, run.id)?.status ?? ""))
+    if (index.status === "running" && (!options.healthOnly || context.db.get("SELECT 1 FROM sqlite_schema WHERE name = 'run_recovery_guards'")) && ["draining", "sealed", "resuming"].includes(recoveryGuard(context, project.id, run.id)?.status ?? ""))
         index.status = "paused";
+    if (!options.healthOnly)
+        delete index.health;
     const runtimePath = run.runtime_path || projectRunJsonPath(context.ddFlowHome, project.id, run.id);
     context.db.writeTransaction(() => {
         const stored = context.db.get("SELECT index_json FROM runs WHERE project_id = ? AND id = ?", [project.id, run.id]);
@@ -1344,7 +1392,7 @@ function persistRunState(context, project, run, index, options = {}) {
         if (options.flagRevision)
             history.push(options.flagRevision);
         index.flag_revision_history = history.slice(-32);
-        const updated = context.db.run(`UPDATE runs SET status = ?, verdict = ?, next_action = ?, index_json = ?, runtime_path = ?, run_index_path = ?, updated_at = ?, completed_at = ?
+        const updated = options.healthOnly ? context.db.run("UPDATE runs SET status = ?, verdict = ?, next_action = ?, index_json = ?, updated_at = ?, completed_at = ? WHERE project_id = ? AND id = ? AND index_json = ?", [index.status, index.verdict, index.next_action, JSON.stringify(persistedVnextIndex(index)), index.updated_at, index.completed_at ?? null, project.id, run.id, stored.index_json]) : context.db.run(`UPDATE runs SET status = ?, verdict = ?, next_action = ?, index_json = ?, runtime_path = ?, run_index_path = ?, updated_at = ?, completed_at = ?
        WHERE project_id = ? AND id = ? AND index_json = ?`, [index.status, index.verdict, index.next_action, JSON.stringify(persistedVnextIndex(index)), runtimePath, runtimePath, index.updated_at, index.completed_at ?? null, project.id, run.id, stored.index_json]);
         if (updated.changes !== 1)
             throw new AppError("run_revision_changed", "RUN changed during preparation", 1, { run_id: run.id });
@@ -1431,7 +1479,7 @@ export function publishRunProjections(context, projectId, runId) {
         if (!latest)
             throw new AppError("not_found", "RUN is not registered", 1, { run_id: runId });
         const index = authoritativeIndex(latest);
-        const mutations = context.db.all("SELECT resulting_revision, result_json, created_at, idempotency_key FROM flow_run_flag_mutations WHERE project_id = ? AND run_id = ? ORDER BY resulting_revision DESC LIMIT 32", [projectId, runId]);
+        const mutations = context.db.get("SELECT 1 FROM sqlite_schema WHERE name = 'flow_run_flag_mutations'") ? context.db.all("SELECT resulting_revision, result_json, created_at, idempotency_key FROM flow_run_flag_mutations WHERE project_id = ? AND run_id = ? ORDER BY resulting_revision DESC LIMIT 32", [projectId, runId]) : [];
         const history = mutations.reverse().map((row) => ({
             revision: row.resulting_revision,
             checksum: String(JSON.parse(row.result_json).snapshot_checksum ?? ""),

package/dist/services/stage-lifecycle.js

@@ -741,7 +741,8 @@ function buildStageReport(context, view, stage, status, semantic, stageRoot, lin
             next_action: stringValue(semantic.next_action) ?? "Resolve the stage result before continuing.",
             ...(Array.isArray(semantic.questions) ? { questions: semantic.questions } : {}),
             ...(Array.isArray(semantic.reviewer_findings) ? { reviewer_findings: semantic.reviewer_findings } : {}),
-            ...(Array.isArray(semantic.def_outcomes) ? { def_outcomes: semantic.def_outcomes } : {})
+            ...(Array.isArray(semantic.def_outcomes) ? { def_outcomes: semantic.def_outcomes } : {}),
+            ...(semantic.materialization !== undefined ? { materialization: semantic.materialization } : {})
         },
         mechanical: {
             started_at: startedAt,

package/dist/services/stage-report-renderer.js

@@ -164,6 +164,19 @@ export function reconcileRunStageReports(context, projectId, runId, exceptStage)
 }
 function normalizeStageReport(report) {
     const semantic = object(report.semantic);
+    if (semantic.materialization !== undefined) {
+        const value = object(semantic.materialization);
+        const authored = object(value.authored_result);
+        const relative = (item) => typeof item === "string" && item.length > 0 && !path.isAbsolute(item) && !item.split(/[\\/]/).includes("..");
+        if (value.schema_id !== "dd-flow/protocolize-materialization@1" || value.run_id !== report.run_id || report.stage !== "protocolize" || value.stage !== "protocolize"
+            || !["create", "link", "update", "not_applicable"].includes(String(value.action)) || value.status !== "passed"
+            || authored.role !== "authored_input" || !relative(authored.run_relative_path) || !/^[a-f0-9]{64}$/.test(String(authored.sha256))
+            || !(value.feature_path === null || relative(value.feature_path)) || !Array.isArray(value.protocol_ids) || value.protocol_ids.some(id => typeof id !== "string")
+            || !Array.isArray(value.documents) || value.documents.some(item => { const ref = object(item); return !relative(ref.path) || !/^[a-f0-9]{64}$/.test(String(ref.sha256)); })
+            || !Array.isArray(value.owned_links) || value.owned_links.some(item => { const edge = object(item); return !relative(edge.feature_path) || !relative(edge.protocol_path) || !value.protocol_ids.includes(edge.protocol_id); })) {
+            throw new AppError("stage_report_materialization_invalid", "PROTOCOLIZE materialization receipt is malformed or out of scope", 1);
+        }
+    }
     const sourceArtifacts = object(report.artifacts);
     return {
         ...report,

package/dist/services/vnext-fanout.js

@@ -32,6 +32,62 @@ export function getStageFanoutCapacity(context, input) {
     const available = variables.variables[subagentCapacityKey];
     return { run_key: subagentCapacityKey, available_slots: typeof available === "number" && Number.isInteger(available) && available >= 0 ? available : null, source: "harness_qualification" };
 }
+function nativeFailureController(context, projectId, runId, stage, parent) {
+    if (!context.db.get("SELECT 1 FROM sqlite_master WHERE type='table' AND name='run_controllers'"))
+        return null;
+    const rows = context.db.all("SELECT controller_id,generation,state_json FROM run_controllers WHERE project_id=? AND run_id=? AND status NOT IN ('completed','cancelled','superseded')", [projectId, runId]);
+    const matching = rows.filter(row => {
+        try {
+            const state = JSON.parse(row.state_json);
+            return state.stage === stage && state.sessions?.[state.current_session]?.id === parent;
+        }
+        catch {
+            return false;
+        }
+    });
+    return matching.length === 1 ? matching[0] : null;
+}
+/** Failure metadata authored by a Work is not native evidence. Require the
+ * independently recorded reconciliation and its exact current registry links. */
+export function verifiedNativeWorkFailure(context, input) {
+    const b = input.receipt.native_binding;
+    if (!b || b.schema_id !== "dd-flow/native-work-failure@1")
+        return;
+    const project = requireProjectByRoot(context, input.projectRoot);
+    const scope = stageGraphScope(context, project.id, input.runId, input.stage);
+    const controller = nativeFailureController(context, project.id, input.runId, input.stage, input.parentSessionId);
+    if (!controller || b.controller_id !== controller.controller_id || b.generation !== controller.generation
+        || b.project_id !== project.id || b.run_id !== input.runId || b.stage !== input.stage
+        || b.stage_attempt_id !== scope.attempt || b.stage_cycle !== scope.cycle
+        || b.work_id !== input.workId || b.parent_work_id !== input.parentWorkId
+        || b.parent_provider_session_id !== input.parentSessionId || b.terminal_turn_id !== input.receipt.terminal_turn_id
+        || input.receipt.source !== "native_child_observation" || input.receipt.outcome !== "failed")
+        return;
+    const link = context.db.get(`SELECT ws.id FROM work_sessions ws JOIN works w ON w.work_id=ws.work_id JOIN sessions s ON s.project_id=w.project_id AND s.session_id=ws.session_id
+    WHERE ws.id=? AND w.project_id=? AND w.run_id=? AND w.work_id=? AND w.parent_work_id=? AND ws.status='failed' AND w.status='failed' AND s.parent_session_id=? AND s.provider_session_id=? AND s.harness=?`, [b.work_session_id, project.id, input.runId, input.workId, input.parentWorkId, storageSessionId({ harness_id: b.harness_id, session_id: input.parentSessionId }), b.provider_session_id, b.harness_id]);
+    if (!link)
+        return;
+    const run = context.db.get("SELECT run_root FROM runs WHERE project_id=? AND id=?", [project.id, input.runId]);
+    try {
+        const events = fs.readFileSync(path.join(run.run_root, "timeline.jsonl"), "utf8").trim().split("\n");
+        const proof = events.some(line => {
+            const event = JSON.parse(line);
+            if (event.type !== "native_child_reconciliation" || event.stage !== input.stage)
+                return false;
+            const terminal = event.observations?.children?.find((child) => child.session_id === b.provider_session_id && child.terminal_turn_id === b.terminal_turn_id);
+            if (event.observations?.parent_session_id !== b.parent_provider_session_id || event.observations?.harness_id !== b.harness_id
+                || terminal?.status !== "failed" || terminal.terminal_observed_at !== input.receipt.terminal_observed_at
+                || JSON.stringify(safeNativeFailure(terminal.provider_error)) !== JSON.stringify(safeNativeFailure(input.receipt.provider_error)))
+                return false;
+            return event.reconciled?.some((entry) => entry.outcome === "failed"
+                && entry.native_binding && Object.keys(b).every(key => b[key] === entry.native_binding[key]));
+        });
+        return proof ? b : undefined;
+    }
+    catch {
+        return;
+    }
+}
 export function fanoutState(input) {
     if (!input.hasWork)
         return input.dispatch === "none" ? "coordinator_required" : "dispatch_required";
@@ -177,12 +233,21 @@ export async function reconcileVnextFanout(context, input) {
             continue;
         }
         const providerError = safeNativeFailure(child.provider_error);
+        const controller = nativeFailureController(context, project.id, input.runId, input.stage, value.parent_session_id);
+        const scope = stageGraphScope(context, project.id, input.runId, input.stage);
+        const terminalTime = typeof child.terminal_observed_at === "string" ? Date.parse(child.terminal_observed_at) : NaN;
+        const nativeBinding = child.status === "failed" && controller && typeof child.terminal_turn_id === "string" && child.terminal_turn_id.length > 0 && child.terminal_turn_id.length <= 256 && Number.isFinite(terminalTime)
+            ? { schema_id: "dd-flow/native-work-failure@1", project_id: project.id, run_id: input.runId, stage: input.stage, stage_attempt_id: scope.attempt, stage_cycle: scope.cycle,
+                parent_work_id: status.orchestration.parent_work_id, work_id: link.work_id, work_session_id: link.id, harness_id: value.harness_id,
+                parent_provider_session_id: value.parent_session_id, provider_session_id: String(child.session_id), terminal_turn_id: child.terminal_turn_id,
+                controller_id: controller.controller_id, generation: controller.generation } : undefined;
         const reason = JSON.stringify({ source: "native_child_observation", harness_id: value.harness_id, session_id: child.session_id, work_session_id: link.id, outcome: child.status,
+            ...(nativeBinding ? { native_binding: nativeBinding } : {}),
             ...(typeof child.terminal_turn_id === "string" ? { terminal_turn_id: child.terminal_turn_id } : {}),
             ...(typeof child.terminal_observed_at === "string" ? { terminal_observed_at: child.terminal_observed_at } : {}),
             ...(providerError ? { provider_error: providerError } : {}) });
         const result = await (child.status === "failed" ? failWork : cancelWork)(context, link.work_id, reason);
-        reconciled.push({ work_id: link.work_id, work_session_id: link.id, outcome: child.status, result });
+        reconciled.push({ work_id: link.work_id, work_session_id: link.id, outcome: child.status, ...(nativeBinding ? { native_binding: nativeBinding } : {}), result });
     }
     appendFlowRunTimelineEvent(context, project.id, input.runId, { type: "native_child_reconciliation", stage: input.stage, observations: value, issues, reconciled });
     return { ok: true, issues, reconciled };