@deksden-com/dd-flow-cli 0.9.0-beta.152 → 0.9.0-beta.156

package/dist/services/run-controller-adapter.js

@@ -13,6 +13,8 @@ import { retainedDaemonReply } from "../harness-runtime/lib/daemon-observation.m
 import { readOperationRecord } from "../harness-runtime/lib/operation-progress.mjs";
 import { readNativeDaemonState } from "./managed-daemon-binding.js";
 import { validateCheckEnvironment } from "./check-environment.js";
+import { controllerObligationAccepted, lateNativeFailure } from "./controller-result-authority.js";
+import { recordTechnicalNote } from "./technical-notes.js";
 /** A paused RUN admits only its controller's exact accepted answer Turn. */
 export function controllerHitlAdmission(context, record, input) {
     if (input.operation !== "session.prompt" || !input.operationId || !input.promptSha256)
@@ -286,11 +288,16 @@ async function awaitSettlement(context, row, manifest, session, executable, args
     }
     return adapterSessionId(current) ? { ...initial, ...current, ...(initial.assistant_text !== undefined ? { assistant_text: initial.assistant_text } : {}) } : { ...initial, settlement: settlement(current) };
 }
-export async function controllerAdapter(context, row, manifest, session, args, assertOwner, retainedOperationId, signal, reconfirmOwner) {
+export async function controllerAdapter(context, row, manifest, session, args, assertOwner, retainedOperationId, signal, reconfirmOwner, obligation) {
     const executable = manifest.adapters[runtimeHarness(session.profile)].executable;
     const promptFile = args.indexOf("--prompt-file"), inlinePrompt = args.indexOf("--prompt");
     const promptSha256 = promptFile >= 0 ? digestBytes(fs.readFileSync(args[promptFile + 1])) : inlinePrompt >= 0 ? digestBytes(args[inlinePrompt + 1] ?? "") : undefined;
-    const input = { executable, args, owner: session.owner.owner_id, state_dir: session.state_dir, ...(args[0] === "session" && args[1] === "prompt" && promptSha256 ? { prompt_sha256: promptSha256 } : {}) };
+    const priorInput = retainedOperationId ? context.db.get("SELECT input_json FROM run_controller_operations WHERE operation_id=? AND controller_id=? AND generation=?", [retainedOperationId, row.controller_id, row.generation]) : undefined;
+    const effectiveObligation = priorInput ? JSON.parse(priorInput.input_json).obligation : obligation;
+    // Committing an answer resumes a Stage; it does not finish the semantic
+    // work owed by that prompt. Preserve its provider error for same-Session recovery.
+    const acceptedPromptBoundary = (expected) => controllerObligationAccepted(context, row, expected.action === "resume" ? { ...expected, action: "finish" } : expected);
+    const input = { executable, args, owner: session.owner.owner_id, state_dir: session.state_dir, ...(args[0] === "session" && args[1] === "prompt" && promptSha256 ? { prompt_sha256: promptSha256 } : {}), ...(effectiveObligation ? { obligation: effectiveObligation } : {}) };
     const prepared = retainedOperationId ? [] : context.db.all("SELECT operation_id FROM run_controller_operations WHERE controller_id = ? AND generation = ? AND input_hash = ? AND status = 'dispatching' AND json_extract(input_json, '$.dispatch_phase') = 'prepared'", [row.controller_id, row.generation, digest(input)]);
     if (prepared.length > 1)
         throw new AppError("controller_operation_conflict", "Multiple prepared intents require explicit reconciliation", 1);
@@ -369,6 +376,14 @@ export async function controllerAdapter(context, row, manifest, session, args, a
             recoveredOperation = true;
         }
         validateReceipt(receipt, session, args);
+        const hookDiagnostics = Array.isArray(receipt.native_hook_diagnostics) ? receipt.native_hook_diagnostics : [];
+        if (hookDiagnostics.some(item => item && typeof item === "object" && ["native_hook_timeout", "native_hook_failed"].includes(String(item.code)))) {
+            recordTechnicalNote(context, row.project_id, row.run_id, { code: "native_hook_observation_unavailable",
+                message: "Native hook observation failed; the actual lifecycle command remains authoritative.", phase: "native_observation",
+                generation: row.generation, ...(effectiveObligation?.stage ? { stage: effectiveObligation.stage } : {}),
+                harness: runtimeHarness(session.profile), source_operation_id: operationId, resource_identity: session.state_dir,
+                diagnostic_reference: session.journal, consequence: "none" });
+        }
         retainPhase("receipt_observed");
         const persistReceipt = () => {
             const changed = context.db.run("UPDATE run_controller_operations SET receipt_json = ?, updated_at = ? WHERE operation_id = ? AND controller_id = ? AND generation = ? AND EXISTS (SELECT 1 FROM run_controllers WHERE controller_id = ? AND owner_token = ? AND generation = ?)", [JSON.stringify(receipt), context.now(), operationId, row.controller_id, row.generation, row.controller_id, row.owner_token, row.generation]);
@@ -376,7 +391,13 @@ export async function controllerAdapter(context, row, manifest, session, args, a
                 throw new AppError("controller_owner_mismatch", "Controller lost ownership before receipt retention", 1);
         };
         persistReceipt();
-        if (["pending", "blocked"].includes(String(settlement(receipt)?.state))) {
+        if (["pending", "blocked"].includes(String(settlement(receipt)?.state)) && effectiveObligation && acceptedPromptBoundary(effectiveObligation)) {
+            // Required semantic work is already committed. Keep the honest pending
+            // native proof for resource cleanup without holding its accepted result.
+            recordTechnicalNote(context, row.project_id, row.run_id, { code: "accepted_boundary_native_settlement_pending", message: "The assigned result is committed; native settlement remains pending.", phase: "native_observation", generation: row.generation, stage: effectiveObligation.stage, harness: runtimeHarness(session.profile), source_operation_id: operationId, diagnostic_reference: session.state_dir, consequence: "cleanup_pending" });
+            receipt = { ...receipt, accepted_boundary: effectiveObligation };
+        }
+        else if (["pending", "blocked"].includes(String(settlement(receipt)?.state))) {
             const recovered = await awaitSettlement(context, row, manifest, session, executable, args, operationId, receipt, assertOwner, signal);
             receipt = recovered;
         }
@@ -389,6 +410,19 @@ export async function controllerAdapter(context, row, manifest, session, args, a
         return receipt;
     }
     catch (error) {
+        // A lost provider/transport acknowledgement cannot revoke the exact
+        // boundary already accepted by the engine. Identity/integrity failures
+        // and unfinished siblings remain on the existing recovery path.
+        const expected = effectiveObligation;
+        if (expected && lateNativeFailure(error, session.id) && acceptedPromptBoundary(expected)) {
+            assertOwner?.();
+            const receipt = { adapter_session_id: session.id, accepted_boundary: expected, native_observation_unavailable: true, settlement: { state: "pending", reason: "native_observation_unavailable" } };
+            const completed = context.db.run("UPDATE run_controller_operations SET status='completed',receipt_json=?,error_json=NULL,updated_at=? WHERE operation_id=? AND controller_id=? AND generation=? AND EXISTS (SELECT 1 FROM run_controllers WHERE controller_id=? AND owner_token=? AND generation=?)", [JSON.stringify(receipt), context.now(), operationId, row.controller_id, row.generation, row.controller_id, row.owner_token, row.generation]);
+            if (completed.changes !== 1)
+                throw new AppError("controller_owner_mismatch", "Controller lost ownership before accepting the committed boundary", 1);
+            recordTechnicalNote(context, row.project_id, row.run_id, { code: "accepted_boundary_native_observation_unavailable", message: "The assigned result is committed; the native acknowledgement is unavailable.", phase: "native_observation", generation: row.generation, stage: expected.stage, harness: runtimeHarness(session.profile), source_operation_id: operationId, diagnostic_reference: session.state_dir, consequence: "cleanup_pending" });
+            return receipt;
+        }
         try {
             context.db.run("UPDATE run_controller_operations SET status = CASE WHEN status = 'dispatching' AND ? <> 'prepared' THEN 'outcome_unknown' ELSE status END, error_json = ?, updated_at = ? WHERE operation_id = ? AND controller_id = ? AND generation = ? AND EXISTS (SELECT 1 FROM run_controllers WHERE controller_id = ? AND owner_token = ? AND generation = ?)", [phase ?? "legacy_unknown", JSON.stringify(errorRecord(error)), context.now(), operationId, row.controller_id, row.generation, row.controller_id, row.owner_token, row.generation]);
         }

package/dist/services/run-controller-process.js

@@ -7,6 +7,7 @@ import { validateCheckEnvironment } from "./check-environment.js";
 import { getResourceDatabase } from "../storage/database.js";
 import { finishManagedProcess, managedProcessRuntimeHome, managedProcessStatus, processIsAlive, processStartedAt, processTreeIsAlive, registerManagedProcess } from "./managed-processes.js";
 import { setFlowRunRecoveryPaused } from "./runs.js";
+import { physicalStartup } from "./physical-startup.js";
 import { assertRunMutationAllowed, assertRunRecoveryAllowed, recoveryGuard } from "./run-recovery.js";
 import { requireRun, requireController, controllerReceipt, appendEvent, digest, errorRecord } from "./run-controller-state.js";
 const launchId = (row) => `${row.controller_id}:owner-launch:${digest(row.owner_token)}`;
@@ -15,6 +16,8 @@ const birth = () => ({ pid: process.pid, pid_started_at: processStartedAt(proces
 const dead = (value) => Boolean(value && Number.isSafeInteger(value.pid) && value.pid > 0 && typeof value.pid_started_at === "string" && value.pid_started_at && !processIsAlive(value));
 /** Called inside the RUN intent transaction, before the client can disappear. */
 export function recordControllerLauncher(context, row, retired, identity = birth()) {
+    if (!identity.pid_started_at)
+        throw new AppError("process_ownership_unknown", "Controller launcher requires an observed physical identity", 1, { phase: "process_confirmation", effect: "no_effect", native_dispatch_started: false, reason: "physical_identity_unavailable" });
     const request = { action: "controller_owner_launch", owner_id: launchId(row) };
     const receipt = { ...identity, registration: "controller_self@1", ...(retired ? { retired_process: { id: retired.id, lease_token: retired.lease_token, pid: retired.pid, pid_started_at: retired.pid_started_at } } : {}) };
     const now = context.now();
@@ -72,12 +75,31 @@ function retire(context, row, identity) {
         throw new AppError("controller_owner_unsettled", "Prior controller identity is not proven dead and retired", 1);
 }
 /** Retry only before productive admission, with exact prior birth and journal CAS. */
-function prepareSpawn(context, row) {
+async function prepareSpawn(context, row) {
     const launch = launchRecord(context, row);
     if (!launch || launch.value.dispatch_ready || !["starting", "running", "recovery_required"].includes(row.status))
         return null;
+    const superseded = new AppError("controller_owner_mismatch", "Controller launch owner changed during physical observation", 1);
+    let launcher;
+    try {
+        launcher = await physicalStartup(context.databaseDeadlineAt).birth(() => {
+            const current = requireController(context, row.controller_id);
+            if (current.generation !== row.generation)
+                throw new AppError("controller_owner_mismatch", "Controller launch generation changed during physical observation", 1);
+            if (current.owner_token !== row.owner_token)
+                throw superseded;
+            assertLaunchAllowed(context, current);
+        });
+    }
+    catch (error) {
+        // Like a lost launch CAS below, a concurrent winner is observed, not
+        // retried or replaced. Other ownership/admission errors remain refusals.
+        if (error === superseded)
+            return null;
+        throw error;
+    }
     const resourceContext = { ...context, env: { ...context.env, DD_FLOW_RESOURCE_HOME: JSON.parse(row.manifest_json).resource_home } };
-    const ours = launch.value.pid === process.pid && launch.value.pid_started_at === processStartedAt(process.pid);
+    const ours = launch.value.pid === launcher.pid && launch.value.pid_started_at === launcher.pid_started_at;
     let token = row.owner_token;
     if (!ours) {
         if (row.owner_pid) {
@@ -115,7 +137,6 @@ function prepareSpawn(context, row) {
         return null;
     if (launch.value.retired_process)
         retire(resourceContext, row, launch.value.retired_process);
-    const launcher = birth();
     return context.db.writeTransaction(() => {
         assertLaunchAllowed(context, row);
         const changed = context.db.run(`UPDATE run_controllers SET owner_token = ?, owner_pid = NULL, process_id = NULL, process_lease_token = NULL, status = 'starting', last_error_json = NULL, updated_at = ?
@@ -165,7 +186,7 @@ export function markControllerDispatchReady(context, row) {
     context.db.run("UPDATE run_controller_operations SET receipt_json = json_set(receipt_json, '$.dispatch_ready', json('true')), updated_at = ? WHERE operation_id = ?", [context.now(), launchId(row)]);
 }
 export async function spawnController(context, row) {
-    const admitted = prepareSpawn(context, row);
+    const admitted = await prepareSpawn(context, row);
     if (!admitted)
         return controllerReceipt(requireController(context, row.controller_id));
     row = admitted;

package/dist/services/run-controller-recovery.js

@@ -18,16 +18,19 @@ import { controllerAdapter, controllerEnvironment, sessionArguments } from "./ru
 import { runHarnessAdapter } from "./harness-adapter.js";
 import { continueProviderCapacity } from "./provider-capacity.js";
 import { spawnController, recordControllerLauncher } from "./run-controller-process.js";
-import { readNativeDaemonState, requireManagedDaemonBinding } from "./managed-daemon-binding.js";
+import { readNativeDaemonState, requireManagedDaemonBinding, managedDaemonExecutionAbsent } from "./managed-daemon-binding.js";
 import { managedInvocationContext } from "./lifecycle-invocations.js";
 import { runControlStatus } from "./run-control.js";
 import { runtimeBudgetStatus, waitForRuntimeScopeRecovery } from "./runtime-budget.js";
 import { prepareControllerPublication } from "./run-controller-capture.js";
+import { technicalDependencyControl } from "./controller-result-authority.js";
+import { recordTechnicalNote } from "./technical-notes.js";
+import { setFlowRunRecoveryPaused } from "./runs.js";
 /** CLI observation is bounded; the physical controller owns the recovery work. */
-export async function resumeRunController(context, input) {
+export async function resumeRunController(context, input, validate) {
     if (input.waitMs !== undefined && (!Number.isSafeInteger(input.waitMs) || input.waitMs < 0 || input.waitMs > 60_000))
         throw new AppError("usage", "--wait-ms must be between 0 and 60000", 2);
-    const intent = recordControllerResume(context, input);
+    const intent = prepareControllerResume(context, input, validate)();
     const manifest = JSON.parse(intent.controller.manifest_json);
     const resourceContext = { ...context, env: { ...context.env, DD_FLOW_RESOURCE_HOME: manifest.resource_home } };
     await spawnController(resourceContext, intent.controller);
@@ -44,7 +47,7 @@ export function recordControllerResume(context, input) {
 }
 /** Verify immutable recovery evidence before a caller reserves the registry
  * writer. The returned operation rechecks SQL ownership under the RUN CAS. */
-export function prepareControllerResume(context, input) {
+export function prepareControllerResume(context, input, validate) {
     if (!/^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/.test(input.requestId) || !input.controlId.trim())
         throw new AppError("usage", "Resume requires request-id and the exact source control", 2);
     const project = requireProjectByRoot(context, resolveProjectRoot(input.projectRoot));
@@ -97,6 +100,8 @@ export function prepareControllerResume(context, input) {
                     throw new AppError("runtime_request_conflict", "Resume request id has different inputs", 1);
                 return { controller: requireController(context, row.controller_id), operationId };
             }
+            if (validate && !validate())
+                throw new AppError("run_control_superseded", "Technical recovery evidence changed before resume admission", 1);
             const current = recoveryGuard(context, project.id, run.id);
             if (current?.recovery_id !== guard.recovery_id || current.generation !== guard.generation || current.status !== "sealed" || current.capture_path !== guard.capture_path)
                 throw new AppError("run_control_superseded", "Control changed before resume could be accepted", 1);
@@ -146,6 +151,39 @@ export async function resumeControllerSession(context, row, manifest, state, ass
     assertRecoveryOwner();
     const readDaemon = () => readNativeDaemonState(session.state_dir);
     const { state: retired, process: record } = requireManagedDaemonBinding(executionContext, { stateDir: session.state_dir, projectId: row.project_id, runId: row.run_id, ownerId: session.owner.owner_id });
+    const run = requireRun(context, row.project_id, row.run_id);
+    const boundary = JSON.parse(run.index_json).stage_runs.find(stage => stage.stage === state.stage && ["done", "skipped"].includes(stage.status));
+    if (boundary && technicalDependencyControl(context, row.project_id, row.run_id, recovery.control_id) && !scopeControl) {
+        if (!managedDaemonExecutionAbsent(executionContext, { stateDir: session.state_dir, projectId: row.project_id, runId: row.run_id, ownerId: session.owner.owner_id }))
+            throw new AppError("recovery_native_shutdown_unproven", "Technical boundary recovery still has an unknown or live owned writer", 1);
+        const assertPublication = prepareControllerPublication(executionContext, row, assertPhysicalOwner);
+        context.db.writeTransaction(() => {
+            assertPublication();
+            assertRecoveryOwner(false);
+            if (requireRun(context, row.project_id, row.run_id).index_json !== run.index_json)
+                throw new AppError("controller_boundary_changed", "Accepted boundary changed before technical continuation", 1);
+            const now = context.now();
+            if (context.db.run("UPDATE run_recovery_guards SET status='recovered',resumed_at=?,updated_at=? WHERE recovery_id=? AND generation=? AND status='sealed'", [now, now, recovery.recovery_id, row.generation]).changes !== 1)
+                throw new AppError("run_recovery_generation_stale", "Technical boundary no longer owns its sealed recovery", 1);
+            if (context.db.run("UPDATE run_controls SET status='resumed',updated_at=? WHERE control_id=? AND status='resuming'", [now, recovery.control_id]).changes !== 1)
+                throw new AppError("run_control_superseded", "Technical recovery control changed before continuation", 1);
+            const receipt = { schema_id: "dd-flow/technical-boundary-resume@1", resumed: true, recovery_id: recovery.recovery_id, stage: boundary.stage, attempt: boundary.attempt, native_cleanup_confirmed: retired.shutdown_state === "clean", native_model_call: false };
+            if (context.db.run("UPDATE run_controller_operations SET status='completed',receipt_json=?,updated_at=? WHERE operation_id=? AND controller_id=? AND generation=? AND status='accepted'", [JSON.stringify(receipt), now, recovery.operation_id, row.controller_id, row.generation]).changes !== 1)
+                throw new AppError("run_control_superseded", "Technical recovery operation changed before continuation", 1);
+            session.retired = true;
+            session.cleanup_pending = retired.shutdown_state !== "clean";
+            state.current_session = null;
+            state.last_continuation = null;
+            delete state.last_receipt;
+            delete state.recovery;
+            setFlowRunRecoveryPaused(context, row.project_id, row.run_id, false);
+            saveState(context, row, state);
+            appendEvent(context, row.controller_id, "technical_boundary_resumed", { ...receipt, control_id: recovery.control_id, state_dir: session.state_dir });
+        });
+        if (retired.shutdown_state !== "clean")
+            recordTechnicalNote(context, row.project_id, row.run_id, { code: "native_cleanup_pending_after_execution_exit", message: "The accepted Stage continues in a fresh Session; old native cleanup remains unconfirmed.", phase: "recovery", generation: row.generation, resource_identity: session.state_dir, diagnostic_reference: session.state_dir, consequence: "cleanup_pending" });
+        return;
+    }
     const call = (args, operationId) => controllerAdapter(executionContext, row, manifest, session, args, assertRecoveryOwner, operationId);
     if (processTreeIsAlive(record)) {
         const stopped = await call(["daemon", "stop", "--state-dir", session.state_dir, "--json"]);
@@ -157,8 +195,10 @@ export async function resumeControllerSession(context, row, manifest, state, ass
             assertRecoveryOwner();
         }
     }
-    if (processTreeIsAlive(record) || readDaemon().shutdown_state !== "clean")
-        throw new AppError("recovery_native_shutdown_unproven", "The previous native daemon is not proven clean and closed", 1);
+    if (processTreeIsAlive(record))
+        throw new AppError("recovery_native_shutdown_unproven", "The previous native daemon still owns execution", 1);
+    if (readDaemon().shutdown_state !== "clean")
+        throw new AppError("recovery_native_shutdown_unproven", "An unfinished coordinator requires its retained native Session to be cleanly resumable", 1);
     assertRecoveryOwner();
     session.stopped = false;
     saveState(context, row, state);

package/dist/services/run-controller-state.js

@@ -2,6 +2,7 @@ import crypto from "node:crypto";
 import { AppError, errorRecord as structuredErrorRecord } from "../shared/errors.js";
 import { runtimeHarness } from "./execution-policy.js";
 import { managedProcessRuntimeHome, managedProcessStatus, processTreeIsAlive } from "./managed-processes.js";
+import { readTechnicalNotes } from "./technical-notes.js";
 export function requireRun(context, projectId, runId) {
     const run = context.db.get("SELECT id, project_id, flow_kind, project_root, workspace_root, run_root, status, index_json FROM runs WHERE project_id = ? AND id = ?", [projectId, runId]);
     if (!run?.run_root)
@@ -14,10 +15,10 @@ export function requireController(context, id) {
         throw new AppError("not_found", "Managed controller was not found", 1, { controller_id: id });
     return row;
 }
-export function controllerReceipt(row) {
+export function controllerReceipt(row, context) {
     const state = JSON.parse(row.state_json);
     const manifest = JSON.parse(row.manifest_json);
-    return { ok: true, schema_id: "dd-flow/run-controller-receipt@1", controller: { controller_id: row.controller_id, request_id: row.request_id, project_id: row.project_id, run_id: row.run_id, generation: row.generation, runtime_budget: manifest.budget ?? null, status: row.status, process_id: row.process_id, owner_pid: row.owner_pid, state_dir: row.state_dir, stage: state.stage, sessions: state.sessions.map(session => ({ session_id: session.id, harness: runtimeHarness(session.profile), profile_id: session.profile.id, state_dir: session.state_dir, journal: session.journal, stopped: session.stopped === true })), error: row.last_error_json ? JSON.parse(row.last_error_json) : null } };
+    return { ok: true, schema_id: "dd-flow/run-controller-receipt@1", ...(context ? { technical_notes: readTechnicalNotes(context, row.project_id, row.run_id) } : {}), controller: { controller_id: row.controller_id, request_id: row.request_id, project_id: row.project_id, run_id: row.run_id, generation: row.generation, runtime_budget: manifest.budget ?? null, status: row.status, process_id: row.process_id, owner_pid: row.owner_pid, state_dir: row.state_dir, stage: state.stage, sessions: state.sessions.map(session => ({ session_id: session.id, harness: runtimeHarness(session.profile), profile_id: session.profile.id, state_dir: session.state_dir, journal: session.journal, stopped: session.stopped === true, ...(session.cleanup_pending ? { cleanup_pending: true } : {}) })), error: row.last_error_json ? JSON.parse(row.last_error_json) : null } };
 }
 /** Read-only causal diagnostics, not a settlement or capture authorization. */
 export function terminalControllerFinalization(context, project, runId, runStatus) {