@deksden-com/dd-flow-cli 0.9.0-beta.124 → 0.9.0-beta.127

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (113) hide show
  1. package/CHANGELOG.md +16 -0
  2. package/README.md +10 -1
  3. package/dist/build-info.json +7 -7
  4. package/dist/cli/argument-values.js +38 -0
  5. package/dist/cli/command-inputs.js +4 -3
  6. package/dist/cli/help.js +1 -0
  7. package/dist/cli/hook-ingress.js +1 -1
  8. package/dist/cli/run-cli.js +126 -193
  9. package/dist/cli/runtime-maintenance-diagnostics.js +46 -0
  10. package/dist/cli/runtime-maintenance-ingress.js +51 -0
  11. package/dist/cli/runtime-process.js +128 -0
  12. package/dist/cli.js +8 -5
  13. package/dist/harness-runtime/bin/dd-agy.mjs +3 -2
  14. package/dist/harness-runtime/bin/dd-codex.mjs +3 -2
  15. package/dist/harness-runtime/bin/dd-droid.mjs +3 -2
  16. package/dist/harness-runtime/bin/dd-grok.mjs +16 -6
  17. package/dist/harness-runtime/bin/dd-opencode.mjs +3 -2
  18. package/dist/harness-runtime/bin/dd-zcode.mjs +20 -16
  19. package/dist/harness-runtime/lib/adapter-timeouts.d.mts +5 -0
  20. package/dist/harness-runtime/lib/adapter-timeouts.mjs +34 -0
  21. package/dist/harness-runtime/lib/daemon-observation.d.mts +2 -0
  22. package/dist/harness-runtime/lib/daemon-observation.mjs +163 -0
  23. package/dist/harness-runtime/lib/daemon-operations.mjs +60 -13
  24. package/dist/harness-runtime/lib/dd-agy-daemon.mjs +151 -72
  25. package/dist/harness-runtime/lib/dd-agy.mjs +11 -23
  26. package/dist/harness-runtime/lib/dd-codex-daemon.mjs +9 -14
  27. package/dist/harness-runtime/lib/dd-codex.mjs +252 -80
  28. package/dist/harness-runtime/lib/dd-droid-daemon.mjs +14 -19
  29. package/dist/harness-runtime/lib/dd-droid.mjs +117 -31
  30. package/dist/harness-runtime/lib/dd-grok-daemon.mjs +56 -30
  31. package/dist/harness-runtime/lib/dd-grok.mjs +15 -13
  32. package/dist/harness-runtime/lib/dd-opencode-daemon.mjs +18 -27
  33. package/dist/harness-runtime/lib/dd-opencode.mjs +44 -14
  34. package/dist/harness-runtime/lib/dd-zcode-daemon.mjs +56 -53
  35. package/dist/harness-runtime/lib/dd-zcode.mjs +200 -111
  36. package/dist/harness-runtime/lib/lease-renewal.d.mts +8 -0
  37. package/dist/harness-runtime/lib/lease-renewal.mjs +23 -0
  38. package/dist/harness-runtime/lib/lifecycle-command.d.mts +3 -3
  39. package/dist/harness-runtime/lib/lifecycle-command.mjs +74 -20
  40. package/dist/harness-runtime/lib/managed-daemon.d.mts +10 -1
  41. package/dist/harness-runtime/lib/managed-daemon.mjs +228 -40
  42. package/dist/harness-runtime/lib/native-children.d.mts +13 -0
  43. package/dist/harness-runtime/lib/native-children.mjs +78 -0
  44. package/dist/harness-runtime/lib/native-hook-command.d.mts +4 -0
  45. package/dist/harness-runtime/lib/native-hook-command.mjs +115 -11
  46. package/dist/harness-runtime/lib/native-streams.mjs +45 -0
  47. package/dist/harness-runtime/lib/observation-clock.d.mts +18 -0
  48. package/dist/harness-runtime/lib/observation-clock.mjs +73 -17
  49. package/dist/harness-runtime/lib/operation-errors.d.mts +14 -0
  50. package/dist/harness-runtime/lib/operation-errors.mjs +105 -8
  51. package/dist/harness-runtime/lib/operation-progress.d.mts +7 -0
  52. package/dist/harness-runtime/lib/operation-progress.mjs +112 -0
  53. package/dist/harness-runtime/lib/process-json.mjs +136 -38
  54. package/dist/harness-runtime/lib/tool-observations.d.mts +1 -0
  55. package/dist/harness-runtime/lib/tool-observations.mjs +5 -2
  56. package/dist/harness-runtime/lib/zcode-usage.mjs +3 -2
  57. package/dist/runtime/context.js +8 -4
  58. package/dist/schemas/run-control-receipt.schema.json +19 -0
  59. package/dist/services/code-checks.js +79 -9
  60. package/dist/services/codex-capacity.js +117 -41
  61. package/dist/services/codex-hook-delivery.js +2 -2
  62. package/dist/services/compatibility-preflight.js +4 -1
  63. package/dist/services/controller-fanout.js +9 -76
  64. package/dist/services/engines.js +70 -20
  65. package/dist/services/eval-snapshots.js +179 -40
  66. package/dist/services/external-work-launch.js +13 -8
  67. package/dist/services/harness-adapter.js +175 -11
  68. package/dist/services/hooks.js +65 -21
  69. package/dist/services/lifecycle-invocations.js +404 -32
  70. package/dist/services/managed-lease-monitor.js +128 -0
  71. package/dist/services/managed-processes.js +103 -47
  72. package/dist/services/merge-server.js +58 -21
  73. package/dist/services/recovery-observation-budget.js +67 -7
  74. package/dist/services/recovery-snapshot-database.js +7 -4
  75. package/dist/services/run-control-receipt.js +1 -1
  76. package/dist/services/run-control-worker.js +177 -118
  77. package/dist/services/run-control.js +99 -8
  78. package/dist/services/run-controller-adapter.js +169 -46
  79. package/dist/services/run-controller-capture.js +14 -2
  80. package/dist/services/run-controller-process.js +11 -4
  81. package/dist/services/run-controller-state.js +68 -7
  82. package/dist/services/run-controller.js +166 -94
  83. package/dist/services/run-fork.js +8 -3
  84. package/dist/services/run-recovery-guard.js +75 -0
  85. package/dist/services/run-recovery-runtime.js +1 -56
  86. package/dist/services/run-recovery.js +17 -13
  87. package/dist/services/runs.js +39 -6
  88. package/dist/services/runtime-budget.js +32 -15
  89. package/dist/services/runtime-scope-control.js +8 -0
  90. package/dist/services/runtime-scope-resume.js +30 -8
  91. package/dist/services/runtime-scope-worker.js +118 -73
  92. package/dist/services/runtime-service.js +6 -7
  93. package/dist/services/scope-writer-cleanup.js +3 -2
  94. package/dist/services/stage-context.js +135 -0
  95. package/dist/services/stage-lifecycle.js +3 -15
  96. package/dist/services/stage-pause.js +3 -2
  97. package/dist/services/usage.js +14 -3
  98. package/dist/services/vnext-code-review.js +10 -9
  99. package/dist/services/vnext-code.js +10 -8
  100. package/dist/services/vnext-contracts.js +10 -1
  101. package/dist/services/vnext-execution-profile.js +75 -2
  102. package/dist/services/vnext-fanout.js +5 -0
  103. package/dist/services/vnext-merge.js +10 -9
  104. package/dist/services/vnext-plan-review.js +21 -17
  105. package/dist/services/vnext-plan.js +9 -7
  106. package/dist/services/vnext-protocolize.js +15 -9
  107. package/dist/services/vnext-specify.js +6 -6
  108. package/dist/services/work-registry.js +34 -4
  109. package/dist/services/workspace-bootstrap.js +14 -3
  110. package/dist/shared/errors.js +11 -6
  111. package/dist/storage/agent-output.js +26 -0
  112. package/dist/storage/database.js +80 -21
  113. package/package.json +7 -4
@@ -14,6 +14,9 @@ import { finishManagedProcess, managedLeaseReleased, managedProcessBinding, mana
14
14
  import { reconcileUnfinishedChecks, terminalCheckSettled } from "./code-checks.js";
15
15
  import { controlStatusReceipt } from "./run-control-receipt.js";
16
16
  import { assertNativeCancellationReceipt, cancelControlledNativeSession, inspectControlledNativeSession } from "./native-session-control.js";
17
+ import { terminalControllerFinalization } from "./run-controller-state.js";
18
+ import { canonicalNativeOperation } from "../harness-runtime/lib/adapter-timeouts.mjs";
19
+ import { requireManagedDaemonBinding } from "./managed-daemon-binding.js";
17
20
  /** Durable projection only: status never probes a provider or acquires an owner. */
18
21
  export function runControlStatus(context, input) {
19
22
  const project = requireProjectByRoot(context, resolveProjectRoot(input.projectRoot));
@@ -26,7 +29,7 @@ function readControlStatus(context, input, project) {
26
29
  if (!run)
27
30
  throw new AppError("not_found", "RUN is not registered", 1);
28
31
  const scope = { project_id: project.id, run_id: input.runId };
29
- const empty = controlStatusReceipt({ ok: true, scope, run_status: run.status, control: null, operations: [], worker: null, resume_operation: null }, input);
32
+ const empty = controlStatusReceipt({ ok: true, scope, run_status: run.status, finalization: terminalControllerFinalization(context, project, input.runId, run.status), control: null, operations: [], worker: null, resume_operation: null }, input);
30
33
  if (input.controlId === null)
31
34
  return empty;
32
35
  if (!context.db.get("SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'run_controls'"))
@@ -79,6 +82,7 @@ async function reconcileOwnedSessions(context, input, interrupt, boundary) {
79
82
  throw new AppError("run_control_invalid_state", "Interrupt requires the current stop intent or its explicitly elapsed pause escalation", 1);
80
83
  const observationId = boundary?.operationId ?? control.control_id;
81
84
  const assertCurrent = () => {
85
+ context.assertControlOwner?.();
82
86
  if (boundary) {
83
87
  boundary.assertCurrent();
84
88
  return;
@@ -88,7 +92,7 @@ async function reconcileOwnedSessions(context, input, interrupt, boundary) {
88
92
  throw new AppError("run_control_superseded", "Stop intent no longer owns this RUN", 1);
89
93
  };
90
94
  assertCurrent();
91
- const databaseVersion = () => JSON.stringify([context.db.get("PRAGMA data_version"), context.db.get("SELECT total_changes() AS changes")]);
95
+ const databaseVersion = () => runSettlementVersion(context, project.id, input.runId, boundary?.operationId);
92
96
  const initialDatabaseVersion = databaseVersion();
93
97
  const readBindings = () => context.db.all("SELECT project_id, session_id, provider_session_id, harness, parent_session_id, provider_parent_session_id, run_id, status, stop_reason, metadata_json FROM sessions");
94
98
  const foreign = (binding) => binding.project_id !== project.id || Boolean(binding.run_id && binding.run_id !== input.runId);
@@ -382,8 +386,8 @@ async function reconcileOwnedSessions(context, input, interrupt, boundary) {
382
386
  const args = saved.args;
383
387
  if (!Array.isArray(args) || args.some(value => typeof value !== "string") || !saved.state_dir || !saved.executable || saved.owner !== operation.controller_id || conflicts.has(saved.state_dir) || directoryAdapters.get(saved.state_dir) !== saved.executable)
384
388
  throw new AppError("controller_operation_binding_unknown", "Unfinished controller operation has no matching frozen runtime binding", 1);
385
- const method = `${args[0]}.${args[1]}`;
386
- if (!["session.create", "session.prompt", "session.start", "session.fork", "session.cancel", "session.resume", "daemon.start", "daemon.stop"].includes(method))
389
+ const method = `${args[0]}.${args[1] === "status" ? "inspect" : args[1]}`;
390
+ if (!["session.create", "session.prompt", "session.start", "session.fork", "session.cancel", "session.resume", "session.inspect", "daemon.start", "daemon.stop"].includes(method))
387
391
  throw new AppError("controller_operation_receipt_required", "Unfinished controller operation needs its original receipt", 1);
388
392
  const target = args.includes("--session-id") ? args[args.indexOf("--session-id") + 1] : null;
389
393
  if (!args.includes("--state-dir") || args[args.indexOf("--state-dir") + 1] !== saved.state_dir || args.includes("--session-id") && (!target || !targets.has(JSON.stringify([saved.state_dir, target]))))
@@ -402,6 +406,23 @@ async function reconcileOwnedSessions(context, input, interrupt, boundary) {
402
406
  delete env.DD_FLOW_CONTROLLER_TOKEN;
403
407
  delete env.DD_FLOW_CONTROL_TOKEN;
404
408
  assertCurrent();
409
+ if (canonicalNativeOperation(harness, method) === "session.inspect") {
410
+ if (!target)
411
+ throw new AppError("controller_operation_binding_unknown", "Native reobservation has no frozen Session target", 1);
412
+ const physicalContext = { ...context, env: { ...context.env, DD_FLOW_RESOURCE_HOME: retained.owner.resource_home ?? resourceHome(context) } };
413
+ const native = requireManagedDaemonBinding(physicalContext, { stateDir: saved.state_dir, projectId: project.id, runId: input.runId, ownerId: operation.controller_id });
414
+ const binding = managedProcessBinding(native.process, resourceHome(physicalContext));
415
+ const assertNative = () => {
416
+ assertCurrent();
417
+ const current = requireManagedDaemonBinding(physicalContext, { stateDir: saved.state_dir, projectId: project.id, runId: input.runId, ownerId: operation.controller_id });
418
+ if (current.state.daemon_id !== native.state.daemon_id || JSON.stringify(managedProcessBinding(current.process, resourceHome(physicalContext))) !== JSON.stringify(binding) || harnessKey(current.process.kind.replace(/-daemon$/, "")) !== harness || !processIsAlive(current.process) || ["starting", "stopping", "clean", "unclean", "cleanup_failed", "failed"].includes(current.state.shutdown_state ?? ""))
419
+ throw new AppError("daemon_owner_mismatch", "Native reobservation lost its live frozen daemon authority", 1);
420
+ };
421
+ const receipt = await inspectControlledNativeSession({ executable: saved.executable, sessionId: target, stateDir: saved.state_dir, harness, env, operationId: `${observationId}:observe:${operation.operation_id}`, assertCurrent: assertNative });
422
+ assertNative();
423
+ context.db.run("UPDATE run_controller_operations SET status = 'completed', receipt_json = ?, error_json = NULL, updated_at = ? WHERE operation_id = ? AND status IN ('dispatching','outcome_unknown')", [JSON.stringify(receipt), context.now(), operation.operation_id]);
424
+ return { operation_id: operation.operation_id, status: "completed", receipt, source: "native_reobservation" };
425
+ }
405
426
  const receipt = await runHarnessAdapter({ executable: saved.executable, args: ["daemon", "operation", "--operation-id", operation.operation_id, "--state-dir", saved.state_dir, "--json"], env, operationId: `${observationId}:observe:${operation.operation_id}`, timeoutMs: 30_000 });
406
427
  if (receipt.operation_id !== operation.operation_id || receipt.operation !== method || target && receipt.session_id !== target)
407
428
  throw new AppError("controller_operation_mismatch", "Operation observation belongs to another retained dispatch", 1);
@@ -761,8 +782,14 @@ async function reconcileOwnedSessions(context, input, interrupt, boundary) {
761
782
  if (fs.existsSync(path.join(directory, "daemon.json")) || JSON.stringify(retainedDaemonOperations(directory)) !== journal)
762
783
  throw new AppError("owned_inventory_changed", "Rejected startup acquired new runtime evidence", 1);
763
784
  const currentProcesses = [...resourceHomes].flatMap(home => managedProcessStatus({ ...context, env: { ...context.env, DD_FLOW_RESOURCE_HOME: home } }).filter(record => record.project_id === project.id && record.run_id === input.runId && [null, context.ddFlowHome].includes(managedProcessRuntimeHome(record, context))).map(record => ({ home, record })));
764
- if (initialDatabaseVersion !== databaseVersion() || processVersion(registeredProcesses) !== processVersion(currentProcesses) || [...nativeVersions].some(([directory, version]) => version !== nativeInventoryVersion(directory)) || [...importVersions].some(([file, version]) => crypto.createHash("sha256").update(fs.readFileSync(file)).digest("hex") !== version))
765
- throw new AppError("owned_inventory_changed", "Owned writers changed after their settlement observation", 1);
785
+ const changed = [
786
+ ...(initialDatabaseVersion !== databaseVersion() ? ["run_bindings"] : []),
787
+ ...(processVersion(registeredProcesses) !== processVersion(currentProcesses) ? ["process_owners"] : []),
788
+ ...[...nativeVersions].filter(([directory, version]) => version !== nativeInventoryVersion(directory)).map(([directory]) => `native:${directory}`),
789
+ ...[...importVersions].filter(([file, version]) => crypto.createHash("sha256").update(fs.readFileSync(file)).digest("hex") !== version).map(([file]) => `import:${file}`)
790
+ ];
791
+ if (changed.length)
792
+ throw new AppError("owned_inventory_changed", "Owned writers changed after their settlement observation", 1, { components: changed });
766
793
  };
767
794
  try {
768
795
  verifyInventory();
@@ -792,7 +819,69 @@ async function reconcileOwnedSessions(context, input, interrupt, boundary) {
792
819
  export function nativeInventoryVersion(directory, operations = retainedDaemonOperations(directory)) {
793
820
  const state = JSON.parse(fs.readFileSync(path.join(directory, "daemon.json"), "utf8"));
794
821
  const keys = ["schema_id", "daemon_id", "config", "pid", "provider_pid", "shutdown_state", "active_operation", "active_tree", "sessions", "descendants", "root_session_id", "resource_process", "provider_process", "owned_processes"];
795
- return crypto.createHash("sha256").update(JSON.stringify([Object.fromEntries(keys.map(key => [key, state[key]])), operations])).digest("hex");
822
+ // Only these native verbs are read-only. Unknown verbs remain guarded.
823
+ const productive = operations.filter(operation => !isObservationOperation(operation.operation));
824
+ return crypto.createHash("sha256").update(JSON.stringify([Object.fromEntries(keys.map(key => [key, state[key]])), productive])).digest("hex");
825
+ }
826
+ export function isObservationOperation(operation) {
827
+ return ["session.inspect", "daemon.status"].includes(String(operation));
828
+ }
829
+ /** Compare authority and productive state of this RUN, not writes anywhere
830
+ * in the shared database. Heartbeat and observer timestamps are not authority. */
831
+ export function runSettlementVersion(context, projectId, runId, observerOperationId) {
832
+ return context.db.readSnapshot(() => {
833
+ const exists = (table) => Boolean(context.db.get("SELECT name FROM sqlite_master WHERE type='table' AND name=?", [table]));
834
+ const rows = (table, where, values) => exists(table) ? context.db.all(`SELECT * FROM ${table} WHERE ${where}`, values) : [];
835
+ const scoped = [projectId, runId];
836
+ const sessions = rows("sessions", "1 = 1", []);
837
+ const identity = (row) => JSON.stringify([row.project_id, row.session_id]);
838
+ const selected = new Set(sessions.filter(row => row.project_id === projectId && row.run_id === runId).map(identity));
839
+ // Include newly registered children, even before their RUN id is attached.
840
+ // ponytail: retained Session inventories are small; index ancestry if scans become costly.
841
+ let previous = -1;
842
+ while (previous !== selected.size) {
843
+ previous = selected.size;
844
+ for (const row of sessions)
845
+ if (sessions.some(parent => selected.has(identity(parent)) && parent.harness === row.harness && (parent.project_id === row.project_id && parent.session_id === row.parent_session_id || parent.provider_session_id && parent.provider_session_id === row.provider_parent_session_id)))
846
+ selected.add(identity(row));
847
+ }
848
+ // Match the protocol relationships retained by recovery-snapshot-database.
849
+ const protocolIds = "SELECT subject_id FROM runs WHERE project_id = ? AND id = ? UNION SELECT value FROM merge_requests, json_each(protocol_ids_json) WHERE project_id = ? AND run_id = ? UNION SELECT protocol_id FROM flow_jobs WHERE project_id = ? AND run_id = ?";
850
+ const protocolScope = [projectId, ...scoped, ...scoped, ...scoped];
851
+ const projection = {
852
+ runs: rows("runs", "project_id = ? AND id = ?", scoped),
853
+ project: rows("projects", "id = ?", [projectId]),
854
+ project_config: rows("project_config", "project_id = ?", [projectId]),
855
+ protocols: rows("protocols", `project_id = ? AND id IN (${protocolIds})`, protocolScope),
856
+ plan_bindings: rows("plan_bindings", `project_id = ? AND protocol_id IN (${protocolIds})`, protocolScope),
857
+ plan_progress: rows("plan_progress", `project_id = ? AND protocol_id IN (${protocolIds})`, protocolScope),
858
+ worktrees: rows("worktree_records", `project_id = ? AND protocol_id IN (${protocolIds})`, protocolScope),
859
+ sessions: sessions.filter(row => selected.has(identity(row))),
860
+ works: rows("works", "project_id = ? AND run_id = ?", scoped),
861
+ checks: rows("check_receipts", "project_id = ? AND run_id = ?", scoped),
862
+ merges: rows("merge_requests", "project_id = ? AND run_id = ?", scoped),
863
+ memberships: rows("stage_work_memberships", "project_id = ? AND run_id = ?", scoped),
864
+ repairs: rows("repair_intents", "project_id = ? AND run_id = ?", scoped),
865
+ lifecycle: rows("lifecycle_invocations", "json_extract(scope_json, '$.runId') = ? AND json_extract(scope_json, '$.projectRoot') = (SELECT root FROM projects WHERE id = ?)", [runId, projectId]),
866
+ packets: rows("lifecycle_packet_assignments", "invocation_id IN (SELECT id FROM lifecycle_invocations WHERE json_extract(scope_json, '$.runId') = ? AND json_extract(scope_json, '$.projectRoot') = (SELECT root FROM projects WHERE id = ?))", [runId, projectId]),
867
+ flags: rows("flow_run_flag_mutations", "project_id = ? AND run_id = ?", scoped),
868
+ jobs: rows("flow_jobs", "project_id = ? AND run_id = ?", scoped),
869
+ work_sessions: rows("work_sessions", "work_id IN (SELECT work_id FROM works WHERE project_id = ? AND run_id = ?)", scoped),
870
+ work_launches: rows("work_launches", "work_id IN (SELECT work_id FROM works WHERE project_id = ? AND run_id = ?)", scoped),
871
+ controls: rows("run_controls", "project_id = ? AND run_id = ?", scoped),
872
+ recovery: rows("run_recovery_guards", "project_id = ? AND run_id = ?", scoped),
873
+ operations: rows("run_controller_operations", "controller_id IN (SELECT controller_id FROM run_controllers WHERE project_id = ? AND run_id = ?)", scoped).filter(row => {
874
+ if (row.operation_id === observerOperationId)
875
+ return false;
876
+ const input = JSON.parse(String(row.input_json));
877
+ const operation = input.args?.slice(0, 2).join(".");
878
+ // CLI session status is an alias for the native session.inspect read.
879
+ return operation !== "session.status" && !isObservationOperation(operation);
880
+ }).map(row => ({ operation_id: row.operation_id, controller_id: row.controller_id, generation: row.generation, input_hash: row.input_hash, input_json: row.input_json, status: row.status, receipt_json: row.receipt_json, error_json: row.error_json })),
881
+ controllers: rows("run_controllers", "project_id = ? AND run_id = ?", scoped).map(row => ({ controller_id: row.controller_id, generation: row.generation, status: row.status, owner_token: row.owner_token, process_id: row.process_id, process_lease_token: row.process_lease_token, state_dir: row.state_dir, manifest_json: row.manifest_json, sessions: JSON.parse(String(row.state_json)).sessions }))
882
+ };
883
+ return JSON.stringify(Object.fromEntries(Object.entries(projection).map(([table, records]) => [table, records.map(row => Object.fromEntries(Object.entries(row).filter(([key]) => !["created_at", "updated_at", "heartbeat_at"].includes(key)))).sort((a, b) => JSON.stringify(a).localeCompare(JSON.stringify(b)))])));
884
+ });
796
885
  }
797
886
  /** Read the bundled runtime's durable operation journal, including lost replies. */
798
887
  export function retainedDaemonOperations(directory) {
@@ -858,7 +947,9 @@ export function recordRunControlIntent(context, input) {
858
947
  const run = context.db.get("SELECT status FROM runs WHERE project_id = ? AND id = ?", [project.id, input.runId]);
859
948
  if (!run)
860
949
  throw new AppError("not_found", "RUN is not registered", 1);
861
- if (["done", "failed", "cancelled"].includes(run.status)) {
950
+ // Terminal subject state does not prove that its native tree is closed.
951
+ // Stop still needs the same fenced settlement/capture as a running RUN.
952
+ if (input.action === "pause" && ["done", "failed", "cancelled"].includes(run.status)) {
862
953
  return { ok: true, terminal: true, status: run.status, control_id: null };
863
954
  }
864
955
  const prior = recoveryGuard(context, project.id, input.runId);
@@ -4,8 +4,14 @@ import path from "node:path";
4
4
  import { AppError } from "../shared/errors.js";
5
5
  import { adapterSessionId, runHarnessAdapter } from "./harness-adapter.js";
6
6
  import { runtimeHarness } from "./execution-policy.js";
7
- import { appendEvent, digest, digestBytes, errorRecord } from "./run-controller-state.js";
7
+ import { appendEvent, digest, digestBytes, errorRecord, requireRun } from "./run-controller-state.js";
8
+ import { managedProcessStatus } from "./managed-processes.js";
8
9
  import { CAPACITY_POLICY, parseCapacityOperation } from "../harness-runtime/lib/codex-capacity-policy.mjs";
10
+ import { recoveryObservationBudget } from "./recovery-observation-budget.js";
11
+ import { canonicalNativeOperation } from "../harness-runtime/lib/adapter-timeouts.mjs";
12
+ import { retainedDaemonReply } from "../harness-runtime/lib/daemon-observation.mjs";
13
+ import { readOperationRecord } from "../harness-runtime/lib/operation-progress.mjs";
14
+ import { readNativeDaemonState } from "./managed-daemon-binding.js";
9
15
  /** A paused RUN admits only its controller's exact accepted answer Turn. */
10
16
  export function controllerHitlAdmission(context, record, input) {
11
17
  if (input.operation !== "session.prompt" || !input.operationId || !input.promptSha256)
@@ -50,7 +56,10 @@ export function controllerHitlAdmission(context, record, input) {
50
56
  if (!answer || !prompt)
51
57
  return false;
52
58
  const accepted = JSON.parse(answer.input_json);
53
- const dispatch = JSON.parse(prompt.input_json);
59
+ // The phase is a mutable journal projection, not part of the frozen input.
60
+ const { dispatch_phase, ...dispatch } = JSON.parse(prompt.input_json);
61
+ if (dispatch_phase !== "external_dispatch_possible")
62
+ return false;
54
63
  const run = context.db.get("SELECT index_json FROM runs WHERE project_id = ? AND id = ?", [record.project_id, record.run_id]);
55
64
  const stage = run && JSON.parse(run.index_json).stage_runs.find(item => item.stage === accepted.stage);
56
65
  if (accepted.action !== "user_answer" || state.stage !== accepted.stage || stage?.status !== "paused" || !stage.pause || stage.pause.answer_path || stage.pause.id !== accepted.pause_id || stage.pause.work_id !== accepted.work_id)
@@ -102,8 +111,47 @@ function operationInspectArgs(args, operationId) {
102
111
  throw new AppError("controller_operation_binding_unknown", "Harness operation has no state directory", 1, { operation_id: operationId });
103
112
  return ["daemon", "operation", "--state-dir", stateDir, "--operation-id", operationId, "--json"];
104
113
  }
105
- function expectedMethod(args) {
106
- return `${args[0]}.${args[1] === "status" ? "inspect" : args[1]}`;
114
+ function expectedMethod(args, session) {
115
+ return canonicalNativeOperation(runtimeHarness(session.profile), `${args[0]}.${args[1] === "status" ? "inspect" : args[1]}`);
116
+ }
117
+ function recoveryAuthorityError(error) {
118
+ return ["session_identity_mismatch", "profile_integrity_violation", "controller_operation_mismatch", "operation_conflict", "operation_cancelled", "settlement_recovery_required"].includes(String(error?.code));
119
+ }
120
+ /** A retained native effect is not permission to continue under a new daemon. */
121
+ async function recoveredSessionReceipt(context, session, args, operationId, receipt, binding = {}, signal, inspection) {
122
+ const method = expectedMethod(args, session);
123
+ if (!["session.create", "session.prompt", "session.start", "session.fork", "session.resume"].includes(method))
124
+ return receipt;
125
+ const requested = await readOperationRecord(session.state_dir, operationId, "requested.json");
126
+ signal?.throwIfAborted();
127
+ const daemon = fs.existsSync(path.join(session.state_dir, "daemon.json")) ? readNativeDaemonState(session.state_dir) : null;
128
+ // Old minimal adapters have neither owned operation records nor daemon
129
+ // ownership metadata. Missing a modern record cannot downgrade to that path.
130
+ const modern = requested?.owner_id || requested && Object.hasOwn(requested, "params_sha256") || requested?.generation != null || daemon?.config?.runtime_owner?.owner_id
131
+ || managedProcessStatus({ ...context, env: { ...context.env, ...(session.owner.resource_home ? { DD_FLOW_RESOURCE_HOME: session.owner.resource_home } : {}) } }).some(record => {
132
+ const metadata = JSON.parse(record.metadata_json);
133
+ return record.kind.endsWith("-daemon") && record.owner_id === session.owner.owner_id && record.project_id === session.owner.project_id && record.run_id === session.owner.run_id
134
+ && metadata.state_dir === session.state_dir && metadata.dd_flow_home === session.owner.dd_flow_home && metadata.adapter_executable === session.owner.adapter_executable;
135
+ });
136
+ if (!modern)
137
+ return receipt;
138
+ const target = args.includes("--session-id") ? args[args.indexOf("--session-id") + 1] : undefined;
139
+ const retained = await retainedDaemonReply(session.state_dir, operationId, { ...binding, operation: method, owner_id: session.owner.owner_id, ...(target ? { session_id: target } : {}) });
140
+ signal?.throwIfAborted();
141
+ if (!retained || retained.reason)
142
+ throw new AppError("settlement_recovery_required", "Retained native outcome has no current dispatch authority", 1, { operation_id: operationId, native_outcome: "completed", native_result: retained?.outcome ?? receipt, settlement: retained?.settlement, reason: retained?.reason ?? "native_receipt_unavailable" });
143
+ if (inspection && settlement(inspection)?.state === "settled" && !retained.ready) {
144
+ const proof = inspection.settlement;
145
+ if (proof.source !== "inspection" || !Array.isArray(proof.operation_ids) || !proof.operation_ids.includes(operationId))
146
+ throw new AppError("settlement_recovery_required", "Fresh inspection does not settle the original retained operation", 1, { operation_id: operationId, native_outcome: "completed", native_result: retained.outcome, reason: "inspection_settlement_unbound", settlement: proof });
147
+ }
148
+ const result = retained.ready ? retained.value : retained.outcome;
149
+ if (!result || typeof result !== "object" || Array.isArray(result))
150
+ throw new AppError("controller_operation_mismatch", "Retained native outcome has no Session result", 1, { operation_id: operationId });
151
+ validateReceipt(result, session, args);
152
+ if (method === "session.create" && !adapterSessionId(result))
153
+ throw new AppError("session_identity_mismatch", "Retained creation outcome has no native Session identity", 1);
154
+ return { ...receipt, ...result, settlement: retained.ready ? retained.settlement ?? { state: "not_required" } : ["pending", "blocked"].includes(String(retained.settlement?.state)) ? retained.settlement : { ...retained.settlement, state: "pending", reason: retained.settlement?.reason ?? "settlement_evidence_required" } };
107
155
  }
108
156
  function validateReceipt(receipt, session, args) {
109
157
  // `daemon start --session-id` names the retained state being reopened; the
@@ -122,11 +170,9 @@ function validateReceipt(receipt, session, args) {
122
170
  }
123
171
  }
124
172
  async function recoverCompletedOperation(context, manifest, session, executable, args, operationId, signal) {
125
- // Creation has no stable client Session id yet. Its durable daemon journal
126
- // is reconciled by run-control after the owner is fenced; probing it from the
127
- // failing transport path would be a second adapter invocation and can itself
128
- // be mistaken for replay.
129
- if (args[0] === "session" && ["create", "inspect", "status"].includes(args[1]))
173
+ // Creation has no Session ID yet, but its original operation ID is already
174
+ // retained. Inspect that receipt, never invoke native creation a second time.
175
+ if (expectedMethod(args, session) === "session.inspect")
130
176
  return null;
131
177
  // A detached adapter without a native daemon has no durable operation
132
178
  // journal to inspect. Avoid a second adapter invocation in that case; the
@@ -142,8 +188,8 @@ async function recoverCompletedOperation(context, manifest, session, executable,
142
188
  }
143
189
  const target = args.includes("--session-id") ? args[args.indexOf("--session-id") + 1] : null;
144
190
  const result = observed.result;
145
- const primary = observed.primary_error;
146
- if (observed.operation_id === operationId && observed.operation === expectedMethod(args)
191
+ const primary = observed.primary_error ?? (["failed", "interrupted"].includes(String(observed.state)) ? observed.error : undefined);
192
+ if (observed.operation_id === operationId && observed.operation === expectedMethod(args, session)
147
193
  && observed.state !== "completed" && (!target || observed.session_id === target)
148
194
  && primary && typeof primary === "object" && !Array.isArray(primary)) {
149
195
  const failure = primary;
@@ -151,69 +197,105 @@ async function recoverCompletedOperation(context, manifest, session, executable,
151
197
  throw new AppError(failure.code, failure.message, 1, { ...(failure.details && typeof failure.details === "object" && !Array.isArray(failure.details) ? failure.details : {}), operation_id: operationId, effect: "unknown" });
152
198
  }
153
199
  }
154
- if (observed.operation_id !== operationId || observed.operation !== expectedMethod(args) || observed.state !== "completed"
200
+ if (observed.operation_id !== operationId || observed.operation !== expectedMethod(args, session) || observed.state !== "completed"
155
201
  || !result || typeof result !== "object" || Array.isArray(result)
156
202
  || target && (observed.session_id !== target || adapterSessionId(result) !== target))
157
203
  return null;
158
- return { ...result, ...(settlement(observed) ? { settlement: settlement(observed) } : {}) };
204
+ return await recoveredSessionReceipt(context, session, args, operationId, { ...result, ...(settlement(observed) ? { settlement: settlement(observed) } : {}) }, Object.fromEntries(["generation", "params_sha256"].filter(key => observed[key] !== undefined).map(key => [key, observed[key]])), signal);
159
205
  }
160
206
  async function awaitSettlement(context, row, manifest, session, executable, args, operationId, initial, assertOwner, signal) {
161
207
  let current = initial, delay = 250;
162
- const deadline = Date.now() + 120_000;
208
+ const observation = recoveryObservationBudget(initial);
209
+ observation.observe(initial);
210
+ const retainObservation = () => {
211
+ assertOwner?.();
212
+ const receipt = { ...initial, ...current, recovery_observation: observation.state() };
213
+ const changed = context.db.run("UPDATE run_controller_operations SET receipt_json = ?, updated_at = ? WHERE operation_id = ? AND controller_id = ? AND generation = ? AND EXISTS (SELECT 1 FROM run_controllers WHERE controller_id = ? AND owner_token = ? AND generation = ?)", [JSON.stringify(receipt), context.now(), operationId, row.controller_id, row.generation, row.controller_id, row.owner_token, row.generation]);
214
+ if (changed.changes !== 1)
215
+ throw new AppError("controller_owner_mismatch", "Controller lost ownership before settlement observation retention", 1);
216
+ };
163
217
  while (["pending", "blocked"].includes(String(settlement(current)?.state))) {
164
218
  if (signal?.aborted)
165
219
  throw new AppError("harness_settlement_interrupted", "Harness settlement was interrupted", 1, { operation_id: operationId });
166
220
  // A completed parent with live children is normal controller waiting, not
167
221
  // failed bookkeeping. Keep its budget reserved until a later inspection.
168
- if (args[0] === "session" && ["prompt", "inspect", "status"].includes(args[1]) && settlement(current)?.reason === "tree_unsettled")
222
+ if (["session.prompt", "session.inspect"].includes(expectedMethod(args, session)) && settlement(current)?.reason === "tree_unsettled")
169
223
  return { ...initial, ...current, ...(initial.assistant_text !== undefined ? { assistant_text: initial.assistant_text } : {}) };
170
- if (Date.now() >= deadline)
171
- throw new AppError("settlement_recovery_required", "Completed provider operation requires settlement recovery", 1, { operation_id: operationId, settlement: settlement(current) });
172
- await new Promise(resolve => setTimeout(resolve, Math.min(delay, Math.max(0, deadline - Date.now()))));
224
+ const exhausted = observation.exhausted();
225
+ retainObservation();
226
+ if (exhausted) {
227
+ // Final bounded read is observation, never another settlement mutation.
228
+ if (args.includes("--session-id")) {
229
+ assertOwner?.();
230
+ try {
231
+ const terminal = await runHarnessAdapter({ executable, args: settlementInspectArgs(args), env: controllerEnvironment(context, manifest, session), operationId: `${operationId}:settlement-final-inspect`, runtimeOwner: session.owner, timeoutMs: 30_000, ...(signal ? { signal } : {}) });
232
+ validateReceipt(terminal, session, args);
233
+ if (settlement(terminal)?.state === "settled") {
234
+ await recoveredSessionReceipt(context, session, args, operationId, initial, {}, signal, terminal);
235
+ assertOwner?.();
236
+ current = terminal;
237
+ break;
238
+ }
239
+ }
240
+ catch (error) {
241
+ if (recoveryAuthorityError(error))
242
+ throw error;
243
+ }
244
+ }
245
+ throw new AppError("settlement_recovery_required", "Completed provider operation requires settlement recovery", 1, { operation_id: operationId, settlement: settlement(current), recovery_observation: observation.state() });
246
+ }
247
+ await new Promise(resolve => setTimeout(resolve, Math.min(delay, observation.timeout())));
173
248
  delay = Math.min(delay * 2, 8_000);
174
249
  assertOwner?.();
175
- if (Date.now() >= deadline)
250
+ if (observation.exhausted())
176
251
  continue;
177
252
  try {
178
253
  // This only retries durable bookkeeping against the same operation
179
254
  // receipt. It cannot replay the provider request.
180
- const inspect = args[0] === "session" && ["inspect", "status"].includes(args[1]);
181
- current = await runHarnessAdapter({ executable, args: inspect ? settlementInspectArgs(args) : settlementRetryArgs(args, operationId), env: controllerEnvironment(context, manifest, session), operationId: `${operationId}:settlement`, runtimeOwner: session.owner, timeoutMs: Math.min(30_000, Math.max(1, deadline - Date.now())), ...(signal ? { signal } : {}) });
255
+ const inspect = expectedMethod(args, session) === "session.inspect";
256
+ current = await runHarnessAdapter({ executable, args: inspect ? settlementInspectArgs(args) : settlementRetryArgs(args, operationId), env: controllerEnvironment(context, manifest, session), operationId: `${operationId}:settlement`, runtimeOwner: session.owner, timeoutMs: observation.timeout(), ...(signal ? { signal } : {}) });
182
257
  if (inspect)
183
258
  validateReceipt(current, session, args);
259
+ else
260
+ current = await recoveredSessionReceipt(context, session, args, operationId, current, Object.fromEntries(["generation", "params_sha256"].filter(key => current[key] !== undefined).map(key => [key, current[key]])), signal);
184
261
  }
185
262
  catch (error) {
186
- if (error instanceof AppError && ["session_identity_mismatch", "profile_integrity_violation"].includes(error.code))
263
+ if (recoveryAuthorityError(error))
187
264
  throw error;
188
265
  current = { settlement: { state: "pending", reason: "inspection_error", error: errorRecord(error) } };
189
266
  }
267
+ observation.observe(current);
190
268
  if (!(["pending", "blocked"].includes(String(settlement(current)?.state))))
191
269
  continue;
192
270
  // A receipt without final tree proof needs a fresh native inspection. Do
193
271
  // it only after the receipt-only retry, and only when this is a session.
194
- if (!args.includes("--session-id") || !["tree_unsettled", "settlement_evidence_required"].includes(String(settlement(current)?.reason)) || Date.now() >= deadline)
272
+ if (!args.includes("--session-id") || !["tree_unsettled", "settlement_evidence_required"].includes(String(settlement(current)?.reason)) || observation.exhausted())
195
273
  continue;
196
274
  try {
197
- current = await runHarnessAdapter({ executable, args: settlementInspectArgs(args), env: controllerEnvironment(context, manifest, session), operationId: `${operationId}:settlement-inspect`, runtimeOwner: session.owner, timeoutMs: Math.min(30_000, Math.max(1, deadline - Date.now())), ...(signal ? { signal } : {}) });
275
+ current = await runHarnessAdapter({ executable, args: settlementInspectArgs(args), env: controllerEnvironment(context, manifest, session), operationId: `${operationId}:settlement-inspect`, runtimeOwner: session.owner, timeoutMs: observation.timeout(), ...(signal ? { signal } : {}) });
198
276
  validateReceipt(current, session, args);
277
+ await recoveredSessionReceipt(context, session, args, operationId, initial, {}, signal, current);
199
278
  }
200
279
  catch (error) {
201
- if (error instanceof AppError && ["session_identity_mismatch", "profile_integrity_violation"].includes(error.code))
280
+ if (recoveryAuthorityError(error))
202
281
  throw error;
203
282
  current = { settlement: { state: "pending", reason: "inspection_error", error: errorRecord(error) } };
204
283
  }
284
+ observation.observe(current);
205
285
  }
206
286
  return adapterSessionId(current) ? { ...initial, ...current, ...(initial.assistant_text !== undefined ? { assistant_text: initial.assistant_text } : {}) } : { ...initial, settlement: settlement(current) };
207
287
  }
208
- export async function controllerAdapter(context, row, manifest, session, args, assertOwner, retainedOperationId, signal) {
209
- assertOwner?.();
210
- const operationId = retainedOperationId ?? `${row.controller_id}:${crypto.randomUUID()}`;
288
+ export async function controllerAdapter(context, row, manifest, session, args, assertOwner, retainedOperationId, signal, reconfirmOwner) {
211
289
  const executable = manifest.adapters[runtimeHarness(session.profile)].executable;
212
290
  const promptFile = args.indexOf("--prompt-file"), inlinePrompt = args.indexOf("--prompt");
213
291
  const promptSha256 = promptFile >= 0 ? digestBytes(fs.readFileSync(args[promptFile + 1])) : inlinePrompt >= 0 ? digestBytes(args[inlinePrompt + 1] ?? "") : undefined;
214
292
  const input = { executable, args, owner: session.owner.owner_id, state_dir: session.state_dir, ...(args[0] === "session" && args[1] === "prompt" && promptSha256 ? { prompt_sha256: promptSha256 } : {}) };
293
+ const prepared = retainedOperationId ? [] : context.db.all("SELECT operation_id FROM run_controller_operations WHERE controller_id = ? AND generation = ? AND input_hash = ? AND status = 'dispatching' AND json_extract(input_json, '$.dispatch_phase') = 'prepared'", [row.controller_id, row.generation, digest(input)]);
294
+ if (prepared.length > 1)
295
+ throw new AppError("controller_operation_conflict", "Multiple prepared intents require explicit reconciliation", 1);
296
+ const operationId = retainedOperationId ?? prepared[0]?.operation_id ?? `${row.controller_id}:${crypto.randomUUID()}`;
215
297
  const now = context.now();
216
- const existing = context.db.get("SELECT input_hash,status,receipt_json,error_json FROM run_controller_operations WHERE operation_id = ? AND controller_id = ? AND generation = ?", [operationId, row.controller_id, row.generation]);
298
+ const existing = context.db.get("SELECT input_hash,input_json,status,receipt_json,error_json FROM run_controller_operations WHERE operation_id = ? AND controller_id = ? AND generation = ?", [operationId, row.controller_id, row.generation]);
217
299
  if (existing?.input_hash && existing.input_hash !== digest(input))
218
300
  throw new AppError("controller_operation_conflict", "Retained operation has different dispatch parameters", 1, { operation_id: operationId });
219
301
  if (existing?.status === "completed" && existing.receipt_json) {
@@ -222,29 +304,63 @@ export async function controllerAdapter(context, row, manifest, session, args, a
222
304
  return receipt;
223
305
  }
224
306
  if (!existing) {
225
- context.db.run("INSERT INTO run_controller_operations (operation_id, controller_id, generation, input_hash, input_json, status, created_at, updated_at) VALUES (?, ?, ?, ?, ?, 'dispatching', ?, ?)", [operationId, row.controller_id, row.generation, digest(input), JSON.stringify(input), now, now]);
307
+ context.db.run("INSERT INTO run_controller_operations (operation_id, controller_id, generation, input_hash, input_json, status, created_at, updated_at) SELECT ?, ?, ?, ?, ?, 'dispatching', ?, ? WHERE EXISTS (SELECT 1 FROM run_controllers WHERE controller_id = ? AND owner_token = ? AND generation = ?)", [operationId, row.controller_id, row.generation, digest(input), JSON.stringify({ ...input, dispatch_phase: "prepared" }), now, now, row.controller_id, row.owner_token, row.generation]);
226
308
  appendEvent(context, row.controller_id, "operation_requested", { operation_id: operationId, operation: args.slice(0, 2), session_id: session.id || null });
227
309
  }
310
+ let phase = existing ? JSON.parse(existing.input_json).dispatch_phase : "prepared";
311
+ const retainPhase = (next) => {
312
+ const changed = context.db.run("UPDATE run_controller_operations SET input_json = ?, updated_at = ? WHERE operation_id = ? AND controller_id = ? AND generation = ? AND status <> 'completed' AND EXISTS (SELECT 1 FROM run_controllers WHERE controller_id = ? AND owner_token = ? AND generation = ?)", [JSON.stringify({ ...input, dispatch_phase: next }), context.now(), operationId, row.controller_id, row.generation, row.controller_id, row.owner_token, row.generation]);
313
+ if (changed.changes !== 1)
314
+ throw new AppError("controller_owner_mismatch", "Controller lost ownership before durable dispatch transition", 1, { operation_id: operationId });
315
+ phase = next;
316
+ };
228
317
  try {
318
+ await reconfirmOwner?.();
229
319
  assertOwner?.();
230
320
  let receipt;
231
321
  let recoveredOperation = false;
232
322
  try {
233
- const timeoutMs = args[0] === "session" && args[1] === "prompt" ? null
234
- : args[0] === "session" && args[1] === "create" && runtimeHarness(session.profile) === "zcode-acp" ? 180_000 : 120_000;
235
- if (existing) {
236
- const observed = await recoverCompletedOperation(context, manifest, session, executable, args, operationId, signal);
237
- if (!observed) {
238
- const cause = existing.error_json ? JSON.parse(existing.error_json) : { code: "controller_operation_outcome_unknown", message: "Retained dispatch has no observed terminal result", details: { operation_id: operationId } };
239
- throw new AppError(cause.code, cause.message, 1, cause.details);
240
- }
241
- receipt = observed;
323
+ const timeoutMs = 120_000;
324
+ if (existing && phase === "receipt_observed" && existing.receipt_json) {
325
+ // The exact input hash and committed receipt survive a controller
326
+ // crash. Complete settlement without replaying productive native IO.
327
+ receipt = JSON.parse(existing.receipt_json);
328
+ validateReceipt(receipt, session, args);
329
+ if (args[0] === "session" && args[1] === "create" && !adapterSessionId(receipt))
330
+ throw new AppError("session_identity_mismatch", "Retained creation receipt has no native Session identity", 1);
331
+ receipt = await recoveredSessionReceipt(context, session, args, operationId, receipt, {}, signal);
242
332
  recoveredOperation = true;
243
333
  }
244
- else
245
- receipt = await runHarnessAdapter({ executable, args, env: controllerEnvironment(context, manifest, session), operationId, runtimeOwner: session.owner, timeoutMs, ...(signal ? { signal } : {}) });
334
+ else if (existing && phase !== "prepared") {
335
+ if (expectedMethod(args, session) === "session.inspect") {
336
+ // A failed read may be explicitly reobserved. Never do this for a
337
+ // materializing resume, prompt, create or fork.
338
+ retainPhase("external_dispatch_possible");
339
+ receipt = await runHarnessAdapter({ executable, args: settlementInspectArgs(args), harness: runtimeHarness(session.profile), env: controllerEnvironment(context, manifest, session), operationId, runtimeOwner: session.owner, timeoutMs, ...(signal ? { signal } : {}) });
340
+ }
341
+ else {
342
+ const observed = await recoverCompletedOperation(context, manifest, session, executable, args, operationId, signal);
343
+ if (!observed) {
344
+ const cause = existing.error_json ? JSON.parse(existing.error_json) : { code: "controller_operation_outcome_unknown", message: "Retained dispatch has no observed terminal result", details: { operation_id: operationId } };
345
+ throw new AppError(cause.code, cause.message, 1, cause.details);
346
+ }
347
+ receipt = observed;
348
+ recoveredOperation = true;
349
+ }
350
+ }
351
+ else {
352
+ // Keep observation/settlement of retained operations available after
353
+ // terminal RUN, but never admit fresh native productive dispatch.
354
+ const productive = ["daemon.start", "session.create", "session.start", "session.fork", "session.prompt", "session.resume"].includes(expectedMethod(args, session));
355
+ if (productive && ["done", "failed", "cancelled"].includes(requireRun(context, row.project_id, row.run_id).status))
356
+ throw new AppError("invalid_run_state", "A terminal RUN cannot admit a new native controller operation", 1);
357
+ retainPhase("external_dispatch_possible");
358
+ receipt = await runHarnessAdapter({ executable, args, harness: runtimeHarness(session.profile), env: controllerEnvironment(context, manifest, session), operationId, runtimeOwner: session.owner, timeoutMs, ...(signal ? { signal } : {}) });
359
+ }
246
360
  }
247
361
  catch (error) {
362
+ if (phase === "prepared" || phase === "receipt_observed" && existing?.receipt_json)
363
+ throw error;
248
364
  const retained = await recoverCompletedOperation(context, manifest, session, executable, args, operationId, signal);
249
365
  if (!retained)
250
366
  throw error;
@@ -252,21 +368,28 @@ export async function controllerAdapter(context, row, manifest, session, args, a
252
368
  recoveredOperation = true;
253
369
  }
254
370
  validateReceipt(receipt, session, args);
255
- context.db.run("UPDATE run_controller_operations SET receipt_json = ?, updated_at = ? WHERE operation_id = ?", [JSON.stringify(receipt), context.now(), operationId]);
371
+ retainPhase("receipt_observed");
372
+ const persistReceipt = () => {
373
+ const changed = context.db.run("UPDATE run_controller_operations SET receipt_json = ?, updated_at = ? WHERE operation_id = ? AND controller_id = ? AND generation = ? AND EXISTS (SELECT 1 FROM run_controllers WHERE controller_id = ? AND owner_token = ? AND generation = ?)", [JSON.stringify(receipt), context.now(), operationId, row.controller_id, row.generation, row.controller_id, row.owner_token, row.generation]);
374
+ if (changed.changes !== 1)
375
+ throw new AppError("controller_owner_mismatch", "Controller lost ownership before receipt retention", 1);
376
+ };
377
+ persistReceipt();
256
378
  if (["pending", "blocked"].includes(String(settlement(receipt)?.state))) {
257
379
  const recovered = await awaitSettlement(context, row, manifest, session, executable, args, operationId, receipt, assertOwner, signal);
258
380
  receipt = recovered;
259
381
  }
260
- context.db.run("UPDATE run_controller_operations SET receipt_json = ?, updated_at = ? WHERE operation_id = ?", [JSON.stringify(receipt), context.now(), operationId]);
382
+ persistReceipt();
261
383
  validateReceipt(receipt, session, args);
262
- context.db.run("UPDATE run_controller_operations SET status = 'completed', updated_at = ? WHERE operation_id = ?", [context.now(), operationId]);
263
- assertOwner?.();
384
+ const completed = context.db.run("UPDATE run_controller_operations SET status = 'completed', updated_at = ? WHERE operation_id = ? AND controller_id = ? AND generation = ? AND EXISTS (SELECT 1 FROM run_controllers WHERE controller_id = ? AND owner_token = ? AND generation = ?)", [context.now(), operationId, row.controller_id, row.generation, row.controller_id, row.owner_token, row.generation]);
385
+ if (completed.changes !== 1)
386
+ throw new AppError("controller_owner_mismatch", "Controller lost ownership before operation completion", 1);
264
387
  appendEvent(context, row.controller_id, recoveredOperation ? "operation_recovered" : "operation_completed", { operation_id: operationId, operation: args.slice(0, 2), session_id: adapterSessionId(receipt) });
265
388
  return receipt;
266
389
  }
267
390
  catch (error) {
268
391
  try {
269
- context.db.run("UPDATE run_controller_operations SET status = CASE WHEN status = 'dispatching' THEN 'outcome_unknown' ELSE status END, error_json = ?, updated_at = ? WHERE operation_id = ?", [JSON.stringify(errorRecord(error)), context.now(), operationId]);
392
+ context.db.run("UPDATE run_controller_operations SET status = CASE WHEN status = 'dispatching' AND ? <> 'prepared' THEN 'outcome_unknown' ELSE status END, error_json = ?, updated_at = ? WHERE operation_id = ? AND controller_id = ? AND generation = ? AND EXISTS (SELECT 1 FROM run_controllers WHERE controller_id = ? AND owner_token = ? AND generation = ?)", [phase ?? "legacy_unknown", JSON.stringify(errorRecord(error)), context.now(), operationId, row.controller_id, row.generation, row.controller_id, row.owner_token, row.generation]);
270
393
  }
271
394
  catch (persistence) {
272
395
  process.stderr.write(`${JSON.stringify({ operation_id: operationId, primary: errorRecord(error), persistence_error: errorRecord(persistence) })}\n`);
@@ -83,7 +83,8 @@ export async function captureControllerBoundary(context, row, manifest, stage, n
83
83
  throw new AppError("controller_boundary_changed", "Stage identity changed after its capture fence", 1);
84
84
  if (physical) {
85
85
  const continuation = getFlowRunStatus(context, scope).continuation;
86
- if (nextStage ? continuation.kind !== "start_stage" || continuation.stage !== nextStage : continuation.kind !== "terminal")
86
+ const stoppedAtTarget = nextStage === null && manifest.request.stop_after === stage.stage && continuation.kind === "start_stage";
87
+ if (nextStage ? continuation.kind !== "start_stage" || continuation.stage !== nextStage : continuation.kind !== "terminal" && !stoppedAtTarget)
87
88
  throw new AppError("controller_boundary_changed", "Stage successor changed after its capture fence", 1);
88
89
  }
89
90
  };
@@ -105,11 +106,22 @@ export async function captureControllerBoundary(context, row, manifest, stage, n
105
106
  let lastPending = "";
106
107
  let lastCaptureMismatch = null;
107
108
  for (;;) {
109
+ assertCurrent();
110
+ if (fs.existsSync(output)) {
111
+ receipt = verifyControllerBoundarySnapshot(output, expected);
112
+ break;
113
+ }
108
114
  if (observationBudget.exhausted()) {
115
+ assertCurrent();
116
+ if (fs.existsSync(output)) {
117
+ receipt = verifyControllerBoundarySnapshot(output, expected);
118
+ break;
119
+ }
109
120
  context.db.run("UPDATE run_controller_operations SET receipt_json = json_set(json_set(COALESCE(receipt_json, '{}'), '$.recovery_observation', json(?)), '$.last_capture_mismatch', json(?)), updated_at = ? WHERE operation_id = ?", [JSON.stringify(observationBudget.state()), JSON.stringify(lastCaptureMismatch), context.now(), operationId]);
110
121
  throw new AppError("recovery_observation_budget_exhausted", "Boundary capture has no stable settlement within the active observation budget", 1, { recovery_observation: observationBudget.state(), last_capture_mismatch: lastCaptureMismatch });
111
122
  }
112
123
  assertCurrent();
124
+ context.db.run("UPDATE run_controller_operations SET receipt_json = json_set(COALESCE(receipt_json, '{}'), '$.recovery_observation', json(?)), updated_at = ? WHERE operation_id = ?", [JSON.stringify(observationBudget.state()), context.now(), operationId]);
113
125
  let verify = () => { throw new AppError("controller_boundary_unsettled", "Boundary has no stable writer observation", 1); };
114
126
  const observation = await withRecoveryObservation(observationBudget, () => observeControllerBoundary(context, { ...scope, controllerId: row.controller_id, operationId, assertCurrent, retainVerification: value => { verify = value; } }));
115
127
  if (observation.settled === true) {
@@ -126,7 +138,7 @@ export async function captureControllerBoundary(context, row, manifest, stage, n
126
138
  lastCaptureMismatch = { code: error.code, components: error.details?.components ?? [], changes: error.details?.changes ?? [] };
127
139
  }
128
140
  }
129
- observationBudget.exhausted();
141
+ observationBudget.observe(observation);
130
142
  const published = { ...observation, recovery_observation: observationBudget.state(), last_capture_mismatch: lastCaptureMismatch };
131
143
  const pending = digest(observation.pending_reasons);
132
144
  context.db.run("UPDATE run_controller_operations SET receipt_json = ?, updated_at = ? WHERE operation_id = ?", [JSON.stringify(published), context.now(), operationId]);
@@ -7,7 +7,7 @@ import { AppError } from "../shared/errors.js";
7
7
  import { getResourceDatabase } from "../storage/database.js";
8
8
  import { finishManagedProcess, managedProcessRuntimeHome, managedProcessStatus, processIsAlive, processStartedAt, processTreeIsAlive, registerManagedProcess } from "./managed-processes.js";
9
9
  import { setFlowRunRecoveryPaused } from "./runs.js";
10
- import { assertRunMutationAllowed, recoveryGuard } from "./run-recovery.js";
10
+ import { assertRunMutationAllowed, assertRunRecoveryAllowed, recoveryGuard } from "./run-recovery.js";
11
11
  import { requireRun, requireController, controllerReceipt, appendEvent, digest, errorRecord } from "./run-controller-state.js";
12
12
  const launchId = (row) => `${row.controller_id}:owner-launch:${digest(row.owner_token)}`;
13
13
  const processOperation = (context, row) => `controller:${digest({ database: context.db.path, project: row.project_id, run: row.run_id })}`;
@@ -27,17 +27,24 @@ function launchRecord(context, row) {
27
27
  const value = JSON.parse(record.receipt_json);
28
28
  return value?.registration === "controller_self@1" ? { ...record, value } : null;
29
29
  }
30
- function assertLaunchAllowed(context, row) {
30
+ export function assertLaunchAllowed(context, row) {
31
31
  const run = requireRun(context, row.project_id, row.run_id);
32
32
  if (!["running", "paused", "waiting_for_user"].includes(run.status))
33
33
  throw new AppError("invalid_run_state", "A terminal RUN cannot start a physical controller", 1);
34
+ if (JSON.parse(row.state_json).recovery)
35
+ assertControllerRecoveryAllowed(context, row);
36
+ else
37
+ assertRunMutationAllowed(context, row.project_id, row.run_id, row.generation);
38
+ }
39
+ /** Existing owners must retain the recovery fence while finishing terminal capture/settlement. */
40
+ export function assertControllerRecoveryAllowed(context, row) {
34
41
  const state = JSON.parse(row.state_json);
35
42
  if (!state.recovery) {
36
- assertRunMutationAllowed(context, row.project_id, row.run_id, row.generation);
43
+ assertRunRecoveryAllowed(context, row.project_id, row.run_id, row.generation);
37
44
  return;
38
45
  }
39
46
  const guard = recoveryGuard(context, row.project_id, row.run_id);
40
- if (guard?.generation !== row.generation || guard.recovery_id !== state.recovery.recovery_id || !["sealed", "resuming"].includes(guard.status)
47
+ if (guard?.generation !== row.generation || guard.recovery_id !== state.recovery.recovery_id || !["sealed", "resuming", "recovered"].includes(guard.status)
41
48
  || !context.db.get("SELECT 1 FROM run_controls WHERE control_id = ? AND recovery_id = ? AND status = 'resuming'", [state.recovery.control_id, state.recovery.recovery_id]))
42
49
  throw new AppError("run_control_superseded", "Controller launch no longer owns the accepted recovery", 1);
43
50
  }