@deksden-com/dd-flow-cli 0.9.0-beta.56 → 0.9.0-beta.63

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/CHANGELOG.md CHANGED
@@ -1,5 +1,49 @@
1
1
  # @deksden-com/dd-flow-cli
2
2
 
3
+ ## 0.9.0-beta.63
4
+
5
+ ### Patch Changes
6
+
7
+ - Accept the qualified retained-subagent ZCode bridge and settle a control only
8
+ when a durable startup rejection proves that it created no native resources.
9
+
10
+ ## 0.9.0-beta.62
11
+
12
+ ### Patch Changes
13
+
14
+ - Verify a controlled native Session with a fresh inspection after cancellation before treating its tree as settled.
15
+
16
+ ## 0.9.0-beta.61
17
+
18
+ ### Patch Changes
19
+
20
+ - Keep the scope reconciliation contract test type-safe.
21
+
22
+ ## 0.9.0-beta.60
23
+
24
+ ### Patch Changes
25
+
26
+ - Document and verify explicit reconciliation of a drained runtime scope without automatic observer respawn.
27
+
28
+ ## 0.9.0-beta.59
29
+
30
+ ### Patch Changes
31
+
32
+ - Keep bundled ZCode close-contract fixtures aligned with retained native topology checks.
33
+
34
+ ## 0.9.0-beta.58
35
+
36
+ ### Patch Changes
37
+
38
+ - Keep local task artifacts outside the release lint scope.
39
+
40
+ ## 0.9.0-beta.57
41
+
42
+ ### Patch Changes
43
+
44
+ - Bound recovery observation, expose safe external reconciliation, and require
45
+ complete non-resident ZCode session-tree evidence before a stopped run settles.
46
+
3
47
  ## 0.9.0-beta.56
4
48
 
5
49
  ### Patch Changes
@@ -1,15 +1,15 @@
1
1
  {
2
2
  "cli_package": "@deksden-com/dd-flow-cli",
3
- "cli_version": "0.9.0-beta.56",
4
- "cli_commit": "ae4bbaa5b2299aeedf3ee90bccc8ed3db19a6191",
5
- "built_at": "2026-09-13T21:08:32.108Z",
3
+ "cli_version": "0.9.0-beta.63",
4
+ "cli_commit": "4975ba184bd847fd34db995cd3d9e1feb3db9f9f",
5
+ "built_at": "2026-09-14T10:36:19.793Z",
6
6
  "built_with_canon": {
7
7
  "version": "4.1.1",
8
8
  "commit": "97f811d33c212ae3497020178b1ed825c7c3ebac",
9
9
  "flow_contract": "dd-flow-canonical-2026-08",
10
- "repo_root": "/home/runner/work/dd-flow-cli/dd-flow-cli/dd-memorybank",
11
- "memorybank_root": "/home/runner/work/dd-flow-cli/dd-flow-cli/dd-memorybank/.memory-bank",
12
- "flow_root": "/home/runner/work/dd-flow-cli/dd-flow-cli/dd-memorybank/.memory-bank/dd-flow",
10
+ "repo_root": "/Users/deksden/Documents/_Projects/dd-memorybank",
11
+ "memorybank_root": "/Users/deksden/Documents/_Projects/dd-memorybank/.memory-bank",
12
+ "flow_root": "/Users/deksden/Documents/_Projects/dd-memorybank/.memory-bank/dd-flow",
13
13
  "layout": "dot_memory_bank"
14
14
  }
15
15
  }
package/dist/cli/help.js CHANGED
@@ -5,6 +5,7 @@ const helpPages = new Map([
5
5
  dd-flow runtime scope stop --scope-id EVAL_ID --request-id ID --json
6
6
  dd-flow runtime scope control --scope-id EVAL_ID --request-id ID --mode pause|stop [--manifest-json JSON] --json
7
7
  dd-flow runtime scope resume --scope-id EVAL_ID --request-id ID --generation N --capture-key SHA256 --json
8
+ dd-flow runtime scope reconcile --scope-id EVAL_ID --generation N --request-id ID --json
8
9
 
9
10
  Use the EVAL manifest's retained DD_FLOW_RESOURCE_HOME. Commands select exactly
10
11
  one scope across its registered runtime homes; neighboring scopes are untouched.
@@ -188,6 +189,7 @@ Usage:
188
189
  dd-flow id next --type protocol|run --project-root <root> --slug <slug> --json
189
190
  dd-flow run status <RUN-ID|RUN-short-id> --project-root <root> --json
190
191
  dd-flow run control status --run <RUN-ID> --project-root <root> --json
192
+ dd-flow run control reconcile --run <RUN-ID> --project-root <root> --control-id <ID> --request-id <ID> --json
191
193
  dd-flow run drive launch --run <RUN-ID> --project-root <root> --request-id <id> --context-file <file> --context-sha256 <sha256> [--stop-after <stage>] [--capture-root <directory>] --json
192
194
  dd-flow run drive status --run <RUN-ID> --project-root <root> [--after <sequence>] --json
193
195
  dd-flow run list --project-root <root> --json
@@ -59,11 +59,11 @@ import { loadExternalStageContext } from "../services/stage-context.js";
59
59
  import { stopManagedProcess, confirmManagedProcess, finishManagedProcess, heartbeatManagedProcess, managedProcessRuntimeHome, managedProcessStatus, processIsAlive, reconcileExpiredManagedProcesses, registerManagedProcess } from "../services/managed-processes.js";
60
60
  import { fenceRuntimeBudget, runtimeBudgetStatus, admitRuntimeCheck, parseRuntimeBudget, reserveProviderTurn, releaseProviderTurns } from "../services/runtime-budget.js";
61
61
  import { stopRuntimeScope } from "../services/runtime-scope-stop.js";
62
- import { launchRuntimeScopeControl, launchRuntimeScopeResume, serveRuntimeScopeControl } from "../services/runtime-scope-worker.js";
62
+ import { launchRuntimeScopeControl, launchRuntimeScopeResume, reconcileRuntimeScopeControl, serveRuntimeScopeControl } from "../services/runtime-scope-worker.js";
63
63
  import { launchRunController, resumeRunController, serveRunController, runControllerStatus, answerRunController, supplyRunControllerContext } from "../services/run-controller.js";
64
64
  import { controllerHitlAdmission } from "../services/run-controller-adapter.js";
65
65
  import { controlLifecycleReply, runControlStatus } from "../services/run-control.js";
66
- import { requestRunControl, serveRunControl } from "../services/run-control-worker.js";
66
+ import { reconcileRunControl, requestRunControl, serveRunControl } from "../services/run-control-worker.js";
67
67
  import { startRuntimeService } from "../services/runtime-service.js";
68
68
  const defaultIo = {
69
69
  stdout: process.stdout,
@@ -1142,6 +1142,11 @@ async function dispatch(args, context, io, scopeProjectRoot = null, classificati
1142
1142
  const scope = { projectRoot: requiredOption(parsed, "project-root"), runId: requiredOption(parsed, "run") };
1143
1143
  if (action === "status")
1144
1144
  return runControlStatus(context, scope);
1145
+ if (action === "reconcile") {
1146
+ if ([...parsed.options.keys()].some(key => !["run", "project-root", "control-id", "request-id"].includes(key)) || [...parsed.options.values()].some(values => values.length !== 1))
1147
+ throw new AppError("usage", "Use run control reconcile --run ID --project-root PATH --control-id ID --request-id ID", 2);
1148
+ return await reconcileRunControl(context, { ...scope, controlId: requiredOption(parsed, "control-id"), requestId: requiredOption(parsed, "request-id") });
1149
+ }
1145
1150
  if (action === "resume") {
1146
1151
  if (["force", "grace-ms", "escalate"].some(option => hasOption(parsed, option)))
1147
1152
  throw new AppError("usage", "Resume does not accept stop or pause escalation options", 2);
@@ -1157,7 +1162,7 @@ async function dispatch(args, context, io, scopeProjectRoot = null, classificati
1157
1162
  ...(hasOption(parsed, "escalate") ? { escalate: requiredOption(parsed, "escalate") } : {}),
1158
1163
  ...(hasOption(parsed, "wait-ms") ? { waitMs: Number(requiredOption(parsed, "wait-ms")) } : {})
1159
1164
  });
1160
- throw new AppError("usage", "Use: dd-flow run control pause|stop|resume|status", 2);
1165
+ throw new AppError("usage", "Use: dd-flow run control pause|stop|resume|reconcile|status", 2);
1161
1166
  }
1162
1167
  if (command === "drive") {
1163
1168
  const action = requiredPosition(parsed, 0, "drive action");
@@ -1790,6 +1795,11 @@ function dispatchRun(context, command, parsed) {
1790
1795
  }
1791
1796
  async function dispatchRuntime(context, command, parsed, progress) {
1792
1797
  if (command === "scope") {
1798
+ if (parsed.positional.length === 1 && parsed.positional[0] === "reconcile") {
1799
+ if ([...parsed.options.keys()].some(key => !["scope-id", "request-id", "generation"].includes(key)) || [...parsed.options.values()].some(values => values.length !== 1))
1800
+ throw new AppError("usage", "Use runtime scope reconcile --scope-id ID --generation N --request-id ID", 2);
1801
+ return { ok: true, ...await reconcileRuntimeScopeControl(context, requiredOption(parsed, "scope-id"), Number(requiredOption(parsed, "generation")), requiredOption(parsed, "request-id")) };
1802
+ }
1793
1803
  if (parsed.positional.length === 1 && parsed.positional[0] === "resume") {
1794
1804
  if ([...parsed.options.keys()].some(key => !["scope-id", "request-id", "generation", "capture-key"].includes(key)) || [...parsed.options.values()].some(values => values.length !== 1))
1795
1805
  throw new AppError("usage", "Use runtime scope resume --scope-id ID --request-id ID --generation N --capture-key SHA256", 2);
@@ -16,6 +16,8 @@ import {
16
16
  forkSessionWithBridge,
17
17
  inspectSessionWithBridge,
18
18
  promptSessionWithBridge,
19
+ zcodeChildSessionIds,
20
+ observeClosedZcodeTree,
19
21
  } from "./dd-zcode.mjs";
20
22
 
21
23
  const REQUEST_SCHEMA = "dd-zcode/daemon-request@1";
@@ -152,7 +154,7 @@ export async function startDaemon(options) {
152
154
  await removeSocket(paths.socket, paths.dir);
153
155
  const versions = await doctor({ bin: config.bin, zcodePath: config.zcodePath });
154
156
  if (hasFlowContext && versions.compatible !== true) {
155
- throw new DaemonError("zcode_lifecycle_unqualified", "ZCode native event identity is not qualified for managed CLI admission; inspect lifecycle_qualification before starting E2E.", false, { observed_runtime: versions.observed_runtime, lifecycle_qualification: versions.lifecycle_qualification });
157
+ throw new DaemonError("zcode_lifecycle_unqualified", "ZCode native event identity is not qualified for managed CLI admission; inspect lifecycle_qualification before starting E2E.", false, { startup_resources: "not_created", observed_runtime: versions.observed_runtime, lifecycle_qualification: versions.lifecycle_qualification });
156
158
  }
157
159
  const resourceProcess = await registerDaemonProcess(config, {
158
160
  kind: "zcode-daemon", owner: `zcode:${path.basename(paths.dir)}`, operation: `zcode-daemon:${path.basename(paths.dir)}`,
@@ -228,7 +230,7 @@ export class DaemonRuntime {
228
230
  const known = this.sessions.get(params.sessionId);
229
231
  const adapter = params.adapterSessionId ?? known?.adapter_session_id ?? params.sessionId;
230
232
  const generation = this.activeGeneration ?? this.controlGeneration ?? 0;
231
- return { ...this.state.config, ...params, adapterSessionId: adapter, daemonId: this.state.daemon_id, allowBackground: true, liveSession: this.liveAdapters.has(adapter), assertDispatch: () => { if (generation !== (this.controlGeneration ?? 0)) throw new DaemonError("operation_cancelled", "ZCode prompt was cancelled before native dispatch"); } };
233
+ return { ...this.state.config, ...params, knownChildSessionIds: known?.known_child_session_ids ?? [], adapterSessionId: adapter, daemonId: this.state.daemon_id, allowBackground: true, liveSession: this.liveAdapters.has(adapter), assertDispatch: () => { if (generation !== (this.controlGeneration ?? 0)) throw new DaemonError("operation_cancelled", "ZCode prompt was cancelled before native dispatch"); } };
232
234
  }
233
235
 
234
236
  track(result, topology = result?.evidence?.subagents) {
@@ -244,6 +246,7 @@ export class DaemonRuntime {
244
246
  parent_provider_session_id: parent,
245
247
  root_provider_session_id: existing?.root_provider_session_id ?? (parent ? parentRoot : result.provider_session_id),
246
248
  topology: topology ?? existing?.topology ?? null,
249
+ known_child_session_ids: [...new Set([...(existing?.known_child_session_ids ?? zcodeChildSessionIds(existing?.topology)), ...zcodeChildSessionIds(topology)])],
247
250
  completed_turns: existing?.completed_turns ?? 0,
248
251
  close_receipt: result.close?.closed === true ? result : existing?.close_receipt ?? null,
249
252
  }); }
@@ -321,7 +324,17 @@ export class DaemonRuntime {
321
324
  const closed = this.sessions.get(params.sessionId)?.close_receipt;
322
325
  if (closed) {
323
326
  if (operation === "session.resume") throw new DaemonError("session_closed", "Closed Session requires explicit recovery");
324
- return { ...closed, read: { projection: { status: "stopped" } }, subagents: closed.after, settled: closed.settled === true };
327
+ const topology = closed.topology;
328
+ if (topology?.root_session_id !== params.sessionId || !Array.isArray(topology.child_session_ids) || topology.child_session_ids.some(id => typeof id !== "string" || !id)) return { ...closed, settled: false, observation_error: { code: "topology_incomplete", message: "closed Session has no native topology identity" } };
329
+ const previous = this.state.settlement_observations?.[params.sessionId];
330
+ const known = [...new Set([...topology.child_session_ids, ...(previous?.topology?.child_session_ids ?? [])])];
331
+ const observation = { ...closed, ...await observeClosedZcodeTree(this.bridge, params.sessionId, known), observed_at: new Date().toISOString() };
332
+ // Observations are not ownership inventory: their timestamps must not
333
+ // invalidate a drain fence on every read.
334
+ this.state.settlement_observations ??= {};
335
+ this.state.settlement_observations[params.sessionId] = observation;
336
+ await this.persist();
337
+ return observation;
325
338
  }
326
339
  const adapter = params.adapterSessionId ?? params.sessionId;
327
340
  if (!this.liveAdapters.has(adapter) && this.activeProductive) {
@@ -367,7 +380,8 @@ export class DaemonRuntime {
367
380
  const unsettled = [];
368
381
  for (const session of this.sessions.values()) {
369
382
  if (session.close_receipt) {
370
- if (!session.close_receipt.settled) unsettled.push(session.provider_session_id);
383
+ const observed = await this.dispatch("session.inspect", { sessionId: session.provider_session_id });
384
+ if (observed.settled !== true) unsettled.push(session.provider_session_id);
371
385
  continue;
372
386
  }
373
387
  const options = this.options({ sessionId: session.provider_session_id, adapterSessionId: session.adapter_session_id });
@@ -199,11 +199,14 @@ export class AcpBridge {
199
199
  if (message.method === "session/update" && this.options.onLifecycleNotification) {
200
200
  const observer = this.options.onLifecycleNotification;
201
201
  const pending = Promise.resolve().then(() => observer(message)).catch(async error => {
202
- if (!this.notificationError || (fatalLifecycleError(error) && !fatalLifecycleError(this.notificationError))) this.notificationError = error;
202
+ // A validated no-effect rejection is returned through the invocation
203
+ // ledger. It is evidence, not a poisoned ACP Session.
204
+ const fatal = fatalLifecycleError(error);
205
+ if (!this.notificationError || fatal) this.notificationError = error;
203
206
  // A missing/foreign authority is not a recoverable CLI validation
204
207
  // result. Stop admitting productive requests immediately; the managed
205
208
  // controller still owns physical stop/drain and evidence capture.
206
- if (fatalLifecycleError(error)) this.rejectAll(error);
209
+ if (fatal) this.rejectAll(error);
207
210
  try {
208
211
  await this.journal.write("notification_failure", { code: error.code ?? "invocation_observation_failed", message: error.message,
209
212
  session_id: message.params?.sessionId ?? null, tool_call_id: update?.toolCallId ?? null, event_time: now(), delivery_stage: "lifecycle_receipt", completeness: "incomplete" });
@@ -429,7 +432,50 @@ export function latestAssistantText(read) {
429
432
  return null;
430
433
  }
431
434
 
432
- async function cancelTree(bridge, sessionId, before, providerSessionId) {
435
+ export function zcodeChildSessionIds(snapshot) {
436
+ return [...new Set([
437
+ ...(Array.isArray(snapshot?.childSessionIds) ? snapshot.childSessionIds : []),
438
+ ...(Array.isArray(snapshot?.running) ? snapshot.running : []).map(child => child?.childSessionId),
439
+ ...(Array.isArray(snapshot?.ended?.items) ? snapshot.ended.items : []).map(child => child?.childSessionId),
440
+ ].filter(id => typeof id === "string" && id))];
441
+ }
442
+
443
+ // Read residency BEFORE each durable topology snapshot. An inactive node cannot
444
+ // add descendants; a resident node never proves settlement, even when idle.
445
+ // No resolve/read/subagents extension here: those may implicitly resume.
446
+ export async function observeClosedZcodeTree(bridge, root, retainedChildren = []) {
447
+ const pending = new Set([root, ...retainedChildren]), residents = [], snapshots = [];
448
+ const deadline = performance.now() + 30_000;
449
+ try {
450
+ for (const sessionId of pending) {
451
+ if (pending.size > 1024 || performance.now() >= deadline) throw new Error("Closed tree observation limit exceeded");
452
+ const timeout = () => Math.max(1, Math.min(5000, deadline - performance.now()));
453
+ const resident = await bridge.request("zcode/session/resident", { sessionId }, timeout());
454
+ if (resident?.sessionId !== sessionId || typeof resident.resident !== "boolean") throw new Error("Invalid residency identity or state");
455
+ residents.push(resident);
456
+ const retained = await bridge.request("zcode/session/retainedSubagents", { sessionId }, timeout());
457
+ const tree = retained?.topology;
458
+ if (retained?.sessionId !== sessionId || !Array.isArray(tree?.childSessionIds)
459
+ || tree.childSessionIds.some(id => typeof id !== "string" || !id || id === sessionId)
460
+ || !Array.isArray(tree?.running) || !Array.isArray(tree?.ended?.items)
461
+ || !Number.isSafeInteger(tree.ended.total) || tree.ended.total < 0
462
+ || tree.ended.total > tree.childSessionIds.length
463
+ || [...tree.running, ...tree.ended.items].some(child => !tree.childSessionIds.includes(child?.childSessionId))) throw new Error("Incomplete retained native topology");
464
+ snapshots.push({ sessionId, ...tree });
465
+ for (const child of tree.childSessionIds) pending.add(child);
466
+ }
467
+ return { topology: { root_session_id: root, child_session_ids: [...pending].filter(id => id !== root), complete: true }, residents, snapshots, settled: residents.every(item => item.resident === false) };
468
+ } catch (error) {
469
+ return { topology: { root_session_id: root, child_session_ids: [...pending].filter(id => id !== root), complete: false }, residents, snapshots, settled: false, observation_error: { code: error.code ?? "topology_observation_failed", message: error.message } };
470
+ }
471
+ }
472
+
473
+ async function cancelTree(bridge, sessionId, before, providerSessionId, retainedChildren = []) {
474
+ const knownChildren = new Set(retainedChildren);
475
+ const remember = snapshot => {
476
+ for (const id of zcodeChildSessionIds(snapshot)) knownChildren.add(id);
477
+ };
478
+ remember(before);
433
479
  const cancellations = [];
434
480
  for (const child of before.running ?? []) {
435
481
  const taskId = child.taskId ?? child.agentId;
@@ -452,27 +498,20 @@ async function cancelTree(bridge, sessionId, before, providerSessionId) {
452
498
  bridge.request("zcode/session/read", { sessionId }, timeout).catch(error => ({ observation_error: { code: error.code, message: error.message } })),
453
499
  ]);
454
500
  root_status = read?.projection?.status ?? read?.session?.status ?? null;
501
+ remember(after);
455
502
  if (after.observation_error || read?.observation_error) break;
456
503
  if (Array.isArray(after.running) && !after.running.length && !(read?.projection?.activeToolCalls ?? []).length && ["idle", "completed", "cancelled", "failed", "stopped"].includes(root_status)) {
457
- return { cancellations, before, after, root_status, cancellation_requested: false, settled: true, close_requested: false };
504
+ // Idle is only a turn-level observation. Close the owned resident before
505
+ // proving the transitive tree cannot create another child.
506
+ break;
458
507
  }
459
508
  }
460
509
  let close = null;
461
510
  try { close = await bridge.request("zcode/session/close", { sessionId }); }
462
511
  catch (error) { close = { closed: false, error: { code: error.code ?? "close_failed", message: error.message } }; }
463
- const knownChildren = [...new Set([...(before.running ?? []), ...(after.running ?? [])].map(child => child.childSessionId).filter(Boolean))];
464
- const residents = close?.closed === true ? await Promise.all([providerSessionId, ...knownChildren].map(async id => {
465
- try {
466
- const observed = await bridge.request("zcode/session/resident", { sessionId: id }, 5000);
467
- if (observed.sessionId !== id) throw Object.assign(new Error("Residency evidence belongs to another Session"), { code: "session_identity_mismatch" });
468
- return observed;
469
- }
470
- catch (error) { return { sessionId: id, observation_error: { code: error.code, message: error.message } }; }
471
- })) : [];
472
- const completeTopology = !before.observation_error && Array.isArray(before.running) && !after.observation_error && Array.isArray(after.running)
473
- && [...(before.running ?? []), ...after.running].every(child => typeof child.childSessionId === "string");
474
- return { cancellations, before, after, root_status, cancellation_requested: true, close_requested: true, close, residents,
475
- settled: close?.closed === true && completeTopology && residents.length > 0 && residents.every(item => item.resident === false) };
512
+ const observation = close?.closed === true ? await observeClosedZcodeTree(bridge, providerSessionId, [...knownChildren])
513
+ : { topology: { root_session_id: providerSessionId, child_session_ids: [...knownChildren], complete: false }, settled: false };
514
+ return { cancellations, before, after, root_status, cancellation_requested: true, close_requested: true, close, ...observation };
476
515
  }
477
516
 
478
517
  // Cancel one known child without cancelling its parent turn. The parent keeps
@@ -642,8 +681,9 @@ async function forwardUsage(options, providerSessionId, usage, toolCalls) {
642
681
  export function zcodeLifecycleQualification(sha256, bridgeCommit, harnessContract) {
643
682
  // Behavioural evidence: dd-eval production-invocations probe, 2026-09-13.
644
683
  // Pin both providers of native event identity, not a string in the bundle.
684
+ // 60af0d3 adds retainedSubagents without changing native event forwarding.
645
685
  const qualified = sha256 === "e9f1868c0fdb863537ed910ee3828b9be96b8c2fd805473f63b439e1113266b8"
646
- && bridgeCommit === "e0600fe3c46e215695f257e65aa6f674ced998d8" && harnessContract === ZCODE_HARNESS_CONTRACT;
686
+ && ["e0600fe3c46e215695f257e65aa6f674ced998d8", "60af0d31e13076a313d9770f10aa70f7c94742cf"].includes(bridgeCommit) && harnessContract === ZCODE_HARNESS_CONTRACT;
647
687
  return { status: qualified ? "qualified" : "unqualified", mechanism: "cli-invocation@1", sha256,
648
688
  capabilities: qualified ? ["root", "concurrent_children", "child_continuation"] : [],
649
689
  native_nested_children: false, reason: qualified ? null : "native_event_contract_unqualified" };
@@ -749,7 +789,7 @@ export async function cancelSessionWithBridge(bridge, options) {
749
789
  requireJournal(options);
750
790
  const identity = await controlledIdentity(bridge, options);
751
791
  const before = await bridge.request("zcode/session/subagents", { sessionId: identity.adapterSessionId }).catch(error => ({ observation_error: { code: error.code, message: error.message } }));
752
- return receipt(options, { harness: "zcode-acp", provider_session_id: identity.providerSessionId, adapter_session_id: identity.adapterSessionId, ...await cancelTree(bridge, identity.adapterSessionId, before, identity.providerSessionId) });
792
+ return receipt(options, { harness: "zcode-acp", provider_session_id: identity.providerSessionId, adapter_session_id: identity.adapterSessionId, ...await cancelTree(bridge, identity.adapterSessionId, before, identity.providerSessionId, options.knownChildSessionIds) });
753
793
  }
754
794
 
755
795
  export async function forkSession(options) {
@@ -3,6 +3,7 @@ import fs from "node:fs";
3
3
  import path from "node:path";
4
4
  import { fileURLToPath, pathToFileURL } from "node:url";
5
5
  import { AppError } from "../shared/errors.js";
6
+ import { recoveryRpcTimeout } from "./recovery-observation-budget.js";
6
7
  const maxAdapterOutputBytes = 1_000_000;
7
8
  /** Native hooks need a real executable, not a PATH-dependent shell expression. */
8
9
  export function prepareHarnessFlowExecutable(context, stateDir) {
@@ -37,7 +38,7 @@ export function prepareHarnessFlowExecutable(context, stateDir) {
37
38
  }
38
39
  /** Runs one harness command with the lifecycle guarantees shared by agent services. */
39
40
  export function runHarnessAdapter(input) {
40
- const timeoutMs = input.timeoutMs ?? 120_000;
41
+ const timeoutMs = recoveryRpcTimeout(input.timeoutMs ?? 120_000);
41
42
  if (!input.operationId.trim())
42
43
  throw new AppError("operation_id_required", "Harness calls require a scoped operation id", 2);
43
44
  // The current adapter transport uses this name; its identity is runtime-owned,
@@ -9,7 +9,7 @@ import { ensureDir, resolveProjectRoot } from "../storage/paths.js";
9
9
  import { appendAudit } from "./audit.js";
10
10
  import { nativeSessionIdentity, storageSessionId } from "./session-identity.js";
11
11
  import { assertRecoveryHookCurrent } from "./run-recovery.js";
12
- import { observeLifecycleInvocation } from "./lifecycle-invocations.js";
12
+ import { observeLifecycleInvocation, settleLifecycleNoEffectRejection } from "./lifecycle-invocations.js";
13
13
  import { commandHasOption, commandOption, commandPosition, shellSuffixOffset, parseLifecycleCommand, unwrapShellCommand } from "./lifecycle-command.js";
14
14
  import { registerProject, requireProjectByRoot } from "./projects.js";
15
15
  import { activeFlowSessionsForProject, bindObservedFlowSession, flowSessionPayloadFromRegisterCommand, recordFlowSessionObservation } from "./sessions.js";
@@ -433,9 +433,9 @@ export function handleZcodeEvent(context, input) {
433
433
  }) }, commandOption(parsed.invocation, "invocation-id"));
434
434
  }
435
435
  /**
436
- * Accept the synchronous native ZCode PreToolUse callback. ACP notifications
437
- * are retained as evidence, but cannot be an admission signal: they arrive
438
- * after the tool call and may be serialized behind unrelated work.
436
+ * Accept the synchronous native ZCode PreToolUse callback. Invocation-ID
437
+ * commands instead rendezvous with their ACP event while the CLI awaits
438
+ * admission; the native hook must not create a competing receipt for them.
439
439
  */
440
440
  export function handleZcodeHook(context, input) {
441
441
  return recordZcodeLifecycle(context, input);
@@ -455,9 +455,6 @@ function recordZcodeLifecycle(context, input, invocationId) {
455
455
  return { ok: true, observed: false, reason: "event_not_participating" };
456
456
  if (!invocationId && commandOption(lifecycle.invocation, "invocation-id"))
457
457
  return { ok: true, observed: false, reason: "cli_invocation_owns_admission" };
458
- if (lifecycle.analysis.kind === "compound") {
459
- throw new AppError("compound_lifecycle_command", "dd-flow lifecycle commands must be a standalone ZCode Bash tool call", 1, { standalone_command: lifecycle.invocation.command });
460
- }
461
458
  const expectedRoot = resolveProjectRoot(input.projectRoot);
462
459
  if (lifecycle.commandProjectRoot && resolveProjectRoot(lifecycle.commandProjectRoot) !== expectedRoot) {
463
460
  throw new AppError("project_mismatch", "ZCode lifecycle command project root does not match the controlled workspace", 1, { expected: expectedRoot, actual: lifecycle.commandProjectRoot });
@@ -477,6 +474,17 @@ function recordZcodeLifecycle(context, input, invocationId) {
477
474
  const toolUseId = stringValue(hook.tool_use_id) ?? stringValue(hook.toolUseId) ?? stringValue(hook.event_id) ?? stringValue(hook.eventId);
478
475
  if (!toolUseId)
479
476
  throw new AppError("zcode_hook_event_missing", "ZCode native hook has no tool-use ID", 1);
477
+ if (lifecycle.analysis.kind === "compound") {
478
+ if (lifecycle.analysis.reason !== "shell_composition")
479
+ throw new AppError("compound_lifecycle_command", "Ambiguous shell cannot authorize a lifecycle replacement", 1, { reason: lifecycle.analysis.reason });
480
+ if (invocationId) {
481
+ const error = { code: "compound_lifecycle_command", message: "dd-flow lifecycle commands must be a standalone ZCode Bash tool call", exitCode: 1,
482
+ details: { standalone_command: lifecycle.invocation.command } };
483
+ settleLifecycleNoEffectRejection(context, invocationId, lifecycle.invocation.command, error, { daemonId, rootSessionId: rootProviderSessionId, sessionId: providerSessionId, parentSessionId: parentProviderSessionId ?? null, toolCallId: toolUseId });
484
+ return { ok: true, observed: false, invocation_id: invocationId, no_effect: true, error };
485
+ }
486
+ throw new AppError("compound_lifecycle_command", "dd-flow lifecycle commands must be a standalone ZCode Bash tool call", 1, { standalone_command: lifecycle.invocation.command });
487
+ }
480
488
  const eventKey = crypto.createHash("sha256").update(JSON.stringify(["zcode-acp", daemonId, providerSessionId, toolUseId])).digest("hex");
481
489
  const rewritten = invocationId ? command : commandWithHookEvent(command, eventKey);
482
490
  if (!invocationId && rewritten === command && commandOption(lifecycle.invocation, "hook-event-id") !== eventKey)
@@ -69,7 +69,7 @@ export function lifecycleRetryCommands(context, stage) {
69
69
  return [];
70
70
  const successor = load(context, commandOption(invocation(retry), "invocation-id") ?? "");
71
71
  const latest = context.db.get("SELECT * FROM lifecycle_invocations WHERE scope_json = ? AND fingerprint = ? ORDER BY rowid DESC LIMIT 1", [row.scope_json, row.fingerprint]);
72
- return successor.id === latest?.id && successor.status === "issued" ? [retry] : [];
72
+ return successor.id === latest?.id && successor.status === "issued" && (successor.deadline === null || successor.deadline > Date.now()) ? [retry] : [];
73
73
  }
74
74
  catch {
75
75
  return [];
@@ -109,6 +109,7 @@ export function managedLifecycleCommand(context, command) {
109
109
  throw new AppError("invocation_scope_mismatch", "Lifecycle attempt belongs to another managed scope", 1, { invocation_id: suppliedId });
110
110
  }
111
111
  checkCommand(retained, command);
112
+ assertRenderableInvocation(retained);
112
113
  return command;
113
114
  }
114
115
  if (!context.db.writable || context.env.DD_FLOW_INVOCATION_READONLY === "1") {
@@ -143,21 +144,67 @@ export function retryLifecycleInvocationCommand(context, id) {
143
144
  const prior = load(context, id);
144
145
  if (prior.status !== "executing")
145
146
  throw new AppError("invocation_retry_not_authorized", "Only the executing lifecycle operation can publish its next attempt", 1);
147
+ return successorLifecycleInvocationCommand(context, prior);
148
+ }
149
+ /** A predecessor may publish exactly one successor only while the caller has
150
+ * proved that no lifecycle effect was admitted. This is shared by ordinary
151
+ * command validation and the native pre-execution rejection path. */
152
+ function successorLifecycleInvocationCommand(context, prior) {
146
153
  const parsed = invocation(prior.command);
147
154
  const argv = [...parsed.argv];
148
155
  const at = argv.indexOf("--invocation-id");
149
- if (at < 0 || argv[at + 1] !== id)
156
+ if (at < 0 || argv[at + 1] !== prior.id)
150
157
  throw new AppError("invocation_command_invalid", "Retained issued command lost its invocation marker", 1);
151
158
  // The parser proves the marker belongs to argv. Remove that one rendered
152
159
  // token from the retained text so response-file/stdin/heredoc presentation
153
160
  // remains byte-for-byte stable across a retry.
154
- const marker = ` --invocation-id ${id}`;
161
+ const marker = ` --invocation-id ${prior.id}`;
155
162
  const markerAt = prior.command.indexOf(marker);
156
163
  if (markerAt < 0 || prior.command.indexOf(marker, markerAt + marker.length) >= 0)
157
164
  throw new AppError("invocation_command_invalid", "Retained issued command has an ambiguous invocation marker", 1);
158
165
  const command = `${prior.command.slice(0, markerAt)}${prior.command.slice(markerAt + marker.length)}`;
159
- return managedLifecycleCommand({ ...context, env: { ...context.env, DD_FLOW_INVOCATION_SCOPE: prior.scope_json, DD_FLOW_CURRENT_INVOCATION: id } }, command);
166
+ return managedLifecycleCommand({ ...context, env: { ...context.env, DD_FLOW_INVOCATION_SCOPE: prior.scope_json, DD_FLOW_CURRENT_INVOCATION: prior.id } }, command);
167
+ }
168
+ /** A native observer can reject a malformed tool call before the CLI claims
169
+ * execution. Persist that fact and its sole replacement atomically: waiting
170
+ * callers must never time out merely because the observer rejected syntax. */
171
+ export function settleLifecycleNoEffectRejection(context, id, command, error, identity) {
172
+ storage(context);
173
+ context.db.exec("BEGIN IMMEDIATE");
174
+ try {
175
+ const prior = load(context, id);
176
+ checkCommand(prior, command);
177
+ const scope = JSON.parse(prior.scope_json);
178
+ checkNativeIdentity(scope, identity, id);
179
+ if (prior.status === "settled") {
180
+ context.db.exec("COMMIT");
181
+ return;
182
+ }
183
+ if (scope.runId)
184
+ assertLifecycleInvocationCurrent(context, scope, command);
185
+ if (prior.status !== "issued")
186
+ throw new AppError("invocation_rejection_conflict", "A lifecycle rejection arrived after execution admission", 1, { invocation_id: id });
187
+ settleUnadmittedInvocation(context, prior, error);
188
+ context.db.exec("COMMIT");
189
+ }
190
+ catch (error) {
191
+ context.db.exec("ROLLBACK");
192
+ throw error;
193
+ }
160
194
  }
195
+ // Caller holds the write transaction and has checked authority. Issued (or
196
+ // legacy expired) attempts have never admitted an effect; observed/executing
197
+ // attempts must never receive an automatic replacement.
198
+ function settleUnadmittedInvocation(context, prior, error) {
199
+ if (!["issued", "expired"].includes(prior.status))
200
+ throw new AppError("invocation_rejection_conflict", "Attempt already admitted native execution", 1);
201
+ const details = { ...error.details, retry_command: successorLifecycleInvocationCommand(context, prior),
202
+ retry_instruction: "Use retry_command as one standalone lifecycle command. The rejected command did not change lifecycle state." };
203
+ if (context.db.run("UPDATE lifecycle_invocations SET status = 'settled', outcome_json = ?, updated_at = ? WHERE id = ? AND status = ?", [JSON.stringify({ error: { ...error, details } }), context.now(), prior.id, prior.status]).changes !== 1) {
204
+ throw new AppError("invocation_settlement_conflict", "Lifecycle rejection could not be committed", 1, { invocation_id: prior.id });
205
+ }
206
+ }
207
+ const receiptTimeout = (id) => ({ code: "invocation_receipt_timeout", message: "No native confirmation before the receipt deadline; lifecycle state was not changed", exitCode: 1, details: { invocation_id: id } });
161
208
  /** Settle a conclusive rejection and publish its only legal successor atomically.
162
209
  * A crash must leave either the old attempt executing, or both the settled
163
210
  * outcome and its successor; never a usable successor beside an unknown effect. */
@@ -291,7 +338,13 @@ function commandWithInvocationId(command, id) {
291
338
  /** The attempt owns authority; the caller owns presentation-only shell input.
292
339
  * Preserve current stdin/heredoc/response-file text while validating that it
293
340
  * still names the exact persisted operation and project. */
341
+ function assertRenderableInvocation(row) {
342
+ if (row.status === "expired" || row.status === "issued" && row.deadline !== null && row.deadline <= Date.now()) {
343
+ throw new AppError("invocation_expired", "Retained lifecycle attempt has expired; inspection cannot grant a new attempt", 1, { invocation_id: row.id, status: row.status, outcome: row.outcome_json ? JSON.parse(row.outcome_json) : null });
344
+ }
345
+ }
294
346
  function renderInvocationCommand(row, command) {
347
+ assertRenderableInvocation(row);
295
348
  const parsed = invocation(command);
296
349
  if (commandOption(parsed, "invocation-id"))
297
350
  throw new AppError("invocation_command_invalid", "Rendered lifecycle command already has an invocation id", 1);
@@ -301,6 +354,13 @@ function renderInvocationCommand(row, command) {
301
354
  }
302
355
  /** Native observer and receipt writer commit together. Never await I/O inside
303
356
  * this transaction; the callback must write to this same SQLite connection. */
357
+ function checkNativeIdentity(scope, identity, id) {
358
+ if (identity.daemonId !== scope.daemonId || identity.rootSessionId !== scope.rootSessionId
359
+ || !identity.sessionId || !identity.toolCallId
360
+ || (identity.sessionId === scope.rootSessionId ? identity.parentSessionId !== null : !identity.parentSessionId)) {
361
+ throw new AppError("invocation_identity_mismatch", "Native identity does not belong to the issued managed scope", 1, { invocation_id: id });
362
+ }
363
+ }
304
364
  export function observeLifecycleInvocation(context, input) {
305
365
  const initial = load(context, input.id);
306
366
  checkCommand(initial, input.command);
@@ -311,11 +371,7 @@ export function observeLifecycleInvocation(context, input) {
311
371
  return { eventKey: initial.event_key, duplicate: true };
312
372
  const scope = JSON.parse(initial.scope_json);
313
373
  const identity = input.identity;
314
- if (identity.daemonId !== scope.daemonId || identity.rootSessionId !== scope.rootSessionId
315
- || !identity.sessionId || !identity.toolCallId
316
- || (identity.sessionId === scope.rootSessionId ? identity.parentSessionId !== null : !identity.parentSessionId)) {
317
- throw new AppError("invocation_identity_mismatch", "Native identity does not belong to the issued managed scope", 1, { invocation_id: input.id });
318
- }
374
+ checkNativeIdentity(scope, identity, input.id);
319
375
  const identityJson = JSON.stringify([identity.daemonId, identity.rootSessionId, identity.sessionId, identity.parentSessionId, identity.toolCallId]);
320
376
  context.db.exec("BEGIN IMMEDIATE");
321
377
  try {
@@ -325,7 +381,11 @@ export function observeLifecycleInvocation(context, input) {
325
381
  return { eventKey: row.event_key, duplicate: true };
326
382
  }
327
383
  if (row.status === "expired" || (row.deadline !== null && row.deadline <= Date.now() && !row.event_key)) {
328
- throw new AppError("invocation_expired", "Late observation cannot revive an expired lifecycle attempt", 1, { invocation_id: input.id });
384
+ if (scope.runId)
385
+ assertLifecycleInvocationCurrent(context, scope, input.command);
386
+ settleUnadmittedInvocation(context, row, receiptTimeout(input.id));
387
+ context.db.exec("COMMIT");
388
+ return { eventKey: row.event_key, duplicate: true };
329
389
  }
330
390
  if (row.identity_json) {
331
391
  if (row.identity_json !== identityJson)
@@ -370,10 +430,20 @@ export async function awaitLifecycleInvocation(context, input) {
370
430
  continue;
371
431
  }
372
432
  if (row.status === "expired" || (row.deadline !== null && row.deadline <= Date.now())) {
373
- const expired = context.db.run("UPDATE lifecycle_invocations SET status = 'expired', updated_at = ? WHERE id = ? AND status = 'issued'", [context.now(), input.id]);
374
- if (!expired.changes && row.status !== "expired")
375
- continue;
376
- throw new AppError("invocation_receipt_timeout", "No native confirmation before the receipt deadline; lifecycle state was not changed", 1, { invocation_id: input.id });
433
+ context.db.exec("BEGIN IMMEDIATE");
434
+ try {
435
+ const current = load(context, input.id);
436
+ if (["issued", "expired"].includes(current.status)) {
437
+ input.assertCurrent(JSON.parse(current.scope_json));
438
+ settleUnadmittedInvocation(context, current, receiptTimeout(input.id));
439
+ }
440
+ context.db.exec("COMMIT");
441
+ }
442
+ catch (error) {
443
+ context.db.exec("ROLLBACK");
444
+ throw error;
445
+ }
446
+ continue;
377
447
  }
378
448
  await delay(Math.min(25, Math.max(1, (row.deadline ?? Date.now() + 25) - Date.now())));
379
449
  }
@@ -8,15 +8,17 @@ export async function cancelControlledNativeSession(input) {
8
8
  if (input.observe && (reply.operation_id !== input.operationId || reply.operation !== "session.cancel" || reply.session_id !== input.sessionId))
9
9
  throw new AppError("controller_operation_mismatch", "Cancellation observation belongs to another operation or Session", 1);
10
10
  const result = input.observe ? reply.state === "completed" ? reply.result : null : reply;
11
+ // A completed cancel command is an immutable ACK, not proof that the
12
+ // provider tree has stopped. Callers must use a separate fresh inspection.
11
13
  if (!input.observe || reply.state === "completed")
12
- assertNativeCancellationReceipt(result, input.sessionId, input.requireSettled);
14
+ assertNativeCancellationReceipt(result, input.sessionId);
13
15
  input.assertCurrent();
14
16
  return { status: result ? "completed" : "outcome_unknown", receipt: (result ?? reply) };
15
17
  }
16
- export function assertNativeCancellationReceipt(value, sessionId, requireSettled = false) {
18
+ export function assertNativeCancellationReceipt(value, sessionId) {
17
19
  const identity = adapterSessionId(value);
18
- if (!value || typeof value !== "object" || Array.isArray(value) || identity && identity !== sessionId || (requireSettled && value.settled !== true))
19
- throw new AppError("controller_operation_mismatch", requireSettled ? "Cancellation receipt has no terminal settlement for its target Session" : "Cancellation receipt has no valid result for its target Session", 1);
20
+ if (!value || typeof value !== "object" || Array.isArray(value) || identity && identity !== sessionId)
21
+ throw new AppError("controller_operation_mismatch", "Cancellation receipt has no valid result for its target Session", 1);
20
22
  }
21
23
  /** Native observation is shared by RUN and non-Work EVAL owners. */
22
24
  export async function inspectControlledNativeSession(input) {
@@ -0,0 +1,53 @@
1
+ import { AsyncLocalStorage } from "node:async_hooks";
2
+ import { AppError } from "../shared/errors.js";
3
+ export const recoveryObservationBudgetMs = 120_000;
4
+ const activeObservation = new AsyncLocalStorage();
5
+ export const withRecoveryObservation = (budget, action) => activeObservation.run(budget, action);
6
+ export const recoveryRpcTimeout = (maximum) => activeObservation.getStore()?.timeout(maximum) ?? maximum;
7
+ /** Call only while holding the existing owner-claim transaction. A replay of
8
+ * an explicit request never replenishes time, even after another request. */
9
+ export function renewRecoveryObservation(snapshot, requestId) {
10
+ if (!requestId.trim() || requestId.length > 200)
11
+ throw new AppError("usage", "Reconciliation requires a nonempty request ID of at most 200 characters", 2);
12
+ const previous = snapshot.recovery_observation;
13
+ const ids = previous?.reconciliation_ids ?? [];
14
+ if (!Array.isArray(ids) || ids.some(id => typeof id !== "string"))
15
+ throw new AppError("recovery_observation_invalid", "Retained reconciliation identities are malformed", 1);
16
+ if (ids.includes(requestId))
17
+ return snapshot;
18
+ return { ...snapshot, recovery_observation: { budget_ms: recoveryObservationBudgetMs, remaining_ms: recoveryObservationBudgetMs, reconciliation_ids: [...ids, requestId], poll_index: 0 } };
19
+ }
20
+ /**
21
+ * Measures only time this owner can actually observe. Large scheduler gaps
22
+ * reset the sample instead of turning host sleep into evidence about a remote
23
+ * process. Persisting the remaining budget makes a recovered owner continue
24
+ * reconciliation rather than silently receiving a new unlimited wait.
25
+ */
26
+ export function recoveryObservationBudget(saved, monotonic = () => performance.now()) {
27
+ const prior = saved && typeof saved === "object" ? saved.recovery_observation : null;
28
+ let remaining = typeof prior?.remaining_ms === "number" && Number.isFinite(prior.remaining_ms)
29
+ ? Math.max(0, Math.min(recoveryObservationBudgetMs, prior.remaining_ms))
30
+ : prior == null ? recoveryObservationBudgetMs : 0;
31
+ let last = monotonic();
32
+ let pollIndex = Number.isSafeInteger(prior?.poll_index) && prior.poll_index >= 0 ? prior.poll_index : 0;
33
+ const observation = {
34
+ exhausted: () => {
35
+ const current = monotonic();
36
+ const elapsed = current - last;
37
+ last = current;
38
+ // A gap means this process was not observing; it never proves progress
39
+ // or death and therefore does not consume the reconciliation budget.
40
+ if (elapsed >= 0 && elapsed <= 60_000)
41
+ remaining = Math.max(0, remaining - elapsed);
42
+ return remaining === 0;
43
+ },
44
+ state: () => ({ budget_ms: recoveryObservationBudgetMs, remaining_ms: Math.ceil(remaining), reconciliation_ids: prior?.reconciliation_ids ?? [], poll_index: pollIndex }),
45
+ timeout: (maximum = 30_000) => {
46
+ if (observation.exhausted())
47
+ throw new AppError("recovery_observation_budget_exhausted", "Recovery observation budget exhausted; explicit reconciliation is required", 1);
48
+ return Math.max(1, Math.min(maximum, Math.ceil(remaining)));
49
+ },
50
+ nextDelay: () => Math.min([1000, 2000, 5000, 10000][Math.min(pollIndex++, 3)], Math.ceil(remaining)),
51
+ };
52
+ return observation;
53
+ }
@@ -13,6 +13,17 @@ import { confirmManagedProcess, finishManagedProcess, heartbeatManagedProcess, m
13
13
  import { interruptControlledSessions, observeControlledSessions, recordRunControlIntent, runControlStatus } from "./run-control.js";
14
14
  import { recoveryGuard, recordRecoveryCapture, sealRunRecovery } from "./run-recovery.js";
15
15
  import { createEvalRunSnapshot, verifyRecoverySnapshot } from "./eval-snapshots.js";
16
+ import { recoveryObservationBudget, renewRecoveryObservation, withRecoveryObservation } from "./recovery-observation-budget.js";
17
+ export async function reconcileRunControl(context, input) {
18
+ const status = runControlStatus(context, input);
19
+ const control = status.control;
20
+ if (!control?.current || !["requested", "settled"].includes(control.status))
21
+ throw new AppError("run_control_state_changed", "Reconciliation requires the current draining/sealed control", 1);
22
+ renewRecoveryObservation({}, input.requestId); // Validate before claiming anything.
23
+ if (control.status === "requested")
24
+ await launchControlWorker(context, { ...input, reconcileId: input.requestId });
25
+ return runControlStatus(context, input);
26
+ }
16
27
  /** The caller publishes intent and launches a control owner; it never owns drain. */
17
28
  export async function requestRunControl(context, input) {
18
29
  if (input.waitMs !== undefined && (!Number.isSafeInteger(input.waitMs) || input.waitMs < 0 || input.waitMs > 60_000))
@@ -24,7 +35,7 @@ export async function requestRunControl(context, input) {
24
35
  };
25
36
  if (!receipt.control_id || receipt.status !== "requested")
26
37
  return projectReceipt();
27
- const scope = { projectRoot: input.projectRoot, runId: input.runId, controlId: String(receipt.control_id) };
38
+ const scope = { projectRoot: input.projectRoot, runId: input.runId, controlId: String(receipt.control_id), ...(input.reconcileId ? { reconcileId: input.reconcileId } : {}) };
28
39
  await launchControlWorker(context, scope);
29
40
  const deadline = performance.now() + (input.waitMs ?? 0);
30
41
  while (performance.now() < deadline) {
@@ -49,8 +60,11 @@ async function launchControlWorker(context, input) {
49
60
  context.db.exec("BEGIN IMMEDIATE");
50
61
  try {
51
62
  claimed = context.db.run("INSERT OR IGNORE INTO run_control_workers (control_id, owner_token, status, resource_home, state_dir, created_at, updated_at) VALUES (?, ?, 'starting', ?, ?, ?, ?)", [input.controlId, token, home, directory, now, now]).changes === 1;
52
- if (claimed)
63
+ if (claimed) {
53
64
  recordControlLauncher(context, input.controlId, token);
65
+ if (input.reconcileId)
66
+ context.db.run("UPDATE run_control_workers SET snapshot_json = ? WHERE control_id = ?", [JSON.stringify(renewRecoveryObservation({}, input.reconcileId)), input.controlId]);
67
+ }
54
68
  context.db.exec("COMMIT");
55
69
  }
56
70
  catch (error) {
@@ -93,6 +107,10 @@ function reclaimControlWorker(context, input, projectId, token, home) {
93
107
  const previous = context.db.get("SELECT * FROM run_control_workers WHERE control_id = ?", [input.controlId]);
94
108
  if (!previous || !["starting", "running", "failed"].includes(previous.status) || previous.resource_home !== home)
95
109
  return false;
110
+ const saved = JSON.parse(previous.snapshot_json ?? "{}");
111
+ const requested = input.reconcileId ? renewRecoveryObservation(saved, input.reconcileId) : saved;
112
+ if (recoveryObservationBudget(requested).state().remaining_ms === 0)
113
+ return false;
96
114
  const registry = fs.existsSync(path.join(home, "runtime.sqlite")) ? getResourceDatabase(home) : null;
97
115
  let record = previous.process_id ? registry?.get("SELECT * FROM managed_processes WHERE id = ?", [previous.process_id]) : null;
98
116
  const operationId = launchOperationId(input.controlId, previous.owner_token);
@@ -157,6 +175,10 @@ function reclaimControlWorker(context, input, projectId, token, home) {
157
175
  context.db.exec("ROLLBACK");
158
176
  return false;
159
177
  }
178
+ if (input.reconcileId) {
179
+ const snapshot = renewRecoveryObservation(JSON.parse(previous.snapshot_json ?? "{}"), input.reconcileId);
180
+ context.db.run("UPDATE run_control_workers SET snapshot_json = ?, error_json = NULL WHERE control_id = ? AND owner_token = ?", [JSON.stringify(snapshot), input.controlId, token]);
181
+ }
160
182
  const now = context.now();
161
183
  context.db.run("INSERT INTO run_control_operations (operation_id, control_id, target_json, status, receipt_json, created_at, updated_at) VALUES (?, ?, ?, 'completed', ?, ?, ?)", [`${launchOperationId(input.controlId, previous.owner_token)}:reconciled`, input.controlId, JSON.stringify({ kind: "control_worker_reconciliation", process_id: record?.id ?? null, resource_home: home }), JSON.stringify({ process_id: record?.id ?? null, pid: record?.pid ?? null, pid_started_at: record?.pid_started_at ?? null, prior_status: previous.status, snapshot: JSON.parse(previous.snapshot_json ?? "null"), error: JSON.parse(previous.error_json ?? "null") }), now, now]);
162
184
  recordControlLauncher(context, input.controlId, token);
@@ -216,6 +238,14 @@ export async function serveRunControl(context, input) {
216
238
  if (!worker.process_lease_token)
217
239
  throw new AppError("run_control_owner_mismatch", "Control owner has no resource lease", 1);
218
240
  confirmManagedProcess(resourceContext, { id: worker.process_id, leaseToken: worker.process_lease_token, pid: process.pid, processGroupId: process.pid, ownerPid: process.pid });
241
+ let priorObservation = null;
242
+ try {
243
+ priorObservation = worker.snapshot_json ? JSON.parse(worker.snapshot_json) : null;
244
+ }
245
+ catch { /* A malformed prior snapshot cannot grant a new budget. */
246
+ priorObservation = { recovery_observation: { remaining_ms: 0 } };
247
+ }
248
+ const observation = recoveryObservationBudget(priorObservation);
219
249
  for (;;) {
220
250
  if (leaseLost)
221
251
  throw new AppError("run_control_lease_lost", "Control worker lost its physical ownership lease", 1);
@@ -226,6 +256,18 @@ export async function serveRunControl(context, input) {
226
256
  context.db.run("UPDATE run_control_workers SET status = ?, updated_at = ? WHERE control_id = ? AND owner_token = ?", [status, context.now(), input.controlId, worker.owner_token]);
227
257
  return { ok: true, control_id: input.controlId, status };
228
258
  }
259
+ if (observation.exhausted()) {
260
+ const latest = context.db.get("SELECT snapshot_json FROM run_control_workers WHERE control_id = ?", [input.controlId]);
261
+ let retained = {};
262
+ try {
263
+ retained = JSON.parse(latest?.snapshot_json ?? "{}") ?? {};
264
+ }
265
+ catch { /* Retain a blocked status even when the old projection is malformed. */ }
266
+ const snapshot = { ...retained, status: "recovery_blocked", settled: false, pending_reasons: ["recovery_observation_budget_exhausted"],
267
+ error: { code: "recovery_observation_budget_exhausted", message: "Control settlement has no confirmed result within the active observation budget" }, recovery_observation: observation.state() };
268
+ context.db.run("UPDATE run_control_workers SET status = 'failed', snapshot_json = ?, error_json = ?, updated_at = ? WHERE control_id = ? AND owner_token = ? AND status = 'running'", [JSON.stringify(snapshot), JSON.stringify(snapshot.error), context.now(), input.controlId, worker.owner_token]);
269
+ return { ok: true, control_id: input.controlId, ...snapshot };
270
+ }
229
271
  if (guard.status === "sealed") {
230
272
  // Capture retries are local evidence work. Never dispatch another
231
273
  // native interrupt/inspect merely because publication previously failed.
@@ -250,14 +292,15 @@ export async function serveRunControl(context, input) {
250
292
  AND EXISTS (SELECT 1 FROM run_recovery_guards g WHERE g.recovery_id = run_controls.recovery_id AND g.status = 'sealed' AND g.capture_path = ? AND g.generation = (SELECT MAX(generation) FROM run_recovery_guards WHERE project_id = g.project_id AND run_id = g.run_id))`, [context.now(), input.controlId, guard.recovery_id, worker.owner_token, capture]);
251
293
  if (applied.changes !== 1)
252
294
  throw new AppError("run_control_state_changed", "Control changed before its capture could be acknowledged", 1);
253
- const snapshot = { ...settlement, status: "settled", capture_path: capture };
295
+ const snapshot = { ...settlement, status: "settled", capture_path: capture, recovery_observation: observation.state() };
254
296
  context.db.run("UPDATE run_control_workers SET status = 'completed', snapshot_json = ?, error_json = NULL, updated_at = ? WHERE control_id = ? AND owner_token = ?", [JSON.stringify(snapshot), context.now(), input.controlId, worker.owner_token]);
255
297
  return { ok: true, control_id: input.controlId, ...snapshot };
256
298
  }
257
299
  catch (error) {
258
- const snapshot = { ...settlement, status: "requested", capture: { status: "pending", error: failure(error) }, pending_reasons: ["recovery_capture_required"] };
300
+ observation.exhausted();
301
+ const snapshot = { ...settlement, status: "requested", capture: { status: "pending", error: failure(error) }, pending_reasons: ["recovery_capture_required"], recovery_observation: observation.state() };
259
302
  context.db.run("UPDATE run_control_workers SET snapshot_json = ?, error_json = ?, updated_at = ? WHERE control_id = ? AND owner_token = ? AND status = 'running'", [JSON.stringify(snapshot), JSON.stringify(failure(error)), context.now(), input.controlId, worker.owner_token]);
260
- await delay(5_000);
303
+ await delay(observation.nextDelay());
261
304
  continue;
262
305
  }
263
306
  }
@@ -265,12 +308,16 @@ export async function serveRunControl(context, input) {
265
308
  const interrupt = current.action === "stop" || request.escalate === "interrupt" && Date.now() >= Date.parse(current.created_at) + (request.grace_ms ?? 30_000);
266
309
  let snapshot;
267
310
  try {
268
- const cancellation = interrupt ? await interruptControlledSessions(context, input) : null;
269
- snapshot = { ...await observeControlledSessions(context, input), ...(cancellation ? { cancellation } : {}) };
311
+ snapshot = await withRecoveryObservation(observation, async () => {
312
+ const cancellation = interrupt ? await interruptControlledSessions(context, input) : null;
313
+ return { ...await observeControlledSessions(context, input), ...(cancellation ? { cancellation } : {}) };
314
+ });
270
315
  }
271
316
  catch (error) {
272
317
  snapshot = { ok: true, control_id: input.controlId, status: "requested", settled: false, pending_reasons: ["control_observation_failed"], error: failure(error) };
273
318
  }
319
+ observation.exhausted();
320
+ snapshot = { ...snapshot, recovery_observation: observation.state() };
274
321
  context.db.run("UPDATE run_control_workers SET snapshot_json = ?, updated_at = ? WHERE control_id = ? AND owner_token = ? AND status = 'running'", [JSON.stringify(snapshot), context.now(), input.controlId, worker.owner_token]);
275
322
  if (snapshot.settled === true) {
276
323
  try {
@@ -281,7 +328,7 @@ export async function serveRunControl(context, input) {
281
328
  context.db.run("UPDATE run_control_workers SET snapshot_json = ?, error_json = ?, updated_at = ? WHERE control_id = ? AND owner_token = ? AND status = 'running'", [JSON.stringify({ ...snapshot, pending_reasons: ["recovery_seal_required"], error: failure(error) }), JSON.stringify(failure(error)), context.now(), input.controlId, worker.owner_token]);
282
329
  }
283
330
  }
284
- await delay(1_000);
331
+ await delay(observation.nextDelay());
285
332
  }
286
333
  }
287
334
  catch (error) {
@@ -413,10 +413,29 @@ async function reconcileOwnedSessions(context, input, interrupt, boundary) {
413
413
  }
414
414
  unresolved.push({ check_id: check.id, status: "pending", reason });
415
415
  }
416
+ const uncreatedDirectories = new Map();
416
417
  const sessionNodes = Promise.all([...targets.values()].map(async ({ session, executable, harness }) => {
417
418
  const env = { ...context.env, DD_FLOW_HOME: context.ddFlowHome };
418
419
  delete env.DD_FLOW_CONTROLLER_TOKEN;
419
420
  delete env.DD_FLOW_CONTROL_TOKEN;
421
+ // An explicit pre-registration rejection proves there is no Session to
422
+ // cancel. A missing id or an arbitrary startup failure does not.
423
+ if (!session.id && !session.create_operation_id && !daemonInventories.has(session.state_dir) && !unsafeDaemons.has(canonicalPath(session.state_dir, false)) && !fs.existsSync(path.join(session.state_dir, "daemon.json"))) {
424
+ const operations = context.db.all("SELECT o.operation_id, o.input_json FROM run_controller_operations o JOIN run_controllers c ON c.controller_id = o.controller_id WHERE c.project_id = ? AND c.run_id = ? AND json_extract(o.input_json, '$.state_dir') = ?", [project.id, input.runId, session.state_dir]);
425
+ if (operations.length === 1)
426
+ try {
427
+ const operation = operations[0];
428
+ const saved = JSON.parse(operation.input_json);
429
+ const journal = retainedDaemonOperations(session.state_dir);
430
+ const receipt = journal.length === 1 ? journal[0] : null;
431
+ const error = receipt?.error;
432
+ if (saved.executable === executable && saved.owner === session.owner.owner_id && saved.args?.[0] === "daemon" && saved.args?.[1] === "start" && receipt?.operation_id === operation.operation_id && receipt.operation === "daemon.start" && receipt.predecessor_daemon_id === null && receipt.state === "failed" && error?.details?.startup_resources === "not_created" && !registeredProcesses.some(({ record }) => JSON.parse(record.metadata_json).state_dir === session.state_dir)) {
433
+ uncreatedDirectories.set(session.state_dir, JSON.stringify(journal));
434
+ return { state_dir: session.state_dir, status: "completed", receipt: { source: "startup_resources_not_created", operation_id: operation.operation_id, error } };
435
+ }
436
+ }
437
+ catch { /* Incomplete evidence retains the normal unknown outcome. */ }
438
+ }
420
439
  if (!session.id && session.create_operation_id) {
421
440
  try {
422
441
  assertCurrent();
@@ -457,20 +476,32 @@ async function reconcileOwnedSessions(context, input, interrupt, boundary) {
457
476
  assertCurrent();
458
477
  const inserted = context.db.run("INSERT OR IGNORE INTO run_control_operations (operation_id, control_id, target_json, status, created_at, updated_at) VALUES (?, ?, ?, 'dispatching', ?, ?)", [operationId, observationId, JSON.stringify({ session_id: session.id, state_dir: session.state_dir, executable }), now, now]);
459
478
  const prior = context.db.get("SELECT status, receipt_json FROM run_control_operations WHERE operation_id = ?", [operationId]);
479
+ let cancellation;
480
+ let cancellationError;
460
481
  try {
461
482
  if (!inserted.changes && prior.status === "completed") {
462
483
  const saved = JSON.parse(prior.receipt_json);
463
- assertNativeCancellationReceipt(saved, session.id, harness === "zcode-acp");
464
- return { operation_id: operationId, session_id: session.id, status: prior.status, receipt: saved, reused: true };
484
+ assertNativeCancellationReceipt(saved, session.id);
485
+ cancellation = saved;
486
+ }
487
+ else {
488
+ const { status, receipt } = await cancelControlledNativeSession({ executable, sessionId: session.id, stateDir: session.state_dir, env, operationId, observe: !inserted.changes, assertCurrent });
489
+ context.db.run("UPDATE run_control_operations SET status = ?, receipt_json = ?, error_json = NULL, updated_at = ? WHERE operation_id = ?", [status, JSON.stringify(receipt), context.now(), operationId]);
490
+ cancellation = receipt;
465
491
  }
466
- const { status, receipt } = await cancelControlledNativeSession({ executable, sessionId: session.id, stateDir: session.state_dir, env, operationId, observe: !inserted.changes, assertCurrent, requireSettled: harness === "zcode-acp" });
467
- context.db.run("UPDATE run_control_operations SET status = ?, receipt_json = ?, error_json = NULL, updated_at = ? WHERE operation_id = ?", [status, JSON.stringify(receipt), context.now(), operationId]);
468
- return { operation_id: operationId, session_id: session.id, status, receipt };
469
492
  }
470
493
  catch (error) {
471
- const failure = { code: error instanceof AppError ? error.code : "control_observation_failed", message: String(error) };
472
- context.db.run("UPDATE run_control_operations SET status = 'outcome_unknown', error_json = ?, updated_at = ? WHERE operation_id = ?", [JSON.stringify(failure), context.now(), operationId]);
473
- return { operation_id: operationId, session_id: session.id, status: "outcome_unknown", error: failure };
494
+ cancellationError = { code: error instanceof AppError ? error.code : "control_observation_failed", message: String(error) };
495
+ context.db.run("UPDATE run_control_operations SET status = 'outcome_unknown', error_json = ?, updated_at = ? WHERE operation_id = ? AND status != 'completed'", [JSON.stringify(cancellationError), context.now(), operationId]);
496
+ if (cancellationError.code === "controller_operation_mismatch")
497
+ return { operation_id: operationId, session_id: session.id, status: "outcome_unknown", error: cancellationError };
498
+ }
499
+ try {
500
+ const observed = await inspectControlledNativeSession({ executable, sessionId: session.id, stateDir: session.state_dir, harness, env, operationId: `${observationId}:inspect:${crypto.randomUUID()}`, assertCurrent });
501
+ return { operation_id: operationId, session_id: session.id, status: observed.settled === true ? "completed" : "pending", receipt: observed, cancellation, reused: !inserted.changes, ...(cancellationError ? { cancellation_error: cancellationError } : {}) };
502
+ }
503
+ catch (error) {
504
+ return { operation_id: operationId, session_id: session.id, status: "pending", cancellation, cancellation_error: cancellationError, error: { code: error instanceof AppError ? error.code : "control_observation_failed", message: String(error) } };
474
505
  }
475
506
  }));
476
507
  const processResults = Promise.all(processTargets.map(async ({ home, record }) => {
@@ -533,7 +564,17 @@ async function reconcileOwnedSessions(context, input, interrupt, boundary) {
533
564
  const executable = directoryAdapters.get(directory) ?? (leader && JSON.parse(leader.metadata_json).adapter_executable);
534
565
  if (!daemonId || !leader || !executable)
535
566
  return { state_dir: directory, status: "pending", reason: "daemon_close_owner_unproven" };
536
- const operationId = `${observationId}:close-daemon:${crypto.createHash("sha256").update(JSON.stringify([directory, daemonId])).digest("hex")}`;
567
+ let operationId = `${observationId}:close-daemon:${crypto.createHash("sha256").update(JSON.stringify([directory, daemonId])).digest("hex")}`;
568
+ const nativeOperations = retainedDaemonOperations(directory);
569
+ const targetList = [...targets.values()];
570
+ const sessionsSettled = ids.every(id => nodes.some((node, index) => targetList[index]?.session.state_dir === directory && targetList[index]?.session.id === id && node.status === "completed"));
571
+ // A confirmed failed tree guard admits a new shutdown attempt only after
572
+ // fresh tree settlement. Unknown outcomes retain their original ID.
573
+ if (sessionsSettled) {
574
+ while (nativeOperations.some(operation => operation.operation_id === operationId && operation.daemon_id === daemonId && operation.operation === "daemon.stop"
575
+ && operation.state === "failed" && operation.error?.code === "tree_not_settled"))
576
+ operationId += ":finalize";
577
+ }
537
578
  assertCurrent();
538
579
  const inserted = context.db.run("INSERT OR IGNORE INTO run_control_operations (operation_id, control_id, target_json, status, created_at, updated_at) VALUES (?, ?, ?, 'dispatching', ?, ?)", [operationId, observationId, JSON.stringify({ state_dir: directory, daemon_id: daemonId, executable, cancel_tree: true }), context.now(), context.now()]);
539
580
  const prior = context.db.get("SELECT status, receipt_json FROM run_control_operations WHERE operation_id = ?", [operationId]);
@@ -600,7 +641,7 @@ async function reconcileOwnedSessions(context, input, interrupt, boundary) {
600
641
  };
601
642
  bindings = readBindings();
602
643
  for (const target of targets.values())
603
- if (!daemonInventories.has(target.session.state_dir))
644
+ if (!daemonInventories.has(target.session.state_dir) && !uncreatedDirectories.has(target.session.state_dir))
604
645
  unresolved.push({ state_dir: target.session.state_dir, status: "pending", reason: "runtime_inventory_unverified" });
605
646
  for (const node of nodes)
606
647
  if ("harness" in node && "session_id" in node && typeof node.session_id === "string" && bindings.some(binding => foreign(binding) && underRoot(binding, node.session_id, node.harness ?? null)))
@@ -625,6 +666,9 @@ async function reconcileOwnedSessions(context, input, interrupt, boundary) {
625
666
  const processVersion = (items) => JSON.stringify(items.map(({ home, record }) => [home, record.id, record.kind, record.owner_id, record.project_id, record.run_id, record.work_id, record.check_id, record.operation_id, record.pid, record.pid_started_at, record.lease_token, record.state, record.metadata_json]).sort((left, right) => JSON.stringify(left).localeCompare(JSON.stringify(right))));
626
667
  verifyInventory = () => {
627
668
  assertCurrent();
669
+ for (const [directory, journal] of uncreatedDirectories)
670
+ if (fs.existsSync(path.join(directory, "daemon.json")) || JSON.stringify(retainedDaemonOperations(directory)) !== journal)
671
+ throw new AppError("owned_inventory_changed", "Rejected startup acquired new runtime evidence", 1);
628
672
  const currentProcesses = [...resourceHomes].flatMap(home => managedProcessStatus({ ...context, env: { ...context.env, DD_FLOW_RESOURCE_HOME: home } }).filter(record => record.project_id === project.id && record.run_id === input.runId && [null, context.ddFlowHome].includes(managedProcessRuntimeHome(record, context))).map(record => ({ home, record })));
629
673
  if (initialDatabaseVersion !== databaseVersion() || processVersion(registeredProcesses) !== processVersion(currentProcesses) || [...nativeVersions].some(([directory, version]) => version !== nativeInventoryVersion(directory)) || [...importVersions].some(([file, version]) => crypto.createHash("sha256").update(fs.readFileSync(file)).digest("hex") !== version))
630
674
  throw new AppError("owned_inventory_changed", "Owned writers changed after their settlement observation", 1);
@@ -7,6 +7,7 @@ import { observeControllerBoundary } from "./run-control.js";
7
7
  import { assertRunMutationAllowed, recoveryGuard } from "./run-recovery.js";
8
8
  import { appendEvent, digest, errorRecord, requireController, requireRun } from "./run-controller-state.js";
9
9
  import { getFlowRunStatus } from "./runs.js";
10
+ import { recoveryObservationBudget, withRecoveryObservation } from "./recovery-observation-budget.js";
10
11
  /** Capture is performed by the productive owner, before it releases the next
11
12
  * Stage. An observer never races the controller to copy a completed boundary. */
12
13
  export async function captureControllerBoundary(context, row, manifest, stage, nextStage, assertPhysicalOwner) {
@@ -63,6 +64,14 @@ export async function captureControllerBoundary(context, row, manifest, stage, n
63
64
  if (nextStage ? continuation.kind !== "start_stage" || continuation.stage !== nextStage : continuation.kind !== "terminal")
64
65
  throw new AppError("controller_boundary_changed", "Stage successor changed after its capture fence", 1);
65
66
  };
67
+ let priorObservation = null;
68
+ try {
69
+ priorObservation = prior?.receipt_json ? JSON.parse(prior.receipt_json) : null;
70
+ }
71
+ catch {
72
+ priorObservation = { recovery_observation: { remaining_ms: 0 } };
73
+ }
74
+ const observationBudget = recoveryObservationBudget(priorObservation);
66
75
  try {
67
76
  let receipt;
68
77
  if (fs.existsSync(output)) {
@@ -72,9 +81,13 @@ export async function captureControllerBoundary(context, row, manifest, stage, n
72
81
  else {
73
82
  let lastPending = "";
74
83
  for (;;) {
84
+ if (observationBudget.exhausted()) {
85
+ context.db.run("UPDATE run_controller_operations SET receipt_json = json_set(COALESCE(receipt_json, '{}'), '$.recovery_observation', json(?)), updated_at = ? WHERE operation_id = ?", [JSON.stringify(observationBudget.state()), context.now(), operationId]);
86
+ throw new AppError("recovery_observation_budget_exhausted", "Boundary capture has no stable settlement within the active observation budget", 1, { recovery_observation: observationBudget.state() });
87
+ }
75
88
  assertCurrent();
76
89
  let verify = () => { throw new AppError("controller_boundary_unsettled", "Boundary has no stable writer observation", 1); };
77
- const observation = await observeControllerBoundary(context, { ...scope, controllerId: row.controller_id, operationId, assertCurrent, retainVerification: value => { verify = value; } });
90
+ const observation = await withRecoveryObservation(observationBudget, () => observeControllerBoundary(context, { ...scope, controllerId: row.controller_id, operationId, assertCurrent, retainVerification: value => { verify = value; } }));
78
91
  if (observation.settled === true) {
79
92
  try {
80
93
  createEvalRunSnapshot(context, { ...scope, output, ...(nextStage ? { stageEntry: nextStage } : { candidate: true }), boundary: { controllerId: row.controller_id, operationId, boundaryKey, observation, verify } });
@@ -88,13 +101,15 @@ export async function captureControllerBoundary(context, row, manifest, stage, n
88
101
  observation.pending_reasons = [error.code];
89
102
  }
90
103
  }
104
+ observationBudget.exhausted();
105
+ const published = { ...observation, recovery_observation: observationBudget.state() };
91
106
  const pending = digest(observation.pending_reasons);
107
+ context.db.run("UPDATE run_controller_operations SET receipt_json = ?, updated_at = ? WHERE operation_id = ?", [JSON.stringify(published), context.now(), operationId]);
92
108
  if (pending !== lastPending) {
93
- context.db.run("UPDATE run_controller_operations SET receipt_json = ?, updated_at = ? WHERE operation_id = ?", [JSON.stringify(observation), context.now(), operationId]);
94
109
  appendEvent(context, row.controller_id, "boundary_capture_pending", { operation_id: operationId, pending_reasons: observation.pending_reasons });
95
110
  lastPending = pending;
96
111
  }
97
- await delay(1000);
112
+ await delay(observationBudget.nextDelay());
98
113
  }
99
114
  }
100
115
  context.db.exec("BEGIN IMMEDIATE");
@@ -111,6 +126,8 @@ export async function captureControllerBoundary(context, row, manifest, stage, n
111
126
  }
112
127
  }
113
128
  catch (error) {
129
+ observationBudget.exhausted();
130
+ context.db.run("UPDATE run_controller_operations SET receipt_json = json_set(COALESCE(receipt_json, '{}'), '$.recovery_observation', json(?)) WHERE operation_id = ?", [JSON.stringify(observationBudget.state()), operationId]);
114
131
  context.db.run("UPDATE run_controller_operations SET status = 'failed', error_json = ?, updated_at = ? WHERE operation_id = ?", [JSON.stringify(errorRecord(error)), context.now(), operationId]);
115
132
  throw error;
116
133
  }
@@ -16,7 +16,7 @@ export function requireController(context, id) {
16
16
  export function controllerReceipt(row) {
17
17
  const state = JSON.parse(row.state_json);
18
18
  const manifest = JSON.parse(row.manifest_json);
19
- return { ok: true, schema_id: "dd-flow/run-controller-receipt@1", controller: { controller_id: row.controller_id, request_id: row.request_id, project_id: row.project_id, run_id: row.run_id, generation: row.generation, runtime_budget: manifest.budget ?? null, status: row.status, process_id: row.process_id, owner_pid: row.owner_pid, state_dir: row.state_dir, stage: state.stage, sessions: state.sessions.map(session => ({ session_id: session.id, harness: runtimeHarness(session.profile), profile_id: session.profile.id, state_dir: session.state_dir, stopped: session.stopped === true })), error: row.last_error_json ? JSON.parse(row.last_error_json) : null } };
19
+ return { ok: true, schema_id: "dd-flow/run-controller-receipt@1", controller: { controller_id: row.controller_id, request_id: row.request_id, project_id: row.project_id, run_id: row.run_id, generation: row.generation, runtime_budget: manifest.budget ?? null, status: row.status, process_id: row.process_id, owner_pid: row.owner_pid, state_dir: row.state_dir, stage: state.stage, sessions: state.sessions.map(session => ({ session_id: session.id, harness: runtimeHarness(session.profile), profile_id: session.profile.id, state_dir: session.state_dir, journal: session.journal, stopped: session.stopped === true })), error: row.last_error_json ? JSON.parse(row.last_error_json) : null } };
20
20
  }
21
21
  export function appendEvent(context, id, type, data) { context.db.run("INSERT INTO run_controller_events (controller_id, event_type, data_json, created_at) VALUES (?, ?, ?, ?)", [id, type, JSON.stringify(data), context.now()]); }
22
22
  export function saveState(context, row, state) { context.db.run("UPDATE run_controllers SET state_json = ?, updated_at = ? WHERE controller_id = ? AND owner_token = ?", [JSON.stringify(state), context.now(), row.controller_id, row.owner_token]); }
@@ -235,7 +235,7 @@ export async function controlRuntimeScopeNativeOwners(context, scopeId, generati
235
235
  const sessions = await Promise.all(ids.map(async (id) => {
236
236
  try {
237
237
  let cancellation;
238
- let cancellationAcknowledged = action !== "stop";
238
+ let cancellationError;
239
239
  if (action === "stop") {
240
240
  const operationId = `scope-cancel:${crypto.createHash("sha256").update(JSON.stringify([scopeId, generation, record.id, state.daemon_id, id])).digest("hex")}`;
241
241
  const now = context.now();
@@ -255,22 +255,34 @@ export async function controlRuntimeScopeNativeOwners(context, scopeId, generati
255
255
  if (saved.status === "completed") {
256
256
  assertNativeCancellationReceipt(saved.receipt, id);
257
257
  cancellation = saved.receipt;
258
- cancellationAcknowledged = true;
258
+ }
259
+ else if (saved.status === "failed") {
260
+ cancellation = saved.receipt;
261
+ try {
262
+ cancellationError = JSON.parse(saved.error ?? "");
263
+ }
264
+ catch {
265
+ cancellationError = { code: "cancellation_outcome_unknown", message: saved.error ?? "Retained cancellation has no conclusive outcome" };
266
+ }
259
267
  }
260
268
  else {
261
269
  const result = await cancelControlledNativeSession({ executable: metadata.adapter_executable, sessionId: id, stateDir: metadata.state_dir, env, operationId, observe: !inserted, assertCurrent: assertOwner });
262
270
  registry.run("UPDATE managed_resources SET metadata_json = json_set(metadata_json, '$.status', ?, '$.receipt', json(?)), updated_at = ? WHERE resource_kind = 'scope-control-operation' AND resource_key = ?", [result.status, JSON.stringify(result.receipt), context.now(), operationId]);
263
271
  cancellation = result.receipt;
264
- cancellationAcknowledged = result.status === "completed";
265
272
  }
266
273
  }
267
274
  catch (error) {
268
- registry.run("UPDATE managed_resources SET metadata_json = json_set(metadata_json, '$.status', 'outcome_unknown', '$.error', ?), updated_at = ? WHERE resource_kind = 'scope-control-operation' AND resource_key = ?", [String(error), context.now(), operationId]);
269
- throw error;
275
+ cancellationError = { code: error instanceof AppError ? error.code : "cancellation_outcome_unknown", message: String(error) };
276
+ registry.run("UPDATE managed_resources SET metadata_json = json_set(metadata_json, '$.status', 'outcome_unknown', '$.error', ?), updated_at = ? WHERE resource_kind = 'scope-control-operation' AND resource_key = ?", [JSON.stringify(cancellationError), context.now(), operationId]);
270
277
  }
278
+ if (cancellationError?.code === "controller_operation_mismatch")
279
+ throw new AppError("controller_operation_mismatch", "Cancellation evidence belongs to another operation or Session", 1);
271
280
  }
272
281
  const receipt = await inspectControlledNativeSession({ executable: metadata.adapter_executable, sessionId: id, stateDir: metadata.state_dir, env, operationId: `scope:${scopeId}:${generation}:inspect:${crypto.randomUUID()}`, assertCurrent: assertOwner });
273
- return { session_id: id, settled: receipt.settled === true && cancellationAcknowledged, receipt, ...(cancellation ? { cancellation } : {}) };
282
+ // Physical settlement is determined by this fresh inspection. A past
283
+ // cancel ACK (or an inconclusive cancel outcome) is retained as
284
+ // evidence, never substituted for the state of the native tree.
285
+ return { session_id: id, settled: receipt.settled === true, receipt, ...(cancellation ? { cancellation } : {}), ...(cancellationError ? { cancellation_error: cancellationError } : {}) };
274
286
  }
275
287
  catch (error) {
276
288
  return { session_id: id, settled: false, reason: error instanceof AppError ? error.code : "runtime_scope_session_control_failed", message: String(error) };
@@ -294,7 +306,7 @@ export async function controlRuntimeScopeNativeOwners(context, scopeId, generati
294
306
  }));
295
307
  }
296
308
  /** Scope routing only: existing RUN workers own physical drain and capture. */
297
- export async function controlRuntimeScopeRuns(context, scopeId, requestId, mode) {
309
+ export async function controlRuntimeScopeRuns(context, scopeId, requestId, mode, reconcileId) {
298
310
  const control = requestRuntimeScopeControl(context, scopeId, requestId, mode);
299
311
  const registry = getResourceDatabase(resourceHome(context));
300
312
  const inventory = runtimeBudgetStatus(context, scopeId).processes;
@@ -341,7 +353,7 @@ export async function controlRuntimeScopeRuns(context, scopeId, requestId, mode)
341
353
  throw error;
342
354
  }
343
355
  seen.add(key);
344
- const receipt = await requestRunControl(target, { ...input, waitMs: 0 });
356
+ const receipt = await requestRunControl(target, { ...input, waitMs: 0, ...(reconcileId ? { reconcileId: crypto.createHash("sha256").update(JSON.stringify([scopeId, control.generation, reconcileId])).digest("hex") } : {}) });
345
357
  runs.push({ runtime_home: metadata.dd_flow_home, project_id: record.project_id, run_id: record.run_id, request_id: input.requestId, receipt });
346
358
  }
347
359
  catch (error) {
@@ -10,6 +10,7 @@ import { confirmManagedProcess, finishManagedProcess, heartbeatManagedProcess, p
10
10
  import { publicRuntimeScopeWorker, requestRuntimeScopeControl, runtimeBudgetStatus } from "./runtime-budget.js";
11
11
  import { controlRuntimeScopeRuns } from "./runtime-scope-control.js";
12
12
  import { prepareRuntimeScopeResume, recordRuntimeScopeResume } from "./runtime-scope-resume.js";
13
+ import { recoveryObservationBudget, renewRecoveryObservation, withRecoveryObservation } from "./recovery-observation-budget.js";
13
14
  export async function launchRuntimeScopeResume(context, input) {
14
15
  const resume = recordRuntimeScopeResume(context, input);
15
16
  if (!resume.current)
@@ -19,7 +20,7 @@ export async function launchRuntimeScopeResume(context, input) {
19
20
  const control = runtimeBudgetStatus(context, input.scopeId).control;
20
21
  if (control?.generation !== input.generation)
21
22
  throw new AppError("runtime_scope_control_superseded", "Scope resume source changed before owner launch", 1);
22
- const owner = await launchRuntimeScopeControl(context, input.scopeId, control.request_id, control.requested_mode);
23
+ const owner = await launchRuntimeScopeControl(context, input.scopeId, control.request_id, control.requested_mode, undefined, undefined, input.requestId);
23
24
  return { ...owner, resume, dispatch_blocked: true, settled: false };
24
25
  }
25
26
  const keyFor = (scope, generation) => crypto.createHash("sha256").update(JSON.stringify([scope, generation])).digest("hex");
@@ -29,7 +30,14 @@ function birth() {
29
30
  throw new AppError("runtime_scope_owner_unknown", "Scope owner birth identity is unavailable", 1);
30
31
  return { pid: process.pid, pid_started_at: started };
31
32
  }
32
- export async function launchRuntimeScopeControl(context, scopeId, requestId, mode, manifest) {
33
+ export async function reconcileRuntimeScopeControl(context, scopeId, generation, requestId) {
34
+ const control = runtimeBudgetStatus(context, scopeId).control;
35
+ if (!control || control.generation !== generation)
36
+ throw new AppError("runtime_scope_control_superseded", "Reconciliation requires the current controlled scope generation", 1);
37
+ renewRecoveryObservation({}, requestId);
38
+ return launchRuntimeScopeControl(context, scopeId, control.request_id, control.requested_mode, undefined, requestId);
39
+ }
40
+ export async function launchRuntimeScopeControl(context, scopeId, requestId, mode, manifest, reconcileId, resumeId) {
33
41
  const control = requestRuntimeScopeControl(context, scopeId, requestId, mode, manifest), db = getResourceDatabase(resourceHome(context)), key = keyFor(scopeId, control.generation);
34
42
  let worker;
35
43
  db.exec("BEGIN IMMEDIATE");
@@ -50,7 +58,18 @@ export async function launchRuntimeScopeControl(context, scopeId, requestId, mod
50
58
  db.exec("COMMIT");
51
59
  return { scope_id: scopeId, control, current: true, settled: false, worker: publicRuntimeScopeWorker(existing) };
52
60
  }
53
- worker = { scope_id: scopeId, generation: control.generation, request_id: requestId, mode, token: crypto.randomUUID(), status: "starting", launcher: birth() };
61
+ if (existing?.status === "completed" && !reconcileId && !resumeId) {
62
+ db.exec("COMMIT");
63
+ return { scope_id: scopeId, control, current: true, settled: false, worker: publicRuntimeScopeWorker(existing) };
64
+ }
65
+ const renewal = resumeId ? `resume:${resumeId}` : reconcileId;
66
+ const snapshot = renewal ? renewRecoveryObservation(existing?.snapshot ?? {}, renewal) : existing?.snapshot;
67
+ if (existing && recoveryObservationBudget(snapshot).state().remaining_ms === 0) {
68
+ db.exec("COMMIT");
69
+ return { scope_id: scopeId, control, current: true, settled: false, worker: publicRuntimeScopeWorker(existing) };
70
+ }
71
+ const cleanupRequest = resumeId ? undefined : reconcileId ?? existing?.reconcile_id;
72
+ worker = { scope_id: scopeId, generation: control.generation, request_id: requestId, mode, token: crypto.randomUUID(), status: "starting", launcher: birth(), ...(snapshot ? { snapshot } : {}), ...(cleanupRequest ? { reconcile_id: cleanupRequest } : {}) };
54
73
  const now = context.now();
55
74
  db.run("INSERT INTO managed_resources (resource_kind,resource_key,owner_id,lease_token,lease_expires_at,process_id,metadata_json,created_at,updated_at) VALUES ('scope-control-worker',?,?,?,?,NULL,?,?,?) ON CONFLICT(resource_kind,resource_key) DO UPDATE SET lease_token = excluded.lease_token, metadata_json = excluded.metadata_json, updated_at = excluded.updated_at", [key, scopeId, worker.token, now, JSON.stringify(worker), now, now]);
56
75
  db.exec("COMMIT");
@@ -107,6 +126,10 @@ export async function serveRuntimeScopeControl(context, scopeId, generation) {
107
126
  throw error;
108
127
  }
109
128
  let heartbeat;
129
+ // A fence must never be held indefinitely by an observer that cannot obtain
130
+ // fresh provider evidence. A later operator request continues the retained
131
+ // budget; this owner records why it stopped without claiming settlement.
132
+ const observation = recoveryObservationBudget(worker.snapshot);
110
133
  try {
111
134
  const record = registerManagedProcess(context, { kind: "scope-control", ownerId: key, operationId: `scope-control:${key}:${token}`, ownerPid: process.pid, metadata: { role: "control", scope_id: scopeId, generation } });
112
135
  worker.process_id = record.id;
@@ -125,6 +148,19 @@ export async function serveRuntimeScopeControl(context, scopeId, generation) {
125
148
  for (;;) {
126
149
  if (leaseLost)
127
150
  throw new AppError("runtime_scope_owner_changed", "Scope worker lost its managed lease", 1);
151
+ if (observation.exhausted()) {
152
+ worker.status = "failed";
153
+ worker.error = "recovery_observation_budget_exhausted";
154
+ worker.snapshot = {
155
+ ...(worker.snapshot ?? {}),
156
+ status: "recovery_blocked",
157
+ settled: false,
158
+ pending_reasons: ["recovery_observation_budget_exhausted"],
159
+ error: { code: "recovery_observation_budget_exhausted", message: "Scope recovery observation exceeded its 120 second budget" },
160
+ recovery_observation: observation.state(),
161
+ };
162
+ return { scope_id: scopeId, generation, status: "recovery_blocked", settled: false, error: worker.error };
163
+ }
128
164
  const status = runtimeBudgetStatus(context, scopeId);
129
165
  const release = status.release;
130
166
  if (release?.current === true && release.generation === generation) {
@@ -137,9 +173,16 @@ export async function serveRuntimeScopeControl(context, scopeId, generation) {
137
173
  break;
138
174
  }
139
175
  const resume = status.resume;
140
- worker.snapshot = await prepareRuntimeScopeResume(context, scopeId, generation, resume?.generation === generation ? resume.request_id : undefined) ?? await controlRuntimeScopeRuns(context, scopeId, worker.request_id, worker.mode);
176
+ const outcome = await withRecoveryObservation(observation, async () => (worker.reconcile_id ? null : await prepareRuntimeScopeResume(context, scopeId, generation, resume?.generation === generation ? resume.request_id : undefined))
177
+ ?? await controlRuntimeScopeRuns(context, scopeId, worker.request_id, worker.mode, worker.reconcile_id));
178
+ observation.exhausted();
179
+ worker.snapshot = { ...outcome, recovery_observation: observation.state() };
141
180
  db.run("UPDATE managed_resources SET metadata_json = ?, updated_at = ? WHERE resource_kind = 'scope-control-worker' AND resource_key = ? AND lease_token = ?", [JSON.stringify(worker), context.now(), key, token]);
142
- await delay(1_000);
181
+ if (worker.snapshot.drain?.physical_settled === true) {
182
+ worker.status = "completed";
183
+ break;
184
+ }
185
+ await delay(observation.nextDelay());
143
186
  }
144
187
  return { scope_id: scopeId, generation, status: worker.status };
145
188
  }
@@ -158,6 +201,8 @@ export async function serveRuntimeScopeControl(context, scopeId, generation) {
158
201
  finally {
159
202
  if (heartbeat)
160
203
  clearInterval(heartbeat);
204
+ observation.exhausted();
205
+ worker.snapshot = { ...(worker.snapshot ?? {}), recovery_observation: observation.state() };
161
206
  db.run("UPDATE managed_resources SET metadata_json = ?, updated_at = ? WHERE resource_kind = 'scope-control-worker' AND resource_key = ? AND lease_token = ?", [JSON.stringify(worker), context.now(), key, token]);
162
207
  if (worker.process_id && worker.process_lease_token)
163
208
  finishManagedProcess(context, { id: worker.process_id, leaseToken: worker.process_lease_token, state: worker.status === "failed" ? "failed" : "stopped", reason: "scope_owner_exited" });
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@deksden-com/dd-flow-cli",
3
- "version": "0.9.0-beta.56",
3
+ "version": "0.9.0-beta.63",
4
4
  "description": "Mechanical runtime CLI for dd-flow workflows.",
5
5
  "type": "module",
6
6
  "bin": {