@deksden-com/dd-flow-cli 0.9.0-beta.123 → 0.9.0-beta.124

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (41) hide show
  1. package/CHANGELOG.md +10 -0
  2. package/README.md +23 -0
  3. package/dist/build-info.json +4 -4
  4. package/dist/cli/run-cli.js +1 -1
  5. package/dist/domain/stage-catalog.js +1 -1
  6. package/dist/harness-runtime/lib/codex-capacity-policy.d.mts +10 -0
  7. package/dist/harness-runtime/lib/codex-capacity-policy.mjs +89 -0
  8. package/dist/harness-runtime/lib/daemon-operations.mjs +100 -3
  9. package/dist/harness-runtime/lib/dd-agy-daemon.mjs +66 -16
  10. package/dist/harness-runtime/lib/dd-agy.mjs +13 -3
  11. package/dist/harness-runtime/lib/dd-codex-daemon.mjs +39 -10
  12. package/dist/harness-runtime/lib/dd-codex.mjs +11 -25
  13. package/dist/harness-runtime/lib/dd-droid-daemon.mjs +19 -12
  14. package/dist/harness-runtime/lib/dd-grok-daemon.mjs +42 -10
  15. package/dist/harness-runtime/lib/dd-opencode-daemon.mjs +57 -14
  16. package/dist/harness-runtime/lib/dd-zcode-daemon.mjs +25 -23
  17. package/dist/harness-runtime/lib/dd-zcode.mjs +9 -10
  18. package/dist/harness-runtime/lib/driver-recovery.mjs +12 -2
  19. package/dist/harness-runtime/lib/managed-daemon.d.mts +2 -0
  20. package/dist/harness-runtime/lib/managed-daemon.mjs +100 -32
  21. package/dist/schemas/flow-run.schema.json +1 -0
  22. package/dist/schemas/plan-review-decision.schema.json +3 -3
  23. package/dist/services/code-checks.js +14 -18
  24. package/dist/services/codex-capacity.js +149 -48
  25. package/dist/services/controller-fanout.js +1 -0
  26. package/dist/services/external-work-launch.js +10 -3
  27. package/dist/services/harness-adapter.js +21 -15
  28. package/dist/services/managed-processes.js +14 -6
  29. package/dist/services/merge-server.js +10 -4
  30. package/dist/services/review-deferral.js +23 -0
  31. package/dist/services/run-controller-adapter.js +8 -5
  32. package/dist/services/run-controller-recovery.js +17 -0
  33. package/dist/services/run-controller.js +42 -14
  34. package/dist/services/run-recovery-runtime.js +49 -3
  35. package/dist/services/runs.js +1 -1
  36. package/dist/services/verification-matrix.js +30 -1
  37. package/dist/services/vnext-code-review.js +16 -4
  38. package/dist/services/vnext-merge.js +20 -2
  39. package/dist/services/vnext-plan-review.js +81 -18
  40. package/dist/services/work-registry.js +9 -2
  41. package/package.json +1 -1
@@ -1,16 +1,57 @@
1
1
  import { setTimeout as delay } from "node:timers/promises";
2
2
  import crypto from "node:crypto";
3
- import { terminalCodexOverload } from "../harness-runtime/lib/dd-codex.mjs";
3
+ import { CAPACITY_POLICY, capacityBackoff, nativeItemsSettled, normalizeCodexFailure, overloadBurst, parseCapacityOperation, refusalObservation, terminalCodexOverload } from "../harness-runtime/lib/codex-capacity-policy.mjs";
4
4
  import { AppError } from "../shared/errors.js";
5
5
  /** Permit acquisition must still belong to the original task, not merely an
6
6
  * open RUN. Controls/inspection and ordinary successor prompts are unaffected. */
7
- export function assertCapacityOwnerAdmission(context, record, operationId) {
7
+ export function assertCapacityOwnerAdmission(context, record, operationId, binding) {
8
8
  if (!operationId || !record.project_id || !record.run_id)
9
9
  return;
10
- const root = operationId.replace(/:capacity:[12]:session\.prompt$/, "");
11
- if (root === operationId && !/:child:[a-f0-9]+:followup:[12]$/.test(root))
10
+ const root = parseCapacityOperation(operationId).root;
11
+ const childOrdinal = /:child:[a-f0-9]+:followup:(\d+)$/.exec(root)?.[1];
12
+ if (childOrdinal && (!Number.isSafeInteger(Number(childOrdinal)) || Number(childOrdinal) < 1 || String(Number(childOrdinal)) !== childOrdinal))
13
+ throw new AppError("capacity_continuation_conflict", "Invalid native child continuation ordinal", 1);
14
+ if (root === operationId && !childOrdinal)
12
15
  return;
13
16
  const reject = () => { throw new AppError("capacity_owner_boundary_changed", "Queued continuation no longer belongs to an unfinished task; reconcile its durable state", 1, { operation_id: operationId, work_id: record.work_id, run_id: record.run_id }); };
17
+ const metadata = JSON.parse(record.metadata_json);
18
+ const hasTable = (name) => Boolean(context.db.get("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?", [name]));
19
+ const parsed = parseCapacityOperation(operationId);
20
+ if (parsed.ordinal > 0 && !["judge", "probe"].includes(metadata.role ?? "")) {
21
+ if (!binding?.nativeSessionId || !binding.promptSha256)
22
+ reject();
23
+ let intent;
24
+ const predecessor = parsed.ordinal === 1 ? root : `${root}:capacity:${parsed.ordinal - 1}:session.prompt`;
25
+ if (hasTable("run_controller_operations")) {
26
+ const saved = context.db.get("SELECT receipt_json FROM run_controller_operations WHERE operation_id = ? AND controller_id = ? AND generation = ? AND status = 'failed'", [predecessor, record.owner_id, metadata.run_generation ?? 0]);
27
+ if (saved)
28
+ intent = JSON.parse(saved.receipt_json).capacity_intent;
29
+ }
30
+ if (!intent && record.work_id)
31
+ for (const table of ["work_launches", "merge_requests"]) {
32
+ if (!hasTable(table))
33
+ continue;
34
+ const rows = table === "work_launches"
35
+ ? context.db.all("SELECT l.adapter_receipt_json FROM work_launches l JOIN works w ON w.work_id = l.work_id WHERE l.work_id = ? AND w.project_id = ? AND w.run_id = ?", [record.work_id, record.project_id, record.run_id])
36
+ : context.db.all("SELECT adapter_receipt_json FROM merge_requests WHERE executor_work_id = ? AND project_id = ? AND run_id = ?", [record.work_id, record.project_id, record.run_id]);
37
+ for (const saved of rows) {
38
+ const retained = JSON.parse(saved.adapter_receipt_json ?? "null");
39
+ intent = retained?.capacity_intents?.find((item) => item.operation_id === operationId);
40
+ if (intent)
41
+ break;
42
+ }
43
+ if (intent)
44
+ break;
45
+ }
46
+ if (!intent || intent.policy !== CAPACITY_POLICY || intent.root_operation_id !== root || intent.ordinal !== parsed.ordinal
47
+ || intent.predecessor !== predecessor || intent.operation_id !== operationId || intent.refusal?.turn_id !== intent.turn_id
48
+ || intent.refusal?.ordinal !== parsed.ordinal - 1 || intent.refusal?.session_id !== intent.session_id
49
+ || !Number.isFinite(Date.parse(intent.not_before)) || Date.parse(intent.not_before) > Date.parse(context.now())
50
+ || intent.deadline !== null && (typeof intent.deadline !== "number" || intent.deadline <= Date.parse(context.now()))
51
+ || binding?.nativeSessionId !== undefined && intent.session_id !== binding.nativeSessionId
52
+ || binding?.promptSha256 !== undefined && intent.prompt_sha256 !== binding.promptSha256)
53
+ reject();
54
+ }
14
55
  if (context.db.get("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'run_recovery_bindings'")) {
15
56
  if (context.db.get("SELECT 1 FROM run_recovery_bindings b JOIN run_recovery_guards g ON g.recovery_id = b.recovery_id WHERE g.project_id = ? AND g.run_id = ? AND g.status = 'recovered' AND b.status = 'accepted' AND json_extract(b.runtime_dispatch_json, '$.operation_id') = ?", [record.project_id, record.run_id, root]))
16
57
  reject();
@@ -22,7 +63,7 @@ export function assertCapacityOwnerAdmission(context, record, operationId) {
22
63
  reject();
23
64
  // MERGE owns the Stage/request boundary: finishing its Work does not
24
65
  // necessarily commit stage finish yet. External workers own only Work.
25
- const mergeOwner = JSON.parse(record.metadata_json).role === "merge";
66
+ const mergeOwner = metadata.role === "merge";
26
67
  if (!work || !(merge && mergeOwner ? ["created", "running", "completed"] : ["created", "running"]).includes(work.status))
27
68
  reject();
28
69
  }
@@ -37,30 +78,56 @@ export function assertCapacityOwnerAdmission(context, record, operationId) {
37
78
  if (state.stage && (index.current_stage && index.current_stage !== state.stage || index.stage_runs?.some(stage => stage.stage === state.stage && ["done", "skipped"].includes(stage.status))))
38
79
  reject();
39
80
  const child = Object.entries(state.child_capacity ?? {}).find(([, intent]) => intent.operation_id === root)?.[0];
81
+ if (childOrdinal && !child)
82
+ reject();
40
83
  if (child && context.db.get("SELECT 1 FROM work_sessions ws JOIN works w ON w.work_id = ws.work_id JOIN sessions s ON s.session_id = ws.session_id WHERE w.project_id = ? AND w.run_id = ? AND s.provider_session_id = ? AND w.status <> 'running'", [record.project_id, record.run_id, child]))
41
84
  reject();
42
85
  }
43
86
  export function capacityDelay(error, ordinal, now = Date.now()) {
44
- const native = error?.details?.provider_error;
45
- const headers = native?.headers;
46
- const header = headers && Object.entries(headers).find(([key]) => key.toLowerCase() === "retry-after")?.[1];
47
- const value = header ?? native?.retryAfter ?? native?.retry_after;
48
- const seconds = typeof value === "number" || typeof value === "string" && /^\d+(\.\d+)?$/.test(value.trim()) ? Number(value) * 1000 : typeof value === "string" ? Date.parse(value) - now : NaN;
49
- return Math.max(ordinal === 1 ? 5_000 : 15_000, Number.isFinite(seconds) ? Math.max(0, seconds) : 0);
87
+ return capacityBackoff(error, ordinal, now);
50
88
  }
51
89
  /** Continue a confirmed terminal Turn, never replay its completed effects. */
52
90
  export async function continueCodexCapacity(input) {
53
91
  let { error, operationId } = input;
54
- const history = input.retainedIntents?.() ?? [];
55
- const rootId = input.operationId.replace(/:capacity:[12]:session\.prompt$/, "");
92
+ const history = [...(input.retainedIntents?.() ?? [])];
93
+ const parsed = parseCapacityOperation(input.operationId);
94
+ const rootId = parsed.root;
56
95
  const text = input.continuationText ?? "Continue the same assigned task in this Session. Check current durable RUN, Work and Stage state first. Never repeat an observed lifecycle command, accepted HITL answer, check or merge apply; use only an exact issued successor. Stop at the assigned boundary.";
57
96
  const promptSha = crypto.createHash("sha256").update(text).digest("hex");
58
- let ordinal = Number(/:capacity:([12]):session\.prompt$/.exec(operationId)?.[1] ?? 0) + 1;
97
+ let ordinal = parsed.ordinal + 1;
98
+ const conflict = (message) => { throw new AppError("capacity_continuation_conflict", message, 1, { operation_id: operationId, root_operation_id: rootId, session_id: input.sessionId }); };
99
+ const refusals = [...(input.retainedFailures?.() ?? [])];
100
+ if (parsed.ordinal > history.length)
101
+ conflict("Continuation has no complete retained predecessor chain");
102
+ for (let index = 0; index < history.length; index++) {
103
+ const intent = history[index];
104
+ const expected = index + 1;
105
+ if (intent.ordinal !== expected || intent.operation_id !== `${rootId}:capacity:${expected}:session.prompt`
106
+ || intent.predecessor !== (expected === 1 ? rootId : `${rootId}:capacity:${expected - 1}:session.prompt`)
107
+ || intent.prompt_sha256 !== promptSha || intent.policy !== CAPACITY_POLICY || intent.root_operation_id !== rootId || intent.session_id !== input.sessionId
108
+ || intent.deadline !== (input.deadline ?? null) || !intent.refusal || intent.refusal.ordinal !== expected - 1 || intent.refusal.turn_id !== intent.turn_id
109
+ || !Number.isFinite(Date.parse(intent.not_before)))
110
+ conflict("Retained capacity chain is incomplete or belongs to another policy/task");
111
+ const previous = refusals.find(item => item.ordinal === intent.refusal.ordinal);
112
+ if (previous && JSON.stringify(previous) !== JSON.stringify(intent.refusal))
113
+ conflict("Retained failure observation changed");
114
+ if (!previous)
115
+ refusals.push(intent.refusal);
116
+ }
117
+ refusals.sort((a, b) => a.ordinal - b.ordinal);
118
+ for (let index = 0; index < refusals.length; index++) {
119
+ const refusal = refusals[index];
120
+ if (refusal.ordinal !== index || refusal.policy !== CAPACITY_POLICY || refusal.session_id !== input.sessionId || !refusal.turn_id
121
+ || refusals.slice(0, index).some(item => item.turn_id === refusal.turn_id) || !Number.isFinite(Date.parse(refusal.first_observed_at)))
122
+ conflict("Retained refusal chain is invalid");
123
+ }
59
124
  // Reconcile a retained dispatch before preparing another. Native absence is
60
125
  // the only safe way to reuse a prepared intent; a lost reply is not absence.
61
126
  for (const intent of history) {
62
- if (intent.predecessor !== operationId || intent.ordinal !== ordinal || intent.operation_id !== `${rootId}:capacity:${ordinal}:session.prompt`)
127
+ if (intent.ordinal < ordinal)
63
128
  continue;
129
+ if (intent.predecessor !== operationId || intent.ordinal !== ordinal || intent.operation_id !== `${rootId}:capacity:${ordinal}:session.prompt`)
130
+ conflict("Retained successor skips its predecessor");
64
131
  if (intent.prompt_sha256 !== promptSha || !intent.turn_id || !Number.isFinite(Date.parse(intent.not_before)))
65
132
  throw error;
66
133
  let observed;
@@ -72,7 +139,7 @@ export async function continueCodexCapacity(input) {
72
139
  for (let depth = 0; depth < 4 && code?.code === "harness_adapter_failed"; depth++)
73
140
  code = code.details?.cause;
74
141
  if (code?.code !== "operation_not_found")
75
- throw error;
142
+ throw cause;
76
143
  break;
77
144
  }
78
145
  if (observed.operation_id !== intent.operation_id || observed.session_id !== input.sessionId || observed.operation !== "session.prompt" || observed.primary_error)
@@ -80,7 +147,12 @@ export async function continueCodexCapacity(input) {
80
147
  if (observed.state === "completed" && observed.result && typeof observed.result === "object") {
81
148
  input.signal?.throwIfAborted();
82
149
  input.assertCurrent();
83
- return observed.result;
150
+ const boundary = await input.decision?.() ?? "continue";
151
+ input.signal?.throwIfAborted();
152
+ input.assertCurrent();
153
+ if (boundary === "reconciliation_blocked")
154
+ throw new AppError("capacity_continuation_blocked", "Retained completion cannot be reconciled with its owner boundary", 1);
155
+ return boundary === "paused" || boundary === "completed" ? { provider_session_id: input.sessionId, capacity_decision: boundary } : observed.result;
84
156
  }
85
157
  if (observed.state !== "failed" || !terminalCodexOverload(observed.error, input.sessionId))
86
158
  throw error;
@@ -95,12 +167,14 @@ export async function continueCodexCapacity(input) {
95
167
  input.signal?.throwIfAborted();
96
168
  input.assertCurrent();
97
169
  if (decision === "reconciliation_blocked")
98
- throw error;
170
+ throw new AppError("capacity_continuation_blocked", "Native refusal cannot be reconciled with its durable owner boundary", 1, { primary_error: error, root_operation_id: rootId, provider_session_id: input.sessionId });
99
171
  if (decision === "continue" && input.deadline !== undefined && Date.now() >= input.deadline)
100
- throw error;
172
+ throw new AppError("capacity_owner_deadline_exceeded", "Continuation exhausted its original owner deadline", 1, { primary_error: error, deadline: input.deadline, root_operation_id: rootId, provider_session_id: input.sessionId });
101
173
  return decision === "continue" ? null : { provider_session_id: input.sessionId, capacity_decision: decision };
102
174
  };
103
- for (; ordinal <= 2; ordinal += 1) {
175
+ for (;; ordinal += 1) {
176
+ if (!Number.isSafeInteger(ordinal))
177
+ conflict("Continuation ordinal overflow");
104
178
  if (!terminalCodexOverload(error, input.sessionId)) {
105
179
  if (ordinal === 1)
106
180
  return null;
@@ -109,17 +183,12 @@ export async function continueCodexCapacity(input) {
109
183
  const completed = await decide();
110
184
  if (completed)
111
185
  return completed;
112
- let prior;
113
- try {
114
- prior = await input.inspectOperation(operationId);
115
- }
116
- catch {
117
- if (ordinal === 1)
118
- return null;
119
- throw error;
120
- }
186
+ const prior = await input.inspectOperation(operationId);
121
187
  const failure = prior.error;
122
- const turnId = failure?.details?.turn_id;
188
+ const confirmed = normalizeCodexFailure(failure, input.sessionId);
189
+ const turnId = confirmed?.turn_id;
190
+ input.signal?.throwIfAborted();
191
+ input.assertCurrent();
123
192
  if (prior.operation_id !== operationId || prior.operation !== "session.prompt" || prior.session_id !== input.sessionId
124
193
  || prior.state !== "failed" || prior.primary_error || !turnId || !terminalCodexOverload(failure, input.sessionId, turnId)) {
125
194
  if (ordinal === 1)
@@ -127,9 +196,15 @@ export async function continueCodexCapacity(input) {
127
196
  throw error;
128
197
  }
129
198
  const settled = async () => {
199
+ input.signal?.throwIfAborted();
200
+ input.assertCurrent();
130
201
  const seen = await input.inspectSession(turnId);
131
- const items = seen.terminal_turn_id === turnId && seen.terminal_status === "failed" ? seen.native_turn_items : failure.details?.native_turn_items;
132
- if (items?.observed !== true || typeof items.possible_effects !== "boolean" || items.pending === true || items.possible_effects === true && items.pending !== false)
202
+ input.signal?.throwIfAborted();
203
+ input.assertCurrent();
204
+ if (seen.terminal_turn_id !== undefined && seen.terminal_turn_id !== turnId)
205
+ return false;
206
+ const items = seen.terminal_turn_id === turnId && seen.terminal_status === "failed" ? seen.native_turn_items : confirmed?.native_turn_items;
207
+ if (!nativeItemsSettled(items))
133
208
  return false;
134
209
  const settlement = seen.settlement;
135
210
  const ids = Array.isArray(seen.tree_session_ids) ? seen.tree_session_ids : [];
@@ -142,33 +217,60 @@ export async function continueCodexCapacity(input) {
142
217
  return seen.provider_session_id === input.sessionId && seen.settled === true && treeProven
143
218
  && (!settlement || settlement.state === "not_required" || settlement.state === "settled" && settlement.operation_ids?.includes(operationId) === true);
144
219
  };
145
- try {
146
- if (!await settled()) {
147
- if (ordinal === 1)
148
- return null;
149
- throw error;
150
- }
151
- }
152
- catch {
220
+ if (!await settled()) {
153
221
  if (ordinal === 1)
154
222
  return null;
155
223
  throw error;
156
224
  }
225
+ const native = normalizeCodexFailure(failure, input.sessionId, turnId);
226
+ let refusal = refusals.find(item => item.ordinal === ordinal - 1);
227
+ if (refusal && (refusal.turn_id !== turnId || refusal.session_id !== input.sessionId || refusal.policy !== CAPACITY_POLICY))
228
+ conflict("Failure ordinal identifies a different native Turn");
229
+ if (refusal) {
230
+ const expected = refusalObservation(failure, ordinal - 1, refusal.first_observed_at);
231
+ if (refusal.native_terminal_at !== expected.native_terminal_at || refusal.native_terminal_basis !== expected.native_terminal_basis)
232
+ conflict("Retained refusal time differs from its native receipt");
233
+ }
234
+ if (!refusal) {
235
+ if (refusals.some(item => item.turn_id === turnId))
236
+ conflict("A cached native Turn cannot be a second refusal");
237
+ // A cached legacy failure cannot be assigned a new time during reattach.
238
+ if (ordinal > 1 && parsed.ordinal > 0 && !native.observed_at)
239
+ conflict("Retained continuation has no original failure time");
240
+ const observedAt = typeof native.observed_at === "string" && Number.isFinite(Date.parse(native.observed_at)) ? native.observed_at : new Date().toISOString();
241
+ refusal = refusalObservation(failure, ordinal - 1, observedAt);
242
+ input.retainFailure?.(operationId, refusal);
243
+ refusals.push(refusal);
244
+ }
245
+ const burst = overloadBurst(refusals);
246
+ if (burst)
247
+ throw new AppError("provider_overload_burst", "Two consecutive continuation Turns were overloaded within two minutes", 1, { ...burst, root_operation_id: rootId, provider_session_id: input.sessionId, continuation_count: ordinal - 1, primary_error: failure });
157
248
  const nextId = `${rootId}:capacity:${ordinal}:session.prompt`;
158
249
  const existing = history.find(item => item.operation_id === nextId);
159
250
  if (existing && (existing.predecessor !== operationId || existing.turn_id !== turnId || existing.prompt_sha256 !== promptSha || existing.ordinal !== ordinal))
160
251
  throw error;
161
252
  const wait = existing ? Math.max(0, Date.parse(existing.not_before) - Date.now()) : capacityDelay(failure, ordinal);
162
- if (!Number.isFinite(wait) || wait > 30_000 || input.deadline !== undefined && Date.now() + wait >= input.deadline)
163
- throw error;
164
- if (!existing)
165
- input.retainIntent({ predecessor: operationId, operation_id: nextId, turn_id: turnId, ordinal, prompt_sha256: promptSha, not_before: new Date(Date.now() + wait).toISOString() });
166
- await delay(wait, undefined, { signal: input.signal });
253
+ if (!Number.isFinite(wait) || Date.now() + wait > 8.64e15 || input.deadline !== undefined && Date.now() + wait >= input.deadline)
254
+ throw new AppError("capacity_owner_deadline_exceeded", "Continuation wait exceeds its original owner deadline", 1, { primary_error: failure, deadline: input.deadline ?? null });
255
+ if (!existing) {
256
+ const intent = { predecessor: operationId, operation_id: nextId, turn_id: turnId, ordinal, prompt_sha256: promptSha, not_before: new Date(Date.now() + wait).toISOString(), policy: CAPACITY_POLICY, root_operation_id: rootId, session_id: input.sessionId, deadline: input.deadline ?? null, refusal };
257
+ input.retainIntent(intent);
258
+ history.push(intent);
259
+ }
260
+ // Node timers truncate large delays. Bounded slices preserve cancellation
261
+ // and owner checks; monotonic elapsed time does not renew a Retry-After.
262
+ const until = performance.now() + wait;
263
+ while (performance.now() < until) {
264
+ await delay(Math.min(250, Math.max(0, until - performance.now())), undefined, { signal: input.signal });
265
+ const boundary = await decide();
266
+ if (boundary)
267
+ return boundary;
268
+ }
167
269
  const afterWait = await decide();
168
270
  if (afterWait)
169
271
  return afterWait;
170
- if (!await settled().catch(() => false))
171
- throw error;
272
+ if (!await settled())
273
+ throw new AppError("capacity_continuation_blocked", "Native predecessor settlement changed during capacity wait", 1, { primary_error: failure });
172
274
  const beforeDispatch = await decide();
173
275
  if (beforeDispatch)
174
276
  return beforeDispatch;
@@ -186,5 +288,4 @@ export async function continueCodexCapacity(input) {
186
288
  operationId = nextId;
187
289
  }
188
290
  }
189
- throw error;
190
291
  }
@@ -138,6 +138,7 @@ export function controllerNativeChildren(receipt, rootSessionId) {
138
138
  const settled = rootChild && status === "unknown" && settledTree.has(id) ? "settled_by_root" : status;
139
139
  candidates.push({ session_id: id, parent_session_id: parent ?? "", status: settled, source, ...(rootChild ? {} : { provenance: "parent_mismatch" }),
140
140
  ...(text(value, ["terminal_turn_id"]) ? { terminal_turn_id: text(value, ["terminal_turn_id"]) } : {}),
141
+ ...(text(value, ["terminal_observed_at"]) ? { terminal_observed_at: text(value, ["terminal_observed_at"]) } : {}),
141
142
  ...(value.provider_error !== undefined ? { provider_error: value.provider_error } : {}),
142
143
  ...(value.native_turn_items && typeof value.native_turn_items === "object" ? { native_turn_items: value.native_turn_items } : {}) });
143
144
  };
@@ -112,6 +112,7 @@ export async function launchExternalWork(context, input) {
112
112
  fs.writeFileSync(promptFile, `Start the assigned Work now. Your first tool call must be this exact standalone command:\n\n${workStartCommand(runtimeContext, work)}\n\nUse the packet returned by that command. Do not claim another Work or run this command more than once.\n\n${lifecycleRetryInstruction}${isolatedWorkspace ? `\n\nThe review copy has no .git. Read the captured status/diff or gitless note in ${path.join(isolatedWorkspace, ".dd-flow", "review-evidence")}; do not run Git inside the copy.\n` : "\n"}`);
113
113
  const promptOperationId = `${operationId}:session.prompt`;
114
114
  const promptDeadline = Date.now() + 45 * 60_000;
115
+ context.db.run("UPDATE work_launches SET adapter_receipt_json = ?, updated_at = ? WHERE operation_id = ? AND status = 'launching'", [JSON.stringify({ create: created, prompt_deadline: promptDeadline }), context.now(), operationId]);
115
116
  let prompted;
116
117
  try {
117
118
  prompted = await call(["session", "prompt", "--session-id", sessionId, "--prompt-file", promptFile, ...common], 45 * 60_000);
@@ -121,10 +122,18 @@ export async function launchExternalWork(context, input) {
121
122
  throw error;
122
123
  const retained = context.db.get("SELECT adapter_receipt_json FROM work_launches WHERE operation_id = ?", [operationId]);
123
124
  const capacityIntents = retained?.adapter_receipt_json ? JSON.parse(retained.adapter_receipt_json).capacity_intents ?? [] : [];
125
+ const capacityFailures = retained?.adapter_receipt_json ? JSON.parse(retained.adapter_receipt_json).capacity_failures ?? [] : [];
126
+ const retainCapacity = () => {
127
+ const changed = context.db.run("UPDATE work_launches SET adapter_receipt_json = ?, updated_at = ? WHERE operation_id = ? AND status = 'launching'", [JSON.stringify({ create: created, prompt_deadline: promptDeadline, capacity_intents: capacityIntents, capacity_failures: capacityFailures }), context.now(), operationId]);
128
+ if (changed.changes !== 1)
129
+ throw new AppError("capacity_continuation_blocked", "External Work launch owner changed", 1);
130
+ };
124
131
  const resumed = await continueCodexCapacity({ error, sessionId, operationId: promptOperationId, allowSettledDescendants: true,
125
132
  ...(input.signal ? { signal: input.signal } : {}), deadline: promptDeadline,
126
133
  continuationText: `Continue only Work ${work.work_id} in RUN ${work.run_id}, Stage ${stage ?? "current"}, in the same Session. Inspect its current durable state and the retained Work packet. Never repeat Work start or a completed tool, accepted answer, check or lifecycle command. Use only the current exact issued successor and finish at the original Work boundary.`,
127
134
  retainedIntents: () => capacityIntents,
135
+ retainedFailures: () => capacityFailures,
136
+ retainFailure: (_id, refusal) => { capacityFailures.push(refusal); retainCapacity(); },
128
137
  inspectOperation: id => call(["daemon", "operation", "--state-dir", root, "--operation-id", id, "--json"], 30_000, `${id}:capacity-observe`),
129
138
  inspectSession: turnId => call(["session", "inspect", "--session-id", sessionId, ...(turnId ? ["--turn-id", turnId] : []), ...common], 30_000, `${operationId}:capacity-inspect`),
130
139
  assertCurrent: () => {
@@ -141,9 +150,7 @@ export async function launchExternalWork(context, input) {
141
150
  },
142
151
  retainIntent: intent => {
143
152
  capacityIntents.push(intent);
144
- const changed = context.db.run("UPDATE work_launches SET adapter_receipt_json = ?, updated_at = ? WHERE operation_id = ? AND status = 'launching'", [JSON.stringify({ create: created, capacity_intent: intent, capacity_intents: capacityIntents }), context.now(), operationId]);
145
- if (changed.changes !== 1)
146
- throw new AppError("capacity_continuation_blocked", "External Work launch owner changed", 1);
153
+ retainCapacity();
147
154
  },
148
155
  prompt: (id, text, ordinal) => {
149
156
  const file = path.join(root, `capacity-${ordinal}.md`);
@@ -2,7 +2,8 @@ import { spawn } from "node:child_process";
2
2
  import fs from "node:fs";
3
3
  import path from "node:path";
4
4
  import { fileURLToPath, pathToFileURL } from "node:url";
5
- import { AppError } from "../shared/errors.js";
5
+ import { AppError, errorRecord } from "../shared/errors.js";
6
+ import { stopProcessGroup } from "../harness-runtime/lib/managed-daemon.mjs";
6
7
  import { recoveryRpcTimeout } from "./recovery-observation-budget.js";
7
8
  const maxAdapterOutputBytes = 1_000_000;
8
9
  /** Native hooks need a real executable, not a PATH-dependent shell expression. */
@@ -64,18 +65,26 @@ export function runHarnessAdapter(input) {
64
65
  let outputTooLarge = false;
65
66
  let timedOut = false;
66
67
  let aborted = false;
67
- let killTimer;
68
+ let cleanup;
68
69
  let settled = false;
69
70
  let callbackError;
70
71
  let spawnError;
71
- const terminate = (signal) => { try {
72
- if (detached && child.pid)
73
- process.kill(-child.pid, signal);
74
- else
75
- child.kill(signal);
76
- }
77
- catch { /* The process already settled. */ } };
78
- const stopTransport = () => { terminate("SIGTERM"); killTimer ??= setTimeout(() => terminate("SIGKILL"), 5_000); };
72
+ const stopTransport = () => {
73
+ cleanup ??= stopProcessGroup(child, 5_000, { detached }).catch(error => {
74
+ if (timer)
75
+ clearTimeout(timer);
76
+ if (heartbeat)
77
+ clearInterval(heartbeat);
78
+ input.signal?.removeEventListener("abort", abort);
79
+ const details = { cleanup_error: errorRecord(error) };
80
+ if (callbackError instanceof Error) {
81
+ Object.assign(callbackError, { cleanup_error: details.cleanup_error, details: { ...callbackError.details, ...details } });
82
+ reject(callbackError);
83
+ }
84
+ else
85
+ reject(new AppError(outputTooLarge ? "harness_adapter_output_limit" : timedOut ? "harness_adapter_timeout" : aborted ? "harness_adapter_aborted" : "harness_adapter_cleanup_unconfirmed", "Adapter transport cleanup could not be confirmed", 1, details));
86
+ });
87
+ };
79
88
  const deliver = (callback) => {
80
89
  try {
81
90
  const result = callback();
@@ -109,7 +118,7 @@ export function runHarnessAdapter(input) {
109
118
  child.stdout.setEncoding("utf8").on("data", (chunk) => { stdout = append(stdout, chunk); });
110
119
  child.stderr.setEncoding("utf8").on("data", (chunk) => { stderr = append(stderr, chunk); });
111
120
  child.once("error", (error) => { spawnError = error; });
112
- child.once("close", (status, signal) => {
121
+ child.once("close", async (status, signal) => {
113
122
  if (settled)
114
123
  return;
115
124
  settled = true;
@@ -118,10 +127,7 @@ export function runHarnessAdapter(input) {
118
127
  input.signal?.removeEventListener("abort", abort);
119
128
  if (heartbeat)
120
129
  clearInterval(heartbeat);
121
- if (killTimer) {
122
- clearTimeout(killTimer);
123
- terminate("SIGKILL");
124
- }
130
+ await cleanup;
125
131
  const progressFailure = callbackError;
126
132
  deliver(() => evidence({ status, signal, timed_out: timedOut, aborted, stdout, stderr, ...(spawnError ? { error: spawnError.message } : {}) }));
127
133
  if (spawnError)
@@ -4,7 +4,7 @@ import path from "node:path";
4
4
  import { spawnSync } from "node:child_process";
5
5
  import net from "node:net";
6
6
  import { getResourceDatabase } from "../storage/database.js";
7
- import { AppError } from "../shared/errors.js";
7
+ import { AppError, errorRecord } from "../shared/errors.js";
8
8
  export function managedProcessBinding(record, home) {
9
9
  return { resource_home: home, process_id: record.id, registration_sha256: crypto.createHash("sha256").update(JSON.stringify([home, record.id, record.kind, record.owner_id, record.lease_token, record.pid, record.pid_started_at, record.project_id, record.run_id, record.work_id, record.check_id, record.operation_id, record.metadata_json])).digest("hex") };
10
10
  }
@@ -339,14 +339,19 @@ export async function stopManagedProcess(context, input) {
339
339
  return record;
340
340
  db.run("UPDATE managed_processes SET state = 'stopping', updated_at = ? WHERE id = ? AND lease_token = ? AND state IN ('starting','running','orphaned')", [context.now(), record.id, record.lease_token]);
341
341
  if (processTreeIsAlive(record)) {
342
- terminateOwnedProcess(record, "SIGTERM");
342
+ const signalErrors = [];
343
+ const termError = terminateOwnedProcess(record, "SIGTERM");
344
+ if (termError)
345
+ signalErrors.push(errorRecord(termError));
343
346
  await delay(graceMs);
344
- if (input.force !== false && processTreeIsAlive(record)) {
345
- terminateOwnedProcess(record, "SIGKILL");
347
+ if (!termError && input.force !== false && processTreeIsAlive(record)) {
348
+ const killError = terminateOwnedProcess(record, "SIGKILL");
349
+ if (killError)
350
+ signalErrors.push(errorRecord(killError));
346
351
  await delay(250);
347
352
  }
348
353
  if (processTreeIsAlive(record))
349
- throw Object.assign(new Error("Owned process tree could not be safely stopped; resources retained"), { code: "process_tree_not_settled", details: { process_id: record.id } });
354
+ throw Object.assign(new Error("Owned process tree could not be safely stopped; resources retained"), { code: "process_tree_not_settled", details: { process_id: record.id, signal_errors: signalErrors } });
350
355
  }
351
356
  if (!finishManagedProcess(context, { id: record.id, leaseToken: record.lease_token, state: "stopped", reason: input.reason ?? "requested_stop" }))
352
357
  throw Object.assign(new Error("Managed process ownership changed before settlement"), { code: "process_lease_lost" });
@@ -460,7 +465,10 @@ function terminateOwnedProcess(record, signal) {
460
465
  try {
461
466
  process.kill(process.platform === "win32" || group !== record.pid ? record.pid : -group, signal);
462
467
  }
463
- catch { /* inspect again below */ }
468
+ catch (error) {
469
+ if (error.code !== "ESRCH")
470
+ return Object.assign(error, { details: { process_id: record.id, pid: record.pid, pgid: group ?? null, signal, phase: "owned_process_stop" } });
471
+ }
464
472
  }
465
473
  function parseMetadata(value) { try {
466
474
  const parsed = JSON.parse(value);
@@ -106,7 +106,8 @@ async function dispatch(context, input) {
106
106
  const executable = resolveHarnessCommand(context.ddFlowHome, input.profile.harness, "adapter");
107
107
  const common = ["--dd-flow-bin", flowExecutable, "--state-dir", stateDir, "--journal", adapterJournal, "--cwd", input.request.target_workspace, "--provider", input.profile.provider, "--model", input.profile.model, "--reasoning", input.profile.reasoning, "--mode", input.profile.mode, "--permission", input.profile.permission, "--project-root", run.project_root, "--dd-flow-home", context.ddFlowHome, ...harnessRuntimeArguments(context.ddFlowHome, input.profile.harness), "--json"];
108
108
  const retained = { profile: input.profile, executable, owner: runtimeOwner, session_id: null, state_dir: stateDir, create_operation_id: `${input.request.merge_request_id}:${token}:session.create`, prompt_operation_id: `${input.request.merge_request_id}:${token}:session.prompt` };
109
- const retain = () => { context.db.run("UPDATE merge_requests SET adapter_receipt_json = ?, updated_at = ? WHERE merge_request_id = ? AND dispatch_owner = ? AND dispatch_lease_token = ?", [JSON.stringify(retained), context.now(), input.request.merge_request_id, input.serverId, token]); };
109
+ const retain = () => { const changed = context.db.run("UPDATE merge_requests SET adapter_receipt_json = ?, updated_at = ? WHERE merge_request_id = ? AND dispatch_owner = ? AND dispatch_lease_token = ?", [JSON.stringify(retained), context.now(), input.request.merge_request_id, input.serverId, token]); if (changed.changes !== 1)
110
+ throw new AppError("capacity_continuation_blocked", "MERGE dispatch owner changed before durable receipt retention", 1); };
110
111
  retain();
111
112
  const leaseHeartbeat = setInterval(() => { if (guarded(context, input.request) || (recoveryGuard(context, input.request.project_id, input.request.run_id)?.generation ?? 0) !== generation)
112
113
  return; const now = context.now(); context.db.run("UPDATE merge_requests SET dispatch_lease_expires_at = ?, updated_at = ? WHERE merge_request_id = ? AND status IN ('dispatching','active') AND dispatch_owner = ? AND dispatch_lease_token = ?", [new Date(Date.now() + 120_000).toISOString(), now, input.request.merge_request_id, input.serverId, token]); }, 30_000);
@@ -125,6 +126,8 @@ async function dispatch(context, input) {
125
126
  fs.writeFileSync(promptFile, `Start the assigned MERGE stage now. Your first tool call must be this exact standalone command:\n\n${stageCommand}\n\nTrust and follow the complete stage packet it returns. Continue through merge apply and stage finish. Stop only when the stage reaches a terminal result or explicitly requests user/operator action.\n\n${stageLifecycleInstruction}\n`);
126
127
  let prompted;
127
128
  const promptDeadline = Date.now() + 45 * 60_000;
129
+ retained.prompt_deadline = promptDeadline;
130
+ retain();
128
131
  try {
129
132
  prompted = await adapterCall(["session", "prompt", "--session-id", sessionId, "--prompt-file", promptFile, ...common], 45 * 60 * 1000, `MERGE ${input.request.merge_request_id} Session ${sessionId} is still running`);
130
133
  }
@@ -132,10 +135,13 @@ async function dispatch(context, input) {
132
135
  if (runtimeHarness(input.profile) !== "codex-desktop")
133
136
  throw error;
134
137
  const capacityIntents = (retained.capacity_intents ?? []);
138
+ const capacityFailures = (retained.capacity_failures ?? []);
135
139
  const resumed = await continueCodexCapacity({ error, sessionId, operationId: String(retained.prompt_operation_id), allowSettledDescendants: true,
136
140
  deadline: promptDeadline,
137
141
  continuationText: `Continue only MERGE ${input.request.merge_request_id}, Work ${input.request.executor_work_id}, RUN ${input.request.run_id} in this same Session and target ${input.request.target_workspace}. First inspect current durable Stage/Work/MERGE receipts. Never repeat Stage start, accepted answer, completed apply or checks. Follow only the current exact issued successor and stop at the original boundary.`,
138
142
  retainedIntents: () => capacityIntents,
143
+ retainedFailures: () => capacityFailures,
144
+ retainFailure: (_id, refusal) => { capacityFailures.push(refusal); retained.capacity_failures = capacityFailures; retain(); },
139
145
  inspectOperation: id => adapterCall(["daemon", "operation", "--state-dir", stateDir, "--operation-id", id, "--json"], 30_000, undefined, `${id}:capacity-observe`),
140
146
  inspectSession: turnId => adapterCall(["session", "inspect", "--session-id", sessionId, ...(turnId ? ["--turn-id", turnId] : []), ...common], 30_000, undefined, `${input.request.merge_request_id}:${token}:capacity-inspect`),
141
147
  assertCurrent: () => {
@@ -163,7 +169,7 @@ async function dispatch(context, input) {
163
169
  }
164
170
  else
165
171
  fs.writeFileSync(file, text, { flag: "wx", mode: 0o600 });
166
- return adapterCall(["session", "prompt", "--session-id", sessionId, "--prompt-file", file, ...common], Math.max(1, promptDeadline - Date.now()), `MERGE ${input.request.merge_request_id} Session ${sessionId} capacity continuation ${ordinal}/2`, id);
172
+ return adapterCall(["session", "prompt", "--session-id", sessionId, "--prompt-file", file, ...common], Math.max(1, promptDeadline - Date.now()), `MERGE ${input.request.merge_request_id} Session ${sessionId} capacity continuation ${ordinal}`, id);
167
173
  } });
168
174
  if (!resumed)
169
175
  throw error;
@@ -210,6 +216,6 @@ function processAlive(pid) { if (!pid)
210
216
  process.kill(pid, 0);
211
217
  return true;
212
218
  }
213
- catch {
214
- return false;
219
+ catch (error) {
220
+ return error.code !== "ESRCH";
215
221
  } }
@@ -0,0 +1,23 @@
1
+ import crypto from "node:crypto";
2
+ import fs from "node:fs";
3
+ import path from "node:path";
4
+ import { AppError } from "../shared/errors.js";
5
+ import { assertPortableArtifactRef } from "./portable-refs.js";
6
+ /** A deferral names one project-local file, never an arbitrary path. */
7
+ export function reviewDefEvidence(workspaceRoot, defId) {
8
+ try {
9
+ if (!defId || !/^DEF-[A-Za-z0-9][A-Za-z0-9_-]*$/.test(defId))
10
+ throw new Error("invalid DEF identifier");
11
+ const relative = `.memory-bank/defs/${defId}.md`;
12
+ const file = assertPortableArtifactRef(relative, { workspaceRoot, runHome: workspaceRoot, runId: "" });
13
+ const directory = fs.realpathSync(path.join(workspaceRoot, ".memory-bank", "defs"));
14
+ const root = fs.realpathSync(workspaceRoot);
15
+ const directoryRelative = path.relative(root, directory);
16
+ if (directoryRelative === ".." || directoryRelative.startsWith(`..${path.sep}`) || path.isAbsolute(directoryRelative) || path.dirname(file) !== directory || !fs.statSync(file).isFile())
17
+ throw new Error("DEF must be a regular file in the project defs directory");
18
+ return { def_id: defId, path: relative, sha256: crypto.createHash("sha256").update(fs.readFileSync(file)).digest("hex") };
19
+ }
20
+ catch (error) {
21
+ throw new AppError("deferral_invalid", "A deferral requires a readable project-local named DEF", 2, { def_id: defId ?? null, cause: error instanceof Error ? error.message : String(error) });
22
+ }
23
+ }
@@ -5,6 +5,7 @@ import { AppError } from "../shared/errors.js";
5
5
  import { adapterSessionId, runHarnessAdapter } from "./harness-adapter.js";
6
6
  import { runtimeHarness } from "./execution-policy.js";
7
7
  import { appendEvent, digest, digestBytes, errorRecord } from "./run-controller-state.js";
8
+ import { CAPACITY_POLICY, parseCapacityOperation } from "../harness-runtime/lib/codex-capacity-policy.mjs";
8
9
  /** A paused RUN admits only its controller's exact accepted answer Turn. */
9
10
  export function controllerHitlAdmission(context, record, input) {
10
11
  if (input.operation !== "session.prompt" || !input.operationId || !input.promptSha256)
@@ -15,13 +16,13 @@ export function controllerHitlAdmission(context, record, input) {
15
16
  if (!row || row.generation !== JSON.parse(record.metadata_json).run_generation)
16
17
  return false;
17
18
  const suffix = `:prompt:${row.generation}`;
18
- const successor = /^(.*):capacity:([12]):session\.prompt$/.exec(input.operationId);
19
- const baseId = successor ? successor[1] : input.operationId;
19
+ const successor = parseCapacityOperation(input.operationId);
20
+ const baseId = successor.root;
20
21
  if (!baseId.endsWith(suffix))
21
22
  return false;
22
- if (successor) {
23
- const ordinal = Number(successor[2]);
24
- const predecessorId = ordinal === 1 ? baseId : `${baseId}:capacity:1:session.prompt`;
23
+ if (successor.ordinal > 0) {
24
+ const ordinal = successor.ordinal;
25
+ const predecessorId = ordinal === 1 ? baseId : `${baseId}:capacity:${ordinal - 1}:session.prompt`;
25
26
  const predecessor = context.db.get("SELECT status, receipt_json FROM run_controller_operations WHERE operation_id = ? AND controller_id = ? AND generation = ?", [predecessorId, row.controller_id, row.generation]);
26
27
  if (predecessor?.status !== "failed" || !predecessor.receipt_json)
27
28
  return false;
@@ -34,6 +35,8 @@ export function controllerHitlAdmission(context, record, input) {
34
35
  }
35
36
  if (retained.capacity_intent?.predecessor !== predecessorId || retained.capacity_intent.operation_id !== input.operationId || retained.capacity_intent.ordinal !== ordinal)
36
37
  return false;
38
+ if (retained.capacity_intent.policy !== CAPACITY_POLICY)
39
+ return false;
37
40
  if (!retained.capacity_intent.not_before || !Number.isFinite(Date.parse(retained.capacity_intent.not_before)) || Date.parse(retained.capacity_intent.not_before) > Date.parse(context.now()) || retained.capacity_intent.prompt_sha256 !== input.promptSha256)
38
41
  return false;
39
42
  }
@@ -221,6 +221,23 @@ export async function resumeControllerSession(context, row, manifest, state, ass
221
221
  const binding = context.db.get("SELECT runtime_dispatch_json FROM run_recovery_bindings WHERE recovery_id = ?", [recovery.recovery_id]);
222
222
  return binding?.runtime_dispatch_json ? JSON.parse(binding.runtime_dispatch_json).capacity_intents ?? [] : [];
223
223
  },
224
+ retainedFailures: () => {
225
+ const binding = context.db.get("SELECT runtime_dispatch_json FROM run_recovery_bindings WHERE recovery_id = ?", [recovery.recovery_id]);
226
+ return binding?.runtime_dispatch_json ? JSON.parse(binding.runtime_dispatch_json).capacity_failures ?? [] : [];
227
+ },
228
+ retainFailure: (_id, refusal) => {
229
+ assertRecoveryOwner();
230
+ context.db.writeTransaction(() => {
231
+ const binding = context.db.get("SELECT runtime_dispatch_json FROM run_recovery_bindings WHERE recovery_id = ? AND daemon_id = ? AND status = 'prepared'", [recovery.recovery_id, daemonId]);
232
+ if (!binding)
233
+ throw new AppError("recovery_capacity_owner_changed", "Recovery binding changed before refusal retention", 1);
234
+ const retained = JSON.parse(binding.runtime_dispatch_json);
235
+ retained.capacity_failures = [...(retained.capacity_failures ?? []), refusal];
236
+ const changed = context.db.run("UPDATE run_recovery_bindings SET runtime_dispatch_json = ? WHERE recovery_id = ? AND daemon_id = ? AND status = 'prepared' AND runtime_dispatch_json = ?", [JSON.stringify(retained), recovery.recovery_id, daemonId, binding.runtime_dispatch_json]);
237
+ if (changed.changes !== 1)
238
+ throw new AppError("recovery_capacity_owner_changed", "Recovery binding changed during refusal retention", 1);
239
+ });
240
+ },
224
241
  inspectOperation: id => inspect(["daemon", "operation", "--state-dir", session.state_dir, "--operation-id", id, "--json"], `${id}:capacity-observe`),
225
242
  inspectSession: turnId => inspect(["session", "inspect", "--session-id", session.id, ...(turnId ? ["--turn-id", turnId] : []), ...sessionArguments(context, manifest, session)], `${dispatch.operation_id}:capacity-inspect`),
226
243
  assertCurrent: () => {