@deksden-com/dd-flow-cli 0.9.0-beta.119 → 0.9.0-beta.120
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +7 -0
- package/dist/build-info.json +3 -3
- package/dist/harness-runtime/lib/dd-agy-daemon.mjs +3 -2
- package/dist/harness-runtime/lib/dd-codex.d.mts +1 -0
- package/dist/harness-runtime/lib/dd-codex.mjs +100 -23
- package/dist/harness-runtime/lib/dd-opencode.mjs +2 -2
- package/dist/harness-runtime/lib/dd-zcode.mjs +2 -2
- package/dist/services/codex-capacity.js +71 -0
- package/dist/services/controller-fanout.js +13 -1
- package/dist/services/external-work-launch.js +37 -3
- package/dist/services/merge-server.js +34 -4
- package/dist/services/run-controller-adapter.js +20 -2
- package/dist/services/run-controller-recovery.js +47 -3
- package/dist/services/run-controller.js +105 -10
- package/dist/services/run-recovery-runtime.js +43 -2
- package/package.json +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,12 @@
|
|
|
1
1
|
# @deksden-com/dd-flow-cli
|
|
2
2
|
|
|
3
|
+
## 0.9.0-beta.120
|
|
4
|
+
|
|
5
|
+
### Patch Changes
|
|
6
|
+
|
|
7
|
+
- Continue only confirmed, effect-free Codex `serverOverloaded` Turns in the same Session with bounded backoff and retained owner, budget, HITL, Work, MERGE, and recovery admission. Preserve exact native child Turn evidence and fail closed when effects are unknown.
|
|
8
|
+
- Distinguish hard provider quota from short rate limiting and ambiguous 429 responses across AGY, ZCode, and OpenCode.
|
|
9
|
+
|
|
3
10
|
## 0.9.0-beta.119
|
|
4
11
|
|
|
5
12
|
### Patch Changes
|
package/dist/build-info.json
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
{
|
|
2
2
|
"cli_package": "@deksden-com/dd-flow-cli",
|
|
3
|
-
"cli_version": "0.9.0-beta.
|
|
4
|
-
"cli_commit": "
|
|
5
|
-
"built_at": "2026-09-
|
|
3
|
+
"cli_version": "0.9.0-beta.120",
|
|
4
|
+
"cli_commit": "a9385404a7a9fced5e4cfdcdb656e534c06d1401",
|
|
5
|
+
"built_at": "2026-09-28T22:25:15.635Z",
|
|
6
6
|
"built_with_canon": {
|
|
7
7
|
"version": "4.1.1",
|
|
8
8
|
"commit": "678daa038287c948ada5b2d785a6dcc925c7b891",
|
|
@@ -327,10 +327,11 @@ export class Runtime {
|
|
|
327
327
|
this.active = null;
|
|
328
328
|
const message = String(result.error ?? "Antigravity returned ERROR"); const normalized = message.toLowerCase();
|
|
329
329
|
const code = result.status === "CANCELLED" ? "agy_turn_cancelled" : /quota|insufficient.?credit|billing|usage limit/.test(normalized) ? "agy_provider_quota_exhausted"
|
|
330
|
-
: /rate.?limit|too many requests
|
|
330
|
+
: /rate.?limit|too many requests/.test(normalized) ? "agy_provider_rate_limited"
|
|
331
|
+
: /\b429\b/.test(normalized) ? "agy_provider_limit_unknown"
|
|
331
332
|
: /auth|unauthori[sz]ed|forbidden|account/.test(normalized) ? "agy_provider_account"
|
|
332
333
|
: "agy_provider_failed";
|
|
333
|
-
return current.reject(new DaemonError(code, message,
|
|
334
|
+
return current.reject(new DaemonError(code, message, code === "agy_provider_rate_limited", { provider_result: result, provider_error_scope: current.previousError && current.previousError === result.error ? "repeated_conversation_error" : "terminal_result", usage_ingest: this.usageIngest }));
|
|
334
335
|
}
|
|
335
336
|
await this.completePendingTurn();
|
|
336
337
|
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export function terminalCodexOverload(error: unknown, expectedSessionId: string, expectedTurnId?: string): boolean;
|
|
@@ -52,7 +52,7 @@ class Journal {
|
|
|
52
52
|
}
|
|
53
53
|
|
|
54
54
|
export class CodexBridge {
|
|
55
|
-
constructor(options = {}) { this.options = options; this.nextId = 1; this.pending = new Map(); this.lateResponses = new Map(); this.turns = new Map(); this.finalMessages = new Map(); this.descendants = new Map(); this.journal = new Journal(options.journal); this.stderr = ""; this.requestedProfiles = new Map(); this.modelObservations = Promise.resolve(); }
|
|
55
|
+
constructor(options = {}) { this.options = options; this.nextId = 1; this.pending = new Map(); this.lateResponses = new Map(); this.turns = new Map(); this.nativeErrors = new Map(); this.finalMessages = new Map(); this.descendants = new Map(); this.journal = new Journal(options.journal); this.stderr = ""; this.requestedProfiles = new Map(); this.modelObservations = Promise.resolve(); }
|
|
56
56
|
|
|
57
57
|
async start() {
|
|
58
58
|
const bin = this.options.bin ?? process.env.DD_CODEX_BIN ?? "codex";
|
|
@@ -120,11 +120,16 @@ export class CodexBridge {
|
|
|
120
120
|
message.error ? pending.reject(Object.assign(new Error(message.error.message ?? JSON.stringify(message.error)), { code: "rpc_error", details: message.error })) : pending.resolve(message.result ?? {});
|
|
121
121
|
return;
|
|
122
122
|
}
|
|
123
|
-
if (message.method === "
|
|
124
|
-
|
|
123
|
+
if (message.method === "error" && message.params?.threadId && message.params?.turnId) {
|
|
124
|
+
const key = `${message.params.threadId}:${message.params.turnId}`;
|
|
125
|
+
this.nativeErrors.set(key, { error: message.params.error, willRetry: message.params.willRetry });
|
|
126
|
+
while (this.nativeErrors.size > 128) this.nativeErrors.delete(this.nativeErrors.keys().next().value);
|
|
127
|
+
}
|
|
128
|
+
if (message.method === "turn/completed") this.turns.set(message.params?.turn?.id ?? message.params?.threadId, { status: "completed", value: message.params, threadId: message.params?.threadId });
|
|
129
|
+
if (message.method === "turn/started" && this.turns.get(message.params?.turn?.id)?.status !== "completed") this.turns.set(message.params?.turn?.id ?? message.params?.threadId, { status: "running", value: message.params, threadId: message.params?.threadId });
|
|
125
130
|
const threadId = message.params?.threadId;
|
|
126
131
|
if (threadId && message.method === "item/started") this.finalMessages.delete(threadId);
|
|
127
|
-
if (threadId && message.method === "item/completed" && message.params?.item?.type === "agentMessage") this.finalMessages.set(threadId, { item: message.params.item, completedAt: Date.now() });
|
|
132
|
+
if (threadId && message.method === "item/completed" && message.params?.item?.type === "agentMessage") this.finalMessages.set(threadId, { item: message.params.item, turnId: message.params?.turnId ?? null, completedAt: Date.now() });
|
|
128
133
|
}
|
|
129
134
|
|
|
130
135
|
isManagedHook(run) {
|
|
@@ -217,13 +222,27 @@ export class CodexBridge {
|
|
|
217
222
|
observeTopology(value, depth = 0) {
|
|
218
223
|
if (!value || typeof value !== "object" || depth > 8) return;
|
|
219
224
|
if (Array.isArray(value)) { for (const item of value) this.observeTopology(item, depth + 1); return; }
|
|
225
|
+
if (depth === 0 && ["turn/started", "turn/completed"].includes(value.method)) {
|
|
226
|
+
const { threadId, turn } = value.params ?? {};
|
|
227
|
+
const previous = this.descendants.get(threadId);
|
|
228
|
+
if (previous && typeof turn?.id === "string" && turn.id) {
|
|
229
|
+
if (value.method === "turn/started" && previous.active_turn_id !== turn.id && this.turns.get(turn.id)?.status !== "completed") {
|
|
230
|
+
this.descendants.set(threadId, { ...previous, status: "running", active_turn_id: turn.id, terminal_turn_id: null, provider_error: null });
|
|
231
|
+
} else if (value.method === "turn/completed" && (!previous.active_turn_id || previous.active_turn_id === turn.id)) {
|
|
232
|
+
this.descendants.set(threadId, { ...previous, status: turn.status === "completed" ? "completed" : turn.status ?? "unknown",
|
|
233
|
+
active_turn_id: turn.id, terminal_turn_id: turn.id, terminal_observed_at: now(), provider_error: nativeTurnError(this, threadId, turn), native_turn_items: turnItemEvidence(turn) });
|
|
234
|
+
}
|
|
235
|
+
}
|
|
236
|
+
}
|
|
220
237
|
const activity = value.params?.item;
|
|
221
238
|
const activityParent = value.params?.threadId;
|
|
222
239
|
if (activity?.type === "subAgentActivity" && typeof activity.agentThreadId === "string" && activity.agentThreadId &&
|
|
223
240
|
typeof activityParent === "string" && activityParent && activity.agentThreadId !== activityParent) {
|
|
224
241
|
const previous = this.descendants.get(activity.agentThreadId);
|
|
225
242
|
const observed = activity.kind === "completed" ? "completed" : activity.kind === "interrupted" ? "interrupted" : "running";
|
|
226
|
-
|
|
243
|
+
// Parent activity has no child Turn identity on some app-server builds.
|
|
244
|
+
// It can discover a child, but cannot reopen or close a later known Turn.
|
|
245
|
+
const status = previous?.active_turn_id || previous?.status === "completed" ? previous.status : observed;
|
|
227
246
|
this.descendants.set(activity.agentThreadId, { ...previous, provider_session_id: activity.agentThreadId,
|
|
228
247
|
parent_provider_session_id: activityParent, status, agent_path: activity.agentPath ?? previous?.agent_path ?? null,
|
|
229
248
|
source: "codex.app_server.subAgentActivity" });
|
|
@@ -234,7 +253,7 @@ export class CodexBridge {
|
|
|
234
253
|
if (typeof child !== "string" || !child || child === value.senderThreadId) continue;
|
|
235
254
|
const previous = this.descendants.get(child);
|
|
236
255
|
this.descendants.set(child, { ...previous, provider_session_id: child, parent_provider_session_id: value.senderThreadId,
|
|
237
|
-
status: previous?.status === "completed" ?
|
|
256
|
+
status: previous?.status === "completed" || previous?.active_turn_id ? previous.status : value.agentsStates?.[child]?.status ?? previous?.status ?? "unknown", source: "codex.app_server" });
|
|
238
257
|
}
|
|
239
258
|
}
|
|
240
259
|
const id = value.threadId ?? value.id ?? null;
|
|
@@ -242,7 +261,7 @@ export class CodexBridge {
|
|
|
242
261
|
if (typeof id === "string" && id && typeof parent === "string" && parent) {
|
|
243
262
|
const previous = this.descendants.get(id);
|
|
244
263
|
this.descendants.set(id, { ...previous, provider_session_id: id, parent_provider_session_id: parent,
|
|
245
|
-
status: previous?.status === "completed" ?
|
|
264
|
+
status: previous?.status === "completed" || previous?.active_turn_id ? previous.status : value.status?.type ?? value.status ?? previous?.status ?? "unknown", source: "codex.app_server" });
|
|
246
265
|
}
|
|
247
266
|
for (const child of Object.values(value)) this.observeTopology(child, depth + 1);
|
|
248
267
|
}
|
|
@@ -306,6 +325,18 @@ async function withBridge(options, action) { const bridge = new CodexBridge(opti
|
|
|
306
325
|
function profile(options) { return { harness: "codex-desktop", model: options.model ?? null, reasoning: options.reasoning ?? null }; }
|
|
307
326
|
function threadId(result) { const id = result?.thread?.id ?? result?.threadId; return required(id, "Codex thread id"); }
|
|
308
327
|
function directChildren(bridge, sessionId) { return typeof bridge?.directChildren === "function" ? bridge.directChildren(sessionId) : []; }
|
|
328
|
+
async function observedChildren(bridge, sessionId) {
|
|
329
|
+
const children = directChildren(bridge, sessionId);
|
|
330
|
+
let reads = 0;
|
|
331
|
+
return await Promise.all(children.map(async child => {
|
|
332
|
+
if (child.status !== "failed" || !child.terminal_turn_id || reads++ >= 16) return child;
|
|
333
|
+
const observed = await readTurn(bridge, child.provider_session_id, child.terminal_turn_id, true);
|
|
334
|
+
const turn = observed?.turn;
|
|
335
|
+
if (turn?.id !== child.terminal_turn_id || turn.status !== child.status) return child;
|
|
336
|
+
return { ...child, provider_error: nativeTurnError(bridge, child.provider_session_id, turn) ?? child.provider_error,
|
|
337
|
+
native_turn_items: turnItemEvidence(turn) };
|
|
338
|
+
}));
|
|
339
|
+
}
|
|
309
340
|
function codexSessionConfig() { return { ...CODEX_SESSION_CONFIG }; }
|
|
310
341
|
|
|
311
342
|
async function requestConfiguredThread(bridge, method, params, timeoutMs) {
|
|
@@ -399,11 +430,24 @@ export async function promptSessionWithBridge(bridge, options) {
|
|
|
399
430
|
bridge.assertHookHealthy?.(sessionId);
|
|
400
431
|
const started = await bridge.request("turn/start", { threadId: sessionId, input: [{ type: "text", text: prompt }], cwd, model: options.model ?? null, effort: options.reasoning ?? null, approvalPolicy: "never", sandboxPolicy: { type: "dangerFullAccess" } }, options.timeoutMs ?? 1_800_000);
|
|
401
432
|
const turn = started?.turn ?? started; const turnId = turn?.id ?? null;
|
|
402
|
-
|
|
433
|
+
let completed;
|
|
434
|
+
try { completed = await waitForTurn(bridge, sessionId, turnId, options.timeoutMs ?? 1_800_000, options, cwd); }
|
|
435
|
+
catch (error) {
|
|
436
|
+
if (error?.code === "turn_interrupted") {
|
|
437
|
+
try {
|
|
438
|
+
const observed = await readTurn(bridge, sessionId, turnId, true);
|
|
439
|
+
if (observed?.turn?.id === turnId && observed.turn.status === error.details?.terminal_status) {
|
|
440
|
+
error.details.native_turn_items = turnItemEvidence(observed.turn);
|
|
441
|
+
error.details.provider_error ??= observed.turn.error ?? null;
|
|
442
|
+
}
|
|
443
|
+
} catch { /* The primary native terminal failure remains authoritative. */ }
|
|
444
|
+
}
|
|
445
|
+
throw error;
|
|
446
|
+
}
|
|
403
447
|
const thread = await readCompletedThread(bridge, sessionId);
|
|
404
448
|
bridge.assertHookHealthy?.(sessionId);
|
|
405
|
-
const assistantText = latestAssistantText(thread) ?? latestAssistantText(completed);
|
|
406
|
-
return { harness: "codex-desktop", provider_session_id: sessionId, adapter_session_id: sessionId, turn_id: turnId, status: "completed", turn: completed, ...(thread ? { thread: threadReceipt(thread) } : {}), descendants:
|
|
449
|
+
const assistantText = latestAssistantText(thread?.turns?.find(item => item?.id === turnId)) ?? latestAssistantText(completed);
|
|
450
|
+
return { harness: "codex-desktop", provider_session_id: sessionId, adapter_session_id: sessionId, turn_id: turnId, status: "completed", turn: completed, ...(thread ? { thread: threadReceipt(thread) } : {}), descendants: await observedChildren(bridge, sessionId), ...(assistantText ? { assistant_text: assistantText } : {}), requested_profile: profile(options), observed_profile: await observeCodexProfile(bridge, options, sessionId, { model: thread?.model ?? null, reasoning: thread?.reasoningEffort ?? null, provider: thread?.modelProvider ?? null }, "thread/read.after_turn") };
|
|
407
451
|
}
|
|
408
452
|
|
|
409
453
|
async function ensureThreadLoaded(bridge, sessionId, options, cwd) {
|
|
@@ -428,23 +472,23 @@ async function waitForTurn(bridge, sessionId, turnId, timeoutMs, options, cwd) {
|
|
|
428
472
|
let nextStoredReadAt = Date.now();
|
|
429
473
|
while (!clock.sample(bridge.activity?.get(sessionId))) {
|
|
430
474
|
bridge.assertHookHealthy?.(sessionId);
|
|
431
|
-
const known = bridge.turns.get(turnId) ?? bridge.turns.get(sessionId); if (known?.status === "completed") return terminalTurn(known.value, turnId, bridge);
|
|
475
|
+
const known = bridge.turns.get(turnId) ?? bridge.turns.get(sessionId); if (known?.status === "completed" && (!known.threadId || known.threadId === sessionId)) return terminalTurn(known.value, sessionId, turnId, bridge);
|
|
432
476
|
if (Date.now() >= nextStoredReadAt) {
|
|
433
477
|
const stored = await readTurn(bridge, sessionId, turnId, false);
|
|
434
|
-
if (stored?.turn && isAuthoritativeTerminal(bridge, turnId, stored.turn.status, stored.threadStatus, failureNotBefore)) return terminalTurn({ turn: stored.turn }, turnId, bridge);
|
|
478
|
+
if (stored?.turn && isAuthoritativeTerminal(bridge, turnId, stored.turn.status, stored.threadStatus, failureNotBefore)) return terminalTurn({ turn: stored.turn }, sessionId, turnId, bridge);
|
|
435
479
|
// A loaded app-server can retain a stale `active` Thread state after a
|
|
436
480
|
// provider-side interruption. The persisted turn page is independent of
|
|
437
481
|
// that live subscription and is the compact authority for its terminal
|
|
438
482
|
// status; unlike `includeTurns`, it never hydrates the whole history.
|
|
439
483
|
const persisted = await readPersistedTurn(bridge, sessionId, turnId);
|
|
440
|
-
if (persisted && isAuthoritativeTerminal(bridge, turnId, persisted.status, stored?.threadStatus, failureNotBefore)) return terminalTurn({ turn: persisted }, turnId, bridge);
|
|
484
|
+
if (persisted && isAuthoritativeTerminal(bridge, turnId, persisted.status, stored?.threadStatus, failureNotBefore)) return terminalTurn({ turn: persisted }, sessionId, turnId, bridge);
|
|
441
485
|
// Desktop may unload an idle Thread immediately after an interrupted
|
|
442
486
|
// Turn. Reload it once before polling again: otherwise the compact
|
|
443
487
|
// status remains `notLoaded` forever and hides the terminal Turn.
|
|
444
488
|
if (stored?.threadStatus === "notLoaded") {
|
|
445
489
|
await ensureThreadLoaded(bridge, sessionId, options, cwd);
|
|
446
490
|
const hydrated = await readTurn(bridge, sessionId, turnId, true);
|
|
447
|
-
if (hydrated?.turn && isAuthoritativeTerminal(bridge, turnId, hydrated.turn.status, hydrated.threadStatus, failureNotBefore)) return terminalTurn({ turn: hydrated.turn }, turnId, bridge);
|
|
491
|
+
if (hydrated?.turn && isAuthoritativeTerminal(bridge, turnId, hydrated.turn.status, hydrated.threadStatus, failureNotBefore)) return terminalTurn({ turn: hydrated.turn }, sessionId, turnId, bridge);
|
|
448
492
|
}
|
|
449
493
|
// Some app-server versions correctly omit turns for a compact read.
|
|
450
494
|
// Once the thread is idle, hydrate exactly once to recover the terminal
|
|
@@ -455,7 +499,7 @@ async function waitForTurn(bridge, sessionId, turnId, timeoutMs, options, cwd) {
|
|
|
455
499
|
// an inProgress Turn after emitting the final agent message; treating
|
|
456
500
|
// that stale Turn as terminal incorrectly aborts an otherwise usable
|
|
457
501
|
// worker session.
|
|
458
|
-
if (hydrated?.turn && isAuthoritativeTerminal(bridge, turnId, hydrated.turn.status, hydrated.threadStatus, failureNotBefore)) return terminalTurn({ turn: hydrated.turn }, turnId, bridge);
|
|
502
|
+
if (hydrated?.turn && isAuthoritativeTerminal(bridge, turnId, hydrated.turn.status, hydrated.threadStatus, failureNotBefore)) return terminalTurn({ turn: hydrated.turn }, sessionId, turnId, bridge);
|
|
459
503
|
}
|
|
460
504
|
// Some app-server builds emit a final agent message but omit both the
|
|
461
505
|
// terminal event and the compact turn record. Wait briefly, hydrate once,
|
|
@@ -464,10 +508,10 @@ async function waitForTurn(bridge, sessionId, turnId, timeoutMs, options, cwd) {
|
|
|
464
508
|
// Agent messages are streamed as completed items while a Turn is still
|
|
465
509
|
// reasoning and calling tools. They are terminal evidence only after
|
|
466
510
|
// app-server reports the Thread idle.
|
|
467
|
-
if (final && stored?.threadStatus === "idle" && Date.now() - final.completedAt >= 1_000) {
|
|
511
|
+
if (final?.turnId === turnId && stored?.threadStatus === "idle" && Date.now() - final.completedAt >= 1_000) {
|
|
468
512
|
const hydrated = await readTurn(bridge, sessionId, turnId, true);
|
|
469
|
-
if (hydrated?.turn && isAuthoritativeTerminal(bridge, turnId, hydrated.turn.status, hydrated.threadStatus, failureNotBefore)) return terminalTurn({ turn: hydrated.turn }, turnId, bridge);
|
|
470
|
-
return terminalTurn({ turn: { id: turnId, status: "completed", synthetic: true, final_message: final.item } }, turnId, bridge);
|
|
513
|
+
if (hydrated?.turn && isAuthoritativeTerminal(bridge, turnId, hydrated.turn.status, hydrated.threadStatus, failureNotBefore)) return terminalTurn({ turn: hydrated.turn }, sessionId, turnId, bridge);
|
|
514
|
+
return terminalTurn({ turn: { id: turnId, status: "completed", synthetic: true, final_message: final.item } }, sessionId, turnId, bridge);
|
|
471
515
|
}
|
|
472
516
|
nextStoredReadAt = Date.now() < failureNotBefore ? failureNotBefore : Date.now() + 15_000;
|
|
473
517
|
}
|
|
@@ -476,15 +520,48 @@ async function waitForTurn(bridge, sessionId, turnId, timeoutMs, options, cwd) {
|
|
|
476
520
|
fail("Codex turn timed out", "turn_timeout");
|
|
477
521
|
}
|
|
478
522
|
|
|
479
|
-
function terminalTurn(value, turnId, bridge) {
|
|
523
|
+
function terminalTurn(value, sessionId, turnId, bridge) {
|
|
480
524
|
// A terminal event may arrive while an older snapshot RPC is in flight.
|
|
481
525
|
const observed = bridge.turns.get(turnId);
|
|
482
|
-
if (observed?.status === "completed") value = observed.value;
|
|
526
|
+
if (observed?.status === "completed" && (!observed.threadId || observed.threadId === sessionId)) value = observed.value;
|
|
483
527
|
const turn = value?.turn ?? value;
|
|
484
|
-
if (turn?.
|
|
528
|
+
if (turn?.id && turn.id !== turnId || value?.threadId && value.threadId !== sessionId) fail("Codex terminal Turn identity does not match the requested Session/Turn", "turn_identity_mismatch", { provider_session_id: sessionId, turn_id: turnId, native_thread_id: value?.threadId ?? null, native_turn_id: turn?.id ?? null });
|
|
529
|
+
const providerError = nativeTurnError(bridge, sessionId, turn);
|
|
530
|
+
if (turn?.status && turn.status !== "completed") fail(`Codex turn ${turnId ?? "unknown"} ended as ${turn.status}`, "turn_interrupted", { provider_session_id: sessionId, turn_id: turnId, native_turn_id: turn.id ?? null, terminal_status: turn.status, observed_at: now(),
|
|
531
|
+
provider_error: providerError, native_turn_items: turnItemEvidence(turn) });
|
|
485
532
|
return value;
|
|
486
533
|
}
|
|
487
534
|
|
|
535
|
+
function nativeTurnError(bridge, sessionId, turn) {
|
|
536
|
+
const native = bridge.nativeErrors?.get(`${sessionId}:${turn?.id}`);
|
|
537
|
+
const error = turn?.error && typeof turn.error === "object" && native?.error && typeof native.error === "object"
|
|
538
|
+
? { ...native.error, ...turn.error } : turn?.error ?? native?.error ?? (turn?.codexErrorInfo ? { codexErrorInfo: turn.codexErrorInfo } : null);
|
|
539
|
+
return error && native?.willRetry !== undefined && typeof error === "object" ? { ...error, willRetry: native.willRetry } : error;
|
|
540
|
+
}
|
|
541
|
+
|
|
542
|
+
function turnItemEvidence(turn) {
|
|
543
|
+
const items = turn?.items;
|
|
544
|
+
return Array.isArray(items) && turn?.itemsView !== "notLoaded" ? { observed: true, count: items.length, types: items.map(item => item?.type ?? "unknown"),
|
|
545
|
+
possible_effects: items.some(item => !["agentMessage", "reasoning", "userMessage"].includes(item?.type)) } : { observed: false };
|
|
546
|
+
}
|
|
547
|
+
|
|
548
|
+
export function terminalCodexOverload(error, expectedSessionId, expectedTurnId) {
|
|
549
|
+
let current = error;
|
|
550
|
+
for (let depth = 0; depth < 4 && current && typeof current === "object"; depth += 1) {
|
|
551
|
+
if (current.code === "turn_interrupted") {
|
|
552
|
+
const details = current.details ?? {}, native = details.provider_error;
|
|
553
|
+
const code = native?.codexErrorInfo;
|
|
554
|
+
return details.terminal_status === "failed" && typeof details.provider_session_id === "string" && details.provider_session_id === expectedSessionId
|
|
555
|
+
&& typeof details.turn_id === "string" && Boolean(details.turn_id) && details.native_turn_id === details.turn_id
|
|
556
|
+
&& (expectedTurnId === undefined || details.turn_id === expectedTurnId)
|
|
557
|
+
&& (code === "serverOverloaded" || code?.code === "serverOverloaded");
|
|
558
|
+
}
|
|
559
|
+
if (current.code !== "harness_adapter_failed") return false;
|
|
560
|
+
current = current.details?.cause ?? current.cause;
|
|
561
|
+
}
|
|
562
|
+
return false;
|
|
563
|
+
}
|
|
564
|
+
|
|
488
565
|
function isTerminalTurnStatus(status) { return ["completed", "failed", "interrupted", "cancelled"].includes(status); }
|
|
489
566
|
function isAuthoritativeTerminal(bridge, turnId, status, threadStatus, failureNotBefore) {
|
|
490
567
|
if (!isTerminalTurnStatus(status)) return false;
|
|
@@ -518,7 +595,7 @@ async function readCompletedThread(bridge, sessionId) {
|
|
|
518
595
|
}
|
|
519
596
|
|
|
520
597
|
function latestAssistantText(value) {
|
|
521
|
-
const messages = value?.messages ?? value?.items ?? value?.turn?.items ?? [];
|
|
598
|
+
const messages = value?.messages ?? value?.items ?? value?.turn?.items ?? (value?.turn?.final_message ? [value.turn.final_message] : []);
|
|
522
599
|
if (!Array.isArray(messages)) return null;
|
|
523
600
|
for (const message of [...messages].reverse()) {
|
|
524
601
|
if (message?.role !== "assistant" && message?.type !== "agentMessage") continue;
|
|
@@ -538,7 +615,7 @@ export async function inspectSession(options) {
|
|
|
538
615
|
|
|
539
616
|
export async function inspectSessionWithBridge(bridge, options) {
|
|
540
617
|
const sessionId = required(options.sessionId, "--session-id"); const value = await bridge.request("thread/read", { threadId: sessionId, includeTurns: false }); const thread = value?.thread ?? value;
|
|
541
|
-
return { harness: "codex-desktop", provider_session_id: sessionId, adapter_session_id: sessionId, status: thread.status ?? null, thread: threadReceipt(thread), descendants:
|
|
618
|
+
return { harness: "codex-desktop", provider_session_id: sessionId, adapter_session_id: sessionId, status: thread.status ?? null, thread: threadReceipt(thread), descendants: await observedChildren(bridge, sessionId), requested_profile: profile(options), observed_profile: await observeCodexProfile(bridge, options, sessionId, { model: thread.model ?? null, reasoning: thread.reasoningEffort ?? null, provider: thread.modelProvider ?? null }, "thread/read") };
|
|
542
619
|
}
|
|
543
620
|
|
|
544
621
|
export async function cancelSession(options) {
|
|
@@ -68,8 +68,8 @@ export class OpenCodeClient {
|
|
|
68
68
|
const native = reply.info.error;
|
|
69
69
|
const message = native.data?.message ?? native.message ?? native.name ?? "OpenCode provider failed";
|
|
70
70
|
const status = native.data?.statusCode;
|
|
71
|
-
const code = status ===
|
|
72
|
-
throw new OpenCodeError(code, message,
|
|
71
|
+
const code = status === 402 || /payment required|quota|usage balance exhausted|insufficient.?credit|billing/i.test(message) ? "provider_quota_exhausted" : /rate.?limit|too many requests/i.test(message) ? "provider_rate_limited" : status === 429 ? "provider_limit_unknown" : "opencode_provider_failed";
|
|
72
|
+
throw new OpenCodeError(code, message, code === "provider_rate_limited", { session_id: id, message_id: reply.info.id, provider_error: native });
|
|
73
73
|
}
|
|
74
74
|
return reply;
|
|
75
75
|
}
|
|
@@ -268,8 +268,8 @@ export class AcpBridge {
|
|
|
268
268
|
const error = new Error(messageText);
|
|
269
269
|
const nativeCode = typeof data?.code === "string" ? data.code : typeof data?.native_code === "string" ? data.native_code : null;
|
|
270
270
|
const unknown = ["native_timeout", "native_backend_dead", "native_backend_pipe_broken", "native_outcome_unknown"].includes(nativeCode);
|
|
271
|
-
error.code = unknown ? nativeCode : /
|
|
272
|
-
error.retryable =
|
|
271
|
+
error.code = unknown ? nativeCode : /payment required|quota|usage balance exhausted|insufficient.?credit|billing/i.test(messageText) ? "provider_quota_exhausted" : /rate.?limit|too many requests/i.test(messageText) ? "provider_rate_limited" : /\b429\b/.test(messageText) ? "provider_limit_unknown" : "acp_request_failed";
|
|
272
|
+
error.retryable = error.code === "provider_rate_limited";
|
|
273
273
|
error.details = { acp_code: message.error.code ?? null, ...(data ? { native: data } : {}), ...(details ? { provider_details: details } : {}), ...(providerDetails ? { provider_update: provider } : {}) };
|
|
274
274
|
pending.reject(error);
|
|
275
275
|
}
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
import { setTimeout as delay } from "node:timers/promises";
|
|
2
|
+
import { terminalCodexOverload } from "../harness-runtime/lib/dd-codex.mjs";
|
|
3
|
+
/** Only a terminal, effect-free native Turn may get another productive Turn. */
|
|
4
|
+
export async function continueCodexCapacity(input) {
|
|
5
|
+
let { error, operationId } = input;
|
|
6
|
+
for (let ordinal = 1; ordinal <= 2; ordinal += 1) {
|
|
7
|
+
if (!terminalCodexOverload(error, input.sessionId)) {
|
|
8
|
+
if (ordinal === 1)
|
|
9
|
+
return null;
|
|
10
|
+
throw error;
|
|
11
|
+
}
|
|
12
|
+
let prior;
|
|
13
|
+
try {
|
|
14
|
+
prior = await input.inspectOperation(operationId);
|
|
15
|
+
}
|
|
16
|
+
catch {
|
|
17
|
+
if (ordinal === 1)
|
|
18
|
+
return null;
|
|
19
|
+
throw error;
|
|
20
|
+
}
|
|
21
|
+
const failure = prior.error;
|
|
22
|
+
const turnId = failure?.details?.turn_id;
|
|
23
|
+
if (prior.operation_id !== operationId || prior.operation !== "session.prompt" || prior.session_id !== input.sessionId
|
|
24
|
+
|| prior.state !== "failed" || prior.primary_error || !turnId || !terminalCodexOverload(failure, input.sessionId, turnId)
|
|
25
|
+
|| failure.details?.native_turn_items?.observed !== true || failure.details.native_turn_items.possible_effects !== false) {
|
|
26
|
+
if (ordinal === 1)
|
|
27
|
+
return null;
|
|
28
|
+
throw error;
|
|
29
|
+
}
|
|
30
|
+
const settled = async () => {
|
|
31
|
+
const seen = await input.inspectSession();
|
|
32
|
+
const settlement = seen.settlement;
|
|
33
|
+
const ids = Array.isArray(seen.tree_session_ids) ? seen.tree_session_ids : [];
|
|
34
|
+
const observations = Array.isArray(seen.tree_observations) ? seen.tree_observations : [];
|
|
35
|
+
const treeProven = input.allowSettledDescendants
|
|
36
|
+
? ids.length > 0 && ids[0] === input.sessionId && observations.length === ids.length
|
|
37
|
+
&& ids.every(id => observations.some(item => item.provider_session_id === id && item.settled === true))
|
|
38
|
+
&& Array.isArray(seen.unsettled_sessions) && seen.unsettled_sessions.length === 0
|
|
39
|
+
: ids.length === 1 && ids[0] === input.sessionId;
|
|
40
|
+
return seen.provider_session_id === input.sessionId && seen.settled === true && treeProven
|
|
41
|
+
&& (!settlement || ["settled", "not_required"].includes(settlement.state ?? ""));
|
|
42
|
+
};
|
|
43
|
+
input.assertCurrent();
|
|
44
|
+
try {
|
|
45
|
+
if (!await settled()) {
|
|
46
|
+
if (ordinal === 1)
|
|
47
|
+
return null;
|
|
48
|
+
throw error;
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
catch {
|
|
52
|
+
if (ordinal === 1)
|
|
53
|
+
return null;
|
|
54
|
+
throw error;
|
|
55
|
+
}
|
|
56
|
+
const nextId = `${input.operationId}:capacity:${ordinal}:session.prompt`;
|
|
57
|
+
input.retainIntent({ predecessor: operationId, operation_id: nextId, turn_id: turnId, ordinal, not_before: new Date(Date.now() + (ordinal === 1 ? 5_000 : 15_000)).toISOString() });
|
|
58
|
+
await delay(ordinal === 1 ? 5_000 : 15_000);
|
|
59
|
+
input.assertCurrent();
|
|
60
|
+
if (!await settled().catch(() => false))
|
|
61
|
+
throw error;
|
|
62
|
+
try {
|
|
63
|
+
return await input.prompt(nextId, input.continuationText ?? "Continue the same assigned task in this Session. Check current durable RUN, Work and Stage state first. Never repeat an observed lifecycle command, accepted HITL answer, check or merge apply; use only an exact issued successor. Stop at the assigned boundary.", ordinal);
|
|
64
|
+
}
|
|
65
|
+
catch (nextError) {
|
|
66
|
+
error = nextError;
|
|
67
|
+
operationId = nextId;
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
throw error;
|
|
71
|
+
}
|
|
@@ -136,7 +136,10 @@ export function controllerNativeChildren(receipt, rootSessionId) {
|
|
|
136
136
|
const status = ["completed", "complete", "done", "success", "succeeded"].includes(state) ? "completed" : state === "settled_by_root" ? "settled_by_root" : ["failed", "error"].includes(state) ? "failed" : ["cancelled", "canceled", "interrupted"].includes(state) ? "cancelled" : ["running", "active", "pending"].includes(state) ? "running" : fallback;
|
|
137
137
|
const rootChild = parent === rootSessionId;
|
|
138
138
|
const settled = rootChild && status === "unknown" && settledTree.has(id) ? "settled_by_root" : status;
|
|
139
|
-
candidates.push({ session_id: id, parent_session_id: parent ?? "", status: settled, source, ...(rootChild ? {} : { provenance: "parent_mismatch" })
|
|
139
|
+
candidates.push({ session_id: id, parent_session_id: parent ?? "", status: settled, source, ...(rootChild ? {} : { provenance: "parent_mismatch" }),
|
|
140
|
+
...(text(value, ["terminal_turn_id"]) ? { terminal_turn_id: text(value, ["terminal_turn_id"]) } : {}),
|
|
141
|
+
...(value.provider_error !== undefined ? { provider_error: value.provider_error } : {}),
|
|
142
|
+
...(value.native_turn_items && typeof value.native_turn_items === "object" ? { native_turn_items: value.native_turn_items } : {}) });
|
|
140
143
|
};
|
|
141
144
|
for (const value of list(receipt.descendants))
|
|
142
145
|
add(value, "adapter.descendants");
|
|
@@ -166,6 +169,15 @@ export function controllerNativeChildren(receipt, rootSessionId) {
|
|
|
166
169
|
}
|
|
167
170
|
if (previous && rank[previous.status] === 3 && rank[child.status] === 3 && previous.status !== child.status)
|
|
168
171
|
throw new AppError("native_child_outcome_conflict", "Native child has contradictory terminal outcomes", 1, { session_id: child.session_id, observations: [previous, child] });
|
|
172
|
+
if (previous && previous.status === child.status && previous.terminal_turn_id && child.terminal_turn_id && previous.terminal_turn_id !== child.terminal_turn_id)
|
|
173
|
+
throw new AppError("native_child_turn_conflict", "Native child has contradictory terminal Turn identities", 1, { session_id: child.session_id, observations: [previous, child] });
|
|
174
|
+
if (previous && previous.status === child.status) {
|
|
175
|
+
unique.set(child.session_id, { ...previous, ...child,
|
|
176
|
+
...(child.terminal_turn_id ?? previous.terminal_turn_id ? { terminal_turn_id: child.terminal_turn_id ?? previous.terminal_turn_id } : {}),
|
|
177
|
+
...(child.provider_error ?? previous.provider_error ? { provider_error: child.provider_error ?? previous.provider_error } : {}),
|
|
178
|
+
...(child.native_turn_items ?? previous.native_turn_items ? { native_turn_items: child.native_turn_items ?? previous.native_turn_items } : {}) });
|
|
179
|
+
continue;
|
|
180
|
+
}
|
|
169
181
|
if (!previous || rank[child.status] > rank[previous.status])
|
|
170
182
|
unique.set(child.session_id, child);
|
|
171
183
|
}
|
|
@@ -16,6 +16,8 @@ import { reviewExecutionCopyHash, reviewSourceFilter, reviewSourceHash, reviewSo
|
|
|
16
16
|
import { workspaceGitFiles } from "./workspace-files.js";
|
|
17
17
|
import { qualifyAgyHooks } from "../harness-runtime/lib/dd-agy-daemon.mjs";
|
|
18
18
|
import { lifecycleRetryInstruction } from "../harness-runtime/lib/delegation-instructions.mjs";
|
|
19
|
+
import { continueCodexCapacity } from "./codex-capacity.js";
|
|
20
|
+
import { errorRecord } from "./run-controller-state.js";
|
|
19
21
|
/**
|
|
20
22
|
* Creates an external provider root for one already-created Work. The launch
|
|
21
23
|
* is not a Work claim: only the worker's trusted `work start` hook binds it.
|
|
@@ -91,7 +93,7 @@ export async function launchExternalWork(context, input) {
|
|
|
91
93
|
const common = ["--dd-flow-bin", flowExecutable, "--state-dir", root, "--journal", journal, "--cwd", isolatedWorkspace ?? run.workspace_root, "--provider", resolved.worker.provider, "--model", resolved.worker.model, "--reasoning", resolved.worker.reasoning, "--mode", resolved.worker.mode, "--permission", resolved.worker.permission, "--project-root", run.project_root, "--dd-flow-home", context.ddFlowHome, ...harnessRuntimeArguments(context.ddFlowHome, runtimeHarness(resolved.worker)), "--json"];
|
|
92
94
|
const evidence = (value) => fs.appendFileSync(journal, `${JSON.stringify({ type: "adapter_call", at: context.now(), ...value })}\n`);
|
|
93
95
|
try {
|
|
94
|
-
const call = (args, timeoutMs) => runHarnessAdapter({ executable, args, env: runtimeContext.env, operationId:
|
|
96
|
+
const call = (args, timeoutMs, id = `${operationId}:${args[0]}.${args[1]}`) => runHarnessAdapter({ executable, args, env: runtimeContext.env, operationId: id, runtimeOwner, ...(timeoutMs ? { timeoutMs } : {}), ...(input.progress ? { progress: input.progress } : {}), onEvidence: evidence });
|
|
95
97
|
await call(["daemon", "start", ...common]);
|
|
96
98
|
assertRunMutationAllowed(context, work.project_id, work.run_id, generation);
|
|
97
99
|
const created = await call(["session", "create", ...common], runtimeHarness(resolved.worker) === "zcode-acp" ? 180_000 : undefined);
|
|
@@ -108,13 +110,45 @@ export async function launchExternalWork(context, input) {
|
|
|
108
110
|
throw new AppError("read_only_input_changed", "Authoritative review workspace changed after the execution copy was made", 2, { work_id: work.work_id });
|
|
109
111
|
const promptFile = path.join(root, "launch.md");
|
|
110
112
|
fs.writeFileSync(promptFile, `Start the assigned Work now. Your first tool call must be this exact standalone command:\n\n${workStartCommand(runtimeContext, work)}\n\nUse the packet returned by that command. Do not claim another Work or run this command more than once.\n\n${lifecycleRetryInstruction}${isolatedWorkspace ? `\n\nThe review copy has no .git. Read the captured status/diff or gitless note in ${path.join(isolatedWorkspace, ".dd-flow", "review-evidence")}; do not run Git inside the copy.\n` : "\n"}`);
|
|
111
|
-
const
|
|
113
|
+
const promptOperationId = `${operationId}:session.prompt`;
|
|
114
|
+
let prompted;
|
|
115
|
+
try {
|
|
116
|
+
prompted = await call(["session", "prompt", "--session-id", sessionId, "--prompt-file", promptFile, ...common], 45 * 60_000);
|
|
117
|
+
}
|
|
118
|
+
catch (error) {
|
|
119
|
+
if (runtimeHarness(resolved.worker) !== "codex-desktop")
|
|
120
|
+
throw error;
|
|
121
|
+
const resumed = await continueCodexCapacity({ error, sessionId, operationId: promptOperationId,
|
|
122
|
+
inspectOperation: id => call(["daemon", "operation", "--state-dir", root, "--operation-id", id, "--json"], 30_000, `${id}:capacity-observe`),
|
|
123
|
+
inspectSession: () => call(["session", "inspect", "--session-id", sessionId, ...common], 30_000, `${operationId}:capacity-inspect`),
|
|
124
|
+
assertCurrent: () => {
|
|
125
|
+
assertRunMutationAllowed(context, work.project_id, work.run_id, generation);
|
|
126
|
+
const current = context.db.get("SELECT w.status, l.status AS launch_status FROM works w JOIN work_launches l ON l.work_id = w.work_id WHERE w.work_id = ? AND l.operation_id = ?", [work.work_id, operationId]);
|
|
127
|
+
if (current?.status !== "created" || current.launch_status !== "launching")
|
|
128
|
+
throw new AppError("capacity_continuation_blocked", "External Work changed before its native continuation", 1, { work_id: work.work_id, state: current });
|
|
129
|
+
if (reviewCopy && reviewSourceHash(run.workspace_root, reviewInputs, reviewBase) !== reviewCopy.source_sha256)
|
|
130
|
+
throw new AppError("read_only_input_changed", "Authoritative review workspace changed during capacity wait", 2);
|
|
131
|
+
},
|
|
132
|
+
retainIntent: intent => {
|
|
133
|
+
const changed = context.db.run("UPDATE work_launches SET adapter_receipt_json = ?, updated_at = ? WHERE operation_id = ? AND status = 'launching'", [JSON.stringify({ create: created, capacity_intent: intent }), context.now(), operationId]);
|
|
134
|
+
if (changed.changes !== 1)
|
|
135
|
+
throw new AppError("capacity_continuation_blocked", "External Work launch owner changed", 1);
|
|
136
|
+
},
|
|
137
|
+
prompt: (id, text, ordinal) => {
|
|
138
|
+
const file = path.join(root, `capacity-${ordinal}.md`);
|
|
139
|
+
fs.writeFileSync(file, `${text}\n`, { flag: "wx", mode: 0o600 });
|
|
140
|
+
return call(["session", "prompt", "--session-id", sessionId, "--prompt-file", file, ...common], 45 * 60_000, id);
|
|
141
|
+
} });
|
|
142
|
+
if (!resumed)
|
|
143
|
+
throw error;
|
|
144
|
+
prompted = resumed;
|
|
145
|
+
}
|
|
112
146
|
assertRunMutationAllowed(context, work.project_id, work.run_id, generation);
|
|
113
147
|
context.db.run("UPDATE work_launches SET status = 'start_requested', provider_session_id = ?, adapter_receipt_json = ?, updated_at = ? WHERE operation_id = ?", [sessionId, JSON.stringify({ create: created, prompt: prompted }), context.now(), operationId]);
|
|
114
148
|
return { ok: true, work_id: work.work_id, operation_id: operationId, status: "start_requested", session: { harness_id: runtimeHarness(resolved.worker), session_id: sessionId }, profile_hash: resolved.profile_hash, journal_path: journal };
|
|
115
149
|
}
|
|
116
150
|
catch (error) {
|
|
117
|
-
context.db.run("UPDATE work_launches SET status = 'failed', adapter_receipt_json = ?, updated_at = ? WHERE operation_id = ?", [JSON.stringify({ error:
|
|
151
|
+
context.db.run("UPDATE work_launches SET status = 'failed', adapter_receipt_json = ?, updated_at = ? WHERE operation_id = ?", [JSON.stringify({ error: errorRecord(error) }), context.now(), operationId]);
|
|
118
152
|
throw error;
|
|
119
153
|
}
|
|
120
154
|
}
|
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
import { managedInvocationContext, managedLifecycleCommand } from "./lifecycle-invocations.js";
|
|
2
2
|
import { stageLifecycleInstruction } from "../harness-runtime/lib/delegation-instructions.mjs";
|
|
3
|
+
import { continueCodexCapacity } from "./codex-capacity.js";
|
|
4
|
+
import { errorRecord } from "./run-controller-state.js";
|
|
3
5
|
import crypto from "node:crypto";
|
|
4
6
|
import fs from "node:fs";
|
|
5
7
|
import path from "node:path";
|
|
@@ -110,7 +112,7 @@ async function dispatch(context, input) {
|
|
|
110
112
|
return; const now = context.now(); context.db.run("UPDATE merge_requests SET dispatch_lease_expires_at = ?, updated_at = ? WHERE merge_request_id = ? AND status = 'dispatching' AND dispatch_owner = ? AND dispatch_lease_token = ?", [new Date(Date.now() + 120_000).toISOString(), now, input.request.merge_request_id, input.serverId, token]); }, 30_000);
|
|
111
113
|
leaseHeartbeat.unref();
|
|
112
114
|
try {
|
|
113
|
-
const adapterCall = (args, timeoutMs, progressMessage) => { assertRunMutationAllowed(context, input.request.project_id, input.request.run_id, generation); return runHarnessAdapter({ executable, args, env: runtimeContext.env, runtimeOwner, operationId:
|
|
115
|
+
const adapterCall = (args, timeoutMs, progressMessage, id = `${input.request.merge_request_id}:${token}:${args[0]}.${args[1]}`) => { assertRunMutationAllowed(context, input.request.project_id, input.request.run_id, generation); return runHarnessAdapter({ executable, args, env: runtimeContext.env, runtimeOwner, operationId: id, ...(timeoutMs ? { timeoutMs } : {}), ...(input.progress ? { progress: input.progress } : {}), ...(progressMessage ? { progressMessage } : {}), onEvidence: (value) => event(input.journal, { type: "adapter_call", ...value }) }); };
|
|
114
116
|
await adapterCall(["daemon", "start", ...common]);
|
|
115
117
|
const created = await adapterCall(["session", "create", ...common], runtimeHarness(input.profile) === "zcode-acp" ? 180_000 : undefined);
|
|
116
118
|
const sessionId = adapterSessionId(created);
|
|
@@ -121,7 +123,35 @@ async function dispatch(context, input) {
|
|
|
121
123
|
runtimeContext = managedInvocationContext(runtimeContext, { harness: runtimeHarness(input.profile), stateDir, sessionId, projectRoot: run.project_root, runId: input.request.run_id, generation });
|
|
122
124
|
const stageCommand = managedLifecycleCommand(runtimeContext, `${flowCommand(runtimeContext)} stage start ${input.request.run_id} --stage merge --project-root ${JSON.stringify(run.project_root)} --json --progress-jsonl`);
|
|
123
125
|
fs.writeFileSync(promptFile, `Start the assigned MERGE stage now. Your first tool call must be this exact standalone command:\n\n${stageCommand}\n\nTrust and follow the complete stage packet it returns. Continue through merge apply and stage finish. Stop only when the stage reaches a terminal result or explicitly requests user/operator action.\n\n${stageLifecycleInstruction}\n`);
|
|
124
|
-
|
|
126
|
+
let prompted;
|
|
127
|
+
try {
|
|
128
|
+
prompted = await adapterCall(["session", "prompt", "--session-id", sessionId, "--prompt-file", promptFile, ...common], 45 * 60 * 1000, `MERGE ${input.request.merge_request_id} Session ${sessionId} is still running`);
|
|
129
|
+
}
|
|
130
|
+
catch (error) {
|
|
131
|
+
if (runtimeHarness(input.profile) !== "codex-desktop")
|
|
132
|
+
throw error;
|
|
133
|
+
const resumed = await continueCodexCapacity({ error, sessionId, operationId: String(retained.prompt_operation_id),
|
|
134
|
+
inspectOperation: id => adapterCall(["daemon", "operation", "--state-dir", stateDir, "--operation-id", id, "--json"], 30_000, undefined, `${id}:capacity-observe`),
|
|
135
|
+
inspectSession: () => adapterCall(["session", "inspect", "--session-id", sessionId, ...common], 30_000, undefined, `${input.request.merge_request_id}:${token}:capacity-inspect`),
|
|
136
|
+
assertCurrent: () => {
|
|
137
|
+
assertRunMutationAllowed(context, input.request.project_id, input.request.run_id, generation);
|
|
138
|
+
const current = context.db.get("SELECT status, dispatch_owner, dispatch_lease_token, dispatch_lease_expires_at, target_workspace FROM merge_requests WHERE merge_request_id = ?", [input.request.merge_request_id]);
|
|
139
|
+
if (current?.status !== "dispatching" || current.dispatch_owner !== input.serverId || current.dispatch_lease_token !== token
|
|
140
|
+
|| current.dispatch_lease_expires_at <= context.now() || current.target_workspace !== input.request.target_workspace)
|
|
141
|
+
throw new AppError("capacity_continuation_blocked", "MERGE dispatch owner or target changed during capacity wait", 1, { request_id: input.request.merge_request_id });
|
|
142
|
+
if (context.db.get("SELECT status FROM works WHERE work_id = ?", [input.request.executor_work_id])?.status !== "created")
|
|
143
|
+
throw new AppError("capacity_continuation_blocked", "MERGE Work already changed state", 1);
|
|
144
|
+
},
|
|
145
|
+
retainIntent: intent => { retained.capacity_intent = intent; retain(); },
|
|
146
|
+
prompt: (id, text, ordinal) => {
|
|
147
|
+
const file = path.join(stateDir, `capacity-${ordinal}.md`);
|
|
148
|
+
fs.writeFileSync(file, `${text}\n`, { flag: "wx", mode: 0o600 });
|
|
149
|
+
return adapterCall(["session", "prompt", "--session-id", sessionId, "--prompt-file", file, ...common], 45 * 60 * 1000, `MERGE ${input.request.merge_request_id} Session ${sessionId} capacity continuation ${ordinal}/2`, id);
|
|
150
|
+
} });
|
|
151
|
+
if (!resumed)
|
|
152
|
+
throw error;
|
|
153
|
+
prompted = resumed;
|
|
154
|
+
}
|
|
125
155
|
Object.assign(retained, { response: prompted });
|
|
126
156
|
retain();
|
|
127
157
|
assertRunMutationAllowed(context, input.request.project_id, input.request.run_id, generation);
|
|
@@ -137,9 +167,9 @@ async function dispatch(context, input) {
|
|
|
137
167
|
}
|
|
138
168
|
const request = context.db.get("SELECT status FROM merge_requests WHERE merge_request_id = ?", [input.request.merge_request_id]);
|
|
139
169
|
if (request?.status === "dispatching")
|
|
140
|
-
context.db.run("UPDATE merge_requests SET status = 'recovery_required', last_error_json = ?, updated_at = ? WHERE merge_request_id = ?", [JSON.stringify({ code: "adapter_launch_failed", error:
|
|
170
|
+
context.db.run("UPDATE merge_requests SET status = 'recovery_required', last_error_json = ?, updated_at = ? WHERE merge_request_id = ?", [JSON.stringify({ code: "adapter_launch_failed", error: errorRecord(error) }), context.now(), input.request.merge_request_id]);
|
|
141
171
|
else if (request?.status === "active")
|
|
142
|
-
context.db.run("UPDATE merge_requests SET status = 'recovery_required', last_error_json = ?, updated_at = ? WHERE merge_request_id = ?", [JSON.stringify({ code: "adapter_lost_after_stage_start", error:
|
|
172
|
+
context.db.run("UPDATE merge_requests SET status = 'recovery_required', last_error_json = ?, updated_at = ? WHERE merge_request_id = ?", [JSON.stringify({ code: "adapter_lost_after_stage_start", error: errorRecord(error) }), context.now(), input.request.merge_request_id]);
|
|
143
173
|
throw error;
|
|
144
174
|
}
|
|
145
175
|
finally {
|
|
@@ -15,14 +15,32 @@ export function controllerHitlAdmission(context, record, input) {
|
|
|
15
15
|
if (!row || row.generation !== JSON.parse(record.metadata_json).run_generation)
|
|
16
16
|
return false;
|
|
17
17
|
const suffix = `:prompt:${row.generation}`;
|
|
18
|
-
|
|
18
|
+
const successor = /^(.*):capacity:([12]):session\.prompt$/.exec(input.operationId);
|
|
19
|
+
const baseId = successor ? successor[1] : input.operationId;
|
|
20
|
+
if (!baseId.endsWith(suffix))
|
|
19
21
|
return false;
|
|
22
|
+
if (successor) {
|
|
23
|
+
const ordinal = Number(successor[2]);
|
|
24
|
+
const predecessorId = ordinal === 1 ? baseId : `${baseId}:capacity:1:session.prompt`;
|
|
25
|
+
const predecessor = context.db.get("SELECT status, receipt_json FROM run_controller_operations WHERE operation_id = ? AND controller_id = ? AND generation = ?", [predecessorId, row.controller_id, row.generation]);
|
|
26
|
+
if (predecessor?.status !== "failed" || !predecessor.receipt_json)
|
|
27
|
+
return false;
|
|
28
|
+
let retained;
|
|
29
|
+
try {
|
|
30
|
+
retained = JSON.parse(predecessor.receipt_json);
|
|
31
|
+
}
|
|
32
|
+
catch {
|
|
33
|
+
return false;
|
|
34
|
+
}
|
|
35
|
+
if (retained.capacity_intent?.predecessor !== predecessorId || retained.capacity_intent.operation_id !== input.operationId || retained.capacity_intent.ordinal !== ordinal)
|
|
36
|
+
return false;
|
|
37
|
+
}
|
|
20
38
|
const state = JSON.parse(row.state_json);
|
|
21
39
|
const session = state.current_session === null ? undefined : state.sessions[state.current_session];
|
|
22
40
|
if (!session || session.stopped || session.id !== input.nativeSessionId || session.state_dir !== input.stateDir || session.owner.owner_id !== record.owner_id)
|
|
23
41
|
return false;
|
|
24
42
|
const operation = (id) => context.db.get("SELECT input_json, input_hash FROM run_controller_operations WHERE operation_id = ? AND controller_id = ? AND generation = ? AND status = 'dispatching'", [id, row.controller_id, row.generation]);
|
|
25
|
-
const answer = context.db.get("SELECT input_json, input_hash FROM run_controller_operations WHERE operation_id = ? AND controller_id = ? AND generation <= ? AND status = 'accepted'", [
|
|
43
|
+
const answer = context.db.get("SELECT input_json, input_hash FROM run_controller_operations WHERE operation_id = ? AND controller_id = ? AND generation <= ? AND status = 'accepted'", [baseId.slice(0, -suffix.length), row.controller_id, row.generation]);
|
|
26
44
|
const prompt = operation(input.operationId);
|
|
27
45
|
if (!answer || !prompt)
|
|
28
46
|
return false;
|
|
@@ -8,11 +8,13 @@ import { requireProjectByRoot } from "./projects.js";
|
|
|
8
8
|
import { loadExternalStageContext } from "./stage-context.js";
|
|
9
9
|
import { verifyRecoverySnapshot } from "./eval-snapshots.js";
|
|
10
10
|
import { recoveryGuard, resumeRunRecovery } from "./run-recovery.js";
|
|
11
|
-
import { prepareRunRecoveryDispatch } from "./run-recovery-runtime.js";
|
|
11
|
+
import { prepareRunRecoveryDispatch, retainRecoveryCapacityIntent } from "./run-recovery-runtime.js";
|
|
12
12
|
import { runtimeHarness } from "./execution-policy.js";
|
|
13
13
|
import { managedProcessBinding, managedProcessRuntimeHome, managedProcessStatus, processIsAlive, processTreeIsAlive, processStartedAt } from "./managed-processes.js";
|
|
14
14
|
import { requireRun, requireController, controllerReceipt, appendEvent, saveState, digest, errorRecord } from "./run-controller-state.js";
|
|
15
|
-
import { controllerAdapter, sessionArguments } from "./run-controller-adapter.js";
|
|
15
|
+
import { controllerAdapter, controllerEnvironment, sessionArguments } from "./run-controller-adapter.js";
|
|
16
|
+
import { runHarnessAdapter } from "./harness-adapter.js";
|
|
17
|
+
import { continueCodexCapacity } from "./codex-capacity.js";
|
|
16
18
|
import { spawnController, recordControllerLauncher } from "./run-controller-process.js";
|
|
17
19
|
import { readNativeDaemonState, requireManagedDaemonBinding } from "./managed-daemon-binding.js";
|
|
18
20
|
import { managedInvocationContext } from "./lifecycle-invocations.js";
|
|
@@ -194,7 +196,49 @@ export async function resumeControllerSession(context, row, manifest, state, ass
|
|
|
194
196
|
fs.writeFileSync(file, dispatch.prompt, { flag: "wx", mode: 0o600 });
|
|
195
197
|
state.turns += 1;
|
|
196
198
|
saveState(context, row, state);
|
|
197
|
-
const
|
|
199
|
+
const sendAck = (operationId, prompt) => {
|
|
200
|
+
let promptFile = file;
|
|
201
|
+
if (operationId !== dispatch.operation_id) {
|
|
202
|
+
promptFile = path.join(row.state_dir, `turn-${String(state.turns + 1).padStart(5, "0")}.md`);
|
|
203
|
+
fs.writeFileSync(promptFile, prompt, { flag: "wx", mode: 0o600 });
|
|
204
|
+
state.turns += 1;
|
|
205
|
+
saveState(context, row, state);
|
|
206
|
+
}
|
|
207
|
+
return call(["session", "prompt", "--session-id", session.id, "--prompt-file", promptFile, ...sessionArguments(context, manifest, session)], operationId);
|
|
208
|
+
};
|
|
209
|
+
let receipt;
|
|
210
|
+
try {
|
|
211
|
+
receipt = await sendAck(dispatch.operation_id, dispatch.prompt);
|
|
212
|
+
}
|
|
213
|
+
catch (error) {
|
|
214
|
+
if (runtimeHarness(session.profile) !== "codex-desktop")
|
|
215
|
+
throw error;
|
|
216
|
+
const continuationText = `The preceding native recovery ACK Turn failed before any observed tool or file effect. This is the same retained Session and recovery generation. First inspect the current RUN recovery state; only if the exact ACK still awaits acceptance, execute the exact standalone acceptance command below once. Never deliver it again after acceptance. Then stop at the original Stage or Work-graph boundary.\n\n${dispatch.prompt}`;
|
|
217
|
+
const inspect = (args, operationId) => runHarnessAdapter({ executable: manifest.adapters["codex-desktop"].executable,
|
|
218
|
+
args, env: controllerEnvironment(executionContext, manifest, session), runtimeOwner: session.owner, operationId, timeoutMs: 30_000 });
|
|
219
|
+
const resumed = await continueCodexCapacity({ error, sessionId: session.id, operationId: dispatch.operation_id, allowSettledDescendants: true, continuationText,
|
|
220
|
+
inspectOperation: id => inspect(["daemon", "operation", "--state-dir", session.state_dir, "--operation-id", id, "--json"], `${id}:capacity-observe`),
|
|
221
|
+
inspectSession: () => inspect(["session", "inspect", "--session-id", session.id, ...sessionArguments(context, manifest, session)], `${dispatch.operation_id}:capacity-inspect`),
|
|
222
|
+
assertCurrent: () => {
|
|
223
|
+
assertRecoveryOwner();
|
|
224
|
+
const guard = recoveryGuard(context, row.project_id, row.run_id);
|
|
225
|
+
const binding = context.db.get("SELECT status, daemon_id FROM run_recovery_bindings WHERE recovery_id = ?", [recovery.recovery_id]);
|
|
226
|
+
const ack = context.db.get("SELECT status FROM run_controller_operations WHERE operation_id = ? AND controller_id = ? AND generation = ?", [recovery.operation_id, row.controller_id, row.generation]);
|
|
227
|
+
if (guard?.status !== "resuming" || binding?.status !== "prepared" || binding.daemon_id !== daemonId || ack?.status !== "accepted")
|
|
228
|
+
throw new AppError("recovery_capacity_owner_changed", "Recovery ACK was accepted or superseded during capacity wait", 1);
|
|
229
|
+
const current = requireManagedDaemonBinding(executionContext, { stateDir: session.state_dir, projectId: row.project_id, runId: row.run_id, ownerId: session.owner.owner_id });
|
|
230
|
+
const metadata = JSON.parse(current.process.metadata_json);
|
|
231
|
+
if (current.process.id !== dispatch.process_id || current.state.daemon_id !== daemonId || current.process.state !== "running"
|
|
232
|
+
|| current.process.lease_expires_at <= context.now() || metadata.run_generation !== row.generation || !processIsAlive(current.process))
|
|
233
|
+
throw new AppError("recovery_capacity_owner_changed", "Replacement native owner changed during capacity wait", 1);
|
|
234
|
+
},
|
|
235
|
+
retainIntent: intent => retainRecoveryCapacityIntent(executionContext, { projectId: row.project_id, runId: row.run_id, recoveryId: recovery.recovery_id,
|
|
236
|
+
controllerId: row.controller_id, generation: row.generation, daemonId, intent, prompt: continuationText }),
|
|
237
|
+
prompt: (operationId, prompt) => sendAck(operationId, prompt) });
|
|
238
|
+
if (!resumed)
|
|
239
|
+
throw error;
|
|
240
|
+
receipt = resumed;
|
|
241
|
+
}
|
|
198
242
|
assertRecoveryOwner();
|
|
199
243
|
if (recoveryGuard(context, row.project_id, row.run_id)?.status !== "recovered")
|
|
200
244
|
throw new AppError("trusted_recovery_acceptance_required", "Recovery Turn ended without its native acknowledgement", 1);
|
|
@@ -30,9 +30,12 @@ import { requireRun, requireController, controllerReceipt, appendEvent, saveStat
|
|
|
30
30
|
import { assertNoCorrelatedRejection } from "./continuation-outcome.js";
|
|
31
31
|
import { spawnController, recordControllerLauncher, claimControllerProcess, markControllerDispatchReady } from "./run-controller-process.js";
|
|
32
32
|
import { controllerAdapter, controllerEnvironment, sessionArguments } from "./run-controller-adapter.js";
|
|
33
|
+
import { continueCodexCapacity } from "./codex-capacity.js";
|
|
33
34
|
import { resumeControllerSession, failControllerRecovery } from "./run-controller-recovery.js";
|
|
34
35
|
import { captureControllerBoundary } from "./run-controller-capture.js";
|
|
35
36
|
import { requestRunControl } from "./run-control-worker.js";
|
|
37
|
+
import { storageSessionId } from "./session-identity.js";
|
|
38
|
+
import { terminalCodexOverload } from "../harness-runtime/lib/dd-codex.mjs";
|
|
36
39
|
export { resumeRunController } from "./run-controller-recovery.js";
|
|
37
40
|
export function prepareRunControllerLaunch(context, input) {
|
|
38
41
|
if (!/^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/.test(input.requestId) || !/^[a-f0-9]{64}$/.test(input.contextSha256))
|
|
@@ -526,16 +529,61 @@ async function executeController(context, row, manifest, state, assertOwnership,
|
|
|
526
529
|
throw new AppError("settlement_reconciliation_required", "The managed Session still has an unsettled productive predecessor", 1, { session_id: session.id, settlement: current });
|
|
527
530
|
}
|
|
528
531
|
};
|
|
529
|
-
const prompt = async (session, text, operationId) => {
|
|
532
|
+
const prompt = async (session, text, operationId, kind) => {
|
|
533
|
+
const hitl = kind === "hitl";
|
|
530
534
|
await reconcilePromptPredecessor(session);
|
|
531
|
-
const
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
+
const firstOperationId = operationId ?? `${row.controller_id}:prompt:${crypto.randomUUID()}`;
|
|
536
|
+
const send = (value, id) => {
|
|
537
|
+
const file = path.join(row.state_dir, `turn-${String(state.turns + 1).padStart(5, "0")}.md`);
|
|
538
|
+
fs.writeFileSync(file, `${value}\n`, { flag: "wx", mode: 0o600 });
|
|
539
|
+
state.turns += 1;
|
|
540
|
+
saveState(context, row, state);
|
|
541
|
+
return controllerAdapter(executionContext, row, manifest, session, ["session", "prompt", "--session-id", session.id, "--prompt-file", file, ...sessionArguments(context, manifest, session)], assertOwner, id, cancellation.signal);
|
|
542
|
+
};
|
|
535
543
|
let receipt;
|
|
536
|
-
const adapter = controllerAdapter(executionContext, row, manifest, session, ["session", "prompt", "--session-id", session.id, "--prompt-file", file, ...sessionArguments(context, manifest, session)], assertOwner, operationId, cancellation.signal);
|
|
537
544
|
try {
|
|
538
|
-
|
|
545
|
+
try {
|
|
546
|
+
receipt = await send(text, firstOperationId);
|
|
547
|
+
}
|
|
548
|
+
catch (error) {
|
|
549
|
+
if (runtimeHarness(session.profile) !== "codex-desktop")
|
|
550
|
+
throw error;
|
|
551
|
+
const resumeCommand = hitl ? text.split("\n\n")[1] : undefined;
|
|
552
|
+
if (hitl && !resumeCommand?.includes(" stage resume "))
|
|
553
|
+
throw error;
|
|
554
|
+
const inspect = (args, id) => runHarnessAdapter({ executable: manifest.adapters["codex-desktop"].executable,
|
|
555
|
+
args, env: controllerEnvironment(context, manifest, session), runtimeOwner: session.owner, operationId: id, timeoutMs: 30_000, signal: cancellation.signal });
|
|
556
|
+
const resumed = await continueCodexCapacity({ error, sessionId: session.id, operationId: firstOperationId,
|
|
557
|
+
inspectOperation: id => inspect(["daemon", "operation", "--state-dir", session.state_dir, "--operation-id", id, "--json"], `${id}:capacity-observe`),
|
|
558
|
+
inspectSession: () => inspect(["session", "inspect", "--session-id", session.id, ...sessionArguments(context, manifest, session)], `${firstOperationId}:capacity-inspect`),
|
|
559
|
+
assertCurrent: () => {
|
|
560
|
+
assertOwner();
|
|
561
|
+
const view = getFlowRunStatus(executionContext, { projectRoot: manifest.request.project_root, runId: row.run_id });
|
|
562
|
+
const next = view.continuation;
|
|
563
|
+
if (hitl) {
|
|
564
|
+
const answerId = firstOperationId.slice(0, -`:prompt:${row.generation}`.length);
|
|
565
|
+
const answer = context.db.get("SELECT input_json FROM run_controller_operations WHERE operation_id = ? AND controller_id = ? AND status = 'accepted'", [answerId, row.controller_id]);
|
|
566
|
+
const accepted = answer && JSON.parse(answer.input_json);
|
|
567
|
+
const pause = view.index.stage_runs.find(item => item.stage === accepted?.stage)?.pause;
|
|
568
|
+
if (next.kind !== "paused" || next.stage !== state.stage || !accepted || pause?.id !== accepted.pause_id || pause.work_id !== accepted.work_id || pause.answer_path)
|
|
569
|
+
throw new AppError("capacity_continuation_blocked", "Accepted HITL pause changed during capacity wait", 1, { continuation: next });
|
|
570
|
+
}
|
|
571
|
+
else if (next.stage !== state.stage || ["terminal", "paused", "blocked", "wait"].includes(next.kind)) {
|
|
572
|
+
throw new AppError("capacity_continuation_blocked", "RUN changed while native capacity was unavailable", 1, { continuation: next });
|
|
573
|
+
}
|
|
574
|
+
},
|
|
575
|
+
retainIntent: intent => context.db.writeTransaction(() => {
|
|
576
|
+
const changed = context.db.run("UPDATE run_controller_operations SET status = 'failed', receipt_json = ?, updated_at = ? WHERE operation_id = ? AND controller_id = ? AND generation = ? AND status = 'outcome_unknown'", [JSON.stringify({ capacity_intent: intent }), context.now(), intent.predecessor, row.controller_id, row.generation]);
|
|
577
|
+
if (changed.changes !== 1)
|
|
578
|
+
throw new AppError("capacity_continuation_blocked", "Failed native operation changed before continuation", 1);
|
|
579
|
+
appendEvent(context, row.controller_id, "capacity_continuation", intent);
|
|
580
|
+
}),
|
|
581
|
+
prompt: (id, value) => send(value, id),
|
|
582
|
+
...(resumeCommand ? { continuationText: `The preceding Turn failed before any tool call. Continue the same accepted HITL answer in this Session. If the pause is still current, run this exact previously issued standalone command once without changing the answer:\n\n${resumeCommand}\n\nDo not accept or deliver the answer again if the pause has changed.` } : {}) });
|
|
583
|
+
if (!resumed)
|
|
584
|
+
throw error;
|
|
585
|
+
receipt = resumed;
|
|
586
|
+
}
|
|
539
587
|
checkOutcomes();
|
|
540
588
|
if (fatal)
|
|
541
589
|
throw fatal;
|
|
@@ -545,7 +593,6 @@ async function executeController(context, row, manifest, state, assertOwnership,
|
|
|
545
593
|
throw error;
|
|
546
594
|
await stopAfterFailure(error);
|
|
547
595
|
cancellation.abort();
|
|
548
|
-
await adapter.catch(() => { });
|
|
549
596
|
throw fatal ?? error;
|
|
550
597
|
}
|
|
551
598
|
state.last_receipt = receipt;
|
|
@@ -613,7 +660,7 @@ async function executeController(context, row, manifest, state, assertOwnership,
|
|
|
613
660
|
// The accepted answer is logical intent; only its generation-bound
|
|
614
661
|
// adapter operation represents a physical dispatch that stop must drain.
|
|
615
662
|
const command = managedLifecycleCommand(executionContext, `${flowCommand(executionContext)} stage resume ${run.id} --stage ${next.stage} --work ${input.work_id} --project-root ${quote([run.project_root])} --answer-file ${quote([input.file])} --json`, { replaceUnobservedAnswer: true });
|
|
616
|
-
const receipt = await prompt(state.sessions[state.current_session], controllerStageResumePrompt(pause.id, command), `${answer.operation_id}:prompt:${row.generation}
|
|
663
|
+
const receipt = await prompt(state.sessions[state.current_session], controllerStageResumePrompt(pause.id, command), `${answer.operation_id}:prompt:${row.generation}`, "hitl");
|
|
617
664
|
const afterAnswer = getFlowRunStatus(executionContext, { projectRoot: run.project_root, runId: run.id });
|
|
618
665
|
if (afterAnswer.index.stage_runs.some(stage => stage.status === "paused" && stage.pause?.id === pause.id)) {
|
|
619
666
|
throw new AppError("controller_answer_not_applied", "The answer Turn completed without resuming the assigned HITL pause", 1, { pause_id: pause.id, operation_id: answer.operation_id, stage: next.stage, effect: "no_progress" });
|
|
@@ -712,7 +759,55 @@ async function executeController(context, row, manifest, state, assertOwnership,
|
|
|
712
759
|
throw new AppError("controller_attach_required", "An unfinished Stage has no managed coordinator", 1);
|
|
713
760
|
const session = state.sessions[state.current_session];
|
|
714
761
|
const baseline = new Set(state.native_baseline ?? []);
|
|
715
|
-
|
|
762
|
+
let children = controllerNativeChildren(state.last_receipt ?? {}, session.id).filter(child => !baseline.has(child.session_id));
|
|
763
|
+
if (runtimeHarness(session.profile) === "codex-desktop" && children.some(child => child.status === "failed")) {
|
|
764
|
+
const inspected = await call(session, ["session", "inspect", "--session-id", session.id, ...sessionArguments(context, manifest, session)], false);
|
|
765
|
+
state.last_receipt = inspected;
|
|
766
|
+
saveState(context, row, state);
|
|
767
|
+
children = controllerNativeChildren(inspected, session.id).filter(child => !baseline.has(child.session_id));
|
|
768
|
+
const failed = children.filter(child => child.status === "failed");
|
|
769
|
+
const child = failed.length === 1 && children.every(item => item === failed[0] || item.status === "completed") ? failed[0] : undefined;
|
|
770
|
+
const turnId = child?.terminal_turn_id;
|
|
771
|
+
const native = child && turnId ? { code: "turn_interrupted", details: { provider_session_id: child.session_id, turn_id: turnId, native_turn_id: turnId,
|
|
772
|
+
terminal_status: "failed", provider_error: child.provider_error } } : null;
|
|
773
|
+
const prior = child && state.child_capacity?.[child.session_id];
|
|
774
|
+
const links = child ? context.db.all("SELECT w.status FROM work_sessions ws JOIN works w ON w.work_id = ws.work_id WHERE w.project_id = ? AND w.run_id = ? AND ws.session_id = ?", [row.project_id, row.run_id, storageSessionId({ harness_id: "codex-desktop", session_id: child.session_id })]) : [];
|
|
775
|
+
if (child && turnId && native && terminalCodexOverload(native, child.session_id, turnId)
|
|
776
|
+
&& child.native_turn_items?.observed === true && child.native_turn_items.possible_effects === false
|
|
777
|
+
&& inspected.settled === true && child.parent_session_id === session.id && links.length <= 1
|
|
778
|
+
&& (!links.length || links[0].status === "running") && (prior?.attempts ?? 0) < 2) {
|
|
779
|
+
if (prior?.turn_id === turnId)
|
|
780
|
+
throw new AppError("capacity_child_followup_unconfirmed", "Parent continuation did not start a new native child Turn", 1, { child_session_id: child.session_id, turn_id: turnId });
|
|
781
|
+
const attempt = (prior?.attempts ?? 0) + 1;
|
|
782
|
+
state.child_capacity ??= {};
|
|
783
|
+
state.child_capacity[child.session_id] = { turn_id: turnId, attempts: attempt };
|
|
784
|
+
saveState(context, row, state);
|
|
785
|
+
const operationId = `${row.controller_id}:child:${digest([child.session_id, turnId]).slice(0, 16)}:followup:${attempt}`;
|
|
786
|
+
appendEvent(context, row.controller_id, "capacity_child_followup", { child_session_id: child.session_id, turn_id: turnId, attempt, operation_id: operationId });
|
|
787
|
+
await delay(attempt === 1 ? 5_000 : 15_000);
|
|
788
|
+
assertOwner();
|
|
789
|
+
const current = getFlowRunStatus(executionContext, { projectRoot: manifest.request.project_root, runId: row.run_id });
|
|
790
|
+
if (current.continuation.stage !== state.stage || ["terminal", "paused", "blocked", "wait"].includes(current.continuation.kind))
|
|
791
|
+
throw new AppError("capacity_child_followup_blocked", "RUN changed during native child capacity wait", 1, { child_session_id: child.session_id });
|
|
792
|
+
const fresh = await call(session, ["session", "inspect", "--session-id", session.id, ...sessionArguments(context, manifest, session)], false);
|
|
793
|
+
const still = controllerNativeChildren(fresh, session.id).find(item => item.session_id === child.session_id);
|
|
794
|
+
const currentLinks = context.db.all("SELECT w.status FROM work_sessions ws JOIN works w ON w.work_id = ws.work_id WHERE w.project_id = ? AND w.run_id = ? AND ws.session_id = ?", [row.project_id, row.run_id, storageSessionId({ harness_id: "codex-desktop", session_id: child.session_id })]);
|
|
795
|
+
if (fresh.settled !== true || !still || still.parent_session_id !== session.id || still.status !== "failed" || still.terminal_turn_id !== turnId
|
|
796
|
+
|| still.native_turn_items?.observed !== true || still.native_turn_items.possible_effects !== false || currentLinks.length > 1
|
|
797
|
+
|| currentLinks.some(link => link.status !== "running"))
|
|
798
|
+
throw new AppError("capacity_child_followup_blocked", "Native child or Work changed during capacity wait", 1, { child_session_id: child.session_id, turn_id: turnId });
|
|
799
|
+
assertOwner();
|
|
800
|
+
const text = `The existing native child ${child.session_id} has a confirmed effect-free failed Turn. Use the native followup_task tool exactly once for that same child ID with this short message: "Continue your assigned Work from its current durable state. Do not repeat an observed lifecycle command. Finish only the original Work." Do not spawn another child, claim another Work, or perform the child's Work in this parent Turn. End this Turn after the followup_task response.`;
|
|
801
|
+
await prompt(session, text, operationId, "child");
|
|
802
|
+
const after = await call(session, ["session", "inspect", "--session-id", session.id, ...sessionArguments(context, manifest, session)], false);
|
|
803
|
+
state.last_receipt = after;
|
|
804
|
+
saveState(context, row, state);
|
|
805
|
+
const observed = controllerNativeChildren(after, session.id).find(item => item.session_id === child.session_id);
|
|
806
|
+
if (!observed || observed.status === "failed" && observed.terminal_turn_id === turnId)
|
|
807
|
+
throw new AppError("capacity_child_followup_unconfirmed", "Native child did not advance to a new Turn after parent followup", 1, { child_session_id: child.session_id, turn_id: turnId });
|
|
808
|
+
continue;
|
|
809
|
+
}
|
|
810
|
+
}
|
|
716
811
|
const fanout = await nextControllerFanout(executionContext, { projectRoot: run.project_root, runId: run.id, stage: next.stage, harness: runtimeHarness(session.profile), sessionId: session.id, children, policy, profiles });
|
|
717
812
|
assertOwner();
|
|
718
813
|
if (fanout.kind === "none") {
|
|
@@ -56,6 +56,42 @@ export function prepareRunRecoveryDispatch(context, input) {
|
|
|
56
56
|
return dispatch;
|
|
57
57
|
});
|
|
58
58
|
}
|
|
59
|
+
/** Publish only an exact successor to a failed, effect-free recovery ACK Turn. */
|
|
60
|
+
export function retainRecoveryCapacityIntent(context, input) {
|
|
61
|
+
const { intent } = input;
|
|
62
|
+
if (![1, 2].includes(intent.ordinal) || !intent.turn_id || !intent.not_before)
|
|
63
|
+
throw new AppError("recovery_capacity_intent_invalid", "Recovery capacity successor is invalid", 1);
|
|
64
|
+
context.db.writeTransaction(() => {
|
|
65
|
+
const guard = recoveryGuard(context, input.projectId, input.runId);
|
|
66
|
+
const binding = context.db.get("SELECT * FROM run_recovery_bindings WHERE recovery_id = ? AND daemon_id = ? AND status = 'prepared'", [input.recoveryId, input.daemonId]);
|
|
67
|
+
if (!guard || guard.recovery_id !== input.recoveryId || guard.generation !== input.generation || guard.status !== "resuming" || !binding?.runtime_dispatch_json)
|
|
68
|
+
throw new AppError("recovery_capacity_owner_changed", "Recovery ACK no longer owns the current generation", 1);
|
|
69
|
+
const controller = context.db.get("SELECT generation, status FROM run_controllers WHERE controller_id = ? AND project_id = ? AND run_id = ?", [input.controllerId, input.projectId, input.runId]);
|
|
70
|
+
const control = context.db.get("SELECT status FROM run_controls WHERE recovery_id = ?", [input.recoveryId]);
|
|
71
|
+
if (controller?.generation !== input.generation || ["completed", "cancelled", "superseded"].includes(controller.status) || control?.status !== "resuming")
|
|
72
|
+
throw new AppError("recovery_capacity_owner_changed", "Recovery controller or control changed before successor publication", 1);
|
|
73
|
+
const dispatch = JSON.parse(binding.runtime_dispatch_json);
|
|
74
|
+
const currentProcess = getResourceDatabase(resourceHome(context)).get("SELECT * FROM managed_processes WHERE id = ?", [dispatch.process_id]);
|
|
75
|
+
if (!currentProcess || currentProcess.state !== "running" || currentProcess.lease_expires_at <= context.now() || currentProcess.owner_id !== dispatch.owner_id
|
|
76
|
+
|| currentProcess.project_id !== input.projectId || currentProcess.run_id !== input.runId)
|
|
77
|
+
throw new AppError("recovery_capacity_owner_changed", "Replacement daemon lease changed before successor publication", 1);
|
|
78
|
+
const prior = dispatch.capacity_intents ?? [];
|
|
79
|
+
const predecessor = intent.ordinal === 1 ? dispatch.operation_id : prior[0]?.operation_id;
|
|
80
|
+
if (prior.length !== intent.ordinal - 1 || predecessor !== intent.predecessor || intent.operation_id !== `${dispatch.operation_id}:capacity:${intent.ordinal}:session.prompt`)
|
|
81
|
+
throw new AppError("recovery_capacity_intent_conflict", "Recovery capacity successor does not extend the pinned ACK", 1);
|
|
82
|
+
const failed = context.db.get("SELECT status FROM run_controller_operations WHERE operation_id = ? AND controller_id = ? AND generation = ?", [intent.predecessor, input.controllerId, input.generation]);
|
|
83
|
+
if (failed?.status !== "outcome_unknown")
|
|
84
|
+
throw new AppError("recovery_capacity_intent_conflict", "Recovery predecessor is not the failed native operation", 1);
|
|
85
|
+
const next = { ...intent, prompt_sha256: crypto.createHash("sha256").update(input.prompt).digest("hex") };
|
|
86
|
+
const updated = { ...dispatch, capacity_intents: [...prior, next] };
|
|
87
|
+
const changed = context.db.run("UPDATE run_recovery_bindings SET runtime_dispatch_json = ? WHERE recovery_id = ? AND daemon_id = ? AND status = 'prepared' AND runtime_dispatch_json = ?", [JSON.stringify(updated), input.recoveryId, input.daemonId, binding.runtime_dispatch_json]);
|
|
88
|
+
if (changed.changes !== 1)
|
|
89
|
+
throw new AppError("recovery_capacity_intent_conflict", "Recovery dispatch changed before successor publication", 1);
|
|
90
|
+
const marked = context.db.run("UPDATE run_controller_operations SET status = 'failed', receipt_json = ?, updated_at = ? WHERE operation_id = ? AND controller_id = ? AND generation = ? AND status = 'outcome_unknown'", [JSON.stringify({ capacity_intent: next }), context.now(), intent.predecessor, input.controllerId, input.generation]);
|
|
91
|
+
if (marked.changes !== 1)
|
|
92
|
+
throw new AppError("recovery_capacity_intent_conflict", "Recovery predecessor changed before successor publication", 1);
|
|
93
|
+
});
|
|
94
|
+
}
|
|
59
95
|
/** The caller has already checked PID/start/lease and runtime-home identity. */
|
|
60
96
|
export function recoveryRuntimeAdmission(context, record, input) {
|
|
61
97
|
if (!record.project_id || !record.run_id || !input.daemonId)
|
|
@@ -73,8 +109,13 @@ export function recoveryRuntimeAdmission(context, record, input) {
|
|
|
73
109
|
if (metadata.run_generation !== guard.generation)
|
|
74
110
|
return false;
|
|
75
111
|
if (guard.status === "recovered" && binding.status === "accepted")
|
|
76
|
-
return
|
|
112
|
+
return !input.operationId?.startsWith(`${dispatch.operation_id}:capacity:`);
|
|
77
113
|
if (guard.status === "resuming" && binding.status === "prepared" && input.operation === "session.resume")
|
|
78
114
|
return typeof dispatch.resume_operation_id === "string" && input.operationId === dispatch.resume_operation_id && input.nativeSessionId === binding.provider_session_id;
|
|
79
|
-
|
|
115
|
+
const active = dispatch.capacity_intents?.at(-1);
|
|
116
|
+
if (active && active.not_before > context.now())
|
|
117
|
+
return false;
|
|
118
|
+
return guard.status === "resuming" && binding.status === "prepared" && input.operation === "session.prompt"
|
|
119
|
+
&& input.operationId === (active?.operation_id ?? dispatch.operation_id) && input.nativeSessionId === binding.provider_session_id
|
|
120
|
+
&& input.promptSha256 === (active?.prompt_sha256 ?? dispatch.prompt_sha256);
|
|
80
121
|
}
|