agentxchain 2.125.0 → 2.126.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/commands/run.js +26 -1
- package/src/commands/status.js +48 -7
- package/src/commands/turn.js +29 -5
- package/src/lib/dashboard/timeout-status.js +11 -2
- package/src/lib/run-loop.js +129 -4
- package/src/lib/timeout-evaluator.js +87 -0
package/package.json
CHANGED
package/src/commands/run.js
CHANGED
|
@@ -300,7 +300,7 @@ export async function executeGovernedRun(context, opts = {}) {
|
|
|
300
300
|
|
|
301
301
|
// ── Route to adapter ──────────────────────────────────────────────
|
|
302
302
|
const adapterOpts = {
|
|
303
|
-
signal: controller.signal,
|
|
303
|
+
signal: combineAbortSignals(controller.signal, ctx.dispatchAbortSignal),
|
|
304
304
|
onStatus: (msg) => log(chalk.dim(` ${msg}`)),
|
|
305
305
|
verifyManifest: true,
|
|
306
306
|
turnId: turn.turn_id,
|
|
@@ -557,3 +557,28 @@ function printManualQaFallback(log = console.log) {
|
|
|
557
557
|
log(chalk.dim(' - Then recover the retained QA turn with: agentxchain step --resume'));
|
|
558
558
|
log(chalk.dim(' - Guide: https://agentxchain.dev/docs/getting-started'));
|
|
559
559
|
}
|
|
560
|
+
|
|
561
|
+
function combineAbortSignals(primarySignal, secondarySignal) {
|
|
562
|
+
if (!secondarySignal) {
|
|
563
|
+
return primarySignal;
|
|
564
|
+
}
|
|
565
|
+
if (!primarySignal) {
|
|
566
|
+
return secondarySignal;
|
|
567
|
+
}
|
|
568
|
+
if (typeof AbortSignal.any === 'function') {
|
|
569
|
+
return AbortSignal.any([primarySignal, secondarySignal]);
|
|
570
|
+
}
|
|
571
|
+
|
|
572
|
+
const combined = new AbortController();
|
|
573
|
+
const forward = (signal) => {
|
|
574
|
+
if (!signal) return;
|
|
575
|
+
if (signal.aborted) {
|
|
576
|
+
combined.abort(signal.reason);
|
|
577
|
+
return;
|
|
578
|
+
}
|
|
579
|
+
signal.addEventListener('abort', () => combined.abort(signal.reason), { once: true });
|
|
580
|
+
};
|
|
581
|
+
forward(primarySignal);
|
|
582
|
+
forward(secondarySignal);
|
|
583
|
+
return combined.signal;
|
|
584
|
+
}
|
package/src/commands/status.js
CHANGED
|
@@ -13,7 +13,7 @@ import { getContinuityStatus } from '../lib/continuity-status.js';
|
|
|
13
13
|
import { getConnectorHealth } from '../lib/connector-health.js';
|
|
14
14
|
import { readRepoDecisions, summarizeRepoDecisions } from '../lib/repo-decisions.js';
|
|
15
15
|
import { deriveWorkflowKitArtifacts } from '../lib/workflow-kit-artifacts.js';
|
|
16
|
-
import { evaluateTimeouts } from '../lib/timeout-evaluator.js';
|
|
16
|
+
import { evaluateTimeouts, computeTimeoutBudget } from '../lib/timeout-evaluator.js';
|
|
17
17
|
import { evaluateApprovalSlaReminders } from '../lib/notification-runner.js';
|
|
18
18
|
import { summarizeRunProvenance } from '../lib/run-provenance.js';
|
|
19
19
|
import { readRecentRunEventSummary } from '../lib/recent-event-summary.js';
|
|
@@ -283,7 +283,19 @@ function renderGovernedStatus(context, opts) {
|
|
|
283
283
|
elapsedTag = m > 0 ? ` — ${m}m ${s % 60}s` : ` — ${s}s`;
|
|
284
284
|
}
|
|
285
285
|
}
|
|
286
|
-
|
|
286
|
+
let budgetTag = '';
|
|
287
|
+
if (config.timeouts?.per_turn_minutes && turn.started_at) {
|
|
288
|
+
const tb = computeTimeoutBudget({ config, state, turn, now: new Date() });
|
|
289
|
+
const tBudget = tb.find((b) => b.scope === 'turn');
|
|
290
|
+
if (tBudget) {
|
|
291
|
+
if (tBudget.exceeded) {
|
|
292
|
+
budgetTag = ` ${chalk.red('[TIMEOUT]')}`;
|
|
293
|
+
} else {
|
|
294
|
+
budgetTag = ` ${chalk.dim(`[${tBudget.remaining_minutes}m left]`)}`;
|
|
295
|
+
}
|
|
296
|
+
}
|
|
297
|
+
}
|
|
298
|
+
console.log(` ${marker} ${turn.turn_id} — ${chalk.bold(turn.assigned_role)} (${statusLabel}) [attempt ${turn.attempt}]${elapsedTag}${budgetTag}`);
|
|
287
299
|
if (turn.status === 'conflicted' && turn.conflict_state) {
|
|
288
300
|
const cs = turn.conflict_state;
|
|
289
301
|
const files = cs.conflict_error?.conflicting_files || [];
|
|
@@ -314,6 +326,21 @@ function renderGovernedStatus(context, opts) {
|
|
|
314
326
|
console.log(` ${chalk.dim('Elapsed:')} ${elapsed}`);
|
|
315
327
|
}
|
|
316
328
|
}
|
|
329
|
+
// Turn-level timeout budget inline with turn info
|
|
330
|
+
if (config.timeouts?.per_turn_minutes && singleActiveTurn.started_at) {
|
|
331
|
+
const turnBudgets = computeTimeoutBudget({ config, state, turn: singleActiveTurn, now: new Date() });
|
|
332
|
+
const turnBudget = turnBudgets.find((b) => b.scope === 'turn');
|
|
333
|
+
if (turnBudget) {
|
|
334
|
+
if (turnBudget.exceeded) {
|
|
335
|
+
console.log(` ${chalk.dim('Budget:')} ${chalk.red(`EXCEEDED — was ${turnBudget.limit_minutes}m, over by ${turnBudget.elapsed_minutes - turnBudget.limit_minutes}m`)}`);
|
|
336
|
+
} else {
|
|
337
|
+
const remMins = Math.floor(turnBudget.remaining_seconds / 60);
|
|
338
|
+
const remSecs = turnBudget.remaining_seconds % 60;
|
|
339
|
+
const remLabel = remMins > 0 ? `${remMins}m ${remSecs}s` : `${remSecs}s`;
|
|
340
|
+
console.log(` ${chalk.dim('Budget:')} ${chalk.green(`${remLabel} remaining`)} of ${turnBudget.limit_minutes}m (deadline ${new Date(turnBudget.deadline_iso).toLocaleTimeString()})`);
|
|
341
|
+
}
|
|
342
|
+
}
|
|
343
|
+
}
|
|
317
344
|
if (singleActiveTurn.status === 'conflicted' && singleActiveTurn.conflict_state) {
|
|
318
345
|
const cs = singleActiveTurn.conflict_state;
|
|
319
346
|
const files = cs.conflict_error?.conflicting_files || [];
|
|
@@ -474,19 +501,22 @@ function renderGovernedStatus(context, opts) {
|
|
|
474
501
|
renderWorkflowKitArtifactsSection(workflowKitArtifacts);
|
|
475
502
|
|
|
476
503
|
if (config.timeouts && (state?.status === 'active' || approvalPending)) {
|
|
504
|
+
const nowDate = new Date();
|
|
477
505
|
const activeTurn = state?.status === 'active' ? getActiveTurn(state) : null;
|
|
478
506
|
const turnResult = activeTurn ? { role: activeTurn.assigned_role } : undefined;
|
|
479
|
-
const timeoutEval = evaluateTimeouts({ config, state, turn: activeTurn, turnResult, now:
|
|
507
|
+
const timeoutEval = evaluateTimeouts({ config, state, turn: activeTurn, turnResult, now: nowDate.toISOString() });
|
|
480
508
|
const allItems = [...timeoutEval.exceeded, ...timeoutEval.warnings];
|
|
481
|
-
|
|
509
|
+
// Compute full budget for phase/run scopes (turn budget is shown inline with turn info above)
|
|
510
|
+
const budgets = computeTimeoutBudget({ config, state, turn: activeTurn, now: nowDate })
|
|
511
|
+
.filter((b) => b.scope !== 'turn'); // turn budget already shown inline
|
|
512
|
+
|
|
513
|
+
if (allItems.length > 0 || budgets.length > 0 || approvalPending) {
|
|
482
514
|
console.log('');
|
|
483
515
|
console.log(` ${chalk.dim('Timeouts:')}`);
|
|
484
516
|
if (approvalPending) {
|
|
485
517
|
console.log(` ${chalk.yellow('◷')} approval wait does not mutate timeout state; phase/run clocks keep ticking until the next accepted turn`);
|
|
486
518
|
}
|
|
487
|
-
|
|
488
|
-
console.log(` ${chalk.dim('No current phase/run timeout pressure.')}`);
|
|
489
|
-
}
|
|
519
|
+
// Show exceeded/warned items
|
|
490
520
|
for (const item of allItems) {
|
|
491
521
|
const isExceeded = timeoutEval.exceeded.includes(item);
|
|
492
522
|
const elapsed = item.elapsed_minutes != null ? `${item.elapsed_minutes}m` : '?';
|
|
@@ -495,6 +525,17 @@ function renderGovernedStatus(context, opts) {
|
|
|
495
525
|
const label = isExceeded ? chalk.red(`EXCEEDED ${item.scope}`) : chalk.yellow(`${item.scope}`);
|
|
496
526
|
console.log(` ${icon} ${label}: ${elapsed}/${limit} (action: ${item.action || 'n/a'})`);
|
|
497
527
|
}
|
|
528
|
+
// Show remaining budget for non-exceeded phase/run scopes
|
|
529
|
+
const exceededScopes = new Set(allItems.map((i) => `${i.scope}:${i.phase || ''}`));
|
|
530
|
+
for (const b of budgets) {
|
|
531
|
+
const key = `${b.scope}:${b.phase || ''}`;
|
|
532
|
+
if (exceededScopes.has(key)) continue; // already shown as exceeded above
|
|
533
|
+
const scopeLabel = b.scope === 'phase' ? `phase (${b.phase})` : b.scope;
|
|
534
|
+
console.log(` ${chalk.green('✓')} ${scopeLabel}: ${b.elapsed_minutes}m/${b.limit_minutes}m — ${chalk.green(`${b.remaining_minutes}m remaining`)}`);
|
|
535
|
+
}
|
|
536
|
+
if (approvalPending && allItems.length === 0 && budgets.length === 0) {
|
|
537
|
+
console.log(` ${chalk.dim('No current phase/run timeout pressure.')}`);
|
|
538
|
+
}
|
|
498
539
|
}
|
|
499
540
|
}
|
|
500
541
|
|
package/src/commands/turn.js
CHANGED
|
@@ -3,6 +3,7 @@ import { join } from 'path';
|
|
|
3
3
|
import chalk from 'chalk';
|
|
4
4
|
import { loadProjectContext, loadProjectState } from '../lib/config.js';
|
|
5
5
|
import { getActiveTurnCount, getActiveTurns } from '../lib/governed-state.js';
|
|
6
|
+
import { computeTimeoutBudget } from '../lib/timeout-evaluator.js';
|
|
6
7
|
import {
|
|
7
8
|
getDispatchAssignmentPath,
|
|
8
9
|
getDispatchContextPath,
|
|
@@ -54,11 +55,11 @@ export function turnShowCommand(turnId, opts) {
|
|
|
54
55
|
}
|
|
55
56
|
|
|
56
57
|
if (opts.json) {
|
|
57
|
-
console.log(JSON.stringify(buildTurnPayload(selectedTurnId, turn, state, artifacts, assignment), null, 2));
|
|
58
|
+
console.log(JSON.stringify(buildTurnPayload(selectedTurnId, turn, state, artifacts, assignment, context.config), null, 2));
|
|
58
59
|
return;
|
|
59
60
|
}
|
|
60
61
|
|
|
61
|
-
printTurnSummary(selectedTurnId, turn, state, artifacts, assignment);
|
|
62
|
+
printTurnSummary(selectedTurnId, turn, state, artifacts, assignment, context.config);
|
|
62
63
|
}
|
|
63
64
|
|
|
64
65
|
function requireGovernedContext() {
|
|
@@ -118,9 +119,9 @@ function buildArtifactIndex(root, turnId) {
|
|
|
118
119
|
);
|
|
119
120
|
}
|
|
120
121
|
|
|
121
|
-
function buildTurnPayload(turnId, turn, state, artifacts, assignment) {
|
|
122
|
+
function buildTurnPayload(turnId, turn, state, artifacts, assignment, config) {
|
|
122
123
|
const elapsedMs = getElapsedMs(turn.started_at);
|
|
123
|
-
|
|
124
|
+
const payload = {
|
|
124
125
|
turn_id: turnId,
|
|
125
126
|
run_id: state.run_id || assignment?.run_id || null,
|
|
126
127
|
phase: state.phase || assignment?.phase || null,
|
|
@@ -140,9 +141,17 @@ function buildTurnPayload(turnId, turn, state, artifacts, assignment) {
|
|
|
140
141
|
}]),
|
|
141
142
|
),
|
|
142
143
|
};
|
|
144
|
+
// Add timeout budget if configured
|
|
145
|
+
if (config?.timeouts) {
|
|
146
|
+
const budgets = computeTimeoutBudget({ config, state, turn, now: new Date() });
|
|
147
|
+
if (budgets.length > 0) {
|
|
148
|
+
payload.timeout_budget = budgets;
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
return payload;
|
|
143
152
|
}
|
|
144
153
|
|
|
145
|
-
function printTurnSummary(turnId, turn, state, artifacts, assignment) {
|
|
154
|
+
function printTurnSummary(turnId, turn, state, artifacts, assignment, config) {
|
|
146
155
|
const elapsedMs = getElapsedMs(turn.started_at);
|
|
147
156
|
console.log('');
|
|
148
157
|
console.log(chalk.bold(` Turn: ${chalk.cyan(turnId)}`));
|
|
@@ -159,6 +168,21 @@ function printTurnSummary(turnId, turn, state, artifacts, assignment) {
|
|
|
159
168
|
if (elapsedMs != null) {
|
|
160
169
|
console.log(` ${chalk.dim('Elapsed:')} ${formatElapsed(elapsedMs)}`);
|
|
161
170
|
}
|
|
171
|
+
// Timeout budget per scope
|
|
172
|
+
if (config?.timeouts) {
|
|
173
|
+
const budgets = computeTimeoutBudget({ config, state, turn, now: new Date() });
|
|
174
|
+
for (const b of budgets) {
|
|
175
|
+
const scopeLabel = b.scope === 'phase' ? `phase (${b.phase})` : b.scope;
|
|
176
|
+
if (b.exceeded) {
|
|
177
|
+
console.log(` ${chalk.dim('Timeout:')} ${chalk.red(`${scopeLabel} EXCEEDED`)} — was ${b.limit_minutes}m, over by ${b.elapsed_minutes - b.limit_minutes}m`);
|
|
178
|
+
} else {
|
|
179
|
+
const remMins = Math.floor(b.remaining_seconds / 60);
|
|
180
|
+
const remSecs = b.remaining_seconds % 60;
|
|
181
|
+
const remLabel = remMins > 0 ? `${remMins}m ${remSecs}s` : `${remSecs}s`;
|
|
182
|
+
console.log(` ${chalk.dim('Timeout:')} ${scopeLabel} — ${chalk.green(`${remLabel} remaining`)} of ${b.limit_minutes}m`);
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
}
|
|
162
186
|
console.log(` ${chalk.dim('Dispatch:')} ${getDispatchTurnDir(turnId)}`);
|
|
163
187
|
if (assignment?.staging_result_path) {
|
|
164
188
|
console.log(` ${chalk.dim('Staging:')} ${assignment.staging_result_path}`);
|
|
@@ -9,7 +9,7 @@
|
|
|
9
9
|
|
|
10
10
|
import { join } from 'path';
|
|
11
11
|
import { loadProjectContext, loadProjectState } from '../config.js';
|
|
12
|
-
import { evaluateTimeouts } from '../timeout-evaluator.js';
|
|
12
|
+
import { evaluateTimeouts, computeTimeoutBudget } from '../timeout-evaluator.js';
|
|
13
13
|
import { readJsonlFile } from './state-reader.js';
|
|
14
14
|
|
|
15
15
|
/**
|
|
@@ -209,7 +209,15 @@ export function readTimeoutStatus(workspacePath) {
|
|
|
209
209
|
const configSummary = buildTimeoutConfigSummary(timeouts, config.routing);
|
|
210
210
|
|
|
211
211
|
// Live timeout evaluation — only meaningful when the run is active
|
|
212
|
-
const
|
|
212
|
+
const nowDate = new Date();
|
|
213
|
+
const live = evaluateDashboardTimeoutPressure(config, state, nowDate);
|
|
214
|
+
|
|
215
|
+
// Compute remaining budget for all configured scopes
|
|
216
|
+
const activeTurnsList = getActiveTurns(state);
|
|
217
|
+
const primaryTurn = activeTurnsList.length === 1 ? activeTurnsList[0] : null;
|
|
218
|
+
const budget = (state?.status === 'active' || Boolean(state?.pending_phase_transition || state?.pending_run_completion))
|
|
219
|
+
? computeTimeoutBudget({ config, state, turn: primaryTurn, now: nowDate })
|
|
220
|
+
: [];
|
|
213
221
|
|
|
214
222
|
// Persisted timeout events from the decision ledger
|
|
215
223
|
const ledger = readJsonlFile(agentxchainDir, 'decision-ledger.jsonl');
|
|
@@ -223,6 +231,7 @@ export function readTimeoutStatus(workspacePath) {
|
|
|
223
231
|
configured: true,
|
|
224
232
|
config: configSummary,
|
|
225
233
|
live,
|
|
234
|
+
budget,
|
|
226
235
|
live_context: buildLiveContext(state),
|
|
227
236
|
events,
|
|
228
237
|
},
|
package/src/lib/run-loop.js
CHANGED
|
@@ -12,7 +12,7 @@
|
|
|
12
12
|
* - Never calls process dot exit
|
|
13
13
|
* - No stdout/stderr
|
|
14
14
|
* - No adapter dispatch (caller provides dispatch callback)
|
|
15
|
-
* -
|
|
15
|
+
* - Governed lifecycle operations import through runner-interface.js
|
|
16
16
|
*/
|
|
17
17
|
|
|
18
18
|
import {
|
|
@@ -21,6 +21,7 @@ import {
|
|
|
21
21
|
assignTurn,
|
|
22
22
|
acceptTurn,
|
|
23
23
|
rejectTurn,
|
|
24
|
+
markRunBlocked,
|
|
24
25
|
writeDispatchBundle,
|
|
25
26
|
getTurnStagingResultPath,
|
|
26
27
|
approvePhaseGate,
|
|
@@ -33,10 +34,11 @@ import {
|
|
|
33
34
|
} from './runner-interface.js';
|
|
34
35
|
|
|
35
36
|
import { runAdmissionControl } from './admission-control.js';
|
|
36
|
-
import { mkdirSync, writeFileSync } from 'fs';
|
|
37
|
+
import { appendFileSync, mkdirSync, writeFileSync } from 'fs';
|
|
37
38
|
import { join, dirname } from 'path';
|
|
38
39
|
import { evaluateApprovalSlaReminders } from './notification-runner.js';
|
|
39
40
|
import { readPreemptionMarker } from './intake.js';
|
|
41
|
+
import { buildTimeoutBlockedReason, evaluateTimeouts } from './timeout-evaluator.js';
|
|
40
42
|
|
|
41
43
|
const DEFAULT_MAX_TURNS = 50;
|
|
42
44
|
|
|
@@ -214,6 +216,7 @@ async function executeSequentialTurn(root, config, state, callbacks, emit, error
|
|
|
214
216
|
async function executeParallelTurns(root, config, state, maxConcurrent, callbacks, emit, errors) {
|
|
215
217
|
const history = [];
|
|
216
218
|
let acceptedCount = 0;
|
|
219
|
+
const timedOutDispatches = [];
|
|
217
220
|
|
|
218
221
|
// ── Collect active turns that need dispatch (retries) ────────────────
|
|
219
222
|
const activeTurns = getActiveTurns(state);
|
|
@@ -332,7 +335,7 @@ async function executeParallelTurns(root, config, state, maxConcurrent, callback
|
|
|
332
335
|
const dispatchResults = await Promise.allSettled(
|
|
333
336
|
contexts.map(async (ctx) => {
|
|
334
337
|
try {
|
|
335
|
-
return { ctx, result: await callbacks.dispatch
|
|
338
|
+
return { ctx, result: await dispatchWithTimeout(ctx, config, callbacks.dispatch) };
|
|
336
339
|
} catch (err) {
|
|
337
340
|
return { ctx, result: { accept: false, reason: `dispatch threw: ${err.message}` } };
|
|
338
341
|
}
|
|
@@ -350,6 +353,11 @@ async function executeParallelTurns(root, config, state, maxConcurrent, callback
|
|
|
350
353
|
const { turn } = ctx;
|
|
351
354
|
const roleId = turn.assigned_role;
|
|
352
355
|
|
|
356
|
+
if (dispatchResult?.timed_out === true) {
|
|
357
|
+
timedOutDispatches.push({ ctx, dispatchResult });
|
|
358
|
+
continue;
|
|
359
|
+
}
|
|
360
|
+
|
|
353
361
|
if (dispatchResult.accept) {
|
|
354
362
|
const absStaging = join(root, ctx.stagingPath);
|
|
355
363
|
mkdirSync(dirname(absStaging), { recursive: true });
|
|
@@ -405,6 +413,13 @@ async function executeParallelTurns(root, config, state, maxConcurrent, callback
|
|
|
405
413
|
}
|
|
406
414
|
}
|
|
407
415
|
|
|
416
|
+
if (timedOutDispatches.length > 0) {
|
|
417
|
+
const timedOut = timedOutDispatches[0];
|
|
418
|
+
const blocked = persistDispatchTimeout(root, config, timedOut.ctx.turn, timedOut.dispatchResult.timeout_result, errors);
|
|
419
|
+
emit({ type: 'blocked', state: blocked.state, reason: 'timeout:turn' });
|
|
420
|
+
return { terminal: true, ok: false, stop_reason: 'blocked', history, acceptedCount };
|
|
421
|
+
}
|
|
422
|
+
|
|
408
423
|
// ── Stall detection: if no turns were accepted and no new roles were ──
|
|
409
424
|
// ── assignable, terminate to avoid infinite re-dispatch loops. ────────
|
|
410
425
|
if (acceptedCount === 0 && history.length > 0) {
|
|
@@ -445,12 +460,18 @@ async function dispatchAndProcess(root, config, turn, assignState, callbacks, em
|
|
|
445
460
|
|
|
446
461
|
let dispatchResult;
|
|
447
462
|
try {
|
|
448
|
-
dispatchResult = await callbacks.dispatch
|
|
463
|
+
dispatchResult = await dispatchWithTimeout(context, config, callbacks.dispatch);
|
|
449
464
|
} catch (err) {
|
|
450
465
|
errors.push(`dispatch threw for ${roleId}: ${err.message}`);
|
|
451
466
|
return { terminal: true, ok: false, stop_reason: 'dispatch_error', history };
|
|
452
467
|
}
|
|
453
468
|
|
|
469
|
+
if (dispatchResult?.timed_out === true) {
|
|
470
|
+
const blocked = persistDispatchTimeout(root, config, turn, dispatchResult.timeout_result, errors);
|
|
471
|
+
emit({ type: 'blocked', state: blocked.state, reason: 'timeout:turn' });
|
|
472
|
+
return { terminal: true, ok: false, stop_reason: 'blocked', history };
|
|
473
|
+
}
|
|
474
|
+
|
|
454
475
|
if (dispatchResult.accept) {
|
|
455
476
|
const absStaging = join(root, stagingPath);
|
|
456
477
|
mkdirSync(dirname(absStaging), { recursive: true });
|
|
@@ -574,3 +595,107 @@ function makeResult(ok, stop_reason, state, turns_executed, turn_history, gates_
|
|
|
574
595
|
function noop() {}
|
|
575
596
|
|
|
576
597
|
export { DEFAULT_MAX_TURNS };
|
|
598
|
+
|
|
599
|
+
function buildTimeoutLedgerEntry(timeoutResult, timestamp, turnId, phase) {
|
|
600
|
+
return {
|
|
601
|
+
type: 'timeout',
|
|
602
|
+
scope: timeoutResult.scope,
|
|
603
|
+
phase: timeoutResult.phase || phase || null,
|
|
604
|
+
turn_id: turnId || null,
|
|
605
|
+
limit_minutes: timeoutResult.limit_minutes,
|
|
606
|
+
elapsed_minutes: timeoutResult.elapsed_minutes,
|
|
607
|
+
exceeded_by_minutes: timeoutResult.exceeded_by_minutes,
|
|
608
|
+
action: timeoutResult.action,
|
|
609
|
+
timestamp,
|
|
610
|
+
};
|
|
611
|
+
}
|
|
612
|
+
|
|
613
|
+
function appendJsonl(root, relPath, value) {
|
|
614
|
+
appendFileSync(join(root, relPath), `${JSON.stringify(value)}\n`);
|
|
615
|
+
}
|
|
616
|
+
|
|
617
|
+
function getDispatchTimeoutResult(config, state, turn, now = new Date()) {
|
|
618
|
+
const evaluation = evaluateTimeouts({ config, state, turn, now });
|
|
619
|
+
return evaluation.exceeded.find((entry) => entry.scope === 'turn' && entry.action === 'escalate') || null;
|
|
620
|
+
}
|
|
621
|
+
|
|
622
|
+
async function dispatchWithTimeout(context, config, dispatchFn) {
|
|
623
|
+
const timeoutResult = getDispatchTimeoutResult(config, context.state, context.turn);
|
|
624
|
+
if (!timeoutResult) {
|
|
625
|
+
return await dispatchFn(context);
|
|
626
|
+
}
|
|
627
|
+
|
|
628
|
+
const remainingMs = Math.max(
|
|
629
|
+
0,
|
|
630
|
+
timeoutResult.limit_minutes * 60 * 1000
|
|
631
|
+
- Math.max(0, new Date() - new Date(context.turn.started_at || context.turn.assigned_at || new Date())),
|
|
632
|
+
);
|
|
633
|
+
const abortController = new AbortController();
|
|
634
|
+
if (remainingMs === 0) {
|
|
635
|
+
abortController.abort(new Error(`Turn timeout exceeded after ${timeoutResult.limit_minutes}m`));
|
|
636
|
+
return {
|
|
637
|
+
timed_out: true,
|
|
638
|
+
timeout_result: timeoutResult,
|
|
639
|
+
};
|
|
640
|
+
}
|
|
641
|
+
const enrichedContext = {
|
|
642
|
+
...context,
|
|
643
|
+
dispatchTimeoutMs: remainingMs,
|
|
644
|
+
dispatchDeadlineAt: new Date(Date.now() + remainingMs).toISOString(),
|
|
645
|
+
dispatchAbortSignal: abortController.signal,
|
|
646
|
+
};
|
|
647
|
+
|
|
648
|
+
let timer = null;
|
|
649
|
+
const dispatchPromise = Promise.resolve(dispatchFn(enrichedContext))
|
|
650
|
+
.then((result) => ({ kind: 'result', result }))
|
|
651
|
+
.catch((error) => ({ kind: 'error', error }));
|
|
652
|
+
const timeoutPromise = new Promise((resolve) => {
|
|
653
|
+
timer = setTimeout(() => {
|
|
654
|
+
abortController.abort(new Error(`Turn timeout exceeded after ${timeoutResult.limit_minutes}m`));
|
|
655
|
+
resolve({ kind: 'timeout', timeoutResult });
|
|
656
|
+
}, remainingMs);
|
|
657
|
+
});
|
|
658
|
+
|
|
659
|
+
const winner = await Promise.race([dispatchPromise, timeoutPromise]);
|
|
660
|
+
clearTimeout(timer);
|
|
661
|
+
|
|
662
|
+
if (winner.kind === 'timeout') {
|
|
663
|
+
return {
|
|
664
|
+
timed_out: true,
|
|
665
|
+
timeout_result: winner.timeoutResult,
|
|
666
|
+
};
|
|
667
|
+
}
|
|
668
|
+
|
|
669
|
+
if (winner.kind === 'error') {
|
|
670
|
+
throw winner.error;
|
|
671
|
+
}
|
|
672
|
+
|
|
673
|
+
return winner.result;
|
|
674
|
+
}
|
|
675
|
+
|
|
676
|
+
function persistDispatchTimeout(root, config, turn, timeoutResult, errors) {
|
|
677
|
+
const blockedAt = new Date().toISOString();
|
|
678
|
+
const blockedReason = buildTimeoutBlockedReason(timeoutResult, { turnRetained: true });
|
|
679
|
+
const blocked = markRunBlocked(root, {
|
|
680
|
+
blockedOn: `timeout:${timeoutResult.scope}`,
|
|
681
|
+
category: blockedReason.category,
|
|
682
|
+
recovery: blockedReason.recovery,
|
|
683
|
+
turnId: turn.turn_id,
|
|
684
|
+
blockedAt,
|
|
685
|
+
notificationConfig: config,
|
|
686
|
+
});
|
|
687
|
+
|
|
688
|
+
if (!blocked.ok) {
|
|
689
|
+
errors.push(`markRunBlocked(timeout): ${blocked.error}`);
|
|
690
|
+
return { state: loadState(root, config) };
|
|
691
|
+
}
|
|
692
|
+
|
|
693
|
+
try {
|
|
694
|
+
appendJsonl(root, '.agentxchain/decision-ledger.jsonl', buildTimeoutLedgerEntry(timeoutResult, blockedAt, turn.turn_id, blocked.state?.phase));
|
|
695
|
+
} catch (err) {
|
|
696
|
+
errors.push(`timeout ledger append failed: ${err.message}`);
|
|
697
|
+
}
|
|
698
|
+
|
|
699
|
+
errors.push(`dispatch timed out for ${turn.assigned_role} after ${timeoutResult.limit_minutes}m`);
|
|
700
|
+
return blocked;
|
|
701
|
+
}
|
|
@@ -210,6 +210,93 @@ export function validateTimeoutsConfig(timeouts, routing) {
|
|
|
210
210
|
return { ok: errors.length === 0, errors };
|
|
211
211
|
}
|
|
212
212
|
|
|
213
|
+
/**
|
|
214
|
+
* Compute remaining timeout budget for an active turn/phase/run.
|
|
215
|
+
*
|
|
216
|
+
* Unlike evaluateTimeouts() which only returns items when exceeded,
|
|
217
|
+
* this returns budget info for ALL configured timeout scopes regardless
|
|
218
|
+
* of whether the deadline has passed.
|
|
219
|
+
*
|
|
220
|
+
* @param {object} options
|
|
221
|
+
* @param {object} options.config - Normalized config with optional `timeouts` section
|
|
222
|
+
* @param {object} options.state - Current governed state
|
|
223
|
+
* @param {object} [options.turn] - Active turn metadata
|
|
224
|
+
* @param {Date|string} [options.now] - Override for current time (testing)
|
|
225
|
+
* @returns {Array<TimeoutBudget>} Array of { scope, limit_minutes, elapsed_minutes, remaining_minutes, deadline_iso, exceeded, action }
|
|
226
|
+
*/
|
|
227
|
+
export function computeTimeoutBudget({ config, state, turn = null, now = new Date() }) {
|
|
228
|
+
const timeouts = config?.timeouts;
|
|
229
|
+
if (!timeouts) return [];
|
|
230
|
+
|
|
231
|
+
const nowMs = typeof now === 'string' ? new Date(now).getTime() : now.getTime();
|
|
232
|
+
const budgets = [];
|
|
233
|
+
|
|
234
|
+
// Per-turn budget
|
|
235
|
+
if (timeouts.per_turn_minutes && turn) {
|
|
236
|
+
const startedAt = turn.started_at || turn.assigned_at;
|
|
237
|
+
if (startedAt) {
|
|
238
|
+
const dispatchMs = new Date(startedAt).getTime();
|
|
239
|
+
const limitMs = timeouts.per_turn_minutes * 60 * 1000;
|
|
240
|
+
const elapsedMs = nowMs - dispatchMs;
|
|
241
|
+
const remainingMs = limitMs - elapsedMs;
|
|
242
|
+
budgets.push({
|
|
243
|
+
scope: 'turn',
|
|
244
|
+
limit_minutes: timeouts.per_turn_minutes,
|
|
245
|
+
elapsed_minutes: Math.round(elapsedMs / 60000),
|
|
246
|
+
remaining_minutes: Math.max(0, Math.round(remainingMs / 60000)),
|
|
247
|
+
remaining_seconds: Math.max(0, Math.round(remainingMs / 1000)),
|
|
248
|
+
deadline_iso: new Date(dispatchMs + limitMs).toISOString(),
|
|
249
|
+
exceeded: elapsedMs > limitMs,
|
|
250
|
+
action: resolveAction(timeouts.action, 'turn'),
|
|
251
|
+
});
|
|
252
|
+
}
|
|
253
|
+
}
|
|
254
|
+
|
|
255
|
+
// Per-phase budget
|
|
256
|
+
const phaseLimit = resolvePhaseLimit(timeouts, config.routing, state.phase);
|
|
257
|
+
const phaseAction = resolvePhaseAction(timeouts, config.routing, state.phase);
|
|
258
|
+
if (phaseLimit) {
|
|
259
|
+
const phaseEnteredAt = findPhaseEntryTime(state);
|
|
260
|
+
if (phaseEnteredAt) {
|
|
261
|
+
const entryMs = new Date(phaseEnteredAt).getTime();
|
|
262
|
+
const limitMs = phaseLimit * 60 * 1000;
|
|
263
|
+
const elapsedMs = nowMs - entryMs;
|
|
264
|
+
const remainingMs = limitMs - elapsedMs;
|
|
265
|
+
budgets.push({
|
|
266
|
+
scope: 'phase',
|
|
267
|
+
phase: state.phase,
|
|
268
|
+
limit_minutes: phaseLimit,
|
|
269
|
+
elapsed_minutes: Math.round(elapsedMs / 60000),
|
|
270
|
+
remaining_minutes: Math.max(0, Math.round(remainingMs / 60000)),
|
|
271
|
+
remaining_seconds: Math.max(0, Math.round(remainingMs / 1000)),
|
|
272
|
+
deadline_iso: new Date(entryMs + limitMs).toISOString(),
|
|
273
|
+
exceeded: elapsedMs > limitMs,
|
|
274
|
+
action: phaseAction,
|
|
275
|
+
});
|
|
276
|
+
}
|
|
277
|
+
}
|
|
278
|
+
|
|
279
|
+
// Per-run budget
|
|
280
|
+
if (timeouts.per_run_minutes && state.created_at) {
|
|
281
|
+
const createMs = new Date(state.created_at).getTime();
|
|
282
|
+
const limitMs = timeouts.per_run_minutes * 60 * 1000;
|
|
283
|
+
const elapsedMs = nowMs - createMs;
|
|
284
|
+
const remainingMs = limitMs - elapsedMs;
|
|
285
|
+
budgets.push({
|
|
286
|
+
scope: 'run',
|
|
287
|
+
limit_minutes: timeouts.per_run_minutes,
|
|
288
|
+
elapsed_minutes: Math.round(elapsedMs / 60000),
|
|
289
|
+
remaining_minutes: Math.max(0, Math.round(remainingMs / 60000)),
|
|
290
|
+
remaining_seconds: Math.max(0, Math.round(remainingMs / 1000)),
|
|
291
|
+
deadline_iso: new Date(createMs + limitMs).toISOString(),
|
|
292
|
+
exceeded: elapsedMs > limitMs,
|
|
293
|
+
action: resolveAction(timeouts.action, 'run'),
|
|
294
|
+
});
|
|
295
|
+
}
|
|
296
|
+
|
|
297
|
+
return budgets;
|
|
298
|
+
}
|
|
299
|
+
|
|
213
300
|
/**
|
|
214
301
|
* Build a blocked_reason descriptor for a timeout.
|
|
215
302
|
*/
|