agentxchain 2.125.0 → 2.127.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dashboard/components/timeline.js +29 -0
- package/package.json +1 -1
- package/src/commands/run.js +122 -19
- package/src/commands/status.js +101 -8
- package/src/commands/turn.js +29 -5
- package/src/lib/dashboard/state-reader.js +5 -0
- package/src/lib/dashboard/timeout-status.js +11 -2
- package/src/lib/dispatch-progress.js +298 -0
- package/src/lib/repo-observer.js +1 -0
- package/src/lib/run-events.js +1 -0
- package/src/lib/run-loop.js +129 -4
- package/src/lib/timeout-evaluator.js +87 -0
|
@@ -95,6 +95,30 @@ function formatPercent(value) {
|
|
|
95
95
|
return `${Math.round(value * 100)}%`;
|
|
96
96
|
}
|
|
97
97
|
|
|
98
|
+
function formatDispatchActivity(progress) {
|
|
99
|
+
if (!progress || typeof progress !== 'object') return null;
|
|
100
|
+
const lastActivityAt = progress.last_activity_at ? new Date(progress.last_activity_at) : null;
|
|
101
|
+
const agoSec = lastActivityAt && !Number.isNaN(lastActivityAt.getTime())
|
|
102
|
+
? Math.round((Date.now() - lastActivityAt.getTime()) / 1000)
|
|
103
|
+
: null;
|
|
104
|
+
|
|
105
|
+
if (progress.activity_type === 'silent') {
|
|
106
|
+
const silentSince = progress.silent_since ? new Date(progress.silent_since) : null;
|
|
107
|
+
const silentSec = silentSince && !Number.isNaN(silentSince.getTime())
|
|
108
|
+
? Math.round((Date.now() - silentSince.getTime()) / 1000)
|
|
109
|
+
: agoSec;
|
|
110
|
+
return `Silent for ${silentSec ?? 0}s (${progress.output_lines || 0} lines total, last output ${agoSec ?? 0}s ago)`;
|
|
111
|
+
}
|
|
112
|
+
if (progress.activity_type === 'request') {
|
|
113
|
+
return `API request in flight (${agoSec ?? 0}s ago)`;
|
|
114
|
+
}
|
|
115
|
+
if (progress.activity_type === 'response') {
|
|
116
|
+
return 'API response received';
|
|
117
|
+
}
|
|
118
|
+
const agoLabel = agoSec != null && agoSec > 0 ? `, last ${agoSec}s ago` : '';
|
|
119
|
+
return `Producing output (${progress.output_lines || 0} lines${agoLabel})`;
|
|
120
|
+
}
|
|
121
|
+
|
|
98
122
|
function statusBadge(status) {
|
|
99
123
|
const colors = {
|
|
100
124
|
running: 'var(--green)',
|
|
@@ -411,6 +435,9 @@ export function render({ state, continuity, history, events = null, annotations,
|
|
|
411
435
|
|
|
412
436
|
const turnCount = Array.isArray(history) ? history.length : 0;
|
|
413
437
|
const activeTurns = state.active_turns ? Object.values(state.active_turns) : [];
|
|
438
|
+
const dispatchProgress = state.dispatch_progress && typeof state.dispatch_progress === 'object'
|
|
439
|
+
? state.dispatch_progress
|
|
440
|
+
: {};
|
|
414
441
|
|
|
415
442
|
let html = `<div class="timeline-view">`;
|
|
416
443
|
|
|
@@ -436,6 +463,7 @@ export function render({ state, continuity, history, events = null, annotations,
|
|
|
436
463
|
for (const turn of activeTurns) {
|
|
437
464
|
const elapsedMs = computeElapsed(turn.started_at);
|
|
438
465
|
const elapsedStr = formatDuration(elapsedMs);
|
|
466
|
+
const activity = formatDispatchActivity(dispatchProgress[turn.turn_id]);
|
|
439
467
|
html += `<div class="turn-card active">
|
|
440
468
|
<div class="turn-header">
|
|
441
469
|
${roleBadge(getRole(turn))}
|
|
@@ -445,6 +473,7 @@ export function render({ state, continuity, history, events = null, annotations,
|
|
|
445
473
|
</div>
|
|
446
474
|
${renderDelegationContext(turn.delegation_context)}
|
|
447
475
|
${renderDelegationReview(turn.delegation_review)}
|
|
476
|
+
${activity ? `<div class="turn-detail"><span class="detail-label">Activity:</span> ${esc(activity)}</div>` : ''}
|
|
448
477
|
</div>`;
|
|
449
478
|
}
|
|
450
479
|
html += `</div></div>`;
|
package/package.json
CHANGED
package/src/commands/run.js
CHANGED
|
@@ -46,6 +46,8 @@ import {
|
|
|
46
46
|
} from '../lib/turn-paths.js';
|
|
47
47
|
import { resolveChainOptions, executeChainedRun } from '../lib/run-chain.js';
|
|
48
48
|
import { resolveContinuousOptions, executeContinuousRun } from '../lib/continuous-run.js';
|
|
49
|
+
import { createDispatchProgressTracker } from '../lib/dispatch-progress.js';
|
|
50
|
+
import { emitRunEvent } from '../lib/run-events.js';
|
|
49
51
|
|
|
50
52
|
export async function runCommand(opts) {
|
|
51
53
|
const context = loadProjectContext();
|
|
@@ -299,38 +301,114 @@ export async function executeGovernedRun(context, opts = {}) {
|
|
|
299
301
|
}
|
|
300
302
|
|
|
301
303
|
// ── Route to adapter ──────────────────────────────────────────────
|
|
304
|
+
const tracker = createDispatchProgressTracker(projectRoot, turn, {
|
|
305
|
+
adapter_type: runtimeType,
|
|
306
|
+
});
|
|
307
|
+
|
|
302
308
|
const adapterOpts = {
|
|
303
|
-
signal: controller.signal,
|
|
309
|
+
signal: combineAbortSignals(controller.signal, ctx.dispatchAbortSignal),
|
|
304
310
|
onStatus: (msg) => log(chalk.dim(` ${msg}`)),
|
|
305
311
|
verifyManifest: true,
|
|
306
312
|
turnId: turn.turn_id,
|
|
307
313
|
};
|
|
308
314
|
|
|
315
|
+
const recordOutputActivity = (stream, text) => {
|
|
316
|
+
const lines = text.split('\n').length - 1 || 1;
|
|
317
|
+
const wasSilent = tracker.onOutput(stream, lines);
|
|
318
|
+
if (wasSilent) {
|
|
319
|
+
const progressState = tracker.getState();
|
|
320
|
+
emitRunEvent(projectRoot, 'dispatch_progress', {
|
|
321
|
+
run_id: state.run_id,
|
|
322
|
+
phase: state.phase,
|
|
323
|
+
status: state.status,
|
|
324
|
+
turn: { turn_id: turn.turn_id, assigned_role: roleId },
|
|
325
|
+
payload: {
|
|
326
|
+
milestone: 'output_resumed',
|
|
327
|
+
output_lines: progressState.output_lines,
|
|
328
|
+
elapsed_seconds: Math.round((Date.now() - new Date(progressState.started_at)) / 1000),
|
|
329
|
+
silent_seconds: 0,
|
|
330
|
+
},
|
|
331
|
+
});
|
|
332
|
+
}
|
|
333
|
+
};
|
|
334
|
+
|
|
309
335
|
if (verbose) {
|
|
310
|
-
adapterOpts.onStdout = (text) =>
|
|
311
|
-
|
|
336
|
+
adapterOpts.onStdout = (text) => {
|
|
337
|
+
process.stdout.write(chalk.dim(text));
|
|
338
|
+
recordOutputActivity('stdout', text);
|
|
339
|
+
};
|
|
340
|
+
adapterOpts.onStderr = (text) => {
|
|
341
|
+
process.stderr.write(chalk.yellow(text));
|
|
342
|
+
recordOutputActivity('stderr', text);
|
|
343
|
+
};
|
|
344
|
+
} else {
|
|
345
|
+
// Even in non-verbose mode, track output activity for progress visibility
|
|
346
|
+
adapterOpts.onStdout = (text) => {
|
|
347
|
+
recordOutputActivity('stdout', text);
|
|
348
|
+
};
|
|
349
|
+
adapterOpts.onStderr = (text) => {
|
|
350
|
+
recordOutputActivity('stderr', text);
|
|
351
|
+
};
|
|
312
352
|
}
|
|
313
353
|
|
|
314
354
|
let adapterResult;
|
|
315
355
|
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
356
|
+
// Emit dispatch_progress started event and begin tracking
|
|
357
|
+
tracker.start();
|
|
358
|
+
emitRunEvent(projectRoot, 'dispatch_progress', {
|
|
359
|
+
run_id: state.run_id,
|
|
360
|
+
phase: state.phase,
|
|
361
|
+
status: state.status,
|
|
362
|
+
turn: { turn_id: turn.turn_id, assigned_role: roleId },
|
|
363
|
+
payload: { milestone: 'started', output_lines: 0, elapsed_seconds: 0, silent_seconds: 0 },
|
|
364
|
+
});
|
|
365
|
+
|
|
366
|
+
try {
|
|
367
|
+
if (runtimeType === 'api_proxy') {
|
|
368
|
+
log(chalk.dim(` Dispatching to API proxy: ${runtime?.provider || '?'} / ${runtime?.model || '?'}`));
|
|
369
|
+
tracker.requestStarted();
|
|
370
|
+
adapterResult = await dispatchApiProxy(projectRoot, state, cfg, adapterOpts);
|
|
371
|
+
if (adapterResult.ok) tracker.responseReceived();
|
|
372
|
+
} else if (runtimeType === 'mcp') {
|
|
373
|
+
const transport = resolveMcpTransport(runtime);
|
|
374
|
+
log(chalk.dim(` Dispatching to MCP ${transport}: ${describeMcpRuntimeTarget(runtime)}`));
|
|
375
|
+
tracker.requestStarted();
|
|
376
|
+
adapterResult = await dispatchMcp(projectRoot, state, cfg, adapterOpts);
|
|
377
|
+
if (adapterResult.ok) tracker.responseReceived();
|
|
378
|
+
} else if (runtimeType === 'local_cli') {
|
|
379
|
+
const transport = runtime ? resolvePromptTransport(runtime) : 'dispatch_bundle_only';
|
|
380
|
+
log(chalk.dim(` Dispatching to local CLI: ${runtime?.command || '(default)'} transport: ${transport}`));
|
|
381
|
+
adapterResult = await dispatchLocalCli(projectRoot, state, cfg, adapterOpts);
|
|
382
|
+
} else if (runtimeType === 'remote_agent') {
|
|
383
|
+
log(chalk.dim(` Dispatching to remote agent: ${describeRemoteAgentTarget(runtime)}`));
|
|
384
|
+
tracker.requestStarted();
|
|
385
|
+
adapterResult = await dispatchRemoteAgent(projectRoot, state, cfg, adapterOpts);
|
|
386
|
+
if (adapterResult.ok) tracker.responseReceived();
|
|
387
|
+
} else {
|
|
388
|
+
tracker.fail();
|
|
389
|
+
return { accept: false, reason: `unknown runtime type "${runtimeType}"` };
|
|
390
|
+
}
|
|
391
|
+
} catch (err) {
|
|
392
|
+
tracker.fail();
|
|
393
|
+
emitRunEvent(projectRoot, 'dispatch_progress', {
|
|
394
|
+
run_id: state.run_id, phase: state.phase, status: state.status,
|
|
395
|
+
turn: { turn_id: turn.turn_id, assigned_role: roleId },
|
|
396
|
+
payload: { milestone: 'failed', output_lines: tracker.getState().output_lines, elapsed_seconds: Math.round((Date.now() - new Date(tracker.getState().started_at)) / 1000), silent_seconds: 0 },
|
|
397
|
+
});
|
|
398
|
+
throw err;
|
|
332
399
|
}
|
|
333
400
|
|
|
401
|
+
// Emit completion/failure progress event and clean up tracker
|
|
402
|
+
const progressState = tracker.getState();
|
|
403
|
+
const elapsedSec = Math.round((Date.now() - new Date(progressState.started_at)) / 1000);
|
|
404
|
+
const milestone = adapterResult.ok ? 'completed' : (adapterResult.timedOut ? 'timed_out' : 'failed');
|
|
405
|
+
if (adapterResult.ok) { tracker.complete(); } else { tracker.fail(); }
|
|
406
|
+
emitRunEvent(projectRoot, 'dispatch_progress', {
|
|
407
|
+
run_id: state.run_id, phase: state.phase, status: state.status,
|
|
408
|
+
turn: { turn_id: turn.turn_id, assigned_role: roleId },
|
|
409
|
+
payload: { milestone, output_lines: progressState.output_lines, elapsed_seconds: elapsedSec, silent_seconds: progressState.silent_since ? Math.round((Date.now() - new Date(progressState.silent_since)) / 1000) : 0 },
|
|
410
|
+
});
|
|
411
|
+
|
|
334
412
|
// Save adapter logs
|
|
335
413
|
if (adapterResult.logs?.length) {
|
|
336
414
|
saveDispatchLogs(projectRoot, turn.turn_id, adapterResult.logs);
|
|
@@ -557,3 +635,28 @@ function printManualQaFallback(log = console.log) {
|
|
|
557
635
|
log(chalk.dim(' - Then recover the retained QA turn with: agentxchain step --resume'));
|
|
558
636
|
log(chalk.dim(' - Guide: https://agentxchain.dev/docs/getting-started'));
|
|
559
637
|
}
|
|
638
|
+
|
|
639
|
+
function combineAbortSignals(primarySignal, secondarySignal) {
|
|
640
|
+
if (!secondarySignal) {
|
|
641
|
+
return primarySignal;
|
|
642
|
+
}
|
|
643
|
+
if (!primarySignal) {
|
|
644
|
+
return secondarySignal;
|
|
645
|
+
}
|
|
646
|
+
if (typeof AbortSignal.any === 'function') {
|
|
647
|
+
return AbortSignal.any([primarySignal, secondarySignal]);
|
|
648
|
+
}
|
|
649
|
+
|
|
650
|
+
const combined = new AbortController();
|
|
651
|
+
const forward = (signal) => {
|
|
652
|
+
if (!signal) return;
|
|
653
|
+
if (signal.aborted) {
|
|
654
|
+
combined.abort(signal.reason);
|
|
655
|
+
return;
|
|
656
|
+
}
|
|
657
|
+
signal.addEventListener('abort', () => combined.abort(signal.reason), { once: true });
|
|
658
|
+
};
|
|
659
|
+
forward(primarySignal);
|
|
660
|
+
forward(secondarySignal);
|
|
661
|
+
return combined.signal;
|
|
662
|
+
}
|
package/src/commands/status.js
CHANGED
|
@@ -13,7 +13,7 @@ import { getContinuityStatus } from '../lib/continuity-status.js';
|
|
|
13
13
|
import { getConnectorHealth } from '../lib/connector-health.js';
|
|
14
14
|
import { readRepoDecisions, summarizeRepoDecisions } from '../lib/repo-decisions.js';
|
|
15
15
|
import { deriveWorkflowKitArtifacts } from '../lib/workflow-kit-artifacts.js';
|
|
16
|
-
import { evaluateTimeouts } from '../lib/timeout-evaluator.js';
|
|
16
|
+
import { evaluateTimeouts, computeTimeoutBudget } from '../lib/timeout-evaluator.js';
|
|
17
17
|
import { evaluateApprovalSlaReminders } from '../lib/notification-runner.js';
|
|
18
18
|
import { summarizeRunProvenance } from '../lib/run-provenance.js';
|
|
19
19
|
import { readRecentRunEventSummary } from '../lib/recent-event-summary.js';
|
|
@@ -23,6 +23,7 @@ import { findCurrentHumanEscalation } from '../lib/human-escalations.js';
|
|
|
23
23
|
import { getDashboardPid, getDashboardSession } from './dashboard.js';
|
|
24
24
|
import { readPreemptionMarker } from '../lib/intake.js';
|
|
25
25
|
import { readContinuousSession } from '../lib/continuous-run.js';
|
|
26
|
+
import { readAllDispatchProgress } from '../lib/dispatch-progress.js';
|
|
26
27
|
|
|
27
28
|
export async function statusCommand(opts) {
|
|
28
29
|
const context = loadStatusContext();
|
|
@@ -142,6 +143,9 @@ function renderGovernedStatus(context, opts) {
|
|
|
142
143
|
// Fire approval SLA reminders as a side effect (webhook-only, no CLI output)
|
|
143
144
|
evaluateApprovalSlaReminders(root, config, state);
|
|
144
145
|
|
|
146
|
+
const activeTurns = getActiveTurns(state);
|
|
147
|
+
const dispatchProgress = filterDispatchProgressForActiveTurns(readAllDispatchProgress(root), activeTurns);
|
|
148
|
+
|
|
145
149
|
if (opts.json) {
|
|
146
150
|
const dashPid = getDashboardPid(root);
|
|
147
151
|
const dashSession = getDashboardSession(root);
|
|
@@ -168,6 +172,7 @@ function renderGovernedStatus(context, opts) {
|
|
|
168
172
|
next_actions: nextActions,
|
|
169
173
|
connector_health: connectorHealth,
|
|
170
174
|
recent_event_summary: recentEventSummary,
|
|
175
|
+
dispatch_progress: dispatchProgress,
|
|
171
176
|
human_escalation: humanEscalation,
|
|
172
177
|
preemption_marker: preemptionMarker,
|
|
173
178
|
continuous_session: continuousSession,
|
|
@@ -262,7 +267,6 @@ function renderGovernedStatus(context, opts) {
|
|
|
262
267
|
renderRecentEventSummary(recentEventSummary);
|
|
263
268
|
|
|
264
269
|
const activeTurnCount = getActiveTurnCount(state);
|
|
265
|
-
const activeTurns = getActiveTurns(state);
|
|
266
270
|
const singleActiveTurn = getActiveTurn(state);
|
|
267
271
|
const approvalPending = Boolean(state?.pending_phase_transition || state?.pending_run_completion);
|
|
268
272
|
if (activeTurnCount > 1) {
|
|
@@ -283,7 +287,23 @@ function renderGovernedStatus(context, opts) {
|
|
|
283
287
|
elapsedTag = m > 0 ? ` — ${m}m ${s % 60}s` : ` — ${s}s`;
|
|
284
288
|
}
|
|
285
289
|
}
|
|
286
|
-
|
|
290
|
+
let budgetTag = '';
|
|
291
|
+
if (config.timeouts?.per_turn_minutes && turn.started_at) {
|
|
292
|
+
const tb = computeTimeoutBudget({ config, state, turn, now: new Date() });
|
|
293
|
+
const tBudget = tb.find((b) => b.scope === 'turn');
|
|
294
|
+
if (tBudget) {
|
|
295
|
+
if (tBudget.exceeded) {
|
|
296
|
+
budgetTag = ` ${chalk.red('[TIMEOUT]')}`;
|
|
297
|
+
} else {
|
|
298
|
+
budgetTag = ` ${chalk.dim(`[${tBudget.remaining_minutes}m left]`)}`;
|
|
299
|
+
}
|
|
300
|
+
}
|
|
301
|
+
}
|
|
302
|
+
console.log(` ${marker} ${turn.turn_id} — ${chalk.bold(turn.assigned_role)} (${statusLabel}) [attempt ${turn.attempt}]${elapsedTag}${budgetTag}`);
|
|
303
|
+
const activityLine = formatDispatchActivityLine(dispatchProgress[turn.turn_id]);
|
|
304
|
+
if (activityLine) {
|
|
305
|
+
console.log(` ${chalk.dim('Activity:')} ${activityLine}`);
|
|
306
|
+
}
|
|
287
307
|
if (turn.status === 'conflicted' && turn.conflict_state) {
|
|
288
308
|
const cs = turn.conflict_state;
|
|
289
309
|
const files = cs.conflict_error?.conflicting_files || [];
|
|
@@ -314,6 +334,26 @@ function renderGovernedStatus(context, opts) {
|
|
|
314
334
|
console.log(` ${chalk.dim('Elapsed:')} ${elapsed}`);
|
|
315
335
|
}
|
|
316
336
|
}
|
|
337
|
+
// Turn-level timeout budget inline with turn info
|
|
338
|
+
if (config.timeouts?.per_turn_minutes && singleActiveTurn.started_at) {
|
|
339
|
+
const turnBudgets = computeTimeoutBudget({ config, state, turn: singleActiveTurn, now: new Date() });
|
|
340
|
+
const turnBudget = turnBudgets.find((b) => b.scope === 'turn');
|
|
341
|
+
if (turnBudget) {
|
|
342
|
+
if (turnBudget.exceeded) {
|
|
343
|
+
console.log(` ${chalk.dim('Budget:')} ${chalk.red(`EXCEEDED — was ${turnBudget.limit_minutes}m, over by ${turnBudget.elapsed_minutes - turnBudget.limit_minutes}m`)}`);
|
|
344
|
+
} else {
|
|
345
|
+
const remMins = Math.floor(turnBudget.remaining_seconds / 60);
|
|
346
|
+
const remSecs = turnBudget.remaining_seconds % 60;
|
|
347
|
+
const remLabel = remMins > 0 ? `${remMins}m ${remSecs}s` : `${remSecs}s`;
|
|
348
|
+
console.log(` ${chalk.dim('Budget:')} ${chalk.green(`${remLabel} remaining`)} of ${turnBudget.limit_minutes}m (deadline ${new Date(turnBudget.deadline_iso).toLocaleTimeString()})`);
|
|
349
|
+
}
|
|
350
|
+
}
|
|
351
|
+
}
|
|
352
|
+
// Dispatch progress activity line (DEC-DISPATCH-PROGRESS-001)
|
|
353
|
+
const activityLine = formatDispatchActivityLine(dispatchProgress[singleActiveTurn.turn_id]);
|
|
354
|
+
if (activityLine) {
|
|
355
|
+
console.log(` ${chalk.dim('Activity:')} ${activityLine}`);
|
|
356
|
+
}
|
|
317
357
|
if (singleActiveTurn.status === 'conflicted' && singleActiveTurn.conflict_state) {
|
|
318
358
|
const cs = singleActiveTurn.conflict_state;
|
|
319
359
|
const files = cs.conflict_error?.conflicting_files || [];
|
|
@@ -474,19 +514,22 @@ function renderGovernedStatus(context, opts) {
|
|
|
474
514
|
renderWorkflowKitArtifactsSection(workflowKitArtifacts);
|
|
475
515
|
|
|
476
516
|
if (config.timeouts && (state?.status === 'active' || approvalPending)) {
|
|
517
|
+
const nowDate = new Date();
|
|
477
518
|
const activeTurn = state?.status === 'active' ? getActiveTurn(state) : null;
|
|
478
519
|
const turnResult = activeTurn ? { role: activeTurn.assigned_role } : undefined;
|
|
479
|
-
const timeoutEval = evaluateTimeouts({ config, state, turn: activeTurn, turnResult, now:
|
|
520
|
+
const timeoutEval = evaluateTimeouts({ config, state, turn: activeTurn, turnResult, now: nowDate.toISOString() });
|
|
480
521
|
const allItems = [...timeoutEval.exceeded, ...timeoutEval.warnings];
|
|
481
|
-
|
|
522
|
+
// Compute full budget for phase/run scopes (turn budget is shown inline with turn info above)
|
|
523
|
+
const budgets = computeTimeoutBudget({ config, state, turn: activeTurn, now: nowDate })
|
|
524
|
+
.filter((b) => b.scope !== 'turn'); // turn budget already shown inline
|
|
525
|
+
|
|
526
|
+
if (allItems.length > 0 || budgets.length > 0 || approvalPending) {
|
|
482
527
|
console.log('');
|
|
483
528
|
console.log(` ${chalk.dim('Timeouts:')}`);
|
|
484
529
|
if (approvalPending) {
|
|
485
530
|
console.log(` ${chalk.yellow('◷')} approval wait does not mutate timeout state; phase/run clocks keep ticking until the next accepted turn`);
|
|
486
531
|
}
|
|
487
|
-
|
|
488
|
-
console.log(` ${chalk.dim('No current phase/run timeout pressure.')}`);
|
|
489
|
-
}
|
|
532
|
+
// Show exceeded/warned items
|
|
490
533
|
for (const item of allItems) {
|
|
491
534
|
const isExceeded = timeoutEval.exceeded.includes(item);
|
|
492
535
|
const elapsed = item.elapsed_minutes != null ? `${item.elapsed_minutes}m` : '?';
|
|
@@ -495,6 +538,17 @@ function renderGovernedStatus(context, opts) {
|
|
|
495
538
|
const label = isExceeded ? chalk.red(`EXCEEDED ${item.scope}`) : chalk.yellow(`${item.scope}`);
|
|
496
539
|
console.log(` ${icon} ${label}: ${elapsed}/${limit} (action: ${item.action || 'n/a'})`);
|
|
497
540
|
}
|
|
541
|
+
// Show remaining budget for non-exceeded phase/run scopes
|
|
542
|
+
const exceededScopes = new Set(allItems.map((i) => `${i.scope}:${i.phase || ''}`));
|
|
543
|
+
for (const b of budgets) {
|
|
544
|
+
const key = `${b.scope}:${b.phase || ''}`;
|
|
545
|
+
if (exceededScopes.has(key)) continue; // already shown as exceeded above
|
|
546
|
+
const scopeLabel = b.scope === 'phase' ? `phase (${b.phase})` : b.scope;
|
|
547
|
+
console.log(` ${chalk.green('✓')} ${scopeLabel}: ${b.elapsed_minutes}m/${b.limit_minutes}m — ${chalk.green(`${b.remaining_minutes}m remaining`)}`);
|
|
548
|
+
}
|
|
549
|
+
if (approvalPending && allItems.length === 0 && budgets.length === 0) {
|
|
550
|
+
console.log(` ${chalk.dim('No current phase/run timeout pressure.')}`);
|
|
551
|
+
}
|
|
498
552
|
}
|
|
499
553
|
}
|
|
500
554
|
|
|
@@ -702,6 +756,45 @@ function pluralizeRepoDecisionCount(count, singular, plural) {
|
|
|
702
756
|
return `${count} ${count === 1 ? singular : plural}`;
|
|
703
757
|
}
|
|
704
758
|
|
|
759
|
+
function filterDispatchProgressForActiveTurns(progressByTurn, activeTurns) {
|
|
760
|
+
const filtered = {};
|
|
761
|
+
if (!progressByTurn || typeof progressByTurn !== 'object') {
|
|
762
|
+
return filtered;
|
|
763
|
+
}
|
|
764
|
+
for (const turn of Object.values(activeTurns || {})) {
|
|
765
|
+
const turnId = turn?.turn_id;
|
|
766
|
+
if (turnId && progressByTurn[turnId]) {
|
|
767
|
+
filtered[turnId] = progressByTurn[turnId];
|
|
768
|
+
}
|
|
769
|
+
}
|
|
770
|
+
return filtered;
|
|
771
|
+
}
|
|
772
|
+
|
|
773
|
+
function formatDispatchActivityLine(progress) {
|
|
774
|
+
if (!progress || typeof progress !== 'object') return null;
|
|
775
|
+
const lastAct = progress.last_activity_at ? new Date(progress.last_activity_at) : null;
|
|
776
|
+
const agoSec = lastAct && !Number.isNaN(lastAct.getTime())
|
|
777
|
+
? Math.round((Date.now() - lastAct.getTime()) / 1000)
|
|
778
|
+
: null;
|
|
779
|
+
|
|
780
|
+
if (progress.activity_type === 'silent') {
|
|
781
|
+
const silentAt = progress.silent_since ? new Date(progress.silent_since) : null;
|
|
782
|
+
const silentSec = silentAt && !Number.isNaN(silentAt.getTime())
|
|
783
|
+
? Math.round((Date.now() - silentAt.getTime()) / 1000)
|
|
784
|
+
: agoSec;
|
|
785
|
+
return chalk.yellow(`Silent for ${silentSec}s`) +
|
|
786
|
+
` (${progress.output_lines || 0} lines total, last output ${agoSec ?? 0}s ago)`;
|
|
787
|
+
}
|
|
788
|
+
if (progress.activity_type === 'request') {
|
|
789
|
+
return chalk.cyan('API request in flight') + ` (${agoSec ?? 0}s ago)`;
|
|
790
|
+
}
|
|
791
|
+
if (progress.activity_type === 'response') {
|
|
792
|
+
return chalk.green('API response received');
|
|
793
|
+
}
|
|
794
|
+
const agoLabel = agoSec != null && agoSec > 0 ? `, last ${agoSec}s ago` : '';
|
|
795
|
+
return chalk.green('Producing output') + ` (${progress.output_lines || 0} lines${agoLabel})`;
|
|
796
|
+
}
|
|
797
|
+
|
|
705
798
|
function renderLastGateFailure(failure, config) {
|
|
706
799
|
const entryRole = config?.routing?.[failure.phase]?.entry_role || null;
|
|
707
800
|
const suggestedCommand = entryRole ? `agentxchain step --role ${entryRole}` : 'agentxchain step --role <role>';
|
package/src/commands/turn.js
CHANGED
|
@@ -3,6 +3,7 @@ import { join } from 'path';
|
|
|
3
3
|
import chalk from 'chalk';
|
|
4
4
|
import { loadProjectContext, loadProjectState } from '../lib/config.js';
|
|
5
5
|
import { getActiveTurnCount, getActiveTurns } from '../lib/governed-state.js';
|
|
6
|
+
import { computeTimeoutBudget } from '../lib/timeout-evaluator.js';
|
|
6
7
|
import {
|
|
7
8
|
getDispatchAssignmentPath,
|
|
8
9
|
getDispatchContextPath,
|
|
@@ -54,11 +55,11 @@ export function turnShowCommand(turnId, opts) {
|
|
|
54
55
|
}
|
|
55
56
|
|
|
56
57
|
if (opts.json) {
|
|
57
|
-
console.log(JSON.stringify(buildTurnPayload(selectedTurnId, turn, state, artifacts, assignment), null, 2));
|
|
58
|
+
console.log(JSON.stringify(buildTurnPayload(selectedTurnId, turn, state, artifacts, assignment, context.config), null, 2));
|
|
58
59
|
return;
|
|
59
60
|
}
|
|
60
61
|
|
|
61
|
-
printTurnSummary(selectedTurnId, turn, state, artifacts, assignment);
|
|
62
|
+
printTurnSummary(selectedTurnId, turn, state, artifacts, assignment, context.config);
|
|
62
63
|
}
|
|
63
64
|
|
|
64
65
|
function requireGovernedContext() {
|
|
@@ -118,9 +119,9 @@ function buildArtifactIndex(root, turnId) {
|
|
|
118
119
|
);
|
|
119
120
|
}
|
|
120
121
|
|
|
121
|
-
function buildTurnPayload(turnId, turn, state, artifacts, assignment) {
|
|
122
|
+
function buildTurnPayload(turnId, turn, state, artifacts, assignment, config) {
|
|
122
123
|
const elapsedMs = getElapsedMs(turn.started_at);
|
|
123
|
-
|
|
124
|
+
const payload = {
|
|
124
125
|
turn_id: turnId,
|
|
125
126
|
run_id: state.run_id || assignment?.run_id || null,
|
|
126
127
|
phase: state.phase || assignment?.phase || null,
|
|
@@ -140,9 +141,17 @@ function buildTurnPayload(turnId, turn, state, artifacts, assignment) {
|
|
|
140
141
|
}]),
|
|
141
142
|
),
|
|
142
143
|
};
|
|
144
|
+
// Add timeout budget if configured
|
|
145
|
+
if (config?.timeouts) {
|
|
146
|
+
const budgets = computeTimeoutBudget({ config, state, turn, now: new Date() });
|
|
147
|
+
if (budgets.length > 0) {
|
|
148
|
+
payload.timeout_budget = budgets;
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
return payload;
|
|
143
152
|
}
|
|
144
153
|
|
|
145
|
-
function printTurnSummary(turnId, turn, state, artifacts, assignment) {
|
|
154
|
+
function printTurnSummary(turnId, turn, state, artifacts, assignment, config) {
|
|
146
155
|
const elapsedMs = getElapsedMs(turn.started_at);
|
|
147
156
|
console.log('');
|
|
148
157
|
console.log(chalk.bold(` Turn: ${chalk.cyan(turnId)}`));
|
|
@@ -159,6 +168,21 @@ function printTurnSummary(turnId, turn, state, artifacts, assignment) {
|
|
|
159
168
|
if (elapsedMs != null) {
|
|
160
169
|
console.log(` ${chalk.dim('Elapsed:')} ${formatElapsed(elapsedMs)}`);
|
|
161
170
|
}
|
|
171
|
+
// Timeout budget per scope
|
|
172
|
+
if (config?.timeouts) {
|
|
173
|
+
const budgets = computeTimeoutBudget({ config, state, turn, now: new Date() });
|
|
174
|
+
for (const b of budgets) {
|
|
175
|
+
const scopeLabel = b.scope === 'phase' ? `phase (${b.phase})` : b.scope;
|
|
176
|
+
if (b.exceeded) {
|
|
177
|
+
console.log(` ${chalk.dim('Timeout:')} ${chalk.red(`${scopeLabel} EXCEEDED`)} — was ${b.limit_minutes}m, over by ${b.elapsed_minutes - b.limit_minutes}m`);
|
|
178
|
+
} else {
|
|
179
|
+
const remMins = Math.floor(b.remaining_seconds / 60);
|
|
180
|
+
const remSecs = b.remaining_seconds % 60;
|
|
181
|
+
const remLabel = remMins > 0 ? `${remMins}m ${remSecs}s` : `${remSecs}s`;
|
|
182
|
+
console.log(` ${chalk.dim('Timeout:')} ${scopeLabel} — ${chalk.green(`${remLabel} remaining`)} of ${b.limit_minutes}m`);
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
}
|
|
162
186
|
console.log(` ${chalk.dim('Dispatch:')} ${getDispatchTurnDir(turnId)}`);
|
|
163
187
|
if (assignment?.staging_result_path) {
|
|
164
188
|
console.log(` ${chalk.dim('Staging:')} ${assignment.staging_result_path}`);
|
|
@@ -15,6 +15,7 @@ import {
|
|
|
15
15
|
import { loadProjectContext } from '../config.js';
|
|
16
16
|
import { getContinuityStatus } from '../continuity-status.js';
|
|
17
17
|
import { readRepoDecisions, summarizeRepoDecisions } from '../repo-decisions.js';
|
|
18
|
+
import { readAllDispatchProgress } from '../dispatch-progress.js';
|
|
18
19
|
|
|
19
20
|
const STATE_FILE = 'state.json';
|
|
20
21
|
const SESSION_FILE = 'session.json';
|
|
@@ -80,6 +81,9 @@ export function normalizeRelativePath(filePath) {
|
|
|
80
81
|
|
|
81
82
|
export function resourcesForRelativePath(filePath) {
|
|
82
83
|
const normalized = normalizeRelativePath(filePath);
|
|
84
|
+
if (/^dispatch-progress-[^/]+\.json$/.test(normalized)) {
|
|
85
|
+
return ['/api/state'];
|
|
86
|
+
}
|
|
83
87
|
if (normalized.startsWith('missions/plans/') && normalized.endsWith('.json')) {
|
|
84
88
|
return ['/api/plans', '/api/missions'];
|
|
85
89
|
}
|
|
@@ -136,6 +140,7 @@ function enrichGovernedState(agentxchainDir, state) {
|
|
|
136
140
|
...state,
|
|
137
141
|
runtime_guidance: deriveRuntimeBlockedGuidance(state, context.config),
|
|
138
142
|
next_actions: deriveGovernedRunNextActions(state, context.config),
|
|
143
|
+
dispatch_progress: readAllDispatchProgress(workspacePath),
|
|
139
144
|
};
|
|
140
145
|
}
|
|
141
146
|
|
|
@@ -9,7 +9,7 @@
|
|
|
9
9
|
|
|
10
10
|
import { join } from 'path';
|
|
11
11
|
import { loadProjectContext, loadProjectState } from '../config.js';
|
|
12
|
-
import { evaluateTimeouts } from '../timeout-evaluator.js';
|
|
12
|
+
import { evaluateTimeouts, computeTimeoutBudget } from '../timeout-evaluator.js';
|
|
13
13
|
import { readJsonlFile } from './state-reader.js';
|
|
14
14
|
|
|
15
15
|
/**
|
|
@@ -209,7 +209,15 @@ export function readTimeoutStatus(workspacePath) {
|
|
|
209
209
|
const configSummary = buildTimeoutConfigSummary(timeouts, config.routing);
|
|
210
210
|
|
|
211
211
|
// Live timeout evaluation — only meaningful when the run is active
|
|
212
|
-
const
|
|
212
|
+
const nowDate = new Date();
|
|
213
|
+
const live = evaluateDashboardTimeoutPressure(config, state, nowDate);
|
|
214
|
+
|
|
215
|
+
// Compute remaining budget for all configured scopes
|
|
216
|
+
const activeTurnsList = getActiveTurns(state);
|
|
217
|
+
const primaryTurn = activeTurnsList.length === 1 ? activeTurnsList[0] : null;
|
|
218
|
+
const budget = (state?.status === 'active' || Boolean(state?.pending_phase_transition || state?.pending_run_completion))
|
|
219
|
+
? computeTimeoutBudget({ config, state, turn: primaryTurn, now: nowDate })
|
|
220
|
+
: [];
|
|
213
221
|
|
|
214
222
|
// Persisted timeout events from the decision ledger
|
|
215
223
|
const ledger = readJsonlFile(agentxchainDir, 'decision-ledger.jsonl');
|
|
@@ -223,6 +231,7 @@ export function readTimeoutStatus(workspacePath) {
|
|
|
223
231
|
configured: true,
|
|
224
232
|
config: configSummary,
|
|
225
233
|
live,
|
|
234
|
+
budget,
|
|
226
235
|
live_context: buildLiveContext(state),
|
|
227
236
|
events,
|
|
228
237
|
},
|
|
@@ -0,0 +1,298 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* dispatch-progress.js — Real-time adapter dispatch progress tracking.
|
|
3
|
+
*
|
|
4
|
+
* Writes `.agentxchain/dispatch-progress.json` during in-flight adapter
|
|
5
|
+
* dispatch so operators can distinguish "adapter is working" from "adapter
|
|
6
|
+
* is hung" via `agentxchain status` and the dashboard file-watcher.
|
|
7
|
+
*
|
|
8
|
+
* DEC-DISPATCH-PROGRESS-001: progress writes are best-effort and never
|
|
9
|
+
* block or delay the governed turn.
|
|
10
|
+
*/
|
|
11
|
+
|
|
12
|
+
import { writeFileSync, unlinkSync, readFileSync, existsSync, mkdirSync, readdirSync } from 'node:fs';
|
|
13
|
+
import { join, dirname, basename } from 'node:path';
|
|
14
|
+
|
|
15
|
+
export const LEGACY_DISPATCH_PROGRESS_PATH = '.agentxchain/dispatch-progress.json';
|
|
16
|
+
export const DISPATCH_PROGRESS_FILE_PREFIX = '.agentxchain/dispatch-progress-';
|
|
17
|
+
|
|
18
|
+
export function getDispatchProgressRelativePath(turnId) {
|
|
19
|
+
return `${DISPATCH_PROGRESS_FILE_PREFIX}${turnId}.json`;
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
function getDispatchProgressFilePath(root, turnId) {
|
|
23
|
+
return join(root, getDispatchProgressRelativePath(turnId));
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
function listDispatchProgressFiles(root) {
|
|
27
|
+
const agentxchainDir = join(root, '.agentxchain');
|
|
28
|
+
if (!existsSync(agentxchainDir)) return [];
|
|
29
|
+
try {
|
|
30
|
+
return readdirSync(agentxchainDir)
|
|
31
|
+
.filter((entry) => entry.startsWith('dispatch-progress-') && entry.endsWith('.json'))
|
|
32
|
+
.map((entry) => join(agentxchainDir, entry));
|
|
33
|
+
} catch {
|
|
34
|
+
return [];
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
/**
|
|
39
|
+
* Create a dispatch progress tracker for a single turn.
|
|
40
|
+
*
|
|
41
|
+
* Usage:
|
|
42
|
+
* const tracker = createDispatchProgressTracker(root, turn, runtime);
|
|
43
|
+
* tracker.start();
|
|
44
|
+
* // ... during dispatch:
|
|
45
|
+
* tracker.onOutput('stdout', lineCount);
|
|
46
|
+
* tracker.onOutput('stderr', lineCount);
|
|
47
|
+
* // ... when done:
|
|
48
|
+
* tracker.complete(); // or tracker.fail();
|
|
49
|
+
*
|
|
50
|
+
* @param {string} root - project root
|
|
51
|
+
* @param {object} turn - turn object with turn_id, runtime_id, assigned_role
|
|
52
|
+
* @param {object} options
|
|
53
|
+
* @param {string} options.adapter_type - 'local_cli' | 'api_proxy' | 'mcp' | 'remote_agent'
|
|
54
|
+
* @param {number} [options.pid] - subprocess PID (local_cli only)
|
|
55
|
+
* @param {number} [options.writeIntervalMs=1000] - min interval between file writes
|
|
56
|
+
* @param {number} [options.silenceThresholdMs=30000] - silence detection threshold
|
|
57
|
+
* @returns {DispatchProgressTracker}
|
|
58
|
+
*/
|
|
59
|
+
export function createDispatchProgressTracker(root, turn, options = {}) {
|
|
60
|
+
const {
|
|
61
|
+
adapter_type = 'local_cli',
|
|
62
|
+
pid = null,
|
|
63
|
+
writeIntervalMs = 1000,
|
|
64
|
+
silenceThresholdMs = 30000,
|
|
65
|
+
} = options;
|
|
66
|
+
|
|
67
|
+
const filePath = getDispatchProgressFilePath(root, turn.turn_id);
|
|
68
|
+
|
|
69
|
+
let state = {
|
|
70
|
+
turn_id: turn.turn_id,
|
|
71
|
+
runtime_id: turn.runtime_id || null,
|
|
72
|
+
adapter_type,
|
|
73
|
+
started_at: null,
|
|
74
|
+
last_activity_at: null,
|
|
75
|
+
activity_type: 'output',
|
|
76
|
+
activity_summary: 'Dispatch starting',
|
|
77
|
+
output_lines: 0,
|
|
78
|
+
stderr_lines: 0,
|
|
79
|
+
silent_since: null,
|
|
80
|
+
pid,
|
|
81
|
+
};
|
|
82
|
+
|
|
83
|
+
let lastWriteAt = 0;
|
|
84
|
+
let silenceTimer = null;
|
|
85
|
+
let dirty = false;
|
|
86
|
+
|
|
87
|
+
function writeProgress() {
|
|
88
|
+
try {
|
|
89
|
+
const dir = dirname(filePath);
|
|
90
|
+
if (!existsSync(dir)) {
|
|
91
|
+
mkdirSync(dir, { recursive: true });
|
|
92
|
+
}
|
|
93
|
+
writeFileSync(filePath, JSON.stringify(state, null, 2) + '\n');
|
|
94
|
+
lastWriteAt = Date.now();
|
|
95
|
+
dirty = false;
|
|
96
|
+
} catch {
|
|
97
|
+
// Best-effort — never interrupt dispatch.
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
function maybeWrite() {
|
|
102
|
+
if (!dirty) return;
|
|
103
|
+
const now = Date.now();
|
|
104
|
+
if (now - lastWriteAt >= writeIntervalMs) {
|
|
105
|
+
writeProgress();
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
function resetSilenceTimer() {
|
|
110
|
+
if (silenceTimer) clearTimeout(silenceTimer);
|
|
111
|
+
silenceTimer = setTimeout(() => {
|
|
112
|
+
state.activity_type = 'silent';
|
|
113
|
+
state.silent_since = state.silent_since || new Date().toISOString();
|
|
114
|
+
state.activity_summary = `No output for ${Math.round(silenceThresholdMs / 1000)}s`;
|
|
115
|
+
dirty = true;
|
|
116
|
+
writeProgress();
|
|
117
|
+
}, silenceThresholdMs);
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
return {
|
|
121
|
+
/** Start tracking — call once at dispatch start. */
|
|
122
|
+
start() {
|
|
123
|
+
const now = new Date().toISOString();
|
|
124
|
+
state.started_at = now;
|
|
125
|
+
state.last_activity_at = now;
|
|
126
|
+
state.activity_type = 'output';
|
|
127
|
+
state.activity_summary = 'Subprocess started';
|
|
128
|
+
dirty = true;
|
|
129
|
+
writeProgress();
|
|
130
|
+
if (adapter_type === 'local_cli') {
|
|
131
|
+
resetSilenceTimer();
|
|
132
|
+
}
|
|
133
|
+
},
|
|
134
|
+
|
|
135
|
+
/** Record output activity from the subprocess. */
|
|
136
|
+
onOutput(stream, lineCount = 1) {
|
|
137
|
+
const now = new Date().toISOString();
|
|
138
|
+
const wasSilent = state.activity_type === 'silent';
|
|
139
|
+
state.last_activity_at = now;
|
|
140
|
+
state.activity_type = 'output';
|
|
141
|
+
state.silent_since = null;
|
|
142
|
+
if (stream === 'stderr') {
|
|
143
|
+
state.stderr_lines += lineCount;
|
|
144
|
+
} else {
|
|
145
|
+
state.output_lines += lineCount;
|
|
146
|
+
}
|
|
147
|
+
state.activity_summary = `Producing output (${state.output_lines} lines)`;
|
|
148
|
+
dirty = true;
|
|
149
|
+
maybeWrite();
|
|
150
|
+
if (adapter_type === 'local_cli') {
|
|
151
|
+
resetSilenceTimer();
|
|
152
|
+
}
|
|
153
|
+
return wasSilent; // caller can use this to emit a "resumed" event
|
|
154
|
+
},
|
|
155
|
+
|
|
156
|
+
/** Mark as API request in flight (api_proxy, mcp, remote_agent). */
|
|
157
|
+
requestStarted() {
|
|
158
|
+
state.activity_type = 'request';
|
|
159
|
+
state.activity_summary = 'API request in flight';
|
|
160
|
+
state.last_activity_at = new Date().toISOString();
|
|
161
|
+
dirty = true;
|
|
162
|
+
writeProgress();
|
|
163
|
+
},
|
|
164
|
+
|
|
165
|
+
/** Mark API response received. */
|
|
166
|
+
responseReceived() {
|
|
167
|
+
state.activity_type = 'response';
|
|
168
|
+
state.activity_summary = 'API response received';
|
|
169
|
+
state.last_activity_at = new Date().toISOString();
|
|
170
|
+
dirty = true;
|
|
171
|
+
writeProgress();
|
|
172
|
+
},
|
|
173
|
+
|
|
174
|
+
/** Update PID after spawn (local_cli). */
|
|
175
|
+
setPid(newPid) {
|
|
176
|
+
state.pid = newPid;
|
|
177
|
+
dirty = true;
|
|
178
|
+
maybeWrite();
|
|
179
|
+
},
|
|
180
|
+
|
|
181
|
+
/** Get current progress state snapshot. */
|
|
182
|
+
getState() {
|
|
183
|
+
return { ...state };
|
|
184
|
+
},
|
|
185
|
+
|
|
186
|
+
/** Clean up — dispatch completed successfully. */
|
|
187
|
+
complete() {
|
|
188
|
+
if (silenceTimer) clearTimeout(silenceTimer);
|
|
189
|
+
deleteProgressFile(root, turn.turn_id);
|
|
190
|
+
},
|
|
191
|
+
|
|
192
|
+
/** Clean up — dispatch failed. */
|
|
193
|
+
fail() {
|
|
194
|
+
if (silenceTimer) clearTimeout(silenceTimer);
|
|
195
|
+
deleteProgressFile(root, turn.turn_id);
|
|
196
|
+
},
|
|
197
|
+
|
|
198
|
+
/** Clean up timers without deleting file (for abort paths). */
|
|
199
|
+
dispose() {
|
|
200
|
+
if (silenceTimer) clearTimeout(silenceTimer);
|
|
201
|
+
},
|
|
202
|
+
};
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
/**
|
|
206
|
+
* Delete the dispatch progress file.
|
|
207
|
+
* @param {string} root - project root
|
|
208
|
+
*/
|
|
209
|
+
export function deleteProgressFile(root, turnId = null) {
|
|
210
|
+
try {
|
|
211
|
+
if (turnId) {
|
|
212
|
+
const filePath = getDispatchProgressFilePath(root, turnId);
|
|
213
|
+
if (existsSync(filePath)) {
|
|
214
|
+
unlinkSync(filePath);
|
|
215
|
+
}
|
|
216
|
+
return;
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
const legacyPath = join(root, LEGACY_DISPATCH_PROGRESS_PATH);
|
|
220
|
+
if (existsSync(legacyPath)) {
|
|
221
|
+
unlinkSync(legacyPath);
|
|
222
|
+
}
|
|
223
|
+
for (const filePath of listDispatchProgressFiles(root)) {
|
|
224
|
+
if (existsSync(filePath)) {
|
|
225
|
+
unlinkSync(filePath);
|
|
226
|
+
}
|
|
227
|
+
}
|
|
228
|
+
} catch {
|
|
229
|
+
// Best-effort.
|
|
230
|
+
}
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
/**
|
|
234
|
+
* Read the current dispatch progress file.
|
|
235
|
+
* Returns null if no file exists or it's malformed.
|
|
236
|
+
*
|
|
237
|
+
* @param {string} root - project root
|
|
238
|
+
* @returns {object|null}
|
|
239
|
+
*/
|
|
240
|
+
export function readDispatchProgress(root, turnId = null) {
|
|
241
|
+
try {
|
|
242
|
+
let filePath;
|
|
243
|
+
if (turnId) {
|
|
244
|
+
filePath = getDispatchProgressFilePath(root, turnId);
|
|
245
|
+
if (!existsSync(filePath)) return null;
|
|
246
|
+
} else {
|
|
247
|
+
const files = listDispatchProgressFiles(root);
|
|
248
|
+
if (files.length === 0) {
|
|
249
|
+
const legacyPath = join(root, LEGACY_DISPATCH_PROGRESS_PATH);
|
|
250
|
+
if (!existsSync(legacyPath)) return null;
|
|
251
|
+
filePath = legacyPath;
|
|
252
|
+
} else if (files.length === 1) {
|
|
253
|
+
filePath = files[0];
|
|
254
|
+
} else {
|
|
255
|
+
return null;
|
|
256
|
+
}
|
|
257
|
+
}
|
|
258
|
+
const raw = readFileSync(filePath, 'utf8');
|
|
259
|
+
const data = JSON.parse(raw);
|
|
260
|
+
if (!data.turn_id || !data.started_at) return null;
|
|
261
|
+
return data;
|
|
262
|
+
} catch {
|
|
263
|
+
return null;
|
|
264
|
+
}
|
|
265
|
+
}
|
|
266
|
+
|
|
267
|
+
/**
|
|
268
|
+
* Read all current per-turn dispatch progress files.
|
|
269
|
+
*
|
|
270
|
+
* @param {string} root - project root
|
|
271
|
+
* @returns {Record<string, object>}
|
|
272
|
+
*/
|
|
273
|
+
export function readAllDispatchProgress(root) {
|
|
274
|
+
const progressByTurn = {};
|
|
275
|
+
|
|
276
|
+
for (const filePath of listDispatchProgressFiles(root)) {
|
|
277
|
+
try {
|
|
278
|
+
const raw = readFileSync(filePath, 'utf8');
|
|
279
|
+
const data = JSON.parse(raw);
|
|
280
|
+
const turnId = typeof data?.turn_id === 'string' && data.turn_id.length > 0
|
|
281
|
+
? data.turn_id
|
|
282
|
+
: basename(filePath).replace(/^dispatch-progress-/, '').replace(/\.json$/, '');
|
|
283
|
+
if (!turnId || !data?.started_at) continue;
|
|
284
|
+
progressByTurn[turnId] = data;
|
|
285
|
+
} catch {
|
|
286
|
+
// Ignore malformed files.
|
|
287
|
+
}
|
|
288
|
+
}
|
|
289
|
+
|
|
290
|
+
if (Object.keys(progressByTurn).length === 0) {
|
|
291
|
+
const legacy = readDispatchProgress(root);
|
|
292
|
+
if (legacy?.turn_id) {
|
|
293
|
+
progressByTurn[legacy.turn_id] = legacy;
|
|
294
|
+
}
|
|
295
|
+
}
|
|
296
|
+
|
|
297
|
+
return progressByTurn;
|
|
298
|
+
}
|
package/src/lib/repo-observer.js
CHANGED
package/src/lib/run-events.js
CHANGED
package/src/lib/run-loop.js
CHANGED
|
@@ -12,7 +12,7 @@
|
|
|
12
12
|
* - Never calls process dot exit
|
|
13
13
|
* - No stdout/stderr
|
|
14
14
|
* - No adapter dispatch (caller provides dispatch callback)
|
|
15
|
-
* -
|
|
15
|
+
* - Governed lifecycle operations import through runner-interface.js
|
|
16
16
|
*/
|
|
17
17
|
|
|
18
18
|
import {
|
|
@@ -21,6 +21,7 @@ import {
|
|
|
21
21
|
assignTurn,
|
|
22
22
|
acceptTurn,
|
|
23
23
|
rejectTurn,
|
|
24
|
+
markRunBlocked,
|
|
24
25
|
writeDispatchBundle,
|
|
25
26
|
getTurnStagingResultPath,
|
|
26
27
|
approvePhaseGate,
|
|
@@ -33,10 +34,11 @@ import {
|
|
|
33
34
|
} from './runner-interface.js';
|
|
34
35
|
|
|
35
36
|
import { runAdmissionControl } from './admission-control.js';
|
|
36
|
-
import { mkdirSync, writeFileSync } from 'fs';
|
|
37
|
+
import { appendFileSync, mkdirSync, writeFileSync } from 'fs';
|
|
37
38
|
import { join, dirname } from 'path';
|
|
38
39
|
import { evaluateApprovalSlaReminders } from './notification-runner.js';
|
|
39
40
|
import { readPreemptionMarker } from './intake.js';
|
|
41
|
+
import { buildTimeoutBlockedReason, evaluateTimeouts } from './timeout-evaluator.js';
|
|
40
42
|
|
|
41
43
|
const DEFAULT_MAX_TURNS = 50;
|
|
42
44
|
|
|
@@ -214,6 +216,7 @@ async function executeSequentialTurn(root, config, state, callbacks, emit, error
|
|
|
214
216
|
async function executeParallelTurns(root, config, state, maxConcurrent, callbacks, emit, errors) {
|
|
215
217
|
const history = [];
|
|
216
218
|
let acceptedCount = 0;
|
|
219
|
+
const timedOutDispatches = [];
|
|
217
220
|
|
|
218
221
|
// ── Collect active turns that need dispatch (retries) ────────────────
|
|
219
222
|
const activeTurns = getActiveTurns(state);
|
|
@@ -332,7 +335,7 @@ async function executeParallelTurns(root, config, state, maxConcurrent, callback
|
|
|
332
335
|
const dispatchResults = await Promise.allSettled(
|
|
333
336
|
contexts.map(async (ctx) => {
|
|
334
337
|
try {
|
|
335
|
-
return { ctx, result: await callbacks.dispatch
|
|
338
|
+
return { ctx, result: await dispatchWithTimeout(ctx, config, callbacks.dispatch) };
|
|
336
339
|
} catch (err) {
|
|
337
340
|
return { ctx, result: { accept: false, reason: `dispatch threw: ${err.message}` } };
|
|
338
341
|
}
|
|
@@ -350,6 +353,11 @@ async function executeParallelTurns(root, config, state, maxConcurrent, callback
|
|
|
350
353
|
const { turn } = ctx;
|
|
351
354
|
const roleId = turn.assigned_role;
|
|
352
355
|
|
|
356
|
+
if (dispatchResult?.timed_out === true) {
|
|
357
|
+
timedOutDispatches.push({ ctx, dispatchResult });
|
|
358
|
+
continue;
|
|
359
|
+
}
|
|
360
|
+
|
|
353
361
|
if (dispatchResult.accept) {
|
|
354
362
|
const absStaging = join(root, ctx.stagingPath);
|
|
355
363
|
mkdirSync(dirname(absStaging), { recursive: true });
|
|
@@ -405,6 +413,13 @@ async function executeParallelTurns(root, config, state, maxConcurrent, callback
|
|
|
405
413
|
}
|
|
406
414
|
}
|
|
407
415
|
|
|
416
|
+
if (timedOutDispatches.length > 0) {
|
|
417
|
+
const timedOut = timedOutDispatches[0];
|
|
418
|
+
const blocked = persistDispatchTimeout(root, config, timedOut.ctx.turn, timedOut.dispatchResult.timeout_result, errors);
|
|
419
|
+
emit({ type: 'blocked', state: blocked.state, reason: 'timeout:turn' });
|
|
420
|
+
return { terminal: true, ok: false, stop_reason: 'blocked', history, acceptedCount };
|
|
421
|
+
}
|
|
422
|
+
|
|
408
423
|
// ── Stall detection: if no turns were accepted and no new roles were ──
|
|
409
424
|
// ── assignable, terminate to avoid infinite re-dispatch loops. ────────
|
|
410
425
|
if (acceptedCount === 0 && history.length > 0) {
|
|
@@ -445,12 +460,18 @@ async function dispatchAndProcess(root, config, turn, assignState, callbacks, em
|
|
|
445
460
|
|
|
446
461
|
let dispatchResult;
|
|
447
462
|
try {
|
|
448
|
-
dispatchResult = await callbacks.dispatch
|
|
463
|
+
dispatchResult = await dispatchWithTimeout(context, config, callbacks.dispatch);
|
|
449
464
|
} catch (err) {
|
|
450
465
|
errors.push(`dispatch threw for ${roleId}: ${err.message}`);
|
|
451
466
|
return { terminal: true, ok: false, stop_reason: 'dispatch_error', history };
|
|
452
467
|
}
|
|
453
468
|
|
|
469
|
+
if (dispatchResult?.timed_out === true) {
|
|
470
|
+
const blocked = persistDispatchTimeout(root, config, turn, dispatchResult.timeout_result, errors);
|
|
471
|
+
emit({ type: 'blocked', state: blocked.state, reason: 'timeout:turn' });
|
|
472
|
+
return { terminal: true, ok: false, stop_reason: 'blocked', history };
|
|
473
|
+
}
|
|
474
|
+
|
|
454
475
|
if (dispatchResult.accept) {
|
|
455
476
|
const absStaging = join(root, stagingPath);
|
|
456
477
|
mkdirSync(dirname(absStaging), { recursive: true });
|
|
@@ -574,3 +595,107 @@ function makeResult(ok, stop_reason, state, turns_executed, turn_history, gates_
|
|
|
574
595
|
function noop() {}
|
|
575
596
|
|
|
576
597
|
export { DEFAULT_MAX_TURNS };
|
|
598
|
+
|
|
599
|
+
function buildTimeoutLedgerEntry(timeoutResult, timestamp, turnId, phase) {
|
|
600
|
+
return {
|
|
601
|
+
type: 'timeout',
|
|
602
|
+
scope: timeoutResult.scope,
|
|
603
|
+
phase: timeoutResult.phase || phase || null,
|
|
604
|
+
turn_id: turnId || null,
|
|
605
|
+
limit_minutes: timeoutResult.limit_minutes,
|
|
606
|
+
elapsed_minutes: timeoutResult.elapsed_minutes,
|
|
607
|
+
exceeded_by_minutes: timeoutResult.exceeded_by_minutes,
|
|
608
|
+
action: timeoutResult.action,
|
|
609
|
+
timestamp,
|
|
610
|
+
};
|
|
611
|
+
}
|
|
612
|
+
|
|
613
|
+
function appendJsonl(root, relPath, value) {
|
|
614
|
+
appendFileSync(join(root, relPath), `${JSON.stringify(value)}\n`);
|
|
615
|
+
}
|
|
616
|
+
|
|
617
|
+
function getDispatchTimeoutResult(config, state, turn, now = new Date()) {
|
|
618
|
+
const evaluation = evaluateTimeouts({ config, state, turn, now });
|
|
619
|
+
return evaluation.exceeded.find((entry) => entry.scope === 'turn' && entry.action === 'escalate') || null;
|
|
620
|
+
}
|
|
621
|
+
|
|
622
|
+
async function dispatchWithTimeout(context, config, dispatchFn) {
|
|
623
|
+
const timeoutResult = getDispatchTimeoutResult(config, context.state, context.turn);
|
|
624
|
+
if (!timeoutResult) {
|
|
625
|
+
return await dispatchFn(context);
|
|
626
|
+
}
|
|
627
|
+
|
|
628
|
+
const remainingMs = Math.max(
|
|
629
|
+
0,
|
|
630
|
+
timeoutResult.limit_minutes * 60 * 1000
|
|
631
|
+
- Math.max(0, new Date() - new Date(context.turn.started_at || context.turn.assigned_at || new Date())),
|
|
632
|
+
);
|
|
633
|
+
const abortController = new AbortController();
|
|
634
|
+
if (remainingMs === 0) {
|
|
635
|
+
abortController.abort(new Error(`Turn timeout exceeded after ${timeoutResult.limit_minutes}m`));
|
|
636
|
+
return {
|
|
637
|
+
timed_out: true,
|
|
638
|
+
timeout_result: timeoutResult,
|
|
639
|
+
};
|
|
640
|
+
}
|
|
641
|
+
const enrichedContext = {
|
|
642
|
+
...context,
|
|
643
|
+
dispatchTimeoutMs: remainingMs,
|
|
644
|
+
dispatchDeadlineAt: new Date(Date.now() + remainingMs).toISOString(),
|
|
645
|
+
dispatchAbortSignal: abortController.signal,
|
|
646
|
+
};
|
|
647
|
+
|
|
648
|
+
let timer = null;
|
|
649
|
+
const dispatchPromise = Promise.resolve(dispatchFn(enrichedContext))
|
|
650
|
+
.then((result) => ({ kind: 'result', result }))
|
|
651
|
+
.catch((error) => ({ kind: 'error', error }));
|
|
652
|
+
const timeoutPromise = new Promise((resolve) => {
|
|
653
|
+
timer = setTimeout(() => {
|
|
654
|
+
abortController.abort(new Error(`Turn timeout exceeded after ${timeoutResult.limit_minutes}m`));
|
|
655
|
+
resolve({ kind: 'timeout', timeoutResult });
|
|
656
|
+
}, remainingMs);
|
|
657
|
+
});
|
|
658
|
+
|
|
659
|
+
const winner = await Promise.race([dispatchPromise, timeoutPromise]);
|
|
660
|
+
clearTimeout(timer);
|
|
661
|
+
|
|
662
|
+
if (winner.kind === 'timeout') {
|
|
663
|
+
return {
|
|
664
|
+
timed_out: true,
|
|
665
|
+
timeout_result: winner.timeoutResult,
|
|
666
|
+
};
|
|
667
|
+
}
|
|
668
|
+
|
|
669
|
+
if (winner.kind === 'error') {
|
|
670
|
+
throw winner.error;
|
|
671
|
+
}
|
|
672
|
+
|
|
673
|
+
return winner.result;
|
|
674
|
+
}
|
|
675
|
+
|
|
676
|
+
function persistDispatchTimeout(root, config, turn, timeoutResult, errors) {
|
|
677
|
+
const blockedAt = new Date().toISOString();
|
|
678
|
+
const blockedReason = buildTimeoutBlockedReason(timeoutResult, { turnRetained: true });
|
|
679
|
+
const blocked = markRunBlocked(root, {
|
|
680
|
+
blockedOn: `timeout:${timeoutResult.scope}`,
|
|
681
|
+
category: blockedReason.category,
|
|
682
|
+
recovery: blockedReason.recovery,
|
|
683
|
+
turnId: turn.turn_id,
|
|
684
|
+
blockedAt,
|
|
685
|
+
notificationConfig: config,
|
|
686
|
+
});
|
|
687
|
+
|
|
688
|
+
if (!blocked.ok) {
|
|
689
|
+
errors.push(`markRunBlocked(timeout): ${blocked.error}`);
|
|
690
|
+
return { state: loadState(root, config) };
|
|
691
|
+
}
|
|
692
|
+
|
|
693
|
+
try {
|
|
694
|
+
appendJsonl(root, '.agentxchain/decision-ledger.jsonl', buildTimeoutLedgerEntry(timeoutResult, blockedAt, turn.turn_id, blocked.state?.phase));
|
|
695
|
+
} catch (err) {
|
|
696
|
+
errors.push(`timeout ledger append failed: ${err.message}`);
|
|
697
|
+
}
|
|
698
|
+
|
|
699
|
+
errors.push(`dispatch timed out for ${turn.assigned_role} after ${timeoutResult.limit_minutes}m`);
|
|
700
|
+
return blocked;
|
|
701
|
+
}
|
|
@@ -210,6 +210,93 @@ export function validateTimeoutsConfig(timeouts, routing) {
|
|
|
210
210
|
return { ok: errors.length === 0, errors };
|
|
211
211
|
}
|
|
212
212
|
|
|
213
|
+
/**
|
|
214
|
+
* Compute remaining timeout budget for an active turn/phase/run.
|
|
215
|
+
*
|
|
216
|
+
* Unlike evaluateTimeouts() which only returns items when exceeded,
|
|
217
|
+
* this returns budget info for ALL configured timeout scopes regardless
|
|
218
|
+
* of whether the deadline has passed.
|
|
219
|
+
*
|
|
220
|
+
* @param {object} options
|
|
221
|
+
* @param {object} options.config - Normalized config with optional `timeouts` section
|
|
222
|
+
* @param {object} options.state - Current governed state
|
|
223
|
+
* @param {object} [options.turn] - Active turn metadata
|
|
224
|
+
* @param {Date|string} [options.now] - Override for current time (testing)
|
|
225
|
+
* @returns {Array<TimeoutBudget>} Array of { scope, limit_minutes, elapsed_minutes, remaining_minutes, deadline_iso, exceeded, action }
|
|
226
|
+
*/
|
|
227
|
+
export function computeTimeoutBudget({ config, state, turn = null, now = new Date() }) {
|
|
228
|
+
const timeouts = config?.timeouts;
|
|
229
|
+
if (!timeouts) return [];
|
|
230
|
+
|
|
231
|
+
const nowMs = typeof now === 'string' ? new Date(now).getTime() : now.getTime();
|
|
232
|
+
const budgets = [];
|
|
233
|
+
|
|
234
|
+
// Per-turn budget
|
|
235
|
+
if (timeouts.per_turn_minutes && turn) {
|
|
236
|
+
const startedAt = turn.started_at || turn.assigned_at;
|
|
237
|
+
if (startedAt) {
|
|
238
|
+
const dispatchMs = new Date(startedAt).getTime();
|
|
239
|
+
const limitMs = timeouts.per_turn_minutes * 60 * 1000;
|
|
240
|
+
const elapsedMs = nowMs - dispatchMs;
|
|
241
|
+
const remainingMs = limitMs - elapsedMs;
|
|
242
|
+
budgets.push({
|
|
243
|
+
scope: 'turn',
|
|
244
|
+
limit_minutes: timeouts.per_turn_minutes,
|
|
245
|
+
elapsed_minutes: Math.round(elapsedMs / 60000),
|
|
246
|
+
remaining_minutes: Math.max(0, Math.round(remainingMs / 60000)),
|
|
247
|
+
remaining_seconds: Math.max(0, Math.round(remainingMs / 1000)),
|
|
248
|
+
deadline_iso: new Date(dispatchMs + limitMs).toISOString(),
|
|
249
|
+
exceeded: elapsedMs > limitMs,
|
|
250
|
+
action: resolveAction(timeouts.action, 'turn'),
|
|
251
|
+
});
|
|
252
|
+
}
|
|
253
|
+
}
|
|
254
|
+
|
|
255
|
+
// Per-phase budget
|
|
256
|
+
const phaseLimit = resolvePhaseLimit(timeouts, config.routing, state.phase);
|
|
257
|
+
const phaseAction = resolvePhaseAction(timeouts, config.routing, state.phase);
|
|
258
|
+
if (phaseLimit) {
|
|
259
|
+
const phaseEnteredAt = findPhaseEntryTime(state);
|
|
260
|
+
if (phaseEnteredAt) {
|
|
261
|
+
const entryMs = new Date(phaseEnteredAt).getTime();
|
|
262
|
+
const limitMs = phaseLimit * 60 * 1000;
|
|
263
|
+
const elapsedMs = nowMs - entryMs;
|
|
264
|
+
const remainingMs = limitMs - elapsedMs;
|
|
265
|
+
budgets.push({
|
|
266
|
+
scope: 'phase',
|
|
267
|
+
phase: state.phase,
|
|
268
|
+
limit_minutes: phaseLimit,
|
|
269
|
+
elapsed_minutes: Math.round(elapsedMs / 60000),
|
|
270
|
+
remaining_minutes: Math.max(0, Math.round(remainingMs / 60000)),
|
|
271
|
+
remaining_seconds: Math.max(0, Math.round(remainingMs / 1000)),
|
|
272
|
+
deadline_iso: new Date(entryMs + limitMs).toISOString(),
|
|
273
|
+
exceeded: elapsedMs > limitMs,
|
|
274
|
+
action: phaseAction,
|
|
275
|
+
});
|
|
276
|
+
}
|
|
277
|
+
}
|
|
278
|
+
|
|
279
|
+
// Per-run budget
|
|
280
|
+
if (timeouts.per_run_minutes && state.created_at) {
|
|
281
|
+
const createMs = new Date(state.created_at).getTime();
|
|
282
|
+
const limitMs = timeouts.per_run_minutes * 60 * 1000;
|
|
283
|
+
const elapsedMs = nowMs - createMs;
|
|
284
|
+
const remainingMs = limitMs - elapsedMs;
|
|
285
|
+
budgets.push({
|
|
286
|
+
scope: 'run',
|
|
287
|
+
limit_minutes: timeouts.per_run_minutes,
|
|
288
|
+
elapsed_minutes: Math.round(elapsedMs / 60000),
|
|
289
|
+
remaining_minutes: Math.max(0, Math.round(remainingMs / 60000)),
|
|
290
|
+
remaining_seconds: Math.max(0, Math.round(remainingMs / 1000)),
|
|
291
|
+
deadline_iso: new Date(createMs + limitMs).toISOString(),
|
|
292
|
+
exceeded: elapsedMs > limitMs,
|
|
293
|
+
action: resolveAction(timeouts.action, 'run'),
|
|
294
|
+
});
|
|
295
|
+
}
|
|
296
|
+
|
|
297
|
+
return budgets;
|
|
298
|
+
}
|
|
299
|
+
|
|
213
300
|
/**
|
|
214
301
|
* Build a blocked_reason descriptor for a timeout.
|
|
215
302
|
*/
|