agentxchain 2.145.0 → 2.147.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dashboard/app.js +3 -0
- package/dashboard/components/notifications.js +127 -0
- package/dashboard/index.html +1 -0
- package/package.json +1 -1
- package/scripts/publish-npm.sh +16 -0
- package/scripts/release-downstream-truth.sh +16 -8
- package/scripts/sync-homebrew.sh +14 -1
- package/scripts/verify-post-publish.sh +55 -4
- package/src/commands/init.js +66 -31
- package/src/commands/reissue-turn.js +16 -0
- package/src/commands/reject-turn.js +14 -1
- package/src/commands/restart.js +33 -3
- package/src/commands/resume.js +78 -66
- package/src/commands/run.js +67 -10
- package/src/commands/schedule.js +34 -7
- package/src/commands/status.js +38 -5
- package/src/commands/step.js +117 -34
- package/src/lib/adapters/api-proxy-adapter.js +8 -0
- package/src/lib/adapters/local-cli-adapter.js +131 -13
- package/src/lib/adapters/manual-adapter.js +9 -10
- package/src/lib/adapters/mcp-adapter.js +3 -5
- package/src/lib/adapters/remote-agent-adapter.js +3 -5
- package/src/lib/config.js +4 -1
- package/src/lib/continuous-run.js +71 -6
- package/src/lib/dashboard/actions.js +9 -3
- package/src/lib/dashboard/bridge-server.js +11 -0
- package/src/lib/dashboard/notifications-reader.js +91 -0
- package/src/lib/dashboard/state-reader.js +16 -4
- package/src/lib/dispatch-bundle.js +1 -1
- package/src/lib/dispatch-progress.js +5 -3
- package/src/lib/governed-state.js +355 -13
- package/src/lib/intake.js +10 -1
- package/src/lib/normalized-config.js +51 -1
- package/src/lib/recent-event-summary.js +12 -0
- package/src/lib/run-events.js +4 -0
- package/src/lib/run-loop.js +67 -2
- package/src/lib/runner-interface.js +1 -0
- package/src/lib/schema.js +7 -0
- package/src/lib/schemas/agentxchain-config.schema.json +15 -1
- package/src/lib/staged-result-proof.js +43 -0
- package/src/lib/stale-turn-watchdog.js +308 -34
- package/src/lib/turn-result-shape.js +38 -0
- package/src/lib/turn-result-validator.js +4 -1
package/src/commands/resume.js
CHANGED
|
@@ -8,8 +8,8 @@
|
|
|
8
8
|
* - resolves target role from routing or --role override
|
|
9
9
|
* - if idle + no run_id → initializeGovernedRun() + assign
|
|
10
10
|
* - if paused + run_id exists → resume same run + assign
|
|
11
|
-
* - if
|
|
12
|
-
* - if active +
|
|
11
|
+
* - if blocked + retained active turn with failed status → re-dispatch same turn
|
|
12
|
+
* - if active + an active turn already exists → reject (no double assignment)
|
|
13
13
|
* - materializes a turn-scoped dispatch bundle under .agentxchain/dispatch/turns/<turn_id>/
|
|
14
14
|
* - exits without waiting for turn completion
|
|
15
15
|
*/
|
|
@@ -26,6 +26,8 @@ import {
|
|
|
26
26
|
getActiveTurns,
|
|
27
27
|
getActiveTurnCount,
|
|
28
28
|
reactivateGovernedRun,
|
|
29
|
+
reconcilePhaseAdvanceBeforeDispatch,
|
|
30
|
+
transitionActiveTurnLifecycle,
|
|
29
31
|
STATE_PATH,
|
|
30
32
|
} from '../lib/governed-state.js';
|
|
31
33
|
import { writeDispatchBundle, getDispatchTurnDir, getTurnStagingResultPath } from '../lib/dispatch-bundle.js';
|
|
@@ -78,6 +80,10 @@ export async function resumeCommand(opts) {
|
|
|
78
80
|
|
|
79
81
|
const staleReconciliation = reconcileStaleTurns(root, state, config);
|
|
80
82
|
state = staleReconciliation.state || state;
|
|
83
|
+
if (staleReconciliation.ghost_turns.length > 0) {
|
|
84
|
+
printGhostTurnRecovery(staleReconciliation.ghost_turns);
|
|
85
|
+
process.exit(1);
|
|
86
|
+
}
|
|
81
87
|
if (staleReconciliation.stale_turns.length > 0) {
|
|
82
88
|
printStaleTurnRecovery(staleReconciliation.stale_turns);
|
|
83
89
|
process.exit(1);
|
|
@@ -116,70 +122,25 @@ export async function resumeCommand(opts) {
|
|
|
116
122
|
process.exit(1);
|
|
117
123
|
}
|
|
118
124
|
|
|
119
|
-
// §47
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
retainedTurn = Object.values(activeTurns)[0];
|
|
139
|
-
}
|
|
140
|
-
|
|
141
|
-
const turnStatus = retainedTurn.status;
|
|
142
|
-
if (turnStatus === 'failed' || turnStatus === 'retrying') {
|
|
143
|
-
printResumeRunContext({ root, state, config });
|
|
144
|
-
console.log(chalk.yellow(`Re-dispatching failed turn: ${retainedTurn.turn_id}`));
|
|
145
|
-
console.log(` Role: ${retainedTurn.assigned_role}`);
|
|
146
|
-
console.log(` Attempt: ${retainedTurn.attempt}`);
|
|
147
|
-
console.log('');
|
|
148
|
-
|
|
149
|
-
const reactivated = reactivateGovernedRun(root, state, { via: turnResumeVia, notificationConfig: config });
|
|
150
|
-
if (!reactivated.ok) {
|
|
151
|
-
console.log(chalk.red(`Failed to reactivate run: ${reactivated.error}`));
|
|
152
|
-
process.exit(1);
|
|
153
|
-
}
|
|
154
|
-
state = reactivated.state;
|
|
155
|
-
if (reactivated.migration_notice) {
|
|
156
|
-
console.log(chalk.yellow(reactivated.migration_notice));
|
|
157
|
-
}
|
|
158
|
-
if (reactivated.phantom_notice) {
|
|
159
|
-
console.log(chalk.yellow(reactivated.phantom_notice));
|
|
160
|
-
}
|
|
161
|
-
|
|
162
|
-
// Write dispatch bundle for the existing turn
|
|
163
|
-
const bundleResult = writeDispatchBundle(root, state, config);
|
|
164
|
-
if (!bundleResult.ok) {
|
|
165
|
-
console.log(chalk.red(`Failed to write dispatch bundle: ${bundleResult.error}`));
|
|
166
|
-
process.exit(1);
|
|
167
|
-
}
|
|
168
|
-
printDispatchBundleWarnings(bundleResult);
|
|
169
|
-
|
|
170
|
-
// after_dispatch hooks with bundle-core tamper protection
|
|
171
|
-
const hooksConfig = config.hooks || {};
|
|
172
|
-
if (hooksConfig.after_dispatch?.length > 0) {
|
|
173
|
-
const afterDispatchResult = runAfterDispatchHooks(root, hooksConfig, state, retainedTurn);
|
|
174
|
-
if (!afterDispatchResult.ok) {
|
|
175
|
-
process.exit(1);
|
|
176
|
-
}
|
|
177
|
-
}
|
|
178
|
-
|
|
179
|
-
printDispatchSummary(state, config, retainedTurn);
|
|
180
|
-
return;
|
|
181
|
-
}
|
|
182
|
-
}
|
|
125
|
+
// Removed (Turn 25): the §47 `paused + retained turn → re-dispatch failed/retrying`
|
|
126
|
+
// branch is provably unreachable under the current schema and migration contract:
|
|
127
|
+
//
|
|
128
|
+
// 1. `cli/src/lib/schema.js:184` rejects `status: 'paused'` unless
|
|
129
|
+
// `pending_phase_transition` or `pending_run_completion` is set.
|
|
130
|
+
// 2. The guard above (line 119) short-circuits with `printRecoverySummary`
|
|
131
|
+
// whenever either pending field is set — so any schema-valid paused state
|
|
132
|
+
// exits before reaching this point.
|
|
133
|
+
// 3. Legacy on-disk shapes that pre-date the schema constraint (paused +
|
|
134
|
+
// `blocked_on: 'human:...'` / `blocked_on: 'escalation:...'` with no
|
|
135
|
+
// pending approval) are auto-migrated to `status: 'blocked'` by
|
|
136
|
+
// `normalizeStateForRead` in `governed-state.js:2191-2204` before
|
|
137
|
+
// `loadProjectState` returns.
|
|
138
|
+
//
|
|
139
|
+
// The reachable retained-turn re-dispatch path is the `blocked + activeCount > 0`
|
|
140
|
+
// branch immediately below, which legacy paused-pause shapes are migrated into.
|
|
141
|
+
// Per `DEC-UNREACHABLE-BRANCH-COVERAGE-001`, dead branches are removed (not
|
|
142
|
+
// patched defensively) once the schema citation + migration citation are
|
|
143
|
+
// documented in code and the coverage matrix.
|
|
183
144
|
|
|
184
145
|
if (state.status === 'blocked' && activeCount > 0) {
|
|
185
146
|
let retainedTurn = null;
|
|
@@ -240,6 +201,21 @@ export async function resumeCommand(opts) {
|
|
|
240
201
|
}
|
|
241
202
|
}
|
|
242
203
|
|
|
204
|
+
// BUG-51 follow-up: see comment in paused/failed retained-turn branch.
|
|
205
|
+
// The blocked re-dispatch path has the same watchdog/manifest invariant.
|
|
206
|
+
const manifestResult = finalizeDispatchManifest(root, retainedTurn.turn_id, {
|
|
207
|
+
run_id: state.run_id,
|
|
208
|
+
role: retainedTurn.assigned_role,
|
|
209
|
+
});
|
|
210
|
+
if (!manifestResult.ok) {
|
|
211
|
+
console.log(chalk.red(`Failed to finalize dispatch manifest: ${manifestResult.error}`));
|
|
212
|
+
process.exit(1);
|
|
213
|
+
}
|
|
214
|
+
const dispatched = transitionActiveTurnLifecycle(root, retainedTurn.turn_id, 'dispatched');
|
|
215
|
+
if (dispatched.ok) {
|
|
216
|
+
state = dispatched.state;
|
|
217
|
+
}
|
|
218
|
+
|
|
243
219
|
printDispatchSummary(state, config, retainedTurn);
|
|
244
220
|
return;
|
|
245
221
|
}
|
|
@@ -295,6 +271,24 @@ export async function resumeCommand(opts) {
|
|
|
295
271
|
}
|
|
296
272
|
}
|
|
297
273
|
|
|
274
|
+
const phaseReconciliation = reconcilePhaseAdvanceBeforeDispatch(root, config, state);
|
|
275
|
+
if (!phaseReconciliation.ok && !phaseReconciliation.state) {
|
|
276
|
+
console.log(chalk.red(`Failed to reconcile phase gate before dispatch: ${phaseReconciliation.error}`));
|
|
277
|
+
process.exit(1);
|
|
278
|
+
}
|
|
279
|
+
state = phaseReconciliation.state || state;
|
|
280
|
+
if (phaseReconciliation.advanced) {
|
|
281
|
+
console.log(chalk.green(`Advanced phase before dispatch: ${phaseReconciliation.from_phase} → ${phaseReconciliation.to_phase}`));
|
|
282
|
+
}
|
|
283
|
+
if (state.pending_phase_transition || state.pending_run_completion) {
|
|
284
|
+
printRecoverySummary(state, 'This run is awaiting approval.', config);
|
|
285
|
+
process.exit(1);
|
|
286
|
+
}
|
|
287
|
+
if (state.status === 'blocked') {
|
|
288
|
+
printRecoverySummary(state, 'This run is blocked.', config);
|
|
289
|
+
process.exit(1);
|
|
290
|
+
}
|
|
291
|
+
|
|
298
292
|
// Print run-context header before dispatch
|
|
299
293
|
printResumeRunContext({ root, state, config });
|
|
300
294
|
|
|
@@ -356,9 +350,27 @@ export async function resumeCommand(opts) {
|
|
|
356
350
|
process.exit(1);
|
|
357
351
|
}
|
|
358
352
|
|
|
353
|
+
const dispatched = transitionActiveTurnLifecycle(root, turn.turn_id, 'dispatched');
|
|
354
|
+
if (dispatched.ok) {
|
|
355
|
+
state = dispatched.state;
|
|
356
|
+
}
|
|
357
|
+
|
|
359
358
|
printDispatchSummary(state, config);
|
|
360
359
|
}
|
|
361
360
|
|
|
361
|
+
function printGhostTurnRecovery(ghostTurns) {
|
|
362
|
+
console.log(chalk.red.bold('Ghost turn detected — subprocess never started.'));
|
|
363
|
+
console.log('');
|
|
364
|
+
for (const ghost of ghostTurns) {
|
|
365
|
+
const secs = Math.floor(ghost.running_ms / 1000);
|
|
366
|
+
console.log(` Turn: ${ghost.turn_id} (${ghost.role})`);
|
|
367
|
+
console.log(` Runtime: ${ghost.runtime_id}`);
|
|
368
|
+
console.log(` Age: ${secs}s with no subprocess output`);
|
|
369
|
+
console.log(` Recover: ${chalk.cyan(`agentxchain reissue-turn --turn ${ghost.turn_id} --reason ghost`)}`);
|
|
370
|
+
console.log('');
|
|
371
|
+
}
|
|
372
|
+
}
|
|
373
|
+
|
|
362
374
|
function printStaleTurnRecovery(staleTurns) {
|
|
363
375
|
console.log(chalk.red.bold('Stale turn detected.'));
|
|
364
376
|
console.log('');
|
package/src/commands/run.js
CHANGED
|
@@ -18,6 +18,7 @@ import { readFileSync, existsSync, mkdirSync, writeFileSync } from 'fs';
|
|
|
18
18
|
import { join } from 'path';
|
|
19
19
|
import { loadProjectContext, loadProjectState } from '../lib/config.js';
|
|
20
20
|
import { runLoop } from '../lib/run-loop.js';
|
|
21
|
+
import { transitionActiveTurnLifecycle } from '../lib/runner-interface.js';
|
|
21
22
|
import { buildRunExport } from '../lib/export.js';
|
|
22
23
|
import { buildGovernanceReport, formatGovernanceReportMarkdown } from '../lib/report.js';
|
|
23
24
|
import { validateParentRun } from '../lib/run-history.js';
|
|
@@ -49,6 +50,8 @@ import { resolveContinuousOptions, executeContinuousRun } from '../lib/continuou
|
|
|
49
50
|
import { createDispatchProgressTracker } from '../lib/dispatch-progress.js';
|
|
50
51
|
import { emitRunEvent } from '../lib/run-events.js';
|
|
51
52
|
import { checkpointAcceptedTurn } from '../lib/turn-checkpoint.js';
|
|
53
|
+
import { failTurnStartup } from '../lib/stale-turn-watchdog.js';
|
|
54
|
+
import { hasMinimumTurnResultShape } from '../lib/turn-result-shape.js';
|
|
52
55
|
|
|
53
56
|
export async function runCommand(opts) {
|
|
54
57
|
const context = loadProjectContext();
|
|
@@ -314,20 +317,49 @@ export async function executeGovernedRun(context, opts = {}) {
|
|
|
314
317
|
if (!manifestResult.ok) {
|
|
315
318
|
return { accept: false, reason: `dispatch manifest failed: ${manifestResult.error}` };
|
|
316
319
|
}
|
|
320
|
+
transitionActiveTurnLifecycle(projectRoot, turn.turn_id, 'dispatched');
|
|
317
321
|
|
|
318
322
|
// ── Route to adapter ──────────────────────────────────────────────
|
|
319
323
|
const tracker = createDispatchProgressTracker(projectRoot, turn, {
|
|
320
324
|
adapter_type: runtimeType,
|
|
321
325
|
});
|
|
326
|
+
let startupStarted = false;
|
|
327
|
+
let runningMarked = false;
|
|
328
|
+
|
|
329
|
+
const ensureStartingState = (pid = null, at = new Date().toISOString()) => {
|
|
330
|
+
if (startupStarted) return;
|
|
331
|
+
startupStarted = true;
|
|
332
|
+
transitionActiveTurnLifecycle(projectRoot, turn.turn_id, 'starting', { pid, at });
|
|
333
|
+
tracker.start();
|
|
334
|
+
if (pid != null) {
|
|
335
|
+
tracker.setPid(pid);
|
|
336
|
+
}
|
|
337
|
+
emitRunEvent(projectRoot, 'dispatch_progress', {
|
|
338
|
+
run_id: state.run_id,
|
|
339
|
+
phase: state.phase,
|
|
340
|
+
status: state.status,
|
|
341
|
+
turn: { turn_id: turn.turn_id, assigned_role: roleId },
|
|
342
|
+
payload: { milestone: 'started', output_lines: 0, elapsed_seconds: 0, silent_seconds: 0 },
|
|
343
|
+
});
|
|
344
|
+
};
|
|
345
|
+
|
|
346
|
+
const ensureRunningState = (stream = 'stdout', at = new Date().toISOString()) => {
|
|
347
|
+
if (runningMarked) return;
|
|
348
|
+
runningMarked = true;
|
|
349
|
+
transitionActiveTurnLifecycle(projectRoot, turn.turn_id, 'running', { stream, at });
|
|
350
|
+
};
|
|
322
351
|
|
|
323
352
|
const adapterOpts = {
|
|
324
353
|
signal: combineAbortSignals(controller.signal, ctx.dispatchAbortSignal),
|
|
325
354
|
onStatus: (msg) => log(chalk.dim(` ${msg}`)),
|
|
326
355
|
verifyManifest: true,
|
|
327
356
|
turnId: turn.turn_id,
|
|
357
|
+
onSpawnAttached: ({ pid, at }) => ensureStartingState(pid, at),
|
|
358
|
+
onFirstOutput: ({ at, stream }) => ensureRunningState(stream, at),
|
|
328
359
|
};
|
|
329
360
|
|
|
330
361
|
const recordOutputActivity = (stream, text) => {
|
|
362
|
+
ensureRunningState(stream);
|
|
331
363
|
const lines = text.split('\n').length - 1 || 1;
|
|
332
364
|
const wasSilent = tracker.onOutput(stream, lines);
|
|
333
365
|
if (wasSilent) {
|
|
@@ -368,23 +400,17 @@ export async function executeGovernedRun(context, opts = {}) {
|
|
|
368
400
|
|
|
369
401
|
let adapterResult;
|
|
370
402
|
|
|
371
|
-
// Emit dispatch_progress started event and begin tracking
|
|
372
|
-
tracker.start();
|
|
373
|
-
emitRunEvent(projectRoot, 'dispatch_progress', {
|
|
374
|
-
run_id: state.run_id,
|
|
375
|
-
phase: state.phase,
|
|
376
|
-
status: state.status,
|
|
377
|
-
turn: { turn_id: turn.turn_id, assigned_role: roleId },
|
|
378
|
-
payload: { milestone: 'started', output_lines: 0, elapsed_seconds: 0, silent_seconds: 0 },
|
|
379
|
-
});
|
|
380
|
-
|
|
381
403
|
try {
|
|
382
404
|
if (runtimeType === 'api_proxy') {
|
|
405
|
+
ensureStartingState(null);
|
|
406
|
+
ensureRunningState('request');
|
|
383
407
|
log(chalk.dim(` Dispatching to API proxy: ${runtime?.provider || '?'} / ${runtime?.model || '?'}`));
|
|
384
408
|
tracker.requestStarted();
|
|
385
409
|
adapterResult = await dispatchApiProxy(projectRoot, state, cfg, adapterOpts);
|
|
386
410
|
if (adapterResult.ok) tracker.responseReceived();
|
|
387
411
|
} else if (runtimeType === 'mcp') {
|
|
412
|
+
ensureStartingState(null);
|
|
413
|
+
ensureRunningState('request');
|
|
388
414
|
const transport = resolveMcpTransport(runtime);
|
|
389
415
|
log(chalk.dim(` Dispatching to MCP ${transport}: ${describeMcpRuntimeTarget(runtime)}`));
|
|
390
416
|
tracker.requestStarted();
|
|
@@ -395,6 +421,8 @@ export async function executeGovernedRun(context, opts = {}) {
|
|
|
395
421
|
log(chalk.dim(` Dispatching to local CLI: ${runtime?.command || '(default)'} transport: ${transport}`));
|
|
396
422
|
adapterResult = await dispatchLocalCli(projectRoot, state, cfg, adapterOpts);
|
|
397
423
|
} else if (runtimeType === 'remote_agent') {
|
|
424
|
+
ensureStartingState(null);
|
|
425
|
+
ensureRunningState('request');
|
|
398
426
|
log(chalk.dim(` Dispatching to remote agent: ${describeRemoteAgentTarget(runtime)}`));
|
|
399
427
|
tracker.requestStarted();
|
|
400
428
|
adapterResult = await dispatchRemoteAgent(projectRoot, state, cfg, adapterOpts);
|
|
@@ -413,6 +441,10 @@ export async function executeGovernedRun(context, opts = {}) {
|
|
|
413
441
|
throw err;
|
|
414
442
|
}
|
|
415
443
|
|
|
444
|
+
if (adapterResult.ok && runtimeType === 'local_cli' && !runningMarked) {
|
|
445
|
+
ensureRunningState('staged_result', adapterResult.firstOutputAt || new Date().toISOString());
|
|
446
|
+
}
|
|
447
|
+
|
|
416
448
|
// Emit completion/failure progress event and clean up tracker
|
|
417
449
|
const progressState = tracker.getState();
|
|
418
450
|
const elapsedSec = Math.round((Date.now() - new Date(progressState.started_at)) / 1000);
|
|
@@ -439,6 +471,19 @@ export async function executeGovernedRun(context, opts = {}) {
|
|
|
439
471
|
return { accept: false, reason: 'dispatch timed out' };
|
|
440
472
|
}
|
|
441
473
|
|
|
474
|
+
if (adapterResult.startupFailure) {
|
|
475
|
+
const freshState = loadProjectState(projectRoot, cfg) || state;
|
|
476
|
+
failTurnStartup(projectRoot, freshState, cfg, turn.turn_id, {
|
|
477
|
+
failure_type: adapterResult.startupFailureType || 'no_subprocess_output',
|
|
478
|
+
threshold_ms: cfg?.run_loop?.startup_watchdog_ms ?? 30_000,
|
|
479
|
+
running_ms: freshState?.active_turns?.[turn.turn_id]?.started_at
|
|
480
|
+
? Math.max(0, Date.now() - new Date(freshState.active_turns[turn.turn_id].started_at).getTime())
|
|
481
|
+
: 0,
|
|
482
|
+
recommendation: `Turn ${turn.turn_id} failed to start within the startup watchdog window. Run \`agentxchain reissue-turn --turn ${turn.turn_id} --reason ghost\` to recover.`,
|
|
483
|
+
});
|
|
484
|
+
return { accept: false, blocked: true, reason: adapterResult.error || 'turn startup failed' };
|
|
485
|
+
}
|
|
486
|
+
|
|
442
487
|
// Adapter failure
|
|
443
488
|
if (!adapterResult.ok) {
|
|
444
489
|
if (shouldSuggestManualQaFallback({
|
|
@@ -472,6 +517,18 @@ export async function executeGovernedRun(context, opts = {}) {
|
|
|
472
517
|
return { accept: false, reason: `failed to parse staged result: ${err.message}` };
|
|
473
518
|
}
|
|
474
519
|
|
|
520
|
+
// Per DEC-MINIMUM-TURN-RESULT-SHAPE-001: the staged-result read shortcut
|
|
521
|
+
// must refuse payloads that lack the minimum governed envelope. Adapter
|
|
522
|
+
// pre-stage guards already reject these, but this is the final boundary
|
|
523
|
+
// before acceptance projection — fail closed on tampered or legacy
|
|
524
|
+
// adapter output rather than trust upstream.
|
|
525
|
+
if (!hasMinimumTurnResultShape(turnResult)) {
|
|
526
|
+
return {
|
|
527
|
+
accept: false,
|
|
528
|
+
reason: 'staged result missing minimum governed envelope (schema_version + identity + lifecycle fields)',
|
|
529
|
+
};
|
|
530
|
+
}
|
|
531
|
+
|
|
475
532
|
return { accept: true, turnResult };
|
|
476
533
|
},
|
|
477
534
|
|
package/src/commands/schedule.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import chalk from 'chalk';
|
|
2
|
-
import { loadProjectContext } from '../lib/config.js';
|
|
2
|
+
import { loadProjectContext, loadProjectState } from '../lib/config.js';
|
|
3
3
|
import {
|
|
4
4
|
SCHEDULE_STATE_PATH,
|
|
5
5
|
DAEMON_STATE_PATH,
|
|
@@ -97,19 +97,37 @@ function buildScheduleProvenance(entry) {
|
|
|
97
97
|
};
|
|
98
98
|
}
|
|
99
99
|
|
|
100
|
-
function buildScheduleExecutionResult(entryId, execution, fallbackState, action = 'ran') {
|
|
101
|
-
const state = execution.result?.state ||
|
|
100
|
+
export function buildScheduleExecutionResult(entryId, execution, fallbackState, action = 'ran') {
|
|
101
|
+
const state = fallbackState || execution.result?.state || null;
|
|
102
|
+
const blockedReason = state?.blocked_reason || null;
|
|
103
|
+
const recoveryAction = blockedReason?.recovery?.recovery_action || null;
|
|
104
|
+
const blockedCategory = blockedReason?.category || null;
|
|
102
105
|
return {
|
|
103
106
|
id: entryId,
|
|
104
107
|
action,
|
|
105
108
|
run_id: state?.run_id || null,
|
|
106
109
|
stop_reason: execution.result?.stop_reason || null,
|
|
107
110
|
exit_code: execution.exitCode,
|
|
111
|
+
recovery_action: recoveryAction,
|
|
112
|
+
blocked_category: blockedCategory,
|
|
108
113
|
};
|
|
109
114
|
}
|
|
110
115
|
|
|
116
|
+
function resolveScheduleExecutionState(root, config, execution, fallbackState) {
|
|
117
|
+
const executionState = execution.result?.state || null;
|
|
118
|
+
const liveState = loadProjectState(root, config);
|
|
119
|
+
|
|
120
|
+
if (execution.result?.stop_reason === 'blocked' || execution.result?.stop_reason === 'reject_exhausted') {
|
|
121
|
+
if (liveState?.status === 'blocked' && liveState?.blocked_reason) {
|
|
122
|
+
return liveState;
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
return executionState || liveState || fallbackState || null;
|
|
127
|
+
}
|
|
128
|
+
|
|
111
129
|
function recordScheduleExecution(context, entryId, execution, fallbackState, nowIso, action = 'ran') {
|
|
112
|
-
const state =
|
|
130
|
+
const state = resolveScheduleExecutionState(context.root, context.config, execution, fallbackState);
|
|
113
131
|
const runId = state?.run_id || null;
|
|
114
132
|
const startedAt = state?.created_at || nowIso;
|
|
115
133
|
|
|
@@ -197,7 +215,7 @@ async function continueActiveScheduledRun(context, opts = {}) {
|
|
|
197
215
|
}
|
|
198
216
|
|
|
199
217
|
const blocked = execution.result?.stop_reason === 'blocked';
|
|
200
|
-
const action = blocked
|
|
218
|
+
const action = blocked ? 'blocked' : 'continued';
|
|
201
219
|
const result = recordScheduleExecution(context, scheduleId, execution, state, opts.at || new Date().toISOString(), action);
|
|
202
220
|
|
|
203
221
|
if (execution.exitCode !== 0 && !(opts.tolerateBlockedRun && blocked)) {
|
|
@@ -312,7 +330,7 @@ async function runDueSchedules(context, opts = {}) {
|
|
|
312
330
|
execution,
|
|
313
331
|
execution.result?.state || null,
|
|
314
332
|
nowIso,
|
|
315
|
-
blocked
|
|
333
|
+
blocked ? 'blocked' : 'ran',
|
|
316
334
|
));
|
|
317
335
|
|
|
318
336
|
if (execution.exitCode !== 0) {
|
|
@@ -489,6 +507,8 @@ async function advanceScheduleContinuousSession(context, entry, opts = {}) {
|
|
|
489
507
|
run_id: step.run_id || null,
|
|
490
508
|
intent_id: step.intent_id || null,
|
|
491
509
|
runs_completed: session.runs_completed,
|
|
510
|
+
recovery_action: step.recovery_action || null,
|
|
511
|
+
blocked_category: step.blocked_category || null,
|
|
492
512
|
};
|
|
493
513
|
}
|
|
494
514
|
|
|
@@ -536,7 +556,12 @@ export async function scheduleRunDueCommand(opts) {
|
|
|
536
556
|
} else if (entry.action === 'preemption_failed') {
|
|
537
557
|
console.log(chalk.red(`Schedule preemption failed: ${entry.id} (${entry.error || 'unknown error'})`));
|
|
538
558
|
} else if (entry.action === 'blocked') {
|
|
539
|
-
|
|
559
|
+
if (entry.recovery_action) {
|
|
560
|
+
const categorySuffix = entry.blocked_category ? ` (${entry.blocked_category})` : '';
|
|
561
|
+
console.log(chalk.yellow(`Schedule blocked: ${entry.id}${categorySuffix}. Recovery: ${entry.recovery_action}`));
|
|
562
|
+
} else {
|
|
563
|
+
console.log(chalk.yellow(`Schedule waiting on unblock: ${entry.id}`));
|
|
564
|
+
}
|
|
540
565
|
} else if (entry.action === 'skipped') {
|
|
541
566
|
console.log(chalk.yellow(`Schedule skipped: ${entry.id} (${entry.reason})`));
|
|
542
567
|
} else if (entry.action === 'not_due') {
|
|
@@ -709,6 +734,8 @@ export async function scheduleDaemonCommand(opts) {
|
|
|
709
734
|
runs_completed: contResult.runs_completed ?? null,
|
|
710
735
|
};
|
|
711
736
|
if (contResult.reason) contResultEntry.reason = contResult.reason;
|
|
737
|
+
if (contResult.recovery_action) contResultEntry.recovery_action = contResult.recovery_action;
|
|
738
|
+
if (contResult.blocked_category) contResultEntry.blocked_category = contResult.blocked_category;
|
|
712
739
|
|
|
713
740
|
result = {
|
|
714
741
|
ok: contResult.ok !== false && nonContResult.ok,
|
package/src/commands/status.js
CHANGED
|
@@ -136,6 +136,10 @@ function loadStatusContext(dir = process.cwd()) {
|
|
|
136
136
|
function renderGovernedStatus(context, opts) {
|
|
137
137
|
const { root, config, version } = context;
|
|
138
138
|
let state = loadProjectState(root, config);
|
|
139
|
+
const staleReconciliation = reconcileStaleTurns(root, state, config);
|
|
140
|
+
state = staleReconciliation.state || state;
|
|
141
|
+
const staleTurns = staleReconciliation.stale_turns;
|
|
142
|
+
const ghostTurns = staleReconciliation.ghost_turns || [];
|
|
139
143
|
const stateRunId = state?.run_id || readRawStateRunId(root, config);
|
|
140
144
|
const continuity = getContinuityStatus(root, state);
|
|
141
145
|
const connectorHealth = getConnectorHealth(root, config, state);
|
|
@@ -166,11 +170,6 @@ function renderGovernedStatus(context, opts) {
|
|
|
166
170
|
// Coordinator warning surfacing — DEC-COORD-RETRY-PROJECTION-EVENT-001
|
|
167
171
|
const coordinatorWarnings = readCoordinatorWarnings(root, { runId: stateRunId || null });
|
|
168
172
|
|
|
169
|
-
// BUG-47: detect stale running turns and emit turn_stalled events
|
|
170
|
-
const staleReconciliation = reconcileStaleTurns(root, state, config);
|
|
171
|
-
state = staleReconciliation.state || state;
|
|
172
|
-
const staleTurns = staleReconciliation.stale_turns;
|
|
173
|
-
|
|
174
173
|
if (opts.json) {
|
|
175
174
|
const dashPid = getDashboardPid(root);
|
|
176
175
|
const dashSession = getDashboardSession(root);
|
|
@@ -209,6 +208,7 @@ function renderGovernedStatus(context, opts) {
|
|
|
209
208
|
bundle_integrity: detectStateBundleDesync(root, state),
|
|
210
209
|
coordinator_warnings: coordinatorWarnings,
|
|
211
210
|
stale_turns: staleTurns,
|
|
211
|
+
ghost_turns: ghostTurns,
|
|
212
212
|
}, null, 2));
|
|
213
213
|
return;
|
|
214
214
|
}
|
|
@@ -382,6 +382,16 @@ function renderGovernedStatus(context, opts) {
|
|
|
382
382
|
console.log(` ${chalk.dim('Recover:')} ${chalk.cyan(`agentxchain reject-turn --turn ${turn.turn_id}`)} — reject and retry`);
|
|
383
383
|
console.log(` ${chalk.dim(' or:')} ${chalk.cyan(`agentxchain accept-turn --turn ${turn.turn_id}`)} — re-attempt acceptance`);
|
|
384
384
|
}
|
|
385
|
+
if (turn.status === 'failed_start') {
|
|
386
|
+
console.log(` ${chalk.dim('Reason:')} ${turn.failed_start_reason || 'no_subprocess_output'}`);
|
|
387
|
+
const recover = turn.recovery_command || `agentxchain reissue-turn --turn ${turn.turn_id} --reason ghost`;
|
|
388
|
+
console.log(` ${chalk.dim('Recover:')} ${chalk.cyan(recover)}`);
|
|
389
|
+
}
|
|
390
|
+
if (turn.status === 'stalled') {
|
|
391
|
+
console.log(` ${chalk.dim('Reason:')} ${turn.stalled_reason || 'no_output_within_threshold'}`);
|
|
392
|
+
const recover = turn.recovery_command || `agentxchain reissue-turn --turn ${turn.turn_id} --reason stale`;
|
|
393
|
+
console.log(` ${chalk.dim('Recover:')} ${chalk.cyan(recover)}`);
|
|
394
|
+
}
|
|
385
395
|
}
|
|
386
396
|
} else if (singleActiveTurn) {
|
|
387
397
|
console.log(` ${chalk.dim('Turn:')} ${singleActiveTurn.turn_id}`);
|
|
@@ -432,6 +442,16 @@ function renderGovernedStatus(context, opts) {
|
|
|
432
442
|
console.log(` ${chalk.dim('Resolve:')} ${chalk.cyan(reassignAction.command)}`);
|
|
433
443
|
console.log(` ${chalk.dim(' or:')} ${chalk.cyan(mergeAction.command)}`);
|
|
434
444
|
}
|
|
445
|
+
if (singleActiveTurn.status === 'failed_start') {
|
|
446
|
+
console.log(` ${chalk.dim('Reason:')} ${singleActiveTurn.failed_start_reason || 'no_subprocess_output'}`);
|
|
447
|
+
const recover = singleActiveTurn.recovery_command || `agentxchain reissue-turn --turn ${singleActiveTurn.turn_id} --reason ghost`;
|
|
448
|
+
console.log(` ${chalk.dim('Recover:')} ${chalk.cyan(recover)}`);
|
|
449
|
+
}
|
|
450
|
+
if (singleActiveTurn.status === 'stalled') {
|
|
451
|
+
console.log(` ${chalk.dim('Reason:')} ${singleActiveTurn.stalled_reason || 'no_output_within_threshold'}`);
|
|
452
|
+
const recover = singleActiveTurn.recovery_command || `agentxchain reissue-turn --turn ${singleActiveTurn.turn_id} --reason stale`;
|
|
453
|
+
console.log(` ${chalk.dim('Recover:')} ${chalk.cyan(recover)}`);
|
|
454
|
+
}
|
|
435
455
|
} else {
|
|
436
456
|
console.log(` ${chalk.dim('Turn:')} ${chalk.yellow('No active turn')}`);
|
|
437
457
|
}
|
|
@@ -453,6 +473,19 @@ function renderGovernedStatus(context, opts) {
|
|
|
453
473
|
}
|
|
454
474
|
}
|
|
455
475
|
|
|
476
|
+
// BUG-51: Ghost turn warning (subprocess never started)
|
|
477
|
+
if (ghostTurns.length > 0) {
|
|
478
|
+
console.log('');
|
|
479
|
+
for (const gt of ghostTurns) {
|
|
480
|
+
const secs = Math.floor(gt.running_ms / 1000);
|
|
481
|
+
console.log(` ${chalk.red.bold('⚠ Ghost turn detected — subprocess never started')}`);
|
|
482
|
+
console.log(` ${chalk.dim('Turn:')} ${gt.turn_id} (${gt.role})`);
|
|
483
|
+
console.log(` ${chalk.dim('Runtime:')} ${gt.runtime_id}`);
|
|
484
|
+
console.log(` ${chalk.dim('Age:')} ${secs}s with no subprocess output`);
|
|
485
|
+
console.log(` ${chalk.dim('Recover:')} ${chalk.cyan(`agentxchain reissue-turn --turn ${gt.turn_id} --reason ghost`)}`);
|
|
486
|
+
}
|
|
487
|
+
}
|
|
488
|
+
|
|
456
489
|
// BUG-47: Stale turn warning
|
|
457
490
|
if (staleTurns.length > 0) {
|
|
458
491
|
console.log('');
|