agentxchain 2.155.72 → 2.155.73
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -8
- package/package.json +3 -5
- package/scripts/migrate-node-test-to-vitest.mjs +98 -0
- package/scripts/release-postflight.sh +1 -1
- package/scripts/release-preflight.sh +5 -5
- package/scripts/verify-post-publish.sh +1 -1
- package/src/commands/run.js +3 -0
- package/src/commands/step.js +47 -1
- package/src/lib/adapters/local-cli-adapter.js +179 -2
- package/src/lib/claude-local-auth.js +14 -0
- package/src/lib/continuous-run.js +42 -3
- package/src/lib/dispatch-bundle.js +7 -3
- package/src/lib/dispatch-progress.js +9 -0
- package/src/lib/governed-state.js +4 -3
- package/src/lib/normalized-config.js +2 -0
- package/src/lib/recovery-classification.js +158 -0
- package/src/lib/report.js +91 -0
- package/src/lib/run-events.js +7 -1
- package/src/lib/schemas/agentxchain-config.schema.json +10 -0
- package/src/lib/turn-checkpoint.js +12 -3
- package/src/lib/turn-result-validator.js +42 -6
- package/src/lib/vision-reader.js +11 -1
|
@@ -47,6 +47,7 @@ import { getDispatchLogPath, getTurnStagingResultPath } from './turn-paths.js';
|
|
|
47
47
|
import { reconcileOperatorHead } from './operator-commit-reconcile.js';
|
|
48
48
|
import { getContinuityStatus } from './continuity-status.js';
|
|
49
49
|
import { resolveGovernedRole } from './role-resolution.js';
|
|
50
|
+
import { writeSessionCheckpoint } from './session-checkpoint.js';
|
|
50
51
|
import {
|
|
51
52
|
archiveStaleIntentsForRun,
|
|
52
53
|
formatLegacyIntentMigrationNotice,
|
|
@@ -63,7 +64,7 @@ import {
|
|
|
63
64
|
|
|
64
65
|
const CONTINUOUS_SESSION_PATH = '.agentxchain/continuous-session.json';
|
|
65
66
|
const PRODUCTIVE_TIMEOUT_RETRY_MAX_PER_RUN = 1;
|
|
66
|
-
const PRODUCTIVE_TIMEOUT_RETRY_DEADLINE_MINUTES =
|
|
67
|
+
const PRODUCTIVE_TIMEOUT_RETRY_DEADLINE_MINUTES = 120;
|
|
67
68
|
const PROVIDER_REQUEST_TIMEOUT_RE = /request timed out|timed out waiting for (?:provider|api)|provider request timed out/i;
|
|
68
69
|
|
|
69
70
|
function getRoadmapReplenishmentTriageHints(root) {
|
|
@@ -636,9 +637,21 @@ function clearGhostBlockerAfterReissue(root, state) {
|
|
|
636
637
|
escalation: null,
|
|
637
638
|
};
|
|
638
639
|
writeGovernedState(root, nextState);
|
|
640
|
+
writeSessionCheckpoint(root, nextState, 'blocker_cleared');
|
|
639
641
|
return nextState;
|
|
640
642
|
}
|
|
641
643
|
|
|
644
|
+
function isGovernedRunStillActiveForSession(root, config, session) {
|
|
645
|
+
try {
|
|
646
|
+
const state = loadProjectState(root, config);
|
|
647
|
+
if (!state || state.status !== 'active') return false;
|
|
648
|
+
if (session.current_run_id && state.run_id && session.current_run_id !== state.run_id) return false;
|
|
649
|
+
return true;
|
|
650
|
+
} catch {
|
|
651
|
+
return false;
|
|
652
|
+
}
|
|
653
|
+
}
|
|
654
|
+
|
|
642
655
|
/**
|
|
643
656
|
* Slice 2d (Turn 201): read the per-turn adapter dispatch log and return the
|
|
644
657
|
* most recent stderr excerpt + exit code + signal from `process_exit` or
|
|
@@ -1253,7 +1266,7 @@ function reconcileContinuousStartupState(context, session, contOpts, log) {
|
|
|
1253
1266
|
* @param {string} root
|
|
1254
1267
|
* @param {string} visionPath - Absolute path to VISION.md
|
|
1255
1268
|
* @param {{ triageApproval?: string }} options
|
|
1256
|
-
* @returns {{ ok: boolean, intentId?: string, section?: string, goal?: string, error?: string, idle?: boolean }}
|
|
1269
|
+
* @returns {{ ok: boolean, intentId?: string, section?: string, goal?: string, source?: string, error?: string, idle?: boolean }}
|
|
1257
1270
|
*/
|
|
1258
1271
|
export function seedFromVision(root, visionPath, options = {}) {
|
|
1259
1272
|
const roadmapResult = deriveRoadmapCandidates(root);
|
|
@@ -1402,6 +1415,15 @@ export function seedFromVision(root, visionPath, options = {}) {
|
|
|
1402
1415
|
};
|
|
1403
1416
|
}
|
|
1404
1417
|
|
|
1418
|
+
if (exhaustion.reason === 'vision_fully_mapped' || exhaustion.reason === 'vision_no_actionable_scope') {
|
|
1419
|
+
return {
|
|
1420
|
+
ok: true,
|
|
1421
|
+
idle: true,
|
|
1422
|
+
source: 'vision_exhausted',
|
|
1423
|
+
reason: exhaustion.reason,
|
|
1424
|
+
};
|
|
1425
|
+
}
|
|
1426
|
+
|
|
1405
1427
|
const result = deriveVisionCandidates(root, visionPath);
|
|
1406
1428
|
if (!result.ok) {
|
|
1407
1429
|
return { ok: false, error: result.error };
|
|
@@ -2289,7 +2311,7 @@ export async function advanceContinuousRunOnce(context, session, contOpts, execu
|
|
|
2289
2311
|
if (seeded.source === 'roadmap_open_work') {
|
|
2290
2312
|
log(`Roadmap-derived: ${visionObjective}`);
|
|
2291
2313
|
} else if (seeded.source === 'roadmap_replenishment') {
|
|
2292
|
-
log(`Roadmap
|
|
2314
|
+
log(`Roadmap exhausted, vision still open, deriving next increment: ${visionObjective}`);
|
|
2293
2315
|
} else {
|
|
2294
2316
|
log(`Vision-derived: ${visionObjective}`);
|
|
2295
2317
|
}
|
|
@@ -2551,6 +2573,23 @@ export async function executeContinuousRun(context, contOpts, executeGovernedRun
|
|
|
2551
2573
|
while (!stopping) {
|
|
2552
2574
|
const step = await advanceContinuousRunOnce(context, session, contOpts, executeGovernedRun, log);
|
|
2553
2575
|
|
|
2576
|
+
if (step.status === 'failed' && isGovernedRunStillActiveForSession(root, context.config, session)) {
|
|
2577
|
+
session.status = 'running';
|
|
2578
|
+
writeContinuousSession(root, session);
|
|
2579
|
+
emitRunEvent(root, 'session_failed_recovered_active_run', {
|
|
2580
|
+
run_id: session.current_run_id || null,
|
|
2581
|
+
phase: null,
|
|
2582
|
+
status: 'active',
|
|
2583
|
+
payload: {
|
|
2584
|
+
session_id: session.session_id,
|
|
2585
|
+
failed_action: step.action || null,
|
|
2586
|
+
failed_reason: step.stop_reason || null,
|
|
2587
|
+
},
|
|
2588
|
+
});
|
|
2589
|
+
log('Session failure recovered - governed run still active, continuing.');
|
|
2590
|
+
continue;
|
|
2591
|
+
}
|
|
2592
|
+
|
|
2554
2593
|
// Terminal states
|
|
2555
2594
|
if (step.status === 'completed' || step.status === 'idle_exit' || step.status === 'failed' || step.status === 'blocked' || step.status === 'stopped' || step.status === 'vision_exhausted' || step.status === 'vision_expansion_exhausted' || step.status === 'session_budget') {
|
|
2556
2595
|
const terminalMessage = describeContinuousTerminalStep(step, contOpts);
|
|
@@ -797,6 +797,9 @@ function renderContext(state, config, root, turn, role) {
|
|
|
797
797
|
lines.push('');
|
|
798
798
|
lines.push(`- **Turn:** ${lastTurn.turn_id}`);
|
|
799
799
|
lines.push(`- **Role:** ${lastTurn.role}`);
|
|
800
|
+
if (lastTurn.runtime_id) {
|
|
801
|
+
lines.push(`- **Runtime:** ${lastTurn.runtime_id}`);
|
|
802
|
+
}
|
|
800
803
|
lines.push(`- **Summary:** ${lastTurn.summary}`);
|
|
801
804
|
if (lastTurn.decisions?.length) {
|
|
802
805
|
lines.push('- **Decisions:**');
|
|
@@ -1412,12 +1415,13 @@ function renderDecisionHistory(root, warnings = []) {
|
|
|
1412
1415
|
const lines = [];
|
|
1413
1416
|
lines.push('## Decision History');
|
|
1414
1417
|
lines.push('');
|
|
1415
|
-
lines.push('| ID | Phase | Role | Statement |');
|
|
1416
|
-
lines.push('
|
|
1418
|
+
lines.push('| ID | Phase | Role | Runtime | Statement |');
|
|
1419
|
+
lines.push('|----|-------|------|---------|-----------|');
|
|
1417
1420
|
for (const d of displayed) {
|
|
1418
1421
|
// Escape pipes in statement to avoid breaking the table
|
|
1419
1422
|
const stmt = (d.statement || '').replace(/\|/g, '\\|').replace(/\n/g, ' ');
|
|
1420
|
-
|
|
1423
|
+
const runtimeId = (d.runtime_id || '').replace(/\|/g, '\\|').replace(/\n/g, ' ');
|
|
1424
|
+
lines.push(`| ${d.id} | ${d.phase || ''} | ${d.role || ''} | ${runtimeId} | ${stmt} |`);
|
|
1421
1425
|
}
|
|
1422
1426
|
if (totalCount > DECISION_HISTORY_MAX_ENTRIES) {
|
|
1423
1427
|
lines.push('');
|
|
@@ -199,6 +199,15 @@ export function createDispatchProgressTracker(root, turn, options = {}) {
|
|
|
199
199
|
writeProgress();
|
|
200
200
|
},
|
|
201
201
|
|
|
202
|
+
/** Record adapter keepalive without treating it as governed startup proof. */
|
|
203
|
+
heartbeat(summary = 'Adapter keepalive') {
|
|
204
|
+
state.activity_type = 'heartbeat';
|
|
205
|
+
state.activity_summary = summary;
|
|
206
|
+
state.last_activity_at = new Date().toISOString();
|
|
207
|
+
dirty = true;
|
|
208
|
+
maybeWrite();
|
|
209
|
+
},
|
|
210
|
+
|
|
202
211
|
/** Update PID after spawn (local_cli). */
|
|
203
212
|
setPid(newPid) {
|
|
204
213
|
state.pid = newPid;
|
|
@@ -3672,7 +3672,7 @@ export function assignGovernedTurn(root, config, roleId, options = {}) {
|
|
|
3672
3672
|
const baseline = captureBaseline(root);
|
|
3673
3673
|
|
|
3674
3674
|
const now = new Date().toISOString();
|
|
3675
|
-
const timeoutMinutes =
|
|
3675
|
+
const timeoutMinutes = config?.timeouts?.per_turn_minutes || 120;
|
|
3676
3676
|
const nextSequence = (state.turn_sequence || 0) + 1;
|
|
3677
3677
|
|
|
3678
3678
|
// Record which turns are concurrent siblings (for conflict detection context)
|
|
@@ -3954,7 +3954,7 @@ export function reissueTurn(root, config, opts = {}) {
|
|
|
3954
3954
|
|
|
3955
3955
|
// Create the new turn
|
|
3956
3956
|
const newTurnId = `turn_${randomBytes(8).toString('hex')}`;
|
|
3957
|
-
const timeoutMinutes =
|
|
3957
|
+
const timeoutMinutes = config?.timeouts?.per_turn_minutes || 120;
|
|
3958
3958
|
const nextSequence = (state.turn_sequence || 0) + 1;
|
|
3959
3959
|
|
|
3960
3960
|
const newTurn = {
|
|
@@ -5238,6 +5238,7 @@ function _acceptGovernedTurnLocked(root, config, opts) {
|
|
|
5238
5238
|
turn_id: turnResult.turn_id,
|
|
5239
5239
|
role: turnResult.role,
|
|
5240
5240
|
phase: state.phase,
|
|
5241
|
+
runtime_id: turnResult.runtime_id,
|
|
5241
5242
|
category: decision.category,
|
|
5242
5243
|
statement: decision.statement,
|
|
5243
5244
|
rationale: decision.rationale,
|
|
@@ -6488,7 +6489,7 @@ export function rejectGovernedTurn(root, config, validationResult, reasonOrOptio
|
|
|
6488
6489
|
const retryStartedAt = new Date().toISOString();
|
|
6489
6490
|
retryTurn.baseline = captureBaseline(root);
|
|
6490
6491
|
retryTurn.started_at = retryStartedAt;
|
|
6491
|
-
retryTurn.deadline_at = new Date(Date.now() +
|
|
6492
|
+
retryTurn.deadline_at = new Date(Date.now() + (config?.timeouts?.per_turn_minutes || 120) * 60 * 1000).toISOString();
|
|
6492
6493
|
|
|
6493
6494
|
if (isConflictReject) {
|
|
6494
6495
|
retryTurn.assigned_sequence = Math.max(
|
|
@@ -444,6 +444,7 @@ export function validateV4Config(data, projectRoot) {
|
|
|
444
444
|
}
|
|
445
445
|
if (rt.type === 'local_cli') {
|
|
446
446
|
validateRuntimePositiveInteger(`Runtime "${id}": startup_watchdog_ms`, rt.startup_watchdog_ms, errors);
|
|
447
|
+
validateRuntimePositiveInteger(`Runtime "${id}": startup_heartbeat_ms`, rt.startup_heartbeat_ms, errors);
|
|
447
448
|
}
|
|
448
449
|
// Validate prompt_transport for local_cli runtimes
|
|
449
450
|
if (rt.type === 'local_cli' && rt.prompt_transport) {
|
|
@@ -639,6 +640,7 @@ export function validateRunLoopConfig(runLoop) {
|
|
|
639
640
|
return errors;
|
|
640
641
|
}
|
|
641
642
|
validateRunLoopPositiveInteger('run_loop.startup_watchdog_ms', runLoop.startup_watchdog_ms, errors);
|
|
643
|
+
validateRunLoopPositiveInteger('run_loop.startup_heartbeat_ms', runLoop.startup_heartbeat_ms, errors);
|
|
642
644
|
validateRunLoopPositiveInteger('run_loop.stale_turn_threshold_ms', runLoop.stale_turn_threshold_ms, errors);
|
|
643
645
|
if (runLoop.continuous !== undefined && runLoop.continuous !== null) {
|
|
644
646
|
validateRunLoopContinuousConfig('run_loop.continuous', runLoop.continuous, errors);
|
|
@@ -0,0 +1,158 @@
|
|
|
1
|
+
const DOMAINS = ['ghost', 'budget', 'credential', 'crash'];
|
|
2
|
+
const OUTCOMES = ['recovered', 'exhausted', 'manual', 'pending'];
|
|
3
|
+
|
|
4
|
+
const EVENT_CLASSIFICATIONS = {
|
|
5
|
+
auto_retried_ghost: {
|
|
6
|
+
domain: 'ghost',
|
|
7
|
+
severity: 'medium',
|
|
8
|
+
outcome: 'recovered',
|
|
9
|
+
mechanism: 'auto_retry',
|
|
10
|
+
summary: 'Ghost turn reissued automatically',
|
|
11
|
+
},
|
|
12
|
+
ghost_retry_exhausted: {
|
|
13
|
+
domain: 'ghost',
|
|
14
|
+
severity: 'high',
|
|
15
|
+
outcome: 'exhausted',
|
|
16
|
+
mechanism: 'auto_retry',
|
|
17
|
+
summary: 'Ghost retry budget exhausted',
|
|
18
|
+
},
|
|
19
|
+
auto_retried_productive_timeout: {
|
|
20
|
+
domain: 'ghost',
|
|
21
|
+
severity: 'medium',
|
|
22
|
+
outcome: 'recovered',
|
|
23
|
+
mechanism: 'auto_retry',
|
|
24
|
+
summary: 'Productive timeout reissued automatically',
|
|
25
|
+
},
|
|
26
|
+
productive_timeout_retry_exhausted: {
|
|
27
|
+
domain: 'ghost',
|
|
28
|
+
severity: 'high',
|
|
29
|
+
outcome: 'exhausted',
|
|
30
|
+
mechanism: 'auto_retry',
|
|
31
|
+
summary: 'Productive timeout retry budget exhausted',
|
|
32
|
+
},
|
|
33
|
+
budget_exceeded_warn: {
|
|
34
|
+
domain: 'budget',
|
|
35
|
+
severity: 'medium',
|
|
36
|
+
outcome: 'pending',
|
|
37
|
+
mechanism: 'config_change',
|
|
38
|
+
summary: 'Budget warning threshold exceeded',
|
|
39
|
+
},
|
|
40
|
+
retained_claude_auth_escalation_reclassified: {
|
|
41
|
+
domain: 'credential',
|
|
42
|
+
severity: 'medium',
|
|
43
|
+
outcome: 'pending',
|
|
44
|
+
mechanism: 'env_refresh',
|
|
45
|
+
summary: 'Claude credential escalation reclassified',
|
|
46
|
+
},
|
|
47
|
+
continuous_paused_active_run_recovered: {
|
|
48
|
+
domain: 'crash',
|
|
49
|
+
severity: 'medium',
|
|
50
|
+
outcome: 'recovered',
|
|
51
|
+
mechanism: 'loop_guard',
|
|
52
|
+
summary: 'Paused continuous session recovered active run',
|
|
53
|
+
},
|
|
54
|
+
session_failed_recovered_active_run: {
|
|
55
|
+
domain: 'crash',
|
|
56
|
+
severity: 'medium',
|
|
57
|
+
outcome: 'recovered',
|
|
58
|
+
mechanism: 'loop_guard',
|
|
59
|
+
summary: 'Failed continuous step recovered active run',
|
|
60
|
+
},
|
|
61
|
+
};
|
|
62
|
+
|
|
63
|
+
function getEventPayload(event) {
|
|
64
|
+
return event && typeof event.payload === 'object' && !Array.isArray(event.payload)
|
|
65
|
+
? event.payload
|
|
66
|
+
: {};
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
function escalateSeverity(eventType, payload, severity) {
|
|
70
|
+
if (eventType === 'ghost_retry_exhausted' && payload.exhaustion_reason === 'same_signature_repeat') {
|
|
71
|
+
return 'critical';
|
|
72
|
+
}
|
|
73
|
+
if (eventType === 'budget_exceeded_warn' && typeof payload.remaining_usd === 'number' && payload.remaining_usd <= 0) {
|
|
74
|
+
return 'high';
|
|
75
|
+
}
|
|
76
|
+
return severity;
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
function emptyOutcomeCounts() {
|
|
80
|
+
return Object.fromEntries(OUTCOMES.map((outcome) => [outcome, 0]));
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
function emptyDomainCounts() {
|
|
84
|
+
return Object.fromEntries(DOMAINS.map((domain) => [domain, { total: 0, ...emptyOutcomeCounts() }]));
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
function formatSummary(eventType, payload, fallback) {
|
|
88
|
+
if (payload.recovery_class && typeof payload.recovery_class === 'string') return payload.recovery_class;
|
|
89
|
+
if (payload.warning && typeof payload.warning === 'string') return payload.warning;
|
|
90
|
+
if (payload.recovery_action && typeof payload.recovery_action === 'string') return payload.recovery_action;
|
|
91
|
+
return fallback || eventType;
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
export function classifyRecoveryEvent(event) {
|
|
95
|
+
if (!event || typeof event !== 'object' || Array.isArray(event)) return null;
|
|
96
|
+
const eventType = event.event_type || event.type;
|
|
97
|
+
const base = EVENT_CLASSIFICATIONS[eventType];
|
|
98
|
+
if (!base) return null;
|
|
99
|
+
|
|
100
|
+
const payload = getEventPayload(event);
|
|
101
|
+
return {
|
|
102
|
+
domain: base.domain,
|
|
103
|
+
severity: escalateSeverity(eventType, payload, base.severity),
|
|
104
|
+
outcome: base.outcome,
|
|
105
|
+
mechanism: base.mechanism,
|
|
106
|
+
};
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
export function buildRecoveryClassificationReport(events) {
|
|
110
|
+
const byDomain = emptyDomainCounts();
|
|
111
|
+
const byOutcome = emptyOutcomeCounts();
|
|
112
|
+
const timeline = [];
|
|
113
|
+
|
|
114
|
+
for (const event of Array.isArray(events) ? events : []) {
|
|
115
|
+
const classification = classifyRecoveryEvent(event);
|
|
116
|
+
if (!classification) continue;
|
|
117
|
+
|
|
118
|
+
byDomain[classification.domain].total += 1;
|
|
119
|
+
byDomain[classification.domain][classification.outcome] += 1;
|
|
120
|
+
byOutcome[classification.outcome] += 1;
|
|
121
|
+
|
|
122
|
+
const eventType = event.event_type || event.type;
|
|
123
|
+
const payload = getEventPayload(event);
|
|
124
|
+
timeline.push({
|
|
125
|
+
event_id: event.event_id || null,
|
|
126
|
+
timestamp: event.timestamp || null,
|
|
127
|
+
event_type: eventType,
|
|
128
|
+
domain: classification.domain,
|
|
129
|
+
severity: classification.severity,
|
|
130
|
+
outcome: classification.outcome,
|
|
131
|
+
mechanism: classification.mechanism,
|
|
132
|
+
summary: formatSummary(eventType, payload, EVENT_CLASSIFICATIONS[eventType]?.summary),
|
|
133
|
+
});
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
timeline.sort((left, right) => {
|
|
137
|
+
const leftTime = Date.parse(left.timestamp || '');
|
|
138
|
+
const rightTime = Date.parse(right.timestamp || '');
|
|
139
|
+
const normalizedLeft = Number.isNaN(leftTime) ? Number.POSITIVE_INFINITY : leftTime;
|
|
140
|
+
const normalizedRight = Number.isNaN(rightTime) ? Number.POSITIVE_INFINITY : rightTime;
|
|
141
|
+
if (normalizedLeft !== normalizedRight) return normalizedLeft - normalizedRight;
|
|
142
|
+
return String(left.event_id || '').localeCompare(String(right.event_id || ''), 'en');
|
|
143
|
+
});
|
|
144
|
+
|
|
145
|
+
const totalRecoveryEvents = timeline.length;
|
|
146
|
+
const hasCritical = timeline.some((entry) => entry.severity === 'critical');
|
|
147
|
+
const healthScore = hasCritical || byOutcome.exhausted > byOutcome.recovered
|
|
148
|
+
? 'critical'
|
|
149
|
+
: (byOutcome.exhausted > 0 || byOutcome.manual > 0 ? 'degraded' : 'healthy');
|
|
150
|
+
|
|
151
|
+
return {
|
|
152
|
+
total_recovery_events: totalRecoveryEvents,
|
|
153
|
+
by_domain: byDomain,
|
|
154
|
+
by_outcome: byOutcome,
|
|
155
|
+
timeline,
|
|
156
|
+
health_score: healthScore,
|
|
157
|
+
};
|
|
158
|
+
}
|
package/src/lib/report.js
CHANGED
|
@@ -12,6 +12,7 @@ import {
|
|
|
12
12
|
import { buildCoordinatorRepoStatusEntries } from './coordinator-repo-status-presentation.js';
|
|
13
13
|
import { summarizeCoordinatorEvent } from './coordinator-event-narrative.js';
|
|
14
14
|
import { extractGateActionDigest } from './gate-actions.js';
|
|
15
|
+
import { buildRecoveryClassificationReport } from './recovery-classification.js';
|
|
15
16
|
|
|
16
17
|
export const GOVERNANCE_REPORT_VERSION = '0.1';
|
|
17
18
|
|
|
@@ -703,6 +704,19 @@ function extractRecoverySummary(artifact) {
|
|
|
703
704
|
};
|
|
704
705
|
}
|
|
705
706
|
|
|
707
|
+
function extractRecoveryClassification(artifact) {
|
|
708
|
+
const report = buildRecoveryClassificationReport(extractRunEventTimeline(artifact));
|
|
709
|
+
return report.total_recovery_events > 0 ? report : null;
|
|
710
|
+
}
|
|
711
|
+
|
|
712
|
+
function formatRecoveryOutcomeCounts(counts) {
|
|
713
|
+
return `${counts.recovered} recovered, ${counts.exhausted} exhausted, ${counts.manual} manual, ${counts.pending} pending`;
|
|
714
|
+
}
|
|
715
|
+
|
|
716
|
+
function formatRecoveryDomainLabel(domain) {
|
|
717
|
+
return domain.charAt(0).toUpperCase() + domain.slice(1);
|
|
718
|
+
}
|
|
719
|
+
|
|
706
720
|
function extractCoordinatorTimeline(artifact) {
|
|
707
721
|
const data = extractFileData(artifact, '.agentxchain/multirepo/history.jsonl');
|
|
708
722
|
if (!Array.isArray(data) || data.length === 0) return [];
|
|
@@ -1008,6 +1022,7 @@ function buildRunSubject(artifact) {
|
|
|
1008
1022
|
const gateSummary = extractGateSummary(artifact);
|
|
1009
1023
|
const intakeLinks = extractIntakeLinks(artifact);
|
|
1010
1024
|
const recoverySummary = extractRecoverySummary(artifact);
|
|
1025
|
+
const recoveryClassification = extractRecoveryClassification(artifact);
|
|
1011
1026
|
const nextActions = deriveGovernedRunNextActions(artifact.state, artifact.config);
|
|
1012
1027
|
const continuity = extractContinuityMetadata(artifact);
|
|
1013
1028
|
const governanceEvents = extractGovernanceEventDigest(artifact);
|
|
@@ -1057,6 +1072,7 @@ function buildRunSubject(artifact) {
|
|
|
1057
1072
|
gate_summary: gateSummary,
|
|
1058
1073
|
intake_links: intakeLinks,
|
|
1059
1074
|
recovery_summary: recoverySummary,
|
|
1075
|
+
recovery_classification: recoveryClassification,
|
|
1060
1076
|
next_actions: nextActions,
|
|
1061
1077
|
continuity,
|
|
1062
1078
|
workflow_kit_artifacts: extractWorkflowKitArtifacts(artifact),
|
|
@@ -1539,6 +1555,26 @@ export function formatGovernanceReportText(report) {
|
|
|
1539
1555
|
}
|
|
1540
1556
|
}
|
|
1541
1557
|
|
|
1558
|
+
if (run.recovery_classification) {
|
|
1559
|
+
const rc = run.recovery_classification;
|
|
1560
|
+
lines.push('', 'Recovery Classification:');
|
|
1561
|
+
lines.push(` Health: ${rc.health_score}`);
|
|
1562
|
+
lines.push(` Events: ${rc.total_recovery_events} total (${formatRecoveryOutcomeCounts(rc.by_outcome)})`);
|
|
1563
|
+
lines.push(' By Domain:');
|
|
1564
|
+
for (const [domain, counts] of Object.entries(rc.by_domain)) {
|
|
1565
|
+
lines.push(` ${formatRecoveryDomainLabel(domain)}: ${counts.total} (${formatRecoveryOutcomeCounts(counts)})`);
|
|
1566
|
+
}
|
|
1567
|
+
if (rc.timeline.length > 0) {
|
|
1568
|
+
const { items: boundedRecoveryEvents, omitted: recoveryEventsOmitted } = boundedSlice(rc.timeline);
|
|
1569
|
+
lines.push(' Timeline:');
|
|
1570
|
+
for (let i = 0; i < boundedRecoveryEvents.length; i++) {
|
|
1571
|
+
const evt = boundedRecoveryEvents[i];
|
|
1572
|
+
lines.push(` ${i + 1}. ${evt.timestamp || 'n/a'} | ${evt.domain} | ${evt.severity} | ${evt.outcome} | ${evt.mechanism} | ${evt.summary}`);
|
|
1573
|
+
}
|
|
1574
|
+
if (recoveryEventsOmitted > 0) lines.push(` (${recoveryEventsOmitted} more recovery events omitted)`);
|
|
1575
|
+
}
|
|
1576
|
+
}
|
|
1577
|
+
|
|
1542
1578
|
if (run.next_actions && run.next_actions.length > 0) {
|
|
1543
1579
|
lines.push('', 'Next Actions:');
|
|
1544
1580
|
for (let i = 0; i < run.next_actions.length; i++) {
|
|
@@ -2137,6 +2173,27 @@ export function formatGovernanceReportMarkdown(report) {
|
|
|
2137
2173
|
}
|
|
2138
2174
|
}
|
|
2139
2175
|
|
|
2176
|
+
if (run.recovery_classification) {
|
|
2177
|
+
const rc = run.recovery_classification;
|
|
2178
|
+
lines.push('', '## Recovery Classification', '');
|
|
2179
|
+
lines.push(`- Health: \`${rc.health_score}\``);
|
|
2180
|
+
lines.push(`- Events: ${rc.total_recovery_events} total (${formatRecoveryOutcomeCounts(rc.by_outcome)})`);
|
|
2181
|
+
lines.push('', '| Domain | Total | Recovered | Exhausted | Manual | Pending |', '|--------|-------|-----------|-----------|--------|---------|');
|
|
2182
|
+
for (const [domain, counts] of Object.entries(rc.by_domain)) {
|
|
2183
|
+
lines.push(`| ${formatRecoveryDomainLabel(domain)} | ${counts.total} | ${counts.recovered} | ${counts.exhausted} | ${counts.manual} | ${counts.pending} |`);
|
|
2184
|
+
}
|
|
2185
|
+
if (rc.timeline.length > 0) {
|
|
2186
|
+
const { items: boundedRecoveryEvents, omitted: recoveryEventsOmitted } = boundedSlice(rc.timeline);
|
|
2187
|
+
lines.push('', '| # | Time | Domain | Severity | Outcome | Mechanism | Summary |', '|---|------|--------|----------|---------|-----------|---------|');
|
|
2188
|
+
for (let i = 0; i < boundedRecoveryEvents.length; i++) {
|
|
2189
|
+
const evt = boundedRecoveryEvents[i];
|
|
2190
|
+
const summary = (evt.summary || '').replace(/\|/g, '\\|');
|
|
2191
|
+
lines.push(`| ${i + 1} | \`${evt.timestamp || 'n/a'}\` | \`${evt.domain}\` | \`${evt.severity}\` | \`${evt.outcome}\` | \`${evt.mechanism}\` | ${summary} |`);
|
|
2192
|
+
}
|
|
2193
|
+
if (recoveryEventsOmitted > 0) lines.push('', `*(${recoveryEventsOmitted} more recovery events omitted)*`);
|
|
2194
|
+
}
|
|
2195
|
+
}
|
|
2196
|
+
|
|
2140
2197
|
if (run.next_actions && run.next_actions.length > 0) {
|
|
2141
2198
|
lines.push('', '## Next Actions', '');
|
|
2142
2199
|
for (let i = 0; i < run.next_actions.length; i++) {
|
|
@@ -2877,6 +2934,40 @@ function renderRunHtml(report) {
|
|
|
2877
2934
|
sections.push(`<div class="section">${htmlSection('Recovery', recoveryHtml)}</div>`);
|
|
2878
2935
|
}
|
|
2879
2936
|
|
|
2937
|
+
if (run.recovery_classification) {
|
|
2938
|
+
const rc = run.recovery_classification;
|
|
2939
|
+
const domainRows = Object.entries(rc.by_domain).map(([domain, counts]) => [
|
|
2940
|
+
esc(formatRecoveryDomainLabel(domain)),
|
|
2941
|
+
String(counts.total),
|
|
2942
|
+
String(counts.recovered),
|
|
2943
|
+
String(counts.exhausted),
|
|
2944
|
+
String(counts.manual),
|
|
2945
|
+
String(counts.pending),
|
|
2946
|
+
]);
|
|
2947
|
+
const { items: boundedRecoveryEvents, omitted: recoveryEventsOmitted } = boundedSlice(rc.timeline);
|
|
2948
|
+
const timelineRows = boundedRecoveryEvents.map((evt, index) => [
|
|
2949
|
+
String(index + 1),
|
|
2950
|
+
`<code>${esc(evt.timestamp || 'n/a')}</code>`,
|
|
2951
|
+
`<code>${esc(evt.domain)}</code>`,
|
|
2952
|
+
`<code>${esc(evt.severity)}</code>`,
|
|
2953
|
+
`<code>${esc(evt.outcome)}</code>`,
|
|
2954
|
+
`<code>${esc(evt.mechanism)}</code>`,
|
|
2955
|
+
esc(evt.summary || ''),
|
|
2956
|
+
]);
|
|
2957
|
+
let classificationHtml = htmlDl([
|
|
2958
|
+
['Health', `<code>${esc(rc.health_score)}</code>`],
|
|
2959
|
+
['Events', `${rc.total_recovery_events} total (${esc(formatRecoveryOutcomeCounts(rc.by_outcome))})`],
|
|
2960
|
+
]);
|
|
2961
|
+
classificationHtml += htmlSection('By Domain', htmlTable(['Domain', 'Total', 'Recovered', 'Exhausted', 'Manual', 'Pending'], domainRows), 3);
|
|
2962
|
+
if (timelineRows.length > 0) {
|
|
2963
|
+
classificationHtml += htmlSection('Timeline', htmlTable(['#', 'Time', 'Domain', 'Severity', 'Outcome', 'Mechanism', 'Summary'], timelineRows), 3);
|
|
2964
|
+
}
|
|
2965
|
+
if (recoveryEventsOmitted > 0) {
|
|
2966
|
+
classificationHtml += `<p><em>(${recoveryEventsOmitted} more recovery events omitted)</em></p>`;
|
|
2967
|
+
}
|
|
2968
|
+
sections.push(`<div class="section">${htmlSection('Recovery Classification', classificationHtml)}</div>`);
|
|
2969
|
+
}
|
|
2970
|
+
|
|
2880
2971
|
if (run.next_actions?.length > 0) {
|
|
2881
2972
|
const nextHtml = '<ol>' + run.next_actions.map((action) =>
|
|
2882
2973
|
`<li><code>${esc(action.command)}</code>: ${esc(action.reason)}</li>`
|
package/src/lib/run-events.js
CHANGED
|
@@ -8,6 +8,7 @@
|
|
|
8
8
|
import { appendFileSync, readFileSync, existsSync, mkdirSync } from 'node:fs';
|
|
9
9
|
import { join, dirname } from 'node:path';
|
|
10
10
|
import { randomBytes } from 'node:crypto';
|
|
11
|
+
import { classifyRecoveryEvent } from './recovery-classification.js';
|
|
11
12
|
|
|
12
13
|
export const RUN_EVENTS_PATH = '.agentxchain/events.jsonl';
|
|
13
14
|
|
|
@@ -72,6 +73,11 @@ export const VALID_RUN_EVENTS = [
|
|
|
72
73
|
*/
|
|
73
74
|
export function emitRunEvent(root, eventType, details = {}) {
|
|
74
75
|
const event_id = `evt_${randomBytes(8).toString('hex')}`;
|
|
76
|
+
const payload = details.payload || {};
|
|
77
|
+
const recoveryClassification = classifyRecoveryEvent({ event_type: eventType, payload });
|
|
78
|
+
const classifiedPayload = recoveryClassification && !payload.recovery_classification
|
|
79
|
+
? { ...payload, recovery_classification: recoveryClassification }
|
|
80
|
+
: payload;
|
|
75
81
|
const entry = {
|
|
76
82
|
event_id,
|
|
77
83
|
event_type: eventType,
|
|
@@ -81,7 +87,7 @@ export function emitRunEvent(root, eventType, details = {}) {
|
|
|
81
87
|
status: details.status || null,
|
|
82
88
|
turn: details.turn || null,
|
|
83
89
|
intent_id: details.intent_id || null,
|
|
84
|
-
payload:
|
|
90
|
+
payload: classifiedPayload,
|
|
85
91
|
};
|
|
86
92
|
|
|
87
93
|
try {
|
|
@@ -100,6 +100,11 @@
|
|
|
100
100
|
"minimum": 1,
|
|
101
101
|
"description": "Milliseconds to wait after dispatch for worker attach/first-output proof before retaining the turn as failed_start. Default 180000."
|
|
102
102
|
},
|
|
103
|
+
"startup_heartbeat_ms": {
|
|
104
|
+
"type": "integer",
|
|
105
|
+
"minimum": 1,
|
|
106
|
+
"description": "Milliseconds between local_cli adapter startup keepalive diagnostics while a spawned subprocess is silent before first-output proof. Default 30000."
|
|
107
|
+
},
|
|
103
108
|
"stale_turn_threshold_ms": {
|
|
104
109
|
"type": "integer",
|
|
105
110
|
"minimum": 1,
|
|
@@ -409,6 +414,11 @@
|
|
|
409
414
|
"minimum": 1,
|
|
410
415
|
"description": "Optional local_cli-specific override for the startup watchdog. When set, this runtime uses the declared threshold before falling back to run_loop.startup_watchdog_ms."
|
|
411
416
|
},
|
|
417
|
+
"startup_heartbeat_ms": {
|
|
418
|
+
"type": "integer",
|
|
419
|
+
"minimum": 1,
|
|
420
|
+
"description": "Optional local_cli-specific startup keepalive diagnostic interval. When set, this runtime uses the declared interval before falling back to run_loop.startup_heartbeat_ms."
|
|
421
|
+
},
|
|
412
422
|
"prompt_transport": {
|
|
413
423
|
"enum": ["argv", "stdin", "dispatch_bundle_only"]
|
|
414
424
|
},
|
|
@@ -201,14 +201,21 @@ function extractGitError(err) {
|
|
|
201
201
|
return stderr || stdout || err?.message || 'git command failed';
|
|
202
202
|
}
|
|
203
203
|
|
|
204
|
+
function normalizeRuntimeId(entry) {
|
|
205
|
+
return typeof entry?.runtime_id === 'string' && entry.runtime_id.trim()
|
|
206
|
+
? entry.runtime_id.trim()
|
|
207
|
+
: null;
|
|
208
|
+
}
|
|
209
|
+
|
|
204
210
|
function buildCheckpointCommit(entry) {
|
|
205
|
-
const
|
|
211
|
+
const runtimeId = normalizeRuntimeId(entry) || '(unknown)';
|
|
212
|
+
const subject = `checkpoint: ${entry.turn_id} (role=${entry.role}, phase=${entry.phase}, runtime=${runtimeId})`;
|
|
206
213
|
const bodyLines = [
|
|
207
214
|
`Summary: ${entry.summary || '(none)'}`,
|
|
208
215
|
`Turn-ID: ${entry.turn_id}`,
|
|
209
216
|
`Role: ${entry.role || '(unknown)'}`,
|
|
210
217
|
`Phase: ${entry.phase || '(unknown)'}`,
|
|
211
|
-
`Runtime: ${
|
|
218
|
+
`Runtime: ${runtimeId}`,
|
|
212
219
|
];
|
|
213
220
|
if (entry.intent_id) bodyLines.push(`Intent-ID: ${entry.intent_id}`);
|
|
214
221
|
if (entry.accepted_at) bodyLines.push(`Accepted-At: ${entry.accepted_at}`);
|
|
@@ -450,6 +457,7 @@ export function checkpointAcceptedTurn(root, opts = {}) {
|
|
|
450
457
|
|
|
451
458
|
const checkpointSha = git(root, ['rev-parse', 'HEAD']);
|
|
452
459
|
const checkpointedAt = new Date().toISOString();
|
|
460
|
+
const runtimeId = normalizeRuntimeId(entry);
|
|
453
461
|
|
|
454
462
|
const historyEntries = readHistoryEntries(root).map((historyEntry) => (
|
|
455
463
|
historyEntry.turn_id === entry.turn_id
|
|
@@ -470,6 +478,7 @@ export function checkpointAcceptedTurn(root, opts = {}) {
|
|
|
470
478
|
turn_id: entry.turn_id,
|
|
471
479
|
role: entry.role || null,
|
|
472
480
|
phase: entry.phase || null,
|
|
481
|
+
runtime_id: runtimeId,
|
|
473
482
|
checkpoint_sha: checkpointSha,
|
|
474
483
|
checkpointed_at: checkpointedAt,
|
|
475
484
|
intent_id: entry.intent_id || null,
|
|
@@ -480,7 +489,7 @@ export function checkpointAcceptedTurn(root, opts = {}) {
|
|
|
480
489
|
run_id: state.run_id || null,
|
|
481
490
|
phase: state.phase || null,
|
|
482
491
|
status: state.status || null,
|
|
483
|
-
turn: { turn_id: entry.turn_id, role_id: entry.role || null },
|
|
492
|
+
turn: { turn_id: entry.turn_id, role_id: entry.role || null, runtime_id: runtimeId },
|
|
484
493
|
intent_id: entry.intent_id || null,
|
|
485
494
|
payload: { checkpoint_sha: checkpointSha, checkpointed_at: checkpointedAt },
|
|
486
495
|
});
|