@chllming/wave-orchestration 0.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +41 -0
- package/README.md +549 -0
- package/docs/agents/wave-deploy-verifier-role.md +34 -0
- package/docs/agents/wave-documentation-role.md +30 -0
- package/docs/agents/wave-evaluator-role.md +43 -0
- package/docs/agents/wave-infra-role.md +34 -0
- package/docs/agents/wave-integration-role.md +32 -0
- package/docs/agents/wave-launcher-role.md +37 -0
- package/docs/context7/bundles.json +91 -0
- package/docs/plans/component-cutover-matrix.json +112 -0
- package/docs/plans/component-cutover-matrix.md +49 -0
- package/docs/plans/context7-wave-orchestrator.md +130 -0
- package/docs/plans/current-state.md +44 -0
- package/docs/plans/master-plan.md +16 -0
- package/docs/plans/migration.md +23 -0
- package/docs/plans/wave-orchestrator.md +254 -0
- package/docs/plans/waves/wave-0.md +165 -0
- package/docs/reference/github-packages-setup.md +52 -0
- package/docs/reference/migration-0.2-to-0.5.md +622 -0
- package/docs/reference/npmjs-trusted-publishing.md +55 -0
- package/docs/reference/repository-guidance.md +18 -0
- package/docs/reference/runtime-config/README.md +85 -0
- package/docs/reference/runtime-config/claude.md +105 -0
- package/docs/reference/runtime-config/codex.md +81 -0
- package/docs/reference/runtime-config/opencode.md +93 -0
- package/docs/research/agent-context-sources.md +57 -0
- package/docs/roadmap.md +626 -0
- package/package.json +53 -0
- package/releases/manifest.json +101 -0
- package/scripts/context7-api-check.sh +21 -0
- package/scripts/context7-export-env.sh +52 -0
- package/scripts/research/agent-context-archive.mjs +472 -0
- package/scripts/research/generate-agent-context-indexes.mjs +85 -0
- package/scripts/research/import-agent-context-archive.mjs +793 -0
- package/scripts/research/manifests/harness-and-blackboard-2026-03-21.mjs +201 -0
- package/scripts/wave-autonomous.mjs +13 -0
- package/scripts/wave-cli-bootstrap.mjs +27 -0
- package/scripts/wave-dashboard.mjs +11 -0
- package/scripts/wave-human-feedback.mjs +11 -0
- package/scripts/wave-launcher.mjs +11 -0
- package/scripts/wave-local-executor.mjs +13 -0
- package/scripts/wave-orchestrator/agent-state.mjs +416 -0
- package/scripts/wave-orchestrator/autonomous.mjs +367 -0
- package/scripts/wave-orchestrator/clarification-triage.mjs +605 -0
- package/scripts/wave-orchestrator/config.mjs +848 -0
- package/scripts/wave-orchestrator/context7.mjs +464 -0
- package/scripts/wave-orchestrator/coord-cli.mjs +286 -0
- package/scripts/wave-orchestrator/coordination-store.mjs +987 -0
- package/scripts/wave-orchestrator/coordination.mjs +768 -0
- package/scripts/wave-orchestrator/dashboard-renderer.mjs +254 -0
- package/scripts/wave-orchestrator/dashboard-state.mjs +473 -0
- package/scripts/wave-orchestrator/dep-cli.mjs +219 -0
- package/scripts/wave-orchestrator/docs-queue.mjs +75 -0
- package/scripts/wave-orchestrator/executors.mjs +385 -0
- package/scripts/wave-orchestrator/feedback.mjs +372 -0
- package/scripts/wave-orchestrator/install.mjs +540 -0
- package/scripts/wave-orchestrator/launcher.mjs +3879 -0
- package/scripts/wave-orchestrator/ledger.mjs +332 -0
- package/scripts/wave-orchestrator/local-executor.mjs +263 -0
- package/scripts/wave-orchestrator/replay.mjs +246 -0
- package/scripts/wave-orchestrator/roots.mjs +10 -0
- package/scripts/wave-orchestrator/routing-state.mjs +542 -0
- package/scripts/wave-orchestrator/shared.mjs +405 -0
- package/scripts/wave-orchestrator/terminals.mjs +209 -0
- package/scripts/wave-orchestrator/traces.mjs +1094 -0
- package/scripts/wave-orchestrator/wave-files.mjs +1923 -0
- package/scripts/wave.mjs +103 -0
- package/wave.config.json +115 -0
|
@@ -0,0 +1,3879 @@
|
|
|
1
|
+
import { spawnSync } from "node:child_process";
|
|
2
|
+
import crypto from "node:crypto";
|
|
3
|
+
import fs from "node:fs";
|
|
4
|
+
import path from "node:path";
|
|
5
|
+
import {
|
|
6
|
+
CODEX_SANDBOX_MODES,
|
|
7
|
+
DEFAULT_CODEX_SANDBOX_MODE,
|
|
8
|
+
normalizeCodexSandboxMode,
|
|
9
|
+
normalizeExecutorMode,
|
|
10
|
+
SUPPORTED_EXECUTOR_MODES,
|
|
11
|
+
} from "./config.mjs";
|
|
12
|
+
import {
|
|
13
|
+
appendOrchestratorBoardEntry,
|
|
14
|
+
buildExecutionPrompt,
|
|
15
|
+
ensureOrchestratorBoard,
|
|
16
|
+
feedbackStateSignature,
|
|
17
|
+
readWaveHumanFeedbackRequests,
|
|
18
|
+
} from "./coordination.mjs";
|
|
19
|
+
import {
|
|
20
|
+
appendCoordinationRecord,
|
|
21
|
+
compileAgentInbox,
|
|
22
|
+
compileSharedSummary,
|
|
23
|
+
isOpenCoordinationStatus,
|
|
24
|
+
openClarificationLinkedRequests,
|
|
25
|
+
readMaterializedCoordinationState,
|
|
26
|
+
renderCoordinationBoardProjection,
|
|
27
|
+
updateSeedRecords,
|
|
28
|
+
writeCompiledInbox,
|
|
29
|
+
writeCoordinationBoardProjection,
|
|
30
|
+
writeJsonArtifact,
|
|
31
|
+
} from "./coordination-store.mjs";
|
|
32
|
+
import {
|
|
33
|
+
applyContext7SelectionsToWave,
|
|
34
|
+
describeContext7Libraries,
|
|
35
|
+
hashAgentPromptFingerprint,
|
|
36
|
+
loadContext7BundleIndex,
|
|
37
|
+
prefetchContext7ForSelection,
|
|
38
|
+
} from "./context7.mjs";
|
|
39
|
+
import {
|
|
40
|
+
buildGlobalDashboardState,
|
|
41
|
+
buildWaveDashboardState,
|
|
42
|
+
getGlobalWaveEntry,
|
|
43
|
+
parseStructuredSignalsFromLog,
|
|
44
|
+
readStatusCodeIfPresent,
|
|
45
|
+
recordGlobalDashboardEvent,
|
|
46
|
+
recordWaveDashboardEvent,
|
|
47
|
+
refreshWaveDashboardAgentStates,
|
|
48
|
+
setWaveDashboardAgent,
|
|
49
|
+
syncGlobalWaveFromWaveDashboard,
|
|
50
|
+
updateWaveDashboardMessageBoard,
|
|
51
|
+
writeGlobalDashboard,
|
|
52
|
+
writeWaveDashboard,
|
|
53
|
+
} from "./dashboard-state.mjs";
|
|
54
|
+
import {
|
|
55
|
+
DEFAULT_AGENT_LAUNCH_STAGGER_MS,
|
|
56
|
+
DEFAULT_AGENT_RATE_LIMIT_BASE_DELAY_SECONDS,
|
|
57
|
+
DEFAULT_AGENT_RATE_LIMIT_MAX_DELAY_SECONDS,
|
|
58
|
+
DEFAULT_AGENT_RATE_LIMIT_RETRIES,
|
|
59
|
+
DEFAULT_MAX_RETRIES_PER_WAVE,
|
|
60
|
+
DEFAULT_TIMEOUT_MINUTES,
|
|
61
|
+
DEFAULT_WAIT_PROGRESS_INTERVAL_MS,
|
|
62
|
+
DEFAULT_WAVE_LANE,
|
|
63
|
+
compactSingleLine,
|
|
64
|
+
parseVerdictFromText,
|
|
65
|
+
readStatusRecordIfPresent,
|
|
66
|
+
REPO_ROOT,
|
|
67
|
+
buildLanePaths,
|
|
68
|
+
ensureDirectory,
|
|
69
|
+
parseNonNegativeInt,
|
|
70
|
+
parsePositiveInt,
|
|
71
|
+
readFileTail,
|
|
72
|
+
readJsonOrNull,
|
|
73
|
+
REPORT_VERDICT_REGEX,
|
|
74
|
+
sanitizeOrchestratorId,
|
|
75
|
+
shellQuote,
|
|
76
|
+
sleep,
|
|
77
|
+
PACKAGE_ROOT,
|
|
78
|
+
TMUX_COMMAND_TIMEOUT_MS,
|
|
79
|
+
WAVE_VERDICT_REGEX,
|
|
80
|
+
WAVE_TERMINAL_STATES,
|
|
81
|
+
toIsoTimestamp,
|
|
82
|
+
writeJsonAtomic,
|
|
83
|
+
writeTextAtomic,
|
|
84
|
+
} from "./shared.mjs";
|
|
85
|
+
import {
|
|
86
|
+
appendTerminalEntries,
|
|
87
|
+
createGlobalDashboardTerminalEntry,
|
|
88
|
+
createTemporaryTerminalEntries,
|
|
89
|
+
killTmuxSessionIfExists,
|
|
90
|
+
pruneOrphanLaneTemporaryTerminalEntries,
|
|
91
|
+
removeLaneTemporaryTerminalEntries,
|
|
92
|
+
removeTerminalEntries,
|
|
93
|
+
} from "./terminals.mjs";
|
|
94
|
+
import {
|
|
95
|
+
buildCodexExecInvocation,
|
|
96
|
+
buildExecutorLaunchSpec,
|
|
97
|
+
commandForExecutor,
|
|
98
|
+
isExecutorCommandAvailable,
|
|
99
|
+
} from "./executors.mjs";
|
|
100
|
+
import {
|
|
101
|
+
buildManifest,
|
|
102
|
+
applyExecutorSelectionsToWave,
|
|
103
|
+
markWaveCompleted,
|
|
104
|
+
parseWaveFiles,
|
|
105
|
+
reconcileRunStateFromStatusFiles,
|
|
106
|
+
resolveAutoNextWaveStart,
|
|
107
|
+
validateWaveRuntimeMixAssignments,
|
|
108
|
+
validateWaveComponentMatrixCurrentLevels,
|
|
109
|
+
validateWaveComponentPromotions,
|
|
110
|
+
validateWaveDefinition,
|
|
111
|
+
writeManifest,
|
|
112
|
+
} from "./wave-files.mjs";
|
|
113
|
+
import {
|
|
114
|
+
agentSummaryPathFromStatusPath,
|
|
115
|
+
buildAgentExecutionSummary,
|
|
116
|
+
readAgentExecutionSummary,
|
|
117
|
+
validateDocumentationClosureSummary,
|
|
118
|
+
validateEvaluatorSummary,
|
|
119
|
+
validateIntegrationSummary,
|
|
120
|
+
validateImplementationSummary,
|
|
121
|
+
writeAgentExecutionSummary,
|
|
122
|
+
} from "./agent-state.mjs";
|
|
123
|
+
import { buildDocsQueue, readDocsQueue, writeDocsQueue } from "./docs-queue.mjs";
|
|
124
|
+
import { deriveWaveLedger, readWaveLedger, writeWaveLedger } from "./ledger.mjs";
|
|
125
|
+
import { buildQualityMetrics, writeTraceBundle } from "./traces.mjs";
|
|
126
|
+
import { triageClarificationRequests } from "./clarification-triage.mjs";
|
|
127
|
+
import {
|
|
128
|
+
buildDependencySnapshot,
|
|
129
|
+
buildRequestAssignments,
|
|
130
|
+
renderDependencySnapshotMarkdown,
|
|
131
|
+
syncAssignmentRecords,
|
|
132
|
+
writeDependencySnapshotMarkdown,
|
|
133
|
+
} from "./routing-state.mjs";
|
|
134
|
+
export { CODEX_SANDBOX_MODES, DEFAULT_CODEX_SANDBOX_MODE, normalizeCodexSandboxMode, buildCodexExecInvocation };
|
|
135
|
+
|
|
136
|
+
function printUsage(lanePaths) {
|
|
137
|
+
console.log(`Usage: pnpm exec wave launch [options]
|
|
138
|
+
|
|
139
|
+
Options:
|
|
140
|
+
--lane <name> Wave lane name (default: ${DEFAULT_WAVE_LANE})
|
|
141
|
+
--start-wave <n> Start from wave number (default: 0)
|
|
142
|
+
--end-wave <n> End at wave number (default: last available)
|
|
143
|
+
--auto-next Start from the next unfinished wave and continue forward
|
|
144
|
+
--reconcile-status Reconcile run-state from agent status files and exit
|
|
145
|
+
--state-file <path> Path to run-state JSON (default: ${path.relative(REPO_ROOT, lanePaths.defaultRunStatePath)})
|
|
146
|
+
--timeout-minutes <n> Max minutes to wait per wave (default: ${DEFAULT_TIMEOUT_MINUTES})
|
|
147
|
+
--max-retries-per-wave <n>
|
|
148
|
+
Relaunch failed or missing jobs per wave (default: ${DEFAULT_MAX_RETRIES_PER_WAVE})
|
|
149
|
+
--agent-rate-limit-retries <n>
|
|
150
|
+
Per-agent retries for 429 or rate-limit errors (default: ${DEFAULT_AGENT_RATE_LIMIT_RETRIES})
|
|
151
|
+
--agent-rate-limit-base-delay-seconds <n>
|
|
152
|
+
Base exponential backoff delay for 429 retries (default: ${DEFAULT_AGENT_RATE_LIMIT_BASE_DELAY_SECONDS})
|
|
153
|
+
--agent-rate-limit-max-delay-seconds <n>
|
|
154
|
+
Max backoff delay for 429 retries (default: ${DEFAULT_AGENT_RATE_LIMIT_MAX_DELAY_SECONDS})
|
|
155
|
+
--agent-launch-stagger-ms <n>
|
|
156
|
+
Delay between agent launches (default: ${DEFAULT_AGENT_LAUNCH_STAGGER_MS})
|
|
157
|
+
--executor <mode> Default agent executor mode: ${SUPPORTED_EXECUTOR_MODES.join(" | ")} (default: ${lanePaths.executors.default})
|
|
158
|
+
--codex-sandbox <mode> Codex sandbox mode: ${CODEX_SANDBOX_MODES.join(" | ")} (default: ${DEFAULT_CODEX_SANDBOX_MODE})
|
|
159
|
+
--manifest-out <path> Write parsed wave manifest JSON (default: ${path.relative(REPO_ROOT, lanePaths.defaultManifestPath)})
|
|
160
|
+
--dry-run Parse waves and update manifest only
|
|
161
|
+
--no-dashboard Disable per-wave tmux dashboard session
|
|
162
|
+
--cleanup-sessions Kill lane tmux sessions after each wave (default: on)
|
|
163
|
+
--keep-sessions Keep lane tmux sessions after each wave
|
|
164
|
+
--keep-terminals Do not remove temporary terminal entries after each wave
|
|
165
|
+
--orchestrator-id <id> Stable orchestrator identity for cross-lane coordination
|
|
166
|
+
--orchestrator-board <path>
|
|
167
|
+
Path to shared orchestrator coordination board (default: ${path.relative(REPO_ROOT, lanePaths.defaultOrchestratorBoardPath)})
|
|
168
|
+
--no-orchestrator-board
|
|
169
|
+
Disable orchestrator coordination board updates for this run
|
|
170
|
+
--coordination-note <text>
|
|
171
|
+
Optional startup intent note appended to orchestrator board
|
|
172
|
+
--no-context7 Disable launcher-side Context7 prefetch/injection
|
|
173
|
+
--help Show this help message
|
|
174
|
+
`);
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
function parseArgs(argv) {
|
|
178
|
+
let lanePaths = buildLanePaths(DEFAULT_WAVE_LANE);
|
|
179
|
+
const options = {
|
|
180
|
+
lane: DEFAULT_WAVE_LANE,
|
|
181
|
+
startWave: 0,
|
|
182
|
+
endWave: null,
|
|
183
|
+
autoNext: false,
|
|
184
|
+
reconcileStatus: false,
|
|
185
|
+
runStatePath: lanePaths.defaultRunStatePath,
|
|
186
|
+
timeoutMinutes: DEFAULT_TIMEOUT_MINUTES,
|
|
187
|
+
maxRetriesPerWave: DEFAULT_MAX_RETRIES_PER_WAVE,
|
|
188
|
+
agentRateLimitRetries: DEFAULT_AGENT_RATE_LIMIT_RETRIES,
|
|
189
|
+
agentRateLimitBaseDelaySeconds: DEFAULT_AGENT_RATE_LIMIT_BASE_DELAY_SECONDS,
|
|
190
|
+
agentRateLimitMaxDelaySeconds: DEFAULT_AGENT_RATE_LIMIT_MAX_DELAY_SECONDS,
|
|
191
|
+
agentLaunchStaggerMs: DEFAULT_AGENT_LAUNCH_STAGGER_MS,
|
|
192
|
+
executorMode: lanePaths.executors.default,
|
|
193
|
+
codexSandboxMode: DEFAULT_CODEX_SANDBOX_MODE,
|
|
194
|
+
manifestOut: lanePaths.defaultManifestPath,
|
|
195
|
+
dryRun: false,
|
|
196
|
+
dashboard: true,
|
|
197
|
+
cleanupSessions: true,
|
|
198
|
+
keepTerminals: false,
|
|
199
|
+
context7Enabled: true,
|
|
200
|
+
orchestratorId: null,
|
|
201
|
+
orchestratorBoardPath: null,
|
|
202
|
+
coordinationNote: "",
|
|
203
|
+
};
|
|
204
|
+
let stateFileProvided = false;
|
|
205
|
+
let manifestOutProvided = false;
|
|
206
|
+
let orchestratorBoardProvided = false;
|
|
207
|
+
let executorProvided = false;
|
|
208
|
+
|
|
209
|
+
for (let i = 0; i < argv.length; i += 1) {
|
|
210
|
+
const arg = argv[i];
|
|
211
|
+
if (arg === "--") {
|
|
212
|
+
continue;
|
|
213
|
+
}
|
|
214
|
+
if (arg === "--help" || arg === "-h") {
|
|
215
|
+
return { help: true, lanePaths, options };
|
|
216
|
+
}
|
|
217
|
+
if (arg === "--dry-run") {
|
|
218
|
+
options.dryRun = true;
|
|
219
|
+
} else if (arg === "--no-dashboard") {
|
|
220
|
+
options.dashboard = false;
|
|
221
|
+
} else if (arg === "--cleanup-sessions") {
|
|
222
|
+
options.cleanupSessions = true;
|
|
223
|
+
} else if (arg === "--keep-sessions") {
|
|
224
|
+
options.cleanupSessions = false;
|
|
225
|
+
} else if (arg === "--auto-next") {
|
|
226
|
+
options.autoNext = true;
|
|
227
|
+
} else if (arg === "--reconcile-status") {
|
|
228
|
+
options.reconcileStatus = true;
|
|
229
|
+
} else if (arg === "--keep-terminals") {
|
|
230
|
+
options.keepTerminals = true;
|
|
231
|
+
} else if (arg === "--no-context7") {
|
|
232
|
+
options.context7Enabled = false;
|
|
233
|
+
} else if (arg === "--no-orchestrator-board") {
|
|
234
|
+
options.orchestratorBoardPath = null;
|
|
235
|
+
orchestratorBoardProvided = true;
|
|
236
|
+
} else if (arg === "--lane") {
|
|
237
|
+
options.lane = String(argv[++i] || "").trim();
|
|
238
|
+
lanePaths = buildLanePaths(options.lane);
|
|
239
|
+
} else if (arg === "--orchestrator-id") {
|
|
240
|
+
options.orchestratorId = sanitizeOrchestratorId(argv[++i]);
|
|
241
|
+
} else if (arg === "--orchestrator-board") {
|
|
242
|
+
options.orchestratorBoardPath = path.resolve(REPO_ROOT, argv[++i] || "");
|
|
243
|
+
orchestratorBoardProvided = true;
|
|
244
|
+
} else if (arg === "--coordination-note") {
|
|
245
|
+
options.coordinationNote = String(argv[++i] || "").trim();
|
|
246
|
+
} else if (arg === "--state-file") {
|
|
247
|
+
options.runStatePath = path.resolve(REPO_ROOT, argv[++i] || "");
|
|
248
|
+
stateFileProvided = true;
|
|
249
|
+
} else if (arg === "--start-wave") {
|
|
250
|
+
options.startWave = parseNonNegativeInt(argv[++i], "--start-wave");
|
|
251
|
+
} else if (arg === "--end-wave") {
|
|
252
|
+
options.endWave = parseNonNegativeInt(argv[++i], "--end-wave");
|
|
253
|
+
} else if (arg === "--timeout-minutes") {
|
|
254
|
+
options.timeoutMinutes = parsePositiveInt(argv[++i], "--timeout-minutes");
|
|
255
|
+
} else if (arg === "--max-retries-per-wave") {
|
|
256
|
+
options.maxRetriesPerWave = parseNonNegativeInt(argv[++i], "--max-retries-per-wave");
|
|
257
|
+
} else if (arg === "--agent-rate-limit-retries") {
|
|
258
|
+
options.agentRateLimitRetries = parseNonNegativeInt(argv[++i], "--agent-rate-limit-retries");
|
|
259
|
+
} else if (arg === "--agent-rate-limit-base-delay-seconds") {
|
|
260
|
+
options.agentRateLimitBaseDelaySeconds = parsePositiveInt(
|
|
261
|
+
argv[++i],
|
|
262
|
+
"--agent-rate-limit-base-delay-seconds",
|
|
263
|
+
);
|
|
264
|
+
} else if (arg === "--agent-rate-limit-max-delay-seconds") {
|
|
265
|
+
options.agentRateLimitMaxDelaySeconds = parsePositiveInt(
|
|
266
|
+
argv[++i],
|
|
267
|
+
"--agent-rate-limit-max-delay-seconds",
|
|
268
|
+
);
|
|
269
|
+
} else if (arg === "--agent-launch-stagger-ms") {
|
|
270
|
+
options.agentLaunchStaggerMs = parseNonNegativeInt(argv[++i], "--agent-launch-stagger-ms");
|
|
271
|
+
} else if (arg === "--executor") {
|
|
272
|
+
options.executorMode = normalizeExecutorMode(argv[++i], "--executor");
|
|
273
|
+
executorProvided = true;
|
|
274
|
+
} else if (arg === "--codex-sandbox") {
|
|
275
|
+
options.codexSandboxMode = normalizeCodexSandboxMode(argv[++i], "--codex-sandbox");
|
|
276
|
+
} else if (arg === "--manifest-out") {
|
|
277
|
+
options.manifestOut = path.resolve(REPO_ROOT, argv[++i] || "");
|
|
278
|
+
manifestOutProvided = true;
|
|
279
|
+
} else {
|
|
280
|
+
throw new Error(`Unknown argument: ${arg}`);
|
|
281
|
+
}
|
|
282
|
+
}
|
|
283
|
+
|
|
284
|
+
lanePaths = buildLanePaths(options.lane, {
|
|
285
|
+
runVariant: options.dryRun ? "dry-run" : undefined,
|
|
286
|
+
});
|
|
287
|
+
if (!stateFileProvided) {
|
|
288
|
+
options.runStatePath = lanePaths.defaultRunStatePath;
|
|
289
|
+
}
|
|
290
|
+
if (!manifestOutProvided) {
|
|
291
|
+
options.manifestOut = lanePaths.defaultManifestPath;
|
|
292
|
+
}
|
|
293
|
+
if (!orchestratorBoardProvided) {
|
|
294
|
+
options.orchestratorBoardPath = lanePaths.defaultOrchestratorBoardPath;
|
|
295
|
+
}
|
|
296
|
+
if (!executorProvided) {
|
|
297
|
+
options.executorMode = lanePaths.executors.default;
|
|
298
|
+
}
|
|
299
|
+
options.orchestratorId ||= sanitizeOrchestratorId(`${lanePaths.lane}-orch-${process.pid}`);
|
|
300
|
+
if (options.agentRateLimitMaxDelaySeconds < options.agentRateLimitBaseDelaySeconds) {
|
|
301
|
+
throw new Error(
|
|
302
|
+
"--agent-rate-limit-max-delay-seconds must be >= --agent-rate-limit-base-delay-seconds",
|
|
303
|
+
);
|
|
304
|
+
}
|
|
305
|
+
if (!options.autoNext && options.endWave !== null && options.endWave < options.startWave) {
|
|
306
|
+
throw new Error("--end-wave must be >= --start-wave");
|
|
307
|
+
}
|
|
308
|
+
return { help: false, lanePaths, options };
|
|
309
|
+
}
|
|
310
|
+
|
|
311
|
+
function isProcessAlive(pid) {
|
|
312
|
+
if (!Number.isInteger(pid) || pid <= 0) {
|
|
313
|
+
return false;
|
|
314
|
+
}
|
|
315
|
+
try {
|
|
316
|
+
process.kill(pid, 0);
|
|
317
|
+
return true;
|
|
318
|
+
} catch {
|
|
319
|
+
return false;
|
|
320
|
+
}
|
|
321
|
+
}
|
|
322
|
+
|
|
323
|
+
export function readWaveEvaluatorGate(wave, agentRuns, options = {}) {
|
|
324
|
+
const evaluatorAgentId = options.evaluatorAgentId || wave.evaluatorAgentId || "A0";
|
|
325
|
+
const evaluatorRun =
|
|
326
|
+
agentRuns.find((run) => run.agent.agentId === evaluatorAgentId) ?? null;
|
|
327
|
+
if (!evaluatorRun) {
|
|
328
|
+
return {
|
|
329
|
+
ok: false,
|
|
330
|
+
agentId: evaluatorAgentId,
|
|
331
|
+
statusCode: "missing-evaluator",
|
|
332
|
+
detail: `Agent ${evaluatorAgentId} is missing.`,
|
|
333
|
+
logPath: null,
|
|
334
|
+
};
|
|
335
|
+
}
|
|
336
|
+
const summary = readRunExecutionSummary(evaluatorRun);
|
|
337
|
+
if (summary) {
|
|
338
|
+
const validation = validateEvaluatorSummary(evaluatorRun.agent, summary);
|
|
339
|
+
return {
|
|
340
|
+
ok: validation.ok,
|
|
341
|
+
agentId: evaluatorRun.agent.agentId,
|
|
342
|
+
statusCode: validation.statusCode,
|
|
343
|
+
detail: validation.detail,
|
|
344
|
+
logPath: summary.logPath || path.relative(REPO_ROOT, evaluatorRun.logPath),
|
|
345
|
+
};
|
|
346
|
+
}
|
|
347
|
+
const evaluatorReportPath = wave.evaluatorReportPath
|
|
348
|
+
? path.resolve(REPO_ROOT, wave.evaluatorReportPath)
|
|
349
|
+
: null;
|
|
350
|
+
const reportText =
|
|
351
|
+
evaluatorReportPath && fs.existsSync(evaluatorReportPath)
|
|
352
|
+
? fs.readFileSync(evaluatorReportPath, "utf8")
|
|
353
|
+
: "";
|
|
354
|
+
const reportVerdict = parseVerdictFromText(reportText, REPORT_VERDICT_REGEX);
|
|
355
|
+
if (reportVerdict.verdict) {
|
|
356
|
+
return {
|
|
357
|
+
ok: reportVerdict.verdict === "pass",
|
|
358
|
+
agentId: evaluatorRun.agent.agentId,
|
|
359
|
+
statusCode: reportVerdict.verdict === "pass" ? "pass" : `evaluator-${reportVerdict.verdict}`,
|
|
360
|
+
detail: reportVerdict.detail || "Verdict read from evaluator report.",
|
|
361
|
+
logPath: path.relative(REPO_ROOT, evaluatorRun.logPath),
|
|
362
|
+
};
|
|
363
|
+
}
|
|
364
|
+
const logVerdict = parseVerdictFromText(
|
|
365
|
+
readFileTail(evaluatorRun.logPath, 30000),
|
|
366
|
+
WAVE_VERDICT_REGEX,
|
|
367
|
+
);
|
|
368
|
+
if (logVerdict.verdict) {
|
|
369
|
+
return {
|
|
370
|
+
ok: logVerdict.verdict === "pass",
|
|
371
|
+
agentId: evaluatorRun.agent.agentId,
|
|
372
|
+
statusCode: logVerdict.verdict === "pass" ? "pass" : `evaluator-${logVerdict.verdict}`,
|
|
373
|
+
detail: logVerdict.detail || "Verdict read from evaluator log marker.",
|
|
374
|
+
logPath: path.relative(REPO_ROOT, evaluatorRun.logPath),
|
|
375
|
+
};
|
|
376
|
+
}
|
|
377
|
+
return {
|
|
378
|
+
ok: false,
|
|
379
|
+
agentId: evaluatorRun.agent.agentId,
|
|
380
|
+
statusCode: "missing-evaluator-verdict",
|
|
381
|
+
detail: evaluatorReportPath
|
|
382
|
+
? `Missing Verdict line in ${path.relative(REPO_ROOT, evaluatorReportPath)} and no [wave-verdict] marker in ${path.relative(REPO_ROOT, evaluatorRun.logPath)}.`
|
|
383
|
+
: `Missing evaluator report path and no [wave-verdict] marker in ${path.relative(REPO_ROOT, evaluatorRun.logPath)}.`,
|
|
384
|
+
logPath: path.relative(REPO_ROOT, evaluatorRun.logPath),
|
|
385
|
+
};
|
|
386
|
+
}
|
|
387
|
+
|
|
388
|
+
function materializeAgentExecutionSummaryForRun(wave, runInfo) {
|
|
389
|
+
const statusRecord = readStatusRecordIfPresent(runInfo.statusPath);
|
|
390
|
+
if (!statusRecord) {
|
|
391
|
+
return null;
|
|
392
|
+
}
|
|
393
|
+
const reportPath =
|
|
394
|
+
runInfo.agent.agentId === (wave.evaluatorAgentId || "A0") && wave.evaluatorReportPath
|
|
395
|
+
? path.resolve(REPO_ROOT, wave.evaluatorReportPath)
|
|
396
|
+
: null;
|
|
397
|
+
const summary = buildAgentExecutionSummary({
|
|
398
|
+
agent: runInfo.agent,
|
|
399
|
+
statusRecord,
|
|
400
|
+
logPath: runInfo.logPath,
|
|
401
|
+
reportPath,
|
|
402
|
+
});
|
|
403
|
+
writeAgentExecutionSummary(runInfo.statusPath, summary);
|
|
404
|
+
return summary;
|
|
405
|
+
}
|
|
406
|
+
|
|
407
|
+
function readRunExecutionSummary(runInfo) {
|
|
408
|
+
if (runInfo?.summary && typeof runInfo.summary === "object") {
|
|
409
|
+
return runInfo.summary;
|
|
410
|
+
}
|
|
411
|
+
if (runInfo?.summaryPath && fs.existsSync(runInfo.summaryPath)) {
|
|
412
|
+
return readAgentExecutionSummary(runInfo.summaryPath);
|
|
413
|
+
}
|
|
414
|
+
if (runInfo?.statusPath && fs.existsSync(agentSummaryPathFromStatusPath(runInfo.statusPath))) {
|
|
415
|
+
return readAgentExecutionSummary(runInfo.statusPath);
|
|
416
|
+
}
|
|
417
|
+
return null;
|
|
418
|
+
}
|
|
419
|
+
|
|
420
|
+
function materializeAgentExecutionSummaries(wave, agentRuns) {
|
|
421
|
+
return Object.fromEntries(
|
|
422
|
+
agentRuns.map((runInfo) => [runInfo.agent.agentId, materializeAgentExecutionSummaryForRun(wave, runInfo)]),
|
|
423
|
+
);
|
|
424
|
+
}
|
|
425
|
+
|
|
426
|
+
function waveCoordinationLogPath(lanePaths, waveNumber) {
|
|
427
|
+
return path.join(lanePaths.coordinationDir, `wave-${waveNumber}.jsonl`);
|
|
428
|
+
}
|
|
429
|
+
|
|
430
|
+
function waveInboxDir(lanePaths, waveNumber) {
|
|
431
|
+
return path.join(lanePaths.inboxesDir, `wave-${waveNumber}`);
|
|
432
|
+
}
|
|
433
|
+
|
|
434
|
+
function waveAssignmentsPath(lanePaths, waveNumber) {
|
|
435
|
+
return path.join(lanePaths.assignmentsDir, `wave-${waveNumber}.json`);
|
|
436
|
+
}
|
|
437
|
+
|
|
438
|
+
function waveLedgerPath(lanePaths, waveNumber) {
|
|
439
|
+
return path.join(lanePaths.ledgerDir, `wave-${waveNumber}.json`);
|
|
440
|
+
}
|
|
441
|
+
|
|
442
|
+
function waveDependencySnapshotPath(lanePaths, waveNumber) {
|
|
443
|
+
return path.join(lanePaths.dependencySnapshotsDir, `wave-${waveNumber}.json`);
|
|
444
|
+
}
|
|
445
|
+
|
|
446
|
+
function waveDependencySnapshotMarkdownPath(lanePaths, waveNumber) {
|
|
447
|
+
return path.join(lanePaths.dependencySnapshotsDir, `wave-${waveNumber}.md`);
|
|
448
|
+
}
|
|
449
|
+
|
|
450
|
+
function waveDocsQueuePath(lanePaths, waveNumber) {
|
|
451
|
+
return path.join(lanePaths.docsQueueDir, `wave-${waveNumber}.json`);
|
|
452
|
+
}
|
|
453
|
+
|
|
454
|
+
function waveIntegrationPath(lanePaths, waveNumber) {
|
|
455
|
+
return path.join(lanePaths.integrationDir, `wave-${waveNumber}.json`);
|
|
456
|
+
}
|
|
457
|
+
|
|
458
|
+
function waveIntegrationMarkdownPath(lanePaths, waveNumber) {
|
|
459
|
+
return path.join(lanePaths.integrationDir, `wave-${waveNumber}.md`);
|
|
460
|
+
}
|
|
461
|
+
|
|
462
|
+
function uniqueStringEntries(values) {
|
|
463
|
+
return Array.from(
|
|
464
|
+
new Set(
|
|
465
|
+
(Array.isArray(values) ? values : [])
|
|
466
|
+
.map((value) => String(value || "").trim())
|
|
467
|
+
.filter(Boolean),
|
|
468
|
+
),
|
|
469
|
+
);
|
|
470
|
+
}
|
|
471
|
+
|
|
472
|
+
function summarizeIntegrationRecord(record, options = {}) {
|
|
473
|
+
const summary = compactSingleLine(
|
|
474
|
+
record?.summary || record?.detail || record?.kind || "coordination item",
|
|
475
|
+
options.maxChars || 180,
|
|
476
|
+
);
|
|
477
|
+
return `${record.id}: ${summary}`;
|
|
478
|
+
}
|
|
479
|
+
|
|
480
|
+
function summarizeDocsQueueItem(item) {
|
|
481
|
+
return `${item.id}: ${compactSingleLine(item.summary || item.path || item.detail || "documentation update required", 180)}`;
|
|
482
|
+
}
|
|
483
|
+
|
|
484
|
+
function summarizeGap(agentId, detail, fallback) {
|
|
485
|
+
return `${agentId}: ${compactSingleLine(detail || fallback, 180)}`;
|
|
486
|
+
}
|
|
487
|
+
|
|
488
|
+
function textMentionsAnyKeyword(value, keywords) {
|
|
489
|
+
const text = String(value || "").trim().toLowerCase();
|
|
490
|
+
if (!text) {
|
|
491
|
+
return false;
|
|
492
|
+
}
|
|
493
|
+
return keywords.some((keyword) => text.includes(String(keyword || "").trim().toLowerCase()));
|
|
494
|
+
}
|
|
495
|
+
|
|
496
|
+
function actionableIntegrationRecords(coordinationState) {
|
|
497
|
+
return (coordinationState?.latestRecords || []).filter(
|
|
498
|
+
(record) =>
|
|
499
|
+
!["cancelled", "superseded"].includes(String(record?.status || "").trim().toLowerCase()) &&
|
|
500
|
+
![
|
|
501
|
+
"human-feedback",
|
|
502
|
+
"human-escalation",
|
|
503
|
+
"orchestrator-guidance",
|
|
504
|
+
"resolved-by-policy",
|
|
505
|
+
"integration-summary",
|
|
506
|
+
].includes(record?.kind),
|
|
507
|
+
);
|
|
508
|
+
}
|
|
509
|
+
|
|
510
|
+
function normalizeOwnedReference(value) {
|
|
511
|
+
return String(value || "").trim().replace(/\/+$/, "");
|
|
512
|
+
}
|
|
513
|
+
|
|
514
|
+
function matchesOwnedPathArtifact(artifactRef, ownedPath) {
|
|
515
|
+
const normalizedArtifact = normalizeOwnedReference(artifactRef);
|
|
516
|
+
const normalizedOwnedPath = normalizeOwnedReference(ownedPath);
|
|
517
|
+
if (!normalizedArtifact || !normalizedOwnedPath) {
|
|
518
|
+
return false;
|
|
519
|
+
}
|
|
520
|
+
return (
|
|
521
|
+
normalizedArtifact === normalizedOwnedPath ||
|
|
522
|
+
normalizedArtifact.startsWith(`${normalizedOwnedPath}/`)
|
|
523
|
+
);
|
|
524
|
+
}
|
|
525
|
+
|
|
526
|
+
function resolveArtifactOwners(artifactRef, agents) {
|
|
527
|
+
const owners = [];
|
|
528
|
+
const normalizedArtifact = normalizeOwnedReference(artifactRef);
|
|
529
|
+
if (!normalizedArtifact) {
|
|
530
|
+
return owners;
|
|
531
|
+
}
|
|
532
|
+
for (const agent of agents || []) {
|
|
533
|
+
const ownedComponents = Array.isArray(agent?.components) ? agent.components : [];
|
|
534
|
+
const ownedPaths = Array.isArray(agent?.ownedPaths) ? agent.ownedPaths : [];
|
|
535
|
+
if (
|
|
536
|
+
ownedComponents.some((componentId) => normalizeOwnedReference(componentId) === normalizedArtifact) ||
|
|
537
|
+
ownedPaths.some((ownedPath) => matchesOwnedPathArtifact(normalizedArtifact, ownedPath))
|
|
538
|
+
) {
|
|
539
|
+
owners.push(agent.agentId);
|
|
540
|
+
}
|
|
541
|
+
}
|
|
542
|
+
return owners;
|
|
543
|
+
}
|
|
544
|
+
|
|
545
|
+
function inferIntegrationRecommendation(evidence) {
|
|
546
|
+
if ((evidence.unresolvedBlockers || []).length > 0) {
|
|
547
|
+
return {
|
|
548
|
+
recommendation: "needs-more-work",
|
|
549
|
+
detail: `${evidence.unresolvedBlockers.length} unresolved blocker(s) remain.`,
|
|
550
|
+
};
|
|
551
|
+
}
|
|
552
|
+
if ((evidence.conflictingClaims || []).length > 0) {
|
|
553
|
+
return {
|
|
554
|
+
recommendation: "needs-more-work",
|
|
555
|
+
detail: `${evidence.conflictingClaims.length} conflicting claim(s) remain.`,
|
|
556
|
+
};
|
|
557
|
+
}
|
|
558
|
+
if ((evidence.proofGaps || []).length > 0) {
|
|
559
|
+
return {
|
|
560
|
+
recommendation: "needs-more-work",
|
|
561
|
+
detail: `${evidence.proofGaps.length} proof gap(s) remain.`,
|
|
562
|
+
};
|
|
563
|
+
}
|
|
564
|
+
if ((evidence.deployRisks || []).length > 0) {
|
|
565
|
+
return {
|
|
566
|
+
recommendation: "needs-more-work",
|
|
567
|
+
detail: `${evidence.deployRisks.length} deploy or ops risk(s) remain.`,
|
|
568
|
+
};
|
|
569
|
+
}
|
|
570
|
+
return {
|
|
571
|
+
recommendation: "ready-for-doc-closure",
|
|
572
|
+
detail:
|
|
573
|
+
"No unresolved blockers, contradictions, proof gaps, or deploy risks remain in integration state.",
|
|
574
|
+
};
|
|
575
|
+
}
|
|
576
|
+
|
|
577
|
+
function padReportedEntries(entries, minimumCount, label) {
|
|
578
|
+
const padded = [...entries];
|
|
579
|
+
for (let index = padded.length + 1; index <= minimumCount; index += 1) {
|
|
580
|
+
padded.push(`${label} #${index}`);
|
|
581
|
+
}
|
|
582
|
+
return padded;
|
|
583
|
+
}
|
|
584
|
+
|
|
585
|
+
function buildIntegrationEvidence({
|
|
586
|
+
lanePaths,
|
|
587
|
+
wave,
|
|
588
|
+
coordinationState,
|
|
589
|
+
summariesByAgentId,
|
|
590
|
+
docsQueue,
|
|
591
|
+
agentRuns,
|
|
592
|
+
dependencySnapshot = null,
|
|
593
|
+
capabilityAssignments = [],
|
|
594
|
+
}) {
|
|
595
|
+
const openClaims = (coordinationState?.claims || [])
|
|
596
|
+
.filter((record) => isOpenCoordinationStatus(record.status))
|
|
597
|
+
.map((record) => summarizeIntegrationRecord(record));
|
|
598
|
+
const conflictingClaims = (coordinationState?.claims || [])
|
|
599
|
+
.filter(
|
|
600
|
+
(record) =>
|
|
601
|
+
isOpenCoordinationStatus(record.status) &&
|
|
602
|
+
/conflict|contradict/i.test(`${record.summary || ""}\n${record.detail || ""}`),
|
|
603
|
+
)
|
|
604
|
+
.map((record) => summarizeIntegrationRecord(record));
|
|
605
|
+
const unresolvedBlockers = (coordinationState?.blockers || [])
|
|
606
|
+
.filter((record) => isOpenCoordinationStatus(record.status))
|
|
607
|
+
.map((record) => summarizeIntegrationRecord(record));
|
|
608
|
+
|
|
609
|
+
const interfaceKeywords = ["interface", "contract", "api", "schema", "migration", "signature"];
|
|
610
|
+
const changedInterfaces = actionableIntegrationRecords(coordinationState)
|
|
611
|
+
.filter((record) =>
|
|
612
|
+
textMentionsAnyKeyword(
|
|
613
|
+
[record.summary, record.detail, ...(record.artifactRefs || [])].join("\n"),
|
|
614
|
+
interfaceKeywords,
|
|
615
|
+
),
|
|
616
|
+
)
|
|
617
|
+
.map((record) => summarizeIntegrationRecord(record));
|
|
618
|
+
|
|
619
|
+
const crossComponentImpacts = actionableIntegrationRecords(coordinationState)
|
|
620
|
+
.flatMap((record) => {
|
|
621
|
+
const owners = new Set();
|
|
622
|
+
for (const artifactRef of record.artifactRefs || []) {
|
|
623
|
+
for (const owner of resolveArtifactOwners(artifactRef, wave.agents)) {
|
|
624
|
+
owners.add(owner);
|
|
625
|
+
}
|
|
626
|
+
}
|
|
627
|
+
for (const target of record.targets || []) {
|
|
628
|
+
if (String(target).startsWith("agent:")) {
|
|
629
|
+
owners.add(String(target).slice("agent:".length));
|
|
630
|
+
} else if ((wave.agents || []).some((agent) => agent.agentId === target)) {
|
|
631
|
+
owners.add(String(target));
|
|
632
|
+
}
|
|
633
|
+
}
|
|
634
|
+
if (owners.size <= 1) {
|
|
635
|
+
return [];
|
|
636
|
+
}
|
|
637
|
+
return [
|
|
638
|
+
`${summarizeIntegrationRecord(record)} [owners: ${Array.from(owners).toSorted().join(", ")}]`,
|
|
639
|
+
];
|
|
640
|
+
});
|
|
641
|
+
|
|
642
|
+
const proofGapEntries = [];
|
|
643
|
+
const docGapEntries = Array.isArray(docsQueue?.items)
|
|
644
|
+
? docsQueue.items.map((item) => summarizeDocsQueueItem(item))
|
|
645
|
+
: [];
|
|
646
|
+
const deployRiskEntries = [];
|
|
647
|
+
for (const agent of wave.agents || []) {
|
|
648
|
+
const summary = summariesByAgentId?.[agent.agentId] || null;
|
|
649
|
+
if (
|
|
650
|
+
![
|
|
651
|
+
lanePaths.evaluatorAgentId,
|
|
652
|
+
lanePaths.integrationAgentId,
|
|
653
|
+
lanePaths.documentationAgentId,
|
|
654
|
+
].includes(agent.agentId)
|
|
655
|
+
) {
|
|
656
|
+
const validation = validateImplementationSummary(agent, summary);
|
|
657
|
+
if (!validation.ok) {
|
|
658
|
+
const entry = summarizeGap(agent.agentId, validation.detail, "Implementation validation failed.");
|
|
659
|
+
if (["missing-doc-delta", "doc-impact-gap"].includes(validation.statusCode)) {
|
|
660
|
+
docGapEntries.push(entry);
|
|
661
|
+
} else {
|
|
662
|
+
proofGapEntries.push(entry);
|
|
663
|
+
}
|
|
664
|
+
}
|
|
665
|
+
}
|
|
666
|
+
for (const gap of summary?.gaps || []) {
|
|
667
|
+
const entry = summarizeGap(
|
|
668
|
+
agent.agentId,
|
|
669
|
+
gap.detail,
|
|
670
|
+
`${gap.kind || "unknown"} gap reported.`,
|
|
671
|
+
);
|
|
672
|
+
if (gap.kind === "docs") {
|
|
673
|
+
docGapEntries.push(entry);
|
|
674
|
+
} else if (gap.kind === "ops") {
|
|
675
|
+
deployRiskEntries.push(entry);
|
|
676
|
+
} else {
|
|
677
|
+
proofGapEntries.push(entry);
|
|
678
|
+
}
|
|
679
|
+
}
|
|
680
|
+
}
|
|
681
|
+
|
|
682
|
+
for (const run of agentRuns || []) {
|
|
683
|
+
const signals = parseStructuredSignalsFromLog(run.logPath);
|
|
684
|
+
if (signals?.deployment && signals.deployment.state !== "healthy") {
|
|
685
|
+
deployRiskEntries.push(
|
|
686
|
+
summarizeGap(
|
|
687
|
+
run.agent.agentId,
|
|
688
|
+
`Deployment ${signals.deployment.service} ended in state ${signals.deployment.state}${signals.deployment.detail ? ` (${signals.deployment.detail})` : ""}.`,
|
|
689
|
+
"Deployment did not finish healthy.",
|
|
690
|
+
),
|
|
691
|
+
);
|
|
692
|
+
}
|
|
693
|
+
if (
|
|
694
|
+
signals?.infra &&
|
|
695
|
+
!["conformant", "action-complete"].includes(
|
|
696
|
+
String(signals.infra.state || "").trim().toLowerCase(),
|
|
697
|
+
)
|
|
698
|
+
) {
|
|
699
|
+
deployRiskEntries.push(
|
|
700
|
+
summarizeGap(
|
|
701
|
+
run.agent.agentId,
|
|
702
|
+
`Infra ${signals.infra.kind || "unknown"} on ${signals.infra.target || "unknown"} ended in state ${signals.infra.state || "unknown"}${signals.infra.detail ? ` (${signals.infra.detail})` : ""}.`,
|
|
703
|
+
"Infra risk remains open.",
|
|
704
|
+
),
|
|
705
|
+
);
|
|
706
|
+
}
|
|
707
|
+
}
|
|
708
|
+
|
|
709
|
+
const inboundDependencies = (dependencySnapshot?.openInbound || []).map(
|
|
710
|
+
(record) =>
|
|
711
|
+
`${record.id}: ${compactSingleLine(record.summary || record.detail || "inbound dependency", 180)}${record.assignedAgentId ? ` -> ${record.assignedAgentId}` : ""}`,
|
|
712
|
+
);
|
|
713
|
+
const outboundDependencies = (dependencySnapshot?.openOutbound || []).map(
|
|
714
|
+
(record) =>
|
|
715
|
+
`${record.id}: ${compactSingleLine(record.summary || record.detail || "outbound dependency", 180)}`,
|
|
716
|
+
);
|
|
717
|
+
const helperAssignments = (capabilityAssignments || [])
|
|
718
|
+
.filter((assignment) => assignment.blocking)
|
|
719
|
+
.map(
|
|
720
|
+
(assignment) =>
|
|
721
|
+
`${assignment.requestId}: ${assignment.target}${assignment.assignedAgentId ? ` -> ${assignment.assignedAgentId}` : " -> unresolved"} (${assignment.assignmentReason || "n/a"})`,
|
|
722
|
+
);
|
|
723
|
+
|
|
724
|
+
return {
|
|
725
|
+
openClaims: uniqueStringEntries(openClaims),
|
|
726
|
+
conflictingClaims: uniqueStringEntries(conflictingClaims),
|
|
727
|
+
unresolvedBlockers: uniqueStringEntries(unresolvedBlockers),
|
|
728
|
+
changedInterfaces: uniqueStringEntries(changedInterfaces),
|
|
729
|
+
crossComponentImpacts: uniqueStringEntries(crossComponentImpacts),
|
|
730
|
+
proofGaps: uniqueStringEntries(proofGapEntries),
|
|
731
|
+
docGaps: uniqueStringEntries(docGapEntries),
|
|
732
|
+
deployRisks: uniqueStringEntries(deployRiskEntries),
|
|
733
|
+
inboundDependencies: uniqueStringEntries(inboundDependencies),
|
|
734
|
+
outboundDependencies: uniqueStringEntries(outboundDependencies),
|
|
735
|
+
helperAssignments: uniqueStringEntries(helperAssignments),
|
|
736
|
+
};
|
|
737
|
+
}
|
|
738
|
+
|
|
739
|
+
export function buildWaveIntegrationSummary({
|
|
740
|
+
lanePaths,
|
|
741
|
+
wave,
|
|
742
|
+
attempt,
|
|
743
|
+
coordinationState,
|
|
744
|
+
summariesByAgentId,
|
|
745
|
+
docsQueue,
|
|
746
|
+
runtimeAssignments,
|
|
747
|
+
agentRuns,
|
|
748
|
+
capabilityAssignments = [],
|
|
749
|
+
dependencySnapshot = null,
|
|
750
|
+
}) {
|
|
751
|
+
const explicitIntegration = summariesByAgentId[lanePaths.integrationAgentId]?.integration || null;
|
|
752
|
+
const evidence = buildIntegrationEvidence({
|
|
753
|
+
lanePaths,
|
|
754
|
+
wave,
|
|
755
|
+
coordinationState,
|
|
756
|
+
summariesByAgentId,
|
|
757
|
+
docsQueue,
|
|
758
|
+
agentRuns,
|
|
759
|
+
capabilityAssignments,
|
|
760
|
+
dependencySnapshot,
|
|
761
|
+
});
|
|
762
|
+
if (explicitIntegration) {
|
|
763
|
+
return {
|
|
764
|
+
wave: wave.wave,
|
|
765
|
+
lane: lanePaths.lane,
|
|
766
|
+
agentId: lanePaths.integrationAgentId,
|
|
767
|
+
attempt,
|
|
768
|
+
openClaims: padReportedEntries(
|
|
769
|
+
evidence.openClaims,
|
|
770
|
+
explicitIntegration.claims || 0,
|
|
771
|
+
"Integration steward reported unresolved claim",
|
|
772
|
+
),
|
|
773
|
+
conflictingClaims: padReportedEntries(
|
|
774
|
+
evidence.conflictingClaims,
|
|
775
|
+
explicitIntegration.conflicts || 0,
|
|
776
|
+
"Integration steward reported unresolved conflict",
|
|
777
|
+
),
|
|
778
|
+
unresolvedBlockers: padReportedEntries(
|
|
779
|
+
evidence.unresolvedBlockers,
|
|
780
|
+
explicitIntegration.blockers || 0,
|
|
781
|
+
"Integration steward reported unresolved blocker",
|
|
782
|
+
),
|
|
783
|
+
changedInterfaces: evidence.changedInterfaces,
|
|
784
|
+
crossComponentImpacts: evidence.crossComponentImpacts,
|
|
785
|
+
proofGaps: evidence.proofGaps,
|
|
786
|
+
docGaps: evidence.docGaps,
|
|
787
|
+
deployRisks: evidence.deployRisks,
|
|
788
|
+
inboundDependencies: evidence.inboundDependencies,
|
|
789
|
+
outboundDependencies: evidence.outboundDependencies,
|
|
790
|
+
helperAssignments: evidence.helperAssignments,
|
|
791
|
+
runtimeAssignments,
|
|
792
|
+
recommendation: explicitIntegration.state,
|
|
793
|
+
detail: explicitIntegration.detail || "",
|
|
794
|
+
createdAt: toIsoTimestamp(),
|
|
795
|
+
updatedAt: toIsoTimestamp(),
|
|
796
|
+
};
|
|
797
|
+
}
|
|
798
|
+
const inferred = inferIntegrationRecommendation(evidence);
|
|
799
|
+
return {
|
|
800
|
+
wave: wave.wave,
|
|
801
|
+
lane: lanePaths.lane,
|
|
802
|
+
agentId: "launcher",
|
|
803
|
+
attempt,
|
|
804
|
+
...evidence,
|
|
805
|
+
runtimeAssignments,
|
|
806
|
+
recommendation: inferred.recommendation,
|
|
807
|
+
detail: inferred.detail,
|
|
808
|
+
createdAt: toIsoTimestamp(),
|
|
809
|
+
updatedAt: toIsoTimestamp(),
|
|
810
|
+
};
|
|
811
|
+
}
|
|
812
|
+
|
|
813
|
+
function renderIntegrationSection(title, items) {
|
|
814
|
+
return [
|
|
815
|
+
title,
|
|
816
|
+
...((items || []).length > 0 ? items.map((item) => `- ${item}`) : ["- None."]),
|
|
817
|
+
"",
|
|
818
|
+
];
|
|
819
|
+
}
|
|
820
|
+
|
|
821
|
+
function renderIntegrationSummaryMarkdown(integrationSummary) {
|
|
822
|
+
return [
|
|
823
|
+
`# Wave ${integrationSummary.wave} Integration Summary`,
|
|
824
|
+
"",
|
|
825
|
+
`- Recommendation: ${integrationSummary.recommendation || "unknown"}`,
|
|
826
|
+
`- Detail: ${integrationSummary.detail || "n/a"}`,
|
|
827
|
+
`- Open claims: ${(integrationSummary.openClaims || []).length}`,
|
|
828
|
+
`- Conflicting claims: ${(integrationSummary.conflictingClaims || []).length}`,
|
|
829
|
+
`- Unresolved blockers: ${(integrationSummary.unresolvedBlockers || []).length}`,
|
|
830
|
+
`- Changed interfaces: ${(integrationSummary.changedInterfaces || []).length}`,
|
|
831
|
+
`- Cross-component impacts: ${(integrationSummary.crossComponentImpacts || []).length}`,
|
|
832
|
+
`- Proof gaps: ${(integrationSummary.proofGaps || []).length}`,
|
|
833
|
+
`- Deploy risks: ${(integrationSummary.deployRisks || []).length}`,
|
|
834
|
+
`- Documentation gaps: ${(integrationSummary.docGaps || []).length}`,
|
|
835
|
+
`- Inbound dependencies: ${(integrationSummary.inboundDependencies || []).length}`,
|
|
836
|
+
`- Outbound dependencies: ${(integrationSummary.outboundDependencies || []).length}`,
|
|
837
|
+
`- Helper assignments: ${(integrationSummary.helperAssignments || []).length}`,
|
|
838
|
+
"",
|
|
839
|
+
...renderIntegrationSection("## Open Claims", integrationSummary.openClaims),
|
|
840
|
+
...renderIntegrationSection("## Conflicting Claims", integrationSummary.conflictingClaims),
|
|
841
|
+
...renderIntegrationSection("## Unresolved Blockers", integrationSummary.unresolvedBlockers),
|
|
842
|
+
...renderIntegrationSection("## Changed Interfaces", integrationSummary.changedInterfaces),
|
|
843
|
+
...renderIntegrationSection(
|
|
844
|
+
"## Cross-Component Impacts",
|
|
845
|
+
integrationSummary.crossComponentImpacts,
|
|
846
|
+
),
|
|
847
|
+
...renderIntegrationSection("## Proof Gaps", integrationSummary.proofGaps),
|
|
848
|
+
...renderIntegrationSection("## Deploy Risks", integrationSummary.deployRisks),
|
|
849
|
+
...renderIntegrationSection("## Inbound Dependencies", integrationSummary.inboundDependencies),
|
|
850
|
+
...renderIntegrationSection("## Outbound Dependencies", integrationSummary.outboundDependencies),
|
|
851
|
+
...renderIntegrationSection("## Helper Assignments", integrationSummary.helperAssignments),
|
|
852
|
+
"## Runtime Assignments",
|
|
853
|
+
...((integrationSummary.runtimeAssignments || []).length > 0
|
|
854
|
+
? integrationSummary.runtimeAssignments.map(
|
|
855
|
+
(assignment) =>
|
|
856
|
+
`- ${assignment.agentId}: executor=${assignment.executorId || "n/a"} role=${assignment.role || "n/a"} profile=${assignment.profile || "none"} fallback_used=${assignment.fallbackUsed ? "yes" : "no"}`,
|
|
857
|
+
)
|
|
858
|
+
: ["- None."]),
|
|
859
|
+
"",
|
|
860
|
+
...renderIntegrationSection("## Documentation Gaps", integrationSummary.docGaps),
|
|
861
|
+
].join("\n");
|
|
862
|
+
}
|
|
863
|
+
|
|
864
|
+
function writeWaveDerivedState({
|
|
865
|
+
lanePaths,
|
|
866
|
+
wave,
|
|
867
|
+
agentRuns = [],
|
|
868
|
+
summariesByAgentId = {},
|
|
869
|
+
feedbackRequests = [],
|
|
870
|
+
attempt = 0,
|
|
871
|
+
orchestratorId = null,
|
|
872
|
+
}) {
|
|
873
|
+
const coordinationLogPath = waveCoordinationLogPath(lanePaths, wave.wave);
|
|
874
|
+
const existingDocsQueue = readDocsQueue(waveDocsQueuePath(lanePaths, wave.wave));
|
|
875
|
+
const existingIntegrationSummary = readJsonOrNull(waveIntegrationPath(lanePaths, wave.wave));
|
|
876
|
+
const existingLedger = readWaveLedger(waveLedgerPath(lanePaths, wave.wave));
|
|
877
|
+
updateSeedRecords(coordinationLogPath, {
|
|
878
|
+
lane: lanePaths.lane,
|
|
879
|
+
wave: wave.wave,
|
|
880
|
+
agents: wave.agents,
|
|
881
|
+
componentPromotions: wave.componentPromotions,
|
|
882
|
+
sharedPlanDocs: lanePaths.sharedPlanDocs,
|
|
883
|
+
evaluatorAgentId: lanePaths.evaluatorAgentId,
|
|
884
|
+
integrationAgentId: lanePaths.integrationAgentId,
|
|
885
|
+
documentationAgentId: lanePaths.documentationAgentId,
|
|
886
|
+
feedbackRequests,
|
|
887
|
+
});
|
|
888
|
+
let coordinationState = readMaterializedCoordinationState(coordinationLogPath);
|
|
889
|
+
const clarificationTriage = triageClarificationRequests({
|
|
890
|
+
lanePaths,
|
|
891
|
+
wave,
|
|
892
|
+
coordinationLogPath,
|
|
893
|
+
coordinationState,
|
|
894
|
+
orchestratorId,
|
|
895
|
+
attempt,
|
|
896
|
+
resolutionContext: {
|
|
897
|
+
docsQueue: existingDocsQueue,
|
|
898
|
+
integrationSummary: existingIntegrationSummary,
|
|
899
|
+
ledger: existingLedger,
|
|
900
|
+
summariesByAgentId,
|
|
901
|
+
},
|
|
902
|
+
});
|
|
903
|
+
if (clarificationTriage.changed) {
|
|
904
|
+
coordinationState = readMaterializedCoordinationState(coordinationLogPath);
|
|
905
|
+
}
|
|
906
|
+
const capabilityAssignments = buildRequestAssignments({
|
|
907
|
+
coordinationState,
|
|
908
|
+
agents: wave.agents,
|
|
909
|
+
ledger: existingLedger,
|
|
910
|
+
capabilityRouting: lanePaths.capabilityRouting,
|
|
911
|
+
});
|
|
912
|
+
syncAssignmentRecords(coordinationLogPath, {
|
|
913
|
+
lane: lanePaths.lane,
|
|
914
|
+
wave: wave.wave,
|
|
915
|
+
assignments: capabilityAssignments,
|
|
916
|
+
});
|
|
917
|
+
coordinationState = readMaterializedCoordinationState(coordinationLogPath);
|
|
918
|
+
const dependencySnapshot = buildDependencySnapshot({
|
|
919
|
+
dirPath: lanePaths.crossLaneDependenciesDir,
|
|
920
|
+
lane: lanePaths.lane,
|
|
921
|
+
waveNumber: wave.wave,
|
|
922
|
+
agents: wave.agents,
|
|
923
|
+
ledger: existingLedger,
|
|
924
|
+
capabilityRouting: lanePaths.capabilityRouting,
|
|
925
|
+
});
|
|
926
|
+
writeJsonArtifact(waveAssignmentsPath(lanePaths, wave.wave), capabilityAssignments);
|
|
927
|
+
writeJsonArtifact(waveDependencySnapshotPath(lanePaths, wave.wave), dependencySnapshot);
|
|
928
|
+
writeDependencySnapshotMarkdown(
|
|
929
|
+
waveDependencySnapshotMarkdownPath(lanePaths, wave.wave),
|
|
930
|
+
dependencySnapshot,
|
|
931
|
+
);
|
|
932
|
+
const runtimeAssignments = wave.agents.map((agent) => ({
|
|
933
|
+
agentId: agent.agentId,
|
|
934
|
+
role: agent.executorResolved?.role || null,
|
|
935
|
+
initialExecutorId: agent.executorResolved?.initialExecutorId || null,
|
|
936
|
+
executorId: agent.executorResolved?.id || null,
|
|
937
|
+
profile: agent.executorResolved?.profile || null,
|
|
938
|
+
selectedBy: agent.executorResolved?.selectedBy || null,
|
|
939
|
+
fallbacks: agent.executorResolved?.fallbacks || [],
|
|
940
|
+
fallbackUsed: agent.executorResolved?.fallbackUsed === true,
|
|
941
|
+
fallbackReason: agent.executorResolved?.fallbackReason || null,
|
|
942
|
+
executorHistory: agent.executorResolved?.executorHistory || [],
|
|
943
|
+
}));
|
|
944
|
+
const docsQueue = buildDocsQueue({
|
|
945
|
+
lane: lanePaths.lane,
|
|
946
|
+
wave,
|
|
947
|
+
summariesByAgentId,
|
|
948
|
+
sharedPlanDocs: lanePaths.sharedPlanDocs,
|
|
949
|
+
componentPromotions: wave.componentPromotions,
|
|
950
|
+
runtimeAssignments,
|
|
951
|
+
});
|
|
952
|
+
writeDocsQueue(waveDocsQueuePath(lanePaths, wave.wave), docsQueue);
|
|
953
|
+
const integrationSummary = buildWaveIntegrationSummary({
|
|
954
|
+
lanePaths,
|
|
955
|
+
wave,
|
|
956
|
+
attempt,
|
|
957
|
+
coordinationState,
|
|
958
|
+
summariesByAgentId,
|
|
959
|
+
docsQueue,
|
|
960
|
+
runtimeAssignments,
|
|
961
|
+
agentRuns,
|
|
962
|
+
capabilityAssignments,
|
|
963
|
+
dependencySnapshot,
|
|
964
|
+
});
|
|
965
|
+
writeJsonArtifact(waveIntegrationPath(lanePaths, wave.wave), integrationSummary);
|
|
966
|
+
writeTextAtomic(
|
|
967
|
+
waveIntegrationMarkdownPath(lanePaths, wave.wave),
|
|
968
|
+
`${renderIntegrationSummaryMarkdown(integrationSummary)}\n`,
|
|
969
|
+
);
|
|
970
|
+
const ledger = deriveWaveLedger({
|
|
971
|
+
lane: lanePaths.lane,
|
|
972
|
+
wave,
|
|
973
|
+
summariesByAgentId,
|
|
974
|
+
coordinationState,
|
|
975
|
+
integrationSummary,
|
|
976
|
+
docsQueue,
|
|
977
|
+
attempt,
|
|
978
|
+
evaluatorAgentId: lanePaths.evaluatorAgentId,
|
|
979
|
+
integrationAgentId: lanePaths.integrationAgentId,
|
|
980
|
+
documentationAgentId: lanePaths.documentationAgentId,
|
|
981
|
+
capabilityAssignments,
|
|
982
|
+
dependencySnapshot,
|
|
983
|
+
});
|
|
984
|
+
writeWaveLedger(waveLedgerPath(lanePaths, wave.wave), ledger);
|
|
985
|
+
const inboxDir = waveInboxDir(lanePaths, wave.wave);
|
|
986
|
+
ensureDirectory(inboxDir);
|
|
987
|
+
const sharedSummary = compileSharedSummary({
|
|
988
|
+
wave,
|
|
989
|
+
state: coordinationState,
|
|
990
|
+
ledger,
|
|
991
|
+
integrationSummary,
|
|
992
|
+
capabilityAssignments,
|
|
993
|
+
dependencySnapshot,
|
|
994
|
+
});
|
|
995
|
+
const sharedSummaryPath = path.join(inboxDir, "shared-summary.md");
|
|
996
|
+
writeCompiledInbox(sharedSummaryPath, sharedSummary.text);
|
|
997
|
+
const inboxesByAgentId = {};
|
|
998
|
+
for (const agent of wave.agents) {
|
|
999
|
+
const inbox = compileAgentInbox({
|
|
1000
|
+
wave,
|
|
1001
|
+
agent,
|
|
1002
|
+
state: coordinationState,
|
|
1003
|
+
ledger,
|
|
1004
|
+
docsQueue,
|
|
1005
|
+
integrationSummary,
|
|
1006
|
+
capabilityAssignments,
|
|
1007
|
+
dependencySnapshot,
|
|
1008
|
+
});
|
|
1009
|
+
const inboxPath = path.join(inboxDir, `${agent.agentId}.md`);
|
|
1010
|
+
writeCompiledInbox(inboxPath, inbox.text);
|
|
1011
|
+
inboxesByAgentId[agent.agentId] = { path: inboxPath, text: inbox.text, truncated: inbox.truncated };
|
|
1012
|
+
}
|
|
1013
|
+
const boardText = renderCoordinationBoardProjection({
|
|
1014
|
+
wave: wave.wave,
|
|
1015
|
+
waveFile: wave.file,
|
|
1016
|
+
agents: wave.agents,
|
|
1017
|
+
state: coordinationState,
|
|
1018
|
+
capabilityAssignments,
|
|
1019
|
+
dependencySnapshot,
|
|
1020
|
+
});
|
|
1021
|
+
const messageBoardPath = path.join(lanePaths.messageboardsDir, `wave-${wave.wave}.md`);
|
|
1022
|
+
writeCoordinationBoardProjection(messageBoardPath, {
|
|
1023
|
+
wave: wave.wave,
|
|
1024
|
+
waveFile: wave.file,
|
|
1025
|
+
agents: wave.agents,
|
|
1026
|
+
state: coordinationState,
|
|
1027
|
+
capabilityAssignments,
|
|
1028
|
+
dependencySnapshot,
|
|
1029
|
+
});
|
|
1030
|
+
return {
|
|
1031
|
+
coordinationLogPath,
|
|
1032
|
+
coordinationState,
|
|
1033
|
+
clarificationTriage,
|
|
1034
|
+
docsQueue,
|
|
1035
|
+
capabilityAssignments,
|
|
1036
|
+
dependencySnapshot,
|
|
1037
|
+
integrationSummary,
|
|
1038
|
+
integrationMarkdownPath: waveIntegrationMarkdownPath(lanePaths, wave.wave),
|
|
1039
|
+
ledger,
|
|
1040
|
+
sharedSummaryPath,
|
|
1041
|
+
sharedSummaryText: sharedSummary.text,
|
|
1042
|
+
inboxesByAgentId,
|
|
1043
|
+
messageBoardPath,
|
|
1044
|
+
messageBoardText: boardText,
|
|
1045
|
+
};
|
|
1046
|
+
}
|
|
1047
|
+
|
|
1048
|
+
function applyDerivedStateToDashboard(dashboardState, derivedState) {
|
|
1049
|
+
if (!dashboardState || !derivedState) {
|
|
1050
|
+
return;
|
|
1051
|
+
}
|
|
1052
|
+
dashboardState.helperAssignmentsOpen = (derivedState.capabilityAssignments || []).filter(
|
|
1053
|
+
(assignment) => assignment.blocking,
|
|
1054
|
+
).length;
|
|
1055
|
+
dashboardState.inboundDependenciesOpen = (derivedState.dependencySnapshot?.openInbound || []).length;
|
|
1056
|
+
dashboardState.outboundDependenciesOpen = (derivedState.dependencySnapshot?.openOutbound || []).length;
|
|
1057
|
+
}
|
|
1058
|
+
|
|
1059
|
+
export function readWaveImplementationGate(wave, agentRuns) {
|
|
1060
|
+
const evaluatorAgentId = wave.evaluatorAgentId || "A0";
|
|
1061
|
+
const integrationAgentId = wave.integrationAgentId || "A8";
|
|
1062
|
+
const documentationAgentId = wave.documentationAgentId || "A9";
|
|
1063
|
+
for (const runInfo of agentRuns) {
|
|
1064
|
+
if ([evaluatorAgentId, integrationAgentId, documentationAgentId].includes(runInfo.agent.agentId)) {
|
|
1065
|
+
continue;
|
|
1066
|
+
}
|
|
1067
|
+
const summary = readRunExecutionSummary(runInfo);
|
|
1068
|
+
const validation = validateImplementationSummary(runInfo.agent, summary);
|
|
1069
|
+
if (!validation.ok) {
|
|
1070
|
+
return {
|
|
1071
|
+
ok: false,
|
|
1072
|
+
agentId: runInfo.agent.agentId,
|
|
1073
|
+
statusCode: validation.statusCode,
|
|
1074
|
+
detail: validation.detail,
|
|
1075
|
+
logPath: summary?.logPath || path.relative(REPO_ROOT, runInfo.logPath),
|
|
1076
|
+
};
|
|
1077
|
+
}
|
|
1078
|
+
}
|
|
1079
|
+
return {
|
|
1080
|
+
ok: true,
|
|
1081
|
+
agentId: null,
|
|
1082
|
+
statusCode: "pass",
|
|
1083
|
+
detail: "All implementation exit contracts are satisfied.",
|
|
1084
|
+
logPath: null,
|
|
1085
|
+
};
|
|
1086
|
+
}
|
|
1087
|
+
|
|
1088
|
+
export function readWaveComponentGate(wave, agentRuns, options = {}) {
|
|
1089
|
+
const summariesByAgentId = Object.fromEntries(
|
|
1090
|
+
agentRuns.map((runInfo) => [runInfo.agent.agentId, readRunExecutionSummary(runInfo)]),
|
|
1091
|
+
);
|
|
1092
|
+
const validation = validateWaveComponentPromotions(wave, summariesByAgentId, options);
|
|
1093
|
+
if (validation.ok) {
|
|
1094
|
+
return {
|
|
1095
|
+
ok: true,
|
|
1096
|
+
agentId: null,
|
|
1097
|
+
componentId: null,
|
|
1098
|
+
statusCode: validation.statusCode,
|
|
1099
|
+
detail: validation.detail,
|
|
1100
|
+
logPath: null,
|
|
1101
|
+
};
|
|
1102
|
+
}
|
|
1103
|
+
const ownerRun =
|
|
1104
|
+
agentRuns.find((runInfo) => runInfo.agent.components?.includes(validation.componentId)) ?? null;
|
|
1105
|
+
return {
|
|
1106
|
+
ok: false,
|
|
1107
|
+
agentId: ownerRun?.agent?.agentId || null,
|
|
1108
|
+
componentId: validation.componentId || null,
|
|
1109
|
+
statusCode: validation.statusCode,
|
|
1110
|
+
detail: validation.detail,
|
|
1111
|
+
logPath: ownerRun ? path.relative(REPO_ROOT, ownerRun.logPath) : null,
|
|
1112
|
+
};
|
|
1113
|
+
}
|
|
1114
|
+
|
|
1115
|
+
export function readWaveComponentMatrixGate(wave, agentRuns, options = {}) {
|
|
1116
|
+
const validation = validateWaveComponentMatrixCurrentLevels(wave, options);
|
|
1117
|
+
if (validation.ok) {
|
|
1118
|
+
return {
|
|
1119
|
+
ok: true,
|
|
1120
|
+
agentId: null,
|
|
1121
|
+
componentId: null,
|
|
1122
|
+
statusCode: validation.statusCode,
|
|
1123
|
+
detail: validation.detail,
|
|
1124
|
+
logPath: null,
|
|
1125
|
+
};
|
|
1126
|
+
}
|
|
1127
|
+
const documentationAgentId =
|
|
1128
|
+
options.documentationAgentId || wave.documentationAgentId || "A9";
|
|
1129
|
+
const docRun =
|
|
1130
|
+
agentRuns.find((runInfo) => runInfo.agent.agentId === documentationAgentId) ?? null;
|
|
1131
|
+
return {
|
|
1132
|
+
ok: false,
|
|
1133
|
+
agentId: docRun?.agent?.agentId || null,
|
|
1134
|
+
componentId: validation.componentId || null,
|
|
1135
|
+
statusCode: validation.statusCode,
|
|
1136
|
+
detail: validation.detail,
|
|
1137
|
+
logPath: docRun ? path.relative(REPO_ROOT, docRun.logPath) : null,
|
|
1138
|
+
};
|
|
1139
|
+
}
|
|
1140
|
+
|
|
1141
|
+
export function readWaveDocumentationGate(wave, agentRuns) {
|
|
1142
|
+
const documentationAgentId = wave.documentationAgentId || "A9";
|
|
1143
|
+
const docRun =
|
|
1144
|
+
agentRuns.find((run) => run.agent.agentId === documentationAgentId) ?? null;
|
|
1145
|
+
if (!docRun) {
|
|
1146
|
+
return {
|
|
1147
|
+
ok: true,
|
|
1148
|
+
agentId: null,
|
|
1149
|
+
statusCode: "pass",
|
|
1150
|
+
detail: "No documentation steward declared for this wave.",
|
|
1151
|
+
logPath: null,
|
|
1152
|
+
};
|
|
1153
|
+
}
|
|
1154
|
+
const summary = readRunExecutionSummary(docRun);
|
|
1155
|
+
const validation = validateDocumentationClosureSummary(docRun.agent, summary);
|
|
1156
|
+
return {
|
|
1157
|
+
ok: validation.ok,
|
|
1158
|
+
agentId: docRun.agent.agentId,
|
|
1159
|
+
statusCode: validation.statusCode,
|
|
1160
|
+
detail: validation.detail,
|
|
1161
|
+
logPath: summary?.logPath || path.relative(REPO_ROOT, docRun.logPath),
|
|
1162
|
+
};
|
|
1163
|
+
}
|
|
1164
|
+
|
|
1165
|
+
export function readWaveIntegrationGate(wave, agentRuns, options = {}) {
|
|
1166
|
+
const integrationAgentId =
|
|
1167
|
+
options.integrationAgentId || wave.integrationAgentId || "A8";
|
|
1168
|
+
const requireIntegration =
|
|
1169
|
+
options.requireIntegrationSteward === true ||
|
|
1170
|
+
(options.requireIntegrationStewardFromWave !== null &&
|
|
1171
|
+
options.requireIntegrationStewardFromWave !== undefined &&
|
|
1172
|
+
wave.wave >= options.requireIntegrationStewardFromWave);
|
|
1173
|
+
const integrationRun =
|
|
1174
|
+
agentRuns.find((run) => run.agent.agentId === integrationAgentId) ?? null;
|
|
1175
|
+
if (!integrationRun) {
|
|
1176
|
+
return {
|
|
1177
|
+
ok: !requireIntegration,
|
|
1178
|
+
agentId: requireIntegration ? integrationAgentId : null,
|
|
1179
|
+
statusCode: requireIntegration ? "missing-integration" : "pass",
|
|
1180
|
+
detail: requireIntegration
|
|
1181
|
+
? `Agent ${integrationAgentId} is missing.`
|
|
1182
|
+
: "No explicit integration steward declared for this wave.",
|
|
1183
|
+
logPath: null,
|
|
1184
|
+
};
|
|
1185
|
+
}
|
|
1186
|
+
const summary = readRunExecutionSummary(integrationRun);
|
|
1187
|
+
const validation = validateIntegrationSummary(integrationRun.agent, summary);
|
|
1188
|
+
return {
|
|
1189
|
+
ok: validation.ok,
|
|
1190
|
+
agentId: integrationRun.agent.agentId,
|
|
1191
|
+
statusCode: validation.statusCode,
|
|
1192
|
+
detail: validation.detail,
|
|
1193
|
+
logPath: summary?.logPath || path.relative(REPO_ROOT, integrationRun.logPath),
|
|
1194
|
+
};
|
|
1195
|
+
}
|
|
1196
|
+
|
|
1197
|
+
export function readWaveIntegrationBarrier(wave, agentRuns, derivedState, options = {}) {
|
|
1198
|
+
const markerGate = readWaveIntegrationGate(wave, agentRuns, options);
|
|
1199
|
+
if (!markerGate.ok) {
|
|
1200
|
+
return markerGate;
|
|
1201
|
+
}
|
|
1202
|
+
const integrationSummary = derivedState?.integrationSummary || null;
|
|
1203
|
+
if (!integrationSummary) {
|
|
1204
|
+
return {
|
|
1205
|
+
ok: false,
|
|
1206
|
+
agentId: markerGate.agentId,
|
|
1207
|
+
statusCode: "missing-integration-summary",
|
|
1208
|
+
detail: `Missing integration summary artifact for wave ${wave.wave}.`,
|
|
1209
|
+
logPath: markerGate.logPath,
|
|
1210
|
+
};
|
|
1211
|
+
}
|
|
1212
|
+
if (integrationSummary.recommendation !== "ready-for-doc-closure") {
|
|
1213
|
+
return {
|
|
1214
|
+
ok: false,
|
|
1215
|
+
agentId: markerGate.agentId,
|
|
1216
|
+
statusCode: "integration-needs-more-work",
|
|
1217
|
+
detail:
|
|
1218
|
+
integrationSummary.detail ||
|
|
1219
|
+
`Integration summary still reports ${integrationSummary.recommendation}.`,
|
|
1220
|
+
logPath: markerGate.logPath,
|
|
1221
|
+
};
|
|
1222
|
+
}
|
|
1223
|
+
return markerGate;
|
|
1224
|
+
}
|
|
1225
|
+
|
|
1226
|
+
function failureResultFromGate(gate, fallbackLogPath) {
|
|
1227
|
+
return {
|
|
1228
|
+
failures: [
|
|
1229
|
+
{
|
|
1230
|
+
agentId: gate.agentId,
|
|
1231
|
+
statusCode: gate.statusCode,
|
|
1232
|
+
logPath: gate.logPath || fallbackLogPath,
|
|
1233
|
+
detail: gate.detail,
|
|
1234
|
+
},
|
|
1235
|
+
],
|
|
1236
|
+
timedOut: false,
|
|
1237
|
+
};
|
|
1238
|
+
}
|
|
1239
|
+
|
|
1240
|
+
function recordClosureGateFailure({
|
|
1241
|
+
wave,
|
|
1242
|
+
lanePaths,
|
|
1243
|
+
gate,
|
|
1244
|
+
label,
|
|
1245
|
+
recordCombinedEvent,
|
|
1246
|
+
appendCoordination,
|
|
1247
|
+
actionRequested,
|
|
1248
|
+
}) {
|
|
1249
|
+
recordCombinedEvent({
|
|
1250
|
+
level: "error",
|
|
1251
|
+
agentId: gate.agentId,
|
|
1252
|
+
message: `${label} blocked wave ${wave.wave}: ${gate.detail}`,
|
|
1253
|
+
});
|
|
1254
|
+
appendCoordination({
|
|
1255
|
+
event: "wave_gate_blocked",
|
|
1256
|
+
waves: [wave.wave],
|
|
1257
|
+
status: "blocked",
|
|
1258
|
+
details: `agent=${gate.agentId}; reason=${gate.statusCode}; ${gate.detail}`,
|
|
1259
|
+
actionRequested:
|
|
1260
|
+
actionRequested ||
|
|
1261
|
+
`Lane ${lanePaths.lane} owners should resolve the ${label.toLowerCase()} before wave progression.`,
|
|
1262
|
+
});
|
|
1263
|
+
}
|
|
1264
|
+
|
|
1265
|
+
export async function runClosureSweepPhase({
|
|
1266
|
+
lanePaths,
|
|
1267
|
+
wave,
|
|
1268
|
+
closureRuns,
|
|
1269
|
+
coordinationLogPath,
|
|
1270
|
+
refreshDerivedState,
|
|
1271
|
+
dashboardState,
|
|
1272
|
+
recordCombinedEvent,
|
|
1273
|
+
flushDashboards,
|
|
1274
|
+
options,
|
|
1275
|
+
feedbackStateByRequestId,
|
|
1276
|
+
appendCoordination,
|
|
1277
|
+
launchAgentSessionFn = launchAgentSession,
|
|
1278
|
+
waitForWaveCompletionFn = waitForWaveCompletion,
|
|
1279
|
+
}) {
|
|
1280
|
+
const evaluatorAgentId = wave.evaluatorAgentId || "A0";
|
|
1281
|
+
const integrationAgentId = wave.integrationAgentId || lanePaths.integrationAgentId || "A8";
|
|
1282
|
+
const documentationAgentId = wave.documentationAgentId || "A9";
|
|
1283
|
+
const stagedRuns = [
|
|
1284
|
+
{
|
|
1285
|
+
agentId: integrationAgentId,
|
|
1286
|
+
label: "Integration gate",
|
|
1287
|
+
runs: closureRuns.filter((run) => run.agent.agentId === integrationAgentId),
|
|
1288
|
+
validate: () =>
|
|
1289
|
+
readWaveIntegrationBarrier(wave, closureRuns, refreshDerivedState?.(dashboardState?.attempt || 0), {
|
|
1290
|
+
integrationAgentId: lanePaths.integrationAgentId,
|
|
1291
|
+
requireIntegrationStewardFromWave: lanePaths.requireIntegrationStewardFromWave,
|
|
1292
|
+
}),
|
|
1293
|
+
actionRequested:
|
|
1294
|
+
`Lane ${lanePaths.lane} owners should resolve integration contradictions or blockers before doc/evaluator closure.`,
|
|
1295
|
+
},
|
|
1296
|
+
{
|
|
1297
|
+
agentId: documentationAgentId,
|
|
1298
|
+
label: "Documentation closure",
|
|
1299
|
+
runs: closureRuns.filter((run) => run.agent.agentId === documentationAgentId),
|
|
1300
|
+
validate: () => {
|
|
1301
|
+
const documentationGate = readWaveDocumentationGate(wave, closureRuns);
|
|
1302
|
+
if (!documentationGate.ok) {
|
|
1303
|
+
return documentationGate;
|
|
1304
|
+
}
|
|
1305
|
+
return readWaveComponentMatrixGate(wave, closureRuns, {
|
|
1306
|
+
laneProfile: lanePaths.laneProfile,
|
|
1307
|
+
documentationAgentId: lanePaths.documentationAgentId,
|
|
1308
|
+
});
|
|
1309
|
+
},
|
|
1310
|
+
actionRequested:
|
|
1311
|
+
`Lane ${lanePaths.lane} owners should resolve the shared-plan or component-matrix closure state before evaluator progression.`,
|
|
1312
|
+
},
|
|
1313
|
+
{
|
|
1314
|
+
agentId: evaluatorAgentId,
|
|
1315
|
+
label: "Evaluator gate",
|
|
1316
|
+
runs: closureRuns.filter((run) => run.agent.agentId === evaluatorAgentId),
|
|
1317
|
+
validate: () => readWaveEvaluatorGate(wave, closureRuns),
|
|
1318
|
+
actionRequested:
|
|
1319
|
+
`Lane ${lanePaths.lane} owners should resolve the evaluator gate before wave progression.`,
|
|
1320
|
+
},
|
|
1321
|
+
];
|
|
1322
|
+
for (const stage of stagedRuns) {
|
|
1323
|
+
if (stage.runs.length === 0) {
|
|
1324
|
+
continue;
|
|
1325
|
+
}
|
|
1326
|
+
const runInfo = stage.runs[0];
|
|
1327
|
+
const existing = dashboardState.agents.find((entry) => entry.agentId === runInfo.agent.agentId);
|
|
1328
|
+
setWaveDashboardAgent(dashboardState, runInfo.agent.agentId, {
|
|
1329
|
+
state: "launching",
|
|
1330
|
+
attempts: (existing?.attempts || 0) + 1,
|
|
1331
|
+
startedAt: existing?.startedAt || toIsoTimestamp(),
|
|
1332
|
+
completedAt: null,
|
|
1333
|
+
exitCode: null,
|
|
1334
|
+
detail: "Launching closure sweep",
|
|
1335
|
+
});
|
|
1336
|
+
flushDashboards();
|
|
1337
|
+
const launchResult = await launchAgentSessionFn(lanePaths, {
|
|
1338
|
+
wave: wave.wave,
|
|
1339
|
+
agent: runInfo.agent,
|
|
1340
|
+
sessionName: runInfo.sessionName,
|
|
1341
|
+
promptPath: runInfo.promptPath,
|
|
1342
|
+
logPath: runInfo.logPath,
|
|
1343
|
+
statusPath: runInfo.statusPath,
|
|
1344
|
+
messageBoardPath: runInfo.messageBoardPath,
|
|
1345
|
+
messageBoardSnapshot: runInfo.messageBoardSnapshot || "",
|
|
1346
|
+
sharedSummaryPath: runInfo.sharedSummaryPath,
|
|
1347
|
+
sharedSummaryText: runInfo.sharedSummaryText,
|
|
1348
|
+
inboxPath: runInfo.inboxPath,
|
|
1349
|
+
inboxText: runInfo.inboxText,
|
|
1350
|
+
orchestratorId: options.orchestratorId,
|
|
1351
|
+
executorMode: options.executorMode,
|
|
1352
|
+
codexSandboxMode: options.codexSandboxMode,
|
|
1353
|
+
agentRateLimitRetries: options.agentRateLimitRetries,
|
|
1354
|
+
agentRateLimitBaseDelaySeconds: options.agentRateLimitBaseDelaySeconds,
|
|
1355
|
+
agentRateLimitMaxDelaySeconds: options.agentRateLimitMaxDelaySeconds,
|
|
1356
|
+
context7Enabled: options.context7Enabled,
|
|
1357
|
+
});
|
|
1358
|
+
runInfo.lastLaunchAttempt = dashboardState?.attempt || null;
|
|
1359
|
+
runInfo.lastPromptHash = launchResult?.promptHash || null;
|
|
1360
|
+
runInfo.lastContext7 = launchResult?.context7 || null;
|
|
1361
|
+
runInfo.lastExecutorId = launchResult?.executorId || runInfo.agent.executorResolved?.id || null;
|
|
1362
|
+
setWaveDashboardAgent(dashboardState, runInfo.agent.agentId, {
|
|
1363
|
+
state: "running",
|
|
1364
|
+
detail: `Closure sweep launched${launchResult?.context7?.mode ? ` (${launchResult.context7.mode})` : ""}`,
|
|
1365
|
+
});
|
|
1366
|
+
recordCombinedEvent({
|
|
1367
|
+
agentId: runInfo.agent.agentId,
|
|
1368
|
+
message: `Closure sweep launched in tmux session ${runInfo.sessionName}`,
|
|
1369
|
+
});
|
|
1370
|
+
flushDashboards();
|
|
1371
|
+
const result = await waitForWaveCompletionFn(
|
|
1372
|
+
lanePaths,
|
|
1373
|
+
[runInfo],
|
|
1374
|
+
options.timeoutMinutes,
|
|
1375
|
+
({ pendingAgentIds }) => {
|
|
1376
|
+
refreshWaveDashboardAgentStates(dashboardState, [runInfo], pendingAgentIds, (event) =>
|
|
1377
|
+
recordCombinedEvent(event),
|
|
1378
|
+
);
|
|
1379
|
+
monitorWaveHumanFeedback({
|
|
1380
|
+
lanePaths,
|
|
1381
|
+
waveNumber: wave.wave,
|
|
1382
|
+
agentRuns: [runInfo],
|
|
1383
|
+
orchestratorId: options.orchestratorId,
|
|
1384
|
+
coordinationLogPath,
|
|
1385
|
+
feedbackStateByRequestId,
|
|
1386
|
+
recordCombinedEvent,
|
|
1387
|
+
appendCoordination,
|
|
1388
|
+
});
|
|
1389
|
+
updateWaveDashboardMessageBoard(dashboardState, runInfo.messageBoardPath);
|
|
1390
|
+
flushDashboards();
|
|
1391
|
+
},
|
|
1392
|
+
);
|
|
1393
|
+
materializeAgentExecutionSummaryForRun(wave, runInfo);
|
|
1394
|
+
refreshDerivedState?.(dashboardState?.attempt || 0);
|
|
1395
|
+
if (result.failures.length > 0) {
|
|
1396
|
+
return result;
|
|
1397
|
+
}
|
|
1398
|
+
const gate = stage.validate();
|
|
1399
|
+
if (!gate.ok) {
|
|
1400
|
+
recordClosureGateFailure({
|
|
1401
|
+
wave,
|
|
1402
|
+
lanePaths,
|
|
1403
|
+
gate,
|
|
1404
|
+
label: stage.label,
|
|
1405
|
+
recordCombinedEvent,
|
|
1406
|
+
appendCoordination,
|
|
1407
|
+
actionRequested: stage.actionRequested,
|
|
1408
|
+
});
|
|
1409
|
+
return failureResultFromGate(gate, path.relative(REPO_ROOT, runInfo.logPath));
|
|
1410
|
+
}
|
|
1411
|
+
}
|
|
1412
|
+
return { failures: [], timedOut: false };
|
|
1413
|
+
}
|
|
1414
|
+
|
|
1415
|
+
const NON_BLOCKING_INFRA_SIGNAL_STATES = new Set([
|
|
1416
|
+
"conformant",
|
|
1417
|
+
"setup-required",
|
|
1418
|
+
"setup-in-progress",
|
|
1419
|
+
"action-required",
|
|
1420
|
+
"action-approved",
|
|
1421
|
+
"action-complete",
|
|
1422
|
+
]);
|
|
1423
|
+
|
|
1424
|
+
export function readWaveInfraGate(agentRuns) {
|
|
1425
|
+
for (const run of agentRuns) {
|
|
1426
|
+
const signals = parseStructuredSignalsFromLog(run.logPath);
|
|
1427
|
+
if (!signals?.infra) {
|
|
1428
|
+
continue;
|
|
1429
|
+
}
|
|
1430
|
+
const infra = signals.infra;
|
|
1431
|
+
const normalizedState = String(infra.state || "")
|
|
1432
|
+
.trim()
|
|
1433
|
+
.toLowerCase();
|
|
1434
|
+
if (NON_BLOCKING_INFRA_SIGNAL_STATES.has(normalizedState)) {
|
|
1435
|
+
continue;
|
|
1436
|
+
}
|
|
1437
|
+
return {
|
|
1438
|
+
ok: false,
|
|
1439
|
+
agentId: run.agent.agentId,
|
|
1440
|
+
statusCode: `infra-${normalizedState || "blocked"}`,
|
|
1441
|
+
detail: `Infra signal ${infra.kind || "unknown"} on ${infra.target || "unknown"} ended in state ${normalizedState || "unknown"}${infra.detail ? ` (${infra.detail})` : ""}.`,
|
|
1442
|
+
logPath: path.relative(REPO_ROOT, run.logPath),
|
|
1443
|
+
};
|
|
1444
|
+
}
|
|
1445
|
+
return {
|
|
1446
|
+
ok: true,
|
|
1447
|
+
agentId: null,
|
|
1448
|
+
statusCode: "pass",
|
|
1449
|
+
detail: "",
|
|
1450
|
+
logPath: null,
|
|
1451
|
+
};
|
|
1452
|
+
}
|
|
1453
|
+
|
|
1454
|
+
export function markLauncherFailed(
|
|
1455
|
+
globalDashboard,
|
|
1456
|
+
lanePaths,
|
|
1457
|
+
selectedWaves,
|
|
1458
|
+
appendCoordination,
|
|
1459
|
+
error,
|
|
1460
|
+
) {
|
|
1461
|
+
if (globalDashboard) {
|
|
1462
|
+
globalDashboard.status = "failed";
|
|
1463
|
+
recordGlobalDashboardEvent(globalDashboard, {
|
|
1464
|
+
level: "error",
|
|
1465
|
+
message: error instanceof Error ? error.message : String(error),
|
|
1466
|
+
});
|
|
1467
|
+
writeGlobalDashboard(lanePaths.globalDashboardPath, globalDashboard);
|
|
1468
|
+
}
|
|
1469
|
+
appendCoordination({
|
|
1470
|
+
event: "launcher_finish",
|
|
1471
|
+
waves: selectedWaves,
|
|
1472
|
+
status: "failed",
|
|
1473
|
+
details: error instanceof Error ? error.message : String(error),
|
|
1474
|
+
actionRequested: `Lane ${lanePaths.lane} owners should inspect the failing wave logs and dashboards before retrying.`,
|
|
1475
|
+
});
|
|
1476
|
+
}
|
|
1477
|
+
|
|
1478
|
+
export function acquireLauncherLock(lockPath, options) {
|
|
1479
|
+
ensureDirectory(path.dirname(lockPath));
|
|
1480
|
+
const payload = {
|
|
1481
|
+
lane: options.lane,
|
|
1482
|
+
pid: process.pid,
|
|
1483
|
+
startedAt: toIsoTimestamp(),
|
|
1484
|
+
argv: process.argv.slice(2),
|
|
1485
|
+
cwd: REPO_ROOT,
|
|
1486
|
+
mode: options.reconcileStatus ? "reconcile" : "launch",
|
|
1487
|
+
};
|
|
1488
|
+
try {
|
|
1489
|
+
const fd = fs.openSync(lockPath, "wx");
|
|
1490
|
+
fs.writeFileSync(fd, `${JSON.stringify(payload, null, 2)}\n`, "utf8");
|
|
1491
|
+
fs.closeSync(fd);
|
|
1492
|
+
return payload;
|
|
1493
|
+
} catch (error) {
|
|
1494
|
+
if (error?.code !== "EEXIST") {
|
|
1495
|
+
throw error;
|
|
1496
|
+
}
|
|
1497
|
+
const existing = readJsonOrNull(lockPath);
|
|
1498
|
+
const existingPid = Number.parseInt(String(existing?.pid ?? ""), 10);
|
|
1499
|
+
if (isProcessAlive(existingPid)) {
|
|
1500
|
+
const lockError = new Error(
|
|
1501
|
+
`Another launcher is active (pid ${existingPid}, started ${existing?.startedAt || "unknown"}). Lock: ${path.relative(REPO_ROOT, lockPath)}`,
|
|
1502
|
+
{ cause: error },
|
|
1503
|
+
);
|
|
1504
|
+
lockError.exitCode = 32;
|
|
1505
|
+
throw lockError;
|
|
1506
|
+
}
|
|
1507
|
+
fs.rmSync(lockPath, { force: true });
|
|
1508
|
+
const retryFd = fs.openSync(lockPath, "wx");
|
|
1509
|
+
fs.writeFileSync(retryFd, `${JSON.stringify(payload, null, 2)}\n`, "utf8");
|
|
1510
|
+
fs.closeSync(retryFd);
|
|
1511
|
+
return payload;
|
|
1512
|
+
}
|
|
1513
|
+
}
|
|
1514
|
+
|
|
1515
|
+
export function releaseLauncherLock(lockPath) {
|
|
1516
|
+
fs.rmSync(lockPath, { force: true });
|
|
1517
|
+
}
|
|
1518
|
+
|
|
1519
|
+
function isLaneSessionName(lanePaths, sessionName) {
|
|
1520
|
+
return (
|
|
1521
|
+
sessionName.startsWith(lanePaths.tmuxSessionPrefix) ||
|
|
1522
|
+
sessionName.startsWith(lanePaths.tmuxDashboardSessionPrefix) ||
|
|
1523
|
+
sessionName.startsWith(lanePaths.tmuxGlobalDashboardSessionPrefix)
|
|
1524
|
+
);
|
|
1525
|
+
}
|
|
1526
|
+
|
|
1527
|
+
function listLaneTmuxSessionNames(lanePaths) {
|
|
1528
|
+
return listTmuxSessionNames(lanePaths).filter((sessionName) =>
|
|
1529
|
+
isLaneSessionName(lanePaths, sessionName),
|
|
1530
|
+
);
|
|
1531
|
+
}
|
|
1532
|
+
|
|
1533
|
+
function isWaveDashboardBackedByLiveSession(lanePaths, dashboardPath, activeSessionNames) {
|
|
1534
|
+
const waveMatch = path.basename(dashboardPath).match(/^wave-(\d+)\.json$/);
|
|
1535
|
+
if (!waveMatch) {
|
|
1536
|
+
return false;
|
|
1537
|
+
}
|
|
1538
|
+
const waveNumber = Number.parseInt(waveMatch[1], 10);
|
|
1539
|
+
if (!Number.isFinite(waveNumber)) {
|
|
1540
|
+
return false;
|
|
1541
|
+
}
|
|
1542
|
+
const dashboardState = readJsonOrNull(dashboardPath);
|
|
1543
|
+
const runTag = String(dashboardState?.runTag || "").trim();
|
|
1544
|
+
const agentPrefix = `${lanePaths.tmuxSessionPrefix}${waveNumber}_`;
|
|
1545
|
+
const dashboardPrefix = `${lanePaths.tmuxDashboardSessionPrefix}${waveNumber}_`;
|
|
1546
|
+
for (const sessionName of activeSessionNames) {
|
|
1547
|
+
if (!(sessionName.startsWith(agentPrefix) || sessionName.startsWith(dashboardPrefix))) {
|
|
1548
|
+
continue;
|
|
1549
|
+
}
|
|
1550
|
+
if (!runTag || sessionName.endsWith(`_${runTag}`)) {
|
|
1551
|
+
return true;
|
|
1552
|
+
}
|
|
1553
|
+
}
|
|
1554
|
+
return false;
|
|
1555
|
+
}
|
|
1556
|
+
|
|
1557
|
+
function removeOrphanWaveDashboards(lanePaths, activeSessionNames) {
|
|
1558
|
+
if (!fs.existsSync(lanePaths.dashboardsDir)) {
|
|
1559
|
+
return [];
|
|
1560
|
+
}
|
|
1561
|
+
const removedDashboardPaths = [];
|
|
1562
|
+
for (const fileName of fs.readdirSync(lanePaths.dashboardsDir)) {
|
|
1563
|
+
if (!/^wave-\d+\.json$/.test(fileName)) {
|
|
1564
|
+
continue;
|
|
1565
|
+
}
|
|
1566
|
+
const dashboardPath = path.join(lanePaths.dashboardsDir, fileName);
|
|
1567
|
+
if (isWaveDashboardBackedByLiveSession(lanePaths, dashboardPath, activeSessionNames)) {
|
|
1568
|
+
continue;
|
|
1569
|
+
}
|
|
1570
|
+
fs.rmSync(dashboardPath, { force: true });
|
|
1571
|
+
removedDashboardPaths.push(path.relative(REPO_ROOT, dashboardPath));
|
|
1572
|
+
}
|
|
1573
|
+
return removedDashboardPaths;
|
|
1574
|
+
}
|
|
1575
|
+
|
|
1576
|
+
export function reconcileStaleLauncherArtifacts(lanePaths) {
|
|
1577
|
+
const outcome = {
|
|
1578
|
+
removedLock: false,
|
|
1579
|
+
removedSessions: [],
|
|
1580
|
+
removedTerminalNames: [],
|
|
1581
|
+
clearedDashboards: false,
|
|
1582
|
+
removedDashboardPaths: [],
|
|
1583
|
+
staleWaves: [],
|
|
1584
|
+
activeLockPid: null,
|
|
1585
|
+
};
|
|
1586
|
+
|
|
1587
|
+
if (fs.existsSync(lanePaths.launcherLockPath)) {
|
|
1588
|
+
const existing = readJsonOrNull(lanePaths.launcherLockPath);
|
|
1589
|
+
const existingPid = Number.parseInt(String(existing?.pid ?? ""), 10);
|
|
1590
|
+
if (isProcessAlive(existingPid)) {
|
|
1591
|
+
outcome.activeLockPid = existingPid;
|
|
1592
|
+
return outcome;
|
|
1593
|
+
}
|
|
1594
|
+
fs.rmSync(lanePaths.launcherLockPath, { force: true });
|
|
1595
|
+
outcome.removedLock = true;
|
|
1596
|
+
}
|
|
1597
|
+
|
|
1598
|
+
outcome.removedSessions = cleanupLaneTmuxSessions(lanePaths);
|
|
1599
|
+
const activeSessionNames = new Set(listLaneTmuxSessionNames(lanePaths));
|
|
1600
|
+
const terminalCleanup = pruneOrphanLaneTemporaryTerminalEntries(
|
|
1601
|
+
lanePaths.terminalsPath,
|
|
1602
|
+
lanePaths,
|
|
1603
|
+
activeSessionNames,
|
|
1604
|
+
);
|
|
1605
|
+
outcome.removedTerminalNames = terminalCleanup.removedNames;
|
|
1606
|
+
|
|
1607
|
+
const globalDashboard = readJsonOrNull(lanePaths.globalDashboardPath);
|
|
1608
|
+
if (globalDashboard && typeof globalDashboard === "object" && Array.isArray(globalDashboard.waves)) {
|
|
1609
|
+
const staleWaves = new Set();
|
|
1610
|
+
for (const waveEntry of globalDashboard.waves) {
|
|
1611
|
+
const waveNumber = Number.parseInt(String(waveEntry?.wave ?? ""), 10);
|
|
1612
|
+
if (Number.isFinite(waveNumber)) {
|
|
1613
|
+
staleWaves.add(waveNumber);
|
|
1614
|
+
}
|
|
1615
|
+
}
|
|
1616
|
+
outcome.staleWaves = Array.from(staleWaves).toSorted((a, b) => a - b);
|
|
1617
|
+
}
|
|
1618
|
+
|
|
1619
|
+
if (fs.existsSync(lanePaths.globalDashboardPath)) {
|
|
1620
|
+
fs.rmSync(lanePaths.globalDashboardPath, { force: true });
|
|
1621
|
+
outcome.removedDashboardPaths.push(path.relative(REPO_ROOT, lanePaths.globalDashboardPath));
|
|
1622
|
+
}
|
|
1623
|
+
outcome.removedDashboardPaths.push(
|
|
1624
|
+
...removeOrphanWaveDashboards(lanePaths, activeSessionNames),
|
|
1625
|
+
);
|
|
1626
|
+
outcome.clearedDashboards = outcome.removedDashboardPaths.length > 0;
|
|
1627
|
+
return outcome;
|
|
1628
|
+
}
|
|
1629
|
+
|
|
1630
|
+
function runTmux(lanePaths, args, description) {
|
|
1631
|
+
const result = spawnSync("tmux", ["-L", lanePaths.tmuxSocketName, ...args], {
|
|
1632
|
+
cwd: REPO_ROOT,
|
|
1633
|
+
encoding: "utf8",
|
|
1634
|
+
env: { ...process.env, TMUX: "" },
|
|
1635
|
+
timeout: TMUX_COMMAND_TIMEOUT_MS,
|
|
1636
|
+
});
|
|
1637
|
+
if (result.error) {
|
|
1638
|
+
if (result.error.code === "ETIMEDOUT") {
|
|
1639
|
+
throw new Error(
|
|
1640
|
+
`${description} failed: tmux command timed out after ${TMUX_COMMAND_TIMEOUT_MS}ms`,
|
|
1641
|
+
);
|
|
1642
|
+
}
|
|
1643
|
+
throw new Error(`${description} failed: ${result.error.message}`);
|
|
1644
|
+
}
|
|
1645
|
+
if (result.status !== 0) {
|
|
1646
|
+
throw new Error(
|
|
1647
|
+
`${description} failed: ${(result.stderr || "").trim() || "tmux command failed"}`,
|
|
1648
|
+
);
|
|
1649
|
+
}
|
|
1650
|
+
}
|
|
1651
|
+
|
|
1652
|
+
function listTmuxSessionNames(lanePaths) {
|
|
1653
|
+
const result = spawnSync(
|
|
1654
|
+
"tmux",
|
|
1655
|
+
["-L", lanePaths.tmuxSocketName, "list-sessions", "-F", "#{session_name}"],
|
|
1656
|
+
{
|
|
1657
|
+
cwd: REPO_ROOT,
|
|
1658
|
+
encoding: "utf8",
|
|
1659
|
+
env: { ...process.env, TMUX: "" },
|
|
1660
|
+
timeout: TMUX_COMMAND_TIMEOUT_MS,
|
|
1661
|
+
},
|
|
1662
|
+
);
|
|
1663
|
+
if (result.error) {
|
|
1664
|
+
if (result.error.code === "ENOENT") {
|
|
1665
|
+
return [];
|
|
1666
|
+
}
|
|
1667
|
+
if (result.error.code === "ETIMEDOUT") {
|
|
1668
|
+
throw new Error(`list tmux sessions failed: timed out after ${TMUX_COMMAND_TIMEOUT_MS}ms`);
|
|
1669
|
+
}
|
|
1670
|
+
throw new Error(`list tmux sessions failed: ${result.error.message}`);
|
|
1671
|
+
}
|
|
1672
|
+
if (result.status !== 0) {
|
|
1673
|
+
const combined = `${String(result.stderr || "").toLowerCase()}\n${String(result.stdout || "").toLowerCase()}`;
|
|
1674
|
+
if (
|
|
1675
|
+
combined.includes("no server running") ||
|
|
1676
|
+
combined.includes("failed to connect") ||
|
|
1677
|
+
combined.includes("error connecting")
|
|
1678
|
+
) {
|
|
1679
|
+
return [];
|
|
1680
|
+
}
|
|
1681
|
+
throw new Error(
|
|
1682
|
+
`list tmux sessions failed: ${(result.stderr || "").trim() || "unknown error"}`,
|
|
1683
|
+
);
|
|
1684
|
+
}
|
|
1685
|
+
return String(result.stdout || "")
|
|
1686
|
+
.split(/\r?\n/)
|
|
1687
|
+
.map((line) => line.trim())
|
|
1688
|
+
.filter(Boolean);
|
|
1689
|
+
}
|
|
1690
|
+
|
|
1691
|
+
function cleanupLaneTmuxSessions(lanePaths, { excludeSessionNames = new Set() } = {}) {
|
|
1692
|
+
const sessionNames = listTmuxSessionNames(lanePaths);
|
|
1693
|
+
const killed = [];
|
|
1694
|
+
for (const sessionName of sessionNames) {
|
|
1695
|
+
if (excludeSessionNames.has(sessionName) || !isLaneSessionName(lanePaths, sessionName)) {
|
|
1696
|
+
continue;
|
|
1697
|
+
}
|
|
1698
|
+
killTmuxSessionIfExists(lanePaths.tmuxSocketName, sessionName);
|
|
1699
|
+
killed.push(sessionName);
|
|
1700
|
+
}
|
|
1701
|
+
return killed;
|
|
1702
|
+
}
|
|
1703
|
+
|
|
1704
|
+
export function collectUnexpectedSessionFailures(lanePaths, agentRuns, pendingAgentIds) {
|
|
1705
|
+
const activeSessionNames = new Set(listLaneTmuxSessionNames(lanePaths));
|
|
1706
|
+
const failures = [];
|
|
1707
|
+
for (const run of agentRuns) {
|
|
1708
|
+
if (!pendingAgentIds.has(run.agent.agentId) || fs.existsSync(run.statusPath)) {
|
|
1709
|
+
continue;
|
|
1710
|
+
}
|
|
1711
|
+
if (activeSessionNames.has(run.sessionName)) {
|
|
1712
|
+
continue;
|
|
1713
|
+
}
|
|
1714
|
+
failures.push({
|
|
1715
|
+
agentId: run.agent.agentId,
|
|
1716
|
+
statusCode: "session-missing",
|
|
1717
|
+
logPath: path.relative(REPO_ROOT, run.logPath),
|
|
1718
|
+
detail: `tmux session ${run.sessionName} disappeared before ${path.relative(REPO_ROOT, run.statusPath)} was written.`,
|
|
1719
|
+
});
|
|
1720
|
+
}
|
|
1721
|
+
return failures;
|
|
1722
|
+
}
|
|
1723
|
+
|
|
1724
|
+
function launchWaveDashboardSession(lanePaths, { sessionName, dashboardPath, messageBoardPath }) {
|
|
1725
|
+
killTmuxSessionIfExists(lanePaths.tmuxSocketName, sessionName);
|
|
1726
|
+
const messageBoardArg = messageBoardPath
|
|
1727
|
+
? ` --message-board ${shellQuote(messageBoardPath)}`
|
|
1728
|
+
: "";
|
|
1729
|
+
const command = [
|
|
1730
|
+
`cd ${shellQuote(REPO_ROOT)}`,
|
|
1731
|
+
`node ${shellQuote(path.join(PACKAGE_ROOT, "scripts", "wave-dashboard.mjs"))} --dashboard-file ${shellQuote(
|
|
1732
|
+
dashboardPath,
|
|
1733
|
+
)}${messageBoardArg} --lane ${shellQuote(lanePaths.lane)} --watch`,
|
|
1734
|
+
"exec bash -l",
|
|
1735
|
+
].join("; ");
|
|
1736
|
+
runTmux(
|
|
1737
|
+
lanePaths,
|
|
1738
|
+
["new-session", "-d", "-s", sessionName, `bash -lc ${shellQuote(command)}`],
|
|
1739
|
+
`launch dashboard session ${sessionName}`,
|
|
1740
|
+
);
|
|
1741
|
+
}
|
|
1742
|
+
|
|
1743
|
+
async function launchAgentSession(lanePaths, params) {
|
|
1744
|
+
const {
|
|
1745
|
+
wave,
|
|
1746
|
+
agent,
|
|
1747
|
+
sessionName,
|
|
1748
|
+
promptPath,
|
|
1749
|
+
logPath,
|
|
1750
|
+
statusPath,
|
|
1751
|
+
messageBoardPath,
|
|
1752
|
+
messageBoardSnapshot,
|
|
1753
|
+
sharedSummaryPath,
|
|
1754
|
+
sharedSummaryText,
|
|
1755
|
+
inboxPath,
|
|
1756
|
+
inboxText,
|
|
1757
|
+
orchestratorId,
|
|
1758
|
+
agentRateLimitRetries,
|
|
1759
|
+
agentRateLimitBaseDelaySeconds,
|
|
1760
|
+
agentRateLimitMaxDelaySeconds,
|
|
1761
|
+
context7Enabled,
|
|
1762
|
+
dryRun = false,
|
|
1763
|
+
} = params;
|
|
1764
|
+
ensureDirectory(path.dirname(promptPath));
|
|
1765
|
+
ensureDirectory(path.dirname(logPath));
|
|
1766
|
+
ensureDirectory(path.dirname(statusPath));
|
|
1767
|
+
fs.rmSync(statusPath, { force: true });
|
|
1768
|
+
|
|
1769
|
+
const context7 = await prefetchContext7ForSelection(agent.context7Resolved, {
|
|
1770
|
+
cacheDir: lanePaths.context7CacheDir,
|
|
1771
|
+
disabled: !context7Enabled,
|
|
1772
|
+
});
|
|
1773
|
+
const prompt = buildExecutionPrompt({
|
|
1774
|
+
lane: lanePaths.lane,
|
|
1775
|
+
wave,
|
|
1776
|
+
agent,
|
|
1777
|
+
orchestratorId,
|
|
1778
|
+
messageBoardPath,
|
|
1779
|
+
messageBoardSnapshot,
|
|
1780
|
+
sharedSummaryPath,
|
|
1781
|
+
sharedSummaryText,
|
|
1782
|
+
inboxPath,
|
|
1783
|
+
inboxText,
|
|
1784
|
+
context7,
|
|
1785
|
+
componentPromotions: wave.componentPromotions,
|
|
1786
|
+
sharedPlanDocs: lanePaths.sharedPlanDocs,
|
|
1787
|
+
evaluatorAgentId: lanePaths.evaluatorAgentId,
|
|
1788
|
+
integrationAgentId: lanePaths.integrationAgentId,
|
|
1789
|
+
documentationAgentId: lanePaths.documentationAgentId,
|
|
1790
|
+
});
|
|
1791
|
+
const promptHash = hashAgentPromptFingerprint(agent);
|
|
1792
|
+
fs.writeFileSync(promptPath, `${prompt}\n`, "utf8");
|
|
1793
|
+
const overlayDir = path.join(lanePaths.executorOverlaysDir, `wave-${wave}`, agent.slug);
|
|
1794
|
+
const launchSpec = buildExecutorLaunchSpec({
|
|
1795
|
+
agent,
|
|
1796
|
+
promptPath,
|
|
1797
|
+
logPath,
|
|
1798
|
+
overlayDir,
|
|
1799
|
+
});
|
|
1800
|
+
const resolvedExecutorMode = launchSpec.executorId || agent.executorResolved?.id || "codex";
|
|
1801
|
+
if (dryRun) {
|
|
1802
|
+
writeJsonAtomic(path.join(overlayDir, "launch-preview.json"), {
|
|
1803
|
+
executorId: resolvedExecutorMode,
|
|
1804
|
+
command: launchSpec.command,
|
|
1805
|
+
env: launchSpec.env || {},
|
|
1806
|
+
useRateLimitRetries: launchSpec.useRateLimitRetries === true,
|
|
1807
|
+
invocationLines: launchSpec.invocationLines,
|
|
1808
|
+
});
|
|
1809
|
+
return { promptHash, context7, executorId: resolvedExecutorMode, launchSpec, dryRun: true };
|
|
1810
|
+
}
|
|
1811
|
+
killTmuxSessionIfExists(lanePaths.tmuxSocketName, sessionName);
|
|
1812
|
+
|
|
1813
|
+
const executionLines = [];
|
|
1814
|
+
if (launchSpec.env) {
|
|
1815
|
+
for (const [key, value] of Object.entries(launchSpec.env)) {
|
|
1816
|
+
executionLines.push(`export ${key}=${shellQuote(value)}`);
|
|
1817
|
+
}
|
|
1818
|
+
}
|
|
1819
|
+
if (!launchSpec.useRateLimitRetries) {
|
|
1820
|
+
executionLines.push(...launchSpec.invocationLines);
|
|
1821
|
+
executionLines.push("status=$?");
|
|
1822
|
+
} else {
|
|
1823
|
+
executionLines.push(`: > ${shellQuote(logPath)}`);
|
|
1824
|
+
executionLines.push(
|
|
1825
|
+
`max_rate_attempts=${Math.max(1, Number.parseInt(String(agentRateLimitRetries || 0), 10) + 1)}`,
|
|
1826
|
+
);
|
|
1827
|
+
executionLines.push(
|
|
1828
|
+
`rate_delay_base=${Math.max(1, Number.parseInt(String(agentRateLimitBaseDelaySeconds || DEFAULT_AGENT_RATE_LIMIT_BASE_DELAY_SECONDS), 10))}`,
|
|
1829
|
+
);
|
|
1830
|
+
executionLines.push(
|
|
1831
|
+
`rate_delay_max=${Math.max(1, Number.parseInt(String(agentRateLimitMaxDelaySeconds || DEFAULT_AGENT_RATE_LIMIT_MAX_DELAY_SECONDS), 10))}`,
|
|
1832
|
+
);
|
|
1833
|
+
executionLines.push("rate_attempt=1");
|
|
1834
|
+
executionLines.push("status=1");
|
|
1835
|
+
executionLines.push('while [ "$rate_attempt" -le "$max_rate_attempts" ]; do');
|
|
1836
|
+
for (const line of launchSpec.invocationLines) {
|
|
1837
|
+
executionLines.push(` ${line}`);
|
|
1838
|
+
}
|
|
1839
|
+
executionLines.push(" status=$?");
|
|
1840
|
+
executionLines.push(' if [ "$status" -eq 0 ]; then');
|
|
1841
|
+
executionLines.push(" break");
|
|
1842
|
+
executionLines.push(" fi");
|
|
1843
|
+
executionLines.push(' if [ "$rate_attempt" -ge "$max_rate_attempts" ]; then');
|
|
1844
|
+
executionLines.push(" break");
|
|
1845
|
+
executionLines.push(" fi");
|
|
1846
|
+
executionLines.push(
|
|
1847
|
+
` if tail -n 120 ${shellQuote(logPath)} | grep -Eqi '429 Too Many Requests|exceeded retry limit|last status: 429|rate limit'; then`,
|
|
1848
|
+
);
|
|
1849
|
+
executionLines.push(" sleep_seconds=$((rate_delay_base * (2 ** (rate_attempt - 1))))");
|
|
1850
|
+
executionLines.push(
|
|
1851
|
+
' if [ "$sleep_seconds" -gt "$rate_delay_max" ]; then sleep_seconds=$rate_delay_max; fi',
|
|
1852
|
+
);
|
|
1853
|
+
executionLines.push(" jitter=$((RANDOM % 5))");
|
|
1854
|
+
executionLines.push(" sleep_seconds=$((sleep_seconds + jitter))");
|
|
1855
|
+
executionLines.push(
|
|
1856
|
+
` echo "[${lanePaths.lane}-wave-launcher] rate-limit detected for ${agent.agentId}; retry \${rate_attempt}/\${max_rate_attempts} after \${sleep_seconds}s" | tee -a ${shellQuote(logPath)}`,
|
|
1857
|
+
);
|
|
1858
|
+
executionLines.push(' sleep "$sleep_seconds"');
|
|
1859
|
+
executionLines.push(" rate_attempt=$((rate_attempt + 1))");
|
|
1860
|
+
executionLines.push(" continue");
|
|
1861
|
+
executionLines.push(" fi");
|
|
1862
|
+
executionLines.push(" break");
|
|
1863
|
+
executionLines.push("done");
|
|
1864
|
+
}
|
|
1865
|
+
|
|
1866
|
+
const command = [
|
|
1867
|
+
`cd ${shellQuote(REPO_ROOT)}`,
|
|
1868
|
+
"set -o pipefail",
|
|
1869
|
+
`export WAVE_ORCHESTRATOR_ID=${shellQuote(orchestratorId || "")}`,
|
|
1870
|
+
`export WAVE_EXECUTOR_MODE=${shellQuote(resolvedExecutorMode)}`,
|
|
1871
|
+
...executionLines,
|
|
1872
|
+
`node -e ${shellQuote(
|
|
1873
|
+
"const fs=require('node:fs'); const statusPath=process.argv[1]; const payload={code:Number(process.argv[2]),promptHash:process.argv[3]||null,orchestratorId:process.argv[4]||null,completedAt:new Date().toISOString()}; fs.writeFileSync(statusPath, JSON.stringify(payload, null, 2)+'\\n', 'utf8');",
|
|
1874
|
+
)} ${shellQuote(statusPath)} "$status" ${shellQuote(promptHash)} ${shellQuote(orchestratorId || "")}`,
|
|
1875
|
+
`echo "[${lanePaths.lane}-wave-launcher] ${sessionName} finished with code $status"`,
|
|
1876
|
+
"exec bash -l",
|
|
1877
|
+
].join("\n");
|
|
1878
|
+
|
|
1879
|
+
runTmux(
|
|
1880
|
+
lanePaths,
|
|
1881
|
+
["new-session", "-d", "-s", sessionName, `bash -lc ${shellQuote(command)}`],
|
|
1882
|
+
`launch session ${sessionName}`,
|
|
1883
|
+
);
|
|
1884
|
+
return { promptHash, context7, executorId: resolvedExecutorMode };
|
|
1885
|
+
}
|
|
1886
|
+
|
|
1887
|
+
async function waitForWaveCompletion(lanePaths, agentRuns, timeoutMinutes, onProgress = null) {
|
|
1888
|
+
const defaultTimeoutMs = timeoutMinutes * 60 * 1000;
|
|
1889
|
+
const startedAt = Date.now();
|
|
1890
|
+
const timeoutAtByAgentId = new Map(
|
|
1891
|
+
agentRuns.map((run) => {
|
|
1892
|
+
const budgetMinutes = Number(run.agent.executorResolved?.budget?.minutes || 0);
|
|
1893
|
+
const effectiveBudgetMs =
|
|
1894
|
+
Number.isFinite(budgetMinutes) && budgetMinutes > 0
|
|
1895
|
+
? Math.min(defaultTimeoutMs, budgetMinutes * 60 * 1000)
|
|
1896
|
+
: defaultTimeoutMs;
|
|
1897
|
+
return [run.agent.agentId, startedAt + effectiveBudgetMs];
|
|
1898
|
+
}),
|
|
1899
|
+
);
|
|
1900
|
+
const pending = new Set(agentRuns.map((run) => run.agent.agentId));
|
|
1901
|
+
const timedOutAgentIds = new Set();
|
|
1902
|
+
let sessionFailures = [];
|
|
1903
|
+
|
|
1904
|
+
const refreshPending = () => {
|
|
1905
|
+
for (const run of agentRuns) {
|
|
1906
|
+
if (pending.has(run.agent.agentId) && fs.existsSync(run.statusPath)) {
|
|
1907
|
+
pending.delete(run.agent.agentId);
|
|
1908
|
+
}
|
|
1909
|
+
}
|
|
1910
|
+
};
|
|
1911
|
+
|
|
1912
|
+
await new Promise((resolve) => {
|
|
1913
|
+
const interval = setInterval(() => {
|
|
1914
|
+
refreshPending();
|
|
1915
|
+
onProgress?.({ pendingAgentIds: new Set(pending), timedOut: false });
|
|
1916
|
+
if (pending.size === 0) {
|
|
1917
|
+
clearInterval(interval);
|
|
1918
|
+
resolve();
|
|
1919
|
+
return;
|
|
1920
|
+
}
|
|
1921
|
+
sessionFailures = collectUnexpectedSessionFailures(lanePaths, agentRuns, pending);
|
|
1922
|
+
if (sessionFailures.length > 0) {
|
|
1923
|
+
onProgress?.({
|
|
1924
|
+
pendingAgentIds: new Set(pending),
|
|
1925
|
+
timedOut: false,
|
|
1926
|
+
failures: sessionFailures,
|
|
1927
|
+
});
|
|
1928
|
+
clearInterval(interval);
|
|
1929
|
+
resolve();
|
|
1930
|
+
return;
|
|
1931
|
+
}
|
|
1932
|
+
const now = Date.now();
|
|
1933
|
+
for (const run of agentRuns) {
|
|
1934
|
+
if (!pending.has(run.agent.agentId)) {
|
|
1935
|
+
continue;
|
|
1936
|
+
}
|
|
1937
|
+
const deadline = timeoutAtByAgentId.get(run.agent.agentId) || startedAt + defaultTimeoutMs;
|
|
1938
|
+
if (now <= deadline) {
|
|
1939
|
+
continue;
|
|
1940
|
+
}
|
|
1941
|
+
timedOutAgentIds.add(run.agent.agentId);
|
|
1942
|
+
pending.delete(run.agent.agentId);
|
|
1943
|
+
killTmuxSessionIfExists(lanePaths.tmuxSocketName, run.sessionName);
|
|
1944
|
+
}
|
|
1945
|
+
if (pending.size === 0) {
|
|
1946
|
+
clearInterval(interval);
|
|
1947
|
+
resolve();
|
|
1948
|
+
}
|
|
1949
|
+
}, DEFAULT_WAIT_PROGRESS_INTERVAL_MS);
|
|
1950
|
+
refreshPending();
|
|
1951
|
+
onProgress?.({ pendingAgentIds: new Set(pending), timedOut: false });
|
|
1952
|
+
});
|
|
1953
|
+
|
|
1954
|
+
if (sessionFailures.length > 0) {
|
|
1955
|
+
onProgress?.({ pendingAgentIds: new Set(), timedOut: false, failures: sessionFailures });
|
|
1956
|
+
return { failures: sessionFailures, timedOut: false };
|
|
1957
|
+
}
|
|
1958
|
+
|
|
1959
|
+
const failures = [];
|
|
1960
|
+
for (const run of agentRuns) {
|
|
1961
|
+
const code = readStatusCodeIfPresent(run.statusPath);
|
|
1962
|
+
if (code === 0) {
|
|
1963
|
+
continue;
|
|
1964
|
+
}
|
|
1965
|
+
if (code === null || timedOutAgentIds.has(run.agent.agentId)) {
|
|
1966
|
+
failures.push({
|
|
1967
|
+
agentId: run.agent.agentId,
|
|
1968
|
+
statusCode: timedOutAgentIds.has(run.agent.agentId) ? "timeout-no-status" : "missing-status",
|
|
1969
|
+
logPath: path.relative(REPO_ROOT, run.logPath),
|
|
1970
|
+
});
|
|
1971
|
+
continue;
|
|
1972
|
+
}
|
|
1973
|
+
failures.push({
|
|
1974
|
+
agentId: run.agent.agentId,
|
|
1975
|
+
statusCode: String(code),
|
|
1976
|
+
logPath: path.relative(REPO_ROOT, run.logPath),
|
|
1977
|
+
});
|
|
1978
|
+
}
|
|
1979
|
+
onProgress?.({ pendingAgentIds: new Set(), timedOut: timedOutAgentIds.size > 0 });
|
|
1980
|
+
return { failures, timedOut: timedOutAgentIds.size > 0 };
|
|
1981
|
+
}
|
|
1982
|
+
|
|
1983
|
+
function monitorWaveHumanFeedback({
|
|
1984
|
+
lanePaths,
|
|
1985
|
+
waveNumber,
|
|
1986
|
+
agentRuns,
|
|
1987
|
+
orchestratorId,
|
|
1988
|
+
coordinationLogPath,
|
|
1989
|
+
feedbackStateByRequestId,
|
|
1990
|
+
recordCombinedEvent,
|
|
1991
|
+
appendCoordination,
|
|
1992
|
+
}) {
|
|
1993
|
+
const triageLogPath = path.join(lanePaths.feedbackTriageDir, `wave-${waveNumber}.jsonl`);
|
|
1994
|
+
const requests = readWaveHumanFeedbackRequests({
|
|
1995
|
+
feedbackRequestsDir: lanePaths.feedbackRequestsDir,
|
|
1996
|
+
lane: lanePaths.lane,
|
|
1997
|
+
waveNumber,
|
|
1998
|
+
agentIds: agentRuns.map((run) => run.agent.agentId),
|
|
1999
|
+
orchestratorId,
|
|
2000
|
+
});
|
|
2001
|
+
for (const request of requests) {
|
|
2002
|
+
const signature = feedbackStateSignature(request);
|
|
2003
|
+
if (feedbackStateByRequestId.get(request.id) === signature) {
|
|
2004
|
+
continue;
|
|
2005
|
+
}
|
|
2006
|
+
feedbackStateByRequestId.set(request.id, signature);
|
|
2007
|
+
const question = request.question || "n/a";
|
|
2008
|
+
const context = request.context ? `; context=${request.context}` : "";
|
|
2009
|
+
const responseOperator = request.responseOperator || "human-operator";
|
|
2010
|
+
const responseText = request.responseText || "(empty response)";
|
|
2011
|
+
if (request.status === "pending") {
|
|
2012
|
+
recordCombinedEvent({
|
|
2013
|
+
level: "warn",
|
|
2014
|
+
agentId: request.agentId,
|
|
2015
|
+
message: `Human feedback requested (${request.id}): ${question}`,
|
|
2016
|
+
});
|
|
2017
|
+
console.warn(
|
|
2018
|
+
`[human-feedback] wave=${waveNumber} agent=${request.agentId} request=${request.id} pending: ${question}`,
|
|
2019
|
+
);
|
|
2020
|
+
console.warn(
|
|
2021
|
+
`[human-feedback] respond with: pnpm exec wave-feedback respond --id ${request.id} --response "<answer>" --operator "<name>"`,
|
|
2022
|
+
);
|
|
2023
|
+
appendCoordination({
|
|
2024
|
+
event: "human_feedback_requested",
|
|
2025
|
+
waves: [waveNumber],
|
|
2026
|
+
status: "waiting-human",
|
|
2027
|
+
details: `request_id=${request.id}; agent=${request.agentId}; question=${question}${context}`,
|
|
2028
|
+
actionRequested: `Launcher operator should ask or answer in the parent session, then run: pnpm exec wave-feedback respond --id ${request.id} --response "<answer>" --operator "<name>"`,
|
|
2029
|
+
});
|
|
2030
|
+
if (coordinationLogPath) {
|
|
2031
|
+
appendCoordinationRecord(coordinationLogPath, {
|
|
2032
|
+
id: request.id,
|
|
2033
|
+
lane: lanePaths.lane,
|
|
2034
|
+
wave: waveNumber,
|
|
2035
|
+
agentId: request.agentId || "human",
|
|
2036
|
+
kind: "human-feedback",
|
|
2037
|
+
targets: request.agentId ? [`agent:${request.agentId}`] : [],
|
|
2038
|
+
priority: "high",
|
|
2039
|
+
summary: question,
|
|
2040
|
+
detail: request.context || "",
|
|
2041
|
+
status: "open",
|
|
2042
|
+
source: "feedback",
|
|
2043
|
+
});
|
|
2044
|
+
}
|
|
2045
|
+
} else if (request.status === "answered") {
|
|
2046
|
+
recordCombinedEvent({
|
|
2047
|
+
level: "info",
|
|
2048
|
+
agentId: request.agentId,
|
|
2049
|
+
message: `Human feedback answered (${request.id}) by ${responseOperator}: ${responseText}`,
|
|
2050
|
+
});
|
|
2051
|
+
appendCoordination({
|
|
2052
|
+
event: "human_feedback_answered",
|
|
2053
|
+
waves: [waveNumber],
|
|
2054
|
+
status: "resolved",
|
|
2055
|
+
details: `request_id=${request.id}; agent=${request.agentId}; operator=${responseOperator}; response=${responseText}`,
|
|
2056
|
+
});
|
|
2057
|
+
if (coordinationLogPath) {
|
|
2058
|
+
const escalationId = `escalation-${request.id}`;
|
|
2059
|
+
const existingEscalation =
|
|
2060
|
+
(fs.existsSync(triageLogPath)
|
|
2061
|
+
? readMaterializedCoordinationState(triageLogPath).byId.get(escalationId)
|
|
2062
|
+
: null) ||
|
|
2063
|
+
readMaterializedCoordinationState(coordinationLogPath).byId.get(escalationId) ||
|
|
2064
|
+
null;
|
|
2065
|
+
if (fs.existsSync(triageLogPath)) {
|
|
2066
|
+
appendCoordinationRecord(triageLogPath, {
|
|
2067
|
+
id: escalationId,
|
|
2068
|
+
lane: lanePaths.lane,
|
|
2069
|
+
wave: waveNumber,
|
|
2070
|
+
agentId: request.agentId || "human",
|
|
2071
|
+
kind: "human-escalation",
|
|
2072
|
+
targets:
|
|
2073
|
+
existingEscalation?.targets ||
|
|
2074
|
+
(request.agentId ? [`agent:${request.agentId}`] : []),
|
|
2075
|
+
dependsOn: existingEscalation?.dependsOn || [],
|
|
2076
|
+
closureCondition: existingEscalation?.closureCondition || "",
|
|
2077
|
+
priority: "high",
|
|
2078
|
+
summary: question,
|
|
2079
|
+
detail: responseText,
|
|
2080
|
+
artifactRefs: [request.id],
|
|
2081
|
+
status: "resolved",
|
|
2082
|
+
source: "feedback",
|
|
2083
|
+
});
|
|
2084
|
+
}
|
|
2085
|
+
appendCoordinationRecord(coordinationLogPath, {
|
|
2086
|
+
id: escalationId,
|
|
2087
|
+
lane: lanePaths.lane,
|
|
2088
|
+
wave: waveNumber,
|
|
2089
|
+
agentId: request.agentId || "human",
|
|
2090
|
+
kind: "human-escalation",
|
|
2091
|
+
targets:
|
|
2092
|
+
existingEscalation?.targets ||
|
|
2093
|
+
(request.agentId ? [`agent:${request.agentId}`] : []),
|
|
2094
|
+
dependsOn: existingEscalation?.dependsOn || [],
|
|
2095
|
+
closureCondition: existingEscalation?.closureCondition || "",
|
|
2096
|
+
priority: "high",
|
|
2097
|
+
summary: question,
|
|
2098
|
+
detail: responseText,
|
|
2099
|
+
artifactRefs: [request.id],
|
|
2100
|
+
status: "resolved",
|
|
2101
|
+
source: "feedback",
|
|
2102
|
+
});
|
|
2103
|
+
appendCoordinationRecord(coordinationLogPath, {
|
|
2104
|
+
id: request.id,
|
|
2105
|
+
lane: lanePaths.lane,
|
|
2106
|
+
wave: waveNumber,
|
|
2107
|
+
agentId: request.agentId || "human",
|
|
2108
|
+
kind: "human-feedback",
|
|
2109
|
+
targets: request.agentId ? [`agent:${request.agentId}`] : [],
|
|
2110
|
+
priority: "high",
|
|
2111
|
+
summary: question,
|
|
2112
|
+
detail: responseText,
|
|
2113
|
+
status: "resolved",
|
|
2114
|
+
source: "feedback",
|
|
2115
|
+
});
|
|
2116
|
+
}
|
|
2117
|
+
}
|
|
2118
|
+
}
|
|
2119
|
+
}
|
|
2120
|
+
|
|
2121
|
+
export function hasReusableSuccessStatus(agent, statusPath) {
|
|
2122
|
+
const statusRecord = readStatusRecordIfPresent(statusPath);
|
|
2123
|
+
return Boolean(
|
|
2124
|
+
statusRecord && statusRecord.code === 0 && statusRecord.promptHash === hashAgentPromptFingerprint(agent),
|
|
2125
|
+
);
|
|
2126
|
+
}
|
|
2127
|
+
|
|
2128
|
+
function isClosureAgentId(agentId, lanePaths) {
|
|
2129
|
+
return [
|
|
2130
|
+
lanePaths.integrationAgentId,
|
|
2131
|
+
lanePaths.documentationAgentId,
|
|
2132
|
+
lanePaths.evaluatorAgentId,
|
|
2133
|
+
].includes(agentId);
|
|
2134
|
+
}
|
|
2135
|
+
|
|
2136
|
+
function isLauncherSeedRequest(record) {
|
|
2137
|
+
return (
|
|
2138
|
+
record?.source === "launcher" &&
|
|
2139
|
+
/^wave-\d+-agent-[^-]+-request$/.test(String(record.id || "")) &&
|
|
2140
|
+
!String(record.closureCondition || "").trim() &&
|
|
2141
|
+
(!Array.isArray(record.dependsOn) || record.dependsOn.length === 0)
|
|
2142
|
+
);
|
|
2143
|
+
}
|
|
2144
|
+
|
|
2145
|
+
function runtimeMixValidationForRuns(agentRuns, lanePaths) {
|
|
2146
|
+
return validateWaveRuntimeMixAssignments(
|
|
2147
|
+
{
|
|
2148
|
+
wave: 0,
|
|
2149
|
+
agents: agentRuns.map((run) => run.agent),
|
|
2150
|
+
},
|
|
2151
|
+
{ laneProfile: lanePaths.laneProfile },
|
|
2152
|
+
);
|
|
2153
|
+
}
|
|
2154
|
+
|
|
2155
|
+
function nextExecutorModel(executorState, executorId) {
|
|
2156
|
+
if (executorId === "claude") {
|
|
2157
|
+
return executorState?.claude?.model || null;
|
|
2158
|
+
}
|
|
2159
|
+
if (executorId === "opencode") {
|
|
2160
|
+
return executorState?.opencode?.model || null;
|
|
2161
|
+
}
|
|
2162
|
+
return null;
|
|
2163
|
+
}
|
|
2164
|
+
|
|
2165
|
+
function executorFallbackChain(executorState) {
|
|
2166
|
+
return Array.isArray(executorState?.fallbacks)
|
|
2167
|
+
? executorState.fallbacks.filter(Boolean)
|
|
2168
|
+
: [];
|
|
2169
|
+
}
|
|
2170
|
+
|
|
2171
|
+
function buildFallbackExecutorState(executorState, executorId, attempt, reason) {
|
|
2172
|
+
const history = Array.isArray(executorState?.executorHistory)
|
|
2173
|
+
? executorState.executorHistory
|
|
2174
|
+
: [];
|
|
2175
|
+
return {
|
|
2176
|
+
...executorState,
|
|
2177
|
+
id: executorId,
|
|
2178
|
+
model: nextExecutorModel(executorState, executorId),
|
|
2179
|
+
selectedBy: "retry-fallback",
|
|
2180
|
+
fallbackUsed: true,
|
|
2181
|
+
fallbackReason: reason,
|
|
2182
|
+
initialExecutorId: executorState?.initialExecutorId || executorState?.id || executorId,
|
|
2183
|
+
executorHistory: [
|
|
2184
|
+
...history,
|
|
2185
|
+
{
|
|
2186
|
+
attempt,
|
|
2187
|
+
executorId,
|
|
2188
|
+
reason,
|
|
2189
|
+
},
|
|
2190
|
+
],
|
|
2191
|
+
};
|
|
2192
|
+
}
|
|
2193
|
+
|
|
2194
|
+
function applyRetryFallbacks(agentRuns, failures, lanePaths, attemptNumber) {
|
|
2195
|
+
const failedAgentIds = new Set(failures.map((failure) => failure.agentId));
|
|
2196
|
+
let changed = false;
|
|
2197
|
+
const outcomes = new Map();
|
|
2198
|
+
for (const run of agentRuns) {
|
|
2199
|
+
if (!failedAgentIds.has(run.agent.agentId)) {
|
|
2200
|
+
continue;
|
|
2201
|
+
}
|
|
2202
|
+
const executorState = run.agent.executorResolved;
|
|
2203
|
+
if (!executorState) {
|
|
2204
|
+
outcomes.set(run.agent.agentId, {
|
|
2205
|
+
applied: false,
|
|
2206
|
+
blocking: false,
|
|
2207
|
+
statusCode: "no-executor-state",
|
|
2208
|
+
detail: `Agent ${run.agent.agentId} has no resolved executor state.`,
|
|
2209
|
+
});
|
|
2210
|
+
continue;
|
|
2211
|
+
}
|
|
2212
|
+
const fallbackChain = executorFallbackChain(executorState);
|
|
2213
|
+
if (fallbackChain.length === 0) {
|
|
2214
|
+
outcomes.set(run.agent.agentId, {
|
|
2215
|
+
applied: false,
|
|
2216
|
+
blocking: false,
|
|
2217
|
+
statusCode: "no-fallback-configured",
|
|
2218
|
+
detail: `Agent ${run.agent.agentId} has no configured fallback executors.`,
|
|
2219
|
+
});
|
|
2220
|
+
continue;
|
|
2221
|
+
}
|
|
2222
|
+
const attemptedExecutors = new Set(
|
|
2223
|
+
Array.isArray(executorState.executorHistory)
|
|
2224
|
+
? executorState.executorHistory.map((entry) => entry.executorId)
|
|
2225
|
+
: [executorState.id],
|
|
2226
|
+
);
|
|
2227
|
+
const fallbackReason = failures.find((failure) => failure.agentId === run.agent.agentId);
|
|
2228
|
+
const blockedCandidates = [];
|
|
2229
|
+
for (const candidate of fallbackChain) {
|
|
2230
|
+
if (!candidate || candidate === executorState.id || attemptedExecutors.has(candidate)) {
|
|
2231
|
+
if (candidate) {
|
|
2232
|
+
blockedCandidates.push(`${candidate}: already tried`);
|
|
2233
|
+
}
|
|
2234
|
+
continue;
|
|
2235
|
+
}
|
|
2236
|
+
const command = commandForExecutor(executorState, candidate);
|
|
2237
|
+
if (!isExecutorCommandAvailable(command)) {
|
|
2238
|
+
blockedCandidates.push(`${candidate}: command unavailable`);
|
|
2239
|
+
continue;
|
|
2240
|
+
}
|
|
2241
|
+
const nextState = buildFallbackExecutorState(
|
|
2242
|
+
executorState,
|
|
2243
|
+
candidate,
|
|
2244
|
+
attemptNumber,
|
|
2245
|
+
`retry:${fallbackReason?.statusCode || "failed-attempt"}`,
|
|
2246
|
+
);
|
|
2247
|
+
const validation = runtimeMixValidationForRuns(
|
|
2248
|
+
agentRuns.map((entry) =>
|
|
2249
|
+
entry.agent.agentId === run.agent.agentId
|
|
2250
|
+
? { ...entry, agent: { ...entry.agent, executorResolved: nextState } }
|
|
2251
|
+
: entry,
|
|
2252
|
+
),
|
|
2253
|
+
lanePaths,
|
|
2254
|
+
);
|
|
2255
|
+
if (!validation.ok) {
|
|
2256
|
+
blockedCandidates.push(`${candidate}: ${validation.detail}`);
|
|
2257
|
+
continue;
|
|
2258
|
+
}
|
|
2259
|
+
run.agent.executorResolved = nextState;
|
|
2260
|
+
changed = true;
|
|
2261
|
+
outcomes.set(run.agent.agentId, {
|
|
2262
|
+
applied: true,
|
|
2263
|
+
blocking: false,
|
|
2264
|
+
statusCode: "fallback-applied",
|
|
2265
|
+
detail: `Agent ${run.agent.agentId} will retry on ${candidate}.`,
|
|
2266
|
+
executorId: candidate,
|
|
2267
|
+
});
|
|
2268
|
+
break;
|
|
2269
|
+
}
|
|
2270
|
+
if (!outcomes.has(run.agent.agentId)) {
|
|
2271
|
+
outcomes.set(run.agent.agentId, {
|
|
2272
|
+
applied: false,
|
|
2273
|
+
blocking: true,
|
|
2274
|
+
statusCode: "retry-fallback-blocked",
|
|
2275
|
+
detail: `Agent ${run.agent.agentId} cannot retry safely on a configured fallback (${blockedCandidates.join("; ") || "no safe fallback remained"}).`,
|
|
2276
|
+
});
|
|
2277
|
+
}
|
|
2278
|
+
}
|
|
2279
|
+
return {
|
|
2280
|
+
changed,
|
|
2281
|
+
outcomes,
|
|
2282
|
+
};
|
|
2283
|
+
}
|
|
2284
|
+
|
|
2285
|
+
function retryBarrierFromOutcomes(outcomes, failures) {
|
|
2286
|
+
const blockingFailures = [];
|
|
2287
|
+
for (const failure of failures) {
|
|
2288
|
+
const outcome = outcomes.get(failure.agentId);
|
|
2289
|
+
if (!outcome?.blocking) {
|
|
2290
|
+
continue;
|
|
2291
|
+
}
|
|
2292
|
+
blockingFailures.push({
|
|
2293
|
+
agentId: failure.agentId,
|
|
2294
|
+
statusCode: outcome.statusCode,
|
|
2295
|
+
logPath: failure.logPath,
|
|
2296
|
+
detail: outcome.detail,
|
|
2297
|
+
});
|
|
2298
|
+
}
|
|
2299
|
+
if (blockingFailures.length === 0) {
|
|
2300
|
+
return null;
|
|
2301
|
+
}
|
|
2302
|
+
return {
|
|
2303
|
+
statusCode: "retry-fallback-blocked",
|
|
2304
|
+
detail: blockingFailures.map((failure) => failure.detail).join(" "),
|
|
2305
|
+
failures: blockingFailures,
|
|
2306
|
+
};
|
|
2307
|
+
}
|
|
2308
|
+
|
|
2309
|
+
export function readClarificationBarrier(derivedState) {
|
|
2310
|
+
const openClarifications = (derivedState?.coordinationState?.clarifications || []).filter(
|
|
2311
|
+
(record) => isOpenCoordinationStatus(record.status),
|
|
2312
|
+
);
|
|
2313
|
+
if (openClarifications.length > 0) {
|
|
2314
|
+
return {
|
|
2315
|
+
ok: false,
|
|
2316
|
+
statusCode: "clarification-open",
|
|
2317
|
+
detail: `Open clarifications remain (${openClarifications.map((record) => record.id).join(", ")}).`,
|
|
2318
|
+
};
|
|
2319
|
+
}
|
|
2320
|
+
const openClarificationRequests = openClarificationLinkedRequests(
|
|
2321
|
+
derivedState?.coordinationState,
|
|
2322
|
+
);
|
|
2323
|
+
if (openClarificationRequests.length > 0) {
|
|
2324
|
+
return {
|
|
2325
|
+
ok: false,
|
|
2326
|
+
statusCode: "clarification-follow-up-open",
|
|
2327
|
+
detail: `Clarification follow-up requests remain open (${openClarificationRequests.map((record) => record.id).join(", ")}).`,
|
|
2328
|
+
};
|
|
2329
|
+
}
|
|
2330
|
+
const pendingHuman = [
|
|
2331
|
+
...((derivedState?.coordinationState?.humanEscalations || []).filter((record) =>
|
|
2332
|
+
isOpenCoordinationStatus(record.status),
|
|
2333
|
+
)),
|
|
2334
|
+
...((derivedState?.coordinationState?.humanFeedback || []).filter((record) =>
|
|
2335
|
+
isOpenCoordinationStatus(record.status),
|
|
2336
|
+
)),
|
|
2337
|
+
];
|
|
2338
|
+
if (pendingHuman.length > 0) {
|
|
2339
|
+
return {
|
|
2340
|
+
ok: false,
|
|
2341
|
+
statusCode: "human-feedback-open",
|
|
2342
|
+
detail: `Pending human input remains (${pendingHuman.map((record) => record.id).join(", ")}).`,
|
|
2343
|
+
};
|
|
2344
|
+
}
|
|
2345
|
+
return {
|
|
2346
|
+
ok: true,
|
|
2347
|
+
statusCode: "pass",
|
|
2348
|
+
detail: "",
|
|
2349
|
+
};
|
|
2350
|
+
}
|
|
2351
|
+
|
|
2352
|
+
export function readWaveAssignmentBarrier(derivedState) {
|
|
2353
|
+
const blockingAssignments = (derivedState?.capabilityAssignments || []).filter(
|
|
2354
|
+
(assignment) => assignment.blocking,
|
|
2355
|
+
);
|
|
2356
|
+
if (blockingAssignments.length === 0) {
|
|
2357
|
+
return {
|
|
2358
|
+
ok: true,
|
|
2359
|
+
statusCode: "pass",
|
|
2360
|
+
detail: "",
|
|
2361
|
+
};
|
|
2362
|
+
}
|
|
2363
|
+
const unresolvedAssignments = blockingAssignments.filter((assignment) => !assignment.assignedAgentId);
|
|
2364
|
+
if (unresolvedAssignments.length > 0) {
|
|
2365
|
+
return {
|
|
2366
|
+
ok: false,
|
|
2367
|
+
statusCode: "helper-assignment-unresolved",
|
|
2368
|
+
detail: `Helper assignments remain unresolved (${unresolvedAssignments.map((assignment) => assignment.requestId).join(", ")}).`,
|
|
2369
|
+
};
|
|
2370
|
+
}
|
|
2371
|
+
return {
|
|
2372
|
+
ok: false,
|
|
2373
|
+
statusCode: "helper-assignment-open",
|
|
2374
|
+
detail: `Helper assignments remain open (${blockingAssignments.map((assignment) => assignment.requestId).join(", ")}).`,
|
|
2375
|
+
};
|
|
2376
|
+
}
|
|
2377
|
+
|
|
2378
|
+
export function readWaveDependencyBarrier(derivedState) {
|
|
2379
|
+
const requiredInbound = derivedState?.dependencySnapshot?.requiredInbound || [];
|
|
2380
|
+
const requiredOutbound = derivedState?.dependencySnapshot?.requiredOutbound || [];
|
|
2381
|
+
const unresolvedInboundAssignments =
|
|
2382
|
+
derivedState?.dependencySnapshot?.unresolvedInboundAssignments || [];
|
|
2383
|
+
if (unresolvedInboundAssignments.length > 0) {
|
|
2384
|
+
return {
|
|
2385
|
+
ok: false,
|
|
2386
|
+
statusCode: "dependency-assignment-unresolved",
|
|
2387
|
+
detail: `Required inbound dependencies are unassigned (${unresolvedInboundAssignments.map((record) => record.id).join(", ")}).`,
|
|
2388
|
+
};
|
|
2389
|
+
}
|
|
2390
|
+
if (requiredInbound.length > 0 || requiredOutbound.length > 0) {
|
|
2391
|
+
return {
|
|
2392
|
+
ok: false,
|
|
2393
|
+
statusCode: "dependency-open",
|
|
2394
|
+
detail: `Open required dependencies remain (${[...requiredInbound, ...requiredOutbound].map((record) => record.id).join(", ")}).`,
|
|
2395
|
+
};
|
|
2396
|
+
}
|
|
2397
|
+
return {
|
|
2398
|
+
ok: true,
|
|
2399
|
+
statusCode: "pass",
|
|
2400
|
+
detail: "",
|
|
2401
|
+
};
|
|
2402
|
+
}
|
|
2403
|
+
|
|
2404
|
+
export function buildGateSnapshot({
|
|
2405
|
+
wave,
|
|
2406
|
+
agentRuns,
|
|
2407
|
+
derivedState,
|
|
2408
|
+
lanePaths,
|
|
2409
|
+
componentMatrixPayload,
|
|
2410
|
+
componentMatrixJsonPath,
|
|
2411
|
+
}) {
|
|
2412
|
+
const implementationGate = readWaveImplementationGate(wave, agentRuns);
|
|
2413
|
+
const componentGate = readWaveComponentGate(wave, agentRuns, {
|
|
2414
|
+
laneProfile: lanePaths?.laneProfile,
|
|
2415
|
+
});
|
|
2416
|
+
const integrationGate = readWaveIntegrationGate(wave, agentRuns, {
|
|
2417
|
+
integrationAgentId: lanePaths?.integrationAgentId,
|
|
2418
|
+
requireIntegrationStewardFromWave: lanePaths?.requireIntegrationStewardFromWave,
|
|
2419
|
+
});
|
|
2420
|
+
const integrationBarrier = readWaveIntegrationBarrier(wave, agentRuns, derivedState, {
|
|
2421
|
+
integrationAgentId: lanePaths?.integrationAgentId,
|
|
2422
|
+
requireIntegrationStewardFromWave: lanePaths?.requireIntegrationStewardFromWave,
|
|
2423
|
+
});
|
|
2424
|
+
const documentationGate = readWaveDocumentationGate(wave, agentRuns);
|
|
2425
|
+
const componentMatrixGate = readWaveComponentMatrixGate(wave, agentRuns, {
|
|
2426
|
+
laneProfile: lanePaths?.laneProfile,
|
|
2427
|
+
documentationAgentId: lanePaths?.documentationAgentId,
|
|
2428
|
+
componentMatrixPayload,
|
|
2429
|
+
componentMatrixJsonPath,
|
|
2430
|
+
});
|
|
2431
|
+
const evaluatorGate = readWaveEvaluatorGate(wave, agentRuns, {
|
|
2432
|
+
evaluatorAgentId: lanePaths?.evaluatorAgentId,
|
|
2433
|
+
});
|
|
2434
|
+
const infraGate = readWaveInfraGate(agentRuns);
|
|
2435
|
+
const clarificationBarrier = readClarificationBarrier(derivedState);
|
|
2436
|
+
const helperAssignmentBarrier = readWaveAssignmentBarrier(derivedState);
|
|
2437
|
+
const dependencyBarrier = readWaveDependencyBarrier(derivedState);
|
|
2438
|
+
const orderedGates = [
|
|
2439
|
+
["implementationGate", implementationGate],
|
|
2440
|
+
["componentGate", componentGate],
|
|
2441
|
+
["helperAssignmentBarrier", helperAssignmentBarrier],
|
|
2442
|
+
["dependencyBarrier", dependencyBarrier],
|
|
2443
|
+
["integrationBarrier", integrationBarrier],
|
|
2444
|
+
["documentationGate", documentationGate],
|
|
2445
|
+
["componentMatrixGate", componentMatrixGate],
|
|
2446
|
+
["evaluatorGate", evaluatorGate],
|
|
2447
|
+
["infraGate", infraGate],
|
|
2448
|
+
["clarificationBarrier", clarificationBarrier],
|
|
2449
|
+
];
|
|
2450
|
+
const firstFailure = orderedGates.find(([, gate]) => gate?.ok === false);
|
|
2451
|
+
return {
|
|
2452
|
+
implementationGate,
|
|
2453
|
+
componentGate,
|
|
2454
|
+
integrationGate,
|
|
2455
|
+
integrationBarrier,
|
|
2456
|
+
documentationGate,
|
|
2457
|
+
componentMatrixGate,
|
|
2458
|
+
evaluatorGate,
|
|
2459
|
+
infraGate,
|
|
2460
|
+
clarificationBarrier,
|
|
2461
|
+
helperAssignmentBarrier,
|
|
2462
|
+
dependencyBarrier,
|
|
2463
|
+
overall: firstFailure
|
|
2464
|
+
? {
|
|
2465
|
+
ok: false,
|
|
2466
|
+
gate: firstFailure[0],
|
|
2467
|
+
statusCode: firstFailure[1].statusCode,
|
|
2468
|
+
detail: firstFailure[1].detail,
|
|
2469
|
+
agentId: firstFailure[1].agentId || null,
|
|
2470
|
+
}
|
|
2471
|
+
: {
|
|
2472
|
+
ok: true,
|
|
2473
|
+
gate: "pass",
|
|
2474
|
+
statusCode: "pass",
|
|
2475
|
+
detail: "All replayed wave gates passed.",
|
|
2476
|
+
agentId: null,
|
|
2477
|
+
},
|
|
2478
|
+
};
|
|
2479
|
+
}
|
|
2480
|
+
|
|
2481
|
+
export function resolveRelaunchRuns(agentRuns, failures, derivedState, lanePaths) {
|
|
2482
|
+
const runsByAgentId = new Map(agentRuns.map((run) => [run.agent.agentId, run]));
|
|
2483
|
+
const pendingFeedback = (derivedState?.coordinationState?.humanFeedback || []).filter((record) =>
|
|
2484
|
+
isOpenCoordinationStatus(record.status),
|
|
2485
|
+
);
|
|
2486
|
+
const pendingHumanEscalations = (derivedState?.coordinationState?.humanEscalations || []).filter(
|
|
2487
|
+
(record) => isOpenCoordinationStatus(record.status),
|
|
2488
|
+
);
|
|
2489
|
+
if (pendingFeedback.length > 0 || pendingHumanEscalations.length > 0) {
|
|
2490
|
+
return { runs: [], barrier: null };
|
|
2491
|
+
}
|
|
2492
|
+
const nextAttemptNumber = Number(derivedState?.ledger?.attempt || 0) + 1;
|
|
2493
|
+
const fallbackResolution = applyRetryFallbacks(
|
|
2494
|
+
agentRuns,
|
|
2495
|
+
failures,
|
|
2496
|
+
lanePaths,
|
|
2497
|
+
nextAttemptNumber,
|
|
2498
|
+
);
|
|
2499
|
+
const retryBarrier = retryBarrierFromOutcomes(fallbackResolution.outcomes, failures);
|
|
2500
|
+
if (retryBarrier) {
|
|
2501
|
+
return { runs: [], barrier: retryBarrier };
|
|
2502
|
+
}
|
|
2503
|
+
const clarificationTargets = new Set();
|
|
2504
|
+
for (const record of openClarificationLinkedRequests(derivedState?.coordinationState)) {
|
|
2505
|
+
for (const target of record.targets || []) {
|
|
2506
|
+
if (String(target).startsWith("agent:")) {
|
|
2507
|
+
clarificationTargets.add(String(target).slice("agent:".length));
|
|
2508
|
+
} else if (runsByAgentId.has(target)) {
|
|
2509
|
+
clarificationTargets.add(target);
|
|
2510
|
+
}
|
|
2511
|
+
}
|
|
2512
|
+
}
|
|
2513
|
+
if (clarificationTargets.size > 0) {
|
|
2514
|
+
return {
|
|
2515
|
+
runs: Array.from(clarificationTargets)
|
|
2516
|
+
.map((agentId) => runsByAgentId.get(agentId))
|
|
2517
|
+
.filter(Boolean),
|
|
2518
|
+
barrier: null,
|
|
2519
|
+
};
|
|
2520
|
+
}
|
|
2521
|
+
const blockingAssignments = (derivedState?.capabilityAssignments || []).filter(
|
|
2522
|
+
(assignment) => assignment.blocking,
|
|
2523
|
+
);
|
|
2524
|
+
const effectiveAssignments =
|
|
2525
|
+
blockingAssignments.length > 0
|
|
2526
|
+
? blockingAssignments
|
|
2527
|
+
: buildRequestAssignments({
|
|
2528
|
+
coordinationState: derivedState?.coordinationState,
|
|
2529
|
+
agents: agentRuns.map((run) => run.agent),
|
|
2530
|
+
ledger: derivedState?.ledger,
|
|
2531
|
+
capabilityRouting: lanePaths?.capabilityRouting,
|
|
2532
|
+
}).filter((assignment) => assignment.blocking);
|
|
2533
|
+
const assignmentSource = effectiveAssignments.length > 0 ? effectiveAssignments : blockingAssignments;
|
|
2534
|
+
const unresolvedFromSource = assignmentSource.filter((assignment) => !assignment.assignedAgentId);
|
|
2535
|
+
if (unresolvedFromSource.length > 0) {
|
|
2536
|
+
return {
|
|
2537
|
+
runs: [],
|
|
2538
|
+
barrier: {
|
|
2539
|
+
statusCode: "helper-assignment-unresolved",
|
|
2540
|
+
detail: `No matching assignee exists for helper requests (${unresolvedFromSource.map((assignment) => assignment.requestId).join(", ")}).`,
|
|
2541
|
+
failures: unresolvedFromSource.map((assignment) => ({
|
|
2542
|
+
agentId: null,
|
|
2543
|
+
statusCode: "helper-assignment-unresolved",
|
|
2544
|
+
logPath: null,
|
|
2545
|
+
detail: assignment.assignmentDetail || assignment.summary || assignment.requestId,
|
|
2546
|
+
})),
|
|
2547
|
+
},
|
|
2548
|
+
};
|
|
2549
|
+
}
|
|
2550
|
+
const assignedAgentIds = new Set(
|
|
2551
|
+
assignmentSource.map((assignment) => assignment.assignedAgentId).filter(Boolean),
|
|
2552
|
+
);
|
|
2553
|
+
if (assignedAgentIds.size > 0) {
|
|
2554
|
+
return {
|
|
2555
|
+
runs: Array.from(assignedAgentIds)
|
|
2556
|
+
.map((agentId) => runsByAgentId.get(agentId))
|
|
2557
|
+
.filter(Boolean),
|
|
2558
|
+
barrier: null,
|
|
2559
|
+
};
|
|
2560
|
+
}
|
|
2561
|
+
const unresolvedInboundAssignments =
|
|
2562
|
+
derivedState?.dependencySnapshot?.unresolvedInboundAssignments || [];
|
|
2563
|
+
if (unresolvedInboundAssignments.length > 0) {
|
|
2564
|
+
return {
|
|
2565
|
+
runs: [],
|
|
2566
|
+
barrier: {
|
|
2567
|
+
statusCode: "dependency-assignment-unresolved",
|
|
2568
|
+
detail: `Required inbound dependencies are not assigned (${unresolvedInboundAssignments.map((record) => record.id).join(", ")}).`,
|
|
2569
|
+
failures: unresolvedInboundAssignments.map((record) => ({
|
|
2570
|
+
agentId: null,
|
|
2571
|
+
statusCode: "dependency-assignment-unresolved",
|
|
2572
|
+
logPath: null,
|
|
2573
|
+
detail: record.assignmentDetail || record.summary || record.id,
|
|
2574
|
+
})),
|
|
2575
|
+
},
|
|
2576
|
+
};
|
|
2577
|
+
}
|
|
2578
|
+
const inboundDependencyAgentIds = new Set(
|
|
2579
|
+
(derivedState?.dependencySnapshot?.openInbound || [])
|
|
2580
|
+
.map((record) => record.assignedAgentId)
|
|
2581
|
+
.filter(Boolean),
|
|
2582
|
+
);
|
|
2583
|
+
if (inboundDependencyAgentIds.size > 0) {
|
|
2584
|
+
return {
|
|
2585
|
+
runs: Array.from(inboundDependencyAgentIds)
|
|
2586
|
+
.map((agentId) => runsByAgentId.get(agentId))
|
|
2587
|
+
.filter(Boolean),
|
|
2588
|
+
barrier: null,
|
|
2589
|
+
};
|
|
2590
|
+
}
|
|
2591
|
+
const blockerAgentIds = new Set();
|
|
2592
|
+
for (const record of derivedState?.coordinationState?.blockers || []) {
|
|
2593
|
+
if (!isOpenCoordinationStatus(record.status)) {
|
|
2594
|
+
continue;
|
|
2595
|
+
}
|
|
2596
|
+
blockerAgentIds.add(record.agentId);
|
|
2597
|
+
for (const target of record.targets || []) {
|
|
2598
|
+
if (String(target).startsWith("agent:")) {
|
|
2599
|
+
blockerAgentIds.add(String(target).slice("agent:".length));
|
|
2600
|
+
}
|
|
2601
|
+
}
|
|
2602
|
+
}
|
|
2603
|
+
if (blockerAgentIds.size > 0) {
|
|
2604
|
+
return {
|
|
2605
|
+
runs: Array.from(blockerAgentIds)
|
|
2606
|
+
.map((agentId) => runsByAgentId.get(agentId))
|
|
2607
|
+
.filter(Boolean),
|
|
2608
|
+
barrier: null,
|
|
2609
|
+
};
|
|
2610
|
+
}
|
|
2611
|
+
if (derivedState?.ledger?.phase === "docs-closure") {
|
|
2612
|
+
return {
|
|
2613
|
+
runs: [runsByAgentId.get(lanePaths.documentationAgentId)].filter(Boolean),
|
|
2614
|
+
barrier: null,
|
|
2615
|
+
};
|
|
2616
|
+
}
|
|
2617
|
+
if (derivedState?.ledger?.phase === "evaluator-closure") {
|
|
2618
|
+
return {
|
|
2619
|
+
runs: [runsByAgentId.get(lanePaths.evaluatorAgentId)].filter(Boolean),
|
|
2620
|
+
barrier: null,
|
|
2621
|
+
};
|
|
2622
|
+
}
|
|
2623
|
+
if (derivedState?.ledger?.phase === "integrating") {
|
|
2624
|
+
return {
|
|
2625
|
+
runs: [runsByAgentId.get(lanePaths.integrationAgentId)].filter(Boolean),
|
|
2626
|
+
barrier: null,
|
|
2627
|
+
};
|
|
2628
|
+
}
|
|
2629
|
+
const failedAgentIds = new Set(failures.map((failure) => failure.agentId));
|
|
2630
|
+
return {
|
|
2631
|
+
runs: agentRuns.filter((run) => failedAgentIds.has(run.agent.agentId)),
|
|
2632
|
+
barrier: null,
|
|
2633
|
+
};
|
|
2634
|
+
}
|
|
2635
|
+
|
|
2636
|
+
function preflightWavesForExecutorAvailability(waves, lanePaths) {
|
|
2637
|
+
for (const wave of waves) {
|
|
2638
|
+
const mixValidation = validateWaveRuntimeMixAssignments(wave, {
|
|
2639
|
+
laneProfile: lanePaths.laneProfile,
|
|
2640
|
+
});
|
|
2641
|
+
if (!mixValidation.ok) {
|
|
2642
|
+
throw new Error(
|
|
2643
|
+
`Wave ${wave.wave} exceeds lane runtime mix targets (${mixValidation.detail})`,
|
|
2644
|
+
);
|
|
2645
|
+
}
|
|
2646
|
+
for (const agent of wave.agents) {
|
|
2647
|
+
const executorState = agent.executorResolved;
|
|
2648
|
+
if (!executorState) {
|
|
2649
|
+
continue;
|
|
2650
|
+
}
|
|
2651
|
+
const chain = [executorState.id, ...executorFallbackChain(executorState)];
|
|
2652
|
+
const availableExecutorId = chain.find((executorId) =>
|
|
2653
|
+
isExecutorCommandAvailable(commandForExecutor(executorState, executorId)),
|
|
2654
|
+
);
|
|
2655
|
+
if (!availableExecutorId) {
|
|
2656
|
+
throw new Error(
|
|
2657
|
+
`Agent ${agent.agentId} has no available executor command in its configured chain (${chain.join(" -> ")})`,
|
|
2658
|
+
);
|
|
2659
|
+
}
|
|
2660
|
+
}
|
|
2661
|
+
}
|
|
2662
|
+
}
|
|
2663
|
+
|
|
2664
|
+
export async function runLauncherCli(argv) {
|
|
2665
|
+
const parsed = parseArgs(argv);
|
|
2666
|
+
if (parsed.help) {
|
|
2667
|
+
printUsage(parsed.lanePaths);
|
|
2668
|
+
return;
|
|
2669
|
+
}
|
|
2670
|
+
const { lanePaths, options } = parsed;
|
|
2671
|
+
let lockHeld = false;
|
|
2672
|
+
let globalDashboard = null;
|
|
2673
|
+
let globalDashboardTerminalEntry = null;
|
|
2674
|
+
let globalDashboardTerminalAppended = false;
|
|
2675
|
+
let selectedWavesForCoordination = [];
|
|
2676
|
+
|
|
2677
|
+
const appendCoordination = ({
|
|
2678
|
+
event,
|
|
2679
|
+
waves = [],
|
|
2680
|
+
status = "info",
|
|
2681
|
+
details = "",
|
|
2682
|
+
actionRequested = "None",
|
|
2683
|
+
}) =>
|
|
2684
|
+
appendOrchestratorBoardEntry({
|
|
2685
|
+
boardPath: options.orchestratorBoardPath,
|
|
2686
|
+
lane: lanePaths.lane,
|
|
2687
|
+
orchestratorId: options.orchestratorId,
|
|
2688
|
+
event,
|
|
2689
|
+
waves,
|
|
2690
|
+
status,
|
|
2691
|
+
details,
|
|
2692
|
+
actionRequested,
|
|
2693
|
+
});
|
|
2694
|
+
|
|
2695
|
+
ensureDirectory(lanePaths.stateDir);
|
|
2696
|
+
ensureDirectory(lanePaths.promptsDir);
|
|
2697
|
+
ensureDirectory(lanePaths.logsDir);
|
|
2698
|
+
ensureDirectory(lanePaths.statusDir);
|
|
2699
|
+
ensureDirectory(lanePaths.messageboardsDir);
|
|
2700
|
+
ensureDirectory(lanePaths.dashboardsDir);
|
|
2701
|
+
ensureDirectory(lanePaths.coordinationDir);
|
|
2702
|
+
ensureDirectory(lanePaths.assignmentsDir);
|
|
2703
|
+
ensureDirectory(lanePaths.inboxesDir);
|
|
2704
|
+
ensureDirectory(lanePaths.ledgerDir);
|
|
2705
|
+
ensureDirectory(lanePaths.integrationDir);
|
|
2706
|
+
ensureDirectory(lanePaths.dependencySnapshotsDir);
|
|
2707
|
+
ensureDirectory(lanePaths.docsQueueDir);
|
|
2708
|
+
ensureDirectory(lanePaths.tracesDir);
|
|
2709
|
+
ensureDirectory(lanePaths.context7CacheDir);
|
|
2710
|
+
ensureDirectory(lanePaths.executorOverlaysDir);
|
|
2711
|
+
ensureDirectory(lanePaths.feedbackRequestsDir);
|
|
2712
|
+
ensureDirectory(lanePaths.feedbackTriageDir);
|
|
2713
|
+
ensureDirectory(lanePaths.crossLaneDependenciesDir);
|
|
2714
|
+
if (options.orchestratorBoardPath) {
|
|
2715
|
+
ensureOrchestratorBoard(options.orchestratorBoardPath);
|
|
2716
|
+
}
|
|
2717
|
+
|
|
2718
|
+
if (!options.reconcileStatus) {
|
|
2719
|
+
try {
|
|
2720
|
+
acquireLauncherLock(lanePaths.launcherLockPath, options);
|
|
2721
|
+
lockHeld = true;
|
|
2722
|
+
appendCoordination({
|
|
2723
|
+
event: "launcher_lock_acquired",
|
|
2724
|
+
status: "running",
|
|
2725
|
+
details: `lock=${path.relative(REPO_ROOT, lanePaths.launcherLockPath)}; pid=${process.pid}`,
|
|
2726
|
+
});
|
|
2727
|
+
} catch (error) {
|
|
2728
|
+
appendCoordination({
|
|
2729
|
+
event: "launcher_lock_blocked",
|
|
2730
|
+
status: "blocked",
|
|
2731
|
+
details: error instanceof Error ? error.message : String(error),
|
|
2732
|
+
actionRequested: `Lane ${lanePaths.lane} owner should wait for the active launcher to finish or choose another lane.`,
|
|
2733
|
+
});
|
|
2734
|
+
throw error;
|
|
2735
|
+
}
|
|
2736
|
+
}
|
|
2737
|
+
|
|
2738
|
+
try {
|
|
2739
|
+
const staleArtifactCleanup = reconcileStaleLauncherArtifacts(lanePaths);
|
|
2740
|
+
const context7BundleIndex = loadContext7BundleIndex(lanePaths.context7BundleIndexPath);
|
|
2741
|
+
const allWaves = parseWaveFiles(lanePaths.wavesDir, { laneProfile: lanePaths.laneProfile })
|
|
2742
|
+
.map((wave) =>
|
|
2743
|
+
applyExecutorSelectionsToWave(wave, {
|
|
2744
|
+
laneProfile: lanePaths.laneProfile,
|
|
2745
|
+
executorMode: options.executorMode,
|
|
2746
|
+
codexSandboxMode: options.codexSandboxMode,
|
|
2747
|
+
}),
|
|
2748
|
+
)
|
|
2749
|
+
.map((wave) =>
|
|
2750
|
+
({
|
|
2751
|
+
...applyContext7SelectionsToWave(wave, {
|
|
2752
|
+
lane: lanePaths.lane,
|
|
2753
|
+
bundleIndex: context7BundleIndex,
|
|
2754
|
+
}),
|
|
2755
|
+
evaluatorAgentId: lanePaths.evaluatorAgentId,
|
|
2756
|
+
integrationAgentId: lanePaths.integrationAgentId,
|
|
2757
|
+
documentationAgentId: lanePaths.documentationAgentId,
|
|
2758
|
+
}),
|
|
2759
|
+
)
|
|
2760
|
+
.map((wave) => validateWaveDefinition(wave, { laneProfile: lanePaths.laneProfile }));
|
|
2761
|
+
const reconciliation = reconcileRunStateFromStatusFiles(
|
|
2762
|
+
allWaves,
|
|
2763
|
+
options.runStatePath,
|
|
2764
|
+
lanePaths.statusDir,
|
|
2765
|
+
{
|
|
2766
|
+
logsDir: lanePaths.logsDir,
|
|
2767
|
+
coordinationDir: lanePaths.coordinationDir,
|
|
2768
|
+
evaluatorAgentId: lanePaths.evaluatorAgentId,
|
|
2769
|
+
integrationAgentId: lanePaths.integrationAgentId,
|
|
2770
|
+
documentationAgentId: lanePaths.documentationAgentId,
|
|
2771
|
+
requireExitContractsFromWave: lanePaths.requireExitContractsFromWave,
|
|
2772
|
+
requireIntegrationStewardFromWave: lanePaths.requireIntegrationStewardFromWave,
|
|
2773
|
+
requireComponentPromotionsFromWave: lanePaths.requireComponentPromotionsFromWave,
|
|
2774
|
+
laneProfile: lanePaths.laneProfile,
|
|
2775
|
+
},
|
|
2776
|
+
);
|
|
2777
|
+
if (options.reconcileStatus) {
|
|
2778
|
+
if (staleArtifactCleanup.removedLock) {
|
|
2779
|
+
console.log(
|
|
2780
|
+
`[reconcile] removed stale launcher lock: ${path.relative(REPO_ROOT, lanePaths.launcherLockPath)}`,
|
|
2781
|
+
);
|
|
2782
|
+
}
|
|
2783
|
+
if (staleArtifactCleanup.removedSessions.length > 0) {
|
|
2784
|
+
console.log(
|
|
2785
|
+
`[reconcile] removed stale lane tmux sessions: ${staleArtifactCleanup.removedSessions.join(", ")}`,
|
|
2786
|
+
);
|
|
2787
|
+
}
|
|
2788
|
+
if (staleArtifactCleanup.removedTerminalNames.length > 0) {
|
|
2789
|
+
console.log(
|
|
2790
|
+
`[reconcile] pruned stale lane terminal entries: ${staleArtifactCleanup.removedTerminalNames.join(", ")}`,
|
|
2791
|
+
);
|
|
2792
|
+
}
|
|
2793
|
+
if (staleArtifactCleanup.clearedDashboards) {
|
|
2794
|
+
const removedDashboards = staleArtifactCleanup.removedDashboardPaths.join(", ");
|
|
2795
|
+
const staleWaves =
|
|
2796
|
+
staleArtifactCleanup.staleWaves.length > 0
|
|
2797
|
+
? staleArtifactCleanup.staleWaves.join(", ")
|
|
2798
|
+
: "unknown";
|
|
2799
|
+
console.log(
|
|
2800
|
+
`[reconcile] cleared stale dashboard artifacts for wave(s) ${staleWaves}: ${removedDashboards}`,
|
|
2801
|
+
);
|
|
2802
|
+
} else if (staleArtifactCleanup.activeLockPid) {
|
|
2803
|
+
console.log(
|
|
2804
|
+
`[reconcile] stale dashboard cleanup skipped because launcher lock is active for pid ${staleArtifactCleanup.activeLockPid}`,
|
|
2805
|
+
);
|
|
2806
|
+
}
|
|
2807
|
+
const addedSummary =
|
|
2808
|
+
reconciliation.addedFromBefore.length > 0
|
|
2809
|
+
? reconciliation.addedFromBefore.join(", ")
|
|
2810
|
+
: "none";
|
|
2811
|
+
const completedSummary =
|
|
2812
|
+
reconciliation.state.completedWaves.length > 0
|
|
2813
|
+
? reconciliation.state.completedWaves.join(", ")
|
|
2814
|
+
: "none";
|
|
2815
|
+
console.log(`[reconcile] added from status files: ${addedSummary}`);
|
|
2816
|
+
console.log(`[reconcile] completed waves now: ${completedSummary}`);
|
|
2817
|
+
return;
|
|
2818
|
+
}
|
|
2819
|
+
|
|
2820
|
+
let effectiveStartWave = options.startWave;
|
|
2821
|
+
if (options.autoNext) {
|
|
2822
|
+
const { nextWave, state } = resolveAutoNextWaveStart(allWaves, options.runStatePath);
|
|
2823
|
+
if (nextWave === null) {
|
|
2824
|
+
console.log(
|
|
2825
|
+
`[auto-next] All known waves are already marked complete in ${path.relative(REPO_ROOT, options.runStatePath)}.`,
|
|
2826
|
+
);
|
|
2827
|
+
return;
|
|
2828
|
+
}
|
|
2829
|
+
const lastCompleted = state.completedWaves.at(-1) ?? "none";
|
|
2830
|
+
effectiveStartWave = nextWave;
|
|
2831
|
+
console.log(
|
|
2832
|
+
`[auto-next] last completed wave: ${lastCompleted}; starting from wave ${effectiveStartWave}.`,
|
|
2833
|
+
);
|
|
2834
|
+
}
|
|
2835
|
+
|
|
2836
|
+
const filteredWaves = allWaves.filter((wave) => {
|
|
2837
|
+
if (wave.wave < effectiveStartWave) {
|
|
2838
|
+
return false;
|
|
2839
|
+
}
|
|
2840
|
+
if (options.endWave !== null && wave.wave > options.endWave) {
|
|
2841
|
+
return false;
|
|
2842
|
+
}
|
|
2843
|
+
return true;
|
|
2844
|
+
});
|
|
2845
|
+
if (filteredWaves.length === 0) {
|
|
2846
|
+
throw new Error(
|
|
2847
|
+
`No waves available for range start=${effectiveStartWave}, end=${options.endWave ?? "last"}`,
|
|
2848
|
+
);
|
|
2849
|
+
}
|
|
2850
|
+
selectedWavesForCoordination = filteredWaves.map((wave) => wave.wave);
|
|
2851
|
+
|
|
2852
|
+
const manifest = buildManifest(lanePaths, allWaves);
|
|
2853
|
+
writeManifest(options.manifestOut, manifest);
|
|
2854
|
+
console.log(`Manifest written: ${path.relative(REPO_ROOT, options.manifestOut)}`);
|
|
2855
|
+
console.log(`Loaded ${manifest.docs.length} docs files and ${allWaves.length} wave files.`);
|
|
2856
|
+
appendCoordination({
|
|
2857
|
+
event: "launcher_start",
|
|
2858
|
+
waves: selectedWavesForCoordination,
|
|
2859
|
+
status: options.dryRun ? "dry-run" : "running",
|
|
2860
|
+
details: `pid=${process.pid}; range=${filteredWaves[0]?.wave ?? "?"}..${filteredWaves.at(-1)?.wave ?? "?"}; timeout_minutes=${options.timeoutMinutes}; retries=${options.maxRetriesPerWave}; ${options.coordinationNote ? `note=${options.coordinationNote}` : "note=n/a"}`,
|
|
2861
|
+
});
|
|
2862
|
+
|
|
2863
|
+
if (options.dryRun) {
|
|
2864
|
+
for (const wave of filteredWaves) {
|
|
2865
|
+
const derivedState = writeWaveDerivedState({
|
|
2866
|
+
lanePaths,
|
|
2867
|
+
wave,
|
|
2868
|
+
summariesByAgentId: {},
|
|
2869
|
+
feedbackRequests: [],
|
|
2870
|
+
attempt: 0,
|
|
2871
|
+
orchestratorId: options.orchestratorId,
|
|
2872
|
+
});
|
|
2873
|
+
const agentRuns = wave.agents.map((agent) => {
|
|
2874
|
+
const safeName = `wave-${wave.wave}-${agent.slug}`;
|
|
2875
|
+
return {
|
|
2876
|
+
agent,
|
|
2877
|
+
sessionName: `dry-run-wave-${wave.wave}-${agent.slug}`,
|
|
2878
|
+
promptPath: path.join(lanePaths.promptsDir, `${safeName}.prompt.md`),
|
|
2879
|
+
logPath: path.join(lanePaths.logsDir, `${safeName}.log`),
|
|
2880
|
+
statusPath: path.join(lanePaths.statusDir, `${safeName}.status`),
|
|
2881
|
+
messageBoardPath: derivedState.messageBoardPath,
|
|
2882
|
+
messageBoardSnapshot: derivedState.messageBoardText,
|
|
2883
|
+
sharedSummaryPath: derivedState.sharedSummaryPath,
|
|
2884
|
+
sharedSummaryText: derivedState.sharedSummaryText,
|
|
2885
|
+
inboxPath: derivedState.inboxesByAgentId[agent.agentId]?.path || null,
|
|
2886
|
+
inboxText: derivedState.inboxesByAgentId[agent.agentId]?.text || "",
|
|
2887
|
+
};
|
|
2888
|
+
});
|
|
2889
|
+
for (const runInfo of agentRuns) {
|
|
2890
|
+
await launchAgentSession(lanePaths, {
|
|
2891
|
+
wave: wave.wave,
|
|
2892
|
+
agent: runInfo.agent,
|
|
2893
|
+
sessionName: runInfo.sessionName,
|
|
2894
|
+
promptPath: runInfo.promptPath,
|
|
2895
|
+
logPath: runInfo.logPath,
|
|
2896
|
+
statusPath: runInfo.statusPath,
|
|
2897
|
+
messageBoardPath: runInfo.messageBoardPath,
|
|
2898
|
+
messageBoardSnapshot: runInfo.messageBoardSnapshot || "",
|
|
2899
|
+
sharedSummaryPath: runInfo.sharedSummaryPath,
|
|
2900
|
+
sharedSummaryText: runInfo.sharedSummaryText,
|
|
2901
|
+
inboxPath: runInfo.inboxPath,
|
|
2902
|
+
inboxText: runInfo.inboxText,
|
|
2903
|
+
orchestratorId: options.orchestratorId,
|
|
2904
|
+
agentRateLimitRetries: options.agentRateLimitRetries,
|
|
2905
|
+
agentRateLimitBaseDelaySeconds: options.agentRateLimitBaseDelaySeconds,
|
|
2906
|
+
agentRateLimitMaxDelaySeconds: options.agentRateLimitMaxDelaySeconds,
|
|
2907
|
+
context7Enabled: false,
|
|
2908
|
+
dryRun: true,
|
|
2909
|
+
});
|
|
2910
|
+
}
|
|
2911
|
+
}
|
|
2912
|
+
console.log(`[dry-run] state root: ${path.relative(REPO_ROOT, lanePaths.stateDir)}`);
|
|
2913
|
+
console.log(
|
|
2914
|
+
`[dry-run] prompts and executor overlays written: ${path.relative(REPO_ROOT, lanePaths.executorOverlaysDir)}`,
|
|
2915
|
+
);
|
|
2916
|
+
console.log("Dry run enabled, skipping tmux and executor launch.");
|
|
2917
|
+
return;
|
|
2918
|
+
}
|
|
2919
|
+
|
|
2920
|
+
preflightWavesForExecutorAvailability(filteredWaves, lanePaths);
|
|
2921
|
+
|
|
2922
|
+
globalDashboard = buildGlobalDashboardState({
|
|
2923
|
+
lane: lanePaths.lane,
|
|
2924
|
+
selectedWaves: filteredWaves,
|
|
2925
|
+
options,
|
|
2926
|
+
runStatePath: options.runStatePath,
|
|
2927
|
+
manifestOut: options.manifestOut,
|
|
2928
|
+
feedbackRequestsDir: lanePaths.feedbackRequestsDir,
|
|
2929
|
+
});
|
|
2930
|
+
writeGlobalDashboard(lanePaths.globalDashboardPath, globalDashboard);
|
|
2931
|
+
|
|
2932
|
+
if (!options.keepTerminals) {
|
|
2933
|
+
const removed = removeLaneTemporaryTerminalEntries(lanePaths.terminalsPath, lanePaths);
|
|
2934
|
+
if (removed > 0) {
|
|
2935
|
+
recordGlobalDashboardEvent(globalDashboard, {
|
|
2936
|
+
message: `Removed ${removed} stale temporary terminal entries for lane ${lanePaths.lane}.`,
|
|
2937
|
+
});
|
|
2938
|
+
}
|
|
2939
|
+
}
|
|
2940
|
+
|
|
2941
|
+
if (options.cleanupSessions) {
|
|
2942
|
+
const killed = cleanupLaneTmuxSessions(lanePaths);
|
|
2943
|
+
if (killed.length > 0) {
|
|
2944
|
+
recordGlobalDashboardEvent(globalDashboard, {
|
|
2945
|
+
message: `Pre-run cleanup removed ${killed.length} stale tmux sessions for lane ${lanePaths.lane}.`,
|
|
2946
|
+
});
|
|
2947
|
+
}
|
|
2948
|
+
}
|
|
2949
|
+
writeGlobalDashboard(lanePaths.globalDashboardPath, globalDashboard);
|
|
2950
|
+
|
|
2951
|
+
if (options.dashboard) {
|
|
2952
|
+
globalDashboardTerminalEntry = createGlobalDashboardTerminalEntry(
|
|
2953
|
+
lanePaths,
|
|
2954
|
+
globalDashboard.runId || "global",
|
|
2955
|
+
);
|
|
2956
|
+
appendTerminalEntries(lanePaths.terminalsPath, [globalDashboardTerminalEntry]);
|
|
2957
|
+
globalDashboardTerminalAppended = true;
|
|
2958
|
+
launchWaveDashboardSession(lanePaths, {
|
|
2959
|
+
sessionName: globalDashboardTerminalEntry.sessionName,
|
|
2960
|
+
dashboardPath: lanePaths.globalDashboardPath,
|
|
2961
|
+
});
|
|
2962
|
+
console.log(
|
|
2963
|
+
`[dashboard] tmux -L ${lanePaths.tmuxSocketName} attach -t ${globalDashboardTerminalEntry.sessionName}`,
|
|
2964
|
+
);
|
|
2965
|
+
}
|
|
2966
|
+
|
|
2967
|
+
for (const wave of filteredWaves) {
|
|
2968
|
+
console.log(`\n=== Wave ${wave.wave} (${wave.file}) ===`);
|
|
2969
|
+
console.log(`Agents: ${wave.agents.map((agent) => agent.agentId).join(", ")}`);
|
|
2970
|
+
|
|
2971
|
+
const globalWave = getGlobalWaveEntry(globalDashboard, wave.wave);
|
|
2972
|
+
if (globalWave) {
|
|
2973
|
+
globalWave.status = "running";
|
|
2974
|
+
globalWave.startedAt ||= toIsoTimestamp();
|
|
2975
|
+
}
|
|
2976
|
+
recordGlobalDashboardEvent(globalDashboard, {
|
|
2977
|
+
wave: wave.wave,
|
|
2978
|
+
message: `Starting wave ${wave.wave}.`,
|
|
2979
|
+
});
|
|
2980
|
+
appendCoordination({
|
|
2981
|
+
event: "wave_start",
|
|
2982
|
+
waves: [wave.wave],
|
|
2983
|
+
status: "running",
|
|
2984
|
+
details: `agents=${wave.agents.map((agent) => agent.agentId).join(", ")}; wave_file=${wave.file}`,
|
|
2985
|
+
});
|
|
2986
|
+
writeGlobalDashboard(lanePaths.globalDashboardPath, globalDashboard);
|
|
2987
|
+
|
|
2988
|
+
const runTag = crypto.randomBytes(3).toString("hex");
|
|
2989
|
+
let derivedState = writeWaveDerivedState({
|
|
2990
|
+
lanePaths,
|
|
2991
|
+
wave,
|
|
2992
|
+
summariesByAgentId: {},
|
|
2993
|
+
feedbackRequests: [],
|
|
2994
|
+
attempt: 0,
|
|
2995
|
+
orchestratorId: options.orchestratorId,
|
|
2996
|
+
});
|
|
2997
|
+
const messageBoardPath = derivedState.messageBoardPath;
|
|
2998
|
+
console.log(`Wave message board: ${path.relative(REPO_ROOT, messageBoardPath)}`);
|
|
2999
|
+
|
|
3000
|
+
const dashboardPath = path.join(lanePaths.dashboardsDir, `wave-${wave.wave}.json`);
|
|
3001
|
+
let dashboardState = null;
|
|
3002
|
+
let terminalEntries = [];
|
|
3003
|
+
let terminalsAppended = false;
|
|
3004
|
+
|
|
3005
|
+
const flushDashboards = () => {
|
|
3006
|
+
if (!dashboardState) {
|
|
3007
|
+
return;
|
|
3008
|
+
}
|
|
3009
|
+
writeWaveDashboard(dashboardPath, dashboardState);
|
|
3010
|
+
syncGlobalWaveFromWaveDashboard(globalDashboard, dashboardState);
|
|
3011
|
+
writeGlobalDashboard(lanePaths.globalDashboardPath, globalDashboard);
|
|
3012
|
+
};
|
|
3013
|
+
|
|
3014
|
+
const recordCombinedEvent = ({ level = "info", agentId = null, message }) => {
|
|
3015
|
+
if (!dashboardState) {
|
|
3016
|
+
return;
|
|
3017
|
+
}
|
|
3018
|
+
const globalMessagePrefix =
|
|
3019
|
+
typeof agentId === "string" && agentId.length > 0 ? "[" + agentId + "] " : "";
|
|
3020
|
+
recordWaveDashboardEvent(dashboardState, { level, agentId, message });
|
|
3021
|
+
recordGlobalDashboardEvent(globalDashboard, {
|
|
3022
|
+
level,
|
|
3023
|
+
wave: wave.wave,
|
|
3024
|
+
message: `${globalMessagePrefix}${message}`,
|
|
3025
|
+
});
|
|
3026
|
+
};
|
|
3027
|
+
|
|
3028
|
+
try {
|
|
3029
|
+
terminalEntries = createTemporaryTerminalEntries(
|
|
3030
|
+
lanePaths,
|
|
3031
|
+
wave.wave,
|
|
3032
|
+
wave.agents,
|
|
3033
|
+
runTag,
|
|
3034
|
+
options.dashboard,
|
|
3035
|
+
);
|
|
3036
|
+
appendTerminalEntries(lanePaths.terminalsPath, terminalEntries);
|
|
3037
|
+
terminalsAppended = true;
|
|
3038
|
+
|
|
3039
|
+
const agentRuns = wave.agents.map((agent) => {
|
|
3040
|
+
const safeName = `wave-${wave.wave}-${agent.slug}`;
|
|
3041
|
+
const terminalName = `${lanePaths.terminalNamePrefix}${wave.wave}-${agent.slug}`;
|
|
3042
|
+
const sessionName = terminalEntries.find(
|
|
3043
|
+
(entry) => entry.terminalName === terminalName,
|
|
3044
|
+
)?.sessionName;
|
|
3045
|
+
if (!sessionName) {
|
|
3046
|
+
throw new Error(`Failed to resolve session name for ${agent.agentId}`);
|
|
3047
|
+
}
|
|
3048
|
+
return {
|
|
3049
|
+
agent,
|
|
3050
|
+
sessionName,
|
|
3051
|
+
promptPath: path.join(lanePaths.promptsDir, `${safeName}.prompt.md`),
|
|
3052
|
+
logPath: path.join(lanePaths.logsDir, `${safeName}.log`),
|
|
3053
|
+
statusPath: path.join(lanePaths.statusDir, `${safeName}.status`),
|
|
3054
|
+
messageBoardPath,
|
|
3055
|
+
messageBoardSnapshot: derivedState.messageBoardText,
|
|
3056
|
+
sharedSummaryPath: derivedState.sharedSummaryPath,
|
|
3057
|
+
sharedSummaryText: derivedState.sharedSummaryText,
|
|
3058
|
+
inboxPath: derivedState.inboxesByAgentId[agent.agentId]?.path || null,
|
|
3059
|
+
inboxText: derivedState.inboxesByAgentId[agent.agentId]?.text || "",
|
|
3060
|
+
};
|
|
3061
|
+
});
|
|
3062
|
+
|
|
3063
|
+
const refreshDerivedState = (attemptNumber = 0) => {
|
|
3064
|
+
const summariesByAgentId = Object.fromEntries(
|
|
3065
|
+
agentRuns
|
|
3066
|
+
.map((run) => [run.agent.agentId, readRunExecutionSummary(run)])
|
|
3067
|
+
.filter(([, summary]) => summary),
|
|
3068
|
+
);
|
|
3069
|
+
const feedbackRequests = readWaveHumanFeedbackRequests({
|
|
3070
|
+
feedbackRequestsDir: lanePaths.feedbackRequestsDir,
|
|
3071
|
+
lane: lanePaths.lane,
|
|
3072
|
+
waveNumber: wave.wave,
|
|
3073
|
+
agentIds: agentRuns.map((run) => run.agent.agentId),
|
|
3074
|
+
orchestratorId: options.orchestratorId,
|
|
3075
|
+
});
|
|
3076
|
+
derivedState = writeWaveDerivedState({
|
|
3077
|
+
lanePaths,
|
|
3078
|
+
wave,
|
|
3079
|
+
agentRuns,
|
|
3080
|
+
summariesByAgentId,
|
|
3081
|
+
feedbackRequests,
|
|
3082
|
+
attempt: attemptNumber,
|
|
3083
|
+
orchestratorId: options.orchestratorId,
|
|
3084
|
+
});
|
|
3085
|
+
for (const run of agentRuns) {
|
|
3086
|
+
run.messageBoardSnapshot = derivedState.messageBoardText;
|
|
3087
|
+
run.sharedSummaryPath = derivedState.sharedSummaryPath;
|
|
3088
|
+
run.sharedSummaryText = derivedState.sharedSummaryText;
|
|
3089
|
+
run.inboxPath = derivedState.inboxesByAgentId[run.agent.agentId]?.path || null;
|
|
3090
|
+
run.inboxText = derivedState.inboxesByAgentId[run.agent.agentId]?.text || "";
|
|
3091
|
+
}
|
|
3092
|
+
applyDerivedStateToDashboard(dashboardState, derivedState);
|
|
3093
|
+
return derivedState;
|
|
3094
|
+
};
|
|
3095
|
+
|
|
3096
|
+
refreshDerivedState(0);
|
|
3097
|
+
|
|
3098
|
+
dashboardState = buildWaveDashboardState({
|
|
3099
|
+
lane: lanePaths.lane,
|
|
3100
|
+
wave: wave.wave,
|
|
3101
|
+
waveFile: wave.file,
|
|
3102
|
+
runTag,
|
|
3103
|
+
maxAttempts: options.maxRetriesPerWave + 1,
|
|
3104
|
+
messageBoardPath,
|
|
3105
|
+
agentRuns,
|
|
3106
|
+
});
|
|
3107
|
+
applyDerivedStateToDashboard(dashboardState, derivedState);
|
|
3108
|
+
|
|
3109
|
+
const preCompletedAgentIds = new Set(
|
|
3110
|
+
agentRuns
|
|
3111
|
+
.filter(
|
|
3112
|
+
(run) =>
|
|
3113
|
+
!isClosureAgentId(run.agent.agentId, lanePaths) &&
|
|
3114
|
+
hasReusableSuccessStatus(run.agent, run.statusPath),
|
|
3115
|
+
)
|
|
3116
|
+
.map((run) => run.agent.agentId),
|
|
3117
|
+
);
|
|
3118
|
+
for (const agentId of preCompletedAgentIds) {
|
|
3119
|
+
setWaveDashboardAgent(dashboardState, agentId, {
|
|
3120
|
+
state: "completed",
|
|
3121
|
+
exitCode: 0,
|
|
3122
|
+
completedAt: toIsoTimestamp(),
|
|
3123
|
+
detail: "Pre-existing status=0",
|
|
3124
|
+
});
|
|
3125
|
+
}
|
|
3126
|
+
const staleCompletedAgentIds = agentRuns
|
|
3127
|
+
.filter(
|
|
3128
|
+
(run) =>
|
|
3129
|
+
!preCompletedAgentIds.has(run.agent.agentId) &&
|
|
3130
|
+
readStatusCodeIfPresent(run.statusPath) === 0,
|
|
3131
|
+
)
|
|
3132
|
+
.map((run) => run.agent.agentId);
|
|
3133
|
+
for (const agentId of staleCompletedAgentIds) {
|
|
3134
|
+
setWaveDashboardAgent(dashboardState, agentId, {
|
|
3135
|
+
state: "pending",
|
|
3136
|
+
detail: "Stale status=0 ignored due to prompt drift or missing metadata",
|
|
3137
|
+
});
|
|
3138
|
+
}
|
|
3139
|
+
flushDashboards();
|
|
3140
|
+
|
|
3141
|
+
const dashboardEntry = terminalEntries.find(
|
|
3142
|
+
(entry) => entry.terminalName === `${lanePaths.dashboardTerminalNamePrefix}${wave.wave}`,
|
|
3143
|
+
);
|
|
3144
|
+
if (options.dashboard && dashboardEntry) {
|
|
3145
|
+
launchWaveDashboardSession(lanePaths, {
|
|
3146
|
+
sessionName: dashboardEntry.sessionName,
|
|
3147
|
+
dashboardPath,
|
|
3148
|
+
messageBoardPath,
|
|
3149
|
+
});
|
|
3150
|
+
}
|
|
3151
|
+
|
|
3152
|
+
let runsToLaunch = agentRuns.filter((run) => !preCompletedAgentIds.has(run.agent.agentId));
|
|
3153
|
+
let attempt = 1;
|
|
3154
|
+
const feedbackStateByRequestId = new Map();
|
|
3155
|
+
|
|
3156
|
+
while (attempt <= options.maxRetriesPerWave + 1) {
|
|
3157
|
+
refreshDerivedState(attempt - 1);
|
|
3158
|
+
dashboardState.attempt = attempt;
|
|
3159
|
+
updateWaveDashboardMessageBoard(dashboardState, messageBoardPath);
|
|
3160
|
+
flushDashboards();
|
|
3161
|
+
recordCombinedEvent({
|
|
3162
|
+
message: `Attempt ${attempt}/${options.maxRetriesPerWave + 1}; launching agents: ${runsToLaunch.map((run) => run.agent.agentId).join(", ") || "none"}`,
|
|
3163
|
+
});
|
|
3164
|
+
|
|
3165
|
+
const launchedImplementationRuns = runsToLaunch.filter(
|
|
3166
|
+
(run) =>
|
|
3167
|
+
![
|
|
3168
|
+
lanePaths.evaluatorAgentId,
|
|
3169
|
+
lanePaths.integrationAgentId,
|
|
3170
|
+
lanePaths.documentationAgentId,
|
|
3171
|
+
].includes(
|
|
3172
|
+
run.agent.agentId,
|
|
3173
|
+
),
|
|
3174
|
+
);
|
|
3175
|
+
const evaluatorOnlyRetry =
|
|
3176
|
+
runsToLaunch.length > 0 &&
|
|
3177
|
+
launchedImplementationRuns.length === 0 &&
|
|
3178
|
+
runsToLaunch.every((run) =>
|
|
3179
|
+
[
|
|
3180
|
+
lanePaths.evaluatorAgentId,
|
|
3181
|
+
lanePaths.integrationAgentId,
|
|
3182
|
+
lanePaths.documentationAgentId,
|
|
3183
|
+
].includes(
|
|
3184
|
+
run.agent.agentId,
|
|
3185
|
+
),
|
|
3186
|
+
);
|
|
3187
|
+
|
|
3188
|
+
let failures = [];
|
|
3189
|
+
let timedOut = false;
|
|
3190
|
+
if (evaluatorOnlyRetry) {
|
|
3191
|
+
const closureResult = await runClosureSweepPhase({
|
|
3192
|
+
lanePaths,
|
|
3193
|
+
wave,
|
|
3194
|
+
closureRuns: runsToLaunch,
|
|
3195
|
+
coordinationLogPath: derivedState.coordinationLogPath,
|
|
3196
|
+
refreshDerivedState,
|
|
3197
|
+
dashboardState,
|
|
3198
|
+
recordCombinedEvent,
|
|
3199
|
+
flushDashboards,
|
|
3200
|
+
options,
|
|
3201
|
+
feedbackStateByRequestId,
|
|
3202
|
+
appendCoordination,
|
|
3203
|
+
});
|
|
3204
|
+
failures = closureResult.failures;
|
|
3205
|
+
timedOut = closureResult.timedOut;
|
|
3206
|
+
} else {
|
|
3207
|
+
for (const runInfo of runsToLaunch) {
|
|
3208
|
+
const existing = dashboardState.agents.find(
|
|
3209
|
+
(entry) => entry.agentId === runInfo.agent.agentId,
|
|
3210
|
+
);
|
|
3211
|
+
setWaveDashboardAgent(dashboardState, runInfo.agent.agentId, {
|
|
3212
|
+
state: "launching",
|
|
3213
|
+
attempts: (existing?.attempts || 0) + 1,
|
|
3214
|
+
startedAt: existing?.startedAt || toIsoTimestamp(),
|
|
3215
|
+
completedAt: null,
|
|
3216
|
+
exitCode: null,
|
|
3217
|
+
detail: `Launching (attempt ${attempt})`,
|
|
3218
|
+
});
|
|
3219
|
+
flushDashboards();
|
|
3220
|
+
const launchResult = await launchAgentSession(lanePaths, {
|
|
3221
|
+
wave: wave.wave,
|
|
3222
|
+
agent: runInfo.agent,
|
|
3223
|
+
sessionName: runInfo.sessionName,
|
|
3224
|
+
promptPath: runInfo.promptPath,
|
|
3225
|
+
logPath: runInfo.logPath,
|
|
3226
|
+
statusPath: runInfo.statusPath,
|
|
3227
|
+
messageBoardPath: runInfo.messageBoardPath,
|
|
3228
|
+
messageBoardSnapshot: runInfo.messageBoardSnapshot || "",
|
|
3229
|
+
sharedSummaryPath: runInfo.sharedSummaryPath,
|
|
3230
|
+
sharedSummaryText: runInfo.sharedSummaryText,
|
|
3231
|
+
inboxPath: runInfo.inboxPath,
|
|
3232
|
+
inboxText: runInfo.inboxText,
|
|
3233
|
+
orchestratorId: options.orchestratorId,
|
|
3234
|
+
executorMode: options.executorMode,
|
|
3235
|
+
codexSandboxMode: options.codexSandboxMode,
|
|
3236
|
+
agentRateLimitRetries: options.agentRateLimitRetries,
|
|
3237
|
+
agentRateLimitBaseDelaySeconds: options.agentRateLimitBaseDelaySeconds,
|
|
3238
|
+
agentRateLimitMaxDelaySeconds: options.agentRateLimitMaxDelaySeconds,
|
|
3239
|
+
context7Enabled: options.context7Enabled,
|
|
3240
|
+
});
|
|
3241
|
+
runInfo.lastLaunchAttempt = attempt;
|
|
3242
|
+
runInfo.lastPromptHash = launchResult?.promptHash || null;
|
|
3243
|
+
runInfo.lastContext7 = launchResult?.context7 || null;
|
|
3244
|
+
runInfo.lastExecutorId = launchResult?.executorId || runInfo.agent.executorResolved?.id || null;
|
|
3245
|
+
setWaveDashboardAgent(dashboardState, runInfo.agent.agentId, {
|
|
3246
|
+
state: "running",
|
|
3247
|
+
detail: "Session launched",
|
|
3248
|
+
});
|
|
3249
|
+
recordCombinedEvent({
|
|
3250
|
+
agentId: runInfo.agent.agentId,
|
|
3251
|
+
message: `Launched in tmux session ${runInfo.sessionName}`,
|
|
3252
|
+
});
|
|
3253
|
+
const context7Mode = launchResult?.context7?.mode || "none";
|
|
3254
|
+
if (runInfo.agent.context7Resolved?.bundleId !== "none") {
|
|
3255
|
+
const librarySummary = describeContext7Libraries(runInfo.agent.context7Resolved);
|
|
3256
|
+
recordCombinedEvent({
|
|
3257
|
+
level:
|
|
3258
|
+
context7Mode === "fetched" || context7Mode === "cached" ? "info" : "warn",
|
|
3259
|
+
agentId: runInfo.agent.agentId,
|
|
3260
|
+
message:
|
|
3261
|
+
context7Mode === "fetched" || context7Mode === "cached"
|
|
3262
|
+
? `Context7 bundle ${runInfo.agent.context7Resolved.bundleId} attached (${context7Mode}); libraries=${librarySummary || "none"}`
|
|
3263
|
+
: `Context7 bundle ${runInfo.agent.context7Resolved.bundleId} not attached (${context7Mode})${launchResult?.context7?.warning ? `: ${launchResult.context7.warning}` : ""}`,
|
|
3264
|
+
});
|
|
3265
|
+
}
|
|
3266
|
+
flushDashboards();
|
|
3267
|
+
if (options.agentLaunchStaggerMs > 0) {
|
|
3268
|
+
await sleep(options.agentLaunchStaggerMs);
|
|
3269
|
+
}
|
|
3270
|
+
}
|
|
3271
|
+
|
|
3272
|
+
const waitResult = await waitForWaveCompletion(
|
|
3273
|
+
lanePaths,
|
|
3274
|
+
agentRuns,
|
|
3275
|
+
options.timeoutMinutes,
|
|
3276
|
+
({ pendingAgentIds }) => {
|
|
3277
|
+
refreshWaveDashboardAgentStates(dashboardState, agentRuns, pendingAgentIds, (event) =>
|
|
3278
|
+
recordCombinedEvent(event),
|
|
3279
|
+
);
|
|
3280
|
+
monitorWaveHumanFeedback({
|
|
3281
|
+
lanePaths,
|
|
3282
|
+
waveNumber: wave.wave,
|
|
3283
|
+
agentRuns,
|
|
3284
|
+
orchestratorId: options.orchestratorId,
|
|
3285
|
+
coordinationLogPath: derivedState.coordinationLogPath,
|
|
3286
|
+
feedbackStateByRequestId,
|
|
3287
|
+
recordCombinedEvent,
|
|
3288
|
+
appendCoordination,
|
|
3289
|
+
});
|
|
3290
|
+
updateWaveDashboardMessageBoard(dashboardState, messageBoardPath);
|
|
3291
|
+
flushDashboards();
|
|
3292
|
+
},
|
|
3293
|
+
);
|
|
3294
|
+
failures = waitResult.failures;
|
|
3295
|
+
timedOut = waitResult.timedOut;
|
|
3296
|
+
}
|
|
3297
|
+
|
|
3298
|
+
materializeAgentExecutionSummaries(wave, agentRuns);
|
|
3299
|
+
refreshDerivedState(attempt);
|
|
3300
|
+
|
|
3301
|
+
if (failures.length > 0) {
|
|
3302
|
+
for (const failure of failures) {
|
|
3303
|
+
if (!failure.detail) {
|
|
3304
|
+
continue;
|
|
3305
|
+
}
|
|
3306
|
+
recordCombinedEvent({
|
|
3307
|
+
level: "error",
|
|
3308
|
+
agentId: failure.agentId,
|
|
3309
|
+
message: failure.detail,
|
|
3310
|
+
});
|
|
3311
|
+
}
|
|
3312
|
+
}
|
|
3313
|
+
|
|
3314
|
+
if (failures.length === 0) {
|
|
3315
|
+
const implementationGate = readWaveImplementationGate(wave, agentRuns);
|
|
3316
|
+
if (!implementationGate.ok) {
|
|
3317
|
+
failures = [
|
|
3318
|
+
{
|
|
3319
|
+
agentId: implementationGate.agentId,
|
|
3320
|
+
statusCode: implementationGate.statusCode,
|
|
3321
|
+
logPath: implementationGate.logPath || path.relative(REPO_ROOT, messageBoardPath),
|
|
3322
|
+
},
|
|
3323
|
+
];
|
|
3324
|
+
recordCombinedEvent({
|
|
3325
|
+
level: "error",
|
|
3326
|
+
agentId: implementationGate.agentId,
|
|
3327
|
+
message: `Implementation exit contract blocked wave ${wave.wave}: ${implementationGate.detail}`,
|
|
3328
|
+
});
|
|
3329
|
+
appendCoordination({
|
|
3330
|
+
event: "wave_gate_blocked",
|
|
3331
|
+
waves: [wave.wave],
|
|
3332
|
+
status: "blocked",
|
|
3333
|
+
details: `agent=${implementationGate.agentId}; reason=${implementationGate.statusCode}; ${implementationGate.detail}`,
|
|
3334
|
+
actionRequested: `Lane ${lanePaths.lane} owners should resolve the implementation contract gap before wave progression.`,
|
|
3335
|
+
});
|
|
3336
|
+
} else {
|
|
3337
|
+
const componentGate = readWaveComponentGate(wave, agentRuns, {
|
|
3338
|
+
laneProfile: lanePaths.laneProfile,
|
|
3339
|
+
});
|
|
3340
|
+
if (!componentGate.ok) {
|
|
3341
|
+
failures = [
|
|
3342
|
+
{
|
|
3343
|
+
agentId: componentGate.agentId,
|
|
3344
|
+
statusCode: componentGate.statusCode,
|
|
3345
|
+
logPath:
|
|
3346
|
+
componentGate.logPath || path.relative(REPO_ROOT, messageBoardPath),
|
|
3347
|
+
},
|
|
3348
|
+
];
|
|
3349
|
+
recordCombinedEvent({
|
|
3350
|
+
level: "error",
|
|
3351
|
+
agentId: componentGate.agentId,
|
|
3352
|
+
message: `Component promotion blocked wave ${wave.wave}: ${componentGate.detail}`,
|
|
3353
|
+
});
|
|
3354
|
+
appendCoordination({
|
|
3355
|
+
event: "wave_gate_blocked",
|
|
3356
|
+
waves: [wave.wave],
|
|
3357
|
+
status: "blocked",
|
|
3358
|
+
details: `component=${componentGate.componentId || "unknown"}; reason=${componentGate.statusCode}; ${componentGate.detail}`,
|
|
3359
|
+
actionRequested: `Lane ${lanePaths.lane} owners should close the component promotion gap before wave progression.`,
|
|
3360
|
+
});
|
|
3361
|
+
} else if (launchedImplementationRuns.length > 0) {
|
|
3362
|
+
const helperAssignmentBarrier = readWaveAssignmentBarrier(derivedState);
|
|
3363
|
+
const dependencyBarrier = readWaveDependencyBarrier(derivedState);
|
|
3364
|
+
if (!helperAssignmentBarrier.ok) {
|
|
3365
|
+
failures = [
|
|
3366
|
+
{
|
|
3367
|
+
agentId: null,
|
|
3368
|
+
statusCode: helperAssignmentBarrier.statusCode,
|
|
3369
|
+
logPath: path.relative(REPO_ROOT, messageBoardPath),
|
|
3370
|
+
detail: helperAssignmentBarrier.detail,
|
|
3371
|
+
},
|
|
3372
|
+
];
|
|
3373
|
+
recordCombinedEvent({
|
|
3374
|
+
level: "error",
|
|
3375
|
+
message: `Helper assignment barrier blocked wave ${wave.wave}: ${helperAssignmentBarrier.detail}`,
|
|
3376
|
+
});
|
|
3377
|
+
appendCoordination({
|
|
3378
|
+
event: "wave_gate_blocked",
|
|
3379
|
+
waves: [wave.wave],
|
|
3380
|
+
status: "blocked",
|
|
3381
|
+
details: `reason=${helperAssignmentBarrier.statusCode}; ${helperAssignmentBarrier.detail}`,
|
|
3382
|
+
actionRequested: `Lane ${lanePaths.lane} owners should resolve helper assignments before closure.`,
|
|
3383
|
+
});
|
|
3384
|
+
} else if (!dependencyBarrier.ok) {
|
|
3385
|
+
failures = [
|
|
3386
|
+
{
|
|
3387
|
+
agentId: null,
|
|
3388
|
+
statusCode: dependencyBarrier.statusCode,
|
|
3389
|
+
logPath: path.relative(REPO_ROOT, messageBoardPath),
|
|
3390
|
+
detail: dependencyBarrier.detail,
|
|
3391
|
+
},
|
|
3392
|
+
];
|
|
3393
|
+
recordCombinedEvent({
|
|
3394
|
+
level: "error",
|
|
3395
|
+
message: `Dependency barrier blocked wave ${wave.wave}: ${dependencyBarrier.detail}`,
|
|
3396
|
+
});
|
|
3397
|
+
appendCoordination({
|
|
3398
|
+
event: "wave_gate_blocked",
|
|
3399
|
+
waves: [wave.wave],
|
|
3400
|
+
status: "blocked",
|
|
3401
|
+
details: `reason=${dependencyBarrier.statusCode}; ${dependencyBarrier.detail}`,
|
|
3402
|
+
actionRequested: `Lane ${lanePaths.lane} owners should resolve required dependency tickets before closure.`,
|
|
3403
|
+
});
|
|
3404
|
+
} else {
|
|
3405
|
+
recordCombinedEvent({
|
|
3406
|
+
message: `Implementation pass complete; running closure sweep for ${wave.wave}.`,
|
|
3407
|
+
});
|
|
3408
|
+
const closureResult = await runClosureSweepPhase({
|
|
3409
|
+
lanePaths,
|
|
3410
|
+
wave,
|
|
3411
|
+
closureRuns: agentRuns.filter((run) =>
|
|
3412
|
+
[
|
|
3413
|
+
lanePaths.evaluatorAgentId,
|
|
3414
|
+
lanePaths.integrationAgentId,
|
|
3415
|
+
lanePaths.documentationAgentId,
|
|
3416
|
+
].includes(
|
|
3417
|
+
run.agent.agentId,
|
|
3418
|
+
),
|
|
3419
|
+
),
|
|
3420
|
+
coordinationLogPath: derivedState.coordinationLogPath,
|
|
3421
|
+
refreshDerivedState,
|
|
3422
|
+
dashboardState,
|
|
3423
|
+
recordCombinedEvent,
|
|
3424
|
+
flushDashboards,
|
|
3425
|
+
options,
|
|
3426
|
+
feedbackStateByRequestId,
|
|
3427
|
+
appendCoordination,
|
|
3428
|
+
});
|
|
3429
|
+
failures = closureResult.failures;
|
|
3430
|
+
timedOut = timedOut || closureResult.timedOut;
|
|
3431
|
+
materializeAgentExecutionSummaries(wave, agentRuns);
|
|
3432
|
+
refreshDerivedState(attempt);
|
|
3433
|
+
}
|
|
3434
|
+
} else {
|
|
3435
|
+
recordCombinedEvent({
|
|
3436
|
+
message: "Implementation exit contracts and component promotions are satisfied.",
|
|
3437
|
+
});
|
|
3438
|
+
}
|
|
3439
|
+
}
|
|
3440
|
+
}
|
|
3441
|
+
|
|
3442
|
+
if (failures.length === 0) {
|
|
3443
|
+
const helperAssignmentBarrier = readWaveAssignmentBarrier(derivedState);
|
|
3444
|
+
if (!helperAssignmentBarrier.ok) {
|
|
3445
|
+
failures = [
|
|
3446
|
+
{
|
|
3447
|
+
agentId: null,
|
|
3448
|
+
statusCode: helperAssignmentBarrier.statusCode,
|
|
3449
|
+
logPath: path.relative(REPO_ROOT, messageBoardPath),
|
|
3450
|
+
detail: helperAssignmentBarrier.detail,
|
|
3451
|
+
},
|
|
3452
|
+
];
|
|
3453
|
+
recordCombinedEvent({
|
|
3454
|
+
level: "error",
|
|
3455
|
+
message: `Helper assignment barrier blocked wave ${wave.wave}: ${helperAssignmentBarrier.detail}`,
|
|
3456
|
+
});
|
|
3457
|
+
appendCoordination({
|
|
3458
|
+
event: "wave_gate_blocked",
|
|
3459
|
+
waves: [wave.wave],
|
|
3460
|
+
status: "blocked",
|
|
3461
|
+
details: `reason=${helperAssignmentBarrier.statusCode}; ${helperAssignmentBarrier.detail}`,
|
|
3462
|
+
actionRequested: `Lane ${lanePaths.lane} owners should resolve helper assignments before wave progression.`,
|
|
3463
|
+
});
|
|
3464
|
+
}
|
|
3465
|
+
}
|
|
3466
|
+
|
|
3467
|
+
if (failures.length === 0) {
|
|
3468
|
+
const dependencyBarrier = readWaveDependencyBarrier(derivedState);
|
|
3469
|
+
if (!dependencyBarrier.ok) {
|
|
3470
|
+
failures = [
|
|
3471
|
+
{
|
|
3472
|
+
agentId: null,
|
|
3473
|
+
statusCode: dependencyBarrier.statusCode,
|
|
3474
|
+
logPath: path.relative(REPO_ROOT, messageBoardPath),
|
|
3475
|
+
detail: dependencyBarrier.detail,
|
|
3476
|
+
},
|
|
3477
|
+
];
|
|
3478
|
+
recordCombinedEvent({
|
|
3479
|
+
level: "error",
|
|
3480
|
+
message: `Dependency barrier blocked wave ${wave.wave}: ${dependencyBarrier.detail}`,
|
|
3481
|
+
});
|
|
3482
|
+
appendCoordination({
|
|
3483
|
+
event: "wave_gate_blocked",
|
|
3484
|
+
waves: [wave.wave],
|
|
3485
|
+
status: "blocked",
|
|
3486
|
+
details: `reason=${dependencyBarrier.statusCode}; ${dependencyBarrier.detail}`,
|
|
3487
|
+
actionRequested: `Lane ${lanePaths.lane} owners should resolve required dependencies before wave progression.`,
|
|
3488
|
+
});
|
|
3489
|
+
}
|
|
3490
|
+
}
|
|
3491
|
+
|
|
3492
|
+
if (failures.length === 0) {
|
|
3493
|
+
const integrationGate = readWaveIntegrationGate(wave, agentRuns, {
|
|
3494
|
+
integrationAgentId: lanePaths.integrationAgentId,
|
|
3495
|
+
requireIntegrationStewardFromWave: lanePaths.requireIntegrationStewardFromWave,
|
|
3496
|
+
});
|
|
3497
|
+
if (!integrationGate.ok) {
|
|
3498
|
+
failures = [
|
|
3499
|
+
{
|
|
3500
|
+
agentId: integrationGate.agentId,
|
|
3501
|
+
statusCode: integrationGate.statusCode,
|
|
3502
|
+
logPath: integrationGate.logPath || path.relative(REPO_ROOT, messageBoardPath),
|
|
3503
|
+
},
|
|
3504
|
+
];
|
|
3505
|
+
recordCombinedEvent({
|
|
3506
|
+
level: "error",
|
|
3507
|
+
agentId: integrationGate.agentId,
|
|
3508
|
+
message: `Integration gate blocked wave ${wave.wave}: ${integrationGate.detail}`,
|
|
3509
|
+
});
|
|
3510
|
+
appendCoordination({
|
|
3511
|
+
event: "wave_gate_blocked",
|
|
3512
|
+
waves: [wave.wave],
|
|
3513
|
+
status: "blocked",
|
|
3514
|
+
details: `agent=${integrationGate.agentId}; reason=${integrationGate.statusCode}; ${integrationGate.detail}`,
|
|
3515
|
+
actionRequested: `Lane ${lanePaths.lane} owners should resolve integration contradictions or blockers before doc/evaluator closure.`,
|
|
3516
|
+
});
|
|
3517
|
+
}
|
|
3518
|
+
}
|
|
3519
|
+
|
|
3520
|
+
if (failures.length === 0) {
|
|
3521
|
+
const documentationGate = readWaveDocumentationGate(wave, agentRuns);
|
|
3522
|
+
if (!documentationGate.ok) {
|
|
3523
|
+
failures = [
|
|
3524
|
+
{
|
|
3525
|
+
agentId: documentationGate.agentId,
|
|
3526
|
+
statusCode: documentationGate.statusCode,
|
|
3527
|
+
logPath: documentationGate.logPath || path.relative(REPO_ROOT, messageBoardPath),
|
|
3528
|
+
},
|
|
3529
|
+
];
|
|
3530
|
+
recordCombinedEvent({
|
|
3531
|
+
level: "error",
|
|
3532
|
+
agentId: documentationGate.agentId,
|
|
3533
|
+
message: `Documentation closure blocked wave ${wave.wave}: ${documentationGate.detail}`,
|
|
3534
|
+
});
|
|
3535
|
+
appendCoordination({
|
|
3536
|
+
event: "wave_gate_blocked",
|
|
3537
|
+
waves: [wave.wave],
|
|
3538
|
+
status: "blocked",
|
|
3539
|
+
details: `agent=${documentationGate.agentId}; reason=${documentationGate.statusCode}; ${documentationGate.detail}`,
|
|
3540
|
+
actionRequested: `Lane ${lanePaths.lane} owners should resolve the shared-plan closure state before wave progression.`,
|
|
3541
|
+
});
|
|
3542
|
+
}
|
|
3543
|
+
}
|
|
3544
|
+
|
|
3545
|
+
if (failures.length === 0) {
|
|
3546
|
+
const componentMatrixGate = readWaveComponentMatrixGate(wave, agentRuns, {
|
|
3547
|
+
laneProfile: lanePaths.laneProfile,
|
|
3548
|
+
documentationAgentId: lanePaths.documentationAgentId,
|
|
3549
|
+
});
|
|
3550
|
+
if (!componentMatrixGate.ok) {
|
|
3551
|
+
failures = [
|
|
3552
|
+
{
|
|
3553
|
+
agentId: componentMatrixGate.agentId,
|
|
3554
|
+
statusCode: componentMatrixGate.statusCode,
|
|
3555
|
+
logPath:
|
|
3556
|
+
componentMatrixGate.logPath || path.relative(REPO_ROOT, messageBoardPath),
|
|
3557
|
+
},
|
|
3558
|
+
];
|
|
3559
|
+
recordCombinedEvent({
|
|
3560
|
+
level: "error",
|
|
3561
|
+
agentId: componentMatrixGate.agentId,
|
|
3562
|
+
message: `Component matrix update blocked wave ${wave.wave}: ${componentMatrixGate.detail}`,
|
|
3563
|
+
});
|
|
3564
|
+
appendCoordination({
|
|
3565
|
+
event: "wave_gate_blocked",
|
|
3566
|
+
waves: [wave.wave],
|
|
3567
|
+
status: "blocked",
|
|
3568
|
+
details: `component=${componentMatrixGate.componentId || "unknown"}; reason=${componentMatrixGate.statusCode}; ${componentMatrixGate.detail}`,
|
|
3569
|
+
actionRequested: `Lane ${lanePaths.lane} owners should update the component cutover matrix current levels before wave progression.`,
|
|
3570
|
+
});
|
|
3571
|
+
}
|
|
3572
|
+
}
|
|
3573
|
+
|
|
3574
|
+
if (failures.length === 0) {
|
|
3575
|
+
const evaluatorGate = readWaveEvaluatorGate(wave, agentRuns);
|
|
3576
|
+
if (!evaluatorGate.ok) {
|
|
3577
|
+
failures = [
|
|
3578
|
+
{
|
|
3579
|
+
agentId: evaluatorGate.agentId,
|
|
3580
|
+
statusCode: evaluatorGate.statusCode,
|
|
3581
|
+
logPath: evaluatorGate.logPath || path.relative(REPO_ROOT, messageBoardPath),
|
|
3582
|
+
},
|
|
3583
|
+
];
|
|
3584
|
+
recordCombinedEvent({
|
|
3585
|
+
level: "error",
|
|
3586
|
+
agentId: evaluatorGate.agentId,
|
|
3587
|
+
message: `Evaluator gate blocked wave ${wave.wave}: ${evaluatorGate.detail}`,
|
|
3588
|
+
});
|
|
3589
|
+
appendCoordination({
|
|
3590
|
+
event: "wave_gate_blocked",
|
|
3591
|
+
waves: [wave.wave],
|
|
3592
|
+
status: "blocked",
|
|
3593
|
+
details: `agent=${evaluatorGate.agentId}; reason=${evaluatorGate.statusCode}; ${evaluatorGate.detail}`,
|
|
3594
|
+
actionRequested: `Lane ${lanePaths.lane} owners should resolve the evaluator gate before wave progression.`,
|
|
3595
|
+
});
|
|
3596
|
+
} else {
|
|
3597
|
+
setWaveDashboardAgent(dashboardState, evaluatorGate.agentId, {
|
|
3598
|
+
detail: evaluatorGate.detail
|
|
3599
|
+
? `Exit 0; evaluator PASS (${evaluatorGate.detail})`
|
|
3600
|
+
: "Exit 0; evaluator PASS",
|
|
3601
|
+
});
|
|
3602
|
+
recordCombinedEvent({
|
|
3603
|
+
agentId: evaluatorGate.agentId,
|
|
3604
|
+
message: evaluatorGate.detail
|
|
3605
|
+
? `Evaluator verdict PASS: ${evaluatorGate.detail}`
|
|
3606
|
+
: "Evaluator verdict PASS.",
|
|
3607
|
+
});
|
|
3608
|
+
}
|
|
3609
|
+
}
|
|
3610
|
+
|
|
3611
|
+
if (failures.length === 0) {
|
|
3612
|
+
const infraGate = readWaveInfraGate(agentRuns);
|
|
3613
|
+
if (!infraGate.ok) {
|
|
3614
|
+
failures = [
|
|
3615
|
+
{
|
|
3616
|
+
agentId: infraGate.agentId,
|
|
3617
|
+
statusCode: infraGate.statusCode,
|
|
3618
|
+
logPath: infraGate.logPath || path.relative(REPO_ROOT, messageBoardPath),
|
|
3619
|
+
},
|
|
3620
|
+
];
|
|
3621
|
+
recordCombinedEvent({
|
|
3622
|
+
level: "error",
|
|
3623
|
+
agentId: infraGate.agentId,
|
|
3624
|
+
message: `Infra gate blocked wave ${wave.wave}: ${infraGate.detail}`,
|
|
3625
|
+
});
|
|
3626
|
+
appendCoordination({
|
|
3627
|
+
event: "wave_gate_blocked",
|
|
3628
|
+
waves: [wave.wave],
|
|
3629
|
+
status: "blocked",
|
|
3630
|
+
details: `agent=${infraGate.agentId}; reason=${infraGate.statusCode}; ${infraGate.detail}`,
|
|
3631
|
+
actionRequested: `Lane ${lanePaths.lane} owners should resolve the infra gate before wave progression.`,
|
|
3632
|
+
});
|
|
3633
|
+
}
|
|
3634
|
+
}
|
|
3635
|
+
|
|
3636
|
+
if (failures.length === 0) {
|
|
3637
|
+
const clarificationBarrier = readClarificationBarrier(derivedState);
|
|
3638
|
+
if (!clarificationBarrier.ok) {
|
|
3639
|
+
failures = [
|
|
3640
|
+
{
|
|
3641
|
+
agentId: lanePaths.integrationAgentId,
|
|
3642
|
+
statusCode: clarificationBarrier.statusCode,
|
|
3643
|
+
logPath: path.relative(REPO_ROOT, messageBoardPath),
|
|
3644
|
+
detail: clarificationBarrier.detail,
|
|
3645
|
+
},
|
|
3646
|
+
];
|
|
3647
|
+
recordCombinedEvent({
|
|
3648
|
+
level: "error",
|
|
3649
|
+
agentId: lanePaths.integrationAgentId,
|
|
3650
|
+
message: `Clarification barrier blocked wave ${wave.wave}: ${clarificationBarrier.detail}`,
|
|
3651
|
+
});
|
|
3652
|
+
appendCoordination({
|
|
3653
|
+
event: "wave_gate_blocked",
|
|
3654
|
+
waves: [wave.wave],
|
|
3655
|
+
status: "blocked",
|
|
3656
|
+
details: `reason=${clarificationBarrier.statusCode}; ${clarificationBarrier.detail}`,
|
|
3657
|
+
actionRequested: `Lane ${lanePaths.lane} owners should resolve open clarification chains before wave progression.`,
|
|
3658
|
+
});
|
|
3659
|
+
}
|
|
3660
|
+
}
|
|
3661
|
+
|
|
3662
|
+
const structuredSignals = Object.fromEntries(
|
|
3663
|
+
agentRuns.map((run) => [run.agent.agentId, parseStructuredSignalsFromLog(run.logPath)]),
|
|
3664
|
+
);
|
|
3665
|
+
const summariesByAgentId = Object.fromEntries(
|
|
3666
|
+
agentRuns
|
|
3667
|
+
.map((run) => [run.agent.agentId, readRunExecutionSummary(run)])
|
|
3668
|
+
.filter(([, summary]) => summary),
|
|
3669
|
+
);
|
|
3670
|
+
const gateSnapshot = buildGateSnapshot({
|
|
3671
|
+
wave,
|
|
3672
|
+
agentRuns,
|
|
3673
|
+
derivedState,
|
|
3674
|
+
lanePaths,
|
|
3675
|
+
});
|
|
3676
|
+
const traceDir = writeTraceBundle({
|
|
3677
|
+
tracesDir: lanePaths.tracesDir,
|
|
3678
|
+
lanePaths,
|
|
3679
|
+
launcherOptions: options,
|
|
3680
|
+
wave,
|
|
3681
|
+
attempt,
|
|
3682
|
+
manifest: buildManifest(lanePaths, [wave]),
|
|
3683
|
+
coordinationLogPath: derivedState.coordinationLogPath,
|
|
3684
|
+
coordinationState: derivedState.coordinationState,
|
|
3685
|
+
ledger: derivedState.ledger,
|
|
3686
|
+
docsQueue: derivedState.docsQueue,
|
|
3687
|
+
capabilityAssignments: derivedState.capabilityAssignments,
|
|
3688
|
+
dependencySnapshot: derivedState.dependencySnapshot,
|
|
3689
|
+
integrationSummary: derivedState.integrationSummary,
|
|
3690
|
+
integrationMarkdownPath: derivedState.integrationMarkdownPath,
|
|
3691
|
+
clarificationTriage: derivedState.clarificationTriage,
|
|
3692
|
+
agentRuns,
|
|
3693
|
+
structuredSignals,
|
|
3694
|
+
gateSnapshot,
|
|
3695
|
+
quality: buildQualityMetrics({
|
|
3696
|
+
tracesDir: lanePaths.tracesDir,
|
|
3697
|
+
wave,
|
|
3698
|
+
coordinationState: derivedState.coordinationState,
|
|
3699
|
+
integrationSummary: derivedState.integrationSummary,
|
|
3700
|
+
ledger: derivedState.ledger,
|
|
3701
|
+
docsQueue: derivedState.docsQueue,
|
|
3702
|
+
capabilityAssignments: derivedState.capabilityAssignments,
|
|
3703
|
+
dependencySnapshot: derivedState.dependencySnapshot,
|
|
3704
|
+
summariesByAgentId,
|
|
3705
|
+
agentRuns,
|
|
3706
|
+
gateSnapshot,
|
|
3707
|
+
attempt,
|
|
3708
|
+
coordinationLogPath: derivedState.coordinationLogPath,
|
|
3709
|
+
}),
|
|
3710
|
+
});
|
|
3711
|
+
|
|
3712
|
+
if (failures.length === 0) {
|
|
3713
|
+
dashboardState.status = "completed";
|
|
3714
|
+
recordCombinedEvent({ message: `Wave ${wave.wave} completed successfully.` });
|
|
3715
|
+
refreshWaveDashboardAgentStates(dashboardState, agentRuns, new Set(), (event) =>
|
|
3716
|
+
recordCombinedEvent(event),
|
|
3717
|
+
);
|
|
3718
|
+
updateWaveDashboardMessageBoard(dashboardState, messageBoardPath);
|
|
3719
|
+
flushDashboards();
|
|
3720
|
+
break;
|
|
3721
|
+
}
|
|
3722
|
+
|
|
3723
|
+
if (attempt >= options.maxRetriesPerWave + 1) {
|
|
3724
|
+
dashboardState.status = timedOut ? "timed_out" : "failed";
|
|
3725
|
+
for (const failure of failures) {
|
|
3726
|
+
setWaveDashboardAgent(dashboardState, failure.agentId, {
|
|
3727
|
+
state: timedOut ? "timed_out" : "failed",
|
|
3728
|
+
detail: failure.detail || `Exit ${failure.statusCode}`,
|
|
3729
|
+
});
|
|
3730
|
+
}
|
|
3731
|
+
flushDashboards();
|
|
3732
|
+
const details = failures
|
|
3733
|
+
.map(
|
|
3734
|
+
(failure) =>
|
|
3735
|
+
` - ${failure.agentId}: exit=${failure.statusCode}, log=${failure.logPath}${failure.detail ? `, detail=${failure.detail}` : ""}`,
|
|
3736
|
+
)
|
|
3737
|
+
.join("\n");
|
|
3738
|
+
const error = new Error(
|
|
3739
|
+
`Wave ${wave.wave} failed after ${attempt} attempt(s):\n${details}`,
|
|
3740
|
+
);
|
|
3741
|
+
if (
|
|
3742
|
+
failures.every(
|
|
3743
|
+
(failure) =>
|
|
3744
|
+
String(failure.statusCode).startsWith("evaluator-") ||
|
|
3745
|
+
String(failure.statusCode) === "missing-evaluator-verdict",
|
|
3746
|
+
)
|
|
3747
|
+
) {
|
|
3748
|
+
error.exitCode = 42;
|
|
3749
|
+
}
|
|
3750
|
+
throw error;
|
|
3751
|
+
}
|
|
3752
|
+
|
|
3753
|
+
const failedAgentIds = new Set(failures.map((failure) => failure.agentId));
|
|
3754
|
+
const failedList = Array.from(failedAgentIds).join(", ");
|
|
3755
|
+
console.warn(
|
|
3756
|
+
`[retry] Wave ${wave.wave} had failures for agents: ${failedList}. Evaluating safe relaunch targets.`,
|
|
3757
|
+
);
|
|
3758
|
+
appendCoordination({
|
|
3759
|
+
event: "wave_retry",
|
|
3760
|
+
waves: [wave.wave],
|
|
3761
|
+
status: "retrying",
|
|
3762
|
+
details: `attempt=${attempt + 1}/${options.maxRetriesPerWave + 1}; failed_agents=${failedList}; timed_out=${timedOut ? "yes" : "no"}`,
|
|
3763
|
+
actionRequested: `Lane ${lanePaths.lane} owners should inspect failed agent logs before retry completion.`,
|
|
3764
|
+
});
|
|
3765
|
+
const relaunchResolution = resolveRelaunchRuns(
|
|
3766
|
+
agentRuns,
|
|
3767
|
+
failures,
|
|
3768
|
+
derivedState,
|
|
3769
|
+
lanePaths,
|
|
3770
|
+
);
|
|
3771
|
+
if (relaunchResolution.barrier) {
|
|
3772
|
+
for (const failure of relaunchResolution.barrier.failures) {
|
|
3773
|
+
recordCombinedEvent({
|
|
3774
|
+
level: "error",
|
|
3775
|
+
agentId: failure.agentId,
|
|
3776
|
+
message: failure.detail,
|
|
3777
|
+
});
|
|
3778
|
+
setWaveDashboardAgent(dashboardState, failure.agentId, {
|
|
3779
|
+
state: "failed",
|
|
3780
|
+
detail: failure.detail,
|
|
3781
|
+
});
|
|
3782
|
+
}
|
|
3783
|
+
flushDashboards();
|
|
3784
|
+
appendCoordination({
|
|
3785
|
+
event: "wave_retry_blocked",
|
|
3786
|
+
waves: [wave.wave],
|
|
3787
|
+
status: "blocked",
|
|
3788
|
+
details: relaunchResolution.barrier.detail,
|
|
3789
|
+
actionRequested: `Lane ${lanePaths.lane} owners should resolve runtime policy or executor availability before retrying.`,
|
|
3790
|
+
});
|
|
3791
|
+
const error = new Error(
|
|
3792
|
+
`Wave ${wave.wave} retry blocked: ${relaunchResolution.barrier.detail}`,
|
|
3793
|
+
);
|
|
3794
|
+
error.exitCode = 43;
|
|
3795
|
+
throw error;
|
|
3796
|
+
}
|
|
3797
|
+
runsToLaunch = relaunchResolution.runs;
|
|
3798
|
+
if (runsToLaunch.length === 0) {
|
|
3799
|
+
const error = new Error(
|
|
3800
|
+
`Wave ${wave.wave} is waiting on human feedback or unresolved coordination state; no safe relaunch target is available.`,
|
|
3801
|
+
);
|
|
3802
|
+
error.exitCode = 43;
|
|
3803
|
+
throw error;
|
|
3804
|
+
}
|
|
3805
|
+
for (const run of runsToLaunch) {
|
|
3806
|
+
setWaveDashboardAgent(dashboardState, run.agent.agentId, {
|
|
3807
|
+
state: "pending",
|
|
3808
|
+
detail: "Queued for retry",
|
|
3809
|
+
});
|
|
3810
|
+
}
|
|
3811
|
+
flushDashboards();
|
|
3812
|
+
attempt += 1;
|
|
3813
|
+
}
|
|
3814
|
+
|
|
3815
|
+
const runState = markWaveCompleted(options.runStatePath, wave.wave);
|
|
3816
|
+
console.log(
|
|
3817
|
+
`[state] completed waves (${path.relative(REPO_ROOT, options.runStatePath)}): ${runState.completedWaves.join(", ") || "none"}`,
|
|
3818
|
+
);
|
|
3819
|
+
appendCoordination({
|
|
3820
|
+
event: "wave_complete",
|
|
3821
|
+
waves: [wave.wave],
|
|
3822
|
+
status: "completed",
|
|
3823
|
+
details: `attempts_used=${dashboardState?.attempt ?? "n/a"}; completed_waves=${runState.completedWaves.join(", ") || "none"}`,
|
|
3824
|
+
});
|
|
3825
|
+
} finally {
|
|
3826
|
+
if (terminalsAppended && !options.keepTerminals) {
|
|
3827
|
+
removeTerminalEntries(lanePaths.terminalsPath, terminalEntries);
|
|
3828
|
+
}
|
|
3829
|
+
if (options.cleanupSessions) {
|
|
3830
|
+
const excludeSessionNames = new Set();
|
|
3831
|
+
if (globalDashboardTerminalEntry) {
|
|
3832
|
+
excludeSessionNames.add(globalDashboardTerminalEntry.sessionName);
|
|
3833
|
+
}
|
|
3834
|
+
cleanupLaneTmuxSessions(lanePaths, { excludeSessionNames });
|
|
3835
|
+
}
|
|
3836
|
+
if (globalWave && globalWave.status === "running") {
|
|
3837
|
+
globalWave.status = dashboardState?.status || "failed";
|
|
3838
|
+
if (WAVE_TERMINAL_STATES.has(globalWave.status)) {
|
|
3839
|
+
globalWave.completedAt = toIsoTimestamp();
|
|
3840
|
+
}
|
|
3841
|
+
writeGlobalDashboard(lanePaths.globalDashboardPath, globalDashboard);
|
|
3842
|
+
}
|
|
3843
|
+
}
|
|
3844
|
+
}
|
|
3845
|
+
|
|
3846
|
+
globalDashboard.status = "completed";
|
|
3847
|
+
recordGlobalDashboardEvent(globalDashboard, { message: "All selected waves completed." });
|
|
3848
|
+
writeGlobalDashboard(lanePaths.globalDashboardPath, globalDashboard);
|
|
3849
|
+
appendCoordination({
|
|
3850
|
+
event: "launcher_finish",
|
|
3851
|
+
waves: selectedWavesForCoordination,
|
|
3852
|
+
status: "completed",
|
|
3853
|
+
details: "All selected waves completed successfully.",
|
|
3854
|
+
});
|
|
3855
|
+
} catch (error) {
|
|
3856
|
+
markLauncherFailed(
|
|
3857
|
+
globalDashboard,
|
|
3858
|
+
lanePaths,
|
|
3859
|
+
selectedWavesForCoordination,
|
|
3860
|
+
appendCoordination,
|
|
3861
|
+
error,
|
|
3862
|
+
);
|
|
3863
|
+
throw error;
|
|
3864
|
+
} finally {
|
|
3865
|
+
if (globalDashboardTerminalAppended && globalDashboardTerminalEntry && !options.keepTerminals) {
|
|
3866
|
+
removeTerminalEntries(lanePaths.terminalsPath, [globalDashboardTerminalEntry]);
|
|
3867
|
+
}
|
|
3868
|
+
if (options.cleanupSessions && globalDashboardTerminalEntry) {
|
|
3869
|
+
try {
|
|
3870
|
+
killTmuxSessionIfExists(lanePaths.tmuxSocketName, globalDashboardTerminalEntry.sessionName);
|
|
3871
|
+
} catch {
|
|
3872
|
+
// no-op
|
|
3873
|
+
}
|
|
3874
|
+
}
|
|
3875
|
+
if (lockHeld) {
|
|
3876
|
+
releaseLauncherLock(lanePaths.launcherLockPath);
|
|
3877
|
+
}
|
|
3878
|
+
}
|
|
3879
|
+
}
|