@weldr/runr 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +216 -0
- package/LICENSE +190 -0
- package/NOTICE +4 -0
- package/README.md +200 -0
- package/dist/cli.js +464 -0
- package/dist/commands/__tests__/report.test.js +202 -0
- package/dist/commands/compare.js +168 -0
- package/dist/commands/doctor.js +124 -0
- package/dist/commands/follow.js +251 -0
- package/dist/commands/gc.js +161 -0
- package/dist/commands/guards-only.js +89 -0
- package/dist/commands/metrics.js +441 -0
- package/dist/commands/orchestrate.js +800 -0
- package/dist/commands/paths.js +31 -0
- package/dist/commands/preflight.js +152 -0
- package/dist/commands/report.js +478 -0
- package/dist/commands/resume.js +149 -0
- package/dist/commands/run.js +538 -0
- package/dist/commands/status.js +189 -0
- package/dist/commands/summarize.js +220 -0
- package/dist/commands/version.js +82 -0
- package/dist/commands/wait.js +170 -0
- package/dist/config/__tests__/presets.test.js +104 -0
- package/dist/config/load.js +66 -0
- package/dist/config/schema.js +160 -0
- package/dist/context/__tests__/artifact.test.js +130 -0
- package/dist/context/__tests__/pack.test.js +191 -0
- package/dist/context/artifact.js +67 -0
- package/dist/context/index.js +2 -0
- package/dist/context/pack.js +273 -0
- package/dist/diagnosis/analyzer.js +678 -0
- package/dist/diagnosis/formatter.js +136 -0
- package/dist/diagnosis/index.js +6 -0
- package/dist/diagnosis/types.js +7 -0
- package/dist/env/__tests__/fingerprint.test.js +116 -0
- package/dist/env/fingerprint.js +111 -0
- package/dist/orchestrator/__tests__/policy.test.js +185 -0
- package/dist/orchestrator/__tests__/schema-version.test.js +65 -0
- package/dist/orchestrator/artifacts.js +405 -0
- package/dist/orchestrator/state-machine.js +646 -0
- package/dist/orchestrator/types.js +88 -0
- package/dist/ownership/normalize.js +45 -0
- package/dist/repo/context.js +90 -0
- package/dist/repo/git.js +13 -0
- package/dist/repo/worktree.js +239 -0
- package/dist/store/run-store.js +107 -0
- package/dist/store/run-utils.js +69 -0
- package/dist/store/runs-root.js +126 -0
- package/dist/supervisor/__tests__/evidence-gate.test.js +111 -0
- package/dist/supervisor/__tests__/ownership.test.js +103 -0
- package/dist/supervisor/__tests__/state-machine.test.js +290 -0
- package/dist/supervisor/collision.js +240 -0
- package/dist/supervisor/evidence-gate.js +98 -0
- package/dist/supervisor/planner.js +18 -0
- package/dist/supervisor/runner.js +1562 -0
- package/dist/supervisor/scope-guard.js +55 -0
- package/dist/supervisor/state-machine.js +121 -0
- package/dist/supervisor/verification-policy.js +64 -0
- package/dist/tasks/task-metadata.js +72 -0
- package/dist/types/schemas.js +1 -0
- package/dist/verification/engine.js +49 -0
- package/dist/workers/__tests__/claude.test.js +88 -0
- package/dist/workers/__tests__/codex.test.js +81 -0
- package/dist/workers/claude.js +119 -0
- package/dist/workers/codex.js +162 -0
- package/dist/workers/json.js +22 -0
- package/dist/workers/mock.js +193 -0
- package/dist/workers/prompts.js +98 -0
- package/dist/workers/schemas.js +39 -0
- package/package.json +47 -0
- package/templates/prompts/implementer.md +70 -0
- package/templates/prompts/planner.md +62 -0
- package/templates/prompts/reviewer.md +77 -0
|
@@ -0,0 +1,800 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* agent orchestrate - Run multiple tracks of tasks in parallel with collision-aware scheduling.
|
|
3
|
+
*
|
|
4
|
+
* Usage:
|
|
5
|
+
* agent orchestrate --config tracks.yaml --repo .
|
|
6
|
+
*
|
|
7
|
+
* The orchestrator:
|
|
8
|
+
* 1. Loads track configuration
|
|
9
|
+
* 2. Launches tracks in parallel (subject to collision policy)
|
|
10
|
+
* 3. Waits for runs to complete
|
|
11
|
+
* 4. Advances tracks to next step
|
|
12
|
+
* 5. Repeats until all tracks complete or fail
|
|
13
|
+
*/
|
|
14
|
+
import fs from 'node:fs';
|
|
15
|
+
import path from 'node:path';
|
|
16
|
+
import { spawn } from 'node:child_process';
|
|
17
|
+
import { createInterface } from 'node:readline';
|
|
18
|
+
import { loadOrchestrationConfig, createInitialOrchestratorState, makeScheduleDecision, startTrackRun, completeTrackStep, failTrack, getOrchestratorSummary, loadOrchestratorState, saveOrchestratorState, findLatestOrchestrationId, reconcileState, getEffectivePolicy, reserveOwnershipClaims } from '../orchestrator/state-machine.js';
|
|
19
|
+
import { writeTerminalArtifacts, getOrchestrationDir, buildWaitResult } from '../orchestrator/artifacts.js';
|
|
20
|
+
import { loadTaskMetadata } from '../tasks/task-metadata.js';
|
|
21
|
+
const POLL_INTERVAL_MS = 2000;
|
|
22
|
+
const RUN_LAUNCH_TIMEOUT_MS = 30000;
|
|
23
|
+
/**
|
|
24
|
+
* Run a shell command and capture output.
|
|
25
|
+
*/
|
|
26
|
+
function runAgentCommand(args, cwd) {
|
|
27
|
+
return new Promise((resolve) => {
|
|
28
|
+
const proc = spawn('npx', ['agent', ...args], {
|
|
29
|
+
cwd,
|
|
30
|
+
stdio: ['ignore', 'pipe', 'pipe'],
|
|
31
|
+
shell: true
|
|
32
|
+
});
|
|
33
|
+
let stdout = '';
|
|
34
|
+
let stderr = '';
|
|
35
|
+
proc.stdout.on('data', (data) => {
|
|
36
|
+
stdout += data.toString();
|
|
37
|
+
});
|
|
38
|
+
proc.stderr.on('data', (data) => {
|
|
39
|
+
stderr += data.toString();
|
|
40
|
+
});
|
|
41
|
+
proc.on('close', (code) => {
|
|
42
|
+
resolve({
|
|
43
|
+
stdout: stdout.trim(),
|
|
44
|
+
stderr: stderr.trim(),
|
|
45
|
+
exitCode: code ?? 1
|
|
46
|
+
});
|
|
47
|
+
});
|
|
48
|
+
proc.on('error', (err) => {
|
|
49
|
+
resolve({
|
|
50
|
+
stdout: '',
|
|
51
|
+
stderr: err.message,
|
|
52
|
+
exitCode: 1
|
|
53
|
+
});
|
|
54
|
+
});
|
|
55
|
+
});
|
|
56
|
+
}
|
|
57
|
+
function applyTaskOwnershipMetadata(state, repoPath) {
|
|
58
|
+
const errors = [];
|
|
59
|
+
const tracks = state.tracks.map((track) => {
|
|
60
|
+
const steps = track.steps.map((step) => {
|
|
61
|
+
if (step.owns_normalized !== undefined && step.owns_raw !== undefined) {
|
|
62
|
+
return step;
|
|
63
|
+
}
|
|
64
|
+
const taskPath = path.resolve(repoPath, step.task_path);
|
|
65
|
+
try {
|
|
66
|
+
const metadata = loadTaskMetadata(taskPath);
|
|
67
|
+
return {
|
|
68
|
+
...step,
|
|
69
|
+
owns_raw: metadata.owns_raw,
|
|
70
|
+
owns_normalized: metadata.owns_normalized
|
|
71
|
+
};
|
|
72
|
+
}
|
|
73
|
+
catch (err) {
|
|
74
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
75
|
+
errors.push(`${step.task_path}: ${message}`);
|
|
76
|
+
return step;
|
|
77
|
+
}
|
|
78
|
+
});
|
|
79
|
+
return { ...track, steps };
|
|
80
|
+
});
|
|
81
|
+
return { state: { ...state, tracks }, errors };
|
|
82
|
+
}
|
|
83
|
+
function collectMissingOwnership(state) {
|
|
84
|
+
const missing = [];
|
|
85
|
+
for (const track of state.tracks) {
|
|
86
|
+
for (const step of track.steps) {
|
|
87
|
+
if (!step.owns_normalized || step.owns_normalized.length === 0) {
|
|
88
|
+
missing.push({ track: track.name, task: step.task_path });
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
return missing;
|
|
93
|
+
}
|
|
94
|
+
function formatOwnershipMissingMessage(missing) {
|
|
95
|
+
const lines = [
|
|
96
|
+
'Parallel runs without worktrees require ownership declarations.',
|
|
97
|
+
'Single-task runs and --worktree runs do not require owns.',
|
|
98
|
+
'',
|
|
99
|
+
'Fix: Add YAML frontmatter to each task file:',
|
|
100
|
+
'',
|
|
101
|
+
' ---',
|
|
102
|
+
' owns:',
|
|
103
|
+
' - src/courses/my-course/',
|
|
104
|
+
' ---',
|
|
105
|
+
'',
|
|
106
|
+
'Or use --worktree for full isolation (recommended).',
|
|
107
|
+
'',
|
|
108
|
+
`Missing owns (${missing.length} task${missing.length === 1 ? '' : 's'}):`
|
|
109
|
+
];
|
|
110
|
+
for (const entry of missing) {
|
|
111
|
+
lines.push(` ${entry.task}`);
|
|
112
|
+
}
|
|
113
|
+
return lines.join('\n');
|
|
114
|
+
}
|
|
115
|
+
/**
|
|
116
|
+
* Launch an agent run and resolve when the JSON run_id is emitted.
|
|
117
|
+
* The process keeps running; we continue to drain output to avoid backpressure.
|
|
118
|
+
*/
|
|
119
|
+
function launchAgentRun(args, cwd, timeoutMs = RUN_LAUNCH_TIMEOUT_MS) {
|
|
120
|
+
return new Promise((resolve, reject) => {
|
|
121
|
+
const proc = spawn('npx', ['agent', ...args], {
|
|
122
|
+
cwd,
|
|
123
|
+
stdio: ['ignore', 'pipe', 'pipe'],
|
|
124
|
+
shell: true
|
|
125
|
+
});
|
|
126
|
+
let resolved = false;
|
|
127
|
+
let stderr = '';
|
|
128
|
+
const timeout = setTimeout(() => {
|
|
129
|
+
if (resolved)
|
|
130
|
+
return;
|
|
131
|
+
resolved = true;
|
|
132
|
+
reject(new Error('Timed out waiting for agent run_id'));
|
|
133
|
+
}, timeoutMs);
|
|
134
|
+
const stdoutLines = createInterface({ input: proc.stdout });
|
|
135
|
+
stdoutLines.on('line', (line) => {
|
|
136
|
+
if (resolved)
|
|
137
|
+
return;
|
|
138
|
+
const trimmed = line.trim();
|
|
139
|
+
if (!trimmed.startsWith('{'))
|
|
140
|
+
return;
|
|
141
|
+
try {
|
|
142
|
+
const output = JSON.parse(trimmed);
|
|
143
|
+
if (output.run_id && output.run_dir) {
|
|
144
|
+
resolved = true;
|
|
145
|
+
clearTimeout(timeout);
|
|
146
|
+
resolve({ runId: output.run_id, runDir: output.run_dir });
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
catch {
|
|
150
|
+
// Ignore non-JSON lines
|
|
151
|
+
}
|
|
152
|
+
});
|
|
153
|
+
proc.stderr.on('data', (data) => {
|
|
154
|
+
if (resolved)
|
|
155
|
+
return;
|
|
156
|
+
const chunk = data.toString();
|
|
157
|
+
if (stderr.length < 4096) {
|
|
158
|
+
stderr = (stderr + chunk).slice(-4096);
|
|
159
|
+
}
|
|
160
|
+
});
|
|
161
|
+
proc.on('error', (err) => {
|
|
162
|
+
if (resolved)
|
|
163
|
+
return;
|
|
164
|
+
resolved = true;
|
|
165
|
+
clearTimeout(timeout);
|
|
166
|
+
reject(err);
|
|
167
|
+
});
|
|
168
|
+
proc.on('close', (code) => {
|
|
169
|
+
if (resolved)
|
|
170
|
+
return;
|
|
171
|
+
resolved = true;
|
|
172
|
+
clearTimeout(timeout);
|
|
173
|
+
const message = stderr.trim() || `Exit code ${code ?? 1}`;
|
|
174
|
+
reject(new Error(message));
|
|
175
|
+
});
|
|
176
|
+
});
|
|
177
|
+
}
|
|
178
|
+
/**
|
|
179
|
+
* Launch a run for a track step.
|
|
180
|
+
*/
|
|
181
|
+
async function launchRun(taskPath, repoPath, options) {
|
|
182
|
+
const args = [
|
|
183
|
+
'run',
|
|
184
|
+
'--task', taskPath,
|
|
185
|
+
'--repo', repoPath,
|
|
186
|
+
'--time', String(options.time),
|
|
187
|
+
'--max-ticks', String(options.maxTicks),
|
|
188
|
+
'--json'
|
|
189
|
+
];
|
|
190
|
+
if (options.allowDeps)
|
|
191
|
+
args.push('--allow-deps');
|
|
192
|
+
if (options.worktree)
|
|
193
|
+
args.push('--worktree');
|
|
194
|
+
if (options.fast)
|
|
195
|
+
args.push('--fast');
|
|
196
|
+
if (options.forceParallel)
|
|
197
|
+
args.push('--force-parallel');
|
|
198
|
+
if (options.skipDoctor)
|
|
199
|
+
args.push('--skip-doctor');
|
|
200
|
+
if (options.autoResume)
|
|
201
|
+
args.push('--auto-resume');
|
|
202
|
+
try {
|
|
203
|
+
const output = await launchAgentRun(args, repoPath);
|
|
204
|
+
return {
|
|
205
|
+
runId: output.runId,
|
|
206
|
+
runDir: output.runDir
|
|
207
|
+
};
|
|
208
|
+
}
|
|
209
|
+
catch (err) {
|
|
210
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
211
|
+
return { error: message };
|
|
212
|
+
}
|
|
213
|
+
}
|
|
214
|
+
/**
|
|
215
|
+
* Wait for a run to complete.
|
|
216
|
+
*/
|
|
217
|
+
async function waitForRun(runId, repoPath) {
|
|
218
|
+
const args = [
|
|
219
|
+
'wait',
|
|
220
|
+
runId,
|
|
221
|
+
'--repo', repoPath,
|
|
222
|
+
'--for', 'terminal',
|
|
223
|
+
'--json'
|
|
224
|
+
];
|
|
225
|
+
const result = await runAgentCommand(args, repoPath);
|
|
226
|
+
try {
|
|
227
|
+
const output = JSON.parse(result.stdout);
|
|
228
|
+
return {
|
|
229
|
+
status: output.status,
|
|
230
|
+
stop_reason: output.stop_reason,
|
|
231
|
+
elapsed_ms: output.elapsed_ms
|
|
232
|
+
};
|
|
233
|
+
}
|
|
234
|
+
catch {
|
|
235
|
+
return {
|
|
236
|
+
status: 'stopped',
|
|
237
|
+
stop_reason: 'wait_parse_error',
|
|
238
|
+
elapsed_ms: 0
|
|
239
|
+
};
|
|
240
|
+
}
|
|
241
|
+
}
|
|
242
|
+
/**
|
|
243
|
+
* Save orchestrator state to disk and write terminal artifacts if complete.
|
|
244
|
+
*/
|
|
245
|
+
function saveState(state, repoPath) {
|
|
246
|
+
saveOrchestratorState(state, repoPath);
|
|
247
|
+
// Write terminal artifacts if orchestration is complete or stopped
|
|
248
|
+
if (state.status !== 'running') {
|
|
249
|
+
writeTerminalArtifacts(state, repoPath);
|
|
250
|
+
}
|
|
251
|
+
}
|
|
252
|
+
/**
|
|
253
|
+
* Print orchestrator status summary.
|
|
254
|
+
*/
|
|
255
|
+
function printStatus(state) {
|
|
256
|
+
const summary = getOrchestratorSummary(state);
|
|
257
|
+
const elapsed = Date.now() - new Date(state.started_at).getTime();
|
|
258
|
+
const elapsedMin = Math.round(elapsed / 60000);
|
|
259
|
+
console.log('');
|
|
260
|
+
console.log(`Orchestrator: ${state.orchestrator_id} (${elapsedMin}m elapsed)`);
|
|
261
|
+
console.log(`Status: ${state.status.toUpperCase()}`);
|
|
262
|
+
console.log(`Progress: ${summary.completed_steps}/${summary.total_steps} steps`);
|
|
263
|
+
console.log(`Tracks: ${summary.complete} complete, ${summary.running} running, ${summary.pending} pending, ${summary.stopped} stopped, ${summary.failed} failed`);
|
|
264
|
+
console.log('');
|
|
265
|
+
for (const track of state.tracks) {
|
|
266
|
+
const step = track.steps[track.current_step] ?? track.steps[track.steps.length - 1];
|
|
267
|
+
const stepInfo = step?.run_id ? ` (${step.run_id})` : '';
|
|
268
|
+
const errorInfo = track.error ? ` - ${track.error}` : '';
|
|
269
|
+
console.log(` ${track.name}: ${track.status}${stepInfo}${errorInfo}`);
|
|
270
|
+
}
|
|
271
|
+
console.log('');
|
|
272
|
+
}
|
|
273
|
+
function sleep(ms) {
|
|
274
|
+
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
275
|
+
}
|
|
276
|
+
export async function orchestrateCommand(options) {
|
|
277
|
+
const configPath = path.resolve(options.config);
|
|
278
|
+
const repoPath = path.resolve(options.repo);
|
|
279
|
+
// Load and validate config
|
|
280
|
+
let config;
|
|
281
|
+
try {
|
|
282
|
+
config = loadOrchestrationConfig(configPath);
|
|
283
|
+
}
|
|
284
|
+
catch (err) {
|
|
285
|
+
console.error(`Failed to load orchestration config: ${err}`);
|
|
286
|
+
process.exitCode = 1;
|
|
287
|
+
return;
|
|
288
|
+
}
|
|
289
|
+
console.log(`Loaded ${config.tracks.length} track(s) from ${configPath}`);
|
|
290
|
+
// Create initial state
|
|
291
|
+
const ownershipRequired = !options.worktree && config.tracks.length > 1;
|
|
292
|
+
let state = createInitialOrchestratorState(config, repoPath, {
|
|
293
|
+
timeBudgetMinutes: options.time,
|
|
294
|
+
maxTicks: options.maxTicks,
|
|
295
|
+
collisionPolicy: options.collisionPolicy,
|
|
296
|
+
fast: options.fast,
|
|
297
|
+
ownershipRequired
|
|
298
|
+
});
|
|
299
|
+
console.log(`Orchestrator ID: ${state.orchestrator_id}`);
|
|
300
|
+
console.log(`Collision policy: ${options.collisionPolicy}`);
|
|
301
|
+
console.log('');
|
|
302
|
+
if (options.dryRun) {
|
|
303
|
+
console.log('Dry run - showing planned execution:');
|
|
304
|
+
for (const track of state.tracks) {
|
|
305
|
+
console.log(` Track "${track.name}":`);
|
|
306
|
+
for (const step of track.steps) {
|
|
307
|
+
console.log(` - ${step.task_path}`);
|
|
308
|
+
}
|
|
309
|
+
}
|
|
310
|
+
return;
|
|
311
|
+
}
|
|
312
|
+
const ownershipApplied = applyTaskOwnershipMetadata(state, repoPath);
|
|
313
|
+
if (ownershipApplied.errors.length > 0) {
|
|
314
|
+
console.error('Failed to parse task ownership metadata:');
|
|
315
|
+
for (const err of ownershipApplied.errors) {
|
|
316
|
+
console.error(` - ${err}`);
|
|
317
|
+
}
|
|
318
|
+
process.exitCode = 1;
|
|
319
|
+
return;
|
|
320
|
+
}
|
|
321
|
+
state = ownershipApplied.state;
|
|
322
|
+
if (ownershipRequired) {
|
|
323
|
+
const missing = collectMissingOwnership(state);
|
|
324
|
+
if (missing.length > 0) {
|
|
325
|
+
console.error(formatOwnershipMissingMessage(missing));
|
|
326
|
+
process.exitCode = 1;
|
|
327
|
+
return;
|
|
328
|
+
}
|
|
329
|
+
}
|
|
330
|
+
saveState(state, repoPath);
|
|
331
|
+
// Track active run promises for concurrent waiting
|
|
332
|
+
const activeWaits = new Map();
|
|
333
|
+
// Main orchestration loop
|
|
334
|
+
while (state.status === 'running') {
|
|
335
|
+
const decision = makeScheduleDecision(state);
|
|
336
|
+
switch (decision.action) {
|
|
337
|
+
case 'done':
|
|
338
|
+
state.status = state.tracks.every((t) => t.status === 'complete')
|
|
339
|
+
? 'complete'
|
|
340
|
+
: 'stopped';
|
|
341
|
+
state.ended_at = new Date().toISOString();
|
|
342
|
+
break;
|
|
343
|
+
case 'blocked':
|
|
344
|
+
console.error(`BLOCKED: ${decision.reason}`);
|
|
345
|
+
if (decision.track_id) {
|
|
346
|
+
state = failTrack(state, decision.track_id, decision.reason ?? 'blocked');
|
|
347
|
+
}
|
|
348
|
+
break;
|
|
349
|
+
case 'launch': {
|
|
350
|
+
const trackId = decision.track_id;
|
|
351
|
+
const track = state.tracks.find((t) => t.id === trackId);
|
|
352
|
+
const step = track.steps[track.current_step];
|
|
353
|
+
const policy = getEffectivePolicy(state);
|
|
354
|
+
if (policy.ownership_required) {
|
|
355
|
+
const ownsRaw = step.owns_raw ?? [];
|
|
356
|
+
const ownsNormalized = step.owns_normalized ?? [];
|
|
357
|
+
const reservation = reserveOwnershipClaims(state, trackId, ownsRaw, ownsNormalized);
|
|
358
|
+
if (reservation.conflicts.length > 0) {
|
|
359
|
+
const conflictList = reservation.conflicts.join(', ');
|
|
360
|
+
const message = `Ownership claim conflict for ${step.task_path}: ${conflictList}`;
|
|
361
|
+
console.error(` ${message}`);
|
|
362
|
+
state = failTrack(state, trackId, message);
|
|
363
|
+
saveState(state, repoPath);
|
|
364
|
+
break;
|
|
365
|
+
}
|
|
366
|
+
state = reservation.state;
|
|
367
|
+
}
|
|
368
|
+
console.log(`Launching: ${track.name} - ${step.task_path} (fast=${options.fast})`);
|
|
369
|
+
const launchResult = await launchRun(step.task_path, repoPath, {
|
|
370
|
+
time: options.time,
|
|
371
|
+
maxTicks: options.maxTicks,
|
|
372
|
+
allowDeps: options.allowDeps,
|
|
373
|
+
worktree: options.worktree,
|
|
374
|
+
fast: options.fast,
|
|
375
|
+
forceParallel: options.collisionPolicy === 'force',
|
|
376
|
+
autoResume: options.autoResume
|
|
377
|
+
});
|
|
378
|
+
if ('error' in launchResult) {
|
|
379
|
+
console.error(` Failed to launch: ${launchResult.error}`);
|
|
380
|
+
state = failTrack(state, trackId, launchResult.error);
|
|
381
|
+
}
|
|
382
|
+
else {
|
|
383
|
+
console.log(` Started run: ${launchResult.runId}`);
|
|
384
|
+
state = startTrackRun(state, trackId, launchResult.runId, launchResult.runDir);
|
|
385
|
+
// Start waiting for this run in the background
|
|
386
|
+
const waitPromise = waitForRun(launchResult.runId, repoPath);
|
|
387
|
+
activeWaits.set(trackId, waitPromise);
|
|
388
|
+
}
|
|
389
|
+
saveState(state, repoPath);
|
|
390
|
+
break;
|
|
391
|
+
}
|
|
392
|
+
case 'wait':
|
|
393
|
+
// Check if any active waits have completed
|
|
394
|
+
if (activeWaits.size > 0) {
|
|
395
|
+
// Race all active waits
|
|
396
|
+
const entries = [...activeWaits.entries()];
|
|
397
|
+
const raceResult = await Promise.race(entries.map(async ([trackId, promise]) => {
|
|
398
|
+
const result = await promise;
|
|
399
|
+
return { trackId, result };
|
|
400
|
+
}));
|
|
401
|
+
const { trackId, result } = raceResult;
|
|
402
|
+
const track = state.tracks.find((t) => t.id === trackId);
|
|
403
|
+
console.log(`Completed: ${track.name} - ${result.status}`);
|
|
404
|
+
if (result.stop_reason) {
|
|
405
|
+
console.log(` Stop reason: ${result.stop_reason}`);
|
|
406
|
+
}
|
|
407
|
+
state = completeTrackStep(state, trackId, result);
|
|
408
|
+
activeWaits.delete(trackId);
|
|
409
|
+
saveState(state, repoPath);
|
|
410
|
+
}
|
|
411
|
+
else {
|
|
412
|
+
// No active waits, just poll
|
|
413
|
+
await sleep(POLL_INTERVAL_MS);
|
|
414
|
+
}
|
|
415
|
+
break;
|
|
416
|
+
}
|
|
417
|
+
printStatus(state);
|
|
418
|
+
}
|
|
419
|
+
// Final summary
|
|
420
|
+
console.log('='.repeat(60));
|
|
421
|
+
console.log('ORCHESTRATION COMPLETE');
|
|
422
|
+
printStatus(state);
|
|
423
|
+
const summary = getOrchestratorSummary(state);
|
|
424
|
+
if (summary.failed > 0 || summary.stopped > 0) {
|
|
425
|
+
process.exitCode = 1;
|
|
426
|
+
}
|
|
427
|
+
}
|
|
428
|
+
/**
|
|
429
|
+
* Apply policy overrides to state.
|
|
430
|
+
* Returns the updated state and a list of what was changed.
|
|
431
|
+
*/
|
|
432
|
+
function applyPolicyOverrides(state, overrides) {
|
|
433
|
+
const applied = [];
|
|
434
|
+
const policy = getEffectivePolicy(state);
|
|
435
|
+
// Create a mutable copy of policy
|
|
436
|
+
const newPolicy = { ...policy };
|
|
437
|
+
if (overrides.time !== undefined && overrides.time !== policy.time_budget_minutes) {
|
|
438
|
+
applied.push({
|
|
439
|
+
field: 'time_budget_minutes',
|
|
440
|
+
from: policy.time_budget_minutes,
|
|
441
|
+
to: overrides.time
|
|
442
|
+
});
|
|
443
|
+
newPolicy.time_budget_minutes = overrides.time;
|
|
444
|
+
}
|
|
445
|
+
if (overrides.maxTicks !== undefined && overrides.maxTicks !== policy.max_ticks) {
|
|
446
|
+
applied.push({
|
|
447
|
+
field: 'max_ticks',
|
|
448
|
+
from: policy.max_ticks,
|
|
449
|
+
to: overrides.maxTicks
|
|
450
|
+
});
|
|
451
|
+
newPolicy.max_ticks = overrides.maxTicks;
|
|
452
|
+
}
|
|
453
|
+
if (overrides.fast !== undefined && overrides.fast !== policy.fast) {
|
|
454
|
+
applied.push({
|
|
455
|
+
field: 'fast',
|
|
456
|
+
from: policy.fast,
|
|
457
|
+
to: overrides.fast
|
|
458
|
+
});
|
|
459
|
+
newPolicy.fast = overrides.fast;
|
|
460
|
+
}
|
|
461
|
+
if (overrides.collisionPolicy !== undefined && overrides.collisionPolicy !== policy.collision_policy) {
|
|
462
|
+
applied.push({
|
|
463
|
+
field: 'collision_policy',
|
|
464
|
+
from: policy.collision_policy,
|
|
465
|
+
to: overrides.collisionPolicy
|
|
466
|
+
});
|
|
467
|
+
newPolicy.collision_policy = overrides.collisionPolicy;
|
|
468
|
+
}
|
|
469
|
+
if (applied.length === 0) {
|
|
470
|
+
return { state, applied: [] };
|
|
471
|
+
}
|
|
472
|
+
// Update state with new policy
|
|
473
|
+
const newState = {
|
|
474
|
+
...state,
|
|
475
|
+
policy: newPolicy,
|
|
476
|
+
// Also update legacy fields for backward compat
|
|
477
|
+
collision_policy: newPolicy.collision_policy,
|
|
478
|
+
time_budget_minutes: newPolicy.time_budget_minutes,
|
|
479
|
+
max_ticks: newPolicy.max_ticks,
|
|
480
|
+
fast: newPolicy.fast
|
|
481
|
+
};
|
|
482
|
+
return { state: newState, applied };
|
|
483
|
+
}
|
|
484
|
+
/**
|
|
485
|
+
* Resume a previously started orchestration.
|
|
486
|
+
*
|
|
487
|
+
* On resume:
|
|
488
|
+
* 1. Load saved state from disk
|
|
489
|
+
* 2. Reconcile active runs (probe for completion)
|
|
490
|
+
* 3. Continue scheduling from current state
|
|
491
|
+
*/
|
|
492
|
+
export async function resumeOrchestrationCommand(options) {
|
|
493
|
+
const repoPath = path.resolve(options.repo);
|
|
494
|
+
// Resolve "latest" to actual ID
|
|
495
|
+
let orchestratorId = options.orchestratorId;
|
|
496
|
+
if (orchestratorId === 'latest') {
|
|
497
|
+
const latest = findLatestOrchestrationId(repoPath);
|
|
498
|
+
if (!latest) {
|
|
499
|
+
console.error('No orchestrations found');
|
|
500
|
+
process.exitCode = 1;
|
|
501
|
+
return;
|
|
502
|
+
}
|
|
503
|
+
orchestratorId = latest;
|
|
504
|
+
}
|
|
505
|
+
// Load saved state
|
|
506
|
+
let state = loadOrchestratorState(orchestratorId, repoPath);
|
|
507
|
+
if (!state) {
|
|
508
|
+
console.error(`Orchestration not found: ${orchestratorId}`);
|
|
509
|
+
process.exitCode = 1;
|
|
510
|
+
return;
|
|
511
|
+
}
|
|
512
|
+
console.log(`Resuming orchestration: ${orchestratorId}`);
|
|
513
|
+
// Apply policy overrides if provided
|
|
514
|
+
if (options.overrides) {
|
|
515
|
+
const { state: updatedState, applied } = applyPolicyOverrides(state, options.overrides);
|
|
516
|
+
state = updatedState;
|
|
517
|
+
if (applied.length > 0) {
|
|
518
|
+
console.log('Policy overrides applied:');
|
|
519
|
+
for (const override of applied) {
|
|
520
|
+
console.log(` ${override.field}: ${override.from} → ${override.to}`);
|
|
521
|
+
}
|
|
522
|
+
saveState(state, repoPath);
|
|
523
|
+
}
|
|
524
|
+
}
|
|
525
|
+
const ownershipApplied = applyTaskOwnershipMetadata(state, repoPath);
|
|
526
|
+
if (ownershipApplied.errors.length > 0) {
|
|
527
|
+
console.error('Failed to parse task ownership metadata:');
|
|
528
|
+
for (const err of ownershipApplied.errors) {
|
|
529
|
+
console.error(` - ${err}`);
|
|
530
|
+
}
|
|
531
|
+
process.exitCode = 1;
|
|
532
|
+
return;
|
|
533
|
+
}
|
|
534
|
+
state = ownershipApplied.state;
|
|
535
|
+
const resumePolicy = getEffectivePolicy(state);
|
|
536
|
+
if (resumePolicy.ownership_required) {
|
|
537
|
+
const missing = collectMissingOwnership(state);
|
|
538
|
+
if (missing.length > 0) {
|
|
539
|
+
console.error(formatOwnershipMissingMessage(missing));
|
|
540
|
+
process.exitCode = 1;
|
|
541
|
+
return;
|
|
542
|
+
}
|
|
543
|
+
}
|
|
544
|
+
// Check if already terminal
|
|
545
|
+
if (state.status !== 'running') {
|
|
546
|
+
console.log(`Orchestration already ${state.status}`);
|
|
547
|
+
printStatus(state);
|
|
548
|
+
process.exitCode = state.status === 'complete' ? 0 : 1;
|
|
549
|
+
return;
|
|
550
|
+
}
|
|
551
|
+
// Reconcile active runs
|
|
552
|
+
console.log('Reconciling active runs...');
|
|
553
|
+
const { state: reconciledState, reconciled } = await reconcileState(state);
|
|
554
|
+
state = reconciledState;
|
|
555
|
+
for (const r of reconciled) {
|
|
556
|
+
const status = r.status === 'still_running' ? 'still running' : r.status;
|
|
557
|
+
console.log(` ${r.trackId} (${r.runId}): ${status}`);
|
|
558
|
+
}
|
|
559
|
+
saveState(state, repoPath);
|
|
560
|
+
// If all runs finished during reconciliation
|
|
561
|
+
if (state.status !== 'running') {
|
|
562
|
+
console.log('All runs completed during reconciliation');
|
|
563
|
+
printStatus(state);
|
|
564
|
+
process.exitCode = state.status === 'complete' ? 0 : 1;
|
|
565
|
+
return;
|
|
566
|
+
}
|
|
567
|
+
// Resume tracking active waits
|
|
568
|
+
const activeWaits = new Map();
|
|
569
|
+
// Start waiting for any still-running runs
|
|
570
|
+
for (const r of reconciled) {
|
|
571
|
+
if (r.status === 'still_running') {
|
|
572
|
+
const waitPromise = waitForRun(r.runId, repoPath);
|
|
573
|
+
activeWaits.set(r.trackId, waitPromise);
|
|
574
|
+
}
|
|
575
|
+
}
|
|
576
|
+
console.log('Continuing orchestration...');
|
|
577
|
+
console.log('');
|
|
578
|
+
// Main orchestration loop (same as orchestrateCommand)
|
|
579
|
+
while (state.status === 'running') {
|
|
580
|
+
const decision = makeScheduleDecision(state);
|
|
581
|
+
switch (decision.action) {
|
|
582
|
+
case 'done':
|
|
583
|
+
state.status = state.tracks.every((t) => t.status === 'complete')
|
|
584
|
+
? 'complete'
|
|
585
|
+
: 'stopped';
|
|
586
|
+
state.ended_at = new Date().toISOString();
|
|
587
|
+
break;
|
|
588
|
+
case 'blocked':
|
|
589
|
+
console.error(`BLOCKED: ${decision.reason}`);
|
|
590
|
+
if (decision.track_id) {
|
|
591
|
+
state = failTrack(state, decision.track_id, decision.reason ?? 'blocked');
|
|
592
|
+
}
|
|
593
|
+
break;
|
|
594
|
+
case 'launch': {
|
|
595
|
+
const trackId = decision.track_id;
|
|
596
|
+
const track = state.tracks.find((t) => t.id === trackId);
|
|
597
|
+
const step = track.steps[track.current_step];
|
|
598
|
+
// Use effective policy (from policy block or legacy fields)
|
|
599
|
+
const policy = getEffectivePolicy(state);
|
|
600
|
+
if (policy.ownership_required) {
|
|
601
|
+
const ownsRaw = step.owns_raw ?? [];
|
|
602
|
+
const ownsNormalized = step.owns_normalized ?? [];
|
|
603
|
+
const reservation = reserveOwnershipClaims(state, trackId, ownsRaw, ownsNormalized);
|
|
604
|
+
if (reservation.conflicts.length > 0) {
|
|
605
|
+
const conflictList = reservation.conflicts.join(', ');
|
|
606
|
+
const message = `Ownership claim conflict for ${step.task_path}: ${conflictList}`;
|
|
607
|
+
console.error(` ${message}`);
|
|
608
|
+
state = failTrack(state, trackId, message);
|
|
609
|
+
saveState(state, repoPath);
|
|
610
|
+
break;
|
|
611
|
+
}
|
|
612
|
+
state = reservation.state;
|
|
613
|
+
}
|
|
614
|
+
console.log(`Launching: ${track.name} - ${step.task_path} (fast=${policy.fast})`);
|
|
615
|
+
const launchResult = await launchRun(step.task_path, repoPath, {
|
|
616
|
+
time: policy.time_budget_minutes,
|
|
617
|
+
maxTicks: policy.max_ticks,
|
|
618
|
+
allowDeps: false, // Default for resume
|
|
619
|
+
worktree: false,
|
|
620
|
+
fast: policy.fast,
|
|
621
|
+
forceParallel: policy.collision_policy === 'force'
|
|
622
|
+
});
|
|
623
|
+
if ('error' in launchResult) {
|
|
624
|
+
console.error(` Failed to launch: ${launchResult.error}`);
|
|
625
|
+
state = failTrack(state, trackId, launchResult.error);
|
|
626
|
+
}
|
|
627
|
+
else {
|
|
628
|
+
console.log(` Started run: ${launchResult.runId}`);
|
|
629
|
+
state = startTrackRun(state, trackId, launchResult.runId, launchResult.runDir);
|
|
630
|
+
const waitPromise = waitForRun(launchResult.runId, repoPath);
|
|
631
|
+
activeWaits.set(trackId, waitPromise);
|
|
632
|
+
}
|
|
633
|
+
saveState(state, repoPath);
|
|
634
|
+
break;
|
|
635
|
+
}
|
|
636
|
+
case 'wait':
|
|
637
|
+
if (activeWaits.size > 0) {
|
|
638
|
+
const entries = [...activeWaits.entries()];
|
|
639
|
+
const raceResult = await Promise.race(entries.map(async ([trackId, promise]) => {
|
|
640
|
+
const result = await promise;
|
|
641
|
+
return { trackId, result };
|
|
642
|
+
}));
|
|
643
|
+
const { trackId, result } = raceResult;
|
|
644
|
+
const track = state.tracks.find((t) => t.id === trackId);
|
|
645
|
+
console.log(`Completed: ${track.name} - ${result.status}`);
|
|
646
|
+
if (result.stop_reason) {
|
|
647
|
+
console.log(` Stop reason: ${result.stop_reason}`);
|
|
648
|
+
}
|
|
649
|
+
state = completeTrackStep(state, trackId, result);
|
|
650
|
+
activeWaits.delete(trackId);
|
|
651
|
+
saveState(state, repoPath);
|
|
652
|
+
}
|
|
653
|
+
else {
|
|
654
|
+
await sleep(POLL_INTERVAL_MS);
|
|
655
|
+
}
|
|
656
|
+
break;
|
|
657
|
+
}
|
|
658
|
+
printStatus(state);
|
|
659
|
+
}
|
|
660
|
+
// Final summary
|
|
661
|
+
console.log('='.repeat(60));
|
|
662
|
+
console.log('ORCHESTRATION COMPLETE');
|
|
663
|
+
printStatus(state);
|
|
664
|
+
const summary = getOrchestratorSummary(state);
|
|
665
|
+
if (summary.failed > 0 || summary.stopped > 0) {
|
|
666
|
+
process.exitCode = 1;
|
|
667
|
+
}
|
|
668
|
+
}
|
|
669
|
+
const WAIT_POLL_INTERVAL_MS = 500;
|
|
670
|
+
const WAIT_BACKOFF_MAX_MS = 2000;
|
|
671
|
+
/**
|
|
672
|
+
* Wait for an orchestration to reach a terminal state.
|
|
673
|
+
*/
|
|
674
|
+
export async function waitOrchestrationCommand(options) {
|
|
675
|
+
const repoPath = path.resolve(options.repo);
|
|
676
|
+
// Resolve "latest" to actual ID
|
|
677
|
+
let orchestratorId = options.orchestratorId;
|
|
678
|
+
if (orchestratorId === 'latest') {
|
|
679
|
+
const latest = findLatestOrchestrationId(repoPath);
|
|
680
|
+
if (!latest) {
|
|
681
|
+
if (options.json) {
|
|
682
|
+
console.log(JSON.stringify({ error: 'no_orchestrations', message: 'No orchestrations found' }));
|
|
683
|
+
}
|
|
684
|
+
else {
|
|
685
|
+
console.error('No orchestrations found');
|
|
686
|
+
}
|
|
687
|
+
process.exitCode = 1;
|
|
688
|
+
return;
|
|
689
|
+
}
|
|
690
|
+
orchestratorId = latest;
|
|
691
|
+
}
|
|
692
|
+
const orchDir = getOrchestrationDir(repoPath, orchestratorId);
|
|
693
|
+
const handoffsDir = path.join(orchDir, 'handoffs');
|
|
694
|
+
// Check if orchestration exists
|
|
695
|
+
const state = loadOrchestratorState(orchestratorId, repoPath);
|
|
696
|
+
if (!state) {
|
|
697
|
+
if (options.json) {
|
|
698
|
+
console.log(JSON.stringify({
|
|
699
|
+
error: 'orchestration_not_found',
|
|
700
|
+
orchestrator_id: orchestratorId,
|
|
701
|
+
message: `Orchestration not found: ${orchestratorId}`
|
|
702
|
+
}));
|
|
703
|
+
}
|
|
704
|
+
else {
|
|
705
|
+
console.error(`Orchestration not found: ${orchestratorId}`);
|
|
706
|
+
}
|
|
707
|
+
process.exitCode = 1;
|
|
708
|
+
return;
|
|
709
|
+
}
|
|
710
|
+
const startTime = Date.now();
|
|
711
|
+
const timeoutMs = options.timeout ?? Infinity;
|
|
712
|
+
let pollInterval = WAIT_POLL_INTERVAL_MS;
|
|
713
|
+
while (true) {
|
|
714
|
+
const elapsed = Date.now() - startTime;
|
|
715
|
+
// Check timeout
|
|
716
|
+
if (elapsed >= timeoutMs) {
|
|
717
|
+
const currentState = loadOrchestratorState(orchestratorId, repoPath);
|
|
718
|
+
if (options.json && currentState) {
|
|
719
|
+
const result = buildWaitResultFromState(currentState, repoPath, true);
|
|
720
|
+
console.log(JSON.stringify(result, null, 2));
|
|
721
|
+
}
|
|
722
|
+
else {
|
|
723
|
+
console.log(`Timeout after ${Math.round(elapsed / 1000)}s`);
|
|
724
|
+
}
|
|
725
|
+
process.exitCode = 124;
|
|
726
|
+
return;
|
|
727
|
+
}
|
|
728
|
+
// Fast path: check for terminal artifact (most reliable)
|
|
729
|
+
const completePath = path.join(handoffsDir, 'complete.json');
|
|
730
|
+
const stopPath = path.join(handoffsDir, 'stop.json');
|
|
731
|
+
if (fs.existsSync(completePath)) {
|
|
732
|
+
const artifact = JSON.parse(fs.readFileSync(completePath, 'utf-8'));
|
|
733
|
+
if (options.for !== 'stop') {
|
|
734
|
+
outputWaitResult(artifact, options.json, elapsed);
|
|
735
|
+
process.exitCode = 0;
|
|
736
|
+
return;
|
|
737
|
+
}
|
|
738
|
+
}
|
|
739
|
+
if (fs.existsSync(stopPath)) {
|
|
740
|
+
const artifact = JSON.parse(fs.readFileSync(stopPath, 'utf-8'));
|
|
741
|
+
if (options.for !== 'complete') {
|
|
742
|
+
outputWaitResult(artifact, options.json, elapsed);
|
|
743
|
+
process.exitCode = 1;
|
|
744
|
+
return;
|
|
745
|
+
}
|
|
746
|
+
}
|
|
747
|
+
// Slow path: read state.json
|
|
748
|
+
const currentState = loadOrchestratorState(orchestratorId, repoPath);
|
|
749
|
+
if (currentState && currentState.status !== 'running') {
|
|
750
|
+
const isComplete = currentState.status === 'complete';
|
|
751
|
+
const matchesCondition = options.for === 'terminal' ||
|
|
752
|
+
(options.for === 'complete' && isComplete) ||
|
|
753
|
+
(options.for === 'stop' && !isComplete);
|
|
754
|
+
if (matchesCondition) {
|
|
755
|
+
const result = buildWaitResultFromState(currentState, repoPath, false);
|
|
756
|
+
outputWaitResult(result, options.json, elapsed);
|
|
757
|
+
process.exitCode = isComplete ? 0 : 1;
|
|
758
|
+
return;
|
|
759
|
+
}
|
|
760
|
+
}
|
|
761
|
+
// Backoff polling
|
|
762
|
+
pollInterval = Math.min(pollInterval * 1.2, WAIT_BACKOFF_MAX_MS);
|
|
763
|
+
await sleep(pollInterval);
|
|
764
|
+
}
|
|
765
|
+
}
|
|
766
|
+
/**
|
|
767
|
+
* Build wait result from state (when no artifact exists yet).
|
|
768
|
+
*/
|
|
769
|
+
function buildWaitResultFromState(state, repoPath, timedOut) {
|
|
770
|
+
const result = buildWaitResult(state, repoPath);
|
|
771
|
+
if (timedOut) {
|
|
772
|
+
return {
|
|
773
|
+
...result,
|
|
774
|
+
status: 'stopped',
|
|
775
|
+
stop_reason: 'orchestrator_timeout',
|
|
776
|
+
stop_reason_family: 'budget'
|
|
777
|
+
};
|
|
778
|
+
}
|
|
779
|
+
return result;
|
|
780
|
+
}
|
|
781
|
+
/**
|
|
782
|
+
* Output wait result in JSON or human-readable format.
|
|
783
|
+
*/
|
|
784
|
+
function outputWaitResult(result, json, elapsedMs) {
|
|
785
|
+
if (json) {
|
|
786
|
+
console.log(JSON.stringify(result, null, 2));
|
|
787
|
+
}
|
|
788
|
+
else {
|
|
789
|
+
const statusWord = result.status === 'complete' ? 'Completed' : 'Stopped';
|
|
790
|
+
console.log(`${statusWord} after ${Math.round(elapsedMs / 1000)}s`);
|
|
791
|
+
console.log(`Tracks: ${result.tracks.completed}/${result.tracks.total}`);
|
|
792
|
+
console.log(`Steps: ${result.steps.completed}/${result.steps.total}`);
|
|
793
|
+
if (result.stop_reason) {
|
|
794
|
+
console.log(`Reason: ${result.stop_reason}`);
|
|
795
|
+
}
|
|
796
|
+
if (result.resume_command) {
|
|
797
|
+
console.log(`Resume: ${result.resume_command}`);
|
|
798
|
+
}
|
|
799
|
+
}
|
|
800
|
+
}
|