@tagma/sdk 0.1.3 → 0.1.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/engine.ts CHANGED
@@ -1,598 +1,637 @@
1
- import { resolve, dirname } from 'path';
2
- import { mkdir } from 'fs/promises';
3
- import type {
4
- PipelineConfig, TaskConfig, TrackConfig, TaskState, TaskStatus,
5
- TaskResult, DriverPlugin, TriggerPlugin, CompletionPlugin,
6
- MiddlewarePlugin, MiddlewareContext, DriverContext,
7
- OnFailure,
8
- } from './types';
9
- import { buildDag, type Dag, type DagNode } from './dag';
10
- import { getHandler, hasHandler } from './registry';
11
- import { runSpawn, runCommand } from './runner';
12
- import { parseDuration, nowISO, generateRunId } from './utils';
13
- import {
14
- executeHook,
15
- buildPipelineStartContext, buildTaskContext,
16
- buildPipelineCompleteContext, buildPipelineErrorContext,
17
- type PipelineInfo, type TrackInfo, type TaskInfo,
18
- } from './hooks';
19
- import { Logger, tailLines, clip } from './logger';
20
- import { InMemoryApprovalGateway, type ApprovalGateway } from './approval';
21
-
22
- // ═══ Preflight Validation ═══
23
-
24
- function preflight(config: PipelineConfig, dag: Dag): void {
25
- const errors: string[] = [];
26
-
27
- for (const [, node] of dag.nodes) {
28
- const task = node.task;
29
- const track = node.track;
30
- const driverName = task.driver ?? track.driver ?? config.driver ?? 'claude-code';
31
-
32
- if (!hasHandler('drivers', driverName)) {
33
- errors.push(`Task "${node.taskId}": driver "${driverName}" not registered`);
34
- }
35
-
36
- if (task.trigger && !hasHandler('triggers', task.trigger.type)) {
37
- errors.push(`Task "${node.taskId}": trigger type "${task.trigger.type}" not registered`);
38
- }
39
-
40
- if (task.completion && !hasHandler('completions', task.completion.type)) {
41
- errors.push(`Task "${node.taskId}": completion type "${task.completion.type}" not registered`);
42
- }
43
-
44
- const mws = task.middlewares ?? track.middlewares ?? [];
45
- for (const mw of mws) {
46
- if (!hasHandler('middlewares', mw.type)) {
47
- errors.push(`Task "${node.taskId}": middleware type "${mw.type}" not registered`);
48
- }
49
- }
50
-
51
- if (task.continue_from && hasHandler('drivers', driverName)) {
52
- const driver = getHandler<DriverPlugin>('drivers', driverName);
53
- if (!driver.capabilities.sessionResume) {
54
- const upstreamId = resolveRefInDag(dag, task.continue_from, track.id);
55
- if (upstreamId) {
56
- const upstream = dag.nodes.get(upstreamId);
57
- if (upstream && !upstream.task.output) {
58
- errors.push(
59
- `Task "${node.taskId}" uses continue_from: "${task.continue_from}", ` +
60
- `but upstream task "${upstreamId}" has no "output" field. ` +
61
- `Add output to the upstream task, or remove continue_from.`
62
- );
63
- }
64
- }
65
- }
66
- }
67
- }
68
-
69
- if (errors.length > 0) {
70
- throw new Error(`Preflight validation failed:\n - ${errors.join('\n - ')}`);
71
- }
72
- }
73
-
74
- function resolveRefInDag(dag: Dag, ref: string, fromTrackId: string): string | null {
75
- if (dag.nodes.has(ref)) return ref;
76
- const sameTrack = `${fromTrackId}.${ref}`;
77
- if (dag.nodes.has(sameTrack)) return sameTrack;
78
- for (const [id] of dag.nodes) {
79
- if (id.endsWith(`.${ref}`)) return id;
80
- }
81
- return null;
82
- }
83
-
84
- // ═══ Engine ═══
85
-
86
- export interface EngineResult {
87
- readonly success: boolean;
88
- readonly summary: {
89
- total: number; success: number; failed: number;
90
- skipped: number; timeout: number; blocked: number;
91
- };
92
- readonly states: ReadonlyMap<string, TaskState>;
93
- }
94
-
95
- export interface RunPipelineOptions {
96
- readonly approvalGateway?: ApprovalGateway;
97
- }
98
-
99
- export async function runPipeline(
100
- config: PipelineConfig,
101
- workDir: string,
102
- options: RunPipelineOptions = {},
103
- ): Promise<EngineResult> {
104
- const approvalGateway = options.approvalGateway ?? new InMemoryApprovalGateway();
105
- const dag = buildDag(config);
106
- preflight(config, dag);
107
-
108
- const runId = generateRunId();
109
- const startedAt = nowISO();
110
- const pipelineInfo: PipelineInfo = { name: config.name, run_id: runId, started_at: startedAt };
111
- const log = new Logger(workDir, runId);
112
- log.info('[pipeline]', `start "${config.name}" run_id=${runId}`);
113
-
114
- // File-only: dump the resolved pipeline shape + DAG topology for post-mortem.
115
- log.section('Pipeline configuration');
116
- log.quiet(`name: ${config.name}`);
117
- log.quiet(`driver: ${config.driver ?? '(default: claude-code)'}`);
118
- log.quiet(`timeout: ${config.timeout ?? '(none)'}`);
119
- log.quiet(`tracks: ${config.tracks.length}`);
120
- log.quiet(`tasks (total): ${dag.nodes.size}`);
121
- log.quiet(`plugins: ${(config.plugins ?? []).join(', ') || '(none)'}`);
122
- log.quiet(`hooks: ${config.hooks ? Object.keys(config.hooks).join(', ') || '(none)' : '(none)'}`);
123
-
124
- log.section('DAG topology');
125
- for (const [id, node] of dag.nodes) {
126
- const deps = node.dependsOn.length ? node.dependsOn.join(', ') : '(root)';
127
- const kind = node.task.prompt ? 'ai' : 'cmd';
128
- log.quiet(` ${id} [${kind}] track=${node.track.id} deps=[${deps}]`);
129
- }
130
- log.quiet('');
131
-
132
- // Initialize states (before hook, so we can return them even if blocked)
133
- const states = new Map<string, TaskState>();
134
- for (const [id, node] of dag.nodes) {
135
- states.set(id, {
136
- config: node.task,
137
- trackConfig: node.track,
138
- status: 'idle',
139
- result: null,
140
- startedAt: null,
141
- finishedAt: null,
142
- });
143
- }
144
-
145
- // Pipeline start hook (gate)
146
- const startHook = await executeHook(
147
- config.hooks, 'pipeline_start', buildPipelineStartContext(pipelineInfo), workDir,
148
- );
149
- if (!startHook.allowed) {
150
- console.error(`Pipeline blocked by pipeline_start hook (exit code ${startHook.exitCode})`);
151
- await executeHook(config.hooks, 'pipeline_error',
152
- buildPipelineErrorContext(pipelineInfo, 'pipeline_blocked', 'pipeline_blocked'), workDir);
153
- // All tasks stay idle — pipeline never started
154
- return {
155
- success: false,
156
- summary: { total: dag.nodes.size, success: 0, failed: 0, skipped: 0, timeout: 0, blocked: 0 },
157
- states,
158
- };
159
- }
160
-
161
- // Pipeline approved transition all tasks to waiting
162
- for (const [, state] of states) {
163
- state.status = 'waiting';
164
- }
165
-
166
- const sessionMap = new Map<string, string>();
167
- const outputMap = new Map<string, string>();
168
- const normalizedMap = new Map<string, string>();
169
-
170
- // Pipeline timeout
171
- const pipelineTimeoutMs = config.timeout ? parseDuration(config.timeout) : 0;
172
- let pipelineAborted = false;
173
- const abortController = new AbortController();
174
- let pipelineTimer: ReturnType<typeof setTimeout> | null = null;
175
-
176
- if (pipelineTimeoutMs > 0) {
177
- pipelineTimer = setTimeout(() => {
178
- pipelineAborted = true;
179
- abortController.abort();
180
- }, pipelineTimeoutMs);
181
- }
182
-
183
- // When the pipeline is aborted (timeout, external shutdown), drain all
184
- // pending approvals so waiting triggers unblock immediately.
185
- abortController.signal.addEventListener('abort', () => {
186
- approvalGateway.abortAll('pipeline aborted');
187
- });
188
-
189
- // ── Helpers ──
190
-
191
- function getOnFailure(taskId: string): OnFailure {
192
- return dag.nodes.get(taskId)?.track.on_failure ?? 'skip_downstream';
193
- }
194
-
195
- function isDependencySatisfied(depId: string): 'satisfied' | 'unsatisfied' | 'skip' {
196
- const depState = states.get(depId);
197
- if (!depState) return 'skip';
198
- switch (depState.status) {
199
- case 'success': return 'satisfied';
200
- case 'skipped': return 'skip';
201
- case 'failed': case 'timeout': case 'blocked':
202
- return getOnFailure(depId) === 'ignore' ? 'satisfied' : 'skip';
203
- default: return 'unsatisfied';
204
- }
205
- }
206
-
207
- function applyStopAll(trackId: string): void {
208
- for (const [, state] of states) {
209
- const node = dag.nodes.get(state.config.id);
210
- if (state.trackConfig.id === trackId && !isTerminal(state.status)) {
211
- state.status = 'skipped';
212
- state.finishedAt = nowISO();
213
- }
214
- }
215
- }
216
-
217
- function buildTaskInfoObj(taskId: string): TaskInfo {
218
- const state = states.get(taskId)!;
219
- return {
220
- id: taskId,
221
- name: state.config.name,
222
- type: state.config.prompt ? 'ai' : 'command',
223
- status: state.status,
224
- exit_code: state.result?.exitCode ?? null,
225
- duration_ms: state.result?.durationMs ?? null,
226
- output_path: state.result?.outputPath ?? null,
227
- stderr_path: state.result?.stderrPath ?? null,
228
- session_id: state.result?.sessionId ?? null,
229
- started_at: state.startedAt,
230
- finished_at: state.finishedAt,
231
- };
232
- }
233
-
234
- function trackInfoOf(taskId: string): TrackInfo {
235
- const node = dag.nodes.get(taskId)!;
236
- return { id: node.track.id, name: node.track.name };
237
- }
238
-
239
- async function fireHook(taskId: string, event: 'task_success' | 'task_failure'): Promise<void> {
240
- await executeHook(config.hooks, event,
241
- buildTaskContext(event, pipelineInfo, trackInfoOf(taskId), buildTaskInfoObj(taskId)), workDir);
242
- }
243
-
244
- // ── Process a single task ──
245
-
246
- async function processTask(taskId: string): Promise<void> {
247
- const state = states.get(taskId)!;
248
- const node = dag.nodes.get(taskId)!;
249
- const task = node.task;
250
- const track = node.track;
251
-
252
- log.section(`Task ${taskId}`);
253
- log.debug(`[task:${taskId}]`,
254
- `type=${task.prompt ? 'ai' : 'cmd'} track=${track.id} deps=[${node.dependsOn.join(', ') || '(root)'}]`);
255
-
256
- // 1. Check dependencies
257
- for (const depId of node.dependsOn) {
258
- const result = isDependencySatisfied(depId);
259
- if (result === 'skip') {
260
- const depStatus = states.get(depId)?.status ?? 'unknown';
261
- log.debug(`[task:${taskId}]`, `skipped (upstream "${depId}" status=${depStatus})`);
262
- state.status = 'skipped';
263
- state.finishedAt = nowISO();
264
- return;
265
- }
266
- if (result === 'unsatisfied') return; // still waiting
267
- }
268
-
269
- // 2. Check trigger
270
- if (task.trigger) {
271
- log.debug(`[task:${taskId}]`, `trigger wait: type=${task.trigger.type} ${JSON.stringify(task.trigger)}`);
272
- try {
273
- const triggerPlugin = getHandler<TriggerPlugin>('triggers', task.trigger.type);
274
- await triggerPlugin.watch(task.trigger as Record<string, unknown>, {
275
- taskId: node.taskId,
276
- trackId: track.id,
277
- workDir: task.cwd ?? workDir,
278
- signal: abortController.signal,
279
- approvalGateway,
280
- });
281
- log.debug(`[task:${taskId}]`, `trigger fired`);
282
- } catch (err: unknown) {
283
- const msg = err instanceof Error ? err.message : String(err);
284
- // If pipeline was aborted while we were still waiting for the trigger,
285
- // this task never entered running state → skipped, not timeout.
286
- if (pipelineAborted) {
287
- state.status = 'skipped';
288
- } else if (msg.includes('rejected') || msg.includes('denied')) {
289
- state.status = 'blocked'; // user/policy rejection
290
- } else if (msg.includes('timeout')) {
291
- state.status = 'timeout'; // genuine trigger wait timeout
292
- } else {
293
- state.status = 'failed'; // plugin error, watcher crash, etc.
294
- }
295
- state.finishedAt = nowISO();
296
- await fireHook(taskId, 'task_failure');
297
- return;
298
- }
299
- }
300
-
301
- // 3. task_start hook (gate)
302
- const hookResult = await executeHook(config.hooks, 'task_start',
303
- buildTaskContext('task_start', pipelineInfo, trackInfoOf(taskId), buildTaskInfoObj(taskId)), workDir);
304
- if (hookResult.exitCode !== 0 || config.hooks?.task_start) {
305
- log.debug(`[task:${taskId}]`,
306
- `task_start hook exit=${hookResult.exitCode} allowed=${hookResult.allowed}`);
307
- }
308
- if (!hookResult.allowed) {
309
- state.status = 'blocked';
310
- state.finishedAt = nowISO();
311
- await fireHook(taskId, 'task_failure');
312
- return;
313
- }
314
-
315
- // 4. Mark running
316
- state.status = 'running';
317
- state.startedAt = nowISO();
318
- log.info(`[task:${taskId}]`, task.command ? `running: ${task.command}` : `running (driver task)`);
319
-
320
- // File-only: resolved config for this task
321
- const resolvedDriver = task.driver ?? track.driver ?? config.driver ?? 'claude-code';
322
- const resolvedTier = task.model_tier ?? track.model_tier ?? '(default)';
323
- const resolvedPerms = task.permissions ?? track.permissions ?? '(default)';
324
- const resolvedCwd = task.cwd ?? track.cwd ?? workDir;
325
- log.debug(`[task:${taskId}]`,
326
- `resolved: driver=${resolvedDriver} tier=${resolvedTier} cwd=${resolvedCwd}`);
327
- log.debug(`[task:${taskId}]`, `permissions: ${JSON.stringify(resolvedPerms)}`);
328
- if (task.continue_from) {
329
- log.debug(`[task:${taskId}]`, `continue_from: "${task.continue_from}"`);
330
- }
331
- if (task.timeout) {
332
- log.debug(`[task:${taskId}]`, `timeout: ${task.timeout}`);
333
- }
334
-
335
- try {
336
- let result: TaskResult;
337
- const timeoutMs = task.timeout ? parseDuration(task.timeout) : undefined;
338
-
339
- const runOpts = { timeoutMs, signal: abortController.signal };
340
-
341
- if (task.command) {
342
- log.debug(`[task:${taskId}]`, `command: ${task.command}`);
343
- result = await runCommand(task.command, task.cwd ?? workDir, runOpts);
344
- } else {
345
- // AI task: apply middleware chain
346
- const driverName = task.driver ?? track.driver ?? config.driver ?? 'claude-code';
347
- const driver = getHandler<DriverPlugin>('drivers', driverName);
348
-
349
- let prompt = task.prompt!;
350
- const originalLen = prompt.length;
351
- const mws = task.middlewares !== undefined ? task.middlewares : track.middlewares;
352
- if (mws && mws.length > 0) {
353
- log.debug(`[task:${taskId}]`,
354
- `middleware chain: ${mws.map(m => m.type).join(' → ')}`);
355
- const mwCtx: MiddlewareContext = {
356
- task, track, outputMap, workDir: task.cwd ?? workDir,
357
- };
358
- for (const mwConfig of mws) {
359
- const before = prompt.length;
360
- const mwPlugin = getHandler<MiddlewarePlugin>('middlewares', mwConfig.type);
361
- prompt = await mwPlugin.enhance(prompt, mwConfig as Record<string, unknown>, mwCtx);
362
- log.debug(`[task:${taskId}]`,
363
- ` ${mwConfig.type}: ${before} → ${prompt.length} chars`);
364
- }
365
- }
366
- log.debug(`[task:${taskId}]`,
367
- `prompt: ${originalLen} chars (final: ${prompt.length} chars)`);
368
- log.quiet(`--- prompt (final) ---\n${clip(prompt)}\n--- end prompt ---`);
369
-
370
- const enrichedTask: TaskConfig = { ...task, prompt };
371
- const driverCtx: DriverContext = {
372
- sessionMap, outputMap, normalizedMap, workDir: task.cwd ?? workDir,
373
- };
374
- const spec = await driver.buildCommand(enrichedTask, track, driverCtx);
375
- log.debug(`[task:${taskId}]`, `driver=${driverName}`);
376
- log.debug(`[task:${taskId}]`,
377
- `spawn args: ${JSON.stringify(spec.args)}`);
378
- if (spec.cwd) log.debug(`[task:${taskId}]`, `spawn cwd: ${spec.cwd}`);
379
- if (spec.env) log.debug(`[task:${taskId}]`,
380
- `spawn env overrides: ${Object.keys(spec.env).join(', ')}`);
381
- if (spec.stdin) log.debug(`[task:${taskId}]`,
382
- `spawn stdin: ${spec.stdin.length} chars`);
383
- result = await runSpawn(spec, driver, runOpts);
384
- }
385
-
386
- // 5. Determine status
387
- if (result.exitCode === -1) {
388
- state.status = 'timeout';
389
- } else if (result.exitCode !== 0) {
390
- state.status = 'failed';
391
- } else if (task.completion) {
392
- const plugin = getHandler<CompletionPlugin>('completions', task.completion.type);
393
- const completionCtx = { workDir: task.cwd ?? workDir };
394
- const passed = await plugin.check(task.completion as Record<string, unknown>, result, completionCtx);
395
- state.status = passed ? 'success' : 'failed';
396
- } else {
397
- state.status = 'success';
398
- }
399
-
400
- // 6. Write output file with RAW stdout (preserves driver output format).
401
- // The separate normalizedMap holds canonical text for continue_from.
402
- if (task.output) {
403
- const outPath = resolve(workDir, task.output);
404
- await mkdir(dirname(outPath), { recursive: true });
405
- await Bun.write(outPath, result.stdout);
406
- result = { ...result, outputPath: outPath };
407
- outputMap.set(taskId, outPath);
408
- const bareId = taskId.includes('.') ? taskId.split('.').pop()! : taskId;
409
- if (!outputMap.has(bareId)) outputMap.set(bareId, outPath);
410
- }
411
-
412
- // Store normalized text separately (in-memory) for continue_from handoff
413
- if (result.normalizedOutput !== null) {
414
- normalizedMap.set(taskId, result.normalizedOutput);
415
- const bareId = taskId.includes('.') ? taskId.split('.').pop()! : taskId;
416
- if (!normalizedMap.has(bareId)) normalizedMap.set(bareId, result.normalizedOutput);
417
- }
418
-
419
- if (result.stderr) {
420
- const stderrDir = resolve(workDir, './tmp', runId);
421
- await mkdir(stderrDir, { recursive: true });
422
- const stderrPath = resolve(stderrDir, `${taskId.replace(/\./g, '_')}.stderr`);
423
- await Bun.write(stderrPath, result.stderr);
424
- result = { ...result, stderrPath };
425
- }
426
-
427
- if (result.sessionId) {
428
- sessionMap.set(taskId, result.sessionId);
429
- const bareId = taskId.includes('.') ? taskId.split('.').pop()! : taskId;
430
- if (!sessionMap.has(bareId)) sessionMap.set(bareId, result.sessionId);
431
- }
432
-
433
- state.result = result;
434
- state.finishedAt = nowISO();
435
-
436
- // Log task outcome with relevant details
437
- const durSec = (result.durationMs / 1000).toFixed(1);
438
- if (state.status === 'success') {
439
- log.info(`[task:${taskId}]`, `success (${durSec}s)`);
440
- } else {
441
- log.error(`[task:${taskId}]`,
442
- `${state.status} exit=${result.exitCode} duration=${durSec}s`);
443
- if (result.stderr) {
444
- const tail = tailLines(result.stderr, 10);
445
- log.error(`[task:${taskId}]`, `stderr tail:\n${tail}`);
446
- }
447
- }
448
-
449
- // File-only: full stdout/stderr dump (clipped) + extracted metadata
450
- log.debug(`[task:${taskId}]`,
451
- `stdout: ${result.stdout.length} chars, stderr: ${result.stderr.length} chars`);
452
- if (result.sessionId) {
453
- log.debug(`[task:${taskId}]`, `sessionId: ${result.sessionId}`);
454
- }
455
- if (result.outputPath) {
456
- log.debug(`[task:${taskId}]`, `wrote output: ${result.outputPath}`);
457
- }
458
- if (result.stderrPath) {
459
- log.debug(`[task:${taskId}]`, `wrote stderr: ${result.stderrPath}`);
460
- }
461
- if (result.stdout) {
462
- log.quiet(`--- stdout (${taskId}) ---\n${clip(result.stdout)}\n--- end stdout ---`);
463
- }
464
- if (result.stderr) {
465
- log.quiet(`--- stderr (${taskId}) ---\n${clip(result.stderr)}\n--- end stderr ---`);
466
- }
467
- if (task.completion) {
468
- log.debug(`[task:${taskId}]`,
469
- `completion check: type=${task.completion.type} result=${state.status}`);
470
- }
471
-
472
- } catch (err: unknown) {
473
- state.status = 'failed';
474
- state.finishedAt = nowISO();
475
- const errMsg = err instanceof Error ? (err.stack ?? err.message) : String(err);
476
- log.error(`[task:${taskId}]`, `failed before execution: ${errMsg}`);
477
- state.result = {
478
- exitCode: -1,
479
- stdout: '',
480
- stderr: errMsg,
481
- outputPath: null, stderrPath: null, durationMs: 0,
482
- sessionId: null, normalizedOutput: null,
483
- };
484
- }
485
-
486
- // 7. Fire hooks
487
- const finalStatus: TaskStatus = state.status;
488
- await fireHook(taskId, finalStatus === 'success' ? 'task_success' : 'task_failure');
489
-
490
- // 8. Handle stop_all for failure states
491
- if (finalStatus !== 'success' && getOnFailure(taskId) === 'stop_all') {
492
- applyStopAll(node.track.id);
493
- }
494
- }
495
-
496
- // ── Event loop ──
497
- try {
498
- let progress = true;
499
- while (progress && !pipelineAborted) {
500
- progress = false;
501
-
502
- // Collect tasks whose deps are all terminal and that are still waiting
503
- const launchable: string[] = [];
504
- for (const [id, state] of states) {
505
- if (state.status !== 'waiting') continue;
506
- const node = dag.nodes.get(id)!;
507
- const allDepsTerminal = node.dependsOn.length === 0 ||
508
- node.dependsOn.every(d => isTerminal(states.get(d)!.status));
509
- if (allDepsTerminal) launchable.push(id);
510
- }
511
-
512
- if (launchable.length === 0) {
513
- // Check if anything is still running (trigger waits etc.)
514
- const anyNonTerminal = [...states.values()].some(s => !isTerminal(s.status));
515
- if (!anyNonTerminal) break;
516
- await new Promise(r => setTimeout(r, 50));
517
- progress = true;
518
- continue;
519
- }
520
-
521
- // Launch all launchable tasks concurrently
522
- await Promise.all(launchable.map(id => processTask(id)));
523
- progress = true;
524
- }
525
-
526
- if (pipelineAborted) {
527
- for (const [, state] of states) {
528
- if (!isTerminal(state.status)) {
529
- // Running tasks get timeout (they were killed); waiting tasks get skipped
530
- state.status = state.status === 'running' ? 'timeout' : 'skipped';
531
- state.finishedAt = nowISO();
532
- }
533
- }
534
- }
535
- } finally {
536
- if (pipelineTimer) clearTimeout(pipelineTimer);
537
- // Safety net: drain any approvals still pending at shutdown (e.g. crash path).
538
- if (approvalGateway.pending().length > 0) {
539
- approvalGateway.abortAll('pipeline finished');
540
- }
541
- }
542
-
543
- // ── Summary ──
544
- const summary = { total: 0, success: 0, failed: 0, skipped: 0, timeout: 0, blocked: 0 };
545
- for (const [, state] of states) {
546
- summary.total++;
547
- switch (state.status) {
548
- case 'success': summary.success++; break;
549
- case 'failed': summary.failed++; break;
550
- case 'skipped': summary.skipped++; break;
551
- case 'timeout': summary.timeout++; break;
552
- case 'blocked': summary.blocked++; break;
553
- }
554
- }
555
-
556
- const finishedAt = nowISO();
557
- const durationMs = new Date(finishedAt).getTime() - new Date(startedAt).getTime();
558
-
559
- if (pipelineAborted) {
560
- await executeHook(config.hooks, 'pipeline_error',
561
- buildPipelineErrorContext(pipelineInfo, 'Pipeline timeout exceeded'), workDir);
562
- } else {
563
- await executeHook(config.hooks, 'pipeline_complete',
564
- buildPipelineCompleteContext(
565
- { ...pipelineInfo, finished_at: finishedAt, duration_ms: durationMs }, summary), workDir);
566
- }
567
-
568
- const allSuccess = !pipelineAborted
569
- && summary.failed === 0 && summary.timeout === 0 && summary.blocked === 0;
570
-
571
- log.section('Pipeline summary');
572
- log.quiet(`status: ${pipelineAborted ? 'aborted (timeout)' : 'completed'}`);
573
- log.quiet(`duration: ${(durationMs / 1000).toFixed(1)}s`);
574
- log.quiet(
575
- `counts: total=${summary.total} success=${summary.success} ` +
576
- `failed=${summary.failed} skipped=${summary.skipped} ` +
577
- `timeout=${summary.timeout} blocked=${summary.blocked}`);
578
- log.quiet('');
579
- log.quiet('per-task:');
580
- for (const [id, state] of states) {
581
- const dur = state.result?.durationMs != null
582
- ? `${(state.result.durationMs / 1000).toFixed(1)}s` : '-';
583
- const exit = state.result?.exitCode ?? '-';
584
- log.quiet(` ${state.status.padEnd(8)} ${id} (exit=${exit}, ${dur})`);
585
- }
586
-
587
- console.log(`\n[Pipeline "${config.name}"] completed`);
588
- console.log(` Total: ${summary.total} | Success: ${summary.success} | Failed: ${summary.failed} | Skipped: ${summary.skipped} | Timeout: ${summary.timeout} | Blocked: ${summary.blocked}`);
589
- console.log(` Duration: ${(durationMs / 1000).toFixed(1)}s`);
590
- console.log(` Log: ${log.path}`);
591
-
592
- return { success: allSuccess, summary, states };
593
- }
594
-
595
- function isTerminal(status: TaskStatus): boolean {
596
- return status === 'success' || status === 'failed' || status === 'timeout'
597
- || status === 'skipped' || status === 'blocked';
598
- }
1
+ import { resolve, dirname } from 'path';
2
+ import { mkdir, readdir, rm } from 'fs/promises';
3
+ import type {
4
+ PipelineConfig, TaskConfig, TrackConfig, TaskState, TaskStatus,
5
+ TaskResult, DriverPlugin, TriggerPlugin, CompletionPlugin,
6
+ MiddlewarePlugin, MiddlewareContext, DriverContext,
7
+ OnFailure,
8
+ } from './types';
9
+ import { buildDag, type Dag, type DagNode } from './dag';
10
+ import { getHandler, hasHandler } from './registry';
11
+ import { runSpawn, runCommand } from './runner';
12
+ import { parseDuration, nowISO, generateRunId } from './utils';
13
+ import {
14
+ executeHook,
15
+ buildPipelineStartContext, buildTaskContext,
16
+ buildPipelineCompleteContext, buildPipelineErrorContext,
17
+ type PipelineInfo, type TrackInfo, type TaskInfo,
18
+ } from './hooks';
19
+ import { Logger, tailLines, clip } from './logger';
20
+ import { InMemoryApprovalGateway, type ApprovalGateway } from './approval';
21
+
22
+ // ═══ Preflight Validation ═══
23
+
24
+ function preflight(config: PipelineConfig, dag: Dag): void {
25
+ const errors: string[] = [];
26
+
27
+ for (const [, node] of dag.nodes) {
28
+ const task = node.task;
29
+ const track = node.track;
30
+ const driverName = task.driver ?? track.driver ?? config.driver ?? 'claude-code';
31
+
32
+ if (!hasHandler('drivers', driverName)) {
33
+ errors.push(`Task "${node.taskId}": driver "${driverName}" not registered`);
34
+ }
35
+
36
+ if (task.trigger && !hasHandler('triggers', task.trigger.type)) {
37
+ errors.push(`Task "${node.taskId}": trigger type "${task.trigger.type}" not registered`);
38
+ }
39
+
40
+ if (task.completion && !hasHandler('completions', task.completion.type)) {
41
+ errors.push(`Task "${node.taskId}": completion type "${task.completion.type}" not registered`);
42
+ }
43
+
44
+ const mws = task.middlewares ?? track.middlewares ?? [];
45
+ for (const mw of mws) {
46
+ if (!hasHandler('middlewares', mw.type)) {
47
+ errors.push(`Task "${node.taskId}": middleware type "${mw.type}" not registered`);
48
+ }
49
+ }
50
+
51
+ if (task.continue_from && hasHandler('drivers', driverName)) {
52
+ const driver = getHandler<DriverPlugin>('drivers', driverName);
53
+ if (!driver.capabilities.sessionResume) {
54
+ const upstreamId = resolveRefInDag(dag, task.continue_from, track.id);
55
+ if (upstreamId) {
56
+ const upstream = dag.nodes.get(upstreamId);
57
+ if (upstream && !upstream.task.output) {
58
+ errors.push(
59
+ `Task "${node.taskId}" uses continue_from: "${task.continue_from}", ` +
60
+ `but upstream task "${upstreamId}" has no "output" field. ` +
61
+ `Add output to the upstream task, or remove continue_from.`
62
+ );
63
+ }
64
+ }
65
+ }
66
+ }
67
+ }
68
+
69
+ if (errors.length > 0) {
70
+ throw new Error(`Preflight validation failed:\n - ${errors.join('\n - ')}`);
71
+ }
72
+ }
73
+
74
+ function resolveRefInDag(dag: Dag, ref: string, fromTrackId: string): string | null {
75
+ if (dag.nodes.has(ref)) return ref;
76
+ const sameTrack = `${fromTrackId}.${ref}`;
77
+ if (dag.nodes.has(sameTrack)) return sameTrack;
78
+ for (const [id] of dag.nodes) {
79
+ if (id.endsWith(`.${ref}`)) return id;
80
+ }
81
+ return null;
82
+ }
83
+
84
+ // ═══ Engine ═══
85
+
86
+ export interface EngineResult {
87
+ readonly success: boolean;
88
+ readonly runId: string;
89
+ readonly logPath: string;
90
+ readonly summary: {
91
+ total: number; success: number; failed: number;
92
+ skipped: number; timeout: number; blocked: number;
93
+ };
94
+ readonly states: ReadonlyMap<string, TaskState>;
95
+ }
96
+
97
+ export interface RunPipelineOptions {
98
+ readonly approvalGateway?: ApprovalGateway;
99
+ /**
100
+ * Maximum number of per-run log directories to retain under `<workDir>/logs/`.
101
+ * Oldest directories are deleted after each run. Defaults to 20. Set to 0 to disable cleanup.
102
+ */
103
+ readonly maxLogRuns?: number;
104
+ }
105
+
106
+ export async function runPipeline(
107
+ config: PipelineConfig,
108
+ workDir: string,
109
+ options: RunPipelineOptions = {},
110
+ ): Promise<EngineResult> {
111
+ const approvalGateway = options.approvalGateway ?? new InMemoryApprovalGateway();
112
+ const maxLogRuns = options.maxLogRuns ?? 20;
113
+ const dag = buildDag(config);
114
+ preflight(config, dag);
115
+
116
+ const runId = generateRunId();
117
+ const startedAt = nowISO();
118
+ const pipelineInfo: PipelineInfo = { name: config.name, run_id: runId, started_at: startedAt };
119
+ const log = new Logger(workDir, runId);
120
+ log.info('[pipeline]', `start "${config.name}" run_id=${runId}`);
121
+
122
+ // File-only: dump the resolved pipeline shape + DAG topology for post-mortem.
123
+ log.section('Pipeline configuration');
124
+ log.quiet(`name: ${config.name}`);
125
+ log.quiet(`driver: ${config.driver ?? '(default: claude-code)'}`);
126
+ log.quiet(`timeout: ${config.timeout ?? '(none)'}`);
127
+ log.quiet(`tracks: ${config.tracks.length}`);
128
+ log.quiet(`tasks (total): ${dag.nodes.size}`);
129
+ log.quiet(`plugins: ${(config.plugins ?? []).join(', ') || '(none)'}`);
130
+ log.quiet(`hooks: ${config.hooks ? Object.keys(config.hooks).join(', ') || '(none)' : '(none)'}`);
131
+
132
+ log.section('DAG topology');
133
+ for (const [id, node] of dag.nodes) {
134
+ const deps = node.dependsOn.length ? node.dependsOn.join(', ') : '(root)';
135
+ const kind = node.task.prompt ? 'ai' : 'cmd';
136
+ log.quiet(` • ${id} [${kind}] track=${node.track.id} deps=[${deps}]`);
137
+ }
138
+ log.quiet('');
139
+
140
+ // Initialize states (before hook, so we can return them even if blocked)
141
+ const states = new Map<string, TaskState>();
142
+ for (const [id, node] of dag.nodes) {
143
+ states.set(id, {
144
+ config: node.task,
145
+ trackConfig: node.track,
146
+ status: 'idle',
147
+ result: null,
148
+ startedAt: null,
149
+ finishedAt: null,
150
+ });
151
+ }
152
+
153
+ // Pipeline start hook (gate)
154
+ const startHook = await executeHook(
155
+ config.hooks, 'pipeline_start', buildPipelineStartContext(pipelineInfo), workDir,
156
+ );
157
+ if (!startHook.allowed) {
158
+ console.error(`Pipeline blocked by pipeline_start hook (exit code ${startHook.exitCode})`);
159
+ await executeHook(config.hooks, 'pipeline_error',
160
+ buildPipelineErrorContext(pipelineInfo, 'pipeline_blocked', 'pipeline_blocked'), workDir);
161
+ // All tasks stay idle pipeline never started
162
+ return {
163
+ success: false,
164
+ runId,
165
+ logPath: log.path,
166
+ summary: { total: dag.nodes.size, success: 0, failed: 0, skipped: 0, timeout: 0, blocked: 0 },
167
+ states,
168
+ };
169
+ }
170
+
171
+ // Pipeline approved transition all tasks to waiting
172
+ for (const [, state] of states) {
173
+ state.status = 'waiting';
174
+ }
175
+
176
+ const sessionMap = new Map<string, string>();
177
+ const outputMap = new Map<string, string>();
178
+ const normalizedMap = new Map<string, string>();
179
+
180
+ // Pipeline timeout
181
+ const pipelineTimeoutMs = config.timeout ? parseDuration(config.timeout) : 0;
182
+ let pipelineAborted = false;
183
+ const abortController = new AbortController();
184
+ let pipelineTimer: ReturnType<typeof setTimeout> | null = null;
185
+
186
+ if (pipelineTimeoutMs > 0) {
187
+ pipelineTimer = setTimeout(() => {
188
+ pipelineAborted = true;
189
+ abortController.abort();
190
+ }, pipelineTimeoutMs);
191
+ }
192
+
193
+ // When the pipeline is aborted (timeout, external shutdown), drain all
194
+ // pending approvals so waiting triggers unblock immediately.
195
+ abortController.signal.addEventListener('abort', () => {
196
+ approvalGateway.abortAll('pipeline aborted');
197
+ });
198
+
199
+ // ── Helpers ──
200
+
201
+ function getOnFailure(taskId: string): OnFailure {
202
+ return dag.nodes.get(taskId)?.track.on_failure ?? 'skip_downstream';
203
+ }
204
+
205
+ function isDependencySatisfied(depId: string): 'satisfied' | 'unsatisfied' | 'skip' {
206
+ const depState = states.get(depId);
207
+ if (!depState) return 'skip';
208
+ switch (depState.status) {
209
+ case 'success': return 'satisfied';
210
+ case 'skipped': return 'skip';
211
+ case 'failed': case 'timeout': case 'blocked':
212
+ return getOnFailure(depId) === 'ignore' ? 'satisfied' : 'skip';
213
+ default: return 'unsatisfied';
214
+ }
215
+ }
216
+
217
+ function applyStopAll(trackId: string): void {
218
+ for (const [, state] of states) {
219
+ const node = dag.nodes.get(state.config.id);
220
+ if (state.trackConfig.id === trackId && !isTerminal(state.status)) {
221
+ state.status = 'skipped';
222
+ state.finishedAt = nowISO();
223
+ }
224
+ }
225
+ }
226
+
227
+ function buildTaskInfoObj(taskId: string): TaskInfo {
228
+ const state = states.get(taskId)!;
229
+ return {
230
+ id: taskId,
231
+ name: state.config.name,
232
+ type: state.config.prompt ? 'ai' : 'command',
233
+ status: state.status,
234
+ exit_code: state.result?.exitCode ?? null,
235
+ duration_ms: state.result?.durationMs ?? null,
236
+ output_path: state.result?.outputPath ?? null,
237
+ stderr_path: state.result?.stderrPath ?? null,
238
+ session_id: state.result?.sessionId ?? null,
239
+ started_at: state.startedAt,
240
+ finished_at: state.finishedAt,
241
+ };
242
+ }
243
+
244
+ function trackInfoOf(taskId: string): TrackInfo {
245
+ const node = dag.nodes.get(taskId)!;
246
+ return { id: node.track.id, name: node.track.name };
247
+ }
248
+
249
+ async function fireHook(taskId: string, event: 'task_success' | 'task_failure'): Promise<void> {
250
+ await executeHook(config.hooks, event,
251
+ buildTaskContext(event, pipelineInfo, trackInfoOf(taskId), buildTaskInfoObj(taskId)), workDir);
252
+ }
253
+
254
+ // ── Process a single task ──
255
+
256
+ async function processTask(taskId: string): Promise<void> {
257
+ const state = states.get(taskId)!;
258
+ const node = dag.nodes.get(taskId)!;
259
+ const task = node.task;
260
+ const track = node.track;
261
+
262
+ log.section(`Task ${taskId}`);
263
+ log.debug(`[task:${taskId}]`,
264
+ `type=${task.prompt ? 'ai' : 'cmd'} track=${track.id} deps=[${node.dependsOn.join(', ') || '(root)'}]`);
265
+
266
+ // 1. Check dependencies
267
+ for (const depId of node.dependsOn) {
268
+ const result = isDependencySatisfied(depId);
269
+ if (result === 'skip') {
270
+ const depStatus = states.get(depId)?.status ?? 'unknown';
271
+ log.debug(`[task:${taskId}]`, `skipped (upstream "${depId}" status=${depStatus})`);
272
+ state.status = 'skipped';
273
+ state.finishedAt = nowISO();
274
+ return;
275
+ }
276
+ if (result === 'unsatisfied') return; // still waiting
277
+ }
278
+
279
+ // 2. Check trigger
280
+ if (task.trigger) {
281
+ log.debug(`[task:${taskId}]`, `trigger wait: type=${task.trigger.type} ${JSON.stringify(task.trigger)}`);
282
+ try {
283
+ const triggerPlugin = getHandler<TriggerPlugin>('triggers', task.trigger.type);
284
+ await triggerPlugin.watch(task.trigger as Record<string, unknown>, {
285
+ taskId: node.taskId,
286
+ trackId: track.id,
287
+ workDir: task.cwd ?? workDir,
288
+ signal: abortController.signal,
289
+ approvalGateway,
290
+ });
291
+ log.debug(`[task:${taskId}]`, `trigger fired`);
292
+ } catch (err: unknown) {
293
+ const msg = err instanceof Error ? err.message : String(err);
294
+ // If pipeline was aborted while we were still waiting for the trigger,
295
+ // this task never entered running state skipped, not timeout.
296
+ if (pipelineAborted) {
297
+ state.status = 'skipped';
298
+ } else if (msg.includes('rejected') || msg.includes('denied')) {
299
+ state.status = 'blocked'; // user/policy rejection
300
+ } else if (msg.includes('timeout')) {
301
+ state.status = 'timeout'; // genuine trigger wait timeout
302
+ } else {
303
+ state.status = 'failed'; // plugin error, watcher crash, etc.
304
+ }
305
+ state.finishedAt = nowISO();
306
+ await fireHook(taskId, 'task_failure');
307
+ return;
308
+ }
309
+ }
310
+
311
+ // 3. task_start hook (gate)
312
+ const hookResult = await executeHook(config.hooks, 'task_start',
313
+ buildTaskContext('task_start', pipelineInfo, trackInfoOf(taskId), buildTaskInfoObj(taskId)), workDir);
314
+ if (hookResult.exitCode !== 0 || config.hooks?.task_start) {
315
+ log.debug(`[task:${taskId}]`,
316
+ `task_start hook exit=${hookResult.exitCode} allowed=${hookResult.allowed}`);
317
+ }
318
+ if (!hookResult.allowed) {
319
+ state.status = 'blocked';
320
+ state.finishedAt = nowISO();
321
+ await fireHook(taskId, 'task_failure');
322
+ return;
323
+ }
324
+
325
+ // 4. Mark running
326
+ state.status = 'running';
327
+ state.startedAt = nowISO();
328
+ log.info(`[task:${taskId}]`, task.command ? `running: ${task.command}` : `running (driver task)`);
329
+
330
+ // File-only: resolved config for this task
331
+ const resolvedDriver = task.driver ?? track.driver ?? config.driver ?? 'claude-code';
332
+ const resolvedTier = task.model_tier ?? track.model_tier ?? '(default)';
333
+ const resolvedPerms = task.permissions ?? track.permissions ?? '(default)';
334
+ const resolvedCwd = task.cwd ?? track.cwd ?? workDir;
335
+ log.debug(`[task:${taskId}]`,
336
+ `resolved: driver=${resolvedDriver} tier=${resolvedTier} cwd=${resolvedCwd}`);
337
+ log.debug(`[task:${taskId}]`, `permissions: ${JSON.stringify(resolvedPerms)}`);
338
+ if (task.continue_from) {
339
+ log.debug(`[task:${taskId}]`, `continue_from: "${task.continue_from}"`);
340
+ }
341
+ if (task.timeout) {
342
+ log.debug(`[task:${taskId}]`, `timeout: ${task.timeout}`);
343
+ }
344
+
345
+ try {
346
+ let result: TaskResult;
347
+ const timeoutMs = task.timeout ? parseDuration(task.timeout) : undefined;
348
+
349
+ const runOpts = { timeoutMs, signal: abortController.signal };
350
+
351
+ if (task.command) {
352
+ log.debug(`[task:${taskId}]`, `command: ${task.command}`);
353
+ result = await runCommand(task.command, task.cwd ?? workDir, runOpts);
354
+ } else {
355
+ // AI task: apply middleware chain
356
+ const driverName = task.driver ?? track.driver ?? config.driver ?? 'claude-code';
357
+ const driver = getHandler<DriverPlugin>('drivers', driverName);
358
+
359
+ let prompt = task.prompt!;
360
+ const originalLen = prompt.length;
361
+ const mws = task.middlewares !== undefined ? task.middlewares : track.middlewares;
362
+ if (mws && mws.length > 0) {
363
+ log.debug(`[task:${taskId}]`,
364
+ `middleware chain: ${mws.map(m => m.type).join(' → ')}`);
365
+ const mwCtx: MiddlewareContext = {
366
+ task, track, outputMap, workDir: task.cwd ?? workDir,
367
+ };
368
+ for (const mwConfig of mws) {
369
+ const before = prompt.length;
370
+ const mwPlugin = getHandler<MiddlewarePlugin>('middlewares', mwConfig.type);
371
+ prompt = await mwPlugin.enhance(prompt, mwConfig as Record<string, unknown>, mwCtx);
372
+ log.debug(`[task:${taskId}]`,
373
+ ` ${mwConfig.type}: ${before} → ${prompt.length} chars`);
374
+ }
375
+ }
376
+ log.debug(`[task:${taskId}]`,
377
+ `prompt: ${originalLen} chars (final: ${prompt.length} chars)`);
378
+ log.quiet(`--- prompt (final) ---\n${clip(prompt)}\n--- end prompt ---`);
379
+
380
+ const enrichedTask: TaskConfig = { ...task, prompt };
381
+ const driverCtx: DriverContext = {
382
+ sessionMap, outputMap, normalizedMap, workDir: task.cwd ?? workDir,
383
+ };
384
+ const spec = await driver.buildCommand(enrichedTask, track, driverCtx);
385
+ log.debug(`[task:${taskId}]`, `driver=${driverName}`);
386
+ log.debug(`[task:${taskId}]`,
387
+ `spawn args: ${JSON.stringify(spec.args)}`);
388
+ if (spec.cwd) log.debug(`[task:${taskId}]`, `spawn cwd: ${spec.cwd}`);
389
+ if (spec.env) log.debug(`[task:${taskId}]`,
390
+ `spawn env overrides: ${Object.keys(spec.env).join(', ')}`);
391
+ if (spec.stdin) log.debug(`[task:${taskId}]`,
392
+ `spawn stdin: ${spec.stdin.length} chars`);
393
+ result = await runSpawn(spec, driver, runOpts);
394
+ }
395
+
396
+ // 5. Determine status
397
+ if (result.exitCode === -1) {
398
+ state.status = 'timeout';
399
+ } else if (result.exitCode !== 0) {
400
+ state.status = 'failed';
401
+ } else if (task.completion) {
402
+ const plugin = getHandler<CompletionPlugin>('completions', task.completion.type);
403
+ const completionCtx = { workDir: task.cwd ?? workDir };
404
+ const passed = await plugin.check(task.completion as Record<string, unknown>, result, completionCtx);
405
+ state.status = passed ? 'success' : 'failed';
406
+ } else {
407
+ state.status = 'success';
408
+ }
409
+
410
+ // 6. Write output file with RAW stdout (preserves driver output format).
411
+ // The separate normalizedMap holds canonical text for continue_from.
412
+ if (task.output) {
413
+ const outPath = resolve(workDir, task.output);
414
+ await mkdir(dirname(outPath), { recursive: true });
415
+ await Bun.write(outPath, result.stdout);
416
+ result = { ...result, outputPath: outPath };
417
+ outputMap.set(taskId, outPath);
418
+ const bareId = taskId.includes('.') ? taskId.split('.').pop()! : taskId;
419
+ if (!outputMap.has(bareId)) outputMap.set(bareId, outPath);
420
+ }
421
+
422
+ // Store normalized text separately (in-memory) for continue_from handoff
423
+ if (result.normalizedOutput !== null) {
424
+ normalizedMap.set(taskId, result.normalizedOutput);
425
+ const bareId = taskId.includes('.') ? taskId.split('.').pop()! : taskId;
426
+ if (!normalizedMap.has(bareId)) normalizedMap.set(bareId, result.normalizedOutput);
427
+ }
428
+
429
+ if (result.stderr) {
430
+ const stderrPath = resolve(log.dir, `${taskId.replace(/\./g, '_')}.stderr`);
431
+ await Bun.write(stderrPath, result.stderr);
432
+ result = { ...result, stderrPath };
433
+ }
434
+
435
+ if (result.sessionId) {
436
+ sessionMap.set(taskId, result.sessionId);
437
+ const bareId = taskId.includes('.') ? taskId.split('.').pop()! : taskId;
438
+ if (!sessionMap.has(bareId)) sessionMap.set(bareId, result.sessionId);
439
+ }
440
+
441
+ state.result = result;
442
+ state.finishedAt = nowISO();
443
+
444
+ // Log task outcome with relevant details
445
+ const durSec = (result.durationMs / 1000).toFixed(1);
446
+ if (state.status === 'success') {
447
+ log.info(`[task:${taskId}]`, `success (${durSec}s)`);
448
+ } else {
449
+ log.error(`[task:${taskId}]`,
450
+ `${state.status} exit=${result.exitCode} duration=${durSec}s`);
451
+ if (result.stderr) {
452
+ const tail = tailLines(result.stderr, 10);
453
+ log.error(`[task:${taskId}]`, `stderr tail:\n${tail}`);
454
+ }
455
+ }
456
+
457
+ // File-only: full stdout/stderr dump (clipped) + extracted metadata
458
+ log.debug(`[task:${taskId}]`,
459
+ `stdout: ${result.stdout.length} chars, stderr: ${result.stderr.length} chars`);
460
+ if (result.sessionId) {
461
+ log.debug(`[task:${taskId}]`, `sessionId: ${result.sessionId}`);
462
+ }
463
+ if (result.outputPath) {
464
+ log.debug(`[task:${taskId}]`, `wrote output: ${result.outputPath}`);
465
+ }
466
+ if (result.stderrPath) {
467
+ log.debug(`[task:${taskId}]`, `wrote stderr: ${result.stderrPath}`);
468
+ }
469
+ if (result.stdout) {
470
+ log.quiet(`--- stdout (${taskId}) ---\n${clip(result.stdout)}\n--- end stdout ---`);
471
+ }
472
+ if (result.stderr) {
473
+ log.quiet(`--- stderr (${taskId}) ---\n${clip(result.stderr)}\n--- end stderr ---`);
474
+ }
475
+ if (task.completion) {
476
+ log.debug(`[task:${taskId}]`,
477
+ `completion check: type=${task.completion.type} result=${state.status}`);
478
+ }
479
+
480
+ } catch (err: unknown) {
481
+ state.status = 'failed';
482
+ state.finishedAt = nowISO();
483
+ const errMsg = err instanceof Error ? (err.stack ?? err.message) : String(err);
484
+ log.error(`[task:${taskId}]`, `failed before execution: ${errMsg}`);
485
+ state.result = {
486
+ exitCode: -1,
487
+ stdout: '',
488
+ stderr: errMsg,
489
+ outputPath: null, stderrPath: null, durationMs: 0,
490
+ sessionId: null, normalizedOutput: null,
491
+ };
492
+ }
493
+
494
+ // 7. Fire hooks
495
+ const finalStatus: TaskStatus = state.status;
496
+ await fireHook(taskId, finalStatus === 'success' ? 'task_success' : 'task_failure');
497
+
498
+ // 8. Handle stop_all for failure states
499
+ if (finalStatus !== 'success' && getOnFailure(taskId) === 'stop_all') {
500
+ applyStopAll(node.track.id);
501
+ }
502
+ }
503
+
504
+ // ── Event loop ──
505
+ try {
506
+ let progress = true;
507
+ while (progress && !pipelineAborted) {
508
+ progress = false;
509
+
510
+ // Collect tasks whose deps are all terminal and that are still waiting
511
+ const launchable: string[] = [];
512
+ for (const [id, state] of states) {
513
+ if (state.status !== 'waiting') continue;
514
+ const node = dag.nodes.get(id)!;
515
+ const allDepsTerminal = node.dependsOn.length === 0 ||
516
+ node.dependsOn.every(d => isTerminal(states.get(d)!.status));
517
+ if (allDepsTerminal) launchable.push(id);
518
+ }
519
+
520
+ if (launchable.length === 0) {
521
+ // Check if anything is still running (trigger waits etc.)
522
+ const anyNonTerminal = [...states.values()].some(s => !isTerminal(s.status));
523
+ if (!anyNonTerminal) break;
524
+ await new Promise(r => setTimeout(r, 50));
525
+ progress = true;
526
+ continue;
527
+ }
528
+
529
+ // Launch all launchable tasks concurrently
530
+ await Promise.all(launchable.map(id => processTask(id)));
531
+ progress = true;
532
+ }
533
+
534
+ if (pipelineAborted) {
535
+ for (const [, state] of states) {
536
+ if (!isTerminal(state.status)) {
537
+ // Running tasks get timeout (they were killed); waiting tasks get skipped
538
+ state.status = state.status === 'running' ? 'timeout' : 'skipped';
539
+ state.finishedAt = nowISO();
540
+ }
541
+ }
542
+ }
543
+ } finally {
544
+ if (pipelineTimer) clearTimeout(pipelineTimer);
545
+ // Safety net: drain any approvals still pending at shutdown (e.g. crash path).
546
+ if (approvalGateway.pending().length > 0) {
547
+ approvalGateway.abortAll('pipeline finished');
548
+ }
549
+ }
550
+
551
+ // ── Summary ──
552
+ const summary = { total: 0, success: 0, failed: 0, skipped: 0, timeout: 0, blocked: 0 };
553
+ for (const [, state] of states) {
554
+ summary.total++;
555
+ switch (state.status) {
556
+ case 'success': summary.success++; break;
557
+ case 'failed': summary.failed++; break;
558
+ case 'skipped': summary.skipped++; break;
559
+ case 'timeout': summary.timeout++; break;
560
+ case 'blocked': summary.blocked++; break;
561
+ }
562
+ }
563
+
564
+ const finishedAt = nowISO();
565
+ const durationMs = new Date(finishedAt).getTime() - new Date(startedAt).getTime();
566
+
567
+ if (pipelineAborted) {
568
+ await executeHook(config.hooks, 'pipeline_error',
569
+ buildPipelineErrorContext(pipelineInfo, 'Pipeline timeout exceeded'), workDir);
570
+ } else {
571
+ await executeHook(config.hooks, 'pipeline_complete',
572
+ buildPipelineCompleteContext(
573
+ { ...pipelineInfo, finished_at: finishedAt, duration_ms: durationMs }, summary), workDir);
574
+ }
575
+
576
+ const allSuccess = !pipelineAborted
577
+ && summary.failed === 0 && summary.timeout === 0 && summary.blocked === 0;
578
+
579
+ log.section('Pipeline summary');
580
+ log.quiet(`status: ${pipelineAborted ? 'aborted (timeout)' : 'completed'}`);
581
+ log.quiet(`duration: ${(durationMs / 1000).toFixed(1)}s`);
582
+ log.quiet(
583
+ `counts: total=${summary.total} success=${summary.success} ` +
584
+ `failed=${summary.failed} skipped=${summary.skipped} ` +
585
+ `timeout=${summary.timeout} blocked=${summary.blocked}`);
586
+ log.quiet('');
587
+ log.quiet('per-task:');
588
+ for (const [id, state] of states) {
589
+ const dur = state.result?.durationMs != null
590
+ ? `${(state.result.durationMs / 1000).toFixed(1)}s` : '-';
591
+ const exit = state.result?.exitCode ?? '-';
592
+ log.quiet(` ${state.status.padEnd(8)} ${id} (exit=${exit}, ${dur})`);
593
+ }
594
+
595
+ console.log(`\n[Pipeline "${config.name}"] completed`);
596
+ console.log(` Total: ${summary.total} | Success: ${summary.success} | Failed: ${summary.failed} | Skipped: ${summary.skipped} | Timeout: ${summary.timeout} | Blocked: ${summary.blocked}`);
597
+ console.log(` Duration: ${(durationMs / 1000).toFixed(1)}s`);
598
+ console.log(` Log: ${log.path}`);
599
+
600
+ // Prune old per-run log directories, keeping only the most recent maxLogRuns.
601
+ if (maxLogRuns > 0) {
602
+ await pruneLogDirs(resolve(workDir, 'logs'), maxLogRuns);
603
+ }
604
+
605
+ return { success: allSuccess, runId, logPath: log.path, summary, states };
606
+ }
607
+
608
+ /**
609
+ * Delete the oldest subdirectories under `logsDir`, keeping only the most recent `keep`.
610
+ * Directories are sorted lexicographically; because runIds are prefixed with a base-36
611
+ * timestamp, lexicographic order equals chronological order.
612
+ */
613
+ async function pruneLogDirs(logsDir: string, keep: number): Promise<void> {
614
+ let entries: string[];
615
+ try {
616
+ entries = await readdir(logsDir);
617
+ } catch {
618
+ return; // logsDir doesn't exist yet — nothing to prune
619
+ }
620
+
621
+ // Only consider directories that look like run IDs (run_<...>)
622
+ const runDirs = entries.filter(e => e.startsWith('run_')).sort();
623
+ const toDelete = runDirs.slice(0, Math.max(0, runDirs.length - keep));
624
+
625
+ await Promise.all(
626
+ toDelete.map(dir =>
627
+ rm(resolve(logsDir, dir), { recursive: true, force: true }).catch(() => {
628
+ // Ignore deletion errors — stale dirs are better than a crash
629
+ })
630
+ )
631
+ );
632
+ }
633
+
634
+ function isTerminal(status: TaskStatus): boolean {
635
+ return status === 'success' || status === 'failed' || status === 'timeout'
636
+ || status === 'skipped' || status === 'blocked';
637
+ }