@tagma/sdk 0.6.0 → 0.6.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -21
- package/README.md +573 -573
- package/dist/drivers/opencode.d.ts.map +1 -1
- package/dist/drivers/opencode.js +47 -17
- package/dist/drivers/opencode.js.map +1 -1
- package/package.json +2 -2
- package/src/bootstrap.ts +37 -37
- package/src/completions/output-check.ts +92 -92
- package/src/dag.ts +245 -245
- package/src/drivers/opencode.ts +410 -371
- package/src/engine.ts +1220 -1220
- package/src/hooks.ts +193 -193
- package/src/middlewares/static-context.ts +49 -49
- package/src/pipeline-runner.ts +173 -173
- package/src/prompt-doc.ts +49 -49
- package/src/registry.ts +267 -267
- package/src/runner.ts +460 -460
- package/src/schema.test.ts +101 -101
- package/src/schema.ts +338 -338
- package/src/sdk.ts +118 -118
- package/src/task-ref.test.ts +401 -401
- package/src/task-ref.ts +120 -120
- package/src/validate-raw.ts +412 -412
- package/dist/drivers/claude-code.d.ts +0 -3
- package/dist/drivers/claude-code.d.ts.map +0 -1
- package/dist/drivers/claude-code.js +0 -225
- package/dist/drivers/claude-code.js.map +0 -1
package/src/engine.ts
CHANGED
|
@@ -1,1220 +1,1220 @@
|
|
|
1
|
-
import { resolve } from 'path';
|
|
2
|
-
import { readdir, rm } from 'fs/promises';
|
|
3
|
-
import type {
|
|
4
|
-
PipelineConfig,
|
|
5
|
-
TaskConfig,
|
|
6
|
-
TaskState,
|
|
7
|
-
TaskStatus,
|
|
8
|
-
TaskResult,
|
|
9
|
-
DriverPlugin,
|
|
10
|
-
TriggerPlugin,
|
|
11
|
-
CompletionPlugin,
|
|
12
|
-
MiddlewarePlugin,
|
|
13
|
-
MiddlewareContext,
|
|
14
|
-
DriverContext,
|
|
15
|
-
OnFailure,
|
|
16
|
-
PromptDocument,
|
|
17
|
-
Permissions,
|
|
18
|
-
AbortReason,
|
|
19
|
-
RunEventPayload,
|
|
20
|
-
RunTaskState,
|
|
21
|
-
} from './types';
|
|
22
|
-
import { buildDag, type Dag } from './dag';
|
|
23
|
-
import { getHandler, hasHandler, loadPlugins } from './registry';
|
|
24
|
-
import { runSpawn, runCommand } from './runner';
|
|
25
|
-
import { parseDuration, nowISO, generateRunId } from './utils';
|
|
26
|
-
import { promptDocumentFromString, serializePromptDocument } from './prompt-doc';
|
|
27
|
-
import {
|
|
28
|
-
executeHook,
|
|
29
|
-
buildPipelineStartContext,
|
|
30
|
-
buildTaskContext,
|
|
31
|
-
buildPipelineCompleteContext,
|
|
32
|
-
buildPipelineErrorContext,
|
|
33
|
-
type PipelineInfo,
|
|
34
|
-
type TrackInfo,
|
|
35
|
-
type TaskInfo,
|
|
36
|
-
} from './hooks';
|
|
37
|
-
import { Logger, tailLines, clip } from './logger';
|
|
38
|
-
import { InMemoryApprovalGateway, type ApprovalGateway } from './approval';
|
|
39
|
-
|
|
40
|
-
// ═══ A7: Typed trigger errors ═══
|
|
41
|
-
// Replace string-matching on error messages with structured error types so
|
|
42
|
-
// coincidental substrings don't cause misclassification.
|
|
43
|
-
|
|
44
|
-
export class TriggerBlockedError extends Error {
|
|
45
|
-
readonly code = 'TRIGGER_BLOCKED' as const;
|
|
46
|
-
constructor(message: string) {
|
|
47
|
-
super(message);
|
|
48
|
-
this.name = 'TriggerBlockedError';
|
|
49
|
-
}
|
|
50
|
-
}
|
|
51
|
-
|
|
52
|
-
export class TriggerTimeoutError extends Error {
|
|
53
|
-
readonly code = 'TRIGGER_TIMEOUT' as const;
|
|
54
|
-
constructor(message: string) {
|
|
55
|
-
super(message);
|
|
56
|
-
this.name = 'TriggerTimeoutError';
|
|
57
|
-
}
|
|
58
|
-
}
|
|
59
|
-
|
|
60
|
-
// ═══ Preflight Validation ═══
|
|
61
|
-
|
|
62
|
-
function preflight(config: PipelineConfig, dag: Dag): void {
|
|
63
|
-
const errors: string[] = [];
|
|
64
|
-
|
|
65
|
-
for (const [, node] of dag.nodes) {
|
|
66
|
-
const task = node.task;
|
|
67
|
-
const track = node.track;
|
|
68
|
-
const driverName = task.driver ?? track.driver ?? config.driver ?? 'opencode';
|
|
69
|
-
|
|
70
|
-
// Pure command tasks don't use a driver — skip driver registration check.
|
|
71
|
-
const isCommandOnly = task.command && !task.prompt;
|
|
72
|
-
|
|
73
|
-
if (!isCommandOnly && !hasHandler('drivers', driverName)) {
|
|
74
|
-
errors.push(`Task "${node.taskId}": driver "${driverName}" not registered`);
|
|
75
|
-
}
|
|
76
|
-
|
|
77
|
-
if (task.trigger && !hasHandler('triggers', task.trigger.type)) {
|
|
78
|
-
errors.push(`Task "${node.taskId}": trigger type "${task.trigger.type}" not registered`);
|
|
79
|
-
}
|
|
80
|
-
|
|
81
|
-
if (task.completion && !hasHandler('completions', task.completion.type)) {
|
|
82
|
-
errors.push(
|
|
83
|
-
`Task "${node.taskId}": completion type "${task.completion.type}" not registered`,
|
|
84
|
-
);
|
|
85
|
-
}
|
|
86
|
-
|
|
87
|
-
const mws = task.middlewares ?? track.middlewares ?? [];
|
|
88
|
-
for (const mw of mws) {
|
|
89
|
-
if (!hasHandler('middlewares', mw.type)) {
|
|
90
|
-
errors.push(`Task "${node.taskId}": middleware type "${mw.type}" not registered`);
|
|
91
|
-
}
|
|
92
|
-
}
|
|
93
|
-
|
|
94
|
-
if (task.continue_from && hasHandler('drivers', driverName)) {
|
|
95
|
-
const driver = getHandler<DriverPlugin>('drivers', driverName);
|
|
96
|
-
if (!driver.capabilities.sessionResume) {
|
|
97
|
-
// buildDag has already qualified `continue_from` and stored the result
|
|
98
|
-
// on the node; preflight runs after buildDag, so the upstream id is
|
|
99
|
-
// always available here without re-resolving.
|
|
100
|
-
const upstreamId = node.resolvedContinueFrom;
|
|
101
|
-
if (upstreamId) {
|
|
102
|
-
const upstream = dag.nodes.get(upstreamId);
|
|
103
|
-
if (upstream) {
|
|
104
|
-
// A handoff is possible via session resume (already ruled out above),
|
|
105
|
-
// OR in-memory text injection through normalizedMap
|
|
106
|
-
// (when the upstream driver implements parseResult and returns normalizedOutput).
|
|
107
|
-
const upstreamDriverName =
|
|
108
|
-
upstream.task.driver ?? upstream.track.driver ?? config.driver ?? 'opencode';
|
|
109
|
-
const upstreamDriver = hasHandler('drivers', upstreamDriverName)
|
|
110
|
-
? getHandler<DriverPlugin>('drivers', upstreamDriverName)
|
|
111
|
-
: null;
|
|
112
|
-
const canNormalize = typeof upstreamDriver?.parseResult === 'function';
|
|
113
|
-
|
|
114
|
-
if (!canNormalize) {
|
|
115
|
-
errors.push(
|
|
116
|
-
`Task "${node.taskId}" uses continue_from: "${task.continue_from}", ` +
|
|
117
|
-
`but upstream task "${upstreamId}" its driver ` +
|
|
118
|
-
`does not implement parseResult for text-injection handoff. ` +
|
|
119
|
-
`Use a driver with parseResult, or remove continue_from.`,
|
|
120
|
-
);
|
|
121
|
-
}
|
|
122
|
-
}
|
|
123
|
-
}
|
|
124
|
-
}
|
|
125
|
-
}
|
|
126
|
-
}
|
|
127
|
-
|
|
128
|
-
if (errors.length > 0) {
|
|
129
|
-
throw new Error(`Preflight validation failed:\n - ${errors.join('\n - ')}`);
|
|
130
|
-
}
|
|
131
|
-
}
|
|
132
|
-
|
|
133
|
-
// ═══ Engine ═══
|
|
134
|
-
|
|
135
|
-
export interface EngineResult {
|
|
136
|
-
readonly success: boolean;
|
|
137
|
-
readonly runId: string;
|
|
138
|
-
readonly logPath: string;
|
|
139
|
-
readonly summary: {
|
|
140
|
-
total: number;
|
|
141
|
-
success: number;
|
|
142
|
-
failed: number;
|
|
143
|
-
skipped: number;
|
|
144
|
-
timeout: number;
|
|
145
|
-
blocked: number;
|
|
146
|
-
};
|
|
147
|
-
readonly states: ReadonlyMap<string, TaskState>;
|
|
148
|
-
}
|
|
149
|
-
|
|
150
|
-
// ═══ Pipeline Events ═══
|
|
151
|
-
//
|
|
152
|
-
// The engine emits RunEventPayload values (defined in @tagma/types) via
|
|
153
|
-
// `onEvent`. Every payload carries `runId`; the editor server stamps a
|
|
154
|
-
// per-run `seq` before broadcasting. There is one event vocabulary
|
|
155
|
-
// end-to-end — no server-side translation layer.
|
|
156
|
-
|
|
157
|
-
// Re-export so SDK consumers can import the event type without reaching
|
|
158
|
-
// into @tagma/types directly.
|
|
159
|
-
export type { RunEventPayload } from './types';
|
|
160
|
-
|
|
161
|
-
// ═══ Helpers ═══
|
|
162
|
-
|
|
163
|
-
/**
|
|
164
|
-
* Project the engine's internal TaskState onto the wire RunTaskState
|
|
165
|
-
* shape. `logs` / `totalLogCount` default to empty — they are populated
|
|
166
|
-
* on the server side from streamed `task_log` events, not from state.
|
|
167
|
-
*/
|
|
168
|
-
function toRunTaskState(
|
|
169
|
-
taskId: string,
|
|
170
|
-
trackId: string,
|
|
171
|
-
taskName: string,
|
|
172
|
-
state: TaskState,
|
|
173
|
-
): RunTaskState {
|
|
174
|
-
const result = state.result;
|
|
175
|
-
const cfg = state.config;
|
|
176
|
-
return {
|
|
177
|
-
taskId,
|
|
178
|
-
trackId,
|
|
179
|
-
taskName,
|
|
180
|
-
status: state.status,
|
|
181
|
-
startedAt: state.startedAt,
|
|
182
|
-
finishedAt: state.finishedAt,
|
|
183
|
-
durationMs: result?.durationMs ?? null,
|
|
184
|
-
exitCode: result?.exitCode ?? null,
|
|
185
|
-
stdout: result?.stdout ?? '',
|
|
186
|
-
stderr: result?.stderr ?? '',
|
|
187
|
-
stderrPath: result?.stderrPath ?? null,
|
|
188
|
-
sessionId: result?.sessionId ?? null,
|
|
189
|
-
normalizedOutput: result?.normalizedOutput ?? null,
|
|
190
|
-
resolvedDriver: cfg.driver ?? null,
|
|
191
|
-
resolvedModel: cfg.model ?? null,
|
|
192
|
-
resolvedPermissions: (cfg.permissions as Permissions | undefined) ?? null,
|
|
193
|
-
logs: [],
|
|
194
|
-
totalLogCount: 0,
|
|
195
|
-
};
|
|
196
|
-
}
|
|
197
|
-
|
|
198
|
-
export interface RunPipelineOptions {
|
|
199
|
-
readonly approvalGateway?: ApprovalGateway;
|
|
200
|
-
/**
|
|
201
|
-
* Maximum number of per-run log directories to retain under `<workDir>/.tagma/logs/`.
|
|
202
|
-
* Oldest directories are deleted after each run. Defaults to 20. Set to 0 to disable cleanup.
|
|
203
|
-
*/
|
|
204
|
-
readonly maxLogRuns?: number;
|
|
205
|
-
/**
|
|
206
|
-
* Caller-supplied run ID. When provided the engine uses this instead of
|
|
207
|
-
* generating its own via `generateRunId()`, keeping the editor and SDK
|
|
208
|
-
* log directories aligned on the same ID.
|
|
209
|
-
*/
|
|
210
|
-
readonly runId?: string;
|
|
211
|
-
/**
|
|
212
|
-
* External AbortSignal — aborting it cancels the pipeline immediately.
|
|
213
|
-
* Equivalent to the pipeline timeout firing, but caller-controlled.
|
|
214
|
-
*/
|
|
215
|
-
readonly signal?: AbortSignal;
|
|
216
|
-
/**
|
|
217
|
-
* Called on every pipeline/task status transition.
|
|
218
|
-
* Use for real-time UI updates (e.g. updating a visual workflow graph).
|
|
219
|
-
*/
|
|
220
|
-
readonly onEvent?: (event: RunEventPayload) => void;
|
|
221
|
-
/**
|
|
222
|
-
* Skip the engine's built-in `loadPlugins(config.plugins)` call.
|
|
223
|
-
* Use this when the host has already pre-loaded plugins from a custom
|
|
224
|
-
* resolution path (e.g. a user workspace's node_modules) so the engine
|
|
225
|
-
* doesn't re-resolve them via Node's default cwd-based import.
|
|
226
|
-
*/
|
|
227
|
-
readonly skipPluginLoading?: boolean;
|
|
228
|
-
}
|
|
229
|
-
|
|
230
|
-
// Poll interval when no tasks are in-flight but non-terminal tasks remain
|
|
231
|
-
// (e.g. tasks waiting on a file or manual trigger).
|
|
232
|
-
const POLL_INTERVAL_MS = 50;
|
|
233
|
-
|
|
234
|
-
// R15: cap on each normalized-output entry stored in normalizedMap so a
|
|
235
|
-
// runaway parseResult can't accumulate hundreds of MB across tasks. 1 MB
|
|
236
|
-
// is generous for any text-context handoff between AI tasks.
|
|
237
|
-
const MAX_NORMALIZED_BYTES = 1_000_000;
|
|
238
|
-
|
|
239
|
-
export async function runPipeline(
|
|
240
|
-
config: PipelineConfig,
|
|
241
|
-
workDir: string,
|
|
242
|
-
options: RunPipelineOptions = {},
|
|
243
|
-
): Promise<EngineResult> {
|
|
244
|
-
const approvalGateway = options.approvalGateway ?? new InMemoryApprovalGateway();
|
|
245
|
-
const maxLogRuns = options.maxLogRuns ?? 20;
|
|
246
|
-
|
|
247
|
-
// Load any plugins declared in the pipeline config before preflight so that
|
|
248
|
-
// drivers, completions, and middlewares referenced in YAML are registered.
|
|
249
|
-
// Hosts that pre-load plugins from a custom path (e.g. the editor loading
|
|
250
|
-
// from the user's workspace node_modules) pass skipPluginLoading: true so
|
|
251
|
-
// we don't re-resolve via Node's cwd-based default import.
|
|
252
|
-
if (!options.skipPluginLoading && config.plugins?.length) {
|
|
253
|
-
await loadPlugins(config.plugins);
|
|
254
|
-
}
|
|
255
|
-
|
|
256
|
-
const dag = buildDag(config);
|
|
257
|
-
const runId = options.runId ?? generateRunId();
|
|
258
|
-
preflight(config, dag);
|
|
259
|
-
|
|
260
|
-
const startedAt = nowISO();
|
|
261
|
-
const pipelineInfo: PipelineInfo = { name: config.name, run_id: runId, started_at: startedAt };
|
|
262
|
-
// Forward every structured log line to subscribers as task_log events.
|
|
263
|
-
// Reading options.onEvent inside the callback (vs. capturing it once) keeps
|
|
264
|
-
// the SDK behavior correct if callers pass a fresh onEvent on each run.
|
|
265
|
-
const log = new Logger(workDir, runId, (record) => {
|
|
266
|
-
options.onEvent?.({
|
|
267
|
-
type: 'task_log',
|
|
268
|
-
runId,
|
|
269
|
-
taskId: record.taskId,
|
|
270
|
-
level: record.level,
|
|
271
|
-
timestamp: record.timestamp,
|
|
272
|
-
text: record.text,
|
|
273
|
-
});
|
|
274
|
-
});
|
|
275
|
-
|
|
276
|
-
try {
|
|
277
|
-
log.info('[pipeline]', `start "${config.name}" run_id=${runId}`);
|
|
278
|
-
|
|
279
|
-
// File-only: dump the resolved pipeline shape + DAG topology for post-mortem.
|
|
280
|
-
log.section('Pipeline configuration');
|
|
281
|
-
log.quiet(`name: ${config.name}`);
|
|
282
|
-
log.quiet(`driver: ${config.driver ?? '(default: opencode)'}`);
|
|
283
|
-
log.quiet(`timeout: ${config.timeout ?? '(none)'}`);
|
|
284
|
-
log.quiet(`tracks: ${config.tracks.length}`);
|
|
285
|
-
log.quiet(`tasks (total): ${dag.nodes.size}`);
|
|
286
|
-
log.quiet(`plugins: ${(config.plugins ?? []).join(', ') || '(none)'}`);
|
|
287
|
-
log.quiet(
|
|
288
|
-
`hooks: ${config.hooks ? Object.keys(config.hooks).join(', ') || '(none)' : '(none)'}`,
|
|
289
|
-
);
|
|
290
|
-
|
|
291
|
-
log.section('DAG topology');
|
|
292
|
-
for (const [id, node] of dag.nodes) {
|
|
293
|
-
const deps = node.dependsOn.length ? node.dependsOn.join(', ') : '(root)';
|
|
294
|
-
const kind = node.task.prompt ? 'ai' : 'cmd';
|
|
295
|
-
log.quiet(` • ${id} [${kind}] track=${node.track.id} deps=[${deps}]`);
|
|
296
|
-
}
|
|
297
|
-
log.quiet('');
|
|
298
|
-
|
|
299
|
-
// Initialize states (before hook, so we can return them even if blocked)
|
|
300
|
-
const states = new Map<string, TaskState>();
|
|
301
|
-
for (const [id, node] of dag.nodes) {
|
|
302
|
-
states.set(id, {
|
|
303
|
-
config: node.task,
|
|
304
|
-
trackConfig: node.track,
|
|
305
|
-
status: 'idle',
|
|
306
|
-
result: null,
|
|
307
|
-
startedAt: null,
|
|
308
|
-
finishedAt: null,
|
|
309
|
-
});
|
|
310
|
-
}
|
|
311
|
-
|
|
312
|
-
// Pipeline start hook (gate). Runs BEFORE the engine emits run_start so
|
|
313
|
-
// a blocked pipeline produces zero wire events (the server treats the
|
|
314
|
-
// thrown error as run_error). Hosts get a rich error message; nothing
|
|
315
|
-
// is ever half-broadcast.
|
|
316
|
-
const startHook = await executeHook(
|
|
317
|
-
config.hooks,
|
|
318
|
-
'pipeline_start',
|
|
319
|
-
buildPipelineStartContext(pipelineInfo),
|
|
320
|
-
workDir,
|
|
321
|
-
);
|
|
322
|
-
if (!startHook.allowed) {
|
|
323
|
-
console.error(`Pipeline blocked by pipeline_start hook (exit code ${startHook.exitCode})`);
|
|
324
|
-
await executeHook(
|
|
325
|
-
config.hooks,
|
|
326
|
-
'pipeline_error',
|
|
327
|
-
buildPipelineErrorContext(pipelineInfo, 'pipeline_blocked', 'pipeline_blocked'),
|
|
328
|
-
workDir,
|
|
329
|
-
);
|
|
330
|
-
return {
|
|
331
|
-
success: false,
|
|
332
|
-
runId,
|
|
333
|
-
logPath: log.path,
|
|
334
|
-
summary: {
|
|
335
|
-
total: dag.nodes.size,
|
|
336
|
-
success: 0,
|
|
337
|
-
failed: 0,
|
|
338
|
-
skipped: 0,
|
|
339
|
-
timeout: 0,
|
|
340
|
-
blocked: 0,
|
|
341
|
-
},
|
|
342
|
-
states: freezeStates(states),
|
|
343
|
-
};
|
|
344
|
-
}
|
|
345
|
-
|
|
346
|
-
// Pipeline approved — transition all tasks to waiting.
|
|
347
|
-
for (const [, state] of states) {
|
|
348
|
-
state.status = 'waiting';
|
|
349
|
-
}
|
|
350
|
-
// Emit run_start with a wire-shape snapshot so SSE subscribers can
|
|
351
|
-
// initialize their task maps on the same event stream that carries
|
|
352
|
-
// updates. No separate "server pre-broadcasts run_start" ceremony —
|
|
353
|
-
// the engine owns the lifecycle boundary.
|
|
354
|
-
const runStartTasks: RunTaskState[] = [];
|
|
355
|
-
for (const [id, node] of dag.nodes) {
|
|
356
|
-
const s = states.get(id)!;
|
|
357
|
-
runStartTasks.push(toRunTaskState(id, node.track.id, node.task.name ?? id, s));
|
|
358
|
-
}
|
|
359
|
-
emit({ type: 'run_start', runId, tasks: runStartTasks });
|
|
360
|
-
|
|
361
|
-
const sessionMap = new Map<string, string>();
|
|
362
|
-
const normalizedMap = new Map<string, string>();
|
|
363
|
-
|
|
364
|
-
// Pipeline timeout + abort reason tracking.
|
|
365
|
-
//
|
|
366
|
-
// `abortReason` replaces the previous `pipelineAborted: boolean`: it
|
|
367
|
-
// carries the concrete cause (timeout / stop_all / external) through
|
|
368
|
-
// to run_end and the pipeline_error hook so downstream consumers can
|
|
369
|
-
// distinguish them without scraping message strings.
|
|
370
|
-
const pipelineTimeoutMs = config.timeout ? parseDuration(config.timeout) : 0;
|
|
371
|
-
let abortReason: AbortReason | null = null;
|
|
372
|
-
const abortController = new AbortController();
|
|
373
|
-
let pipelineTimer: ReturnType<typeof setTimeout> | null = null;
|
|
374
|
-
|
|
375
|
-
if (pipelineTimeoutMs > 0) {
|
|
376
|
-
pipelineTimer = setTimeout(() => {
|
|
377
|
-
if (abortReason === null) abortReason = 'timeout';
|
|
378
|
-
abortController.abort();
|
|
379
|
-
}, pipelineTimeoutMs);
|
|
380
|
-
}
|
|
381
|
-
|
|
382
|
-
// When the pipeline is aborted (timeout, stop_all, external), drain
|
|
383
|
-
// all pending approvals so waiting triggers unblock immediately.
|
|
384
|
-
abortController.signal.addEventListener('abort', () => {
|
|
385
|
-
approvalGateway.abortAll('pipeline aborted');
|
|
386
|
-
});
|
|
387
|
-
|
|
388
|
-
// Wire external cancel signal into the internal abort controller.
|
|
389
|
-
const externalAbortHandler = () => {
|
|
390
|
-
if (abortReason === null) abortReason = 'external';
|
|
391
|
-
abortController.abort();
|
|
392
|
-
};
|
|
393
|
-
if (options.signal) {
|
|
394
|
-
if (options.signal.aborted) {
|
|
395
|
-
externalAbortHandler();
|
|
396
|
-
} else {
|
|
397
|
-
options.signal.addEventListener('abort', externalAbortHandler, { once: true });
|
|
398
|
-
}
|
|
399
|
-
}
|
|
400
|
-
|
|
401
|
-
// Bridge approval gateway events onto the wire stream so hosts (editor
|
|
402
|
-
// server, CLI adapters) see approvals on the same channel as task
|
|
403
|
-
// updates. The server no longer needs its own gateway subscription.
|
|
404
|
-
const unsubscribeApprovals = approvalGateway.subscribe((ev) => {
|
|
405
|
-
if (ev.type === 'requested') {
|
|
406
|
-
emit({
|
|
407
|
-
type: 'approval_request',
|
|
408
|
-
runId,
|
|
409
|
-
request: {
|
|
410
|
-
id: ev.request.id,
|
|
411
|
-
taskId: ev.request.taskId,
|
|
412
|
-
trackId: ev.request.trackId,
|
|
413
|
-
message: ev.request.message,
|
|
414
|
-
createdAt: ev.request.createdAt,
|
|
415
|
-
timeoutMs: ev.request.timeoutMs,
|
|
416
|
-
metadata: ev.request.metadata,
|
|
417
|
-
},
|
|
418
|
-
});
|
|
419
|
-
return;
|
|
420
|
-
}
|
|
421
|
-
if (ev.type === 'resolved' || ev.type === 'expired' || ev.type === 'aborted') {
|
|
422
|
-
const outcome =
|
|
423
|
-
ev.type === 'resolved'
|
|
424
|
-
? ev.decision.outcome
|
|
425
|
-
: ev.type === 'expired'
|
|
426
|
-
? 'timeout'
|
|
427
|
-
: 'aborted';
|
|
428
|
-
emit({
|
|
429
|
-
type: 'approval_resolved',
|
|
430
|
-
runId,
|
|
431
|
-
requestId: ev.request.id,
|
|
432
|
-
outcome,
|
|
433
|
-
});
|
|
434
|
-
}
|
|
435
|
-
});
|
|
436
|
-
|
|
437
|
-
// ── Helpers ──
|
|
438
|
-
|
|
439
|
-
function emit(event: RunEventPayload): void {
|
|
440
|
-
options.onEvent?.(event);
|
|
441
|
-
}
|
|
442
|
-
|
|
443
|
-
function setTaskStatus(taskId: string, newStatus: TaskStatus): void {
|
|
444
|
-
const state = states.get(taskId)!;
|
|
445
|
-
// Terminal lock: once a task reaches a terminal state it must not be
|
|
446
|
-
// re-transitioned. This prevents stop_all from marking running tasks as
|
|
447
|
-
// skipped and then having their in-flight processTask promise overwrite
|
|
448
|
-
// that with success/failed, producing an invalid double transition.
|
|
449
|
-
if (isTerminal(state.status)) return;
|
|
450
|
-
state.status = newStatus;
|
|
451
|
-
const result = state.result;
|
|
452
|
-
const cfg = state.config;
|
|
453
|
-
emit({
|
|
454
|
-
type: 'task_update',
|
|
455
|
-
runId,
|
|
456
|
-
taskId,
|
|
457
|
-
status: newStatus,
|
|
458
|
-
startedAt: state.startedAt ?? undefined,
|
|
459
|
-
finishedAt: state.finishedAt ?? undefined,
|
|
460
|
-
durationMs: result?.durationMs,
|
|
461
|
-
exitCode: result?.exitCode,
|
|
462
|
-
stdout: result?.stdout,
|
|
463
|
-
stderr: result?.stderr,
|
|
464
|
-
stderrPath: result?.stderrPath ?? null,
|
|
465
|
-
sessionId: result?.sessionId ?? null,
|
|
466
|
-
normalizedOutput: result?.normalizedOutput ?? null,
|
|
467
|
-
resolvedDriver: cfg.driver ?? null,
|
|
468
|
-
resolvedModel: cfg.model ?? null,
|
|
469
|
-
resolvedPermissions: (cfg.permissions as Permissions | undefined) ?? null,
|
|
470
|
-
});
|
|
471
|
-
}
|
|
472
|
-
|
|
473
|
-
function getOnFailure(taskId: string): OnFailure {
|
|
474
|
-
return dag.nodes.get(taskId)?.track.on_failure ?? 'skip_downstream';
|
|
475
|
-
}
|
|
476
|
-
|
|
477
|
-
function isDependencySatisfied(depId: string): 'satisfied' | 'unsatisfied' | 'skip' {
|
|
478
|
-
const depState = states.get(depId);
|
|
479
|
-
if (!depState) return 'skip';
|
|
480
|
-
switch (depState.status) {
|
|
481
|
-
case 'success':
|
|
482
|
-
return 'satisfied';
|
|
483
|
-
case 'skipped':
|
|
484
|
-
return 'skip';
|
|
485
|
-
case 'failed':
|
|
486
|
-
case 'timeout':
|
|
487
|
-
case 'blocked':
|
|
488
|
-
return getOnFailure(depId) === 'ignore' ? 'satisfied' : 'skip';
|
|
489
|
-
default:
|
|
490
|
-
return 'unsatisfied';
|
|
491
|
-
}
|
|
492
|
-
}
|
|
493
|
-
|
|
494
|
-
/**
|
|
495
|
-
* H3: "stop_all" historically only stopped tasks within the same track,
|
|
496
|
-
* which contradicted both its name and user expectations. It now stops
|
|
497
|
-
* the **entire pipeline**:
|
|
498
|
-
* - In-flight tasks are signalled via the shared abort controller so
|
|
499
|
-
* drivers / runner.ts can cancel cooperatively (returning
|
|
500
|
-
* `failureKind: 'timeout'`).
|
|
501
|
-
* - Still-waiting tasks across every track are immediately marked
|
|
502
|
-
* skipped so the run completes promptly.
|
|
503
|
-
* The terminal lock in setTaskStatus prevents any later re-transition
|
|
504
|
-
* should a completed running task try to overwrite the skipped state.
|
|
505
|
-
*/
|
|
506
|
-
function applyStopAll(_failedTrackId: string): void {
|
|
507
|
-
if (abortReason === null) abortReason = 'stop_all';
|
|
508
|
-
abortController.abort();
|
|
509
|
-
for (const [id, state] of states) {
|
|
510
|
-
if (state.status === 'waiting') {
|
|
511
|
-
state.finishedAt = nowISO();
|
|
512
|
-
setTaskStatus(id, 'skipped');
|
|
513
|
-
}
|
|
514
|
-
}
|
|
515
|
-
}
|
|
516
|
-
|
|
517
|
-
function buildTaskInfoObj(taskId: string): TaskInfo {
|
|
518
|
-
const state = states.get(taskId)!;
|
|
519
|
-
return {
|
|
520
|
-
id: taskId,
|
|
521
|
-
name: state.config.name,
|
|
522
|
-
type: state.config.prompt ? 'ai' : 'command',
|
|
523
|
-
status: state.status,
|
|
524
|
-
exit_code: state.result?.exitCode ?? null,
|
|
525
|
-
duration_ms: state.result?.durationMs ?? null,
|
|
526
|
-
stderr_path: state.result?.stderrPath ?? null,
|
|
527
|
-
session_id: state.result?.sessionId ?? null,
|
|
528
|
-
started_at: state.startedAt,
|
|
529
|
-
finished_at: state.finishedAt,
|
|
530
|
-
};
|
|
531
|
-
}
|
|
532
|
-
|
|
533
|
-
function trackInfoOf(taskId: string): TrackInfo {
|
|
534
|
-
const node = dag.nodes.get(taskId)!;
|
|
535
|
-
return { id: node.track.id, name: node.track.name };
|
|
536
|
-
}
|
|
537
|
-
|
|
538
|
-
async function fireHook(taskId: string, event: 'task_success' | 'task_failure'): Promise<void> {
|
|
539
|
-
await executeHook(
|
|
540
|
-
config.hooks,
|
|
541
|
-
event,
|
|
542
|
-
buildTaskContext(event, pipelineInfo, trackInfoOf(taskId), buildTaskInfoObj(taskId)),
|
|
543
|
-
workDir,
|
|
544
|
-
abortController.signal,
|
|
545
|
-
);
|
|
546
|
-
}
|
|
547
|
-
|
|
548
|
-
// ── Process a single task ──
|
|
549
|
-
|
|
550
|
-
async function processTask(taskId: string): Promise<void> {
|
|
551
|
-
const state = states.get(taskId)!;
|
|
552
|
-
const node = dag.nodes.get(taskId)!;
|
|
553
|
-
const task = node.task;
|
|
554
|
-
const track = node.track;
|
|
555
|
-
|
|
556
|
-
log.section(`Task ${taskId}`, taskId);
|
|
557
|
-
log.debug(
|
|
558
|
-
`[task:${taskId}]`,
|
|
559
|
-
`type=${task.prompt ? 'ai' : 'cmd'} track=${track.id} deps=[${node.dependsOn.join(', ') || '(root)'}]`,
|
|
560
|
-
);
|
|
561
|
-
|
|
562
|
-
// 1. Check dependencies
|
|
563
|
-
for (const depId of node.dependsOn) {
|
|
564
|
-
const result = isDependencySatisfied(depId);
|
|
565
|
-
if (result === 'skip') {
|
|
566
|
-
const depStatus = states.get(depId)?.status ?? 'unknown';
|
|
567
|
-
log.debug(`[task:${taskId}]`, `skipped (upstream "${depId}" status=${depStatus})`);
|
|
568
|
-
state.finishedAt = nowISO();
|
|
569
|
-
setTaskStatus(taskId, 'skipped');
|
|
570
|
-
return;
|
|
571
|
-
}
|
|
572
|
-
if (result === 'unsatisfied') return; // still waiting
|
|
573
|
-
}
|
|
574
|
-
|
|
575
|
-
// 2. Check trigger
|
|
576
|
-
if (task.trigger) {
|
|
577
|
-
log.debug(
|
|
578
|
-
`[task:${taskId}]`,
|
|
579
|
-
`trigger wait: type=${task.trigger.type} ${JSON.stringify(task.trigger)}`,
|
|
580
|
-
);
|
|
581
|
-
try {
|
|
582
|
-
const triggerPlugin = getHandler<TriggerPlugin>('triggers', task.trigger.type);
|
|
583
|
-
// R6: race the plugin's watch() against the pipeline's abort signal.
|
|
584
|
-
// Third-party triggers may forget to wire up ctx.signal — without
|
|
585
|
-
// this race, an aborted pipeline would hang forever waiting for the
|
|
586
|
-
// plugin's watch promise to resolve. The race resolves on whichever
|
|
587
|
-
// path settles first, and the cleanup paths in finally never run on
|
|
588
|
-
// the orphaned plugin promise (it's allowed to leak a watcher; the
|
|
589
|
-
// pipeline is being torn down anyway).
|
|
590
|
-
await new Promise<unknown>((resolve, reject) => {
|
|
591
|
-
let settled = false;
|
|
592
|
-
const onAbort = () => {
|
|
593
|
-
if (settled) return;
|
|
594
|
-
settled = true;
|
|
595
|
-
abortController.signal.removeEventListener('abort', onAbort);
|
|
596
|
-
reject(new Error('Pipeline aborted'));
|
|
597
|
-
};
|
|
598
|
-
if (abortController.signal.aborted) {
|
|
599
|
-
onAbort();
|
|
600
|
-
return;
|
|
601
|
-
}
|
|
602
|
-
abortController.signal.addEventListener('abort', onAbort, { once: true });
|
|
603
|
-
triggerPlugin
|
|
604
|
-
.watch(task.trigger as Record<string, unknown>, {
|
|
605
|
-
taskId: node.taskId,
|
|
606
|
-
trackId: track.id,
|
|
607
|
-
workDir: task.cwd ?? workDir,
|
|
608
|
-
signal: abortController.signal,
|
|
609
|
-
approvalGateway,
|
|
610
|
-
})
|
|
611
|
-
.then(
|
|
612
|
-
(v) => {
|
|
613
|
-
if (settled) return;
|
|
614
|
-
settled = true;
|
|
615
|
-
abortController.signal.removeEventListener('abort', onAbort);
|
|
616
|
-
resolve(v);
|
|
617
|
-
},
|
|
618
|
-
(e) => {
|
|
619
|
-
if (settled) return;
|
|
620
|
-
settled = true;
|
|
621
|
-
abortController.signal.removeEventListener('abort', onAbort);
|
|
622
|
-
reject(e);
|
|
623
|
-
},
|
|
624
|
-
);
|
|
625
|
-
});
|
|
626
|
-
log.debug(`[task:${taskId}]`, `trigger fired`);
|
|
627
|
-
} catch (err: unknown) {
|
|
628
|
-
// If pipeline was aborted while we were still waiting for the trigger,
|
|
629
|
-
// this task never entered running state → skipped, not timeout.
|
|
630
|
-
state.finishedAt = nowISO();
|
|
631
|
-
if (abortReason !== null) {
|
|
632
|
-
setTaskStatus(taskId, 'skipped');
|
|
633
|
-
} else if (err instanceof TriggerBlockedError) {
|
|
634
|
-
setTaskStatus(taskId, 'blocked'); // user/policy rejection
|
|
635
|
-
} else if (err instanceof TriggerTimeoutError) {
|
|
636
|
-
setTaskStatus(taskId, 'timeout'); // genuine trigger wait timeout
|
|
637
|
-
} else {
|
|
638
|
-
// A7 fallback: also check message strings for backward-compat with
|
|
639
|
-
// third-party trigger plugins that don't throw typed errors yet.
|
|
640
|
-
const msg = err instanceof Error ? err.message : String(err);
|
|
641
|
-
if (msg.includes('rejected') || msg.includes('denied')) {
|
|
642
|
-
setTaskStatus(taskId, 'blocked');
|
|
643
|
-
} else if (msg.includes('timeout')) {
|
|
644
|
-
setTaskStatus(taskId, 'timeout');
|
|
645
|
-
} else {
|
|
646
|
-
setTaskStatus(taskId, 'failed'); // plugin error, watcher crash, etc.
|
|
647
|
-
}
|
|
648
|
-
}
|
|
649
|
-
try {
|
|
650
|
-
await fireHook(taskId, 'task_failure');
|
|
651
|
-
} catch (hookErr) {
|
|
652
|
-
log.error(
|
|
653
|
-
`[task:${taskId}]`,
|
|
654
|
-
`hook execution failed: ${hookErr instanceof Error ? hookErr.message : String(hookErr)}`,
|
|
655
|
-
);
|
|
656
|
-
}
|
|
657
|
-
return;
|
|
658
|
-
}
|
|
659
|
-
}
|
|
660
|
-
|
|
661
|
-
// 3. task_start hook (gate)
|
|
662
|
-
const hookResult = await executeHook(
|
|
663
|
-
config.hooks,
|
|
664
|
-
'task_start',
|
|
665
|
-
buildTaskContext('task_start', pipelineInfo, trackInfoOf(taskId), buildTaskInfoObj(taskId)),
|
|
666
|
-
workDir,
|
|
667
|
-
abortController.signal,
|
|
668
|
-
);
|
|
669
|
-
if (hookResult.exitCode !== 0 || config.hooks?.task_start) {
|
|
670
|
-
log.debug(
|
|
671
|
-
`[task:${taskId}]`,
|
|
672
|
-
`task_start hook exit=${hookResult.exitCode} allowed=${hookResult.allowed}`,
|
|
673
|
-
);
|
|
674
|
-
}
|
|
675
|
-
if (!hookResult.allowed) {
|
|
676
|
-
state.finishedAt = nowISO();
|
|
677
|
-
setTaskStatus(taskId, 'blocked');
|
|
678
|
-
try {
|
|
679
|
-
await fireHook(taskId, 'task_failure');
|
|
680
|
-
} catch (hookErr) {
|
|
681
|
-
log.error(
|
|
682
|
-
`[task:${taskId}]`,
|
|
683
|
-
`hook execution failed: ${hookErr instanceof Error ? hookErr.message : String(hookErr)}`,
|
|
684
|
-
);
|
|
685
|
-
}
|
|
686
|
-
return;
|
|
687
|
-
}
|
|
688
|
-
|
|
689
|
-
// 4. Mark running — set startedAt before emitting so subscribers see a
|
|
690
|
-
// complete task_update (startedAt non-null) on the status transition.
|
|
691
|
-
state.startedAt = nowISO();
|
|
692
|
-
setTaskStatus(taskId, 'running');
|
|
693
|
-
log.info(
|
|
694
|
-
`[task:${taskId}]`,
|
|
695
|
-
task.command ? `running: ${task.command}` : `running (driver task)`,
|
|
696
|
-
);
|
|
697
|
-
|
|
698
|
-
// File-only: resolved config for this task
|
|
699
|
-
const resolvedDriver = task.driver ?? track.driver ?? config.driver ?? 'opencode';
|
|
700
|
-
const resolvedModel = task.model ?? track.model ?? config.model ?? '(default)';
|
|
701
|
-
const resolvedPerms = task.permissions ?? track.permissions ?? '(default)';
|
|
702
|
-
const resolvedCwd = task.cwd ?? track.cwd ?? workDir;
|
|
703
|
-
log.debug(
|
|
704
|
-
`[task:${taskId}]`,
|
|
705
|
-
`resolved: driver=${resolvedDriver} model=${resolvedModel} cwd=${resolvedCwd}`,
|
|
706
|
-
);
|
|
707
|
-
log.debug(`[task:${taskId}]`, `permissions: ${JSON.stringify(resolvedPerms)}`);
|
|
708
|
-
if (task.continue_from) {
|
|
709
|
-
log.debug(`[task:${taskId}]`, `continue_from: "${task.continue_from}"`);
|
|
710
|
-
}
|
|
711
|
-
if (task.timeout) {
|
|
712
|
-
log.debug(`[task:${taskId}]`, `timeout: ${task.timeout}`);
|
|
713
|
-
}
|
|
714
|
-
|
|
715
|
-
try {
|
|
716
|
-
let result: TaskResult;
|
|
717
|
-
const timeoutMs = task.timeout ? parseDuration(task.timeout) : undefined;
|
|
718
|
-
|
|
719
|
-
const runOpts = { timeoutMs, signal: abortController.signal };
|
|
720
|
-
|
|
721
|
-
if (task.command) {
|
|
722
|
-
log.debug(`[task:${taskId}]`, `command: ${task.command}`);
|
|
723
|
-
result = await runCommand(task.command, task.cwd ?? workDir, runOpts);
|
|
724
|
-
} else {
|
|
725
|
-
// AI task: apply middleware chain against a structured PromptDocument.
|
|
726
|
-
const driverName = task.driver ?? track.driver ?? config.driver ?? 'opencode';
|
|
727
|
-
const driver = getHandler<DriverPlugin>('drivers', driverName);
|
|
728
|
-
|
|
729
|
-
const originalLen = task.prompt!.length;
|
|
730
|
-
let doc: PromptDocument = promptDocumentFromString(task.prompt!);
|
|
731
|
-
const mws = task.middlewares !== undefined ? task.middlewares : track.middlewares;
|
|
732
|
-
if (mws && mws.length > 0) {
|
|
733
|
-
log.debug(
|
|
734
|
-
`[task:${taskId}]`,
|
|
735
|
-
`middleware chain: ${mws.map((m) => m.type).join(' → ')}`,
|
|
736
|
-
);
|
|
737
|
-
const mwCtx: MiddlewareContext = {
|
|
738
|
-
task,
|
|
739
|
-
track,
|
|
740
|
-
workDir: task.cwd ?? workDir,
|
|
741
|
-
};
|
|
742
|
-
for (const mwConfig of mws) {
|
|
743
|
-
const mwPlugin = getHandler<MiddlewarePlugin>('middlewares', mwConfig.type);
|
|
744
|
-
const beforeBlocks = doc.contexts.length;
|
|
745
|
-
const beforeLen = serializePromptDocument(doc).length;
|
|
746
|
-
|
|
747
|
-
// Prefer the structured API. Fall back to the legacy
|
|
748
|
-
// `enhance(string) → string` path so v0.x plugins keep
|
|
749
|
-
// working — that fallback loses context structure (the
|
|
750
|
-
// middleware's output becomes the new task body) but never
|
|
751
|
-
// silently drops content.
|
|
752
|
-
if (typeof mwPlugin.enhanceDoc === 'function') {
|
|
753
|
-
const next = await mwPlugin.enhanceDoc(
|
|
754
|
-
doc,
|
|
755
|
-
mwConfig as Record<string, unknown>,
|
|
756
|
-
mwCtx,
|
|
757
|
-
);
|
|
758
|
-
if (
|
|
759
|
-
!next ||
|
|
760
|
-
typeof next !== 'object' ||
|
|
761
|
-
!Array.isArray((next as PromptDocument).contexts) ||
|
|
762
|
-
typeof (next as PromptDocument).task !== 'string'
|
|
763
|
-
) {
|
|
764
|
-
throw new Error(
|
|
765
|
-
`middleware "${mwConfig.type}".enhanceDoc() returned a malformed PromptDocument`,
|
|
766
|
-
);
|
|
767
|
-
}
|
|
768
|
-
doc = next as PromptDocument;
|
|
769
|
-
} else if (typeof mwPlugin.enhance === 'function') {
|
|
770
|
-
const asString = serializePromptDocument(doc);
|
|
771
|
-
const next = await mwPlugin.enhance(
|
|
772
|
-
asString,
|
|
773
|
-
mwConfig as Record<string, unknown>,
|
|
774
|
-
mwCtx,
|
|
775
|
-
);
|
|
776
|
-
// R3: a middleware that returns undefined / null / a non-string
|
|
777
|
-
// would silently corrupt the prompt. Fail loud.
|
|
778
|
-
if (typeof next !== 'string') {
|
|
779
|
-
throw new Error(
|
|
780
|
-
`middleware "${mwConfig.type}".enhance() returned ${next === null ? 'null' : typeof next}, expected string`,
|
|
781
|
-
);
|
|
782
|
-
}
|
|
783
|
-
// Legacy fallback: collapse the returned string into a
|
|
784
|
-
// fresh doc. Earlier structure is folded into the string
|
|
785
|
-
// (serializePromptDocument just ran), so bytes the driver
|
|
786
|
-
// sees match the old string pipeline.
|
|
787
|
-
doc = { contexts: [], task: next };
|
|
788
|
-
} else {
|
|
789
|
-
throw new Error(
|
|
790
|
-
`middleware "${mwConfig.type}" provides neither enhanceDoc nor enhance`,
|
|
791
|
-
);
|
|
792
|
-
}
|
|
793
|
-
const afterLen = serializePromptDocument(doc).length;
|
|
794
|
-
const addedBlocks = doc.contexts.length - beforeBlocks;
|
|
795
|
-
log.debug(
|
|
796
|
-
`[task:${taskId}]`,
|
|
797
|
-
` ${mwConfig.type}: ${beforeLen} → ${afterLen} chars` +
|
|
798
|
-
(addedBlocks > 0
|
|
799
|
-
? ` (+${addedBlocks} context block${addedBlocks > 1 ? 's' : ''})`
|
|
800
|
-
: ''),
|
|
801
|
-
);
|
|
802
|
-
}
|
|
803
|
-
}
|
|
804
|
-
const prompt = serializePromptDocument(doc);
|
|
805
|
-
log.debug(
|
|
806
|
-
`[task:${taskId}]`,
|
|
807
|
-
`prompt: ${originalLen} chars (final: ${prompt.length} chars, ${doc.contexts.length} block${doc.contexts.length === 1 ? '' : 's'})`,
|
|
808
|
-
);
|
|
809
|
-
log.quiet(`--- prompt (final) ---\n${clip(prompt)}\n--- end prompt ---`, taskId);
|
|
810
|
-
|
|
811
|
-
// H1: hand the driver a continue_from that has already been
|
|
812
|
-
// qualified by dag.ts. Without this, drivers like codex/opencode/
|
|
813
|
-
// claude-code look up maps directly with
|
|
814
|
-
// the user's raw (possibly bare) string, which races whenever two
|
|
815
|
-
// tracks share a task name. dag.ts has the only authoritative
|
|
816
|
-
// resolver, so we use its precomputed answer here.
|
|
817
|
-
// Drivers key sessionMap/normalizedMap by fully-qualified id. buildDag
|
|
818
|
-
// guarantees `resolvedContinueFrom` is set for every task that has a
|
|
819
|
-
// `continue_from`, so if we see the bare form here something upstream
|
|
820
|
-
// is broken — fail loud instead of silently miskeying the lookup.
|
|
821
|
-
if (task.continue_from && !node.resolvedContinueFrom) {
|
|
822
|
-
throw new Error(
|
|
823
|
-
`Internal: task "${taskId}" has continue_from "${task.continue_from}" ` +
|
|
824
|
-
`but no resolvedContinueFrom. buildDag should have qualified it.`,
|
|
825
|
-
);
|
|
826
|
-
}
|
|
827
|
-
const enrichedTask: TaskConfig = {
|
|
828
|
-
...task,
|
|
829
|
-
prompt,
|
|
830
|
-
continue_from: node.resolvedContinueFrom,
|
|
831
|
-
};
|
|
832
|
-
const driverCtx: DriverContext = {
|
|
833
|
-
sessionMap,
|
|
834
|
-
normalizedMap,
|
|
835
|
-
workDir: task.cwd ?? workDir,
|
|
836
|
-
// Structured view for drivers that want fine-grained control
|
|
837
|
-
// over serialization (e.g. inserting [Previous Output] between
|
|
838
|
-
// contexts and task). Drivers that read task.prompt see the
|
|
839
|
-
// default serialization and need no changes.
|
|
840
|
-
promptDoc: doc,
|
|
841
|
-
};
|
|
842
|
-
const spec = await driver.buildCommand(enrichedTask, track, driverCtx);
|
|
843
|
-
log.debug(`[task:${taskId}]`, `driver=${driverName}`);
|
|
844
|
-
log.debug(`[task:${taskId}]`, `spawn args: ${JSON.stringify(spec.args)}`);
|
|
845
|
-
if (spec.cwd) log.debug(`[task:${taskId}]`, `spawn cwd: ${spec.cwd}`);
|
|
846
|
-
if (spec.env)
|
|
847
|
-
log.debug(
|
|
848
|
-
`[task:${taskId}]`,
|
|
849
|
-
`spawn env overrides: ${Object.keys(spec.env).join(', ')}`,
|
|
850
|
-
);
|
|
851
|
-
if (spec.stdin) log.debug(`[task:${taskId}]`, `spawn stdin: ${spec.stdin.length} chars`);
|
|
852
|
-
result = await runSpawn(spec, driver, runOpts);
|
|
853
|
-
}
|
|
854
|
-
|
|
855
|
-
// 6. Determine terminal status (without emitting yet — result must be complete first)
|
|
856
|
-
// H2: branch on failureKind so spawn errors no longer masquerade as
|
|
857
|
-
// timeouts. Old runners that don't set failureKind still work — we
|
|
858
|
-
// fall back to the historical `exitCode === -1 → timeout` heuristic so
|
|
859
|
-
// pre-existing third-party drivers don't regress.
|
|
860
|
-
let terminalStatus: TaskStatus;
|
|
861
|
-
const kind = result.failureKind;
|
|
862
|
-
if (kind === 'timeout') {
|
|
863
|
-
terminalStatus = 'timeout';
|
|
864
|
-
} else if (kind === 'spawn_error') {
|
|
865
|
-
terminalStatus = 'failed';
|
|
866
|
-
} else if (kind === undefined && result.exitCode === -1) {
|
|
867
|
-
// Legacy path: pre-H2 driver returned -1 with no kind. Treat as
|
|
868
|
-
// timeout for backward compatibility (the previous behaviour).
|
|
869
|
-
terminalStatus = 'timeout';
|
|
870
|
-
} else if (result.exitCode !== 0) {
|
|
871
|
-
terminalStatus = 'failed';
|
|
872
|
-
} else if (task.completion) {
|
|
873
|
-
const plugin = getHandler<CompletionPlugin>('completions', task.completion.type);
|
|
874
|
-
const completionCtx = { workDir: task.cwd ?? workDir, signal: abortController.signal };
|
|
875
|
-
const passed = await plugin.check(
|
|
876
|
-
task.completion as Record<string, unknown>,
|
|
877
|
-
result,
|
|
878
|
-
completionCtx,
|
|
879
|
-
);
|
|
880
|
-
// R4: strict boolean check. Truthy strings/numbers used to be coerced
|
|
881
|
-
// to success — a check returning "ok" would let a failing task pass.
|
|
882
|
-
if (typeof passed !== 'boolean') {
|
|
883
|
-
throw new Error(
|
|
884
|
-
`completion "${task.completion.type}".check() returned ${passed === null ? 'null' : typeof passed}, expected boolean`,
|
|
885
|
-
);
|
|
886
|
-
}
|
|
887
|
-
terminalStatus = passed ? 'success' : 'failed';
|
|
888
|
-
} else {
|
|
889
|
-
terminalStatus = 'success';
|
|
890
|
-
}
|
|
891
|
-
|
|
892
|
-
// Store normalized text separately (in-memory) for continue_from handoff.
|
|
893
|
-
// R15: clip oversized values so a runaway parseResult can't accumulate
|
|
894
|
-
// hundreds of MB across tasks.
|
|
895
|
-
if (result.normalizedOutput !== null) {
|
|
896
|
-
const clipped =
|
|
897
|
-
result.normalizedOutput.length > MAX_NORMALIZED_BYTES
|
|
898
|
-
? result.normalizedOutput.slice(0, MAX_NORMALIZED_BYTES) +
|
|
899
|
-
`\n[…clipped at ${MAX_NORMALIZED_BYTES} bytes]`
|
|
900
|
-
: result.normalizedOutput;
|
|
901
|
-
normalizedMap.set(taskId, clipped);
|
|
902
|
-
}
|
|
903
|
-
|
|
904
|
-
if (result.stderr) {
|
|
905
|
-
const stderrPath = resolve(log.dir, `${taskId.replace(/\./g, '_')}.stderr`);
|
|
906
|
-
await Bun.write(stderrPath, result.stderr);
|
|
907
|
-
result = { ...result, stderrPath };
|
|
908
|
-
}
|
|
909
|
-
|
|
910
|
-
if (result.sessionId) {
|
|
911
|
-
// H1: qualified-only key.
|
|
912
|
-
sessionMap.set(taskId, result.sessionId);
|
|
913
|
-
}
|
|
914
|
-
|
|
915
|
-
// Set result and finishedAt before emitting terminal status so listeners see complete state
|
|
916
|
-
state.result = result;
|
|
917
|
-
state.finishedAt = nowISO();
|
|
918
|
-
setTaskStatus(taskId, terminalStatus);
|
|
919
|
-
|
|
920
|
-
// Log task outcome with relevant details
|
|
921
|
-
const durSec = (result.durationMs / 1000).toFixed(1);
|
|
922
|
-
if (terminalStatus === 'success') {
|
|
923
|
-
log.info(`[task:${taskId}]`, `success (${durSec}s)`);
|
|
924
|
-
} else {
|
|
925
|
-
log.error(
|
|
926
|
-
`[task:${taskId}]`,
|
|
927
|
-
`${terminalStatus} exit=${result.exitCode} duration=${durSec}s`,
|
|
928
|
-
);
|
|
929
|
-
if (result.stderr) {
|
|
930
|
-
const tail = tailLines(result.stderr, 10);
|
|
931
|
-
log.error(`[task:${taskId}]`, `stderr tail:\n${tail}`);
|
|
932
|
-
}
|
|
933
|
-
}
|
|
934
|
-
|
|
935
|
-
// File-only: full stdout/stderr dump (clipped) + extracted metadata
|
|
936
|
-
log.debug(
|
|
937
|
-
`[task:${taskId}]`,
|
|
938
|
-
`stdout: ${result.stdout.length} chars, stderr: ${result.stderr.length} chars`,
|
|
939
|
-
);
|
|
940
|
-
if (result.sessionId) {
|
|
941
|
-
log.debug(`[task:${taskId}]`, `sessionId: ${result.sessionId}`);
|
|
942
|
-
}
|
|
943
|
-
if (result.stderrPath) {
|
|
944
|
-
log.debug(`[task:${taskId}]`, `wrote stderr: ${result.stderrPath}`);
|
|
945
|
-
}
|
|
946
|
-
if (result.stdout) {
|
|
947
|
-
log.quiet(
|
|
948
|
-
`--- stdout (${taskId}) ---\n${clip(result.stdout)}\n--- end stdout ---`,
|
|
949
|
-
taskId,
|
|
950
|
-
);
|
|
951
|
-
}
|
|
952
|
-
if (result.stderr) {
|
|
953
|
-
log.quiet(
|
|
954
|
-
`--- stderr (${taskId}) ---\n${clip(result.stderr)}\n--- end stderr ---`,
|
|
955
|
-
taskId,
|
|
956
|
-
);
|
|
957
|
-
}
|
|
958
|
-
if (task.completion) {
|
|
959
|
-
log.debug(
|
|
960
|
-
`[task:${taskId}]`,
|
|
961
|
-
`completion check: type=${task.completion.type} result=${terminalStatus}`,
|
|
962
|
-
);
|
|
963
|
-
}
|
|
964
|
-
} catch (err: unknown) {
|
|
965
|
-
const errMsg = err instanceof Error ? (err.stack ?? err.message) : String(err);
|
|
966
|
-
log.error(`[task:${taskId}]`, `failed before execution: ${errMsg}`);
|
|
967
|
-
state.result = {
|
|
968
|
-
exitCode: -1,
|
|
969
|
-
stdout: '',
|
|
970
|
-
stderr: errMsg,
|
|
971
|
-
stderrPath: null,
|
|
972
|
-
durationMs: 0,
|
|
973
|
-
sessionId: null,
|
|
974
|
-
normalizedOutput: null,
|
|
975
|
-
// H2: Engine-level pre-execution errors (driver throw, middleware
|
|
976
|
-
// throw, getHandler 404) classify as spawn_error — the process never
|
|
977
|
-
// ran, so calling them "timeout" was actively misleading.
|
|
978
|
-
failureKind: 'spawn_error',
|
|
979
|
-
};
|
|
980
|
-
state.finishedAt = nowISO();
|
|
981
|
-
setTaskStatus(taskId, 'failed');
|
|
982
|
-
}
|
|
983
|
-
|
|
984
|
-
// 7. Fire hooks
|
|
985
|
-
const finalStatus: TaskStatus = state.status;
|
|
986
|
-
try {
|
|
987
|
-
await fireHook(taskId, finalStatus === 'success' ? 'task_success' : 'task_failure');
|
|
988
|
-
} catch (hookErr) {
|
|
989
|
-
log.error(
|
|
990
|
-
`[task:${taskId}]`,
|
|
991
|
-
`hook execution failed: ${hookErr instanceof Error ? hookErr.message : String(hookErr)}`,
|
|
992
|
-
);
|
|
993
|
-
}
|
|
994
|
-
|
|
995
|
-
// 8. Handle stop_all for failure states
|
|
996
|
-
if (finalStatus !== 'success' && getOnFailure(taskId) === 'stop_all') {
|
|
997
|
-
applyStopAll(node.track.id);
|
|
998
|
-
}
|
|
999
|
-
}
|
|
1000
|
-
|
|
1001
|
-
// ── Event loop ──
|
|
1002
|
-
// Each task is launched as soon as ALL its deps reach a terminal state.
|
|
1003
|
-
// We track in-flight tasks in `running` so a task completing mid-batch
|
|
1004
|
-
// immediately unblocks its dependents without waiting for sibling tasks.
|
|
1005
|
-
const running = new Map<string, Promise<void>>();
|
|
1006
|
-
|
|
1007
|
-
try {
|
|
1008
|
-
while (abortReason === null) {
|
|
1009
|
-
// Launch every task whose deps are all terminal and that isn't already in-flight
|
|
1010
|
-
for (const [id, state] of states) {
|
|
1011
|
-
if (state.status !== 'waiting' || running.has(id)) continue;
|
|
1012
|
-
const node = dag.nodes.get(id)!;
|
|
1013
|
-
const allDepsTerminal =
|
|
1014
|
-
node.dependsOn.length === 0 ||
|
|
1015
|
-
node.dependsOn.every((d) => isTerminal(states.get(d)!.status));
|
|
1016
|
-
if (!allDepsTerminal) continue;
|
|
1017
|
-
const p = processTask(id).finally(() => running.delete(id));
|
|
1018
|
-
running.set(id, p);
|
|
1019
|
-
}
|
|
1020
|
-
|
|
1021
|
-
// All tasks terminal — done
|
|
1022
|
-
if ([...states.values()].every((s) => isTerminal(s.status))) break;
|
|
1023
|
-
|
|
1024
|
-
if (running.size === 0) {
|
|
1025
|
-
// Nothing in-flight but non-terminal tasks exist (e.g. trigger-wait states
|
|
1026
|
-
// that processTask hasn't been called for yet). Poll briefly.
|
|
1027
|
-
await new Promise((r) => setTimeout(r, POLL_INTERVAL_MS));
|
|
1028
|
-
} else {
|
|
1029
|
-
// Wait for any one task to finish, then re-scan for new launchables.
|
|
1030
|
-
await Promise.race(running.values());
|
|
1031
|
-
}
|
|
1032
|
-
}
|
|
1033
|
-
|
|
1034
|
-
if (abortReason !== null) {
|
|
1035
|
-
// Wait for in-flight tasks to honour the abort signal before marking states.
|
|
1036
|
-
if (running.size > 0) await Promise.allSettled(running.values());
|
|
1037
|
-
for (const [id, state] of states) {
|
|
1038
|
-
if (!isTerminal(state.status)) {
|
|
1039
|
-
// By the time allSettled resolves, processTask's try/finally has already
|
|
1040
|
-
// set running tasks to success/failed/timeout. The only non-terminal
|
|
1041
|
-
// statuses remaining here are waiting/idle tasks that were never started.
|
|
1042
|
-
state.finishedAt = nowISO();
|
|
1043
|
-
setTaskStatus(id, 'skipped');
|
|
1044
|
-
}
|
|
1045
|
-
}
|
|
1046
|
-
}
|
|
1047
|
-
} finally {
|
|
1048
|
-
if (pipelineTimer) clearTimeout(pipelineTimer);
|
|
1049
|
-
// Clean up the external abort signal listener to prevent dead references
|
|
1050
|
-
// accumulating on long-lived shared AbortControllers.
|
|
1051
|
-
if (options.signal) {
|
|
1052
|
-
options.signal.removeEventListener('abort', externalAbortHandler);
|
|
1053
|
-
}
|
|
1054
|
-
// Safety net: drain any approvals still pending at shutdown (e.g. crash path).
|
|
1055
|
-
if (approvalGateway.pending().length > 0) {
|
|
1056
|
-
approvalGateway.abortAll('pipeline finished');
|
|
1057
|
-
}
|
|
1058
|
-
// Detach gateway → onEvent bridge so a long-lived gateway (host-supplied)
|
|
1059
|
-
// doesn't keep firing into a dead run.
|
|
1060
|
-
unsubscribeApprovals();
|
|
1061
|
-
}
|
|
1062
|
-
|
|
1063
|
-
// ── Summary ──
|
|
1064
|
-
const summary = { total: 0, success: 0, failed: 0, skipped: 0, timeout: 0, blocked: 0 };
|
|
1065
|
-
for (const [, state] of states) {
|
|
1066
|
-
summary.total++;
|
|
1067
|
-
switch (state.status) {
|
|
1068
|
-
case 'success':
|
|
1069
|
-
summary.success++;
|
|
1070
|
-
break;
|
|
1071
|
-
case 'failed':
|
|
1072
|
-
summary.failed++;
|
|
1073
|
-
break;
|
|
1074
|
-
case 'skipped':
|
|
1075
|
-
summary.skipped++;
|
|
1076
|
-
break;
|
|
1077
|
-
case 'timeout':
|
|
1078
|
-
summary.timeout++;
|
|
1079
|
-
break;
|
|
1080
|
-
case 'blocked':
|
|
1081
|
-
summary.blocked++;
|
|
1082
|
-
break;
|
|
1083
|
-
}
|
|
1084
|
-
}
|
|
1085
|
-
|
|
1086
|
-
const finishedAt = nowISO();
|
|
1087
|
-
const durationMs = new Date(finishedAt).getTime() - new Date(startedAt).getTime();
|
|
1088
|
-
|
|
1089
|
-
if (abortReason !== null) {
|
|
1090
|
-
const reasonText =
|
|
1091
|
-
abortReason === 'timeout'
|
|
1092
|
-
? 'Pipeline timeout exceeded'
|
|
1093
|
-
: abortReason === 'stop_all'
|
|
1094
|
-
? 'Pipeline stopped (on_failure: stop_all)'
|
|
1095
|
-
: 'Pipeline aborted by host';
|
|
1096
|
-
await executeHook(
|
|
1097
|
-
config.hooks,
|
|
1098
|
-
'pipeline_error',
|
|
1099
|
-
buildPipelineErrorContext(pipelineInfo, reasonText, undefined, abortReason),
|
|
1100
|
-
workDir,
|
|
1101
|
-
);
|
|
1102
|
-
} else {
|
|
1103
|
-
await executeHook(
|
|
1104
|
-
config.hooks,
|
|
1105
|
-
'pipeline_complete',
|
|
1106
|
-
buildPipelineCompleteContext(
|
|
1107
|
-
{ ...pipelineInfo, finished_at: finishedAt, duration_ms: durationMs },
|
|
1108
|
-
summary,
|
|
1109
|
-
),
|
|
1110
|
-
workDir,
|
|
1111
|
-
);
|
|
1112
|
-
}
|
|
1113
|
-
|
|
1114
|
-
const allSuccess =
|
|
1115
|
-
abortReason === null &&
|
|
1116
|
-
summary.failed === 0 &&
|
|
1117
|
-
summary.timeout === 0 &&
|
|
1118
|
-
summary.blocked === 0;
|
|
1119
|
-
|
|
1120
|
-
log.section('Pipeline summary');
|
|
1121
|
-
log.quiet(
|
|
1122
|
-
`status: ${abortReason !== null ? `aborted (${abortReason})` : 'completed'}`,
|
|
1123
|
-
);
|
|
1124
|
-
log.quiet(`duration: ${(durationMs / 1000).toFixed(1)}s`);
|
|
1125
|
-
log.quiet(
|
|
1126
|
-
`counts: total=${summary.total} success=${summary.success} ` +
|
|
1127
|
-
`failed=${summary.failed} skipped=${summary.skipped} ` +
|
|
1128
|
-
`timeout=${summary.timeout} blocked=${summary.blocked}`,
|
|
1129
|
-
);
|
|
1130
|
-
log.quiet('');
|
|
1131
|
-
log.quiet('per-task:');
|
|
1132
|
-
for (const [id, state] of states) {
|
|
1133
|
-
const dur =
|
|
1134
|
-
state.result?.durationMs != null ? `${(state.result.durationMs / 1000).toFixed(1)}s` : '-';
|
|
1135
|
-
const exit = state.result?.exitCode ?? '-';
|
|
1136
|
-
log.quiet(` ${state.status.padEnd(8)} ${id} (exit=${exit}, ${dur})`);
|
|
1137
|
-
}
|
|
1138
|
-
|
|
1139
|
-
log.info('[pipeline]', `completed "${config.name}"`);
|
|
1140
|
-
log.info(
|
|
1141
|
-
'[pipeline]',
|
|
1142
|
-
`Total: ${summary.total} | Success: ${summary.success} | Failed: ${summary.failed} | Skipped: ${summary.skipped} | Timeout: ${summary.timeout} | Blocked: ${summary.blocked}`,
|
|
1143
|
-
);
|
|
1144
|
-
log.info('[pipeline]', `Duration: ${(durationMs / 1000).toFixed(1)}s`);
|
|
1145
|
-
log.info('[pipeline]', `Log: ${log.path}`);
|
|
1146
|
-
|
|
1147
|
-
emit({ type: 'run_end', runId, success: allSuccess, abortReason });
|
|
1148
|
-
return { success: allSuccess, runId, logPath: log.path, summary, states: freezeStates(states) };
|
|
1149
|
-
} finally {
|
|
1150
|
-
// Close the persistent log file handle before pruning.
|
|
1151
|
-
log.close();
|
|
1152
|
-
// Prune old per-run log directories on every exit path (normal, blocked, or thrown).
|
|
1153
|
-
// Exclude the current runId so a concurrent run cannot delete its own live directory.
|
|
1154
|
-
if (maxLogRuns > 0) {
|
|
1155
|
-
await pruneLogDirs(resolve(workDir, '.tagma', 'logs'), maxLogRuns, runId);
|
|
1156
|
-
}
|
|
1157
|
-
}
|
|
1158
|
-
}
|
|
1159
|
-
|
|
1160
|
-
/**
|
|
1161
|
-
* Delete the oldest subdirectories under `logsDir`, keeping only the most recent `keep`
|
|
1162
|
-
* total runs (including the currently-live run identified by `excludeRunId`).
|
|
1163
|
-
* Directories are sorted lexicographically; because runIds are prefixed with a base-36
|
|
1164
|
-
* timestamp, lexicographic order equals chronological order.
|
|
1165
|
-
*
|
|
1166
|
-
* `excludeRunId` is always skipped from deletion even if it would otherwise be pruned —
|
|
1167
|
-
* this prevents a concurrent run from removing a live log directory that is still in use.
|
|
1168
|
-
*
|
|
1169
|
-
* D10: The live run occupies one slot out of `keep`, so the maximum number of
|
|
1170
|
-
* *historical* dirs to retain is `keep - 1`. Without this adjustment the function
|
|
1171
|
-
* kept `keep` historical dirs plus 1 live dir = `keep + 1` total on disk.
|
|
1172
|
-
*/
|
|
1173
|
-
async function pruneLogDirs(logsDir: string, keep: number, excludeRunId: string): Promise<void> {
|
|
1174
|
-
let entries: string[];
|
|
1175
|
-
try {
|
|
1176
|
-
entries = await readdir(logsDir);
|
|
1177
|
-
} catch {
|
|
1178
|
-
return; // logsDir doesn't exist yet — nothing to prune
|
|
1179
|
-
}
|
|
1180
|
-
|
|
1181
|
-
// Only consider directories that look like run IDs (run_<...>), excluding the live run.
|
|
1182
|
-
const runDirs = entries.filter((e) => e.startsWith('run_') && e !== excludeRunId).sort();
|
|
1183
|
-
// keep - 1 historical slots (1 slot is reserved for the live excludeRunId).
|
|
1184
|
-
const historyKeep = Math.max(0, keep - 1);
|
|
1185
|
-
const toDelete = runDirs.slice(0, Math.max(0, runDirs.length - historyKeep));
|
|
1186
|
-
|
|
1187
|
-
await Promise.all(
|
|
1188
|
-
toDelete.map((dir) =>
|
|
1189
|
-
rm(resolve(logsDir, dir), { recursive: true, force: true }).catch(() => {
|
|
1190
|
-
// Ignore deletion errors — stale dirs are better than a crash
|
|
1191
|
-
}),
|
|
1192
|
-
),
|
|
1193
|
-
);
|
|
1194
|
-
}
|
|
1195
|
-
|
|
1196
|
-
function isTerminal(status: TaskStatus): boolean {
|
|
1197
|
-
return (
|
|
1198
|
-
status === 'success' ||
|
|
1199
|
-
status === 'failed' ||
|
|
1200
|
-
status === 'timeout' ||
|
|
1201
|
-
status === 'skipped' ||
|
|
1202
|
-
status === 'blocked'
|
|
1203
|
-
);
|
|
1204
|
-
}
|
|
1205
|
-
|
|
1206
|
-
/** Return a deep-copied, caller-safe snapshot of the states map. */
|
|
1207
|
-
function freezeStates(states: Map<string, TaskState>): ReadonlyMap<string, TaskState> {
|
|
1208
|
-
const copy = new Map<string, TaskState>();
|
|
1209
|
-
for (const [id, s] of states) {
|
|
1210
|
-
copy.set(id, {
|
|
1211
|
-
config: { ...s.config },
|
|
1212
|
-
trackConfig: { ...s.trackConfig },
|
|
1213
|
-
status: s.status,
|
|
1214
|
-
result: s.result ? { ...s.result } : null,
|
|
1215
|
-
startedAt: s.startedAt,
|
|
1216
|
-
finishedAt: s.finishedAt,
|
|
1217
|
-
});
|
|
1218
|
-
}
|
|
1219
|
-
return copy;
|
|
1220
|
-
}
|
|
1
|
+
import { resolve } from 'path';
|
|
2
|
+
import { readdir, rm } from 'fs/promises';
|
|
3
|
+
import type {
|
|
4
|
+
PipelineConfig,
|
|
5
|
+
TaskConfig,
|
|
6
|
+
TaskState,
|
|
7
|
+
TaskStatus,
|
|
8
|
+
TaskResult,
|
|
9
|
+
DriverPlugin,
|
|
10
|
+
TriggerPlugin,
|
|
11
|
+
CompletionPlugin,
|
|
12
|
+
MiddlewarePlugin,
|
|
13
|
+
MiddlewareContext,
|
|
14
|
+
DriverContext,
|
|
15
|
+
OnFailure,
|
|
16
|
+
PromptDocument,
|
|
17
|
+
Permissions,
|
|
18
|
+
AbortReason,
|
|
19
|
+
RunEventPayload,
|
|
20
|
+
RunTaskState,
|
|
21
|
+
} from './types';
|
|
22
|
+
import { buildDag, type Dag } from './dag';
|
|
23
|
+
import { getHandler, hasHandler, loadPlugins } from './registry';
|
|
24
|
+
import { runSpawn, runCommand } from './runner';
|
|
25
|
+
import { parseDuration, nowISO, generateRunId } from './utils';
|
|
26
|
+
import { promptDocumentFromString, serializePromptDocument } from './prompt-doc';
|
|
27
|
+
import {
|
|
28
|
+
executeHook,
|
|
29
|
+
buildPipelineStartContext,
|
|
30
|
+
buildTaskContext,
|
|
31
|
+
buildPipelineCompleteContext,
|
|
32
|
+
buildPipelineErrorContext,
|
|
33
|
+
type PipelineInfo,
|
|
34
|
+
type TrackInfo,
|
|
35
|
+
type TaskInfo,
|
|
36
|
+
} from './hooks';
|
|
37
|
+
import { Logger, tailLines, clip } from './logger';
|
|
38
|
+
import { InMemoryApprovalGateway, type ApprovalGateway } from './approval';
|
|
39
|
+
|
|
40
|
+
// ═══ A7: Typed trigger errors ═══
|
|
41
|
+
// Replace string-matching on error messages with structured error types so
|
|
42
|
+
// coincidental substrings don't cause misclassification.
|
|
43
|
+
|
|
44
|
+
export class TriggerBlockedError extends Error {
|
|
45
|
+
readonly code = 'TRIGGER_BLOCKED' as const;
|
|
46
|
+
constructor(message: string) {
|
|
47
|
+
super(message);
|
|
48
|
+
this.name = 'TriggerBlockedError';
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
export class TriggerTimeoutError extends Error {
|
|
53
|
+
readonly code = 'TRIGGER_TIMEOUT' as const;
|
|
54
|
+
constructor(message: string) {
|
|
55
|
+
super(message);
|
|
56
|
+
this.name = 'TriggerTimeoutError';
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
// ═══ Preflight Validation ═══
|
|
61
|
+
|
|
62
|
+
function preflight(config: PipelineConfig, dag: Dag): void {
|
|
63
|
+
const errors: string[] = [];
|
|
64
|
+
|
|
65
|
+
for (const [, node] of dag.nodes) {
|
|
66
|
+
const task = node.task;
|
|
67
|
+
const track = node.track;
|
|
68
|
+
const driverName = task.driver ?? track.driver ?? config.driver ?? 'opencode';
|
|
69
|
+
|
|
70
|
+
// Pure command tasks don't use a driver — skip driver registration check.
|
|
71
|
+
const isCommandOnly = task.command && !task.prompt;
|
|
72
|
+
|
|
73
|
+
if (!isCommandOnly && !hasHandler('drivers', driverName)) {
|
|
74
|
+
errors.push(`Task "${node.taskId}": driver "${driverName}" not registered`);
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
if (task.trigger && !hasHandler('triggers', task.trigger.type)) {
|
|
78
|
+
errors.push(`Task "${node.taskId}": trigger type "${task.trigger.type}" not registered`);
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
if (task.completion && !hasHandler('completions', task.completion.type)) {
|
|
82
|
+
errors.push(
|
|
83
|
+
`Task "${node.taskId}": completion type "${task.completion.type}" not registered`,
|
|
84
|
+
);
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
const mws = task.middlewares ?? track.middlewares ?? [];
|
|
88
|
+
for (const mw of mws) {
|
|
89
|
+
if (!hasHandler('middlewares', mw.type)) {
|
|
90
|
+
errors.push(`Task "${node.taskId}": middleware type "${mw.type}" not registered`);
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
if (task.continue_from && hasHandler('drivers', driverName)) {
|
|
95
|
+
const driver = getHandler<DriverPlugin>('drivers', driverName);
|
|
96
|
+
if (!driver.capabilities.sessionResume) {
|
|
97
|
+
// buildDag has already qualified `continue_from` and stored the result
|
|
98
|
+
// on the node; preflight runs after buildDag, so the upstream id is
|
|
99
|
+
// always available here without re-resolving.
|
|
100
|
+
const upstreamId = node.resolvedContinueFrom;
|
|
101
|
+
if (upstreamId) {
|
|
102
|
+
const upstream = dag.nodes.get(upstreamId);
|
|
103
|
+
if (upstream) {
|
|
104
|
+
// A handoff is possible via session resume (already ruled out above),
|
|
105
|
+
// OR in-memory text injection through normalizedMap
|
|
106
|
+
// (when the upstream driver implements parseResult and returns normalizedOutput).
|
|
107
|
+
const upstreamDriverName =
|
|
108
|
+
upstream.task.driver ?? upstream.track.driver ?? config.driver ?? 'opencode';
|
|
109
|
+
const upstreamDriver = hasHandler('drivers', upstreamDriverName)
|
|
110
|
+
? getHandler<DriverPlugin>('drivers', upstreamDriverName)
|
|
111
|
+
: null;
|
|
112
|
+
const canNormalize = typeof upstreamDriver?.parseResult === 'function';
|
|
113
|
+
|
|
114
|
+
if (!canNormalize) {
|
|
115
|
+
errors.push(
|
|
116
|
+
`Task "${node.taskId}" uses continue_from: "${task.continue_from}", ` +
|
|
117
|
+
`but upstream task "${upstreamId}" its driver ` +
|
|
118
|
+
`does not implement parseResult for text-injection handoff. ` +
|
|
119
|
+
`Use a driver with parseResult, or remove continue_from.`,
|
|
120
|
+
);
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
if (errors.length > 0) {
|
|
129
|
+
throw new Error(`Preflight validation failed:\n - ${errors.join('\n - ')}`);
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
// ═══ Engine ═══
|
|
134
|
+
|
|
135
|
+
export interface EngineResult {
|
|
136
|
+
readonly success: boolean;
|
|
137
|
+
readonly runId: string;
|
|
138
|
+
readonly logPath: string;
|
|
139
|
+
readonly summary: {
|
|
140
|
+
total: number;
|
|
141
|
+
success: number;
|
|
142
|
+
failed: number;
|
|
143
|
+
skipped: number;
|
|
144
|
+
timeout: number;
|
|
145
|
+
blocked: number;
|
|
146
|
+
};
|
|
147
|
+
readonly states: ReadonlyMap<string, TaskState>;
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
// ═══ Pipeline Events ═══
|
|
151
|
+
//
|
|
152
|
+
// The engine emits RunEventPayload values (defined in @tagma/types) via
|
|
153
|
+
// `onEvent`. Every payload carries `runId`; the editor server stamps a
|
|
154
|
+
// per-run `seq` before broadcasting. There is one event vocabulary
|
|
155
|
+
// end-to-end — no server-side translation layer.
|
|
156
|
+
|
|
157
|
+
// Re-export so SDK consumers can import the event type without reaching
|
|
158
|
+
// into @tagma/types directly.
|
|
159
|
+
export type { RunEventPayload } from './types';
|
|
160
|
+
|
|
161
|
+
// ═══ Helpers ═══
|
|
162
|
+
|
|
163
|
+
/**
|
|
164
|
+
* Project the engine's internal TaskState onto the wire RunTaskState
|
|
165
|
+
* shape. `logs` / `totalLogCount` default to empty — they are populated
|
|
166
|
+
* on the server side from streamed `task_log` events, not from state.
|
|
167
|
+
*/
|
|
168
|
+
function toRunTaskState(
|
|
169
|
+
taskId: string,
|
|
170
|
+
trackId: string,
|
|
171
|
+
taskName: string,
|
|
172
|
+
state: TaskState,
|
|
173
|
+
): RunTaskState {
|
|
174
|
+
const result = state.result;
|
|
175
|
+
const cfg = state.config;
|
|
176
|
+
return {
|
|
177
|
+
taskId,
|
|
178
|
+
trackId,
|
|
179
|
+
taskName,
|
|
180
|
+
status: state.status,
|
|
181
|
+
startedAt: state.startedAt,
|
|
182
|
+
finishedAt: state.finishedAt,
|
|
183
|
+
durationMs: result?.durationMs ?? null,
|
|
184
|
+
exitCode: result?.exitCode ?? null,
|
|
185
|
+
stdout: result?.stdout ?? '',
|
|
186
|
+
stderr: result?.stderr ?? '',
|
|
187
|
+
stderrPath: result?.stderrPath ?? null,
|
|
188
|
+
sessionId: result?.sessionId ?? null,
|
|
189
|
+
normalizedOutput: result?.normalizedOutput ?? null,
|
|
190
|
+
resolvedDriver: cfg.driver ?? null,
|
|
191
|
+
resolvedModel: cfg.model ?? null,
|
|
192
|
+
resolvedPermissions: (cfg.permissions as Permissions | undefined) ?? null,
|
|
193
|
+
logs: [],
|
|
194
|
+
totalLogCount: 0,
|
|
195
|
+
};
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
export interface RunPipelineOptions {
|
|
199
|
+
readonly approvalGateway?: ApprovalGateway;
|
|
200
|
+
/**
|
|
201
|
+
* Maximum number of per-run log directories to retain under `<workDir>/.tagma/logs/`.
|
|
202
|
+
* Oldest directories are deleted after each run. Defaults to 20. Set to 0 to disable cleanup.
|
|
203
|
+
*/
|
|
204
|
+
readonly maxLogRuns?: number;
|
|
205
|
+
/**
|
|
206
|
+
* Caller-supplied run ID. When provided the engine uses this instead of
|
|
207
|
+
* generating its own via `generateRunId()`, keeping the editor and SDK
|
|
208
|
+
* log directories aligned on the same ID.
|
|
209
|
+
*/
|
|
210
|
+
readonly runId?: string;
|
|
211
|
+
/**
|
|
212
|
+
* External AbortSignal — aborting it cancels the pipeline immediately.
|
|
213
|
+
* Equivalent to the pipeline timeout firing, but caller-controlled.
|
|
214
|
+
*/
|
|
215
|
+
readonly signal?: AbortSignal;
|
|
216
|
+
/**
|
|
217
|
+
* Called on every pipeline/task status transition.
|
|
218
|
+
* Use for real-time UI updates (e.g. updating a visual workflow graph).
|
|
219
|
+
*/
|
|
220
|
+
readonly onEvent?: (event: RunEventPayload) => void;
|
|
221
|
+
/**
|
|
222
|
+
* Skip the engine's built-in `loadPlugins(config.plugins)` call.
|
|
223
|
+
* Use this when the host has already pre-loaded plugins from a custom
|
|
224
|
+
* resolution path (e.g. a user workspace's node_modules) so the engine
|
|
225
|
+
* doesn't re-resolve them via Node's default cwd-based import.
|
|
226
|
+
*/
|
|
227
|
+
readonly skipPluginLoading?: boolean;
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
// Poll interval when no tasks are in-flight but non-terminal tasks remain
|
|
231
|
+
// (e.g. tasks waiting on a file or manual trigger).
|
|
232
|
+
const POLL_INTERVAL_MS = 50;
|
|
233
|
+
|
|
234
|
+
// R15: cap on each normalized-output entry stored in normalizedMap so a
|
|
235
|
+
// runaway parseResult can't accumulate hundreds of MB across tasks. 1 MB
|
|
236
|
+
// is generous for any text-context handoff between AI tasks.
|
|
237
|
+
const MAX_NORMALIZED_BYTES = 1_000_000;
|
|
238
|
+
|
|
239
|
+
export async function runPipeline(
|
|
240
|
+
config: PipelineConfig,
|
|
241
|
+
workDir: string,
|
|
242
|
+
options: RunPipelineOptions = {},
|
|
243
|
+
): Promise<EngineResult> {
|
|
244
|
+
const approvalGateway = options.approvalGateway ?? new InMemoryApprovalGateway();
|
|
245
|
+
const maxLogRuns = options.maxLogRuns ?? 20;
|
|
246
|
+
|
|
247
|
+
// Load any plugins declared in the pipeline config before preflight so that
|
|
248
|
+
// drivers, completions, and middlewares referenced in YAML are registered.
|
|
249
|
+
// Hosts that pre-load plugins from a custom path (e.g. the editor loading
|
|
250
|
+
// from the user's workspace node_modules) pass skipPluginLoading: true so
|
|
251
|
+
// we don't re-resolve via Node's cwd-based default import.
|
|
252
|
+
if (!options.skipPluginLoading && config.plugins?.length) {
|
|
253
|
+
await loadPlugins(config.plugins);
|
|
254
|
+
}
|
|
255
|
+
|
|
256
|
+
const dag = buildDag(config);
|
|
257
|
+
const runId = options.runId ?? generateRunId();
|
|
258
|
+
preflight(config, dag);
|
|
259
|
+
|
|
260
|
+
const startedAt = nowISO();
|
|
261
|
+
const pipelineInfo: PipelineInfo = { name: config.name, run_id: runId, started_at: startedAt };
|
|
262
|
+
// Forward every structured log line to subscribers as task_log events.
|
|
263
|
+
// Reading options.onEvent inside the callback (vs. capturing it once) keeps
|
|
264
|
+
// the SDK behavior correct if callers pass a fresh onEvent on each run.
|
|
265
|
+
const log = new Logger(workDir, runId, (record) => {
|
|
266
|
+
options.onEvent?.({
|
|
267
|
+
type: 'task_log',
|
|
268
|
+
runId,
|
|
269
|
+
taskId: record.taskId,
|
|
270
|
+
level: record.level,
|
|
271
|
+
timestamp: record.timestamp,
|
|
272
|
+
text: record.text,
|
|
273
|
+
});
|
|
274
|
+
});
|
|
275
|
+
|
|
276
|
+
try {
|
|
277
|
+
log.info('[pipeline]', `start "${config.name}" run_id=${runId}`);
|
|
278
|
+
|
|
279
|
+
// File-only: dump the resolved pipeline shape + DAG topology for post-mortem.
|
|
280
|
+
log.section('Pipeline configuration');
|
|
281
|
+
log.quiet(`name: ${config.name}`);
|
|
282
|
+
log.quiet(`driver: ${config.driver ?? '(default: opencode)'}`);
|
|
283
|
+
log.quiet(`timeout: ${config.timeout ?? '(none)'}`);
|
|
284
|
+
log.quiet(`tracks: ${config.tracks.length}`);
|
|
285
|
+
log.quiet(`tasks (total): ${dag.nodes.size}`);
|
|
286
|
+
log.quiet(`plugins: ${(config.plugins ?? []).join(', ') || '(none)'}`);
|
|
287
|
+
log.quiet(
|
|
288
|
+
`hooks: ${config.hooks ? Object.keys(config.hooks).join(', ') || '(none)' : '(none)'}`,
|
|
289
|
+
);
|
|
290
|
+
|
|
291
|
+
log.section('DAG topology');
|
|
292
|
+
for (const [id, node] of dag.nodes) {
|
|
293
|
+
const deps = node.dependsOn.length ? node.dependsOn.join(', ') : '(root)';
|
|
294
|
+
const kind = node.task.prompt ? 'ai' : 'cmd';
|
|
295
|
+
log.quiet(` • ${id} [${kind}] track=${node.track.id} deps=[${deps}]`);
|
|
296
|
+
}
|
|
297
|
+
log.quiet('');
|
|
298
|
+
|
|
299
|
+
// Initialize states (before hook, so we can return them even if blocked)
|
|
300
|
+
const states = new Map<string, TaskState>();
|
|
301
|
+
for (const [id, node] of dag.nodes) {
|
|
302
|
+
states.set(id, {
|
|
303
|
+
config: node.task,
|
|
304
|
+
trackConfig: node.track,
|
|
305
|
+
status: 'idle',
|
|
306
|
+
result: null,
|
|
307
|
+
startedAt: null,
|
|
308
|
+
finishedAt: null,
|
|
309
|
+
});
|
|
310
|
+
}
|
|
311
|
+
|
|
312
|
+
// Pipeline start hook (gate). Runs BEFORE the engine emits run_start so
|
|
313
|
+
// a blocked pipeline produces zero wire events (the server treats the
|
|
314
|
+
// thrown error as run_error). Hosts get a rich error message; nothing
|
|
315
|
+
// is ever half-broadcast.
|
|
316
|
+
const startHook = await executeHook(
|
|
317
|
+
config.hooks,
|
|
318
|
+
'pipeline_start',
|
|
319
|
+
buildPipelineStartContext(pipelineInfo),
|
|
320
|
+
workDir,
|
|
321
|
+
);
|
|
322
|
+
if (!startHook.allowed) {
|
|
323
|
+
console.error(`Pipeline blocked by pipeline_start hook (exit code ${startHook.exitCode})`);
|
|
324
|
+
await executeHook(
|
|
325
|
+
config.hooks,
|
|
326
|
+
'pipeline_error',
|
|
327
|
+
buildPipelineErrorContext(pipelineInfo, 'pipeline_blocked', 'pipeline_blocked'),
|
|
328
|
+
workDir,
|
|
329
|
+
);
|
|
330
|
+
return {
|
|
331
|
+
success: false,
|
|
332
|
+
runId,
|
|
333
|
+
logPath: log.path,
|
|
334
|
+
summary: {
|
|
335
|
+
total: dag.nodes.size,
|
|
336
|
+
success: 0,
|
|
337
|
+
failed: 0,
|
|
338
|
+
skipped: 0,
|
|
339
|
+
timeout: 0,
|
|
340
|
+
blocked: 0,
|
|
341
|
+
},
|
|
342
|
+
states: freezeStates(states),
|
|
343
|
+
};
|
|
344
|
+
}
|
|
345
|
+
|
|
346
|
+
// Pipeline approved — transition all tasks to waiting.
|
|
347
|
+
for (const [, state] of states) {
|
|
348
|
+
state.status = 'waiting';
|
|
349
|
+
}
|
|
350
|
+
// Emit run_start with a wire-shape snapshot so SSE subscribers can
|
|
351
|
+
// initialize their task maps on the same event stream that carries
|
|
352
|
+
// updates. No separate "server pre-broadcasts run_start" ceremony —
|
|
353
|
+
// the engine owns the lifecycle boundary.
|
|
354
|
+
const runStartTasks: RunTaskState[] = [];
|
|
355
|
+
for (const [id, node] of dag.nodes) {
|
|
356
|
+
const s = states.get(id)!;
|
|
357
|
+
runStartTasks.push(toRunTaskState(id, node.track.id, node.task.name ?? id, s));
|
|
358
|
+
}
|
|
359
|
+
emit({ type: 'run_start', runId, tasks: runStartTasks });
|
|
360
|
+
|
|
361
|
+
const sessionMap = new Map<string, string>();
|
|
362
|
+
const normalizedMap = new Map<string, string>();
|
|
363
|
+
|
|
364
|
+
// Pipeline timeout + abort reason tracking.
|
|
365
|
+
//
|
|
366
|
+
// `abortReason` replaces the previous `pipelineAborted: boolean`: it
|
|
367
|
+
// carries the concrete cause (timeout / stop_all / external) through
|
|
368
|
+
// to run_end and the pipeline_error hook so downstream consumers can
|
|
369
|
+
// distinguish them without scraping message strings.
|
|
370
|
+
const pipelineTimeoutMs = config.timeout ? parseDuration(config.timeout) : 0;
|
|
371
|
+
let abortReason: AbortReason | null = null;
|
|
372
|
+
const abortController = new AbortController();
|
|
373
|
+
let pipelineTimer: ReturnType<typeof setTimeout> | null = null;
|
|
374
|
+
|
|
375
|
+
if (pipelineTimeoutMs > 0) {
|
|
376
|
+
pipelineTimer = setTimeout(() => {
|
|
377
|
+
if (abortReason === null) abortReason = 'timeout';
|
|
378
|
+
abortController.abort();
|
|
379
|
+
}, pipelineTimeoutMs);
|
|
380
|
+
}
|
|
381
|
+
|
|
382
|
+
// When the pipeline is aborted (timeout, stop_all, external), drain
|
|
383
|
+
// all pending approvals so waiting triggers unblock immediately.
|
|
384
|
+
abortController.signal.addEventListener('abort', () => {
|
|
385
|
+
approvalGateway.abortAll('pipeline aborted');
|
|
386
|
+
});
|
|
387
|
+
|
|
388
|
+
// Wire external cancel signal into the internal abort controller.
|
|
389
|
+
const externalAbortHandler = () => {
|
|
390
|
+
if (abortReason === null) abortReason = 'external';
|
|
391
|
+
abortController.abort();
|
|
392
|
+
};
|
|
393
|
+
if (options.signal) {
|
|
394
|
+
if (options.signal.aborted) {
|
|
395
|
+
externalAbortHandler();
|
|
396
|
+
} else {
|
|
397
|
+
options.signal.addEventListener('abort', externalAbortHandler, { once: true });
|
|
398
|
+
}
|
|
399
|
+
}
|
|
400
|
+
|
|
401
|
+
// Bridge approval gateway events onto the wire stream so hosts (editor
|
|
402
|
+
// server, CLI adapters) see approvals on the same channel as task
|
|
403
|
+
// updates. The server no longer needs its own gateway subscription.
|
|
404
|
+
const unsubscribeApprovals = approvalGateway.subscribe((ev) => {
|
|
405
|
+
if (ev.type === 'requested') {
|
|
406
|
+
emit({
|
|
407
|
+
type: 'approval_request',
|
|
408
|
+
runId,
|
|
409
|
+
request: {
|
|
410
|
+
id: ev.request.id,
|
|
411
|
+
taskId: ev.request.taskId,
|
|
412
|
+
trackId: ev.request.trackId,
|
|
413
|
+
message: ev.request.message,
|
|
414
|
+
createdAt: ev.request.createdAt,
|
|
415
|
+
timeoutMs: ev.request.timeoutMs,
|
|
416
|
+
metadata: ev.request.metadata,
|
|
417
|
+
},
|
|
418
|
+
});
|
|
419
|
+
return;
|
|
420
|
+
}
|
|
421
|
+
if (ev.type === 'resolved' || ev.type === 'expired' || ev.type === 'aborted') {
|
|
422
|
+
const outcome =
|
|
423
|
+
ev.type === 'resolved'
|
|
424
|
+
? ev.decision.outcome
|
|
425
|
+
: ev.type === 'expired'
|
|
426
|
+
? 'timeout'
|
|
427
|
+
: 'aborted';
|
|
428
|
+
emit({
|
|
429
|
+
type: 'approval_resolved',
|
|
430
|
+
runId,
|
|
431
|
+
requestId: ev.request.id,
|
|
432
|
+
outcome,
|
|
433
|
+
});
|
|
434
|
+
}
|
|
435
|
+
});
|
|
436
|
+
|
|
437
|
+
// ── Helpers ──
|
|
438
|
+
|
|
439
|
+
function emit(event: RunEventPayload): void {
|
|
440
|
+
options.onEvent?.(event);
|
|
441
|
+
}
|
|
442
|
+
|
|
443
|
+
function setTaskStatus(taskId: string, newStatus: TaskStatus): void {
|
|
444
|
+
const state = states.get(taskId)!;
|
|
445
|
+
// Terminal lock: once a task reaches a terminal state it must not be
|
|
446
|
+
// re-transitioned. This prevents stop_all from marking running tasks as
|
|
447
|
+
// skipped and then having their in-flight processTask promise overwrite
|
|
448
|
+
// that with success/failed, producing an invalid double transition.
|
|
449
|
+
if (isTerminal(state.status)) return;
|
|
450
|
+
state.status = newStatus;
|
|
451
|
+
const result = state.result;
|
|
452
|
+
const cfg = state.config;
|
|
453
|
+
emit({
|
|
454
|
+
type: 'task_update',
|
|
455
|
+
runId,
|
|
456
|
+
taskId,
|
|
457
|
+
status: newStatus,
|
|
458
|
+
startedAt: state.startedAt ?? undefined,
|
|
459
|
+
finishedAt: state.finishedAt ?? undefined,
|
|
460
|
+
durationMs: result?.durationMs,
|
|
461
|
+
exitCode: result?.exitCode,
|
|
462
|
+
stdout: result?.stdout,
|
|
463
|
+
stderr: result?.stderr,
|
|
464
|
+
stderrPath: result?.stderrPath ?? null,
|
|
465
|
+
sessionId: result?.sessionId ?? null,
|
|
466
|
+
normalizedOutput: result?.normalizedOutput ?? null,
|
|
467
|
+
resolvedDriver: cfg.driver ?? null,
|
|
468
|
+
resolvedModel: cfg.model ?? null,
|
|
469
|
+
resolvedPermissions: (cfg.permissions as Permissions | undefined) ?? null,
|
|
470
|
+
});
|
|
471
|
+
}
|
|
472
|
+
|
|
473
|
+
function getOnFailure(taskId: string): OnFailure {
|
|
474
|
+
return dag.nodes.get(taskId)?.track.on_failure ?? 'skip_downstream';
|
|
475
|
+
}
|
|
476
|
+
|
|
477
|
+
function isDependencySatisfied(depId: string): 'satisfied' | 'unsatisfied' | 'skip' {
|
|
478
|
+
const depState = states.get(depId);
|
|
479
|
+
if (!depState) return 'skip';
|
|
480
|
+
switch (depState.status) {
|
|
481
|
+
case 'success':
|
|
482
|
+
return 'satisfied';
|
|
483
|
+
case 'skipped':
|
|
484
|
+
return 'skip';
|
|
485
|
+
case 'failed':
|
|
486
|
+
case 'timeout':
|
|
487
|
+
case 'blocked':
|
|
488
|
+
return getOnFailure(depId) === 'ignore' ? 'satisfied' : 'skip';
|
|
489
|
+
default:
|
|
490
|
+
return 'unsatisfied';
|
|
491
|
+
}
|
|
492
|
+
}
|
|
493
|
+
|
|
494
|
+
/**
|
|
495
|
+
* H3: "stop_all" historically only stopped tasks within the same track,
|
|
496
|
+
* which contradicted both its name and user expectations. It now stops
|
|
497
|
+
* the **entire pipeline**:
|
|
498
|
+
* - In-flight tasks are signalled via the shared abort controller so
|
|
499
|
+
* drivers / runner.ts can cancel cooperatively (returning
|
|
500
|
+
* `failureKind: 'timeout'`).
|
|
501
|
+
* - Still-waiting tasks across every track are immediately marked
|
|
502
|
+
* skipped so the run completes promptly.
|
|
503
|
+
* The terminal lock in setTaskStatus prevents any later re-transition
|
|
504
|
+
* should a completed running task try to overwrite the skipped state.
|
|
505
|
+
*/
|
|
506
|
+
function applyStopAll(_failedTrackId: string): void {
|
|
507
|
+
if (abortReason === null) abortReason = 'stop_all';
|
|
508
|
+
abortController.abort();
|
|
509
|
+
for (const [id, state] of states) {
|
|
510
|
+
if (state.status === 'waiting') {
|
|
511
|
+
state.finishedAt = nowISO();
|
|
512
|
+
setTaskStatus(id, 'skipped');
|
|
513
|
+
}
|
|
514
|
+
}
|
|
515
|
+
}
|
|
516
|
+
|
|
517
|
+
function buildTaskInfoObj(taskId: string): TaskInfo {
|
|
518
|
+
const state = states.get(taskId)!;
|
|
519
|
+
return {
|
|
520
|
+
id: taskId,
|
|
521
|
+
name: state.config.name,
|
|
522
|
+
type: state.config.prompt ? 'ai' : 'command',
|
|
523
|
+
status: state.status,
|
|
524
|
+
exit_code: state.result?.exitCode ?? null,
|
|
525
|
+
duration_ms: state.result?.durationMs ?? null,
|
|
526
|
+
stderr_path: state.result?.stderrPath ?? null,
|
|
527
|
+
session_id: state.result?.sessionId ?? null,
|
|
528
|
+
started_at: state.startedAt,
|
|
529
|
+
finished_at: state.finishedAt,
|
|
530
|
+
};
|
|
531
|
+
}
|
|
532
|
+
|
|
533
|
+
function trackInfoOf(taskId: string): TrackInfo {
|
|
534
|
+
const node = dag.nodes.get(taskId)!;
|
|
535
|
+
return { id: node.track.id, name: node.track.name };
|
|
536
|
+
}
|
|
537
|
+
|
|
538
|
+
async function fireHook(taskId: string, event: 'task_success' | 'task_failure'): Promise<void> {
|
|
539
|
+
await executeHook(
|
|
540
|
+
config.hooks,
|
|
541
|
+
event,
|
|
542
|
+
buildTaskContext(event, pipelineInfo, trackInfoOf(taskId), buildTaskInfoObj(taskId)),
|
|
543
|
+
workDir,
|
|
544
|
+
abortController.signal,
|
|
545
|
+
);
|
|
546
|
+
}
|
|
547
|
+
|
|
548
|
+
// ── Process a single task ──
|
|
549
|
+
|
|
550
|
+
async function processTask(taskId: string): Promise<void> {
|
|
551
|
+
const state = states.get(taskId)!;
|
|
552
|
+
const node = dag.nodes.get(taskId)!;
|
|
553
|
+
const task = node.task;
|
|
554
|
+
const track = node.track;
|
|
555
|
+
|
|
556
|
+
log.section(`Task ${taskId}`, taskId);
|
|
557
|
+
log.debug(
|
|
558
|
+
`[task:${taskId}]`,
|
|
559
|
+
`type=${task.prompt ? 'ai' : 'cmd'} track=${track.id} deps=[${node.dependsOn.join(', ') || '(root)'}]`,
|
|
560
|
+
);
|
|
561
|
+
|
|
562
|
+
// 1. Check dependencies
|
|
563
|
+
for (const depId of node.dependsOn) {
|
|
564
|
+
const result = isDependencySatisfied(depId);
|
|
565
|
+
if (result === 'skip') {
|
|
566
|
+
const depStatus = states.get(depId)?.status ?? 'unknown';
|
|
567
|
+
log.debug(`[task:${taskId}]`, `skipped (upstream "${depId}" status=${depStatus})`);
|
|
568
|
+
state.finishedAt = nowISO();
|
|
569
|
+
setTaskStatus(taskId, 'skipped');
|
|
570
|
+
return;
|
|
571
|
+
}
|
|
572
|
+
if (result === 'unsatisfied') return; // still waiting
|
|
573
|
+
}
|
|
574
|
+
|
|
575
|
+
// 2. Check trigger
|
|
576
|
+
if (task.trigger) {
|
|
577
|
+
log.debug(
|
|
578
|
+
`[task:${taskId}]`,
|
|
579
|
+
`trigger wait: type=${task.trigger.type} ${JSON.stringify(task.trigger)}`,
|
|
580
|
+
);
|
|
581
|
+
try {
|
|
582
|
+
const triggerPlugin = getHandler<TriggerPlugin>('triggers', task.trigger.type);
|
|
583
|
+
// R6: race the plugin's watch() against the pipeline's abort signal.
|
|
584
|
+
// Third-party triggers may forget to wire up ctx.signal — without
|
|
585
|
+
// this race, an aborted pipeline would hang forever waiting for the
|
|
586
|
+
// plugin's watch promise to resolve. The race resolves on whichever
|
|
587
|
+
// path settles first, and the cleanup paths in finally never run on
|
|
588
|
+
// the orphaned plugin promise (it's allowed to leak a watcher; the
|
|
589
|
+
// pipeline is being torn down anyway).
|
|
590
|
+
await new Promise<unknown>((resolve, reject) => {
|
|
591
|
+
let settled = false;
|
|
592
|
+
const onAbort = () => {
|
|
593
|
+
if (settled) return;
|
|
594
|
+
settled = true;
|
|
595
|
+
abortController.signal.removeEventListener('abort', onAbort);
|
|
596
|
+
reject(new Error('Pipeline aborted'));
|
|
597
|
+
};
|
|
598
|
+
if (abortController.signal.aborted) {
|
|
599
|
+
onAbort();
|
|
600
|
+
return;
|
|
601
|
+
}
|
|
602
|
+
abortController.signal.addEventListener('abort', onAbort, { once: true });
|
|
603
|
+
triggerPlugin
|
|
604
|
+
.watch(task.trigger as Record<string, unknown>, {
|
|
605
|
+
taskId: node.taskId,
|
|
606
|
+
trackId: track.id,
|
|
607
|
+
workDir: task.cwd ?? workDir,
|
|
608
|
+
signal: abortController.signal,
|
|
609
|
+
approvalGateway,
|
|
610
|
+
})
|
|
611
|
+
.then(
|
|
612
|
+
(v) => {
|
|
613
|
+
if (settled) return;
|
|
614
|
+
settled = true;
|
|
615
|
+
abortController.signal.removeEventListener('abort', onAbort);
|
|
616
|
+
resolve(v);
|
|
617
|
+
},
|
|
618
|
+
(e) => {
|
|
619
|
+
if (settled) return;
|
|
620
|
+
settled = true;
|
|
621
|
+
abortController.signal.removeEventListener('abort', onAbort);
|
|
622
|
+
reject(e);
|
|
623
|
+
},
|
|
624
|
+
);
|
|
625
|
+
});
|
|
626
|
+
log.debug(`[task:${taskId}]`, `trigger fired`);
|
|
627
|
+
} catch (err: unknown) {
|
|
628
|
+
// If pipeline was aborted while we were still waiting for the trigger,
|
|
629
|
+
// this task never entered running state → skipped, not timeout.
|
|
630
|
+
state.finishedAt = nowISO();
|
|
631
|
+
if (abortReason !== null) {
|
|
632
|
+
setTaskStatus(taskId, 'skipped');
|
|
633
|
+
} else if (err instanceof TriggerBlockedError) {
|
|
634
|
+
setTaskStatus(taskId, 'blocked'); // user/policy rejection
|
|
635
|
+
} else if (err instanceof TriggerTimeoutError) {
|
|
636
|
+
setTaskStatus(taskId, 'timeout'); // genuine trigger wait timeout
|
|
637
|
+
} else {
|
|
638
|
+
// A7 fallback: also check message strings for backward-compat with
|
|
639
|
+
// third-party trigger plugins that don't throw typed errors yet.
|
|
640
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
641
|
+
if (msg.includes('rejected') || msg.includes('denied')) {
|
|
642
|
+
setTaskStatus(taskId, 'blocked');
|
|
643
|
+
} else if (msg.includes('timeout')) {
|
|
644
|
+
setTaskStatus(taskId, 'timeout');
|
|
645
|
+
} else {
|
|
646
|
+
setTaskStatus(taskId, 'failed'); // plugin error, watcher crash, etc.
|
|
647
|
+
}
|
|
648
|
+
}
|
|
649
|
+
try {
|
|
650
|
+
await fireHook(taskId, 'task_failure');
|
|
651
|
+
} catch (hookErr) {
|
|
652
|
+
log.error(
|
|
653
|
+
`[task:${taskId}]`,
|
|
654
|
+
`hook execution failed: ${hookErr instanceof Error ? hookErr.message : String(hookErr)}`,
|
|
655
|
+
);
|
|
656
|
+
}
|
|
657
|
+
return;
|
|
658
|
+
}
|
|
659
|
+
}
|
|
660
|
+
|
|
661
|
+
// 3. task_start hook (gate)
|
|
662
|
+
const hookResult = await executeHook(
|
|
663
|
+
config.hooks,
|
|
664
|
+
'task_start',
|
|
665
|
+
buildTaskContext('task_start', pipelineInfo, trackInfoOf(taskId), buildTaskInfoObj(taskId)),
|
|
666
|
+
workDir,
|
|
667
|
+
abortController.signal,
|
|
668
|
+
);
|
|
669
|
+
if (hookResult.exitCode !== 0 || config.hooks?.task_start) {
|
|
670
|
+
log.debug(
|
|
671
|
+
`[task:${taskId}]`,
|
|
672
|
+
`task_start hook exit=${hookResult.exitCode} allowed=${hookResult.allowed}`,
|
|
673
|
+
);
|
|
674
|
+
}
|
|
675
|
+
if (!hookResult.allowed) {
|
|
676
|
+
state.finishedAt = nowISO();
|
|
677
|
+
setTaskStatus(taskId, 'blocked');
|
|
678
|
+
try {
|
|
679
|
+
await fireHook(taskId, 'task_failure');
|
|
680
|
+
} catch (hookErr) {
|
|
681
|
+
log.error(
|
|
682
|
+
`[task:${taskId}]`,
|
|
683
|
+
`hook execution failed: ${hookErr instanceof Error ? hookErr.message : String(hookErr)}`,
|
|
684
|
+
);
|
|
685
|
+
}
|
|
686
|
+
return;
|
|
687
|
+
}
|
|
688
|
+
|
|
689
|
+
// 4. Mark running — set startedAt before emitting so subscribers see a
|
|
690
|
+
// complete task_update (startedAt non-null) on the status transition.
|
|
691
|
+
state.startedAt = nowISO();
|
|
692
|
+
setTaskStatus(taskId, 'running');
|
|
693
|
+
log.info(
|
|
694
|
+
`[task:${taskId}]`,
|
|
695
|
+
task.command ? `running: ${task.command}` : `running (driver task)`,
|
|
696
|
+
);
|
|
697
|
+
|
|
698
|
+
// File-only: resolved config for this task
|
|
699
|
+
const resolvedDriver = task.driver ?? track.driver ?? config.driver ?? 'opencode';
|
|
700
|
+
const resolvedModel = task.model ?? track.model ?? config.model ?? '(default)';
|
|
701
|
+
const resolvedPerms = task.permissions ?? track.permissions ?? '(default)';
|
|
702
|
+
const resolvedCwd = task.cwd ?? track.cwd ?? workDir;
|
|
703
|
+
log.debug(
|
|
704
|
+
`[task:${taskId}]`,
|
|
705
|
+
`resolved: driver=${resolvedDriver} model=${resolvedModel} cwd=${resolvedCwd}`,
|
|
706
|
+
);
|
|
707
|
+
log.debug(`[task:${taskId}]`, `permissions: ${JSON.stringify(resolvedPerms)}`);
|
|
708
|
+
if (task.continue_from) {
|
|
709
|
+
log.debug(`[task:${taskId}]`, `continue_from: "${task.continue_from}"`);
|
|
710
|
+
}
|
|
711
|
+
if (task.timeout) {
|
|
712
|
+
log.debug(`[task:${taskId}]`, `timeout: ${task.timeout}`);
|
|
713
|
+
}
|
|
714
|
+
|
|
715
|
+
try {
|
|
716
|
+
let result: TaskResult;
|
|
717
|
+
const timeoutMs = task.timeout ? parseDuration(task.timeout) : undefined;
|
|
718
|
+
|
|
719
|
+
const runOpts = { timeoutMs, signal: abortController.signal };
|
|
720
|
+
|
|
721
|
+
if (task.command) {
|
|
722
|
+
log.debug(`[task:${taskId}]`, `command: ${task.command}`);
|
|
723
|
+
result = await runCommand(task.command, task.cwd ?? workDir, runOpts);
|
|
724
|
+
} else {
|
|
725
|
+
// AI task: apply middleware chain against a structured PromptDocument.
|
|
726
|
+
const driverName = task.driver ?? track.driver ?? config.driver ?? 'opencode';
|
|
727
|
+
const driver = getHandler<DriverPlugin>('drivers', driverName);
|
|
728
|
+
|
|
729
|
+
const originalLen = task.prompt!.length;
|
|
730
|
+
let doc: PromptDocument = promptDocumentFromString(task.prompt!);
|
|
731
|
+
const mws = task.middlewares !== undefined ? task.middlewares : track.middlewares;
|
|
732
|
+
if (mws && mws.length > 0) {
|
|
733
|
+
log.debug(
|
|
734
|
+
`[task:${taskId}]`,
|
|
735
|
+
`middleware chain: ${mws.map((m) => m.type).join(' → ')}`,
|
|
736
|
+
);
|
|
737
|
+
const mwCtx: MiddlewareContext = {
|
|
738
|
+
task,
|
|
739
|
+
track,
|
|
740
|
+
workDir: task.cwd ?? workDir,
|
|
741
|
+
};
|
|
742
|
+
for (const mwConfig of mws) {
|
|
743
|
+
const mwPlugin = getHandler<MiddlewarePlugin>('middlewares', mwConfig.type);
|
|
744
|
+
const beforeBlocks = doc.contexts.length;
|
|
745
|
+
const beforeLen = serializePromptDocument(doc).length;
|
|
746
|
+
|
|
747
|
+
// Prefer the structured API. Fall back to the legacy
|
|
748
|
+
// `enhance(string) → string` path so v0.x plugins keep
|
|
749
|
+
// working — that fallback loses context structure (the
|
|
750
|
+
// middleware's output becomes the new task body) but never
|
|
751
|
+
// silently drops content.
|
|
752
|
+
if (typeof mwPlugin.enhanceDoc === 'function') {
|
|
753
|
+
const next = await mwPlugin.enhanceDoc(
|
|
754
|
+
doc,
|
|
755
|
+
mwConfig as Record<string, unknown>,
|
|
756
|
+
mwCtx,
|
|
757
|
+
);
|
|
758
|
+
if (
|
|
759
|
+
!next ||
|
|
760
|
+
typeof next !== 'object' ||
|
|
761
|
+
!Array.isArray((next as PromptDocument).contexts) ||
|
|
762
|
+
typeof (next as PromptDocument).task !== 'string'
|
|
763
|
+
) {
|
|
764
|
+
throw new Error(
|
|
765
|
+
`middleware "${mwConfig.type}".enhanceDoc() returned a malformed PromptDocument`,
|
|
766
|
+
);
|
|
767
|
+
}
|
|
768
|
+
doc = next as PromptDocument;
|
|
769
|
+
} else if (typeof mwPlugin.enhance === 'function') {
|
|
770
|
+
const asString = serializePromptDocument(doc);
|
|
771
|
+
const next = await mwPlugin.enhance(
|
|
772
|
+
asString,
|
|
773
|
+
mwConfig as Record<string, unknown>,
|
|
774
|
+
mwCtx,
|
|
775
|
+
);
|
|
776
|
+
// R3: a middleware that returns undefined / null / a non-string
|
|
777
|
+
// would silently corrupt the prompt. Fail loud.
|
|
778
|
+
if (typeof next !== 'string') {
|
|
779
|
+
throw new Error(
|
|
780
|
+
`middleware "${mwConfig.type}".enhance() returned ${next === null ? 'null' : typeof next}, expected string`,
|
|
781
|
+
);
|
|
782
|
+
}
|
|
783
|
+
// Legacy fallback: collapse the returned string into a
|
|
784
|
+
// fresh doc. Earlier structure is folded into the string
|
|
785
|
+
// (serializePromptDocument just ran), so bytes the driver
|
|
786
|
+
// sees match the old string pipeline.
|
|
787
|
+
doc = { contexts: [], task: next };
|
|
788
|
+
} else {
|
|
789
|
+
throw new Error(
|
|
790
|
+
`middleware "${mwConfig.type}" provides neither enhanceDoc nor enhance`,
|
|
791
|
+
);
|
|
792
|
+
}
|
|
793
|
+
const afterLen = serializePromptDocument(doc).length;
|
|
794
|
+
const addedBlocks = doc.contexts.length - beforeBlocks;
|
|
795
|
+
log.debug(
|
|
796
|
+
`[task:${taskId}]`,
|
|
797
|
+
` ${mwConfig.type}: ${beforeLen} → ${afterLen} chars` +
|
|
798
|
+
(addedBlocks > 0
|
|
799
|
+
? ` (+${addedBlocks} context block${addedBlocks > 1 ? 's' : ''})`
|
|
800
|
+
: ''),
|
|
801
|
+
);
|
|
802
|
+
}
|
|
803
|
+
}
|
|
804
|
+
const prompt = serializePromptDocument(doc);
|
|
805
|
+
log.debug(
|
|
806
|
+
`[task:${taskId}]`,
|
|
807
|
+
`prompt: ${originalLen} chars (final: ${prompt.length} chars, ${doc.contexts.length} block${doc.contexts.length === 1 ? '' : 's'})`,
|
|
808
|
+
);
|
|
809
|
+
log.quiet(`--- prompt (final) ---\n${clip(prompt)}\n--- end prompt ---`, taskId);
|
|
810
|
+
|
|
811
|
+
// H1: hand the driver a continue_from that has already been
|
|
812
|
+
// qualified by dag.ts. Without this, drivers like codex/opencode/
|
|
813
|
+
// claude-code look up maps directly with
|
|
814
|
+
// the user's raw (possibly bare) string, which races whenever two
|
|
815
|
+
// tracks share a task name. dag.ts has the only authoritative
|
|
816
|
+
// resolver, so we use its precomputed answer here.
|
|
817
|
+
// Drivers key sessionMap/normalizedMap by fully-qualified id. buildDag
|
|
818
|
+
// guarantees `resolvedContinueFrom` is set for every task that has a
|
|
819
|
+
// `continue_from`, so if we see the bare form here something upstream
|
|
820
|
+
// is broken — fail loud instead of silently miskeying the lookup.
|
|
821
|
+
if (task.continue_from && !node.resolvedContinueFrom) {
|
|
822
|
+
throw new Error(
|
|
823
|
+
`Internal: task "${taskId}" has continue_from "${task.continue_from}" ` +
|
|
824
|
+
`but no resolvedContinueFrom. buildDag should have qualified it.`,
|
|
825
|
+
);
|
|
826
|
+
}
|
|
827
|
+
const enrichedTask: TaskConfig = {
|
|
828
|
+
...task,
|
|
829
|
+
prompt,
|
|
830
|
+
continue_from: node.resolvedContinueFrom,
|
|
831
|
+
};
|
|
832
|
+
const driverCtx: DriverContext = {
|
|
833
|
+
sessionMap,
|
|
834
|
+
normalizedMap,
|
|
835
|
+
workDir: task.cwd ?? workDir,
|
|
836
|
+
// Structured view for drivers that want fine-grained control
|
|
837
|
+
// over serialization (e.g. inserting [Previous Output] between
|
|
838
|
+
// contexts and task). Drivers that read task.prompt see the
|
|
839
|
+
// default serialization and need no changes.
|
|
840
|
+
promptDoc: doc,
|
|
841
|
+
};
|
|
842
|
+
const spec = await driver.buildCommand(enrichedTask, track, driverCtx);
|
|
843
|
+
log.debug(`[task:${taskId}]`, `driver=${driverName}`);
|
|
844
|
+
log.debug(`[task:${taskId}]`, `spawn args: ${JSON.stringify(spec.args)}`);
|
|
845
|
+
if (spec.cwd) log.debug(`[task:${taskId}]`, `spawn cwd: ${spec.cwd}`);
|
|
846
|
+
if (spec.env)
|
|
847
|
+
log.debug(
|
|
848
|
+
`[task:${taskId}]`,
|
|
849
|
+
`spawn env overrides: ${Object.keys(spec.env).join(', ')}`,
|
|
850
|
+
);
|
|
851
|
+
if (spec.stdin) log.debug(`[task:${taskId}]`, `spawn stdin: ${spec.stdin.length} chars`);
|
|
852
|
+
result = await runSpawn(spec, driver, runOpts);
|
|
853
|
+
}
|
|
854
|
+
|
|
855
|
+
// 6. Determine terminal status (without emitting yet — result must be complete first)
|
|
856
|
+
// H2: branch on failureKind so spawn errors no longer masquerade as
|
|
857
|
+
// timeouts. Old runners that don't set failureKind still work — we
|
|
858
|
+
// fall back to the historical `exitCode === -1 → timeout` heuristic so
|
|
859
|
+
// pre-existing third-party drivers don't regress.
|
|
860
|
+
let terminalStatus: TaskStatus;
|
|
861
|
+
const kind = result.failureKind;
|
|
862
|
+
if (kind === 'timeout') {
|
|
863
|
+
terminalStatus = 'timeout';
|
|
864
|
+
} else if (kind === 'spawn_error') {
|
|
865
|
+
terminalStatus = 'failed';
|
|
866
|
+
} else if (kind === undefined && result.exitCode === -1) {
|
|
867
|
+
// Legacy path: pre-H2 driver returned -1 with no kind. Treat as
|
|
868
|
+
// timeout for backward compatibility (the previous behaviour).
|
|
869
|
+
terminalStatus = 'timeout';
|
|
870
|
+
} else if (result.exitCode !== 0) {
|
|
871
|
+
terminalStatus = 'failed';
|
|
872
|
+
} else if (task.completion) {
|
|
873
|
+
const plugin = getHandler<CompletionPlugin>('completions', task.completion.type);
|
|
874
|
+
const completionCtx = { workDir: task.cwd ?? workDir, signal: abortController.signal };
|
|
875
|
+
const passed = await plugin.check(
|
|
876
|
+
task.completion as Record<string, unknown>,
|
|
877
|
+
result,
|
|
878
|
+
completionCtx,
|
|
879
|
+
);
|
|
880
|
+
// R4: strict boolean check. Truthy strings/numbers used to be coerced
|
|
881
|
+
// to success — a check returning "ok" would let a failing task pass.
|
|
882
|
+
if (typeof passed !== 'boolean') {
|
|
883
|
+
throw new Error(
|
|
884
|
+
`completion "${task.completion.type}".check() returned ${passed === null ? 'null' : typeof passed}, expected boolean`,
|
|
885
|
+
);
|
|
886
|
+
}
|
|
887
|
+
terminalStatus = passed ? 'success' : 'failed';
|
|
888
|
+
} else {
|
|
889
|
+
terminalStatus = 'success';
|
|
890
|
+
}
|
|
891
|
+
|
|
892
|
+
// Store normalized text separately (in-memory) for continue_from handoff.
|
|
893
|
+
// R15: clip oversized values so a runaway parseResult can't accumulate
|
|
894
|
+
// hundreds of MB across tasks.
|
|
895
|
+
if (result.normalizedOutput !== null) {
|
|
896
|
+
const clipped =
|
|
897
|
+
result.normalizedOutput.length > MAX_NORMALIZED_BYTES
|
|
898
|
+
? result.normalizedOutput.slice(0, MAX_NORMALIZED_BYTES) +
|
|
899
|
+
`\n[…clipped at ${MAX_NORMALIZED_BYTES} bytes]`
|
|
900
|
+
: result.normalizedOutput;
|
|
901
|
+
normalizedMap.set(taskId, clipped);
|
|
902
|
+
}
|
|
903
|
+
|
|
904
|
+
if (result.stderr) {
|
|
905
|
+
const stderrPath = resolve(log.dir, `${taskId.replace(/\./g, '_')}.stderr`);
|
|
906
|
+
await Bun.write(stderrPath, result.stderr);
|
|
907
|
+
result = { ...result, stderrPath };
|
|
908
|
+
}
|
|
909
|
+
|
|
910
|
+
if (result.sessionId) {
|
|
911
|
+
// H1: qualified-only key.
|
|
912
|
+
sessionMap.set(taskId, result.sessionId);
|
|
913
|
+
}
|
|
914
|
+
|
|
915
|
+
// Set result and finishedAt before emitting terminal status so listeners see complete state
|
|
916
|
+
state.result = result;
|
|
917
|
+
state.finishedAt = nowISO();
|
|
918
|
+
setTaskStatus(taskId, terminalStatus);
|
|
919
|
+
|
|
920
|
+
// Log task outcome with relevant details
|
|
921
|
+
const durSec = (result.durationMs / 1000).toFixed(1);
|
|
922
|
+
if (terminalStatus === 'success') {
|
|
923
|
+
log.info(`[task:${taskId}]`, `success (${durSec}s)`);
|
|
924
|
+
} else {
|
|
925
|
+
log.error(
|
|
926
|
+
`[task:${taskId}]`,
|
|
927
|
+
`${terminalStatus} exit=${result.exitCode} duration=${durSec}s`,
|
|
928
|
+
);
|
|
929
|
+
if (result.stderr) {
|
|
930
|
+
const tail = tailLines(result.stderr, 10);
|
|
931
|
+
log.error(`[task:${taskId}]`, `stderr tail:\n${tail}`);
|
|
932
|
+
}
|
|
933
|
+
}
|
|
934
|
+
|
|
935
|
+
// File-only: full stdout/stderr dump (clipped) + extracted metadata
|
|
936
|
+
log.debug(
|
|
937
|
+
`[task:${taskId}]`,
|
|
938
|
+
`stdout: ${result.stdout.length} chars, stderr: ${result.stderr.length} chars`,
|
|
939
|
+
);
|
|
940
|
+
if (result.sessionId) {
|
|
941
|
+
log.debug(`[task:${taskId}]`, `sessionId: ${result.sessionId}`);
|
|
942
|
+
}
|
|
943
|
+
if (result.stderrPath) {
|
|
944
|
+
log.debug(`[task:${taskId}]`, `wrote stderr: ${result.stderrPath}`);
|
|
945
|
+
}
|
|
946
|
+
if (result.stdout) {
|
|
947
|
+
log.quiet(
|
|
948
|
+
`--- stdout (${taskId}) ---\n${clip(result.stdout)}\n--- end stdout ---`,
|
|
949
|
+
taskId,
|
|
950
|
+
);
|
|
951
|
+
}
|
|
952
|
+
if (result.stderr) {
|
|
953
|
+
log.quiet(
|
|
954
|
+
`--- stderr (${taskId}) ---\n${clip(result.stderr)}\n--- end stderr ---`,
|
|
955
|
+
taskId,
|
|
956
|
+
);
|
|
957
|
+
}
|
|
958
|
+
if (task.completion) {
|
|
959
|
+
log.debug(
|
|
960
|
+
`[task:${taskId}]`,
|
|
961
|
+
`completion check: type=${task.completion.type} result=${terminalStatus}`,
|
|
962
|
+
);
|
|
963
|
+
}
|
|
964
|
+
} catch (err: unknown) {
|
|
965
|
+
const errMsg = err instanceof Error ? (err.stack ?? err.message) : String(err);
|
|
966
|
+
log.error(`[task:${taskId}]`, `failed before execution: ${errMsg}`);
|
|
967
|
+
state.result = {
|
|
968
|
+
exitCode: -1,
|
|
969
|
+
stdout: '',
|
|
970
|
+
stderr: errMsg,
|
|
971
|
+
stderrPath: null,
|
|
972
|
+
durationMs: 0,
|
|
973
|
+
sessionId: null,
|
|
974
|
+
normalizedOutput: null,
|
|
975
|
+
// H2: Engine-level pre-execution errors (driver throw, middleware
|
|
976
|
+
// throw, getHandler 404) classify as spawn_error — the process never
|
|
977
|
+
// ran, so calling them "timeout" was actively misleading.
|
|
978
|
+
failureKind: 'spawn_error',
|
|
979
|
+
};
|
|
980
|
+
state.finishedAt = nowISO();
|
|
981
|
+
setTaskStatus(taskId, 'failed');
|
|
982
|
+
}
|
|
983
|
+
|
|
984
|
+
// 7. Fire hooks
|
|
985
|
+
const finalStatus: TaskStatus = state.status;
|
|
986
|
+
try {
|
|
987
|
+
await fireHook(taskId, finalStatus === 'success' ? 'task_success' : 'task_failure');
|
|
988
|
+
} catch (hookErr) {
|
|
989
|
+
log.error(
|
|
990
|
+
`[task:${taskId}]`,
|
|
991
|
+
`hook execution failed: ${hookErr instanceof Error ? hookErr.message : String(hookErr)}`,
|
|
992
|
+
);
|
|
993
|
+
}
|
|
994
|
+
|
|
995
|
+
// 8. Handle stop_all for failure states
|
|
996
|
+
if (finalStatus !== 'success' && getOnFailure(taskId) === 'stop_all') {
|
|
997
|
+
applyStopAll(node.track.id);
|
|
998
|
+
}
|
|
999
|
+
}
|
|
1000
|
+
|
|
1001
|
+
// ── Event loop ──
|
|
1002
|
+
// Each task is launched as soon as ALL its deps reach a terminal state.
|
|
1003
|
+
// We track in-flight tasks in `running` so a task completing mid-batch
|
|
1004
|
+
// immediately unblocks its dependents without waiting for sibling tasks.
|
|
1005
|
+
const running = new Map<string, Promise<void>>();
|
|
1006
|
+
|
|
1007
|
+
try {
|
|
1008
|
+
while (abortReason === null) {
|
|
1009
|
+
// Launch every task whose deps are all terminal and that isn't already in-flight
|
|
1010
|
+
for (const [id, state] of states) {
|
|
1011
|
+
if (state.status !== 'waiting' || running.has(id)) continue;
|
|
1012
|
+
const node = dag.nodes.get(id)!;
|
|
1013
|
+
const allDepsTerminal =
|
|
1014
|
+
node.dependsOn.length === 0 ||
|
|
1015
|
+
node.dependsOn.every((d) => isTerminal(states.get(d)!.status));
|
|
1016
|
+
if (!allDepsTerminal) continue;
|
|
1017
|
+
const p = processTask(id).finally(() => running.delete(id));
|
|
1018
|
+
running.set(id, p);
|
|
1019
|
+
}
|
|
1020
|
+
|
|
1021
|
+
// All tasks terminal — done
|
|
1022
|
+
if ([...states.values()].every((s) => isTerminal(s.status))) break;
|
|
1023
|
+
|
|
1024
|
+
if (running.size === 0) {
|
|
1025
|
+
// Nothing in-flight but non-terminal tasks exist (e.g. trigger-wait states
|
|
1026
|
+
// that processTask hasn't been called for yet). Poll briefly.
|
|
1027
|
+
await new Promise((r) => setTimeout(r, POLL_INTERVAL_MS));
|
|
1028
|
+
} else {
|
|
1029
|
+
// Wait for any one task to finish, then re-scan for new launchables.
|
|
1030
|
+
await Promise.race(running.values());
|
|
1031
|
+
}
|
|
1032
|
+
}
|
|
1033
|
+
|
|
1034
|
+
if (abortReason !== null) {
|
|
1035
|
+
// Wait for in-flight tasks to honour the abort signal before marking states.
|
|
1036
|
+
if (running.size > 0) await Promise.allSettled(running.values());
|
|
1037
|
+
for (const [id, state] of states) {
|
|
1038
|
+
if (!isTerminal(state.status)) {
|
|
1039
|
+
// By the time allSettled resolves, processTask's try/finally has already
|
|
1040
|
+
// set running tasks to success/failed/timeout. The only non-terminal
|
|
1041
|
+
// statuses remaining here are waiting/idle tasks that were never started.
|
|
1042
|
+
state.finishedAt = nowISO();
|
|
1043
|
+
setTaskStatus(id, 'skipped');
|
|
1044
|
+
}
|
|
1045
|
+
}
|
|
1046
|
+
}
|
|
1047
|
+
} finally {
|
|
1048
|
+
if (pipelineTimer) clearTimeout(pipelineTimer);
|
|
1049
|
+
// Clean up the external abort signal listener to prevent dead references
|
|
1050
|
+
// accumulating on long-lived shared AbortControllers.
|
|
1051
|
+
if (options.signal) {
|
|
1052
|
+
options.signal.removeEventListener('abort', externalAbortHandler);
|
|
1053
|
+
}
|
|
1054
|
+
// Safety net: drain any approvals still pending at shutdown (e.g. crash path).
|
|
1055
|
+
if (approvalGateway.pending().length > 0) {
|
|
1056
|
+
approvalGateway.abortAll('pipeline finished');
|
|
1057
|
+
}
|
|
1058
|
+
// Detach gateway → onEvent bridge so a long-lived gateway (host-supplied)
|
|
1059
|
+
// doesn't keep firing into a dead run.
|
|
1060
|
+
unsubscribeApprovals();
|
|
1061
|
+
}
|
|
1062
|
+
|
|
1063
|
+
// ── Summary ──
|
|
1064
|
+
const summary = { total: 0, success: 0, failed: 0, skipped: 0, timeout: 0, blocked: 0 };
|
|
1065
|
+
for (const [, state] of states) {
|
|
1066
|
+
summary.total++;
|
|
1067
|
+
switch (state.status) {
|
|
1068
|
+
case 'success':
|
|
1069
|
+
summary.success++;
|
|
1070
|
+
break;
|
|
1071
|
+
case 'failed':
|
|
1072
|
+
summary.failed++;
|
|
1073
|
+
break;
|
|
1074
|
+
case 'skipped':
|
|
1075
|
+
summary.skipped++;
|
|
1076
|
+
break;
|
|
1077
|
+
case 'timeout':
|
|
1078
|
+
summary.timeout++;
|
|
1079
|
+
break;
|
|
1080
|
+
case 'blocked':
|
|
1081
|
+
summary.blocked++;
|
|
1082
|
+
break;
|
|
1083
|
+
}
|
|
1084
|
+
}
|
|
1085
|
+
|
|
1086
|
+
const finishedAt = nowISO();
|
|
1087
|
+
const durationMs = new Date(finishedAt).getTime() - new Date(startedAt).getTime();
|
|
1088
|
+
|
|
1089
|
+
if (abortReason !== null) {
|
|
1090
|
+
const reasonText =
|
|
1091
|
+
abortReason === 'timeout'
|
|
1092
|
+
? 'Pipeline timeout exceeded'
|
|
1093
|
+
: abortReason === 'stop_all'
|
|
1094
|
+
? 'Pipeline stopped (on_failure: stop_all)'
|
|
1095
|
+
: 'Pipeline aborted by host';
|
|
1096
|
+
await executeHook(
|
|
1097
|
+
config.hooks,
|
|
1098
|
+
'pipeline_error',
|
|
1099
|
+
buildPipelineErrorContext(pipelineInfo, reasonText, undefined, abortReason),
|
|
1100
|
+
workDir,
|
|
1101
|
+
);
|
|
1102
|
+
} else {
|
|
1103
|
+
await executeHook(
|
|
1104
|
+
config.hooks,
|
|
1105
|
+
'pipeline_complete',
|
|
1106
|
+
buildPipelineCompleteContext(
|
|
1107
|
+
{ ...pipelineInfo, finished_at: finishedAt, duration_ms: durationMs },
|
|
1108
|
+
summary,
|
|
1109
|
+
),
|
|
1110
|
+
workDir,
|
|
1111
|
+
);
|
|
1112
|
+
}
|
|
1113
|
+
|
|
1114
|
+
const allSuccess =
|
|
1115
|
+
abortReason === null &&
|
|
1116
|
+
summary.failed === 0 &&
|
|
1117
|
+
summary.timeout === 0 &&
|
|
1118
|
+
summary.blocked === 0;
|
|
1119
|
+
|
|
1120
|
+
log.section('Pipeline summary');
|
|
1121
|
+
log.quiet(
|
|
1122
|
+
`status: ${abortReason !== null ? `aborted (${abortReason})` : 'completed'}`,
|
|
1123
|
+
);
|
|
1124
|
+
log.quiet(`duration: ${(durationMs / 1000).toFixed(1)}s`);
|
|
1125
|
+
log.quiet(
|
|
1126
|
+
`counts: total=${summary.total} success=${summary.success} ` +
|
|
1127
|
+
`failed=${summary.failed} skipped=${summary.skipped} ` +
|
|
1128
|
+
`timeout=${summary.timeout} blocked=${summary.blocked}`,
|
|
1129
|
+
);
|
|
1130
|
+
log.quiet('');
|
|
1131
|
+
log.quiet('per-task:');
|
|
1132
|
+
for (const [id, state] of states) {
|
|
1133
|
+
const dur =
|
|
1134
|
+
state.result?.durationMs != null ? `${(state.result.durationMs / 1000).toFixed(1)}s` : '-';
|
|
1135
|
+
const exit = state.result?.exitCode ?? '-';
|
|
1136
|
+
log.quiet(` ${state.status.padEnd(8)} ${id} (exit=${exit}, ${dur})`);
|
|
1137
|
+
}
|
|
1138
|
+
|
|
1139
|
+
log.info('[pipeline]', `completed "${config.name}"`);
|
|
1140
|
+
log.info(
|
|
1141
|
+
'[pipeline]',
|
|
1142
|
+
`Total: ${summary.total} | Success: ${summary.success} | Failed: ${summary.failed} | Skipped: ${summary.skipped} | Timeout: ${summary.timeout} | Blocked: ${summary.blocked}`,
|
|
1143
|
+
);
|
|
1144
|
+
log.info('[pipeline]', `Duration: ${(durationMs / 1000).toFixed(1)}s`);
|
|
1145
|
+
log.info('[pipeline]', `Log: ${log.path}`);
|
|
1146
|
+
|
|
1147
|
+
emit({ type: 'run_end', runId, success: allSuccess, abortReason });
|
|
1148
|
+
return { success: allSuccess, runId, logPath: log.path, summary, states: freezeStates(states) };
|
|
1149
|
+
} finally {
|
|
1150
|
+
// Close the persistent log file handle before pruning.
|
|
1151
|
+
log.close();
|
|
1152
|
+
// Prune old per-run log directories on every exit path (normal, blocked, or thrown).
|
|
1153
|
+
// Exclude the current runId so a concurrent run cannot delete its own live directory.
|
|
1154
|
+
if (maxLogRuns > 0) {
|
|
1155
|
+
await pruneLogDirs(resolve(workDir, '.tagma', 'logs'), maxLogRuns, runId);
|
|
1156
|
+
}
|
|
1157
|
+
}
|
|
1158
|
+
}
|
|
1159
|
+
|
|
1160
|
+
/**
|
|
1161
|
+
* Delete the oldest subdirectories under `logsDir`, keeping only the most recent `keep`
|
|
1162
|
+
* total runs (including the currently-live run identified by `excludeRunId`).
|
|
1163
|
+
* Directories are sorted lexicographically; because runIds are prefixed with a base-36
|
|
1164
|
+
* timestamp, lexicographic order equals chronological order.
|
|
1165
|
+
*
|
|
1166
|
+
* `excludeRunId` is always skipped from deletion even if it would otherwise be pruned —
|
|
1167
|
+
* this prevents a concurrent run from removing a live log directory that is still in use.
|
|
1168
|
+
*
|
|
1169
|
+
* D10: The live run occupies one slot out of `keep`, so the maximum number of
|
|
1170
|
+
* *historical* dirs to retain is `keep - 1`. Without this adjustment the function
|
|
1171
|
+
* kept `keep` historical dirs plus 1 live dir = `keep + 1` total on disk.
|
|
1172
|
+
*/
|
|
1173
|
+
async function pruneLogDirs(logsDir: string, keep: number, excludeRunId: string): Promise<void> {
|
|
1174
|
+
let entries: string[];
|
|
1175
|
+
try {
|
|
1176
|
+
entries = await readdir(logsDir);
|
|
1177
|
+
} catch {
|
|
1178
|
+
return; // logsDir doesn't exist yet — nothing to prune
|
|
1179
|
+
}
|
|
1180
|
+
|
|
1181
|
+
// Only consider directories that look like run IDs (run_<...>), excluding the live run.
|
|
1182
|
+
const runDirs = entries.filter((e) => e.startsWith('run_') && e !== excludeRunId).sort();
|
|
1183
|
+
// keep - 1 historical slots (1 slot is reserved for the live excludeRunId).
|
|
1184
|
+
const historyKeep = Math.max(0, keep - 1);
|
|
1185
|
+
const toDelete = runDirs.slice(0, Math.max(0, runDirs.length - historyKeep));
|
|
1186
|
+
|
|
1187
|
+
await Promise.all(
|
|
1188
|
+
toDelete.map((dir) =>
|
|
1189
|
+
rm(resolve(logsDir, dir), { recursive: true, force: true }).catch(() => {
|
|
1190
|
+
// Ignore deletion errors — stale dirs are better than a crash
|
|
1191
|
+
}),
|
|
1192
|
+
),
|
|
1193
|
+
);
|
|
1194
|
+
}
|
|
1195
|
+
|
|
1196
|
+
function isTerminal(status: TaskStatus): boolean {
|
|
1197
|
+
return (
|
|
1198
|
+
status === 'success' ||
|
|
1199
|
+
status === 'failed' ||
|
|
1200
|
+
status === 'timeout' ||
|
|
1201
|
+
status === 'skipped' ||
|
|
1202
|
+
status === 'blocked'
|
|
1203
|
+
);
|
|
1204
|
+
}
|
|
1205
|
+
|
|
1206
|
+
/** Return a deep-copied, caller-safe snapshot of the states map. */
|
|
1207
|
+
function freezeStates(states: Map<string, TaskState>): ReadonlyMap<string, TaskState> {
|
|
1208
|
+
const copy = new Map<string, TaskState>();
|
|
1209
|
+
for (const [id, s] of states) {
|
|
1210
|
+
copy.set(id, {
|
|
1211
|
+
config: { ...s.config },
|
|
1212
|
+
trackConfig: { ...s.trackConfig },
|
|
1213
|
+
status: s.status,
|
|
1214
|
+
result: s.result ? { ...s.result } : null,
|
|
1215
|
+
startedAt: s.startedAt,
|
|
1216
|
+
finishedAt: s.finishedAt,
|
|
1217
|
+
});
|
|
1218
|
+
}
|
|
1219
|
+
return copy;
|
|
1220
|
+
}
|