@async/pipeline 0.1.5 → 0.2.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +2 -0
- package/dist/cli.d.ts +1 -1
- package/dist/cli.d.ts.map +1 -1
- package/dist/cli.js +5 -1
- package/dist/cli.js.map +1 -1
- package/dist/internal/core/index.d.ts +77 -41
- package/dist/internal/core/index.d.ts.map +1 -1
- package/dist/internal/core/index.js +150 -26
- package/dist/internal/core/index.js.map +1 -1
- package/dist/internal/lima/index.d.ts +1 -1
- package/dist/internal/lima/index.d.ts.map +1 -1
- package/dist/internal/lima/index.js +1 -1
- package/dist/internal/lima/index.js.map +1 -1
- package/dist/internal/node/cli.d.ts +5 -2
- package/dist/internal/node/cli.d.ts.map +1 -1
- package/dist/internal/node/cli.js +396 -89
- package/dist/internal/node/cli.js.map +1 -1
- package/dist/internal/node/doctor.d.ts +1 -1
- package/dist/internal/node/doctor.d.ts.map +1 -1
- package/dist/internal/node/doctor.js +50 -1
- package/dist/internal/node/doctor.js.map +1 -1
- package/dist/internal/node/github.d.ts +2 -0
- package/dist/internal/node/github.d.ts.map +1 -1
- package/dist/internal/node/github.js +59 -11
- package/dist/internal/node/github.js.map +1 -1
- package/dist/internal/node/index.d.ts +1 -0
- package/dist/internal/node/index.d.ts.map +1 -1
- package/dist/internal/node/index.js +1 -0
- package/dist/internal/node/index.js.map +1 -1
- package/dist/internal/node/loader.d.ts.map +1 -1
- package/dist/internal/node/loader.js +4 -0
- package/dist/internal/node/loader.js.map +1 -1
- package/dist/internal/node/mcp.d.ts +16 -0
- package/dist/internal/node/mcp.d.ts.map +1 -0
- package/dist/internal/node/mcp.js +243 -0
- package/dist/internal/node/mcp.js.map +1 -0
- package/dist/internal/node/runner.d.ts +64 -49
- package/dist/internal/node/runner.d.ts.map +1 -1
- package/dist/internal/node/runner.js +648 -138
- package/dist/internal/node/runner.js.map +1 -1
- package/dist/internal/node/sources.d.ts +3 -0
- package/dist/internal/node/sources.d.ts.map +1 -1
- package/dist/internal/node/sources.js +24 -7
- package/dist/internal/node/sources.js.map +1 -1
- package/dist/internal/node/store.d.ts +86 -0
- package/dist/internal/node/store.d.ts.map +1 -1
- package/dist/internal/node/store.js +322 -24
- package/dist/internal/node/store.js.map +1 -1
- package/package.json +7 -5
|
@@ -1,38 +1,44 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
-
import { existsSync } from "node:fs";
|
|
3
|
-
import {
|
|
2
|
+
import { existsSync, realpathSync } from "node:fs";
|
|
3
|
+
import { readFile, readdir, rm } from "node:fs/promises";
|
|
4
|
+
import { basename, dirname, join, resolve } from "node:path";
|
|
4
5
|
import { pathToFileURL } from "node:url";
|
|
5
6
|
import { buildGraph, composePipelines, tasksForJob } from "../core/index.js";
|
|
6
7
|
import { runDoctor } from "./doctor.js";
|
|
7
8
|
import { checkGitHubWorkflow, jobsForGitHubEvent, readGitHubEventContext, renderGitHubWorkflow, writeGitHubWorkflow } from "./github.js";
|
|
8
9
|
import { loadPipeline } from "./loader.js";
|
|
9
|
-
import {
|
|
10
|
-
import {
|
|
10
|
+
import { beginShutdown, commandProxy, planJob, runJob, runSingleTask, shutdownExitCode } from "./runner.js";
|
|
11
|
+
import { runMcpServer } from "./mcp.js";
|
|
12
|
+
import { computeTaskInputManifest, createStore, diffInputManifests, pruneCacheEntries, readCacheInputManifest, readContextPacks, readTaskBaseline } from "./store.js";
|
|
11
13
|
import { matrixForJob, readPipelineMetadata, resolveSources, sourceContext } from "./sources.js";
|
|
12
14
|
import { checkTaskSync, describeTaskSync, renderTaskSync, writeTaskSync } from "./sync.js";
|
|
13
15
|
export async function runPipelineCli(options) {
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
};
|
|
19
|
-
const writeStderr = (text) => {
|
|
20
|
-
stderr += text;
|
|
21
|
-
};
|
|
16
|
+
const sinkStdout = options.stdout ?? ((text) => { process.stdout.write(text); });
|
|
17
|
+
const sinkStderr = options.stderr ?? ((text) => { process.stderr.write(text); });
|
|
18
|
+
let streamedStdout = 0;
|
|
19
|
+
let streamedStderr = 0;
|
|
22
20
|
const result = await runPipelineCliBuffered({
|
|
23
21
|
args: options.args,
|
|
24
|
-
|
|
22
|
+
cwd: options.cwd ?? process.cwd(),
|
|
23
|
+
env: options.env ?? process.env,
|
|
24
|
+
commands: options.commands,
|
|
25
25
|
program: options.program,
|
|
26
|
-
|
|
27
|
-
|
|
26
|
+
onStdout(text) {
|
|
27
|
+
streamedStdout += text.length;
|
|
28
|
+
sinkStdout(text);
|
|
29
|
+
},
|
|
30
|
+
onStderr(text) {
|
|
31
|
+
streamedStderr += text.length;
|
|
32
|
+
sinkStderr(text);
|
|
33
|
+
},
|
|
28
34
|
applyCommandPolicy: true
|
|
29
35
|
});
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
if (
|
|
33
|
-
|
|
34
|
-
if (
|
|
35
|
-
|
|
36
|
+
// Streamed text is always a prefix of the final result text; replaced output
|
|
37
|
+
// (command policy deny/mock, or errors) was never streamed, so flush the rest.
|
|
38
|
+
if (result.stdout.length > streamedStdout)
|
|
39
|
+
sinkStdout(result.stdout.slice(streamedStdout));
|
|
40
|
+
if (result.stderr.length > streamedStderr)
|
|
41
|
+
sinkStderr(result.stderr.slice(streamedStderr));
|
|
36
42
|
return result;
|
|
37
43
|
}
|
|
38
44
|
async function runPipelineCliBuffered(options) {
|
|
@@ -40,18 +46,20 @@ async function runPipelineCliBuffered(options) {
|
|
|
40
46
|
const parsed = parseGlobalOptions(options.args);
|
|
41
47
|
const [commandName, ...args] = parsed.args;
|
|
42
48
|
const program = options.program ?? programName();
|
|
43
|
-
const cwd = options.
|
|
49
|
+
const cwd = options.cwd;
|
|
44
50
|
const configPath = findPipelineConfig(cwd);
|
|
45
51
|
let stdout = "";
|
|
46
52
|
let stderr = "";
|
|
47
53
|
const out = (text) => {
|
|
48
54
|
stdout += text;
|
|
55
|
+
options.onStdout?.(text);
|
|
49
56
|
};
|
|
50
57
|
const err = (text) => {
|
|
51
58
|
stderr += text;
|
|
59
|
+
options.onStderr?.(text);
|
|
52
60
|
};
|
|
53
61
|
if (commandName === "doctor") {
|
|
54
|
-
const checks = await runDoctor();
|
|
62
|
+
const checks = await runDoctor(cwd);
|
|
55
63
|
for (const check of checks)
|
|
56
64
|
out(`${check.status.toUpperCase()} ${check.name}: ${check.message}\n`);
|
|
57
65
|
return { code: checks.some((check) => check.status === "fail") ? 1 : 0, stdout, stderr };
|
|
@@ -64,25 +72,36 @@ async function runPipelineCliBuffered(options) {
|
|
|
64
72
|
throw new Error(`No pipeline.ts, pipeline.mjs, or pipeline.js found in ${cwd}.`);
|
|
65
73
|
}
|
|
66
74
|
const pipeline = await loadPipeline(configPath);
|
|
67
|
-
const
|
|
68
|
-
|
|
69
|
-
|
|
75
|
+
const configDir = dirname(configPath);
|
|
76
|
+
// Validate the sandbox id before the command policy can mock the
|
|
77
|
+
// invocation, so typos fail loudly even under mocked CLI commands.
|
|
78
|
+
if (parsed.sandboxId && parsed.sandboxId !== "host" && !pipeline.sandboxes[parsed.sandboxId]) {
|
|
79
|
+
throw new Error(`Unknown sandbox "${parsed.sandboxId}". Declare it under \`sandboxes\` in the pipeline config.`);
|
|
80
|
+
}
|
|
81
|
+
const commands = options.commands ?? (pipeline.commands ? commandProxy(pipeline.commands) : undefined);
|
|
82
|
+
if (options.applyCommandPolicy && commands) {
|
|
83
|
+
return commands.run({
|
|
70
84
|
argv: ["async-pipeline", ...options.args],
|
|
71
|
-
cwd:
|
|
72
|
-
env:
|
|
85
|
+
cwd: configDir,
|
|
86
|
+
env: options.env
|
|
73
87
|
}, () => runPipelineCliBuffered({
|
|
74
88
|
...options,
|
|
75
89
|
args: options.args,
|
|
76
|
-
|
|
90
|
+
cwd: configDir,
|
|
91
|
+
commands,
|
|
77
92
|
applyCommandPolicy: false
|
|
78
93
|
}));
|
|
79
94
|
}
|
|
80
95
|
const context = {
|
|
81
96
|
concurrency: parsed.concurrency,
|
|
82
|
-
|
|
97
|
+
force: parsed.force,
|
|
98
|
+
dryRun: parsed.dryRun,
|
|
99
|
+
cwd: configDir,
|
|
83
100
|
configPath,
|
|
84
101
|
pipeline,
|
|
85
|
-
|
|
102
|
+
env: options.env,
|
|
103
|
+
commands,
|
|
104
|
+
sandboxId: parsed.sandboxId,
|
|
86
105
|
stdout: out,
|
|
87
106
|
stderr: err
|
|
88
107
|
};
|
|
@@ -119,17 +138,26 @@ async function dispatchCommand(commandName, args, context, program) {
|
|
|
119
138
|
return 0;
|
|
120
139
|
}
|
|
121
140
|
if (subcommand === "run") {
|
|
122
|
-
const
|
|
123
|
-
const
|
|
141
|
+
const explicitJobs = collectFlagValues(args.slice(1), "--job");
|
|
142
|
+
const eventContext = await readGitHubEventContext(context.env);
|
|
143
|
+
const jobs = explicitJobs.length > 0
|
|
144
|
+
? explicitJobs.map((jobId) => {
|
|
145
|
+
const selected = context.pipeline.jobs[jobId];
|
|
146
|
+
if (!selected)
|
|
147
|
+
throw new Error(`Unknown job "${jobId}".`);
|
|
148
|
+
return selected;
|
|
149
|
+
})
|
|
150
|
+
: jobsForGitHubEvent(context.pipeline, eventContext);
|
|
124
151
|
if (jobs.length === 0) {
|
|
125
|
-
context.stdout(`No pipeline jobs matched GitHub event "${eventContext.eventName}".\n`);
|
|
152
|
+
context.stdout(`No pipeline jobs matched GitHub event "${eventContext.eventName}". Jobs without a manual trigger need an explicit --job <id> on workflow_dispatch.\n`);
|
|
126
153
|
return 0;
|
|
127
154
|
}
|
|
128
155
|
let failed = false;
|
|
129
156
|
for (const selectedJob of jobs) {
|
|
130
157
|
const graph = tasksForJob(context.pipeline, selectedJob.id);
|
|
131
158
|
context.stdout(`Running ${context.pipeline.name}:${selectedJob.id} (${graph.executionOrder.join(" -> ")})\n`);
|
|
132
|
-
const result = await runJob(context.pipeline, { id: selectedJob.id, mode: "ci",
|
|
159
|
+
const result = await runJob(context.pipeline, { id: selectedJob.id, mode: "ci", cwd: context.cwd, env: context.env, commands: context.commands, sandbox: context.sandboxId, concurrency: context.concurrency });
|
|
160
|
+
reportFailedTasks(context, result.tasks);
|
|
133
161
|
context.stdout(`Pipeline ${result.status}: ${result.id}\n`);
|
|
134
162
|
if (result.status !== "passed")
|
|
135
163
|
failed = true;
|
|
@@ -176,14 +204,85 @@ async function dispatchCommand(commandName, args, context, program) {
|
|
|
176
204
|
throw new Error(`Unsupported graph format "${format}".`);
|
|
177
205
|
}
|
|
178
206
|
if (commandName === "explain") {
|
|
179
|
-
const taskId = args[0];
|
|
180
|
-
if (!taskId)
|
|
181
|
-
throw new Error(`Usage: ${program} explain <task>`);
|
|
182
207
|
const store = virtualStore(context.cwd);
|
|
208
|
+
const formatIndex = args.indexOf("--format");
|
|
209
|
+
const explainFormat = formatIndex >= 0 ? args[formatIndex + 1] : "text";
|
|
210
|
+
if (explainFormat !== "text" && explainFormat !== "json")
|
|
211
|
+
throw new Error(`Unsupported explain format "${explainFormat}".`);
|
|
212
|
+
const runIndex = args.indexOf("--run");
|
|
213
|
+
if (runIndex >= 0) {
|
|
214
|
+
const runId = args[runIndex + 1];
|
|
215
|
+
if (!runId || runId.startsWith("--"))
|
|
216
|
+
throw new Error(`Usage: ${program} explain --run <run-id> [--format json]`);
|
|
217
|
+
const packs = await readContextPacks(store, runId);
|
|
218
|
+
if (explainFormat === "json") {
|
|
219
|
+
context.stdout(`${JSON.stringify(packs, null, 2)}\n`);
|
|
220
|
+
return 0;
|
|
221
|
+
}
|
|
222
|
+
if (packs.length === 0) {
|
|
223
|
+
context.stdout(`No context packs recorded for run ${runId}.\n`);
|
|
224
|
+
return 0;
|
|
225
|
+
}
|
|
226
|
+
for (const pack of packs) {
|
|
227
|
+
context.stdout(`Task ${pack.task} failed after ${pack.attempts} attempt${pack.attempts === 1 ? "" : "s"}: ${pack.error}\n`);
|
|
228
|
+
if ("baselineMissing" in pack.inputDiff) {
|
|
229
|
+
context.stdout(" inputs: no passing baseline recorded\n");
|
|
230
|
+
}
|
|
231
|
+
else {
|
|
232
|
+
const diff = pack.inputDiff;
|
|
233
|
+
context.stdout(` inputs vs last pass: ${diff.changed.length} changed, ${diff.added.length} added, ${diff.removed.length} removed\n`);
|
|
234
|
+
for (const path of diff.changed)
|
|
235
|
+
context.stdout(` ~ ${path}\n`);
|
|
236
|
+
for (const path of diff.added)
|
|
237
|
+
context.stdout(` + ${path}\n`);
|
|
238
|
+
for (const path of diff.removed)
|
|
239
|
+
context.stdout(` - ${path}\n`);
|
|
240
|
+
}
|
|
241
|
+
if (pack.claims?.length)
|
|
242
|
+
context.stdout(` claims touched: ${pack.claims.join(", ")}\n`);
|
|
243
|
+
context.stdout(` reproduce: ${pack.reproduce}\n`);
|
|
244
|
+
}
|
|
245
|
+
return 0;
|
|
246
|
+
}
|
|
247
|
+
const taskId = args[0];
|
|
248
|
+
if (!taskId || taskId.startsWith("--"))
|
|
249
|
+
throw new Error(`Usage: ${program} explain <task> [--diff-inputs] | explain --run <run-id>`);
|
|
183
250
|
const explainPipeline = await loadAvailableSourceGraph(context.pipeline, context.cwd, store);
|
|
184
251
|
const task = explainPipeline.tasks[taskId];
|
|
185
252
|
if (!task)
|
|
186
253
|
throw new Error(`Unknown task "${taskId}".`);
|
|
254
|
+
if (args.includes("--diff-inputs")) {
|
|
255
|
+
const baseline = await readTaskBaseline(store, taskId);
|
|
256
|
+
const baselineManifest = baseline ? await readCacheInputManifest(store, baseline.cacheKey) : null;
|
|
257
|
+
const current = await computeTaskInputManifest(explainPipeline, task, context.cwd);
|
|
258
|
+
if (!baseline || !baselineManifest) {
|
|
259
|
+
if (explainFormat === "json") {
|
|
260
|
+
context.stdout(`${JSON.stringify({ task: taskId, baselineMissing: true, current }, null, 2)}\n`);
|
|
261
|
+
}
|
|
262
|
+
else {
|
|
263
|
+
context.stdout(`No passing baseline recorded for task "${taskId}" (${Object.keys(current.files).length} current input file(s)). Run the task to record one.\n`);
|
|
264
|
+
}
|
|
265
|
+
return 0;
|
|
266
|
+
}
|
|
267
|
+
const diff = diffInputManifests(baselineManifest, current);
|
|
268
|
+
if (explainFormat === "json") {
|
|
269
|
+
context.stdout(`${JSON.stringify({ task: taskId, baselineCacheKey: baseline.cacheKey, baselineRecordedAt: baseline.recordedAt, ...diff }, null, 2)}\n`);
|
|
270
|
+
return 0;
|
|
271
|
+
}
|
|
272
|
+
const total = diff.changed.length + diff.added.length + diff.removed.length;
|
|
273
|
+
context.stdout(`Inputs for "${taskId}" vs last passing entry (recorded ${baseline.recordedAt}):\n`);
|
|
274
|
+
if (total === 0) {
|
|
275
|
+
context.stdout(" unchanged\n");
|
|
276
|
+
return 0;
|
|
277
|
+
}
|
|
278
|
+
for (const path of diff.changed)
|
|
279
|
+
context.stdout(` ~ ${path}\n`);
|
|
280
|
+
for (const path of diff.added)
|
|
281
|
+
context.stdout(` + ${path}\n`);
|
|
282
|
+
for (const path of diff.removed)
|
|
283
|
+
context.stdout(` - ${path}\n`);
|
|
284
|
+
return 0;
|
|
285
|
+
}
|
|
187
286
|
context.stdout(`${JSON.stringify(task, jsonReplacer, 2)}\n`);
|
|
188
287
|
return 0;
|
|
189
288
|
}
|
|
@@ -201,6 +300,19 @@ async function dispatchCommand(commandName, args, context, program) {
|
|
|
201
300
|
context.stdout(`${JSON.stringify(metadata, jsonReplacer, 2)}\n`);
|
|
202
301
|
return 0;
|
|
203
302
|
}
|
|
303
|
+
if (commandName === "mcp") {
|
|
304
|
+
return runMcpServer({
|
|
305
|
+
pipeline: context.pipeline,
|
|
306
|
+
configPath: context.configPath,
|
|
307
|
+
cwd: context.cwd,
|
|
308
|
+
env: context.env,
|
|
309
|
+
store: virtualStore(context.cwd),
|
|
310
|
+
allowRun: args.includes("--allow-run"),
|
|
311
|
+
serverVersion: await ownPackageVersion(),
|
|
312
|
+
input: process.stdin,
|
|
313
|
+
write: (line) => context.stdout(`${line}\n`)
|
|
314
|
+
});
|
|
315
|
+
}
|
|
204
316
|
if (commandName === "matrix") {
|
|
205
317
|
const jobId = args[0];
|
|
206
318
|
if (!jobId)
|
|
@@ -216,22 +328,168 @@ async function dispatchCommand(commandName, args, context, program) {
|
|
|
216
328
|
const jobId = args[0];
|
|
217
329
|
if (!jobId)
|
|
218
330
|
throw new Error(`Usage: ${program} run <job>`);
|
|
331
|
+
const format = runOutputFormat(args, program);
|
|
219
332
|
const graph = tasksForJob(context.pipeline, jobId);
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
333
|
+
if (context.dryRun) {
|
|
334
|
+
return printDryRun(context, format, jobId);
|
|
335
|
+
}
|
|
336
|
+
if (format === "text")
|
|
337
|
+
context.stdout(`Running ${context.pipeline.name}:${jobId} (${graph.executionOrder.join(" -> ")})\n`);
|
|
338
|
+
const result = await runJob(context.pipeline, { id: jobId, mode: context.env.CI ? "ci" : "manual", cwd: context.cwd, env: context.env, commands: context.commands, sandbox: context.sandboxId, concurrency: context.concurrency, force: context.force, echo: format === "text" });
|
|
339
|
+
if (format === "json") {
|
|
340
|
+
context.stdout(`${JSON.stringify(result, null, 2)}\n`);
|
|
341
|
+
}
|
|
342
|
+
else {
|
|
343
|
+
reportFailedTasks(context, result.tasks);
|
|
344
|
+
context.stdout(`Pipeline ${result.status}: ${result.id}\n`);
|
|
345
|
+
}
|
|
346
|
+
await pruneRunsAfterRun(context);
|
|
223
347
|
return result.status === "passed" ? 0 : 1;
|
|
224
348
|
}
|
|
225
349
|
if (commandName === "run-task") {
|
|
226
350
|
const taskId = args[0];
|
|
227
351
|
if (!taskId)
|
|
228
352
|
throw new Error(`Usage: ${program} run-task <task>`);
|
|
229
|
-
const
|
|
230
|
-
context.
|
|
353
|
+
const format = runOutputFormat(args, program);
|
|
354
|
+
if (context.dryRun) {
|
|
355
|
+
return printDryRun(context, format, undefined, taskId);
|
|
356
|
+
}
|
|
357
|
+
const result = await runSingleTask(context.pipeline, taskId, { mode: context.env.CI ? "ci" : "manual", cwd: context.cwd, env: context.env, commands: context.commands, sandbox: context.sandboxId, concurrency: context.concurrency, force: context.force, echo: format === "text" });
|
|
358
|
+
if (format === "json") {
|
|
359
|
+
context.stdout(`${JSON.stringify(result, null, 2)}\n`);
|
|
360
|
+
}
|
|
361
|
+
else {
|
|
362
|
+
reportFailedTasks(context, result.tasks);
|
|
363
|
+
context.stdout(`Task run ${result.status}: ${result.id}\n`);
|
|
364
|
+
}
|
|
365
|
+
await pruneRunsAfterRun(context);
|
|
231
366
|
return result.status === "passed" ? 0 : 1;
|
|
232
367
|
}
|
|
368
|
+
if (commandName === "cache") {
|
|
369
|
+
const subcommand = args[0];
|
|
370
|
+
if (subcommand === "clear") {
|
|
371
|
+
await rm(join(context.cwd, ".async", "cache", "tasks"), { recursive: true, force: true });
|
|
372
|
+
context.stdout("Cleared task cache.\n");
|
|
373
|
+
return 0;
|
|
374
|
+
}
|
|
375
|
+
throw new Error(`Unknown cache command "${subcommand ?? ""}". Use: ${program} cache clear`);
|
|
376
|
+
}
|
|
377
|
+
if (commandName === "gc") {
|
|
378
|
+
const keep = parsePositiveInteger(args, "--keep", 20);
|
|
379
|
+
const cacheDays = parsePositiveInteger(args, "--cache-days", 30);
|
|
380
|
+
const removed = await pruneRuns(context.cwd, keep);
|
|
381
|
+
const remaining = await countRuns(context.cwd);
|
|
382
|
+
const removedCacheEntries = await pruneCacheEntries(context.cwd, cacheDays);
|
|
383
|
+
context.stdout(`Removed ${removed} run record${removed === 1 ? "" : "s"}; kept ${remaining}.\n`);
|
|
384
|
+
context.stdout(cacheDays > 0
|
|
385
|
+
? `Removed ${removedCacheEntries} cache entr${removedCacheEntries === 1 ? "y" : "ies"} unused for ${cacheDays}+ days.\n`
|
|
386
|
+
: "Cache pruning disabled (--cache-days 0).\n");
|
|
387
|
+
return 0;
|
|
388
|
+
}
|
|
233
389
|
throw new Error(`Unknown command "${commandName}".`);
|
|
234
390
|
}
|
|
391
|
+
async function printDryRun(context, format, jobId, taskId) {
|
|
392
|
+
let pipeline = context.pipeline;
|
|
393
|
+
let id = jobId;
|
|
394
|
+
if (taskId !== undefined) {
|
|
395
|
+
id = `task:${taskId}`;
|
|
396
|
+
pipeline = {
|
|
397
|
+
...pipeline,
|
|
398
|
+
jobs: { ...pipeline.jobs, [id]: { id, target: [taskId], trigger: [] } }
|
|
399
|
+
};
|
|
400
|
+
}
|
|
401
|
+
if (!id)
|
|
402
|
+
throw new Error("Dry run requires a job or task.");
|
|
403
|
+
const plan = await planJob(pipeline, { id, cwd: context.cwd, env: context.env, sandbox: context.sandboxId });
|
|
404
|
+
if (format === "json") {
|
|
405
|
+
context.stdout(`${JSON.stringify(plan, null, 2)}\n`);
|
|
406
|
+
return 0;
|
|
407
|
+
}
|
|
408
|
+
context.stdout(`Plan ${context.pipeline.name}:${jobId ?? taskId} (${plan.executionOrder.join(" -> ")})\n`);
|
|
409
|
+
for (const entry of plan.entries) {
|
|
410
|
+
const note = context.force && entry.predicted === "cached"
|
|
411
|
+
? "run (cache ignored by --force)"
|
|
412
|
+
: entry.predicted;
|
|
413
|
+
context.stdout(` ${entry.id}\t${note}${entry.reason ? `\t${entry.reason}` : ""}\n`);
|
|
414
|
+
}
|
|
415
|
+
context.stdout("Dry run: no tasks executed. Cached predictions do not verify restored output files.\n");
|
|
416
|
+
return 0;
|
|
417
|
+
}
|
|
418
|
+
function runOutputFormat(args, program) {
|
|
419
|
+
const formatIndex = args.indexOf("--format");
|
|
420
|
+
if (formatIndex < 0)
|
|
421
|
+
return "text";
|
|
422
|
+
const format = args[formatIndex + 1];
|
|
423
|
+
if (format !== "text" && format !== "json") {
|
|
424
|
+
throw new Error(`Usage: ${program} run <job> --format text|json`);
|
|
425
|
+
}
|
|
426
|
+
return format;
|
|
427
|
+
}
|
|
428
|
+
const DEFAULT_KEPT_RUNS = 50;
|
|
429
|
+
async function pruneRunsAfterRun(context) {
|
|
430
|
+
const raw = context.env.ASYNC_PIPELINE_KEEP_RUNS;
|
|
431
|
+
let keep = DEFAULT_KEPT_RUNS;
|
|
432
|
+
if (raw !== undefined && raw !== "") {
|
|
433
|
+
const parsed = Number(raw);
|
|
434
|
+
if (!Number.isInteger(parsed) || parsed < 0)
|
|
435
|
+
return;
|
|
436
|
+
if (parsed === 0)
|
|
437
|
+
return; // 0 disables automatic pruning.
|
|
438
|
+
keep = parsed;
|
|
439
|
+
}
|
|
440
|
+
await pruneRuns(context.cwd, keep);
|
|
441
|
+
}
|
|
442
|
+
async function pruneRuns(cwd, keep) {
|
|
443
|
+
const runsDir = join(cwd, ".async", "runs");
|
|
444
|
+
let runIds = [];
|
|
445
|
+
try {
|
|
446
|
+
runIds = (await readdir(runsDir, { withFileTypes: true }))
|
|
447
|
+
.filter((entry) => entry.isDirectory())
|
|
448
|
+
.map((entry) => entry.name)
|
|
449
|
+
.sort((left, right) => right.localeCompare(left));
|
|
450
|
+
}
|
|
451
|
+
catch {
|
|
452
|
+
return 0;
|
|
453
|
+
}
|
|
454
|
+
const stale = runIds.slice(keep);
|
|
455
|
+
for (const runId of stale) {
|
|
456
|
+
await rm(join(runsDir, runId), { recursive: true, force: true });
|
|
457
|
+
}
|
|
458
|
+
return stale.length;
|
|
459
|
+
}
|
|
460
|
+
async function countRuns(cwd) {
|
|
461
|
+
try {
|
|
462
|
+
const entries = await readdir(join(cwd, ".async", "runs"), { withFileTypes: true });
|
|
463
|
+
return entries.filter((entry) => entry.isDirectory()).length;
|
|
464
|
+
}
|
|
465
|
+
catch {
|
|
466
|
+
return 0;
|
|
467
|
+
}
|
|
468
|
+
}
|
|
469
|
+
function collectFlagValues(args, flag) {
|
|
470
|
+
const values = [];
|
|
471
|
+
for (let index = 0; index < args.length; index += 1) {
|
|
472
|
+
if (args[index] !== flag)
|
|
473
|
+
continue;
|
|
474
|
+
const value = args[index + 1];
|
|
475
|
+
if (!value)
|
|
476
|
+
throw new Error(`${flag} requires a value.`);
|
|
477
|
+
values.push(value);
|
|
478
|
+
index += 1;
|
|
479
|
+
}
|
|
480
|
+
return values;
|
|
481
|
+
}
|
|
482
|
+
function parsePositiveInteger(args, flag, fallback) {
|
|
483
|
+
const index = args.indexOf(flag);
|
|
484
|
+
if (index < 0)
|
|
485
|
+
return fallback;
|
|
486
|
+
const raw = args[index + 1];
|
|
487
|
+
const value = Number(raw);
|
|
488
|
+
if (!Number.isInteger(value) || value < 0) {
|
|
489
|
+
throw new Error(`${flag} requires a non-negative integer.`);
|
|
490
|
+
}
|
|
491
|
+
return value;
|
|
492
|
+
}
|
|
235
493
|
async function handleSourcesCommand(args, context) {
|
|
236
494
|
const subcommand = args[0] ?? "list";
|
|
237
495
|
if (subcommand === "list") {
|
|
@@ -256,11 +514,13 @@ async function handleSourcesCommand(args, context) {
|
|
|
256
514
|
}
|
|
257
515
|
function printHelp(program) {
|
|
258
516
|
return `Usage:
|
|
259
|
-
${program} run <job> [--
|
|
260
|
-
${program} run-task <task> [--
|
|
517
|
+
${program} run <job> [--sandbox <id>] [--concurrency <n>] [--force] [--dry-run] [--format text|json]
|
|
518
|
+
${program} run-task <task> [--sandbox <id>] [--concurrency <n>] [--force] [--dry-run] [--format text|json]
|
|
261
519
|
${program} list
|
|
262
520
|
${program} graph --format json|dot
|
|
263
|
-
${program} explain <task>
|
|
521
|
+
${program} explain <task> [--diff-inputs] [--format text|json]
|
|
522
|
+
${program} explain --run <run-id> [--format text|json]
|
|
523
|
+
${program} mcp [--allow-run]
|
|
264
524
|
${program} sources list
|
|
265
525
|
${program} sources sync
|
|
266
526
|
${program} metadata --format json [--include-sources]
|
|
@@ -276,7 +536,9 @@ function printHelp(program) {
|
|
|
276
536
|
${program} sync tasks check
|
|
277
537
|
${program} github generate [--workflow <path>] [--lock <path>]
|
|
278
538
|
${program} github check [--workflow <path>] [--lock <path>]
|
|
279
|
-
${program} github run [--
|
|
539
|
+
${program} github run [--job <id>] [--sandbox <id>] [--concurrency <n>]
|
|
540
|
+
${program} cache clear
|
|
541
|
+
${program} gc [--keep <n>] [--cache-days <n>]
|
|
280
542
|
${program} doctor\n`;
|
|
281
543
|
}
|
|
282
544
|
async function handleSyncCommand(args, context) {
|
|
@@ -405,15 +667,25 @@ async function handleSyncTasksCommand(subcommand, context, options) {
|
|
|
405
667
|
function parseGlobalOptions(args) {
|
|
406
668
|
const rest = [];
|
|
407
669
|
let concurrency;
|
|
408
|
-
let
|
|
670
|
+
let sandboxId;
|
|
671
|
+
let force = false;
|
|
672
|
+
let dryRun = false;
|
|
409
673
|
for (let index = 0; index < args.length; index += 1) {
|
|
410
674
|
const arg = args[index];
|
|
411
675
|
if (arg === undefined)
|
|
412
676
|
continue;
|
|
413
|
-
if (arg === "--
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
677
|
+
if (arg === "--force") {
|
|
678
|
+
force = true;
|
|
679
|
+
continue;
|
|
680
|
+
}
|
|
681
|
+
if (arg === "--dry-run") {
|
|
682
|
+
dryRun = true;
|
|
683
|
+
continue;
|
|
684
|
+
}
|
|
685
|
+
if (arg === "--sandbox") {
|
|
686
|
+
sandboxId = args[index + 1];
|
|
687
|
+
if (!sandboxId)
|
|
688
|
+
throw new Error("Usage: async-pipeline <command> --sandbox <id>");
|
|
417
689
|
index += 1;
|
|
418
690
|
continue;
|
|
419
691
|
}
|
|
@@ -427,52 +699,56 @@ function parseGlobalOptions(args) {
|
|
|
427
699
|
}
|
|
428
700
|
rest.push(arg);
|
|
429
701
|
}
|
|
430
|
-
return { args: rest, concurrency,
|
|
431
|
-
}
|
|
432
|
-
function selectWorkspace(workspaceId, pipeline, base) {
|
|
433
|
-
const commands = base.commands ?? (pipeline.commands ? commandProxy(pipeline.commands) : undefined);
|
|
434
|
-
if (!workspaceId || workspaceId === "host") {
|
|
435
|
-
return { ...base, commands };
|
|
436
|
-
}
|
|
437
|
-
const definition = pipeline.workspaces[workspaceId];
|
|
438
|
-
if (!definition)
|
|
439
|
-
throw new Error(`Unknown workspace "${workspaceId}".`);
|
|
440
|
-
return createWorkspaceFromDefinition(definition, base, commands);
|
|
441
|
-
}
|
|
442
|
-
function createWorkspaceFromDefinition(definition, base, commands) {
|
|
443
|
-
if (definition.kind === "host")
|
|
444
|
-
return hostWorkspace({ cwd: base.cwd, env: base.env, commands });
|
|
445
|
-
if (definition.kind === "lima")
|
|
446
|
-
return limaWorkspace({ cwd: base.cwd, env: base.env, vm: definition.vm, commands });
|
|
447
|
-
if (definition.kind === "docker") {
|
|
448
|
-
return dockerWorkspace({
|
|
449
|
-
cwd: base.cwd,
|
|
450
|
-
env: base.env,
|
|
451
|
-
image: definition.image,
|
|
452
|
-
workdir: definition.workdir,
|
|
453
|
-
volumes: definition.volumes,
|
|
454
|
-
commands
|
|
455
|
-
});
|
|
456
|
-
}
|
|
457
|
-
throw new Error(`Workspace kind "${definition.kind}" is declared but not executable in this tranche.`);
|
|
702
|
+
return { args: rest, concurrency, sandboxId, force, dryRun };
|
|
458
703
|
}
|
|
459
704
|
function findPipelineConfig(cwd) {
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
705
|
+
let current = resolve(cwd);
|
|
706
|
+
for (;;) {
|
|
707
|
+
for (const fileName of ["pipeline.ts", "pipeline.mjs", "pipeline.js"]) {
|
|
708
|
+
const configPath = join(current, fileName);
|
|
709
|
+
if (existsSync(configPath))
|
|
710
|
+
return configPath;
|
|
711
|
+
}
|
|
712
|
+
const parent = dirname(current);
|
|
713
|
+
if (parent === current)
|
|
714
|
+
return null;
|
|
715
|
+
current = parent;
|
|
464
716
|
}
|
|
465
|
-
return null;
|
|
466
717
|
}
|
|
467
718
|
function programName() {
|
|
468
719
|
const name = basename(process.argv[1] ?? "async-pipeline");
|
|
469
720
|
return name === "cli.js" ? "async-pipeline" : name;
|
|
470
721
|
}
|
|
722
|
+
// Failed runs must name the reason on the terminal, not only inside
|
|
723
|
+
// .async/runs/<id>/execution.json. Task output was already streamed; this
|
|
724
|
+
// repeats the recorded per-task error next to the final status line.
|
|
725
|
+
function reportFailedTasks(context, tasks) {
|
|
726
|
+
for (const failed of tasks.filter((task) => task.status === "failed")) {
|
|
727
|
+
context.stderr(`Task ${failed.id} failed${failed.error ? `: ${failed.error}` : ""}\n`);
|
|
728
|
+
}
|
|
729
|
+
}
|
|
471
730
|
function jsonReplacer(_key, value) {
|
|
472
731
|
if (typeof value === "function")
|
|
473
732
|
return "[function]";
|
|
474
733
|
return value;
|
|
475
734
|
}
|
|
735
|
+
/**
|
|
736
|
+
* The published package's version for MCP serverInfo, falling back to this
|
|
737
|
+
* internal package's own. Best-effort: identity metadata, not behavior.
|
|
738
|
+
*/
|
|
739
|
+
async function ownPackageVersion() {
|
|
740
|
+
for (const candidate of ["../../pipeline/package.json", "../package.json"]) {
|
|
741
|
+
try {
|
|
742
|
+
const manifest = JSON.parse(await readFile(new URL(candidate, import.meta.url), "utf8"));
|
|
743
|
+
if (typeof manifest.version === "string")
|
|
744
|
+
return manifest.version;
|
|
745
|
+
}
|
|
746
|
+
catch {
|
|
747
|
+
// Try the next candidate.
|
|
748
|
+
}
|
|
749
|
+
}
|
|
750
|
+
return undefined;
|
|
751
|
+
}
|
|
476
752
|
function githubGenerationPaths(args) {
|
|
477
753
|
const workflowIndex = args.indexOf("--workflow");
|
|
478
754
|
const lockIndex = args.indexOf("--lock");
|
|
@@ -506,12 +782,43 @@ function virtualStore(root) {
|
|
|
506
782
|
sourcesDir: resolve(root, ".async", "sources")
|
|
507
783
|
};
|
|
508
784
|
}
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
785
|
+
export function runCliMain() {
|
|
786
|
+
// When the downstream pipe closes (e.g. `async-pipeline run x | head`),
|
|
787
|
+
// terminate running task process groups, let the run finalize its
|
|
788
|
+
// execution record, and exit 141 (128 + SIGPIPE) instead of crashing
|
|
789
|
+
// with an unhandled EPIPE or orphaning task processes.
|
|
790
|
+
const shutdownOnEpipe = (error) => {
|
|
791
|
+
if (error.code === "EPIPE") {
|
|
792
|
+
beginShutdown("SIGTERM", 141);
|
|
793
|
+
return;
|
|
794
|
+
}
|
|
795
|
+
throw error;
|
|
796
|
+
};
|
|
797
|
+
process.stdout.on("error", shutdownOnEpipe);
|
|
798
|
+
process.stderr.on("error", shutdownOnEpipe);
|
|
799
|
+
return runPipelineCli({ args: process.argv.slice(2) }).then((result) => {
|
|
800
|
+
process.exitCode = shutdownExitCode() ?? result.code;
|
|
512
801
|
}).catch((error) => {
|
|
513
802
|
console.error(error instanceof Error ? error.message : String(error));
|
|
514
|
-
process.exitCode = 1;
|
|
803
|
+
process.exitCode = shutdownExitCode() ?? 1;
|
|
515
804
|
});
|
|
516
805
|
}
|
|
806
|
+
// True only when this module is the executed entrypoint. The public package
|
|
807
|
+
// bin (`@async/pipeline` dist/cli.js) is a different wrapper module, so it
|
|
808
|
+
// cannot rely on this guard; it imports and calls runCliMain() explicitly.
|
|
809
|
+
// Realpath the argv path so bin shims and node_modules symlinks still count
|
|
810
|
+
// as direct execution of this file.
|
|
811
|
+
function isCliEntrypoint(argvPath) {
|
|
812
|
+
if (!argvPath)
|
|
813
|
+
return false;
|
|
814
|
+
try {
|
|
815
|
+
return pathToFileURL(realpathSync(argvPath)).href === import.meta.url;
|
|
816
|
+
}
|
|
817
|
+
catch {
|
|
818
|
+
return false;
|
|
819
|
+
}
|
|
820
|
+
}
|
|
821
|
+
if (isCliEntrypoint(process.argv[1])) {
|
|
822
|
+
void runCliMain();
|
|
823
|
+
}
|
|
517
824
|
//# sourceMappingURL=cli.js.map
|