@async/pipeline 0.1.3 → 0.1.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -3
- package/dist/internal/core/cache.d.ts +5 -1
- package/dist/internal/core/cache.d.ts.map +1 -1
- package/dist/internal/core/cache.js +19 -9
- package/dist/internal/core/cache.js.map +1 -1
- package/dist/internal/core/index.d.ts +121 -3
- package/dist/internal/core/index.d.ts.map +1 -1
- package/dist/internal/core/index.js +105 -8
- package/dist/internal/core/index.js.map +1 -1
- package/dist/internal/core/runtime.js +1 -1
- package/dist/internal/core/runtime.js.map +1 -1
- package/dist/internal/lima/index.d.ts +1 -14
- package/dist/internal/lima/index.d.ts.map +1 -1
- package/dist/internal/lima/index.js +1 -64
- package/dist/internal/lima/index.js.map +1 -1
- package/dist/internal/node/cli.d.ts +9 -1
- package/dist/internal/node/cli.d.ts.map +1 -1
- package/dist/internal/node/cli.js +279 -172
- package/dist/internal/node/cli.js.map +1 -1
- package/dist/internal/node/doctor.js +2 -2
- package/dist/internal/node/doctor.js.map +1 -1
- package/dist/internal/node/github.d.ts +3 -1
- package/dist/internal/node/github.d.ts.map +1 -1
- package/dist/internal/node/github.js +31 -9
- package/dist/internal/node/github.js.map +1 -1
- package/dist/internal/node/index.d.ts +1 -0
- package/dist/internal/node/index.d.ts.map +1 -1
- package/dist/internal/node/index.js +1 -0
- package/dist/internal/node/index.js.map +1 -1
- package/dist/internal/node/runner.d.ts +98 -7
- package/dist/internal/node/runner.d.ts.map +1 -1
- package/dist/internal/node/runner.js +400 -73
- package/dist/internal/node/runner.js.map +1 -1
- package/dist/internal/node/store.d.ts +29 -5
- package/dist/internal/node/store.d.ts.map +1 -1
- package/dist/internal/node/store.js +225 -20
- package/dist/internal/node/store.js.map +1 -1
- package/package.json +1 -1
|
@@ -1,10 +1,12 @@
|
|
|
1
1
|
import { spawn } from "node:child_process";
|
|
2
2
|
import { randomUUID } from "node:crypto";
|
|
3
|
+
import { availableParallelism } from "node:os";
|
|
4
|
+
import { posix, relative } from "node:path";
|
|
3
5
|
import { setTimeout as delay } from "node:timers/promises";
|
|
4
6
|
import { sh, tasksForJob } from "../core/index.js";
|
|
5
|
-
import { computeTaskCacheKey, createStore, readCacheEntry, writeCacheEntry, writeExecution, writeTaskLog } from "./store.js";
|
|
7
|
+
import { computeTaskCacheKey, createStore, outputFilesExist, readCacheEntry, resolveOutputFiles, restoreCacheOutputs, writeCacheEntry, writeExecution, writeTaskLog } from "./store.js";
|
|
6
8
|
import { createRunPlan, sourceContext } from "./sources.js";
|
|
7
|
-
export class
|
|
9
|
+
export class HostCommandExecutor {
|
|
8
10
|
name = "host";
|
|
9
11
|
runShell(command, options) {
|
|
10
12
|
return runProcess(command, { cwd: options.cwd, env: options.env, timeoutMs: options.timeoutMs });
|
|
@@ -14,94 +16,297 @@ export class HostRunnerAdapter {
|
|
|
14
16
|
return result.code === 0;
|
|
15
17
|
}
|
|
16
18
|
}
|
|
19
|
+
export class DockerCommandExecutor {
|
|
20
|
+
options;
|
|
21
|
+
name = "docker";
|
|
22
|
+
constructor(options) {
|
|
23
|
+
this.options = options;
|
|
24
|
+
}
|
|
25
|
+
runShell(command, options) {
|
|
26
|
+
return runProcess(this.dockerCommand(command, options.cwd, options.env), { cwd: this.options.hostCwd, env: options.env, timeoutMs: options.timeoutMs });
|
|
27
|
+
}
|
|
28
|
+
async checkTool(tool) {
|
|
29
|
+
const result = await runProcess(this.dockerCommand(`command -v ${shellEscape(tool)}`, this.options.hostCwd, process.env), {
|
|
30
|
+
cwd: this.options.hostCwd,
|
|
31
|
+
env: process.env,
|
|
32
|
+
echo: false
|
|
33
|
+
});
|
|
34
|
+
return result.code === 0;
|
|
35
|
+
}
|
|
36
|
+
dockerCommand(command, cwd, env) {
|
|
37
|
+
return [
|
|
38
|
+
"docker",
|
|
39
|
+
"run",
|
|
40
|
+
"--rm",
|
|
41
|
+
"-w",
|
|
42
|
+
shellEscape(this.containerCwd(cwd)),
|
|
43
|
+
...this.options.volumes.flatMap((volume) => ["-v", shellEscape(`${volume.source}:${volume.target}${volume.readonly ? ":ro" : ""}`)]),
|
|
44
|
+
...Object.keys(env).filter((key) => env[key] !== undefined).flatMap((key) => ["-e", shellEscape(key)]),
|
|
45
|
+
shellEscape(this.options.image),
|
|
46
|
+
"bash",
|
|
47
|
+
"-lc",
|
|
48
|
+
shellEscape(command)
|
|
49
|
+
].join(" ");
|
|
50
|
+
}
|
|
51
|
+
containerCwd(cwd) {
|
|
52
|
+
const rel = relative(this.options.hostCwd, cwd);
|
|
53
|
+
if (!rel || rel === ".")
|
|
54
|
+
return this.options.workdir;
|
|
55
|
+
if (rel.startsWith(".."))
|
|
56
|
+
return this.options.workdir;
|
|
57
|
+
return posix.join(this.options.workdir, rel.split(/[\\/]+/).join("/"));
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
export class LimaCommandExecutor {
|
|
61
|
+
vm;
|
|
62
|
+
name = "lima";
|
|
63
|
+
constructor(vm = "async-pipeline") {
|
|
64
|
+
this.vm = vm;
|
|
65
|
+
}
|
|
66
|
+
runShell(command, options) {
|
|
67
|
+
const vm = options.task.environment?.vm ?? this.vm;
|
|
68
|
+
const escaped = shellEscape(`cd ${shellEscape(options.cwd)} && ${command}`);
|
|
69
|
+
return runProcess(`limactl shell ${shellEscape(vm)} -- bash -lc ${escaped}`, { cwd: options.cwd, env: options.env, timeoutMs: options.timeoutMs });
|
|
70
|
+
}
|
|
71
|
+
async checkTool(tool) {
|
|
72
|
+
const result = await runProcess(`limactl shell ${shellEscape(this.vm)} -- bash -lc ${shellEscape(`command -v ${shellEscape(tool)}`)}`, {
|
|
73
|
+
cwd: process.cwd(),
|
|
74
|
+
env: process.env
|
|
75
|
+
});
|
|
76
|
+
return result.code === 0;
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
export function hostWorkspace(options = {}) {
|
|
80
|
+
return {
|
|
81
|
+
cwd: options.cwd ?? process.cwd(),
|
|
82
|
+
env: options.env ?? process.env,
|
|
83
|
+
fs: { kind: "host" },
|
|
84
|
+
executor: options.executor ?? new HostCommandExecutor(),
|
|
85
|
+
commands: options.commands
|
|
86
|
+
};
|
|
87
|
+
}
|
|
88
|
+
export function dockerWorkspace(options) {
|
|
89
|
+
const cwd = options.cwd ?? process.cwd();
|
|
90
|
+
const workdir = options.workdir ?? "/workspace";
|
|
91
|
+
return hostWorkspace({
|
|
92
|
+
cwd,
|
|
93
|
+
env: options.env,
|
|
94
|
+
executor: new DockerCommandExecutor({
|
|
95
|
+
image: options.image,
|
|
96
|
+
hostCwd: cwd,
|
|
97
|
+
workdir,
|
|
98
|
+
volumes: options.volumes ?? [{ source: cwd, target: workdir }]
|
|
99
|
+
}),
|
|
100
|
+
commands: options.commands
|
|
101
|
+
});
|
|
102
|
+
}
|
|
103
|
+
export function limaWorkspace(options = {}) {
|
|
104
|
+
return hostWorkspace({
|
|
105
|
+
cwd: options.cwd,
|
|
106
|
+
env: options.env,
|
|
107
|
+
executor: new LimaCommandExecutor(options.vm),
|
|
108
|
+
commands: options.commands
|
|
109
|
+
});
|
|
110
|
+
}
|
|
111
|
+
export function commandProxy(policy = { rules: [] }) {
|
|
112
|
+
const records = [];
|
|
113
|
+
return {
|
|
114
|
+
async run(invocation, next) {
|
|
115
|
+
const startedAt = new Date().toISOString();
|
|
116
|
+
const started = Date.now();
|
|
117
|
+
const action = matchingAction(policy, invocation.argv);
|
|
118
|
+
const status = commandStatus(action);
|
|
119
|
+
const result = await runCommandAction(action, next);
|
|
120
|
+
const outputPolicy = action.output ?? policy.output ?? {};
|
|
121
|
+
if (policy.record) {
|
|
122
|
+
records.push({
|
|
123
|
+
argv: [...invocation.argv],
|
|
124
|
+
cwd: invocation.cwd,
|
|
125
|
+
status,
|
|
126
|
+
code: result.code,
|
|
127
|
+
stdout: applyOutputPolicy(result.stdout, outputPolicy, invocation.env),
|
|
128
|
+
stderr: applyOutputPolicy(result.stderr, outputPolicy, invocation.env),
|
|
129
|
+
startedAt,
|
|
130
|
+
finishedAt: new Date().toISOString(),
|
|
131
|
+
durationMs: Date.now() - started
|
|
132
|
+
});
|
|
133
|
+
}
|
|
134
|
+
return result;
|
|
135
|
+
},
|
|
136
|
+
records() {
|
|
137
|
+
return records.map((record) => ({
|
|
138
|
+
...record,
|
|
139
|
+
argv: [...record.argv]
|
|
140
|
+
}));
|
|
141
|
+
}
|
|
142
|
+
};
|
|
143
|
+
}
|
|
17
144
|
const memoryCacheEntries = new Map();
|
|
145
|
+
const DEFAULT_MAX_CONCURRENCY = 4;
|
|
18
146
|
export async function runJob(pipeline, options) {
|
|
19
|
-
const
|
|
20
|
-
const store = await createStore(
|
|
21
|
-
const plan = await createRunPlan(pipeline,
|
|
22
|
-
const graph = tasksForJob(plan.pipeline, options.
|
|
147
|
+
const workspace = options.workspace ?? hostWorkspace();
|
|
148
|
+
const store = await createStore(workspace.cwd);
|
|
149
|
+
const plan = await createRunPlan(pipeline, workspace.cwd, store);
|
|
150
|
+
const graph = tasksForJob(plan.pipeline, options.id);
|
|
23
151
|
const record = {
|
|
24
152
|
id: `${new Date().toISOString().replaceAll(/[:.]/g, "-")}-${randomUUID().slice(0, 8)}`,
|
|
25
153
|
pipelineName: plan.pipeline.name,
|
|
26
|
-
jobId: options.
|
|
27
|
-
cwd:
|
|
154
|
+
jobId: options.id,
|
|
155
|
+
cwd: workspace.cwd,
|
|
28
156
|
startedAt: new Date().toISOString(),
|
|
29
157
|
status: "running",
|
|
30
|
-
mode: options.mode ?? "manual",
|
|
158
|
+
mode: options.mode ?? (workspace.env.CI ? "ci" : "manual"),
|
|
31
159
|
tasks: [],
|
|
32
160
|
sources: Object.fromEntries(Object.entries(plan.sources).map(([sourceId, resolved]) => [sourceId, resolved.record]))
|
|
33
161
|
};
|
|
34
162
|
await writeExecution(store, record);
|
|
35
|
-
const
|
|
36
|
-
const jobDefinition = plan.pipeline.jobs[options.jobId];
|
|
163
|
+
const jobDefinition = plan.pipeline.jobs[options.id];
|
|
37
164
|
const envDefinitions = {
|
|
38
165
|
...plan.pipeline.env,
|
|
39
166
|
...(jobDefinition?.env ?? {})
|
|
40
167
|
};
|
|
41
|
-
|
|
168
|
+
const taskFingerprints = new Map();
|
|
169
|
+
const concurrency = normalizeTaskConcurrency(options.concurrency);
|
|
170
|
+
const graphIndex = new Map(graph.executionOrder.map((taskId, index) => [taskId, index]));
|
|
171
|
+
const graphNodes = new Map(graph.tasks.map((node) => [node.id, node]));
|
|
172
|
+
const runnableTaskIds = graph.executionOrder.filter((taskId) => Boolean(plan.pipeline.tasks[taskId]));
|
|
173
|
+
const dependencyCounts = new Map();
|
|
174
|
+
const sourcePrepareOrder = new Map();
|
|
175
|
+
const recordedResults = new Map();
|
|
176
|
+
const sourcePreparePromises = new Map();
|
|
177
|
+
const recordedPrepareSources = new Set();
|
|
178
|
+
for (const taskId of runnableTaskIds) {
|
|
42
179
|
const taskDefinition = plan.pipeline.tasks[taskId];
|
|
43
|
-
if (!taskDefinition)
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
180
|
+
if (taskDefinition?.source?.name && !sourcePrepareOrder.has(taskDefinition.source.name)) {
|
|
181
|
+
sourcePrepareOrder.set(taskDefinition.source.name, (graphIndex.get(taskId) ?? 0) - 0.25);
|
|
182
|
+
}
|
|
183
|
+
const node = graphNodes.get(taskId);
|
|
184
|
+
dependencyCounts.set(taskId, (node?.dependsOn ?? []).filter((dependency) => Boolean(plan.pipeline.tasks[dependency])).length);
|
|
185
|
+
}
|
|
186
|
+
const ready = runnableTaskIds
|
|
187
|
+
.filter((taskId) => dependencyCounts.get(taskId) === 0)
|
|
188
|
+
.sort((left, right) => (graphIndex.get(left) ?? 0) - (graphIndex.get(right) ?? 0));
|
|
189
|
+
const running = new Map();
|
|
190
|
+
let failed = false;
|
|
191
|
+
const updateRecord = async () => {
|
|
192
|
+
record.tasks = [...recordedResults.values()]
|
|
193
|
+
.sort((left, right) => left.order - right.order || left.result.id.localeCompare(right.result.id))
|
|
194
|
+
.map((entry) => entry.result);
|
|
195
|
+
await writeExecution(store, record);
|
|
196
|
+
};
|
|
197
|
+
const ensureSourcePrepared = async (source) => {
|
|
198
|
+
let promise = sourcePreparePromises.get(source.id);
|
|
199
|
+
if (!promise) {
|
|
200
|
+
promise = runSourcePrepare(source, {
|
|
49
201
|
candidate: plan.candidate,
|
|
50
|
-
|
|
202
|
+
executor: workspace.executor,
|
|
203
|
+
rootCwd: workspace.cwd,
|
|
51
204
|
runId: record.id,
|
|
205
|
+
workspaceEnv: workspace.env,
|
|
52
206
|
store
|
|
53
207
|
});
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
208
|
+
sourcePreparePromises.set(source.id, promise);
|
|
209
|
+
}
|
|
210
|
+
const result = await promise;
|
|
211
|
+
const recordResult = Boolean(result && !recordedPrepareSources.has(source.id));
|
|
212
|
+
if (recordResult)
|
|
213
|
+
recordedPrepareSources.add(source.id);
|
|
214
|
+
return { result, recordResult };
|
|
215
|
+
};
|
|
216
|
+
const runScheduledTask = async (taskId) => {
|
|
217
|
+
const taskDefinition = plan.pipeline.tasks[taskId];
|
|
218
|
+
if (!taskDefinition)
|
|
219
|
+
return { taskId, failed: false, results: [] };
|
|
220
|
+
const results = [];
|
|
221
|
+
const taskSource = taskDefinition.source?.name ? plan.sources[taskDefinition.source.name] : undefined;
|
|
222
|
+
if (taskSource) {
|
|
223
|
+
const prepare = await ensureSourcePrepared(taskSource);
|
|
224
|
+
if (prepare.result && prepare.recordResult) {
|
|
225
|
+
results.push({
|
|
226
|
+
order: sourcePrepareOrder.get(taskSource.id) ?? ((graphIndex.get(taskId) ?? 0) - 0.25),
|
|
227
|
+
result: prepare.result
|
|
228
|
+
});
|
|
229
|
+
}
|
|
230
|
+
if (prepare.result?.status === "failed") {
|
|
231
|
+
return { taskId, failed: true, results };
|
|
64
232
|
}
|
|
65
233
|
}
|
|
66
234
|
const result = await runTask(plan.pipeline, taskDefinition, {
|
|
67
|
-
adapter,
|
|
68
235
|
candidate: plan.candidate,
|
|
69
|
-
cwd: taskDefinition.source?.dir ||
|
|
70
|
-
|
|
236
|
+
cwd: taskDefinition.source?.dir || workspace.cwd,
|
|
237
|
+
executor: workspace.executor,
|
|
238
|
+
rootCwd: workspace.cwd,
|
|
71
239
|
runId: record.id,
|
|
72
240
|
source: taskDefinition.source,
|
|
73
241
|
envDefinitions,
|
|
242
|
+
workspaceEnv: workspace.env,
|
|
74
243
|
sourcePrepareCommands: taskSource ? await resolvePrepareCommands(taskSource, {
|
|
75
244
|
candidate: plan.candidate,
|
|
76
|
-
rootCwd:
|
|
77
|
-
runId: record.id
|
|
245
|
+
rootCwd: workspace.cwd,
|
|
246
|
+
runId: record.id,
|
|
247
|
+
workspaceEnv: workspace.env
|
|
78
248
|
}) : [],
|
|
249
|
+
dependencyFingerprints: Object.fromEntries(taskDefinition.dependsOn.map((dependency) => [
|
|
250
|
+
dependency,
|
|
251
|
+
taskFingerprints.get(dependency) ?? null
|
|
252
|
+
])),
|
|
79
253
|
store
|
|
80
254
|
});
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
255
|
+
results.push({ order: graphIndex.get(taskId) ?? results.length, result });
|
|
256
|
+
return { taskId, failed: result.status === "failed", results, taskResult: result };
|
|
257
|
+
};
|
|
258
|
+
while (ready.length > 0 || running.size > 0) {
|
|
259
|
+
while (!failed && ready.length > 0 && running.size < concurrency) {
|
|
260
|
+
const taskId = ready.shift();
|
|
261
|
+
if (!taskId)
|
|
262
|
+
break;
|
|
263
|
+
running.set(taskId, runScheduledTask(taskId));
|
|
88
264
|
}
|
|
265
|
+
if (running.size === 0)
|
|
266
|
+
break;
|
|
267
|
+
const completed = await Promise.race(running.values());
|
|
268
|
+
running.delete(completed.taskId);
|
|
269
|
+
for (const entry of completed.results) {
|
|
270
|
+
recordedResults.set(entry.result.id, entry);
|
|
271
|
+
}
|
|
272
|
+
if (completed.taskResult) {
|
|
273
|
+
taskFingerprints.set(completed.taskId, completed.taskResult.cacheKey ?? `${completed.taskId}:${completed.taskResult.status}`);
|
|
274
|
+
}
|
|
275
|
+
await updateRecord();
|
|
276
|
+
if (completed.failed) {
|
|
277
|
+
failed = true;
|
|
278
|
+
ready.length = 0;
|
|
279
|
+
continue;
|
|
280
|
+
}
|
|
281
|
+
if (failed || !completed.taskResult)
|
|
282
|
+
continue;
|
|
283
|
+
const node = graphNodes.get(completed.taskId);
|
|
284
|
+
for (const dependent of node?.dependents ?? []) {
|
|
285
|
+
if (!plan.pipeline.tasks[dependent])
|
|
286
|
+
continue;
|
|
287
|
+
const remaining = Math.max(0, (dependencyCounts.get(dependent) ?? 0) - 1);
|
|
288
|
+
dependencyCounts.set(dependent, remaining);
|
|
289
|
+
if (remaining === 0) {
|
|
290
|
+
ready.push(dependent);
|
|
291
|
+
}
|
|
292
|
+
}
|
|
293
|
+
ready.sort((left, right) => (graphIndex.get(left) ?? 0) - (graphIndex.get(right) ?? 0));
|
|
89
294
|
}
|
|
90
|
-
record.status = "passed";
|
|
295
|
+
record.status = failed ? "failed" : "passed";
|
|
91
296
|
record.finishedAt = new Date().toISOString();
|
|
92
297
|
await writeExecution(store, record);
|
|
93
298
|
return record;
|
|
94
299
|
}
|
|
95
|
-
export async function runSingleTask(pipeline, taskId, options) {
|
|
300
|
+
export async function runSingleTask(pipeline, taskId, options = {}) {
|
|
96
301
|
const syntheticJobId = `task:${taskId}`;
|
|
97
302
|
const syntheticPipeline = {
|
|
98
303
|
...pipeline,
|
|
99
304
|
jobs: {
|
|
100
305
|
...pipeline.jobs,
|
|
101
|
-
[syntheticJobId]: { id: syntheticJobId, target: [taskId], trigger: []
|
|
306
|
+
[syntheticJobId]: { id: syntheticJobId, target: [taskId], trigger: [] }
|
|
102
307
|
}
|
|
103
308
|
};
|
|
104
|
-
return runJob(syntheticPipeline, { ...options,
|
|
309
|
+
return runJob(syntheticPipeline, { ...options, id: syntheticJobId });
|
|
105
310
|
}
|
|
106
311
|
async function runTask(pipeline, taskDefinition, options) {
|
|
107
312
|
const started = Date.now();
|
|
@@ -110,7 +315,7 @@ async function runTask(pipeline, taskDefinition, options) {
|
|
|
110
315
|
let combinedLog = "";
|
|
111
316
|
let taskEnv;
|
|
112
317
|
try {
|
|
113
|
-
taskEnv = buildTaskEnv(
|
|
318
|
+
taskEnv = buildTaskEnv(options.workspaceEnv, {
|
|
114
319
|
candidate: options.candidate,
|
|
115
320
|
envDefinitions: options.envDefinitions,
|
|
116
321
|
rootCwd: options.rootCwd,
|
|
@@ -148,6 +353,7 @@ async function runTask(pipeline, taskDefinition, options) {
|
|
|
148
353
|
const resolvedSteps = await resolveTaskSteps(taskDefinition.steps, context);
|
|
149
354
|
const cacheKey = await computeTaskCacheKey(pipeline, taskDefinition, options.cwd, {
|
|
150
355
|
candidate: options.candidate,
|
|
356
|
+
dependencyFingerprints: options.dependencyFingerprints,
|
|
151
357
|
prepareCommands: (options.sourcePrepareCommands ?? []).map((command) => command.command),
|
|
152
358
|
source: options.source,
|
|
153
359
|
steps: resolvedSteps
|
|
@@ -155,19 +361,23 @@ async function runTask(pipeline, taskDefinition, options) {
|
|
|
155
361
|
if (taskDefinition.cache.enabled) {
|
|
156
362
|
const cached = await readTaskCacheEntry(taskDefinition, options.store, cacheKey);
|
|
157
363
|
if (cached?.status === "passed") {
|
|
158
|
-
const
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
364
|
+
const cacheHit = await validateTaskCacheHit(taskDefinition, options.store, cacheKey, options.cwd, cached);
|
|
365
|
+
if (cacheHit.ok) {
|
|
366
|
+
const result = {
|
|
367
|
+
...cached,
|
|
368
|
+
id: taskDefinition.id,
|
|
369
|
+
status: "cached",
|
|
370
|
+
startedAt,
|
|
371
|
+
finishedAt: new Date().toISOString(),
|
|
372
|
+
attempts: 0,
|
|
373
|
+
cacheKey,
|
|
374
|
+
cacheHit: true,
|
|
375
|
+
durationMs: Date.now() - started
|
|
376
|
+
};
|
|
377
|
+
await writeTaskLog(options.store, options.runId, taskDefinition.id, `[cache hit] ${cacheKey}\n`);
|
|
378
|
+
return result;
|
|
379
|
+
}
|
|
380
|
+
combinedLog += `[cache miss] ${cacheHit.reason}\n`;
|
|
171
381
|
}
|
|
172
382
|
}
|
|
173
383
|
let attempts = 0;
|
|
@@ -177,13 +387,13 @@ async function runTask(pipeline, taskDefinition, options) {
|
|
|
177
387
|
attempts += 1;
|
|
178
388
|
try {
|
|
179
389
|
for (const requirement of taskDefinition.requires?.tools ?? []) {
|
|
180
|
-
const ok = await options.
|
|
390
|
+
const ok = await options.executor.checkTool?.(requirement);
|
|
181
391
|
if (ok === false) {
|
|
182
392
|
throw new Error(`Required tool "${requirement}" is not available for task "${taskDefinition.id}".`);
|
|
183
393
|
}
|
|
184
394
|
}
|
|
185
395
|
for (const secret of taskDefinition.requires?.secrets ?? []) {
|
|
186
|
-
if (!
|
|
396
|
+
if (!taskEnv[secret]) {
|
|
187
397
|
throw new Error(`Required secret "${secret}" is not available for task "${taskDefinition.id}".`);
|
|
188
398
|
}
|
|
189
399
|
}
|
|
@@ -219,7 +429,7 @@ async function runTask(pipeline, taskDefinition, options) {
|
|
|
219
429
|
};
|
|
220
430
|
await writeTaskLog(options.store, options.runId, taskDefinition.id, combinedLog);
|
|
221
431
|
if (taskDefinition.cache.enabled) {
|
|
222
|
-
await writeTaskCacheEntry(taskDefinition, options.store, cacheKey, result);
|
|
432
|
+
await writeTaskCacheEntry(taskDefinition, options.store, cacheKey, result, options.cwd);
|
|
223
433
|
}
|
|
224
434
|
return result;
|
|
225
435
|
}
|
|
@@ -247,7 +457,7 @@ async function runTask(pipeline, taskDefinition, options) {
|
|
|
247
457
|
return result;
|
|
248
458
|
}
|
|
249
459
|
async function runShellStep(step, taskDefinition, options) {
|
|
250
|
-
return options.
|
|
460
|
+
return options.executor.runShell(step.command, {
|
|
251
461
|
cwd: options.cwd,
|
|
252
462
|
env: options.env,
|
|
253
463
|
task: taskDefinition,
|
|
@@ -265,7 +475,7 @@ async function runSourcePrepare(source, options) {
|
|
|
265
475
|
const context = createTaskContext({ id: taskId }, {
|
|
266
476
|
candidate: options.candidate,
|
|
267
477
|
cwd: source.dir,
|
|
268
|
-
env: buildTaskEnv(
|
|
478
|
+
env: buildTaskEnv(options.workspaceEnv, {
|
|
269
479
|
candidate: options.candidate,
|
|
270
480
|
rootCwd: options.rootCwd,
|
|
271
481
|
source: sourceTaskContext
|
|
@@ -288,9 +498,9 @@ async function runSourcePrepare(source, options) {
|
|
|
288
498
|
if (!isShellCommand(step)) {
|
|
289
499
|
throw new Error(`Deferred shell step for source "${source.id}" was not resolved.`);
|
|
290
500
|
}
|
|
291
|
-
const result = await options.
|
|
501
|
+
const result = await options.executor.runShell(step.command, {
|
|
292
502
|
cwd: source.dir,
|
|
293
|
-
env: buildTaskEnv(
|
|
503
|
+
env: buildTaskEnv(options.workspaceEnv, {
|
|
294
504
|
candidate: options.candidate,
|
|
295
505
|
rootCwd: options.rootCwd,
|
|
296
506
|
source: sourceTaskContext
|
|
@@ -335,7 +545,7 @@ async function resolvePrepareCommands(source, options) {
|
|
|
335
545
|
const context = createTaskContext({ id: `${source.id}:prepare` }, {
|
|
336
546
|
candidate: options.candidate,
|
|
337
547
|
cwd: source.dir,
|
|
338
|
-
env: buildTaskEnv(
|
|
548
|
+
env: buildTaskEnv(options.workspaceEnv, {
|
|
339
549
|
candidate: options.candidate,
|
|
340
550
|
rootCwd: options.rootCwd,
|
|
341
551
|
source: sourceContext(source)
|
|
@@ -441,20 +651,66 @@ async function readTaskCacheEntry(taskDefinition, store, cacheKey) {
|
|
|
441
651
|
if (storeName === "file")
|
|
442
652
|
return readCacheEntry(store, cacheKey);
|
|
443
653
|
if (storeName === "memory")
|
|
444
|
-
return memoryCacheEntries.get(cacheKey) ?? null;
|
|
445
|
-
throw new Error(`Cache store "${storeName}" is registered but this runner cannot execute it. Use "file" or "memory", or provide a runtime-specific
|
|
654
|
+
return memoryCacheEntries.get(cacheKey)?.result ?? null;
|
|
655
|
+
throw new Error(`Cache store "${storeName}" is registered but this runner cannot execute it. Use "file" or "memory", or provide a runtime-specific executor.`);
|
|
446
656
|
}
|
|
447
|
-
async function writeTaskCacheEntry(taskDefinition, store, cacheKey, result) {
|
|
657
|
+
async function writeTaskCacheEntry(taskDefinition, store, cacheKey, result, cwd) {
|
|
448
658
|
const storeName = taskDefinition.cache.store ?? "file";
|
|
449
659
|
if (storeName === "file") {
|
|
450
|
-
await writeCacheEntry(store, cacheKey, result
|
|
660
|
+
await writeCacheEntry(store, cacheKey, result, {
|
|
661
|
+
cwd,
|
|
662
|
+
outputs: taskDefinition.outputs
|
|
663
|
+
});
|
|
451
664
|
return;
|
|
452
665
|
}
|
|
453
666
|
if (storeName === "memory") {
|
|
454
|
-
memoryCacheEntries.set(cacheKey,
|
|
667
|
+
memoryCacheEntries.set(cacheKey, {
|
|
668
|
+
result,
|
|
669
|
+
outputFiles: taskDefinition.outputs.length > 0 ? await resolveOutputFiles(cwd, taskDefinition.outputs) : []
|
|
670
|
+
});
|
|
455
671
|
return;
|
|
456
672
|
}
|
|
457
|
-
throw new Error(`Cache store "${storeName}" is registered but this runner cannot execute it. Use "file" or "memory", or provide a runtime-specific
|
|
673
|
+
throw new Error(`Cache store "${storeName}" is registered but this runner cannot execute it. Use "file" or "memory", or provide a runtime-specific executor.`);
|
|
674
|
+
}
|
|
675
|
+
async function validateTaskCacheHit(taskDefinition, store, cacheKey, cwd, cached) {
|
|
676
|
+
if (!isCacheEntryFresh(cached, taskDefinition.cache.ttlMs)) {
|
|
677
|
+
return { ok: false, reason: "cache entry expired or has an invalid timestamp" };
|
|
678
|
+
}
|
|
679
|
+
if (taskDefinition.outputs.length === 0)
|
|
680
|
+
return { ok: true };
|
|
681
|
+
const storeName = taskDefinition.cache.store ?? "file";
|
|
682
|
+
if (storeName === "file") {
|
|
683
|
+
const restored = await restoreCacheOutputs(store, cacheKey, cwd, taskDefinition.outputs);
|
|
684
|
+
return restored
|
|
685
|
+
? { ok: true }
|
|
686
|
+
: { ok: false, reason: "declared outputs were not available in the file cache" };
|
|
687
|
+
}
|
|
688
|
+
if (storeName === "memory") {
|
|
689
|
+
const entry = memoryCacheEntries.get(cacheKey);
|
|
690
|
+
if (!entry)
|
|
691
|
+
return { ok: false, reason: "memory cache entry was not available" };
|
|
692
|
+
const outputsExist = await outputFilesExist(cwd, entry.outputFiles);
|
|
693
|
+
return outputsExist
|
|
694
|
+
? { ok: true }
|
|
695
|
+
: { ok: false, reason: "declared outputs were missing from the working tree" };
|
|
696
|
+
}
|
|
697
|
+
return { ok: false, reason: `cache store "${storeName}" is not executable by this runner` };
|
|
698
|
+
}
|
|
699
|
+
function isCacheEntryFresh(cached, ttlMs) {
|
|
700
|
+
if (ttlMs === undefined)
|
|
701
|
+
return true;
|
|
702
|
+
if (!Number.isFinite(ttlMs) || ttlMs < 0)
|
|
703
|
+
return false;
|
|
704
|
+
const finishedAtMs = cached.finishedAt ? Date.parse(cached.finishedAt) : Number.NaN;
|
|
705
|
+
return Number.isFinite(finishedAtMs) && Date.now() - finishedAtMs <= ttlMs;
|
|
706
|
+
}
|
|
707
|
+
function normalizeTaskConcurrency(concurrency) {
|
|
708
|
+
if (concurrency === undefined)
|
|
709
|
+
return Math.max(1, Math.min(DEFAULT_MAX_CONCURRENCY, availableParallelism()));
|
|
710
|
+
if (!Number.isInteger(concurrency) || concurrency < 1) {
|
|
711
|
+
throw new Error("Task concurrency must be a positive integer.");
|
|
712
|
+
}
|
|
713
|
+
return concurrency;
|
|
458
714
|
}
|
|
459
715
|
async function runFunctionStep(step, context, timeoutMs) {
|
|
460
716
|
if (!timeoutMs) {
|
|
@@ -521,6 +777,77 @@ function runProcess(command, options) {
|
|
|
521
777
|
});
|
|
522
778
|
});
|
|
523
779
|
}
|
|
780
|
+
function matchingAction(policy, argv) {
|
|
781
|
+
for (const rule of policy.rules) {
|
|
782
|
+
if (matchesCommandRule(rule, argv))
|
|
783
|
+
return rule.action;
|
|
784
|
+
}
|
|
785
|
+
return policy.fallback ?? { kind: "async-pipeline.command.allow" };
|
|
786
|
+
}
|
|
787
|
+
function matchesCommandRule(rule, argv) {
|
|
788
|
+
if (rule.exact && sameArgs(rule.exact, argv))
|
|
789
|
+
return true;
|
|
790
|
+
if (rule.prefix && hasPrefix(argv, rule.prefix))
|
|
791
|
+
return true;
|
|
792
|
+
return false;
|
|
793
|
+
}
|
|
794
|
+
function sameArgs(left, right) {
|
|
795
|
+
return left.length === right.length && left.every((value, index) => right[index] === value);
|
|
796
|
+
}
|
|
797
|
+
function hasPrefix(argv, prefix) {
|
|
798
|
+
return prefix.length <= argv.length && prefix.every((value, index) => argv[index] === value);
|
|
799
|
+
}
|
|
800
|
+
function commandStatus(action) {
|
|
801
|
+
if (action.kind === "async-pipeline.command.mock")
|
|
802
|
+
return "mocked";
|
|
803
|
+
if (action.kind === "async-pipeline.command.deny")
|
|
804
|
+
return "denied";
|
|
805
|
+
if (action.kind === "async-pipeline.command.requireApproval")
|
|
806
|
+
return "approval-required";
|
|
807
|
+
return "allowed";
|
|
808
|
+
}
|
|
809
|
+
async function runCommandAction(action, next) {
|
|
810
|
+
if (action.kind === "async-pipeline.command.allow")
|
|
811
|
+
return next();
|
|
812
|
+
if (action.kind === "async-pipeline.command.requireEnvironment")
|
|
813
|
+
return next();
|
|
814
|
+
if (action.kind === "async-pipeline.command.mock") {
|
|
815
|
+
return {
|
|
816
|
+
code: action.code ?? 0,
|
|
817
|
+
stdout: action.stdout ?? "",
|
|
818
|
+
stderr: action.stderr ?? ""
|
|
819
|
+
};
|
|
820
|
+
}
|
|
821
|
+
if (action.kind === "async-pipeline.command.deny") {
|
|
822
|
+
return {
|
|
823
|
+
code: 1,
|
|
824
|
+
stdout: "",
|
|
825
|
+
stderr: `${action.message ?? "Command denied by async-pipeline command policy."}\n`
|
|
826
|
+
};
|
|
827
|
+
}
|
|
828
|
+
return {
|
|
829
|
+
code: 1,
|
|
830
|
+
stdout: "",
|
|
831
|
+
stderr: `${action.message ?? "Command requires approval by async-pipeline command policy."}\n`
|
|
832
|
+
};
|
|
833
|
+
}
|
|
834
|
+
function applyOutputPolicy(output, policy, env) {
|
|
835
|
+
let next = policy.redactSecrets ? redactSecretValues(output, env) : output;
|
|
836
|
+
const maxBytes = policy.maxBytes;
|
|
837
|
+
if (maxBytes === undefined || Buffer.byteLength(next, "utf8") <= maxBytes)
|
|
838
|
+
return next;
|
|
839
|
+
const truncated = Buffer.from(next).subarray(0, maxBytes).toString("utf8");
|
|
840
|
+
return `${truncated}\n[async-pipeline] output truncated to ${maxBytes} bytes\n`;
|
|
841
|
+
}
|
|
842
|
+
function redactSecretValues(output, env) {
|
|
843
|
+
let redacted = output;
|
|
844
|
+
for (const [key, value] of Object.entries(env)) {
|
|
845
|
+
if (!value || value.length < 4 || !/(SECRET|TOKEN|PASSWORD|AUTH|KEY)/i.test(key))
|
|
846
|
+
continue;
|
|
847
|
+
redacted = redacted.split(value).join("[redacted]");
|
|
848
|
+
}
|
|
849
|
+
return redacted;
|
|
850
|
+
}
|
|
524
851
|
function shellEscape(value) {
|
|
525
852
|
return `'${value.replaceAll("'", "'\\''")}'`;
|
|
526
853
|
}
|