@async/pipeline 0.1.3 → 0.1.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (38) hide show
  1. package/README.md +3 -3
  2. package/dist/internal/core/cache.d.ts +5 -1
  3. package/dist/internal/core/cache.d.ts.map +1 -1
  4. package/dist/internal/core/cache.js +19 -9
  5. package/dist/internal/core/cache.js.map +1 -1
  6. package/dist/internal/core/index.d.ts +121 -3
  7. package/dist/internal/core/index.d.ts.map +1 -1
  8. package/dist/internal/core/index.js +105 -8
  9. package/dist/internal/core/index.js.map +1 -1
  10. package/dist/internal/core/runtime.js +1 -1
  11. package/dist/internal/core/runtime.js.map +1 -1
  12. package/dist/internal/lima/index.d.ts +1 -14
  13. package/dist/internal/lima/index.d.ts.map +1 -1
  14. package/dist/internal/lima/index.js +1 -64
  15. package/dist/internal/lima/index.js.map +1 -1
  16. package/dist/internal/node/cli.d.ts +9 -1
  17. package/dist/internal/node/cli.d.ts.map +1 -1
  18. package/dist/internal/node/cli.js +279 -172
  19. package/dist/internal/node/cli.js.map +1 -1
  20. package/dist/internal/node/doctor.js +2 -2
  21. package/dist/internal/node/doctor.js.map +1 -1
  22. package/dist/internal/node/github.d.ts +3 -1
  23. package/dist/internal/node/github.d.ts.map +1 -1
  24. package/dist/internal/node/github.js +31 -9
  25. package/dist/internal/node/github.js.map +1 -1
  26. package/dist/internal/node/index.d.ts +1 -0
  27. package/dist/internal/node/index.d.ts.map +1 -1
  28. package/dist/internal/node/index.js +1 -0
  29. package/dist/internal/node/index.js.map +1 -1
  30. package/dist/internal/node/runner.d.ts +98 -7
  31. package/dist/internal/node/runner.d.ts.map +1 -1
  32. package/dist/internal/node/runner.js +400 -73
  33. package/dist/internal/node/runner.js.map +1 -1
  34. package/dist/internal/node/store.d.ts +29 -5
  35. package/dist/internal/node/store.d.ts.map +1 -1
  36. package/dist/internal/node/store.js +225 -20
  37. package/dist/internal/node/store.js.map +1 -1
  38. package/package.json +1 -1
@@ -1,10 +1,12 @@
1
1
  import { spawn } from "node:child_process";
2
2
  import { randomUUID } from "node:crypto";
3
+ import { availableParallelism } from "node:os";
4
+ import { posix, relative } from "node:path";
3
5
  import { setTimeout as delay } from "node:timers/promises";
4
6
  import { sh, tasksForJob } from "../core/index.js";
5
- import { computeTaskCacheKey, createStore, readCacheEntry, writeCacheEntry, writeExecution, writeTaskLog } from "./store.js";
7
+ import { computeTaskCacheKey, createStore, outputFilesExist, readCacheEntry, resolveOutputFiles, restoreCacheOutputs, writeCacheEntry, writeExecution, writeTaskLog } from "./store.js";
6
8
  import { createRunPlan, sourceContext } from "./sources.js";
7
- export class HostRunnerAdapter {
9
+ export class HostCommandExecutor {
8
10
  name = "host";
9
11
  runShell(command, options) {
10
12
  return runProcess(command, { cwd: options.cwd, env: options.env, timeoutMs: options.timeoutMs });
@@ -14,94 +16,297 @@ export class HostRunnerAdapter {
14
16
  return result.code === 0;
15
17
  }
16
18
  }
19
+ export class DockerCommandExecutor {
20
+ options;
21
+ name = "docker";
22
+ constructor(options) {
23
+ this.options = options;
24
+ }
25
+ runShell(command, options) {
26
+ return runProcess(this.dockerCommand(command, options.cwd, options.env), { cwd: this.options.hostCwd, env: options.env, timeoutMs: options.timeoutMs });
27
+ }
28
+ async checkTool(tool) {
29
+ const result = await runProcess(this.dockerCommand(`command -v ${shellEscape(tool)}`, this.options.hostCwd, process.env), {
30
+ cwd: this.options.hostCwd,
31
+ env: process.env,
32
+ echo: false
33
+ });
34
+ return result.code === 0;
35
+ }
36
+ dockerCommand(command, cwd, env) {
37
+ return [
38
+ "docker",
39
+ "run",
40
+ "--rm",
41
+ "-w",
42
+ shellEscape(this.containerCwd(cwd)),
43
+ ...this.options.volumes.flatMap((volume) => ["-v", shellEscape(`${volume.source}:${volume.target}${volume.readonly ? ":ro" : ""}`)]),
44
+ ...Object.keys(env).filter((key) => env[key] !== undefined).flatMap((key) => ["-e", shellEscape(key)]),
45
+ shellEscape(this.options.image),
46
+ "bash",
47
+ "-lc",
48
+ shellEscape(command)
49
+ ].join(" ");
50
+ }
51
+ containerCwd(cwd) {
52
+ const rel = relative(this.options.hostCwd, cwd);
53
+ if (!rel || rel === ".")
54
+ return this.options.workdir;
55
+ if (rel.startsWith(".."))
56
+ return this.options.workdir;
57
+ return posix.join(this.options.workdir, rel.split(/[\\/]+/).join("/"));
58
+ }
59
+ }
60
+ export class LimaCommandExecutor {
61
+ vm;
62
+ name = "lima";
63
+ constructor(vm = "async-pipeline") {
64
+ this.vm = vm;
65
+ }
66
+ runShell(command, options) {
67
+ const vm = options.task.environment?.vm ?? this.vm;
68
+ const escaped = shellEscape(`cd ${shellEscape(options.cwd)} && ${command}`);
69
+ return runProcess(`limactl shell ${shellEscape(vm)} -- bash -lc ${escaped}`, { cwd: options.cwd, env: options.env, timeoutMs: options.timeoutMs });
70
+ }
71
+ async checkTool(tool) {
72
+ const result = await runProcess(`limactl shell ${shellEscape(this.vm)} -- bash -lc ${shellEscape(`command -v ${shellEscape(tool)}`)}`, {
73
+ cwd: process.cwd(),
74
+ env: process.env
75
+ });
76
+ return result.code === 0;
77
+ }
78
+ }
79
+ export function hostWorkspace(options = {}) {
80
+ return {
81
+ cwd: options.cwd ?? process.cwd(),
82
+ env: options.env ?? process.env,
83
+ fs: { kind: "host" },
84
+ executor: options.executor ?? new HostCommandExecutor(),
85
+ commands: options.commands
86
+ };
87
+ }
88
+ export function dockerWorkspace(options) {
89
+ const cwd = options.cwd ?? process.cwd();
90
+ const workdir = options.workdir ?? "/workspace";
91
+ return hostWorkspace({
92
+ cwd,
93
+ env: options.env,
94
+ executor: new DockerCommandExecutor({
95
+ image: options.image,
96
+ hostCwd: cwd,
97
+ workdir,
98
+ volumes: options.volumes ?? [{ source: cwd, target: workdir }]
99
+ }),
100
+ commands: options.commands
101
+ });
102
+ }
103
+ export function limaWorkspace(options = {}) {
104
+ return hostWorkspace({
105
+ cwd: options.cwd,
106
+ env: options.env,
107
+ executor: new LimaCommandExecutor(options.vm),
108
+ commands: options.commands
109
+ });
110
+ }
111
+ export function commandProxy(policy = { rules: [] }) {
112
+ const records = [];
113
+ return {
114
+ async run(invocation, next) {
115
+ const startedAt = new Date().toISOString();
116
+ const started = Date.now();
117
+ const action = matchingAction(policy, invocation.argv);
118
+ const status = commandStatus(action);
119
+ const result = await runCommandAction(action, next);
120
+ const outputPolicy = action.output ?? policy.output ?? {};
121
+ if (policy.record) {
122
+ records.push({
123
+ argv: [...invocation.argv],
124
+ cwd: invocation.cwd,
125
+ status,
126
+ code: result.code,
127
+ stdout: applyOutputPolicy(result.stdout, outputPolicy, invocation.env),
128
+ stderr: applyOutputPolicy(result.stderr, outputPolicy, invocation.env),
129
+ startedAt,
130
+ finishedAt: new Date().toISOString(),
131
+ durationMs: Date.now() - started
132
+ });
133
+ }
134
+ return result;
135
+ },
136
+ records() {
137
+ return records.map((record) => ({
138
+ ...record,
139
+ argv: [...record.argv]
140
+ }));
141
+ }
142
+ };
143
+ }
17
144
  const memoryCacheEntries = new Map();
145
+ const DEFAULT_MAX_CONCURRENCY = 4;
18
146
  export async function runJob(pipeline, options) {
19
- const adapter = options.adapter ?? new HostRunnerAdapter();
20
- const store = await createStore(options.cwd);
21
- const plan = await createRunPlan(pipeline, options.cwd, store);
22
- const graph = tasksForJob(plan.pipeline, options.jobId);
147
+ const workspace = options.workspace ?? hostWorkspace();
148
+ const store = await createStore(workspace.cwd);
149
+ const plan = await createRunPlan(pipeline, workspace.cwd, store);
150
+ const graph = tasksForJob(plan.pipeline, options.id);
23
151
  const record = {
24
152
  id: `${new Date().toISOString().replaceAll(/[:.]/g, "-")}-${randomUUID().slice(0, 8)}`,
25
153
  pipelineName: plan.pipeline.name,
26
- jobId: options.jobId,
27
- cwd: options.cwd,
154
+ jobId: options.id,
155
+ cwd: workspace.cwd,
28
156
  startedAt: new Date().toISOString(),
29
157
  status: "running",
30
- mode: options.mode ?? "manual",
158
+ mode: options.mode ?? (workspace.env.CI ? "ci" : "manual"),
31
159
  tasks: [],
32
160
  sources: Object.fromEntries(Object.entries(plan.sources).map(([sourceId, resolved]) => [sourceId, resolved.record]))
33
161
  };
34
162
  await writeExecution(store, record);
35
- const preparedSources = new Set();
36
- const jobDefinition = plan.pipeline.jobs[options.jobId];
163
+ const jobDefinition = plan.pipeline.jobs[options.id];
37
164
  const envDefinitions = {
38
165
  ...plan.pipeline.env,
39
166
  ...(jobDefinition?.env ?? {})
40
167
  };
41
- for (const taskId of graph.executionOrder) {
168
+ const taskFingerprints = new Map();
169
+ const concurrency = normalizeTaskConcurrency(options.concurrency);
170
+ const graphIndex = new Map(graph.executionOrder.map((taskId, index) => [taskId, index]));
171
+ const graphNodes = new Map(graph.tasks.map((node) => [node.id, node]));
172
+ const runnableTaskIds = graph.executionOrder.filter((taskId) => Boolean(plan.pipeline.tasks[taskId]));
173
+ const dependencyCounts = new Map();
174
+ const sourcePrepareOrder = new Map();
175
+ const recordedResults = new Map();
176
+ const sourcePreparePromises = new Map();
177
+ const recordedPrepareSources = new Set();
178
+ for (const taskId of runnableTaskIds) {
42
179
  const taskDefinition = plan.pipeline.tasks[taskId];
43
- if (!taskDefinition)
44
- continue;
45
- const taskSource = taskDefinition.source?.name ? plan.sources[taskDefinition.source.name] : undefined;
46
- if (taskSource && !preparedSources.has(taskSource.id)) {
47
- const prepareResult = await runSourcePrepare(taskSource, {
48
- adapter,
180
+ if (taskDefinition?.source?.name && !sourcePrepareOrder.has(taskDefinition.source.name)) {
181
+ sourcePrepareOrder.set(taskDefinition.source.name, (graphIndex.get(taskId) ?? 0) - 0.25);
182
+ }
183
+ const node = graphNodes.get(taskId);
184
+ dependencyCounts.set(taskId, (node?.dependsOn ?? []).filter((dependency) => Boolean(plan.pipeline.tasks[dependency])).length);
185
+ }
186
+ const ready = runnableTaskIds
187
+ .filter((taskId) => dependencyCounts.get(taskId) === 0)
188
+ .sort((left, right) => (graphIndex.get(left) ?? 0) - (graphIndex.get(right) ?? 0));
189
+ const running = new Map();
190
+ let failed = false;
191
+ const updateRecord = async () => {
192
+ record.tasks = [...recordedResults.values()]
193
+ .sort((left, right) => left.order - right.order || left.result.id.localeCompare(right.result.id))
194
+ .map((entry) => entry.result);
195
+ await writeExecution(store, record);
196
+ };
197
+ const ensureSourcePrepared = async (source) => {
198
+ let promise = sourcePreparePromises.get(source.id);
199
+ if (!promise) {
200
+ promise = runSourcePrepare(source, {
49
201
  candidate: plan.candidate,
50
- rootCwd: options.cwd,
202
+ executor: workspace.executor,
203
+ rootCwd: workspace.cwd,
51
204
  runId: record.id,
205
+ workspaceEnv: workspace.env,
52
206
  store
53
207
  });
54
- preparedSources.add(taskSource.id);
55
- if (prepareResult) {
56
- record.tasks.push(prepareResult);
57
- await writeExecution(store, record);
58
- if (prepareResult.status === "failed") {
59
- record.status = "failed";
60
- record.finishedAt = new Date().toISOString();
61
- await writeExecution(store, record);
62
- return record;
63
- }
208
+ sourcePreparePromises.set(source.id, promise);
209
+ }
210
+ const result = await promise;
211
+ const recordResult = Boolean(result && !recordedPrepareSources.has(source.id));
212
+ if (recordResult)
213
+ recordedPrepareSources.add(source.id);
214
+ return { result, recordResult };
215
+ };
216
+ const runScheduledTask = async (taskId) => {
217
+ const taskDefinition = plan.pipeline.tasks[taskId];
218
+ if (!taskDefinition)
219
+ return { taskId, failed: false, results: [] };
220
+ const results = [];
221
+ const taskSource = taskDefinition.source?.name ? plan.sources[taskDefinition.source.name] : undefined;
222
+ if (taskSource) {
223
+ const prepare = await ensureSourcePrepared(taskSource);
224
+ if (prepare.result && prepare.recordResult) {
225
+ results.push({
226
+ order: sourcePrepareOrder.get(taskSource.id) ?? ((graphIndex.get(taskId) ?? 0) - 0.25),
227
+ result: prepare.result
228
+ });
229
+ }
230
+ if (prepare.result?.status === "failed") {
231
+ return { taskId, failed: true, results };
64
232
  }
65
233
  }
66
234
  const result = await runTask(plan.pipeline, taskDefinition, {
67
- adapter,
68
235
  candidate: plan.candidate,
69
- cwd: taskDefinition.source?.dir || options.cwd,
70
- rootCwd: options.cwd,
236
+ cwd: taskDefinition.source?.dir || workspace.cwd,
237
+ executor: workspace.executor,
238
+ rootCwd: workspace.cwd,
71
239
  runId: record.id,
72
240
  source: taskDefinition.source,
73
241
  envDefinitions,
242
+ workspaceEnv: workspace.env,
74
243
  sourcePrepareCommands: taskSource ? await resolvePrepareCommands(taskSource, {
75
244
  candidate: plan.candidate,
76
- rootCwd: options.cwd,
77
- runId: record.id
245
+ rootCwd: workspace.cwd,
246
+ runId: record.id,
247
+ workspaceEnv: workspace.env
78
248
  }) : [],
249
+ dependencyFingerprints: Object.fromEntries(taskDefinition.dependsOn.map((dependency) => [
250
+ dependency,
251
+ taskFingerprints.get(dependency) ?? null
252
+ ])),
79
253
  store
80
254
  });
81
- record.tasks.push(result);
82
- await writeExecution(store, record);
83
- if (result.status === "failed") {
84
- record.status = "failed";
85
- record.finishedAt = new Date().toISOString();
86
- await writeExecution(store, record);
87
- return record;
255
+ results.push({ order: graphIndex.get(taskId) ?? results.length, result });
256
+ return { taskId, failed: result.status === "failed", results, taskResult: result };
257
+ };
258
+ while (ready.length > 0 || running.size > 0) {
259
+ while (!failed && ready.length > 0 && running.size < concurrency) {
260
+ const taskId = ready.shift();
261
+ if (!taskId)
262
+ break;
263
+ running.set(taskId, runScheduledTask(taskId));
88
264
  }
265
+ if (running.size === 0)
266
+ break;
267
+ const completed = await Promise.race(running.values());
268
+ running.delete(completed.taskId);
269
+ for (const entry of completed.results) {
270
+ recordedResults.set(entry.result.id, entry);
271
+ }
272
+ if (completed.taskResult) {
273
+ taskFingerprints.set(completed.taskId, completed.taskResult.cacheKey ?? `${completed.taskId}:${completed.taskResult.status}`);
274
+ }
275
+ await updateRecord();
276
+ if (completed.failed) {
277
+ failed = true;
278
+ ready.length = 0;
279
+ continue;
280
+ }
281
+ if (failed || !completed.taskResult)
282
+ continue;
283
+ const node = graphNodes.get(completed.taskId);
284
+ for (const dependent of node?.dependents ?? []) {
285
+ if (!plan.pipeline.tasks[dependent])
286
+ continue;
287
+ const remaining = Math.max(0, (dependencyCounts.get(dependent) ?? 0) - 1);
288
+ dependencyCounts.set(dependent, remaining);
289
+ if (remaining === 0) {
290
+ ready.push(dependent);
291
+ }
292
+ }
293
+ ready.sort((left, right) => (graphIndex.get(left) ?? 0) - (graphIndex.get(right) ?? 0));
89
294
  }
90
- record.status = "passed";
295
+ record.status = failed ? "failed" : "passed";
91
296
  record.finishedAt = new Date().toISOString();
92
297
  await writeExecution(store, record);
93
298
  return record;
94
299
  }
95
- export async function runSingleTask(pipeline, taskId, options) {
300
+ export async function runSingleTask(pipeline, taskId, options = {}) {
96
301
  const syntheticJobId = `task:${taskId}`;
97
302
  const syntheticPipeline = {
98
303
  ...pipeline,
99
304
  jobs: {
100
305
  ...pipeline.jobs,
101
- [syntheticJobId]: { id: syntheticJobId, target: [taskId], trigger: [], mode: options.mode }
306
+ [syntheticJobId]: { id: syntheticJobId, target: [taskId], trigger: [] }
102
307
  }
103
308
  };
104
- return runJob(syntheticPipeline, { ...options, jobId: syntheticJobId });
309
+ return runJob(syntheticPipeline, { ...options, id: syntheticJobId });
105
310
  }
106
311
  async function runTask(pipeline, taskDefinition, options) {
107
312
  const started = Date.now();
@@ -110,7 +315,7 @@ async function runTask(pipeline, taskDefinition, options) {
110
315
  let combinedLog = "";
111
316
  let taskEnv;
112
317
  try {
113
- taskEnv = buildTaskEnv(process.env, {
318
+ taskEnv = buildTaskEnv(options.workspaceEnv, {
114
319
  candidate: options.candidate,
115
320
  envDefinitions: options.envDefinitions,
116
321
  rootCwd: options.rootCwd,
@@ -148,6 +353,7 @@ async function runTask(pipeline, taskDefinition, options) {
148
353
  const resolvedSteps = await resolveTaskSteps(taskDefinition.steps, context);
149
354
  const cacheKey = await computeTaskCacheKey(pipeline, taskDefinition, options.cwd, {
150
355
  candidate: options.candidate,
356
+ dependencyFingerprints: options.dependencyFingerprints,
151
357
  prepareCommands: (options.sourcePrepareCommands ?? []).map((command) => command.command),
152
358
  source: options.source,
153
359
  steps: resolvedSteps
@@ -155,19 +361,23 @@ async function runTask(pipeline, taskDefinition, options) {
155
361
  if (taskDefinition.cache.enabled) {
156
362
  const cached = await readTaskCacheEntry(taskDefinition, options.store, cacheKey);
157
363
  if (cached?.status === "passed") {
158
- const result = {
159
- ...cached,
160
- id: taskDefinition.id,
161
- status: "cached",
162
- startedAt,
163
- finishedAt: new Date().toISOString(),
164
- attempts: 0,
165
- cacheKey,
166
- cacheHit: true,
167
- durationMs: Date.now() - started
168
- };
169
- await writeTaskLog(options.store, options.runId, taskDefinition.id, `[cache hit] ${cacheKey}\n`);
170
- return result;
364
+ const cacheHit = await validateTaskCacheHit(taskDefinition, options.store, cacheKey, options.cwd, cached);
365
+ if (cacheHit.ok) {
366
+ const result = {
367
+ ...cached,
368
+ id: taskDefinition.id,
369
+ status: "cached",
370
+ startedAt,
371
+ finishedAt: new Date().toISOString(),
372
+ attempts: 0,
373
+ cacheKey,
374
+ cacheHit: true,
375
+ durationMs: Date.now() - started
376
+ };
377
+ await writeTaskLog(options.store, options.runId, taskDefinition.id, `[cache hit] ${cacheKey}\n`);
378
+ return result;
379
+ }
380
+ combinedLog += `[cache miss] ${cacheHit.reason}\n`;
171
381
  }
172
382
  }
173
383
  let attempts = 0;
@@ -177,13 +387,13 @@ async function runTask(pipeline, taskDefinition, options) {
177
387
  attempts += 1;
178
388
  try {
179
389
  for (const requirement of taskDefinition.requires?.tools ?? []) {
180
- const ok = await options.adapter.checkTool?.(requirement);
390
+ const ok = await options.executor.checkTool?.(requirement);
181
391
  if (ok === false) {
182
392
  throw new Error(`Required tool "${requirement}" is not available for task "${taskDefinition.id}".`);
183
393
  }
184
394
  }
185
395
  for (const secret of taskDefinition.requires?.secrets ?? []) {
186
- if (!process.env[secret]) {
396
+ if (!taskEnv[secret]) {
187
397
  throw new Error(`Required secret "${secret}" is not available for task "${taskDefinition.id}".`);
188
398
  }
189
399
  }
@@ -219,7 +429,7 @@ async function runTask(pipeline, taskDefinition, options) {
219
429
  };
220
430
  await writeTaskLog(options.store, options.runId, taskDefinition.id, combinedLog);
221
431
  if (taskDefinition.cache.enabled) {
222
- await writeTaskCacheEntry(taskDefinition, options.store, cacheKey, result);
432
+ await writeTaskCacheEntry(taskDefinition, options.store, cacheKey, result, options.cwd);
223
433
  }
224
434
  return result;
225
435
  }
@@ -247,7 +457,7 @@ async function runTask(pipeline, taskDefinition, options) {
247
457
  return result;
248
458
  }
249
459
  async function runShellStep(step, taskDefinition, options) {
250
- return options.adapter.runShell(step.command, {
460
+ return options.executor.runShell(step.command, {
251
461
  cwd: options.cwd,
252
462
  env: options.env,
253
463
  task: taskDefinition,
@@ -265,7 +475,7 @@ async function runSourcePrepare(source, options) {
265
475
  const context = createTaskContext({ id: taskId }, {
266
476
  candidate: options.candidate,
267
477
  cwd: source.dir,
268
- env: buildTaskEnv(process.env, {
478
+ env: buildTaskEnv(options.workspaceEnv, {
269
479
  candidate: options.candidate,
270
480
  rootCwd: options.rootCwd,
271
481
  source: sourceTaskContext
@@ -288,9 +498,9 @@ async function runSourcePrepare(source, options) {
288
498
  if (!isShellCommand(step)) {
289
499
  throw new Error(`Deferred shell step for source "${source.id}" was not resolved.`);
290
500
  }
291
- const result = await options.adapter.runShell(step.command, {
501
+ const result = await options.executor.runShell(step.command, {
292
502
  cwd: source.dir,
293
- env: buildTaskEnv(process.env, {
503
+ env: buildTaskEnv(options.workspaceEnv, {
294
504
  candidate: options.candidate,
295
505
  rootCwd: options.rootCwd,
296
506
  source: sourceTaskContext
@@ -335,7 +545,7 @@ async function resolvePrepareCommands(source, options) {
335
545
  const context = createTaskContext({ id: `${source.id}:prepare` }, {
336
546
  candidate: options.candidate,
337
547
  cwd: source.dir,
338
- env: buildTaskEnv(process.env, {
548
+ env: buildTaskEnv(options.workspaceEnv, {
339
549
  candidate: options.candidate,
340
550
  rootCwd: options.rootCwd,
341
551
  source: sourceContext(source)
@@ -441,20 +651,66 @@ async function readTaskCacheEntry(taskDefinition, store, cacheKey) {
441
651
  if (storeName === "file")
442
652
  return readCacheEntry(store, cacheKey);
443
653
  if (storeName === "memory")
444
- return memoryCacheEntries.get(cacheKey) ?? null;
445
- throw new Error(`Cache store "${storeName}" is registered but this runner cannot execute it. Use "file" or "memory", or provide a runtime-specific adapter.`);
654
+ return memoryCacheEntries.get(cacheKey)?.result ?? null;
655
+ throw new Error(`Cache store "${storeName}" is registered but this runner cannot execute it. Use "file" or "memory", or provide a runtime-specific executor.`);
446
656
  }
447
- async function writeTaskCacheEntry(taskDefinition, store, cacheKey, result) {
657
+ async function writeTaskCacheEntry(taskDefinition, store, cacheKey, result, cwd) {
448
658
  const storeName = taskDefinition.cache.store ?? "file";
449
659
  if (storeName === "file") {
450
- await writeCacheEntry(store, cacheKey, result);
660
+ await writeCacheEntry(store, cacheKey, result, {
661
+ cwd,
662
+ outputs: taskDefinition.outputs
663
+ });
451
664
  return;
452
665
  }
453
666
  if (storeName === "memory") {
454
- memoryCacheEntries.set(cacheKey, result);
667
+ memoryCacheEntries.set(cacheKey, {
668
+ result,
669
+ outputFiles: taskDefinition.outputs.length > 0 ? await resolveOutputFiles(cwd, taskDefinition.outputs) : []
670
+ });
455
671
  return;
456
672
  }
457
- throw new Error(`Cache store "${storeName}" is registered but this runner cannot execute it. Use "file" or "memory", or provide a runtime-specific adapter.`);
673
+ throw new Error(`Cache store "${storeName}" is registered but this runner cannot execute it. Use "file" or "memory", or provide a runtime-specific executor.`);
674
+ }
675
+ async function validateTaskCacheHit(taskDefinition, store, cacheKey, cwd, cached) {
676
+ if (!isCacheEntryFresh(cached, taskDefinition.cache.ttlMs)) {
677
+ return { ok: false, reason: "cache entry expired or has an invalid timestamp" };
678
+ }
679
+ if (taskDefinition.outputs.length === 0)
680
+ return { ok: true };
681
+ const storeName = taskDefinition.cache.store ?? "file";
682
+ if (storeName === "file") {
683
+ const restored = await restoreCacheOutputs(store, cacheKey, cwd, taskDefinition.outputs);
684
+ return restored
685
+ ? { ok: true }
686
+ : { ok: false, reason: "declared outputs were not available in the file cache" };
687
+ }
688
+ if (storeName === "memory") {
689
+ const entry = memoryCacheEntries.get(cacheKey);
690
+ if (!entry)
691
+ return { ok: false, reason: "memory cache entry was not available" };
692
+ const outputsExist = await outputFilesExist(cwd, entry.outputFiles);
693
+ return outputsExist
694
+ ? { ok: true }
695
+ : { ok: false, reason: "declared outputs were missing from the working tree" };
696
+ }
697
+ return { ok: false, reason: `cache store "${storeName}" is not executable by this runner` };
698
+ }
699
+ function isCacheEntryFresh(cached, ttlMs) {
700
+ if (ttlMs === undefined)
701
+ return true;
702
+ if (!Number.isFinite(ttlMs) || ttlMs < 0)
703
+ return false;
704
+ const finishedAtMs = cached.finishedAt ? Date.parse(cached.finishedAt) : Number.NaN;
705
+ return Number.isFinite(finishedAtMs) && Date.now() - finishedAtMs <= ttlMs;
706
+ }
707
+ function normalizeTaskConcurrency(concurrency) {
708
+ if (concurrency === undefined)
709
+ return Math.max(1, Math.min(DEFAULT_MAX_CONCURRENCY, availableParallelism()));
710
+ if (!Number.isInteger(concurrency) || concurrency < 1) {
711
+ throw new Error("Task concurrency must be a positive integer.");
712
+ }
713
+ return concurrency;
458
714
  }
459
715
  async function runFunctionStep(step, context, timeoutMs) {
460
716
  if (!timeoutMs) {
@@ -521,6 +777,77 @@ function runProcess(command, options) {
521
777
  });
522
778
  });
523
779
  }
780
+ function matchingAction(policy, argv) {
781
+ for (const rule of policy.rules) {
782
+ if (matchesCommandRule(rule, argv))
783
+ return rule.action;
784
+ }
785
+ return policy.fallback ?? { kind: "async-pipeline.command.allow" };
786
+ }
787
+ function matchesCommandRule(rule, argv) {
788
+ if (rule.exact && sameArgs(rule.exact, argv))
789
+ return true;
790
+ if (rule.prefix && hasPrefix(argv, rule.prefix))
791
+ return true;
792
+ return false;
793
+ }
794
+ function sameArgs(left, right) {
795
+ return left.length === right.length && left.every((value, index) => right[index] === value);
796
+ }
797
+ function hasPrefix(argv, prefix) {
798
+ return prefix.length <= argv.length && prefix.every((value, index) => argv[index] === value);
799
+ }
800
+ function commandStatus(action) {
801
+ if (action.kind === "async-pipeline.command.mock")
802
+ return "mocked";
803
+ if (action.kind === "async-pipeline.command.deny")
804
+ return "denied";
805
+ if (action.kind === "async-pipeline.command.requireApproval")
806
+ return "approval-required";
807
+ return "allowed";
808
+ }
809
+ async function runCommandAction(action, next) {
810
+ if (action.kind === "async-pipeline.command.allow")
811
+ return next();
812
+ if (action.kind === "async-pipeline.command.requireEnvironment")
813
+ return next();
814
+ if (action.kind === "async-pipeline.command.mock") {
815
+ return {
816
+ code: action.code ?? 0,
817
+ stdout: action.stdout ?? "",
818
+ stderr: action.stderr ?? ""
819
+ };
820
+ }
821
+ if (action.kind === "async-pipeline.command.deny") {
822
+ return {
823
+ code: 1,
824
+ stdout: "",
825
+ stderr: `${action.message ?? "Command denied by async-pipeline command policy."}\n`
826
+ };
827
+ }
828
+ return {
829
+ code: 1,
830
+ stdout: "",
831
+ stderr: `${action.message ?? "Command requires approval by async-pipeline command policy."}\n`
832
+ };
833
+ }
834
+ function applyOutputPolicy(output, policy, env) {
835
+ let next = policy.redactSecrets ? redactSecretValues(output, env) : output;
836
+ const maxBytes = policy.maxBytes;
837
+ if (maxBytes === undefined || Buffer.byteLength(next, "utf8") <= maxBytes)
838
+ return next;
839
+ const truncated = Buffer.from(next).subarray(0, maxBytes).toString("utf8");
840
+ return `${truncated}\n[async-pipeline] output truncated to ${maxBytes} bytes\n`;
841
+ }
842
+ function redactSecretValues(output, env) {
843
+ let redacted = output;
844
+ for (const [key, value] of Object.entries(env)) {
845
+ if (!value || value.length < 4 || !/(SECRET|TOKEN|PASSWORD|AUTH|KEY)/i.test(key))
846
+ continue;
847
+ redacted = redacted.split(value).join("[redacted]");
848
+ }
849
+ return redacted;
850
+ }
524
851
  function shellEscape(value) {
525
852
  return `'${value.replaceAll("'", "'\\''")}'`;
526
853
  }