@livingdata/pipex 0.0.9 → 0.0.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (88) hide show
  1. package/README.md +154 -14
  2. package/dist/__tests__/errors.js +162 -0
  3. package/dist/__tests__/helpers.js +41 -0
  4. package/dist/__tests__/types.js +8 -0
  5. package/dist/cli/__tests__/condition.js +23 -0
  6. package/dist/cli/__tests__/dag.js +154 -0
  7. package/dist/cli/__tests__/pipeline-loader.js +267 -0
  8. package/dist/cli/__tests__/pipeline-runner.js +257 -0
  9. package/dist/cli/__tests__/state-persistence.js +80 -0
  10. package/dist/cli/__tests__/state.js +58 -0
  11. package/dist/cli/__tests__/step-runner.js +116 -0
  12. package/dist/cli/commands/bundle.js +35 -0
  13. package/dist/cli/commands/cat.js +54 -0
  14. package/dist/cli/commands/exec.js +89 -0
  15. package/dist/cli/commands/export.js +2 -2
  16. package/dist/cli/commands/inspect.js +1 -1
  17. package/dist/cli/commands/list.js +2 -1
  18. package/dist/cli/commands/logs.js +1 -1
  19. package/dist/cli/commands/prune.js +1 -1
  20. package/dist/cli/commands/rm-step.js +41 -0
  21. package/dist/cli/commands/run-bundle.js +59 -0
  22. package/dist/cli/commands/run.js +9 -4
  23. package/dist/cli/commands/show.js +42 -7
  24. package/dist/cli/condition.js +11 -0
  25. package/dist/cli/dag.js +143 -0
  26. package/dist/cli/index.js +6 -0
  27. package/dist/cli/interactive-reporter.js +227 -0
  28. package/dist/cli/pipeline-loader.js +10 -110
  29. package/dist/cli/pipeline-runner.js +164 -78
  30. package/dist/cli/reporter.js +2 -158
  31. package/dist/cli/state.js +8 -0
  32. package/dist/cli/step-loader.js +25 -0
  33. package/dist/cli/step-resolver.js +111 -0
  34. package/dist/cli/step-runner.js +226 -0
  35. package/dist/cli/utils.js +0 -46
  36. package/dist/core/__tests__/bundle.js +663 -0
  37. package/dist/core/__tests__/condition.js +23 -0
  38. package/dist/core/__tests__/dag.js +154 -0
  39. package/dist/core/__tests__/env-file.test.js +41 -0
  40. package/dist/core/__tests__/event-aggregator.js +244 -0
  41. package/dist/core/__tests__/pipeline-loader.js +267 -0
  42. package/dist/core/__tests__/pipeline-runner.js +257 -0
  43. package/dist/core/__tests__/state-persistence.js +80 -0
  44. package/dist/core/__tests__/state.js +58 -0
  45. package/dist/core/__tests__/step-runner.js +118 -0
  46. package/dist/core/__tests__/stream-reporter.js +142 -0
  47. package/dist/core/__tests__/transport.js +50 -0
  48. package/dist/core/__tests__/utils.js +40 -0
  49. package/dist/core/bundle.js +130 -0
  50. package/dist/core/condition.js +11 -0
  51. package/dist/core/dag.js +143 -0
  52. package/dist/core/env-file.js +6 -0
  53. package/dist/core/event-aggregator.js +114 -0
  54. package/dist/core/index.js +14 -0
  55. package/dist/core/pipeline-loader.js +81 -0
  56. package/dist/core/pipeline-runner.js +360 -0
  57. package/dist/core/reporter.js +11 -0
  58. package/dist/core/state.js +110 -0
  59. package/dist/core/step-loader.js +25 -0
  60. package/dist/core/step-resolver.js +117 -0
  61. package/dist/core/step-runner.js +225 -0
  62. package/dist/core/stream-reporter.js +41 -0
  63. package/dist/core/transport.js +9 -0
  64. package/dist/core/utils.js +56 -0
  65. package/dist/engine/__tests__/workspace.js +288 -0
  66. package/dist/engine/docker-executor.js +10 -2
  67. package/dist/engine/index.js +1 -0
  68. package/dist/engine/workspace.js +76 -12
  69. package/dist/errors.js +122 -0
  70. package/dist/index.js +3 -0
  71. package/dist/kits/__tests__/index.js +23 -0
  72. package/dist/kits/builtin/__tests__/node.js +74 -0
  73. package/dist/kits/builtin/__tests__/python.js +67 -0
  74. package/dist/kits/builtin/__tests__/shell.js +74 -0
  75. package/dist/kits/builtin/node.js +10 -5
  76. package/dist/kits/builtin/python.js +10 -5
  77. package/dist/kits/builtin/shell.js +2 -1
  78. package/dist/kits/index.js +2 -1
  79. package/package.json +6 -3
  80. package/dist/cli/types.js +0 -3
  81. package/dist/engine/docker-runtime.js +0 -65
  82. package/dist/engine/runtime.js +0 -2
  83. package/dist/kits/bash.js +0 -19
  84. package/dist/kits/builtin/bash.js +0 -19
  85. package/dist/kits/node.js +0 -56
  86. package/dist/kits/python.js +0 -51
  87. package/dist/kits/types.js +0 -1
  88. package/dist/reporter.js +0 -13
@@ -0,0 +1,114 @@
1
+ /**
2
+ * Reconstructs pipeline session state from a stream of TransportMessages.
3
+ */
4
+ export class EventAggregator {
5
+ sessions = new Map();
6
+ consume(message) {
7
+ const { event } = message;
8
+ if (event.event === 'STEP_LOG') {
9
+ return;
10
+ }
11
+ const { jobId } = event;
12
+ switch (event.event) {
13
+ case 'PIPELINE_START': {
14
+ const session = {
15
+ workspaceId: event.workspaceId,
16
+ jobId,
17
+ pipelineName: event.pipelineName,
18
+ status: 'running',
19
+ startedAt: message.timestamp,
20
+ steps: new Map()
21
+ };
22
+ for (const step of event.steps) {
23
+ session.steps.set(step.id, {
24
+ id: step.id,
25
+ displayName: step.displayName,
26
+ status: 'pending'
27
+ });
28
+ }
29
+ this.sessions.set(jobId, session);
30
+ break;
31
+ }
32
+ case 'STEP_STARTING': {
33
+ const step = this.getOrCreateStep(jobId, event.step.id, event.step.displayName);
34
+ step.status = 'running';
35
+ break;
36
+ }
37
+ case 'STEP_SKIPPED': {
38
+ const step = this.getOrCreateStep(jobId, event.step.id, event.step.displayName);
39
+ step.status = 'skipped';
40
+ if (event.runId) {
41
+ step.runId = event.runId;
42
+ }
43
+ break;
44
+ }
45
+ case 'STEP_FINISHED': {
46
+ const step = this.getOrCreateStep(jobId, event.step.id, event.step.displayName);
47
+ step.status = 'finished';
48
+ if (event.runId) {
49
+ step.runId = event.runId;
50
+ }
51
+ if (event.durationMs !== undefined) {
52
+ step.durationMs = event.durationMs;
53
+ }
54
+ if (event.artifactSize !== undefined) {
55
+ step.artifactSize = event.artifactSize;
56
+ }
57
+ break;
58
+ }
59
+ case 'STEP_FAILED': {
60
+ const step = this.getOrCreateStep(jobId, event.step.id, event.step.displayName);
61
+ step.status = 'failed';
62
+ step.exitCode = event.exitCode;
63
+ break;
64
+ }
65
+ case 'PIPELINE_FINISHED': {
66
+ const session = this.sessions.get(jobId);
67
+ if (session) {
68
+ session.status = 'completed';
69
+ session.finishedAt = message.timestamp;
70
+ }
71
+ break;
72
+ }
73
+ case 'PIPELINE_FAILED': {
74
+ const session = this.sessions.get(jobId);
75
+ if (session) {
76
+ session.status = 'failed';
77
+ session.finishedAt = message.timestamp;
78
+ }
79
+ break;
80
+ }
81
+ case 'STEP_RETRYING':
82
+ case 'STEP_WOULD_RUN': {
83
+ break;
84
+ }
85
+ }
86
+ }
87
+ getSession(jobId) {
88
+ return this.sessions.get(jobId);
89
+ }
90
+ getAllSessions() {
91
+ return [...this.sessions.values()];
92
+ }
93
+ clear() {
94
+ this.sessions.clear();
95
+ }
96
+ getOrCreateStep(jobId, stepId, displayName) {
97
+ let session = this.sessions.get(jobId);
98
+ if (!session) {
99
+ session = {
100
+ workspaceId: '',
101
+ jobId,
102
+ status: 'running',
103
+ steps: new Map()
104
+ };
105
+ this.sessions.set(jobId, session);
106
+ }
107
+ let step = session.steps.get(stepId);
108
+ if (!step) {
109
+ step = { id: stepId, displayName, status: 'pending' };
110
+ session.steps.set(stepId, step);
111
+ }
112
+ return step;
113
+ }
114
+ }
@@ -0,0 +1,14 @@
1
+ export { PipelineRunner } from './pipeline-runner.js';
2
+ export { StepRunner } from './step-runner.js';
3
+ export { PipelineLoader } from './pipeline-loader.js';
4
+ export { loadStepFile } from './step-loader.js';
5
+ export { resolveStep, validateStep } from './step-resolver.js';
6
+ export { StateManager } from './state.js';
7
+ export { ConsoleReporter } from './reporter.js';
8
+ export { StreamReporter, CompositeReporter } from './stream-reporter.js';
9
+ export { InMemoryTransport } from './transport.js';
10
+ export { EventAggregator } from './event-aggregator.js';
11
+ export { buildGraph, validateGraph, topologicalLevels, subgraph, leafNodes } from './dag.js';
12
+ export { collectDependencies, buildIgnoreFilter, buildBundle, extractBundle } from './bundle.js';
13
+ export { evaluateCondition } from './condition.js';
14
+ export { dirSize, formatSize, formatDuration } from './utils.js';
@@ -0,0 +1,81 @@
1
+ import { readFile } from 'node:fs/promises';
2
+ import { extname } from 'node:path';
3
+ import { deburr } from 'lodash-es';
4
+ import { parse as parseYaml } from 'yaml';
5
+ import { ValidationError } from '../errors.js';
6
+ import { buildGraph, validateGraph } from './dag.js';
7
+ import { resolveStep, validateStep } from './step-resolver.js';
8
+ export class PipelineLoader {
9
+ async load(filePath) {
10
+ const content = await readFile(filePath, 'utf8');
11
+ return this.parse(content, filePath);
12
+ }
13
+ parse(content, filePath) {
14
+ const input = parsePipelineFile(content, filePath);
15
+ if (!input.id && !input.name) {
16
+ throw new ValidationError('Invalid pipeline: at least one of "id" or "name" must be defined');
17
+ }
18
+ const pipelineId = input.id ?? slugify(input.name);
19
+ if (!Array.isArray(input.steps) || input.steps.length === 0) {
20
+ throw new ValidationError('Invalid pipeline: steps must be a non-empty array');
21
+ }
22
+ const steps = input.steps.map(step => resolveStep(step));
23
+ for (const step of steps) {
24
+ validateStep(step);
25
+ }
26
+ this.validateUniqueStepIds(steps);
27
+ const graph = buildGraph(steps);
28
+ validateGraph(graph, steps);
29
+ return { id: pipelineId, name: input.name, steps };
30
+ }
31
+ validateUniqueStepIds(steps) {
32
+ const seen = new Set();
33
+ for (const step of steps) {
34
+ if (seen.has(step.id)) {
35
+ throw new ValidationError(`Duplicate step id: '${step.id}'`);
36
+ }
37
+ seen.add(step.id);
38
+ }
39
+ }
40
+ }
41
+ /** Convert a free-form name into a valid identifier. */
42
+ export function slugify(name) {
43
+ return deburr(name)
44
+ .toLowerCase()
45
+ .replaceAll(/[^\w-]/g, '-')
46
+ .replaceAll(/-{2,}/g, '-')
47
+ .replace(/^-/, '')
48
+ .replace(/-$/, '');
49
+ }
50
+ export function parsePipelineFile(content, filePath) {
51
+ const ext = extname(filePath).toLowerCase();
52
+ if (ext === '.yaml' || ext === '.yml') {
53
+ return parseYaml(content);
54
+ }
55
+ return JSON.parse(content);
56
+ }
57
+ export function mergeEnv(kitEnv, userEnv) {
58
+ if (!kitEnv && !userEnv) {
59
+ return undefined;
60
+ }
61
+ return { ...kitEnv, ...userEnv };
62
+ }
63
+ export function mergeCaches(kitCaches, userCaches) {
64
+ if (!kitCaches && !userCaches) {
65
+ return undefined;
66
+ }
67
+ const map = new Map();
68
+ for (const c of kitCaches ?? []) {
69
+ map.set(c.name, c);
70
+ }
71
+ for (const c of userCaches ?? []) {
72
+ map.set(c.name, c);
73
+ }
74
+ return [...map.values()];
75
+ }
76
+ export function mergeMounts(kitMounts, userMounts) {
77
+ if (!kitMounts && !userMounts) {
78
+ return undefined;
79
+ }
80
+ return [...(kitMounts ?? []), ...(userMounts ?? [])];
81
+ }
@@ -0,0 +1,360 @@
1
+ import process from 'node:process';
2
+ import { randomUUID } from 'node:crypto';
3
+ import { cpus } from 'node:os';
4
+ import { cp, writeFile } from 'node:fs/promises';
5
+ import { setTimeout } from 'node:timers/promises';
6
+ import { createWriteStream } from 'node:fs';
7
+ import { dirname, join, resolve } from 'node:path';
8
+ import { Workspace } from '../engine/index.js';
9
+ import { ContainerCrashError, PipexError } from '../errors.js';
10
+ import { loadEnvFile } from './env-file.js';
11
+ import { buildGraph, topologicalLevels, subgraph, leafNodes } from './dag.js';
12
+ import { evaluateCondition } from './condition.js';
13
+ import { StateManager } from './state.js';
14
+ import { dirSize, resolveHostPath } from './utils.js';
15
+ /**
16
+ * Orchestrates pipeline execution with DAG-based parallel execution and caching.
17
+ */
18
+ export class PipelineRunner {
19
+ loader;
20
+ runtime;
21
+ reporter;
22
+ workdirRoot;
23
+ constructor(loader, runtime, reporter, workdirRoot) {
24
+ this.loader = loader;
25
+ this.runtime = runtime;
26
+ this.reporter = reporter;
27
+ this.workdirRoot = workdirRoot;
28
+ }
29
+ async run(pipelineFilePath, options) {
30
+ const { workspace: workspaceName, force, dryRun, target, concurrency, envFile } = options ?? {};
31
+ const config = await this.loader.load(pipelineFilePath);
32
+ const pipelineRoot = dirname(resolve(pipelineFilePath));
33
+ const workspaceId = workspaceName ?? config.id;
34
+ let workspace;
35
+ try {
36
+ workspace = await Workspace.open(this.workdirRoot, workspaceId);
37
+ }
38
+ catch {
39
+ workspace = await Workspace.create(this.workdirRoot, workspaceId);
40
+ }
41
+ await workspace.cleanupStaging();
42
+ await workspace.cleanupRunning();
43
+ if (!dryRun) {
44
+ await this.runtime.check();
45
+ await this.runtime.cleanupContainers(workspace.id);
46
+ }
47
+ const cliEnv = envFile ? await loadEnvFile(resolve(envFile)) : undefined;
48
+ const state = new StateManager(workspace.root);
49
+ await state.load();
50
+ const stepRuns = new Map();
51
+ let totalArtifactSize = 0;
52
+ const job = { workspaceId: workspace.id, jobId: randomUUID() };
53
+ // Build DAG and determine execution scope
54
+ const graph = buildGraph(config.steps);
55
+ const targets = target ?? leafNodes(graph);
56
+ const activeSteps = subgraph(graph, targets);
57
+ this.reporter.emit({
58
+ ...job,
59
+ event: 'PIPELINE_START',
60
+ pipelineName: config.name ?? config.id,
61
+ steps: config.steps
62
+ .filter(s => activeSteps.has(s.id))
63
+ .map(s => ({ id: s.id, displayName: s.name ?? s.id }))
64
+ });
65
+ const levels = topologicalLevels(graph)
66
+ .map(level => level.filter(id => activeSteps.has(id)))
67
+ .filter(level => level.length > 0);
68
+ const stepMap = new Map(config.steps.map(s => [s.id, s]));
69
+ const failed = new Set();
70
+ const skipped = new Set();
71
+ const maxConcurrency = concurrency ?? cpus().length;
72
+ for (const level of levels) {
73
+ const tasks = level.map(stepId => async () => {
74
+ const step = stepMap.get(stepId);
75
+ const stepRef = { id: step.id, displayName: step.name ?? step.id };
76
+ // Check if blocked by a failed/skipped required dependency
77
+ if (this.isDependencyBlocked(step, failed, skipped)) {
78
+ skipped.add(step.id);
79
+ this.reporter.emit({ ...job, event: 'STEP_SKIPPED', step: stepRef, reason: 'dependency' });
80
+ return 0;
81
+ }
82
+ // Evaluate condition
83
+ if (step.if) {
84
+ const conditionMet = await evaluateCondition(step.if, { env: process.env });
85
+ if (!conditionMet) {
86
+ skipped.add(step.id);
87
+ this.reporter.emit({ ...job, event: 'STEP_SKIPPED', step: stepRef, reason: 'condition' });
88
+ return 0;
89
+ }
90
+ }
91
+ // Resolve env vars: CLI envFile < step envFile < step inline env
92
+ const stepFileEnv = step.envFile
93
+ ? await loadEnvFile(resolve(pipelineRoot, step.envFile))
94
+ : undefined;
95
+ const resolvedEnv = (cliEnv ?? stepFileEnv ?? step.env)
96
+ ? { ...cliEnv, ...stepFileEnv, ...step.env }
97
+ : undefined;
98
+ // Compute fingerprint
99
+ const inputRunIds = step.inputs
100
+ ?.map(i => stepRuns.get(i.step))
101
+ .filter((id) => id !== undefined);
102
+ const resolvedMounts = step.mounts?.map(m => ({
103
+ hostPath: resolveHostPath(pipelineRoot, m.host),
104
+ containerPath: m.container
105
+ }));
106
+ const currentFingerprint = StateManager.fingerprint({
107
+ image: step.image,
108
+ cmd: step.cmd,
109
+ env: resolvedEnv,
110
+ inputRunIds,
111
+ mounts: resolvedMounts
112
+ });
113
+ // Cache check
114
+ const skipCache = force === true || (Array.isArray(force) && force.includes(step.id));
115
+ if (!skipCache && await this.tryUseCache({ workspace, state, step, stepRef, currentFingerprint, stepRuns, job })) {
116
+ return 0;
117
+ }
118
+ // Dry run
119
+ if (dryRun) {
120
+ this.reporter.emit({ ...job, event: 'STEP_WOULD_RUN', step: stepRef });
121
+ return 0;
122
+ }
123
+ // Execute
124
+ this.reporter.emit({ ...job, event: 'STEP_STARTING', step: stepRef });
125
+ return this.executeStep({ workspace, state, step, stepRef, stepRuns, currentFingerprint, resolvedMounts, pipelineRoot, job, resolvedEnv });
126
+ });
127
+ const results = await withConcurrency(tasks, maxConcurrency);
128
+ // Collect results
129
+ for (const [i, result] of results.entries()) {
130
+ const stepId = level[i];
131
+ if (result.status === 'fulfilled') {
132
+ totalArtifactSize += result.value;
133
+ }
134
+ else if (!failed.has(stepId)) {
135
+ // Step threw an error (ContainerCrashError if not allowFailure)
136
+ failed.add(stepId);
137
+ }
138
+ }
139
+ await state.save();
140
+ }
141
+ if (failed.size > 0) {
142
+ this.reporter.emit({ ...job, event: 'PIPELINE_FAILED' });
143
+ // Re-throw the first failure to signal pipeline failure to the CLI
144
+ const firstFailedId = [...failed][0];
145
+ throw new ContainerCrashError(firstFailedId, 1);
146
+ }
147
+ this.reporter.emit({ ...job, event: 'PIPELINE_FINISHED', totalArtifactSize });
148
+ }
149
+ isDependencyBlocked(step, failed, skippedSteps) {
150
+ if (!step.inputs) {
151
+ return false;
152
+ }
153
+ return step.inputs.some(input => !input.optional && (failed.has(input.step) || skippedSteps.has(input.step)));
154
+ }
155
+ async executeStep({ workspace, state, step, stepRef, stepRuns, currentFingerprint, resolvedMounts, pipelineRoot, job, resolvedEnv }) {
156
+ const runId = workspace.generateRunId();
157
+ const stagingPath = await workspace.prepareRun(runId);
158
+ await workspace.markStepRunning(step.id, { startedAt: new Date().toISOString(), pid: process.pid, stepName: step.name });
159
+ await this.prepareStagingWithInputs(workspace, step, workspace.runStagingArtifactsPath(runId), stepRuns);
160
+ if (step.caches) {
161
+ for (const cache of step.caches) {
162
+ await workspace.prepareCache(cache.name);
163
+ }
164
+ }
165
+ const { inputs, output, caches, mounts } = this.buildMounts(step, runId, stepRuns, pipelineRoot);
166
+ const stdoutLog = createWriteStream(join(stagingPath, 'stdout.log'));
167
+ const stderrLog = createWriteStream(join(stagingPath, 'stderr.log'));
168
+ try {
169
+ const maxRetries = step.retries ?? 0;
170
+ const retryDelay = step.retryDelayMs ?? 5000;
171
+ let result;
172
+ for (let attempt = 0; attempt <= maxRetries; attempt++) {
173
+ try {
174
+ result = await this.runtime.run(workspace, {
175
+ name: `pipex-${workspace.id}-${step.id}-${Date.now()}`,
176
+ image: step.image,
177
+ cmd: step.cmd,
178
+ env: resolvedEnv,
179
+ inputs,
180
+ output,
181
+ caches,
182
+ mounts,
183
+ sources: step.sources?.map(m => ({
184
+ hostPath: resolveHostPath(pipelineRoot, m.host),
185
+ containerPath: m.container
186
+ })),
187
+ network: step.allowNetwork ? 'bridge' : 'none',
188
+ timeoutSec: step.timeoutSec
189
+ }, ({ stream, line }) => {
190
+ if (stream === 'stdout') {
191
+ stdoutLog.write(line + '\n');
192
+ }
193
+ else {
194
+ stderrLog.write(line + '\n');
195
+ }
196
+ this.reporter.emit({ ...job, event: 'STEP_LOG', step: stepRef, stream, line });
197
+ });
198
+ break;
199
+ }
200
+ catch (error) {
201
+ if (error instanceof PipexError && error.transient && attempt < maxRetries) {
202
+ this.reporter.emit({ ...job, event: 'STEP_RETRYING', step: stepRef, attempt: attempt + 1, maxRetries });
203
+ await setTimeout(retryDelay);
204
+ continue;
205
+ }
206
+ throw error;
207
+ }
208
+ }
209
+ await closeStream(stdoutLog);
210
+ await closeStream(stderrLog);
211
+ await this.writeRunMeta(stagingPath, { runId, step, stepRuns, resolvedMounts, currentFingerprint, result, resolvedEnv });
212
+ if (result.exitCode === 0 || step.allowFailure) {
213
+ await workspace.commitRun(runId);
214
+ await workspace.linkRun(step.id, runId);
215
+ stepRuns.set(step.id, runId);
216
+ state.setStep(step.id, runId, currentFingerprint);
217
+ const durationMs = result.finishedAt.getTime() - result.startedAt.getTime();
218
+ const artifactSize = await dirSize(workspace.runArtifactsPath(runId));
219
+ this.reporter.emit({ ...job, event: 'STEP_FINISHED', step: stepRef, runId, durationMs, artifactSize });
220
+ return artifactSize;
221
+ }
222
+ await workspace.commitRun(runId);
223
+ await workspace.linkRun(step.id, runId);
224
+ state.setStep(step.id, runId, '');
225
+ this.reporter.emit({ ...job, event: 'STEP_FAILED', step: stepRef, exitCode: result.exitCode });
226
+ throw new ContainerCrashError(step.id, result.exitCode);
227
+ }
228
+ catch (error) {
229
+ await closeStream(stdoutLog);
230
+ await closeStream(stderrLog);
231
+ throw error;
232
+ }
233
+ finally {
234
+ await workspace.markStepDone(step.id);
235
+ }
236
+ }
237
+ async writeRunMeta(stagingPath, { runId, step, stepRuns, resolvedMounts, currentFingerprint, result, resolvedEnv }) {
238
+ const meta = {
239
+ runId,
240
+ stepId: step.id,
241
+ stepName: step.name,
242
+ startedAt: result.startedAt.toISOString(),
243
+ finishedAt: result.finishedAt.toISOString(),
244
+ durationMs: result.finishedAt.getTime() - result.startedAt.getTime(),
245
+ exitCode: result.exitCode,
246
+ image: step.image,
247
+ cmd: step.cmd,
248
+ env: resolvedEnv,
249
+ inputs: step.inputs?.map(i => ({
250
+ step: i.step,
251
+ runId: stepRuns.get(i.step),
252
+ mountedAs: `/input/${i.step}`
253
+ })),
254
+ mounts: resolvedMounts,
255
+ caches: step.caches?.map(c => c.name),
256
+ allowNetwork: step.allowNetwork ?? false,
257
+ fingerprint: currentFingerprint,
258
+ status: result.exitCode === 0 ? 'success' : 'failure'
259
+ };
260
+ await writeFile(join(stagingPath, 'meta.json'), JSON.stringify(meta, null, 2), 'utf8');
261
+ }
262
+ async tryUseCache({ workspace, state, step, stepRef, currentFingerprint, stepRuns, job }) {
263
+ const cached = state.getStep(step.id);
264
+ if (cached?.fingerprint === currentFingerprint) {
265
+ try {
266
+ const runs = await workspace.listRuns();
267
+ if (runs.includes(cached.runId)) {
268
+ stepRuns.set(step.id, cached.runId);
269
+ await workspace.linkRun(step.id, cached.runId);
270
+ this.reporter.emit({ ...job, event: 'STEP_SKIPPED', step: stepRef, runId: cached.runId, reason: 'cached' });
271
+ return true;
272
+ }
273
+ }
274
+ catch {
275
+ // Run missing, proceed with execution
276
+ }
277
+ }
278
+ return false;
279
+ }
280
+ async prepareStagingWithInputs(workspace, step, stagingArtifactsPath, stepRuns) {
281
+ if (!step.inputs) {
282
+ return;
283
+ }
284
+ for (const input of step.inputs) {
285
+ const inputRunId = stepRuns.get(input.step);
286
+ if (!inputRunId) {
287
+ if (input.optional) {
288
+ continue;
289
+ }
290
+ // Non-optional input without a run — this shouldn't happen in DAG mode
291
+ // but keep the continue for safety (the step may still work with bind mounts)
292
+ continue;
293
+ }
294
+ if (input.copyToOutput) {
295
+ await cp(workspace.runArtifactsPath(inputRunId), stagingArtifactsPath, { recursive: true });
296
+ }
297
+ }
298
+ }
299
+ buildMounts(step, outputRunId, stepRuns, pipelineRoot) {
300
+ const inputs = [];
301
+ if (step.inputs) {
302
+ for (const input of step.inputs) {
303
+ const inputRunId = stepRuns.get(input.step);
304
+ if (inputRunId) {
305
+ inputs.push({
306
+ runId: inputRunId,
307
+ containerPath: `/input/${input.step}`
308
+ });
309
+ }
310
+ }
311
+ }
312
+ const output = {
313
+ stagingRunId: outputRunId,
314
+ containerPath: step.outputPath ?? '/output'
315
+ };
316
+ let caches;
317
+ if (step.caches) {
318
+ caches = step.caches.map(c => ({
319
+ name: c.name,
320
+ containerPath: c.path
321
+ }));
322
+ }
323
+ let mounts;
324
+ if (step.mounts) {
325
+ mounts = step.mounts.map(m => ({
326
+ hostPath: resolveHostPath(pipelineRoot, m.host),
327
+ containerPath: m.container
328
+ }));
329
+ }
330
+ return { inputs, output, caches, mounts };
331
+ }
332
+ }
333
+ async function withConcurrency(tasks, limit) {
334
+ const results = Array.from({ length: tasks.length });
335
+ let next = 0;
336
+ async function worker() {
337
+ while (next < tasks.length) {
338
+ const i = next++;
339
+ try {
340
+ results[i] = { status: 'fulfilled', value: await tasks[i]() };
341
+ }
342
+ catch (error) {
343
+ results[i] = { status: 'rejected', reason: error };
344
+ }
345
+ }
346
+ }
347
+ await Promise.all(Array.from({ length: Math.min(limit, tasks.length) }, async () => worker()));
348
+ return results;
349
+ }
350
+ async function closeStream(stream) {
351
+ if (stream.destroyed) {
352
+ return;
353
+ }
354
+ return new Promise((resolve, reject) => {
355
+ stream.end(() => {
356
+ resolve();
357
+ });
358
+ stream.on('error', reject);
359
+ });
360
+ }
@@ -0,0 +1,11 @@
1
+ import pino from 'pino';
2
+ /**
3
+ * Reporter that outputs structured JSON logs via pino.
4
+ * Suitable for CI/CD environments and log aggregation.
5
+ */
6
+ export class ConsoleReporter {
7
+ logger = pino({ level: 'info' });
8
+ emit(event) {
9
+ this.logger.info(event);
10
+ }
11
+ }
@@ -0,0 +1,110 @@
1
+ import { readFile, writeFile } from 'node:fs/promises';
2
+ import { join } from 'node:path';
3
+ import { createHash } from 'node:crypto';
4
+ /**
5
+ * Manages caching state for pipeline execution.
6
+ *
7
+ * The StateManager computes fingerprints for steps and tracks which
8
+ * run was produced by each step. This enables cache hits when
9
+ * a step's configuration hasn't changed.
10
+ *
11
+ * ## Fingerprint Algorithm
12
+ *
13
+ * A step fingerprint is computed as:
14
+ * ```
15
+ * SHA256(image + JSON(cmd) + JSON(sorted env) + JSON(sorted inputRunIds))
16
+ * ```
17
+ *
18
+ * A step is re-executed when:
19
+ * - The fingerprint changes (image, cmd, env, or inputs modified)
20
+ * - The run no longer exists (manually deleted)
21
+ *
22
+ * ## Cache Propagation
23
+ *
24
+ * Changes propagate through dependencies. If step A is modified,
25
+ * all steps depending on A are invalidated automatically (via inputRunIds).
26
+ */
27
+ export class StateManager {
28
+ static fingerprint(config) {
29
+ const hash = createHash('sha256');
30
+ hash.update(config.image);
31
+ hash.update(JSON.stringify(config.cmd));
32
+ if (config.env) {
33
+ hash.update(JSON.stringify(Object.entries(config.env).sort((a, b) => a[0].localeCompare(b[0]))));
34
+ }
35
+ if (config.inputRunIds) {
36
+ hash.update(JSON.stringify([...config.inputRunIds].sort((a, b) => a.localeCompare(b))));
37
+ }
38
+ if (config.mounts && config.mounts.length > 0) {
39
+ const sorted = [...config.mounts].sort((a, b) => a.containerPath.localeCompare(b.containerPath));
40
+ hash.update(JSON.stringify(sorted));
41
+ }
42
+ return hash.digest('hex');
43
+ }
44
+ state = { steps: {} };
45
+ path;
46
+ /**
47
+ * Creates a state manager for the given workspace.
48
+ * @param workspaceRoot - Absolute path to workspace directory
49
+ */
50
+ constructor(workspaceRoot) {
51
+ this.path = join(workspaceRoot, 'state.json');
52
+ }
53
+ /**
54
+ * Loads cached state from state.json.
55
+ * If the file doesn't exist, initializes with empty state.
56
+ */
57
+ async load() {
58
+ try {
59
+ const content = await readFile(this.path, 'utf8');
60
+ Object.assign(this.state, JSON.parse(content));
61
+ }
62
+ catch {
63
+ this.state.steps = {};
64
+ }
65
+ }
66
+ /**
67
+ * Persists current state to state.json.
68
+ */
69
+ async save() {
70
+ await writeFile(this.path, JSON.stringify(this.state, null, 2), 'utf8');
71
+ }
72
+ /**
73
+ * Retrieves cached state for a step.
74
+ * @param stepId - Step identifier
75
+ * @returns Cached state if available, undefined otherwise
76
+ */
77
+ getStep(stepId) {
78
+ return this.state.steps[stepId];
79
+ }
80
+ /**
81
+ * Lists all steps with their run IDs (in insertion order).
82
+ * @returns Array of {stepId, runId} entries
83
+ */
84
+ listSteps() {
85
+ return Object.entries(this.state.steps).map(([stepId, { runId }]) => ({ stepId, runId }));
86
+ }
87
+ /**
88
+ * Returns the set of run IDs currently referenced by state.
89
+ */
90
+ activeRunIds() {
91
+ return new Set(Object.values(this.state.steps).map(s => s.runId));
92
+ }
93
+ /**
94
+ * Updates cached state for a step.
95
+ * @param stepId - Step identifier
96
+ * @param runId - Run produced by the step
97
+ * @param fingerprint - Step configuration fingerprint
98
+ */
99
+ setStep(stepId, runId, fingerprint) {
100
+ this.state.steps[stepId] = { runId, fingerprint };
101
+ }
102
+ /**
103
+ * Removes a step's cached state.
104
+ * @param stepId - Step identifier
105
+ */
106
+ removeStep(stepId) {
107
+ // eslint-disable-next-line @typescript-eslint/no-dynamic-delete
108
+ delete this.state.steps[stepId];
109
+ }
110
+ }