@livingdata/pipex 0.0.8 → 0.0.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +186 -16
- package/dist/__tests__/errors.js +162 -0
- package/dist/__tests__/helpers.js +41 -0
- package/dist/__tests__/types.js +8 -0
- package/dist/cli/__tests__/condition.js +23 -0
- package/dist/cli/__tests__/dag.js +154 -0
- package/dist/cli/__tests__/pipeline-loader.js +267 -0
- package/dist/cli/__tests__/pipeline-runner.js +257 -0
- package/dist/cli/__tests__/state-persistence.js +80 -0
- package/dist/cli/__tests__/state.js +58 -0
- package/dist/cli/__tests__/step-runner.js +116 -0
- package/dist/cli/commands/bundle.js +35 -0
- package/dist/cli/commands/cat.js +54 -0
- package/dist/cli/commands/clean.js +22 -0
- package/dist/cli/commands/exec.js +89 -0
- package/dist/cli/commands/export.js +32 -0
- package/dist/cli/commands/inspect.js +58 -0
- package/dist/cli/commands/list.js +39 -0
- package/dist/cli/commands/logs.js +54 -0
- package/dist/cli/commands/prune.js +26 -0
- package/dist/cli/commands/rm-step.js +41 -0
- package/dist/cli/commands/rm.js +27 -0
- package/dist/cli/commands/run-bundle.js +59 -0
- package/dist/cli/commands/run.js +44 -0
- package/dist/cli/commands/show.js +108 -0
- package/dist/cli/condition.js +11 -0
- package/dist/cli/dag.js +143 -0
- package/dist/cli/index.js +24 -105
- package/dist/cli/interactive-reporter.js +227 -0
- package/dist/cli/pipeline-loader.js +10 -110
- package/dist/cli/pipeline-runner.js +256 -111
- package/dist/cli/reporter.js +2 -107
- package/dist/cli/state.js +30 -9
- package/dist/cli/step-loader.js +25 -0
- package/dist/cli/step-resolver.js +111 -0
- package/dist/cli/step-runner.js +226 -0
- package/dist/cli/utils.js +3 -0
- package/dist/core/__tests__/bundle.js +663 -0
- package/dist/core/__tests__/condition.js +23 -0
- package/dist/core/__tests__/dag.js +154 -0
- package/dist/core/__tests__/env-file.test.js +41 -0
- package/dist/core/__tests__/event-aggregator.js +244 -0
- package/dist/core/__tests__/pipeline-loader.js +267 -0
- package/dist/core/__tests__/pipeline-runner.js +257 -0
- package/dist/core/__tests__/state-persistence.js +80 -0
- package/dist/core/__tests__/state.js +58 -0
- package/dist/core/__tests__/step-runner.js +118 -0
- package/dist/core/__tests__/stream-reporter.js +142 -0
- package/dist/core/__tests__/transport.js +50 -0
- package/dist/core/__tests__/utils.js +40 -0
- package/dist/core/bundle.js +130 -0
- package/dist/core/condition.js +11 -0
- package/dist/core/dag.js +143 -0
- package/dist/core/env-file.js +6 -0
- package/dist/core/event-aggregator.js +114 -0
- package/dist/core/index.js +14 -0
- package/dist/core/pipeline-loader.js +81 -0
- package/dist/core/pipeline-runner.js +360 -0
- package/dist/core/reporter.js +11 -0
- package/dist/core/state.js +110 -0
- package/dist/core/step-loader.js +25 -0
- package/dist/core/step-resolver.js +117 -0
- package/dist/core/step-runner.js +225 -0
- package/dist/core/stream-reporter.js +41 -0
- package/dist/core/transport.js +9 -0
- package/dist/core/utils.js +56 -0
- package/dist/engine/__tests__/workspace.js +288 -0
- package/dist/engine/docker-executor.js +32 -6
- package/dist/engine/index.js +1 -0
- package/dist/engine/workspace.js +164 -66
- package/dist/errors.js +122 -0
- package/dist/index.js +3 -0
- package/dist/kits/__tests__/index.js +23 -0
- package/dist/kits/builtin/__tests__/node.js +74 -0
- package/dist/kits/builtin/__tests__/python.js +67 -0
- package/dist/kits/builtin/__tests__/shell.js +74 -0
- package/dist/kits/builtin/node.js +10 -5
- package/dist/kits/builtin/python.js +10 -5
- package/dist/kits/builtin/shell.js +2 -1
- package/dist/kits/index.js +2 -1
- package/package.json +6 -3
- package/dist/cli/types.js +0 -3
- package/dist/engine/docker-runtime.js +0 -65
- package/dist/engine/runtime.js +0 -2
- package/dist/kits/bash.js +0 -19
- package/dist/kits/builtin/bash.js +0 -19
- package/dist/kits/node.js +0 -56
- package/dist/kits/python.js +0 -51
- package/dist/kits/types.js +0 -1
- package/dist/reporter.js +0 -13
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Reconstructs pipeline session state from a stream of TransportMessages.
|
|
3
|
+
*/
|
|
4
|
+
export class EventAggregator {
|
|
5
|
+
sessions = new Map();
|
|
6
|
+
consume(message) {
|
|
7
|
+
const { event } = message;
|
|
8
|
+
if (event.event === 'STEP_LOG') {
|
|
9
|
+
return;
|
|
10
|
+
}
|
|
11
|
+
const { jobId } = event;
|
|
12
|
+
switch (event.event) {
|
|
13
|
+
case 'PIPELINE_START': {
|
|
14
|
+
const session = {
|
|
15
|
+
workspaceId: event.workspaceId,
|
|
16
|
+
jobId,
|
|
17
|
+
pipelineName: event.pipelineName,
|
|
18
|
+
status: 'running',
|
|
19
|
+
startedAt: message.timestamp,
|
|
20
|
+
steps: new Map()
|
|
21
|
+
};
|
|
22
|
+
for (const step of event.steps) {
|
|
23
|
+
session.steps.set(step.id, {
|
|
24
|
+
id: step.id,
|
|
25
|
+
displayName: step.displayName,
|
|
26
|
+
status: 'pending'
|
|
27
|
+
});
|
|
28
|
+
}
|
|
29
|
+
this.sessions.set(jobId, session);
|
|
30
|
+
break;
|
|
31
|
+
}
|
|
32
|
+
case 'STEP_STARTING': {
|
|
33
|
+
const step = this.getOrCreateStep(jobId, event.step.id, event.step.displayName);
|
|
34
|
+
step.status = 'running';
|
|
35
|
+
break;
|
|
36
|
+
}
|
|
37
|
+
case 'STEP_SKIPPED': {
|
|
38
|
+
const step = this.getOrCreateStep(jobId, event.step.id, event.step.displayName);
|
|
39
|
+
step.status = 'skipped';
|
|
40
|
+
if (event.runId) {
|
|
41
|
+
step.runId = event.runId;
|
|
42
|
+
}
|
|
43
|
+
break;
|
|
44
|
+
}
|
|
45
|
+
case 'STEP_FINISHED': {
|
|
46
|
+
const step = this.getOrCreateStep(jobId, event.step.id, event.step.displayName);
|
|
47
|
+
step.status = 'finished';
|
|
48
|
+
if (event.runId) {
|
|
49
|
+
step.runId = event.runId;
|
|
50
|
+
}
|
|
51
|
+
if (event.durationMs !== undefined) {
|
|
52
|
+
step.durationMs = event.durationMs;
|
|
53
|
+
}
|
|
54
|
+
if (event.artifactSize !== undefined) {
|
|
55
|
+
step.artifactSize = event.artifactSize;
|
|
56
|
+
}
|
|
57
|
+
break;
|
|
58
|
+
}
|
|
59
|
+
case 'STEP_FAILED': {
|
|
60
|
+
const step = this.getOrCreateStep(jobId, event.step.id, event.step.displayName);
|
|
61
|
+
step.status = 'failed';
|
|
62
|
+
step.exitCode = event.exitCode;
|
|
63
|
+
break;
|
|
64
|
+
}
|
|
65
|
+
case 'PIPELINE_FINISHED': {
|
|
66
|
+
const session = this.sessions.get(jobId);
|
|
67
|
+
if (session) {
|
|
68
|
+
session.status = 'completed';
|
|
69
|
+
session.finishedAt = message.timestamp;
|
|
70
|
+
}
|
|
71
|
+
break;
|
|
72
|
+
}
|
|
73
|
+
case 'PIPELINE_FAILED': {
|
|
74
|
+
const session = this.sessions.get(jobId);
|
|
75
|
+
if (session) {
|
|
76
|
+
session.status = 'failed';
|
|
77
|
+
session.finishedAt = message.timestamp;
|
|
78
|
+
}
|
|
79
|
+
break;
|
|
80
|
+
}
|
|
81
|
+
case 'STEP_RETRYING':
|
|
82
|
+
case 'STEP_WOULD_RUN': {
|
|
83
|
+
break;
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
getSession(jobId) {
|
|
88
|
+
return this.sessions.get(jobId);
|
|
89
|
+
}
|
|
90
|
+
getAllSessions() {
|
|
91
|
+
return [...this.sessions.values()];
|
|
92
|
+
}
|
|
93
|
+
clear() {
|
|
94
|
+
this.sessions.clear();
|
|
95
|
+
}
|
|
96
|
+
getOrCreateStep(jobId, stepId, displayName) {
|
|
97
|
+
let session = this.sessions.get(jobId);
|
|
98
|
+
if (!session) {
|
|
99
|
+
session = {
|
|
100
|
+
workspaceId: '',
|
|
101
|
+
jobId,
|
|
102
|
+
status: 'running',
|
|
103
|
+
steps: new Map()
|
|
104
|
+
};
|
|
105
|
+
this.sessions.set(jobId, session);
|
|
106
|
+
}
|
|
107
|
+
let step = session.steps.get(stepId);
|
|
108
|
+
if (!step) {
|
|
109
|
+
step = { id: stepId, displayName, status: 'pending' };
|
|
110
|
+
session.steps.set(stepId, step);
|
|
111
|
+
}
|
|
112
|
+
return step;
|
|
113
|
+
}
|
|
114
|
+
}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
export { PipelineRunner } from './pipeline-runner.js';
|
|
2
|
+
export { StepRunner } from './step-runner.js';
|
|
3
|
+
export { PipelineLoader } from './pipeline-loader.js';
|
|
4
|
+
export { loadStepFile } from './step-loader.js';
|
|
5
|
+
export { resolveStep, validateStep } from './step-resolver.js';
|
|
6
|
+
export { StateManager } from './state.js';
|
|
7
|
+
export { ConsoleReporter } from './reporter.js';
|
|
8
|
+
export { StreamReporter, CompositeReporter } from './stream-reporter.js';
|
|
9
|
+
export { InMemoryTransport } from './transport.js';
|
|
10
|
+
export { EventAggregator } from './event-aggregator.js';
|
|
11
|
+
export { buildGraph, validateGraph, topologicalLevels, subgraph, leafNodes } from './dag.js';
|
|
12
|
+
export { collectDependencies, buildIgnoreFilter, buildBundle, extractBundle } from './bundle.js';
|
|
13
|
+
export { evaluateCondition } from './condition.js';
|
|
14
|
+
export { dirSize, formatSize, formatDuration } from './utils.js';
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
import { readFile } from 'node:fs/promises';
|
|
2
|
+
import { extname } from 'node:path';
|
|
3
|
+
import { deburr } from 'lodash-es';
|
|
4
|
+
import { parse as parseYaml } from 'yaml';
|
|
5
|
+
import { ValidationError } from '../errors.js';
|
|
6
|
+
import { buildGraph, validateGraph } from './dag.js';
|
|
7
|
+
import { resolveStep, validateStep } from './step-resolver.js';
|
|
8
|
+
export class PipelineLoader {
|
|
9
|
+
async load(filePath) {
|
|
10
|
+
const content = await readFile(filePath, 'utf8');
|
|
11
|
+
return this.parse(content, filePath);
|
|
12
|
+
}
|
|
13
|
+
parse(content, filePath) {
|
|
14
|
+
const input = parsePipelineFile(content, filePath);
|
|
15
|
+
if (!input.id && !input.name) {
|
|
16
|
+
throw new ValidationError('Invalid pipeline: at least one of "id" or "name" must be defined');
|
|
17
|
+
}
|
|
18
|
+
const pipelineId = input.id ?? slugify(input.name);
|
|
19
|
+
if (!Array.isArray(input.steps) || input.steps.length === 0) {
|
|
20
|
+
throw new ValidationError('Invalid pipeline: steps must be a non-empty array');
|
|
21
|
+
}
|
|
22
|
+
const steps = input.steps.map(step => resolveStep(step));
|
|
23
|
+
for (const step of steps) {
|
|
24
|
+
validateStep(step);
|
|
25
|
+
}
|
|
26
|
+
this.validateUniqueStepIds(steps);
|
|
27
|
+
const graph = buildGraph(steps);
|
|
28
|
+
validateGraph(graph, steps);
|
|
29
|
+
return { id: pipelineId, name: input.name, steps };
|
|
30
|
+
}
|
|
31
|
+
validateUniqueStepIds(steps) {
|
|
32
|
+
const seen = new Set();
|
|
33
|
+
for (const step of steps) {
|
|
34
|
+
if (seen.has(step.id)) {
|
|
35
|
+
throw new ValidationError(`Duplicate step id: '${step.id}'`);
|
|
36
|
+
}
|
|
37
|
+
seen.add(step.id);
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
/** Convert a free-form name into a valid identifier. */
|
|
42
|
+
export function slugify(name) {
|
|
43
|
+
return deburr(name)
|
|
44
|
+
.toLowerCase()
|
|
45
|
+
.replaceAll(/[^\w-]/g, '-')
|
|
46
|
+
.replaceAll(/-{2,}/g, '-')
|
|
47
|
+
.replace(/^-/, '')
|
|
48
|
+
.replace(/-$/, '');
|
|
49
|
+
}
|
|
50
|
+
export function parsePipelineFile(content, filePath) {
|
|
51
|
+
const ext = extname(filePath).toLowerCase();
|
|
52
|
+
if (ext === '.yaml' || ext === '.yml') {
|
|
53
|
+
return parseYaml(content);
|
|
54
|
+
}
|
|
55
|
+
return JSON.parse(content);
|
|
56
|
+
}
|
|
57
|
+
export function mergeEnv(kitEnv, userEnv) {
|
|
58
|
+
if (!kitEnv && !userEnv) {
|
|
59
|
+
return undefined;
|
|
60
|
+
}
|
|
61
|
+
return { ...kitEnv, ...userEnv };
|
|
62
|
+
}
|
|
63
|
+
export function mergeCaches(kitCaches, userCaches) {
|
|
64
|
+
if (!kitCaches && !userCaches) {
|
|
65
|
+
return undefined;
|
|
66
|
+
}
|
|
67
|
+
const map = new Map();
|
|
68
|
+
for (const c of kitCaches ?? []) {
|
|
69
|
+
map.set(c.name, c);
|
|
70
|
+
}
|
|
71
|
+
for (const c of userCaches ?? []) {
|
|
72
|
+
map.set(c.name, c);
|
|
73
|
+
}
|
|
74
|
+
return [...map.values()];
|
|
75
|
+
}
|
|
76
|
+
export function mergeMounts(kitMounts, userMounts) {
|
|
77
|
+
if (!kitMounts && !userMounts) {
|
|
78
|
+
return undefined;
|
|
79
|
+
}
|
|
80
|
+
return [...(kitMounts ?? []), ...(userMounts ?? [])];
|
|
81
|
+
}
|
|
@@ -0,0 +1,360 @@
|
|
|
1
|
+
import process from 'node:process';
|
|
2
|
+
import { randomUUID } from 'node:crypto';
|
|
3
|
+
import { cpus } from 'node:os';
|
|
4
|
+
import { cp, writeFile } from 'node:fs/promises';
|
|
5
|
+
import { setTimeout } from 'node:timers/promises';
|
|
6
|
+
import { createWriteStream } from 'node:fs';
|
|
7
|
+
import { dirname, join, resolve } from 'node:path';
|
|
8
|
+
import { Workspace } from '../engine/index.js';
|
|
9
|
+
import { ContainerCrashError, PipexError } from '../errors.js';
|
|
10
|
+
import { loadEnvFile } from './env-file.js';
|
|
11
|
+
import { buildGraph, topologicalLevels, subgraph, leafNodes } from './dag.js';
|
|
12
|
+
import { evaluateCondition } from './condition.js';
|
|
13
|
+
import { StateManager } from './state.js';
|
|
14
|
+
import { dirSize, resolveHostPath } from './utils.js';
|
|
15
|
+
/**
|
|
16
|
+
* Orchestrates pipeline execution with DAG-based parallel execution and caching.
|
|
17
|
+
*/
|
|
18
|
+
export class PipelineRunner {
|
|
19
|
+
loader;
|
|
20
|
+
runtime;
|
|
21
|
+
reporter;
|
|
22
|
+
workdirRoot;
|
|
23
|
+
constructor(loader, runtime, reporter, workdirRoot) {
|
|
24
|
+
this.loader = loader;
|
|
25
|
+
this.runtime = runtime;
|
|
26
|
+
this.reporter = reporter;
|
|
27
|
+
this.workdirRoot = workdirRoot;
|
|
28
|
+
}
|
|
29
|
+
async run(pipelineFilePath, options) {
|
|
30
|
+
const { workspace: workspaceName, force, dryRun, target, concurrency, envFile } = options ?? {};
|
|
31
|
+
const config = await this.loader.load(pipelineFilePath);
|
|
32
|
+
const pipelineRoot = dirname(resolve(pipelineFilePath));
|
|
33
|
+
const workspaceId = workspaceName ?? config.id;
|
|
34
|
+
let workspace;
|
|
35
|
+
try {
|
|
36
|
+
workspace = await Workspace.open(this.workdirRoot, workspaceId);
|
|
37
|
+
}
|
|
38
|
+
catch {
|
|
39
|
+
workspace = await Workspace.create(this.workdirRoot, workspaceId);
|
|
40
|
+
}
|
|
41
|
+
await workspace.cleanupStaging();
|
|
42
|
+
await workspace.cleanupRunning();
|
|
43
|
+
if (!dryRun) {
|
|
44
|
+
await this.runtime.check();
|
|
45
|
+
await this.runtime.cleanupContainers(workspace.id);
|
|
46
|
+
}
|
|
47
|
+
const cliEnv = envFile ? await loadEnvFile(resolve(envFile)) : undefined;
|
|
48
|
+
const state = new StateManager(workspace.root);
|
|
49
|
+
await state.load();
|
|
50
|
+
const stepRuns = new Map();
|
|
51
|
+
let totalArtifactSize = 0;
|
|
52
|
+
const job = { workspaceId: workspace.id, jobId: randomUUID() };
|
|
53
|
+
// Build DAG and determine execution scope
|
|
54
|
+
const graph = buildGraph(config.steps);
|
|
55
|
+
const targets = target ?? leafNodes(graph);
|
|
56
|
+
const activeSteps = subgraph(graph, targets);
|
|
57
|
+
this.reporter.emit({
|
|
58
|
+
...job,
|
|
59
|
+
event: 'PIPELINE_START',
|
|
60
|
+
pipelineName: config.name ?? config.id,
|
|
61
|
+
steps: config.steps
|
|
62
|
+
.filter(s => activeSteps.has(s.id))
|
|
63
|
+
.map(s => ({ id: s.id, displayName: s.name ?? s.id }))
|
|
64
|
+
});
|
|
65
|
+
const levels = topologicalLevels(graph)
|
|
66
|
+
.map(level => level.filter(id => activeSteps.has(id)))
|
|
67
|
+
.filter(level => level.length > 0);
|
|
68
|
+
const stepMap = new Map(config.steps.map(s => [s.id, s]));
|
|
69
|
+
const failed = new Set();
|
|
70
|
+
const skipped = new Set();
|
|
71
|
+
const maxConcurrency = concurrency ?? cpus().length;
|
|
72
|
+
for (const level of levels) {
|
|
73
|
+
const tasks = level.map(stepId => async () => {
|
|
74
|
+
const step = stepMap.get(stepId);
|
|
75
|
+
const stepRef = { id: step.id, displayName: step.name ?? step.id };
|
|
76
|
+
// Check if blocked by a failed/skipped required dependency
|
|
77
|
+
if (this.isDependencyBlocked(step, failed, skipped)) {
|
|
78
|
+
skipped.add(step.id);
|
|
79
|
+
this.reporter.emit({ ...job, event: 'STEP_SKIPPED', step: stepRef, reason: 'dependency' });
|
|
80
|
+
return 0;
|
|
81
|
+
}
|
|
82
|
+
// Evaluate condition
|
|
83
|
+
if (step.if) {
|
|
84
|
+
const conditionMet = await evaluateCondition(step.if, { env: process.env });
|
|
85
|
+
if (!conditionMet) {
|
|
86
|
+
skipped.add(step.id);
|
|
87
|
+
this.reporter.emit({ ...job, event: 'STEP_SKIPPED', step: stepRef, reason: 'condition' });
|
|
88
|
+
return 0;
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
// Resolve env vars: CLI envFile < step envFile < step inline env
|
|
92
|
+
const stepFileEnv = step.envFile
|
|
93
|
+
? await loadEnvFile(resolve(pipelineRoot, step.envFile))
|
|
94
|
+
: undefined;
|
|
95
|
+
const resolvedEnv = (cliEnv ?? stepFileEnv ?? step.env)
|
|
96
|
+
? { ...cliEnv, ...stepFileEnv, ...step.env }
|
|
97
|
+
: undefined;
|
|
98
|
+
// Compute fingerprint
|
|
99
|
+
const inputRunIds = step.inputs
|
|
100
|
+
?.map(i => stepRuns.get(i.step))
|
|
101
|
+
.filter((id) => id !== undefined);
|
|
102
|
+
const resolvedMounts = step.mounts?.map(m => ({
|
|
103
|
+
hostPath: resolveHostPath(pipelineRoot, m.host),
|
|
104
|
+
containerPath: m.container
|
|
105
|
+
}));
|
|
106
|
+
const currentFingerprint = StateManager.fingerprint({
|
|
107
|
+
image: step.image,
|
|
108
|
+
cmd: step.cmd,
|
|
109
|
+
env: resolvedEnv,
|
|
110
|
+
inputRunIds,
|
|
111
|
+
mounts: resolvedMounts
|
|
112
|
+
});
|
|
113
|
+
// Cache check
|
|
114
|
+
const skipCache = force === true || (Array.isArray(force) && force.includes(step.id));
|
|
115
|
+
if (!skipCache && await this.tryUseCache({ workspace, state, step, stepRef, currentFingerprint, stepRuns, job })) {
|
|
116
|
+
return 0;
|
|
117
|
+
}
|
|
118
|
+
// Dry run
|
|
119
|
+
if (dryRun) {
|
|
120
|
+
this.reporter.emit({ ...job, event: 'STEP_WOULD_RUN', step: stepRef });
|
|
121
|
+
return 0;
|
|
122
|
+
}
|
|
123
|
+
// Execute
|
|
124
|
+
this.reporter.emit({ ...job, event: 'STEP_STARTING', step: stepRef });
|
|
125
|
+
return this.executeStep({ workspace, state, step, stepRef, stepRuns, currentFingerprint, resolvedMounts, pipelineRoot, job, resolvedEnv });
|
|
126
|
+
});
|
|
127
|
+
const results = await withConcurrency(tasks, maxConcurrency);
|
|
128
|
+
// Collect results
|
|
129
|
+
for (const [i, result] of results.entries()) {
|
|
130
|
+
const stepId = level[i];
|
|
131
|
+
if (result.status === 'fulfilled') {
|
|
132
|
+
totalArtifactSize += result.value;
|
|
133
|
+
}
|
|
134
|
+
else if (!failed.has(stepId)) {
|
|
135
|
+
// Step threw an error (ContainerCrashError if not allowFailure)
|
|
136
|
+
failed.add(stepId);
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
await state.save();
|
|
140
|
+
}
|
|
141
|
+
if (failed.size > 0) {
|
|
142
|
+
this.reporter.emit({ ...job, event: 'PIPELINE_FAILED' });
|
|
143
|
+
// Re-throw the first failure to signal pipeline failure to the CLI
|
|
144
|
+
const firstFailedId = [...failed][0];
|
|
145
|
+
throw new ContainerCrashError(firstFailedId, 1);
|
|
146
|
+
}
|
|
147
|
+
this.reporter.emit({ ...job, event: 'PIPELINE_FINISHED', totalArtifactSize });
|
|
148
|
+
}
|
|
149
|
+
isDependencyBlocked(step, failed, skippedSteps) {
|
|
150
|
+
if (!step.inputs) {
|
|
151
|
+
return false;
|
|
152
|
+
}
|
|
153
|
+
return step.inputs.some(input => !input.optional && (failed.has(input.step) || skippedSteps.has(input.step)));
|
|
154
|
+
}
|
|
155
|
+
async executeStep({ workspace, state, step, stepRef, stepRuns, currentFingerprint, resolvedMounts, pipelineRoot, job, resolvedEnv }) {
|
|
156
|
+
const runId = workspace.generateRunId();
|
|
157
|
+
const stagingPath = await workspace.prepareRun(runId);
|
|
158
|
+
await workspace.markStepRunning(step.id, { startedAt: new Date().toISOString(), pid: process.pid, stepName: step.name });
|
|
159
|
+
await this.prepareStagingWithInputs(workspace, step, workspace.runStagingArtifactsPath(runId), stepRuns);
|
|
160
|
+
if (step.caches) {
|
|
161
|
+
for (const cache of step.caches) {
|
|
162
|
+
await workspace.prepareCache(cache.name);
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
const { inputs, output, caches, mounts } = this.buildMounts(step, runId, stepRuns, pipelineRoot);
|
|
166
|
+
const stdoutLog = createWriteStream(join(stagingPath, 'stdout.log'));
|
|
167
|
+
const stderrLog = createWriteStream(join(stagingPath, 'stderr.log'));
|
|
168
|
+
try {
|
|
169
|
+
const maxRetries = step.retries ?? 0;
|
|
170
|
+
const retryDelay = step.retryDelayMs ?? 5000;
|
|
171
|
+
let result;
|
|
172
|
+
for (let attempt = 0; attempt <= maxRetries; attempt++) {
|
|
173
|
+
try {
|
|
174
|
+
result = await this.runtime.run(workspace, {
|
|
175
|
+
name: `pipex-${workspace.id}-${step.id}-${Date.now()}`,
|
|
176
|
+
image: step.image,
|
|
177
|
+
cmd: step.cmd,
|
|
178
|
+
env: resolvedEnv,
|
|
179
|
+
inputs,
|
|
180
|
+
output,
|
|
181
|
+
caches,
|
|
182
|
+
mounts,
|
|
183
|
+
sources: step.sources?.map(m => ({
|
|
184
|
+
hostPath: resolveHostPath(pipelineRoot, m.host),
|
|
185
|
+
containerPath: m.container
|
|
186
|
+
})),
|
|
187
|
+
network: step.allowNetwork ? 'bridge' : 'none',
|
|
188
|
+
timeoutSec: step.timeoutSec
|
|
189
|
+
}, ({ stream, line }) => {
|
|
190
|
+
if (stream === 'stdout') {
|
|
191
|
+
stdoutLog.write(line + '\n');
|
|
192
|
+
}
|
|
193
|
+
else {
|
|
194
|
+
stderrLog.write(line + '\n');
|
|
195
|
+
}
|
|
196
|
+
this.reporter.emit({ ...job, event: 'STEP_LOG', step: stepRef, stream, line });
|
|
197
|
+
});
|
|
198
|
+
break;
|
|
199
|
+
}
|
|
200
|
+
catch (error) {
|
|
201
|
+
if (error instanceof PipexError && error.transient && attempt < maxRetries) {
|
|
202
|
+
this.reporter.emit({ ...job, event: 'STEP_RETRYING', step: stepRef, attempt: attempt + 1, maxRetries });
|
|
203
|
+
await setTimeout(retryDelay);
|
|
204
|
+
continue;
|
|
205
|
+
}
|
|
206
|
+
throw error;
|
|
207
|
+
}
|
|
208
|
+
}
|
|
209
|
+
await closeStream(stdoutLog);
|
|
210
|
+
await closeStream(stderrLog);
|
|
211
|
+
await this.writeRunMeta(stagingPath, { runId, step, stepRuns, resolvedMounts, currentFingerprint, result, resolvedEnv });
|
|
212
|
+
if (result.exitCode === 0 || step.allowFailure) {
|
|
213
|
+
await workspace.commitRun(runId);
|
|
214
|
+
await workspace.linkRun(step.id, runId);
|
|
215
|
+
stepRuns.set(step.id, runId);
|
|
216
|
+
state.setStep(step.id, runId, currentFingerprint);
|
|
217
|
+
const durationMs = result.finishedAt.getTime() - result.startedAt.getTime();
|
|
218
|
+
const artifactSize = await dirSize(workspace.runArtifactsPath(runId));
|
|
219
|
+
this.reporter.emit({ ...job, event: 'STEP_FINISHED', step: stepRef, runId, durationMs, artifactSize });
|
|
220
|
+
return artifactSize;
|
|
221
|
+
}
|
|
222
|
+
await workspace.commitRun(runId);
|
|
223
|
+
await workspace.linkRun(step.id, runId);
|
|
224
|
+
state.setStep(step.id, runId, '');
|
|
225
|
+
this.reporter.emit({ ...job, event: 'STEP_FAILED', step: stepRef, exitCode: result.exitCode });
|
|
226
|
+
throw new ContainerCrashError(step.id, result.exitCode);
|
|
227
|
+
}
|
|
228
|
+
catch (error) {
|
|
229
|
+
await closeStream(stdoutLog);
|
|
230
|
+
await closeStream(stderrLog);
|
|
231
|
+
throw error;
|
|
232
|
+
}
|
|
233
|
+
finally {
|
|
234
|
+
await workspace.markStepDone(step.id);
|
|
235
|
+
}
|
|
236
|
+
}
|
|
237
|
+
async writeRunMeta(stagingPath, { runId, step, stepRuns, resolvedMounts, currentFingerprint, result, resolvedEnv }) {
|
|
238
|
+
const meta = {
|
|
239
|
+
runId,
|
|
240
|
+
stepId: step.id,
|
|
241
|
+
stepName: step.name,
|
|
242
|
+
startedAt: result.startedAt.toISOString(),
|
|
243
|
+
finishedAt: result.finishedAt.toISOString(),
|
|
244
|
+
durationMs: result.finishedAt.getTime() - result.startedAt.getTime(),
|
|
245
|
+
exitCode: result.exitCode,
|
|
246
|
+
image: step.image,
|
|
247
|
+
cmd: step.cmd,
|
|
248
|
+
env: resolvedEnv,
|
|
249
|
+
inputs: step.inputs?.map(i => ({
|
|
250
|
+
step: i.step,
|
|
251
|
+
runId: stepRuns.get(i.step),
|
|
252
|
+
mountedAs: `/input/${i.step}`
|
|
253
|
+
})),
|
|
254
|
+
mounts: resolvedMounts,
|
|
255
|
+
caches: step.caches?.map(c => c.name),
|
|
256
|
+
allowNetwork: step.allowNetwork ?? false,
|
|
257
|
+
fingerprint: currentFingerprint,
|
|
258
|
+
status: result.exitCode === 0 ? 'success' : 'failure'
|
|
259
|
+
};
|
|
260
|
+
await writeFile(join(stagingPath, 'meta.json'), JSON.stringify(meta, null, 2), 'utf8');
|
|
261
|
+
}
|
|
262
|
+
async tryUseCache({ workspace, state, step, stepRef, currentFingerprint, stepRuns, job }) {
|
|
263
|
+
const cached = state.getStep(step.id);
|
|
264
|
+
if (cached?.fingerprint === currentFingerprint) {
|
|
265
|
+
try {
|
|
266
|
+
const runs = await workspace.listRuns();
|
|
267
|
+
if (runs.includes(cached.runId)) {
|
|
268
|
+
stepRuns.set(step.id, cached.runId);
|
|
269
|
+
await workspace.linkRun(step.id, cached.runId);
|
|
270
|
+
this.reporter.emit({ ...job, event: 'STEP_SKIPPED', step: stepRef, runId: cached.runId, reason: 'cached' });
|
|
271
|
+
return true;
|
|
272
|
+
}
|
|
273
|
+
}
|
|
274
|
+
catch {
|
|
275
|
+
// Run missing, proceed with execution
|
|
276
|
+
}
|
|
277
|
+
}
|
|
278
|
+
return false;
|
|
279
|
+
}
|
|
280
|
+
async prepareStagingWithInputs(workspace, step, stagingArtifactsPath, stepRuns) {
|
|
281
|
+
if (!step.inputs) {
|
|
282
|
+
return;
|
|
283
|
+
}
|
|
284
|
+
for (const input of step.inputs) {
|
|
285
|
+
const inputRunId = stepRuns.get(input.step);
|
|
286
|
+
if (!inputRunId) {
|
|
287
|
+
if (input.optional) {
|
|
288
|
+
continue;
|
|
289
|
+
}
|
|
290
|
+
// Non-optional input without a run — this shouldn't happen in DAG mode
|
|
291
|
+
// but keep the continue for safety (the step may still work with bind mounts)
|
|
292
|
+
continue;
|
|
293
|
+
}
|
|
294
|
+
if (input.copyToOutput) {
|
|
295
|
+
await cp(workspace.runArtifactsPath(inputRunId), stagingArtifactsPath, { recursive: true });
|
|
296
|
+
}
|
|
297
|
+
}
|
|
298
|
+
}
|
|
299
|
+
buildMounts(step, outputRunId, stepRuns, pipelineRoot) {
|
|
300
|
+
const inputs = [];
|
|
301
|
+
if (step.inputs) {
|
|
302
|
+
for (const input of step.inputs) {
|
|
303
|
+
const inputRunId = stepRuns.get(input.step);
|
|
304
|
+
if (inputRunId) {
|
|
305
|
+
inputs.push({
|
|
306
|
+
runId: inputRunId,
|
|
307
|
+
containerPath: `/input/${input.step}`
|
|
308
|
+
});
|
|
309
|
+
}
|
|
310
|
+
}
|
|
311
|
+
}
|
|
312
|
+
const output = {
|
|
313
|
+
stagingRunId: outputRunId,
|
|
314
|
+
containerPath: step.outputPath ?? '/output'
|
|
315
|
+
};
|
|
316
|
+
let caches;
|
|
317
|
+
if (step.caches) {
|
|
318
|
+
caches = step.caches.map(c => ({
|
|
319
|
+
name: c.name,
|
|
320
|
+
containerPath: c.path
|
|
321
|
+
}));
|
|
322
|
+
}
|
|
323
|
+
let mounts;
|
|
324
|
+
if (step.mounts) {
|
|
325
|
+
mounts = step.mounts.map(m => ({
|
|
326
|
+
hostPath: resolveHostPath(pipelineRoot, m.host),
|
|
327
|
+
containerPath: m.container
|
|
328
|
+
}));
|
|
329
|
+
}
|
|
330
|
+
return { inputs, output, caches, mounts };
|
|
331
|
+
}
|
|
332
|
+
}
|
|
333
|
+
async function withConcurrency(tasks, limit) {
|
|
334
|
+
const results = Array.from({ length: tasks.length });
|
|
335
|
+
let next = 0;
|
|
336
|
+
async function worker() {
|
|
337
|
+
while (next < tasks.length) {
|
|
338
|
+
const i = next++;
|
|
339
|
+
try {
|
|
340
|
+
results[i] = { status: 'fulfilled', value: await tasks[i]() };
|
|
341
|
+
}
|
|
342
|
+
catch (error) {
|
|
343
|
+
results[i] = { status: 'rejected', reason: error };
|
|
344
|
+
}
|
|
345
|
+
}
|
|
346
|
+
}
|
|
347
|
+
await Promise.all(Array.from({ length: Math.min(limit, tasks.length) }, async () => worker()));
|
|
348
|
+
return results;
|
|
349
|
+
}
|
|
350
|
+
async function closeStream(stream) {
|
|
351
|
+
if (stream.destroyed) {
|
|
352
|
+
return;
|
|
353
|
+
}
|
|
354
|
+
return new Promise((resolve, reject) => {
|
|
355
|
+
stream.end(() => {
|
|
356
|
+
resolve();
|
|
357
|
+
});
|
|
358
|
+
stream.on('error', reject);
|
|
359
|
+
});
|
|
360
|
+
}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import pino from 'pino';
|
|
2
|
+
/**
|
|
3
|
+
* Reporter that outputs structured JSON logs via pino.
|
|
4
|
+
* Suitable for CI/CD environments and log aggregation.
|
|
5
|
+
*/
|
|
6
|
+
export class ConsoleReporter {
|
|
7
|
+
logger = pino({ level: 'info' });
|
|
8
|
+
emit(event) {
|
|
9
|
+
this.logger.info(event);
|
|
10
|
+
}
|
|
11
|
+
}
|
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
import { readFile, writeFile } from 'node:fs/promises';
|
|
2
|
+
import { join } from 'node:path';
|
|
3
|
+
import { createHash } from 'node:crypto';
|
|
4
|
+
/**
|
|
5
|
+
* Manages caching state for pipeline execution.
|
|
6
|
+
*
|
|
7
|
+
* The StateManager computes fingerprints for steps and tracks which
|
|
8
|
+
* run was produced by each step. This enables cache hits when
|
|
9
|
+
* a step's configuration hasn't changed.
|
|
10
|
+
*
|
|
11
|
+
* ## Fingerprint Algorithm
|
|
12
|
+
*
|
|
13
|
+
* A step fingerprint is computed as:
|
|
14
|
+
* ```
|
|
15
|
+
* SHA256(image + JSON(cmd) + JSON(sorted env) + JSON(sorted inputRunIds))
|
|
16
|
+
* ```
|
|
17
|
+
*
|
|
18
|
+
* A step is re-executed when:
|
|
19
|
+
* - The fingerprint changes (image, cmd, env, or inputs modified)
|
|
20
|
+
* - The run no longer exists (manually deleted)
|
|
21
|
+
*
|
|
22
|
+
* ## Cache Propagation
|
|
23
|
+
*
|
|
24
|
+
* Changes propagate through dependencies. If step A is modified,
|
|
25
|
+
* all steps depending on A are invalidated automatically (via inputRunIds).
|
|
26
|
+
*/
|
|
27
|
+
export class StateManager {
|
|
28
|
+
static fingerprint(config) {
|
|
29
|
+
const hash = createHash('sha256');
|
|
30
|
+
hash.update(config.image);
|
|
31
|
+
hash.update(JSON.stringify(config.cmd));
|
|
32
|
+
if (config.env) {
|
|
33
|
+
hash.update(JSON.stringify(Object.entries(config.env).sort((a, b) => a[0].localeCompare(b[0]))));
|
|
34
|
+
}
|
|
35
|
+
if (config.inputRunIds) {
|
|
36
|
+
hash.update(JSON.stringify([...config.inputRunIds].sort((a, b) => a.localeCompare(b))));
|
|
37
|
+
}
|
|
38
|
+
if (config.mounts && config.mounts.length > 0) {
|
|
39
|
+
const sorted = [...config.mounts].sort((a, b) => a.containerPath.localeCompare(b.containerPath));
|
|
40
|
+
hash.update(JSON.stringify(sorted));
|
|
41
|
+
}
|
|
42
|
+
return hash.digest('hex');
|
|
43
|
+
}
|
|
44
|
+
state = { steps: {} };
|
|
45
|
+
path;
|
|
46
|
+
/**
|
|
47
|
+
* Creates a state manager for the given workspace.
|
|
48
|
+
* @param workspaceRoot - Absolute path to workspace directory
|
|
49
|
+
*/
|
|
50
|
+
constructor(workspaceRoot) {
|
|
51
|
+
this.path = join(workspaceRoot, 'state.json');
|
|
52
|
+
}
|
|
53
|
+
/**
|
|
54
|
+
* Loads cached state from state.json.
|
|
55
|
+
* If the file doesn't exist, initializes with empty state.
|
|
56
|
+
*/
|
|
57
|
+
async load() {
|
|
58
|
+
try {
|
|
59
|
+
const content = await readFile(this.path, 'utf8');
|
|
60
|
+
Object.assign(this.state, JSON.parse(content));
|
|
61
|
+
}
|
|
62
|
+
catch {
|
|
63
|
+
this.state.steps = {};
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
/**
|
|
67
|
+
* Persists current state to state.json.
|
|
68
|
+
*/
|
|
69
|
+
async save() {
|
|
70
|
+
await writeFile(this.path, JSON.stringify(this.state, null, 2), 'utf8');
|
|
71
|
+
}
|
|
72
|
+
/**
|
|
73
|
+
* Retrieves cached state for a step.
|
|
74
|
+
* @param stepId - Step identifier
|
|
75
|
+
* @returns Cached state if available, undefined otherwise
|
|
76
|
+
*/
|
|
77
|
+
getStep(stepId) {
|
|
78
|
+
return this.state.steps[stepId];
|
|
79
|
+
}
|
|
80
|
+
/**
|
|
81
|
+
* Lists all steps with their run IDs (in insertion order).
|
|
82
|
+
* @returns Array of {stepId, runId} entries
|
|
83
|
+
*/
|
|
84
|
+
listSteps() {
|
|
85
|
+
return Object.entries(this.state.steps).map(([stepId, { runId }]) => ({ stepId, runId }));
|
|
86
|
+
}
|
|
87
|
+
/**
|
|
88
|
+
* Returns the set of run IDs currently referenced by state.
|
|
89
|
+
*/
|
|
90
|
+
activeRunIds() {
|
|
91
|
+
return new Set(Object.values(this.state.steps).map(s => s.runId));
|
|
92
|
+
}
|
|
93
|
+
/**
|
|
94
|
+
* Updates cached state for a step.
|
|
95
|
+
* @param stepId - Step identifier
|
|
96
|
+
* @param runId - Run produced by the step
|
|
97
|
+
* @param fingerprint - Step configuration fingerprint
|
|
98
|
+
*/
|
|
99
|
+
setStep(stepId, runId, fingerprint) {
|
|
100
|
+
this.state.steps[stepId] = { runId, fingerprint };
|
|
101
|
+
}
|
|
102
|
+
/**
|
|
103
|
+
* Removes a step's cached state.
|
|
104
|
+
* @param stepId - Step identifier
|
|
105
|
+
*/
|
|
106
|
+
removeStep(stepId) {
|
|
107
|
+
// eslint-disable-next-line @typescript-eslint/no-dynamic-delete
|
|
108
|
+
delete this.state.steps[stepId];
|
|
109
|
+
}
|
|
110
|
+
}
|