@livingdata/pipex 0.0.9 → 0.0.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +154 -14
- package/dist/__tests__/errors.js +162 -0
- package/dist/__tests__/helpers.js +41 -0
- package/dist/__tests__/types.js +8 -0
- package/dist/cli/__tests__/condition.js +23 -0
- package/dist/cli/__tests__/dag.js +154 -0
- package/dist/cli/__tests__/pipeline-loader.js +267 -0
- package/dist/cli/__tests__/pipeline-runner.js +257 -0
- package/dist/cli/__tests__/state-persistence.js +80 -0
- package/dist/cli/__tests__/state.js +58 -0
- package/dist/cli/__tests__/step-runner.js +116 -0
- package/dist/cli/commands/bundle.js +35 -0
- package/dist/cli/commands/cat.js +54 -0
- package/dist/cli/commands/exec.js +89 -0
- package/dist/cli/commands/export.js +2 -2
- package/dist/cli/commands/inspect.js +1 -1
- package/dist/cli/commands/list.js +2 -1
- package/dist/cli/commands/logs.js +1 -1
- package/dist/cli/commands/prune.js +1 -1
- package/dist/cli/commands/rm-step.js +41 -0
- package/dist/cli/commands/run-bundle.js +59 -0
- package/dist/cli/commands/run.js +9 -4
- package/dist/cli/commands/show.js +42 -7
- package/dist/cli/condition.js +11 -0
- package/dist/cli/dag.js +143 -0
- package/dist/cli/index.js +6 -0
- package/dist/cli/interactive-reporter.js +227 -0
- package/dist/cli/pipeline-loader.js +10 -110
- package/dist/cli/pipeline-runner.js +164 -78
- package/dist/cli/reporter.js +2 -158
- package/dist/cli/state.js +8 -0
- package/dist/cli/step-loader.js +25 -0
- package/dist/cli/step-resolver.js +111 -0
- package/dist/cli/step-runner.js +226 -0
- package/dist/cli/utils.js +0 -46
- package/dist/core/__tests__/bundle.js +663 -0
- package/dist/core/__tests__/condition.js +23 -0
- package/dist/core/__tests__/dag.js +154 -0
- package/dist/core/__tests__/env-file.test.js +41 -0
- package/dist/core/__tests__/event-aggregator.js +244 -0
- package/dist/core/__tests__/pipeline-loader.js +267 -0
- package/dist/core/__tests__/pipeline-runner.js +257 -0
- package/dist/core/__tests__/state-persistence.js +80 -0
- package/dist/core/__tests__/state.js +58 -0
- package/dist/core/__tests__/step-runner.js +118 -0
- package/dist/core/__tests__/stream-reporter.js +142 -0
- package/dist/core/__tests__/transport.js +50 -0
- package/dist/core/__tests__/utils.js +40 -0
- package/dist/core/bundle.js +130 -0
- package/dist/core/condition.js +11 -0
- package/dist/core/dag.js +143 -0
- package/dist/core/env-file.js +6 -0
- package/dist/core/event-aggregator.js +114 -0
- package/dist/core/index.js +14 -0
- package/dist/core/pipeline-loader.js +81 -0
- package/dist/core/pipeline-runner.js +360 -0
- package/dist/core/reporter.js +11 -0
- package/dist/core/state.js +110 -0
- package/dist/core/step-loader.js +25 -0
- package/dist/core/step-resolver.js +117 -0
- package/dist/core/step-runner.js +225 -0
- package/dist/core/stream-reporter.js +41 -0
- package/dist/core/transport.js +9 -0
- package/dist/core/utils.js +56 -0
- package/dist/engine/__tests__/workspace.js +288 -0
- package/dist/engine/docker-executor.js +10 -2
- package/dist/engine/index.js +1 -0
- package/dist/engine/workspace.js +76 -12
- package/dist/errors.js +122 -0
- package/dist/index.js +3 -0
- package/dist/kits/__tests__/index.js +23 -0
- package/dist/kits/builtin/__tests__/node.js +74 -0
- package/dist/kits/builtin/__tests__/python.js +67 -0
- package/dist/kits/builtin/__tests__/shell.js +74 -0
- package/dist/kits/builtin/node.js +10 -5
- package/dist/kits/builtin/python.js +10 -5
- package/dist/kits/builtin/shell.js +2 -1
- package/dist/kits/index.js +2 -1
- package/package.json +6 -3
- package/dist/cli/types.js +0 -3
- package/dist/engine/docker-runtime.js +0 -65
- package/dist/engine/runtime.js +0 -2
- package/dist/kits/bash.js +0 -19
- package/dist/kits/builtin/bash.js +0 -19
- package/dist/kits/node.js +0 -56
- package/dist/kits/python.js +0 -51
- package/dist/kits/types.js +0 -1
- package/dist/reporter.js +0 -13
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
import { readFile } from 'node:fs/promises';
|
|
2
|
+
import { join } from 'node:path';
|
|
3
|
+
import test from 'ava';
|
|
4
|
+
import { Workspace } from '../../engine/workspace.js';
|
|
5
|
+
import { DockerCliExecutor } from '../../engine/docker-executor.js';
|
|
6
|
+
import { ContainerCrashError } from '../../errors.js';
|
|
7
|
+
import { StateManager } from '../state.js';
|
|
8
|
+
import { StepRunner } from '../step-runner.js';
|
|
9
|
+
import { createTmpDir, isDockerAvailable, noopReporter, recordingReporter } from '../../__tests__/helpers.js';
|
|
10
|
+
const hasDocker = isDockerAvailable();
|
|
11
|
+
const dockerTest = hasDocker ? test : test.skip;
|
|
12
|
+
// -- helpers -----------------------------------------------------------------
|
|
13
|
+
function makeStep(overrides) {
|
|
14
|
+
return {
|
|
15
|
+
image: 'alpine:3.20',
|
|
16
|
+
cmd: ['sh', '-c', 'echo hello'],
|
|
17
|
+
...overrides
|
|
18
|
+
};
|
|
19
|
+
}
|
|
20
|
+
async function setupWorkspace() {
|
|
21
|
+
const tmpDir = await createTmpDir();
|
|
22
|
+
const workspace = await Workspace.create(tmpDir, 'test-ws');
|
|
23
|
+
const state = new StateManager(workspace.root);
|
|
24
|
+
await state.load();
|
|
25
|
+
return { workspace, state, tmpDir };
|
|
26
|
+
}
|
|
27
|
+
// -- minimal execution -------------------------------------------------------
|
|
28
|
+
dockerTest('minimal step writes artifact and returns exitCode 0', async (t) => {
|
|
29
|
+
const { workspace, state } = await setupWorkspace();
|
|
30
|
+
const runner = new StepRunner(new DockerCliExecutor(), noopReporter);
|
|
31
|
+
const step = makeStep({ id: 'greet', cmd: ['sh', '-c', 'echo hello > /output/greeting.txt'] });
|
|
32
|
+
const result = await runner.run({ workspace, state, step, inputs: new Map(), pipelineRoot: '/' });
|
|
33
|
+
t.is(result.exitCode, 0);
|
|
34
|
+
t.truthy(result.runId);
|
|
35
|
+
// Artifact exists in committed run
|
|
36
|
+
const content = await readFile(join(workspace.runArtifactsPath(result.runId), 'greeting.txt'), 'utf8');
|
|
37
|
+
t.is(content.trim(), 'hello');
|
|
38
|
+
});
|
|
39
|
+
dockerTest('meta.json exists with correct fields', async (t) => {
|
|
40
|
+
const { workspace, state } = await setupWorkspace();
|
|
41
|
+
const runner = new StepRunner(new DockerCliExecutor(), noopReporter);
|
|
42
|
+
const step = makeStep({ id: 'meta-test', cmd: ['sh', '-c', 'echo ok > /output/out.txt'] });
|
|
43
|
+
const result = await runner.run({ workspace, state, step, inputs: new Map(), pipelineRoot: '/' });
|
|
44
|
+
const metaPath = join(workspace.runPath(result.runId), 'meta.json');
|
|
45
|
+
const meta = JSON.parse(await readFile(metaPath, 'utf8'));
|
|
46
|
+
t.is(meta.runId, result.runId);
|
|
47
|
+
t.is(meta.exitCode, 0);
|
|
48
|
+
t.is(meta.image, 'alpine:3.20');
|
|
49
|
+
t.deepEqual(meta.cmd, ['sh', '-c', 'echo ok > /output/out.txt']);
|
|
50
|
+
});
|
|
51
|
+
// -- log capture -------------------------------------------------------------
|
|
52
|
+
dockerTest('stdout and stderr are captured to log files', async (t) => {
|
|
53
|
+
const { workspace, state } = await setupWorkspace();
|
|
54
|
+
const runner = new StepRunner(new DockerCliExecutor(), noopReporter);
|
|
55
|
+
const step = makeStep({ id: 'logs', cmd: ['sh', '-c', 'echo out-line && echo err-line >&2'] });
|
|
56
|
+
const result = await runner.run({ workspace, state, step, inputs: new Map(), pipelineRoot: '/' });
|
|
57
|
+
const stdout = await readFile(join(workspace.runPath(result.runId), 'stdout.log'), 'utf8');
|
|
58
|
+
const stderr = await readFile(join(workspace.runPath(result.runId), 'stderr.log'), 'utf8');
|
|
59
|
+
t.true(stdout.includes('out-line'));
|
|
60
|
+
t.true(stderr.includes('err-line'));
|
|
61
|
+
});
|
|
62
|
+
// -- cache hit ---------------------------------------------------------------
|
|
63
|
+
dockerTest('second run of same step is cached (STEP_SKIPPED)', async (t) => {
|
|
64
|
+
const { workspace, state } = await setupWorkspace();
|
|
65
|
+
const { reporter, events } = recordingReporter();
|
|
66
|
+
const runner = new StepRunner(new DockerCliExecutor(), reporter);
|
|
67
|
+
const step = makeStep({ id: 'cached', cmd: ['sh', '-c', 'echo hi > /output/x.txt'] });
|
|
68
|
+
const first = await runner.run({ workspace, state, step, inputs: new Map(), pipelineRoot: '/' });
|
|
69
|
+
const second = await runner.run({ workspace, state, step, inputs: new Map(), pipelineRoot: '/' });
|
|
70
|
+
t.is(first.runId, second.runId);
|
|
71
|
+
const skipped = events.find(e => e.event === 'STEP_SKIPPED');
|
|
72
|
+
t.truthy(skipped);
|
|
73
|
+
t.is(skipped.reason, 'cached');
|
|
74
|
+
});
|
|
75
|
+
// -- force -------------------------------------------------------------------
|
|
76
|
+
dockerTest('force: true produces new runId', async (t) => {
|
|
77
|
+
const { workspace, state } = await setupWorkspace();
|
|
78
|
+
const runner = new StepRunner(new DockerCliExecutor(), noopReporter);
|
|
79
|
+
const step = makeStep({ id: 'force-test', cmd: ['sh', '-c', 'echo data > /output/f.txt'] });
|
|
80
|
+
const first = await runner.run({ workspace, state, step, inputs: new Map(), pipelineRoot: '/' });
|
|
81
|
+
const second = await runner.run({ workspace, state, step, inputs: new Map(), pipelineRoot: '/', force: true });
|
|
82
|
+
t.not(first.runId, second.runId);
|
|
83
|
+
t.is(second.exitCode, 0);
|
|
84
|
+
});
|
|
85
|
+
// -- ephemeral ---------------------------------------------------------------
|
|
86
|
+
dockerTest('ephemeral: true returns exitCode but no runId', async (t) => {
|
|
87
|
+
const { workspace, state } = await setupWorkspace();
|
|
88
|
+
const runner = new StepRunner(new DockerCliExecutor(), noopReporter);
|
|
89
|
+
const step = makeStep({ id: 'ephemeral', cmd: ['sh', '-c', 'echo temp > /output/t.txt'] });
|
|
90
|
+
const result = await runner.run({ workspace, state, step, inputs: new Map(), pipelineRoot: '/', ephemeral: true });
|
|
91
|
+
t.is(result.exitCode, 0);
|
|
92
|
+
t.is(result.runId, undefined);
|
|
93
|
+
// No run committed
|
|
94
|
+
const runs = await workspace.listRuns();
|
|
95
|
+
t.is(runs.length, 0);
|
|
96
|
+
});
|
|
97
|
+
// -- failure -----------------------------------------------------------------
|
|
98
|
+
dockerTest('non-zero exit throws ContainerCrashError', async (t) => {
|
|
99
|
+
const { workspace, state } = await setupWorkspace();
|
|
100
|
+
const runner = new StepRunner(new DockerCliExecutor(), noopReporter);
|
|
101
|
+
const step = makeStep({ id: 'fail', cmd: ['sh', '-c', 'exit 1'] });
|
|
102
|
+
const error = await t.throwsAsync(async () => runner.run({ workspace, state, step, inputs: new Map(), pipelineRoot: '/' }));
|
|
103
|
+
t.true(error instanceof ContainerCrashError);
|
|
104
|
+
});
|
|
105
|
+
// -- allowFailure ------------------------------------------------------------
|
|
106
|
+
dockerTest('allowFailure: true commits run with non-zero exitCode', async (t) => {
|
|
107
|
+
const { workspace, state } = await setupWorkspace();
|
|
108
|
+
const runner = new StepRunner(new DockerCliExecutor(), noopReporter);
|
|
109
|
+
const step = makeStep({ id: 'allow-fail', cmd: ['sh', '-c', 'exit 1'], allowFailure: true });
|
|
110
|
+
const result = await runner.run({ workspace, state, step, inputs: new Map(), pipelineRoot: '/' });
|
|
111
|
+
t.truthy(result.runId);
|
|
112
|
+
t.is(result.exitCode, 1);
|
|
113
|
+
// Run was committed
|
|
114
|
+
const runs = await workspace.listRuns();
|
|
115
|
+
t.true(runs.includes(result.runId));
|
|
116
|
+
});
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
import process from 'node:process';
|
|
2
|
+
import { writeFile } from 'node:fs/promises';
|
|
3
|
+
import { basename, resolve } from 'node:path';
|
|
4
|
+
import chalk from 'chalk';
|
|
5
|
+
import { buildBundle } from '../../core/bundle.js';
|
|
6
|
+
import { PipelineLoader } from '../../core/pipeline-loader.js';
|
|
7
|
+
import { formatSize } from '../../core/utils.js';
|
|
8
|
+
export function registerBundleCommand(program) {
|
|
9
|
+
program
|
|
10
|
+
.command('bundle')
|
|
11
|
+
.description('Package a pipeline and its local dependencies into a tar.gz archive')
|
|
12
|
+
.argument('<pipeline>', 'Path to the pipeline file')
|
|
13
|
+
.option('-o, --output <path>', 'Output file path (default: <pipeline-id>.tar.gz)')
|
|
14
|
+
.action(async (pipelineFile, options) => {
|
|
15
|
+
try {
|
|
16
|
+
const pipelinePath = resolve(pipelineFile);
|
|
17
|
+
// Determine output path
|
|
18
|
+
let outputPath = options.output;
|
|
19
|
+
if (!outputPath) {
|
|
20
|
+
const loader = new PipelineLoader();
|
|
21
|
+
const pipeline = await loader.load(pipelinePath);
|
|
22
|
+
outputPath = `${pipeline.id}.tar.gz`;
|
|
23
|
+
}
|
|
24
|
+
outputPath = resolve(outputPath);
|
|
25
|
+
const archive = await buildBundle(pipelinePath);
|
|
26
|
+
await writeFile(outputPath, archive);
|
|
27
|
+
console.log(chalk.green(`Bundle created: ${basename(outputPath)} (${formatSize(archive.length)})`));
|
|
28
|
+
}
|
|
29
|
+
catch (error) {
|
|
30
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
31
|
+
console.error(chalk.red(`Bundle failed: ${message}`));
|
|
32
|
+
process.exitCode = 1;
|
|
33
|
+
}
|
|
34
|
+
});
|
|
35
|
+
}
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
import process from 'node:process';
|
|
2
|
+
import { readFile, readdir, stat } from 'node:fs/promises';
|
|
3
|
+
import { join, resolve } from 'node:path';
|
|
4
|
+
import chalk from 'chalk';
|
|
5
|
+
import { Workspace } from '../../engine/workspace.js';
|
|
6
|
+
import { StateManager } from '../../core/state.js';
|
|
7
|
+
import { getGlobalOptions } from '../utils.js';
|
|
8
|
+
export function registerCatCommand(program) {
|
|
9
|
+
program
|
|
10
|
+
.command('cat')
|
|
11
|
+
.description('Read artifact content from a step\'s latest run')
|
|
12
|
+
.argument('<workspace>', 'Workspace name')
|
|
13
|
+
.argument('<step>', 'Step ID')
|
|
14
|
+
.argument('[path]', 'Path within artifacts (omit to list)')
|
|
15
|
+
.action(async (workspaceName, stepId, artifactPath, _options, cmd) => {
|
|
16
|
+
const { workdir } = getGlobalOptions(cmd);
|
|
17
|
+
const workdirRoot = resolve(workdir);
|
|
18
|
+
const workspace = await Workspace.open(workdirRoot, workspaceName);
|
|
19
|
+
const state = new StateManager(workspace.root);
|
|
20
|
+
await state.load();
|
|
21
|
+
const stepState = state.getStep(stepId);
|
|
22
|
+
if (!stepState) {
|
|
23
|
+
console.error(chalk.red(`No run found for step: ${stepId}`));
|
|
24
|
+
process.exitCode = 1;
|
|
25
|
+
return;
|
|
26
|
+
}
|
|
27
|
+
const artifactsDir = workspace.runArtifactsPath(stepState.runId);
|
|
28
|
+
const targetPath = artifactPath ? join(artifactsDir, artifactPath) : artifactsDir;
|
|
29
|
+
// Prevent path traversal
|
|
30
|
+
if (!targetPath.startsWith(artifactsDir)) {
|
|
31
|
+
console.error(chalk.red('Invalid path: must be within artifacts directory'));
|
|
32
|
+
process.exitCode = 1;
|
|
33
|
+
return;
|
|
34
|
+
}
|
|
35
|
+
try {
|
|
36
|
+
const info = await stat(targetPath);
|
|
37
|
+
if (info.isDirectory()) {
|
|
38
|
+
const entries = await readdir(targetPath, { withFileTypes: true });
|
|
39
|
+
for (const entry of entries) {
|
|
40
|
+
const suffix = entry.isDirectory() ? '/' : '';
|
|
41
|
+
console.log(entry.name + suffix);
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
else {
|
|
45
|
+
const content = await readFile(targetPath);
|
|
46
|
+
process.stdout.write(content);
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
catch {
|
|
50
|
+
console.error(chalk.red(`Not found: ${artifactPath ?? '(artifacts directory)'}`));
|
|
51
|
+
process.exitCode = 1;
|
|
52
|
+
}
|
|
53
|
+
});
|
|
54
|
+
}
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
import process from 'node:process';
|
|
2
|
+
import { randomUUID } from 'node:crypto';
|
|
3
|
+
import { dirname, resolve } from 'node:path';
|
|
4
|
+
import { DockerCliExecutor } from '../../engine/docker-executor.js';
|
|
5
|
+
import { Workspace } from '../../engine/workspace.js';
|
|
6
|
+
import { loadStepFile } from '../../core/step-loader.js';
|
|
7
|
+
import { StepRunner } from '../../core/step-runner.js';
|
|
8
|
+
import { StateManager } from '../../core/state.js';
|
|
9
|
+
import { ConsoleReporter } from '../../core/reporter.js';
|
|
10
|
+
import { InteractiveReporter } from '../interactive-reporter.js';
|
|
11
|
+
import { getGlobalOptions } from '../utils.js';
|
|
12
|
+
export function registerExecCommand(program) {
|
|
13
|
+
program
|
|
14
|
+
.command('exec')
|
|
15
|
+
.description('Execute a single step in a workspace')
|
|
16
|
+
.argument('<workspace>', 'Workspace name')
|
|
17
|
+
.requiredOption('-f, --file <path>', 'Step definition file (YAML or JSON)')
|
|
18
|
+
.option('--step <id>', 'Step ID (overrides file\'s id)')
|
|
19
|
+
.option('--input <specs...>', 'Input steps (e.g. "extract" or "data=extract")')
|
|
20
|
+
.option('--ephemeral', 'Don\'t commit run, stream stdout to terminal')
|
|
21
|
+
.option('--force', 'Skip cache check')
|
|
22
|
+
.option('--verbose', 'Stream container logs in real-time')
|
|
23
|
+
.action(async (workspaceName, options, cmd) => {
|
|
24
|
+
const { workdir, json } = getGlobalOptions(cmd);
|
|
25
|
+
const workdirRoot = resolve(workdir);
|
|
26
|
+
const runtime = new DockerCliExecutor();
|
|
27
|
+
const reporter = json ? new ConsoleReporter() : new InteractiveReporter({ verbose: options.verbose });
|
|
28
|
+
// Load step from file
|
|
29
|
+
const stepFilePath = resolve(options.file);
|
|
30
|
+
const step = await loadStepFile(stepFilePath, options.step);
|
|
31
|
+
const pipelineRoot = dirname(stepFilePath);
|
|
32
|
+
// Open or create workspace
|
|
33
|
+
let workspace;
|
|
34
|
+
try {
|
|
35
|
+
workspace = await Workspace.open(workdirRoot, workspaceName);
|
|
36
|
+
}
|
|
37
|
+
catch {
|
|
38
|
+
workspace = await Workspace.create(workdirRoot, workspaceName);
|
|
39
|
+
}
|
|
40
|
+
await workspace.cleanupStaging();
|
|
41
|
+
await runtime.check();
|
|
42
|
+
await runtime.cleanupContainers(workspace.id);
|
|
43
|
+
// Load state and resolve inputs
|
|
44
|
+
const state = new StateManager(workspace.root);
|
|
45
|
+
await state.load();
|
|
46
|
+
const inputs = new Map();
|
|
47
|
+
if (options.input) {
|
|
48
|
+
for (const spec of options.input) {
|
|
49
|
+
const { alias, stepId } = parseInputSpec(spec);
|
|
50
|
+
const stepState = state.getStep(stepId);
|
|
51
|
+
if (!stepState) {
|
|
52
|
+
console.error(`No run found for input step: ${stepId}`);
|
|
53
|
+
process.exitCode = 1;
|
|
54
|
+
return;
|
|
55
|
+
}
|
|
56
|
+
inputs.set(alias, stepState.runId);
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
// Merge parsed inputs into step.inputs
|
|
60
|
+
if (inputs.size > 0 && !step.inputs) {
|
|
61
|
+
step.inputs = [];
|
|
62
|
+
}
|
|
63
|
+
for (const [alias, _runId] of inputs) {
|
|
64
|
+
// Only add if not already declared in step file
|
|
65
|
+
if (!step.inputs.some(i => i.step === alias)) {
|
|
66
|
+
step.inputs.push({ step: alias });
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
const job = { workspaceId: workspace.id, jobId: randomUUID() };
|
|
70
|
+
const runner = new StepRunner(runtime, reporter);
|
|
71
|
+
await runner.run({
|
|
72
|
+
workspace,
|
|
73
|
+
state,
|
|
74
|
+
step,
|
|
75
|
+
inputs,
|
|
76
|
+
pipelineRoot,
|
|
77
|
+
force: options.force,
|
|
78
|
+
ephemeral: options.ephemeral,
|
|
79
|
+
job
|
|
80
|
+
});
|
|
81
|
+
});
|
|
82
|
+
}
|
|
83
|
+
function parseInputSpec(spec) {
|
|
84
|
+
const eqIdx = spec.indexOf('=');
|
|
85
|
+
if (eqIdx > 0) {
|
|
86
|
+
return { alias: spec.slice(0, eqIdx), stepId: spec.slice(eqIdx + 1) };
|
|
87
|
+
}
|
|
88
|
+
return { alias: spec, stepId: spec };
|
|
89
|
+
}
|
|
@@ -3,7 +3,7 @@ import { cp } from 'node:fs/promises';
|
|
|
3
3
|
import { resolve } from 'node:path';
|
|
4
4
|
import chalk from 'chalk';
|
|
5
5
|
import { Workspace } from '../../engine/workspace.js';
|
|
6
|
-
import { StateManager } from '
|
|
6
|
+
import { StateManager } from '../../core/state.js';
|
|
7
7
|
import { getGlobalOptions } from '../utils.js';
|
|
8
8
|
export function registerExportCommand(program) {
|
|
9
9
|
program
|
|
@@ -12,7 +12,7 @@ export function registerExportCommand(program) {
|
|
|
12
12
|
.argument('<workspace>', 'Workspace name')
|
|
13
13
|
.argument('<step>', 'Step ID')
|
|
14
14
|
.argument('<dest>', 'Destination directory')
|
|
15
|
-
.action(async (workspaceName, stepId, dest, cmd) => {
|
|
15
|
+
.action(async (workspaceName, stepId, dest, _options, cmd) => {
|
|
16
16
|
const { workdir } = getGlobalOptions(cmd);
|
|
17
17
|
const workdirRoot = resolve(workdir);
|
|
18
18
|
const workspace = await Workspace.open(workdirRoot, workspaceName);
|
|
@@ -3,7 +3,7 @@ import { readFile } from 'node:fs/promises';
|
|
|
3
3
|
import { join, resolve } from 'node:path';
|
|
4
4
|
import chalk from 'chalk';
|
|
5
5
|
import { Workspace } from '../../engine/workspace.js';
|
|
6
|
-
import { StateManager } from '
|
|
6
|
+
import { StateManager } from '../../core/state.js';
|
|
7
7
|
import { getGlobalOptions } from '../utils.js';
|
|
8
8
|
export function registerInspectCommand(program) {
|
|
9
9
|
program
|
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
import { resolve } from 'node:path';
|
|
2
2
|
import chalk from 'chalk';
|
|
3
3
|
import { Workspace } from '../../engine/workspace.js';
|
|
4
|
-
import { dirSize, formatSize
|
|
4
|
+
import { dirSize, formatSize } from '../../core/utils.js';
|
|
5
|
+
import { getGlobalOptions } from '../utils.js';
|
|
5
6
|
export function registerListCommand(program) {
|
|
6
7
|
program
|
|
7
8
|
.command('list')
|
|
@@ -3,7 +3,7 @@ import { readFile } from 'node:fs/promises';
|
|
|
3
3
|
import { join, resolve } from 'node:path';
|
|
4
4
|
import chalk from 'chalk';
|
|
5
5
|
import { Workspace } from '../../engine/workspace.js';
|
|
6
|
-
import { StateManager } from '
|
|
6
|
+
import { StateManager } from '../../core/state.js';
|
|
7
7
|
import { getGlobalOptions } from '../utils.js';
|
|
8
8
|
export function registerLogsCommand(program) {
|
|
9
9
|
program
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { resolve } from 'node:path';
|
|
2
2
|
import chalk from 'chalk';
|
|
3
3
|
import { Workspace } from '../../engine/workspace.js';
|
|
4
|
-
import { StateManager } from '
|
|
4
|
+
import { StateManager } from '../../core/state.js';
|
|
5
5
|
import { getGlobalOptions } from '../utils.js';
|
|
6
6
|
export function registerPruneCommand(program) {
|
|
7
7
|
program
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
import process from 'node:process';
|
|
2
|
+
import { rm } from 'node:fs/promises';
|
|
3
|
+
import { join, resolve } from 'node:path';
|
|
4
|
+
import chalk from 'chalk';
|
|
5
|
+
import { Workspace } from '../../engine/workspace.js';
|
|
6
|
+
import { StateManager } from '../../core/state.js';
|
|
7
|
+
import { dirSize, formatSize } from '../../core/utils.js';
|
|
8
|
+
import { getGlobalOptions } from '../utils.js';
|
|
9
|
+
export function registerRmStepCommand(program) {
|
|
10
|
+
program
|
|
11
|
+
.command('rm-step')
|
|
12
|
+
.description('Remove a step\'s run and state entry')
|
|
13
|
+
.argument('<workspace>', 'Workspace name')
|
|
14
|
+
.argument('<step>', 'Step ID')
|
|
15
|
+
.action(async (workspaceName, stepId, _options, cmd) => {
|
|
16
|
+
const { workdir } = getGlobalOptions(cmd);
|
|
17
|
+
const workdirRoot = resolve(workdir);
|
|
18
|
+
const workspace = await Workspace.open(workdirRoot, workspaceName);
|
|
19
|
+
const state = new StateManager(workspace.root);
|
|
20
|
+
await state.load();
|
|
21
|
+
const stepState = state.getStep(stepId);
|
|
22
|
+
if (!stepState) {
|
|
23
|
+
console.error(chalk.red(`No run found for step: ${stepId}`));
|
|
24
|
+
process.exitCode = 1;
|
|
25
|
+
return;
|
|
26
|
+
}
|
|
27
|
+
const { runId } = stepState;
|
|
28
|
+
// Measure size before removal
|
|
29
|
+
const runDir = workspace.runPath(runId);
|
|
30
|
+
const size = await dirSize(runDir);
|
|
31
|
+
// Remove run directory
|
|
32
|
+
await rm(runDir, { recursive: true, force: true });
|
|
33
|
+
// Remove step-runs symlink
|
|
34
|
+
const linkPath = join(workspace.root, 'step-runs', stepId);
|
|
35
|
+
await rm(linkPath, { force: true });
|
|
36
|
+
// Remove from state
|
|
37
|
+
state.removeStep(stepId);
|
|
38
|
+
await state.save();
|
|
39
|
+
console.log(chalk.green(`Removed step ${chalk.bold(stepId)} (run ${runId}, freed ${formatSize(size)})`));
|
|
40
|
+
});
|
|
41
|
+
}
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
import process from 'node:process';
|
|
2
|
+
import { readFile, writeFile, rm } from 'node:fs/promises';
|
|
3
|
+
import { tmpdir } from 'node:os';
|
|
4
|
+
import { join, resolve } from 'node:path';
|
|
5
|
+
import { DockerCliExecutor } from '../../engine/docker-executor.js';
|
|
6
|
+
import { extractBundle } from '../../core/bundle.js';
|
|
7
|
+
import { PipelineLoader } from '../../core/pipeline-loader.js';
|
|
8
|
+
import { PipelineRunner } from '../../core/pipeline-runner.js';
|
|
9
|
+
import { ConsoleReporter } from '../../core/reporter.js';
|
|
10
|
+
import { InteractiveReporter } from '../interactive-reporter.js';
|
|
11
|
+
import { getGlobalOptions } from '../utils.js';
|
|
12
|
+
export function registerRunBundleCommand(program) {
|
|
13
|
+
program
|
|
14
|
+
.command('run-bundle')
|
|
15
|
+
.description('Execute a pipeline from a bundle archive')
|
|
16
|
+
.argument('<archive>', 'Bundle archive (.tar.gz)')
|
|
17
|
+
.option('-w, --workspace <name>', 'Workspace name (for caching)')
|
|
18
|
+
.option('-f, --force [steps]', 'Skip cache for all steps, or a comma-separated list')
|
|
19
|
+
.option('--dry-run', 'Validate pipeline and show what would run without executing')
|
|
20
|
+
.option('--verbose', 'Stream container logs in real-time (interactive mode)')
|
|
21
|
+
.option('-t, --target <steps>', 'Execute only these steps and their dependencies (comma-separated)')
|
|
22
|
+
.option('-c, --concurrency <number>', 'Max parallel step executions (default: CPU count)', Number)
|
|
23
|
+
.action(async (archivePath, options, cmd) => {
|
|
24
|
+
const { workdir, json } = getGlobalOptions(cmd);
|
|
25
|
+
const workdirRoot = resolve(workdir);
|
|
26
|
+
const archive = await readFile(resolve(archivePath));
|
|
27
|
+
const extractDir = join(tmpdir(), `pipex-run-bundle-${Date.now()}-${Math.random().toString(36).slice(2)}`);
|
|
28
|
+
try {
|
|
29
|
+
const pipeline = await extractBundle(archive, extractDir);
|
|
30
|
+
// Write a pipeline.json so PipelineRunner can load it normally.
|
|
31
|
+
// The extract directory becomes the pipelineRoot — all relative
|
|
32
|
+
// host paths in mounts/sources resolve against it.
|
|
33
|
+
const pipelineFile = join(extractDir, 'pipeline.json');
|
|
34
|
+
await writeFile(pipelineFile, JSON.stringify(pipeline));
|
|
35
|
+
const loader = new PipelineLoader();
|
|
36
|
+
const runtime = new DockerCliExecutor();
|
|
37
|
+
const reporter = json ? new ConsoleReporter() : new InteractiveReporter({ verbose: options.verbose });
|
|
38
|
+
const runner = new PipelineRunner(loader, runtime, reporter, workdirRoot);
|
|
39
|
+
const force = options.force === true
|
|
40
|
+
? true
|
|
41
|
+
: (typeof options.force === 'string' ? options.force.split(',') : undefined);
|
|
42
|
+
const target = options.target ? options.target.split(',') : undefined;
|
|
43
|
+
await runner.run(pipelineFile, { workspace: options.workspace, force, dryRun: options.dryRun, target, concurrency: options.concurrency });
|
|
44
|
+
if (json) {
|
|
45
|
+
console.log('Pipeline completed');
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
catch (error) {
|
|
49
|
+
if (json) {
|
|
50
|
+
console.error('Pipeline failed:', error instanceof Error ? error.message : error);
|
|
51
|
+
}
|
|
52
|
+
process.exitCode = 1;
|
|
53
|
+
throw error;
|
|
54
|
+
}
|
|
55
|
+
finally {
|
|
56
|
+
await rm(extractDir, { recursive: true, force: true });
|
|
57
|
+
}
|
|
58
|
+
});
|
|
59
|
+
}
|
package/dist/cli/commands/run.js
CHANGED
|
@@ -1,8 +1,9 @@
|
|
|
1
1
|
import { resolve } from 'node:path';
|
|
2
2
|
import { DockerCliExecutor } from '../../engine/docker-executor.js';
|
|
3
|
-
import { PipelineLoader } from '
|
|
4
|
-
import { PipelineRunner } from '
|
|
5
|
-
import { ConsoleReporter
|
|
3
|
+
import { PipelineLoader } from '../../core/pipeline-loader.js';
|
|
4
|
+
import { PipelineRunner } from '../../core/pipeline-runner.js';
|
|
5
|
+
import { ConsoleReporter } from '../../core/reporter.js';
|
|
6
|
+
import { InteractiveReporter } from '../interactive-reporter.js';
|
|
6
7
|
import { getGlobalOptions } from '../utils.js';
|
|
7
8
|
export function registerRunCommand(program) {
|
|
8
9
|
program
|
|
@@ -13,6 +14,9 @@ export function registerRunCommand(program) {
|
|
|
13
14
|
.option('-f, --force [steps]', 'Skip cache for all steps, or a comma-separated list (e.g. --force step1,step2)')
|
|
14
15
|
.option('--dry-run', 'Validate pipeline and show what would run without executing')
|
|
15
16
|
.option('--verbose', 'Stream container logs in real-time (interactive mode)')
|
|
17
|
+
.option('-t, --target <steps>', 'Execute only these steps and their dependencies (comma-separated)')
|
|
18
|
+
.option('-c, --concurrency <number>', 'Max parallel step executions (default: CPU count)', Number)
|
|
19
|
+
.option('--env-file <path>', 'Load environment variables from a dotenv file for all steps')
|
|
16
20
|
.action(async (pipelineFile, options, cmd) => {
|
|
17
21
|
const { workdir, json } = getGlobalOptions(cmd);
|
|
18
22
|
const workdirRoot = resolve(workdir);
|
|
@@ -24,7 +28,8 @@ export function registerRunCommand(program) {
|
|
|
24
28
|
const force = options.force === true
|
|
25
29
|
? true
|
|
26
30
|
: (typeof options.force === 'string' ? options.force.split(',') : undefined);
|
|
27
|
-
|
|
31
|
+
const target = options.target ? options.target.split(',') : undefined;
|
|
32
|
+
await runner.run(pipelineFile, { workspace: options.workspace, force, dryRun: options.dryRun, target, concurrency: options.concurrency, envFile: options.envFile });
|
|
28
33
|
if (json) {
|
|
29
34
|
console.log('Pipeline completed');
|
|
30
35
|
}
|
|
@@ -1,9 +1,20 @@
|
|
|
1
|
+
import process from 'node:process';
|
|
1
2
|
import { readFile } from 'node:fs/promises';
|
|
2
3
|
import { join, resolve } from 'node:path';
|
|
3
4
|
import chalk from 'chalk';
|
|
4
5
|
import { Workspace } from '../../engine/workspace.js';
|
|
5
|
-
import { StateManager } from '
|
|
6
|
-
import { dirSize,
|
|
6
|
+
import { StateManager } from '../../core/state.js';
|
|
7
|
+
import { dirSize, formatDuration, formatSize } from '../../core/utils.js';
|
|
8
|
+
import { getGlobalOptions } from '../utils.js';
|
|
9
|
+
function isProcessAlive(pid) {
|
|
10
|
+
try {
|
|
11
|
+
process.kill(pid, 0);
|
|
12
|
+
return true;
|
|
13
|
+
}
|
|
14
|
+
catch {
|
|
15
|
+
return false;
|
|
16
|
+
}
|
|
17
|
+
}
|
|
7
18
|
export function registerShowCommand(program) {
|
|
8
19
|
program
|
|
9
20
|
.command('show')
|
|
@@ -16,10 +27,7 @@ export function registerShowCommand(program) {
|
|
|
16
27
|
const state = new StateManager(workspace.root);
|
|
17
28
|
await state.load();
|
|
18
29
|
const steps = state.listSteps();
|
|
19
|
-
|
|
20
|
-
console.log(chalk.gray('No runs found in this workspace.'));
|
|
21
|
-
return;
|
|
22
|
-
}
|
|
30
|
+
const runningSteps = await workspace.listRunningSteps();
|
|
23
31
|
const rows = [];
|
|
24
32
|
let totalSize = 0;
|
|
25
33
|
for (const { stepId, runId } of steps) {
|
|
@@ -43,6 +51,30 @@ export function registerShowCommand(program) {
|
|
|
43
51
|
rows.push({ stepId, status: 'unknown', duration: '-', size: '-', date: '-', runId });
|
|
44
52
|
}
|
|
45
53
|
}
|
|
54
|
+
// Add running steps (not already in committed state)
|
|
55
|
+
const committedStepIds = new Set(rows.map(r => r.stepId));
|
|
56
|
+
for (const running of runningSteps) {
|
|
57
|
+
if (committedStepIds.has(running.stepId)) {
|
|
58
|
+
continue;
|
|
59
|
+
}
|
|
60
|
+
if (!isProcessAlive(running.pid)) {
|
|
61
|
+
continue;
|
|
62
|
+
}
|
|
63
|
+
const elapsedMs = Date.now() - new Date(running.startedAt).getTime();
|
|
64
|
+
rows.push({
|
|
65
|
+
stepId: running.stepId,
|
|
66
|
+
stepName: running.stepName,
|
|
67
|
+
status: 'running',
|
|
68
|
+
duration: formatDuration(elapsedMs),
|
|
69
|
+
size: '-',
|
|
70
|
+
date: '-',
|
|
71
|
+
runId: '-'
|
|
72
|
+
});
|
|
73
|
+
}
|
|
74
|
+
if (rows.length === 0) {
|
|
75
|
+
console.log(chalk.gray('No runs found in this workspace.'));
|
|
76
|
+
return;
|
|
77
|
+
}
|
|
46
78
|
if (json) {
|
|
47
79
|
console.log(JSON.stringify(rows, null, 2));
|
|
48
80
|
return;
|
|
@@ -55,7 +87,10 @@ export function registerShowCommand(program) {
|
|
|
55
87
|
console.log(chalk.bold(`${'STEP'.padEnd(stepWidth)} ${'STATUS'.padEnd(statusWidth)} ${'DURATION'.padStart(durationWidth)} ${'SIZE'.padStart(sizeWidth)} ${'FINISHED'.padEnd(dateWidth)}`));
|
|
56
88
|
for (const row of rows) {
|
|
57
89
|
const stepLabel = row.stepName ? `${row.stepId} (${row.stepName})` : row.stepId;
|
|
58
|
-
const
|
|
90
|
+
const statusColor = row.status === 'success'
|
|
91
|
+
? chalk.green
|
|
92
|
+
: (row.status === 'running' ? chalk.yellow : chalk.red);
|
|
93
|
+
const statusText = statusColor(row.status);
|
|
59
94
|
const statusPad = statusWidth + (statusText.length - row.status.length);
|
|
60
95
|
const cols = [
|
|
61
96
|
stepLabel.padEnd(stepWidth),
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import jexlModule from 'jexl';
|
|
2
|
+
const jexl = new jexlModule.Jexl();
|
|
3
|
+
export async function evaluateCondition(expression, context) {
|
|
4
|
+
try {
|
|
5
|
+
const result = await jexl.eval(expression, context);
|
|
6
|
+
return Boolean(result);
|
|
7
|
+
}
|
|
8
|
+
catch {
|
|
9
|
+
return false;
|
|
10
|
+
}
|
|
11
|
+
}
|