@livingdata/pipex 0.0.9 → 0.0.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (88) hide show
  1. package/README.md +154 -14
  2. package/dist/__tests__/errors.js +162 -0
  3. package/dist/__tests__/helpers.js +41 -0
  4. package/dist/__tests__/types.js +8 -0
  5. package/dist/cli/__tests__/condition.js +23 -0
  6. package/dist/cli/__tests__/dag.js +154 -0
  7. package/dist/cli/__tests__/pipeline-loader.js +267 -0
  8. package/dist/cli/__tests__/pipeline-runner.js +257 -0
  9. package/dist/cli/__tests__/state-persistence.js +80 -0
  10. package/dist/cli/__tests__/state.js +58 -0
  11. package/dist/cli/__tests__/step-runner.js +116 -0
  12. package/dist/cli/commands/bundle.js +35 -0
  13. package/dist/cli/commands/cat.js +54 -0
  14. package/dist/cli/commands/exec.js +89 -0
  15. package/dist/cli/commands/export.js +2 -2
  16. package/dist/cli/commands/inspect.js +1 -1
  17. package/dist/cli/commands/list.js +2 -1
  18. package/dist/cli/commands/logs.js +1 -1
  19. package/dist/cli/commands/prune.js +1 -1
  20. package/dist/cli/commands/rm-step.js +41 -0
  21. package/dist/cli/commands/run-bundle.js +59 -0
  22. package/dist/cli/commands/run.js +9 -4
  23. package/dist/cli/commands/show.js +42 -7
  24. package/dist/cli/condition.js +11 -0
  25. package/dist/cli/dag.js +143 -0
  26. package/dist/cli/index.js +6 -0
  27. package/dist/cli/interactive-reporter.js +227 -0
  28. package/dist/cli/pipeline-loader.js +10 -110
  29. package/dist/cli/pipeline-runner.js +164 -78
  30. package/dist/cli/reporter.js +2 -158
  31. package/dist/cli/state.js +8 -0
  32. package/dist/cli/step-loader.js +25 -0
  33. package/dist/cli/step-resolver.js +111 -0
  34. package/dist/cli/step-runner.js +226 -0
  35. package/dist/cli/utils.js +0 -46
  36. package/dist/core/__tests__/bundle.js +663 -0
  37. package/dist/core/__tests__/condition.js +23 -0
  38. package/dist/core/__tests__/dag.js +154 -0
  39. package/dist/core/__tests__/env-file.test.js +41 -0
  40. package/dist/core/__tests__/event-aggregator.js +244 -0
  41. package/dist/core/__tests__/pipeline-loader.js +267 -0
  42. package/dist/core/__tests__/pipeline-runner.js +257 -0
  43. package/dist/core/__tests__/state-persistence.js +80 -0
  44. package/dist/core/__tests__/state.js +58 -0
  45. package/dist/core/__tests__/step-runner.js +118 -0
  46. package/dist/core/__tests__/stream-reporter.js +142 -0
  47. package/dist/core/__tests__/transport.js +50 -0
  48. package/dist/core/__tests__/utils.js +40 -0
  49. package/dist/core/bundle.js +130 -0
  50. package/dist/core/condition.js +11 -0
  51. package/dist/core/dag.js +143 -0
  52. package/dist/core/env-file.js +6 -0
  53. package/dist/core/event-aggregator.js +114 -0
  54. package/dist/core/index.js +14 -0
  55. package/dist/core/pipeline-loader.js +81 -0
  56. package/dist/core/pipeline-runner.js +360 -0
  57. package/dist/core/reporter.js +11 -0
  58. package/dist/core/state.js +110 -0
  59. package/dist/core/step-loader.js +25 -0
  60. package/dist/core/step-resolver.js +117 -0
  61. package/dist/core/step-runner.js +225 -0
  62. package/dist/core/stream-reporter.js +41 -0
  63. package/dist/core/transport.js +9 -0
  64. package/dist/core/utils.js +56 -0
  65. package/dist/engine/__tests__/workspace.js +288 -0
  66. package/dist/engine/docker-executor.js +10 -2
  67. package/dist/engine/index.js +1 -0
  68. package/dist/engine/workspace.js +76 -12
  69. package/dist/errors.js +122 -0
  70. package/dist/index.js +3 -0
  71. package/dist/kits/__tests__/index.js +23 -0
  72. package/dist/kits/builtin/__tests__/node.js +74 -0
  73. package/dist/kits/builtin/__tests__/python.js +67 -0
  74. package/dist/kits/builtin/__tests__/shell.js +74 -0
  75. package/dist/kits/builtin/node.js +10 -5
  76. package/dist/kits/builtin/python.js +10 -5
  77. package/dist/kits/builtin/shell.js +2 -1
  78. package/dist/kits/index.js +2 -1
  79. package/package.json +6 -3
  80. package/dist/cli/types.js +0 -3
  81. package/dist/engine/docker-runtime.js +0 -65
  82. package/dist/engine/runtime.js +0 -2
  83. package/dist/kits/bash.js +0 -19
  84. package/dist/kits/builtin/bash.js +0 -19
  85. package/dist/kits/node.js +0 -56
  86. package/dist/kits/python.js +0 -51
  87. package/dist/kits/types.js +0 -1
  88. package/dist/reporter.js +0 -13
@@ -0,0 +1,116 @@
1
+ import { readFile } from 'node:fs/promises';
2
+ import { join } from 'node:path';
3
+ import test from 'ava';
4
+ import { Workspace } from '../../engine/workspace.js';
5
+ import { DockerCliExecutor } from '../../engine/docker-executor.js';
6
+ import { ContainerCrashError } from '../../errors.js';
7
+ import { StateManager } from '../state.js';
8
+ import { StepRunner } from '../step-runner.js';
9
+ import { createTmpDir, isDockerAvailable, noopReporter, recordingReporter } from '../../__tests__/helpers.js';
10
+ const hasDocker = isDockerAvailable();
11
+ const dockerTest = hasDocker ? test : test.skip;
12
+ // -- helpers -----------------------------------------------------------------
13
+ function makeStep(overrides) {
14
+ return {
15
+ image: 'alpine:3.20',
16
+ cmd: ['sh', '-c', 'echo hello'],
17
+ ...overrides
18
+ };
19
+ }
20
+ async function setupWorkspace() {
21
+ const tmpDir = await createTmpDir();
22
+ const workspace = await Workspace.create(tmpDir, 'test-ws');
23
+ const state = new StateManager(workspace.root);
24
+ await state.load();
25
+ return { workspace, state, tmpDir };
26
+ }
27
+ // -- minimal execution -------------------------------------------------------
28
+ dockerTest('minimal step writes artifact and returns exitCode 0', async (t) => {
29
+ const { workspace, state } = await setupWorkspace();
30
+ const runner = new StepRunner(new DockerCliExecutor(), noopReporter);
31
+ const step = makeStep({ id: 'greet', cmd: ['sh', '-c', 'echo hello > /output/greeting.txt'] });
32
+ const result = await runner.run({ workspace, state, step, inputs: new Map(), pipelineRoot: '/' });
33
+ t.is(result.exitCode, 0);
34
+ t.truthy(result.runId);
35
+ // Artifact exists in committed run
36
+ const content = await readFile(join(workspace.runArtifactsPath(result.runId), 'greeting.txt'), 'utf8');
37
+ t.is(content.trim(), 'hello');
38
+ });
39
+ dockerTest('meta.json exists with correct fields', async (t) => {
40
+ const { workspace, state } = await setupWorkspace();
41
+ const runner = new StepRunner(new DockerCliExecutor(), noopReporter);
42
+ const step = makeStep({ id: 'meta-test', cmd: ['sh', '-c', 'echo ok > /output/out.txt'] });
43
+ const result = await runner.run({ workspace, state, step, inputs: new Map(), pipelineRoot: '/' });
44
+ const metaPath = join(workspace.runPath(result.runId), 'meta.json');
45
+ const meta = JSON.parse(await readFile(metaPath, 'utf8'));
46
+ t.is(meta.runId, result.runId);
47
+ t.is(meta.exitCode, 0);
48
+ t.is(meta.image, 'alpine:3.20');
49
+ t.deepEqual(meta.cmd, ['sh', '-c', 'echo ok > /output/out.txt']);
50
+ });
51
+ // -- log capture -------------------------------------------------------------
52
+ dockerTest('stdout and stderr are captured to log files', async (t) => {
53
+ const { workspace, state } = await setupWorkspace();
54
+ const runner = new StepRunner(new DockerCliExecutor(), noopReporter);
55
+ const step = makeStep({ id: 'logs', cmd: ['sh', '-c', 'echo out-line && echo err-line >&2'] });
56
+ const result = await runner.run({ workspace, state, step, inputs: new Map(), pipelineRoot: '/' });
57
+ const stdout = await readFile(join(workspace.runPath(result.runId), 'stdout.log'), 'utf8');
58
+ const stderr = await readFile(join(workspace.runPath(result.runId), 'stderr.log'), 'utf8');
59
+ t.true(stdout.includes('out-line'));
60
+ t.true(stderr.includes('err-line'));
61
+ });
62
+ // -- cache hit ---------------------------------------------------------------
63
+ dockerTest('second run of same step is cached (STEP_SKIPPED)', async (t) => {
64
+ const { workspace, state } = await setupWorkspace();
65
+ const { reporter, events } = recordingReporter();
66
+ const runner = new StepRunner(new DockerCliExecutor(), reporter);
67
+ const step = makeStep({ id: 'cached', cmd: ['sh', '-c', 'echo hi > /output/x.txt'] });
68
+ const first = await runner.run({ workspace, state, step, inputs: new Map(), pipelineRoot: '/' });
69
+ const second = await runner.run({ workspace, state, step, inputs: new Map(), pipelineRoot: '/' });
70
+ t.is(first.runId, second.runId);
71
+ const skipped = events.find(e => e.event === 'STEP_SKIPPED');
72
+ t.truthy(skipped);
73
+ t.is(skipped.reason, 'cached');
74
+ });
75
+ // -- force -------------------------------------------------------------------
76
+ dockerTest('force: true produces new runId', async (t) => {
77
+ const { workspace, state } = await setupWorkspace();
78
+ const runner = new StepRunner(new DockerCliExecutor(), noopReporter);
79
+ const step = makeStep({ id: 'force-test', cmd: ['sh', '-c', 'echo data > /output/f.txt'] });
80
+ const first = await runner.run({ workspace, state, step, inputs: new Map(), pipelineRoot: '/' });
81
+ const second = await runner.run({ workspace, state, step, inputs: new Map(), pipelineRoot: '/', force: true });
82
+ t.not(first.runId, second.runId);
83
+ t.is(second.exitCode, 0);
84
+ });
85
+ // -- ephemeral ---------------------------------------------------------------
86
+ dockerTest('ephemeral: true returns exitCode but no runId', async (t) => {
87
+ const { workspace, state } = await setupWorkspace();
88
+ const runner = new StepRunner(new DockerCliExecutor(), noopReporter);
89
+ const step = makeStep({ id: 'ephemeral', cmd: ['sh', '-c', 'echo temp > /output/t.txt'] });
90
+ const result = await runner.run({ workspace, state, step, inputs: new Map(), pipelineRoot: '/', ephemeral: true });
91
+ t.is(result.exitCode, 0);
92
+ t.is(result.runId, undefined);
93
+ // No run committed
94
+ const runs = await workspace.listRuns();
95
+ t.is(runs.length, 0);
96
+ });
97
+ // -- failure -----------------------------------------------------------------
98
+ dockerTest('non-zero exit throws ContainerCrashError', async (t) => {
99
+ const { workspace, state } = await setupWorkspace();
100
+ const runner = new StepRunner(new DockerCliExecutor(), noopReporter);
101
+ const step = makeStep({ id: 'fail', cmd: ['sh', '-c', 'exit 1'] });
102
+ const error = await t.throwsAsync(async () => runner.run({ workspace, state, step, inputs: new Map(), pipelineRoot: '/' }));
103
+ t.true(error instanceof ContainerCrashError);
104
+ });
105
+ // -- allowFailure ------------------------------------------------------------
106
+ dockerTest('allowFailure: true commits run with non-zero exitCode', async (t) => {
107
+ const { workspace, state } = await setupWorkspace();
108
+ const runner = new StepRunner(new DockerCliExecutor(), noopReporter);
109
+ const step = makeStep({ id: 'allow-fail', cmd: ['sh', '-c', 'exit 1'], allowFailure: true });
110
+ const result = await runner.run({ workspace, state, step, inputs: new Map(), pipelineRoot: '/' });
111
+ t.truthy(result.runId);
112
+ t.is(result.exitCode, 1);
113
+ // Run was committed
114
+ const runs = await workspace.listRuns();
115
+ t.true(runs.includes(result.runId));
116
+ });
@@ -0,0 +1,35 @@
1
+ import process from 'node:process';
2
+ import { writeFile } from 'node:fs/promises';
3
+ import { basename, resolve } from 'node:path';
4
+ import chalk from 'chalk';
5
+ import { buildBundle } from '../../core/bundle.js';
6
+ import { PipelineLoader } from '../../core/pipeline-loader.js';
7
+ import { formatSize } from '../../core/utils.js';
8
+ export function registerBundleCommand(program) {
9
+ program
10
+ .command('bundle')
11
+ .description('Package a pipeline and its local dependencies into a tar.gz archive')
12
+ .argument('<pipeline>', 'Path to the pipeline file')
13
+ .option('-o, --output <path>', 'Output file path (default: <pipeline-id>.tar.gz)')
14
+ .action(async (pipelineFile, options) => {
15
+ try {
16
+ const pipelinePath = resolve(pipelineFile);
17
+ // Determine output path
18
+ let outputPath = options.output;
19
+ if (!outputPath) {
20
+ const loader = new PipelineLoader();
21
+ const pipeline = await loader.load(pipelinePath);
22
+ outputPath = `${pipeline.id}.tar.gz`;
23
+ }
24
+ outputPath = resolve(outputPath);
25
+ const archive = await buildBundle(pipelinePath);
26
+ await writeFile(outputPath, archive);
27
+ console.log(chalk.green(`Bundle created: ${basename(outputPath)} (${formatSize(archive.length)})`));
28
+ }
29
+ catch (error) {
30
+ const message = error instanceof Error ? error.message : String(error);
31
+ console.error(chalk.red(`Bundle failed: ${message}`));
32
+ process.exitCode = 1;
33
+ }
34
+ });
35
+ }
@@ -0,0 +1,54 @@
1
+ import process from 'node:process';
2
+ import { readFile, readdir, stat } from 'node:fs/promises';
3
+ import { join, resolve } from 'node:path';
4
+ import chalk from 'chalk';
5
+ import { Workspace } from '../../engine/workspace.js';
6
+ import { StateManager } from '../../core/state.js';
7
+ import { getGlobalOptions } from '../utils.js';
8
+ export function registerCatCommand(program) {
9
+ program
10
+ .command('cat')
11
+ .description('Read artifact content from a step\'s latest run')
12
+ .argument('<workspace>', 'Workspace name')
13
+ .argument('<step>', 'Step ID')
14
+ .argument('[path]', 'Path within artifacts (omit to list)')
15
+ .action(async (workspaceName, stepId, artifactPath, _options, cmd) => {
16
+ const { workdir } = getGlobalOptions(cmd);
17
+ const workdirRoot = resolve(workdir);
18
+ const workspace = await Workspace.open(workdirRoot, workspaceName);
19
+ const state = new StateManager(workspace.root);
20
+ await state.load();
21
+ const stepState = state.getStep(stepId);
22
+ if (!stepState) {
23
+ console.error(chalk.red(`No run found for step: ${stepId}`));
24
+ process.exitCode = 1;
25
+ return;
26
+ }
27
+ const artifactsDir = workspace.runArtifactsPath(stepState.runId);
28
+ const targetPath = artifactPath ? join(artifactsDir, artifactPath) : artifactsDir;
29
+ // Prevent path traversal
30
+ if (!targetPath.startsWith(artifactsDir)) {
31
+ console.error(chalk.red('Invalid path: must be within artifacts directory'));
32
+ process.exitCode = 1;
33
+ return;
34
+ }
35
+ try {
36
+ const info = await stat(targetPath);
37
+ if (info.isDirectory()) {
38
+ const entries = await readdir(targetPath, { withFileTypes: true });
39
+ for (const entry of entries) {
40
+ const suffix = entry.isDirectory() ? '/' : '';
41
+ console.log(entry.name + suffix);
42
+ }
43
+ }
44
+ else {
45
+ const content = await readFile(targetPath);
46
+ process.stdout.write(content);
47
+ }
48
+ }
49
+ catch {
50
+ console.error(chalk.red(`Not found: ${artifactPath ?? '(artifacts directory)'}`));
51
+ process.exitCode = 1;
52
+ }
53
+ });
54
+ }
@@ -0,0 +1,89 @@
1
+ import process from 'node:process';
2
+ import { randomUUID } from 'node:crypto';
3
+ import { dirname, resolve } from 'node:path';
4
+ import { DockerCliExecutor } from '../../engine/docker-executor.js';
5
+ import { Workspace } from '../../engine/workspace.js';
6
+ import { loadStepFile } from '../../core/step-loader.js';
7
+ import { StepRunner } from '../../core/step-runner.js';
8
+ import { StateManager } from '../../core/state.js';
9
+ import { ConsoleReporter } from '../../core/reporter.js';
10
+ import { InteractiveReporter } from '../interactive-reporter.js';
11
+ import { getGlobalOptions } from '../utils.js';
12
+ export function registerExecCommand(program) {
13
+ program
14
+ .command('exec')
15
+ .description('Execute a single step in a workspace')
16
+ .argument('<workspace>', 'Workspace name')
17
+ .requiredOption('-f, --file <path>', 'Step definition file (YAML or JSON)')
18
+ .option('--step <id>', 'Step ID (overrides file\'s id)')
19
+ .option('--input <specs...>', 'Input steps (e.g. "extract" or "data=extract")')
20
+ .option('--ephemeral', 'Don\'t commit run, stream stdout to terminal')
21
+ .option('--force', 'Skip cache check')
22
+ .option('--verbose', 'Stream container logs in real-time')
23
+ .action(async (workspaceName, options, cmd) => {
24
+ const { workdir, json } = getGlobalOptions(cmd);
25
+ const workdirRoot = resolve(workdir);
26
+ const runtime = new DockerCliExecutor();
27
+ const reporter = json ? new ConsoleReporter() : new InteractiveReporter({ verbose: options.verbose });
28
+ // Load step from file
29
+ const stepFilePath = resolve(options.file);
30
+ const step = await loadStepFile(stepFilePath, options.step);
31
+ const pipelineRoot = dirname(stepFilePath);
32
+ // Open or create workspace
33
+ let workspace;
34
+ try {
35
+ workspace = await Workspace.open(workdirRoot, workspaceName);
36
+ }
37
+ catch {
38
+ workspace = await Workspace.create(workdirRoot, workspaceName);
39
+ }
40
+ await workspace.cleanupStaging();
41
+ await runtime.check();
42
+ await runtime.cleanupContainers(workspace.id);
43
+ // Load state and resolve inputs
44
+ const state = new StateManager(workspace.root);
45
+ await state.load();
46
+ const inputs = new Map();
47
+ if (options.input) {
48
+ for (const spec of options.input) {
49
+ const { alias, stepId } = parseInputSpec(spec);
50
+ const stepState = state.getStep(stepId);
51
+ if (!stepState) {
52
+ console.error(`No run found for input step: ${stepId}`);
53
+ process.exitCode = 1;
54
+ return;
55
+ }
56
+ inputs.set(alias, stepState.runId);
57
+ }
58
+ }
59
+ // Merge parsed inputs into step.inputs
60
+ if (inputs.size > 0 && !step.inputs) {
61
+ step.inputs = [];
62
+ }
63
+ for (const [alias, _runId] of inputs) {
64
+ // Only add if not already declared in step file
65
+ if (!step.inputs.some(i => i.step === alias)) {
66
+ step.inputs.push({ step: alias });
67
+ }
68
+ }
69
+ const job = { workspaceId: workspace.id, jobId: randomUUID() };
70
+ const runner = new StepRunner(runtime, reporter);
71
+ await runner.run({
72
+ workspace,
73
+ state,
74
+ step,
75
+ inputs,
76
+ pipelineRoot,
77
+ force: options.force,
78
+ ephemeral: options.ephemeral,
79
+ job
80
+ });
81
+ });
82
+ }
83
+ function parseInputSpec(spec) {
84
+ const eqIdx = spec.indexOf('=');
85
+ if (eqIdx > 0) {
86
+ return { alias: spec.slice(0, eqIdx), stepId: spec.slice(eqIdx + 1) };
87
+ }
88
+ return { alias: spec, stepId: spec };
89
+ }
@@ -3,7 +3,7 @@ import { cp } from 'node:fs/promises';
3
3
  import { resolve } from 'node:path';
4
4
  import chalk from 'chalk';
5
5
  import { Workspace } from '../../engine/workspace.js';
6
- import { StateManager } from '../state.js';
6
+ import { StateManager } from '../../core/state.js';
7
7
  import { getGlobalOptions } from '../utils.js';
8
8
  export function registerExportCommand(program) {
9
9
  program
@@ -12,7 +12,7 @@ export function registerExportCommand(program) {
12
12
  .argument('<workspace>', 'Workspace name')
13
13
  .argument('<step>', 'Step ID')
14
14
  .argument('<dest>', 'Destination directory')
15
- .action(async (workspaceName, stepId, dest, cmd) => {
15
+ .action(async (workspaceName, stepId, dest, _options, cmd) => {
16
16
  const { workdir } = getGlobalOptions(cmd);
17
17
  const workdirRoot = resolve(workdir);
18
18
  const workspace = await Workspace.open(workdirRoot, workspaceName);
@@ -3,7 +3,7 @@ import { readFile } from 'node:fs/promises';
3
3
  import { join, resolve } from 'node:path';
4
4
  import chalk from 'chalk';
5
5
  import { Workspace } from '../../engine/workspace.js';
6
- import { StateManager } from '../state.js';
6
+ import { StateManager } from '../../core/state.js';
7
7
  import { getGlobalOptions } from '../utils.js';
8
8
  export function registerInspectCommand(program) {
9
9
  program
@@ -1,7 +1,8 @@
1
1
  import { resolve } from 'node:path';
2
2
  import chalk from 'chalk';
3
3
  import { Workspace } from '../../engine/workspace.js';
4
- import { dirSize, formatSize, getGlobalOptions } from '../utils.js';
4
+ import { dirSize, formatSize } from '../../core/utils.js';
5
+ import { getGlobalOptions } from '../utils.js';
5
6
  export function registerListCommand(program) {
6
7
  program
7
8
  .command('list')
@@ -3,7 +3,7 @@ import { readFile } from 'node:fs/promises';
3
3
  import { join, resolve } from 'node:path';
4
4
  import chalk from 'chalk';
5
5
  import { Workspace } from '../../engine/workspace.js';
6
- import { StateManager } from '../state.js';
6
+ import { StateManager } from '../../core/state.js';
7
7
  import { getGlobalOptions } from '../utils.js';
8
8
  export function registerLogsCommand(program) {
9
9
  program
@@ -1,7 +1,7 @@
1
1
  import { resolve } from 'node:path';
2
2
  import chalk from 'chalk';
3
3
  import { Workspace } from '../../engine/workspace.js';
4
- import { StateManager } from '../state.js';
4
+ import { StateManager } from '../../core/state.js';
5
5
  import { getGlobalOptions } from '../utils.js';
6
6
  export function registerPruneCommand(program) {
7
7
  program
@@ -0,0 +1,41 @@
1
+ import process from 'node:process';
2
+ import { rm } from 'node:fs/promises';
3
+ import { join, resolve } from 'node:path';
4
+ import chalk from 'chalk';
5
+ import { Workspace } from '../../engine/workspace.js';
6
+ import { StateManager } from '../../core/state.js';
7
+ import { dirSize, formatSize } from '../../core/utils.js';
8
+ import { getGlobalOptions } from '../utils.js';
9
+ export function registerRmStepCommand(program) {
10
+ program
11
+ .command('rm-step')
12
+ .description('Remove a step\'s run and state entry')
13
+ .argument('<workspace>', 'Workspace name')
14
+ .argument('<step>', 'Step ID')
15
+ .action(async (workspaceName, stepId, _options, cmd) => {
16
+ const { workdir } = getGlobalOptions(cmd);
17
+ const workdirRoot = resolve(workdir);
18
+ const workspace = await Workspace.open(workdirRoot, workspaceName);
19
+ const state = new StateManager(workspace.root);
20
+ await state.load();
21
+ const stepState = state.getStep(stepId);
22
+ if (!stepState) {
23
+ console.error(chalk.red(`No run found for step: ${stepId}`));
24
+ process.exitCode = 1;
25
+ return;
26
+ }
27
+ const { runId } = stepState;
28
+ // Measure size before removal
29
+ const runDir = workspace.runPath(runId);
30
+ const size = await dirSize(runDir);
31
+ // Remove run directory
32
+ await rm(runDir, { recursive: true, force: true });
33
+ // Remove step-runs symlink
34
+ const linkPath = join(workspace.root, 'step-runs', stepId);
35
+ await rm(linkPath, { force: true });
36
+ // Remove from state
37
+ state.removeStep(stepId);
38
+ await state.save();
39
+ console.log(chalk.green(`Removed step ${chalk.bold(stepId)} (run ${runId}, freed ${formatSize(size)})`));
40
+ });
41
+ }
@@ -0,0 +1,59 @@
1
+ import process from 'node:process';
2
+ import { readFile, writeFile, rm } from 'node:fs/promises';
3
+ import { tmpdir } from 'node:os';
4
+ import { join, resolve } from 'node:path';
5
+ import { DockerCliExecutor } from '../../engine/docker-executor.js';
6
+ import { extractBundle } from '../../core/bundle.js';
7
+ import { PipelineLoader } from '../../core/pipeline-loader.js';
8
+ import { PipelineRunner } from '../../core/pipeline-runner.js';
9
+ import { ConsoleReporter } from '../../core/reporter.js';
10
+ import { InteractiveReporter } from '../interactive-reporter.js';
11
+ import { getGlobalOptions } from '../utils.js';
12
+ export function registerRunBundleCommand(program) {
13
+ program
14
+ .command('run-bundle')
15
+ .description('Execute a pipeline from a bundle archive')
16
+ .argument('<archive>', 'Bundle archive (.tar.gz)')
17
+ .option('-w, --workspace <name>', 'Workspace name (for caching)')
18
+ .option('-f, --force [steps]', 'Skip cache for all steps, or a comma-separated list')
19
+ .option('--dry-run', 'Validate pipeline and show what would run without executing')
20
+ .option('--verbose', 'Stream container logs in real-time (interactive mode)')
21
+ .option('-t, --target <steps>', 'Execute only these steps and their dependencies (comma-separated)')
22
+ .option('-c, --concurrency <number>', 'Max parallel step executions (default: CPU count)', Number)
23
+ .action(async (archivePath, options, cmd) => {
24
+ const { workdir, json } = getGlobalOptions(cmd);
25
+ const workdirRoot = resolve(workdir);
26
+ const archive = await readFile(resolve(archivePath));
27
+ const extractDir = join(tmpdir(), `pipex-run-bundle-${Date.now()}-${Math.random().toString(36).slice(2)}`);
28
+ try {
29
+ const pipeline = await extractBundle(archive, extractDir);
30
+ // Write a pipeline.json so PipelineRunner can load it normally.
31
+ // The extract directory becomes the pipelineRoot — all relative
32
+ // host paths in mounts/sources resolve against it.
33
+ const pipelineFile = join(extractDir, 'pipeline.json');
34
+ await writeFile(pipelineFile, JSON.stringify(pipeline));
35
+ const loader = new PipelineLoader();
36
+ const runtime = new DockerCliExecutor();
37
+ const reporter = json ? new ConsoleReporter() : new InteractiveReporter({ verbose: options.verbose });
38
+ const runner = new PipelineRunner(loader, runtime, reporter, workdirRoot);
39
+ const force = options.force === true
40
+ ? true
41
+ : (typeof options.force === 'string' ? options.force.split(',') : undefined);
42
+ const target = options.target ? options.target.split(',') : undefined;
43
+ await runner.run(pipelineFile, { workspace: options.workspace, force, dryRun: options.dryRun, target, concurrency: options.concurrency });
44
+ if (json) {
45
+ console.log('Pipeline completed');
46
+ }
47
+ }
48
+ catch (error) {
49
+ if (json) {
50
+ console.error('Pipeline failed:', error instanceof Error ? error.message : error);
51
+ }
52
+ process.exitCode = 1;
53
+ throw error;
54
+ }
55
+ finally {
56
+ await rm(extractDir, { recursive: true, force: true });
57
+ }
58
+ });
59
+ }
@@ -1,8 +1,9 @@
1
1
  import { resolve } from 'node:path';
2
2
  import { DockerCliExecutor } from '../../engine/docker-executor.js';
3
- import { PipelineLoader } from '../pipeline-loader.js';
4
- import { PipelineRunner } from '../pipeline-runner.js';
5
- import { ConsoleReporter, InteractiveReporter } from '../reporter.js';
3
+ import { PipelineLoader } from '../../core/pipeline-loader.js';
4
+ import { PipelineRunner } from '../../core/pipeline-runner.js';
5
+ import { ConsoleReporter } from '../../core/reporter.js';
6
+ import { InteractiveReporter } from '../interactive-reporter.js';
6
7
  import { getGlobalOptions } from '../utils.js';
7
8
  export function registerRunCommand(program) {
8
9
  program
@@ -13,6 +14,9 @@ export function registerRunCommand(program) {
13
14
  .option('-f, --force [steps]', 'Skip cache for all steps, or a comma-separated list (e.g. --force step1,step2)')
14
15
  .option('--dry-run', 'Validate pipeline and show what would run without executing')
15
16
  .option('--verbose', 'Stream container logs in real-time (interactive mode)')
17
+ .option('-t, --target <steps>', 'Execute only these steps and their dependencies (comma-separated)')
18
+ .option('-c, --concurrency <number>', 'Max parallel step executions (default: CPU count)', Number)
19
+ .option('--env-file <path>', 'Load environment variables from a dotenv file for all steps')
16
20
  .action(async (pipelineFile, options, cmd) => {
17
21
  const { workdir, json } = getGlobalOptions(cmd);
18
22
  const workdirRoot = resolve(workdir);
@@ -24,7 +28,8 @@ export function registerRunCommand(program) {
24
28
  const force = options.force === true
25
29
  ? true
26
30
  : (typeof options.force === 'string' ? options.force.split(',') : undefined);
27
- await runner.run(pipelineFile, { workspace: options.workspace, force, dryRun: options.dryRun });
31
+ const target = options.target ? options.target.split(',') : undefined;
32
+ await runner.run(pipelineFile, { workspace: options.workspace, force, dryRun: options.dryRun, target, concurrency: options.concurrency, envFile: options.envFile });
28
33
  if (json) {
29
34
  console.log('Pipeline completed');
30
35
  }
@@ -1,9 +1,20 @@
1
+ import process from 'node:process';
1
2
  import { readFile } from 'node:fs/promises';
2
3
  import { join, resolve } from 'node:path';
3
4
  import chalk from 'chalk';
4
5
  import { Workspace } from '../../engine/workspace.js';
5
- import { StateManager } from '../state.js';
6
- import { dirSize, formatSize, getGlobalOptions } from '../utils.js';
6
+ import { StateManager } from '../../core/state.js';
7
+ import { dirSize, formatDuration, formatSize } from '../../core/utils.js';
8
+ import { getGlobalOptions } from '../utils.js';
9
+ function isProcessAlive(pid) {
10
+ try {
11
+ process.kill(pid, 0);
12
+ return true;
13
+ }
14
+ catch {
15
+ return false;
16
+ }
17
+ }
7
18
  export function registerShowCommand(program) {
8
19
  program
9
20
  .command('show')
@@ -16,10 +27,7 @@ export function registerShowCommand(program) {
16
27
  const state = new StateManager(workspace.root);
17
28
  await state.load();
18
29
  const steps = state.listSteps();
19
- if (steps.length === 0) {
20
- console.log(chalk.gray('No runs found in this workspace.'));
21
- return;
22
- }
30
+ const runningSteps = await workspace.listRunningSteps();
23
31
  const rows = [];
24
32
  let totalSize = 0;
25
33
  for (const { stepId, runId } of steps) {
@@ -43,6 +51,30 @@ export function registerShowCommand(program) {
43
51
  rows.push({ stepId, status: 'unknown', duration: '-', size: '-', date: '-', runId });
44
52
  }
45
53
  }
54
+ // Add running steps (not already in committed state)
55
+ const committedStepIds = new Set(rows.map(r => r.stepId));
56
+ for (const running of runningSteps) {
57
+ if (committedStepIds.has(running.stepId)) {
58
+ continue;
59
+ }
60
+ if (!isProcessAlive(running.pid)) {
61
+ continue;
62
+ }
63
+ const elapsedMs = Date.now() - new Date(running.startedAt).getTime();
64
+ rows.push({
65
+ stepId: running.stepId,
66
+ stepName: running.stepName,
67
+ status: 'running',
68
+ duration: formatDuration(elapsedMs),
69
+ size: '-',
70
+ date: '-',
71
+ runId: '-'
72
+ });
73
+ }
74
+ if (rows.length === 0) {
75
+ console.log(chalk.gray('No runs found in this workspace.'));
76
+ return;
77
+ }
46
78
  if (json) {
47
79
  console.log(JSON.stringify(rows, null, 2));
48
80
  return;
@@ -55,7 +87,10 @@ export function registerShowCommand(program) {
55
87
  console.log(chalk.bold(`${'STEP'.padEnd(stepWidth)} ${'STATUS'.padEnd(statusWidth)} ${'DURATION'.padStart(durationWidth)} ${'SIZE'.padStart(sizeWidth)} ${'FINISHED'.padEnd(dateWidth)}`));
56
88
  for (const row of rows) {
57
89
  const stepLabel = row.stepName ? `${row.stepId} (${row.stepName})` : row.stepId;
58
- const statusText = row.status === 'success' ? chalk.green(row.status) : chalk.red(row.status);
90
+ const statusColor = row.status === 'success'
91
+ ? chalk.green
92
+ : (row.status === 'running' ? chalk.yellow : chalk.red);
93
+ const statusText = statusColor(row.status);
59
94
  const statusPad = statusWidth + (statusText.length - row.status.length);
60
95
  const cols = [
61
96
  stepLabel.padEnd(stepWidth),
@@ -0,0 +1,11 @@
1
+ import jexlModule from 'jexl';
2
+ const jexl = new jexlModule.Jexl();
3
+ export async function evaluateCondition(expression, context) {
4
+ try {
5
+ const result = await jexl.eval(expression, context);
6
+ return Boolean(result);
7
+ }
8
+ catch {
9
+ return false;
10
+ }
11
+ }