@livingdata/pipex 0.0.8 → 0.0.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +186 -16
- package/dist/__tests__/errors.js +162 -0
- package/dist/__tests__/helpers.js +41 -0
- package/dist/__tests__/types.js +8 -0
- package/dist/cli/__tests__/condition.js +23 -0
- package/dist/cli/__tests__/dag.js +154 -0
- package/dist/cli/__tests__/pipeline-loader.js +267 -0
- package/dist/cli/__tests__/pipeline-runner.js +257 -0
- package/dist/cli/__tests__/state-persistence.js +80 -0
- package/dist/cli/__tests__/state.js +58 -0
- package/dist/cli/__tests__/step-runner.js +116 -0
- package/dist/cli/commands/bundle.js +35 -0
- package/dist/cli/commands/cat.js +54 -0
- package/dist/cli/commands/clean.js +22 -0
- package/dist/cli/commands/exec.js +89 -0
- package/dist/cli/commands/export.js +32 -0
- package/dist/cli/commands/inspect.js +58 -0
- package/dist/cli/commands/list.js +39 -0
- package/dist/cli/commands/logs.js +54 -0
- package/dist/cli/commands/prune.js +26 -0
- package/dist/cli/commands/rm-step.js +41 -0
- package/dist/cli/commands/rm.js +27 -0
- package/dist/cli/commands/run-bundle.js +59 -0
- package/dist/cli/commands/run.js +44 -0
- package/dist/cli/commands/show.js +108 -0
- package/dist/cli/condition.js +11 -0
- package/dist/cli/dag.js +143 -0
- package/dist/cli/index.js +24 -105
- package/dist/cli/interactive-reporter.js +227 -0
- package/dist/cli/pipeline-loader.js +10 -110
- package/dist/cli/pipeline-runner.js +256 -111
- package/dist/cli/reporter.js +2 -107
- package/dist/cli/state.js +30 -9
- package/dist/cli/step-loader.js +25 -0
- package/dist/cli/step-resolver.js +111 -0
- package/dist/cli/step-runner.js +226 -0
- package/dist/cli/utils.js +3 -0
- package/dist/core/__tests__/bundle.js +663 -0
- package/dist/core/__tests__/condition.js +23 -0
- package/dist/core/__tests__/dag.js +154 -0
- package/dist/core/__tests__/env-file.test.js +41 -0
- package/dist/core/__tests__/event-aggregator.js +244 -0
- package/dist/core/__tests__/pipeline-loader.js +267 -0
- package/dist/core/__tests__/pipeline-runner.js +257 -0
- package/dist/core/__tests__/state-persistence.js +80 -0
- package/dist/core/__tests__/state.js +58 -0
- package/dist/core/__tests__/step-runner.js +118 -0
- package/dist/core/__tests__/stream-reporter.js +142 -0
- package/dist/core/__tests__/transport.js +50 -0
- package/dist/core/__tests__/utils.js +40 -0
- package/dist/core/bundle.js +130 -0
- package/dist/core/condition.js +11 -0
- package/dist/core/dag.js +143 -0
- package/dist/core/env-file.js +6 -0
- package/dist/core/event-aggregator.js +114 -0
- package/dist/core/index.js +14 -0
- package/dist/core/pipeline-loader.js +81 -0
- package/dist/core/pipeline-runner.js +360 -0
- package/dist/core/reporter.js +11 -0
- package/dist/core/state.js +110 -0
- package/dist/core/step-loader.js +25 -0
- package/dist/core/step-resolver.js +117 -0
- package/dist/core/step-runner.js +225 -0
- package/dist/core/stream-reporter.js +41 -0
- package/dist/core/transport.js +9 -0
- package/dist/core/utils.js +56 -0
- package/dist/engine/__tests__/workspace.js +288 -0
- package/dist/engine/docker-executor.js +32 -6
- package/dist/engine/index.js +1 -0
- package/dist/engine/workspace.js +164 -66
- package/dist/errors.js +122 -0
- package/dist/index.js +3 -0
- package/dist/kits/__tests__/index.js +23 -0
- package/dist/kits/builtin/__tests__/node.js +74 -0
- package/dist/kits/builtin/__tests__/python.js +67 -0
- package/dist/kits/builtin/__tests__/shell.js +74 -0
- package/dist/kits/builtin/node.js +10 -5
- package/dist/kits/builtin/python.js +10 -5
- package/dist/kits/builtin/shell.js +2 -1
- package/dist/kits/index.js +2 -1
- package/package.json +6 -3
- package/dist/cli/types.js +0 -3
- package/dist/engine/docker-runtime.js +0 -65
- package/dist/engine/runtime.js +0 -2
- package/dist/kits/bash.js +0 -19
- package/dist/kits/builtin/bash.js +0 -19
- package/dist/kits/node.js +0 -56
- package/dist/kits/python.js +0 -51
- package/dist/kits/types.js +0 -1
- package/dist/reporter.js +0 -13
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
import process from 'node:process';
|
|
2
|
+
import { randomUUID } from 'node:crypto';
|
|
3
|
+
import { dirname, resolve } from 'node:path';
|
|
4
|
+
import { DockerCliExecutor } from '../../engine/docker-executor.js';
|
|
5
|
+
import { Workspace } from '../../engine/workspace.js';
|
|
6
|
+
import { loadStepFile } from '../../core/step-loader.js';
|
|
7
|
+
import { StepRunner } from '../../core/step-runner.js';
|
|
8
|
+
import { StateManager } from '../../core/state.js';
|
|
9
|
+
import { ConsoleReporter } from '../../core/reporter.js';
|
|
10
|
+
import { InteractiveReporter } from '../interactive-reporter.js';
|
|
11
|
+
import { getGlobalOptions } from '../utils.js';
|
|
12
|
+
export function registerExecCommand(program) {
|
|
13
|
+
program
|
|
14
|
+
.command('exec')
|
|
15
|
+
.description('Execute a single step in a workspace')
|
|
16
|
+
.argument('<workspace>', 'Workspace name')
|
|
17
|
+
.requiredOption('-f, --file <path>', 'Step definition file (YAML or JSON)')
|
|
18
|
+
.option('--step <id>', 'Step ID (overrides file\'s id)')
|
|
19
|
+
.option('--input <specs...>', 'Input steps (e.g. "extract" or "data=extract")')
|
|
20
|
+
.option('--ephemeral', 'Don\'t commit run, stream stdout to terminal')
|
|
21
|
+
.option('--force', 'Skip cache check')
|
|
22
|
+
.option('--verbose', 'Stream container logs in real-time')
|
|
23
|
+
.action(async (workspaceName, options, cmd) => {
|
|
24
|
+
const { workdir, json } = getGlobalOptions(cmd);
|
|
25
|
+
const workdirRoot = resolve(workdir);
|
|
26
|
+
const runtime = new DockerCliExecutor();
|
|
27
|
+
const reporter = json ? new ConsoleReporter() : new InteractiveReporter({ verbose: options.verbose });
|
|
28
|
+
// Load step from file
|
|
29
|
+
const stepFilePath = resolve(options.file);
|
|
30
|
+
const step = await loadStepFile(stepFilePath, options.step);
|
|
31
|
+
const pipelineRoot = dirname(stepFilePath);
|
|
32
|
+
// Open or create workspace
|
|
33
|
+
let workspace;
|
|
34
|
+
try {
|
|
35
|
+
workspace = await Workspace.open(workdirRoot, workspaceName);
|
|
36
|
+
}
|
|
37
|
+
catch {
|
|
38
|
+
workspace = await Workspace.create(workdirRoot, workspaceName);
|
|
39
|
+
}
|
|
40
|
+
await workspace.cleanupStaging();
|
|
41
|
+
await runtime.check();
|
|
42
|
+
await runtime.cleanupContainers(workspace.id);
|
|
43
|
+
// Load state and resolve inputs
|
|
44
|
+
const state = new StateManager(workspace.root);
|
|
45
|
+
await state.load();
|
|
46
|
+
const inputs = new Map();
|
|
47
|
+
if (options.input) {
|
|
48
|
+
for (const spec of options.input) {
|
|
49
|
+
const { alias, stepId } = parseInputSpec(spec);
|
|
50
|
+
const stepState = state.getStep(stepId);
|
|
51
|
+
if (!stepState) {
|
|
52
|
+
console.error(`No run found for input step: ${stepId}`);
|
|
53
|
+
process.exitCode = 1;
|
|
54
|
+
return;
|
|
55
|
+
}
|
|
56
|
+
inputs.set(alias, stepState.runId);
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
// Merge parsed inputs into step.inputs
|
|
60
|
+
if (inputs.size > 0 && !step.inputs) {
|
|
61
|
+
step.inputs = [];
|
|
62
|
+
}
|
|
63
|
+
for (const [alias, _runId] of inputs) {
|
|
64
|
+
// Only add if not already declared in step file
|
|
65
|
+
if (!step.inputs.some(i => i.step === alias)) {
|
|
66
|
+
step.inputs.push({ step: alias });
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
const job = { workspaceId: workspace.id, jobId: randomUUID() };
|
|
70
|
+
const runner = new StepRunner(runtime, reporter);
|
|
71
|
+
await runner.run({
|
|
72
|
+
workspace,
|
|
73
|
+
state,
|
|
74
|
+
step,
|
|
75
|
+
inputs,
|
|
76
|
+
pipelineRoot,
|
|
77
|
+
force: options.force,
|
|
78
|
+
ephemeral: options.ephemeral,
|
|
79
|
+
job
|
|
80
|
+
});
|
|
81
|
+
});
|
|
82
|
+
}
|
|
83
|
+
function parseInputSpec(spec) {
|
|
84
|
+
const eqIdx = spec.indexOf('=');
|
|
85
|
+
if (eqIdx > 0) {
|
|
86
|
+
return { alias: spec.slice(0, eqIdx), stepId: spec.slice(eqIdx + 1) };
|
|
87
|
+
}
|
|
88
|
+
return { alias: spec, stepId: spec };
|
|
89
|
+
}
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
import process from 'node:process';
|
|
2
|
+
import { cp } from 'node:fs/promises';
|
|
3
|
+
import { resolve } from 'node:path';
|
|
4
|
+
import chalk from 'chalk';
|
|
5
|
+
import { Workspace } from '../../engine/workspace.js';
|
|
6
|
+
import { StateManager } from '../../core/state.js';
|
|
7
|
+
import { getGlobalOptions } from '../utils.js';
|
|
8
|
+
export function registerExportCommand(program) {
|
|
9
|
+
program
|
|
10
|
+
.command('export')
|
|
11
|
+
.description('Extract artifacts from a step run to the host filesystem')
|
|
12
|
+
.argument('<workspace>', 'Workspace name')
|
|
13
|
+
.argument('<step>', 'Step ID')
|
|
14
|
+
.argument('<dest>', 'Destination directory')
|
|
15
|
+
.action(async (workspaceName, stepId, dest, _options, cmd) => {
|
|
16
|
+
const { workdir } = getGlobalOptions(cmd);
|
|
17
|
+
const workdirRoot = resolve(workdir);
|
|
18
|
+
const workspace = await Workspace.open(workdirRoot, workspaceName);
|
|
19
|
+
const state = new StateManager(workspace.root);
|
|
20
|
+
await state.load();
|
|
21
|
+
const stepState = state.getStep(stepId);
|
|
22
|
+
if (!stepState) {
|
|
23
|
+
console.error(chalk.red(`No run found for step: ${stepId}`));
|
|
24
|
+
process.exitCode = 1;
|
|
25
|
+
return;
|
|
26
|
+
}
|
|
27
|
+
const artifactsPath = workspace.runArtifactsPath(stepState.runId);
|
|
28
|
+
const destPath = resolve(dest);
|
|
29
|
+
await cp(artifactsPath, destPath, { recursive: true });
|
|
30
|
+
console.log(chalk.green(`Exported artifacts from ${stepId} to ${destPath}`));
|
|
31
|
+
});
|
|
32
|
+
}
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
import process from 'node:process';
|
|
2
|
+
import { readFile } from 'node:fs/promises';
|
|
3
|
+
import { join, resolve } from 'node:path';
|
|
4
|
+
import chalk from 'chalk';
|
|
5
|
+
import { Workspace } from '../../engine/workspace.js';
|
|
6
|
+
import { StateManager } from '../../core/state.js';
|
|
7
|
+
import { getGlobalOptions } from '../utils.js';
|
|
8
|
+
export function registerInspectCommand(program) {
|
|
9
|
+
program
|
|
10
|
+
.command('inspect')
|
|
11
|
+
.description('Show metadata from the last run of a step')
|
|
12
|
+
.argument('<workspace>', 'Workspace name')
|
|
13
|
+
.argument('<step>', 'Step ID')
|
|
14
|
+
.action(async (workspaceName, stepId, _options, cmd) => {
|
|
15
|
+
const { workdir, json } = getGlobalOptions(cmd);
|
|
16
|
+
const workdirRoot = resolve(workdir);
|
|
17
|
+
const workspace = await Workspace.open(workdirRoot, workspaceName);
|
|
18
|
+
const state = new StateManager(workspace.root);
|
|
19
|
+
await state.load();
|
|
20
|
+
const stepState = state.getStep(stepId);
|
|
21
|
+
if (!stepState) {
|
|
22
|
+
console.error(chalk.red(`No run found for step: ${stepId}`));
|
|
23
|
+
process.exitCode = 1;
|
|
24
|
+
return;
|
|
25
|
+
}
|
|
26
|
+
const metaPath = join(workspace.runPath(stepState.runId), 'meta.json');
|
|
27
|
+
try {
|
|
28
|
+
const content = await readFile(metaPath, 'utf8');
|
|
29
|
+
if (json) {
|
|
30
|
+
console.log(content);
|
|
31
|
+
}
|
|
32
|
+
else {
|
|
33
|
+
const meta = JSON.parse(content);
|
|
34
|
+
console.log(chalk.bold(`\nRun: ${chalk.cyan(meta.runId)}`));
|
|
35
|
+
console.log(` Step: ${meta.stepId}${meta.stepName ? ` (${meta.stepName})` : ''}`);
|
|
36
|
+
console.log(` Status: ${meta.status === 'success' ? chalk.green('success') : chalk.red('failure')}`);
|
|
37
|
+
console.log(` Image: ${meta.image}`);
|
|
38
|
+
console.log(` Command: ${meta.cmd.join(' ')}`);
|
|
39
|
+
console.log(` Duration: ${meta.durationMs}ms`);
|
|
40
|
+
console.log(` Started: ${meta.startedAt}`);
|
|
41
|
+
console.log(` Finished: ${meta.finishedAt}`);
|
|
42
|
+
console.log(` Exit code: ${meta.exitCode}`);
|
|
43
|
+
console.log(` Fingerprint: ${meta.fingerprint}`);
|
|
44
|
+
if (meta.env && Object.keys(meta.env).length > 0) {
|
|
45
|
+
console.log(` Env: ${JSON.stringify(meta.env)}`);
|
|
46
|
+
}
|
|
47
|
+
if (meta.inputs && meta.inputs.length > 0) {
|
|
48
|
+
console.log(` Inputs: ${JSON.stringify(meta.inputs)}`);
|
|
49
|
+
}
|
|
50
|
+
console.log();
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
catch {
|
|
54
|
+
console.error(chalk.red(`No metadata found for run: ${stepState.runId}`));
|
|
55
|
+
process.exitCode = 1;
|
|
56
|
+
}
|
|
57
|
+
});
|
|
58
|
+
}
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
import { resolve } from 'node:path';
|
|
2
|
+
import chalk from 'chalk';
|
|
3
|
+
import { Workspace } from '../../engine/workspace.js';
|
|
4
|
+
import { dirSize, formatSize } from '../../core/utils.js';
|
|
5
|
+
import { getGlobalOptions } from '../utils.js';
|
|
6
|
+
export function registerListCommand(program) {
|
|
7
|
+
program
|
|
8
|
+
.command('list')
|
|
9
|
+
.alias('ls')
|
|
10
|
+
.description('List workspaces')
|
|
11
|
+
.action(async (_options, cmd) => {
|
|
12
|
+
const { workdir, json } = getGlobalOptions(cmd);
|
|
13
|
+
const workdirRoot = resolve(workdir);
|
|
14
|
+
const names = await Workspace.list(workdirRoot);
|
|
15
|
+
if (json) {
|
|
16
|
+
console.log(JSON.stringify(names));
|
|
17
|
+
return;
|
|
18
|
+
}
|
|
19
|
+
if (names.length === 0) {
|
|
20
|
+
console.log(chalk.gray('No workspaces found.'));
|
|
21
|
+
return;
|
|
22
|
+
}
|
|
23
|
+
const rows = [];
|
|
24
|
+
for (const name of names) {
|
|
25
|
+
const ws = await Workspace.open(workdirRoot, name);
|
|
26
|
+
const runs = await ws.listRuns();
|
|
27
|
+
const caches = await ws.listCaches();
|
|
28
|
+
const wsSize = await dirSize(ws.root);
|
|
29
|
+
rows.push({ name, runs: runs.length, caches: caches.length, size: formatSize(wsSize) });
|
|
30
|
+
}
|
|
31
|
+
const nameWidth = Math.max('WORKSPACE'.length, ...rows.map(r => r.name.length));
|
|
32
|
+
const sizeWidth = Math.max('SIZE'.length, ...rows.map(r => r.size.length));
|
|
33
|
+
const header = `${'WORKSPACE'.padEnd(nameWidth)} RUNS CACHES ${'SIZE'.padStart(sizeWidth)}`;
|
|
34
|
+
console.log(chalk.bold(header));
|
|
35
|
+
for (const row of rows) {
|
|
36
|
+
console.log(`${row.name.padEnd(nameWidth)} ${String(row.runs).padStart(4)} ${String(row.caches).padStart(6)} ${row.size.padStart(sizeWidth)}`);
|
|
37
|
+
}
|
|
38
|
+
});
|
|
39
|
+
}
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
import process from 'node:process';
|
|
2
|
+
import { readFile } from 'node:fs/promises';
|
|
3
|
+
import { join, resolve } from 'node:path';
|
|
4
|
+
import chalk from 'chalk';
|
|
5
|
+
import { Workspace } from '../../engine/workspace.js';
|
|
6
|
+
import { StateManager } from '../../core/state.js';
|
|
7
|
+
import { getGlobalOptions } from '../utils.js';
|
|
8
|
+
export function registerLogsCommand(program) {
|
|
9
|
+
program
|
|
10
|
+
.command('logs')
|
|
11
|
+
.description('Show logs from the last run of a step')
|
|
12
|
+
.argument('<workspace>', 'Workspace name')
|
|
13
|
+
.argument('<step>', 'Step ID')
|
|
14
|
+
.option('-s, --stream <stream>', 'Show only stdout or stderr', 'both')
|
|
15
|
+
.action(async (workspaceName, stepId, options, cmd) => {
|
|
16
|
+
const { workdir } = getGlobalOptions(cmd);
|
|
17
|
+
const workdirRoot = resolve(workdir);
|
|
18
|
+
const workspace = await Workspace.open(workdirRoot, workspaceName);
|
|
19
|
+
const state = new StateManager(workspace.root);
|
|
20
|
+
await state.load();
|
|
21
|
+
const stepState = state.getStep(stepId);
|
|
22
|
+
if (!stepState) {
|
|
23
|
+
console.error(chalk.red(`No run found for step: ${stepId}`));
|
|
24
|
+
process.exitCode = 1;
|
|
25
|
+
return;
|
|
26
|
+
}
|
|
27
|
+
const runDir = workspace.runPath(stepState.runId);
|
|
28
|
+
if (options.stream === 'both' || options.stream === 'stdout') {
|
|
29
|
+
try {
|
|
30
|
+
const stdout = await readFile(join(runDir, 'stdout.log'), 'utf8');
|
|
31
|
+
if (stdout) {
|
|
32
|
+
process.stdout.write(stdout);
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
catch {
|
|
36
|
+
// No stdout log
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
if (options.stream === 'both' || options.stream === 'stderr') {
|
|
40
|
+
try {
|
|
41
|
+
const stderr = await readFile(join(runDir, 'stderr.log'), 'utf8');
|
|
42
|
+
if (stderr) {
|
|
43
|
+
if (options.stream === 'both') {
|
|
44
|
+
console.error(chalk.red('── stderr ──'));
|
|
45
|
+
}
|
|
46
|
+
process.stderr.write(stderr);
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
catch {
|
|
50
|
+
// No stderr log
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
});
|
|
54
|
+
}
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
import { resolve } from 'node:path';
|
|
2
|
+
import chalk from 'chalk';
|
|
3
|
+
import { Workspace } from '../../engine/workspace.js';
|
|
4
|
+
import { StateManager } from '../../core/state.js';
|
|
5
|
+
import { getGlobalOptions } from '../utils.js';
|
|
6
|
+
export function registerPruneCommand(program) {
|
|
7
|
+
program
|
|
8
|
+
.command('prune')
|
|
9
|
+
.description('Remove old runs not referenced by current state')
|
|
10
|
+
.argument('<workspace>', 'Workspace name')
|
|
11
|
+
.action(async (workspaceName, _options, cmd) => {
|
|
12
|
+
const { workdir } = getGlobalOptions(cmd);
|
|
13
|
+
const workdirRoot = resolve(workdir);
|
|
14
|
+
const workspace = await Workspace.open(workdirRoot, workspaceName);
|
|
15
|
+
const state = new StateManager(workspace.root);
|
|
16
|
+
await state.load();
|
|
17
|
+
const activeIds = state.activeRunIds();
|
|
18
|
+
const removed = await workspace.pruneRuns(activeIds);
|
|
19
|
+
if (removed === 0) {
|
|
20
|
+
console.log(chalk.gray('No old runs to remove.'));
|
|
21
|
+
}
|
|
22
|
+
else {
|
|
23
|
+
console.log(chalk.green(`Removed ${removed} old run${removed > 1 ? 's' : ''}.`));
|
|
24
|
+
}
|
|
25
|
+
});
|
|
26
|
+
}
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
import process from 'node:process';
|
|
2
|
+
import { rm } from 'node:fs/promises';
|
|
3
|
+
import { join, resolve } from 'node:path';
|
|
4
|
+
import chalk from 'chalk';
|
|
5
|
+
import { Workspace } from '../../engine/workspace.js';
|
|
6
|
+
import { StateManager } from '../../core/state.js';
|
|
7
|
+
import { dirSize, formatSize } from '../../core/utils.js';
|
|
8
|
+
import { getGlobalOptions } from '../utils.js';
|
|
9
|
+
export function registerRmStepCommand(program) {
|
|
10
|
+
program
|
|
11
|
+
.command('rm-step')
|
|
12
|
+
.description('Remove a step\'s run and state entry')
|
|
13
|
+
.argument('<workspace>', 'Workspace name')
|
|
14
|
+
.argument('<step>', 'Step ID')
|
|
15
|
+
.action(async (workspaceName, stepId, _options, cmd) => {
|
|
16
|
+
const { workdir } = getGlobalOptions(cmd);
|
|
17
|
+
const workdirRoot = resolve(workdir);
|
|
18
|
+
const workspace = await Workspace.open(workdirRoot, workspaceName);
|
|
19
|
+
const state = new StateManager(workspace.root);
|
|
20
|
+
await state.load();
|
|
21
|
+
const stepState = state.getStep(stepId);
|
|
22
|
+
if (!stepState) {
|
|
23
|
+
console.error(chalk.red(`No run found for step: ${stepId}`));
|
|
24
|
+
process.exitCode = 1;
|
|
25
|
+
return;
|
|
26
|
+
}
|
|
27
|
+
const { runId } = stepState;
|
|
28
|
+
// Measure size before removal
|
|
29
|
+
const runDir = workspace.runPath(runId);
|
|
30
|
+
const size = await dirSize(runDir);
|
|
31
|
+
// Remove run directory
|
|
32
|
+
await rm(runDir, { recursive: true, force: true });
|
|
33
|
+
// Remove step-runs symlink
|
|
34
|
+
const linkPath = join(workspace.root, 'step-runs', stepId);
|
|
35
|
+
await rm(linkPath, { force: true });
|
|
36
|
+
// Remove from state
|
|
37
|
+
state.removeStep(stepId);
|
|
38
|
+
await state.save();
|
|
39
|
+
console.log(chalk.green(`Removed step ${chalk.bold(stepId)} (run ${runId}, freed ${formatSize(size)})`));
|
|
40
|
+
});
|
|
41
|
+
}
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
import process from 'node:process';
|
|
2
|
+
import { resolve } from 'node:path';
|
|
3
|
+
import chalk from 'chalk';
|
|
4
|
+
import { Workspace } from '../../engine/workspace.js';
|
|
5
|
+
import { getGlobalOptions } from '../utils.js';
|
|
6
|
+
export function registerRmCommand(program) {
|
|
7
|
+
program
|
|
8
|
+
.command('rm')
|
|
9
|
+
.description('Remove one or more workspaces')
|
|
10
|
+
.argument('<workspace...>', 'Workspace names to remove')
|
|
11
|
+
.action(async (workspaces, _options, cmd) => {
|
|
12
|
+
const { workdir } = getGlobalOptions(cmd);
|
|
13
|
+
const workdirRoot = resolve(workdir);
|
|
14
|
+
const existing = await Workspace.list(workdirRoot);
|
|
15
|
+
for (const name of workspaces) {
|
|
16
|
+
if (!existing.includes(name)) {
|
|
17
|
+
console.error(chalk.red(`Workspace not found: ${name}`));
|
|
18
|
+
process.exitCode = 1;
|
|
19
|
+
return;
|
|
20
|
+
}
|
|
21
|
+
}
|
|
22
|
+
for (const name of workspaces) {
|
|
23
|
+
await Workspace.remove(workdirRoot, name);
|
|
24
|
+
console.log(chalk.green(`Removed ${name}`));
|
|
25
|
+
}
|
|
26
|
+
});
|
|
27
|
+
}
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
import process from 'node:process';
|
|
2
|
+
import { readFile, writeFile, rm } from 'node:fs/promises';
|
|
3
|
+
import { tmpdir } from 'node:os';
|
|
4
|
+
import { join, resolve } from 'node:path';
|
|
5
|
+
import { DockerCliExecutor } from '../../engine/docker-executor.js';
|
|
6
|
+
import { extractBundle } from '../../core/bundle.js';
|
|
7
|
+
import { PipelineLoader } from '../../core/pipeline-loader.js';
|
|
8
|
+
import { PipelineRunner } from '../../core/pipeline-runner.js';
|
|
9
|
+
import { ConsoleReporter } from '../../core/reporter.js';
|
|
10
|
+
import { InteractiveReporter } from '../interactive-reporter.js';
|
|
11
|
+
import { getGlobalOptions } from '../utils.js';
|
|
12
|
+
export function registerRunBundleCommand(program) {
|
|
13
|
+
program
|
|
14
|
+
.command('run-bundle')
|
|
15
|
+
.description('Execute a pipeline from a bundle archive')
|
|
16
|
+
.argument('<archive>', 'Bundle archive (.tar.gz)')
|
|
17
|
+
.option('-w, --workspace <name>', 'Workspace name (for caching)')
|
|
18
|
+
.option('-f, --force [steps]', 'Skip cache for all steps, or a comma-separated list')
|
|
19
|
+
.option('--dry-run', 'Validate pipeline and show what would run without executing')
|
|
20
|
+
.option('--verbose', 'Stream container logs in real-time (interactive mode)')
|
|
21
|
+
.option('-t, --target <steps>', 'Execute only these steps and their dependencies (comma-separated)')
|
|
22
|
+
.option('-c, --concurrency <number>', 'Max parallel step executions (default: CPU count)', Number)
|
|
23
|
+
.action(async (archivePath, options, cmd) => {
|
|
24
|
+
const { workdir, json } = getGlobalOptions(cmd);
|
|
25
|
+
const workdirRoot = resolve(workdir);
|
|
26
|
+
const archive = await readFile(resolve(archivePath));
|
|
27
|
+
const extractDir = join(tmpdir(), `pipex-run-bundle-${Date.now()}-${Math.random().toString(36).slice(2)}`);
|
|
28
|
+
try {
|
|
29
|
+
const pipeline = await extractBundle(archive, extractDir);
|
|
30
|
+
// Write a pipeline.json so PipelineRunner can load it normally.
|
|
31
|
+
// The extract directory becomes the pipelineRoot — all relative
|
|
32
|
+
// host paths in mounts/sources resolve against it.
|
|
33
|
+
const pipelineFile = join(extractDir, 'pipeline.json');
|
|
34
|
+
await writeFile(pipelineFile, JSON.stringify(pipeline));
|
|
35
|
+
const loader = new PipelineLoader();
|
|
36
|
+
const runtime = new DockerCliExecutor();
|
|
37
|
+
const reporter = json ? new ConsoleReporter() : new InteractiveReporter({ verbose: options.verbose });
|
|
38
|
+
const runner = new PipelineRunner(loader, runtime, reporter, workdirRoot);
|
|
39
|
+
const force = options.force === true
|
|
40
|
+
? true
|
|
41
|
+
: (typeof options.force === 'string' ? options.force.split(',') : undefined);
|
|
42
|
+
const target = options.target ? options.target.split(',') : undefined;
|
|
43
|
+
await runner.run(pipelineFile, { workspace: options.workspace, force, dryRun: options.dryRun, target, concurrency: options.concurrency });
|
|
44
|
+
if (json) {
|
|
45
|
+
console.log('Pipeline completed');
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
catch (error) {
|
|
49
|
+
if (json) {
|
|
50
|
+
console.error('Pipeline failed:', error instanceof Error ? error.message : error);
|
|
51
|
+
}
|
|
52
|
+
process.exitCode = 1;
|
|
53
|
+
throw error;
|
|
54
|
+
}
|
|
55
|
+
finally {
|
|
56
|
+
await rm(extractDir, { recursive: true, force: true });
|
|
57
|
+
}
|
|
58
|
+
});
|
|
59
|
+
}
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
import { resolve } from 'node:path';
|
|
2
|
+
import { DockerCliExecutor } from '../../engine/docker-executor.js';
|
|
3
|
+
import { PipelineLoader } from '../../core/pipeline-loader.js';
|
|
4
|
+
import { PipelineRunner } from '../../core/pipeline-runner.js';
|
|
5
|
+
import { ConsoleReporter } from '../../core/reporter.js';
|
|
6
|
+
import { InteractiveReporter } from '../interactive-reporter.js';
|
|
7
|
+
import { getGlobalOptions } from '../utils.js';
|
|
8
|
+
export function registerRunCommand(program) {
|
|
9
|
+
program
|
|
10
|
+
.command('run')
|
|
11
|
+
.description('Execute a pipeline')
|
|
12
|
+
.argument('<pipeline>', 'Pipeline file to execute (JSON or YAML)')
|
|
13
|
+
.option('-w, --workspace <name>', 'Workspace name (for caching)')
|
|
14
|
+
.option('-f, --force [steps]', 'Skip cache for all steps, or a comma-separated list (e.g. --force step1,step2)')
|
|
15
|
+
.option('--dry-run', 'Validate pipeline and show what would run without executing')
|
|
16
|
+
.option('--verbose', 'Stream container logs in real-time (interactive mode)')
|
|
17
|
+
.option('-t, --target <steps>', 'Execute only these steps and their dependencies (comma-separated)')
|
|
18
|
+
.option('-c, --concurrency <number>', 'Max parallel step executions (default: CPU count)', Number)
|
|
19
|
+
.option('--env-file <path>', 'Load environment variables from a dotenv file for all steps')
|
|
20
|
+
.action(async (pipelineFile, options, cmd) => {
|
|
21
|
+
const { workdir, json } = getGlobalOptions(cmd);
|
|
22
|
+
const workdirRoot = resolve(workdir);
|
|
23
|
+
const loader = new PipelineLoader();
|
|
24
|
+
const runtime = new DockerCliExecutor();
|
|
25
|
+
const reporter = json ? new ConsoleReporter() : new InteractiveReporter({ verbose: options.verbose });
|
|
26
|
+
const runner = new PipelineRunner(loader, runtime, reporter, workdirRoot);
|
|
27
|
+
try {
|
|
28
|
+
const force = options.force === true
|
|
29
|
+
? true
|
|
30
|
+
: (typeof options.force === 'string' ? options.force.split(',') : undefined);
|
|
31
|
+
const target = options.target ? options.target.split(',') : undefined;
|
|
32
|
+
await runner.run(pipelineFile, { workspace: options.workspace, force, dryRun: options.dryRun, target, concurrency: options.concurrency, envFile: options.envFile });
|
|
33
|
+
if (json) {
|
|
34
|
+
console.log('Pipeline completed');
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
catch (error) {
|
|
38
|
+
if (json) {
|
|
39
|
+
console.error('Pipeline failed:', error instanceof Error ? error.message : error);
|
|
40
|
+
}
|
|
41
|
+
throw error;
|
|
42
|
+
}
|
|
43
|
+
});
|
|
44
|
+
}
|
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
import process from 'node:process';
|
|
2
|
+
import { readFile } from 'node:fs/promises';
|
|
3
|
+
import { join, resolve } from 'node:path';
|
|
4
|
+
import chalk from 'chalk';
|
|
5
|
+
import { Workspace } from '../../engine/workspace.js';
|
|
6
|
+
import { StateManager } from '../../core/state.js';
|
|
7
|
+
import { dirSize, formatDuration, formatSize } from '../../core/utils.js';
|
|
8
|
+
import { getGlobalOptions } from '../utils.js';
|
|
9
|
+
function isProcessAlive(pid) {
|
|
10
|
+
try {
|
|
11
|
+
process.kill(pid, 0);
|
|
12
|
+
return true;
|
|
13
|
+
}
|
|
14
|
+
catch {
|
|
15
|
+
return false;
|
|
16
|
+
}
|
|
17
|
+
}
|
|
18
|
+
export function registerShowCommand(program) {
|
|
19
|
+
program
|
|
20
|
+
.command('show')
|
|
21
|
+
.description('Show steps and runs in a workspace')
|
|
22
|
+
.argument('<workspace>', 'Workspace name')
|
|
23
|
+
.action(async (workspaceName, _options, cmd) => {
|
|
24
|
+
const { workdir, json } = getGlobalOptions(cmd);
|
|
25
|
+
const workdirRoot = resolve(workdir);
|
|
26
|
+
const workspace = await Workspace.open(workdirRoot, workspaceName);
|
|
27
|
+
const state = new StateManager(workspace.root);
|
|
28
|
+
await state.load();
|
|
29
|
+
const steps = state.listSteps();
|
|
30
|
+
const runningSteps = await workspace.listRunningSteps();
|
|
31
|
+
const rows = [];
|
|
32
|
+
let totalSize = 0;
|
|
33
|
+
for (const { stepId, runId } of steps) {
|
|
34
|
+
const metaPath = join(workspace.runPath(runId), 'meta.json');
|
|
35
|
+
try {
|
|
36
|
+
const content = await readFile(metaPath, 'utf8');
|
|
37
|
+
const meta = JSON.parse(content);
|
|
38
|
+
const artifactBytes = await dirSize(workspace.runArtifactsPath(runId));
|
|
39
|
+
totalSize += artifactBytes;
|
|
40
|
+
rows.push({
|
|
41
|
+
stepId,
|
|
42
|
+
stepName: meta.stepName,
|
|
43
|
+
status: meta.status,
|
|
44
|
+
duration: `${meta.durationMs}ms`,
|
|
45
|
+
size: formatSize(artifactBytes),
|
|
46
|
+
date: meta.finishedAt.replace('T', ' ').replace(/\.\d+Z$/, ''),
|
|
47
|
+
runId
|
|
48
|
+
});
|
|
49
|
+
}
|
|
50
|
+
catch {
|
|
51
|
+
rows.push({ stepId, status: 'unknown', duration: '-', size: '-', date: '-', runId });
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
// Add running steps (not already in committed state)
|
|
55
|
+
const committedStepIds = new Set(rows.map(r => r.stepId));
|
|
56
|
+
for (const running of runningSteps) {
|
|
57
|
+
if (committedStepIds.has(running.stepId)) {
|
|
58
|
+
continue;
|
|
59
|
+
}
|
|
60
|
+
if (!isProcessAlive(running.pid)) {
|
|
61
|
+
continue;
|
|
62
|
+
}
|
|
63
|
+
const elapsedMs = Date.now() - new Date(running.startedAt).getTime();
|
|
64
|
+
rows.push({
|
|
65
|
+
stepId: running.stepId,
|
|
66
|
+
stepName: running.stepName,
|
|
67
|
+
status: 'running',
|
|
68
|
+
duration: formatDuration(elapsedMs),
|
|
69
|
+
size: '-',
|
|
70
|
+
date: '-',
|
|
71
|
+
runId: '-'
|
|
72
|
+
});
|
|
73
|
+
}
|
|
74
|
+
if (rows.length === 0) {
|
|
75
|
+
console.log(chalk.gray('No runs found in this workspace.'));
|
|
76
|
+
return;
|
|
77
|
+
}
|
|
78
|
+
if (json) {
|
|
79
|
+
console.log(JSON.stringify(rows, null, 2));
|
|
80
|
+
return;
|
|
81
|
+
}
|
|
82
|
+
const stepWidth = Math.max('STEP'.length, ...rows.map(r => (r.stepName ? `${r.stepId} (${r.stepName})` : r.stepId).length));
|
|
83
|
+
const statusWidth = Math.max('STATUS'.length, ...rows.map(r => r.status.length));
|
|
84
|
+
const durationWidth = Math.max('DURATION'.length, ...rows.map(r => r.duration.length));
|
|
85
|
+
const sizeWidth = Math.max('SIZE'.length, ...rows.map(r => r.size.length));
|
|
86
|
+
const dateWidth = Math.max('FINISHED'.length, ...rows.map(r => r.date.length));
|
|
87
|
+
console.log(chalk.bold(`${'STEP'.padEnd(stepWidth)} ${'STATUS'.padEnd(statusWidth)} ${'DURATION'.padStart(durationWidth)} ${'SIZE'.padStart(sizeWidth)} ${'FINISHED'.padEnd(dateWidth)}`));
|
|
88
|
+
for (const row of rows) {
|
|
89
|
+
const stepLabel = row.stepName ? `${row.stepId} (${row.stepName})` : row.stepId;
|
|
90
|
+
const statusColor = row.status === 'success'
|
|
91
|
+
? chalk.green
|
|
92
|
+
: (row.status === 'running' ? chalk.yellow : chalk.red);
|
|
93
|
+
const statusText = statusColor(row.status);
|
|
94
|
+
const statusPad = statusWidth + (statusText.length - row.status.length);
|
|
95
|
+
const cols = [
|
|
96
|
+
stepLabel.padEnd(stepWidth),
|
|
97
|
+
statusText.padEnd(statusPad),
|
|
98
|
+
row.duration.padStart(durationWidth),
|
|
99
|
+
row.size.padStart(sizeWidth),
|
|
100
|
+
row.date.padEnd(dateWidth)
|
|
101
|
+
];
|
|
102
|
+
console.log(cols.join(' '));
|
|
103
|
+
}
|
|
104
|
+
if (rows.length > 1) {
|
|
105
|
+
console.log(chalk.gray(`\n Total: ${formatSize(totalSize)}`));
|
|
106
|
+
}
|
|
107
|
+
});
|
|
108
|
+
}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import jexlModule from 'jexl';
|
|
2
|
+
const jexl = new jexlModule.Jexl();
|
|
3
|
+
export async function evaluateCondition(expression, context) {
|
|
4
|
+
try {
|
|
5
|
+
const result = await jexl.eval(expression, context);
|
|
6
|
+
return Boolean(result);
|
|
7
|
+
}
|
|
8
|
+
catch {
|
|
9
|
+
return false;
|
|
10
|
+
}
|
|
11
|
+
}
|