@livingdata/pipex 0.0.7 → 0.0.9

package/README.md

@@ -37,13 +37,36 @@ pipex run pipeline.yaml --json
 pipex run pipeline.yaml --workdir /tmp/builds
 ```
 
+### Inspecting runs
+
+Each step execution produces a **run** containing artifacts, logs (stdout/stderr), and metadata:
+
+```bash
+# Show all steps and their last run (status, duration, size, date)
+pipex show my-pipeline
+
+# Show logs from the last run of a step
+pipex logs my-pipeline download
+pipex logs my-pipeline download --stream stderr
+
+# Show execution metadata (image, cmd, duration, exit code, fingerprint…)
+pipex inspect my-pipeline download
+pipex inspect my-pipeline download --json
+
+# Export artifacts from a step to the host filesystem
+pipex export my-pipeline download ./output-dir
+```
+
 ### Managing workspaces
 
 ```bash
-# List workspaces (with artifact/cache counts)
+# List workspaces (with run/cache counts and disk size)
 pipex list
 pipex ls --json
 
+# Remove old runs (keeps only current ones)
+pipex prune my-pipeline
+
 # Remove specific workspaces
 pipex rm my-build other-build
 
@@ -56,7 +79,12 @@ pipex clean
 | Command | Description |
 |---------|-------------|
 | `run <pipeline>` | Execute a pipeline |
-| `list` (alias `ls`) | List workspaces |
+| `show <workspace>` | Show steps and runs in a workspace (with artifact sizes) |
+| `logs <workspace> <step>` | Show stdout/stderr from last run |
+| `inspect <workspace> <step>` | Show run metadata (meta.json) |
+| `export <workspace> <step> <dest>` | Extract artifacts from a step run to the host filesystem |
+| `prune <workspace>` | Remove old runs not referenced by current state |
+| `list` (alias `ls`) | List workspaces (with disk sizes) |
 | `rm <workspace...>` | Remove one or more workspaces |
 | `clean` | Remove all workspaces |
 
@@ -73,6 +101,8 @@ pipex clean
 |--------|-------|-------------|
 | `--workspace <name>` | `-w` | Workspace name for caching |
 | `--force [steps]` | `-f` | Skip cache for all steps, or a comma-separated list |
+| `--dry-run` | | Validate pipeline, compute fingerprints, show what would run without executing |
+| `--verbose` | | Stream container logs in real-time (interactive mode) |
 
 ## Pipeline Format
 
@@ -372,7 +402,7 @@ pipex rm old-workspace-id
 pipex clean
 ```
 
-### Cached step with missing artifact
+### Cached step with missing run
 
 Force re-execution:
 

package/dist/cli/commands/clean.js

@@ -0,0 +1,22 @@
+import { resolve } from 'node:path';
+import chalk from 'chalk';
+import { Workspace } from '../../engine/workspace.js';
+import { getGlobalOptions } from '../utils.js';
+export function registerCleanCommand(program) {
+    program
+        .command('clean')
+        .description('Remove all workspaces')
+        .action(async (_options, cmd) => {
+        const { workdir } = getGlobalOptions(cmd);
+        const workdirRoot = resolve(workdir);
+        const names = await Workspace.list(workdirRoot);
+        if (names.length === 0) {
+            console.log(chalk.gray('No workspaces to clean.'));
+            return;
+        }
+        for (const name of names) {
+            await Workspace.remove(workdirRoot, name);
+        }
+        console.log(chalk.green(`Removed ${names.length} workspace${names.length > 1 ? 's' : ''}.`));
+    });
+}

package/dist/cli/commands/export.js

@@ -0,0 +1,32 @@
+import process from 'node:process';
+import { cp } from 'node:fs/promises';
+import { resolve } from 'node:path';
+import chalk from 'chalk';
+import { Workspace } from '../../engine/workspace.js';
+import { StateManager } from '../state.js';
+import { getGlobalOptions } from '../utils.js';
+export function registerExportCommand(program) {
+    program
+        .command('export')
+        .description('Extract artifacts from a step run to the host filesystem')
+        .argument('<workspace>', 'Workspace name')
+        .argument('<step>', 'Step ID')
+        .argument('<dest>', 'Destination directory')
+        .action(async (workspaceName, stepId, dest, cmd) => {
+        const { workdir } = getGlobalOptions(cmd);
+        const workdirRoot = resolve(workdir);
+        const workspace = await Workspace.open(workdirRoot, workspaceName);
+        const state = new StateManager(workspace.root);
+        await state.load();
+        const stepState = state.getStep(stepId);
+        if (!stepState) {
+            console.error(chalk.red(`No run found for step: ${stepId}`));
+            process.exitCode = 1;
+            return;
+        }
+        const artifactsPath = workspace.runArtifactsPath(stepState.runId);
+        const destPath = resolve(dest);
+        await cp(artifactsPath, destPath, { recursive: true });
+        console.log(chalk.green(`Exported artifacts from ${stepId} to ${destPath}`));
+    });
+}

package/dist/cli/commands/inspect.js

@@ -0,0 +1,58 @@
+import process from 'node:process';
+import { readFile } from 'node:fs/promises';
+import { join, resolve } from 'node:path';
+import chalk from 'chalk';
+import { Workspace } from '../../engine/workspace.js';
+import { StateManager } from '../state.js';
+import { getGlobalOptions } from '../utils.js';
+export function registerInspectCommand(program) {
+    program
+        .command('inspect')
+        .description('Show metadata from the last run of a step')
+        .argument('<workspace>', 'Workspace name')
+        .argument('<step>', 'Step ID')
+        .action(async (workspaceName, stepId, _options, cmd) => {
+        const { workdir, json } = getGlobalOptions(cmd);
+        const workdirRoot = resolve(workdir);
+        const workspace = await Workspace.open(workdirRoot, workspaceName);
+        const state = new StateManager(workspace.root);
+        await state.load();
+        const stepState = state.getStep(stepId);
+        if (!stepState) {
+            console.error(chalk.red(`No run found for step: ${stepId}`));
+            process.exitCode = 1;
+            return;
+        }
+        const metaPath = join(workspace.runPath(stepState.runId), 'meta.json');
+        try {
+            const content = await readFile(metaPath, 'utf8');
+            if (json) {
+                console.log(content);
+            }
+            else {
+                const meta = JSON.parse(content);
+                console.log(chalk.bold(`\nRun: ${chalk.cyan(meta.runId)}`));
+                console.log(`  Step:       ${meta.stepId}${meta.stepName ? ` (${meta.stepName})` : ''}`);
+                console.log(`  Status:     ${meta.status === 'success' ? chalk.green('success') : chalk.red('failure')}`);
+                console.log(`  Image:      ${meta.image}`);
+                console.log(`  Command:    ${meta.cmd.join(' ')}`);
+                console.log(`  Duration:   ${meta.durationMs}ms`);
+                console.log(`  Started:    ${meta.startedAt}`);
+                console.log(`  Finished:   ${meta.finishedAt}`);
+                console.log(`  Exit code:  ${meta.exitCode}`);
+                console.log(`  Fingerprint: ${meta.fingerprint}`);
+                if (meta.env && Object.keys(meta.env).length > 0) {
+                    console.log(`  Env:        ${JSON.stringify(meta.env)}`);
+                }
+                if (meta.inputs && meta.inputs.length > 0) {
+                    console.log(`  Inputs:     ${JSON.stringify(meta.inputs)}`);
+                }
+                console.log();
+            }
+        }
+        catch {
+            console.error(chalk.red(`No metadata found for run: ${stepState.runId}`));
+            process.exitCode = 1;
+        }
+    });
+}

package/dist/cli/commands/list.js

@@ -0,0 +1,38 @@
+import { resolve } from 'node:path';
+import chalk from 'chalk';
+import { Workspace } from '../../engine/workspace.js';
+import { dirSize, formatSize, getGlobalOptions } from '../utils.js';
+export function registerListCommand(program) {
+    program
+        .command('list')
+        .alias('ls')
+        .description('List workspaces')
+        .action(async (_options, cmd) => {
+        const { workdir, json } = getGlobalOptions(cmd);
+        const workdirRoot = resolve(workdir);
+        const names = await Workspace.list(workdirRoot);
+        if (json) {
+            console.log(JSON.stringify(names));
+            return;
+        }
+        if (names.length === 0) {
+            console.log(chalk.gray('No workspaces found.'));
+            return;
+        }
+        const rows = [];
+        for (const name of names) {
+            const ws = await Workspace.open(workdirRoot, name);
+            const runs = await ws.listRuns();
+            const caches = await ws.listCaches();
+            const wsSize = await dirSize(ws.root);
+            rows.push({ name, runs: runs.length, caches: caches.length, size: formatSize(wsSize) });
+        }
+        const nameWidth = Math.max('WORKSPACE'.length, ...rows.map(r => r.name.length));
+        const sizeWidth = Math.max('SIZE'.length, ...rows.map(r => r.size.length));
+        const header = `${'WORKSPACE'.padEnd(nameWidth)}  RUNS  CACHES  ${'SIZE'.padStart(sizeWidth)}`;
+        console.log(chalk.bold(header));
+        for (const row of rows) {
+            console.log(`${row.name.padEnd(nameWidth)}  ${String(row.runs).padStart(4)}  ${String(row.caches).padStart(6)}  ${row.size.padStart(sizeWidth)}`);
+        }
+    });
+}

package/dist/cli/commands/logs.js

@@ -0,0 +1,54 @@
+import process from 'node:process';
+import { readFile } from 'node:fs/promises';
+import { join, resolve } from 'node:path';
+import chalk from 'chalk';
+import { Workspace } from '../../engine/workspace.js';
+import { StateManager } from '../state.js';
+import { getGlobalOptions } from '../utils.js';
+export function registerLogsCommand(program) {
+    program
+        .command('logs')
+        .description('Show logs from the last run of a step')
+        .argument('<workspace>', 'Workspace name')
+        .argument('<step>', 'Step ID')
+        .option('-s, --stream <stream>', 'Show only stdout or stderr', 'both')
+        .action(async (workspaceName, stepId, options, cmd) => {
+        const { workdir } = getGlobalOptions(cmd);
+        const workdirRoot = resolve(workdir);
+        const workspace = await Workspace.open(workdirRoot, workspaceName);
+        const state = new StateManager(workspace.root);
+        await state.load();
+        const stepState = state.getStep(stepId);
+        if (!stepState) {
+            console.error(chalk.red(`No run found for step: ${stepId}`));
+            process.exitCode = 1;
+            return;
+        }
+        const runDir = workspace.runPath(stepState.runId);
+        if (options.stream === 'both' || options.stream === 'stdout') {
+            try {
+                const stdout = await readFile(join(runDir, 'stdout.log'), 'utf8');
+                if (stdout) {
+                    process.stdout.write(stdout);
+                }
+            }
+            catch {
+                // No stdout log
+            }
+        }
+        if (options.stream === 'both' || options.stream === 'stderr') {
+            try {
+                const stderr = await readFile(join(runDir, 'stderr.log'), 'utf8');
+                if (stderr) {
+                    if (options.stream === 'both') {
+                        console.error(chalk.red('── stderr ──'));
+                    }
+                    process.stderr.write(stderr);
+                }
+            }
+            catch {
+                // No stderr log
+            }
+        }
+    });
+}

package/dist/cli/commands/prune.js

@@ -0,0 +1,26 @@
+import { resolve } from 'node:path';
+import chalk from 'chalk';
+import { Workspace } from '../../engine/workspace.js';
+import { StateManager } from '../state.js';
+import { getGlobalOptions } from '../utils.js';
+export function registerPruneCommand(program) {
+    program
+        .command('prune')
+        .description('Remove old runs not referenced by current state')
+        .argument('<workspace>', 'Workspace name')
+        .action(async (workspaceName, _options, cmd) => {
+        const { workdir } = getGlobalOptions(cmd);
+        const workdirRoot = resolve(workdir);
+        const workspace = await Workspace.open(workdirRoot, workspaceName);
+        const state = new StateManager(workspace.root);
+        await state.load();
+        const activeIds = state.activeRunIds();
+        const removed = await workspace.pruneRuns(activeIds);
+        if (removed === 0) {
+            console.log(chalk.gray('No old runs to remove.'));
+        }
+        else {
+            console.log(chalk.green(`Removed ${removed} old run${removed > 1 ? 's' : ''}.`));
+        }
+    });
+}

package/dist/cli/commands/rm.js

@@ -0,0 +1,27 @@
+import process from 'node:process';
+import { resolve } from 'node:path';
+import chalk from 'chalk';
+import { Workspace } from '../../engine/workspace.js';
+import { getGlobalOptions } from '../utils.js';
+export function registerRmCommand(program) {
+    program
+        .command('rm')
+        .description('Remove one or more workspaces')
+        .argument('<workspace...>', 'Workspace names to remove')
+        .action(async (workspaces, _options, cmd) => {
+        const { workdir } = getGlobalOptions(cmd);
+        const workdirRoot = resolve(workdir);
+        const existing = await Workspace.list(workdirRoot);
+        for (const name of workspaces) {
+            if (!existing.includes(name)) {
+                console.error(chalk.red(`Workspace not found: ${name}`));
+                process.exitCode = 1;
+                return;
+            }
+        }
+        for (const name of workspaces) {
+            await Workspace.remove(workdirRoot, name);
+            console.log(chalk.green(`Removed ${name}`));
+        }
+    });
+}

package/dist/cli/commands/run.js

@@ -0,0 +1,39 @@
+import { resolve } from 'node:path';
+import { DockerCliExecutor } from '../../engine/docker-executor.js';
+import { PipelineLoader } from '../pipeline-loader.js';
+import { PipelineRunner } from '../pipeline-runner.js';
+import { ConsoleReporter, InteractiveReporter } from '../reporter.js';
+import { getGlobalOptions } from '../utils.js';
+export function registerRunCommand(program) {
+    program
+        .command('run')
+        .description('Execute a pipeline')
+        .argument('<pipeline>', 'Pipeline file to execute (JSON or YAML)')
+        .option('-w, --workspace <name>', 'Workspace name (for caching)')
+        .option('-f, --force [steps]', 'Skip cache for all steps, or a comma-separated list (e.g. --force step1,step2)')
+        .option('--dry-run', 'Validate pipeline and show what would run without executing')
+        .option('--verbose', 'Stream container logs in real-time (interactive mode)')
+        .action(async (pipelineFile, options, cmd) => {
+        const { workdir, json } = getGlobalOptions(cmd);
+        const workdirRoot = resolve(workdir);
+        const loader = new PipelineLoader();
+        const runtime = new DockerCliExecutor();
+        const reporter = json ? new ConsoleReporter() : new InteractiveReporter({ verbose: options.verbose });
+        const runner = new PipelineRunner(loader, runtime, reporter, workdirRoot);
+        try {
+            const force = options.force === true
+                ? true
+                : (typeof options.force === 'string' ? options.force.split(',') : undefined);
+            await runner.run(pipelineFile, { workspace: options.workspace, force, dryRun: options.dryRun });
+            if (json) {
+                console.log('Pipeline completed');
+            }
+        }
+        catch (error) {
+            if (json) {
+                console.error('Pipeline failed:', error instanceof Error ? error.message : error);
+            }
+            throw error;
+        }
+    });
+}

package/dist/cli/commands/show.js

@@ -0,0 +1,73 @@
+import { readFile } from 'node:fs/promises';
+import { join, resolve } from 'node:path';
+import chalk from 'chalk';
+import { Workspace } from '../../engine/workspace.js';
+import { StateManager } from '../state.js';
+import { dirSize, formatSize, getGlobalOptions } from '../utils.js';
+export function registerShowCommand(program) {
+    program
+        .command('show')
+        .description('Show steps and runs in a workspace')
+        .argument('<workspace>', 'Workspace name')
+        .action(async (workspaceName, _options, cmd) => {
+        const { workdir, json } = getGlobalOptions(cmd);
+        const workdirRoot = resolve(workdir);
+        const workspace = await Workspace.open(workdirRoot, workspaceName);
+        const state = new StateManager(workspace.root);
+        await state.load();
+        const steps = state.listSteps();
+        if (steps.length === 0) {
+            console.log(chalk.gray('No runs found in this workspace.'));
+            return;
+        }
+        const rows = [];
+        let totalSize = 0;
+        for (const { stepId, runId } of steps) {
+            const metaPath = join(workspace.runPath(runId), 'meta.json');
+            try {
+                const content = await readFile(metaPath, 'utf8');
+                const meta = JSON.parse(content);
+                const artifactBytes = await dirSize(workspace.runArtifactsPath(runId));
+                totalSize += artifactBytes;
+                rows.push({
+                    stepId,
+                    stepName: meta.stepName,
+                    status: meta.status,
+                    duration: `${meta.durationMs}ms`,
+                    size: formatSize(artifactBytes),
+                    date: meta.finishedAt.replace('T', ' ').replace(/\.\d+Z$/, ''),
+                    runId
+                });
+            }
+            catch {
+                rows.push({ stepId, status: 'unknown', duration: '-', size: '-', date: '-', runId });
+            }
+        }
+        if (json) {
+            console.log(JSON.stringify(rows, null, 2));
+            return;
+        }
+        const stepWidth = Math.max('STEP'.length, ...rows.map(r => (r.stepName ? `${r.stepId} (${r.stepName})` : r.stepId).length));
+        const statusWidth = Math.max('STATUS'.length, ...rows.map(r => r.status.length));
+        const durationWidth = Math.max('DURATION'.length, ...rows.map(r => r.duration.length));
+        const sizeWidth = Math.max('SIZE'.length, ...rows.map(r => r.size.length));
+        const dateWidth = Math.max('FINISHED'.length, ...rows.map(r => r.date.length));
+        console.log(chalk.bold(`${'STEP'.padEnd(stepWidth)}  ${'STATUS'.padEnd(statusWidth)}  ${'DURATION'.padStart(durationWidth)}  ${'SIZE'.padStart(sizeWidth)}  ${'FINISHED'.padEnd(dateWidth)}`));
+        for (const row of rows) {
+            const stepLabel = row.stepName ? `${row.stepId} (${row.stepName})` : row.stepId;
+            const statusText = row.status === 'success' ? chalk.green(row.status) : chalk.red(row.status);
+            const statusPad = statusWidth + (statusText.length - row.status.length);
+            const cols = [
+                stepLabel.padEnd(stepWidth),
+                statusText.padEnd(statusPad),
+                row.duration.padStart(durationWidth),
+                row.size.padStart(sizeWidth),
+                row.date.padEnd(dateWidth)
+            ];
+            console.log(cols.join('  '));
+        }
+        if (rows.length > 1) {
+            console.log(chalk.gray(`\n  Total: ${formatSize(totalSize)}`));
+        }
+    });
+}

package/dist/cli/index.js

@@ -1,17 +1,16 @@
 #!/usr/bin/env node
 import 'dotenv/config';
 import process from 'node:process';
-import { resolve } from 'node:path';
-import chalk from 'chalk';
 import { Command } from 'commander';
-import { Workspace } from '../engine/workspace.js';
-import { DockerCliExecutor } from '../engine/docker-executor.js';
-import { PipelineLoader } from './pipeline-loader.js';
-import { PipelineRunner } from './pipeline-runner.js';
-import { ConsoleReporter, InteractiveReporter } from './reporter.js';
-function getGlobalOptions(cmd) {
-    return cmd.optsWithGlobals();
-}
+import { registerRunCommand } from './commands/run.js';
+import { registerLogsCommand } from './commands/logs.js';
+import { registerInspectCommand } from './commands/inspect.js';
+import { registerExportCommand } from './commands/export.js';
+import { registerShowCommand } from './commands/show.js';
+import { registerPruneCommand } from './commands/prune.js';
+import { registerListCommand } from './commands/list.js';
+import { registerRmCommand } from './commands/rm.js';
+import { registerCleanCommand } from './commands/clean.js';
 async function main() {
     const program = new Command();
     program
@@ -20,101 +19,15 @@ async function main() {
         .version('0.1.0')
         .option('--workdir <path>', 'Workspaces root directory', process.env.PIPEX_WORKDIR ?? './workdir')
         .option('--json', 'Output structured JSON logs');
-    program
-        .command('run')
-        .description('Execute a pipeline')
-        .argument('<pipeline>', 'Pipeline file to execute (JSON or YAML)')
-        .option('-w, --workspace <name>', 'Workspace name (for caching)')
-        .option('-f, --force [steps]', 'Skip cache for all steps, or a comma-separated list (e.g. --force step1,step2)')
-        .action(async (pipelineFile, options, cmd) => {
-        const { workdir, json } = getGlobalOptions(cmd);
-        const workdirRoot = resolve(workdir);
-        const loader = new PipelineLoader();
-        const runtime = new DockerCliExecutor();
-        const reporter = json ? new ConsoleReporter() : new InteractiveReporter();
-        const runner = new PipelineRunner(loader, runtime, reporter, workdirRoot);
-        try {
-            const force = options.force === true
-                ? true
-                : (typeof options.force === 'string' ? options.force.split(',') : undefined);
-            await runner.run(pipelineFile, { workspace: options.workspace, force });
-            if (json) {
-                console.log('Pipeline completed');
-            }
-        }
-        catch (error) {
-            if (json) {
-                console.error('Pipeline failed:', error instanceof Error ? error.message : error);
-            }
-            throw error;
-        }
-    });
-    program
-        .command('list')
-        .alias('ls')
-        .description('List workspaces')
-        .action(async (_options, cmd) => {
-        const { workdir, json } = getGlobalOptions(cmd);
-        const workdirRoot = resolve(workdir);
-        const names = await Workspace.list(workdirRoot);
-        if (json) {
-            console.log(JSON.stringify(names));
-            return;
-        }
-        if (names.length === 0) {
-            console.log(chalk.gray('No workspaces found.'));
-            return;
-        }
-        const rows = [];
-        for (const name of names) {
-            const ws = await Workspace.open(workdirRoot, name);
-            const artifacts = await ws.listArtifacts();
-            const caches = await ws.listCaches();
-            rows.push({ name, artifacts: artifacts.length, caches: caches.length });
-        }
-        const nameWidth = Math.max('WORKSPACE'.length, ...rows.map(r => r.name.length));
-        const header = `${'WORKSPACE'.padEnd(nameWidth)}  ARTIFACTS  CACHES`;
-        console.log(chalk.bold(header));
-        for (const row of rows) {
-            console.log(`${row.name.padEnd(nameWidth)}  ${String(row.artifacts).padStart(9)}  ${String(row.caches).padStart(6)}`);
-        }
-    });
-    program
-        .command('rm')
-        .description('Remove one or more workspaces')
-        .argument('<workspace...>', 'Workspace names to remove')
-        .action(async (workspaces, _options, cmd) => {
-        const { workdir } = getGlobalOptions(cmd);
-        const workdirRoot = resolve(workdir);
-        const existing = await Workspace.list(workdirRoot);
-        for (const name of workspaces) {
-            if (!existing.includes(name)) {
-                console.error(chalk.red(`Workspace not found: ${name}`));
-                process.exitCode = 1;
-                return;
-            }
-        }
-        for (const name of workspaces) {
-            await Workspace.remove(workdirRoot, name);
-            console.log(chalk.green(`Removed ${name}`));
-        }
-    });
-    program
-        .command('clean')
-        .description('Remove all workspaces')
-        .action(async (_options, cmd) => {
-        const { workdir } = getGlobalOptions(cmd);
-        const workdirRoot = resolve(workdir);
-        const names = await Workspace.list(workdirRoot);
-        if (names.length === 0) {
-            console.log(chalk.gray('No workspaces to clean.'));
-            return;
-        }
-        for (const name of names) {
-            await Workspace.remove(workdirRoot, name);
-        }
-        console.log(chalk.green(`Removed ${names.length} workspace${names.length > 1 ? 's' : ''}.`));
-    });
+    registerRunCommand(program);
+    registerLogsCommand(program);
+    registerInspectCommand(program);
+    registerExportCommand(program);
+    registerShowCommand(program);
+    registerPruneCommand(program);
+    registerListCommand(program);
+    registerRmCommand(program);
+    registerCleanCommand(program);
     await program.parseAsync();
 }
 try {