@livingdata/pipex 0.0.8 → 0.0.10

package/README.md

@@ -37,13 +37,68 @@ pipex run pipeline.yaml --json
 pipex run pipeline.yaml --workdir /tmp/builds
 ```
 
+### Interactive step execution
+
+Execute individual steps without a full pipeline file — useful for iterative, exploratory, or agent-driven workflows:
+
+```bash
+# Create a step file
+cat > step.yaml <<'EOF'
+uses: shell
+with:
+  run: "echo hello > /output/greeting.txt"
+EOF
+
+# Execute a single step in a workspace
+pipex exec my-workspace -f step.yaml --step greet
+
+# Read artifact content
+pipex cat my-workspace greet                    # list artifacts
+pipex cat my-workspace greet greeting.txt       # read a file
+
+# Ephemeral mode: stream stdout, don't commit run
+pipex exec my-workspace -f step.yaml --step greet --ephemeral
+
+# Chain steps via --input
+pipex exec my-workspace -f process.yaml --step process --input greet
+
+# Aliased inputs (mount under /input/data instead of /input/greet)
+pipex exec my-workspace -f process.yaml --step process --input data=greet
+
+# Remove a step's run and state entry
+pipex rm-step my-workspace greet
+```
+
+### Inspecting runs
+
+Each step execution produces a **run** containing artifacts, logs (stdout/stderr), and metadata:
+
+```bash
+# Show all steps and their last run (status, duration, size, date)
+pipex show my-pipeline
+
+# Show logs from the last run of a step
+pipex logs my-pipeline download
+pipex logs my-pipeline download --stream stderr
+
+# Show execution metadata (image, cmd, duration, exit code, fingerprint…)
+pipex inspect my-pipeline download
+pipex inspect my-pipeline download --json
+
+# Export artifacts from a step to the host filesystem
+pipex export my-pipeline download ./output-dir
+```
+
 ### Managing workspaces
 
 ```bash
-# List workspaces (with artifact/cache counts)
+# List workspaces (with run/cache counts and disk size)
 pipex list
 pipex ls --json
 
+# Remove old runs (keeps only current ones)
+pipex prune my-pipeline
+
 # Remove specific workspaces
 pipex rm my-build other-build
 
@@ -56,8 +111,16 @@ pipex clean
 | Command | Description |
 |---------|-------------|
 | `run <pipeline>` | Execute a pipeline |
-| `list` (alias `ls`) | List workspaces |
+| `exec <workspace> -f <step-file>` | Execute a single step in a workspace |
+| `cat <workspace> <step> [path]` | Read or list artifact content from a step's latest run |
+| `show <workspace>` | Show steps and runs in a workspace |
+| `logs <workspace> <step>` | Show stdout/stderr from last run |
+| `inspect <workspace> <step>` | Show run metadata (meta.json) |
+| `export <workspace> <step> <dest>` | Extract artifacts from a step run to the host filesystem |
+| `prune <workspace>` | Remove old runs not referenced by current state |
+| `list` (alias `ls`) | List workspaces (with disk sizes) |
 | `rm <workspace...>` | Remove one or more workspaces |
+| `rm-step <workspace> <step>` | Remove a step's run and state entry |
 | `clean` | Remove all workspaces |
 
 ### Global Options
@@ -73,6 +136,22 @@ pipex clean
 |--------|-------|-------------|
 | `--workspace <name>` | `-w` | Workspace name for caching |
 | `--force [steps]` | `-f` | Skip cache for all steps, or a comma-separated list |
+| `--dry-run` | | Validate pipeline, compute fingerprints, show what would run without executing |
+| `--target <steps>` | `-t` | Execute only these steps and their dependencies (comma-separated) |
+| `--concurrency <n>` | `-c` | Max parallel step executions (default: CPU count) |
+| `--env-file <path>` | | Load environment variables from a dotenv file for all steps |
+| `--verbose` | | Stream container logs in real-time (interactive mode) |
+
+### Exec Options
+
+| Option | Alias | Description |
+|--------|-------|-------------|
+| `--file <path>` | `-f` | Step definition file (YAML or JSON, required) |
+| `--step <id>` | | Step ID (overrides file's id/name) |
+| `--input <specs...>` | | Input steps (e.g. `extract` or `data=extract`) |
+| `--ephemeral` | | Stream stdout to terminal and discard the run |
+| `--force` | | Skip cache check |
+| `--verbose` | | Stream container logs in real-time |
 
 ## Pipeline Format
 
@@ -117,44 +196,57 @@ Kits are reusable templates that generate the image, command, caches, and mounts
 ```yaml
 name: my-pipeline
 steps:
-  - id: build
+  - id: transform
     uses: node
-    with: { script: build.js, src: src/app }
+    with: { script: transform.js, src: src/app }
+  - id: convert
+    uses: node
+    with: { run: "node /app/convert.js --format csv --output /output/", src: src/app }
   - id: analyze
     uses: python
     with: { script: analyze.py, src: scripts }
+  - id: enrich
+    uses: python
+    with: { run: "python /app/enrich.py --locale fr --input /input/analyze/", src: scripts/ }
+    inputs: [{ step: analyze }]
   - id: extract
     uses: shell
-    with: { packages: [unzip], run: "unzip /input/build/archive.zip -d /output/" }
-    inputs: [{ step: build }]
+    with: { packages: [unzip], run: "unzip /input/transform/archive.zip -d /output/" }
+    inputs: [{ step: transform }]
 ```
 
 `uses` and `image`/`cmd` are mutually exclusive. All other step fields (`env`, `inputs`, `mounts`, `sources`, `caches`, `timeoutSec`, `allowFailure`, `allowNetwork`) remain available and merge with kit defaults (user values take priority). The `src` parameter in `with` copies the host directory into `/app` in the container's writable layer (see [Sources](#sources)).
 
 #### Available Kits
 
-**`node`** -- Run a Node.js script with automatic dependency installation.
+**`node`** -- Run a Node.js command with automatic dependency installation.
 
 | Parameter | Default | Description |
 |-----------|---------|-------------|
-| `script` | *(required)* | Script to run (relative to `/app`) |
+| `script` | -- | Script to run (relative to `/app`). Mutually exclusive with `run`. |
+| `run` | -- | Arbitrary shell command. Mutually exclusive with `script`. |
 | `src` | -- | Host directory to copy into `/app` |
 | `version` | `"24"` | Node.js version |
 | `packageManager` | `"npm"` | `"npm"`, `"pnpm"`, or `"yarn"` |
-| `install` | `true` | Run package install before script |
+| `install` | `true` | Run package install before command |
 | `variant` | `"alpine"` | Image variant |
 
-**`python`** -- Run a Python script with automatic dependency installation from `requirements.txt`.
+Exactly one of `script` or `run` is required. `script: transform.js` is shorthand for `run: node /app/transform.js`. Use `run` to pass arguments or call any command available in the container.
+
+**`python`** -- Run a Python command with automatic dependency installation from `requirements.txt`.
 
 | Parameter | Default | Description |
 |-----------|---------|-------------|
-| `script` | *(required)* | Script to run (relative to `/app`) |
+| `script` | -- | Script to run (relative to `/app`). Mutually exclusive with `run`. |
+| `run` | -- | Arbitrary shell command. Mutually exclusive with `script`. |
 | `src` | -- | Host directory to copy into `/app` |
 | `version` | `"3.12"` | Python version |
 | `packageManager` | `"pip"` | `"pip"` or `"uv"` |
-| `install` | `true` | Run dependency install before script |
+| `install` | `true` | Run dependency install before command |
 | `variant` | `"slim"` | Image variant |
 
+Exactly one of `script` or `run` is required. `script: analyze.py` is shorthand for `run: python /app/analyze.py`. Use `run` to pass arguments or call any command available in the container.
+
 **`shell`** -- Run a shell command in a container, with optional apt package installation.
 
 | Parameter | Default | Description |
@@ -210,15 +302,36 @@ steps:
 | `with` | object | Kit parameters |
 | `inputs` | InputSpec[] | Previous steps to mount as read-only |
 | `env` | Record<string, string> | Environment variables |
+| `envFile` | string | Path to a dotenv file (relative to pipeline file) |
 | `outputPath` | string | Output mount point (default: `/output`) |
 | `mounts` | MountSpec[] | Host directories to bind mount (read-only) |
 | `sources` | MountSpec[] | Host directories copied into the container's writable layer |
 | `caches` | CacheSpec[] | Persistent caches to mount |
+| `if` | string | Condition expression — step is skipped when it evaluates to false |
 | `timeoutSec` | number | Execution timeout |
+| `retries` | number | Number of retry attempts on transient failure |
+| `retryDelayMs` | number | Delay between retries (default: 5000) |
 | `allowFailure` | boolean | Continue pipeline if step fails |
 | `allowNetwork` | boolean | Enable network access |
 
-### Inputs
+### Step Dependencies and Parallel Execution
+
+Steps declare dependencies via `inputs`. Pipex automatically determines which steps can run in parallel based on the dependency graph. Steps at the same level (no dependency between them) execute concurrently, up to `--concurrency` (defaults to CPU count).
+
+```yaml
+steps:
+  - id: download
+    # ...
+  - id: process-a
+    inputs: [{ step: download }]      # waits for download
+  - id: process-b
+    inputs: [{ step: download }]      # waits for download
+  # process-a and process-b run in parallel
+  - id: merge
+    inputs: [{ step: process-a }, { step: process-b }]  # waits for both
+```
+
+#### Inputs
 
 Mount previous steps as read-only:
 
@@ -227,10 +340,45 @@ inputs:
   - step: step1
   - step: step2
     copyToOutput: true
+  - step: step3
+    optional: true
 ```
 
 - Mounted under `/input/{stepName}/`
 - `copyToOutput: true` copies content to output before execution
+- `optional: true` allows the step to run even if the dependency failed or was skipped
+
+#### Targeted Execution
+
+Use `--target` to execute only specific steps and their dependencies:
+
+```bash
+# Only run 'merge' and everything it depends on
+pipex run pipeline.yaml --target merge
+
+# Multiple targets
+pipex run pipeline.yaml --target process-a,process-b
+```
+
+### Conditional Steps
+
+Steps can be conditionally skipped using `if:` with a [JEXL](https://github.com/TomFrost/Jexl) expression. The expression is evaluated against the current environment variables via `env`:
+
+```yaml
+- id: notify
+  if: env.CI
+  uses: shell
+  with:
+    run: echo "Running in CI"
+
+- id: deploy
+  if: env.NODE_ENV == "production"
+  uses: shell
+  with:
+    run: echo "Deploying..."
+```
+
+When a condition evaluates to false, the step is skipped. Steps that depend on a skipped step with a required (non-optional) input are also skipped.
 
 ### Host Mounts
 
@@ -298,7 +446,7 @@ Common use cases:
 
 ### Geodata Processing
 
-The `examples/geodata/` pipeline downloads a shapefile archive, extracts it, and produces a CSV inventory — using the `debian` and `bash` kits:
+The `examples/geodata/` pipeline downloads a shapefile archive, extracts it, and produces a CSV inventory. The last two steps run in parallel:
 
 ```
 examples/geodata/
@@ -311,6 +459,28 @@ Steps: `download` → `extract` → `list-files` / `build-csv`
 pipex run examples/geodata/pipeline.yaml
 ```
 
+### Text Processing
+
+The `examples/text-processing/` pipeline demonstrates parallel branches, conditional steps, and optional inputs:
+
+```
+examples/text-processing/
+└── pipeline.yaml
+```
+
+Steps: `generate` → `stats` + `filter` (parallel) → `report` → `notify` (conditional), with `audit` (conditional, optional input to `notify`)
+
+```bash
+# Default: notify and audit are skipped (conditions not met)
+pipex run examples/text-processing/pipeline.yaml
+
+# Enable notifications
+NOTIFY=1 pipex run examples/text-processing/pipeline.yaml
+
+# Only run stats and its dependencies
+pipex run examples/text-processing/pipeline.yaml --target stats
+```
+
 ### Multi-Language
 
 The `examples/multi-language/` pipeline chains Node.js and Python steps using kits:
@@ -342,7 +512,7 @@ Workspaces enable caching across runs. The workspace ID is determined by:
 1. CLI flag `--workspace` (highest priority)
 2. Pipeline `id` (explicit or derived from `name`)
 
-**Cache behavior**: Steps are skipped if image, cmd, env, inputs, and mounts haven't changed. See code documentation for details.
+**Cache behavior**: Steps are skipped if image, cmd, env (including values from `envFile` and `--env-file`), inputs, and mounts haven't changed. See code documentation for details.
 
 ## Troubleshooting
 
@@ -372,7 +542,7 @@ pipex rm old-workspace-id
 pipex clean
 ```
 
-### Cached step with missing artifact
+### Cached step with missing run
 
 Force re-execution:
 

package/dist/__tests__/errors.js

@@ -0,0 +1,162 @@
+import test from 'ava';
+import { PipexError, DockerError, DockerNotAvailableError, ImagePullError, ContainerTimeoutError, ContainerCrashError, ContainerCleanupError, WorkspaceError, ArtifactNotFoundError, StagingError, PipelineError, ValidationError, CyclicDependencyError, StepNotFoundError, KitError, MissingParameterError } from '../errors.js';
+// -- instanceof chains -------------------------------------------------------
+test('DockerNotAvailableError is instanceof DockerError and PipexError', t => {
+    const error = new DockerNotAvailableError();
+    t.true(error instanceof DockerNotAvailableError);
+    t.true(error instanceof DockerError);
+    t.true(error instanceof PipexError);
+    t.true(error instanceof Error);
+});
+test('ImagePullError is instanceof DockerError and PipexError', t => {
+    const error = new ImagePullError('alpine:latest');
+    t.true(error instanceof ImagePullError);
+    t.true(error instanceof DockerError);
+    t.true(error instanceof PipexError);
+});
+test('ContainerTimeoutError is instanceof DockerError and PipexError', t => {
+    const error = new ContainerTimeoutError(30);
+    t.true(error instanceof ContainerTimeoutError);
+    t.true(error instanceof DockerError);
+    t.true(error instanceof PipexError);
+});
+test('ContainerCrashError is instanceof DockerError and PipexError', t => {
+    const error = new ContainerCrashError('build', 1);
+    t.true(error instanceof ContainerCrashError);
+    t.true(error instanceof DockerError);
+    t.true(error instanceof PipexError);
+});
+test('ContainerCleanupError is instanceof DockerError and PipexError', t => {
+    const error = new ContainerCleanupError();
+    t.true(error instanceof ContainerCleanupError);
+    t.true(error instanceof DockerError);
+    t.true(error instanceof PipexError);
+});
+test('ArtifactNotFoundError is instanceof WorkspaceError and PipexError', t => {
+    const error = new ArtifactNotFoundError('missing artifact');
+    t.true(error instanceof ArtifactNotFoundError);
+    t.true(error instanceof WorkspaceError);
+    t.true(error instanceof PipexError);
+});
+test('StagingError is instanceof WorkspaceError and PipexError', t => {
+    const error = new StagingError('staging failed');
+    t.true(error instanceof StagingError);
+    t.true(error instanceof WorkspaceError);
+    t.true(error instanceof PipexError);
+});
+test('ValidationError is instanceof PipelineError and PipexError', t => {
+    const error = new ValidationError('invalid');
+    t.true(error instanceof ValidationError);
+    t.true(error instanceof PipelineError);
+    t.true(error instanceof PipexError);
+});
+test('CyclicDependencyError is instanceof PipelineError and PipexError', t => {
+    const error = new CyclicDependencyError('cycle detected');
+    t.true(error instanceof CyclicDependencyError);
+    t.true(error instanceof PipelineError);
+    t.true(error instanceof PipexError);
+});
+test('StepNotFoundError is instanceof PipelineError and PipexError', t => {
+    const error = new StepNotFoundError('build', 'compile');
+    t.true(error instanceof StepNotFoundError);
+    t.true(error instanceof PipelineError);
+    t.true(error instanceof PipexError);
+});
+test('MissingParameterError is instanceof KitError and PipexError', t => {
+    const error = new MissingParameterError('node', 'script');
+    t.true(error instanceof MissingParameterError);
+    t.true(error instanceof KitError);
+    t.true(error instanceof PipexError);
+});
+// -- code property -----------------------------------------------------------
+test('DockerNotAvailableError has code DOCKER_NOT_AVAILABLE', t => {
+    t.is(new DockerNotAvailableError().code, 'DOCKER_NOT_AVAILABLE');
+});
+test('ImagePullError has code IMAGE_PULL_FAILED', t => {
+    t.is(new ImagePullError('alpine').code, 'IMAGE_PULL_FAILED');
+});
+test('ContainerTimeoutError has code CONTAINER_TIMEOUT', t => {
+    t.is(new ContainerTimeoutError(30).code, 'CONTAINER_TIMEOUT');
+});
+test('ContainerCrashError has code CONTAINER_CRASH', t => {
+    t.is(new ContainerCrashError('build', 1).code, 'CONTAINER_CRASH');
+});
+test('ContainerCleanupError has code CONTAINER_CLEANUP_FAILED', t => {
+    t.is(new ContainerCleanupError().code, 'CONTAINER_CLEANUP_FAILED');
+});
+test('ArtifactNotFoundError has code ARTIFACT_NOT_FOUND', t => {
+    t.is(new ArtifactNotFoundError('msg').code, 'ARTIFACT_NOT_FOUND');
+});
+test('StagingError has code STAGING_FAILED', t => {
+    t.is(new StagingError('msg').code, 'STAGING_FAILED');
+});
+test('ValidationError has code VALIDATION_ERROR', t => {
+    t.is(new ValidationError('msg').code, 'VALIDATION_ERROR');
+});
+test('StepNotFoundError has code STEP_NOT_FOUND', t => {
+    t.is(new StepNotFoundError('a', 'b').code, 'STEP_NOT_FOUND');
+});
+test('MissingParameterError has code MISSING_PARAMETER', t => {
+    t.is(new MissingParameterError('node', 'script').code, 'MISSING_PARAMETER');
+});
+// -- transient flag ----------------------------------------------------------
+test('DockerNotAvailableError is transient', t => {
+    t.true(new DockerNotAvailableError().transient);
+});
+test('ImagePullError is transient', t => {
+    t.true(new ImagePullError('alpine').transient);
+});
+test('ContainerTimeoutError is not transient', t => {
+    t.false(new ContainerTimeoutError(30).transient);
+});
+test('ContainerCrashError is not transient', t => {
+    t.false(new ContainerCrashError('build', 1).transient);
+});
+test('ValidationError is not transient', t => {
+    t.false(new ValidationError('msg').transient);
+});
+test('StagingError is not transient', t => {
+    t.false(new StagingError('msg').transient);
+});
+// -- cause chaining ----------------------------------------------------------
+test('PipexError supports cause chaining', t => {
+    const cause = new Error('original');
+    const error = new ValidationError('wrapped', { cause });
+    t.is(error.cause, cause);
+});
+test('DockerNotAvailableError supports cause chaining', t => {
+    const cause = new Error('cannot connect');
+    const error = new DockerNotAvailableError({ cause });
+    t.is(error.cause, cause);
+});
+test('ImagePullError supports cause chaining', t => {
+    const cause = new Error('network error');
+    const error = new ImagePullError('alpine', { cause });
+    t.is(error.cause, cause);
+});
+// -- message content ---------------------------------------------------------
+test('ContainerCrashError includes step id and exit code', t => {
+    const error = new ContainerCrashError('build', 137);
+    t.true(error.message.includes('build'));
+    t.true(error.message.includes('137'));
+    t.is(error.stepId, 'build');
+    t.is(error.exitCode, 137);
+});
+test('StepNotFoundError includes step id and referenced step', t => {
+    const error = new StepNotFoundError('deploy', 'build');
+    t.true(error.message.includes('deploy'));
+    t.true(error.message.includes('build'));
+});
+test('MissingParameterError includes kit name and param name', t => {
+    const error = new MissingParameterError('node', 'script');
+    t.true(error.message.includes('node'));
+    t.true(error.message.includes('script'));
+});
+test('ContainerTimeoutError includes timeout value', t => {
+    const error = new ContainerTimeoutError(60);
+    t.true(error.message.includes('60'));
+});
+test('ImagePullError includes image name', t => {
+    const error = new ImagePullError('myregistry/myimage:latest');
+    t.true(error.message.includes('myregistry/myimage:latest'));
+});

package/dist/__tests__/helpers.js

@@ -0,0 +1,41 @@
+import { execSync } from 'node:child_process';
+import { mkdtemp } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+/**
+ * Creates a temporary directory for test isolation.
+ * Each test should use its own tmpdir to avoid interference.
+ */
+export async function createTmpDir() {
+    return mkdtemp(join(tmpdir(), 'pipex-test-'));
+}
+/**
+ * Silent reporter — all methods are no-ops.
+ */
+export const noopReporter = {
+    emit() { }
+};
+/**
+ * Returns a reporter that records emit() calls for assertions.
+ */
+export function recordingReporter() {
+    const events = [];
+    const reporter = {
+        emit(event) {
+            events.push(event);
+        }
+    };
+    return { reporter, events };
+}
+/**
+ * Checks if Docker is available on the host (synchronous for use at module level).
+ */
+export function isDockerAvailable() {
+    try {
+        execSync('docker version', { stdio: 'ignore' });
+        return true;
+    }
+    catch {
+        return false;
+    }
+}

package/dist/__tests__/types.js

@@ -0,0 +1,8 @@
+import test from 'ava';
+import { isKitStep } from '../types.js';
+test('isKitStep returns true when step has uses', t => {
+    t.true(isKitStep({ uses: 'node', name: 'build' }));
+});
+test('isKitStep returns false when step has no uses', t => {
+    t.false(isKitStep({ image: 'alpine', cmd: ['echo', 'hi'], name: 'build' }));
+});

package/dist/cli/__tests__/condition.js

@@ -0,0 +1,23 @@
+import test from 'ava';
+import { evaluateCondition } from '../condition.js';
+test('env.CI truthy when CI is defined', async (t) => {
+    const result = await evaluateCondition('env.CI', { env: { CI: 'true' } });
+    t.true(result);
+});
+test('env.CI falsy when CI is absent', async (t) => {
+    const result = await evaluateCondition('env.CI', { env: {} });
+    t.false(result);
+});
+test('env.NODE_ENV == "production" — equality', async (t) => {
+    t.true(await evaluateCondition('env.NODE_ENV == "production"', { env: { NODE_ENV: 'production' } }));
+    t.false(await evaluateCondition('env.NODE_ENV == "production"', { env: { NODE_ENV: 'development' } }));
+});
+test('env.CI && !env.STAGING — combined logic', async (t) => {
+    t.true(await evaluateCondition('env.CI && !env.STAGING', { env: { CI: 'true' } }));
+    t.false(await evaluateCondition('env.CI && !env.STAGING', { env: { CI: 'true', STAGING: 'true' } }));
+    t.false(await evaluateCondition('env.CI && !env.STAGING', { env: {} }));
+});
+test('invalid expression returns false (fail-closed)', async (t) => {
+    const result = await evaluateCondition(')))invalid(((', { env: {} });
+    t.false(result);
+});