@livingdata/pipex 0.0.4 → 0.0.6

package/README.md

@@ -14,14 +14,14 @@ Runs containers with explicit volume mounts and manages artifacts through a stag
 Run directly without installing:
 
 ```bash
-npx @livingdata/pipex run pipeline.yaml --workspace my-build
+npx @livingdata/pipex run pipeline.yaml
 ```
 
 Or install globally:
 
 ```bash
 npm install -g @livingdata/pipex
-pipex run pipeline.yaml --workspace my-build
+pipex run pipeline.yaml
 ```
 
 ## Usage
@@ -30,9 +30,6 @@ pipex run pipeline.yaml --workspace my-build
 # Interactive mode (default)
 pipex run pipeline.yaml
 
-# With workspace name (enables caching)
-pipex run pipeline.yaml --workspace my-build
-
 # JSON mode (for CI/CD)
 pipex run pipeline.yaml --json
 
@@ -126,6 +123,10 @@ steps:
   - id: analyze
     uses: python
     with: { script: analyze.py, src: scripts }
+  - id: extract
+    uses: shell
+    with: { packages: [unzip], run: "unzip /input/build/archive.zip -d /output/" }
+    inputs: [{ step: build }]
 ```
 
 `uses` and `image`/`cmd` are mutually exclusive. All other step fields (`env`, `inputs`, `mounts`, `caches`, `timeoutSec`, `allowFailure`, `allowNetwork`) remain available and merge with kit defaults (user values take priority). The `src` parameter in `with` generates a read-only mount at `/app` in the container.
@@ -154,13 +155,32 @@ steps:
 | `install` | `true` | Run dependency install before script |
 | `variant` | `"slim"` | Image variant |
 
-**`bash`** -- Run a shell command in a lightweight container.
+**`shell`** -- Run a shell command in a container, with optional apt package installation.
 
 | Parameter | Default | Description |
 |-----------|---------|-------------|
 | `run` | *(required)* | Shell command to execute |
+| `packages` | -- | Apt packages to install before running |
 | `src` | -- | Host directory to mount at `/app` |
-| `image` | `"alpine:3.20"` | Docker image |
+| `image` | `"alpine:3.20"` | Docker image (defaults to `"debian:bookworm-slim"` when `packages` is set) |
+
+When `packages` is provided, the kit automatically switches to a Debian image, enables network access, and provides an `apt-cache` cache. Without packages, it runs on a minimal Alpine image with no network.
+
+```yaml
+# Simple command (alpine, no network)
+- id: list-files
+  uses: shell
+  with:
+    run: ls -lhR /input/data/
+
+# With system packages (debian, network + apt cache)
+- id: extract
+  uses: shell
+  with:
+    packages: [unzip, jq]
+    run: unzip /input/download/data.zip -d /output/
+  inputs: [{ step: download }]
+```
 
 ### Raw Steps
 
@@ -253,12 +273,29 @@ Common use cases:
 
 **Note**: Caches are workspace-scoped (not global). Different workspaces have isolated caches.
 
-## Example
+## Examples
+
+### Geodata Processing
+
+The `examples/geodata/` pipeline downloads a shapefile archive, extracts it, and produces a CSV inventory — using the `debian` and `bash` kits:
+
+```
+examples/geodata/
+└── pipeline.yaml
+```
+
+Steps: `download` → `extract` → `list-files` / `build-csv`
+
+```bash
+pipex run examples/geodata/pipeline.yaml
+```
+
+### Multi-Language
 
-The `example/` directory contains a multi-language pipeline that chains Node.js and Python steps using kits:
+The `examples/multi-language/` pipeline chains Node.js and Python steps using kits:
 
 ```
-example/
+examples/multi-language/
 ├── pipeline.yaml
 └── scripts/
     ├── nodejs/              # lodash-based data analysis
@@ -272,10 +309,10 @@ example/
         └── transform.py
 ```
 
-The pipeline uses the `node` and `python` kits to run 4 steps: `node-analyze` → `node-transform` → `python-analyze` → `python-transform`. Each step passes artifacts to the next via `/input`.
+Steps: `node-analyze` → `node-transform` → `python-analyze` → `python-transform`
 
 ```bash
-pipex run example/pipeline.yaml --workspace example-test
+pipex run examples/multi-language/pipeline.yaml
 ```
 
 ## Caching & Workspaces
@@ -334,7 +371,7 @@ cp .env.example .env
 Run the CLI without building (via tsx):
 
 ```bash
-npm run cli -- run pipeline.yaml --workspace my-build
+npm run cli -- run pipeline.yaml
 npm run cli -- list
 ```
 

package/dist/cli/pipeline-loader.js

@@ -7,6 +7,9 @@ import { isKitStep } from '../types.js';
 export class PipelineLoader {
     async load(filePath) {
         const content = await readFile(filePath, 'utf8');
+        return this.parse(content, filePath);
+    }
+    parse(content, filePath) {
         const input = parsePipelineFile(content, filePath);
         if (!input.id && !input.name) {
             throw new Error('Invalid pipeline: at least one of "id" or "name" must be defined');
@@ -132,7 +135,7 @@ export class PipelineLoader {
     }
 }
 /** Convert a free-form name into a valid identifier. */
-function slugify(name) {
+export function slugify(name) {
     return deburr(name)
         .toLowerCase()
         .replaceAll(/[^\w-]/g, '-')
@@ -140,20 +143,20 @@ function slugify(name) {
         .replace(/^-/, '')
         .replace(/-$/, '');
 }
-function parsePipelineFile(content, filePath) {
+export function parsePipelineFile(content, filePath) {
     const ext = extname(filePath).toLowerCase();
     if (ext === '.yaml' || ext === '.yml') {
         return parseYaml(content);
     }
     return JSON.parse(content);
 }
-function mergeEnv(kitEnv, userEnv) {
+export function mergeEnv(kitEnv, userEnv) {
     if (!kitEnv && !userEnv) {
         return undefined;
     }
     return { ...kitEnv, ...userEnv };
 }
-function mergeCaches(kitCaches, userCaches) {
+export function mergeCaches(kitCaches, userCaches) {
     if (!kitCaches && !userCaches) {
         return undefined;
     }
@@ -166,7 +169,7 @@ function mergeCaches(kitCaches, userCaches) {
     }
     return [...map.values()];
 }
-function mergeMounts(kitMounts, userMounts) {
+export function mergeMounts(kitMounts, userMounts) {
     if (!kitMounts && !userMounts) {
         return undefined;
     }

package/dist/cli/reporter.js

@@ -23,8 +23,10 @@ export class ConsoleReporter {
  * Suitable for local development and manual execution.
  */
 export class InteractiveReporter {
+    static maxStderrLines = 20;
     spinner;
     stepSpinners = new Map();
+    stderrBuffers = new Map();
     state(workspaceId, event, step, meta) {
         if (event === 'PIPELINE_START') {
             const displayName = meta?.pipelineName ?? workspaceId;
@@ -50,6 +52,7 @@ export class InteractiveReporter {
                 spinner.stopAndPersist({ symbol: chalk.green('✓'), text: chalk.green(step.displayName) });
                 this.stepSpinners.delete(step.id);
             }
+            this.stderrBuffers.delete(step.id);
         }
         if (event === 'STEP_FAILED' && step) {
             const spinner = this.stepSpinners.get(step.id);
@@ -62,6 +65,14 @@ export class InteractiveReporter {
                 });
                 this.stepSpinners.delete(step.id);
             }
+            const stderr = this.stderrBuffers.get(step.id);
+            if (stderr && stderr.length > 0) {
+                console.log(chalk.red('  ── stderr ──'));
+                for (const line of stderr) {
+                    console.log(chalk.red(`  ${line}`));
+                }
+            }
+            this.stderrBuffers.delete(step.id);
         }
         if (event === 'PIPELINE_FINISHED') {
             console.log(chalk.bold.green('\n✓ Pipeline completed\n'));
@@ -70,8 +81,18 @@ export class InteractiveReporter {
             console.log(chalk.bold.red('\n✗ Pipeline failed\n'));
         }
     }
-    log(_workspaceId, _step, _stream, _line) {
-        // Suppress logs in interactive mode
+    log(_workspaceId, step, stream, line) {
+        if (stream === 'stderr') {
+            let buffer = this.stderrBuffers.get(step.id);
+            if (!buffer) {
+                buffer = [];
+                this.stderrBuffers.set(step.id, buffer);
+            }
+            buffer.push(line);
+            if (buffer.length > InteractiveReporter.maxStderrLines) {
+                buffer.shift();
+            }
+        }
     }
     result(_workspaceId, _step, _result) {
         // Results shown via state updates

package/dist/kits/builtin/shell.js

@@ -0,0 +1,31 @@
+export const shellKit = {
+    name: 'shell',
+    resolve(params) {
+        const run = params.run;
+        if (!run || typeof run !== 'string') {
+            throw new Error('Kit "shell": "run" parameter is required');
+        }
+        const packages = params.packages;
+        const hasPackages = packages && packages.length > 0;
+        const src = params.src;
+        const defaultImage = hasPackages ? 'debian:bookworm-slim' : 'alpine:3.20';
+        const image = params.image ?? defaultImage;
+        const parts = [];
+        if (hasPackages) {
+            parts.push(`apt-get update && apt-get install -y --no-install-recommends ${packages.join(' ')} && rm -rf /var/lib/apt/lists/*`);
+        }
+        parts.push(run);
+        const output = {
+            image,
+            cmd: ['sh', '-c', parts.join(' && ')]
+        };
+        if (hasPackages) {
+            output.caches = [{ name: 'apt-cache', path: '/var/cache/apt' }];
+            output.allowNetwork = true;
+        }
+        if (src) {
+            output.mounts = [{ host: src, container: '/app' }];
+        }
+        return output;
+    }
+};

package/dist/kits/index.js

@@ -1,10 +1,10 @@
-import { bashKit } from './builtin/bash.js';
 import { nodeKit } from './builtin/node.js';
 import { pythonKit } from './builtin/python.js';
+import { shellKit } from './builtin/shell.js';
 const kits = new Map([
-    [bashKit.name, bashKit],
     [nodeKit.name, nodeKit],
-    [pythonKit.name, pythonKit]
+    [pythonKit.name, pythonKit],
+    [shellKit.name, shellKit]
 ]);
 export function getKit(name) {
     const kit = kits.get(name);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@livingdata/pipex",
-  "version": "0.0.4",
+  "version": "0.0.6",
   "description": "Execution engine for containerized pipeline steps",
   "author": "Jérôme Desboeufs <jerome@livingdata.co>",
   "type": "module",
@@ -21,6 +21,7 @@
   },
   "scripts": {
     "cli": "tsx src/cli/index.ts",
+    "test": "ava",
     "lint": "xo",
     "lint:fix": "xo --fix",
     "build": "tsc"
@@ -36,8 +37,10 @@
     "yaml": "^2.8.2"
   },
   "devDependencies": {
+    "@ava/typescript": "^6.0.0",
     "@types/lodash-es": "^4.17.12",
     "@types/node": "^25.2.0",
+    "ava": "^6.4.1",
     "tsx": "^4.21.0",
     "xo": "^1.2.3"
   }