@livingdata/pipex 0.0.6 → 0.0.8

package/README.md

@@ -129,7 +129,7 @@ steps:
     inputs: [{ step: build }]
 ```
 
-`uses` and `image`/`cmd` are mutually exclusive. All other step fields (`env`, `inputs`, `mounts`, `caches`, `timeoutSec`, `allowFailure`, `allowNetwork`) remain available and merge with kit defaults (user values take priority). The `src` parameter in `with` generates a read-only mount at `/app` in the container.
+`uses` and `image`/`cmd` are mutually exclusive. All other step fields (`env`, `inputs`, `mounts`, `sources`, `caches`, `timeoutSec`, `allowFailure`, `allowNetwork`) remain available and merge with kit defaults (user values take priority). The `src` parameter in `with` copies the host directory into `/app` in the container's writable layer (see [Sources](#sources)).
 
 #### Available Kits
 
@@ -138,7 +138,7 @@ steps:
 | Parameter | Default | Description |
 |-----------|---------|-------------|
 | `script` | *(required)* | Script to run (relative to `/app`) |
-| `src` | -- | Host directory to mount at `/app` |
+| `src` | -- | Host directory to copy into `/app` |
 | `version` | `"24"` | Node.js version |
 | `packageManager` | `"npm"` | `"npm"`, `"pnpm"`, or `"yarn"` |
 | `install` | `true` | Run package install before script |
@@ -149,7 +149,7 @@ steps:
 | Parameter | Default | Description |
 |-----------|---------|-------------|
 | `script` | *(required)* | Script to run (relative to `/app`) |
-| `src` | -- | Host directory to mount at `/app` |
+| `src` | -- | Host directory to copy into `/app` |
 | `version` | `"3.12"` | Python version |
 | `packageManager` | `"pip"` | `"pip"` or `"uv"` |
 | `install` | `true` | Run dependency install before script |
@@ -161,7 +161,7 @@ steps:
 |-----------|---------|-------------|
 | `run` | *(required)* | Shell command to execute |
 | `packages` | -- | Apt packages to install before running |
-| `src` | -- | Host directory to mount at `/app` |
+| `src` | -- | Host directory to mount read-only at `/app` |
 | `image` | `"alpine:3.20"` | Docker image (defaults to `"debian:bookworm-slim"` when `packages` is set) |
 
 When `packages` is provided, the kit automatically switches to a Debian image, enables network access, and provides an `apt-cache` cache. Without packages, it runs on a minimal Alpine image with no network.
@@ -212,6 +212,7 @@ steps:
 | `env` | Record<string, string> | Environment variables |
 | `outputPath` | string | Output mount point (default: `/output`) |
 | `mounts` | MountSpec[] | Host directories to bind mount (read-only) |
+| `sources` | MountSpec[] | Host directories copied into the container's writable layer |
 | `caches` | CacheSpec[] | Persistent caches to mount |
 | `timeoutSec` | number | Execution timeout |
 | `allowFailure` | boolean | Continue pipeline if step fails |
@@ -250,6 +251,26 @@ mounts:
 
 This means a pipeline at `/project/ci/pipeline.yaml` can only mount subdirectories of `/project/ci/`. Use `/tmp` or `/output` inside the container for writes.
 
+### Sources
+
+Copy host directories into the container's **writable layer**. Unlike bind mounts, copied files live inside the container so the step can create new files and subdirectories alongside them (e.g. `node_modules` after `npm install`).
+
+```yaml
+sources:
+  - host: src/app
+    container: /app
+```
+
+- Same path rules as `mounts` (`host` relative, `container` absolute, no `..`)
+- Files are snapshotted at step start -- changes on the host during execution are not reflected
+- The container can write next to source files without affecting the host
+
+**When to use `sources` vs `mounts`**:
+- Use `sources` when the step needs to write alongside the source files (install dependencies, generate build artifacts next to sources)
+- Use `mounts` when read-only access is sufficient (config files, static data)
+
+Kits use `sources` internally: the `node` kit's `src` parameter copies into `/app` so that `npm install` can create `node_modules`.
+
 ### Caches
 
 Persistent read-write directories shared across steps and executions:

package/dist/cli/pipeline-loader.js

@@ -49,6 +49,7 @@ export class PipelineLoader {
             outputPath: step.outputPath,
             caches: mergeCaches(kitOutput.caches, step.caches),
             mounts: mergeMounts(kitOutput.mounts, step.mounts),
+            sources: mergeMounts(kitOutput.sources, step.sources),
             timeoutSec: step.timeoutSec,
             allowFailure: step.allowFailure,
             allowNetwork: step.allowNetwork ?? kitOutput.allowNetwork
@@ -70,6 +71,9 @@ export class PipelineLoader {
         if (step.mounts) {
             this.validateMounts(step.id, step.mounts);
         }
+        if (step.sources) {
+            this.validateMounts(step.id, step.sources);
+        }
         if (step.caches) {
             this.validateCaches(step.id, step.caches);
         }

package/dist/cli/pipeline-runner.js

@@ -102,6 +102,10 @@ export class PipelineRunner {
                 output,
                 caches,
                 mounts,
+                sources: step.sources?.map(m => ({
+                    hostPath: resolve(pipelineRoot, m.host),
+                    containerPath: m.container
+                })),
                 network: step.allowNetwork ? 'bridge' : 'none',
                 timeoutSec: step.timeoutSec
             }, ({ stream, line }) => {
@@ -110,6 +114,7 @@ export class PipelineRunner {
             this.reporter.result(workspace.id, stepRef, result);
             if (result.exitCode === 0 || step.allowFailure) {
                 await workspace.commitArtifact(artifactId);
+                await workspace.linkArtifact(step.id, artifactId);
                 stepArtifacts.set(step.id, artifactId);
                 state.setStep(step.id, artifactId, currentFingerprint);
                 await state.save();
@@ -131,6 +136,7 @@ export class PipelineRunner {
                 const artifacts = await workspace.listArtifacts();
                 if (artifacts.includes(cached.artifactId)) {
                     stepArtifacts.set(step.id, cached.artifactId);
+                    await workspace.linkArtifact(step.id, cached.artifactId);
                     this.reporter.state(workspace.id, 'STEP_SKIPPED', stepRef, { artifactId: cached.artifactId, reason: 'cached' });
                     return true;
                 }

package/dist/cli/reporter.js

@@ -23,62 +23,64 @@ export class ConsoleReporter {
  * Suitable for local development and manual execution.
  */
 export class InteractiveReporter {
-    static maxStderrLines = 20;
+    static get maxStderrLines() {
+        return 20;
+    }
     spinner;
     stepSpinners = new Map();
     stderrBuffers = new Map();
     state(workspaceId, event, step, meta) {
-        if (event === 'PIPELINE_START') {
-            const displayName = meta?.pipelineName ?? workspaceId;
-            console.log(chalk.bold(`\n▶ Pipeline: ${chalk.cyan(displayName)}\n`));
-        }
-        if (event === 'STEP_STARTING' && step) {
-            const spinner = ora({ text: step.displayName, prefixText: '  ' }).start();
-            this.stepSpinners.set(step.id, spinner);
-        }
-        if (event === 'STEP_SKIPPED' && step) {
-            const spinner = this.stepSpinners.get(step.id);
-            if (spinner) {
-                spinner.stopAndPersist({ symbol: chalk.gray('⊙'), text: chalk.gray(`${step.displayName} (cached)`) });
-                this.stepSpinners.delete(step.id);
+        switch (event) {
+            case 'PIPELINE_START': {
+                const displayName = meta?.pipelineName ?? workspaceId;
+                console.log(chalk.bold(`\n▶ Pipeline: ${chalk.cyan(displayName)}\n`));
+                break;
             }
-            else {
-                console.log(`  ${chalk.gray('⊙')} ${chalk.gray(`${step.displayName} (cached)`)}`);
+            case 'STEP_STARTING': {
+                if (step) {
+                    const spinner = ora({ text: step.displayName, prefixText: '  ' }).start();
+                    this.stepSpinners.set(step.id, spinner);
+                }
+                break;
             }
-        }
-        if (event === 'STEP_FINISHED' && step) {
-            const spinner = this.stepSpinners.get(step.id);
-            if (spinner) {
-                spinner.stopAndPersist({ symbol: chalk.green('✓'), text: chalk.green(step.displayName) });
-                this.stepSpinners.delete(step.id);
+            case 'STEP_SKIPPED': {
+                if (step) {
+                    const spinner = this.stepSpinners.get(step.id);
+                    if (spinner) {
+                        spinner.stopAndPersist({ symbol: chalk.gray('⊙'), text: chalk.gray(`${step.displayName} (cached)`) });
+                        this.stepSpinners.delete(step.id);
+                    }
+                    else {
+                        console.log(`  ${chalk.gray('⊙')} ${chalk.gray(`${step.displayName} (cached)`)}`);
+                    }
+                }
+                break;
             }
-            this.stderrBuffers.delete(step.id);
-        }
-        if (event === 'STEP_FAILED' && step) {
-            const spinner = this.stepSpinners.get(step.id);
-            const exitCode = meta?.exitCode;
-            if (spinner) {
-                const exitInfo = exitCode === undefined ? '' : ` (exit ${exitCode})`;
-                spinner.stopAndPersist({
-                    symbol: chalk.red('✗'),
-                    text: chalk.red(`${step.displayName}${exitInfo}`)
-                });
-                this.stepSpinners.delete(step.id);
+            case 'STEP_FINISHED': {
+                if (step) {
+                    const spinner = this.stepSpinners.get(step.id);
+                    if (spinner) {
+                        spinner.stopAndPersist({ symbol: chalk.green('✓'), text: chalk.green(step.displayName) });
+                        this.stepSpinners.delete(step.id);
+                    }
+                    this.stderrBuffers.delete(step.id);
+                }
+                break;
             }
-            const stderr = this.stderrBuffers.get(step.id);
-            if (stderr && stderr.length > 0) {
-                console.log(chalk.red('  ── stderr ──'));
-                for (const line of stderr) {
-                    console.log(chalk.red(`  ${line}`));
+            case 'STEP_FAILED': {
+                if (step) {
+                    this.handleStepFailed(step, meta);
                 }
+                break;
+            }
+            case 'PIPELINE_FINISHED': {
+                console.log(chalk.bold.green('\n✓ Pipeline completed\n'));
+                break;
+            }
+            case 'PIPELINE_FAILED': {
+                console.log(chalk.bold.red('\n✗ Pipeline failed\n'));
+                break;
             }
-            this.stderrBuffers.delete(step.id);
-        }
-        if (event === 'PIPELINE_FINISHED') {
-            console.log(chalk.bold.green('\n✓ Pipeline completed\n'));
-        }
-        if (event === 'PIPELINE_FAILED') {
-            console.log(chalk.bold.red('\n✗ Pipeline failed\n'));
         }
     }
     log(_workspaceId, step, stream, line) {
@@ -97,4 +99,24 @@ export class InteractiveReporter {
     result(_workspaceId, _step, _result) {
         // Results shown via state updates
     }
+    handleStepFailed(step, meta) {
+        const spinner = this.stepSpinners.get(step.id);
+        const exitCode = meta?.exitCode;
+        if (spinner) {
+            const exitInfo = exitCode === undefined ? '' : ` (exit ${exitCode})`;
+            spinner.stopAndPersist({
+                symbol: chalk.red('✗'),
+                text: chalk.red(`${step.displayName}${exitInfo}`)
+            });
+            this.stepSpinners.delete(step.id);
+        }
+        const stderr = this.stderrBuffers.get(step.id);
+        if (stderr && stderr.length > 0) {
+            console.log(chalk.red('  ── stderr ──'));
+            for (const line of stderr) {
+                console.log(chalk.red(`  ${line}`));
+            }
+        }
+        this.stderrBuffers.delete(step.id);
+    }
 }

package/dist/engine/docker-executor.js

@@ -1,6 +1,5 @@
 import process from 'node:process';
 import { execa } from 'execa';
-import { createInterface } from 'node:readline';
 import { ContainerExecutor } from './executor.js';
 /**
  * Build a minimal environment for the Docker CLI process.
@@ -29,7 +28,10 @@ export class DockerCliExecutor extends ContainerExecutor {
     }
     async run(workspace, request, onLogLine) {
         const startedAt = new Date();
-        const args = ['run', '--name', request.name, '--network', request.network];
+        // Use create+start instead of run: docker run cannot create mountpoints
+        // for anonymous volumes inside read-only bind mounts (shadow paths).
+        // docker create sets up the filesystem layer before readonly applies.
+        const args = ['create', '--name', request.name, '--network', request.network];
         if (request.env) {
             for (const [key, value] of Object.entries(request.env)) {
                 args.push('-e', `${key}=${value}`);
@@ -59,24 +61,32 @@ export class DockerCliExecutor extends ContainerExecutor {
         let exitCode = 0;
         let error;
         try {
-            const proc = execa('docker', args, {
+            await execa('docker', args, { env: this.env });
+            // Copy source directories into the container's writable layer.
+            // Unlike bind mounts, copied files live in the container layer,
+            // so the container can create subdirectories (e.g. node_modules).
+            if (request.sources) {
+                for (const source of request.sources) {
+                    await execa('docker', ['cp', `${source.hostPath}/.`, `${request.name}:${source.containerPath}`], { env: this.env });
+                }
+            }
+            const proc = execa('docker', ['start', '-a', request.name], {
                 env: this.env,
                 reject: false,
                 timeout: request.timeoutSec ? request.timeoutSec * 1000 : undefined
             });
-            if (proc.stdout) {
-                const rl = createInterface({ input: proc.stdout });
-                rl.on('line', line => {
+            const stdoutDone = (async () => {
+                for await (const line of proc.iterable({ from: 'stdout' })) {
                     onLogLine({ stream: 'stdout', line });
-                });
-            }
-            if (proc.stderr) {
-                const rl = createInterface({ input: proc.stderr });
-                rl.on('line', line => {
+                }
+            })();
+            const stderrDone = (async () => {
+                for await (const line of proc.iterable({ from: 'stderr' })) {
                     onLogLine({ stream: 'stderr', line });
-                });
-            }
+                }
+            })();
             const result = await proc;
+            await Promise.all([stdoutDone, stderrDone]);
             exitCode = result.exitCode ?? 0;
         }
         catch (error_) {
@@ -85,7 +95,7 @@ export class DockerCliExecutor extends ContainerExecutor {
         }
         finally {
             try {
-                await execa('docker', ['rm', '-f', request.name], { env: this.env, reject: false });
+                await execa('docker', ['rm', '-f', '-v', request.name], { env: this.env, reject: false });
             }
             catch {
                 // Best effort cleanup

package/dist/engine/workspace.js

@@ -1,4 +1,4 @@
-import { access, mkdir, readdir, rename, rm } from 'node:fs/promises';
+import { access, mkdir, readdir, rename, rm, symlink } from 'node:fs/promises';
 import { randomUUID } from 'node:crypto';
 import { join } from 'node:path';
 /**
@@ -159,6 +159,19 @@ export class Workspace {
     async commitArtifact(artifactId) {
         await rename(this.stagingPath(artifactId), this.artifactPath(artifactId));
     }
+    /**
+     * Creates a symlink from `step-artifacts/{stepId}` to the committed artifact.
+     * Replaces any existing symlink for the same step.
+     * @param stepId - Step identifier
+     * @param artifactId - Committed artifact identifier
+     */
+    async linkArtifact(stepId, artifactId) {
+        const dir = join(this.root, 'step-artifacts');
+        await mkdir(dir, { recursive: true });
+        const linkPath = join(dir, stepId);
+        await rm(linkPath, { force: true });
+        await symlink(join('..', 'artifacts', artifactId), linkPath);
+    }
     /**
      * Discards a staging artifact (on execution failure).
      * @param artifactId - Artifact identifier

package/dist/kits/builtin/node.js

@@ -6,13 +6,13 @@ const cacheMap = {
 function buildInstallCommand(packageManager) {
     switch (packageManager) {
         case 'npm': {
-            return 'cd /tmp && cp /app/package*.json . && npm install --no-audit --no-fund 2>&1';
+            return 'cd /app && npm install --no-package-lock --no-audit --no-fund 2>&1';
         }
         case 'pnpm': {
-            return 'cd /tmp && cp /app/package.json . && cp /app/pnpm-lock.yaml . 2>/dev/null; pnpm install --no-frozen-lockfile 2>&1';
+            return 'cd /app && pnpm install --no-frozen-lockfile 2>&1';
         }
         case 'yarn': {
-            return 'cd /tmp && cp /app/package.json . && cp /app/yarn.lock . 2>/dev/null; yarn install 2>&1';
+            return 'cd /app && yarn install 2>&1';
         }
         default: {
             throw new Error(`Kit "node": unsupported packageManager "${packageManager}"`);
@@ -36,8 +36,7 @@ export const nodeKit = {
         if (install) {
             parts.push(buildInstallCommand(packageManager));
         }
-        const nodePathPrefix = install ? 'NODE_PATH=/tmp/node_modules ' : '';
-        parts.push(`${nodePathPrefix}node /app/${script}`);
+        parts.push(`node /app/${script}`);
         const cache = cacheMap[packageManager];
         if (!cache) {
             throw new Error(`Kit "node": unsupported packageManager "${packageManager}"`);
@@ -49,7 +48,7 @@ export const nodeKit = {
             allowNetwork: true
         };
         if (src) {
-            output.mounts = [{ host: src, container: '/app' }];
+            output.sources = [{ host: src, container: '/app' }];
         }
         return output;
     }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@livingdata/pipex",
-  "version": "0.0.6",
+  "version": "0.0.8",
   "description": "Execution engine for containerized pipeline steps",
   "author": "Jérôme Desboeufs <jerome@livingdata.co>",
   "type": "module",