@livingdata/pipex 0.0.8 → 0.0.9

package/dist/cli/pipeline-runner.js

@@ -1,7 +1,9 @@
-import { cp } from 'node:fs/promises';
-import { dirname, resolve } from 'node:path';
+import { cp, writeFile } from 'node:fs/promises';
+import { createWriteStream } from 'node:fs';
+import { dirname, join, resolve } from 'node:path';
 import { Workspace } from '../engine/index.js';
 import { StateManager } from './state.js';
+import { dirSize } from './utils.js';
 /**
  * Orchestrates pipeline execution with dependency resolution and caching.
  *
@@ -10,27 +12,20 @@ import { StateManager } from './state.js';
  * 1. **Workspace Resolution**: Determines workspace ID from CLI flag, config, or filename
  * 2. **State Loading**: Loads cached fingerprints from state.json
  * 3. **Step Execution**: For each step:
- *    a. Computes fingerprint (image + cmd + env + input artifact IDs)
- *    b. Checks cache (fingerprint match + artifact exists)
+ *    a. Computes fingerprint (image + cmd + env + input run IDs)
+ *    b. Checks cache (fingerprint match + run exists)
  *    c. If cached: skips execution
  *    d. If not cached: resolves inputs, prepares staging, executes container
- *    e. On success: commits artifact, saves state
- *    f. On failure: discards artifact, halts pipeline (unless allowFailure)
+ *    e. On success: writes meta.json, commits run, saves state
+ *    f. On failure: discards run, halts pipeline (unless allowFailure)
  * 4. **Completion**: Reports final pipeline status
  *
- * ## Dependencies
+ * ## Runs
  *
- * Steps declare dependencies via `inputs: [{step: "stepId"}]`.
- * The runner:
- * - Mounts input artifacts as read-only volumes
- * - Optionally copies inputs to output staging (if `copyToOutput: true`)
- * - Tracks execution order to resolve step names to artifact IDs
- *
- * ## Caching
- *
- * Cache invalidation is automatic:
- * - Changing a step's configuration re-runs it
- * - Re-running a step invalidates all dependent steps
+ * Each step execution produces a **run** containing:
+ * - `artifacts/` — files produced by the step
+ * - `stdout.log` / `stderr.log` — captured container logs
+ * - `meta.json` — structured execution metadata
  */
 export class PipelineRunner {
     loader;
@@ -44,10 +39,9 @@ export class PipelineRunner {
         this.workdirRoot = workdirRoot;
     }
     async run(pipelineFilePath, options) {
-        const { workspace: workspaceName, force } = options ?? {};
+        const { workspace: workspaceName, force, dryRun } = options ?? {};
         const config = await this.loader.load(pipelineFilePath);
         const pipelineRoot = dirname(resolve(pipelineFilePath));
-        // Workspace ID priority: CLI arg > pipeline id
         const workspaceId = workspaceName ?? config.id;
         let workspace;
         try {
@@ -57,15 +51,19 @@ export class PipelineRunner {
             workspace = await Workspace.create(this.workdirRoot, workspaceId);
         }
         await workspace.cleanupStaging();
-        await this.runtime.check();
+        if (!dryRun) {
+            await this.runtime.check();
+            await this.runtime.cleanupContainers(workspace.id);
+        }
         const state = new StateManager(workspace.root);
         await state.load();
-        const stepArtifacts = new Map();
+        const stepRuns = new Map();
+        let totalArtifactSize = 0;
         this.reporter.state(workspace.id, 'PIPELINE_START', undefined, { pipelineName: config.name ?? config.id });
         for (const step of config.steps) {
             const stepRef = { id: step.id, displayName: step.name ?? step.id };
-            const inputArtifactIds = step.inputs
-                ?.map(i => stepArtifacts.get(i.step))
+            const inputRunIds = step.inputs
+                ?.map(i => stepRuns.get(i.step))
                 .filter((id) => id !== undefined);
             const resolvedMounts = step.mounts?.map(m => ({
                 hostPath: resolve(pipelineRoot, m.host),
@@ -75,24 +73,36 @@ export class PipelineRunner {
                 image: step.image,
                 cmd: step.cmd,
                 env: step.env,
-                inputArtifactIds,
+                inputRunIds,
                 mounts: resolvedMounts
             });
             const skipCache = force === true || (Array.isArray(force) && force.includes(step.id));
-            if (!skipCache && await this.tryUseCache({ workspace, state, step, stepRef, currentFingerprint, stepArtifacts })) {
+            if (!skipCache && await this.tryUseCache({ workspace, state, step, stepRef, currentFingerprint, stepRuns })) {
+                continue;
+            }
+            if (dryRun) {
+                this.reporter.state(workspace.id, 'STEP_WOULD_RUN', stepRef);
                 continue;
             }
             this.reporter.state(workspace.id, 'STEP_STARTING', stepRef);
-            const artifactId = workspace.generateArtifactId();
-            const stagingPath = await workspace.prepareArtifact(artifactId);
-            await this.prepareStagingWithInputs(workspace, step, stagingPath, stepArtifacts);
-            // Prepare caches
-            if (step.caches) {
-                for (const cache of step.caches) {
-                    await workspace.prepareCache(cache.name);
-                }
+            const stepArtifactSize = await this.executeStep({ workspace, state, step, stepRef, stepRuns, currentFingerprint, resolvedMounts, pipelineRoot });
+            totalArtifactSize += stepArtifactSize;
+        }
+        this.reporter.state(workspace.id, 'PIPELINE_FINISHED', undefined, { totalArtifactSize });
+    }
+    async executeStep({ workspace, state, step, stepRef, stepRuns, currentFingerprint, resolvedMounts, pipelineRoot }) {
+        const runId = workspace.generateRunId();
+        const stagingPath = await workspace.prepareRun(runId);
+        await this.prepareStagingWithInputs(workspace, step, workspace.runStagingArtifactsPath(runId), stepRuns);
+        if (step.caches) {
+            for (const cache of step.caches) {
+                await workspace.prepareCache(cache.name);
             }
-            const { inputs, output, caches, mounts } = this.buildMounts(step, artifactId, stepArtifacts, pipelineRoot);
+        }
+        const { inputs, output, caches, mounts } = this.buildMounts(step, runId, stepRuns, pipelineRoot);
+        const stdoutLog = createWriteStream(join(stagingPath, 'stdout.log'));
+        const stderrLog = createWriteStream(join(stagingPath, 'stderr.log'));
+        try {
             const result = await this.runtime.run(workspace, {
                 name: `pipex-${workspace.id}-${step.id}-${Date.now()}`,
                 image: step.image,
@@ -109,76 +119,114 @@ export class PipelineRunner {
                 network: step.allowNetwork ? 'bridge' : 'none',
                 timeoutSec: step.timeoutSec
             }, ({ stream, line }) => {
+                if (stream === 'stdout') {
+                    stdoutLog.write(line + '\n');
+                }
+                else {
+                    stderrLog.write(line + '\n');
+                }
                 this.reporter.log(workspace.id, stepRef, stream, line);
             });
             this.reporter.result(workspace.id, stepRef, result);
+            await closeStream(stdoutLog);
+            await closeStream(stderrLog);
+            await this.writeRunMeta(stagingPath, { runId, step, stepRuns, resolvedMounts, currentFingerprint, result });
             if (result.exitCode === 0 || step.allowFailure) {
-                await workspace.commitArtifact(artifactId);
-                await workspace.linkArtifact(step.id, artifactId);
-                stepArtifacts.set(step.id, artifactId);
-                state.setStep(step.id, artifactId, currentFingerprint);
+                await workspace.commitRun(runId);
+                await workspace.linkRun(step.id, runId);
+                stepRuns.set(step.id, runId);
+                state.setStep(step.id, runId, currentFingerprint);
                 await state.save();
-                this.reporter.state(workspace.id, 'STEP_FINISHED', stepRef, { artifactId });
-            }
-            else {
-                await workspace.discardArtifact(artifactId);
-                this.reporter.state(workspace.id, 'STEP_FAILED', stepRef, { exitCode: result.exitCode });
-                this.reporter.state(workspace.id, 'PIPELINE_FAILED');
-                throw new Error(`Step ${step.id} failed with exit code ${result.exitCode}`);
+                const durationMs = result.finishedAt.getTime() - result.startedAt.getTime();
+                const artifactSize = await dirSize(workspace.runArtifactsPath(runId));
+                this.reporter.state(workspace.id, 'STEP_FINISHED', stepRef, { runId, durationMs, artifactSize });
+                return artifactSize;
             }
+            await workspace.discardRun(runId);
+            this.reporter.state(workspace.id, 'STEP_FAILED', stepRef, { exitCode: result.exitCode });
+            this.reporter.state(workspace.id, 'PIPELINE_FAILED');
+            throw new Error(`Step ${step.id} failed with exit code ${result.exitCode}`);
+        }
+        catch (error) {
+            await closeStream(stdoutLog);
+            await closeStream(stderrLog);
+            throw error;
         }
-        this.reporter.state(workspace.id, 'PIPELINE_FINISHED');
     }
-    async tryUseCache({ workspace, state, step, stepRef, currentFingerprint, stepArtifacts }) {
+    async writeRunMeta(stagingPath, { runId, step, stepRuns, resolvedMounts, currentFingerprint, result }) {
+        const meta = {
+            runId,
+            stepId: step.id,
+            stepName: step.name,
+            startedAt: result.startedAt.toISOString(),
+            finishedAt: result.finishedAt.toISOString(),
+            durationMs: result.finishedAt.getTime() - result.startedAt.getTime(),
+            exitCode: result.exitCode,
+            image: step.image,
+            cmd: step.cmd,
+            env: step.env,
+            inputs: step.inputs?.map(i => ({
+                step: i.step,
+                runId: stepRuns.get(i.step),
+                mountedAs: `/input/${i.step}`
+            })),
+            mounts: resolvedMounts,
+            caches: step.caches?.map(c => c.name),
+            allowNetwork: step.allowNetwork ?? false,
+            fingerprint: currentFingerprint,
+            status: result.exitCode === 0 ? 'success' : 'failure'
+        };
+        await writeFile(join(stagingPath, 'meta.json'), JSON.stringify(meta, null, 2), 'utf8');
+    }
+    async tryUseCache({ workspace, state, step, stepRef, currentFingerprint, stepRuns }) {
         const cached = state.getStep(step.id);
         if (cached?.fingerprint === currentFingerprint) {
             try {
-                const artifacts = await workspace.listArtifacts();
-                if (artifacts.includes(cached.artifactId)) {
-                    stepArtifacts.set(step.id, cached.artifactId);
-                    await workspace.linkArtifact(step.id, cached.artifactId);
-                    this.reporter.state(workspace.id, 'STEP_SKIPPED', stepRef, { artifactId: cached.artifactId, reason: 'cached' });
+                const runs = await workspace.listRuns();
+                if (runs.includes(cached.runId)) {
+                    stepRuns.set(step.id, cached.runId);
+                    await workspace.linkRun(step.id, cached.runId);
+                    this.reporter.state(workspace.id, 'STEP_SKIPPED', stepRef, { runId: cached.runId, reason: 'cached' });
                     return true;
                 }
             }
             catch {
-                // Artifact missing, proceed with execution
+                // Run missing, proceed with execution
             }
         }
         return false;
     }
-    async prepareStagingWithInputs(workspace, step, stagingPath, stepArtifacts) {
+    async prepareStagingWithInputs(workspace, step, stagingArtifactsPath, stepRuns) {
         if (!step.inputs) {
             return;
         }
         for (const input of step.inputs) {
-            const inputArtifactId = stepArtifacts.get(input.step);
-            if (!inputArtifactId) {
+            const inputRunId = stepRuns.get(input.step);
+            if (!inputRunId) {
                 throw new Error(`Step ${step.id}: input step '${input.step}' not found or not yet executed`);
             }
             if (input.copyToOutput) {
-                await cp(workspace.artifactPath(inputArtifactId), stagingPath, { recursive: true });
+                await cp(workspace.runArtifactsPath(inputRunId), stagingArtifactsPath, { recursive: true });
             }
         }
     }
-    buildMounts(step, outputArtifactId, stepArtifacts, pipelineRoot) {
+    buildMounts(step, outputRunId, stepRuns, pipelineRoot) {
         const inputs = [];
         if (step.inputs) {
             for (const input of step.inputs) {
-                const inputArtifactId = stepArtifacts.get(input.step);
-                if (inputArtifactId) {
+                const inputRunId = stepRuns.get(input.step);
+                if (inputRunId) {
                     inputs.push({
-                        artifactId: inputArtifactId,
+                        runId: inputRunId,
                         containerPath: `/input/${input.step}`
                     });
                 }
             }
         }
         const output = {
-            stagingArtifactId: outputArtifactId,
+            stagingRunId: outputRunId,
             containerPath: step.outputPath ?? '/output'
         };
-        // Build cache mounts
         let caches;
         if (step.caches) {
             caches = step.caches.map(c => ({
@@ -196,3 +244,14 @@ export class PipelineRunner {
         return { inputs, output, caches, mounts };
     }
 }
+async function closeStream(stream) {
+    if (stream.destroyed) {
+        return;
+    }
+    return new Promise((resolve, reject) => {
+        stream.end(() => {
+            resolve();
+        });
+        stream.on('error', reject);
+    });
+}

package/dist/cli/reporter.js

@@ -1,6 +1,7 @@
 import pino from 'pino';
 import chalk from 'chalk';
 import ora from 'ora';
+import { formatDuration, formatSize } from './utils.js';
 /**
  * Reporter that outputs structured JSON logs via pino.
  * Suitable for CI/CD environments and log aggregation.
@@ -26,9 +27,13 @@ export class InteractiveReporter {
     static get maxStderrLines() {
         return 20;
     }
+    verbose;
     spinner;
     stepSpinners = new Map();
     stderrBuffers = new Map();
+    constructor(options) {
+        this.verbose = options?.verbose ?? false;
+    }
     state(workspaceId, event, step, meta) {
         switch (event) {
             case 'PIPELINE_START': {
@@ -58,12 +63,7 @@ export class InteractiveReporter {
             }
             case 'STEP_FINISHED': {
                 if (step) {
-                    const spinner = this.stepSpinners.get(step.id);
-                    if (spinner) {
-                        spinner.stopAndPersist({ symbol: chalk.green('✓'), text: chalk.green(step.displayName) });
-                        this.stepSpinners.delete(step.id);
-                    }
-                    this.stderrBuffers.delete(step.id);
+                    this.handleStepFinished(step, meta);
                 }
                 break;
             }
@@ -73,8 +73,14 @@ export class InteractiveReporter {
                 }
                 break;
             }
+            case 'STEP_WOULD_RUN': {
+                if (step) {
+                    this.handleStepWouldRun(step);
+                }
+                break;
+            }
             case 'PIPELINE_FINISHED': {
-                console.log(chalk.bold.green('\n✓ Pipeline completed\n'));
+                this.handlePipelineFinished(meta);
                 break;
             }
             case 'PIPELINE_FAILED': {
@@ -84,6 +90,18 @@ export class InteractiveReporter {
         }
     }
     log(_workspaceId, step, stream, line) {
+        if (this.verbose) {
+            const spinner = this.stepSpinners.get(step.id);
+            const prefix = chalk.gray(`  [${step.id}]`);
+            if (spinner) {
+                spinner.clear();
+                console.log(`${prefix} ${line}`);
+                spinner.render();
+            }
+            else {
+                console.log(`${prefix} ${line}`);
+            }
+        }
         if (stream === 'stderr') {
             let buffer = this.stderrBuffers.get(step.id);
             if (!buffer) {
@@ -99,6 +117,39 @@ export class InteractiveReporter {
     result(_workspaceId, _step, _result) {
         // Results shown via state updates
     }
+    handleStepFinished(step, meta) {
+        const spinner = this.stepSpinners.get(step.id);
+        if (spinner) {
+            const details = [];
+            if (typeof meta?.durationMs === 'number') {
+                details.push(formatDuration(meta.durationMs));
+            }
+            if (typeof meta?.artifactSize === 'number' && meta.artifactSize > 0) {
+                details.push(formatSize(meta.artifactSize));
+            }
+            const suffix = details.length > 0 ? ` (${details.join(', ')})` : '';
+            spinner.stopAndPersist({ symbol: chalk.green('✓'), text: chalk.green(`${step.displayName}${suffix}`) });
+            this.stepSpinners.delete(step.id);
+        }
+        this.stderrBuffers.delete(step.id);
+    }
+    handleStepWouldRun(step) {
+        const spinner = this.stepSpinners.get(step.id);
+        if (spinner) {
+            spinner.stopAndPersist({ symbol: chalk.yellow('○'), text: chalk.yellow(`${step.displayName} (would run)`) });
+            this.stepSpinners.delete(step.id);
+        }
+        else {
+            console.log(`  ${chalk.yellow('○')} ${chalk.yellow(`${step.displayName} (would run)`)}`);
+        }
+    }
+    handlePipelineFinished(meta) {
+        const parts = ['Pipeline completed'];
+        if (typeof meta?.totalArtifactSize === 'number' && meta.totalArtifactSize > 0) {
+            parts.push(`(${formatSize(meta.totalArtifactSize)})`);
+        }
+        console.log(chalk.bold.green(`\n✓ ${parts.join(' ')}\n`));
+    }
     handleStepFailed(step, meta) {
         const spinner = this.stepSpinners.get(step.id);
         const exitCode = meta?.exitCode;

package/dist/cli/state.js

@@ -5,24 +5,24 @@ import { createHash } from 'node:crypto';
  * Manages caching state for pipeline execution.
  *
  * The StateManager computes fingerprints for steps and tracks which
- * artifact was produced by each step. This enables cache hits when
+ * run was produced by each step. This enables cache hits when
  * a step's configuration hasn't changed.
  *
  * ## Fingerprint Algorithm
  *
  * A step fingerprint is computed as:
  * ```
- * SHA256(image + JSON(cmd) + JSON(sorted env) + JSON(sorted inputArtifactIds))
+ * SHA256(image + JSON(cmd) + JSON(sorted env) + JSON(sorted inputRunIds))
  * ```
  *
  * A step is re-executed when:
  * - The fingerprint changes (image, cmd, env, or inputs modified)
- * - The artifact no longer exists (manually deleted)
+ * - The run no longer exists (manually deleted)
  *
  * ## Cache Propagation
  *
  * Changes propagate through dependencies. If step A is modified,
- * all steps depending on A are invalidated automatically (via inputArtifactIds).
+ * all steps depending on A are invalidated automatically (via inputRunIds).
  */
 export class StateManager {
     static fingerprint(config) {
@@ -32,8 +32,8 @@ export class StateManager {
         if (config.env) {
             hash.update(JSON.stringify(Object.entries(config.env).sort((a, b) => a[0].localeCompare(b[0]))));
         }
-        if (config.inputArtifactIds) {
-            hash.update(JSON.stringify([...config.inputArtifactIds].sort((a, b) => a.localeCompare(b))));
+        if (config.inputRunIds) {
+            hash.update(JSON.stringify([...config.inputRunIds].sort((a, b) => a.localeCompare(b))));
         }
         if (config.mounts && config.mounts.length > 0) {
             const sorted = [...config.mounts].sort((a, b) => a.containerPath.localeCompare(b.containerPath));
@@ -77,13 +77,26 @@ export class StateManager {
     getStep(stepId) {
         return this.state.steps[stepId];
     }
+    /**
+     * Lists all steps with their run IDs (in insertion order).
+     * @returns Array of {stepId, runId} entries
+     */
+    listSteps() {
+        return Object.entries(this.state.steps).map(([stepId, { runId }]) => ({ stepId, runId }));
+    }
+    /**
+     * Returns the set of run IDs currently referenced by state.
+     */
+    activeRunIds() {
+        return new Set(Object.values(this.state.steps).map(s => s.runId));
+    }
     /**
      * Updates cached state for a step.
      * @param stepId - Step identifier
-     * @param artifactId - Artifact produced by the step
+     * @param runId - Run produced by the step
      * @param fingerprint - Step configuration fingerprint
      */
-    setStep(stepId, artifactId, fingerprint) {
-        this.state.steps[stepId] = { artifactId, fingerprint };
+    setStep(stepId, runId, fingerprint) {
+        this.state.steps[stepId] = { runId, fingerprint };
     }
 }

package/dist/cli/utils.js

@@ -0,0 +1,49 @@
+import { readdir, stat } from 'node:fs/promises';
+import { join } from 'node:path';
+export function getGlobalOptions(cmd) {
+    return cmd.optsWithGlobals();
+}
+export async function dirSize(dirPath) {
+    let total = 0;
+    try {
+        const entries = await readdir(dirPath, { withFileTypes: true });
+        for (const entry of entries) {
+            const fullPath = join(dirPath, entry.name);
+            if (entry.isDirectory()) {
+                total += await dirSize(fullPath);
+            }
+            else if (entry.isFile()) {
+                const s = await stat(fullPath);
+                total += s.size;
+            }
+        }
+    }
+    catch {
+        // Directory doesn't exist or isn't readable
+    }
+    return total;
+}
+export function formatSize(bytes) {
+    if (bytes < 1024) {
+        return `${bytes} B`;
+    }
+    if (bytes < 1024 * 1024) {
+        return `${(bytes / 1024).toFixed(1)} KB`;
+    }
+    if (bytes < 1024 * 1024 * 1024) {
+        return `${(bytes / (1024 * 1024)).toFixed(1)} MB`;
+    }
+    return `${(bytes / (1024 * 1024 * 1024)).toFixed(1)} GB`;
+}
+export function formatDuration(ms) {
+    if (ms < 1000) {
+        return `${ms}ms`;
+    }
+    const seconds = ms / 1000;
+    if (seconds < 60) {
+        return `${seconds.toFixed(1)}s`;
+    }
+    const minutes = Math.floor(seconds / 60);
+    const remainingSeconds = Math.round(seconds % 60);
+    return `${minutes}m ${remainingSeconds}s`;
+}

package/dist/engine/docker-executor.js

@@ -26,20 +26,38 @@ export class DockerCliExecutor extends ContainerExecutor {
             throw new Error('Docker CLI not found. Please install Docker.');
         }
     }
+    /**
+     * Remove any leftover pipex containers for the given workspace.
+     * Called before pipeline execution to clean up after crashes.
+     */
+    async cleanupContainers(workspaceId) {
+        try {
+            const { stdout } = await execa('docker', [
+                'ps', '-a', '--filter', `label=pipex.workspace=${workspaceId}`, '-q'
+            ], { env: this.env });
+            const ids = stdout.trim().split('\n').filter(Boolean);
+            if (ids.length > 0) {
+                await execa('docker', ['rm', '-f', ...ids], { env: this.env, reject: false });
+            }
+        }
+        catch {
+            // Best effort
+        }
+    }
     async run(workspace, request, onLogLine) {
         const startedAt = new Date();
         // Use create+start instead of run: docker run cannot create mountpoints
         // for anonymous volumes inside read-only bind mounts (shadow paths).
         // docker create sets up the filesystem layer before readonly applies.
-        const args = ['create', '--name', request.name, '--network', request.network];
+        const args = ['create', '--name', request.name, '--network', request.network, '--label', 'pipex=true', '--label', `pipex.workspace=${workspace.id}`];
         if (request.env) {
             for (const [key, value] of Object.entries(request.env)) {
                 args.push('-e', `${key}=${value}`);
             }
         }
-        // Mount inputs (committed artifacts, read-only)
+        // Mount inputs (committed run artifacts, read-only)
         for (const input of request.inputs) {
-            const hostPath = workspace.artifactPath(input.artifactId);
+            const hostPath = workspace.runArtifactsPath(input.runId);
             args.push('-v', `${hostPath}:${input.containerPath}:ro`);
         }
         // Mount caches (persistent, read-write)
@@ -55,8 +73,8 @@ export class DockerCliExecutor extends ContainerExecutor {
                 args.push('-v', `${mount.hostPath}:${mount.containerPath}:ro`);
             }
         }
-        // Mount output (staging artifact, read-write)
-        const outputHostPath = workspace.stagingPath(request.output.stagingArtifactId);
+        // Mount output (staging run artifacts, read-write)
+        const outputHostPath = workspace.runStagingArtifactsPath(request.output.stagingRunId);
         args.push('-v', `${outputHostPath}:${request.output.containerPath}:rw`, request.image, ...request.cmd);
         let exitCode = 0;
         let error;