@livingdata/pipex 0.0.1

package/dist/cli/reporter.js

@@ -0,0 +1,78 @@
+import pino from 'pino';
+import chalk from 'chalk';
+import ora from 'ora';
+/**
+ * Reporter that outputs structured JSON logs via pino.
+ * Suitable for CI/CD environments and log aggregation.
+ */
+export class ConsoleReporter {
+    logger = pino({ level: 'info' });
+    state(workspaceId, event, stepId, meta) {
+        this.logger.info({ workspaceId, event, stepId, ...meta });
+    }
+    log(workspaceId, stepId, stream, line) {
+        this.logger.info({ workspaceId, stepId, stream, line });
+    }
+    result(workspaceId, stepId, result) {
+        this.logger.info({ workspaceId, stepId, result });
+    }
+}
+/**
+ * Reporter with interactive terminal UI using spinners and colors.
+ * Suitable for local development and manual execution.
+ */
+export class InteractiveReporter {
+    spinner;
+    stepSpinners = new Map();
+    state(workspaceId, event, stepId, meta) {
+        if (event === 'PIPELINE_START') {
+            console.log(chalk.bold(`\n▶ Pipeline: ${chalk.cyan(workspaceId)}\n`));
+        }
+        if (event === 'STEP_STARTING' && stepId) {
+            const spinner = ora({ text: stepId, prefixText: '  ' }).start();
+            this.stepSpinners.set(stepId, spinner);
+        }
+        if (event === 'STEP_SKIPPED' && stepId) {
+            const spinner = this.stepSpinners.get(stepId);
+            if (spinner) {
+                spinner.stopAndPersist({ symbol: chalk.gray('⊙'), text: chalk.gray(`${stepId} (cached)`) });
+                this.stepSpinners.delete(stepId);
+            }
+            else {
+                // Step was skipped before spinner was created
+                console.log(`  ${chalk.gray('⊙')} ${chalk.gray(`${stepId} (cached)`)}`);
+            }
+        }
+        if (event === 'STEP_FINISHED' && stepId) {
+            const spinner = this.stepSpinners.get(stepId);
+            if (spinner) {
+                spinner.stopAndPersist({ symbol: chalk.green('✓'), text: chalk.green(stepId) });
+                this.stepSpinners.delete(stepId);
+            }
+        }
+        if (event === 'STEP_FAILED' && stepId) {
+            const spinner = this.stepSpinners.get(stepId);
+            const exitCode = meta?.exitCode;
+            if (spinner) {
+                const exitInfo = exitCode === undefined ? '' : ` (exit ${exitCode})`;
+                spinner.stopAndPersist({
+                    symbol: chalk.red('✗'),
+                    text: chalk.red(`${stepId}${exitInfo}`)
+                });
+                this.stepSpinners.delete(stepId);
+            }
+        }
+        if (event === 'PIPELINE_FINISHED') {
+            console.log(chalk.bold.green('\n✓ Pipeline completed\n'));
+        }
+        if (event === 'PIPELINE_FAILED') {
+            console.log(chalk.bold.red('\n✗ Pipeline failed\n'));
+        }
+    }
+    log(_workspaceId, _stepId, _stream, _line) {
+        // Suppress logs in interactive mode
+    }
+    result(_workspaceId, _stepId, _result) {
+        // Results shown via state updates
+    }
+}

package/dist/cli/state.js

@@ -0,0 +1,89 @@
+import { readFile, writeFile } from 'node:fs/promises';
+import { join } from 'node:path';
+import { createHash } from 'node:crypto';
+/**
+ * Manages caching state for pipeline execution.
+ *
+ * The StateManager computes fingerprints for steps and tracks which
+ * artifact was produced by each step. This enables cache hits when
+ * a step's configuration hasn't changed.
+ *
+ * ## Fingerprint Algorithm
+ *
+ * A step fingerprint is computed as:
+ * ```
+ * SHA256(image + JSON(cmd) + JSON(sorted env) + JSON(sorted inputArtifactIds))
+ * ```
+ *
+ * A step is re-executed when:
+ * - The fingerprint changes (image, cmd, env, or inputs modified)
+ * - The artifact no longer exists (manually deleted)
+ *
+ * ## Cache Propagation
+ *
+ * Changes propagate through dependencies. If step A is modified,
+ * all steps depending on A are invalidated automatically (via inputArtifactIds).
+ */
+export class StateManager {
+    static fingerprint(config) {
+        const hash = createHash('sha256');
+        hash.update(config.image);
+        hash.update(JSON.stringify(config.cmd));
+        if (config.env) {
+            hash.update(JSON.stringify(Object.entries(config.env).sort((a, b) => a[0].localeCompare(b[0]))));
+        }
+        if (config.inputArtifactIds) {
+            hash.update(JSON.stringify([...config.inputArtifactIds].sort((a, b) => a.localeCompare(b))));
+        }
+        if (config.mounts && config.mounts.length > 0) {
+            const sorted = [...config.mounts].sort((a, b) => a.containerPath.localeCompare(b.containerPath));
+            hash.update(JSON.stringify(sorted));
+        }
+        return hash.digest('hex');
+    }
+    state = { steps: {} };
+    path;
+    /**
+     * Creates a state manager for the given workspace.
+     * @param workspaceRoot - Absolute path to workspace directory
+     */
+    constructor(workspaceRoot) {
+        this.path = join(workspaceRoot, 'state.json');
+    }
+    /**
+     * Loads cached state from state.json.
+     * If the file doesn't exist, initializes with empty state.
+     */
+    async load() {
+        try {
+            const content = await readFile(this.path, 'utf8');
+            Object.assign(this.state, JSON.parse(content));
+        }
+        catch {
+            this.state.steps = {};
+        }
+    }
+    /**
+     * Persists current state to state.json.
+     */
+    async save() {
+        await writeFile(this.path, JSON.stringify(this.state, null, 2), 'utf8');
+    }
+    /**
+     * Retrieves cached state for a step.
+     * @param stepId - Step identifier
+     * @returns Cached state if available, undefined otherwise
+     */
+    getStep(stepId) {
+        return this.state.steps[stepId];
+    }
+    /**
+     * Updates cached state for a step.
+     * @param stepId - Step identifier
+     * @param artifactId - Artifact produced by the step
+     * @param fingerprint - Step configuration fingerprint
+     */
+    setStep(stepId, artifactId, fingerprint) {
+        this.state.steps[stepId] = { artifactId, fingerprint };
+    }
+}

package/dist/cli/types.js

@@ -0,0 +1 @@
+export {};

package/dist/engine/docker-executor.js

@@ -0,0 +1,96 @@
+import process from 'node:process';
+import { execa } from 'execa';
+import { createInterface } from 'node:readline';
+import { ContainerExecutor } from './executor.js';
+/**
+ * Build a minimal environment for the Docker CLI process.
+ * Only PATH, HOME, and DOCKER_* are kept — everything else is stripped
+ * so that host secrets (API keys, tokens, credentials) never leak,
+ * even if a `-e KEY` (without value) were accidentally added.
+ */
+function dockerCliEnv() {
+    const env = {};
+    for (const [key, value] of Object.entries(process.env)) {
+        if (value !== undefined && (key === 'PATH' || key === 'HOME' || key.startsWith('DOCKER_'))) {
+            env[key] = value;
+        }
+    }
+    return env;
+}
+export class DockerCliExecutor extends ContainerExecutor {
+    env = dockerCliEnv();
+    async check() {
+        try {
+            await execa('docker', ['--version'], { env: this.env });
+        }
+        catch {
+            throw new Error('Docker CLI not found. Please install Docker.');
+        }
+    }
+    async run(workspace, request, onLogLine) {
+        const startedAt = new Date();
+        const args = ['run', '--name', request.name, '--network', request.network];
+        if (request.env) {
+            for (const [key, value] of Object.entries(request.env)) {
+                args.push('-e', `${key}=${value}`);
+            }
+        }
+        // Mount inputs (committed artifacts, read-only)
+        for (const input of request.inputs) {
+            const hostPath = workspace.artifactPath(input.artifactId);
+            args.push('-v', `${hostPath}:${input.containerPath}:ro`);
+        }
+        // Mount caches (persistent, read-write)
+        if (request.caches) {
+            for (const cache of request.caches) {
+                const hostPath = workspace.cachePath(cache.name);
+                args.push('-v', `${hostPath}:${cache.containerPath}:rw`);
+            }
+        }
+        // Mount host bind mounts (always read-only)
+        if (request.mounts) {
+            for (const mount of request.mounts) {
+                args.push('-v', `${mount.hostPath}:${mount.containerPath}:ro`);
+            }
+        }
+        // Mount output (staging artifact, read-write)
+        const outputHostPath = workspace.stagingPath(request.output.stagingArtifactId);
+        args.push('-v', `${outputHostPath}:${request.output.containerPath}:rw`, request.image, ...request.cmd);
+        let exitCode = 0;
+        let error;
+        try {
+            const proc = execa('docker', args, {
+                env: this.env,
+                reject: false,
+                timeout: request.timeoutSec ? request.timeoutSec * 1000 : undefined
+            });
+            if (proc.stdout) {
+                const rl = createInterface({ input: proc.stdout });
+                rl.on('line', line => {
+                    onLogLine({ stream: 'stdout', line });
+                });
+            }
+            if (proc.stderr) {
+                const rl = createInterface({ input: proc.stderr });
+                rl.on('line', line => {
+                    onLogLine({ stream: 'stderr', line });
+                });
+            }
+            const result = await proc;
+            exitCode = result.exitCode ?? 0;
+        }
+        catch (error_) {
+            exitCode = 1;
+            error = error_ instanceof Error ? error_.message : String(error_);
+        }
+        finally {
+            try {
+                await execa('docker', ['rm', '-f', request.name], { env: this.env, reject: false });
+            }
+            catch {
+                // Best effort cleanup
+            }
+        }
+        return { exitCode, startedAt, finishedAt: new Date(), error };
+    }
+}

package/dist/engine/docker-runtime.js

@@ -0,0 +1,65 @@
+import { execa } from 'execa';
+import { createInterface } from 'node:readline';
+import { ContainerRuntime } from './runtime.js';
+export class DockerCliRuntime extends ContainerRuntime {
+    async check() {
+        try {
+            await execa('docker', ['--version']);
+        }
+        catch {
+            throw new Error('Docker CLI not found. Please install Docker.');
+        }
+    }
+    async run(workspace, request, onLogLine) {
+        const startedAt = new Date();
+        const args = ['run', '--name', request.name, '--network', request.network];
+        if (request.env) {
+            for (const [key, value] of Object.entries(request.env)) {
+                args.push('-e', `${key}=${value}`);
+            }
+        }
+        // Mount inputs (committed artifacts, read-only)
+        for (const input of request.inputs) {
+            const hostPath = workspace.artifactPath(input.artifactId);
+            args.push('-v', `${hostPath}:${input.containerPath}:ro`);
+        }
+        // Mount output (staging artifact, read-write)
+        const outputHostPath = workspace.stagingPath(request.output.stagingArtifactId);
+        args.push('-v', `${outputHostPath}:${request.output.containerPath}:rw`, request.image, ...request.cmd);
+        let exitCode = 0;
+        let error;
+        try {
+            const proc = execa('docker', args, {
+                reject: false,
+                timeout: request.timeoutSec ? request.timeoutSec * 1000 : undefined
+            });
+            if (proc.stdout) {
+                const rl = createInterface({ input: proc.stdout });
+                rl.on('line', line => {
+                    onLogLine({ stream: 'stdout', line });
+                });
+            }
+            if (proc.stderr) {
+                const rl = createInterface({ input: proc.stderr });
+                rl.on('line', line => {
+                    onLogLine({ stream: 'stderr', line });
+                });
+            }
+            const result = await proc;
+            exitCode = result.exitCode ?? 0;
+        }
+        catch (error_) {
+            exitCode = 1;
+            error = error_ instanceof Error ? error_.message : String(error_);
+        }
+        finally {
+            try {
+                await execa('docker', ['rm', '-f', request.name], { reject: false });
+            }
+            catch {
+                // Best effort cleanup
+            }
+        }
+        return { exitCode, startedAt, finishedAt: new Date(), error };
+    }
+}

package/dist/engine/executor.js

@@ -0,0 +1,16 @@
+/**
+ * Abstract interface for executing containers.
+ *
+ * Implementations:
+ * - `DockerCliExecutor`: Uses Docker CLI
+ * - Future: PodmanExecutor, KubernetesExecutor, etc.
+ *
+ * The executor is responsible for:
+ * - Running containers with specified configuration
+ * - Mounting input artifacts as read-only volumes
+ * - Mounting output staging directory as read-write volume
+ * - Streaming logs in real-time
+ * - Cleaning up containers after execution
+ */
+export class ContainerExecutor {
+}

package/dist/engine/index.js

@@ -0,0 +1,3 @@
+export { Workspace } from './workspace.js';
+export { ContainerExecutor } from './executor.js';
+export { DockerCliExecutor } from './docker-executor.js';

package/dist/engine/runtime.js

@@ -0,0 +1,2 @@
+export class ContainerRuntime {
+}

package/dist/engine/types.js

@@ -0,0 +1 @@
+export {};

package/dist/engine/workspace.js

@@ -0,0 +1,264 @@
+import { access, mkdir, readdir, rename, rm } from 'node:fs/promises';
+import { randomUUID } from 'node:crypto';
+import { join } from 'node:path';
+/**
+ * Isolated execution environment for container runs.
+ *
+ * A workspace provides:
+ * - **staging/**: Temporary write location during execution
+ * - **artifacts/**: Committed outputs (immutable, read-only)
+ * - **caches/**: Persistent read-write caches (shared across steps)
+ * - **state.json**: Managed by orchestration layer (e.g., CLI) for caching
+ *
+ * ## Artifact Lifecycle
+ *
+ * 1. `prepareArtifact()` creates `staging/{artifactId}/`
+ * 2. Container writes to `staging/{artifactId}/` (mounted as `/output`)
+ * 3. Success: `commitArtifact()` atomically moves to `artifacts/{artifactId}/`
+ *    OR Failure: `discardArtifact()` deletes `staging/{artifactId}/`
+ *
+ * Artifacts are immutable once committed.
+ *
+ * ## Cache Lifecycle
+ *
+ * 1. `prepareCache()` creates `caches/{name}/` if it doesn't exist
+ * 2. Container writes/reads from `caches/{name}/` during execution
+ * 3. Cache persists across executions (never deleted automatically)
+ *
+ * Caches are mutable and shared between steps.
+ *
+ * @example
+ * ```typescript
+ * const ws = await Workspace.create('/tmp/workdir', 'my-workspace')
+ * const artifactId = ws.generateArtifactId()
+ * await ws.prepareArtifact(artifactId)
+ * await ws.prepareCache('pnpm-store')
+ * // ... container execution ...
+ * await ws.commitArtifact(artifactId) // On success
+ * // OR await ws.discardArtifact(artifactId) // On failure
+ * ```
+ */
+export class Workspace {
+    id;
+    root;
+    /**
+     * Generates a unique workspace identifier.
+     * @returns Workspace ID in format: `{timestamp}-{uuid-prefix}`
+     */
+    static generateWorkspaceId() {
+        return Workspace.generateId();
+    }
+    /**
+     * Creates a new workspace with staging, artifacts, and caches directories.
+     * @param workdirRoot - Root directory for all workspaces
+     * @param id - Optional workspace ID (auto-generated if omitted)
+     * @returns Newly created workspace
+     */
+    static async create(workdirRoot, id) {
+        const workspaceId = id ?? Workspace.generateWorkspaceId();
+        const root = join(workdirRoot, workspaceId);
+        await mkdir(join(root, 'staging'), { recursive: true });
+        await mkdir(join(root, 'artifacts'), { recursive: true });
+        await mkdir(join(root, 'caches'), { recursive: true });
+        return new Workspace(workspaceId, root);
+    }
+    /**
+     * Opens an existing workspace.
+     * @param workdirRoot - Root directory for all workspaces
+     * @param id - Workspace ID
+     * @returns Existing workspace
+     * @throws If workspace does not exist
+     */
+    static async open(workdirRoot, id) {
+        const root = join(workdirRoot, id);
+        await access(root);
+        return new Workspace(id, root);
+    }
+    /**
+     * Lists all workspace IDs under the given root directory.
+     * @param workdirRoot - Root directory for all workspaces
+     * @returns Sorted array of workspace names (directories)
+     */
+    static async list(workdirRoot) {
+        try {
+            const entries = await readdir(workdirRoot, { withFileTypes: true });
+            return entries.filter(e => e.isDirectory()).map(e => e.name).sort();
+        }
+        catch {
+            return [];
+        }
+    }
+    /**
+     * Removes a workspace directory.
+     * @param workdirRoot - Root directory for all workspaces
+     * @param id - Workspace ID to remove
+     * @throws If the workspace ID is invalid
+     */
+    static async remove(workdirRoot, id) {
+        if (id.includes('..') || id.includes('/')) {
+            throw new Error(`Invalid workspace ID: ${id}`);
+        }
+        await rm(join(workdirRoot, id), { recursive: true, force: true });
+    }
+    /**
+     * Generates a unique identifier using timestamp and UUID.
+     * @returns Unique ID in format: `{timestamp}-{uuid-prefix}`
+     * @internal
+     */
+    static generateId() {
+        return `${Date.now()}-${randomUUID().slice(0, 8)}`;
+    }
+    constructor(id, root) {
+        this.id = id;
+        this.root = root;
+    }
+    /**
+     * Generates a unique artifact identifier.
+     * @returns Artifact ID in format: `{timestamp}-{uuid-prefix}`
+     */
+    generateArtifactId() {
+        return Workspace.generateId();
+    }
+    /**
+     * Returns the staging directory path for an artifact.
+     * Staging is used for temporary writes during execution.
+     * @param artifactId - Artifact identifier
+     * @returns Absolute path to staging directory
+     * @throws If artifact ID is invalid
+     */
+    stagingPath(artifactId) {
+        this.validateArtifactId(artifactId);
+        return join(this.root, 'staging', artifactId);
+    }
+    /**
+     * Returns the committed artifact directory path.
+     * Artifacts are immutable once committed.
+     * @param artifactId - Artifact identifier
+     * @returns Absolute path to artifact directory
+     * @throws If artifact ID is invalid
+     */
+    artifactPath(artifactId) {
+        this.validateArtifactId(artifactId);
+        return join(this.root, 'artifacts', artifactId);
+    }
+    /**
+     * Prepares a staging directory for a new artifact.
+     * @param artifactId - Artifact identifier
+     * @returns Absolute path to the created staging directory
+     */
+    async prepareArtifact(artifactId) {
+        const path = this.stagingPath(artifactId);
+        await mkdir(path, { recursive: true });
+        return path;
+    }
+    /**
+     * Commits a staging artifact to the artifacts directory.
+     * Uses atomic rename operation for consistency.
+     * @param artifactId - Artifact identifier
+     */
+    async commitArtifact(artifactId) {
+        await rename(this.stagingPath(artifactId), this.artifactPath(artifactId));
+    }
+    /**
+     * Discards a staging artifact (on execution failure).
+     * @param artifactId - Artifact identifier
+     */
+    async discardArtifact(artifactId) {
+        await rm(this.stagingPath(artifactId), { recursive: true, force: true });
+    }
+    /**
+     * Removes all staging directories.
+     * Should be called on workspace initialization to clean up incomplete artifacts.
+     */
+    async cleanupStaging() {
+        const stagingDir = join(this.root, 'staging');
+        try {
+            const entries = await readdir(stagingDir, { withFileTypes: true });
+            for (const entry of entries) {
+                if (entry.isDirectory()) {
+                    await rm(join(stagingDir, entry.name), { recursive: true, force: true });
+                }
+            }
+        }
+        catch {
+            // Staging directory doesn't exist yet
+        }
+    }
+    /**
+     * Lists all committed artifact IDs in this workspace.
+     * @returns Array of artifact IDs (directory names in artifacts/)
+     */
+    async listArtifacts() {
+        try {
+            const entries = await readdir(join(this.root, 'artifacts'), { withFileTypes: true });
+            return entries.filter(e => e.isDirectory()).map(e => e.name);
+        }
+        catch {
+            return [];
+        }
+    }
+    /**
+     * Returns the cache directory path.
+     * Caches are persistent read-write directories shared across steps.
+     * @param cacheName - Cache identifier (e.g., "pnpm-store")
+     * @returns Absolute path to cache directory
+     * @throws If cache name is invalid
+     */
+    cachePath(cacheName) {
+        this.validateCacheName(cacheName);
+        return join(this.root, 'caches', cacheName);
+    }
+    /**
+     * Prepares a cache directory.
+     * Creates the directory if it doesn't exist.
+     * @param cacheName - Cache identifier
+     * @returns Absolute path to the cache directory
+     */
+    async prepareCache(cacheName) {
+        const path = this.cachePath(cacheName);
+        await mkdir(path, { recursive: true });
+        return path;
+    }
+    /**
+     * Lists all cache directories in the workspace.
+     * @returns Array of cache names
+     */
+    async listCaches() {
+        try {
+            const entries = await readdir(join(this.root, 'caches'), { withFileTypes: true });
+            return entries.filter(e => e.isDirectory()).map(e => e.name);
+        }
+        catch {
+            return [];
+        }
+    }
+    /**
+     * Validates an artifact ID to prevent path traversal attacks.
+     * @param id - Artifact identifier to validate
+     * @throws If the artifact ID contains invalid characters or path traversal attempts
+     * @internal
+     */
+    validateArtifactId(id) {
+        if (!/^[\w-]+$/.test(id)) {
+            throw new Error(`Invalid artifact ID: ${id}. Must contain only alphanumeric characters, dashes, and underscores.`);
+        }
+        if (id.includes('..')) {
+            throw new Error(`Invalid artifact ID: ${id}. Path traversal is not allowed.`);
+        }
+    }
+    /**
+     * Validates a cache name to prevent path traversal attacks.
+     * Same rules as artifact IDs: alphanumeric, dashes, underscores only.
+     * @param name - Cache name to validate
+     * @throws If the cache name contains invalid characters or path traversal attempts
+     * @internal
+     */
+    validateCacheName(name) {
+        if (!/^[\w-]+$/.test(name)) {
+            throw new Error(`Invalid cache name: ${name}. Must contain only alphanumeric characters, dashes, and underscores.`);
+        }
+        if (name.includes('..')) {
+            throw new Error(`Invalid cache name: ${name}. Path traversal is not allowed.`);
+        }
+    }
+}

package/dist/index.js

@@ -0,0 +1,40 @@
+/**
+ * Engine layer exports for programmatic use.
+ *
+ * The engine provides low-level primitives for container execution
+ * and artifact management, decoupled from pipeline orchestration.
+ *
+ * For CLI usage, see src/cli/index.ts
+ *
+ * @example
+ * ```typescript
+ * import {Workspace, DockerCliExecutor} from '@livingdata/pipex'
+ *
+ * const ws = await Workspace.create('/tmp/workdir', 'my-workspace')
+ * const executor = new DockerCliExecutor()
+ * await executor.check()
+ *
+ * // Prepare cache
+ * await ws.prepareCache('pnpm-store')
+ *
+ * const artifactId = ws.generateArtifactId()
+ * await ws.prepareArtifact(artifactId)
+ *
+ * await executor.run(ws, {
+ *   name: 'my-container',
+ *   image: 'node:20-alpine',
+ *   cmd: ['pnpm', 'install'],
+ *   inputs: [],
+ *   output: {stagingArtifactId: artifactId, containerPath: '/output'},
+ *   caches: [{
+ *     name: 'pnpm-store',
+ *     containerPath: '/root/.local/share/pnpm/store'
+ *   }],
+ *   network: 'none'
+ * }, (log) => console.log(log))
+ *
+ * await ws.commitArtifact(artifactId)
+ * ```
+ */
+// Export engine layer for programmatic use
+export { Workspace, ContainerExecutor, DockerCliExecutor } from './engine/index.js';

package/dist/reporter.js

@@ -0,0 +1,13 @@
+import pino from 'pino';
+export class ConsoleReporter {
+    logger = pino({ level: 'info' });
+    state(workspaceId, event, stepId, meta) {
+        this.logger.info({ workspaceId, event, stepId, ...meta });
+    }
+    log(workspaceId, stepId, stream, line) {
+        this.logger.info({ workspaceId, stepId, stream, line });
+    }
+    result(workspaceId, stepId, result) {
+        this.logger.info({ workspaceId, stepId, result });
+    }
+}