@livingdata/pipex 0.0.9 → 0.0.10

package/dist/cli/dag.js

@@ -0,0 +1,143 @@
+import { CyclicDependencyError, ValidationError } from '../errors.js';
+/** Build a dependency graph from resolved steps. */
+export function buildGraph(steps) {
+    const graph = new Map();
+    for (const step of steps) {
+        const deps = new Set();
+        if (step.inputs) {
+            for (const input of step.inputs) {
+                deps.add(input.step);
+            }
+        }
+        graph.set(step.id, deps);
+    }
+    return graph;
+}
+/** Validate graph: check for missing refs (non-optional) and cycles. */
+export function validateGraph(graph, steps) {
+    validateReferences(graph, steps);
+    detectCycles(graph);
+}
+function validateReferences(graph, steps) {
+    const optionalInputs = new Set();
+    for (const step of steps) {
+        if (step.inputs) {
+            for (const input of step.inputs) {
+                if (input.optional) {
+                    optionalInputs.add(`${step.id}:${input.step}`);
+                }
+            }
+        }
+    }
+    for (const [stepId, deps] of graph) {
+        for (const dep of deps) {
+            if (!graph.has(dep) && !optionalInputs.has(`${stepId}:${dep}`)) {
+                throw new ValidationError(`Step '${stepId}' references unknown step '${dep}'`);
+            }
+        }
+    }
+}
+function detectCycles(graph) {
+    const inDeg = computeInDegree(graph);
+    const queue = [];
+    for (const [id, deg] of inDeg) {
+        if (deg === 0) {
+            queue.push(id);
+        }
+    }
+    let processed = 0;
+    while (queue.length > 0) {
+        const current = queue.shift();
+        processed++;
+        for (const [id, deps] of graph) {
+            if (deps.has(current)) {
+                const newDeg = inDeg.get(id) - 1;
+                inDeg.set(id, newDeg);
+                if (newDeg === 0) {
+                    queue.push(id);
+                }
+            }
+        }
+    }
+    if (processed < graph.size) {
+        throw new CyclicDependencyError('Pipeline contains a dependency cycle');
+    }
+}
+/** Compute in-degree for each node (number of existing deps). */
+function computeInDegree(graph) {
+    const inDeg = new Map();
+    for (const [id, deps] of graph) {
+        let count = 0;
+        for (const dep of deps) {
+            if (graph.has(dep)) {
+                count++;
+            }
+        }
+        inDeg.set(id, count);
+    }
+    return inDeg;
+}
+/** Return steps grouped by topological level (parallelizable groups). */
+export function topologicalLevels(graph) {
+    const inDeg = computeInDegree(graph);
+    const levels = [];
+    const remaining = new Set(graph.keys());
+    while (remaining.size > 0) {
+        const level = [];
+        for (const id of remaining) {
+            if (inDeg.get(id) === 0) {
+                level.push(id);
+            }
+        }
+        if (level.length === 0) {
+            break; // Cycle — should not happen after validateGraph
+        }
+        levels.push(level);
+        for (const id of level) {
+            remaining.delete(id);
+            for (const [nodeId, deps] of graph) {
+                if (deps.has(id) && remaining.has(nodeId)) {
+                    inDeg.set(nodeId, inDeg.get(nodeId) - 1);
+                }
+            }
+        }
+    }
+    return levels;
+}
+/** BFS backward from targets to collect all ancestors + targets. */
+export function subgraph(graph, targets) {
+    const result = new Set();
+    const queue = [...targets];
+    while (queue.length > 0) {
+        const current = queue.shift();
+        if (result.has(current)) {
+            continue;
+        }
+        result.add(current);
+        const deps = graph.get(current);
+        if (deps) {
+            for (const dep of deps) {
+                if (!result.has(dep)) {
+                    queue.push(dep);
+                }
+            }
+        }
+    }
+    return result;
+}
+/** Return steps that no other step depends on (leaf/terminal nodes). */
+export function leafNodes(graph) {
+    const depended = new Set();
+    for (const deps of graph.values()) {
+        for (const dep of deps) {
+            depended.add(dep);
+        }
+    }
+    const leaves = [];
+    for (const id of graph.keys()) {
+        if (!depended.has(id)) {
+            leaves.push(id);
+        }
+    }
+    return leaves;
+}

package/dist/cli/index.js

@@ -11,6 +11,9 @@ import { registerPruneCommand } from './commands/prune.js';
 import { registerListCommand } from './commands/list.js';
 import { registerRmCommand } from './commands/rm.js';
 import { registerCleanCommand } from './commands/clean.js';
+import { registerExecCommand } from './commands/exec.js';
+import { registerCatCommand } from './commands/cat.js';
+import { registerRmStepCommand } from './commands/rm-step.js';
 async function main() {
     const program = new Command();
     program
@@ -28,6 +31,9 @@ async function main() {
     registerListCommand(program);
     registerRmCommand(program);
     registerCleanCommand(program);
+    registerExecCommand(program);
+    registerCatCommand(program);
+    registerRmStepCommand(program);
     await program.parseAsync();
 }
 try {

package/dist/cli/interactive-reporter.js

@@ -0,0 +1,227 @@
+import process from 'node:process';
+import { createLogUpdate } from 'log-update';
+import chalk from 'chalk';
+import { formatDuration, formatSize } from '../core/utils.js';
+const spinnerFrames = ['⠋', '⠙', '⠹', '⠸', '⠼', '⠴', '⠦', '⠧', '⠇', '⠏'];
+/**
+ * Reporter with interactive terminal UI using log-update for multi-step display.
+ * Suitable for local development and manual execution.
+ */
+export class InteractiveReporter {
+    static get maxStderrLines() {
+        return 20;
+    }
+    verbose;
+    logUpdate = createLogUpdate(process.stderr);
+    steps = new Map();
+    stderrBuffers = new Map();
+    frame = 0;
+    timer;
+    constructor(options) {
+        this.verbose = options?.verbose ?? false;
+    }
+    emit(event) {
+        switch (event.event) {
+            case 'PIPELINE_START': {
+                console.error(chalk.bold(`\n▶ Pipeline: ${chalk.cyan(event.pipelineName)}\n`));
+                for (const step of event.steps) {
+                    this.steps.set(step.id, { displayName: step.displayName, status: 'pending' });
+                }
+                this.startRendering();
+                break;
+            }
+            case 'STEP_STARTING': {
+                const step = this.steps.get(event.step.id);
+                if (step) {
+                    step.status = 'running';
+                }
+                else {
+                    this.steps.set(event.step.id, { displayName: event.step.displayName, status: 'running' });
+                    this.startRendering();
+                }
+                break;
+            }
+            case 'STEP_SKIPPED': {
+                const reasonLabel = event.reason === 'cached'
+                    ? '(cached)'
+                    : (event.reason === 'condition'
+                        ? '(condition)'
+                        : '(dependency skipped)');
+                const step = this.steps.get(event.step.id);
+                if (step) {
+                    step.status = 'skipped';
+                    step.detail = ` ${reasonLabel}`;
+                }
+                else {
+                    this.steps.set(event.step.id, { displayName: event.step.displayName, status: 'skipped', detail: ` ${reasonLabel}` });
+                }
+                break;
+            }
+            case 'STEP_FINISHED': {
+                this.handleStepFinished(event);
+                break;
+            }
+            case 'STEP_FAILED': {
+                this.handleStepFailed(event);
+                break;
+            }
+            case 'STEP_RETRYING': {
+                const step = this.steps.get(event.step.id);
+                if (step) {
+                    step.displayName = `${event.step.displayName} (retry ${event.attempt}/${event.maxRetries})`;
+                }
+                break;
+            }
+            case 'STEP_WOULD_RUN': {
+                const step = this.steps.get(event.step.id);
+                if (step) {
+                    step.status = 'would-run';
+                }
+                break;
+            }
+            case 'PIPELINE_FINISHED': {
+                this.stopRendering();
+                const parts = ['Pipeline completed'];
+                if (event.totalArtifactSize > 0) {
+                    parts.push(`(${formatSize(event.totalArtifactSize)})`);
+                }
+                console.error(chalk.bold.green(`\n✓ ${parts.join(' ')}\n`));
+                break;
+            }
+            case 'PIPELINE_FAILED': {
+                this.stopRendering();
+                this.printFailedStderr();
+                console.error(chalk.bold.red('\n✗ Pipeline failed\n'));
+                break;
+            }
+            case 'STEP_LOG': {
+                if (this.verbose) {
+                    const prefix = chalk.gray(`  [${event.step.id}]`);
+                    this.logUpdate.clear();
+                    console.error(`${prefix} ${event.line}`);
+                    this.render();
+                }
+                if (event.stream === 'stderr') {
+                    let buffer = this.stderrBuffers.get(event.step.id);
+                    if (!buffer) {
+                        buffer = [];
+                        this.stderrBuffers.set(event.step.id, buffer);
+                    }
+                    buffer.push(event.line);
+                    if (buffer.length > InteractiveReporter.maxStderrLines) {
+                        buffer.shift();
+                    }
+                }
+                break;
+            }
+        }
+    }
+    render() {
+        const lines = [];
+        for (const step of this.steps.values()) {
+            const symbol = this.symbolFor(step);
+            const text = this.textFor(step);
+            lines.push(`  ${symbol} ${text}`);
+        }
+        this.logUpdate(lines.join('\n'));
+        this.frame++;
+    }
+    symbolFor(step) {
+        switch (step.status) {
+            case 'pending': {
+                return chalk.gray('○');
+            }
+            case 'running': {
+                return chalk.cyan(spinnerFrames[this.frame % spinnerFrames.length]);
+            }
+            case 'done': {
+                return chalk.green('✓');
+            }
+            case 'skipped': {
+                return chalk.gray('⊙');
+            }
+            case 'failed': {
+                return chalk.red('✗');
+            }
+            case 'would-run': {
+                return chalk.yellow('○');
+            }
+        }
+    }
+    textFor(step) {
+        const suffix = step.detail ?? '';
+        switch (step.status) {
+            case 'pending': {
+                return chalk.gray(step.displayName);
+            }
+            case 'running': {
+                return step.displayName;
+            }
+            case 'done': {
+                return chalk.green(`${step.displayName}${suffix}`);
+            }
+            case 'skipped': {
+                return chalk.gray(`${step.displayName}${suffix}`);
+            }
+            case 'failed': {
+                return chalk.red(`${step.displayName}${suffix}`);
+            }
+            case 'would-run': {
+                return chalk.yellow(`${step.displayName} (would run)`);
+            }
+        }
+    }
+    startRendering() {
+        if (!this.timer) {
+            this.render();
+            this.timer = setInterval(() => {
+                this.render();
+            }, 80);
+        }
+    }
+    stopRendering() {
+        if (this.timer) {
+            clearInterval(this.timer);
+            this.timer = undefined;
+        }
+        this.render();
+        this.logUpdate.done();
+    }
+    handleStepFinished(event) {
+        const step = this.steps.get(event.step.id);
+        if (step) {
+            step.status = 'done';
+            const details = [];
+            if (typeof event.durationMs === 'number') {
+                details.push(formatDuration(event.durationMs));
+            }
+            if (typeof event.artifactSize === 'number' && event.artifactSize > 0) {
+                details.push(formatSize(event.artifactSize));
+            }
+            if (details.length > 0) {
+                step.detail = ` (${details.join(', ')})`;
+            }
+        }
+        this.stderrBuffers.delete(event.step.id);
+    }
+    handleStepFailed(event) {
+        const step = this.steps.get(event.step.id);
+        if (step) {
+            step.status = 'failed';
+            step.detail = ` (exit ${event.exitCode})`;
+        }
+    }
+    printFailedStderr() {
+        for (const [stepId, step] of this.steps) {
+            if (step.status === 'failed') {
+                const stderr = this.stderrBuffers.get(stepId);
+                if (stderr?.length) {
+                    console.error(chalk.red(`  ── ${step.displayName} stderr ──`));
+                    for (const line of stderr) {
+                        console.error(chalk.red(`  ${line}`));
+                    }
+                }
+            }
+        }
+    }
+}

package/dist/cli/pipeline-loader.js

@@ -2,8 +2,9 @@ import { readFile } from 'node:fs/promises';
 import { extname } from 'node:path';
 import { deburr } from 'lodash-es';
 import { parse as parseYaml } from 'yaml';
-import { getKit } from '../kits/index.js';
-import { isKitStep } from '../types.js';
+import { ValidationError } from '../errors.js';
+import { buildGraph, validateGraph } from './dag.js';
+import { resolveStep, validateStep } from './step-resolver.js';
 export class PipelineLoader {
     async load(filePath) {
         const content = await readFile(filePath, 'utf8');
@@ -12,127 +13,26 @@ export class PipelineLoader {
     parse(content, filePath) {
         const input = parsePipelineFile(content, filePath);
         if (!input.id && !input.name) {
-            throw new Error('Invalid pipeline: at least one of "id" or "name" must be defined');
+            throw new ValidationError('Invalid pipeline: at least one of "id" or "name" must be defined');
         }
         const pipelineId = input.id ?? slugify(input.name);
         if (!Array.isArray(input.steps) || input.steps.length === 0) {
-            throw new Error('Invalid pipeline: steps must be a non-empty array');
+            throw new ValidationError('Invalid pipeline: steps must be a non-empty array');
         }
-        const steps = input.steps.map(step => this.resolveStep(step));
+        const steps = input.steps.map(step => resolveStep(step));
         for (const step of steps) {
-            this.validateStep(step);
+            validateStep(step);
         }
         this.validateUniqueStepIds(steps);
+        const graph = buildGraph(steps);
+        validateGraph(graph, steps);
         return { id: pipelineId, name: input.name, steps };
     }
-    resolveStep(step) {
-        if (!step.id && !step.name) {
-            throw new Error('Invalid step: at least one of "id" or "name" must be defined');
-        }
-        const id = step.id ?? slugify(step.name);
-        const { name } = step;
-        if (!isKitStep(step)) {
-            return { ...step, id, name };
-        }
-        return this.resolveKitStep(step, id, name);
-    }
-    resolveKitStep(step, id, name) {
-        const kit = getKit(step.uses);
-        const kitOutput = kit.resolve(step.with ?? {});
-        return {
-            id,
-            name,
-            image: kitOutput.image,
-            cmd: kitOutput.cmd,
-            env: mergeEnv(kitOutput.env, step.env),
-            inputs: step.inputs,
-            outputPath: step.outputPath,
-            caches: mergeCaches(kitOutput.caches, step.caches),
-            mounts: mergeMounts(kitOutput.mounts, step.mounts),
-            sources: mergeMounts(kitOutput.sources, step.sources),
-            timeoutSec: step.timeoutSec,
-            allowFailure: step.allowFailure,
-            allowNetwork: step.allowNetwork ?? kitOutput.allowNetwork
-        };
-    }
-    validateStep(step) {
-        this.validateIdentifier(step.id, 'step id');
-        if (!step.image || typeof step.image !== 'string') {
-            throw new Error(`Invalid step ${step.id}: image is required`);
-        }
-        if (!Array.isArray(step.cmd) || step.cmd.length === 0) {
-            throw new Error(`Invalid step ${step.id}: cmd must be a non-empty array`);
-        }
-        if (step.inputs) {
-            for (const input of step.inputs) {
-                this.validateIdentifier(input.step, `input step name in step ${step.id}`);
-            }
-        }
-        if (step.mounts) {
-            this.validateMounts(step.id, step.mounts);
-        }
-        if (step.sources) {
-            this.validateMounts(step.id, step.sources);
-        }
-        if (step.caches) {
-            this.validateCaches(step.id, step.caches);
-        }
-    }
-    validateMounts(stepId, mounts) {
-        if (!Array.isArray(mounts)) {
-            throw new TypeError(`Step ${stepId}: mounts must be an array`);
-        }
-        for (const mount of mounts) {
-            if (!mount.host || typeof mount.host !== 'string') {
-                throw new Error(`Step ${stepId}: mount.host is required and must be a string`);
-            }
-            if (mount.host.startsWith('/')) {
-                throw new Error(`Step ${stepId}: mount.host '${mount.host}' must be a relative path`);
-            }
-            if (mount.host.includes('..')) {
-                throw new Error(`Step ${stepId}: mount.host '${mount.host}' must not contain '..'`);
-            }
-            if (!mount.container || typeof mount.container !== 'string') {
-                throw new Error(`Step ${stepId}: mount.container is required and must be a string`);
-            }
-            if (!mount.container.startsWith('/')) {
-                throw new Error(`Step ${stepId}: mount.container '${mount.container}' must be an absolute path`);
-            }
-            if (mount.container.includes('..')) {
-                throw new Error(`Step ${stepId}: mount.container '${mount.container}' must not contain '..'`);
-            }
-        }
-    }
-    validateCaches(stepId, caches) {
-        if (!Array.isArray(caches)) {
-            throw new TypeError(`Step ${stepId}: caches must be an array`);
-        }
-        for (const cache of caches) {
-            if (!cache.name || typeof cache.name !== 'string') {
-                throw new Error(`Step ${stepId}: cache.name is required and must be a string`);
-            }
-            this.validateIdentifier(cache.name, `cache name in step ${stepId}`);
-            if (!cache.path || typeof cache.path !== 'string') {
-                throw new Error(`Step ${stepId}: cache.path is required and must be a string`);
-            }
-            if (!cache.path.startsWith('/')) {
-                throw new Error(`Step ${stepId}: cache.path '${cache.path}' must be an absolute path`);
-            }
-        }
-    }
-    validateIdentifier(id, context) {
-        if (!/^[\w-]+$/.test(id)) {
-            throw new Error(`Invalid ${context}: '${id}' must contain only alphanumeric characters, underscore, and hyphen`);
-        }
-        if (id.includes('..')) {
-            throw new Error(`Invalid ${context}: '${id}' cannot contain '..'`);
-        }
-    }
     validateUniqueStepIds(steps) {
         const seen = new Set();
         for (const step of steps) {
             if (seen.has(step.id)) {
-                throw new Error(`Duplicate step id: '${step.id}'`);
+                throw new ValidationError(`Duplicate step id: '${step.id}'`);
             }
             seen.add(step.id);
         }