@livingdata/pipex 0.0.8 → 0.0.10

package/dist/cli/pipeline-runner.js

@@ -1,36 +1,17 @@
-import { cp } from 'node:fs/promises';
-import { dirname, resolve } from 'node:path';
+import process from 'node:process';
+import { cpus } from 'node:os';
+import { cp, writeFile } from 'node:fs/promises';
+import { setTimeout } from 'node:timers/promises';
+import { createWriteStream } from 'node:fs';
+import { dirname, join, resolve } from 'node:path';
 import { Workspace } from '../engine/index.js';
+import { ContainerCrashError, PipexError } from '../errors.js';
+import { buildGraph, topologicalLevels, subgraph, leafNodes } from './dag.js';
+import { evaluateCondition } from './condition.js';
 import { StateManager } from './state.js';
+import { dirSize } from './utils.js';
 /**
- * Orchestrates pipeline execution with dependency resolution and caching.
- *
- * ## Workflow
- *
- * 1. **Workspace Resolution**: Determines workspace ID from CLI flag, config, or filename
- * 2. **State Loading**: Loads cached fingerprints from state.json
- * 3. **Step Execution**: For each step:
- *    a. Computes fingerprint (image + cmd + env + input artifact IDs)
- *    b. Checks cache (fingerprint match + artifact exists)
- *    c. If cached: skips execution
- *    d. If not cached: resolves inputs, prepares staging, executes container
- *    e. On success: commits artifact, saves state
- *    f. On failure: discards artifact, halts pipeline (unless allowFailure)
- * 4. **Completion**: Reports final pipeline status
- *
- * ## Dependencies
- *
- * Steps declare dependencies via `inputs: [{step: "stepId"}]`.
- * The runner:
- * - Mounts input artifacts as read-only volumes
- * - Optionally copies inputs to output staging (if `copyToOutput: true`)
- * - Tracks execution order to resolve step names to artifact IDs
- *
- * ## Caching
- *
- * Cache invalidation is automatic:
- * - Changing a step's configuration re-runs it
- * - Re-running a step invalidates all dependent steps
+ * Orchestrates pipeline execution with DAG-based parallel execution and caching.
  */
 export class PipelineRunner {
     loader;
@@ -44,10 +25,9 @@ export class PipelineRunner {
         this.workdirRoot = workdirRoot;
     }
     async run(pipelineFilePath, options) {
-        const { workspace: workspaceName, force } = options ?? {};
+        const { workspace: workspaceName, force, dryRun, target, concurrency } = options ?? {};
         const config = await this.loader.load(pipelineFilePath);
         const pipelineRoot = dirname(resolve(pipelineFilePath));
-        // Workspace ID priority: CLI arg > pipeline id
         const workspaceId = workspaceName ?? config.id;
         let workspace;
         try {
@@ -57,128 +37,265 @@ export class PipelineRunner {
             workspace = await Workspace.create(this.workdirRoot, workspaceId);
         }
         await workspace.cleanupStaging();
-        await this.runtime.check();
+        if (!dryRun) {
+            await this.runtime.check();
+            await this.runtime.cleanupContainers(workspace.id);
+        }
         const state = new StateManager(workspace.root);
         await state.load();
-        const stepArtifacts = new Map();
-        this.reporter.state(workspace.id, 'PIPELINE_START', undefined, { pipelineName: config.name ?? config.id });
-        for (const step of config.steps) {
-            const stepRef = { id: step.id, displayName: step.name ?? step.id };
-            const inputArtifactIds = step.inputs
-                ?.map(i => stepArtifacts.get(i.step))
-                .filter((id) => id !== undefined);
-            const resolvedMounts = step.mounts?.map(m => ({
-                hostPath: resolve(pipelineRoot, m.host),
-                containerPath: m.container
-            }));
-            const currentFingerprint = StateManager.fingerprint({
-                image: step.image,
-                cmd: step.cmd,
-                env: step.env,
-                inputArtifactIds,
-                mounts: resolvedMounts
-            });
-            const skipCache = force === true || (Array.isArray(force) && force.includes(step.id));
-            if (!skipCache && await this.tryUseCache({ workspace, state, step, stepRef, currentFingerprint, stepArtifacts })) {
-                continue;
-            }
-            this.reporter.state(workspace.id, 'STEP_STARTING', stepRef);
-            const artifactId = workspace.generateArtifactId();
-            const stagingPath = await workspace.prepareArtifact(artifactId);
-            await this.prepareStagingWithInputs(workspace, step, stagingPath, stepArtifacts);
-            // Prepare caches
-            if (step.caches) {
-                for (const cache of step.caches) {
-                    await workspace.prepareCache(cache.name);
+        const stepRuns = new Map();
+        let totalArtifactSize = 0;
+        // Build DAG and determine execution scope
+        const graph = buildGraph(config.steps);
+        const targets = target ?? leafNodes(graph);
+        const activeSteps = subgraph(graph, targets);
+        this.reporter.emit({
+            event: 'PIPELINE_START',
+            workspaceId: workspace.id,
+            pipelineName: config.name ?? config.id,
+            steps: config.steps
+                .filter(s => activeSteps.has(s.id))
+                .map(s => ({ id: s.id, displayName: s.name ?? s.id }))
+        });
+        const levels = topologicalLevels(graph)
+            .map(level => level.filter(id => activeSteps.has(id)))
+            .filter(level => level.length > 0);
+        const stepMap = new Map(config.steps.map(s => [s.id, s]));
+        const failed = new Set();
+        const skipped = new Set();
+        const maxConcurrency = concurrency ?? cpus().length;
+        for (const level of levels) {
+            const tasks = level.map(stepId => async () => {
+                const step = stepMap.get(stepId);
+                const stepRef = { id: step.id, displayName: step.name ?? step.id };
+                // Check if blocked by a failed/skipped required dependency
+                if (this.isDependencyBlocked(step, failed, skipped)) {
+                    skipped.add(step.id);
+                    this.reporter.emit({ event: 'STEP_SKIPPED', workspaceId: workspace.id, step: stepRef, reason: 'dependency' });
+                    return 0;
                 }
-            }
-            const { inputs, output, caches, mounts } = this.buildMounts(step, artifactId, stepArtifacts, pipelineRoot);
-            const result = await this.runtime.run(workspace, {
-                name: `pipex-${workspace.id}-${step.id}-${Date.now()}`,
-                image: step.image,
-                cmd: step.cmd,
-                env: step.env,
-                inputs,
-                output,
-                caches,
-                mounts,
-                sources: step.sources?.map(m => ({
+                // Evaluate condition
+                if (step.if) {
+                    const conditionMet = await evaluateCondition(step.if, { env: process.env });
+                    if (!conditionMet) {
+                        skipped.add(step.id);
+                        this.reporter.emit({ event: 'STEP_SKIPPED', workspaceId: workspace.id, step: stepRef, reason: 'condition' });
+                        return 0;
+                    }
+                }
+                // Compute fingerprint
+                const inputRunIds = step.inputs
+                    ?.map(i => stepRuns.get(i.step))
+                    .filter((id) => id !== undefined);
+                const resolvedMounts = step.mounts?.map(m => ({
                     hostPath: resolve(pipelineRoot, m.host),
                     containerPath: m.container
-                })),
-                network: step.allowNetwork ? 'bridge' : 'none',
-                timeoutSec: step.timeoutSec
-            }, ({ stream, line }) => {
-                this.reporter.log(workspace.id, stepRef, stream, line);
+                }));
+                const currentFingerprint = StateManager.fingerprint({
+                    image: step.image,
+                    cmd: step.cmd,
+                    env: step.env,
+                    inputRunIds,
+                    mounts: resolvedMounts
+                });
+                // Cache check
+                const skipCache = force === true || (Array.isArray(force) && force.includes(step.id));
+                if (!skipCache && await this.tryUseCache({ workspace, state, step, stepRef, currentFingerprint, stepRuns })) {
+                    return 0;
+                }
+                // Dry run
+                if (dryRun) {
+                    this.reporter.emit({ event: 'STEP_WOULD_RUN', workspaceId: workspace.id, step: stepRef });
+                    return 0;
+                }
+                // Execute
+                this.reporter.emit({ event: 'STEP_STARTING', workspaceId: workspace.id, step: stepRef });
+                return this.executeStep({ workspace, state, step, stepRef, stepRuns, currentFingerprint, resolvedMounts, pipelineRoot });
             });
+            const results = await withConcurrency(tasks, maxConcurrency);
+            // Collect results
+            for (const [i, result] of results.entries()) {
+                const stepId = level[i];
+                if (result.status === 'fulfilled') {
+                    totalArtifactSize += result.value;
+                }
+                else if (!failed.has(stepId)) {
+                    // Step threw an error (ContainerCrashError if not allowFailure)
+                    failed.add(stepId);
+                }
+            }
+            await state.save();
+        }
+        if (failed.size > 0) {
+            this.reporter.emit({ event: 'PIPELINE_FAILED', workspaceId: workspace.id });
+            // Re-throw the first failure to signal pipeline failure to the CLI
+            const firstFailedId = [...failed][0];
+            throw new ContainerCrashError(firstFailedId, 1);
+        }
+        this.reporter.emit({ event: 'PIPELINE_FINISHED', workspaceId: workspace.id, totalArtifactSize });
+    }
+    isDependencyBlocked(step, failed, skippedSteps) {
+        if (!step.inputs) {
+            return false;
+        }
+        return step.inputs.some(input => !input.optional && (failed.has(input.step) || skippedSteps.has(input.step)));
+    }
+    async executeStep({ workspace, state, step, stepRef, stepRuns, currentFingerprint, resolvedMounts, pipelineRoot }) {
+        const runId = workspace.generateRunId();
+        const stagingPath = await workspace.prepareRun(runId);
+        await this.prepareStagingWithInputs(workspace, step, workspace.runStagingArtifactsPath(runId), stepRuns);
+        if (step.caches) {
+            for (const cache of step.caches) {
+                await workspace.prepareCache(cache.name);
+            }
+        }
+        const { inputs, output, caches, mounts } = this.buildMounts(step, runId, stepRuns, pipelineRoot);
+        const stdoutLog = createWriteStream(join(stagingPath, 'stdout.log'));
+        const stderrLog = createWriteStream(join(stagingPath, 'stderr.log'));
+        try {
+            const maxRetries = step.retries ?? 0;
+            const retryDelay = step.retryDelayMs ?? 5000;
+            let result;
+            for (let attempt = 0; attempt <= maxRetries; attempt++) {
+                try {
+                    result = await this.runtime.run(workspace, {
+                        name: `pipex-${workspace.id}-${step.id}-${Date.now()}`,
+                        image: step.image,
+                        cmd: step.cmd,
+                        env: step.env,
+                        inputs,
+                        output,
+                        caches,
+                        mounts,
+                        sources: step.sources?.map(m => ({
+                            hostPath: resolve(pipelineRoot, m.host),
+                            containerPath: m.container
+                        })),
+                        network: step.allowNetwork ? 'bridge' : 'none',
+                        timeoutSec: step.timeoutSec
+                    }, ({ stream, line }) => {
+                        if (stream === 'stdout') {
+                            stdoutLog.write(line + '\n');
+                        }
+                        else {
+                            stderrLog.write(line + '\n');
+                        }
+                        this.reporter.log(workspace.id, stepRef, stream, line);
+                    });
+                    break;
+                }
+                catch (error) {
+                    if (error instanceof PipexError && error.transient && attempt < maxRetries) {
+                        this.reporter.emit({ event: 'STEP_RETRYING', workspaceId: workspace.id, step: stepRef, attempt: attempt + 1, maxRetries });
+                        await setTimeout(retryDelay);
+                        continue;
+                    }
+                    throw error;
+                }
+            }
             this.reporter.result(workspace.id, stepRef, result);
+            await closeStream(stdoutLog);
+            await closeStream(stderrLog);
+            await this.writeRunMeta(stagingPath, { runId, step, stepRuns, resolvedMounts, currentFingerprint, result });
             if (result.exitCode === 0 || step.allowFailure) {
-                await workspace.commitArtifact(artifactId);
-                await workspace.linkArtifact(step.id, artifactId);
-                stepArtifacts.set(step.id, artifactId);
-                state.setStep(step.id, artifactId, currentFingerprint);
-                await state.save();
-                this.reporter.state(workspace.id, 'STEP_FINISHED', stepRef, { artifactId });
-            }
-            else {
-                await workspace.discardArtifact(artifactId);
-                this.reporter.state(workspace.id, 'STEP_FAILED', stepRef, { exitCode: result.exitCode });
-                this.reporter.state(workspace.id, 'PIPELINE_FAILED');
-                throw new Error(`Step ${step.id} failed with exit code ${result.exitCode}`);
+                await workspace.commitRun(runId);
+                await workspace.linkRun(step.id, runId);
+                stepRuns.set(step.id, runId);
+                state.setStep(step.id, runId, currentFingerprint);
+                const durationMs = result.finishedAt.getTime() - result.startedAt.getTime();
+                const artifactSize = await dirSize(workspace.runArtifactsPath(runId));
+                this.reporter.emit({ event: 'STEP_FINISHED', workspaceId: workspace.id, step: stepRef, runId, durationMs, artifactSize });
+                return artifactSize;
             }
+            await workspace.discardRun(runId);
+            this.reporter.emit({ event: 'STEP_FAILED', workspaceId: workspace.id, step: stepRef, exitCode: result.exitCode });
+            throw new ContainerCrashError(step.id, result.exitCode);
+        }
+        catch (error) {
+            await closeStream(stdoutLog);
+            await closeStream(stderrLog);
+            throw error;
         }
-        this.reporter.state(workspace.id, 'PIPELINE_FINISHED');
     }
-    async tryUseCache({ workspace, state, step, stepRef, currentFingerprint, stepArtifacts }) {
+    async writeRunMeta(stagingPath, { runId, step, stepRuns, resolvedMounts, currentFingerprint, result }) {
+        const meta = {
+            runId,
+            stepId: step.id,
+            stepName: step.name,
+            startedAt: result.startedAt.toISOString(),
+            finishedAt: result.finishedAt.toISOString(),
+            durationMs: result.finishedAt.getTime() - result.startedAt.getTime(),
+            exitCode: result.exitCode,
+            image: step.image,
+            cmd: step.cmd,
+            env: step.env,
+            inputs: step.inputs?.map(i => ({
+                step: i.step,
+                runId: stepRuns.get(i.step),
+                mountedAs: `/input/${i.step}`
+            })),
+            mounts: resolvedMounts,
+            caches: step.caches?.map(c => c.name),
+            allowNetwork: step.allowNetwork ?? false,
+            fingerprint: currentFingerprint,
+            status: result.exitCode === 0 ? 'success' : 'failure'
+        };
+        await writeFile(join(stagingPath, 'meta.json'), JSON.stringify(meta, null, 2), 'utf8');
+    }
+    async tryUseCache({ workspace, state, step, stepRef, currentFingerprint, stepRuns }) {
         const cached = state.getStep(step.id);
         if (cached?.fingerprint === currentFingerprint) {
             try {
-                const artifacts = await workspace.listArtifacts();
-                if (artifacts.includes(cached.artifactId)) {
-                    stepArtifacts.set(step.id, cached.artifactId);
-                    await workspace.linkArtifact(step.id, cached.artifactId);
-                    this.reporter.state(workspace.id, 'STEP_SKIPPED', stepRef, { artifactId: cached.artifactId, reason: 'cached' });
+                const runs = await workspace.listRuns();
+                if (runs.includes(cached.runId)) {
+                    stepRuns.set(step.id, cached.runId);
+                    await workspace.linkRun(step.id, cached.runId);
+                    this.reporter.emit({ event: 'STEP_SKIPPED', workspaceId: workspace.id, step: stepRef, runId: cached.runId, reason: 'cached' });
                     return true;
                 }
             }
             catch {
-                // Artifact missing, proceed with execution
+                // Run missing, proceed with execution
             }
         }
         return false;
     }
-    async prepareStagingWithInputs(workspace, step, stagingPath, stepArtifacts) {
+    async prepareStagingWithInputs(workspace, step, stagingArtifactsPath, stepRuns) {
         if (!step.inputs) {
             return;
         }
         for (const input of step.inputs) {
-            const inputArtifactId = stepArtifacts.get(input.step);
-            if (!inputArtifactId) {
-                throw new Error(`Step ${step.id}: input step '${input.step}' not found or not yet executed`);
+            const inputRunId = stepRuns.get(input.step);
+            if (!inputRunId) {
+                if (input.optional) {
+                    continue;
+                }
+                // Non-optional input without a run — this shouldn't happen in DAG mode
+                // but keep the continue for safety (the step may still work with bind mounts)
+                continue;
             }
             if (input.copyToOutput) {
-                await cp(workspace.artifactPath(inputArtifactId), stagingPath, { recursive: true });
+                await cp(workspace.runArtifactsPath(inputRunId), stagingArtifactsPath, { recursive: true });
             }
         }
     }
-    buildMounts(step, outputArtifactId, stepArtifacts, pipelineRoot) {
+    buildMounts(step, outputRunId, stepRuns, pipelineRoot) {
         const inputs = [];
         if (step.inputs) {
             for (const input of step.inputs) {
-                const inputArtifactId = stepArtifacts.get(input.step);
-                if (inputArtifactId) {
+                const inputRunId = stepRuns.get(input.step);
+                if (inputRunId) {
                     inputs.push({
-                        artifactId: inputArtifactId,
+                        runId: inputRunId,
                         containerPath: `/input/${input.step}`
                     });
                 }
             }
         }
         const output = {
-            stagingArtifactId: outputArtifactId,
+            stagingRunId: outputRunId,
             containerPath: step.outputPath ?? '/output'
         };
-        // Build cache mounts
         let caches;
         if (step.caches) {
             caches = step.caches.map(c => ({
@@ -196,3 +313,31 @@ export class PipelineRunner {
         return { inputs, output, caches, mounts };
     }
 }
+async function withConcurrency(tasks, limit) {
+    const results = Array.from({ length: tasks.length });
+    let next = 0;
+    async function worker() {
+        while (next < tasks.length) {
+            const i = next++;
+            try {
+                results[i] = { status: 'fulfilled', value: await tasks[i]() };
+            }
+            catch (error) {
+                results[i] = { status: 'rejected', reason: error };
+            }
+        }
+    }
+    await Promise.all(Array.from({ length: Math.min(limit, tasks.length) }, async () => worker()));
+    return results;
+}
+async function closeStream(stream) {
+    if (stream.destroyed) {
+        return;
+    }
+    return new Promise((resolve, reject) => {
+        stream.end(() => {
+            resolve();
+        });
+        stream.on('error', reject);
+    });
+}

package/dist/cli/reporter.js

@@ -1,15 +1,12 @@
 import pino from 'pino';
-import chalk from 'chalk';
-import ora from 'ora';
 /**
  * Reporter that outputs structured JSON logs via pino.
  * Suitable for CI/CD environments and log aggregation.
  */
 export class ConsoleReporter {
     logger = pino({ level: 'info' });
-    state(workspaceId, event, step, meta) {
-        const stepName = step?.displayName === step?.id ? undefined : step?.displayName;
-        this.logger.info({ workspaceId, event, stepId: step?.id, stepName, ...meta });
+    emit(event) {
+        this.logger.info(event);
     }
     log(workspaceId, step, stream, line) {
         this.logger.info({ workspaceId, stepId: step.id, stream, line });
@@ -18,105 +15,3 @@ export class ConsoleReporter {
         this.logger.info({ workspaceId, stepId: step.id, result });
     }
 }
-/**
- * Reporter with interactive terminal UI using spinners and colors.
- * Suitable for local development and manual execution.
- */
-export class InteractiveReporter {
-    static get maxStderrLines() {
-        return 20;
-    }
-    spinner;
-    stepSpinners = new Map();
-    stderrBuffers = new Map();
-    state(workspaceId, event, step, meta) {
-        switch (event) {
-            case 'PIPELINE_START': {
-                const displayName = meta?.pipelineName ?? workspaceId;
-                console.log(chalk.bold(`\n▶ Pipeline: ${chalk.cyan(displayName)}\n`));
-                break;
-            }
-            case 'STEP_STARTING': {
-                if (step) {
-                    const spinner = ora({ text: step.displayName, prefixText: '  ' }).start();
-                    this.stepSpinners.set(step.id, spinner);
-                }
-                break;
-            }
-            case 'STEP_SKIPPED': {
-                if (step) {
-                    const spinner = this.stepSpinners.get(step.id);
-                    if (spinner) {
-                        spinner.stopAndPersist({ symbol: chalk.gray('⊙'), text: chalk.gray(`${step.displayName} (cached)`) });
-                        this.stepSpinners.delete(step.id);
-                    }
-                    else {
-                        console.log(`  ${chalk.gray('⊙')} ${chalk.gray(`${step.displayName} (cached)`)}`);
-                    }
-                }
-                break;
-            }
-            case 'STEP_FINISHED': {
-                if (step) {
-                    const spinner = this.stepSpinners.get(step.id);
-                    if (spinner) {
-                        spinner.stopAndPersist({ symbol: chalk.green('✓'), text: chalk.green(step.displayName) });
-                        this.stepSpinners.delete(step.id);
-                    }
-                    this.stderrBuffers.delete(step.id);
-                }
-                break;
-            }
-            case 'STEP_FAILED': {
-                if (step) {
-                    this.handleStepFailed(step, meta);
-                }
-                break;
-            }
-            case 'PIPELINE_FINISHED': {
-                console.log(chalk.bold.green('\n✓ Pipeline completed\n'));
-                break;
-            }
-            case 'PIPELINE_FAILED': {
-                console.log(chalk.bold.red('\n✗ Pipeline failed\n'));
-                break;
-            }
-        }
-    }
-    log(_workspaceId, step, stream, line) {
-        if (stream === 'stderr') {
-            let buffer = this.stderrBuffers.get(step.id);
-            if (!buffer) {
-                buffer = [];
-                this.stderrBuffers.set(step.id, buffer);
-            }
-            buffer.push(line);
-            if (buffer.length > InteractiveReporter.maxStderrLines) {
-                buffer.shift();
-            }
-        }
-    }
-    result(_workspaceId, _step, _result) {
-        // Results shown via state updates
-    }
-    handleStepFailed(step, meta) {
-        const spinner = this.stepSpinners.get(step.id);
-        const exitCode = meta?.exitCode;
-        if (spinner) {
-            const exitInfo = exitCode === undefined ? '' : ` (exit ${exitCode})`;
-            spinner.stopAndPersist({
-                symbol: chalk.red('✗'),
-                text: chalk.red(`${step.displayName}${exitInfo}`)
-            });
-            this.stepSpinners.delete(step.id);
-        }
-        const stderr = this.stderrBuffers.get(step.id);
-        if (stderr && stderr.length > 0) {
-            console.log(chalk.red('  ── stderr ──'));
-            for (const line of stderr) {
-                console.log(chalk.red(`  ${line}`));
-            }
-        }
-        this.stderrBuffers.delete(step.id);
-    }
-}

package/dist/cli/state.js

@@ -5,24 +5,24 @@ import { createHash } from 'node:crypto';
  * Manages caching state for pipeline execution.
  *
  * The StateManager computes fingerprints for steps and tracks which
- * artifact was produced by each step. This enables cache hits when
+ * run was produced by each step. This enables cache hits when
  * a step's configuration hasn't changed.
  *
  * ## Fingerprint Algorithm
  *
  * A step fingerprint is computed as:
  * ```
- * SHA256(image + JSON(cmd) + JSON(sorted env) + JSON(sorted inputArtifactIds))
+ * SHA256(image + JSON(cmd) + JSON(sorted env) + JSON(sorted inputRunIds))
  * ```
  *
  * A step is re-executed when:
  * - The fingerprint changes (image, cmd, env, or inputs modified)
- * - The artifact no longer exists (manually deleted)
+ * - The run no longer exists (manually deleted)
  *
  * ## Cache Propagation
  *
  * Changes propagate through dependencies. If step A is modified,
- * all steps depending on A are invalidated automatically (via inputArtifactIds).
+ * all steps depending on A are invalidated automatically (via inputRunIds).
  */
 export class StateManager {
     static fingerprint(config) {
@@ -32,8 +32,8 @@ export class StateManager {
         if (config.env) {
             hash.update(JSON.stringify(Object.entries(config.env).sort((a, b) => a[0].localeCompare(b[0]))));
         }
-        if (config.inputArtifactIds) {
-            hash.update(JSON.stringify([...config.inputArtifactIds].sort((a, b) => a.localeCompare(b))));
+        if (config.inputRunIds) {
+            hash.update(JSON.stringify([...config.inputRunIds].sort((a, b) => a.localeCompare(b))));
         }
         if (config.mounts && config.mounts.length > 0) {
             const sorted = [...config.mounts].sort((a, b) => a.containerPath.localeCompare(b.containerPath));
@@ -77,13 +77,34 @@ export class StateManager {
     getStep(stepId) {
         return this.state.steps[stepId];
     }
+    /**
+     * Lists all steps with their run IDs (in insertion order).
+     * @returns Array of {stepId, runId} entries
+     */
+    listSteps() {
+        return Object.entries(this.state.steps).map(([stepId, { runId }]) => ({ stepId, runId }));
+    }
+    /**
+     * Returns the set of run IDs currently referenced by state.
+     */
+    activeRunIds() {
+        return new Set(Object.values(this.state.steps).map(s => s.runId));
+    }
     /**
      * Updates cached state for a step.
      * @param stepId - Step identifier
-     * @param artifactId - Artifact produced by the step
+     * @param runId - Run produced by the step
      * @param fingerprint - Step configuration fingerprint
      */
-    setStep(stepId, artifactId, fingerprint) {
-        this.state.steps[stepId] = { artifactId, fingerprint };
+    setStep(stepId, runId, fingerprint) {
+        this.state.steps[stepId] = { runId, fingerprint };
+    }
+    /**
+     * Removes a step's cached state.
+     * @param stepId - Step identifier
+     */
+    removeStep(stepId) {
+        // eslint-disable-next-line @typescript-eslint/no-dynamic-delete
+        delete this.state.steps[stepId];
     }
 }

package/dist/cli/step-loader.js

@@ -0,0 +1,25 @@
+import { readFile } from 'node:fs/promises';
+import { ValidationError } from '../errors.js';
+import { parsePipelineFile } from './pipeline-loader.js';
+import { resolveStep, validateStep } from './step-resolver.js';
+/**
+ * Loads and resolves a single step definition from a file.
+ */
+export async function loadStepFile(filePath, stepIdOverride) {
+    const content = await readFile(filePath, 'utf8');
+    const raw = parsePipelineFile(content, filePath);
+    if (!raw || typeof raw !== 'object') {
+        throw new ValidationError('Step file must contain an object');
+    }
+    // If no id/name provided, require --step override
+    if (!('id' in raw && raw.id) && !('name' in raw && raw.name) && !stepIdOverride) {
+        throw new ValidationError('Step file must have "id" or "name", or use --step to set an ID');
+    }
+    // Apply step ID override
+    if (stepIdOverride) {
+        raw.id = stepIdOverride;
+    }
+    const step = resolveStep(raw);
+    validateStep(step);
+    return step;
+}