@livingdata/pipex 0.0.9 → 0.0.10

package/dist/cli/step-runner.js

@@ -0,0 +1,226 @@
+import process from 'node:process';
+import { cp, writeFile } from 'node:fs/promises';
+import { setTimeout } from 'node:timers/promises';
+import { createWriteStream } from 'node:fs';
+import { join, resolve } from 'node:path';
+import { ContainerCrashError, PipexError } from '../errors.js';
+import { StateManager } from './state.js';
+import { dirSize } from './utils.js';
+/**
+ * Executes a single step in a workspace.
+ * Adapted from PipelineRunner.executeStep() for standalone use.
+ */
+export class StepRunner {
+    runtime;
+    reporter;
+    constructor(runtime, reporter) {
+        this.runtime = runtime;
+        this.reporter = reporter;
+    }
+    async run(options) {
+        const { workspace, state, step, inputs, pipelineRoot, force, ephemeral } = options;
+        const stepRef = { id: step.id, displayName: step.name ?? step.id };
+        const resolvedMounts = step.mounts?.map(m => ({
+            hostPath: resolve(pipelineRoot, m.host),
+            containerPath: m.container
+        }));
+        const currentFingerprint = this.computeFingerprint(step, inputs, resolvedMounts);
+        // Cache check (skip for ephemeral or force)
+        if (!force && !ephemeral) {
+            const cacheResult = await this.tryUseCache({ workspace, state, stepId: step.id, stepRef, fingerprint: currentFingerprint });
+            if (cacheResult) {
+                return cacheResult;
+            }
+        }
+        this.reporter.emit({ event: 'STEP_STARTING', workspaceId: workspace.id, step: stepRef });
+        return this.executeStep({ workspace, state, step, stepRef, inputs, pipelineRoot, ephemeral, currentFingerprint, resolvedMounts });
+    }
+    computeFingerprint(step, inputs, resolvedMounts) {
+        const inputRunIds = step.inputs
+            ?.map(i => inputs.get(i.step))
+            .filter((id) => id !== undefined);
+        return StateManager.fingerprint({
+            image: step.image,
+            cmd: step.cmd,
+            env: step.env,
+            inputRunIds,
+            mounts: resolvedMounts
+        });
+    }
+    async tryUseCache({ workspace, state, stepId, stepRef, fingerprint }) {
+        const cached = state.getStep(stepId);
+        if (cached?.fingerprint === fingerprint) {
+            const runs = await workspace.listRuns();
+            if (runs.includes(cached.runId)) {
+                await workspace.linkRun(stepId, cached.runId);
+                this.reporter.emit({ event: 'STEP_SKIPPED', workspaceId: workspace.id, step: stepRef, runId: cached.runId, reason: 'cached' });
+                return { runId: cached.runId, exitCode: 0 };
+            }
+        }
+        return undefined;
+    }
+    async executeStep(ctx) {
+        const { workspace, step, stepRef, inputs, pipelineRoot, ephemeral } = ctx;
+        const runId = workspace.generateRunId();
+        const stagingPath = await workspace.prepareRun(runId);
+        await this.prepareStagingInputs(workspace, step, runId, inputs);
+        await this.prepareCaches(workspace, step);
+        const { containerInputs, output, caches, mounts } = this.buildMounts(step, runId, inputs, pipelineRoot);
+        const stdoutLog = createWriteStream(join(stagingPath, 'stdout.log'));
+        const stderrLog = createWriteStream(join(stagingPath, 'stderr.log'));
+        try {
+            const result = await this.executeWithRetries({
+                ctx, containerInputs, output, caches, mounts, stdoutLog, stderrLog
+            });
+            this.reporter.result(workspace.id, stepRef, result);
+            await closeStream(stdoutLog);
+            await closeStream(stderrLog);
+            if (ephemeral) {
+                await workspace.discardRun(runId);
+                this.reporter.emit({ event: 'STEP_FINISHED', workspaceId: workspace.id, step: stepRef, ephemeral: true });
+                return { exitCode: result.exitCode };
+            }
+            return await this.commitOrDiscard({ ...ctx, runId, stagingPath, result });
+        }
+        catch (error) {
+            await closeStream(stdoutLog);
+            await closeStream(stderrLog);
+            throw error;
+        }
+    }
+    async prepareStagingInputs(workspace, step, runId, inputs) {
+        if (!step.inputs) {
+            return;
+        }
+        for (const input of step.inputs) {
+            const inputRunId = inputs.get(input.step);
+            if (inputRunId && input.copyToOutput) {
+                await cp(workspace.runArtifactsPath(inputRunId), workspace.runStagingArtifactsPath(runId), { recursive: true });
+            }
+        }
+    }
+    async prepareCaches(workspace, step) {
+        if (!step.caches) {
+            return;
+        }
+        for (const cache of step.caches) {
+            await workspace.prepareCache(cache.name);
+        }
+    }
+    buildMounts(step, runId, inputs, pipelineRoot) {
+        const containerInputs = [];
+        if (step.inputs) {
+            for (const input of step.inputs) {
+                const inputRunId = inputs.get(input.step);
+                if (inputRunId) {
+                    containerInputs.push({ runId: inputRunId, containerPath: `/input/${input.step}` });
+                }
+            }
+        }
+        const output = { stagingRunId: runId, containerPath: step.outputPath ?? '/output' };
+        const caches = step.caches?.map(c => ({ name: c.name, containerPath: c.path }));
+        const mounts = step.mounts?.map(m => ({
+            hostPath: resolve(pipelineRoot, m.host),
+            containerPath: m.container
+        }));
+        return { containerInputs, output, caches, mounts };
+    }
+    async executeWithRetries({ ctx, containerInputs, output, caches, mounts, stdoutLog, stderrLog }) {
+        const { workspace, step, stepRef, pipelineRoot, ephemeral } = ctx;
+        const maxRetries = step.retries ?? 0;
+        const retryDelay = step.retryDelayMs ?? 5000;
+        let result;
+        for (let attempt = 0; attempt <= maxRetries; attempt++) {
+            try {
+                result = await this.runtime.run(workspace, {
+                    name: `pipex-${workspace.id}-${step.id}-${Date.now()}`,
+                    image: step.image,
+                    cmd: step.cmd,
+                    env: step.env,
+                    inputs: containerInputs,
+                    output,
+                    caches,
+                    mounts,
+                    sources: step.sources?.map(m => ({
+                        hostPath: resolve(pipelineRoot, m.host),
+                        containerPath: m.container
+                    })),
+                    network: step.allowNetwork ? 'bridge' : 'none',
+                    timeoutSec: step.timeoutSec
+                }, ({ stream, line }) => {
+                    if (stream === 'stdout') {
+                        stdoutLog.write(line + '\n');
+                    }
+                    else {
+                        stderrLog.write(line + '\n');
+                    }
+                    if (ephemeral && stream === 'stdout') {
+                        process.stdout.write(line + '\n');
+                    }
+                    else {
+                        this.reporter.log(workspace.id, stepRef, stream, line);
+                    }
+                });
+                break;
+            }
+            catch (error) {
+                if (error instanceof PipexError && error.transient && attempt < maxRetries) {
+                    this.reporter.emit({ event: 'STEP_RETRYING', workspaceId: workspace.id, step: stepRef, attempt: attempt + 1, maxRetries });
+                    await setTimeout(retryDelay);
+                    continue;
+                }
+                throw error;
+            }
+        }
+        return result;
+    }
+    async commitOrDiscard({ workspace, state, step, stepRef, inputs, resolvedMounts, currentFingerprint, runId, stagingPath, result }) {
+        const meta = {
+            runId,
+            stepId: step.id,
+            stepName: step.name,
+            startedAt: result.startedAt.toISOString(),
+            finishedAt: result.finishedAt.toISOString(),
+            durationMs: result.finishedAt.getTime() - result.startedAt.getTime(),
+            exitCode: result.exitCode,
+            image: step.image,
+            cmd: step.cmd,
+            env: step.env,
+            inputs: step.inputs?.map(i => ({
+                step: i.step,
+                runId: inputs.get(i.step),
+                mountedAs: `/input/${i.step}`
+            })),
+            mounts: resolvedMounts,
+            caches: step.caches?.map(c => c.name),
+            allowNetwork: step.allowNetwork ?? false,
+            fingerprint: currentFingerprint,
+            status: result.exitCode === 0 ? 'success' : 'failure'
+        };
+        await writeFile(join(stagingPath, 'meta.json'), JSON.stringify(meta, null, 2), 'utf8');
+        if (result.exitCode === 0 || step.allowFailure) {
+            await workspace.commitRun(runId);
+            await workspace.linkRun(step.id, runId);
+            state.setStep(step.id, runId, currentFingerprint);
+            await state.save();
+            const durationMs = result.finishedAt.getTime() - result.startedAt.getTime();
+            const artifactSize = await dirSize(workspace.runArtifactsPath(runId));
+            this.reporter.emit({ event: 'STEP_FINISHED', workspaceId: workspace.id, step: stepRef, runId, durationMs, artifactSize });
+            return { runId, exitCode: result.exitCode };
+        }
+        await workspace.discardRun(runId);
+        this.reporter.emit({ event: 'STEP_FAILED', workspaceId: workspace.id, step: stepRef, exitCode: result.exitCode });
+        throw new ContainerCrashError(step.id, result.exitCode);
+    }
+}
+async function closeStream(stream) {
+    if (stream.destroyed) {
+        return;
+    }
+    return new Promise((resolve, reject) => {
+        stream.end(() => {
+            resolve();
+        });
+        stream.on('error', reject);
+    });
+}

package/dist/cli/utils.js

@@ -1,49 +1,3 @@
-import { readdir, stat } from 'node:fs/promises';
-import { join } from 'node:path';
 export function getGlobalOptions(cmd) {
     return cmd.optsWithGlobals();
 }
-export async function dirSize(dirPath) {
-    let total = 0;
-    try {
-        const entries = await readdir(dirPath, { withFileTypes: true });
-        for (const entry of entries) {
-            const fullPath = join(dirPath, entry.name);
-            if (entry.isDirectory()) {
-                total += await dirSize(fullPath);
-            }
-            else if (entry.isFile()) {
-                const s = await stat(fullPath);
-                total += s.size;
-            }
-        }
-    }
-    catch {
-        // Directory doesn't exist or isn't readable
-    }
-    return total;
-}
-export function formatSize(bytes) {
-    if (bytes < 1024) {
-        return `${bytes} B`;
-    }
-    if (bytes < 1024 * 1024) {
-        return `${(bytes / 1024).toFixed(1)} KB`;
-    }
-    if (bytes < 1024 * 1024 * 1024) {
-        return `${(bytes / (1024 * 1024)).toFixed(1)} MB`;
-    }
-    return `${(bytes / (1024 * 1024 * 1024)).toFixed(1)} GB`;
-}
-export function formatDuration(ms) {
-    if (ms < 1000) {
-        return `${ms}ms`;
-    }
-    const seconds = ms / 1000;
-    if (seconds < 60) {
-        return `${seconds.toFixed(1)}s`;
-    }
-    const minutes = Math.floor(seconds / 60);
-    const remainingSeconds = Math.round(seconds % 60);
-    return `${minutes}m ${remainingSeconds}s`;
-}