@livingdata/pipex 0.0.8 → 0.0.10

package/dist/cli/step-resolver.js

@@ -0,0 +1,111 @@
+import { ValidationError } from '../errors.js';
+import { getKit } from '../kits/index.js';
+import { isKitStep } from '../types.js';
+import { slugify, mergeEnv, mergeCaches, mergeMounts } from './pipeline-loader.js';
+/**
+ * Resolves a step definition into a fully resolved Step.
+ * Kit steps (`uses`) are expanded into image + cmd.
+ */
+export function resolveStep(step) {
+    if (!step.id && !step.name) {
+        throw new ValidationError('Invalid step: at least one of "id" or "name" must be defined');
+    }
+    const id = step.id ?? slugify(step.name);
+    const { name } = step;
+    if (!isKitStep(step)) {
+        return { ...step, id, name };
+    }
+    return resolveKitStep(step, id, name);
+}
+function resolveKitStep(step, id, name) {
+    const kit = getKit(step.uses);
+    const kitOutput = kit.resolve(step.with ?? {});
+    return {
+        id,
+        name,
+        image: kitOutput.image,
+        cmd: kitOutput.cmd,
+        env: mergeEnv(kitOutput.env, step.env),
+        inputs: step.inputs,
+        outputPath: step.outputPath,
+        caches: mergeCaches(kitOutput.caches, step.caches),
+        mounts: mergeMounts(kitOutput.mounts, step.mounts),
+        sources: mergeMounts(kitOutput.sources, step.sources),
+        timeoutSec: step.timeoutSec,
+        allowFailure: step.allowFailure,
+        allowNetwork: step.allowNetwork ?? kitOutput.allowNetwork,
+        retries: step.retries,
+        retryDelayMs: step.retryDelayMs,
+        if: step.if
+    };
+}
+/**
+ * Validates a resolved step for correctness and security.
+ */
+export function validateStep(step) {
+    validateIdentifier(step.id, 'step id');
+    if (!step.image || typeof step.image !== 'string') {
+        throw new ValidationError(`Invalid step ${step.id}: image is required`);
+    }
+    if (!Array.isArray(step.cmd) || step.cmd.length === 0) {
+        throw new ValidationError(`Invalid step ${step.id}: cmd must be a non-empty array`);
+    }
+    if (step.inputs) {
+        for (const input of step.inputs) {
+            validateIdentifier(input.step, `input step name in step ${step.id}`);
+        }
+    }
+    if (step.mounts) {
+        validateMounts(step.id, step.mounts);
+    }
+    if (step.sources) {
+        validateMounts(step.id, step.sources);
+    }
+    if (step.caches) {
+        validateCaches(step.id, step.caches);
+    }
+}
+function validateIdentifier(id, context) {
+    if (!/^[\w-]+$/.test(id)) {
+        throw new ValidationError(`Invalid ${context}: '${id}' must contain only alphanumeric characters, underscore, and hyphen`);
+    }
+    if (id.includes('..')) {
+        throw new ValidationError(`Invalid ${context}: '${id}' cannot contain '..'`);
+    }
+}
+function validateMounts(stepId, mounts) {
+    for (const mount of mounts) {
+        if (!mount.host || typeof mount.host !== 'string') {
+            throw new ValidationError(`Step ${stepId}: mount.host is required and must be a string`);
+        }
+        if (mount.host.startsWith('/')) {
+            throw new ValidationError(`Step ${stepId}: mount.host '${mount.host}' must be a relative path`);
+        }
+        if (mount.host.includes('..')) {
+            throw new ValidationError(`Step ${stepId}: mount.host '${mount.host}' must not contain '..'`);
+        }
+        if (!mount.container || typeof mount.container !== 'string') {
+            throw new ValidationError(`Step ${stepId}: mount.container is required and must be a string`);
+        }
+        if (!mount.container.startsWith('/')) {
+            throw new ValidationError(`Step ${stepId}: mount.container '${mount.container}' must be an absolute path`);
+        }
+        if (mount.container.includes('..')) {
+            throw new ValidationError(`Step ${stepId}: mount.container '${mount.container}' must not contain '..'`);
+        }
+    }
+}
+function validateCaches(stepId, caches) {
+    for (const cache of caches) {
+        if (!cache.name || typeof cache.name !== 'string') {
+            throw new ValidationError(`Step ${stepId}: cache.name is required and must be a string`);
+        }
+        validateIdentifier(cache.name, `cache name in step ${stepId}`);
+        if (!cache.path || typeof cache.path !== 'string') {
+            throw new ValidationError(`Step ${stepId}: cache.path is required and must be a string`);
+        }
+        if (!cache.path.startsWith('/')) {
+            throw new ValidationError(`Step ${stepId}: cache.path '${cache.path}' must be an absolute path`);
+        }
+    }
+}

package/dist/cli/step-runner.js

@@ -0,0 +1,226 @@
+import process from 'node:process';
+import { cp, writeFile } from 'node:fs/promises';
+import { setTimeout } from 'node:timers/promises';
+import { createWriteStream } from 'node:fs';
+import { join, resolve } from 'node:path';
+import { ContainerCrashError, PipexError } from '../errors.js';
+import { StateManager } from './state.js';
+import { dirSize } from './utils.js';
+/**
+ * Executes a single step in a workspace.
+ * Adapted from PipelineRunner.executeStep() for standalone use.
+ */
+export class StepRunner {
+    runtime;
+    reporter;
+    constructor(runtime, reporter) {
+        this.runtime = runtime;
+        this.reporter = reporter;
+    }
+    async run(options) {
+        const { workspace, state, step, inputs, pipelineRoot, force, ephemeral } = options;
+        const stepRef = { id: step.id, displayName: step.name ?? step.id };
+        const resolvedMounts = step.mounts?.map(m => ({
+            hostPath: resolve(pipelineRoot, m.host),
+            containerPath: m.container
+        }));
+        const currentFingerprint = this.computeFingerprint(step, inputs, resolvedMounts);
+        // Cache check (skip for ephemeral or force)
+        if (!force && !ephemeral) {
+            const cacheResult = await this.tryUseCache({ workspace, state, stepId: step.id, stepRef, fingerprint: currentFingerprint });
+            if (cacheResult) {
+                return cacheResult;
+            }
+        }
+        this.reporter.emit({ event: 'STEP_STARTING', workspaceId: workspace.id, step: stepRef });
+        return this.executeStep({ workspace, state, step, stepRef, inputs, pipelineRoot, ephemeral, currentFingerprint, resolvedMounts });
+    }
+    computeFingerprint(step, inputs, resolvedMounts) {
+        const inputRunIds = step.inputs
+            ?.map(i => inputs.get(i.step))
+            .filter((id) => id !== undefined);
+        return StateManager.fingerprint({
+            image: step.image,
+            cmd: step.cmd,
+            env: step.env,
+            inputRunIds,
+            mounts: resolvedMounts
+        });
+    }
+    async tryUseCache({ workspace, state, stepId, stepRef, fingerprint }) {
+        const cached = state.getStep(stepId);
+        if (cached?.fingerprint === fingerprint) {
+            const runs = await workspace.listRuns();
+            if (runs.includes(cached.runId)) {
+                await workspace.linkRun(stepId, cached.runId);
+                this.reporter.emit({ event: 'STEP_SKIPPED', workspaceId: workspace.id, step: stepRef, runId: cached.runId, reason: 'cached' });
+                return { runId: cached.runId, exitCode: 0 };
+            }
+        }
+        return undefined;
+    }
+    async executeStep(ctx) {
+        const { workspace, step, stepRef, inputs, pipelineRoot, ephemeral } = ctx;
+        const runId = workspace.generateRunId();
+        const stagingPath = await workspace.prepareRun(runId);
+        await this.prepareStagingInputs(workspace, step, runId, inputs);
+        await this.prepareCaches(workspace, step);
+        const { containerInputs, output, caches, mounts } = this.buildMounts(step, runId, inputs, pipelineRoot);
+        const stdoutLog = createWriteStream(join(stagingPath, 'stdout.log'));
+        const stderrLog = createWriteStream(join(stagingPath, 'stderr.log'));
+        try {
+            const result = await this.executeWithRetries({
+                ctx, containerInputs, output, caches, mounts, stdoutLog, stderrLog
+            });
+            this.reporter.result(workspace.id, stepRef, result);
+            await closeStream(stdoutLog);
+            await closeStream(stderrLog);
+            if (ephemeral) {
+                await workspace.discardRun(runId);
+                this.reporter.emit({ event: 'STEP_FINISHED', workspaceId: workspace.id, step: stepRef, ephemeral: true });
+                return { exitCode: result.exitCode };
+            }
+            return await this.commitOrDiscard({ ...ctx, runId, stagingPath, result });
+        }
+        catch (error) {
+            await closeStream(stdoutLog);
+            await closeStream(stderrLog);
+            throw error;
+        }
+    }
+    async prepareStagingInputs(workspace, step, runId, inputs) {
+        if (!step.inputs) {
+            return;
+        }
+        for (const input of step.inputs) {
+            const inputRunId = inputs.get(input.step);
+            if (inputRunId && input.copyToOutput) {
+                await cp(workspace.runArtifactsPath(inputRunId), workspace.runStagingArtifactsPath(runId), { recursive: true });
+            }
+        }
+    }
+    async prepareCaches(workspace, step) {
+        if (!step.caches) {
+            return;
+        }
+        for (const cache of step.caches) {
+            await workspace.prepareCache(cache.name);
+        }
+    }
+    buildMounts(step, runId, inputs, pipelineRoot) {
+        const containerInputs = [];
+        if (step.inputs) {
+            for (const input of step.inputs) {
+                const inputRunId = inputs.get(input.step);
+                if (inputRunId) {
+                    containerInputs.push({ runId: inputRunId, containerPath: `/input/${input.step}` });
+                }
+            }
+        }
+        const output = { stagingRunId: runId, containerPath: step.outputPath ?? '/output' };
+        const caches = step.caches?.map(c => ({ name: c.name, containerPath: c.path }));
+        const mounts = step.mounts?.map(m => ({
+            hostPath: resolve(pipelineRoot, m.host),
+            containerPath: m.container
+        }));
+        return { containerInputs, output, caches, mounts };
+    }
+    async executeWithRetries({ ctx, containerInputs, output, caches, mounts, stdoutLog, stderrLog }) {
+        const { workspace, step, stepRef, pipelineRoot, ephemeral } = ctx;
+        const maxRetries = step.retries ?? 0;
+        const retryDelay = step.retryDelayMs ?? 5000;
+        let result;
+        for (let attempt = 0; attempt <= maxRetries; attempt++) {
+            try {
+                result = await this.runtime.run(workspace, {
+                    name: `pipex-${workspace.id}-${step.id}-${Date.now()}`,
+                    image: step.image,
+                    cmd: step.cmd,
+                    env: step.env,
+                    inputs: containerInputs,
+                    output,
+                    caches,
+                    mounts,
+                    sources: step.sources?.map(m => ({
+                        hostPath: resolve(pipelineRoot, m.host),
+                        containerPath: m.container
+                    })),
+                    network: step.allowNetwork ? 'bridge' : 'none',
+                    timeoutSec: step.timeoutSec
+                }, ({ stream, line }) => {
+                    if (stream === 'stdout') {
+                        stdoutLog.write(line + '\n');
+                    }
+                    else {
+                        stderrLog.write(line + '\n');
+                    }
+                    if (ephemeral && stream === 'stdout') {
+                        process.stdout.write(line + '\n');
+                    }
+                    else {
+                        this.reporter.log(workspace.id, stepRef, stream, line);
+                    }
+                });
+                break;
+            }
+            catch (error) {
+                if (error instanceof PipexError && error.transient && attempt < maxRetries) {
+                    this.reporter.emit({ event: 'STEP_RETRYING', workspaceId: workspace.id, step: stepRef, attempt: attempt + 1, maxRetries });
+                    await setTimeout(retryDelay);
+                    continue;
+                }
+                throw error;
+            }
+        }
+        return result;
+    }
+    async commitOrDiscard({ workspace, state, step, stepRef, inputs, resolvedMounts, currentFingerprint, runId, stagingPath, result }) {
+        const meta = {
+            runId,
+            stepId: step.id,
+            stepName: step.name,
+            startedAt: result.startedAt.toISOString(),
+            finishedAt: result.finishedAt.toISOString(),
+            durationMs: result.finishedAt.getTime() - result.startedAt.getTime(),
+            exitCode: result.exitCode,
+            image: step.image,
+            cmd: step.cmd,
+            env: step.env,
+            inputs: step.inputs?.map(i => ({
+                step: i.step,
+                runId: inputs.get(i.step),
+                mountedAs: `/input/${i.step}`
+            })),
+            mounts: resolvedMounts,
+            caches: step.caches?.map(c => c.name),
+            allowNetwork: step.allowNetwork ?? false,
+            fingerprint: currentFingerprint,
+            status: result.exitCode === 0 ? 'success' : 'failure'
+        };
+        await writeFile(join(stagingPath, 'meta.json'), JSON.stringify(meta, null, 2), 'utf8');
+        if (result.exitCode === 0 || step.allowFailure) {
+            await workspace.commitRun(runId);
+            await workspace.linkRun(step.id, runId);
+            state.setStep(step.id, runId, currentFingerprint);
+            await state.save();
+            const durationMs = result.finishedAt.getTime() - result.startedAt.getTime();
+            const artifactSize = await dirSize(workspace.runArtifactsPath(runId));
+            this.reporter.emit({ event: 'STEP_FINISHED', workspaceId: workspace.id, step: stepRef, runId, durationMs, artifactSize });
+            return { runId, exitCode: result.exitCode };
+        }
+        await workspace.discardRun(runId);
+        this.reporter.emit({ event: 'STEP_FAILED', workspaceId: workspace.id, step: stepRef, exitCode: result.exitCode });
+        throw new ContainerCrashError(step.id, result.exitCode);
+    }
+}
+async function closeStream(stream) {
+    if (stream.destroyed) {
+        return;
+    }
+    return new Promise((resolve, reject) => {
+        stream.end(() => {
+            resolve();
+        });
+        stream.on('error', reject);
+    });
+}

package/dist/cli/utils.js

@@ -0,0 +1,3 @@
+export function getGlobalOptions(cmd) {
+    return cmd.optsWithGlobals();
+}