@livingdata/pipex 0.0.9 → 0.0.10

package/dist/core/step-loader.js

@@ -0,0 +1,25 @@
+import { readFile } from 'node:fs/promises';
+import { ValidationError } from '../errors.js';
+import { parsePipelineFile } from './pipeline-loader.js';
+import { resolveStep, validateStep } from './step-resolver.js';
+/**
+ * Loads and resolves a single step definition from a file.
+ */
+export async function loadStepFile(filePath, stepIdOverride) {
+    const content = await readFile(filePath, 'utf8');
+    const raw = parsePipelineFile(content, filePath);
+    if (!raw || typeof raw !== 'object') {
+        throw new ValidationError('Step file must contain an object');
+    }
+    // If no id/name provided, require --step override
+    if (!('id' in raw && raw.id) && !('name' in raw && raw.name) && !stepIdOverride) {
+        throw new ValidationError('Step file must have "id" or "name", or use --step to set an ID');
+    }
+    // Apply step ID override
+    if (stepIdOverride) {
+        raw.id = stepIdOverride;
+    }
+    const step = resolveStep(raw);
+    validateStep(step);
+    return step;
+}

package/dist/core/step-resolver.js

@@ -0,0 +1,117 @@
+import { ValidationError } from '../errors.js';
+import { getKit } from '../kits/index.js';
+import { isKitStep } from '../types.js';
+import { slugify, mergeEnv, mergeCaches, mergeMounts } from './pipeline-loader.js';
+/**
+ * Resolves a step definition into a fully resolved Step.
+ * Kit steps (`uses`) are expanded into image + cmd.
+ */
+export function resolveStep(step) {
+    if (!step.id && !step.name) {
+        throw new ValidationError('Invalid step: at least one of "id" or "name" must be defined');
+    }
+    const id = step.id ?? slugify(step.name);
+    const { name } = step;
+    if (!isKitStep(step)) {
+        return { ...step, id, name };
+    }
+    return resolveKitStep(step, id, name);
+}
+function resolveKitStep(step, id, name) {
+    const kit = getKit(step.uses);
+    const kitOutput = kit.resolve(step.with ?? {});
+    return {
+        id,
+        name,
+        image: kitOutput.image,
+        cmd: kitOutput.cmd,
+        env: mergeEnv(kitOutput.env, step.env),
+        envFile: step.envFile,
+        inputs: step.inputs,
+        outputPath: step.outputPath,
+        caches: mergeCaches(kitOutput.caches, step.caches),
+        mounts: mergeMounts(kitOutput.mounts, step.mounts),
+        sources: mergeMounts(kitOutput.sources, step.sources),
+        timeoutSec: step.timeoutSec,
+        allowFailure: step.allowFailure,
+        allowNetwork: step.allowNetwork ?? kitOutput.allowNetwork,
+        retries: step.retries,
+        retryDelayMs: step.retryDelayMs,
+        if: step.if
+    };
+}
+/**
+ * Validates a resolved step for correctness and security.
+ */
+export function validateStep(step) {
+    validateIdentifier(step.id, 'step id');
+    if (!step.image || typeof step.image !== 'string') {
+        throw new ValidationError(`Invalid step ${step.id}: image is required`);
+    }
+    if (!Array.isArray(step.cmd) || step.cmd.length === 0) {
+        throw new ValidationError(`Invalid step ${step.id}: cmd must be a non-empty array`);
+    }
+    if (step.envFile) {
+        if (typeof step.envFile !== 'string') {
+            throw new ValidationError(`Step ${step.id}: envFile must be a string`);
+        }
+        if (step.envFile.startsWith('/')) {
+            throw new ValidationError(`Step ${step.id}: envFile '${step.envFile}' must be a relative path`);
+        }
+    }
+    if (step.inputs) {
+        for (const input of step.inputs) {
+            validateIdentifier(input.step, `input step name in step ${step.id}`);
+        }
+    }
+    if (step.mounts) {
+        validateMounts(step.id, step.mounts);
+    }
+    if (step.sources) {
+        validateMounts(step.id, step.sources);
+    }
+    if (step.caches) {
+        validateCaches(step.id, step.caches);
+    }
+}
+function validateIdentifier(id, context) {
+    if (!/^[\w-]+$/.test(id)) {
+        throw new ValidationError(`Invalid ${context}: '${id}' must contain only alphanumeric characters, underscore, and hyphen`);
+    }
+    if (id.includes('..')) {
+        throw new ValidationError(`Invalid ${context}: '${id}' cannot contain '..'`);
+    }
+}
+function validateMounts(stepId, mounts) {
+    for (const mount of mounts) {
+        if (!mount.host || typeof mount.host !== 'string') {
+            throw new ValidationError(`Step ${stepId}: mount.host is required and must be a string`);
+        }
+        if (mount.host.startsWith('/')) {
+            throw new ValidationError(`Step ${stepId}: mount.host '${mount.host}' must be a relative path`);
+        }
+        if (!mount.container || typeof mount.container !== 'string') {
+            throw new ValidationError(`Step ${stepId}: mount.container is required and must be a string`);
+        }
+        if (!mount.container.startsWith('/')) {
+            throw new ValidationError(`Step ${stepId}: mount.container '${mount.container}' must be an absolute path`);
+        }
+        if (mount.container.includes('..')) {
+            throw new ValidationError(`Step ${stepId}: mount.container '${mount.container}' must not contain '..'`);
+        }
+    }
+}
+function validateCaches(stepId, caches) {
+    for (const cache of caches) {
+        if (!cache.name || typeof cache.name !== 'string') {
+            throw new ValidationError(`Step ${stepId}: cache.name is required and must be a string`);
+        }
+        validateIdentifier(cache.name, `cache name in step ${stepId}`);
+        if (!cache.path || typeof cache.path !== 'string') {
+            throw new ValidationError(`Step ${stepId}: cache.path is required and must be a string`);
+        }
+        if (!cache.path.startsWith('/')) {
+            throw new ValidationError(`Step ${stepId}: cache.path '${cache.path}' must be an absolute path`);
+        }
+    }
+}

package/dist/core/step-runner.js

@@ -0,0 +1,225 @@
+import process from 'node:process';
+import { cp, writeFile } from 'node:fs/promises';
+import { setTimeout } from 'node:timers/promises';
+import { createWriteStream } from 'node:fs';
+import { join } from 'node:path';
+import { ContainerCrashError, PipexError } from '../errors.js';
+import { StateManager } from './state.js';
+import { dirSize, resolveHostPath } from './utils.js';
+/**
+ * Executes a single step in a workspace.
+ * Adapted from PipelineRunner.executeStep() for standalone use.
+ */
+export class StepRunner {
+    runtime;
+    reporter;
+    constructor(runtime, reporter) {
+        this.runtime = runtime;
+        this.reporter = reporter;
+    }
+    async run(options) {
+        const { workspace, state, step, inputs, pipelineRoot, force, ephemeral, job } = options;
+        const stepRef = { id: step.id, displayName: step.name ?? step.id };
+        const resolvedMounts = step.mounts?.map(m => ({
+            hostPath: resolveHostPath(pipelineRoot, m.host),
+            containerPath: m.container
+        }));
+        const currentFingerprint = this.computeFingerprint(step, inputs, resolvedMounts);
+        // Cache check (skip for ephemeral or force)
+        if (!force && !ephemeral) {
+            const cacheResult = await this.tryUseCache({ workspace, state, stepId: step.id, stepRef, fingerprint: currentFingerprint, job });
+            if (cacheResult) {
+                return cacheResult;
+            }
+        }
+        this.reporter.emit({ ...job, event: 'STEP_STARTING', step: stepRef });
+        return this.executeStep({ workspace, state, step, stepRef, inputs, pipelineRoot, ephemeral, currentFingerprint, resolvedMounts, job });
+    }
+    computeFingerprint(step, inputs, resolvedMounts) {
+        const inputRunIds = step.inputs
+            ?.map(i => inputs.get(i.step))
+            .filter((id) => id !== undefined);
+        return StateManager.fingerprint({
+            image: step.image,
+            cmd: step.cmd,
+            env: step.env,
+            inputRunIds,
+            mounts: resolvedMounts
+        });
+    }
+    async tryUseCache({ workspace, state, stepId, stepRef, fingerprint, job }) {
+        const cached = state.getStep(stepId);
+        if (cached?.fingerprint === fingerprint) {
+            const runs = await workspace.listRuns();
+            if (runs.includes(cached.runId)) {
+                await workspace.linkRun(stepId, cached.runId);
+                this.reporter.emit({ ...job, event: 'STEP_SKIPPED', step: stepRef, runId: cached.runId, reason: 'cached' });
+                return { runId: cached.runId, exitCode: 0 };
+            }
+        }
+        return undefined;
+    }
+    async executeStep(ctx) {
+        const { workspace, step, stepRef, inputs, pipelineRoot, ephemeral, job } = ctx;
+        const runId = workspace.generateRunId();
+        const stagingPath = await workspace.prepareRun(runId);
+        await this.prepareStagingInputs(workspace, step, runId, inputs);
+        await this.prepareCaches(workspace, step);
+        const { containerInputs, output, caches, mounts } = this.buildMounts(step, runId, inputs, pipelineRoot);
+        const stdoutLog = createWriteStream(join(stagingPath, 'stdout.log'));
+        const stderrLog = createWriteStream(join(stagingPath, 'stderr.log'));
+        try {
+            const result = await this.executeWithRetries({
+                ctx, containerInputs, output, caches, mounts, stdoutLog, stderrLog
+            });
+            await closeStream(stdoutLog);
+            await closeStream(stderrLog);
+            if (ephemeral) {
+                await workspace.discardRun(runId);
+                this.reporter.emit({ ...job, event: 'STEP_FINISHED', step: stepRef, ephemeral: true });
+                return { exitCode: result.exitCode };
+            }
+            return await this.commitOrDiscard({ ...ctx, runId, stagingPath, result });
+        }
+        catch (error) {
+            await closeStream(stdoutLog);
+            await closeStream(stderrLog);
+            throw error;
+        }
+    }
+    async prepareStagingInputs(workspace, step, runId, inputs) {
+        if (!step.inputs) {
+            return;
+        }
+        for (const input of step.inputs) {
+            const inputRunId = inputs.get(input.step);
+            if (inputRunId && input.copyToOutput) {
+                await cp(workspace.runArtifactsPath(inputRunId), workspace.runStagingArtifactsPath(runId), { recursive: true });
+            }
+        }
+    }
+    async prepareCaches(workspace, step) {
+        if (!step.caches) {
+            return;
+        }
+        for (const cache of step.caches) {
+            await workspace.prepareCache(cache.name);
+        }
+    }
+    buildMounts(step, runId, inputs, pipelineRoot) {
+        const containerInputs = [];
+        if (step.inputs) {
+            for (const input of step.inputs) {
+                const inputRunId = inputs.get(input.step);
+                if (inputRunId) {
+                    containerInputs.push({ runId: inputRunId, containerPath: `/input/${input.step}` });
+                }
+            }
+        }
+        const output = { stagingRunId: runId, containerPath: step.outputPath ?? '/output' };
+        const caches = step.caches?.map(c => ({ name: c.name, containerPath: c.path }));
+        const mounts = step.mounts?.map(m => ({
+            hostPath: resolveHostPath(pipelineRoot, m.host),
+            containerPath: m.container
+        }));
+        return { containerInputs, output, caches, mounts };
+    }
+    async executeWithRetries({ ctx, containerInputs, output, caches, mounts, stdoutLog, stderrLog }) {
+        const { workspace, step, stepRef, pipelineRoot, ephemeral, job } = ctx;
+        const maxRetries = step.retries ?? 0;
+        const retryDelay = step.retryDelayMs ?? 5000;
+        let result;
+        for (let attempt = 0; attempt <= maxRetries; attempt++) {
+            try {
+                result = await this.runtime.run(workspace, {
+                    name: `pipex-${workspace.id}-${step.id}-${Date.now()}`,
+                    image: step.image,
+                    cmd: step.cmd,
+                    env: step.env,
+                    inputs: containerInputs,
+                    output,
+                    caches,
+                    mounts,
+                    sources: step.sources?.map(m => ({
+                        hostPath: resolveHostPath(pipelineRoot, m.host),
+                        containerPath: m.container
+                    })),
+                    network: step.allowNetwork ? 'bridge' : 'none',
+                    timeoutSec: step.timeoutSec
+                }, ({ stream, line }) => {
+                    if (stream === 'stdout') {
+                        stdoutLog.write(line + '\n');
+                    }
+                    else {
+                        stderrLog.write(line + '\n');
+                    }
+                    if (ephemeral && stream === 'stdout') {
+                        process.stdout.write(line + '\n');
+                    }
+                    else {
+                        this.reporter.emit({ ...job, event: 'STEP_LOG', step: stepRef, stream, line });
+                    }
+                });
+                break;
+            }
+            catch (error) {
+                if (error instanceof PipexError && error.transient && attempt < maxRetries) {
+                    this.reporter.emit({ ...job, event: 'STEP_RETRYING', step: stepRef, attempt: attempt + 1, maxRetries });
+                    await setTimeout(retryDelay);
+                    continue;
+                }
+                throw error;
+            }
+        }
+        return result;
+    }
+    async commitOrDiscard({ workspace, state, step, stepRef, inputs, resolvedMounts, currentFingerprint, runId, stagingPath, result, job }) {
+        const meta = {
+            runId,
+            stepId: step.id,
+            stepName: step.name,
+            startedAt: result.startedAt.toISOString(),
+            finishedAt: result.finishedAt.toISOString(),
+            durationMs: result.finishedAt.getTime() - result.startedAt.getTime(),
+            exitCode: result.exitCode,
+            image: step.image,
+            cmd: step.cmd,
+            env: step.env,
+            inputs: step.inputs?.map(i => ({
+                step: i.step,
+                runId: inputs.get(i.step),
+                mountedAs: `/input/${i.step}`
+            })),
+            mounts: resolvedMounts,
+            caches: step.caches?.map(c => c.name),
+            allowNetwork: step.allowNetwork ?? false,
+            fingerprint: currentFingerprint,
+            status: result.exitCode === 0 ? 'success' : 'failure'
+        };
+        await writeFile(join(stagingPath, 'meta.json'), JSON.stringify(meta, null, 2), 'utf8');
+        if (result.exitCode === 0 || step.allowFailure) {
+            await workspace.commitRun(runId);
+            await workspace.linkRun(step.id, runId);
+            state.setStep(step.id, runId, currentFingerprint);
+            await state.save();
+            const durationMs = result.finishedAt.getTime() - result.startedAt.getTime();
+            const artifactSize = await dirSize(workspace.runArtifactsPath(runId));
+            this.reporter.emit({ ...job, event: 'STEP_FINISHED', step: stepRef, runId, durationMs, artifactSize });
+            return { runId, exitCode: result.exitCode };
+        }
+        await workspace.discardRun(runId);
+        this.reporter.emit({ ...job, event: 'STEP_FAILED', step: stepRef, exitCode: result.exitCode });
+        throw new ContainerCrashError(step.id, result.exitCode);
+    }
+}
+async function closeStream(stream) {
+    if (stream.destroyed) {
+        return;
+    }
+    return new Promise((resolve, reject) => {
+        stream.end(() => {
+            resolve();
+        });
+        stream.on('error', reject);
+    });
+}

package/dist/core/stream-reporter.js

@@ -0,0 +1,41 @@
+/**
+ * Reporter that wraps events into TransportMessages and publishes them.
+ * Ignores STEP_LOG events (high volume, not suitable for transport).
+ */
+export class StreamReporter {
+    transport;
+    seq = 0;
+    constructor(transport) {
+        this.transport = transport;
+    }
+    emit(event) {
+        if (event.event === 'STEP_LOG') {
+            return;
+        }
+        const message = {
+            seq: this.seq++,
+            timestamp: new Date().toISOString(),
+            version: 1,
+            type: event.event,
+            event
+        };
+        void this.transport.publish(message);
+    }
+    async flush() {
+        await this.transport.flush?.();
+    }
+}
+/**
+ * Delegates emit() to multiple reporters.
+ */
+export class CompositeReporter {
+    reporters;
+    constructor(...reporters) {
+        this.reporters = reporters;
+    }
+    emit(event) {
+        for (const reporter of this.reporters) {
+            reporter.emit(event);
+        }
+    }
+}

package/dist/core/transport.js

@@ -0,0 +1,9 @@
+export class InMemoryTransport {
+    messages = [];
+    async publish(message) {
+        this.messages.push(message);
+    }
+    clear() {
+        this.messages.length = 0;
+    }
+}

package/dist/core/utils.js

@@ -0,0 +1,56 @@
+import process from 'node:process';
+import { readdir, stat } from 'node:fs/promises';
+import { join, resolve } from 'node:path';
+import { ValidationError } from '../errors.js';
+export async function dirSize(dirPath) {
+    let total = 0;
+    try {
+        const entries = await readdir(dirPath, { withFileTypes: true });
+        for (const entry of entries) {
+            const fullPath = join(dirPath, entry.name);
+            if (entry.isDirectory()) {
+                total += await dirSize(fullPath);
+            }
+            else if (entry.isFile()) {
+                const s = await stat(fullPath);
+                total += s.size;
+            }
+        }
+    }
+    catch {
+        // Directory doesn't exist or isn't readable
+    }
+    return total;
+}
+export function resolveHostPath(pipelineRoot, hostRelative) {
+    const resolved = resolve(pipelineRoot, hostRelative);
+    const boundary = process.cwd();
+    if (!resolved.startsWith(boundary + '/') && resolved !== boundary) {
+        throw new ValidationError(`Mount host '${hostRelative}' resolves to '${resolved}' which is outside the working directory '${boundary}'`);
+    }
+    return resolved;
+}
+export function formatSize(bytes) {
+    if (bytes < 1024) {
+        return `${bytes} B`;
+    }
+    if (bytes < 1024 * 1024) {
+        return `${(bytes / 1024).toFixed(1)} KB`;
+    }
+    if (bytes < 1024 * 1024 * 1024) {
+        return `${(bytes / (1024 * 1024)).toFixed(1)} MB`;
+    }
+    return `${(bytes / (1024 * 1024 * 1024)).toFixed(1)} GB`;
+}
+export function formatDuration(ms) {
+    if (ms < 1000) {
+        return `${ms}ms`;
+    }
+    const seconds = ms / 1000;
+    if (seconds < 60) {
+        return `${seconds.toFixed(1)}s`;
+    }
+    const minutes = Math.floor(seconds / 60);
+    const remainingSeconds = Math.round(seconds % 60);
+    return `${minutes}m ${remainingSeconds}s`;
+}