@livingdata/pipex 0.0.8 → 0.0.10

package/dist/engine/__tests__/workspace.js

@@ -0,0 +1,288 @@
+import { access, readdir, readlink, writeFile } from 'node:fs/promises';
+import { join } from 'node:path';
+import test from 'ava';
+import { Workspace } from '../workspace.js';
+import { WorkspaceError } from '../../errors.js';
+import { createTmpDir } from '../../__tests__/helpers.js';
+// -- create & open -----------------------------------------------------------
+test('create makes staging/, runs/, caches/ directories', async (t) => {
+    const root = await createTmpDir();
+    const ws = await Workspace.create(root, 'ws-1');
+    const entries = await readdir(ws.root);
+    t.true(entries.includes('staging'));
+    t.true(entries.includes('runs'));
+    t.true(entries.includes('caches'));
+});
+test('create with custom ID uses that ID', async (t) => {
+    const root = await createTmpDir();
+    const ws = await Workspace.create(root, 'my-workspace');
+    t.is(ws.id, 'my-workspace');
+});
+test('create without ID auto-generates one', async (t) => {
+    const root = await createTmpDir();
+    const ws = await Workspace.create(root);
+    t.truthy(ws.id);
+    t.true(ws.id.length > 0);
+});
+test('open returns workspace with correct id and root', async (t) => {
+    const root = await createTmpDir();
+    await Workspace.create(root, 'existing');
+    const ws = await Workspace.open(root, 'existing');
+    t.is(ws.id, 'existing');
+    t.is(ws.root, join(root, 'existing'));
+});
+test('open throws if workspace missing', async (t) => {
+    const root = await createTmpDir();
+    await t.throwsAsync(async () => Workspace.open(root, 'nonexistent'));
+});
+// -- list & remove -----------------------------------------------------------
+test('list returns sorted workspace IDs', async (t) => {
+    const root = await createTmpDir();
+    await Workspace.create(root, 'beta');
+    await Workspace.create(root, 'alpha');
+    await Workspace.create(root, 'gamma');
+    const ids = await Workspace.list(root);
+    t.deepEqual(ids, ['alpha', 'beta', 'gamma']);
+});
+test('list returns empty array if root does not exist', async (t) => {
+    const ids = await Workspace.list('/tmp/nonexistent-pipex-root-' + Date.now());
+    t.deepEqual(ids, []);
+});
+test('remove deletes workspace', async (t) => {
+    const root = await createTmpDir();
+    await Workspace.create(root, 'to-delete');
+    await Workspace.remove(root, 'to-delete');
+    const ids = await Workspace.list(root);
+    t.false(ids.includes('to-delete'));
+});
+test('remove throws WorkspaceError on path traversal with ..', async (t) => {
+    const root = await createTmpDir();
+    const error = await t.throwsAsync(async () => Workspace.remove(root, '../etc'));
+    t.true(error instanceof WorkspaceError);
+});
+test('remove throws WorkspaceError on path traversal with /', async (t) => {
+    const error = await t.throwsAsync(async () => Workspace.remove('/tmp', 'a/b'));
+    t.true(error instanceof WorkspaceError);
+});
+// -- run lifecycle: commit path ----------------------------------------------
+test('prepareRun creates staging/{runId}/artifacts/', async (t) => {
+    const root = await createTmpDir();
+    const ws = await Workspace.create(root, 'run-test');
+    const runId = ws.generateRunId();
+    await ws.prepareRun(runId);
+    const stagingPath = ws.runStagingPath(runId);
+    await t.notThrowsAsync(async () => access(stagingPath));
+    await t.notThrowsAsync(async () => access(join(stagingPath, 'artifacts')));
+});
+test('commitRun moves staging to runs', async (t) => {
+    const root = await createTmpDir();
+    const ws = await Workspace.create(root, 'commit-test');
+    const runId = ws.generateRunId();
+    await ws.prepareRun(runId);
+    // Write a file into staging artifacts
+    await writeFile(join(ws.runStagingArtifactsPath(runId), 'file.txt'), 'hello');
+    await ws.commitRun(runId);
+    // Committed run should exist
+    await t.notThrowsAsync(async () => access(ws.runPath(runId)));
+    await t.notThrowsAsync(async () => access(join(ws.runArtifactsPath(runId), 'file.txt')));
+});
+test('committed run appears in listRuns', async (t) => {
+    const root = await createTmpDir();
+    const ws = await Workspace.create(root, 'list-runs');
+    const runId = ws.generateRunId();
+    await ws.prepareRun(runId);
+    await ws.commitRun(runId);
+    const runs = await ws.listRuns();
+    t.true(runs.includes(runId));
+});
+test('staging is empty after commit', async (t) => {
+    const root = await createTmpDir();
+    const ws = await Workspace.create(root, 'staging-empty');
+    const runId = ws.generateRunId();
+    await ws.prepareRun(runId);
+    await ws.commitRun(runId);
+    const staging = await readdir(join(ws.root, 'staging'));
+    t.deepEqual(staging, []);
+});
+// -- run lifecycle: discard path ---------------------------------------------
+test('discardRun removes staging directory', async (t) => {
+    const root = await createTmpDir();
+    const ws = await Workspace.create(root, 'discard-test');
+    const runId = ws.generateRunId();
+    await ws.prepareRun(runId);
+    await ws.discardRun(runId);
+    const staging = await readdir(join(ws.root, 'staging'));
+    t.deepEqual(staging, []);
+});
+test('discarded run does not appear in listRuns', async (t) => {
+    const root = await createTmpDir();
+    const ws = await Workspace.create(root, 'discard-list');
+    const runId = ws.generateRunId();
+    await ws.prepareRun(runId);
+    await ws.discardRun(runId);
+    const runs = await ws.listRuns();
+    t.false(runs.includes(runId));
+});
+// -- cleanupStaging ----------------------------------------------------------
+test('cleanupStaging removes all staging dirs', async (t) => {
+    const root = await createTmpDir();
+    const ws = await Workspace.create(root, 'cleanup-test');
+    await ws.prepareRun(ws.generateRunId());
+    await ws.prepareRun(ws.generateRunId());
+    await ws.cleanupStaging();
+    const staging = await readdir(join(ws.root, 'staging'));
+    t.deepEqual(staging, []);
+});
+test('cleanupStaging is no-op when empty', async (t) => {
+    const root = await createTmpDir();
+    const ws = await Workspace.create(root, 'cleanup-noop');
+    await t.notThrowsAsync(async () => ws.cleanupStaging());
+});
+// -- linkRun -----------------------------------------------------------------
+test('linkRun creates step-runs/{stepId} symlink', async (t) => {
+    const root = await createTmpDir();
+    const ws = await Workspace.create(root, 'link-test');
+    const runId = ws.generateRunId();
+    await ws.prepareRun(runId);
+    await ws.commitRun(runId);
+    await ws.linkRun('my-step', runId);
+    const linkDir = join(ws.root, 'step-runs');
+    const entries = await readdir(linkDir);
+    t.true(entries.includes('my-step'));
+});
+test('linkRun replaces existing symlink with new target', async (t) => {
+    const root = await createTmpDir();
+    const ws = await Workspace.create(root, 'link-replace');
+    const runId1 = ws.generateRunId();
+    await ws.prepareRun(runId1);
+    await ws.commitRun(runId1);
+    await ws.linkRun('step-a', runId1);
+    const runId2 = ws.generateRunId();
+    await ws.prepareRun(runId2);
+    await ws.commitRun(runId2);
+    await ws.linkRun('step-a', runId2);
+    // Symlink must point to the second run, not the first
+    const target = await readlink(join(ws.root, 'step-runs', 'step-a'));
+    t.true(target.includes(runId2));
+    t.false(target.includes(runId1));
+});
+// -- pruneRuns ---------------------------------------------------------------
+test('pruneRuns removes runs not in active set', async (t) => {
+    const root = await createTmpDir();
+    const ws = await Workspace.create(root, 'prune-test');
+    const keep = ws.generateRunId();
+    await ws.prepareRun(keep);
+    await ws.commitRun(keep);
+    const remove = ws.generateRunId();
+    await ws.prepareRun(remove);
+    await ws.commitRun(remove);
+    const removed = await ws.pruneRuns(new Set([keep]));
+    t.is(removed, 1);
+    const runs = await ws.listRuns();
+    t.true(runs.includes(keep));
+    t.false(runs.includes(remove));
+});
+test('pruneRuns keeps runs in active set', async (t) => {
+    const root = await createTmpDir();
+    const ws = await Workspace.create(root, 'prune-keep');
+    const run1 = ws.generateRunId();
+    await ws.prepareRun(run1);
+    await ws.commitRun(run1);
+    const run2 = ws.generateRunId();
+    await ws.prepareRun(run2);
+    await ws.commitRun(run2);
+    const removed = await ws.pruneRuns(new Set([run1, run2]));
+    t.is(removed, 0);
+});
+// -- caches ------------------------------------------------------------------
+test('prepareCache creates cache directory', async (t) => {
+    const root = await createTmpDir();
+    const ws = await Workspace.create(root, 'cache-test');
+    const path = await ws.prepareCache('npm-cache');
+    await t.notThrowsAsync(async () => access(path));
+});
+test('listCaches lists cache names', async (t) => {
+    const root = await createTmpDir();
+    const ws = await Workspace.create(root, 'cache-list');
+    await ws.prepareCache('cache-a');
+    await ws.prepareCache('cache-b');
+    const caches = await ws.listCaches();
+    t.true(caches.includes('cache-a'));
+    t.true(caches.includes('cache-b'));
+});
+test('invalid cache name throws WorkspaceError', async (t) => {
+    const root = await createTmpDir();
+    const ws = await Workspace.create(root, 'cache-invalid');
+    const error = t.throws(() => ws.cachePath('invalid/name'));
+    t.true(error instanceof WorkspaceError);
+    t.true(error.message.includes('invalid/name'));
+});
+// -- running step markers ----------------------------------------------------
+test('markStepRunning creates running/{stepId} file', async (t) => {
+    const root = await createTmpDir();
+    const ws = await Workspace.create(root, 'running-mark');
+    await ws.markStepRunning('build', { startedAt: '2024-01-01T00:00:00Z', pid: 12345 });
+    const entries = await readdir(join(ws.root, 'running'));
+    t.deepEqual(entries, ['build']);
+});
+test('markStepDone removes the running marker', async (t) => {
+    const root = await createTmpDir();
+    const ws = await Workspace.create(root, 'running-done');
+    await ws.markStepRunning('build', { startedAt: '2024-01-01T00:00:00Z', pid: 12345 });
+    await ws.markStepDone('build');
+    const entries = await readdir(join(ws.root, 'running'));
+    t.deepEqual(entries, []);
+});
+test('markStepDone is no-op for non-existent marker', async (t) => {
+    const root = await createTmpDir();
+    const ws = await Workspace.create(root, 'running-done-noop');
+    await t.notThrowsAsync(async () => ws.markStepDone('nonexistent'));
+});
+test('listRunningSteps returns all running steps with metadata', async (t) => {
+    const root = await createTmpDir();
+    const ws = await Workspace.create(root, 'running-list');
+    await ws.markStepRunning('build', { startedAt: '2024-01-01T00:00:00Z', pid: 111, stepName: 'Build' });
+    await ws.markStepRunning('test', { startedAt: '2024-01-01T00:01:00Z', pid: 222 });
+    const running = await ws.listRunningSteps();
+    t.is(running.length, 2);
+    const build = running.find(r => r.stepId === 'build');
+    t.is(build.pid, 111);
+    t.is(build.stepName, 'Build');
+    t.is(build.startedAt, '2024-01-01T00:00:00Z');
+    const testStep = running.find(r => r.stepId === 'test');
+    t.is(testStep.pid, 222);
+    t.is(testStep.stepName, undefined);
+});
+test('listRunningSteps returns empty array when no running directory', async (t) => {
+    const root = await createTmpDir();
+    const ws = await Workspace.create(root, 'running-empty');
+    const running = await ws.listRunningSteps();
+    t.deepEqual(running, []);
+});
+test('cleanupRunning removes all running markers', async (t) => {
+    const root = await createTmpDir();
+    const ws = await Workspace.create(root, 'running-cleanup');
+    await ws.markStepRunning('a', { startedAt: '2024-01-01T00:00:00Z', pid: 1 });
+    await ws.markStepRunning('b', { startedAt: '2024-01-01T00:00:00Z', pid: 2 });
+    await ws.cleanupRunning();
+    const running = await ws.listRunningSteps();
+    t.deepEqual(running, []);
+});
+test('cleanupRunning is no-op when no running directory', async (t) => {
+    const root = await createTmpDir();
+    const ws = await Workspace.create(root, 'running-cleanup-noop');
+    await t.notThrowsAsync(async () => ws.cleanupRunning());
+});
+// -- validation --------------------------------------------------------------
+test('invalid run ID in runStagingPath throws WorkspaceError', async (t) => {
+    const root = await createTmpDir();
+    const ws = await Workspace.create(root, 'validate-staging');
+    const error = t.throws(() => ws.runStagingPath('bad/id'));
+    t.true(error instanceof WorkspaceError);
+});
+test('invalid run ID in runPath throws WorkspaceError', async (t) => {
+    const root = await createTmpDir();
+    const ws = await Workspace.create(root, 'validate-run');
+    const error = t.throws(() => ws.runPath('bad id'));
+    t.true(error instanceof WorkspaceError);
+});

package/dist/engine/docker-executor.js

@@ -1,5 +1,6 @@
 import process from 'node:process';
 import { execa } from 'execa';
+import { DockerNotAvailableError, ImagePullError, ContainerTimeoutError } from '../errors.js';
 import { ContainerExecutor } from './executor.js';
 /**
  * Build a minimal environment for the Docker CLI process.
@@ -22,8 +23,26 @@ export class DockerCliExecutor extends ContainerExecutor {
         try {
             await execa('docker', ['--version'], { env: this.env });
         }
+        catch (error) {
+            throw new DockerNotAvailableError({ cause: error });
+        }
+    }
+    /**
+     * Remove any leftover pipex containers for the given workspace.
+     * Called before pipeline execution to clean up after crashes.
+     */
+    async cleanupContainers(workspaceId) {
+        try {
+            const { stdout } = await execa('docker', [
+                'ps', '-a', '--filter', `label=pipex.workspace=${workspaceId}`, '-q'
+            ], { env: this.env });
+            const ids = stdout.trim().split('\n').filter(Boolean);
+            if (ids.length > 0) {
+                await execa('docker', ['rm', '-f', ...ids], { env: this.env, reject: false });
+            }
+        }
         catch {
-            throw new Error('Docker CLI not found. Please install Docker.');
+            // Best effort
         }
     }
     async run(workspace, request, onLogLine) {
@@ -31,15 +50,15 @@ export class DockerCliExecutor extends ContainerExecutor {
         // Use create+start instead of run: docker run cannot create mountpoints
         // for anonymous volumes inside read-only bind mounts (shadow paths).
         // docker create sets up the filesystem layer before readonly applies.
-        const args = ['create', '--name', request.name, '--network', request.network];
+        const args = ['create', '--name', request.name, '--network', request.network, '--label', 'pipex=true', '--label', `pipex.workspace=${workspace.id}`];
         if (request.env) {
             for (const [key, value] of Object.entries(request.env)) {
                 args.push('-e', `${key}=${value}`);
             }
         }
-        // Mount inputs (committed artifacts, read-only)
+        // Mount inputs (committed run artifacts, read-only)
         for (const input of request.inputs) {
-            const hostPath = workspace.artifactPath(input.artifactId);
+            const hostPath = workspace.runArtifactsPath(input.runId);
             args.push('-v', `${hostPath}:${input.containerPath}:ro`);
         }
         // Mount caches (persistent, read-write)
@@ -55,8 +74,8 @@ export class DockerCliExecutor extends ContainerExecutor {
                 args.push('-v', `${mount.hostPath}:${mount.containerPath}:ro`);
             }
         }
-        // Mount output (staging artifact, read-write)
-        const outputHostPath = workspace.stagingPath(request.output.stagingArtifactId);
+        // Mount output (staging run artifacts, read-write)
+        const outputHostPath = workspace.runStagingArtifactsPath(request.output.stagingRunId);
         args.push('-v', `${outputHostPath}:${request.output.containerPath}:rw`, request.image, ...request.cmd);
         let exitCode = 0;
         let error;
@@ -90,6 +109,13 @@ export class DockerCliExecutor extends ContainerExecutor {
             exitCode = result.exitCode ?? 0;
         }
         catch (error_) {
+            if (error_ instanceof Error && 'timedOut' in error_ && error_.timedOut) {
+                throw new ContainerTimeoutError(request.timeoutSec ?? 0, { cause: error_ });
+            }
+            const stderr = error_ instanceof Error && 'stderr' in error_ ? String(error_.stderr) : '';
+            if (/unable to find image|pull access denied|manifest unknown/i.test(stderr)) {
+                throw new ImagePullError(request.image, { cause: error_ });
+            }
             exitCode = 1;
             error = error_ instanceof Error ? error_.message : String(error_);
         }

package/dist/engine/index.js

@@ -1,3 +1,4 @@
 export { Workspace } from './workspace.js';
 export { ContainerExecutor } from './executor.js';
 export { DockerCliExecutor } from './docker-executor.js';
+export { PipexError, DockerError, DockerNotAvailableError, ImagePullError, ContainerTimeoutError, ContainerCrashError, ContainerCleanupError, WorkspaceError, ArtifactNotFoundError, StagingError, PipelineError, ValidationError, CyclicDependencyError, StepNotFoundError, KitError, MissingParameterError } from '../errors.js';