@livingdata/pipex 0.0.7 → 0.0.9

package/dist/engine/workspace.js

@@ -1,4 +1,4 @@
-import { access, mkdir, readdir, rename, rm } from 'node:fs/promises';
+import { access, mkdir, readdir, rename, rm, symlink } from 'node:fs/promises';
 import { randomUUID } from 'node:crypto';
 import { join } from 'node:path';
 /**
@@ -6,18 +6,22 @@ import { join } from 'node:path';
  *
  * A workspace provides:
  * - **staging/**: Temporary write location during execution
- * - **artifacts/**: Committed outputs (immutable, read-only)
+ * - **runs/**: Committed run outputs (immutable, read-only)
  * - **caches/**: Persistent read-write caches (shared across steps)
  * - **state.json**: Managed by orchestration layer (e.g., CLI) for caching
  *
- * ## Artifact Lifecycle
+ * ## Run Lifecycle
  *
- * 1. `prepareArtifact()` creates `staging/{artifactId}/`
- * 2. Container writes to `staging/{artifactId}/` (mounted as `/output`)
- * 3. Success: `commitArtifact()` atomically moves to `artifacts/{artifactId}/`
- *    OR Failure: `discardArtifact()` deletes `staging/{artifactId}/`
+ * Each step execution produces a **run**, a structured directory containing
+ * artifacts (files produced by the step), logs (stdout/stderr), and metadata.
  *
- * Artifacts are immutable once committed.
+ * 1. `prepareRun()` creates `staging/{runId}/` with `artifacts/` subdirectory
+ * 2. Container writes to `staging/{runId}/artifacts/` (mounted as `/output`)
+ * 3. Orchestration layer writes logs and metadata to `staging/{runId}/`
+ * 4. Success: `commitRun()` atomically moves to `runs/{runId}/`
+ *    OR Failure: `discardRun()` deletes `staging/{runId}/`
+ *
+ * Runs are immutable once committed.
  *
  * ## Cache Lifecycle
  *
@@ -30,12 +34,12 @@ import { join } from 'node:path';
  * @example
  * ```typescript
  * const ws = await Workspace.create('/tmp/workdir', 'my-workspace')
- * const artifactId = ws.generateArtifactId()
- * await ws.prepareArtifact(artifactId)
+ * const runId = ws.generateRunId()
+ * await ws.prepareRun(runId)
  * await ws.prepareCache('pnpm-store')
  * // ... container execution ...
- * await ws.commitArtifact(artifactId) // On success
- * // OR await ws.discardArtifact(artifactId) // On failure
+ * await ws.commitRun(runId) // On success
+ * // OR await ws.discardRun(runId) // On failure
  * ```
  */
 export class Workspace {
@@ -49,7 +53,7 @@ export class Workspace {
         return Workspace.generateId();
     }
     /**
-     * Creates a new workspace with staging, artifacts, and caches directories.
+     * Creates a new workspace with staging, runs, and caches directories.
      * @param workdirRoot - Root directory for all workspaces
      * @param id - Optional workspace ID (auto-generated if omitted)
      * @returns Newly created workspace
@@ -58,7 +62,7 @@ export class Workspace {
         const workspaceId = id ?? Workspace.generateWorkspaceId();
         const root = join(workdirRoot, workspaceId);
         await mkdir(join(root, 'staging'), { recursive: true });
-        await mkdir(join(root, 'artifacts'), { recursive: true });
+        await mkdir(join(root, 'runs'), { recursive: true });
         await mkdir(join(root, 'caches'), { recursive: true });
         return new Workspace(workspaceId, root);
     }
@@ -113,62 +117,89 @@ export class Workspace {
         this.root = root;
     }
     /**
-     * Generates a unique artifact identifier.
-     * @returns Artifact ID in format: `{timestamp}-{uuid-prefix}`
+     * Generates a unique run identifier.
+     * @returns Run ID in format: `{timestamp}-{uuid-prefix}`
      */
-    generateArtifactId() {
+    generateRunId() {
         return Workspace.generateId();
     }
     /**
-     * Returns the staging directory path for an artifact.
-     * Staging is used for temporary writes during execution.
-     * @param artifactId - Artifact identifier
+     * Returns the staging directory path for a run.
+     * @param runId - Run identifier
      * @returns Absolute path to staging directory
-     * @throws If artifact ID is invalid
      */
-    stagingPath(artifactId) {
-        this.validateArtifactId(artifactId);
-        return join(this.root, 'staging', artifactId);
+    runStagingPath(runId) {
+        this.validateRunId(runId);
+        return join(this.root, 'staging', runId);
+    }
+    /**
+     * Returns the staging artifacts directory path for a run.
+     * @param runId - Run identifier
+     * @returns Absolute path to staging artifacts directory
+     */
+    runStagingArtifactsPath(runId) {
+        return join(this.runStagingPath(runId), 'artifacts');
     }
     /**
-     * Returns the committed artifact directory path.
-     * Artifacts are immutable once committed.
-     * @param artifactId - Artifact identifier
-     * @returns Absolute path to artifact directory
-     * @throws If artifact ID is invalid
+     * Returns the committed run directory path.
+     * @param runId - Run identifier
+     * @returns Absolute path to run directory
      */
-    artifactPath(artifactId) {
-        this.validateArtifactId(artifactId);
-        return join(this.root, 'artifacts', artifactId);
+    runPath(runId) {
+        this.validateRunId(runId);
+        return join(this.root, 'runs', runId);
     }
     /**
-     * Prepares a staging directory for a new artifact.
-     * @param artifactId - Artifact identifier
+     * Returns the artifacts directory path within a committed run.
+     * @param runId - Run identifier
+     * @returns Absolute path to run artifacts directory
+     */
+    runArtifactsPath(runId) {
+        return join(this.runPath(runId), 'artifacts');
+    }
+    /**
+     * Prepares a staging directory for a new run.
+     * Creates both the run directory and its artifacts subdirectory.
+     * @param runId - Run identifier
      * @returns Absolute path to the created staging directory
      */
-    async prepareArtifact(artifactId) {
-        const path = this.stagingPath(artifactId);
+    async prepareRun(runId) {
+        const path = this.runStagingPath(runId);
         await mkdir(path, { recursive: true });
+        await mkdir(join(path, 'artifacts'), { recursive: true });
         return path;
     }
     /**
-     * Commits a staging artifact to the artifacts directory.
+     * Commits a staging run to the runs directory.
      * Uses atomic rename operation for consistency.
-     * @param artifactId - Artifact identifier
+     * @param runId - Run identifier
      */
-    async commitArtifact(artifactId) {
-        await rename(this.stagingPath(artifactId), this.artifactPath(artifactId));
+    async commitRun(runId) {
+        await rename(this.runStagingPath(runId), this.runPath(runId));
     }
     /**
-     * Discards a staging artifact (on execution failure).
-     * @param artifactId - Artifact identifier
+     * Creates a symlink from `step-runs/{stepId}` to the committed run.
+     * Replaces any existing symlink for the same step.
+     * @param stepId - Step identifier
+     * @param runId - Committed run identifier
      */
-    async discardArtifact(artifactId) {
-        await rm(this.stagingPath(artifactId), { recursive: true, force: true });
+    async linkRun(stepId, runId) {
+        const dir = join(this.root, 'step-runs');
+        await mkdir(dir, { recursive: true });
+        const linkPath = join(dir, stepId);
+        await rm(linkPath, { force: true });
+        await symlink(join('..', 'runs', runId), linkPath);
+    }
+    /**
+     * Discards a staging run (on execution failure).
+     * @param runId - Run identifier
+     */
+    async discardRun(runId) {
+        await rm(this.runStagingPath(runId), { recursive: true, force: true });
     }
     /**
      * Removes all staging directories.
-     * Should be called on workspace initialization to clean up incomplete artifacts.
+     * Should be called on workspace initialization to clean up incomplete runs.
      */
     async cleanupStaging() {
         const stagingDir = join(this.root, 'staging');
@@ -185,18 +216,34 @@ export class Workspace {
         }
     }
     /**
-     * Lists all committed artifact IDs in this workspace.
-     * @returns Array of artifact IDs (directory names in artifacts/)
+     * Lists all committed run IDs in this workspace.
+     * @returns Array of run IDs (directory names in runs/)
      */
-    async listArtifacts() {
+    async listRuns() {
         try {
-            const entries = await readdir(join(this.root, 'artifacts'), { withFileTypes: true });
+            const entries = await readdir(join(this.root, 'runs'), { withFileTypes: true });
             return entries.filter(e => e.isDirectory()).map(e => e.name);
         }
         catch {
             return [];
         }
     }
+    /**
+     * Removes runs not in the given set of active run IDs.
+     * @param activeRunIds - Set of run IDs to keep
+     * @returns Number of runs removed
+     */
+    async pruneRuns(activeRunIds) {
+        const allRuns = await this.listRuns();
+        let removed = 0;
+        for (const runId of allRuns) {
+            if (!activeRunIds.has(runId)) {
+                await rm(join(this.root, 'runs', runId), { recursive: true, force: true });
+                removed++;
+            }
+        }
+        return removed;
+    }
     /**
      * Returns the cache directory path.
      * Caches are persistent read-write directories shared across steps.
@@ -233,22 +280,22 @@ export class Workspace {
         }
     }
     /**
-     * Validates an artifact ID to prevent path traversal attacks.
-     * @param id - Artifact identifier to validate
-     * @throws If the artifact ID contains invalid characters or path traversal attempts
+     * Validates a run ID to prevent path traversal attacks.
+     * @param id - Run identifier to validate
+     * @throws If the run ID contains invalid characters or path traversal attempts
      * @internal
      */
-    validateArtifactId(id) {
+    validateRunId(id) {
         if (!/^[\w-]+$/.test(id)) {
-            throw new Error(`Invalid artifact ID: ${id}. Must contain only alphanumeric characters, dashes, and underscores.`);
+            throw new Error(`Invalid run ID: ${id}. Must contain only alphanumeric characters, dashes, and underscores.`);
         }
         if (id.includes('..')) {
-            throw new Error(`Invalid artifact ID: ${id}. Path traversal is not allowed.`);
+            throw new Error(`Invalid run ID: ${id}. Path traversal is not allowed.`);
         }
     }
     /**
      * Validates a cache name to prevent path traversal attacks.
-     * Same rules as artifact IDs: alphanumeric, dashes, underscores only.
+     * Same rules as run IDs: alphanumeric, dashes, underscores only.
      * @param name - Cache name to validate
      * @throws If the cache name contains invalid characters or path traversal attempts
      * @internal

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@livingdata/pipex",
-  "version": "0.0.7",
+  "version": "0.0.9",
   "description": "Execution engine for containerized pipeline steps",
   "author": "Jérôme Desboeufs <jerome@livingdata.co>",
   "type": "module",