@elaraai/e3-core 0.0.2-beta.3 → 0.0.2-beta.31

package/dist/src/dataflow.js

@@ -21,15 +21,60 @@
  * output is written to the workspace and dependents are notified only after the
  * write completes, ensuring downstream tasks see consistent state.
  */
-import { decodeBeast2For } from '@elaraai/east';
+import { decodeBeast2For, encodeBeast2For, variant } from '@elaraai/east';
 import { PackageObjectType, TaskObjectType, WorkspaceStateType, pathToString, } from '@elaraai/e3-types';
-import { objectRead } from './objects.js';
-import { taskExecute, executionGetOutput, inputsHash, } from './executions.js';
+import { executionGetOutput, inputsHash, } from './executions.js';
+import { uuidv7 } from './uuid.js';
+import { taskExecute } from './execution/LocalTaskRunner.js';
 import { workspaceGetDatasetHash, workspaceSetDatasetByHash, } from './trees.js';
-import { E3Error, WorkspaceNotFoundError, WorkspaceNotDeployedError, TaskNotFoundError, DataflowError, DataflowAbortedError, isNotFoundError, } from './errors.js';
-import { acquireWorkspaceLock, } from './workspaceLock.js';
-import * as fs from 'fs/promises';
-import * as path from 'path';
+import { E3Error, WorkspaceNotFoundError, WorkspaceNotDeployedError, WorkspaceLockError, TaskNotFoundError, DataflowError, DataflowAbortedError, } from './errors.js';
+// =============================================================================
+// Path Parsing Helper
+// =============================================================================
+/**
+ * Parse a keypath string (from pathToString) back to TreePath.
+ *
+ * The keypath format is: .field1.field2 (dot-separated field names)
+ * Quoted identifiers use backticks: .field1.`complex/name`
+ *
+ * @param pathStr - The path string in keypath format
+ * @returns TreePath array of path segments
+ */
+export function parsePathString(pathStr) {
+    if (!pathStr.startsWith('.')) {
+        throw new Error(`Invalid path string: expected '.' prefix, got '${pathStr}'`);
+    }
+    const segments = [];
+    let i = 1; // Skip the leading '.'
+    while (i < pathStr.length) {
+        let fieldName;
+        if (pathStr[i] === '`') {
+            // Quoted identifier: find closing backtick
+            const endQuote = pathStr.indexOf('`', i + 1);
+            if (endQuote === -1) {
+                throw new Error(`Invalid path string: unclosed backtick at position ${i}`);
+            }
+            fieldName = pathStr.slice(i + 1, endQuote);
+            i = endQuote + 1;
+        }
+        else {
+            // Unquoted identifier: read until '.' or end
+            let end = pathStr.indexOf('.', i);
+            if (end === -1)
+                end = pathStr.length;
+            fieldName = pathStr.slice(i, end);
+            i = end;
+        }
+        if (fieldName) {
+            segments.push(variant('field', fieldName));
+        }
+        // Skip the '.' separator
+        if (i < pathStr.length && pathStr[i] === '.') {
+            i++;
+        }
+    }
+    return segments;
+}
 // =============================================================================
 // Async Mutex for Workspace Updates
 // =============================================================================
@@ -81,27 +126,20 @@ class AsyncMutex {
 // Workspace State Reader
 // =============================================================================
 /**
- * Read workspace state from file.
+ * Read workspace state.
  * @throws {WorkspaceNotFoundError} If workspace doesn't exist
  * @throws {WorkspaceNotDeployedError} If workspace has no package deployed
  */
-async function readWorkspaceState(repoPath, ws) {
-    const stateFile = path.join(repoPath, 'workspaces', `${ws}.beast2`);
-    let data;
-    try {
-        data = await fs.readFile(stateFile);
-    }
-    catch (err) {
-        if (isNotFoundError(err)) {
-            throw new WorkspaceNotFoundError(ws);
-        }
-        throw err;
+async function readWorkspaceState(storage, repo, ws) {
+    const data = await storage.refs.workspaceRead(repo, ws);
+    if (data === null) {
+        throw new WorkspaceNotFoundError(ws);
     }
     if (data.length === 0) {
         throw new WorkspaceNotDeployedError(ws);
     }
     const decoder = decodeBeast2For(WorkspaceStateType);
-    return decoder(data);
+    return decoder(Buffer.from(data));
 }
 // =============================================================================
 // Dependency Graph Building
@@ -114,11 +152,11 @@ async function readWorkspaceState(repoPath, ws) {
  * - outputToTask: Map of output path string -> task name
  * - taskDependents: Map of task name -> set of dependent task names
  */
-async function buildDependencyGraph(repoPath, ws) {
+async function buildDependencyGraph(storage, repo, ws) {
     // Read workspace state to get package hash
-    const state = await readWorkspaceState(repoPath, ws);
+    const state = await readWorkspaceState(storage, repo, ws);
     // Read package object to get tasks map
-    const pkgData = await objectRead(repoPath, state.packageHash);
+    const pkgData = await storage.objects.read(repo, state.packageHash);
     const pkgDecoder = decodeBeast2For(PackageObjectType);
     const pkgObject = pkgDecoder(Buffer.from(pkgData));
     const taskNodes = new Map();
@@ -126,7 +164,7 @@ async function buildDependencyGraph(repoPath, ws) {
     // First pass: load all tasks and build output->task map
     const taskDecoder = decodeBeast2For(TaskObjectType);
     for (const [taskName, taskHash] of pkgObject.tasks) {
-        const taskData = await objectRead(repoPath, taskHash);
+        const taskData = await storage.objects.read(repo, taskHash);
         const task = taskDecoder(Buffer.from(taskData));
         const outputPathStr = pathToString(task.output);
         outputToTask.set(outputPathStr, taskName);
@@ -159,7 +197,7 @@ async function buildDependencyGraph(repoPath, ws) {
             }
             // If not produced by a task, it's an external input - check if assigned
             else {
-                const { refType } = await workspaceGetDatasetHash(repoPath, ws, inputPath);
+                const { refType } = await workspaceGetDatasetHash(storage, repo, ws, inputPath);
                 if (refType === 'unassigned') {
                     // External input that is unassigned - this task can never run
                     node.unresolvedCount++;
@@ -183,7 +221,8 @@ async function buildDependencyGraph(repoPath, ws) {
  * to prevent concurrent modifications. If options.lock is provided, uses that
  * lock instead (caller is responsible for releasing it).
  *
- * @param repoPath - Path to .e3 repository
+ * @param storage - Storage backend
+ * @param repo - Repository identifier (for local storage, the path to e3 repository directory)
  * @param ws - Workspace name
  * @param options - Execution options
  * @returns Result of the dataflow execution
@@ -193,12 +232,16 @@ async function buildDependencyGraph(repoPath, ws) {
  * @throws {TaskNotFoundError} If filter specifies a task that doesn't exist
  * @throws {DataflowError} If execution fails for other reasons
  */
-export async function dataflowExecute(repoPath, ws, options = {}) {
+export async function dataflowExecute(storage, repo, ws, options = {}) {
     // Acquire lock if not provided externally
     const externalLock = options.lock;
-    const lock = externalLock ?? await acquireWorkspaceLock(repoPath, ws);
+    const lock = externalLock ?? await storage.locks.acquire(repo, ws, variant('dataflow', null));
+    if (!lock) {
+        // Lock couldn't be acquired - the LockService returns null instead of throwing
+        throw new WorkspaceLockError(ws);
+    }
     try {
-        return await dataflowExecuteWithLock(repoPath, ws, options);
+        return await dataflowExecuteWithLock(storage, repo, ws, options);
     }
     finally {
         // Only release the lock if we acquired it internally
@@ -213,7 +256,8 @@ export async function dataflowExecute(repoPath, ws, options = {}) {
  * Returns a promise immediately without awaiting execution. The lock is
  * released automatically when execution completes.
  *
- * @param repoPath - Path to .e3 repository
+ * @param storage - Storage backend
+ * @param repo - Repository identifier (for local storage, the path to e3 repository directory)
  * @param ws - Workspace name
  * @param options - Execution options (lock must be provided)
  * @returns Promise that resolves when execution completes
@@ -222,23 +266,31 @@ export async function dataflowExecute(repoPath, ws, options = {}) {
  * @throws {TaskNotFoundError} If filter specifies a task that doesn't exist
  * @throws {DataflowError} If execution fails for other reasons
  */
-export function dataflowStart(repoPath, ws, options) {
-    return dataflowExecuteWithLock(repoPath, ws, options)
+export function dataflowStart(storage, repo, ws, options) {
+    return dataflowExecuteWithLock(storage, repo, ws, options)
         .finally(() => options.lock.release());
 }
 /**
  * Internal: Execute dataflow with lock already held.
  */
-async function dataflowExecuteWithLock(repoPath, ws, options) {
+async function dataflowExecuteWithLock(storage, repo, ws, options) {
     const startTime = Date.now();
+    const startedAt = new Date();
     const concurrency = options.concurrency ?? 4;
+    // Generate run ID for this execution
+    const runId = uuidv7();
     let taskNodes;
     let taskDependents;
+    let outputToTask;
+    let wsState;
     try {
+        // Read workspace state for run tracking
+        wsState = await readWorkspaceState(storage, repo, ws);
         // Build dependency graph
-        const graph = await buildDependencyGraph(repoPath, ws);
-        taskNodes = graph.taskNodes;
-        taskDependents = graph.taskDependents;
+        const graphResult = await buildDependencyGraph(storage, repo, ws);
+        taskNodes = graphResult.taskNodes;
+        taskDependents = graphResult.taskDependents;
+        outputToTask = graphResult.outputToTask;
     }
     catch (err) {
         // Re-throw E3Errors as-is
@@ -247,6 +299,54 @@ async function dataflowExecuteWithLock(repoPath, ws, options) {
         // Wrap unexpected errors
         throw new DataflowError(`Failed to build dependency graph: ${err instanceof Error ? err.message : err}`);
     }
+    // Clean up all previous runs (we hold the lock, so no concurrent runs)
+    const allRunIds = await storage.refs.dataflowRunList(repo, ws);
+    for (const oldRunId of allRunIds) {
+        await storage.refs.dataflowRunDelete(repo, ws, oldRunId);
+    }
+    // Initialize task execution records map
+    const taskExecutions = new Map();
+    // Create initial DataflowRun record
+    const initialRun = {
+        runId,
+        workspaceName: ws,
+        packageRef: `${wsState.packageName}@${wsState.packageVersion}`,
+        startedAt,
+        completedAt: variant('none', null),
+        status: variant('running', {}),
+        inputSnapshot: wsState.rootHash,
+        outputSnapshot: variant('none', null),
+        taskExecutions: taskExecutions,
+        summary: {
+            total: BigInt(taskNodes.size),
+            completed: 0n,
+            cached: 0n,
+            failed: 0n,
+            skipped: 0n,
+        },
+    };
+    // Write initial run record
+    await storage.refs.dataflowRunWrite(repo, ws, initialRun);
+    // Build DataflowGraph for use with decomposed building blocks
+    const dataflowGraph = {
+        tasks: Array.from(taskNodes.entries()).map(([taskName, node]) => {
+            const dependsOn = [];
+            for (const inputPath of node.inputPaths) {
+                const inputPathStr = pathToString(inputPath);
+                const producerTask = outputToTask.get(inputPathStr);
+                if (producerTask) {
+                    dependsOn.push(producerTask);
+                }
+            }
+            return {
+                name: taskName,
+                hash: node.hash,
+                inputs: node.inputPaths.map(pathToString),
+                output: pathToString(node.outputPath),
+                dependsOn,
+            };
+        }),
+    };
     // Apply filter if specified
     const filteredTaskNames = options.filter
         ? new Set([options.filter])
@@ -278,6 +378,7 @@ async function dataflowExecuteWithLock(repoPath, ws, options) {
     const readyQueue = [];
     const completed = new Set();
     const inProgress = new Set();
+    const skippedTasks = new Set(); // Track skipped tasks separately for dataflowGetDependentsToSkip
     // Initialize ready queue with tasks that have no unresolved dependencies
     // and pass the filter (if any)
     for (const [taskName, node] of taskNodes) {
@@ -288,13 +389,13 @@ async function dataflowExecuteWithLock(repoPath, ws, options) {
         }
     }
     // Check if the task has a valid cached execution for current inputs
-    // Returns the output hash if cached, null if re-execution is needed
+    // Returns the output hash and executionId if cached, null if re-execution is needed
     async function getCachedOutput(taskName) {
         const node = taskNodes.get(taskName);
         // Gather current input hashes
         const currentInputHashes = [];
         for (const inputPath of node.inputPaths) {
-            const { refType, hash } = await workspaceGetDatasetHash(repoPath, ws, inputPath);
+            const { refType, hash } = await workspaceGetDatasetHash(storage, repo, ws, inputPath);
             if (refType !== 'value' || hash === null) {
                 // Input not assigned, can't be cached
                 return null;
@@ -303,20 +404,26 @@ async function dataflowExecuteWithLock(repoPath, ws, options) {
         }
         // Check if there's a cached execution for these inputs
         const inHash = inputsHash(currentInputHashes);
-        const cachedOutputHash = await executionGetOutput(repoPath, node.hash, inHash);
+        const cachedOutputHash = await executionGetOutput(storage, repo, node.hash, inHash);
         if (cachedOutputHash === null) {
             // No cached execution for current inputs
             return null;
         }
+        // Get the latest execution status to retrieve the executionId
+        const latestStatus = await storage.refs.executionGetLatest(repo, node.hash, inHash);
+        if (!latestStatus || latestStatus.type !== 'success') {
+            // Latest execution wasn't a success
+            return null;
+        }
         // Also verify the workspace output matches the cached output
         // (in case the workspace was modified outside of execution)
-        const { refType, hash: wsOutputHash } = await workspaceGetDatasetHash(repoPath, ws, node.outputPath);
+        const { refType, hash: wsOutputHash } = await workspaceGetDatasetHash(storage, repo, ws, node.outputPath);
         if (refType !== 'value' || wsOutputHash !== cachedOutputHash) {
             // Workspace output doesn't match cached output, need to re-execute
             // (or update workspace with cached value)
             return null;
         }
-        return cachedOutputHash;
+        return { outputHash: cachedOutputHash, executionId: latestStatus.value.executionId };
     }
     // Execute a single task (does NOT write to workspace - caller must do that)
     async function executeTask(taskName) {
@@ -326,7 +433,7 @@ async function dataflowExecuteWithLock(repoPath, ws, options) {
         // Gather input hashes
         const inputHashes = [];
         for (const inputPath of node.inputPaths) {
-            const { refType, hash } = await workspaceGetDatasetHash(repoPath, ws, inputPath);
+            const { refType, hash } = await workspaceGetDatasetHash(storage, repo, ws, inputPath);
             if (refType !== 'value' || hash === null) {
                 // Input not available - should not happen if dependency tracking is correct
                 return {
@@ -339,18 +446,22 @@ async function dataflowExecuteWithLock(repoPath, ws, options) {
             }
             inputHashes.push(hash);
         }
-        // Execute the task
+        // Execute the task using either the provided runner or direct taskExecute()
         const execOptions = {
             force: options.force,
             signal: options.signal,
             onStdout: options.onStdout ? (data) => options.onStdout(taskName, data) : undefined,
             onStderr: options.onStderr ? (data) => options.onStderr(taskName, data) : undefined,
         };
-        const result = await taskExecute(repoPath, node.hash, inputHashes, execOptions);
+        // Use provided runner if available, otherwise call taskExecute directly
+        const result = options.runner
+            ? await options.runner.execute(storage, node.hash, inputHashes, execOptions)
+            : await taskExecute(storage, repo, node.hash, inputHashes, execOptions);
         // Build task result (NOTE: workspace update happens later, in mutex-protected section)
         const taskResult = {
             name: taskName,
             cached: result.cached,
+            executionId: result.executionId,
             state: result.state,
             duration: Date.now() - taskStartTime,
         };
@@ -359,7 +470,6 @@ async function dataflowExecuteWithLock(repoPath, ws, options) {
         }
         else if (result.state === 'failed') {
             taskResult.exitCode = result.exitCode ?? undefined;
-            taskResult.error = result.error ?? undefined;
         }
         // Pass output hash to caller for workspace update (if successful)
         if (result.state === 'success' && result.outputHash) {
@@ -383,17 +493,17 @@ async function dataflowExecuteWithLock(repoPath, ws, options) {
             }
         }
     }
-    // Mark dependents as skipped when a task fails
+    // Mark dependents as skipped when a task fails.
+    // Uses dataflowGetDependentsToSkip to find all transitive dependents at once
+    // (shared with distributed execution in e3-aws).
     function skipDependents(taskName) {
-        const dependents = taskDependents.get(taskName) ?? new Set();
-        for (const depName of dependents) {
-            if (completed.has(depName) || inProgress.has(depName))
-                continue;
-            // Skip dependents not in the filter
-            if (filteredTaskNames && !filteredTaskNames.has(depName))
-                continue;
-            // Recursively skip
+        // Get all tasks to skip (excludes already completed, already skipped, and in-progress)
+        const toSkip = dataflowGetDependentsToSkip(dataflowGraph, taskName, completed, skippedTasks)
+            .filter(name => !inProgress.has(name)) // Also exclude in-progress tasks
+            .filter(name => !filteredTaskNames || filteredTaskNames.has(name)); // Apply filter
+        for (const depName of toSkip) {
             completed.add(depName);
+            skippedTasks.add(depName);
             skipped++;
             results.push({
                 name: depName,
@@ -407,7 +517,6 @@ async function dataflowExecuteWithLock(repoPath, ws, options) {
                 state: 'skipped',
                 duration: 0,
             });
-            skipDependents(depName);
         }
     }
     // Main execution loop using a work-stealing approach
@@ -424,8 +533,8 @@ async function dataflowExecuteWithLock(repoPath, ws, options) {
                 if (completed.has(taskName) || inProgress.has(taskName))
                     continue;
                 // Check if there's a valid cached execution for current inputs
-                const cachedOutputHash = await getCachedOutput(taskName);
-                if (cachedOutputHash !== null && !options.force) {
+                const cachedResult = await getCachedOutput(taskName);
+                if (cachedResult !== null && !options.force) {
                     // Valid cached execution exists for current inputs.
                     // No workspace write needed (output already matches), but we still
                     // need mutex protection for state updates to prevent races with
@@ -436,12 +545,18 @@ async function dataflowExecuteWithLock(repoPath, ws, options) {
                         const result = {
                             name: taskName,
                             cached: true,
+                            executionId: cachedResult.executionId,
                             state: 'success',
                             duration: 0,
                         };
                         results.push(result);
                         options.onTaskComplete?.(result);
                         notifyDependents(taskName);
+                        // Track in taskExecutions map
+                        taskExecutions.set(taskName, {
+                            executionId: cachedResult.executionId,
+                            cached: true,
+                        });
                     });
                     continue;
                 }
@@ -456,7 +571,7 @@ async function dataflowExecuteWithLock(repoPath, ws, options) {
                             // Write output to workspace BEFORE notifying dependents
                             if (result.state === 'success' && result.outputHash) {
                                 const node = taskNodes.get(taskName);
-                                await workspaceSetDatasetByHash(repoPath, ws, node.outputPath, result.outputHash);
+                                await workspaceSetDatasetByHash(storage, repo, ws, node.outputPath, result.outputHash);
                             }
                             // Now safe to update execution state and notify dependents
                             inProgress.delete(taskName);
@@ -471,11 +586,25 @@ async function dataflowExecuteWithLock(repoPath, ws, options) {
                                     executed++;
                                 }
                                 notifyDependents(taskName);
+                                // Track in taskExecutions map
+                                if (result.executionId) {
+                                    taskExecutions.set(taskName, {
+                                        executionId: result.executionId,
+                                        cached: result.cached,
+                                    });
+                                }
                             }
                             else {
                                 failed++;
                                 hasFailure = true;
                                 skipDependents(taskName);
+                                // Track failed execution too
+                                if (result.executionId) {
+                                    taskExecutions.set(taskName, {
+                                        executionId: result.executionId,
+                                        cached: false,
+                                    });
+                                }
                             }
                         });
                     }
@@ -489,8 +618,10 @@ async function dataflowExecuteWithLock(repoPath, ws, options) {
             if (runningPromises.size > 0) {
                 await Promise.race(runningPromises.values());
             }
-            else if (readyQueue.length === 0) {
-                // No running tasks and no ready tasks - we might have unresolvable dependencies
+            else if (readyQueue.length === 0 || aborted) {
+                // No running tasks and either:
+                // - no ready tasks (unresolvable dependencies)
+                // - aborted (stop processing)
                 break;
             }
         }
@@ -502,12 +633,79 @@ async function dataflowExecuteWithLock(repoPath, ws, options) {
     }
     // Check for abort one final time
     checkAborted();
-    // If aborted, throw with partial results
+    // If aborted, throw with partial results (also update run record)
     if (aborted) {
+        const finalWsState = await readWorkspaceState(storage, repo, ws);
+        const cancelledRun = {
+            runId,
+            workspaceName: ws,
+            packageRef: `${wsState.packageName}@${wsState.packageVersion}`,
+            startedAt,
+            completedAt: variant('some', new Date()),
+            status: variant('cancelled', {}),
+            inputSnapshot: wsState.rootHash,
+            outputSnapshot: variant('some', finalWsState.rootHash),
+            taskExecutions,
+            summary: {
+                total: BigInt(taskNodes.size),
+                completed: BigInt(executed + cached),
+                cached: BigInt(cached),
+                failed: BigInt(failed),
+                skipped: BigInt(skipped),
+            },
+        };
+        await storage.refs.dataflowRunWrite(repo, ws, cancelledRun);
         throw new DataflowAbortedError(results);
     }
+    // Read final workspace state for output snapshot
+    const finalWsState = await readWorkspaceState(storage, repo, ws);
+    // Determine final status
+    let finalStatus;
+    if (hasFailure) {
+        // Find the failed task
+        const failedTask = results.find(r => r.state === 'failed' || r.state === 'error');
+        finalStatus = variant('failed', {
+            failedTask: failedTask?.name ?? 'unknown',
+            error: failedTask?.error ?? failedTask?.exitCode?.toString() ?? 'Task failed',
+        });
+    }
+    else {
+        finalStatus = variant('completed', {});
+    }
+    // Write final DataflowRun record
+    const finalRun = {
+        runId,
+        workspaceName: ws,
+        packageRef: `${wsState.packageName}@${wsState.packageVersion}`,
+        startedAt,
+        completedAt: variant('some', new Date()),
+        status: finalStatus,
+        inputSnapshot: wsState.rootHash,
+        outputSnapshot: variant('some', finalWsState.rootHash),
+        taskExecutions,
+        summary: {
+            total: BigInt(taskNodes.size),
+            completed: BigInt(executed + cached),
+            cached: BigInt(cached),
+            failed: BigInt(failed),
+            skipped: BigInt(skipped),
+        },
+    };
+    await storage.refs.dataflowRunWrite(repo, ws, finalRun);
+    // Update workspace state with currentRunId on success
+    if (!hasFailure) {
+        // Read, update, write workspace state
+        const currentState = await readWorkspaceState(storage, repo, ws);
+        const updatedState = {
+            ...currentState,
+            currentRunId: variant('some', runId),
+        };
+        const encoder = encodeBeast2For(WorkspaceStateType);
+        await storage.refs.workspaceWrite(repo, ws, encoder(updatedState));
+    }
     return {
         success: !hasFailure,
+        runId,
         executed,
         cached,
         failed,
@@ -519,18 +717,19 @@ async function dataflowExecuteWithLock(repoPath, ws, options) {
 /**
  * Get the dependency graph for a workspace (for visualization/debugging).
  *
- * @param repoPath - Path to .e3 repository
+ * @param storage - Storage backend
+ * @param repo - Repository identifier (for local storage, the path to e3 repository directory)
  * @param ws - Workspace name
  * @returns Graph information
  * @throws {WorkspaceNotFoundError} If workspace doesn't exist
  * @throws {WorkspaceNotDeployedError} If workspace has no package deployed
  * @throws {DataflowError} If graph building fails for other reasons
  */
-export async function dataflowGetGraph(repoPath, ws) {
+export async function dataflowGetGraph(storage, repo, ws) {
     let taskNodes;
     let outputToTask;
     try {
-        const graph = await buildDependencyGraph(repoPath, ws);
+        const graph = await buildDependencyGraph(storage, repo, ws);
         taskNodes = graph.taskNodes;
         outputToTask = graph.outputToTask;
     }
@@ -559,4 +758,168 @@ export async function dataflowGetGraph(repoPath, ws) {
     }
     return { tasks };
 }
+/**
+ * Get tasks that are ready to execute given the set of completed tasks.
+ *
+ * A task is ready when all tasks it depends on have completed.
+ * This is useful for distributed execution (e.g., AWS Step Functions)
+ * where a coordinator needs to determine which tasks can run next.
+ *
+ * @param graph - The dependency graph from dataflowGetGraph
+ * @param completedTasks - Set of task names that have completed
+ * @returns Array of task names that are ready to execute
+ *
+ * @example
+ * ```typescript
+ * const graph = await dataflowGetGraph(storage, repo, 'production');
+ * const ready = dataflowGetReadyTasks(graph, new Set()); // Initial ready tasks
+ * // Execute ready[0]...
+ * const nextReady = dataflowGetReadyTasks(graph, new Set([ready[0]]));
+ * ```
+ */
+export function dataflowGetReadyTasks(graph, completedTasks) {
+    const ready = [];
+    for (const task of graph.tasks) {
+        // Skip already completed tasks
+        if (completedTasks.has(task.name)) {
+            continue;
+        }
+        // Check if all dependencies are satisfied
+        const allDepsCompleted = task.dependsOn.every(dep => completedTasks.has(dep));
+        if (allDepsCompleted) {
+            ready.push(task.name);
+        }
+    }
+    return ready;
+}
+/**
+ * Check if a task execution is cached for the given inputs.
+ *
+ * This is useful for distributed execution where a Lambda handler needs
+ * to check if a task can be skipped before spawning execution.
+ *
+ * @param storage - Storage backend
+ * @param repo - Repository path
+ * @param taskHash - Hash of the TaskObject
+ * @param inputHashes - Array of input dataset hashes (in order)
+ * @returns Output hash if cached, null if execution needed
+ *
+ * @example
+ * ```typescript
+ * const outputHash = await dataflowCheckCache(storage, repo, taskHash, inputHashes);
+ * if (outputHash) {
+ *   // Task is cached, use outputHash directly
+ * } else {
+ *   // Need to execute task
+ * }
+ * ```
+ */
+export async function dataflowCheckCache(storage, repo, taskHash, inputHashes) {
+    const inHash = inputsHash(inputHashes);
+    return executionGetOutput(storage, repo, taskHash, inHash);
+}
+/**
+ * Find tasks that should be skipped when a task fails.
+ *
+ * Returns all tasks that transitively depend on the failed task
+ * (directly or through other tasks), excluding already completed
+ * or already skipped tasks.
+ *
+ * This is useful for distributed execution where the coordinator
+ * needs to mark downstream tasks as skipped after a failure.
+ *
+ * @param graph - The dependency graph from dataflowGetGraph
+ * @param failedTask - Name of the task that failed
+ * @param completedTasks - Set of task names already completed (won't be skipped)
+ * @param skippedTasks - Set of task names already skipped (won't be returned again)
+ * @returns Array of task names that should be skipped
+ *
+ * @example
+ * ```typescript
+ * const graph = await dataflowGetGraph(storage, repo, 'production');
+ * // Task 'etl' failed...
+ * const toSkip = dataflowGetDependentsToSkip(graph, 'etl', completed, skipped);
+ * // toSkip might be ['transform', 'aggregate', 'report'] - all downstream tasks
+ * ```
+ */
+export function dataflowGetDependentsToSkip(graph, failedTask, completedTasks, skippedTasks) {
+    // Build reverse dependency map: task -> tasks that depend on it
+    const dependents = new Map();
+    for (const task of graph.tasks) {
+        dependents.set(task.name, []);
+    }
+    for (const task of graph.tasks) {
+        for (const dep of task.dependsOn) {
+            dependents.get(dep)?.push(task.name);
+        }
+    }
+    // BFS to find all transitive dependents
+    const toSkip = [];
+    const visited = new Set();
+    const queue = [failedTask];
+    while (queue.length > 0) {
+        const current = queue.shift();
+        const deps = dependents.get(current) ?? [];
+        for (const dep of deps) {
+            // Skip if already processed
+            if (visited.has(dep)) {
+                continue;
+            }
+            visited.add(dep);
+            // Skip if already completed (no need to explore further - completed tasks break the chain)
+            if (completedTasks.has(dep)) {
+                continue;
+            }
+            // If already skipped, still explore dependents but don't add to result again
+            if (skippedTasks.has(dep)) {
+                queue.push(dep);
+                continue;
+            }
+            // New task to skip
+            toSkip.push(dep);
+            queue.push(dep);
+        }
+    }
+    return toSkip;
+}
+/**
+ * Resolve input hashes for a task from current workspace state.
+ *
+ * Returns an array of hashes in the same order as the task's inputs.
+ * If any input is unassigned, returns null for that position.
+ *
+ * This is useful for distributed execution where the input hashes
+ * need to be resolved before checking cache or executing.
+ *
+ * @param storage - Storage backend
+ * @param repo - Repository path
+ * @param ws - Workspace name
+ * @param task - Task info from the graph (needs inputs array)
+ * @returns Array of hashes (null if input is unassigned)
+ *
+ * @example
+ * ```typescript
+ * const graph = await dataflowGetGraph(storage, repo, 'production');
+ * const task = graph.tasks.find(t => t.name === 'etl')!;
+ * const inputHashes = await dataflowResolveInputHashes(storage, repo, 'production', task);
+ * if (!inputHashes.includes(null)) {
+ *   const cached = await dataflowCheckCache(storage, repo, task.hash, inputHashes);
+ * }
+ * ```
+ */
+export async function dataflowResolveInputHashes(storage, repo, ws, task) {
+    const hashes = [];
+    for (const inputPathStr of task.inputs) {
+        // Parse the keypath string back to TreePath
+        const inputPath = parsePathString(inputPathStr);
+        const { refType, hash } = await workspaceGetDatasetHash(storage, repo, ws, inputPath);
+        if (refType === 'value' && hash !== null) {
+            hashes.push(hash);
+        }
+        else {
+            hashes.push(null);
+        }
+    }
+    return hashes;
+}
 //# sourceMappingURL=dataflow.js.map