@elaraai/e3-core 0.0.2-beta.5 → 0.0.2-beta.50

package/dist/src/dataflow.js

@@ -5,128 +5,101 @@
 /**
  * Dataflow execution for e3 workspaces.
  *
- * Executes tasks in a workspace based on their dependency graph. Tasks are
- * executed in parallel where possible, respecting a concurrency limit.
+ * Provides the high-level `dataflowExecute` entry point (which delegates
+ * to `LocalOrchestrator`) and shared graph-building utilities used by
+ * both local and cloud execution paths.
  *
- * The execution model is event-driven with a work queue:
- * 1. Build dependency graph from tasks (input paths -> task -> output path)
- * 2. Compute reverse dependencies (which tasks depend on each output)
- * 3. Initialize ready queue with tasks whose inputs are all assigned
- * 4. Execute tasks from ready queue, respecting concurrency limit
- * 5. On task completion, queue workspace update then check dependents for readiness
- * 6. On failure, stop launching new tasks but wait for running ones
- *
- * IMPORTANT: Workspace state updates are serialized through an async queue to
- * prevent race conditions when multiple tasks complete concurrently. Each task's
- * output is written to the workspace and dependents are notified only after the
- * write completes, ensuring downstream tasks see consistent state.
+ * The reactive execution logic (input change detection, task invalidation,
+ * version vector consistency) lives in `dataflow/steps.ts` and is orchestrated
+ * by `dataflow/orchestrator/LocalOrchestrator.ts`.
  */
-import { decodeBeast2For } from '@elaraai/east';
+import { decodeBeast2For, variant } from '@elaraai/east';
 import { PackageObjectType, TaskObjectType, WorkspaceStateType, pathToString, } from '@elaraai/e3-types';
-import { objectRead } from './objects.js';
-import { taskExecute, executionGetOutput, inputsHash, } from './executions.js';
-import { workspaceGetDatasetHash, workspaceSetDatasetByHash, } from './trees.js';
-import { E3Error, WorkspaceNotFoundError, WorkspaceNotDeployedError, TaskNotFoundError, DataflowError, DataflowAbortedError, isNotFoundError, } from './errors.js';
-import { acquireWorkspaceLock, } from './workspaceLock.js';
-import * as fs from 'fs/promises';
-import * as path from 'path';
+import { executionGetOutput, inputsHash, } from './executions.js';
+import { workspaceGetDatasetHash, } from './trees.js';
+import { E3Error, WorkspaceNotFoundError, WorkspaceNotDeployedError, DataflowError, } from './errors.js';
 // =============================================================================
-// Async Mutex for Workspace Updates
+// Path Parsing Helper
 // =============================================================================
 /**
- * Simple async mutex to serialize workspace state updates.
+ * Parse a keypath string (from pathToString) back to TreePath.
+ *
+ * The keypath format is: .field1.field2 (dot-separated field names)
+ * Quoted identifiers use backticks: .field1.`complex/name`
  *
- * When multiple tasks complete concurrently, their workspace writes must be
- * serialized to prevent race conditions (read-modify-write on the workspace
- * root hash). This mutex ensures only one update runs at a time.
+ * @param pathStr - The path string in keypath format
+ * @returns TreePath array of path segments
  */
-class AsyncMutex {
-    queue = [];
-    locked = false;
-    /**
-     * Acquire the mutex, execute the callback, then release.
-     * If the mutex is already held, waits until it's available.
-     */
-    async runExclusive(fn) {
-        await this.acquire();
-        try {
-            return await fn();
-        }
-        finally {
-            this.release();
-        }
+export function parsePathString(pathStr) {
+    if (!pathStr.startsWith('.')) {
+        throw new Error(`Invalid path string: expected '.' prefix, got '${pathStr}'`);
     }
-    acquire() {
-        return new Promise((resolve) => {
-            if (!this.locked) {
-                this.locked = true;
-                resolve();
+    const segments = [];
+    let i = 1; // Skip the leading '.'
+    while (i < pathStr.length) {
+        let fieldName;
+        if (pathStr[i] === '`') {
+            // Quoted identifier: find closing backtick
+            const endQuote = pathStr.indexOf('`', i + 1);
+            if (endQuote === -1) {
+                throw new Error(`Invalid path string: unclosed backtick at position ${i}`);
             }
-            else {
-                this.queue.push(resolve);
-            }
-        });
-    }
-    release() {
-        const next = this.queue.shift();
-        if (next) {
-            next();
+            fieldName = pathStr.slice(i + 1, endQuote);
+            i = endQuote + 1;
         }
         else {
-            this.locked = false;
+            // Unquoted identifier: read until '.' or end
+            let end = pathStr.indexOf('.', i);
+            if (end === -1)
+                end = pathStr.length;
+            fieldName = pathStr.slice(i, end);
+            i = end;
+        }
+        if (fieldName) {
+            segments.push(variant('field', fieldName));
+        }
+        // Skip the '.' separator
+        if (i < pathStr.length && pathStr[i] === '.') {
+            i++;
         }
     }
+    return segments;
 }
 // =============================================================================
 // Workspace State Reader
 // =============================================================================
 /**
- * Read workspace state from file.
+ * Read workspace state.
  * @throws {WorkspaceNotFoundError} If workspace doesn't exist
  * @throws {WorkspaceNotDeployedError} If workspace has no package deployed
  */
-async function readWorkspaceState(repoPath, ws) {
-    const stateFile = path.join(repoPath, 'workspaces', `${ws}.beast2`);
-    let data;
-    try {
-        data = await fs.readFile(stateFile);
-    }
-    catch (err) {
-        if (isNotFoundError(err)) {
-            throw new WorkspaceNotFoundError(ws);
-        }
-        throw err;
+async function readWorkspaceState(storage, repo, ws) {
+    const data = await storage.refs.workspaceRead(repo, ws);
+    if (data === null) {
+        throw new WorkspaceNotFoundError(ws);
     }
     if (data.length === 0) {
         throw new WorkspaceNotDeployedError(ws);
     }
     const decoder = decodeBeast2For(WorkspaceStateType);
-    return decoder(data);
+    return decoder(Buffer.from(data));
 }
 // =============================================================================
 // Dependency Graph Building
 // =============================================================================
 /**
  * Build the dependency graph for a workspace.
- *
- * Returns:
- * - taskNodes: Map of task name -> TaskNode
- * - outputToTask: Map of output path string -> task name
- * - taskDependents: Map of task name -> set of dependent task names
  */
-async function buildDependencyGraph(repoPath, ws) {
-    // Read workspace state to get package hash
-    const state = await readWorkspaceState(repoPath, ws);
-    // Read package object to get tasks map
-    const pkgData = await objectRead(repoPath, state.packageHash);
+async function buildDependencyGraph(storage, repo, ws) {
+    const state = await readWorkspaceState(storage, repo, ws);
+    const pkgData = await storage.objects.read(repo, state.packageHash);
     const pkgDecoder = decodeBeast2For(PackageObjectType);
     const pkgObject = pkgDecoder(Buffer.from(pkgData));
     const taskNodes = new Map();
-    const outputToTask = new Map(); // output path -> task name
-    // First pass: load all tasks and build output->task map
+    const outputToTask = new Map();
     const taskDecoder = decodeBeast2For(TaskObjectType);
     for (const [taskName, taskHash] of pkgObject.tasks) {
-        const taskData = await objectRead(repoPath, taskHash);
+        const taskData = await storage.objects.read(repo, taskHash);
         const task = taskDecoder(Buffer.from(taskData));
         const outputPathStr = pathToString(task.output);
         outputToTask.set(outputPathStr, taskName);
@@ -136,32 +109,24 @@ async function buildDependencyGraph(repoPath, ws) {
             task,
             inputPaths: task.inputs,
             outputPath: task.output,
-            unresolvedCount: 0, // Will be computed below
+            unresolvedCount: 0,
         });
     }
-    // Build reverse dependency map: task -> tasks that depend on it
     const taskDependents = new Map();
     for (const taskName of taskNodes.keys()) {
         taskDependents.set(taskName, new Set());
     }
-    // Second pass: compute dependencies and unresolved counts
     for (const [taskName, node] of taskNodes) {
         for (const inputPath of node.inputPaths) {
             const inputPathStr = pathToString(inputPath);
             const producerTask = outputToTask.get(inputPathStr);
             if (producerTask) {
-                // This input comes from another task's output.
-                // The task cannot run until the producer task completes,
-                // regardless of whether the output is currently assigned
-                // (it might be stale from a previous run).
                 taskDependents.get(producerTask).add(taskName);
                 node.unresolvedCount++;
             }
-            // If not produced by a task, it's an external input - check if assigned
             else {
-                const { refType } = await workspaceGetDatasetHash(repoPath, ws, inputPath);
+                const { refType } = await workspaceGetDatasetHash(storage, repo, ws, inputPath);
                 if (refType === 'unassigned') {
-                    // External input that is unassigned - this task can never run
                     node.unresolvedCount++;
                 }
             }
@@ -175,362 +140,109 @@ async function buildDependencyGraph(repoPath, ws) {
 /**
  * Execute all tasks in a workspace according to the dependency graph.
  *
- * Tasks are executed in parallel where dependencies allow, respecting
- * the concurrency limit. On failure, no new tasks are launched but
- * running tasks are allowed to complete.
+ * Delegates to `LocalOrchestrator` which implements reactive fixpoint
+ * execution using step functions. After each task completes, input changes
+ * are detected and affected tasks are invalidated and re-executed.
  *
- * Acquires an exclusive lock on the workspace for the duration of execution
- * to prevent concurrent modifications. If options.lock is provided, uses that
- * lock instead (caller is responsible for releasing it).
- *
- * @param repoPath - Path to .e3 repository
+ * @param storage - Storage backend
+ * @param repo - Repository identifier
  * @param ws - Workspace name
  * @param options - Execution options
  * @returns Result of the dataflow execution
+ *
  * @throws {WorkspaceLockError} If workspace is locked by another process
  * @throws {WorkspaceNotFoundError} If workspace doesn't exist
  * @throws {WorkspaceNotDeployedError} If workspace has no package deployed
  * @throws {TaskNotFoundError} If filter specifies a task that doesn't exist
  * @throws {DataflowError} If execution fails for other reasons
  */
-export async function dataflowExecute(repoPath, ws, options = {}) {
-    // Acquire lock if not provided externally
-    const externalLock = options.lock;
-    const lock = externalLock ?? await acquireWorkspaceLock(repoPath, ws);
-    try {
-        return await dataflowExecuteWithLock(repoPath, ws, options);
-    }
-    finally {
-        // Only release the lock if we acquired it internally
-        if (!externalLock) {
-            await lock.release();
-        }
-    }
+export async function dataflowExecute(storage, repo, ws, options = {}) {
+    const { LocalOrchestrator } = await import('./dataflow/orchestrator/LocalOrchestrator.js');
+    const orchestrator = new LocalOrchestrator();
+    const taskResults = [];
+    const handle = await orchestrator.start(storage, repo, ws, {
+        concurrency: options.concurrency,
+        force: options.force,
+        filter: options.filter,
+        signal: options.signal,
+        lock: options.lock,
+        runner: options.runner,
+        onTaskStart: options.onTaskStart,
+        onTaskComplete: (result) => {
+            taskResults.push({
+                name: result.name,
+                cached: result.cached,
+                state: result.state,
+                error: result.error,
+                exitCode: result.exitCode,
+                duration: result.duration,
+            });
+            options.onTaskComplete?.({
+                name: result.name,
+                cached: result.cached,
+                state: result.state,
+                error: result.error,
+                exitCode: result.exitCode,
+                duration: result.duration,
+            });
+        },
+        onStdout: options.onStdout,
+        onStderr: options.onStderr,
+        onInputChanged: options.onInputChanged,
+        onTaskInvalidated: options.onTaskInvalidated,
+        onTaskDeferred: options.onTaskDeferred,
+    });
+    const result = await orchestrator.wait(handle);
+    return {
+        success: result.success,
+        runId: result.runId,
+        executed: result.executed,
+        cached: result.cached,
+        failed: result.failed,
+        skipped: result.skipped,
+        reexecuted: result.reexecuted,
+        tasks: taskResults,
+        duration: result.duration,
+    };
 }
 /**
- * Start dataflow execution in the background (non-blocking).
- *
- * Returns a promise immediately without awaiting execution. The lock is
- * released automatically when execution completes.
+ * Execute dataflow with an externally-held lock.
+ * The lock is released automatically when execution completes or fails.
  *
- * @param repoPath - Path to .e3 repository
+ * @param storage - Storage backend
+ * @param repo - Repository identifier
  * @param ws - Workspace name
  * @param options - Execution options (lock must be provided)
  * @returns Promise that resolves when execution completes
- * @throws {WorkspaceNotFoundError} If workspace doesn't exist
- * @throws {WorkspaceNotDeployedError} If workspace has no package deployed
- * @throws {TaskNotFoundError} If filter specifies a task that doesn't exist
- * @throws {DataflowError} If execution fails for other reasons
  */
-export function dataflowStart(repoPath, ws, options) {
-    return dataflowExecuteWithLock(repoPath, ws, options)
-        .finally(() => options.lock.release());
-}
-/**
- * Internal: Execute dataflow with lock already held.
- */
-async function dataflowExecuteWithLock(repoPath, ws, options) {
-    const startTime = Date.now();
-    const concurrency = options.concurrency ?? 4;
-    let taskNodes;
-    let taskDependents;
+export async function dataflowStart(storage, repo, ws, options) {
     try {
-        // Build dependency graph
-        const graph = await buildDependencyGraph(repoPath, ws);
-        taskNodes = graph.taskNodes;
-        taskDependents = graph.taskDependents;
-    }
-    catch (err) {
-        // Re-throw E3Errors as-is
-        if (err instanceof E3Error)
-            throw err;
-        // Wrap unexpected errors
-        throw new DataflowError(`Failed to build dependency graph: ${err instanceof Error ? err.message : err}`);
-    }
-    // Apply filter if specified
-    const filteredTaskNames = options.filter
-        ? new Set([options.filter])
-        : null;
-    // Validate filter
-    if (filteredTaskNames && options.filter && !taskNodes.has(options.filter)) {
-        throw new TaskNotFoundError(options.filter);
-    }
-    // Track execution state
-    const results = [];
-    let executed = 0;
-    let cached = 0;
-    let failed = 0;
-    let skipped = 0;
-    let hasFailure = false;
-    let aborted = false;
-    // Check for abort signal
-    const checkAborted = () => {
-        if (options.signal?.aborted && !aborted) {
-            aborted = true;
-        }
-        return aborted;
-    };
-    // Mutex to serialize workspace state updates.
-    // When multiple tasks complete concurrently, their writes to the workspace
-    // must be serialized to prevent lost updates (read-modify-write race).
-    const workspaceUpdateMutex = new AsyncMutex();
-    // Ready queue: tasks with all dependencies resolved
-    const readyQueue = [];
-    const completed = new Set();
-    const inProgress = new Set();
-    // Initialize ready queue with tasks that have no unresolved dependencies
-    // and pass the filter (if any)
-    for (const [taskName, node] of taskNodes) {
-        if (node.unresolvedCount === 0) {
-            if (!filteredTaskNames || filteredTaskNames.has(taskName)) {
-                readyQueue.push(taskName);
-            }
-        }
-    }
-    // Check if the task has a valid cached execution for current inputs
-    // Returns the output hash if cached, null if re-execution is needed
-    async function getCachedOutput(taskName) {
-        const node = taskNodes.get(taskName);
-        // Gather current input hashes
-        const currentInputHashes = [];
-        for (const inputPath of node.inputPaths) {
-            const { refType, hash } = await workspaceGetDatasetHash(repoPath, ws, inputPath);
-            if (refType !== 'value' || hash === null) {
-                // Input not assigned, can't be cached
-                return null;
-            }
-            currentInputHashes.push(hash);
-        }
-        // Check if there's a cached execution for these inputs
-        const inHash = inputsHash(currentInputHashes);
-        const cachedOutputHash = await executionGetOutput(repoPath, node.hash, inHash);
-        if (cachedOutputHash === null) {
-            // No cached execution for current inputs
-            return null;
-        }
-        // Also verify the workspace output matches the cached output
-        // (in case the workspace was modified outside of execution)
-        const { refType, hash: wsOutputHash } = await workspaceGetDatasetHash(repoPath, ws, node.outputPath);
-        if (refType !== 'value' || wsOutputHash !== cachedOutputHash) {
-            // Workspace output doesn't match cached output, need to re-execute
-            // (or update workspace with cached value)
-            return null;
-        }
-        return cachedOutputHash;
-    }
-    // Execute a single task (does NOT write to workspace - caller must do that)
-    async function executeTask(taskName) {
-        const node = taskNodes.get(taskName);
-        const taskStartTime = Date.now();
-        options.onTaskStart?.(taskName);
-        // Gather input hashes
-        const inputHashes = [];
-        for (const inputPath of node.inputPaths) {
-            const { refType, hash } = await workspaceGetDatasetHash(repoPath, ws, inputPath);
-            if (refType !== 'value' || hash === null) {
-                // Input not available - should not happen if dependency tracking is correct
-                return {
-                    name: taskName,
-                    cached: false,
-                    state: 'error',
-                    error: `Input at ${pathToString(inputPath)} is not assigned (refType: ${refType})`,
-                    duration: Date.now() - taskStartTime,
-                };
-            }
-            inputHashes.push(hash);
-        }
-        // Execute the task
-        const execOptions = {
-            force: options.force,
-            signal: options.signal,
-            onStdout: options.onStdout ? (data) => options.onStdout(taskName, data) : undefined,
-            onStderr: options.onStderr ? (data) => options.onStderr(taskName, data) : undefined,
-        };
-        const result = await taskExecute(repoPath, node.hash, inputHashes, execOptions);
-        // Build task result (NOTE: workspace update happens later, in mutex-protected section)
-        const taskResult = {
-            name: taskName,
-            cached: result.cached,
-            state: result.state,
-            duration: Date.now() - taskStartTime,
-        };
-        if (result.state === 'error') {
-            taskResult.error = result.error ?? undefined;
-        }
-        else if (result.state === 'failed') {
-            taskResult.exitCode = result.exitCode ?? undefined;
-            taskResult.error = result.error ?? undefined;
-        }
-        // Pass output hash to caller for workspace update (if successful)
-        if (result.state === 'success' && result.outputHash) {
-            taskResult.outputHash = result.outputHash;
-        }
-        return taskResult;
-    }
-    // Process dependents when a task completes
-    function notifyDependents(taskName) {
-        const dependents = taskDependents.get(taskName) ?? new Set();
-        for (const depName of dependents) {
-            if (completed.has(depName) || inProgress.has(depName))
-                continue;
-            // Skip dependents not in the filter
-            if (filteredTaskNames && !filteredTaskNames.has(depName))
-                continue;
-            const depNode = taskNodes.get(depName);
-            depNode.unresolvedCount--;
-            if (depNode.unresolvedCount === 0 && !readyQueue.includes(depName)) {
-                readyQueue.push(depName);
-            }
-        }
+        return await dataflowExecute(storage, repo, ws, options);
     }
-    // Mark dependents as skipped when a task fails
-    function skipDependents(taskName) {
-        const dependents = taskDependents.get(taskName) ?? new Set();
-        for (const depName of dependents) {
-            if (completed.has(depName) || inProgress.has(depName))
-                continue;
-            // Skip dependents not in the filter
-            if (filteredTaskNames && !filteredTaskNames.has(depName))
-                continue;
-            // Recursively skip
-            completed.add(depName);
-            skipped++;
-            results.push({
-                name: depName,
-                cached: false,
-                state: 'skipped',
-                duration: 0,
-            });
-            options.onTaskComplete?.({
-                name: depName,
-                cached: false,
-                state: 'skipped',
-                duration: 0,
-            });
-            skipDependents(depName);
-        }
-    }
-    // Main execution loop using a work-stealing approach
-    const runningPromises = new Map();
-    async function processQueue() {
-        while (true) {
-            // Check if we're done
-            if (readyQueue.length === 0 && runningPromises.size === 0) {
-                break;
-            }
-            // Launch tasks up to concurrency limit if no failure and not aborted
-            while (!hasFailure && !checkAborted() && readyQueue.length > 0 && runningPromises.size < concurrency) {
-                const taskName = readyQueue.shift();
-                if (completed.has(taskName) || inProgress.has(taskName))
-                    continue;
-                // Check if there's a valid cached execution for current inputs
-                const cachedOutputHash = await getCachedOutput(taskName);
-                if (cachedOutputHash !== null && !options.force) {
-                    // Valid cached execution exists for current inputs.
-                    // No workspace write needed (output already matches), but we still
-                    // need mutex protection for state updates to prevent races with
-                    // concurrent task completions.
-                    await workspaceUpdateMutex.runExclusive(() => {
-                        completed.add(taskName);
-                        cached++;
-                        const result = {
-                            name: taskName,
-                            cached: true,
-                            state: 'success',
-                            duration: 0,
-                        };
-                        results.push(result);
-                        options.onTaskComplete?.(result);
-                        notifyDependents(taskName);
-                    });
-                    continue;
-                }
-                inProgress.add(taskName);
-                const promise = (async () => {
-                    try {
-                        const result = await executeTask(taskName);
-                        // Use mutex to serialize workspace updates and dependent notifications.
-                        // This prevents race conditions where two tasks complete simultaneously,
-                        // both read the same workspace state, and one overwrites the other's changes.
-                        await workspaceUpdateMutex.runExclusive(async () => {
-                            // Write output to workspace BEFORE notifying dependents
-                            if (result.state === 'success' && result.outputHash) {
-                                const node = taskNodes.get(taskName);
-                                await workspaceSetDatasetByHash(repoPath, ws, node.outputPath, result.outputHash);
-                            }
-                            // Now safe to update execution state and notify dependents
-                            inProgress.delete(taskName);
-                            completed.add(taskName);
-                            results.push(result);
-                            options.onTaskComplete?.(result);
-                            if (result.state === 'success') {
-                                if (result.cached) {
-                                    cached++;
-                                }
-                                else {
-                                    executed++;
-                                }
-                                notifyDependents(taskName);
-                            }
-                            else {
-                                failed++;
-                                hasFailure = true;
-                                skipDependents(taskName);
-                            }
-                        });
-                    }
-                    finally {
-                        runningPromises.delete(taskName);
-                    }
-                })();
-                runningPromises.set(taskName, promise);
-            }
-            // Wait for at least one task to complete if we can't launch more
-            if (runningPromises.size > 0) {
-                await Promise.race(runningPromises.values());
-            }
-            else if (readyQueue.length === 0) {
-                // No running tasks and no ready tasks - we might have unresolvable dependencies
-                break;
-            }
-        }
-    }
-    await processQueue();
-    // Wait for any remaining tasks
-    if (runningPromises.size > 0) {
-        await Promise.all(runningPromises.values());
-    }
-    // Check for abort one final time
-    checkAborted();
-    // If aborted, throw with partial results
-    if (aborted) {
-        throw new DataflowAbortedError(results);
+    finally {
+        await options.lock.release();
     }
-    return {
-        success: !hasFailure,
-        executed,
-        cached,
-        failed,
-        skipped,
-        tasks: results,
-        duration: Date.now() - startTime,
-    };
 }
+// =============================================================================
+// Graph Queries (shared between local and cloud execution)
+// =============================================================================
 /**
  * Get the dependency graph for a workspace (for visualization/debugging).
  *
- * @param repoPath - Path to .e3 repository
+ * @param storage - Storage backend
+ * @param repo - Repository identifier
  * @param ws - Workspace name
  * @returns Graph information
+ *
  * @throws {WorkspaceNotFoundError} If workspace doesn't exist
  * @throws {WorkspaceNotDeployedError} If workspace has no package deployed
  * @throws {DataflowError} If graph building fails for other reasons
  */
-export async function dataflowGetGraph(repoPath, ws) {
+export async function dataflowGetGraph(storage, repo, ws) {
     let taskNodes;
     let outputToTask;
     try {
-        const graph = await buildDependencyGraph(repoPath, ws);
+        const graph = await buildDependencyGraph(storage, repo, ws);
         taskNodes = graph.taskNodes;
         outputToTask = graph.outputToTask;
     }
@@ -559,4 +271,145 @@ export async function dataflowGetGraph(repoPath, ws) {
     }
     return { tasks };
 }
+/**
+ * Find all tasks affected by input changes (transitive dependents).
+ * An affected task is one whose output could change due to the input change.
+ *
+ * @param graph - The dependency graph
+ * @param changes - Array of changed input paths
+ * @returns Array of affected task names
+ */
+export function findAffectedTasks(graph, changes) {
+    const changedPaths = new Set(changes.map(c => c.path));
+    const affected = new Set();
+    const queue = [];
+    // Build forward dep map: task name → tasks that depend on its output
+    const taskToDependents = new Map();
+    for (const task of graph.tasks) {
+        for (const dep of task.dependsOn) {
+            if (!taskToDependents.has(dep))
+                taskToDependents.set(dep, []);
+            taskToDependents.get(dep).push(task.name);
+        }
+    }
+    // Seed: tasks that directly read a changed input
+    for (const task of graph.tasks) {
+        if (task.inputs.some(inp => changedPaths.has(inp))) {
+            queue.push(task.name);
+        }
+    }
+    // BFS through dependency graph
+    while (queue.length > 0) {
+        const name = queue.shift();
+        if (affected.has(name))
+            continue;
+        affected.add(name);
+        for (const dep of taskToDependents.get(name) ?? []) {
+            queue.push(dep);
+        }
+    }
+    return Array.from(affected);
+}
+/**
+ * Get tasks that are ready to execute given the set of completed tasks.
+ *
+ * A task is ready when all tasks it depends on have completed.
+ *
+ * @param graph - The dependency graph from dataflowGetGraph
+ * @param completedTasks - Set of task names that have completed
+ * @returns Array of task names that are ready to execute
+ */
+export function dataflowGetReadyTasks(graph, completedTasks) {
+    const ready = [];
+    for (const task of graph.tasks) {
+        if (completedTasks.has(task.name)) {
+            continue;
+        }
+        const allDepsCompleted = task.dependsOn.every(dep => completedTasks.has(dep));
+        if (allDepsCompleted) {
+            ready.push(task.name);
+        }
+    }
+    return ready;
+}
+/**
+ * Check if a task execution is cached for the given inputs.
+ *
+ * @param storage - Storage backend
+ * @param repo - Repository path
+ * @param taskHash - Hash of the TaskObject
+ * @param inputHashes - Array of input dataset hashes (in order)
+ * @returns Output hash if cached, null if execution needed
+ */
+export async function dataflowCheckCache(storage, repo, taskHash, inputHashes) {
+    const inHash = inputsHash(inputHashes);
+    return executionGetOutput(storage, repo, taskHash, inHash);
+}
+/**
+ * Find tasks that should be skipped when a task fails.
+ *
+ * Returns all tasks that transitively depend on the failed task,
+ * excluding already completed or already skipped tasks.
+ *
+ * @param graph - The dependency graph from dataflowGetGraph
+ * @param failedTask - Name of the task that failed
+ * @param completedTasks - Set of task names already completed
+ * @param skippedTasks - Set of task names already skipped
+ * @returns Array of task names that should be skipped
+ */
+export function dataflowGetDependentsToSkip(graph, failedTask, completedTasks, skippedTasks) {
+    const dependents = new Map();
+    for (const task of graph.tasks) {
+        dependents.set(task.name, []);
+    }
+    for (const task of graph.tasks) {
+        for (const dep of task.dependsOn) {
+            dependents.get(dep)?.push(task.name);
+        }
+    }
+    const toSkip = [];
+    const visited = new Set();
+    const queue = [failedTask];
+    while (queue.length > 0) {
+        const current = queue.shift();
+        const deps = dependents.get(current) ?? [];
+        for (const dep of deps) {
+            if (visited.has(dep))
+                continue;
+            visited.add(dep);
+            if (completedTasks.has(dep))
+                continue;
+            if (skippedTasks.has(dep)) {
+                queue.push(dep);
+                continue;
+            }
+            toSkip.push(dep);
+            queue.push(dep);
+        }
+    }
+    return toSkip;
+}
+/**
+ * Resolve input hashes for a task from current workspace state.
+ *
+ * @param storage - Storage backend
+ * @param repo - Repository path
+ * @param ws - Workspace name
+ * @param task - Task info from the graph
+ * @returns Array of hashes (null if input is unassigned)
+ */
+export async function dataflowResolveInputHashes(storage, repo, ws, task) {
+    const hashes = [];
+    for (const inputPathStr of task.inputs) {
+        const inputPath = parsePathString(inputPathStr);
+        const { refType, hash } = await workspaceGetDatasetHash(storage, repo, ws, inputPath);
+        if (refType === 'value' && hash !== null) {
+            hashes.push(hash);
+        }
+        else {
+            hashes.push(null);
+        }
+    }
+    return hashes;
+}
 //# sourceMappingURL=dataflow.js.map