@elaraai/e3-core 0.0.2-beta.8 → 1.0.0

package/dist/src/dataflow-orchestration.spec.js

@@ -0,0 +1,1025 @@
+/**
+ * Copyright (c) 2025 Elara AI Pty Ltd
+ * Licensed under BSL 1.1. See LICENSE for details.
+ */
+/**
+ * Tests for dataflow orchestration using MockTaskRunner.
+ *
+ * These tests verify the dataflow execution logic (dependency ordering,
+ * concurrency limits, failure propagation, abort handling, caching)
+ * without spawning real processes.
+ */
+import { describe, it, beforeEach, afterEach } from 'node:test';
+import assert from 'node:assert';
+import { join } from 'node:path';
+import { mkdirSync, writeFileSync } from 'node:fs';
+import { variant, StringType, ArrayType, encodeBeast2For, East, IRType } from '@elaraai/east';
+import { TaskObjectType, PackageObjectType, } from '@elaraai/e3-types';
+import { dataflowExecute } from './dataflow.js';
+import { datasetWrite } from './trees.js';
+import { objectWrite } from './storage/local/LocalObjectStore.js';
+import { workspaceDeploy } from './workspaces.js';
+import { workspaceSetDataset } from './trees.js';
+import { createTestRepo, removeTestRepo } from './test-helpers.js';
+import { LocalStorage } from './storage/local/index.js';
+import { MockTaskRunner } from './execution/MockTaskRunner.js';
+import { inputsHash } from './executions.js';
+describe('dataflow orchestration with MockTaskRunner', () => {
+    let testRepo;
+    let storage;
+    let mockRunner;
+    beforeEach(() => {
+        testRepo = createTestRepo();
+        storage = new LocalStorage();
+        mockRunner = new MockTaskRunner();
+    });
+    afterEach(() => {
+        removeTestRepo(testRepo);
+    });
+    /**
+     * Helper to create a command IR object.
+     */
+    async function createCommandIr(repoPath, parts) {
+        const commandFn = East.function([ArrayType(StringType), StringType], ArrayType(StringType), ($, inputs, output) => {
+            const result = [];
+            for (const part of parts) {
+                if (part === '{input}' || part === '{input0}') {
+                    result.push(inputs.get(0n));
+                }
+                else if (part.match(/^\{input(\d+)\}$/)) {
+                    const idx = BigInt(part.match(/^\{input(\d+)\}$/)[1]);
+                    result.push(inputs.get(idx));
+                }
+                else if (part === '{output}') {
+                    result.push(output);
+                }
+                else {
+                    result.push(part);
+                }
+            }
+            return result;
+        });
+        const ir = commandFn.toIR().ir;
+        const encoder = encodeBeast2For(IRType);
+        return objectWrite(repoPath, encoder(ir));
+    }
+    /**
+     * Helper to create a package with tasks.
+     * Returns a map of task names to task hashes.
+     */
+    async function createPackageWithTasks(repoPath, tasks, structure) {
+        const taskEncoder = encodeBeast2For(TaskObjectType);
+        const tasksMap = new Map();
+        for (const t of tasks) {
+            const commandIrHash = await createCommandIr(repoPath, t.command);
+            const taskObj = {
+                commandIr: commandIrHash,
+                inputs: t.inputs,
+                output: t.output,
+                kind: variant('none', null), metadata: variant('none', null),
+            };
+            const taskHash = await objectWrite(repoPath, taskEncoder(taskObj));
+            tasksMap.set(t.name, taskHash);
+        }
+        // Create package object (no root tree — per-dataset refs are used instead)
+        const pkgEncoder = encodeBeast2For(PackageObjectType);
+        const pkgObj = {
+            data: {
+                structure,
+                refs: new Map(),
+            },
+            tasks: tasksMap,
+        };
+        const pkgHash = await objectWrite(repoPath, pkgEncoder(pkgObj));
+        const pkgDir = join(repoPath, 'packages', 'test');
+        mkdirSync(pkgDir, { recursive: true });
+        writeFileSync(join(pkgDir, '1.0.0'), pkgHash + '\n');
+        return tasksMap;
+    }
+    describe('dependency ordering', () => {
+        it('executes tasks in topological order', async () => {
+            // Create package with A -> B -> C chain
+            const structure = {
+                type: 'struct',
+                value: new Map([
+                    ['input', { type: 'value', value: { type: StringType, writable: true } }],
+                    ['middle1', { type: 'value', value: { type: StringType, writable: true } }],
+                    ['middle2', { type: 'value', value: { type: StringType, writable: true } }],
+                    ['output', { type: 'value', value: { type: StringType, writable: true } }],
+                ]),
+            };
+            const inputPath = [variant('field', 'input')];
+            const middle1Path = [variant('field', 'middle1')];
+            const middle2Path = [variant('field', 'middle2')];
+            const outputPath = [variant('field', 'output')];
+            const inputEncoder = encodeBeast2For(StringType);
+            const _inputHash = await objectWrite(testRepo, inputEncoder('test'));
+            const taskHashes = await createPackageWithTasks(testRepo, [
+                { name: 'task-a', command: ['echo'], inputs: [inputPath], output: middle1Path },
+                { name: 'task-b', command: ['echo'], inputs: [middle1Path], output: middle2Path },
+                { name: 'task-c', command: ['echo'], inputs: [middle2Path], output: outputPath },
+            ], structure);
+            await workspaceDeploy(storage, testRepo, 'test-ws', 'test', '1.0.0');
+            await workspaceSetDataset(storage, testRepo, 'test-ws', inputPath, 'test', StringType);
+            // Configure mock to return unique output hashes
+            for (const [name, hash] of taskHashes) {
+                mockRunner.setResult(hash, {
+                    state: 'success',
+                    cached: false,
+                    outputHash: `output-${name}`,
+                });
+            }
+            const completedOrder = [];
+            await dataflowExecute(storage, testRepo, 'test-ws', {
+                runner: mockRunner,
+                onTaskComplete: (r) => completedOrder.push(r.name),
+            });
+            // Verify execution order: A must complete before B, B before C
+            assert.strictEqual(completedOrder.indexOf('task-a') < completedOrder.indexOf('task-b'), true);
+            assert.strictEqual(completedOrder.indexOf('task-b') < completedOrder.indexOf('task-c'), true);
+        });
+        it('executes independent tasks in parallel', async () => {
+            // Create package with diamond: A -> B, A -> C, B+C -> D
+            const structure = {
+                type: 'struct',
+                value: new Map([
+                    ['input', { type: 'value', value: { type: StringType, writable: true } }],
+                    ['out_a', { type: 'value', value: { type: StringType, writable: true } }],
+                    ['out_b', { type: 'value', value: { type: StringType, writable: true } }],
+                    ['out_c', { type: 'value', value: { type: StringType, writable: true } }],
+                ]),
+            };
+            const inputPath = [variant('field', 'input')];
+            const inputEncoder = encodeBeast2For(StringType);
+            const _inputHash = await objectWrite(testRepo, inputEncoder('test'));
+            const taskHashes = await createPackageWithTasks(testRepo, [
+                { name: 'task-a', command: ['echo'], inputs: [inputPath], output: [variant('field', 'out_a')] },
+                { name: 'task-b', command: ['echo'], inputs: [inputPath], output: [variant('field', 'out_b')] },
+                { name: 'task-c', command: ['echo'], inputs: [inputPath], output: [variant('field', 'out_c')] },
+            ], structure);
+            await workspaceDeploy(storage, testRepo, 'test-ws', 'test', '1.0.0');
+            await workspaceSetDataset(storage, testRepo, 'test-ws', inputPath, 'test', StringType);
+            // Configure mock results
+            for (const [name, hash] of taskHashes) {
+                mockRunner.setResult(hash, {
+                    state: 'success',
+                    cached: false,
+                    outputHash: `output-${name}`,
+                });
+            }
+            const result = await dataflowExecute(storage, testRepo, 'test-ws', {
+                runner: mockRunner,
+                concurrency: 4,
+            });
+            assert.strictEqual(result.success, true);
+            assert.strictEqual(result.executed, 3);
+            // Verify all tasks were called
+            const calls = mockRunner.getCalls();
+            assert.strictEqual(calls.length, 3);
+        });
+    });
+    describe('concurrency limit', () => {
+        it('respects concurrency limit with mock runner', async () => {
+            // Create package with 4 independent tasks
+            const structure = {
+                type: 'struct',
+                value: new Map([
+                    ['input', { type: 'value', value: { type: StringType, writable: true } }],
+                    ['out1', { type: 'value', value: { type: StringType, writable: true } }],
+                    ['out2', { type: 'value', value: { type: StringType, writable: true } }],
+                    ['out3', { type: 'value', value: { type: StringType, writable: true } }],
+                    ['out4', { type: 'value', value: { type: StringType, writable: true } }],
+                ]),
+            };
+            const inputPath = [variant('field', 'input')];
+            const inputEncoder = encodeBeast2For(StringType);
+            const _inputHash = await objectWrite(testRepo, inputEncoder('test'));
+            const taskHashes = await createPackageWithTasks(testRepo, [
+                { name: 'task-1', command: ['echo'], inputs: [inputPath], output: [variant('field', 'out1')] },
+                { name: 'task-2', command: ['echo'], inputs: [inputPath], output: [variant('field', 'out2')] },
+                { name: 'task-3', command: ['echo'], inputs: [inputPath], output: [variant('field', 'out3')] },
+                { name: 'task-4', command: ['echo'], inputs: [inputPath], output: [variant('field', 'out4')] },
+            ], structure);
+            await workspaceDeploy(storage, testRepo, 'test-ws', 'test', '1.0.0');
+            await workspaceSetDataset(storage, testRepo, 'test-ws', inputPath, 'test', StringType);
+            // Configure mock to add delay and track concurrency
+            let currentConcurrent = 0;
+            let maxConcurrent = 0;
+            for (const [name, hash] of taskHashes) {
+                mockRunner.setResult(hash, () => {
+                    currentConcurrent++;
+                    maxConcurrent = Math.max(maxConcurrent, currentConcurrent);
+                    // Simulate async work then decrement
+                    return {
+                        state: 'success',
+                        cached: false,
+                        outputHash: `output-${name}`,
+                    };
+                });
+            }
+            // Track via callbacks since mock execute is sync
+            let startCount = 0;
+            await dataflowExecute(storage, testRepo, 'test-ws', {
+                runner: mockRunner,
+                concurrency: 2,
+                onTaskStart: () => {
+                    startCount++;
+                },
+                onTaskComplete: () => {
+                    currentConcurrent--;
+                },
+            });
+            assert.strictEqual(startCount, 4);
+            // Note: With synchronous mock, concurrency tracking through callbacks works differently
+            // The key test is that all tasks were executed
+            const calls = mockRunner.getCalls();
+            assert.strictEqual(calls.length, 4);
+        });
+    });
+    describe('cache behavior', () => {
+        it('counts tasks as cached when runner returns cached: true', async () => {
+            const structure = {
+                type: 'struct',
+                value: new Map([
+                    ['input', { type: 'value', value: { type: StringType, writable: true } }],
+                    ['output', { type: 'value', value: { type: StringType, writable: true } }],
+                ]),
+            };
+            const inputPath = [variant('field', 'input')];
+            const outputPath = [variant('field', 'output')];
+            const inputEncoder = encodeBeast2For(StringType);
+            const _inputHash = await objectWrite(testRepo, inputEncoder('test'));
+            const taskHashes = await createPackageWithTasks(testRepo, [{ name: 'task', command: ['echo'], inputs: [inputPath], output: outputPath }], structure);
+            await workspaceDeploy(storage, testRepo, 'test-ws', 'test', '1.0.0');
+            await workspaceSetDataset(storage, testRepo, 'test-ws', inputPath, 'test', StringType);
+            // First run: not cached
+            for (const [, hash] of taskHashes) {
+                mockRunner.setResult(hash, {
+                    state: 'success',
+                    cached: false,
+                    outputHash: 'output-hash',
+                });
+            }
+            const result1 = await dataflowExecute(storage, testRepo, 'test-ws', {
+                runner: mockRunner,
+            });
+            assert.strictEqual(result1.executed, 1);
+            assert.strictEqual(result1.cached, 0);
+            // Second run: runner returns cached: true
+            mockRunner.clearCalls();
+            for (const [, hash] of taskHashes) {
+                mockRunner.setResult(hash, {
+                    state: 'success',
+                    cached: true,
+                    outputHash: 'output-hash',
+                });
+            }
+            const result2 = await dataflowExecute(storage, testRepo, 'test-ws', {
+                runner: mockRunner,
+            });
+            // Note: The dataflow has its own cache check before calling the runner.
+            // If the workspace output already matches the cached output, runner isn't called.
+            // In this test, we're verifying that if runner IS called and returns cached: true,
+            // it's counted correctly.
+            assert.strictEqual(result2.success, true);
+        });
+    });
+    describe('failure propagation', () => {
+        it('skips downstream tasks when upstream fails', async () => {
+            // Create A -> B -> C, where A fails
+            const structure = {
+                type: 'struct',
+                value: new Map([
+                    ['input', { type: 'value', value: { type: StringType, writable: true } }],
+                    ['middle1', { type: 'value', value: { type: StringType, writable: true } }],
+                    ['middle2', { type: 'value', value: { type: StringType, writable: true } }],
+                    ['output', { type: 'value', value: { type: StringType, writable: true } }],
+                ]),
+            };
+            const inputPath = [variant('field', 'input')];
+            const middle1Path = [variant('field', 'middle1')];
+            const middle2Path = [variant('field', 'middle2')];
+            const outputPath = [variant('field', 'output')];
+            const inputEncoder = encodeBeast2For(StringType);
+            const _inputHash = await objectWrite(testRepo, inputEncoder('test'));
+            const taskHashes = await createPackageWithTasks(testRepo, [
+                { name: 'task-a', command: ['echo'], inputs: [inputPath], output: middle1Path },
+                { name: 'task-b', command: ['echo'], inputs: [middle1Path], output: middle2Path },
+                { name: 'task-c', command: ['echo'], inputs: [middle2Path], output: outputPath },
+            ], structure);
+            await workspaceDeploy(storage, testRepo, 'test-ws', 'test', '1.0.0');
+            await workspaceSetDataset(storage, testRepo, 'test-ws', inputPath, 'test', StringType);
+            // task-a fails, others should succeed if called
+            mockRunner.setResult(taskHashes.get('task-a'), {
+                state: 'failed',
+                cached: false,
+                exitCode: 1,
+            });
+            mockRunner.setResult(taskHashes.get('task-b'), {
+                state: 'success',
+                cached: false,
+                outputHash: 'output-b',
+            });
+            mockRunner.setResult(taskHashes.get('task-c'), {
+                state: 'success',
+                cached: false,
+                outputHash: 'output-c',
+            });
+            const result = await dataflowExecute(storage, testRepo, 'test-ws', {
+                runner: mockRunner,
+            });
+            assert.strictEqual(result.success, false);
+            assert.strictEqual(result.failed, 1);
+            assert.strictEqual(result.skipped, 2); // B and C should be skipped
+            const taskA = result.tasks.find(t => t.name === 'task-a');
+            const taskB = result.tasks.find(t => t.name === 'task-b');
+            const taskC = result.tasks.find(t => t.name === 'task-c');
+            assert.strictEqual(taskA?.state, 'failed');
+            assert.strictEqual(taskB?.state, 'skipped');
+            assert.strictEqual(taskC?.state, 'skipped');
+            // Only task-a should have been called
+            const calls = mockRunner.getCalls();
+            assert.strictEqual(calls.length, 1);
+            assert.strictEqual(calls[0].taskHash, taskHashes.get('task-a'));
+        });
+        it('handles error state from runner', async () => {
+            const structure = {
+                type: 'struct',
+                value: new Map([
+                    ['input', { type: 'value', value: { type: StringType, writable: true } }],
+                    ['output', { type: 'value', value: { type: StringType, writable: true } }],
+                ]),
+            };
+            const inputPath = [variant('field', 'input')];
+            const outputPath = [variant('field', 'output')];
+            const inputEncoder = encodeBeast2For(StringType);
+            const _inputHash = await objectWrite(testRepo, inputEncoder('test'));
+            const taskHashes = await createPackageWithTasks(testRepo, [{ name: 'task', command: ['echo'], inputs: [inputPath], output: outputPath }], structure);
+            await workspaceDeploy(storage, testRepo, 'test-ws', 'test', '1.0.0');
+            await workspaceSetDataset(storage, testRepo, 'test-ws', inputPath, 'test', StringType);
+            // Runner returns error state
+            mockRunner.setResult(taskHashes.get('task'), {
+                state: 'error',
+                cached: false,
+                error: 'Internal error',
+            });
+            const result = await dataflowExecute(storage, testRepo, 'test-ws', {
+                runner: mockRunner,
+            });
+            assert.strictEqual(result.success, false);
+            assert.strictEqual(result.failed, 1);
+            const task = result.tasks.find(t => t.name === 'task');
+            assert.strictEqual(task?.state, 'error');
+            assert.strictEqual(task?.error, 'Internal error');
+        });
+    });
+    describe('abort handling', () => {
+        it('does not start tasks when signal is pre-aborted', async () => {
+            // Create an independent task
+            const structure = {
+                type: 'struct',
+                value: new Map([
+                    ['input', { type: 'value', value: { type: StringType, writable: true } }],
+                    ['output', { type: 'value', value: { type: StringType, writable: true } }],
+                ]),
+            };
+            const inputPath = [variant('field', 'input')];
+            const outputPath = [variant('field', 'output')];
+            const inputEncoder = encodeBeast2For(StringType);
+            const _inputHash = await objectWrite(testRepo, inputEncoder('test'));
+            const taskHashes = await createPackageWithTasks(testRepo, [{ name: 'task', command: ['echo'], inputs: [inputPath], output: outputPath }], structure);
+            await workspaceDeploy(storage, testRepo, 'test-ws', 'test', '1.0.0');
+            await workspaceSetDataset(storage, testRepo, 'test-ws', inputPath, 'test', StringType);
+            mockRunner.setResult(taskHashes.get('task'), {
+                state: 'success',
+                cached: false,
+                outputHash: 'output-hash',
+            });
+            // Pre-abort the signal before execution starts
+            const controller = new AbortController();
+            controller.abort();
+            const { DataflowAbortedError } = await import('./errors.js');
+            await assert.rejects(dataflowExecute(storage, testRepo, 'test-ws', {
+                runner: mockRunner,
+                signal: controller.signal,
+            }), (err) => {
+                assert.ok(err instanceof DataflowAbortedError);
+                return true;
+            });
+            // No tasks should have been executed since signal was pre-aborted
+            const calls = mockRunner.getCalls();
+            assert.strictEqual(calls.length, 0, 'No tasks should execute when signal is pre-aborted');
+        });
+    });
+    describe('input hash passing', () => {
+        it('passes correct input hashes to runner', async () => {
+            const structure = {
+                type: 'struct',
+                value: new Map([
+                    ['input', { type: 'value', value: { type: StringType, writable: true } }],
+                    ['output', { type: 'value', value: { type: StringType, writable: true } }],
+                ]),
+            };
+            const inputPath = [variant('field', 'input')];
+            const outputPath = [variant('field', 'output')];
+            const inputEncoder = encodeBeast2For(StringType);
+            const inputHash = await objectWrite(testRepo, inputEncoder('specific-value'));
+            const taskHashes = await createPackageWithTasks(testRepo, [{ name: 'task', command: ['echo'], inputs: [inputPath], output: outputPath }], structure);
+            await workspaceDeploy(storage, testRepo, 'test-ws', 'test', '1.0.0');
+            await workspaceSetDataset(storage, testRepo, 'test-ws', inputPath, 'specific-value', StringType);
+            // Set up mock to capture input hashes
+            let capturedInputHashes = [];
+            mockRunner.setResult(taskHashes.get('task'), (inputHashes) => {
+                capturedInputHashes = [...inputHashes];
+                return {
+                    state: 'success',
+                    cached: false,
+                    outputHash: 'output-hash',
+                };
+            });
+            await dataflowExecute(storage, testRepo, 'test-ws', {
+                runner: mockRunner,
+            });
+            // Verify the input hash was passed correctly
+            assert.strictEqual(capturedInputHashes.length, 1);
+            assert.strictEqual(capturedInputHashes[0], inputHash);
+        });
+    });
+    describe('callback invocation', () => {
+        it('calls onTaskStart and onTaskComplete callbacks', async () => {
+            const structure = {
+                type: 'struct',
+                value: new Map([
+                    ['input', { type: 'value', value: { type: StringType, writable: true } }],
+                    ['output', { type: 'value', value: { type: StringType, writable: true } }],
+                ]),
+            };
+            const inputPath = [variant('field', 'input')];
+            const outputPath = [variant('field', 'output')];
+            const inputEncoder = encodeBeast2For(StringType);
+            const _inputHash = await objectWrite(testRepo, inputEncoder('test'));
+            const taskHashes = await createPackageWithTasks(testRepo, [{ name: 'my-task', command: ['echo'], inputs: [inputPath], output: outputPath }], structure);
+            await workspaceDeploy(storage, testRepo, 'test-ws', 'test', '1.0.0');
+            await workspaceSetDataset(storage, testRepo, 'test-ws', inputPath, 'test', StringType);
+            mockRunner.setResult(taskHashes.get('my-task'), {
+                state: 'success',
+                cached: false,
+                outputHash: 'output-hash',
+            });
+            const startedTasks = [];
+            const completedTasks = [];
+            await dataflowExecute(storage, testRepo, 'test-ws', {
+                runner: mockRunner,
+                onTaskStart: (name) => startedTasks.push(name),
+                onTaskComplete: (result) => completedTasks.push(result.name),
+            });
+            assert.deepStrictEqual(startedTasks, ['my-task']);
+            assert.deepStrictEqual(completedTasks, ['my-task']);
+        });
+        it('passes stdout/stderr callbacks to runner', async () => {
+            const structure = {
+                type: 'struct',
+                value: new Map([
+                    ['input', { type: 'value', value: { type: StringType, writable: true } }],
+                    ['output', { type: 'value', value: { type: StringType, writable: true } }],
+                ]),
+            };
+            const inputPath = [variant('field', 'input')];
+            const outputPath = [variant('field', 'output')];
+            const inputEncoder = encodeBeast2For(StringType);
+            const _inputHash = await objectWrite(testRepo, inputEncoder('test'));
+            const taskHashes = await createPackageWithTasks(testRepo, [{ name: 'task', command: ['echo'], inputs: [inputPath], output: outputPath }], structure);
+            await workspaceDeploy(storage, testRepo, 'test-ws', 'test', '1.0.0');
+            await workspaceSetDataset(storage, testRepo, 'test-ws', inputPath, 'test', StringType);
+            // Capture the options passed to runner
+            let _capturedOptions;
+            mockRunner.setResult(taskHashes.get('task'), (_inputHashes) => {
+                _capturedOptions = mockRunner.getCalls()[0]?.options;
+                return {
+                    state: 'success',
+                    cached: false,
+                    outputHash: 'output-hash',
+                };
+            });
+            const stdoutCalls = [];
+            const stderrCalls = [];
+            await dataflowExecute(storage, testRepo, 'test-ws', {
+                runner: mockRunner,
+                onStdout: (task, data) => stdoutCalls.push({ task, data }),
+                onStderr: (task, data) => stderrCalls.push({ task, data }),
+            });
+            // Verify callbacks were passed to runner's options
+            const call = mockRunner.getCalls()[0];
+            assert.ok(call.options?.onStdout, 'onStdout should be passed to runner');
+            assert.ok(call.options?.onStderr, 'onStderr should be passed to runner');
+        });
+    });
+    describe('reactive dataflow', () => {
+        it('reaches fixpoint without re-execution when inputs unchanged', async () => {
+            // Normal execution, no input changes → same behavior as before
+            const structure = {
+                type: 'struct',
+                value: new Map([
+                    ['input', { type: 'value', value: { type: StringType, writable: true } }],
+                    ['middle', { type: 'value', value: { type: StringType, writable: true } }],
+                    ['output', { type: 'value', value: { type: StringType, writable: true } }],
+                ]),
+            };
+            const inputPath = [variant('field', 'input')];
+            const middlePath = [variant('field', 'middle')];
+            const outputPath = [variant('field', 'output')];
+            const taskHashes = await createPackageWithTasks(testRepo, [
+                { name: 'task-a', command: ['echo'], inputs: [inputPath], output: middlePath },
+                { name: 'task-b', command: ['echo'], inputs: [middlePath], output: outputPath },
+            ], structure);
+            await workspaceDeploy(storage, testRepo, 'test-ws', 'test', '1.0.0');
+            await workspaceSetDataset(storage, testRepo, 'test-ws', inputPath, 'test', StringType);
+            for (const [name, hash] of taskHashes) {
+                mockRunner.setResult(hash, {
+                    state: 'success',
+                    cached: false,
+                    outputHash: `output-${name}`,
+                });
+            }
+            const result = await dataflowExecute(storage, testRepo, 'test-ws', {
+                runner: mockRunner,
+            });
+            assert.strictEqual(result.success, true);
+            assert.strictEqual(result.executed, 2);
+            assert.strictEqual(result.reexecuted, 0);
+        });
+        it('re-executes downstream tasks when input changes during execution', async () => {
+            // Setup: input → taskA → output
+            // MockTaskRunner for taskA: during execution, write new value to input ref
+            // After taskA completes, reactive loop detects change, invalidates taskA
+            // taskA re-runs with new input
+            const structure = {
+                type: 'struct',
+                value: new Map([
+                    ['input', { type: 'value', value: { type: StringType, writable: true } }],
+                    ['output', { type: 'value', value: { type: StringType, writable: true } }],
+                ]),
+            };
+            const inputPath = [variant('field', 'input')];
+            const outputPath = [variant('field', 'output')];
+            const taskHashes = await createPackageWithTasks(testRepo, [
+                { name: 'task-a', command: ['echo'], inputs: [inputPath], output: outputPath },
+            ], structure);
+            await workspaceDeploy(storage, testRepo, 'test-ws', 'test', '1.0.0');
+            await workspaceSetDataset(storage, testRepo, 'test-ws', inputPath, 'initial-value', StringType);
+            let callCount = 0;
+            const taskAHash = taskHashes.get('task-a');
+            mockRunner.setResult(taskAHash, async (_inputHashes) => {
+                callCount++;
+                if (callCount === 1) {
+                    // On first execution, simulate a concurrent input change
+                    const newHash = await datasetWrite(storage, testRepo, 'changed-value', StringType);
+                    const ref = variant('value', { hash: newHash, versions: new Map() });
+                    await storage.datasets.write(testRepo, 'test-ws', 'input', ref);
+                }
+                return {
+                    state: 'success',
+                    cached: false,
+                    outputHash: `output-v${callCount}`,
+                };
+            });
+            const result = await dataflowExecute(storage, testRepo, 'test-ws', {
+                runner: mockRunner,
+            });
+            assert.strictEqual(result.success, true);
+            // Should have executed task-a twice: once initially, once after input change
+            assert.strictEqual(callCount, 2);
+            assert.strictEqual(result.reexecuted, 1);
+            // executed counts final unique tasks, not total calls
+            assert.strictEqual(result.executed, 1);
+        });
+        it('re-executes chain when input changes during execution', async () => {
+            // Setup: input → taskA → middle → taskB → output
+            // During taskA execution, input changes
+            // taskA should re-run, then taskB should run with new output
+            const structure = {
+                type: 'struct',
+                value: new Map([
+                    ['input', { type: 'value', value: { type: StringType, writable: true } }],
+                    ['middle', { type: 'value', value: { type: StringType, writable: true } }],
+                    ['output', { type: 'value', value: { type: StringType, writable: true } }],
+                ]),
+            };
+            const inputPath = [variant('field', 'input')];
+            const middlePath = [variant('field', 'middle')];
+            const outputPath = [variant('field', 'output')];
+            const taskHashes = await createPackageWithTasks(testRepo, [
+                { name: 'task-a', command: ['echo'], inputs: [inputPath], output: middlePath },
+                { name: 'task-b', command: ['echo'], inputs: [middlePath], output: outputPath },
+            ], structure);
+            await workspaceDeploy(storage, testRepo, 'test-ws', 'test', '1.0.0');
+            await workspaceSetDataset(storage, testRepo, 'test-ws', inputPath, 'initial-value', StringType);
+            let taskACallCount = 0;
+            let taskBCallCount = 0;
+            const taskAHash = taskHashes.get('task-a');
+            const taskBHash = taskHashes.get('task-b');
+            mockRunner.setResult(taskAHash, async (_inputHashes) => {
+                taskACallCount++;
+                if (taskACallCount === 1) {
+                    // On first execution, simulate a concurrent input change
+                    const newHash = await datasetWrite(storage, testRepo, 'changed-value', StringType);
+                    const ref = variant('value', { hash: newHash, versions: new Map() });
+                    await storage.datasets.write(testRepo, 'test-ws', 'input', ref);
+                }
+                return {
+                    state: 'success',
+                    cached: false,
+                    outputHash: `middle-v${taskACallCount}`,
+                };
+            });
+            mockRunner.setResult(taskBHash, (_inputHashes) => {
+                taskBCallCount++;
+                return {
+                    state: 'success',
+                    cached: false,
+                    outputHash: `output-v${taskBCallCount}`,
+                };
+            });
+            const result = await dataflowExecute(storage, testRepo, 'test-ws', {
+                runner: mockRunner,
+            });
+            assert.strictEqual(result.success, true);
+            // taskA should run twice (initial + re-execution after input change)
+            assert.strictEqual(taskACallCount, 2);
+            // taskB should run once (blocked until taskA re-executes, then runs with fresh data)
+            assert.strictEqual(taskBCallCount, 1);
+            // One re-execution (taskA)
+            assert.strictEqual(result.reexecuted, 1);
+        });
+        it('tracks reexecuted count correctly', async () => {
+            const structure = {
+                type: 'struct',
+                value: new Map([
+                    ['input', { type: 'value', value: { type: StringType, writable: true } }],
+                    ['output', { type: 'value', value: { type: StringType, writable: true } }],
+                ]),
+            };
+            const inputPath = [variant('field', 'input')];
+            const outputPath = [variant('field', 'output')];
+            const taskHashes = await createPackageWithTasks(testRepo, [
+                { name: 'task-a', command: ['echo'], inputs: [inputPath], output: outputPath },
+            ], structure);
+            await workspaceDeploy(storage, testRepo, 'test-ws', 'test', '1.0.0');
+            await workspaceSetDataset(storage, testRepo, 'test-ws', inputPath, 'v1', StringType);
+            let callCount = 0;
+            const taskAHash = taskHashes.get('task-a');
+            mockRunner.setResult(taskAHash, async (_inputHashes) => {
+                callCount++;
+                if (callCount <= 2) {
+                    // First two calls: write a new input value
+                    const newHash = await datasetWrite(storage, testRepo, `v${callCount + 1}`, StringType);
+                    const ref = variant('value', { hash: newHash, versions: new Map() });
+                    await storage.datasets.write(testRepo, 'test-ws', 'input', ref);
+                }
+                return {
+                    state: 'success',
+                    cached: false,
+                    outputHash: `output-v${callCount}`,
+                };
+            });
+            const result = await dataflowExecute(storage, testRepo, 'test-ws', {
+                runner: mockRunner,
+            });
+            assert.strictEqual(result.success, true);
+            // Should execute 3 times total: initial + 2 re-executions
+            assert.strictEqual(callCount, 3);
+            assert.strictEqual(result.reexecuted, 2);
+        });
+        it('calls onInputChanged callback', async () => {
+            const structure = {
+                type: 'struct',
+                value: new Map([
+                    ['input', { type: 'value', value: { type: StringType, writable: true } }],
+                    ['output', { type: 'value', value: { type: StringType, writable: true } }],
+                ]),
+            };
+            const inputPath = [variant('field', 'input')];
+            const outputPath = [variant('field', 'output')];
+            const taskHashes = await createPackageWithTasks(testRepo, [
+                { name: 'task-a', command: ['echo'], inputs: [inputPath], output: outputPath },
+            ], structure);
+            await workspaceDeploy(storage, testRepo, 'test-ws', 'test', '1.0.0');
+            await workspaceSetDataset(storage, testRepo, 'test-ws', inputPath, 'initial', StringType);
+            let callCount = 0;
+            const taskAHash = taskHashes.get('task-a');
+            mockRunner.setResult(taskAHash, async (_inputHashes) => {
+                callCount++;
+                if (callCount === 1) {
+                    const newHash = await datasetWrite(storage, testRepo, 'changed', StringType);
+                    const ref = variant('value', { hash: newHash, versions: new Map() });
+                    await storage.datasets.write(testRepo, 'test-ws', 'input', ref);
+                }
+                return {
+                    state: 'success',
+                    cached: false,
+                    outputHash: `output-v${callCount}`,
+                };
+            });
+            const inputChanges = [];
+            await dataflowExecute(storage, testRepo, 'test-ws', {
+                runner: mockRunner,
+                onInputChanged: (path, previousHash, newHash) => {
+                    inputChanges.push({ path, previousHash, newHash });
+                },
+            });
+            assert.strictEqual(inputChanges.length, 1);
+            assert.strictEqual(inputChanges[0].path, '.input');
+            assert.ok(inputChanges[0].previousHash.length > 0);
+            assert.ok(inputChanges[0].newHash.length > 0);
+            assert.notStrictEqual(inputChanges[0].previousHash, inputChanges[0].newHash);
+        });
+        it('calls onTaskInvalidated callback', async () => {
+            const structure = {
+                type: 'struct',
+                value: new Map([
+                    ['input', { type: 'value', value: { type: StringType, writable: true } }],
+                    ['output', { type: 'value', value: { type: StringType, writable: true } }],
+                ]),
+            };
+            const inputPath = [variant('field', 'input')];
+            const outputPath = [variant('field', 'output')];
+            const taskHashes = await createPackageWithTasks(testRepo, [
+                { name: 'task-a', command: ['echo'], inputs: [inputPath], output: outputPath },
+            ], structure);
+            await workspaceDeploy(storage, testRepo, 'test-ws', 'test', '1.0.0');
+            await workspaceSetDataset(storage, testRepo, 'test-ws', inputPath, 'initial', StringType);
+            let callCount = 0;
+            const taskAHash = taskHashes.get('task-a');
+            mockRunner.setResult(taskAHash, async (_inputHashes) => {
+                callCount++;
+                if (callCount === 1) {
+                    const newHash = await datasetWrite(storage, testRepo, 'changed', StringType);
+                    const ref = variant('value', { hash: newHash, versions: new Map() });
+                    await storage.datasets.write(testRepo, 'test-ws', 'input', ref);
+                }
+                return {
+                    state: 'success',
+                    cached: false,
+                    outputHash: `output-v${callCount}`,
+                };
+            });
+            const invalidated = [];
+            await dataflowExecute(storage, testRepo, 'test-ws', {
+                runner: mockRunner,
+                onTaskInvalidated: (name, reason) => {
+                    invalidated.push({ name, reason });
+                },
+            });
+            assert.strictEqual(invalidated.length, 1);
+            assert.strictEqual(invalidated[0].name, 'task-a');
+            assert.ok(invalidated[0].reason.includes('.input'), `Reason should mention input path, got: ${invalidated[0].reason}`);
+        });
+        it('handles no-op change (same hash)', async () => {
+            // Input "changes" but to same hash value → no invalidation
+            const structure = {
+                type: 'struct',
+                value: new Map([
+                    ['input', { type: 'value', value: { type: StringType, writable: true } }],
+                    ['output', { type: 'value', value: { type: StringType, writable: true } }],
+                ]),
+            };
+            const inputPath = [variant('field', 'input')];
+            const outputPath = [variant('field', 'output')];
+            const taskHashes = await createPackageWithTasks(testRepo, [
+                { name: 'task-a', command: ['echo'], inputs: [inputPath], output: outputPath },
+            ], structure);
+            await workspaceDeploy(storage, testRepo, 'test-ws', 'test', '1.0.0');
+            await workspaceSetDataset(storage, testRepo, 'test-ws', inputPath, 'same-value', StringType);
+            let callCount = 0;
+            const taskAHash = taskHashes.get('task-a');
+            mockRunner.setResult(taskAHash, async (_inputHashes) => {
+                callCount++;
+                if (callCount === 1) {
+                    // Write the same value — hash should not change
+                    const sameHash = await datasetWrite(storage, testRepo, 'same-value', StringType);
+                    const ref = variant('value', { hash: sameHash, versions: new Map() });
+                    await storage.datasets.write(testRepo, 'test-ws', 'input', ref);
+                }
+                return {
+                    state: 'success',
+                    cached: false,
+                    outputHash: `output-v${callCount}`,
+                };
+            });
+            const result = await dataflowExecute(storage, testRepo, 'test-ws', {
+                runner: mockRunner,
+            });
+            assert.strictEqual(result.success, true);
+            // Should NOT re-execute since the hash is the same
+            assert.strictEqual(callCount, 1);
+            assert.strictEqual(result.reexecuted, 0);
+        });
+    });
+    describe('DataflowRun recording', () => {
+        it('records correct outputVersions with task output hashes', async () => {
+            const structure = {
+                type: 'struct',
+                value: new Map([
+                    ['input', { type: 'value', value: { type: StringType, writable: true } }],
+                    ['output', { type: 'value', value: { type: StringType, writable: true } }],
+                ]),
+            };
+            const inputPath = [variant('field', 'input')];
+            const outputPath = [variant('field', 'output')];
+            const taskHashes = await createPackageWithTasks(testRepo, [{ name: 'task-a', command: ['echo'], inputs: [inputPath], output: outputPath }], structure);
+            await workspaceDeploy(storage, testRepo, 'test-ws', 'test', '1.0.0');
+            await workspaceSetDataset(storage, testRepo, 'test-ws', inputPath, 'test', StringType);
+            mockRunner.setResult(taskHashes.get('task-a'), {
+                state: 'success',
+                cached: false,
+                outputHash: 'task-a-output-hash',
+            });
+            const result = await dataflowExecute(storage, testRepo, 'test-ws', {
+                runner: mockRunner,
+            });
+            assert.strictEqual(result.success, true);
+            const run = await storage.refs.dataflowRunGetLatest(testRepo, 'test-ws');
+            assert.ok(run, 'DataflowRun should exist');
+            assert.strictEqual(run.outputVersions.type, 'some');
+            const outputVersions = run.outputVersions.value;
+            assert.strictEqual(outputVersions.get('.output'), 'task-a-output-hash');
+            assert.strictEqual(outputVersions.has('.input'), false, 'Input should not appear in outputVersions');
+        });
+        it('records outputVersions for all completed tasks in a chain', async () => {
+            const structure = {
+                type: 'struct',
+                value: new Map([
+                    ['input', { type: 'value', value: { type: StringType, writable: true } }],
+                    ['middle', { type: 'value', value: { type: StringType, writable: true } }],
+                    ['output', { type: 'value', value: { type: StringType, writable: true } }],
+                ]),
+            };
+            const inputPath = [variant('field', 'input')];
+            const middlePath = [variant('field', 'middle')];
+            const outputPath = [variant('field', 'output')];
+            const taskHashes = await createPackageWithTasks(testRepo, [
+                { name: 'task-a', command: ['echo'], inputs: [inputPath], output: middlePath },
+                { name: 'task-b', command: ['echo'], inputs: [middlePath], output: outputPath },
+            ], structure);
+            await workspaceDeploy(storage, testRepo, 'test-ws', 'test', '1.0.0');
+            await workspaceSetDataset(storage, testRepo, 'test-ws', inputPath, 'test', StringType);
+            mockRunner.setResult(taskHashes.get('task-a'), {
+                state: 'success',
+                cached: false,
+                outputHash: 'middle-hash',
+            });
+            mockRunner.setResult(taskHashes.get('task-b'), {
+                state: 'success',
+                cached: false,
+                outputHash: 'output-hash',
+            });
+            const result = await dataflowExecute(storage, testRepo, 'test-ws', {
+                runner: mockRunner,
+            });
+            assert.strictEqual(result.success, true);
+            const run = await storage.refs.dataflowRunGetLatest(testRepo, 'test-ws');
+            assert.ok(run, 'DataflowRun should exist');
+            assert.strictEqual(run.outputVersions.type, 'some');
+            const outputVersions = run.outputVersions.value;
+            assert.strictEqual(outputVersions.get('.middle'), 'middle-hash');
+            assert.strictEqual(outputVersions.get('.output'), 'output-hash');
+        });
+        it('records partial outputVersions when execution is cancelled', async () => {
+            const structure = {
+                type: 'struct',
+                value: new Map([
+                    ['input', { type: 'value', value: { type: StringType, writable: true } }],
+                    ['output', { type: 'value', value: { type: StringType, writable: true } }],
+                ]),
+            };
+            const inputPath = [variant('field', 'input')];
+            const outputPath = [variant('field', 'output')];
+            const taskHashes = await createPackageWithTasks(testRepo, [{ name: 'task-a', command: ['echo'], inputs: [inputPath], output: outputPath }], structure);
+            await workspaceDeploy(storage, testRepo, 'test-ws', 'test', '1.0.0');
+            await workspaceSetDataset(storage, testRepo, 'test-ws', inputPath, 'test', StringType);
+            const controller = new AbortController();
+            // Task aborts during execution
+            mockRunner.setResult(taskHashes.get('task-a'), async () => {
+                controller.abort();
+                // Small delay to let abort propagate
+                await new Promise(resolve => setTimeout(resolve, 10));
+                return {
+                    state: 'success',
+                    cached: false,
+                    outputHash: 'task-a-output-hash',
+                };
+            });
+            const { DataflowAbortedError } = await import('./errors.js');
+            await assert.rejects(dataflowExecute(storage, testRepo, 'test-ws', {
+                runner: mockRunner,
+                signal: controller.signal,
+            }), (err) => {
+                assert.ok(err instanceof DataflowAbortedError);
+                return true;
+            });
+            const run = await storage.refs.dataflowRunGetLatest(testRepo, 'test-ws');
+            assert.ok(run, 'DataflowRun should exist after cancellation');
+            assert.strictEqual(run.status.type, 'cancelled');
+            assert.strictEqual(run.outputVersions.type, 'some');
+            // Input should not appear in outputVersions even on cancellation
+            assert.strictEqual(run.outputVersions.value.has('.input'), false);
+        });
+    });
+    describe('abort cleanup', () => {
+        it('removes abort listener after normal completion', async () => {
+            const structure = {
+                type: 'struct',
+                value: new Map([
+                    ['input', { type: 'value', value: { type: StringType, writable: true } }],
+                    ['output', { type: 'value', value: { type: StringType, writable: true } }],
+                ]),
+            };
+            const inputPath = [variant('field', 'input')];
+            const outputPath = [variant('field', 'output')];
+            const taskHashes = await createPackageWithTasks(testRepo, [{ name: 'task', command: ['echo'], inputs: [inputPath], output: outputPath }], structure);
+            await workspaceDeploy(storage, testRepo, 'test-ws', 'test', '1.0.0');
+            await workspaceSetDataset(storage, testRepo, 'test-ws', inputPath, 'test', StringType);
+            mockRunner.setResult(taskHashes.get('task'), {
+                state: 'success',
+                cached: false,
+                outputHash: 'output-hash',
+            });
+            const controller = new AbortController();
+            const result = await dataflowExecute(storage, testRepo, 'test-ws', {
+                runner: mockRunner,
+                signal: controller.signal,
+            });
+            assert.strictEqual(result.success, true);
+            // After execution completes, aborting should not throw.
+            // If the abort listener were still attached, it could attempt to write
+            // to a cleaned-up state store and throw.
+            assert.doesNotThrow(() => controller.abort());
+        });
+    });
+    describe('cache-hit mutex', () => {
+        it('correctly handles cache hit during concurrent execution', async () => {
+            const structure = {
+                type: 'struct',
+                value: new Map([
+                    ['input', { type: 'value', value: { type: StringType, writable: true } }],
+                    ['out_a', { type: 'value', value: { type: StringType, writable: true } }],
+                    ['out_b', { type: 'value', value: { type: StringType, writable: true } }],
+                ]),
+            };
+            const inputPath = [variant('field', 'input')];
+            const taskHashes = await createPackageWithTasks(testRepo, [
+                { name: 'task-a', command: ['echo'], inputs: [inputPath], output: [variant('field', 'out_a')] },
+                { name: 'task-b', command: ['echo'], inputs: [inputPath], output: [variant('field', 'out_b')] },
+            ], structure);
+            await workspaceDeploy(storage, testRepo, 'test-ws', 'test', '1.0.0');
+            await workspaceSetDataset(storage, testRepo, 'test-ws', inputPath, 'test', StringType);
+            // First run: both tasks execute and capture input hashes
+            const capturedInputHashes = new Map();
+            for (const [name, hash] of taskHashes) {
+                mockRunner.setResult(hash, (inputHashesArr) => {
+                    capturedInputHashes.set(name, [...inputHashesArr]);
+                    return {
+                        state: 'success',
+                        cached: false,
+                        outputHash: `output-${name}`,
+                    };
+                });
+            }
+            const result1 = await dataflowExecute(storage, testRepo, 'test-ws', {
+                runner: mockRunner,
+                concurrency: 4,
+            });
+            assert.strictEqual(result1.success, true);
+            assert.strictEqual(result1.executed, 2);
+            // Write execution cache entries so the second run finds cached outputs.
+            // The orchestrator's stepPrepareTask checks the execution store.
+            // The executionId must be UUIDv7 format for LocalRefStore to find it.
+            const now = new Date();
+            const fakeUuid = '01900000-0000-7000-8000-000000000001';
+            for (const [name, hash] of taskHashes) {
+                const captured = capturedInputHashes.get(name);
+                assert.ok(captured, `Should have captured input hashes for ${name}`);
+                const inHash = inputsHash(captured);
+                await storage.refs.executionWrite(testRepo, hash, inHash, fakeUuid, variant('success', {
+                    executionId: fakeUuid,
+                    inputHashes: captured,
+                    outputHash: `output-${name}`,
+                    startedAt: now,
+                    completedAt: now,
+                }));
+            }
+            // Second run: both tasks should be inline cache hits (workspace output matches)
+            mockRunner.clearCalls();
+            const result2 = await dataflowExecute(storage, testRepo, 'test-ws', {
+                runner: mockRunner,
+                concurrency: 4,
+            });
+            assert.strictEqual(result2.success, true);
+            assert.strictEqual(result2.cached, 2);
+            assert.strictEqual(result2.executed, 0);
+            // MockRunner should not have been called — cache resolved inline
+            assert.strictEqual(mockRunner.getCalls().length, 0);
+        });
+    });
+});
+//# sourceMappingURL=dataflow-orchestration.spec.js.map