@elaraai/e3-core 0.0.2-beta.3 → 0.0.2-beta.30
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +25 -22
- package/dist/src/dataflow/api-compat.d.ts +90 -0
- package/dist/src/dataflow/api-compat.d.ts.map +1 -0
- package/dist/src/dataflow/api-compat.js +134 -0
- package/dist/src/dataflow/api-compat.js.map +1 -0
- package/dist/src/dataflow/index.d.ts +18 -0
- package/dist/src/dataflow/index.d.ts.map +1 -0
- package/dist/src/dataflow/index.js +23 -0
- package/dist/src/dataflow/index.js.map +1 -0
- package/dist/src/dataflow/orchestrator/LocalOrchestrator.d.ts +53 -0
- package/dist/src/dataflow/orchestrator/LocalOrchestrator.d.ts.map +1 -0
- package/dist/src/dataflow/orchestrator/LocalOrchestrator.js +416 -0
- package/dist/src/dataflow/orchestrator/LocalOrchestrator.js.map +1 -0
- package/dist/src/dataflow/orchestrator/index.d.ts +12 -0
- package/dist/src/dataflow/orchestrator/index.d.ts.map +1 -0
- package/dist/src/dataflow/orchestrator/index.js +12 -0
- package/dist/src/dataflow/orchestrator/index.js.map +1 -0
- package/dist/src/dataflow/orchestrator/interfaces.d.ts +157 -0
- package/dist/src/dataflow/orchestrator/interfaces.d.ts.map +1 -0
- package/dist/src/dataflow/orchestrator/interfaces.js +51 -0
- package/dist/src/dataflow/orchestrator/interfaces.js.map +1 -0
- package/dist/src/dataflow/state-store/FileStateStore.d.ts +67 -0
- package/dist/src/dataflow/state-store/FileStateStore.d.ts.map +1 -0
- package/dist/src/dataflow/state-store/FileStateStore.js +286 -0
- package/dist/src/dataflow/state-store/FileStateStore.js.map +1 -0
- package/dist/src/dataflow/state-store/InMemoryStateStore.d.ts +42 -0
- package/dist/src/dataflow/state-store/InMemoryStateStore.d.ts.map +1 -0
- package/dist/src/dataflow/state-store/InMemoryStateStore.js +214 -0
- package/dist/src/dataflow/state-store/InMemoryStateStore.js.map +1 -0
- package/dist/src/dataflow/state-store/index.d.ts +13 -0
- package/dist/src/dataflow/state-store/index.d.ts.map +1 -0
- package/dist/src/dataflow/state-store/index.js +13 -0
- package/dist/src/dataflow/state-store/index.js.map +1 -0
- package/dist/src/dataflow/state-store/interfaces.d.ts +159 -0
- package/dist/src/dataflow/state-store/interfaces.d.ts.map +1 -0
- package/dist/src/dataflow/state-store/interfaces.js +6 -0
- package/dist/src/dataflow/state-store/interfaces.js.map +1 -0
- package/dist/src/dataflow/steps.d.ts +176 -0
- package/dist/src/dataflow/steps.d.ts.map +1 -0
- package/dist/src/dataflow/steps.js +528 -0
- package/dist/src/dataflow/steps.js.map +1 -0
- package/dist/src/dataflow/types.d.ts +116 -0
- package/dist/src/dataflow/types.d.ts.map +1 -0
- package/dist/src/dataflow/types.js +7 -0
- package/dist/src/dataflow/types.js.map +1 -0
- package/dist/src/dataflow.d.ts +142 -9
- package/dist/src/dataflow.d.ts.map +1 -1
- package/dist/src/dataflow.js +427 -64
- package/dist/src/dataflow.js.map +1 -1
- package/dist/src/errors.d.ts +39 -9
- package/dist/src/errors.d.ts.map +1 -1
- package/dist/src/errors.js +51 -8
- package/dist/src/errors.js.map +1 -1
- package/dist/src/execution/LocalTaskRunner.d.ts +73 -0
- package/dist/src/execution/LocalTaskRunner.d.ts.map +1 -0
- package/dist/src/execution/LocalTaskRunner.js +399 -0
- package/dist/src/execution/LocalTaskRunner.js.map +1 -0
- package/dist/src/execution/MockTaskRunner.d.ts +49 -0
- package/dist/src/execution/MockTaskRunner.d.ts.map +1 -0
- package/dist/src/execution/MockTaskRunner.js +55 -0
- package/dist/src/execution/MockTaskRunner.js.map +1 -0
- package/dist/src/execution/index.d.ts +16 -0
- package/dist/src/execution/index.d.ts.map +1 -0
- package/dist/src/execution/index.js +8 -0
- package/dist/src/execution/index.js.map +1 -0
- package/dist/src/execution/interfaces.d.ts +246 -0
- package/dist/src/execution/interfaces.d.ts.map +1 -0
- package/dist/src/execution/interfaces.js +6 -0
- package/dist/src/execution/interfaces.js.map +1 -0
- package/dist/src/execution/processHelpers.d.ts +20 -0
- package/dist/src/execution/processHelpers.d.ts.map +1 -0
- package/dist/src/execution/processHelpers.js +62 -0
- package/dist/src/execution/processHelpers.js.map +1 -0
- package/dist/src/executions.d.ts +71 -104
- package/dist/src/executions.d.ts.map +1 -1
- package/dist/src/executions.js +110 -476
- package/dist/src/executions.js.map +1 -1
- package/dist/src/index.d.ts +17 -9
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +44 -18
- package/dist/src/index.js.map +1 -1
- package/dist/src/objects.d.ts +6 -53
- package/dist/src/objects.d.ts.map +1 -1
- package/dist/src/objects.js +11 -232
- package/dist/src/objects.js.map +1 -1
- package/dist/src/packages.d.ts +22 -14
- package/dist/src/packages.d.ts.map +1 -1
- package/dist/src/packages.js +116 -83
- package/dist/src/packages.js.map +1 -1
- package/dist/src/storage/in-memory/InMemoryRepoStore.d.ts +35 -0
- package/dist/src/storage/in-memory/InMemoryRepoStore.d.ts.map +1 -0
- package/dist/src/storage/in-memory/InMemoryRepoStore.js +107 -0
- package/dist/src/storage/in-memory/InMemoryRepoStore.js.map +1 -0
- package/dist/src/storage/in-memory/InMemoryStorage.d.ts +114 -0
- package/dist/src/storage/in-memory/InMemoryStorage.d.ts.map +1 -0
- package/dist/src/storage/in-memory/InMemoryStorage.js +349 -0
- package/dist/src/storage/in-memory/InMemoryStorage.js.map +1 -0
- package/dist/src/storage/in-memory/index.d.ts +12 -0
- package/dist/src/storage/in-memory/index.d.ts.map +1 -0
- package/dist/src/storage/in-memory/index.js +12 -0
- package/dist/src/storage/in-memory/index.js.map +1 -0
- package/dist/src/storage/index.d.ts +18 -0
- package/dist/src/storage/index.d.ts.map +1 -0
- package/dist/src/storage/index.js +10 -0
- package/dist/src/storage/index.js.map +1 -0
- package/dist/src/storage/interfaces.d.ts +520 -0
- package/dist/src/storage/interfaces.d.ts.map +1 -0
- package/dist/src/storage/interfaces.js +6 -0
- package/dist/src/storage/interfaces.js.map +1 -0
- package/dist/src/storage/local/LocalBackend.d.ts +54 -0
- package/dist/src/storage/local/LocalBackend.d.ts.map +1 -0
- package/dist/src/storage/local/LocalBackend.js +141 -0
- package/dist/src/storage/local/LocalBackend.js.map +1 -0
- package/dist/src/storage/local/LocalLockService.d.ts +105 -0
- package/dist/src/storage/local/LocalLockService.d.ts.map +1 -0
- package/dist/src/storage/local/LocalLockService.js +342 -0
- package/dist/src/storage/local/LocalLockService.js.map +1 -0
- package/dist/src/storage/local/LocalLogStore.d.ts +23 -0
- package/dist/src/storage/local/LocalLogStore.d.ts.map +1 -0
- package/dist/src/storage/local/LocalLogStore.js +66 -0
- package/dist/src/storage/local/LocalLogStore.js.map +1 -0
- package/dist/src/storage/local/LocalObjectStore.d.ts +52 -0
- package/dist/src/storage/local/LocalObjectStore.d.ts.map +1 -0
- package/dist/src/storage/local/LocalObjectStore.js +287 -0
- package/dist/src/storage/local/LocalObjectStore.js.map +1 -0
- package/dist/src/storage/local/LocalRefStore.d.ts +50 -0
- package/dist/src/storage/local/LocalRefStore.d.ts.map +1 -0
- package/dist/src/storage/local/LocalRefStore.js +337 -0
- package/dist/src/storage/local/LocalRefStore.js.map +1 -0
- package/dist/src/storage/local/LocalRepoStore.d.ts +53 -0
- package/dist/src/storage/local/LocalRepoStore.d.ts.map +1 -0
- package/dist/src/storage/local/LocalRepoStore.js +353 -0
- package/dist/src/storage/local/LocalRepoStore.js.map +1 -0
- package/dist/src/storage/local/gc.d.ts +92 -0
- package/dist/src/storage/local/gc.d.ts.map +1 -0
- package/dist/src/storage/local/gc.js +322 -0
- package/dist/src/storage/local/gc.js.map +1 -0
- package/dist/src/storage/local/index.d.ts +17 -0
- package/dist/src/storage/local/index.d.ts.map +1 -0
- package/dist/src/storage/local/index.js +17 -0
- package/dist/src/storage/local/index.js.map +1 -0
- package/dist/src/storage/local/localHelpers.d.ts +25 -0
- package/dist/src/storage/local/localHelpers.d.ts.map +1 -0
- package/dist/src/storage/local/localHelpers.js +69 -0
- package/dist/src/storage/local/localHelpers.js.map +1 -0
- package/dist/src/{repository.d.ts → storage/local/repository.d.ts} +8 -4
- package/dist/src/storage/local/repository.d.ts.map +1 -0
- package/dist/src/{repository.js → storage/local/repository.js} +31 -29
- package/dist/src/storage/local/repository.js.map +1 -0
- package/dist/src/tasks.d.ts +16 -10
- package/dist/src/tasks.d.ts.map +1 -1
- package/dist/src/tasks.js +35 -41
- package/dist/src/tasks.js.map +1 -1
- package/dist/src/test-helpers.d.ts +4 -4
- package/dist/src/test-helpers.d.ts.map +1 -1
- package/dist/src/test-helpers.js +7 -21
- package/dist/src/test-helpers.js.map +1 -1
- package/dist/src/trees.d.ts +89 -27
- package/dist/src/trees.d.ts.map +1 -1
- package/dist/src/trees.js +218 -100
- package/dist/src/trees.js.map +1 -1
- package/dist/src/uuid.d.ts +26 -0
- package/dist/src/uuid.d.ts.map +1 -0
- package/dist/src/uuid.js +80 -0
- package/dist/src/uuid.js.map +1 -0
- package/dist/src/workspaceStatus.d.ts +6 -4
- package/dist/src/workspaceStatus.d.ts.map +1 -1
- package/dist/src/workspaceStatus.js +43 -49
- package/dist/src/workspaceStatus.js.map +1 -1
- package/dist/src/workspaces.d.ts +35 -26
- package/dist/src/workspaces.d.ts.map +1 -1
- package/dist/src/workspaces.js +169 -118
- package/dist/src/workspaces.js.map +1 -1
- package/package.json +4 -4
- package/dist/src/gc.d.ts +0 -54
- package/dist/src/gc.d.ts.map +0 -1
- package/dist/src/gc.js +0 -233
- package/dist/src/gc.js.map +0 -1
- package/dist/src/repository.d.ts.map +0 -1
- package/dist/src/repository.js.map +0 -1
- package/dist/src/workspaceLock.d.ts +0 -67
- package/dist/src/workspaceLock.d.ts.map +0 -1
- package/dist/src/workspaceLock.js +0 -217
- package/dist/src/workspaceLock.js.map +0 -1
|
@@ -0,0 +1,399 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Copyright (c) 2025 Elara AI Pty Ltd
|
|
3
|
+
* Licensed under BSL 1.1. See LICENSE for details.
|
|
4
|
+
*/
|
|
5
|
+
/**
|
|
6
|
+
* Local task execution for e3 repositories.
|
|
7
|
+
*
|
|
8
|
+
* This module handles all local process-specific execution:
|
|
9
|
+
* - Creating temporary scratch directories for task I/O
|
|
10
|
+
* - Spawning runner processes (east-node, east-py, julia)
|
|
11
|
+
* - Capturing stdout/stderr and persisting to logs
|
|
12
|
+
* - Process lifecycle management (signals, timeouts, cleanup)
|
|
13
|
+
*/
|
|
14
|
+
import * as fs from 'fs/promises';
|
|
15
|
+
import * as path from 'path';
|
|
16
|
+
import { spawn } from 'child_process';
|
|
17
|
+
import { tmpdir } from 'os';
|
|
18
|
+
import { decodeBeast2For, variant } from '@elaraai/east';
|
|
19
|
+
import { TaskObjectType } from '@elaraai/e3-types';
|
|
20
|
+
import { inputsHash, evaluateCommandIr } from '../executions.js';
|
|
21
|
+
import { uuidv7 } from '../uuid.js';
|
|
22
|
+
import { getBootId, getPidStartTime } from './processHelpers.js';
|
|
23
|
+
/**
|
|
24
|
+
* TaskRunner implementation for local process execution.
|
|
25
|
+
*
|
|
26
|
+
* Spawns runner processes locally to execute tasks.
|
|
27
|
+
* Used by the local CLI and e3-api-server for task execution.
|
|
28
|
+
*/
|
|
29
|
+
export class LocalTaskRunner {
|
|
30
|
+
repo;
|
|
31
|
+
constructor(repo) {
|
|
32
|
+
this.repo = repo;
|
|
33
|
+
}
|
|
34
|
+
async execute(storage, taskHash, inputHashes, options) {
|
|
35
|
+
const result = await taskExecute(storage, this.repo, taskHash, inputHashes, {
|
|
36
|
+
force: options?.force,
|
|
37
|
+
signal: options?.signal,
|
|
38
|
+
onStdout: options?.onStdout,
|
|
39
|
+
onStderr: options?.onStderr,
|
|
40
|
+
});
|
|
41
|
+
// Convert ExecutionResult to TaskResult
|
|
42
|
+
const taskResult = {
|
|
43
|
+
state: result.state,
|
|
44
|
+
cached: result.cached,
|
|
45
|
+
executionId: result.executionId,
|
|
46
|
+
};
|
|
47
|
+
if (result.state === 'success' && result.outputHash) {
|
|
48
|
+
taskResult.outputHash = result.outputHash;
|
|
49
|
+
}
|
|
50
|
+
else if (result.state === 'failed') {
|
|
51
|
+
taskResult.exitCode = result.exitCode ?? undefined;
|
|
52
|
+
}
|
|
53
|
+
else if (result.state === 'error') {
|
|
54
|
+
taskResult.error = result.error ?? undefined;
|
|
55
|
+
}
|
|
56
|
+
return taskResult;
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
/**
|
|
60
|
+
* Execute a single task.
|
|
61
|
+
*
|
|
62
|
+
* This is the core execution primitive. It:
|
|
63
|
+
* 1. Computes the execution identity from task + inputs
|
|
64
|
+
* 2. Checks cache (unless force=true)
|
|
65
|
+
* 3. Marshals inputs to a scratch directory
|
|
66
|
+
* 4. Evaluates command IR to get exec args
|
|
67
|
+
* 5. Runs the command
|
|
68
|
+
* 6. Stores the output and updates status
|
|
69
|
+
*
|
|
70
|
+
* @param storage - Storage backend
|
|
71
|
+
* @param repo - Repository identifier (for local storage, the path to e3 repository directory)
|
|
72
|
+
* @param taskHash - Hash of the task object
|
|
73
|
+
* @param inputHashes - Array of input dataset hashes
|
|
74
|
+
* @param options - Execution options
|
|
75
|
+
* @returns Execution result
|
|
76
|
+
*/
|
|
77
|
+
export async function taskExecute(storage, repo, taskHash, inputHashes, options = {}) {
|
|
78
|
+
const inHash = inputsHash(inputHashes);
|
|
79
|
+
const startTime = Date.now();
|
|
80
|
+
// Step 1: Check cache (unless force)
|
|
81
|
+
if (!options.force) {
|
|
82
|
+
const existingOutput = await storage.refs.executionGetLatestOutput(repo, taskHash, inHash);
|
|
83
|
+
if (existingOutput !== null) {
|
|
84
|
+
const status = await storage.refs.executionGetLatest(repo, taskHash, inHash);
|
|
85
|
+
if (status && status.type === 'success') {
|
|
86
|
+
return {
|
|
87
|
+
inputsHash: inHash,
|
|
88
|
+
executionId: status.value.executionId,
|
|
89
|
+
cached: true,
|
|
90
|
+
state: 'success',
|
|
91
|
+
outputHash: existingOutput,
|
|
92
|
+
exitCode: 0,
|
|
93
|
+
duration: 0,
|
|
94
|
+
error: null,
|
|
95
|
+
};
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
// Step 2: Generate a new execution ID
|
|
100
|
+
const executionId = uuidv7();
|
|
101
|
+
// Step 3: Read task object
|
|
102
|
+
let task;
|
|
103
|
+
try {
|
|
104
|
+
const taskData = await storage.objects.read(repo, taskHash);
|
|
105
|
+
const decoder = decodeBeast2For(TaskObjectType);
|
|
106
|
+
task = decoder(Buffer.from(taskData));
|
|
107
|
+
}
|
|
108
|
+
catch (err) {
|
|
109
|
+
// Record error with executionId for audit trail
|
|
110
|
+
const status = variant('error', {
|
|
111
|
+
executionId,
|
|
112
|
+
inputHashes,
|
|
113
|
+
startedAt: new Date(startTime),
|
|
114
|
+
completedAt: new Date(),
|
|
115
|
+
message: `Failed to read task object: ${err}`,
|
|
116
|
+
});
|
|
117
|
+
await storage.refs.executionWrite(repo, taskHash, inHash, executionId, status);
|
|
118
|
+
return {
|
|
119
|
+
inputsHash: inHash,
|
|
120
|
+
executionId,
|
|
121
|
+
cached: false,
|
|
122
|
+
state: 'error',
|
|
123
|
+
outputHash: null,
|
|
124
|
+
exitCode: null,
|
|
125
|
+
duration: Date.now() - startTime,
|
|
126
|
+
error: `Failed to read task object: ${err}`,
|
|
127
|
+
};
|
|
128
|
+
}
|
|
129
|
+
// Step 4: Create scratch directory
|
|
130
|
+
// Include PID to prevent collisions when multiple e3 processes run the same
|
|
131
|
+
// task concurrently (e.g., same task in different workspaces at same millisecond)
|
|
132
|
+
const scratchDir = path.join(tmpdir(), `e3-exec-${taskHash.slice(0, 8)}-${inHash.slice(0, 8)}-${process.pid}-${Date.now()}`);
|
|
133
|
+
await fs.mkdir(scratchDir, { recursive: true });
|
|
134
|
+
try {
|
|
135
|
+
// Step 5: Marshal inputs to scratch dir
|
|
136
|
+
const inputPaths = [];
|
|
137
|
+
for (let i = 0; i < inputHashes.length; i++) {
|
|
138
|
+
const inputPath = path.join(scratchDir, `input-${i}.beast2`);
|
|
139
|
+
const inputData = await storage.objects.read(repo, inputHashes[i]);
|
|
140
|
+
await fs.writeFile(inputPath, inputData);
|
|
141
|
+
inputPaths.push(inputPath);
|
|
142
|
+
}
|
|
143
|
+
// Step 6: Evaluate command IR to get exec args
|
|
144
|
+
const outputPath = path.join(scratchDir, 'output.beast2');
|
|
145
|
+
let args;
|
|
146
|
+
try {
|
|
147
|
+
args = await evaluateCommandIr(storage, repo, task.commandIr, inputPaths, outputPath);
|
|
148
|
+
}
|
|
149
|
+
catch (err) {
|
|
150
|
+
const status = variant('error', {
|
|
151
|
+
executionId,
|
|
152
|
+
inputHashes,
|
|
153
|
+
startedAt: new Date(startTime),
|
|
154
|
+
completedAt: new Date(),
|
|
155
|
+
message: `Failed to evaluate command IR: ${err}`,
|
|
156
|
+
});
|
|
157
|
+
await storage.refs.executionWrite(repo, taskHash, inHash, executionId, status);
|
|
158
|
+
return {
|
|
159
|
+
inputsHash: inHash,
|
|
160
|
+
executionId,
|
|
161
|
+
cached: false,
|
|
162
|
+
state: 'error',
|
|
163
|
+
outputHash: null,
|
|
164
|
+
exitCode: null,
|
|
165
|
+
duration: Date.now() - startTime,
|
|
166
|
+
error: `Failed to evaluate command IR: ${err}`,
|
|
167
|
+
};
|
|
168
|
+
}
|
|
169
|
+
if (args.length === 0) {
|
|
170
|
+
const status = variant('error', {
|
|
171
|
+
executionId,
|
|
172
|
+
inputHashes,
|
|
173
|
+
startedAt: new Date(startTime),
|
|
174
|
+
completedAt: new Date(),
|
|
175
|
+
message: 'Command IR produced empty command',
|
|
176
|
+
});
|
|
177
|
+
await storage.refs.executionWrite(repo, taskHash, inHash, executionId, status);
|
|
178
|
+
return {
|
|
179
|
+
inputsHash: inHash,
|
|
180
|
+
executionId,
|
|
181
|
+
cached: false,
|
|
182
|
+
state: 'error',
|
|
183
|
+
outputHash: null,
|
|
184
|
+
exitCode: null,
|
|
185
|
+
duration: Date.now() - startTime,
|
|
186
|
+
error: 'Command IR produced empty command',
|
|
187
|
+
};
|
|
188
|
+
}
|
|
189
|
+
// Step 7: Get boot ID for crash detection
|
|
190
|
+
const bootId = await getBootId();
|
|
191
|
+
// Step 8: Execute command
|
|
192
|
+
const result = await runCommand(storage, repo, taskHash, inHash, executionId, args, inputHashes, bootId, options);
|
|
193
|
+
// Step 9: Handle result
|
|
194
|
+
if (result.exitCode === 0) {
|
|
195
|
+
// Success - read and store output
|
|
196
|
+
try {
|
|
197
|
+
const outputData = await fs.readFile(outputPath);
|
|
198
|
+
const outputHash = await storage.objects.write(repo, outputData);
|
|
199
|
+
// Write success status (output is stored within status.beast2's directory)
|
|
200
|
+
const status = variant('success', {
|
|
201
|
+
executionId,
|
|
202
|
+
inputHashes,
|
|
203
|
+
outputHash,
|
|
204
|
+
startedAt: new Date(startTime),
|
|
205
|
+
completedAt: new Date(),
|
|
206
|
+
});
|
|
207
|
+
await storage.refs.executionWrite(repo, taskHash, inHash, executionId, status);
|
|
208
|
+
return {
|
|
209
|
+
inputsHash: inHash,
|
|
210
|
+
executionId,
|
|
211
|
+
cached: false,
|
|
212
|
+
state: 'success',
|
|
213
|
+
outputHash,
|
|
214
|
+
exitCode: 0,
|
|
215
|
+
duration: Date.now() - startTime,
|
|
216
|
+
error: null,
|
|
217
|
+
};
|
|
218
|
+
}
|
|
219
|
+
catch (err) {
|
|
220
|
+
// Output file missing or unreadable
|
|
221
|
+
const status = variant('error', {
|
|
222
|
+
executionId,
|
|
223
|
+
inputHashes,
|
|
224
|
+
startedAt: new Date(startTime),
|
|
225
|
+
completedAt: new Date(),
|
|
226
|
+
message: `Failed to read output: ${err}`,
|
|
227
|
+
});
|
|
228
|
+
await storage.refs.executionWrite(repo, taskHash, inHash, executionId, status);
|
|
229
|
+
return {
|
|
230
|
+
inputsHash: inHash,
|
|
231
|
+
executionId,
|
|
232
|
+
cached: false,
|
|
233
|
+
state: 'error',
|
|
234
|
+
outputHash: null,
|
|
235
|
+
exitCode: 0,
|
|
236
|
+
duration: Date.now() - startTime,
|
|
237
|
+
error: `Failed to read output: ${err}`,
|
|
238
|
+
};
|
|
239
|
+
}
|
|
240
|
+
}
|
|
241
|
+
else {
|
|
242
|
+
// Failed - write failed status
|
|
243
|
+
const status = variant('failed', {
|
|
244
|
+
executionId,
|
|
245
|
+
inputHashes,
|
|
246
|
+
startedAt: new Date(startTime),
|
|
247
|
+
completedAt: new Date(),
|
|
248
|
+
exitCode: BigInt(result?.exitCode ?? -1),
|
|
249
|
+
});
|
|
250
|
+
await storage.refs.executionWrite(repo, taskHash, inHash, executionId, status);
|
|
251
|
+
return {
|
|
252
|
+
inputsHash: inHash,
|
|
253
|
+
executionId,
|
|
254
|
+
cached: false,
|
|
255
|
+
state: 'failed',
|
|
256
|
+
outputHash: null,
|
|
257
|
+
exitCode: result.exitCode,
|
|
258
|
+
duration: Date.now() - startTime,
|
|
259
|
+
error: result.error,
|
|
260
|
+
};
|
|
261
|
+
}
|
|
262
|
+
}
|
|
263
|
+
finally {
|
|
264
|
+
// Cleanup scratch directory
|
|
265
|
+
try {
|
|
266
|
+
await fs.rm(scratchDir, { recursive: true, force: true });
|
|
267
|
+
}
|
|
268
|
+
catch {
|
|
269
|
+
// Ignore cleanup errors
|
|
270
|
+
}
|
|
271
|
+
}
|
|
272
|
+
}
|
|
273
|
+
/**
|
|
274
|
+
* Run a command and capture output
|
|
275
|
+
*/
|
|
276
|
+
async function runCommand(storage, repo, taskHash, inHash, executionId, args, inputHashes, bootId, options) {
|
|
277
|
+
const [cmd, ...cmdArgs] = args;
|
|
278
|
+
// Process Lifecycle Management
|
|
279
|
+
// ============================
|
|
280
|
+
// We use detached: true to create a new process group, allowing us to kill
|
|
281
|
+
// the entire process tree by signaling the negative PID (process group leader).
|
|
282
|
+
//
|
|
283
|
+
// LIMITATION: Process groups are flat, not hierarchical. If a task spawns a
|
|
284
|
+
// subprocess that creates its own process group (via setsid, daemonization,
|
|
285
|
+
// or another detached spawn), that subprocess will escape our kill signal.
|
|
286
|
+
// This is a fundamental Unix limitation - process groups were designed for
|
|
287
|
+
// terminal job control (Ctrl+C/Ctrl+Z), not process tree management.
|
|
288
|
+
//
|
|
289
|
+
// For most tasks (shell scripts, pipelines, normal child processes) this works
|
|
290
|
+
// fine. A task would have to intentionally call setsid() to escape.
|
|
291
|
+
//
|
|
292
|
+
// Potential improvements for hosted e3:
|
|
293
|
+
// - Linux cgroups: Hierarchical containment with no escape. Requires root or
|
|
294
|
+
// systemd integration (systemd-run --scope). Used by Docker/Kubernetes.
|
|
295
|
+
// - PR_SET_CHILD_SUBREAPER: Makes e3 adopt orphaned processes instead of init,
|
|
296
|
+
// allowing tracking and cleanup. Requires polling to detect orphans.
|
|
297
|
+
// - Firecracker/microVMs: Complete isolation with hardware virtualization.
|
|
298
|
+
// The VM boundary provides bulletproof containment. Best for multi-tenant.
|
|
299
|
+
const child = spawn(cmd, cmdArgs, {
|
|
300
|
+
stdio: ['ignore', 'pipe', 'pipe'],
|
|
301
|
+
detached: true,
|
|
302
|
+
});
|
|
303
|
+
// Set up event listeners IMMEDIATELY before any async work
|
|
304
|
+
// to avoid missing events if the process completes quickly
|
|
305
|
+
const resultPromise = new Promise((resolve) => {
|
|
306
|
+
child.on('error', (err) => {
|
|
307
|
+
resolve({ exitCode: null, error: `Failed to spawn: ${err.message}` });
|
|
308
|
+
});
|
|
309
|
+
child.on('close', (code) => {
|
|
310
|
+
resolve({ exitCode: code, error: code !== 0 ? `Exit code: ${code}` : null });
|
|
311
|
+
});
|
|
312
|
+
});
|
|
313
|
+
// Use promise chains to ensure sequential log writes without overlapping
|
|
314
|
+
let stdoutWriteChain = Promise.resolve();
|
|
315
|
+
let stderrWriteChain = Promise.resolve();
|
|
316
|
+
// Tee stdout - use storage.logs.append for log persistence
|
|
317
|
+
child.stdout?.on('data', (data) => {
|
|
318
|
+
const str = data.toString('utf-8');
|
|
319
|
+
// Chain writes sequentially to avoid overlapping
|
|
320
|
+
stdoutWriteChain = stdoutWriteChain.then(async () => {
|
|
321
|
+
try {
|
|
322
|
+
await storage.logs.append(repo, taskHash, inHash, executionId, 'stdout', str);
|
|
323
|
+
}
|
|
324
|
+
catch (err) {
|
|
325
|
+
console.warn(`Failed to append stdout log: ${err instanceof Error ? err.message : String(err)}`);
|
|
326
|
+
}
|
|
327
|
+
});
|
|
328
|
+
if (options.onStdout) {
|
|
329
|
+
options.onStdout(str);
|
|
330
|
+
}
|
|
331
|
+
});
|
|
332
|
+
// Tee stderr - use storage.logs.append for log persistence
|
|
333
|
+
child.stderr?.on('data', (data) => {
|
|
334
|
+
const str = data.toString('utf-8');
|
|
335
|
+
// Chain writes sequentially to avoid overlapping
|
|
336
|
+
stderrWriteChain = stderrWriteChain.then(async () => {
|
|
337
|
+
try {
|
|
338
|
+
await storage.logs.append(repo, taskHash, inHash, executionId, 'stderr', str);
|
|
339
|
+
}
|
|
340
|
+
catch (err) {
|
|
341
|
+
console.warn(`Failed to append stderr log: ${err instanceof Error ? err.message : String(err)}`);
|
|
342
|
+
}
|
|
343
|
+
});
|
|
344
|
+
if (options.onStderr) {
|
|
345
|
+
options.onStderr(str);
|
|
346
|
+
}
|
|
347
|
+
});
|
|
348
|
+
// Helper to kill the entire process group (child and all its descendants).
|
|
349
|
+
// With detached: true, child.pid is the process group leader, so killing
|
|
350
|
+
// -child.pid sends the signal to all processes in that group.
|
|
351
|
+
const killProcessGroup = () => {
|
|
352
|
+
if (child.pid) {
|
|
353
|
+
try {
|
|
354
|
+
process.kill(-child.pid, 'SIGKILL');
|
|
355
|
+
}
|
|
356
|
+
catch {
|
|
357
|
+
// Process may have already exited
|
|
358
|
+
}
|
|
359
|
+
}
|
|
360
|
+
};
|
|
361
|
+
// Handle timeout
|
|
362
|
+
let timeoutId;
|
|
363
|
+
if (options.timeout) {
|
|
364
|
+
timeoutId = setTimeout(killProcessGroup, options.timeout);
|
|
365
|
+
}
|
|
366
|
+
// Handle abort signal
|
|
367
|
+
if (options.signal) {
|
|
368
|
+
if (options.signal.aborted) {
|
|
369
|
+
// Already aborted before we started
|
|
370
|
+
killProcessGroup();
|
|
371
|
+
}
|
|
372
|
+
else {
|
|
373
|
+
options.signal.addEventListener('abort', killProcessGroup, { once: true });
|
|
374
|
+
}
|
|
375
|
+
}
|
|
376
|
+
// Write running status with actual child PID
|
|
377
|
+
const pidStartTime = await getPidStartTime(child.pid);
|
|
378
|
+
const status = variant('running', {
|
|
379
|
+
executionId,
|
|
380
|
+
inputHashes,
|
|
381
|
+
startedAt: new Date(),
|
|
382
|
+
pid: BigInt(child.pid ?? -1),
|
|
383
|
+
pidStartTime: BigInt(pidStartTime ?? -1),
|
|
384
|
+
bootId,
|
|
385
|
+
});
|
|
386
|
+
await storage.refs.executionWrite(repo, taskHash, inHash, executionId, status);
|
|
387
|
+
// Wait for process to complete
|
|
388
|
+
const result = await resultPromise;
|
|
389
|
+
// Wait for any pending log writes to complete
|
|
390
|
+
await Promise.all([stdoutWriteChain, stderrWriteChain]);
|
|
391
|
+
// Cleanup
|
|
392
|
+
if (timeoutId)
|
|
393
|
+
clearTimeout(timeoutId);
|
|
394
|
+
if (options.signal) {
|
|
395
|
+
options.signal.removeEventListener('abort', killProcessGroup);
|
|
396
|
+
}
|
|
397
|
+
return result;
|
|
398
|
+
}
|
|
399
|
+
//# sourceMappingURL=LocalTaskRunner.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"LocalTaskRunner.js","sourceRoot":"","sources":["../../../src/execution/LocalTaskRunner.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH;;;;;;;;GAQG;AAEH,OAAO,KAAK,EAAE,MAAM,aAAa,CAAC;AAClC,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAC7B,OAAO,EAAE,KAAK,EAAE,MAAM,eAAe,CAAC;AACtC,OAAO,EAAE,MAAM,EAAE,MAAM,IAAI,CAAC;AAC5B,OAAO,EAAE,eAAe,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC;AACzD,OAAO,EAAwB,cAAc,EAAmB,MAAM,mBAAmB,CAAC;AAC1F,OAAO,EAAE,UAAU,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AACjE,OAAO,EAAE,MAAM,EAAE,MAAM,YAAY,CAAC;AAGpC,OAAO,EAAE,SAAS,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAwCjE;;;;;GAKG;AACH,MAAM,OAAO,eAAe;IACG;IAA7B,YAA6B,IAAY;QAAZ,SAAI,GAAJ,IAAI,CAAQ;IAAG,CAAC;IAE7C,KAAK,CAAC,OAAO,CACX,OAAuB,EACvB,QAAgB,EAChB,WAAqB,EACrB,OAA4B;QAE5B,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC,OAAO,EAAE,IAAI,CAAC,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE;YAC1E,KAAK,EAAE,OAAO,EAAE,KAAK;YACrB,MAAM,EAAE,OAAO,EAAE,MAAM;YACvB,QAAQ,EAAE,OAAO,EAAE,QAAQ;YAC3B,QAAQ,EAAE,OAAO,EAAE,QAAQ;SAC5B,CAAC,CAAC;QAEH,wCAAwC;QACxC,MAAM,UAAU,GAAe;YAC7B,KAAK,EAAE,MAAM,CAAC,KAAK;YACnB,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,WAAW,EAAE,MAAM,CAAC,WAAW;SAChC,CAAC;QAEF,IAAI,MAAM,CAAC,KAAK,KAAK,SAAS,IAAI,MAAM,CAAC,UAAU,EAAE,CAAC;YACpD,UAAU,CAAC,UAAU,GAAG,MAAM,CAAC,UAAU,CAAC;QAC5C,CAAC;aAAM,IAAI,MAAM,CAAC,KAAK,KAAK,QAAQ,EAAE,CAAC;YACrC,UAAU,CAAC,QAAQ,GAAG,MAAM,CAAC,QAAQ,IAAI,SAAS,CAAC;QACrD,CAAC;aAAM,IAAI,MAAM,CAAC,KAAK,KAAK,OAAO,EAAE,CAAC;YACpC,UAAU,CAAC,KAAK,GAAG,MAAM,CAAC,KAAK,IAAI,SAAS,CAAC;QAC/C,CAAC;QAED,OAAO,UAAU,CAAC;IACpB,CAAC;CACF;AAED;;;;;;;;;;;;;;;;;GAiBG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,OAAuB,EACvB,IAAY,EACZ,QAAgB,EAChB,WAAqB,EACrB,UAA0B,EAAE;IAE5B,MAAM,MAAM,GAAG,UAAU,CAAC,WAAW,CAAC,CAAC;IACvC,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAE7B,qCAAqC;IACrC,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;QACnB,MAAM,cAAc,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,wBAAwB,CAAC,IAAI,EAAE,QAAQ,EAAE,MAAM,CAAC,CAAC;QAC3F,IAAI,cAAc,KAAK,IAAI,EAAE,CAAC;YAC5B,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,kBAAkB,CAAC,IAAI,EAAE,QAAQ,EAAE,MAAM,CAAC,CAAC;YAC7E,IAAI,MAAM,IAAI,MAAM,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;gBACxC,OAAO;oBACL,UAAU,EAAE,MAAM;oBAClB,WAAW,EAAE,MAAM,CAAC,KAAK,CAAC,WAAW;oBACrC,MAAM,EAAE,IAAI;oBACZ,KAAK,EAAE,SAAS;oBAChB,UAAU,EAAE,cAAc;oBAC1B,QAAQ,EAAE,CAAC;oBACX,QAAQ,EAAE,CAAC;oBACX,KAAK,EAAE,IAAI;iBACZ,CAAC;YACJ,CAAC;QACH,CAAC;IACH,CAAC;IAED,sCAAsC;IACtC,MAAM,WAAW,GAAG,MAAM,EAAE,CAAC;IAE7B,2BAA2B;IAC3B,IAAI,IAAgB,CAAC;IACrB,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;QAC5D,MAAM,OAAO,GAAG,eAAe,CAAC,cAAc,CAAC,CAAC;QAChD,IAAI,GAAG,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC;IACxC,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,gDAAgD;QAChD,MAAM,MAAM,GAAoB,OAAO,CAAC,OAAO,EAAE;YAC/C,WAAW;YACX,WAAW;YACX,SAAS,EAAE,IAAI,IAAI,CAAC,SAAS,CAAC;YAC9B,WAAW,EAAE,IAAI,IAAI,EAAE;YACvB,OAAO,EAAE,+BAA+B,GAAG,EAAE;SAC9C,CAAC,CAAC;QACH,MAAM,OAAO,CAAC,IAAI,CAAC,cAAc,CAAC,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,CAAC,CAAC;QAE/E,OAAO;YACL,UAAU,EAAE,MAAM;YAClB,WAAW;YACX,MAAM,EAAE,KAAK;YACb,KAAK,EAAE,OAAO;YACd,UAAU,EAAE,IAAI;YAChB,QAAQ,EAAE,IAAI;YACd,QAAQ,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;YAChC,KAAK,EAAE,+BAA+B,GAAG,EAAE;SAC5C,CAAC;IACJ,CAAC;IAED,mCAAmC;IACnC,4EAA4E;IAC5E,kFAAkF;IAClF,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAC1B,MAAM,EAAE,EACR,WAAW,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,OAAO,CAAC,GAAG,IAAI,IAAI,CAAC,GAAG,EAAE,EAAE,CACrF,CAAC;IACF,MAAM,EAAE,CAAC,KAAK,CAAC,UAAU,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAEhD,IAAI,CAAC;QACH,wCAAwC;QACxC,MAAM,UAAU,GAAa,EAAE,CAAC;QAChC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,WAAW,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAC5C,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,SAAS,CAAC,SAAS,CAAC,CAAC;YAC7D,MAAM,SAAS,GAAG,MAAM,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,EAAE,WAAW,CAAC,CAAC,CAAE,CAAC,CAAC;YACpE,MAAM,EAAE,CAAC,SAAS,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC;YACzC,UAAU,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QAC7B,CAAC;QAED,+CAA+C;QAC/C,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,eAAe,CAAC,CAAC;QAC1D,IAAI,IAAc,CAAC;QACnB,IAAI,CAAC;YACH,IAAI,GAAG,MAAM,iBAAiB,CAAC,OAAO,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,EAAE,UAAU,EAAE,UAAU,CAAC,CAAC;QACxF,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,MAAM,GAAoB,OAAO,CAAC,OAAO,EAAE;gBAC/C,WAAW;gBACX,WAAW;gBACX,SAAS,EAAE,IAAI,IAAI,CAAC,SAAS,CAAC;gBAC9B,WAAW,EAAE,IAAI,IAAI,EAAE;gBACvB,OAAO,EAAE,kCAAkC,GAAG,EAAE;aACjD,CAAC,CAAC;YACH,MAAM,OAAO,CAAC,IAAI,CAAC,cAAc,CAAC,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,CAAC,CAAC;YAE/E,OAAO;gBACL,UAAU,EAAE,MAAM;gBAClB,WAAW;gBACX,MAAM,EAAE,KAAK;gBACb,KAAK,EAAE,OAAO;gBACd,UAAU,EAAE,IAAI;gBAChB,QAAQ,EAAE,IAAI;gBACd,QAAQ,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;gBAChC,KAAK,EAAE,kCAAkC,GAAG,EAAE;aAC/C,CAAC;QACJ,CAAC;QAED,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACtB,MAAM,MAAM,GAAoB,OAAO,CAAC,OAAO,EAAE;gBAC/C,WAAW;gBACX,WAAW;gBACX,SAAS,EAAE,IAAI,IAAI,CAAC,SAAS,CAAC;gBAC9B,WAAW,EAAE,IAAI,IAAI,EAAE;gBACvB,OAAO,EAAE,mCAAmC;aAC7C,CAAC,CAAC;YACH,MAAM,OAAO,CAAC,IAAI,CAAC,cAAc,CAAC,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,CAAC,CAAC;YAE/E,OAAO;gBACL,UAAU,EAAE,MAAM;gBAClB,WAAW;gBACX,MAAM,EAAE,KAAK;gBACb,KAAK,EAAE,OAAO;gBACd,UAAU,EAAE,IAAI;gBAChB,QAAQ,EAAE,IAAI;gBACd,QAAQ,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;gBAChC,KAAK,EAAE,mCAAmC;aAC3C,CAAC;QACJ,CAAC;QAED,0CAA0C;QAC1C,MAAM,MAAM,GAAG,MAAM,SAAS,EAAE,CAAC;QAEjC,0BAA0B;QAC1B,MAAM,MAAM,GAAG,MAAM,UAAU,CAC7B,OAAO,EACP,IAAI,EACJ,QAAQ,EACR,MAAM,EACN,WAAW,EACX,IAAI,EACJ,WAAW,EACX,MAAM,EACN,OAAO,CACR,CAAC;QAEF,wBAAwB;QACxB,IAAI,MAAM,CAAC,QAAQ,KAAK,CAAC,EAAE,CAAC;YAC1B,kCAAkC;YAClC,IAAI,CAAC;gBACH,MAAM,UAAU,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;gBACjD,MAAM,UAAU,GAAG,MAAM,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC;gBAEjE,2EAA2E;gBAC3E,MAAM,MAAM,GAAoB,OAAO,CAAC,SAAS,EAAE;oBACjD,WAAW;oBACX,WAAW;oBACX,UAAU;oBACV,SAAS,EAAE,IAAI,IAAI,CAAC,SAAS,CAAC;oBAC9B,WAAW,EAAE,IAAI,IAAI,EAAE;iBACxB,CAAC,CAAC;gBACH,MAAM,OAAO,CAAC,IAAI,CAAC,cAAc,CAAC,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,CAAC,CAAC;gBAE/E,OAAO;oBACL,UAAU,EAAE,MAAM;oBAClB,WAAW;oBACX,MAAM,EAAE,KAAK;oBACb,KAAK,EAAE,SAAS;oBAChB,UAAU;oBACV,QAAQ,EAAE,CAAC;oBACX,QAAQ,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;oBAChC,KAAK,EAAE,IAAI;iBACZ,CAAC;YACJ,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,oCAAoC;gBACpC,MAAM,MAAM,GAAoB,OAAO,CAAC,OAAO,EAAE;oBAC/C,WAAW;oBACX,WAAW;oBACX,SAAS,EAAE,IAAI,IAAI,CAAC,SAAS,CAAC;oBAC9B,WAAW,EAAE,IAAI,IAAI,EAAE;oBACvB,OAAO,EAAE,0BAA0B,GAAG,EAAE;iBACzC,CAAC,CAAC;gBACH,MAAM,OAAO,CAAC,IAAI,CAAC,cAAc,CAAC,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,CAAC,CAAC;gBAE/E,OAAO;oBACL,UAAU,EAAE,MAAM;oBAClB,WAAW;oBACX,MAAM,EAAE,KAAK;oBACb,KAAK,EAAE,OAAO;oBACd,UAAU,EAAE,IAAI;oBAChB,QAAQ,EAAE,CAAC;oBACX,QAAQ,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;oBAChC,KAAK,EAAE,0BAA0B,GAAG,EAAE;iBACvC,CAAC;YACJ,CAAC;QACH,CAAC;aAAM,CAAC;YACN,+BAA+B;YAC/B,MAAM,MAAM,GAAoB,OAAO,CAAC,QAAQ,EAAE;gBAChD,WAAW;gBACX,WAAW;gBACX,SAAS,EAAE,IAAI,IAAI,CAAC,SAAS,CAAC;gBAC9B,WAAW,EAAE,IAAI,IAAI,EAAE;gBACvB,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,QAAQ,IAAI,CAAC,CAAC,CAAC;aACzC,CAAC,CAAC;YACH,MAAM,OAAO,CAAC,IAAI,CAAC,cAAc,CAAC,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,CAAC,CAAC;YAE/E,OAAO;gBACL,UAAU,EAAE,MAAM;gBAClB,WAAW;gBACX,MAAM,EAAE,KAAK;gBACb,KAAK,EAAE,QAAQ;gBACf,UAAU,EAAE,IAAI;gBAChB,QAAQ,EAAE,MAAM,CAAC,QAAQ;gBACzB,QAAQ,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;gBAChC,KAAK,EAAE,MAAM,CAAC,KAAK;aACpB,CAAC;QACJ,CAAC;IACH,CAAC;YAAS,CAAC;QACT,4BAA4B;QAC5B,IAAI,CAAC;YACH,MAAM,EAAE,CAAC,EAAE,CAAC,UAAU,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QAC5D,CAAC;QAAC,MAAM,CAAC;YACP,wBAAwB;QAC1B,CAAC;IACH,CAAC;AACH,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,UAAU,CACvB,OAAuB,EACvB,IAAY,EACZ,QAAgB,EAChB,MAAc,EACd,WAAmB,EACnB,IAAc,EACd,WAAqB,EACrB,MAAc,EACd,OAAuB;IAEvB,MAAM,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC,GAAG,IAAI,CAAC;IAE/B,+BAA+B;IAC/B,+BAA+B;IAC/B,2EAA2E;IAC3E,gFAAgF;IAChF,EAAE;IACF,4EAA4E;IAC5E,4EAA4E;IAC5E,2EAA2E;IAC3E,2EAA2E;IAC3E,qEAAqE;IACrE,EAAE;IACF,+EAA+E;IAC/E,oEAAoE;IACpE,EAAE;IACF,wCAAwC;IACxC,6EAA6E;IAC7E,0EAA0E;IAC1E,+EAA+E;IAC/E,uEAAuE;IACvE,2EAA2E;IAC3E,6EAA6E;IAC7E,MAAM,KAAK,GAAG,KAAK,CAAC,GAAG,EAAE,OAAO,EAAE;QAChC,KAAK,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,CAAC;QACjC,QAAQ,EAAE,IAAI;KACf,CAAC,CAAC;IAEH,2DAA2D;IAC3D,2DAA2D;IAC3D,MAAM,aAAa,GAAG,IAAI,OAAO,CAAoD,CAAC,OAAO,EAAE,EAAE;QAC/F,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE;YACxB,OAAO,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,KAAK,EAAE,oBAAoB,GAAG,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;QACxE,CAAC,CAAC,CAAC;QAEH,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,IAAI,EAAE,EAAE;YACzB,OAAO,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,cAAc,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;QAC/E,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,yEAAyE;IACzE,IAAI,gBAAgB,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC;IACzC,IAAI,gBAAgB,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC;IAEzC,2DAA2D;IAC3D,KAAK,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,EAAE,CAAC,IAAY,EAAE,EAAE;QACxC,MAAM,GAAG,GAAG,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;QACnC,iDAAiD;QACjD,gBAAgB,GAAG,gBAAgB,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE;YAClD,IAAI,CAAC;gBACH,MAAM,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,WAAW,EAAE,QAAQ,EAAE,GAAG,CAAC,CAAC;YAChF,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,OAAO,CAAC,IAAI,CAAC,gCAAgC,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YACnG,CAAC;QACH,CAAC,CAAC,CAAC;QACH,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;YACrB,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;QACxB,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,2DAA2D;IAC3D,KAAK,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,EAAE,CAAC,IAAY,EAAE,EAAE;QACxC,MAAM,GAAG,GAAG,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;QACnC,iDAAiD;QACjD,gBAAgB,GAAG,gBAAgB,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE;YAClD,IAAI,CAAC;gBACH,MAAM,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,WAAW,EAAE,QAAQ,EAAE,GAAG,CAAC,CAAC;YAChF,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,OAAO,CAAC,IAAI,CAAC,gCAAgC,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YACnG,CAAC;QACH,CAAC,CAAC,CAAC;QACH,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;YACrB,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;QACxB,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,2EAA2E;IAC3E,yEAAyE;IACzE,8DAA8D;IAC9D,MAAM,gBAAgB,GAAG,GAAG,EAAE;QAC5B,IAAI,KAAK,CAAC,GAAG,EAAE,CAAC;YACd,IAAI,CAAC;gBACH,OAAO,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,SAAS,CAAC,CAAC;YACtC,CAAC;YAAC,MAAM,CAAC;gBACP,kCAAkC;YACpC,CAAC;QACH,CAAC;IACH,CAAC,CAAC;IAEF,iBAAiB;IACjB,IAAI,SAAqC,CAAC;IAC1C,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;QACpB,SAAS,GAAG,UAAU,CAAC,gBAAgB,EAAE,OAAO,CAAC,OAAO,CAAC,CAAC;IAC5D,CAAC;IAED,sBAAsB;IACtB,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;QACnB,IAAI,OAAO,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;YAC3B,oCAAoC;YACpC,gBAAgB,EAAE,CAAC;QACrB,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,MAAM,CAAC,gBAAgB,CAAC,OAAO,EAAE,gBAAgB,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;QAC7E,CAAC;IACH,CAAC;IAED,6CAA6C;IAC7C,MAAM,YAAY,GAAG,MAAM,eAAe,CAAC,KAAK,CAAC,GAAI,CAAC,CAAC;IACvD,MAAM,MAAM,GAAoB,OAAO,CAAC,SAAS,EAAE;QACjD,WAAW;QACX,WAAW;QACX,SAAS,EAAE,IAAI,IAAI,EAAE;QACrB,GAAG,EAAE,MAAM,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC;QAC5B,YAAY,EAAE,MAAM,CAAC,YAAY,IAAI,CAAC,CAAC,CAAC;QACxC,MAAM;KACP,CAAC,CAAC;IACH,MAAM,OAAO,CAAC,IAAI,CAAC,cAAc,CAAC,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,CAAC,CAAC;IAE/E,+BAA+B;IAC/B,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC;IAEnC,8CAA8C;IAC9C,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC,gBAAgB,EAAE,gBAAgB,CAAC,CAAC,CAAC;IAExD,UAAU;IACV,IAAI,SAAS;QAAE,YAAY,CAAC,SAAS,CAAC,CAAC;IACvC,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;QACnB,OAAO,CAAC,MAAM,CAAC,mBAAmB,CAAC,OAAO,EAAE,gBAAgB,CAAC,CAAC;IAChE,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC"}
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Copyright (c) 2025 Elara AI Pty Ltd
|
|
3
|
+
* Licensed under BSL 1.1. See LICENSE for details.
|
|
4
|
+
*/
|
|
5
|
+
import type { StorageBackend } from '../storage/interfaces.js';
|
|
6
|
+
import type { TaskRunner, TaskExecuteOptions, TaskResult } from './interfaces.js';
|
|
7
|
+
/**
|
|
8
|
+
* Record of a single task execution call.
|
|
9
|
+
*/
|
|
10
|
+
export interface MockTaskCall {
|
|
11
|
+
taskHash: string;
|
|
12
|
+
inputHashes: string[];
|
|
13
|
+
options?: TaskExecuteOptions;
|
|
14
|
+
}
|
|
15
|
+
/**
|
|
16
|
+
* TaskRunner mock for testing dataflow orchestration without spawning processes.
|
|
17
|
+
*
|
|
18
|
+
* Allows configuring responses per task and records all calls for assertions.
|
|
19
|
+
*/
|
|
20
|
+
export declare class MockTaskRunner implements TaskRunner {
|
|
21
|
+
private results;
|
|
22
|
+
private calls;
|
|
23
|
+
private defaultResult;
|
|
24
|
+
/**
|
|
25
|
+
* Set result for a specific task hash.
|
|
26
|
+
*
|
|
27
|
+
* @param taskHash - The task hash to configure
|
|
28
|
+
* @param result - Either a static TaskResult or a function that computes result from inputHashes
|
|
29
|
+
*/
|
|
30
|
+
setResult(taskHash: string, result: TaskResult | ((inputHashes: string[]) => TaskResult)): void;
|
|
31
|
+
/**
|
|
32
|
+
* Set default result for tasks without specific results configured.
|
|
33
|
+
*
|
|
34
|
+
* @param result - The default TaskResult to return
|
|
35
|
+
*/
|
|
36
|
+
setDefaultResult(result: TaskResult): void;
|
|
37
|
+
/**
|
|
38
|
+
* Get all recorded calls.
|
|
39
|
+
*
|
|
40
|
+
* @returns Readonly array of all execute() calls
|
|
41
|
+
*/
|
|
42
|
+
getCalls(): readonly MockTaskCall[];
|
|
43
|
+
/**
|
|
44
|
+
* Clear recorded calls.
|
|
45
|
+
*/
|
|
46
|
+
clearCalls(): void;
|
|
47
|
+
execute(_storage: StorageBackend, taskHash: string, inputHashes: string[], options?: TaskExecuteOptions): Promise<TaskResult>;
|
|
48
|
+
}
|
|
49
|
+
//# sourceMappingURL=MockTaskRunner.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"MockTaskRunner.d.ts","sourceRoot":"","sources":["../../../src/execution/MockTaskRunner.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,0BAA0B,CAAC;AAC/D,OAAO,KAAK,EAAE,UAAU,EAAE,kBAAkB,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAElF;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB,OAAO,CAAC,EAAE,kBAAkB,CAAC;CAC9B;AAED;;;;GAIG;AACH,qBAAa,cAAe,YAAW,UAAU;IAC/C,OAAO,CAAC,OAAO,CAA2E;IAC1F,OAAO,CAAC,KAAK,CAAsB;IACnC,OAAO,CAAC,aAAa,CAA4E;IAEjG;;;;;OAKG;IACH,SAAS,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,GAAG,CAAC,CAAC,WAAW,EAAE,MAAM,EAAE,KAAK,UAAU,CAAC,GAAG,IAAI;IAI/F;;;;OAIG;IACH,gBAAgB,CAAC,MAAM,EAAE,UAAU,GAAG,IAAI;IAI1C;;;;OAIG;IACH,QAAQ,IAAI,SAAS,YAAY,EAAE;IAInC;;OAEG;IACH,UAAU,IAAI,IAAI;IAKZ,OAAO,CACX,QAAQ,EAAE,cAAc,EACxB,QAAQ,EAAE,MAAM,EAChB,WAAW,EAAE,MAAM,EAAE,EACrB,OAAO,CAAC,EAAE,kBAAkB,GAC3B,OAAO,CAAC,UAAU,CAAC;CASvB"}
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Copyright (c) 2025 Elara AI Pty Ltd
|
|
3
|
+
* Licensed under BSL 1.1. See LICENSE for details.
|
|
4
|
+
*/
|
|
5
|
+
/**
|
|
6
|
+
* TaskRunner mock for testing dataflow orchestration without spawning processes.
|
|
7
|
+
*
|
|
8
|
+
* Allows configuring responses per task and records all calls for assertions.
|
|
9
|
+
*/
|
|
10
|
+
export class MockTaskRunner {
|
|
11
|
+
results = new Map();
|
|
12
|
+
calls = [];
|
|
13
|
+
defaultResult = { state: 'success', cached: false, outputHash: 'mock-hash' };
|
|
14
|
+
/**
|
|
15
|
+
* Set result for a specific task hash.
|
|
16
|
+
*
|
|
17
|
+
* @param taskHash - The task hash to configure
|
|
18
|
+
* @param result - Either a static TaskResult or a function that computes result from inputHashes
|
|
19
|
+
*/
|
|
20
|
+
setResult(taskHash, result) {
|
|
21
|
+
this.results.set(taskHash, result);
|
|
22
|
+
}
|
|
23
|
+
/**
|
|
24
|
+
* Set default result for tasks without specific results configured.
|
|
25
|
+
*
|
|
26
|
+
* @param result - The default TaskResult to return
|
|
27
|
+
*/
|
|
28
|
+
setDefaultResult(result) {
|
|
29
|
+
this.defaultResult = result;
|
|
30
|
+
}
|
|
31
|
+
/**
|
|
32
|
+
* Get all recorded calls.
|
|
33
|
+
*
|
|
34
|
+
* @returns Readonly array of all execute() calls
|
|
35
|
+
*/
|
|
36
|
+
getCalls() {
|
|
37
|
+
return this.calls;
|
|
38
|
+
}
|
|
39
|
+
/**
|
|
40
|
+
* Clear recorded calls.
|
|
41
|
+
*/
|
|
42
|
+
clearCalls() {
|
|
43
|
+
this.calls = [];
|
|
44
|
+
}
|
|
45
|
+
// eslint-disable-next-line @typescript-eslint/require-await
|
|
46
|
+
async execute(_storage, taskHash, inputHashes, options) {
|
|
47
|
+
this.calls.push({ taskHash, inputHashes, options });
|
|
48
|
+
const configured = this.results.get(taskHash);
|
|
49
|
+
if (configured) {
|
|
50
|
+
return typeof configured === 'function' ? configured(inputHashes) : configured;
|
|
51
|
+
}
|
|
52
|
+
return this.defaultResult;
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
//# sourceMappingURL=MockTaskRunner.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"MockTaskRunner.js","sourceRoot":"","sources":["../../../src/execution/MockTaskRunner.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAcH;;;;GAIG;AACH,MAAM,OAAO,cAAc;IACjB,OAAO,GAAG,IAAI,GAAG,EAAgE,CAAC;IAClF,KAAK,GAAmB,EAAE,CAAC;IAC3B,aAAa,GAAe,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,EAAE,KAAK,EAAE,UAAU,EAAE,WAAW,EAAE,CAAC;IAEjG;;;;;OAKG;IACH,SAAS,CAAC,QAAgB,EAAE,MAA4D;QACtF,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;IACrC,CAAC;IAED;;;;OAIG;IACH,gBAAgB,CAAC,MAAkB;QACjC,IAAI,CAAC,aAAa,GAAG,MAAM,CAAC;IAC9B,CAAC;IAED;;;;OAIG;IACH,QAAQ;QACN,OAAO,IAAI,CAAC,KAAK,CAAC;IACpB,CAAC;IAED;;OAEG;IACH,UAAU;QACR,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC;IAClB,CAAC;IAED,4DAA4D;IAC5D,KAAK,CAAC,OAAO,CACX,QAAwB,EACxB,QAAgB,EAChB,WAAqB,EACrB,OAA4B;QAE5B,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,WAAW,EAAE,OAAO,EAAE,CAAC,CAAC;QAEpD,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAC9C,IAAI,UAAU,EAAE,CAAC;YACf,OAAO,OAAO,UAAU,KAAK,UAAU,CAAC,CAAC,CAAC,UAAU,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC;QACjF,CAAC;QACD,OAAO,IAAI,CAAC,aAAa,CAAC;IAC5B,CAAC;CACF"}
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Copyright (c) 2025 Elara AI Pty Ltd
|
|
3
|
+
* Licensed under BSL 1.1. See LICENSE for details.
|
|
4
|
+
*/
|
|
5
|
+
/**
|
|
6
|
+
* Execution abstraction layer for e3 dataflow.
|
|
7
|
+
*
|
|
8
|
+
* This module provides interfaces that separate orchestration from business
|
|
9
|
+
* logic, enabling different execution strategies:
|
|
10
|
+
* - LocalDataflowExecutor: In-process execution with AsyncMutex (CLI, local dev)
|
|
11
|
+
* - StepFunctionsDataflowExecutor: AWS Step Functions orchestration (cloud)
|
|
12
|
+
*/
|
|
13
|
+
export { type TaskExecuteOptions, type TaskResult, type TaskRunner, type ExecutionHandle, type DataflowStatus, type DataflowExecuteOptions, type DataflowExecuteResult, type DataflowExecutor, type TaskGraph, type DataflowGetGraphFn, type DataflowCheckCacheFn, type DataflowWriteOutputFn, type DataflowGetReadyTasksFn, } from './interfaces.js';
|
|
14
|
+
export { LocalTaskRunner } from './LocalTaskRunner.js';
|
|
15
|
+
export { MockTaskRunner, type MockTaskCall } from './MockTaskRunner.js';
|
|
16
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/execution/index.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH;;;;;;;GAOG;AAEH,OAAO,EAEL,KAAK,kBAAkB,EACvB,KAAK,UAAU,EACf,KAAK,UAAU,EAEf,KAAK,eAAe,EACpB,KAAK,cAAc,EACnB,KAAK,sBAAsB,EAC3B,KAAK,qBAAqB,EAC1B,KAAK,gBAAgB,EAErB,KAAK,SAAS,EAEd,KAAK,kBAAkB,EACvB,KAAK,oBAAoB,EACzB,KAAK,qBAAqB,EAC1B,KAAK,uBAAuB,GAC7B,MAAM,iBAAiB,CAAC;AAGzB,OAAO,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AACvD,OAAO,EAAE,cAAc,EAAE,KAAK,YAAY,EAAE,MAAM,qBAAqB,CAAC"}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Copyright (c) 2025 Elara AI Pty Ltd
|
|
3
|
+
* Licensed under BSL 1.1. See LICENSE for details.
|
|
4
|
+
*/
|
|
5
|
+
// TaskRunner implementations
|
|
6
|
+
export { LocalTaskRunner } from './LocalTaskRunner.js';
|
|
7
|
+
export { MockTaskRunner } from './MockTaskRunner.js';
|
|
8
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/execution/index.ts"],"names":[],"mappings":"AAAA;;;GAGG;AA+BH,6BAA6B;AAC7B,OAAO,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AACvD,OAAO,EAAE,cAAc,EAAqB,MAAM,qBAAqB,CAAC"}
|