@elaraai/e3-core 0.0.2-beta.3 → 0.0.2-beta.31
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +25 -22
- package/dist/src/dataflow/api-compat.d.ts +90 -0
- package/dist/src/dataflow/api-compat.d.ts.map +1 -0
- package/dist/src/dataflow/api-compat.js +134 -0
- package/dist/src/dataflow/api-compat.js.map +1 -0
- package/dist/src/dataflow/index.d.ts +18 -0
- package/dist/src/dataflow/index.d.ts.map +1 -0
- package/dist/src/dataflow/index.js +23 -0
- package/dist/src/dataflow/index.js.map +1 -0
- package/dist/src/dataflow/orchestrator/LocalOrchestrator.d.ts +53 -0
- package/dist/src/dataflow/orchestrator/LocalOrchestrator.d.ts.map +1 -0
- package/dist/src/dataflow/orchestrator/LocalOrchestrator.js +416 -0
- package/dist/src/dataflow/orchestrator/LocalOrchestrator.js.map +1 -0
- package/dist/src/dataflow/orchestrator/index.d.ts +12 -0
- package/dist/src/dataflow/orchestrator/index.d.ts.map +1 -0
- package/dist/src/dataflow/orchestrator/index.js +12 -0
- package/dist/src/dataflow/orchestrator/index.js.map +1 -0
- package/dist/src/dataflow/orchestrator/interfaces.d.ts +157 -0
- package/dist/src/dataflow/orchestrator/interfaces.d.ts.map +1 -0
- package/dist/src/dataflow/orchestrator/interfaces.js +51 -0
- package/dist/src/dataflow/orchestrator/interfaces.js.map +1 -0
- package/dist/src/dataflow/state-store/FileStateStore.d.ts +67 -0
- package/dist/src/dataflow/state-store/FileStateStore.d.ts.map +1 -0
- package/dist/src/dataflow/state-store/FileStateStore.js +286 -0
- package/dist/src/dataflow/state-store/FileStateStore.js.map +1 -0
- package/dist/src/dataflow/state-store/InMemoryStateStore.d.ts +42 -0
- package/dist/src/dataflow/state-store/InMemoryStateStore.d.ts.map +1 -0
- package/dist/src/dataflow/state-store/InMemoryStateStore.js +214 -0
- package/dist/src/dataflow/state-store/InMemoryStateStore.js.map +1 -0
- package/dist/src/dataflow/state-store/index.d.ts +13 -0
- package/dist/src/dataflow/state-store/index.d.ts.map +1 -0
- package/dist/src/dataflow/state-store/index.js +13 -0
- package/dist/src/dataflow/state-store/index.js.map +1 -0
- package/dist/src/dataflow/state-store/interfaces.d.ts +159 -0
- package/dist/src/dataflow/state-store/interfaces.d.ts.map +1 -0
- package/dist/src/dataflow/state-store/interfaces.js +6 -0
- package/dist/src/dataflow/state-store/interfaces.js.map +1 -0
- package/dist/src/dataflow/steps.d.ts +176 -0
- package/dist/src/dataflow/steps.d.ts.map +1 -0
- package/dist/src/dataflow/steps.js +528 -0
- package/dist/src/dataflow/steps.js.map +1 -0
- package/dist/src/dataflow/types.d.ts +116 -0
- package/dist/src/dataflow/types.d.ts.map +1 -0
- package/dist/src/dataflow/types.js +7 -0
- package/dist/src/dataflow/types.js.map +1 -0
- package/dist/src/dataflow.d.ts +142 -9
- package/dist/src/dataflow.d.ts.map +1 -1
- package/dist/src/dataflow.js +427 -64
- package/dist/src/dataflow.js.map +1 -1
- package/dist/src/errors.d.ts +39 -9
- package/dist/src/errors.d.ts.map +1 -1
- package/dist/src/errors.js +51 -8
- package/dist/src/errors.js.map +1 -1
- package/dist/src/execution/LocalTaskRunner.d.ts +73 -0
- package/dist/src/execution/LocalTaskRunner.d.ts.map +1 -0
- package/dist/src/execution/LocalTaskRunner.js +399 -0
- package/dist/src/execution/LocalTaskRunner.js.map +1 -0
- package/dist/src/execution/MockTaskRunner.d.ts +49 -0
- package/dist/src/execution/MockTaskRunner.d.ts.map +1 -0
- package/dist/src/execution/MockTaskRunner.js +55 -0
- package/dist/src/execution/MockTaskRunner.js.map +1 -0
- package/dist/src/execution/index.d.ts +16 -0
- package/dist/src/execution/index.d.ts.map +1 -0
- package/dist/src/execution/index.js +8 -0
- package/dist/src/execution/index.js.map +1 -0
- package/dist/src/execution/interfaces.d.ts +246 -0
- package/dist/src/execution/interfaces.d.ts.map +1 -0
- package/dist/src/execution/interfaces.js +6 -0
- package/dist/src/execution/interfaces.js.map +1 -0
- package/dist/src/execution/processHelpers.d.ts +20 -0
- package/dist/src/execution/processHelpers.d.ts.map +1 -0
- package/dist/src/execution/processHelpers.js +62 -0
- package/dist/src/execution/processHelpers.js.map +1 -0
- package/dist/src/executions.d.ts +71 -104
- package/dist/src/executions.d.ts.map +1 -1
- package/dist/src/executions.js +110 -476
- package/dist/src/executions.js.map +1 -1
- package/dist/src/index.d.ts +17 -9
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +44 -18
- package/dist/src/index.js.map +1 -1
- package/dist/src/objects.d.ts +6 -53
- package/dist/src/objects.d.ts.map +1 -1
- package/dist/src/objects.js +11 -232
- package/dist/src/objects.js.map +1 -1
- package/dist/src/packages.d.ts +22 -14
- package/dist/src/packages.d.ts.map +1 -1
- package/dist/src/packages.js +116 -83
- package/dist/src/packages.js.map +1 -1
- package/dist/src/storage/in-memory/InMemoryRepoStore.d.ts +35 -0
- package/dist/src/storage/in-memory/InMemoryRepoStore.d.ts.map +1 -0
- package/dist/src/storage/in-memory/InMemoryRepoStore.js +107 -0
- package/dist/src/storage/in-memory/InMemoryRepoStore.js.map +1 -0
- package/dist/src/storage/in-memory/InMemoryStorage.d.ts +114 -0
- package/dist/src/storage/in-memory/InMemoryStorage.d.ts.map +1 -0
- package/dist/src/storage/in-memory/InMemoryStorage.js +349 -0
- package/dist/src/storage/in-memory/InMemoryStorage.js.map +1 -0
- package/dist/src/storage/in-memory/index.d.ts +12 -0
- package/dist/src/storage/in-memory/index.d.ts.map +1 -0
- package/dist/src/storage/in-memory/index.js +12 -0
- package/dist/src/storage/in-memory/index.js.map +1 -0
- package/dist/src/storage/index.d.ts +18 -0
- package/dist/src/storage/index.d.ts.map +1 -0
- package/dist/src/storage/index.js +10 -0
- package/dist/src/storage/index.js.map +1 -0
- package/dist/src/storage/interfaces.d.ts +520 -0
- package/dist/src/storage/interfaces.d.ts.map +1 -0
- package/dist/src/storage/interfaces.js +6 -0
- package/dist/src/storage/interfaces.js.map +1 -0
- package/dist/src/storage/local/LocalBackend.d.ts +54 -0
- package/dist/src/storage/local/LocalBackend.d.ts.map +1 -0
- package/dist/src/storage/local/LocalBackend.js +141 -0
- package/dist/src/storage/local/LocalBackend.js.map +1 -0
- package/dist/src/storage/local/LocalLockService.d.ts +105 -0
- package/dist/src/storage/local/LocalLockService.d.ts.map +1 -0
- package/dist/src/storage/local/LocalLockService.js +342 -0
- package/dist/src/storage/local/LocalLockService.js.map +1 -0
- package/dist/src/storage/local/LocalLogStore.d.ts +23 -0
- package/dist/src/storage/local/LocalLogStore.d.ts.map +1 -0
- package/dist/src/storage/local/LocalLogStore.js +66 -0
- package/dist/src/storage/local/LocalLogStore.js.map +1 -0
- package/dist/src/storage/local/LocalObjectStore.d.ts +52 -0
- package/dist/src/storage/local/LocalObjectStore.d.ts.map +1 -0
- package/dist/src/storage/local/LocalObjectStore.js +287 -0
- package/dist/src/storage/local/LocalObjectStore.js.map +1 -0
- package/dist/src/storage/local/LocalRefStore.d.ts +50 -0
- package/dist/src/storage/local/LocalRefStore.d.ts.map +1 -0
- package/dist/src/storage/local/LocalRefStore.js +337 -0
- package/dist/src/storage/local/LocalRefStore.js.map +1 -0
- package/dist/src/storage/local/LocalRepoStore.d.ts +53 -0
- package/dist/src/storage/local/LocalRepoStore.d.ts.map +1 -0
- package/dist/src/storage/local/LocalRepoStore.js +353 -0
- package/dist/src/storage/local/LocalRepoStore.js.map +1 -0
- package/dist/src/storage/local/gc.d.ts +92 -0
- package/dist/src/storage/local/gc.d.ts.map +1 -0
- package/dist/src/storage/local/gc.js +322 -0
- package/dist/src/storage/local/gc.js.map +1 -0
- package/dist/src/storage/local/index.d.ts +17 -0
- package/dist/src/storage/local/index.d.ts.map +1 -0
- package/dist/src/storage/local/index.js +17 -0
- package/dist/src/storage/local/index.js.map +1 -0
- package/dist/src/storage/local/localHelpers.d.ts +25 -0
- package/dist/src/storage/local/localHelpers.d.ts.map +1 -0
- package/dist/src/storage/local/localHelpers.js +69 -0
- package/dist/src/storage/local/localHelpers.js.map +1 -0
- package/dist/src/{repository.d.ts → storage/local/repository.d.ts} +8 -4
- package/dist/src/storage/local/repository.d.ts.map +1 -0
- package/dist/src/{repository.js → storage/local/repository.js} +31 -29
- package/dist/src/storage/local/repository.js.map +1 -0
- package/dist/src/tasks.d.ts +16 -10
- package/dist/src/tasks.d.ts.map +1 -1
- package/dist/src/tasks.js +35 -41
- package/dist/src/tasks.js.map +1 -1
- package/dist/src/test-helpers.d.ts +4 -4
- package/dist/src/test-helpers.d.ts.map +1 -1
- package/dist/src/test-helpers.js +7 -21
- package/dist/src/test-helpers.js.map +1 -1
- package/dist/src/trees.d.ts +89 -27
- package/dist/src/trees.d.ts.map +1 -1
- package/dist/src/trees.js +218 -100
- package/dist/src/trees.js.map +1 -1
- package/dist/src/uuid.d.ts +26 -0
- package/dist/src/uuid.d.ts.map +1 -0
- package/dist/src/uuid.js +80 -0
- package/dist/src/uuid.js.map +1 -0
- package/dist/src/workspaceStatus.d.ts +6 -4
- package/dist/src/workspaceStatus.d.ts.map +1 -1
- package/dist/src/workspaceStatus.js +43 -49
- package/dist/src/workspaceStatus.js.map +1 -1
- package/dist/src/workspaces.d.ts +35 -26
- package/dist/src/workspaces.d.ts.map +1 -1
- package/dist/src/workspaces.js +169 -118
- package/dist/src/workspaces.js.map +1 -1
- package/package.json +4 -4
- package/dist/src/gc.d.ts +0 -54
- package/dist/src/gc.d.ts.map +0 -1
- package/dist/src/gc.js +0 -233
- package/dist/src/gc.js.map +0 -1
- package/dist/src/repository.d.ts.map +0 -1
- package/dist/src/repository.js.map +0 -1
- package/dist/src/workspaceLock.d.ts +0 -67
- package/dist/src/workspaceLock.d.ts.map +0 -1
- package/dist/src/workspaceLock.js +0 -217
- package/dist/src/workspaceLock.js.map +0 -1
|
@@ -0,0 +1,214 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Copyright (c) 2025 Elara AI Pty Ltd
|
|
3
|
+
* Licensed under BSL 1.1. See LICENSE for details.
|
|
4
|
+
*/
|
|
5
|
+
/**
|
|
6
|
+
* In-memory implementation of ExecutionStateStore.
|
|
7
|
+
*
|
|
8
|
+
* Useful for testing and simple cases where persistence is not required.
|
|
9
|
+
* State is lost when the process exits.
|
|
10
|
+
*/
|
|
11
|
+
import { some } from '@elaraai/east';
|
|
12
|
+
/**
|
|
13
|
+
* In-memory state store for testing and simple use cases.
|
|
14
|
+
*
|
|
15
|
+
* @remarks
|
|
16
|
+
* - Thread-safe for concurrent access within a single process
|
|
17
|
+
* - State is lost on process exit
|
|
18
|
+
* - No durability guarantees
|
|
19
|
+
*/
|
|
20
|
+
export class InMemoryStateStore {
|
|
21
|
+
/** Map of "repo::workspace" -> execution ID -> state */
|
|
22
|
+
states = new Map();
|
|
23
|
+
/** Map of "repo::workspace" -> next execution ID counter */
|
|
24
|
+
counters = new Map();
|
|
25
|
+
makeKey(repo, workspace) {
|
|
26
|
+
return `${repo}::${workspace}`;
|
|
27
|
+
}
|
|
28
|
+
// eslint-disable-next-line @typescript-eslint/require-await
|
|
29
|
+
async create(state) {
|
|
30
|
+
const key = this.makeKey(state.repo, state.workspace);
|
|
31
|
+
if (!this.states.has(key)) {
|
|
32
|
+
this.states.set(key, new Map());
|
|
33
|
+
}
|
|
34
|
+
const wsStates = this.states.get(key);
|
|
35
|
+
if (wsStates.has(state.id)) {
|
|
36
|
+
throw new Error(`Execution ${state.id} already exists in ${key}`);
|
|
37
|
+
}
|
|
38
|
+
// Deep clone to prevent external mutation
|
|
39
|
+
wsStates.set(state.id, this.cloneState(state));
|
|
40
|
+
}
|
|
41
|
+
// eslint-disable-next-line @typescript-eslint/require-await
|
|
42
|
+
async read(repo, workspace, id) {
|
|
43
|
+
const key = this.makeKey(repo, workspace);
|
|
44
|
+
const wsStates = this.states.get(key);
|
|
45
|
+
if (!wsStates)
|
|
46
|
+
return null;
|
|
47
|
+
const state = wsStates.get(id);
|
|
48
|
+
if (!state)
|
|
49
|
+
return null;
|
|
50
|
+
return this.cloneState(state);
|
|
51
|
+
}
|
|
52
|
+
// eslint-disable-next-line @typescript-eslint/require-await
|
|
53
|
+
async readLatest(repo, workspace) {
|
|
54
|
+
const key = this.makeKey(repo, workspace);
|
|
55
|
+
const wsStates = this.states.get(key);
|
|
56
|
+
if (!wsStates || wsStates.size === 0)
|
|
57
|
+
return null;
|
|
58
|
+
// Find the highest execution ID (assuming numeric string IDs)
|
|
59
|
+
let latestId = null;
|
|
60
|
+
let latestNum = -1;
|
|
61
|
+
for (const id of wsStates.keys()) {
|
|
62
|
+
const num = parseInt(id, 10);
|
|
63
|
+
if (!isNaN(num) && num > latestNum) {
|
|
64
|
+
latestNum = num;
|
|
65
|
+
latestId = id;
|
|
66
|
+
}
|
|
67
|
+
else if (latestId === null) {
|
|
68
|
+
// Fallback for non-numeric IDs: just take the first one
|
|
69
|
+
latestId = id;
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
if (latestId === null)
|
|
73
|
+
return null;
|
|
74
|
+
return this.cloneState(wsStates.get(latestId));
|
|
75
|
+
}
|
|
76
|
+
// eslint-disable-next-line @typescript-eslint/require-await
|
|
77
|
+
async update(state) {
|
|
78
|
+
const key = this.makeKey(state.repo, state.workspace);
|
|
79
|
+
const wsStates = this.states.get(key);
|
|
80
|
+
if (!wsStates || !wsStates.has(state.id)) {
|
|
81
|
+
throw new Error(`Execution ${state.id} not found in ${key}`);
|
|
82
|
+
}
|
|
83
|
+
wsStates.set(state.id, this.cloneState(state));
|
|
84
|
+
}
|
|
85
|
+
async updateTaskStatus(repo, workspace, executionId, task, status, details) {
|
|
86
|
+
const state = await this.read(repo, workspace, executionId);
|
|
87
|
+
if (!state) {
|
|
88
|
+
throw new Error(`Execution ${executionId} not found in ${repo}::${workspace}`);
|
|
89
|
+
}
|
|
90
|
+
const taskState = state.tasks.get(task);
|
|
91
|
+
if (!taskState) {
|
|
92
|
+
throw new Error(`Task '${task}' not found in execution ${executionId}`);
|
|
93
|
+
}
|
|
94
|
+
taskState.status = status;
|
|
95
|
+
if (details) {
|
|
96
|
+
if (details.cached !== undefined)
|
|
97
|
+
taskState.cached = some(details.cached);
|
|
98
|
+
if (details.outputHash !== undefined)
|
|
99
|
+
taskState.outputHash = some(details.outputHash);
|
|
100
|
+
if (details.error !== undefined)
|
|
101
|
+
taskState.error = some(details.error);
|
|
102
|
+
if (details.exitCode !== undefined)
|
|
103
|
+
taskState.exitCode = some(BigInt(details.exitCode));
|
|
104
|
+
if (details.duration !== undefined)
|
|
105
|
+
taskState.duration = some(BigInt(details.duration));
|
|
106
|
+
}
|
|
107
|
+
taskState.completedAt = some(new Date());
|
|
108
|
+
await this.update(state);
|
|
109
|
+
}
|
|
110
|
+
async updateStatus(repo, workspace, executionId, status, details) {
|
|
111
|
+
const state = await this.read(repo, workspace, executionId);
|
|
112
|
+
if (!state) {
|
|
113
|
+
throw new Error(`Execution ${executionId} not found in ${repo}::${workspace}`);
|
|
114
|
+
}
|
|
115
|
+
const mutableState = state;
|
|
116
|
+
mutableState.status = status;
|
|
117
|
+
if (status !== 'running') {
|
|
118
|
+
mutableState.completedAt = some(new Date());
|
|
119
|
+
}
|
|
120
|
+
if (details?.error) {
|
|
121
|
+
mutableState.error = some(details.error);
|
|
122
|
+
}
|
|
123
|
+
if (details?.summary) {
|
|
124
|
+
mutableState.executed = BigInt(details.summary.executed);
|
|
125
|
+
mutableState.cached = BigInt(details.summary.cached);
|
|
126
|
+
mutableState.failed = BigInt(details.summary.failed);
|
|
127
|
+
mutableState.skipped = BigInt(details.summary.skipped);
|
|
128
|
+
}
|
|
129
|
+
await this.update(state);
|
|
130
|
+
}
|
|
131
|
+
async recordEvent(repo, workspace, executionId, event) {
|
|
132
|
+
const state = await this.read(repo, workspace, executionId);
|
|
133
|
+
if (!state) {
|
|
134
|
+
throw new Error(`Execution ${executionId} not found in ${repo}::${workspace}`);
|
|
135
|
+
}
|
|
136
|
+
// Append event to inline events array (cast to mutable array)
|
|
137
|
+
state.events.push(event);
|
|
138
|
+
await this.update(state);
|
|
139
|
+
}
|
|
140
|
+
// eslint-disable-next-line @typescript-eslint/require-await
|
|
141
|
+
async getEventsSince(repo, workspace, executionId, sinceSeq) {
|
|
142
|
+
const key = this.makeKey(repo, workspace);
|
|
143
|
+
const wsStates = this.states.get(key);
|
|
144
|
+
if (!wsStates)
|
|
145
|
+
return [];
|
|
146
|
+
const state = wsStates.get(executionId);
|
|
147
|
+
if (!state)
|
|
148
|
+
return [];
|
|
149
|
+
// Filter events from inline array
|
|
150
|
+
const sinceSeqBigInt = BigInt(sinceSeq);
|
|
151
|
+
return state.events.filter(e => e.value.seq > sinceSeqBigInt);
|
|
152
|
+
}
|
|
153
|
+
// eslint-disable-next-line @typescript-eslint/require-await
|
|
154
|
+
async nextExecutionId(repo, workspace) {
|
|
155
|
+
const key = this.makeKey(repo, workspace);
|
|
156
|
+
const current = this.counters.get(key) ?? 0;
|
|
157
|
+
const next = current + 1;
|
|
158
|
+
this.counters.set(key, next);
|
|
159
|
+
return String(next);
|
|
160
|
+
}
|
|
161
|
+
// eslint-disable-next-line @typescript-eslint/require-await
|
|
162
|
+
async delete(repo, workspace, executionId) {
|
|
163
|
+
const key = this.makeKey(repo, workspace);
|
|
164
|
+
const wsStates = this.states.get(key);
|
|
165
|
+
if (wsStates) {
|
|
166
|
+
wsStates.delete(executionId);
|
|
167
|
+
}
|
|
168
|
+
}
|
|
169
|
+
/**
|
|
170
|
+
* Clear all state (for testing).
|
|
171
|
+
*/
|
|
172
|
+
clear() {
|
|
173
|
+
this.states.clear();
|
|
174
|
+
this.counters.clear();
|
|
175
|
+
}
|
|
176
|
+
/**
|
|
177
|
+
* Deep clone execution state to prevent external mutation.
|
|
178
|
+
*
|
|
179
|
+
* Note: We use spread and some() to properly clone the branded option types.
|
|
180
|
+
*/
|
|
181
|
+
cloneState(state) {
|
|
182
|
+
const tasks = new Map();
|
|
183
|
+
for (const [name, taskState] of state.tasks) {
|
|
184
|
+
// Shallow clone is sufficient since we use some() for options
|
|
185
|
+
tasks.set(name, { ...taskState });
|
|
186
|
+
}
|
|
187
|
+
// Clone graph if present
|
|
188
|
+
let graph = state.graph;
|
|
189
|
+
if (state.graph.type === 'some') {
|
|
190
|
+
const graphValue = {
|
|
191
|
+
tasks: state.graph.value.tasks.map(t => ({
|
|
192
|
+
...t,
|
|
193
|
+
inputs: [...t.inputs],
|
|
194
|
+
dependsOn: [...t.dependsOn],
|
|
195
|
+
})),
|
|
196
|
+
};
|
|
197
|
+
graph = some(graphValue);
|
|
198
|
+
}
|
|
199
|
+
// Clone completedAt if present
|
|
200
|
+
let completedAt = state.completedAt;
|
|
201
|
+
if (state.completedAt.type === 'some') {
|
|
202
|
+
completedAt = some(new Date(state.completedAt.value.getTime()));
|
|
203
|
+
}
|
|
204
|
+
return {
|
|
205
|
+
...state,
|
|
206
|
+
startedAt: new Date(state.startedAt.getTime()),
|
|
207
|
+
completedAt,
|
|
208
|
+
graph,
|
|
209
|
+
tasks,
|
|
210
|
+
events: [...state.events],
|
|
211
|
+
};
|
|
212
|
+
}
|
|
213
|
+
}
|
|
214
|
+
//# sourceMappingURL=InMemoryStateStore.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"InMemoryStateStore.js","sourceRoot":"","sources":["../../../../src/dataflow/state-store/InMemoryStateStore.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH;;;;;GAKG;AAEH,OAAO,EAAE,IAAI,EAAE,MAAM,eAAe,CAAC;AAiBrC;;;;;;;GAOG;AACH,MAAM,OAAO,kBAAkB;IAC7B,wDAAwD;IAChD,MAAM,GAAG,IAAI,GAAG,EAA+C,CAAC;IAExE,4DAA4D;IACpD,QAAQ,GAAG,IAAI,GAAG,EAAkB,CAAC;IAErC,OAAO,CAAC,IAAY,EAAE,SAAiB;QAC7C,OAAO,GAAG,IAAI,KAAK,SAAS,EAAE,CAAC;IACjC,CAAC;IAED,4DAA4D;IAC5D,KAAK,CAAC,MAAM,CAAC,KAA6B;QACxC,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC;QACtD,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;YAC1B,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,GAAG,EAAE,CAAC,CAAC;QAClC,CAAC;QAED,MAAM,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAE,CAAC;QACvC,IAAI,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,EAAE,CAAC;YAC3B,MAAM,IAAI,KAAK,CAAC,aAAa,KAAK,CAAC,EAAE,sBAAsB,GAAG,EAAE,CAAC,CAAC;QACpE,CAAC;QAED,0CAA0C;QAC1C,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,EAAE,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC;IACjD,CAAC;IAED,4DAA4D;IAC5D,KAAK,CAAC,IAAI,CAAC,IAAY,EAAE,SAAiB,EAAE,EAAU;QACpD,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;QAC1C,MAAM,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACtC,IAAI,CAAC,QAAQ;YAAE,OAAO,IAAI,CAAC;QAE3B,MAAM,KAAK,GAAG,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAC/B,IAAI,CAAC,KAAK;YAAE,OAAO,IAAI,CAAC;QAExB,OAAO,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;IAChC,CAAC;IAED,4DAA4D;IAC5D,KAAK,CAAC,UAAU,CAAC,IAAY,EAAE,SAAiB;QAC9C,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;QAC1C,MAAM,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACtC,IAAI,CAAC,QAAQ,IAAI,QAAQ,CAAC,IAAI,KAAK,CAAC;YAAE,OAAO,IAAI,CAAC;QAElD,8DAA8D;QAC9D,IAAI,QAAQ,GAAkB,IAAI,CAAC;QACnC,IAAI,SAAS,GAAG,CAAC,CAAC,CAAC;QACnB,KAAK,MAAM,EAAE,IAAI,QAAQ,CAAC,IAAI,EAAE,EAAE,CAAC;YACjC,MAAM,GAAG,GAAG,QAAQ,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC;YAC7B,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,GAAG,GAAG,SAAS,EAAE,CAAC;gBACnC,SAAS,GAAG,GAAG,CAAC;gBAChB,QAAQ,GAAG,EAAE,CAAC;YAChB,CAAC;iBAAM,IAAI,QAAQ,KAAK,IAAI,EAAE,CAAC;gBAC7B,wDAAwD;gBACxD,QAAQ,GAAG,EAAE,CAAC;YAChB,CAAC;QACH,CAAC;QAED,IAAI,QAAQ,KAAK,IAAI;YAAE,OAAO,IAAI,CAAC;QACnC,OAAO,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAE,CAAC,CAAC;IAClD,CAAC;IAED,4DAA4D;IAC5D,KAAK,CAAC,MAAM,CAAC,KAA6B;QACxC,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC;QACtD,MAAM,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACtC,IAAI,CAAC,QAAQ,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,EAAE,CAAC;YACzC,MAAM,IAAI,KAAK,CAAC,aAAa,KAAK,CAAC,EAAE,iBAAiB,GAAG,EAAE,CAAC,CAAC;QAC/D,CAAC;QAED,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,EAAE,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC;IACjD,CAAC;IAED,KAAK,CAAC,gBAAgB,CACpB,IAAY,EACZ,SAAiB,EACjB,WAAmB,EACnB,IAAY,EACZ,MAAkB,EAClB,OAA2B;QAE3B,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,SAAS,EAAE,WAAW,CAAC,CAAC;QAC5D,IAAI,CAAC,KAAK,EAAE,CAAC;YACX,MAAM,IAAI,KAAK,CAAC,aAAa,WAAW,iBAAiB,IAAI,KAAK,SAAS,EAAE,CAAC,CAAC;QACjF,CAAC;QAED,MAAM,SAAS,GAAG,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAmC,CAAC;QAC1E,IAAI,CAAC,SAAS,EAAE,CAAC;YACf,MAAM,IAAI,KAAK,CAAC,SAAS,IAAI,4BAA4B,WAAW,EAAE,CAAC,CAAC;QAC1E,CAAC;QAED,SAAS,CAAC,MAAM,GAAG,MAAM,CAAC;QAC1B,IAAI,OAAO,EAAE,CAAC;YACZ,IAAI,OAAO,CAAC,MAAM,KAAK,SAAS;gBAAE,SAAS,CAAC,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;YAC1E,IAAI,OAAO,CAAC,UAAU,KAAK,SAAS;gBAAE,SAAS,CAAC,UAAU,GAAG,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;YACtF,IAAI,OAAO,CAAC,KAAK,KAAK,SAAS;gBAAE,SAAS,CAAC,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;YACvE,IAAI,OAAO,CAAC,QAAQ,KAAK,SAAS;gBAAE,SAAS,CAAC,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC;YACxF,IAAI,OAAO,CAAC,QAAQ,KAAK,SAAS;gBAAE,SAAS,CAAC,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC;QAC1F,CAAC;QACD,SAAS,CAAC,WAAW,GAAG,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC;QAEzC,MAAM,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAC3B,CAAC;IAED,KAAK,CAAC,YAAY,CAChB,IAAY,EACZ,SAAiB,EACjB,WAAmB,EACnB,MAAwD,EACxD,OAAgC;QAEhC,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,SAAS,EAAE,WAAW,CAAC,CAAC;QAC5D,IAAI,CAAC,KAAK,EAAE,CAAC;YACX,MAAM,IAAI,KAAK,CAAC,aAAa,WAAW,iBAAiB,IAAI,KAAK,SAAS,EAAE,CAAC,CAAC;QACjF,CAAC;QAED,MAAM,YAAY,GAAG,KAAwC,CAAC;QAE9D,YAAY,CAAC,MAAM,GAAG,MAAM,CAAC;QAC7B,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;YACzB,YAAY,CAAC,WAAW,GAAG,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC;QAC9C,CAAC;QACD,IAAI,OAAO,EAAE,KAAK,EAAE,CAAC;YACnB,YAAY,CAAC,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;QAC3C,CAAC;QACD,IAAI,OAAO,EAAE,OAAO,EAAE,CAAC;YACrB,YAAY,CAAC,QAAQ,GAAG,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;YACzD,YAAY,CAAC,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;YACrD,YAAY,CAAC,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;YACrD,YAAY,CAAC,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QACzD,CAAC;QAED,MAAM,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAC3B,CAAC;IAED,KAAK,CAAC,WAAW,CACf,IAAY,EACZ,SAAiB,EACjB,WAAmB,EACnB,KAAqB;QAErB,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,SAAS,EAAE,WAAW,CAAC,CAAC;QAC5D,IAAI,CAAC,KAAK,EAAE,CAAC;YACX,MAAM,IAAI,KAAK,CAAC,aAAa,WAAW,iBAAiB,IAAI,KAAK,SAAS,EAAE,CAAC,CAAC;QACjF,CAAC;QAED,8DAA8D;QAC7D,KAAK,CAAC,MAA2B,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAE/C,MAAM,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAC3B,CAAC;IAED,4DAA4D;IAC5D,KAAK,CAAC,cAAc,CAClB,IAAY,EACZ,SAAiB,EACjB,WAAmB,EACnB,QAAgB;QAEhB,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;QAC1C,MAAM,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACtC,IAAI,CAAC,QAAQ;YAAE,OAAO,EAAE,CAAC;QAEzB,MAAM,KAAK,GAAG,QAAQ,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;QACxC,IAAI,CAAC,KAAK;YAAE,OAAO,EAAE,CAAC;QAEtB,kCAAkC;QAClC,MAAM,cAAc,GAAG,MAAM,CAAC,QAAQ,CAAC,CAAC;QACxC,OAAO,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,GAAG,cAAc,CAAC,CAAC;IAChE,CAAC;IAED,4DAA4D;IAC5D,KAAK,CAAC,eAAe,CAAC,IAAY,EAAE,SAAiB;QACnD,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;QAC1C,MAAM,OAAO,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAC5C,MAAM,IAAI,GAAG,OAAO,GAAG,CAAC,CAAC;QACzB,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;QAC7B,OAAO,MAAM,CAAC,IAAI,CAAC,CAAC;IACtB,CAAC;IAED,4DAA4D;IAC5D,KAAK,CAAC,MAAM,CAAC,IAAY,EAAE,SAAiB,EAAE,WAAmB;QAC/D,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;QAC1C,MAAM,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACtC,IAAI,QAAQ,EAAE,CAAC;YACb,QAAQ,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC;QAC/B,CAAC;IACH,CAAC;IAED;;OAEG;IACH,KAAK;QACH,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;QACpB,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAC;IACxB,CAAC;IAED;;;;OAIG;IACK,UAAU,CAAC,KAA6B;QAC9C,MAAM,KAAK,GAAG,IAAI,GAAG,EAAqB,CAAC;QAC3C,KAAK,MAAM,CAAC,IAAI,EAAE,SAAS,CAAC,IAAI,KAAK,CAAC,KAAK,EAAE,CAAC;YAC5C,8DAA8D;YAC9D,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,EAAE,GAAG,SAAS,EAAe,CAAC,CAAC;QACjD,CAAC;QAED,yBAAyB;QACzB,IAAI,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC;QACxB,IAAI,KAAK,CAAC,KAAK,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;YAChC,MAAM,UAAU,GAAkB;gBAChC,KAAK,EAAE,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;oBACvC,GAAG,CAAC;oBACJ,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC;oBACrB,SAAS,EAAE,CAAC,GAAG,CAAC,CAAC,SAAS,CAAC;iBAC5B,CAAC,CAAC;aACJ,CAAC;YACF,KAAK,GAAG,IAAI,CAAC,UAAU,CAAC,CAAC;QAC3B,CAAC;QAED,+BAA+B;QAC/B,IAAI,WAAW,GAAG,KAAK,CAAC,WAAW,CAAC;QACpC,IAAI,KAAK,CAAC,WAAW,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;YACtC,WAAW,GAAG,IAAI,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;QAClE,CAAC;QAED,OAAO;YACL,GAAG,KAAK;YACR,SAAS,EAAE,IAAI,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,OAAO,EAAE,CAAC;YAC9C,WAAW;YACX,KAAK;YACL,KAAK;YACL,MAAM,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC;SACA,CAAC;IAC9B,CAAC;CACF"}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Copyright (c) 2025 Elara AI Pty Ltd
|
|
3
|
+
* Licensed under BSL 1.1. See LICENSE for details.
|
|
4
|
+
*/
|
|
5
|
+
/**
|
|
6
|
+
* State store implementations for dataflow execution.
|
|
7
|
+
*
|
|
8
|
+
* @packageDocumentation
|
|
9
|
+
*/
|
|
10
|
+
export * from './interfaces.js';
|
|
11
|
+
export * from './InMemoryStateStore.js';
|
|
12
|
+
export * from './FileStateStore.js';
|
|
13
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/dataflow/state-store/index.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH;;;;GAIG;AAEH,cAAc,iBAAiB,CAAC;AAChC,cAAc,yBAAyB,CAAC;AACxC,cAAc,qBAAqB,CAAC"}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Copyright (c) 2025 Elara AI Pty Ltd
|
|
3
|
+
* Licensed under BSL 1.1. See LICENSE for details.
|
|
4
|
+
*/
|
|
5
|
+
/**
|
|
6
|
+
* State store implementations for dataflow execution.
|
|
7
|
+
*
|
|
8
|
+
* @packageDocumentation
|
|
9
|
+
*/
|
|
10
|
+
export * from './interfaces.js';
|
|
11
|
+
export * from './InMemoryStateStore.js';
|
|
12
|
+
export * from './FileStateStore.js';
|
|
13
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../../src/dataflow/state-store/index.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH;;;;GAIG;AAEH,cAAc,iBAAiB,CAAC;AAChC,cAAc,yBAAyB,CAAC;AACxC,cAAc,qBAAqB,CAAC"}
|
|
@@ -0,0 +1,159 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Copyright (c) 2025 Elara AI Pty Ltd
|
|
3
|
+
* Licensed under BSL 1.1. See LICENSE for details.
|
|
4
|
+
*/
|
|
5
|
+
/**
|
|
6
|
+
* State store interface for dataflow execution.
|
|
7
|
+
*
|
|
8
|
+
* Abstracts the persistence of execution state, enabling:
|
|
9
|
+
* - InMemoryStateStore: For testing and simple cases
|
|
10
|
+
* - FileStateStore: Local filesystem persistence (workspace/execution.beast2)
|
|
11
|
+
* - DynamoDBStateStore: Cloud execution (in e3-aws)
|
|
12
|
+
*/
|
|
13
|
+
import type { DataflowExecutionState, ExecutionEvent, TaskStatus } from '../types.js';
|
|
14
|
+
/**
|
|
15
|
+
* Details for task status updates.
|
|
16
|
+
*/
|
|
17
|
+
export interface TaskStatusDetails {
|
|
18
|
+
/** Whether the result was from cache */
|
|
19
|
+
cached?: boolean;
|
|
20
|
+
/** Output hash (for completed tasks) */
|
|
21
|
+
outputHash?: string;
|
|
22
|
+
/** Error message (for failed tasks) */
|
|
23
|
+
error?: string;
|
|
24
|
+
/** Exit code (for failed tasks) */
|
|
25
|
+
exitCode?: number;
|
|
26
|
+
/** Duration in milliseconds */
|
|
27
|
+
duration?: number;
|
|
28
|
+
}
|
|
29
|
+
/**
|
|
30
|
+
* Details for execution status updates.
|
|
31
|
+
*/
|
|
32
|
+
export interface ExecutionStatusDetails {
|
|
33
|
+
/** Error message (for failed executions) */
|
|
34
|
+
error?: string;
|
|
35
|
+
/** Summary counts */
|
|
36
|
+
summary?: {
|
|
37
|
+
executed: number;
|
|
38
|
+
cached: number;
|
|
39
|
+
failed: number;
|
|
40
|
+
skipped: number;
|
|
41
|
+
};
|
|
42
|
+
}
|
|
43
|
+
/**
|
|
44
|
+
* Interface for persisting and retrieving execution state.
|
|
45
|
+
*
|
|
46
|
+
* Implementations must be thread-safe for concurrent access within a process.
|
|
47
|
+
* For distributed execution (cloud), implementations should use optimistic
|
|
48
|
+
* concurrency control (e.g., DynamoDB conditional writes).
|
|
49
|
+
*
|
|
50
|
+
* @remarks
|
|
51
|
+
* All methods take repo and workspace parameters because cloud storage
|
|
52
|
+
* (DynamoDB) needs both to identify an execution uniquely across repositories.
|
|
53
|
+
*/
|
|
54
|
+
export interface ExecutionStateStore {
|
|
55
|
+
/**
|
|
56
|
+
* Create a new execution state.
|
|
57
|
+
*
|
|
58
|
+
* @param state - The initial execution state (contains repo and workspace)
|
|
59
|
+
* @throws If an execution with the same ID already exists
|
|
60
|
+
*/
|
|
61
|
+
create(state: DataflowExecutionState): Promise<void>;
|
|
62
|
+
/**
|
|
63
|
+
* Read an execution state by ID.
|
|
64
|
+
*
|
|
65
|
+
* @param repo - Repository identifier
|
|
66
|
+
* @param workspace - Workspace name
|
|
67
|
+
* @param id - Execution ID
|
|
68
|
+
* @returns The execution state, or null if not found
|
|
69
|
+
*/
|
|
70
|
+
read(repo: string, workspace: string, id: string): Promise<DataflowExecutionState | null>;
|
|
71
|
+
/**
|
|
72
|
+
* Read the most recent execution for a workspace.
|
|
73
|
+
*
|
|
74
|
+
* @param repo - Repository identifier
|
|
75
|
+
* @param workspace - Workspace name
|
|
76
|
+
* @returns The most recent execution state, or null if none exists
|
|
77
|
+
*/
|
|
78
|
+
readLatest(repo: string, workspace: string): Promise<DataflowExecutionState | null>;
|
|
79
|
+
/**
|
|
80
|
+
* Update the entire execution state.
|
|
81
|
+
*
|
|
82
|
+
* This is used for bulk updates after a sequence of step functions.
|
|
83
|
+
* Implementations may optimize by only writing changed fields.
|
|
84
|
+
*
|
|
85
|
+
* @param state - The updated execution state (contains repo and workspace)
|
|
86
|
+
*/
|
|
87
|
+
update(state: DataflowExecutionState): Promise<void>;
|
|
88
|
+
/**
|
|
89
|
+
* Update a task's status within an execution.
|
|
90
|
+
*
|
|
91
|
+
* This is a convenience method for updating a single task without
|
|
92
|
+
* reading and writing the entire state.
|
|
93
|
+
*
|
|
94
|
+
* @param repo - Repository identifier
|
|
95
|
+
* @param workspace - Workspace name
|
|
96
|
+
* @param executionId - Execution ID
|
|
97
|
+
* @param task - Task name
|
|
98
|
+
* @param status - New status
|
|
99
|
+
* @param details - Additional details (output hash, error, etc.)
|
|
100
|
+
*/
|
|
101
|
+
updateTaskStatus(repo: string, workspace: string, executionId: string, task: string, status: TaskStatus, details?: TaskStatusDetails): Promise<void>;
|
|
102
|
+
/**
|
|
103
|
+
* Update the execution's overall status.
|
|
104
|
+
*
|
|
105
|
+
* @param repo - Repository identifier
|
|
106
|
+
* @param workspace - Workspace name
|
|
107
|
+
* @param executionId - Execution ID
|
|
108
|
+
* @param status - New status ('running' | 'completed' | 'failed' | 'cancelled')
|
|
109
|
+
* @param details - Additional details (error message, summary)
|
|
110
|
+
*/
|
|
111
|
+
updateStatus(repo: string, workspace: string, executionId: string, status: 'running' | 'completed' | 'failed' | 'cancelled', details?: ExecutionStatusDetails): Promise<void>;
|
|
112
|
+
/**
|
|
113
|
+
* Record an event for an execution.
|
|
114
|
+
*
|
|
115
|
+
* Events are used for monitoring and debugging. They are append-only
|
|
116
|
+
* and can be read with getEventsSince().
|
|
117
|
+
*
|
|
118
|
+
* @param repo - Repository identifier
|
|
119
|
+
* @param workspace - Workspace name
|
|
120
|
+
* @param executionId - Execution ID
|
|
121
|
+
* @param event - The event to record
|
|
122
|
+
*/
|
|
123
|
+
recordEvent(repo: string, workspace: string, executionId: string, event: ExecutionEvent): Promise<void>;
|
|
124
|
+
/**
|
|
125
|
+
* Get events for an execution since a given sequence number.
|
|
126
|
+
*
|
|
127
|
+
* Used for polling/watching execution progress.
|
|
128
|
+
*
|
|
129
|
+
* @param repo - Repository identifier
|
|
130
|
+
* @param workspace - Workspace name
|
|
131
|
+
* @param executionId - Execution ID
|
|
132
|
+
* @param sinceSeq - Only return events with seq > sinceSeq
|
|
133
|
+
* @returns Array of events in sequence order
|
|
134
|
+
*/
|
|
135
|
+
getEventsSince(repo: string, workspace: string, executionId: string, sinceSeq: number): Promise<ExecutionEvent[]>;
|
|
136
|
+
/**
|
|
137
|
+
* Get the next execution ID for a workspace.
|
|
138
|
+
*
|
|
139
|
+
* For local storage, returns auto-incrementing string IDs ("1", "2", ...).
|
|
140
|
+
* For cloud storage, may return UUIDs or other unique identifiers.
|
|
141
|
+
*
|
|
142
|
+
* @param repo - Repository identifier
|
|
143
|
+
* @param workspace - Workspace name
|
|
144
|
+
* @returns The next execution ID as a string
|
|
145
|
+
*/
|
|
146
|
+
nextExecutionId(repo: string, workspace: string): Promise<string>;
|
|
147
|
+
/**
|
|
148
|
+
* Delete an execution state.
|
|
149
|
+
*
|
|
150
|
+
* Used for cleanup after execution completion or for removing
|
|
151
|
+
* abandoned executions.
|
|
152
|
+
*
|
|
153
|
+
* @param repo - Repository identifier
|
|
154
|
+
* @param workspace - Workspace name
|
|
155
|
+
* @param executionId - Execution ID
|
|
156
|
+
*/
|
|
157
|
+
delete(repo: string, workspace: string, executionId: string): Promise<void>;
|
|
158
|
+
}
|
|
159
|
+
//# sourceMappingURL=interfaces.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"interfaces.d.ts","sourceRoot":"","sources":["../../../../src/dataflow/state-store/interfaces.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH;;;;;;;GAOG;AAEH,OAAO,KAAK,EACV,sBAAsB,EACtB,cAAc,EACd,UAAU,EACX,MAAM,aAAa,CAAC;AAErB;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,wCAAwC;IACxC,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,wCAAwC;IACxC,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,uCAAuC;IACvC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,mCAAmC;IACnC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,+BAA+B;IAC/B,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED;;GAEG;AACH,MAAM,WAAW,sBAAsB;IACrC,4CAA4C;IAC5C,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,qBAAqB;IACrB,OAAO,CAAC,EAAE;QACR,QAAQ,EAAE,MAAM,CAAC;QACjB,MAAM,EAAE,MAAM,CAAC;QACf,MAAM,EAAE,MAAM,CAAC;QACf,OAAO,EAAE,MAAM,CAAC;KACjB,CAAC;CACH;AAED;;;;;;;;;;GAUG;AACH,MAAM,WAAW,mBAAmB;IAClC;;;;;OAKG;IACH,MAAM,CAAC,KAAK,EAAE,sBAAsB,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAErD;;;;;;;OAOG;IACH,IAAI,CAAC,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,sBAAsB,GAAG,IAAI,CAAC,CAAC;IAE1F;;;;;;OAMG;IACH,UAAU,CAAC,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,sBAAsB,GAAG,IAAI,CAAC,CAAC;IAEpF;;;;;;;OAOG;IACH,MAAM,CAAC,KAAK,EAAE,sBAAsB,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAErD;;;;;;;;;;;;OAYG;IACH,gBAAgB,CACd,IAAI,EAAE,MAAM,EACZ,SAAS,EAAE,MAAM,EACjB,WAAW,EAAE,MAAM,EACnB,IAAI,EAAE,MAAM,EACZ,MAAM,EAAE,UAAU,EAClB,OAAO,CAAC,EAAE,iBAAiB,GAC1B,OAAO,CAAC,IAAI,CAAC,CAAC;IAEjB;;;;;;;;OAQG;IACH,YAAY,CACV,IAAI,EAAE,MAAM,EACZ,SAAS,EAAE,MAAM,EACjB,WAAW,EAAE,MAAM,EACnB,MAAM,EAAE,SAAS,GAAG,WAAW,GAAG,QAAQ,GAAG,WAAW,EACxD,OAAO,CAAC,EAAE,sBAAsB,GAC/B,OAAO,CAAC,IAAI,CAAC,CAAC;IAEjB;;;;;;;;;;OAUG;IACH,WAAW,CACT,IAAI,EAAE,MAAM,EACZ,SAAS,EAAE,MAAM,EACjB,WAAW,EAAE,MAAM,EACnB,KAAK,EAAE,cAAc,GACpB,OAAO,CAAC,IAAI,CAAC,CAAC;IAEjB;;;;;;;;;;OAUG;IACH,cAAc,CACZ,IAAI,EAAE,MAAM,EACZ,SAAS,EAAE,MAAM,EACjB,WAAW,EAAE,MAAM,EACnB,QAAQ,EAAE,MAAM,GACf,OAAO,CAAC,cAAc,EAAE,CAAC,CAAC;IAE7B;;;;;;;;;OASG;IACH,eAAe,CAAC,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IAElE;;;;;;;;;OASG;IACH,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CAC7E"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"interfaces.js","sourceRoot":"","sources":["../../../../src/dataflow/state-store/interfaces.ts"],"names":[],"mappings":"AAAA;;;GAGG"}
|
|
@@ -0,0 +1,176 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Copyright (c) 2025 Elara AI Pty Ltd
|
|
3
|
+
* Licensed under BSL 1.1. See LICENSE for details.
|
|
4
|
+
*/
|
|
5
|
+
import type { StorageBackend } from '../storage/interfaces.js';
|
|
6
|
+
import type { DataflowExecutionState, InitializeResult, PrepareTaskResult, TaskCompletedResult, TaskFailedResult, FinalizeResult, TreeUpdateResult, ExecutionEvent } from './types.js';
|
|
7
|
+
/**
|
|
8
|
+
* Options for initializing a dataflow execution.
|
|
9
|
+
*/
|
|
10
|
+
export interface StepInitializeOptions {
|
|
11
|
+
/** Maximum concurrent task executions (default: 4) */
|
|
12
|
+
concurrency?: number;
|
|
13
|
+
/** Force re-execution even if cached (default: false) */
|
|
14
|
+
force?: boolean;
|
|
15
|
+
/** Filter to run only specific task(s) by exact name */
|
|
16
|
+
filter?: string;
|
|
17
|
+
}
|
|
18
|
+
/**
|
|
19
|
+
* Initialize a new dataflow execution.
|
|
20
|
+
*
|
|
21
|
+
* Builds the dependency graph and creates the initial execution state.
|
|
22
|
+
* This is an async operation because it reads workspace and package state.
|
|
23
|
+
*
|
|
24
|
+
* @param storage - Storage backend
|
|
25
|
+
* @param repo - Repository identifier
|
|
26
|
+
* @param workspace - Workspace name
|
|
27
|
+
* @param executionId - Unique execution ID
|
|
28
|
+
* @param options - Execution options
|
|
29
|
+
* @returns Initial state and ready tasks
|
|
30
|
+
*
|
|
31
|
+
* @throws {WorkspaceNotFoundError} If workspace doesn't exist
|
|
32
|
+
* @throws {WorkspaceNotDeployedError} If workspace has no package deployed
|
|
33
|
+
* @throws {TaskNotFoundError} If filter specifies a task that doesn't exist
|
|
34
|
+
*/
|
|
35
|
+
export declare function stepInitialize(storage: StorageBackend, repo: string, workspace: string, executionId: string, options?: StepInitializeOptions): Promise<InitializeResult>;
|
|
36
|
+
/**
|
|
37
|
+
* Get tasks that are ready to execute.
|
|
38
|
+
*
|
|
39
|
+
* A task is ready when:
|
|
40
|
+
* 1. All tasks it depends on have completed (not just started)
|
|
41
|
+
* 2. It passes the filter (if any)
|
|
42
|
+
* 3. It is not already completed, in-progress, failed, or skipped
|
|
43
|
+
*
|
|
44
|
+
* This is a pure function - it only reads state.
|
|
45
|
+
*
|
|
46
|
+
* @param state - Current execution state
|
|
47
|
+
* @returns Array of task names that are ready to execute
|
|
48
|
+
*/
|
|
49
|
+
export declare function stepGetReady(state: DataflowExecutionState): string[];
|
|
50
|
+
/**
|
|
51
|
+
* Check if the execution is complete.
|
|
52
|
+
*
|
|
53
|
+
* An execution is complete when:
|
|
54
|
+
* - All tasks are in a terminal state (completed, failed, skipped)
|
|
55
|
+
* - Or there are no more ready tasks and no tasks in progress
|
|
56
|
+
*
|
|
57
|
+
* This is a pure function - it only reads state.
|
|
58
|
+
*
|
|
59
|
+
* @param state - Current execution state
|
|
60
|
+
* @returns True if execution is complete
|
|
61
|
+
*/
|
|
62
|
+
export declare function stepIsComplete(state: DataflowExecutionState): boolean;
|
|
63
|
+
/**
|
|
64
|
+
* Prepare a task for execution by resolving inputs and checking cache.
|
|
65
|
+
*
|
|
66
|
+
* This async operation:
|
|
67
|
+
* 1. Resolves input hashes from current workspace state
|
|
68
|
+
* 2. Checks if there's a valid cached execution
|
|
69
|
+
*
|
|
70
|
+
* @param storage - Storage backend
|
|
71
|
+
* @param state - Current execution state
|
|
72
|
+
* @param taskName - Name of the task to prepare
|
|
73
|
+
* @returns Preparation result with input hashes and cache status
|
|
74
|
+
*/
|
|
75
|
+
export declare function stepPrepareTask(storage: StorageBackend, state: DataflowExecutionState, taskName: string): Promise<PrepareTaskResult>;
|
|
76
|
+
/**
|
|
77
|
+
* Mark a task as started (in-progress).
|
|
78
|
+
*
|
|
79
|
+
* Mutates the execution state to record that a task has begun execution.
|
|
80
|
+
*
|
|
81
|
+
* @param state - Execution state to mutate
|
|
82
|
+
* @param taskName - Name of the task
|
|
83
|
+
* @returns Event to record
|
|
84
|
+
*/
|
|
85
|
+
export declare function stepTaskStarted(state: DataflowExecutionState, taskName: string): ExecutionEvent;
|
|
86
|
+
/**
|
|
87
|
+
* Mark a task as completed successfully.
|
|
88
|
+
*
|
|
89
|
+
* Mutates the execution state and returns the newly ready tasks.
|
|
90
|
+
*
|
|
91
|
+
* @param state - Execution state to mutate
|
|
92
|
+
* @param taskName - Name of the task
|
|
93
|
+
* @param outputHash - Hash of the output dataset
|
|
94
|
+
* @param cached - Whether the result was from cache
|
|
95
|
+
* @param duration - Execution duration in milliseconds
|
|
96
|
+
* @returns Result with newly ready tasks and event
|
|
97
|
+
*/
|
|
98
|
+
export declare function stepTaskCompleted(state: DataflowExecutionState, taskName: string, outputHash: string, cached: boolean, duration: number): {
|
|
99
|
+
result: TaskCompletedResult;
|
|
100
|
+
event: ExecutionEvent;
|
|
101
|
+
};
|
|
102
|
+
/**
|
|
103
|
+
* Mark a task as failed.
|
|
104
|
+
*
|
|
105
|
+
* Mutates the execution state and returns tasks that should be skipped.
|
|
106
|
+
*
|
|
107
|
+
* @param state - Execution state to mutate
|
|
108
|
+
* @param taskName - Name of the failed task
|
|
109
|
+
* @param error - Error message (optional)
|
|
110
|
+
* @param exitCode - Exit code if process failed (optional)
|
|
111
|
+
* @param duration - Execution duration in milliseconds
|
|
112
|
+
* @returns Result with tasks to skip and event
|
|
113
|
+
*/
|
|
114
|
+
export declare function stepTaskFailed(state: DataflowExecutionState, taskName: string, error: string | undefined, exitCode: number | undefined, duration: number): {
|
|
115
|
+
result: TaskFailedResult;
|
|
116
|
+
event: ExecutionEvent;
|
|
117
|
+
};
|
|
118
|
+
/**
|
|
119
|
+
* Mark tasks as skipped due to upstream failure.
|
|
120
|
+
*
|
|
121
|
+
* @param state - Execution state to mutate
|
|
122
|
+
* @param taskNames - Names of tasks to skip
|
|
123
|
+
* @param cause - Name of the task that caused the skip
|
|
124
|
+
* @returns Array of events to record
|
|
125
|
+
*/
|
|
126
|
+
export declare function stepTasksSkipped(state: DataflowExecutionState, taskNames: string[], cause: string): ExecutionEvent[];
|
|
127
|
+
/**
|
|
128
|
+
* Finalize the execution and return the result.
|
|
129
|
+
*
|
|
130
|
+
* Mutates the execution state to mark it as completed or failed.
|
|
131
|
+
*
|
|
132
|
+
* @param state - Execution state to mutate
|
|
133
|
+
* @returns Final result
|
|
134
|
+
*/
|
|
135
|
+
export declare function stepFinalize(state: DataflowExecutionState): {
|
|
136
|
+
result: FinalizeResult;
|
|
137
|
+
event: ExecutionEvent;
|
|
138
|
+
};
|
|
139
|
+
/**
|
|
140
|
+
* Cancel the execution.
|
|
141
|
+
*
|
|
142
|
+
* @param state - Execution state to mutate
|
|
143
|
+
* @param reason - Reason for cancellation
|
|
144
|
+
* @returns Event to record
|
|
145
|
+
*/
|
|
146
|
+
export declare function stepCancel(state: DataflowExecutionState, reason?: string): ExecutionEvent;
|
|
147
|
+
/**
|
|
148
|
+
* Apply a task's output to the workspace tree.
|
|
149
|
+
*
|
|
150
|
+
* This step function handles workspace tree updates, which must be serialized
|
|
151
|
+
* to prevent lost-update race conditions when multiple tasks complete
|
|
152
|
+
* concurrently.
|
|
153
|
+
*
|
|
154
|
+
* For local execution, the LocalOrchestrator handles this internally with
|
|
155
|
+
* an AsyncMutex. For cloud execution (e.g., AWS Step Functions), this should
|
|
156
|
+
* be called in a dedicated "ApplyTreeUpdates" state that processes updates
|
|
157
|
+
* serially.
|
|
158
|
+
*
|
|
159
|
+
* @param storage - Storage backend
|
|
160
|
+
* @param repo - Repository identifier
|
|
161
|
+
* @param workspace - Workspace name
|
|
162
|
+
* @param outputPathStr - Output path as a keypath string (e.g., ".results.data")
|
|
163
|
+
* @param outputHash - Hash of the output dataset to write
|
|
164
|
+
* @returns Result with the new workspace root hash
|
|
165
|
+
*
|
|
166
|
+
* @remarks
|
|
167
|
+
* Tree updates must be serialized to prevent race conditions:
|
|
168
|
+
* - Two tasks complete concurrently, both read the same workspace root
|
|
169
|
+
* - Both compute new roots with their outputs
|
|
170
|
+
* - One write overwrites the other, losing the first task's output
|
|
171
|
+
*
|
|
172
|
+
* Cloud implementations should use a dedicated serialization mechanism
|
|
173
|
+
* (e.g., a single Lambda invocation per update, or DynamoDB transactions).
|
|
174
|
+
*/
|
|
175
|
+
export declare function stepApplyTreeUpdate(storage: StorageBackend, repo: string, workspace: string, outputPathStr: string, outputHash: string): Promise<TreeUpdateResult>;
|
|
176
|
+
//# sourceMappingURL=steps.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"steps.d.ts","sourceRoot":"","sources":["../../../src/dataflow/steps.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAgBH,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,0BAA0B,CAAC;AAa/D,OAAO,KAAK,EACV,sBAAsB,EAEtB,gBAAgB,EAChB,iBAAiB,EACjB,mBAAmB,EACnB,gBAAgB,EAChB,cAAc,EACd,gBAAgB,EAChB,cAAc,EACf,MAAM,YAAY,CAAC;AASpB;;GAEG;AACH,MAAM,WAAW,qBAAqB;IACpC,sDAAsD;IACtD,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,yDAAyD;IACzD,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,wDAAwD;IACxD,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED;;;;;;;;;;;;;;;;GAgBG;AACH,wBAAsB,cAAc,CAClC,OAAO,EAAE,cAAc,EACvB,IAAI,EAAE,MAAM,EACZ,SAAS,EAAE,MAAM,EACjB,WAAW,EAAE,MAAM,EACnB,OAAO,GAAE,qBAA0B,GAClC,OAAO,CAAC,gBAAgB,CAAC,CAqE3B;AAyBD;;;;;;;;;;;;GAYG;AACH,wBAAgB,YAAY,CAAC,KAAK,EAAE,sBAAsB,GAAG,MAAM,EAAE,CA+BpE;AAED;;;;;;;;;;;GAWG;AACH,wBAAgB,cAAc,CAAC,KAAK,EAAE,sBAAsB,GAAG,OAAO,CAqCrE;AAMD;;;;;;;;;;;GAWG;AACH,wBAAsB,eAAe,CACnC,OAAO,EAAE,cAAc,EACvB,KAAK,EAAE,sBAAsB,EAC7B,QAAQ,EAAE,MAAM,GACf,OAAO,CAAC,iBAAiB,CAAC,CA0D5B;AAMD;;;;;;;;GAQG;AACH,wBAAgB,eAAe,CAC7B,KAAK,EAAE,sBAAsB,EAC7B,QAAQ,EAAE,MAAM,GACf,cAAc,CAmBhB;AAED;;;;;;;;;;;GAWG;AACH,wBAAgB,iBAAiB,CAC/B,KAAK,EAAE,sBAAsB,EAC7B,QAAQ,EAAE,MAAM,EAChB,UAAU,EAAE,MAAM,EAClB,MAAM,EAAE,OAAO,EACf,QAAQ,EAAE,MAAM,GACf;IAAE,MAAM,EAAE,mBAAmB,CAAC;IAAC,KAAK,EAAE,cAAc,CAAA;CAAE,CA2CxD;AAED;;;;;;;;;;;GAWG;AACH,wBAAgB,cAAc,CAC5B,KAAK,EAAE,sBAAsB,EAC7B,QAAQ,EAAE,MAAM,EAChB,KAAK,EAAE,MAAM,GAAG,SAAS,EACzB,QAAQ,EAAE,MAAM,GAAG,SAAS,EAC5B,QAAQ,EAAE,MAAM,GACf;IAAE,MAAM,EAAE,gBAAgB,CAAC;IAAC,KAAK,EAAE,cAAc,CAAA;CAAE,CAsDrD;AAED;;;;;;;GAOG;AACH,wBAAgB,gBAAgB,CAC9B,KAAK,EAAE,sBAAsB,EAC7B,SAAS,EAAE,MAAM,EAAE,EACnB,KAAK,EAAE,MAAM,GACZ,cAAc,EAAE,CA2BlB;AAED;;;;;;;GAOG;AACH,wBAAgB,YAAY,CAAC,KAAK,EAAE,sBAAsB,GAAG;IAC3D,MAAM,EAAE,cAAc,CAAC;IACvB,KAAK,EAAE,cAAc,CAAC;CACvB,CAoCA;AAED;;;;;;GAMG;AACH,wBAAgB,UAAU,CACxB,KAAK,EAAE,sBAAsB,EAC7B,MAAM,CAAC,EAAE,MAAM,GACd,cAAc,CAgBhB;AAMD;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AACH,wBAAsB,mBAAmB,CACvC,OAAO,EAAE,cAAc,EACvB,IAAI,EAAE,MAAM,EACZ,SAAS,EAAE,MAAM,EACjB,aAAa,EAAE,MAAM,EACrB,UAAU,EAAE,MAAM,GACjB,OAAO,CAAC,gBAAgB,CAAC,CAc3B"}
|