@elaraai/e3-core 0.0.2-beta.4 → 0.0.2-beta.40
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +25 -22
- package/dist/src/dataflow/api-compat.d.ts +90 -0
- package/dist/src/dataflow/api-compat.d.ts.map +1 -0
- package/dist/src/dataflow/api-compat.js +139 -0
- package/dist/src/dataflow/api-compat.js.map +1 -0
- package/dist/src/dataflow/index.d.ts +18 -0
- package/dist/src/dataflow/index.d.ts.map +1 -0
- package/dist/src/dataflow/index.js +23 -0
- package/dist/src/dataflow/index.js.map +1 -0
- package/dist/src/dataflow/orchestrator/LocalOrchestrator.d.ts +76 -0
- package/dist/src/dataflow/orchestrator/LocalOrchestrator.d.ts.map +1 -0
- package/dist/src/dataflow/orchestrator/LocalOrchestrator.js +695 -0
- package/dist/src/dataflow/orchestrator/LocalOrchestrator.js.map +1 -0
- package/dist/src/dataflow/orchestrator/index.d.ts +12 -0
- package/dist/src/dataflow/orchestrator/index.d.ts.map +1 -0
- package/dist/src/dataflow/orchestrator/index.js +12 -0
- package/dist/src/dataflow/orchestrator/index.js.map +1 -0
- package/dist/src/dataflow/orchestrator/interfaces.d.ts +163 -0
- package/dist/src/dataflow/orchestrator/interfaces.d.ts.map +1 -0
- package/dist/src/dataflow/orchestrator/interfaces.js +52 -0
- package/dist/src/dataflow/orchestrator/interfaces.js.map +1 -0
- package/dist/src/dataflow/state-store/FileStateStore.d.ts +67 -0
- package/dist/src/dataflow/state-store/FileStateStore.d.ts.map +1 -0
- package/dist/src/dataflow/state-store/FileStateStore.js +300 -0
- package/dist/src/dataflow/state-store/FileStateStore.js.map +1 -0
- package/dist/src/dataflow/state-store/InMemoryStateStore.d.ts +42 -0
- package/dist/src/dataflow/state-store/InMemoryStateStore.d.ts.map +1 -0
- package/dist/src/dataflow/state-store/InMemoryStateStore.js +229 -0
- package/dist/src/dataflow/state-store/InMemoryStateStore.js.map +1 -0
- package/dist/src/dataflow/state-store/index.d.ts +13 -0
- package/dist/src/dataflow/state-store/index.d.ts.map +1 -0
- package/dist/src/dataflow/state-store/index.js +13 -0
- package/dist/src/dataflow/state-store/index.js.map +1 -0
- package/dist/src/dataflow/state-store/interfaces.d.ts +159 -0
- package/dist/src/dataflow/state-store/interfaces.d.ts.map +1 -0
- package/dist/src/dataflow/state-store/interfaces.js +6 -0
- package/dist/src/dataflow/state-store/interfaces.js.map +1 -0
- package/dist/src/dataflow/steps.d.ts +222 -0
- package/dist/src/dataflow/steps.d.ts.map +1 -0
- package/dist/src/dataflow/steps.js +707 -0
- package/dist/src/dataflow/steps.js.map +1 -0
- package/dist/src/dataflow/types.d.ts +127 -0
- package/dist/src/dataflow/types.d.ts.map +1 -0
- package/dist/src/dataflow/types.js +7 -0
- package/dist/src/dataflow/types.js.map +1 -0
- package/dist/src/dataflow.d.ts +113 -38
- package/dist/src/dataflow.d.ts.map +1 -1
- package/dist/src/dataflow.js +269 -416
- package/dist/src/dataflow.js.map +1 -1
- package/dist/src/dataset-refs.d.ts +124 -0
- package/dist/src/dataset-refs.d.ts.map +1 -0
- package/dist/src/dataset-refs.js +319 -0
- package/dist/src/dataset-refs.js.map +1 -0
- package/dist/src/errors.d.ts +39 -9
- package/dist/src/errors.d.ts.map +1 -1
- package/dist/src/errors.js +51 -8
- package/dist/src/errors.js.map +1 -1
- package/dist/src/execution/LocalTaskRunner.d.ts +73 -0
- package/dist/src/execution/LocalTaskRunner.d.ts.map +1 -0
- package/dist/src/execution/LocalTaskRunner.js +399 -0
- package/dist/src/execution/LocalTaskRunner.js.map +1 -0
- package/dist/src/execution/MockTaskRunner.d.ts +49 -0
- package/dist/src/execution/MockTaskRunner.d.ts.map +1 -0
- package/dist/src/execution/MockTaskRunner.js +54 -0
- package/dist/src/execution/MockTaskRunner.js.map +1 -0
- package/dist/src/execution/index.d.ts +16 -0
- package/dist/src/execution/index.d.ts.map +1 -0
- package/dist/src/execution/index.js +8 -0
- package/dist/src/execution/index.js.map +1 -0
- package/dist/src/execution/interfaces.d.ts +246 -0
- package/dist/src/execution/interfaces.d.ts.map +1 -0
- package/dist/src/execution/interfaces.js +6 -0
- package/dist/src/execution/interfaces.js.map +1 -0
- package/dist/src/execution/processHelpers.d.ts +20 -0
- package/dist/src/execution/processHelpers.d.ts.map +1 -0
- package/dist/src/execution/processHelpers.js +62 -0
- package/dist/src/execution/processHelpers.js.map +1 -0
- package/dist/src/executions.d.ts +71 -104
- package/dist/src/executions.d.ts.map +1 -1
- package/dist/src/executions.js +110 -476
- package/dist/src/executions.js.map +1 -1
- package/dist/src/index.d.ts +19 -9
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +48 -18
- package/dist/src/index.js.map +1 -1
- package/dist/src/objects.d.ts +8 -51
- package/dist/src/objects.d.ts.map +1 -1
- package/dist/src/objects.js +13 -230
- package/dist/src/objects.js.map +1 -1
- package/dist/src/packages.d.ts +22 -14
- package/dist/src/packages.d.ts.map +1 -1
- package/dist/src/packages.js +134 -88
- package/dist/src/packages.js.map +1 -1
- package/dist/src/storage/in-memory/InMemoryRepoStore.d.ts +35 -0
- package/dist/src/storage/in-memory/InMemoryRepoStore.d.ts.map +1 -0
- package/dist/src/storage/in-memory/InMemoryRepoStore.js +107 -0
- package/dist/src/storage/in-memory/InMemoryRepoStore.js.map +1 -0
- package/dist/src/storage/in-memory/InMemoryStorage.d.ts +139 -0
- package/dist/src/storage/in-memory/InMemoryStorage.d.ts.map +1 -0
- package/dist/src/storage/in-memory/InMemoryStorage.js +439 -0
- package/dist/src/storage/in-memory/InMemoryStorage.js.map +1 -0
- package/dist/src/storage/in-memory/index.d.ts +12 -0
- package/dist/src/storage/in-memory/index.d.ts.map +1 -0
- package/dist/src/storage/in-memory/index.js +12 -0
- package/dist/src/storage/in-memory/index.js.map +1 -0
- package/dist/src/storage/index.d.ts +18 -0
- package/dist/src/storage/index.d.ts.map +1 -0
- package/dist/src/storage/index.js +10 -0
- package/dist/src/storage/index.js.map +1 -0
- package/dist/src/storage/interfaces.d.ts +581 -0
- package/dist/src/storage/interfaces.d.ts.map +1 -0
- package/dist/src/storage/interfaces.js +6 -0
- package/dist/src/storage/interfaces.js.map +1 -0
- package/dist/src/storage/local/LocalBackend.d.ts +56 -0
- package/dist/src/storage/local/LocalBackend.d.ts.map +1 -0
- package/dist/src/storage/local/LocalBackend.js +145 -0
- package/dist/src/storage/local/LocalBackend.js.map +1 -0
- package/dist/src/storage/local/LocalDatasetRefStore.d.ts +22 -0
- package/dist/src/storage/local/LocalDatasetRefStore.d.ts.map +1 -0
- package/dist/src/storage/local/LocalDatasetRefStore.js +118 -0
- package/dist/src/storage/local/LocalDatasetRefStore.js.map +1 -0
- package/dist/src/storage/local/LocalLockService.d.ts +111 -0
- package/dist/src/storage/local/LocalLockService.d.ts.map +1 -0
- package/dist/src/storage/local/LocalLockService.js +355 -0
- package/dist/src/storage/local/LocalLockService.js.map +1 -0
- package/dist/src/storage/local/LocalLogStore.d.ts +23 -0
- package/dist/src/storage/local/LocalLogStore.d.ts.map +1 -0
- package/dist/src/storage/local/LocalLogStore.js +66 -0
- package/dist/src/storage/local/LocalLogStore.js.map +1 -0
- package/dist/src/storage/local/LocalObjectStore.d.ts +55 -0
- package/dist/src/storage/local/LocalObjectStore.d.ts.map +1 -0
- package/dist/src/storage/local/LocalObjectStore.js +300 -0
- package/dist/src/storage/local/LocalObjectStore.js.map +1 -0
- package/dist/src/storage/local/LocalRefStore.d.ts +50 -0
- package/dist/src/storage/local/LocalRefStore.d.ts.map +1 -0
- package/dist/src/storage/local/LocalRefStore.js +337 -0
- package/dist/src/storage/local/LocalRefStore.js.map +1 -0
- package/dist/src/storage/local/LocalRepoStore.d.ts +55 -0
- package/dist/src/storage/local/LocalRepoStore.d.ts.map +1 -0
- package/dist/src/storage/local/LocalRepoStore.js +365 -0
- package/dist/src/storage/local/LocalRepoStore.js.map +1 -0
- package/dist/src/storage/local/gc.d.ts +92 -0
- package/dist/src/storage/local/gc.d.ts.map +1 -0
- package/dist/src/storage/local/gc.js +377 -0
- package/dist/src/storage/local/gc.js.map +1 -0
- package/dist/src/storage/local/index.d.ts +18 -0
- package/dist/src/storage/local/index.d.ts.map +1 -0
- package/dist/src/storage/local/index.js +18 -0
- package/dist/src/storage/local/index.js.map +1 -0
- package/dist/src/storage/local/localHelpers.d.ts +25 -0
- package/dist/src/storage/local/localHelpers.d.ts.map +1 -0
- package/dist/src/storage/local/localHelpers.js +69 -0
- package/dist/src/storage/local/localHelpers.js.map +1 -0
- package/dist/src/{repository.d.ts → storage/local/repository.d.ts} +8 -4
- package/dist/src/storage/local/repository.d.ts.map +1 -0
- package/dist/src/{repository.js → storage/local/repository.js} +31 -29
- package/dist/src/storage/local/repository.js.map +1 -0
- package/dist/src/tasks.d.ts +16 -10
- package/dist/src/tasks.d.ts.map +1 -1
- package/dist/src/tasks.js +35 -41
- package/dist/src/tasks.js.map +1 -1
- package/dist/src/test-helpers.d.ts +5 -4
- package/dist/src/test-helpers.d.ts.map +1 -1
- package/dist/src/test-helpers.js +9 -21
- package/dist/src/test-helpers.js.map +1 -1
- package/dist/src/transfer/InMemoryTransferBackend.d.ts +66 -0
- package/dist/src/transfer/InMemoryTransferBackend.d.ts.map +1 -0
- package/dist/src/transfer/InMemoryTransferBackend.js +166 -0
- package/dist/src/transfer/InMemoryTransferBackend.js.map +1 -0
- package/dist/src/transfer/index.d.ts +8 -0
- package/dist/src/transfer/index.d.ts.map +1 -0
- package/dist/src/transfer/index.js +9 -0
- package/dist/src/transfer/index.js.map +1 -0
- package/dist/src/transfer/interfaces.d.ts +103 -0
- package/dist/src/transfer/interfaces.d.ts.map +1 -0
- package/dist/src/transfer/interfaces.js +6 -0
- package/dist/src/transfer/interfaces.js.map +1 -0
- package/dist/src/transfer/types.d.ts +79 -0
- package/dist/src/transfer/types.d.ts.map +1 -0
- package/dist/src/transfer/types.js +58 -0
- package/dist/src/transfer/types.js.map +1 -0
- package/dist/src/trees.d.ts +147 -59
- package/dist/src/trees.d.ts.map +1 -1
- package/dist/src/trees.js +372 -419
- package/dist/src/trees.js.map +1 -1
- package/dist/src/uuid.d.ts +26 -0
- package/dist/src/uuid.d.ts.map +1 -0
- package/dist/src/uuid.js +80 -0
- package/dist/src/uuid.js.map +1 -0
- package/dist/src/workspaceStatus.d.ts +6 -4
- package/dist/src/workspaceStatus.d.ts.map +1 -1
- package/dist/src/workspaceStatus.js +43 -49
- package/dist/src/workspaceStatus.js.map +1 -1
- package/dist/src/workspaces.d.ts +35 -47
- package/dist/src/workspaces.d.ts.map +1 -1
- package/dist/src/workspaces.js +194 -156
- package/dist/src/workspaces.js.map +1 -1
- package/package.json +4 -4
- package/dist/src/gc.d.ts +0 -54
- package/dist/src/gc.d.ts.map +0 -1
- package/dist/src/gc.js +0 -233
- package/dist/src/gc.js.map +0 -1
- package/dist/src/repository.d.ts.map +0 -1
- package/dist/src/repository.js.map +0 -1
- package/dist/src/workspaceLock.d.ts +0 -67
- package/dist/src/workspaceLock.d.ts.map +0 -1
- package/dist/src/workspaceLock.js +0 -217
- package/dist/src/workspaceLock.js.map +0 -1
|
@@ -0,0 +1,159 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Copyright (c) 2025 Elara AI Pty Ltd
|
|
3
|
+
* Licensed under BSL 1.1. See LICENSE for details.
|
|
4
|
+
*/
|
|
5
|
+
/**
|
|
6
|
+
* State store interface for dataflow execution.
|
|
7
|
+
*
|
|
8
|
+
* Abstracts the persistence of execution state, enabling:
|
|
9
|
+
* - InMemoryStateStore: For testing and simple cases
|
|
10
|
+
* - FileStateStore: Local filesystem persistence (workspace/execution.beast2)
|
|
11
|
+
* - DynamoDBStateStore: Cloud execution (in e3-aws)
|
|
12
|
+
*/
|
|
13
|
+
import type { DataflowExecutionState, ExecutionEvent, TaskStatus } from '../types.js';
|
|
14
|
+
/**
|
|
15
|
+
* Details for task status updates.
|
|
16
|
+
*/
|
|
17
|
+
export interface TaskStatusDetails {
|
|
18
|
+
/** Whether the result was from cache */
|
|
19
|
+
cached?: boolean;
|
|
20
|
+
/** Output hash (for completed tasks) */
|
|
21
|
+
outputHash?: string;
|
|
22
|
+
/** Error message (for failed tasks) */
|
|
23
|
+
error?: string;
|
|
24
|
+
/** Exit code (for failed tasks) */
|
|
25
|
+
exitCode?: number;
|
|
26
|
+
/** Duration in milliseconds */
|
|
27
|
+
duration?: number;
|
|
28
|
+
}
|
|
29
|
+
/**
|
|
30
|
+
* Details for execution status updates.
|
|
31
|
+
*/
|
|
32
|
+
export interface ExecutionStatusDetails {
|
|
33
|
+
/** Error message (for failed executions) */
|
|
34
|
+
error?: string;
|
|
35
|
+
/** Summary counts */
|
|
36
|
+
summary?: {
|
|
37
|
+
executed: number;
|
|
38
|
+
cached: number;
|
|
39
|
+
failed: number;
|
|
40
|
+
skipped: number;
|
|
41
|
+
};
|
|
42
|
+
}
|
|
43
|
+
/**
|
|
44
|
+
* Interface for persisting and retrieving execution state.
|
|
45
|
+
*
|
|
46
|
+
* Implementations must be thread-safe for concurrent access within a process.
|
|
47
|
+
* For distributed execution (cloud), implementations should use optimistic
|
|
48
|
+
* concurrency control (e.g., DynamoDB conditional writes).
|
|
49
|
+
*
|
|
50
|
+
* @remarks
|
|
51
|
+
* All methods take repo and workspace parameters because cloud storage
|
|
52
|
+
* (DynamoDB) needs both to identify an execution uniquely across repositories.
|
|
53
|
+
*/
|
|
54
|
+
export interface ExecutionStateStore {
|
|
55
|
+
/**
|
|
56
|
+
* Create a new execution state.
|
|
57
|
+
*
|
|
58
|
+
* @param state - The initial execution state (contains repo and workspace)
|
|
59
|
+
* @throws If an execution with the same ID already exists
|
|
60
|
+
*/
|
|
61
|
+
create(state: DataflowExecutionState): Promise<void>;
|
|
62
|
+
/**
|
|
63
|
+
* Read an execution state by ID.
|
|
64
|
+
*
|
|
65
|
+
* @param repo - Repository identifier
|
|
66
|
+
* @param workspace - Workspace name
|
|
67
|
+
* @param id - Execution ID
|
|
68
|
+
* @returns The execution state, or null if not found
|
|
69
|
+
*/
|
|
70
|
+
read(repo: string, workspace: string, id: string): Promise<DataflowExecutionState | null>;
|
|
71
|
+
/**
|
|
72
|
+
* Read the most recent execution for a workspace.
|
|
73
|
+
*
|
|
74
|
+
* @param repo - Repository identifier
|
|
75
|
+
* @param workspace - Workspace name
|
|
76
|
+
* @returns The most recent execution state, or null if none exists
|
|
77
|
+
*/
|
|
78
|
+
readLatest(repo: string, workspace: string): Promise<DataflowExecutionState | null>;
|
|
79
|
+
/**
|
|
80
|
+
* Update the entire execution state.
|
|
81
|
+
*
|
|
82
|
+
* This is used for bulk updates after a sequence of step functions.
|
|
83
|
+
* Implementations may optimize by only writing changed fields.
|
|
84
|
+
*
|
|
85
|
+
* @param state - The updated execution state (contains repo and workspace)
|
|
86
|
+
*/
|
|
87
|
+
update(state: DataflowExecutionState): Promise<void>;
|
|
88
|
+
/**
|
|
89
|
+
* Update a task's status within an execution.
|
|
90
|
+
*
|
|
91
|
+
* This is a convenience method for updating a single task without
|
|
92
|
+
* reading and writing the entire state.
|
|
93
|
+
*
|
|
94
|
+
* @param repo - Repository identifier
|
|
95
|
+
* @param workspace - Workspace name
|
|
96
|
+
* @param executionId - Execution ID
|
|
97
|
+
* @param task - Task name
|
|
98
|
+
* @param status - New status
|
|
99
|
+
* @param details - Additional details (output hash, error, etc.)
|
|
100
|
+
*/
|
|
101
|
+
updateTaskStatus(repo: string, workspace: string, executionId: string, task: string, status: TaskStatus, details?: TaskStatusDetails): Promise<void>;
|
|
102
|
+
/**
|
|
103
|
+
* Update the execution's overall status.
|
|
104
|
+
*
|
|
105
|
+
* @param repo - Repository identifier
|
|
106
|
+
* @param workspace - Workspace name
|
|
107
|
+
* @param executionId - Execution ID
|
|
108
|
+
* @param status - New status ('running' | 'completed' | 'failed' | 'cancelled')
|
|
109
|
+
* @param details - Additional details (error message, summary)
|
|
110
|
+
*/
|
|
111
|
+
updateStatus(repo: string, workspace: string, executionId: string, status: 'running' | 'completed' | 'failed' | 'cancelled', details?: ExecutionStatusDetails): Promise<void>;
|
|
112
|
+
/**
|
|
113
|
+
* Record an event for an execution.
|
|
114
|
+
*
|
|
115
|
+
* Events are used for monitoring and debugging. They are append-only
|
|
116
|
+
* and can be read with getEventsSince().
|
|
117
|
+
*
|
|
118
|
+
* @param repo - Repository identifier
|
|
119
|
+
* @param workspace - Workspace name
|
|
120
|
+
* @param executionId - Execution ID
|
|
121
|
+
* @param event - The event to record
|
|
122
|
+
*/
|
|
123
|
+
recordEvent(repo: string, workspace: string, executionId: string, event: ExecutionEvent): Promise<void>;
|
|
124
|
+
/**
|
|
125
|
+
* Get events for an execution since a given sequence number.
|
|
126
|
+
*
|
|
127
|
+
* Used for polling/watching execution progress.
|
|
128
|
+
*
|
|
129
|
+
* @param repo - Repository identifier
|
|
130
|
+
* @param workspace - Workspace name
|
|
131
|
+
* @param executionId - Execution ID
|
|
132
|
+
* @param sinceSeq - Only return events with seq > sinceSeq
|
|
133
|
+
* @returns Array of events in sequence order
|
|
134
|
+
*/
|
|
135
|
+
getEventsSince(repo: string, workspace: string, executionId: string, sinceSeq: number): Promise<ExecutionEvent[]>;
|
|
136
|
+
/**
|
|
137
|
+
* Get the next execution ID for a workspace.
|
|
138
|
+
*
|
|
139
|
+
* For local storage, returns auto-incrementing string IDs ("1", "2", ...).
|
|
140
|
+
* For cloud storage, may return UUIDs or other unique identifiers.
|
|
141
|
+
*
|
|
142
|
+
* @param repo - Repository identifier
|
|
143
|
+
* @param workspace - Workspace name
|
|
144
|
+
* @returns The next execution ID as a string
|
|
145
|
+
*/
|
|
146
|
+
nextExecutionId(repo: string, workspace: string): Promise<string>;
|
|
147
|
+
/**
|
|
148
|
+
* Delete an execution state.
|
|
149
|
+
*
|
|
150
|
+
* Used for cleanup after execution completion or for removing
|
|
151
|
+
* abandoned executions.
|
|
152
|
+
*
|
|
153
|
+
* @param repo - Repository identifier
|
|
154
|
+
* @param workspace - Workspace name
|
|
155
|
+
* @param executionId - Execution ID
|
|
156
|
+
*/
|
|
157
|
+
delete(repo: string, workspace: string, executionId: string): Promise<void>;
|
|
158
|
+
}
|
|
159
|
+
//# sourceMappingURL=interfaces.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"interfaces.d.ts","sourceRoot":"","sources":["../../../../src/dataflow/state-store/interfaces.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH;;;;;;;GAOG;AAEH,OAAO,KAAK,EACV,sBAAsB,EACtB,cAAc,EACd,UAAU,EACX,MAAM,aAAa,CAAC;AAErB;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,wCAAwC;IACxC,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,wCAAwC;IACxC,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,uCAAuC;IACvC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,mCAAmC;IACnC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,+BAA+B;IAC/B,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED;;GAEG;AACH,MAAM,WAAW,sBAAsB;IACrC,4CAA4C;IAC5C,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,qBAAqB;IACrB,OAAO,CAAC,EAAE;QACR,QAAQ,EAAE,MAAM,CAAC;QACjB,MAAM,EAAE,MAAM,CAAC;QACf,MAAM,EAAE,MAAM,CAAC;QACf,OAAO,EAAE,MAAM,CAAC;KACjB,CAAC;CACH;AAED;;;;;;;;;;GAUG;AACH,MAAM,WAAW,mBAAmB;IAClC;;;;;OAKG;IACH,MAAM,CAAC,KAAK,EAAE,sBAAsB,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAErD;;;;;;;OAOG;IACH,IAAI,CAAC,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,sBAAsB,GAAG,IAAI,CAAC,CAAC;IAE1F;;;;;;OAMG;IACH,UAAU,CAAC,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,sBAAsB,GAAG,IAAI,CAAC,CAAC;IAEpF;;;;;;;OAOG;IACH,MAAM,CAAC,KAAK,EAAE,sBAAsB,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAErD;;;;;;;;;;;;OAYG;IACH,gBAAgB,CACd,IAAI,EAAE,MAAM,EACZ,SAAS,EAAE,MAAM,EACjB,WAAW,EAAE,MAAM,EACnB,IAAI,EAAE,MAAM,EACZ,MAAM,EAAE,UAAU,EAClB,OAAO,CAAC,EAAE,iBAAiB,GAC1B,OAAO,CAAC,IAAI,CAAC,CAAC;IAEjB;;;;;;;;OAQG;IACH,YAAY,CACV,IAAI,EAAE,MAAM,EACZ,SAAS,EAAE,MAAM,EACjB,WAAW,EAAE,MAAM,EACnB,MAAM,EAAE,SAAS,GAAG,WAAW,GAAG,QAAQ,GAAG,WAAW,EACxD,OAAO,CAAC,EAAE,sBAAsB,GAC/B,OAAO,CAAC,IAAI,CAAC,CAAC;IAEjB;;;;;;;;;;OAUG;IACH,WAAW,CACT,IAAI,EAAE,MAAM,EACZ,SAAS,EAAE,MAAM,EACjB,WAAW,EAAE,MAAM,EACnB,KAAK,EAAE,cAAc,GACpB,OAAO,CAAC,IAAI,CAAC,CAAC;IAEjB;;;;;;;;;;OAUG;IACH,cAAc,CACZ,IAAI,EAAE,MAAM,EACZ,SAAS,EAAE,MAAM,EACjB,WAAW,EAAE,MAAM,EACnB,QAAQ,EAAE,MAAM,GACf,OAAO,CAAC,cAAc,EAAE,CAAC,CAAC;IAE7B;;;;;;;;;OASG;IACH,eAAe,CAAC,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IAElE;;;;;;;;;OASG;IACH,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CAC7E"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"interfaces.js","sourceRoot":"","sources":["../../../../src/dataflow/state-store/interfaces.ts"],"names":[],"mappings":"AAAA;;;GAGG"}
|
|
@@ -0,0 +1,222 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Copyright (c) 2025 Elara AI Pty Ltd
|
|
3
|
+
* Licensed under BSL 1.1. See LICENSE for details.
|
|
4
|
+
*/
|
|
5
|
+
import type { VersionVector, Structure } from '@elaraai/e3-types';
|
|
6
|
+
import type { StorageBackend } from '../storage/interfaces.js';
|
|
7
|
+
import type { DataflowExecutionState, InitializeResult, PrepareTaskResult, TaskCompletedResult, TaskFailedResult, FinalizeResult, TreeUpdateResult, ExecutionEvent } from './types.js';
|
|
8
|
+
/**
|
|
9
|
+
* Options for initializing a dataflow execution.
|
|
10
|
+
*/
|
|
11
|
+
export interface StepInitializeOptions {
|
|
12
|
+
/** Maximum concurrent task executions (default: 4) */
|
|
13
|
+
concurrency?: number;
|
|
14
|
+
/** Force re-execution even if cached (default: false) */
|
|
15
|
+
force?: boolean;
|
|
16
|
+
/** Filter to run only specific task(s) by exact name */
|
|
17
|
+
filter?: string;
|
|
18
|
+
}
|
|
19
|
+
/**
|
|
20
|
+
* Initialize a new dataflow execution.
|
|
21
|
+
*
|
|
22
|
+
* Builds the dependency graph, snapshots root input versions, initializes
|
|
23
|
+
* version vectors, and creates the initial execution state.
|
|
24
|
+
*
|
|
25
|
+
* @param storage - Storage backend
|
|
26
|
+
* @param repo - Repository identifier
|
|
27
|
+
* @param workspace - Workspace name
|
|
28
|
+
* @param executionId - Unique execution ID
|
|
29
|
+
* @param options - Execution options
|
|
30
|
+
* @returns Initial state and ready tasks
|
|
31
|
+
*
|
|
32
|
+
* @throws {WorkspaceNotFoundError} If workspace doesn't exist
|
|
33
|
+
* @throws {WorkspaceNotDeployedError} If workspace has no package deployed
|
|
34
|
+
* @throws {TaskNotFoundError} If filter specifies a task that doesn't exist
|
|
35
|
+
*/
|
|
36
|
+
export declare function stepInitialize(storage: StorageBackend, repo: string, workspace: string, executionId: string, options?: StepInitializeOptions): Promise<InitializeResult>;
|
|
37
|
+
/**
|
|
38
|
+
* Get tasks that are ready to execute.
|
|
39
|
+
*
|
|
40
|
+
* A task is ready when:
|
|
41
|
+
* 1. All tasks it depends on have completed (not just started)
|
|
42
|
+
* 2. It passes the filter (if any)
|
|
43
|
+
* 3. It is not already completed, in-progress, failed, skipped, or deferred
|
|
44
|
+
*
|
|
45
|
+
* This is a pure function - it only reads state.
|
|
46
|
+
*
|
|
47
|
+
* @param state - Current execution state
|
|
48
|
+
* @returns Array of task names that are ready to execute
|
|
49
|
+
*/
|
|
50
|
+
export declare function stepGetReady(state: DataflowExecutionState): string[];
|
|
51
|
+
/**
|
|
52
|
+
* Check if the execution is complete.
|
|
53
|
+
*
|
|
54
|
+
* An execution is complete when all tasks are in a terminal state
|
|
55
|
+
* (completed, failed, skipped) or have permanently unresolvable
|
|
56
|
+
* dependencies. Returns false if any non-terminal tasks remain.
|
|
57
|
+
*
|
|
58
|
+
* This is a pure function - it only reads state.
|
|
59
|
+
*
|
|
60
|
+
* @param state - Current execution state
|
|
61
|
+
* @returns True if execution is complete
|
|
62
|
+
*/
|
|
63
|
+
export declare function stepIsComplete(state: DataflowExecutionState): boolean;
|
|
64
|
+
/**
|
|
65
|
+
* Detect input changes since the last snapshot.
|
|
66
|
+
*
|
|
67
|
+
* Reads current root input dataset hashes from storage and compares
|
|
68
|
+
* against the snapshot stored in state. For each change, updates the
|
|
69
|
+
* input snapshot and version vectors, and emits input_changed events.
|
|
70
|
+
*
|
|
71
|
+
* @param storage - Storage backend
|
|
72
|
+
* @param state - Execution state to mutate (inputSnapshot, versionVectors)
|
|
73
|
+
* @returns Changes detected and events emitted
|
|
74
|
+
*/
|
|
75
|
+
export declare function stepDetectInputChanges(storage: StorageBackend, state: DataflowExecutionState, cachedStructure?: Structure | null): Promise<{
|
|
76
|
+
changes: Array<{
|
|
77
|
+
path: string;
|
|
78
|
+
previousHash: string | null;
|
|
79
|
+
newHash: string;
|
|
80
|
+
}>;
|
|
81
|
+
events: ExecutionEvent[];
|
|
82
|
+
}>;
|
|
83
|
+
/**
|
|
84
|
+
* Invalidate tasks affected by input changes.
|
|
85
|
+
*
|
|
86
|
+
* For each affected task that is not currently in_progress:
|
|
87
|
+
* - completed: reset to pending, decrement executed/cached counter, emit task_invalidated
|
|
88
|
+
* - deferred: reset to pending
|
|
89
|
+
* - pending/ready: leave as-is (will pick up new inputs naturally)
|
|
90
|
+
* - skipped: leave as-is (upstream failure still applies)
|
|
91
|
+
* - failed: leave as-is (task already failed; orchestrator is winding down)
|
|
92
|
+
*
|
|
93
|
+
* @param state - Execution state to mutate
|
|
94
|
+
* @param changes - Input changes from stepDetectInputChanges
|
|
95
|
+
* @returns Invalidated task names and events
|
|
96
|
+
*/
|
|
97
|
+
export declare function stepInvalidateTasks(state: DataflowExecutionState, changes: Array<{
|
|
98
|
+
path: string;
|
|
99
|
+
}>): {
|
|
100
|
+
invalidated: string[];
|
|
101
|
+
events: ExecutionEvent[];
|
|
102
|
+
};
|
|
103
|
+
/**
|
|
104
|
+
* Check version vector consistency for a task's inputs.
|
|
105
|
+
*
|
|
106
|
+
* All input version vectors must agree on shared keys (same root input path
|
|
107
|
+
* must have the same hash in every vector that contains it).
|
|
108
|
+
*
|
|
109
|
+
* @param state - Execution state
|
|
110
|
+
* @param taskName - Task to check
|
|
111
|
+
* @returns Consistency result with merged VV or conflict path
|
|
112
|
+
*/
|
|
113
|
+
export declare function stepCheckVersionConsistency(state: DataflowExecutionState, taskName: string): {
|
|
114
|
+
consistent: true;
|
|
115
|
+
mergedVV: VersionVector;
|
|
116
|
+
} | {
|
|
117
|
+
consistent: false;
|
|
118
|
+
conflictPath: string;
|
|
119
|
+
};
|
|
120
|
+
/**
|
|
121
|
+
* Prepare a task for execution by resolving inputs and checking cache.
|
|
122
|
+
*
|
|
123
|
+
* This async operation:
|
|
124
|
+
* 1. Resolves input hashes from current workspace state
|
|
125
|
+
* 2. Checks if there's a valid cached execution
|
|
126
|
+
*
|
|
127
|
+
* @param storage - Storage backend
|
|
128
|
+
* @param state - Current execution state
|
|
129
|
+
* @param taskName - Name of the task to prepare
|
|
130
|
+
* @returns Preparation result with input hashes and cache status
|
|
131
|
+
*/
|
|
132
|
+
export declare function stepPrepareTask(storage: StorageBackend, state: DataflowExecutionState, taskName: string): Promise<PrepareTaskResult>;
|
|
133
|
+
/**
|
|
134
|
+
* Mark a task as started (in-progress).
|
|
135
|
+
*
|
|
136
|
+
* Mutates the execution state to record that a task has begun execution.
|
|
137
|
+
*
|
|
138
|
+
* @param state - Execution state to mutate
|
|
139
|
+
* @param taskName - Name of the task
|
|
140
|
+
* @returns Event to record
|
|
141
|
+
*/
|
|
142
|
+
export declare function stepTaskStarted(state: DataflowExecutionState, taskName: string): ExecutionEvent;
|
|
143
|
+
/**
|
|
144
|
+
* Mark a task as completed successfully.
|
|
145
|
+
*
|
|
146
|
+
* Mutates the execution state, computes the merged version vector for the
|
|
147
|
+
* task's output, and returns the newly ready tasks.
|
|
148
|
+
*
|
|
149
|
+
* @param state - Execution state to mutate
|
|
150
|
+
* @param taskName - Name of the task
|
|
151
|
+
* @param outputHash - Hash of the output dataset
|
|
152
|
+
* @param cached - Whether the result was from cache
|
|
153
|
+
* @param duration - Execution duration in milliseconds
|
|
154
|
+
* @returns Result with newly ready tasks and event
|
|
155
|
+
*/
|
|
156
|
+
export declare function stepTaskCompleted(state: DataflowExecutionState, taskName: string, outputHash: string, cached: boolean, duration: number): {
|
|
157
|
+
result: TaskCompletedResult;
|
|
158
|
+
event: ExecutionEvent;
|
|
159
|
+
};
|
|
160
|
+
/**
|
|
161
|
+
* Mark a task as failed.
|
|
162
|
+
*
|
|
163
|
+
* Mutates the execution state and returns tasks that should be skipped.
|
|
164
|
+
*
|
|
165
|
+
* @param state - Execution state to mutate
|
|
166
|
+
* @param taskName - Name of the failed task
|
|
167
|
+
* @param error - Error message (optional)
|
|
168
|
+
* @param exitCode - Exit code if process failed (optional)
|
|
169
|
+
* @param duration - Execution duration in milliseconds
|
|
170
|
+
* @returns Result with tasks to skip and event
|
|
171
|
+
*/
|
|
172
|
+
export declare function stepTaskFailed(state: DataflowExecutionState, taskName: string, error: string | undefined, exitCode: number | undefined, duration: number): {
|
|
173
|
+
result: TaskFailedResult;
|
|
174
|
+
event: ExecutionEvent;
|
|
175
|
+
};
|
|
176
|
+
/**
|
|
177
|
+
* Mark tasks as skipped due to upstream failure.
|
|
178
|
+
*
|
|
179
|
+
* @param state - Execution state to mutate
|
|
180
|
+
* @param taskNames - Names of tasks to skip
|
|
181
|
+
* @param cause - Name of the task that caused the skip
|
|
182
|
+
* @returns Array of events to record
|
|
183
|
+
*/
|
|
184
|
+
export declare function stepTasksSkipped(state: DataflowExecutionState, taskNames: string[], cause: string): ExecutionEvent[];
|
|
185
|
+
/**
|
|
186
|
+
* Finalize the execution and return the result.
|
|
187
|
+
*
|
|
188
|
+
* Mutates the execution state to mark it as completed or failed.
|
|
189
|
+
*
|
|
190
|
+
* @param state - Execution state to mutate
|
|
191
|
+
* @param runId - Dataflow run ID (UUIDv7) from the orchestrator
|
|
192
|
+
* @returns Final result
|
|
193
|
+
*/
|
|
194
|
+
export declare function stepFinalize(state: DataflowExecutionState, runId: string): {
|
|
195
|
+
result: FinalizeResult;
|
|
196
|
+
event: ExecutionEvent;
|
|
197
|
+
};
|
|
198
|
+
/**
|
|
199
|
+
* Cancel the execution.
|
|
200
|
+
*
|
|
201
|
+
* @param state - Execution state to mutate
|
|
202
|
+
* @param reason - Reason for cancellation
|
|
203
|
+
* @returns Event to record
|
|
204
|
+
*/
|
|
205
|
+
export declare function stepCancel(state: DataflowExecutionState, reason?: string): ExecutionEvent;
|
|
206
|
+
/**
|
|
207
|
+
* Apply a task's output to the workspace tree with version vector.
|
|
208
|
+
*
|
|
209
|
+
* Writes the output ref file with the merged version vector from the task's
|
|
210
|
+
* inputs. Per-dataset ref writes are atomic and independent, so no
|
|
211
|
+
* serialization is needed for concurrent writes to different paths.
|
|
212
|
+
*
|
|
213
|
+
* @param storage - Storage backend
|
|
214
|
+
* @param repo - Repository identifier
|
|
215
|
+
* @param workspace - Workspace name
|
|
216
|
+
* @param outputPathStr - Output path as a keypath string (e.g., ".results.data")
|
|
217
|
+
* @param outputHash - Hash of the output dataset to write
|
|
218
|
+
* @param versions - Merged version vector for provenance tracking
|
|
219
|
+
* @returns Result indicating success
|
|
220
|
+
*/
|
|
221
|
+
export declare function stepApplyTreeUpdate(storage: StorageBackend, repo: string, workspace: string, outputPathStr: string, outputHash: string, versions: VersionVector): Promise<TreeUpdateResult>;
|
|
222
|
+
//# sourceMappingURL=steps.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"steps.d.ts","sourceRoot":"","sources":["../../../src/dataflow/steps.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAoBH,OAAO,KAAK,EAAE,aAAa,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AAClE,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,0BAA0B,CAAC;AAqB/D,OAAO,KAAK,EACV,sBAAsB,EAEtB,gBAAgB,EAChB,iBAAiB,EACjB,mBAAmB,EACnB,gBAAgB,EAChB,cAAc,EACd,gBAAgB,EAChB,cAAc,EAEf,MAAM,YAAY,CAAC;AAMpB;;GAEG;AACH,MAAM,WAAW,qBAAqB;IACpC,sDAAsD;IACtD,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,yDAAyD;IACzD,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,wDAAwD;IACxD,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED;;;;;;;;;;;;;;;;GAgBG;AACH,wBAAsB,cAAc,CAClC,OAAO,EAAE,cAAc,EACvB,IAAI,EAAE,MAAM,EACZ,SAAS,EAAE,MAAM,EACjB,WAAW,EAAE,MAAM,EACnB,OAAO,GAAE,qBAA0B,GAClC,OAAO,CAAC,gBAAgB,CAAC,CA8F3B;AAoDD;;;;;;;;;;;;GAYG;AACH,wBAAgB,YAAY,CAAC,KAAK,EAAE,sBAAsB,GAAG,MAAM,EAAE,CAkCpE;AAED;;;;;;;;;;;GAWG;AACH,wBAAgB,cAAc,CAAC,KAAK,EAAE,sBAAsB,GAAG,OAAO,CAoCrE;AAMD;;;;;;;;;;GAUG;AACH,wBAAsB,sBAAsB,CAC1C,OAAO,EAAE,cAAc,EACvB,KAAK,EAAE,sBAAsB,EAC7B,eAAe,CAAC,EAAE,SAAS,GAAG,IAAI,GACjC,OAAO,CAAC;IACT,OAAO,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAC/E,MAAM,EAAE,cAAc,EAAE,CAAC;CAC1B,CAAC,CAiCD;AAED;;;;;;;;;;;;;GAaG;AACH,wBAAgB,mBAAmB,CACjC,KAAK,EAAE,sBAAsB,EAC7B,OAAO,EAAE,KAAK,CAAC;IAAE,IAAI,EAAE,MAAM,CAAA;CAAE,CAAC,GAC/B;IACD,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB,MAAM,EAAE,cAAc,EAAE,CAAC;CAC1B,CAwDA;AAED;;;;;;;;;GASG;AACH,wBAAgB,2BAA2B,CACzC,KAAK,EAAE,sBAAsB,EAC7B,QAAQ,EAAE,MAAM,GACf;IAAE,UAAU,EAAE,IAAI,CAAC;IAAC,QAAQ,EAAE,aAAa,CAAA;CAAE,GAAG;IAAE,UAAU,EAAE,KAAK,CAAC;IAAC,YAAY,EAAE,MAAM,CAAA;CAAE,CAkB7F;AAMD;;;;;;;;;;;GAWG;AACH,wBAAsB,eAAe,CACnC,OAAO,EAAE,cAAc,EACvB,KAAK,EAAE,sBAAsB,EAC7B,QAAQ,EAAE,MAAM,GACf,OAAO,CAAC,iBAAiB,CAAC,CA0D5B;AAMD;;;;;;;;GAQG;AACH,wBAAgB,eAAe,CAC7B,KAAK,EAAE,sBAAsB,EAC7B,QAAQ,EAAE,MAAM,GACf,cAAc,CAmBhB;AAED;;;;;;;;;;;;GAYG;AACH,wBAAgB,iBAAiB,CAC/B,KAAK,EAAE,sBAAsB,EAC7B,QAAQ,EAAE,MAAM,EAChB,UAAU,EAAE,MAAM,EAClB,MAAM,EAAE,OAAO,EACf,QAAQ,EAAE,MAAM,GACf;IAAE,MAAM,EAAE,mBAAmB,CAAC;IAAC,KAAK,EAAE,cAAc,CAAA;CAAE,CAwDxD;AAED;;;;;;;;;;;GAWG;AACH,wBAAgB,cAAc,CAC5B,KAAK,EAAE,sBAAsB,EAC7B,QAAQ,EAAE,MAAM,EAChB,KAAK,EAAE,MAAM,GAAG,SAAS,EACzB,QAAQ,EAAE,MAAM,GAAG,SAAS,EAC5B,QAAQ,EAAE,MAAM,GACf;IAAE,MAAM,EAAE,gBAAgB,CAAC;IAAC,KAAK,EAAE,cAAc,CAAA;CAAE,CAsDrD;AAED;;;;;;;GAOG;AACH,wBAAgB,gBAAgB,CAC9B,KAAK,EAAE,sBAAsB,EAC7B,SAAS,EAAE,MAAM,EAAE,EACnB,KAAK,EAAE,MAAM,GACZ,cAAc,EAAE,CA2BlB;AAED;;;;;;;;GAQG;AACH,wBAAgB,YAAY,CAAC,KAAK,EAAE,sBAAsB,EAAE,KAAK,EAAE,MAAM,GAAG;IAC1E,MAAM,EAAE,cAAc,CAAC;IACvB,KAAK,EAAE,cAAc,CAAC;CACvB,CAsCA;AAED;;;;;;GAMG;AACH,wBAAgB,UAAU,CACxB,KAAK,EAAE,sBAAsB,EAC7B,MAAM,CAAC,EAAE,MAAM,GACd,cAAc,CAgBhB;AAMD;;;;;;;;;;;;;;GAcG;AACH,wBAAsB,mBAAmB,CACvC,OAAO,EAAE,cAAc,EACvB,IAAI,EAAE,MAAM,EACZ,SAAS,EAAE,MAAM,EACjB,aAAa,EAAE,MAAM,EACrB,UAAU,EAAE,MAAM,EAClB,QAAQ,EAAE,aAAa,GACtB,OAAO,CAAC,gBAAgB,CAAC,CAe3B"}
|