@elaraai/e3-core 0.0.2-beta.4 → 0.0.2-beta.41

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (208) hide show
  1. package/README.md +25 -22
  2. package/dist/src/dataflow/api-compat.d.ts +90 -0
  3. package/dist/src/dataflow/api-compat.d.ts.map +1 -0
  4. package/dist/src/dataflow/api-compat.js +139 -0
  5. package/dist/src/dataflow/api-compat.js.map +1 -0
  6. package/dist/src/dataflow/index.d.ts +18 -0
  7. package/dist/src/dataflow/index.d.ts.map +1 -0
  8. package/dist/src/dataflow/index.js +23 -0
  9. package/dist/src/dataflow/index.js.map +1 -0
  10. package/dist/src/dataflow/orchestrator/LocalOrchestrator.d.ts +76 -0
  11. package/dist/src/dataflow/orchestrator/LocalOrchestrator.d.ts.map +1 -0
  12. package/dist/src/dataflow/orchestrator/LocalOrchestrator.js +695 -0
  13. package/dist/src/dataflow/orchestrator/LocalOrchestrator.js.map +1 -0
  14. package/dist/src/dataflow/orchestrator/index.d.ts +12 -0
  15. package/dist/src/dataflow/orchestrator/index.d.ts.map +1 -0
  16. package/dist/src/dataflow/orchestrator/index.js +12 -0
  17. package/dist/src/dataflow/orchestrator/index.js.map +1 -0
  18. package/dist/src/dataflow/orchestrator/interfaces.d.ts +163 -0
  19. package/dist/src/dataflow/orchestrator/interfaces.d.ts.map +1 -0
  20. package/dist/src/dataflow/orchestrator/interfaces.js +52 -0
  21. package/dist/src/dataflow/orchestrator/interfaces.js.map +1 -0
  22. package/dist/src/dataflow/state-store/FileStateStore.d.ts +67 -0
  23. package/dist/src/dataflow/state-store/FileStateStore.d.ts.map +1 -0
  24. package/dist/src/dataflow/state-store/FileStateStore.js +300 -0
  25. package/dist/src/dataflow/state-store/FileStateStore.js.map +1 -0
  26. package/dist/src/dataflow/state-store/InMemoryStateStore.d.ts +42 -0
  27. package/dist/src/dataflow/state-store/InMemoryStateStore.d.ts.map +1 -0
  28. package/dist/src/dataflow/state-store/InMemoryStateStore.js +229 -0
  29. package/dist/src/dataflow/state-store/InMemoryStateStore.js.map +1 -0
  30. package/dist/src/dataflow/state-store/index.d.ts +13 -0
  31. package/dist/src/dataflow/state-store/index.d.ts.map +1 -0
  32. package/dist/src/dataflow/state-store/index.js +13 -0
  33. package/dist/src/dataflow/state-store/index.js.map +1 -0
  34. package/dist/src/dataflow/state-store/interfaces.d.ts +159 -0
  35. package/dist/src/dataflow/state-store/interfaces.d.ts.map +1 -0
  36. package/dist/src/dataflow/state-store/interfaces.js +6 -0
  37. package/dist/src/dataflow/state-store/interfaces.js.map +1 -0
  38. package/dist/src/dataflow/steps.d.ts +222 -0
  39. package/dist/src/dataflow/steps.d.ts.map +1 -0
  40. package/dist/src/dataflow/steps.js +707 -0
  41. package/dist/src/dataflow/steps.js.map +1 -0
  42. package/dist/src/dataflow/types.d.ts +127 -0
  43. package/dist/src/dataflow/types.d.ts.map +1 -0
  44. package/dist/src/dataflow/types.js +7 -0
  45. package/dist/src/dataflow/types.js.map +1 -0
  46. package/dist/src/dataflow.d.ts +113 -38
  47. package/dist/src/dataflow.d.ts.map +1 -1
  48. package/dist/src/dataflow.js +269 -416
  49. package/dist/src/dataflow.js.map +1 -1
  50. package/dist/src/dataset-refs.d.ts +124 -0
  51. package/dist/src/dataset-refs.d.ts.map +1 -0
  52. package/dist/src/dataset-refs.js +319 -0
  53. package/dist/src/dataset-refs.js.map +1 -0
  54. package/dist/src/errors.d.ts +39 -9
  55. package/dist/src/errors.d.ts.map +1 -1
  56. package/dist/src/errors.js +51 -8
  57. package/dist/src/errors.js.map +1 -1
  58. package/dist/src/execution/LocalTaskRunner.d.ts +73 -0
  59. package/dist/src/execution/LocalTaskRunner.d.ts.map +1 -0
  60. package/dist/src/execution/LocalTaskRunner.js +399 -0
  61. package/dist/src/execution/LocalTaskRunner.js.map +1 -0
  62. package/dist/src/execution/MockTaskRunner.d.ts +49 -0
  63. package/dist/src/execution/MockTaskRunner.d.ts.map +1 -0
  64. package/dist/src/execution/MockTaskRunner.js +54 -0
  65. package/dist/src/execution/MockTaskRunner.js.map +1 -0
  66. package/dist/src/execution/index.d.ts +16 -0
  67. package/dist/src/execution/index.d.ts.map +1 -0
  68. package/dist/src/execution/index.js +8 -0
  69. package/dist/src/execution/index.js.map +1 -0
  70. package/dist/src/execution/interfaces.d.ts +246 -0
  71. package/dist/src/execution/interfaces.d.ts.map +1 -0
  72. package/dist/src/execution/interfaces.js +6 -0
  73. package/dist/src/execution/interfaces.js.map +1 -0
  74. package/dist/src/execution/processHelpers.d.ts +20 -0
  75. package/dist/src/execution/processHelpers.d.ts.map +1 -0
  76. package/dist/src/execution/processHelpers.js +62 -0
  77. package/dist/src/execution/processHelpers.js.map +1 -0
  78. package/dist/src/executions.d.ts +71 -104
  79. package/dist/src/executions.d.ts.map +1 -1
  80. package/dist/src/executions.js +110 -476
  81. package/dist/src/executions.js.map +1 -1
  82. package/dist/src/index.d.ts +19 -9
  83. package/dist/src/index.d.ts.map +1 -1
  84. package/dist/src/index.js +48 -18
  85. package/dist/src/index.js.map +1 -1
  86. package/dist/src/objects.d.ts +8 -51
  87. package/dist/src/objects.d.ts.map +1 -1
  88. package/dist/src/objects.js +13 -230
  89. package/dist/src/objects.js.map +1 -1
  90. package/dist/src/packages.d.ts +22 -14
  91. package/dist/src/packages.d.ts.map +1 -1
  92. package/dist/src/packages.js +134 -88
  93. package/dist/src/packages.js.map +1 -1
  94. package/dist/src/storage/in-memory/InMemoryRepoStore.d.ts +35 -0
  95. package/dist/src/storage/in-memory/InMemoryRepoStore.d.ts.map +1 -0
  96. package/dist/src/storage/in-memory/InMemoryRepoStore.js +107 -0
  97. package/dist/src/storage/in-memory/InMemoryRepoStore.js.map +1 -0
  98. package/dist/src/storage/in-memory/InMemoryStorage.d.ts +139 -0
  99. package/dist/src/storage/in-memory/InMemoryStorage.d.ts.map +1 -0
  100. package/dist/src/storage/in-memory/InMemoryStorage.js +439 -0
  101. package/dist/src/storage/in-memory/InMemoryStorage.js.map +1 -0
  102. package/dist/src/storage/in-memory/index.d.ts +12 -0
  103. package/dist/src/storage/in-memory/index.d.ts.map +1 -0
  104. package/dist/src/storage/in-memory/index.js +12 -0
  105. package/dist/src/storage/in-memory/index.js.map +1 -0
  106. package/dist/src/storage/index.d.ts +18 -0
  107. package/dist/src/storage/index.d.ts.map +1 -0
  108. package/dist/src/storage/index.js +10 -0
  109. package/dist/src/storage/index.js.map +1 -0
  110. package/dist/src/storage/interfaces.d.ts +581 -0
  111. package/dist/src/storage/interfaces.d.ts.map +1 -0
  112. package/dist/src/storage/interfaces.js +6 -0
  113. package/dist/src/storage/interfaces.js.map +1 -0
  114. package/dist/src/storage/local/LocalBackend.d.ts +56 -0
  115. package/dist/src/storage/local/LocalBackend.d.ts.map +1 -0
  116. package/dist/src/storage/local/LocalBackend.js +145 -0
  117. package/dist/src/storage/local/LocalBackend.js.map +1 -0
  118. package/dist/src/storage/local/LocalDatasetRefStore.d.ts +22 -0
  119. package/dist/src/storage/local/LocalDatasetRefStore.d.ts.map +1 -0
  120. package/dist/src/storage/local/LocalDatasetRefStore.js +118 -0
  121. package/dist/src/storage/local/LocalDatasetRefStore.js.map +1 -0
  122. package/dist/src/storage/local/LocalLockService.d.ts +111 -0
  123. package/dist/src/storage/local/LocalLockService.d.ts.map +1 -0
  124. package/dist/src/storage/local/LocalLockService.js +355 -0
  125. package/dist/src/storage/local/LocalLockService.js.map +1 -0
  126. package/dist/src/storage/local/LocalLogStore.d.ts +23 -0
  127. package/dist/src/storage/local/LocalLogStore.d.ts.map +1 -0
  128. package/dist/src/storage/local/LocalLogStore.js +66 -0
  129. package/dist/src/storage/local/LocalLogStore.js.map +1 -0
  130. package/dist/src/storage/local/LocalObjectStore.d.ts +55 -0
  131. package/dist/src/storage/local/LocalObjectStore.d.ts.map +1 -0
  132. package/dist/src/storage/local/LocalObjectStore.js +300 -0
  133. package/dist/src/storage/local/LocalObjectStore.js.map +1 -0
  134. package/dist/src/storage/local/LocalRefStore.d.ts +50 -0
  135. package/dist/src/storage/local/LocalRefStore.d.ts.map +1 -0
  136. package/dist/src/storage/local/LocalRefStore.js +337 -0
  137. package/dist/src/storage/local/LocalRefStore.js.map +1 -0
  138. package/dist/src/storage/local/LocalRepoStore.d.ts +55 -0
  139. package/dist/src/storage/local/LocalRepoStore.d.ts.map +1 -0
  140. package/dist/src/storage/local/LocalRepoStore.js +365 -0
  141. package/dist/src/storage/local/LocalRepoStore.js.map +1 -0
  142. package/dist/src/storage/local/gc.d.ts +92 -0
  143. package/dist/src/storage/local/gc.d.ts.map +1 -0
  144. package/dist/src/storage/local/gc.js +377 -0
  145. package/dist/src/storage/local/gc.js.map +1 -0
  146. package/dist/src/storage/local/index.d.ts +18 -0
  147. package/dist/src/storage/local/index.d.ts.map +1 -0
  148. package/dist/src/storage/local/index.js +18 -0
  149. package/dist/src/storage/local/index.js.map +1 -0
  150. package/dist/src/storage/local/localHelpers.d.ts +25 -0
  151. package/dist/src/storage/local/localHelpers.d.ts.map +1 -0
  152. package/dist/src/storage/local/localHelpers.js +69 -0
  153. package/dist/src/storage/local/localHelpers.js.map +1 -0
  154. package/dist/src/{repository.d.ts → storage/local/repository.d.ts} +8 -4
  155. package/dist/src/storage/local/repository.d.ts.map +1 -0
  156. package/dist/src/{repository.js → storage/local/repository.js} +31 -29
  157. package/dist/src/storage/local/repository.js.map +1 -0
  158. package/dist/src/tasks.d.ts +16 -10
  159. package/dist/src/tasks.d.ts.map +1 -1
  160. package/dist/src/tasks.js +35 -41
  161. package/dist/src/tasks.js.map +1 -1
  162. package/dist/src/test-helpers.d.ts +5 -4
  163. package/dist/src/test-helpers.d.ts.map +1 -1
  164. package/dist/src/test-helpers.js +9 -21
  165. package/dist/src/test-helpers.js.map +1 -1
  166. package/dist/src/transfer/InMemoryTransferBackend.d.ts +66 -0
  167. package/dist/src/transfer/InMemoryTransferBackend.d.ts.map +1 -0
  168. package/dist/src/transfer/InMemoryTransferBackend.js +166 -0
  169. package/dist/src/transfer/InMemoryTransferBackend.js.map +1 -0
  170. package/dist/src/transfer/index.d.ts +8 -0
  171. package/dist/src/transfer/index.d.ts.map +1 -0
  172. package/dist/src/transfer/index.js +9 -0
  173. package/dist/src/transfer/index.js.map +1 -0
  174. package/dist/src/transfer/interfaces.d.ts +103 -0
  175. package/dist/src/transfer/interfaces.d.ts.map +1 -0
  176. package/dist/src/transfer/interfaces.js +6 -0
  177. package/dist/src/transfer/interfaces.js.map +1 -0
  178. package/dist/src/transfer/types.d.ts +79 -0
  179. package/dist/src/transfer/types.d.ts.map +1 -0
  180. package/dist/src/transfer/types.js +58 -0
  181. package/dist/src/transfer/types.js.map +1 -0
  182. package/dist/src/trees.d.ts +147 -59
  183. package/dist/src/trees.d.ts.map +1 -1
  184. package/dist/src/trees.js +372 -419
  185. package/dist/src/trees.js.map +1 -1
  186. package/dist/src/uuid.d.ts +26 -0
  187. package/dist/src/uuid.d.ts.map +1 -0
  188. package/dist/src/uuid.js +80 -0
  189. package/dist/src/uuid.js.map +1 -0
  190. package/dist/src/workspaceStatus.d.ts +6 -4
  191. package/dist/src/workspaceStatus.d.ts.map +1 -1
  192. package/dist/src/workspaceStatus.js +43 -49
  193. package/dist/src/workspaceStatus.js.map +1 -1
  194. package/dist/src/workspaces.d.ts +35 -47
  195. package/dist/src/workspaces.d.ts.map +1 -1
  196. package/dist/src/workspaces.js +194 -156
  197. package/dist/src/workspaces.js.map +1 -1
  198. package/package.json +4 -4
  199. package/dist/src/gc.d.ts +0 -54
  200. package/dist/src/gc.d.ts.map +0 -1
  201. package/dist/src/gc.js +0 -233
  202. package/dist/src/gc.js.map +0 -1
  203. package/dist/src/repository.d.ts.map +0 -1
  204. package/dist/src/repository.js.map +0 -1
  205. package/dist/src/workspaceLock.d.ts +0 -67
  206. package/dist/src/workspaceLock.d.ts.map +0 -1
  207. package/dist/src/workspaceLock.js +0 -217
  208. package/dist/src/workspaceLock.js.map +0 -1
@@ -0,0 +1,159 @@
1
+ /**
2
+ * Copyright (c) 2025 Elara AI Pty Ltd
3
+ * Licensed under BSL 1.1. See LICENSE for details.
4
+ */
5
+ /**
6
+ * State store interface for dataflow execution.
7
+ *
8
+ * Abstracts the persistence of execution state, enabling:
9
+ * - InMemoryStateStore: For testing and simple cases
10
+ * - FileStateStore: Local filesystem persistence (workspace/execution.beast2)
11
+ * - DynamoDBStateStore: Cloud execution (in e3-aws)
12
+ */
13
+ import type { DataflowExecutionState, ExecutionEvent, TaskStatus } from '../types.js';
14
+ /**
15
+ * Details for task status updates.
16
+ */
17
+ export interface TaskStatusDetails {
18
+ /** Whether the result was from cache */
19
+ cached?: boolean;
20
+ /** Output hash (for completed tasks) */
21
+ outputHash?: string;
22
+ /** Error message (for failed tasks) */
23
+ error?: string;
24
+ /** Exit code (for failed tasks) */
25
+ exitCode?: number;
26
+ /** Duration in milliseconds */
27
+ duration?: number;
28
+ }
29
+ /**
30
+ * Details for execution status updates.
31
+ */
32
+ export interface ExecutionStatusDetails {
33
+ /** Error message (for failed executions) */
34
+ error?: string;
35
+ /** Summary counts */
36
+ summary?: {
37
+ executed: number;
38
+ cached: number;
39
+ failed: number;
40
+ skipped: number;
41
+ };
42
+ }
43
+ /**
44
+ * Interface for persisting and retrieving execution state.
45
+ *
46
+ * Implementations must be thread-safe for concurrent access within a process.
47
+ * For distributed execution (cloud), implementations should use optimistic
48
+ * concurrency control (e.g., DynamoDB conditional writes).
49
+ *
50
+ * @remarks
51
+ * All methods take repo and workspace parameters because cloud storage
52
+ * (DynamoDB) needs both to identify an execution uniquely across repositories.
53
+ */
54
+ export interface ExecutionStateStore {
55
+ /**
56
+ * Create a new execution state.
57
+ *
58
+ * @param state - The initial execution state (contains repo and workspace)
59
+ * @throws If an execution with the same ID already exists
60
+ */
61
+ create(state: DataflowExecutionState): Promise<void>;
62
+ /**
63
+ * Read an execution state by ID.
64
+ *
65
+ * @param repo - Repository identifier
66
+ * @param workspace - Workspace name
67
+ * @param id - Execution ID
68
+ * @returns The execution state, or null if not found
69
+ */
70
+ read(repo: string, workspace: string, id: string): Promise<DataflowExecutionState | null>;
71
+ /**
72
+ * Read the most recent execution for a workspace.
73
+ *
74
+ * @param repo - Repository identifier
75
+ * @param workspace - Workspace name
76
+ * @returns The most recent execution state, or null if none exists
77
+ */
78
+ readLatest(repo: string, workspace: string): Promise<DataflowExecutionState | null>;
79
+ /**
80
+ * Update the entire execution state.
81
+ *
82
+ * This is used for bulk updates after a sequence of step functions.
83
+ * Implementations may optimize by only writing changed fields.
84
+ *
85
+ * @param state - The updated execution state (contains repo and workspace)
86
+ */
87
+ update(state: DataflowExecutionState): Promise<void>;
88
+ /**
89
+ * Update a task's status within an execution.
90
+ *
91
+ * This is a convenience method for updating a single task without
92
+ * reading and writing the entire state.
93
+ *
94
+ * @param repo - Repository identifier
95
+ * @param workspace - Workspace name
96
+ * @param executionId - Execution ID
97
+ * @param task - Task name
98
+ * @param status - New status
99
+ * @param details - Additional details (output hash, error, etc.)
100
+ */
101
+ updateTaskStatus(repo: string, workspace: string, executionId: string, task: string, status: TaskStatus, details?: TaskStatusDetails): Promise<void>;
102
+ /**
103
+ * Update the execution's overall status.
104
+ *
105
+ * @param repo - Repository identifier
106
+ * @param workspace - Workspace name
107
+ * @param executionId - Execution ID
108
+ * @param status - New status ('running' | 'completed' | 'failed' | 'cancelled')
109
+ * @param details - Additional details (error message, summary)
110
+ */
111
+ updateStatus(repo: string, workspace: string, executionId: string, status: 'running' | 'completed' | 'failed' | 'cancelled', details?: ExecutionStatusDetails): Promise<void>;
112
+ /**
113
+ * Record an event for an execution.
114
+ *
115
+ * Events are used for monitoring and debugging. They are append-only
116
+ * and can be read with getEventsSince().
117
+ *
118
+ * @param repo - Repository identifier
119
+ * @param workspace - Workspace name
120
+ * @param executionId - Execution ID
121
+ * @param event - The event to record
122
+ */
123
+ recordEvent(repo: string, workspace: string, executionId: string, event: ExecutionEvent): Promise<void>;
124
+ /**
125
+ * Get events for an execution since a given sequence number.
126
+ *
127
+ * Used for polling/watching execution progress.
128
+ *
129
+ * @param repo - Repository identifier
130
+ * @param workspace - Workspace name
131
+ * @param executionId - Execution ID
132
+ * @param sinceSeq - Only return events with seq > sinceSeq
133
+ * @returns Array of events in sequence order
134
+ */
135
+ getEventsSince(repo: string, workspace: string, executionId: string, sinceSeq: number): Promise<ExecutionEvent[]>;
136
+ /**
137
+ * Get the next execution ID for a workspace.
138
+ *
139
+ * For local storage, returns auto-incrementing string IDs ("1", "2", ...).
140
+ * For cloud storage, may return UUIDs or other unique identifiers.
141
+ *
142
+ * @param repo - Repository identifier
143
+ * @param workspace - Workspace name
144
+ * @returns The next execution ID as a string
145
+ */
146
+ nextExecutionId(repo: string, workspace: string): Promise<string>;
147
+ /**
148
+ * Delete an execution state.
149
+ *
150
+ * Used for cleanup after execution completion or for removing
151
+ * abandoned executions.
152
+ *
153
+ * @param repo - Repository identifier
154
+ * @param workspace - Workspace name
155
+ * @param executionId - Execution ID
156
+ */
157
+ delete(repo: string, workspace: string, executionId: string): Promise<void>;
158
+ }
159
+ //# sourceMappingURL=interfaces.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"interfaces.d.ts","sourceRoot":"","sources":["../../../../src/dataflow/state-store/interfaces.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH;;;;;;;GAOG;AAEH,OAAO,KAAK,EACV,sBAAsB,EACtB,cAAc,EACd,UAAU,EACX,MAAM,aAAa,CAAC;AAErB;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,wCAAwC;IACxC,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,wCAAwC;IACxC,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,uCAAuC;IACvC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,mCAAmC;IACnC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,+BAA+B;IAC/B,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED;;GAEG;AACH,MAAM,WAAW,sBAAsB;IACrC,4CAA4C;IAC5C,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,qBAAqB;IACrB,OAAO,CAAC,EAAE;QACR,QAAQ,EAAE,MAAM,CAAC;QACjB,MAAM,EAAE,MAAM,CAAC;QACf,MAAM,EAAE,MAAM,CAAC;QACf,OAAO,EAAE,MAAM,CAAC;KACjB,CAAC;CACH;AAED;;;;;;;;;;GAUG;AACH,MAAM,WAAW,mBAAmB;IAClC;;;;;OAKG;IACH,MAAM,CAAC,KAAK,EAAE,sBAAsB,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAErD;;;;;;;OAOG;IACH,IAAI,CAAC,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,sBAAsB,GAAG,IAAI,CAAC,CAAC;IAE1F;;;;;;OAMG;IACH,UAAU,CAAC,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,sBAAsB,GAAG,IAAI,CAAC,CAAC;IAEpF;;;;;;;OAOG;IACH,MAAM,CAAC,KAAK,EAAE,sBAAsB,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAErD;;;;;;;;;;;;OAYG;IACH,gBAAgB,CACd,IAAI,EAAE,MAAM,EACZ,SAAS,EAAE,MAAM,EACjB,WAAW,EAAE,MAAM,EACnB,IAAI,EAAE,MAAM,EACZ,MAAM,EAAE,UAAU,EAClB,OAAO,CAAC,EAAE,iBAAiB,GAC1B,OAAO,CAAC,IAAI,CAAC,CAAC;IAEjB;;;;;;;;OAQG;IACH,YAAY,CACV,IAAI,EAAE,MAAM,EACZ,SAAS,EAAE,MAAM,EACjB,WAAW,EAAE,MAAM,EACnB,MAAM,EAAE,SAAS,GAAG,WAAW,GAAG,QAAQ,GAAG,WAAW,EACxD,OAAO,CAAC,EAAE,sBAAsB,GAC/B,OAAO,CAAC,IAAI,CAAC,CAAC;IAEjB;;;;;;;;;;OAUG;IACH,WAAW,CACT,IAAI,EAAE,MAAM,EACZ,SAAS,EAAE,MAAM,EACjB,WAAW,EAAE,MAAM,EACnB,KAAK,EAAE,cAAc,GACpB,OAAO,CAAC,IAAI,CAAC,CAAC;IAEjB;;;;;;;;;;OAUG;IACH,cAAc,CACZ,IAAI,EAAE,MAAM,EACZ,SAAS,EAAE,MAAM,EACjB,WAAW,EAAE,MAAM,EACnB,QAAQ,EAAE,MAAM,GACf,OAAO,CAAC,cAAc,EAAE,CAAC,CAAC;IAE7B;;;;;;;;;OASG;IACH,eAAe,CAAC,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IAElE;;;;;;;;;OASG;IACH,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CAC7E"}
@@ -0,0 +1,6 @@
1
+ /**
2
+ * Copyright (c) 2025 Elara AI Pty Ltd
3
+ * Licensed under BSL 1.1. See LICENSE for details.
4
+ */
5
+ export {};
6
+ //# sourceMappingURL=interfaces.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"interfaces.js","sourceRoot":"","sources":["../../../../src/dataflow/state-store/interfaces.ts"],"names":[],"mappings":"AAAA;;;GAGG"}
@@ -0,0 +1,222 @@
1
+ /**
2
+ * Copyright (c) 2025 Elara AI Pty Ltd
3
+ * Licensed under BSL 1.1. See LICENSE for details.
4
+ */
5
+ import type { VersionVector, Structure } from '@elaraai/e3-types';
6
+ import type { StorageBackend } from '../storage/interfaces.js';
7
+ import type { DataflowExecutionState, InitializeResult, PrepareTaskResult, TaskCompletedResult, TaskFailedResult, FinalizeResult, TreeUpdateResult, ExecutionEvent } from './types.js';
8
+ /**
9
+ * Options for initializing a dataflow execution.
10
+ */
11
+ export interface StepInitializeOptions {
12
+ /** Maximum concurrent task executions (default: 4) */
13
+ concurrency?: number;
14
+ /** Force re-execution even if cached (default: false) */
15
+ force?: boolean;
16
+ /** Filter to run only specific task(s) by exact name */
17
+ filter?: string;
18
+ }
19
+ /**
20
+ * Initialize a new dataflow execution.
21
+ *
22
+ * Builds the dependency graph, snapshots root input versions, initializes
23
+ * version vectors, and creates the initial execution state.
24
+ *
25
+ * @param storage - Storage backend
26
+ * @param repo - Repository identifier
27
+ * @param workspace - Workspace name
28
+ * @param executionId - Unique execution ID
29
+ * @param options - Execution options
30
+ * @returns Initial state and ready tasks
31
+ *
32
+ * @throws {WorkspaceNotFoundError} If workspace doesn't exist
33
+ * @throws {WorkspaceNotDeployedError} If workspace has no package deployed
34
+ * @throws {TaskNotFoundError} If filter specifies a task that doesn't exist
35
+ */
36
+ export declare function stepInitialize(storage: StorageBackend, repo: string, workspace: string, executionId: string, options?: StepInitializeOptions): Promise<InitializeResult>;
37
+ /**
38
+ * Get tasks that are ready to execute.
39
+ *
40
+ * A task is ready when:
41
+ * 1. All tasks it depends on have completed (not just started)
42
+ * 2. It passes the filter (if any)
43
+ * 3. It is not already completed, in-progress, failed, skipped, or deferred
44
+ *
45
+ * This is a pure function - it only reads state.
46
+ *
47
+ * @param state - Current execution state
48
+ * @returns Array of task names that are ready to execute
49
+ */
50
+ export declare function stepGetReady(state: DataflowExecutionState): string[];
51
+ /**
52
+ * Check if the execution is complete.
53
+ *
54
+ * An execution is complete when all tasks are in a terminal state
55
+ * (completed, failed, skipped) or have permanently unresolvable
56
+ * dependencies. Returns false if any non-terminal tasks remain.
57
+ *
58
+ * This is a pure function - it only reads state.
59
+ *
60
+ * @param state - Current execution state
61
+ * @returns True if execution is complete
62
+ */
63
+ export declare function stepIsComplete(state: DataflowExecutionState): boolean;
64
+ /**
65
+ * Detect input changes since the last snapshot.
66
+ *
67
+ * Reads current root input dataset hashes from storage and compares
68
+ * against the snapshot stored in state. For each change, updates the
69
+ * input snapshot and version vectors, and emits input_changed events.
70
+ *
71
+ * @param storage - Storage backend
72
+ * @param state - Execution state to mutate (inputSnapshot, versionVectors)
73
+ * @returns Changes detected and events emitted
74
+ */
75
+ export declare function stepDetectInputChanges(storage: StorageBackend, state: DataflowExecutionState, cachedStructure?: Structure | null): Promise<{
76
+ changes: Array<{
77
+ path: string;
78
+ previousHash: string | null;
79
+ newHash: string;
80
+ }>;
81
+ events: ExecutionEvent[];
82
+ }>;
83
+ /**
84
+ * Invalidate tasks affected by input changes.
85
+ *
86
+ * For each affected task that is not currently in_progress:
87
+ * - completed: reset to pending, decrement executed/cached counter, emit task_invalidated
88
+ * - deferred: reset to pending
89
+ * - pending/ready: leave as-is (will pick up new inputs naturally)
90
+ * - skipped: leave as-is (upstream failure still applies)
91
+ * - failed: leave as-is (task already failed; orchestrator is winding down)
92
+ *
93
+ * @param state - Execution state to mutate
94
+ * @param changes - Input changes from stepDetectInputChanges
95
+ * @returns Invalidated task names and events
96
+ */
97
+ export declare function stepInvalidateTasks(state: DataflowExecutionState, changes: Array<{
98
+ path: string;
99
+ }>): {
100
+ invalidated: string[];
101
+ events: ExecutionEvent[];
102
+ };
103
+ /**
104
+ * Check version vector consistency for a task's inputs.
105
+ *
106
+ * All input version vectors must agree on shared keys (same root input path
107
+ * must have the same hash in every vector that contains it).
108
+ *
109
+ * @param state - Execution state
110
+ * @param taskName - Task to check
111
+ * @returns Consistency result with merged VV or conflict path
112
+ */
113
+ export declare function stepCheckVersionConsistency(state: DataflowExecutionState, taskName: string): {
114
+ consistent: true;
115
+ mergedVV: VersionVector;
116
+ } | {
117
+ consistent: false;
118
+ conflictPath: string;
119
+ };
120
+ /**
121
+ * Prepare a task for execution by resolving inputs and checking cache.
122
+ *
123
+ * This async operation:
124
+ * 1. Resolves input hashes from current workspace state
125
+ * 2. Checks if there's a valid cached execution
126
+ *
127
+ * @param storage - Storage backend
128
+ * @param state - Current execution state
129
+ * @param taskName - Name of the task to prepare
130
+ * @returns Preparation result with input hashes and cache status
131
+ */
132
+ export declare function stepPrepareTask(storage: StorageBackend, state: DataflowExecutionState, taskName: string): Promise<PrepareTaskResult>;
133
+ /**
134
+ * Mark a task as started (in-progress).
135
+ *
136
+ * Mutates the execution state to record that a task has begun execution.
137
+ *
138
+ * @param state - Execution state to mutate
139
+ * @param taskName - Name of the task
140
+ * @returns Event to record
141
+ */
142
+ export declare function stepTaskStarted(state: DataflowExecutionState, taskName: string): ExecutionEvent;
143
+ /**
144
+ * Mark a task as completed successfully.
145
+ *
146
+ * Mutates the execution state, computes the merged version vector for the
147
+ * task's output, and returns the newly ready tasks.
148
+ *
149
+ * @param state - Execution state to mutate
150
+ * @param taskName - Name of the task
151
+ * @param outputHash - Hash of the output dataset
152
+ * @param cached - Whether the result was from cache
153
+ * @param duration - Execution duration in milliseconds
154
+ * @returns Result with newly ready tasks and event
155
+ */
156
+ export declare function stepTaskCompleted(state: DataflowExecutionState, taskName: string, outputHash: string, cached: boolean, duration: number): {
157
+ result: TaskCompletedResult;
158
+ event: ExecutionEvent;
159
+ };
160
+ /**
161
+ * Mark a task as failed.
162
+ *
163
+ * Mutates the execution state and returns tasks that should be skipped.
164
+ *
165
+ * @param state - Execution state to mutate
166
+ * @param taskName - Name of the failed task
167
+ * @param error - Error message (optional)
168
+ * @param exitCode - Exit code if process failed (optional)
169
+ * @param duration - Execution duration in milliseconds
170
+ * @returns Result with tasks to skip and event
171
+ */
172
+ export declare function stepTaskFailed(state: DataflowExecutionState, taskName: string, error: string | undefined, exitCode: number | undefined, duration: number): {
173
+ result: TaskFailedResult;
174
+ event: ExecutionEvent;
175
+ };
176
+ /**
177
+ * Mark tasks as skipped due to upstream failure.
178
+ *
179
+ * @param state - Execution state to mutate
180
+ * @param taskNames - Names of tasks to skip
181
+ * @param cause - Name of the task that caused the skip
182
+ * @returns Array of events to record
183
+ */
184
+ export declare function stepTasksSkipped(state: DataflowExecutionState, taskNames: string[], cause: string): ExecutionEvent[];
185
+ /**
186
+ * Finalize the execution and return the result.
187
+ *
188
+ * Mutates the execution state to mark it as completed or failed.
189
+ *
190
+ * @param state - Execution state to mutate
191
+ * @param runId - Dataflow run ID (UUIDv7) from the orchestrator
192
+ * @returns Final result
193
+ */
194
+ export declare function stepFinalize(state: DataflowExecutionState, runId: string): {
195
+ result: FinalizeResult;
196
+ event: ExecutionEvent;
197
+ };
198
+ /**
199
+ * Cancel the execution.
200
+ *
201
+ * @param state - Execution state to mutate
202
+ * @param reason - Reason for cancellation
203
+ * @returns Event to record
204
+ */
205
+ export declare function stepCancel(state: DataflowExecutionState, reason?: string): ExecutionEvent;
206
+ /**
207
+ * Apply a task's output to the workspace tree with version vector.
208
+ *
209
+ * Writes the output ref file with the merged version vector from the task's
210
+ * inputs. Per-dataset ref writes are atomic and independent, so no
211
+ * serialization is needed for concurrent writes to different paths.
212
+ *
213
+ * @param storage - Storage backend
214
+ * @param repo - Repository identifier
215
+ * @param workspace - Workspace name
216
+ * @param outputPathStr - Output path as a keypath string (e.g., ".results.data")
217
+ * @param outputHash - Hash of the output dataset to write
218
+ * @param versions - Merged version vector for provenance tracking
219
+ * @returns Result indicating success
220
+ */
221
+ export declare function stepApplyTreeUpdate(storage: StorageBackend, repo: string, workspace: string, outputPathStr: string, outputHash: string, versions: VersionVector): Promise<TreeUpdateResult>;
222
+ //# sourceMappingURL=steps.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"steps.d.ts","sourceRoot":"","sources":["../../../src/dataflow/steps.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAoBH,OAAO,KAAK,EAAE,aAAa,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AAClE,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,0BAA0B,CAAC;AAqB/D,OAAO,KAAK,EACV,sBAAsB,EAEtB,gBAAgB,EAChB,iBAAiB,EACjB,mBAAmB,EACnB,gBAAgB,EAChB,cAAc,EACd,gBAAgB,EAChB,cAAc,EAEf,MAAM,YAAY,CAAC;AAMpB;;GAEG;AACH,MAAM,WAAW,qBAAqB;IACpC,sDAAsD;IACtD,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,yDAAyD;IACzD,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,wDAAwD;IACxD,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED;;;;;;;;;;;;;;;;GAgBG;AACH,wBAAsB,cAAc,CAClC,OAAO,EAAE,cAAc,EACvB,IAAI,EAAE,MAAM,EACZ,SAAS,EAAE,MAAM,EACjB,WAAW,EAAE,MAAM,EACnB,OAAO,GAAE,qBAA0B,GAClC,OAAO,CAAC,gBAAgB,CAAC,CA8F3B;AAoDD;;;;;;;;;;;;GAYG;AACH,wBAAgB,YAAY,CAAC,KAAK,EAAE,sBAAsB,GAAG,MAAM,EAAE,CAkCpE;AAED;;;;;;;;;;;GAWG;AACH,wBAAgB,cAAc,CAAC,KAAK,EAAE,sBAAsB,GAAG,OAAO,CAoCrE;AAMD;;;;;;;;;;GAUG;AACH,wBAAsB,sBAAsB,CAC1C,OAAO,EAAE,cAAc,EACvB,KAAK,EAAE,sBAAsB,EAC7B,eAAe,CAAC,EAAE,SAAS,GAAG,IAAI,GACjC,OAAO,CAAC;IACT,OAAO,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAC/E,MAAM,EAAE,cAAc,EAAE,CAAC;CAC1B,CAAC,CAiCD;AAED;;;;;;;;;;;;;GAaG;AACH,wBAAgB,mBAAmB,CACjC,KAAK,EAAE,sBAAsB,EAC7B,OAAO,EAAE,KAAK,CAAC;IAAE,IAAI,EAAE,MAAM,CAAA;CAAE,CAAC,GAC/B;IACD,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB,MAAM,EAAE,cAAc,EAAE,CAAC;CAC1B,CAwDA;AAED;;;;;;;;;GASG;AACH,wBAAgB,2BAA2B,CACzC,KAAK,EAAE,sBAAsB,EAC7B,QAAQ,EAAE,MAAM,GACf;IAAE,UAAU,EAAE,IAAI,CAAC;IAAC,QAAQ,EAAE,aAAa,CAAA;CAAE,GAAG;IAAE,UAAU,EAAE,KAAK,CAAC;IAAC,YAAY,EAAE,MAAM,CAAA;CAAE,CAkB7F;AAMD;;;;;;;;;;;GAWG;AACH,wBAAsB,eAAe,CACnC,OAAO,EAAE,cAAc,EACvB,KAAK,EAAE,sBAAsB,EAC7B,QAAQ,EAAE,MAAM,GACf,OAAO,CAAC,iBAAiB,CAAC,CA0D5B;AAMD;;;;;;;;GAQG;AACH,wBAAgB,eAAe,CAC7B,KAAK,EAAE,sBAAsB,EAC7B,QAAQ,EAAE,MAAM,GACf,cAAc,CAmBhB;AAED;;;;;;;;;;;;GAYG;AACH,wBAAgB,iBAAiB,CAC/B,KAAK,EAAE,sBAAsB,EAC7B,QAAQ,EAAE,MAAM,EAChB,UAAU,EAAE,MAAM,EAClB,MAAM,EAAE,OAAO,EACf,QAAQ,EAAE,MAAM,GACf;IAAE,MAAM,EAAE,mBAAmB,CAAC;IAAC,KAAK,EAAE,cAAc,CAAA;CAAE,CAwDxD;AAED;;;;;;;;;;;GAWG;AACH,wBAAgB,cAAc,CAC5B,KAAK,EAAE,sBAAsB,EAC7B,QAAQ,EAAE,MAAM,EAChB,KAAK,EAAE,MAAM,GAAG,SAAS,EACzB,QAAQ,EAAE,MAAM,GAAG,SAAS,EAC5B,QAAQ,EAAE,MAAM,GACf;IAAE,MAAM,EAAE,gBAAgB,CAAC;IAAC,KAAK,EAAE,cAAc,CAAA;CAAE,CAsDrD;AAED;;;;;;;GAOG;AACH,wBAAgB,gBAAgB,CAC9B,KAAK,EAAE,sBAAsB,EAC7B,SAAS,EAAE,MAAM,EAAE,EACnB,KAAK,EAAE,MAAM,GACZ,cAAc,EAAE,CA2BlB;AAED;;;;;;;;GAQG;AACH,wBAAgB,YAAY,CAAC,KAAK,EAAE,sBAAsB,EAAE,KAAK,EAAE,MAAM,GAAG;IAC1E,MAAM,EAAE,cAAc,CAAC;IACvB,KAAK,EAAE,cAAc,CAAC;CACvB,CAsCA;AAED;;;;;;GAMG;AACH,wBAAgB,UAAU,CACxB,KAAK,EAAE,sBAAsB,EAC7B,MAAM,CAAC,EAAE,MAAM,GACd,cAAc,CAgBhB;AAMD;;;;;;;;;;;;;;GAcG;AACH,wBAAsB,mBAAmB,CACvC,OAAO,EAAE,cAAc,EACvB,IAAI,EAAE,MAAM,EACZ,SAAS,EAAE,MAAM,EACjB,aAAa,EAAE,MAAM,EACrB,UAAU,EAAE,MAAM,EAClB,QAAQ,EAAE,aAAa,GACtB,OAAO,CAAC,gBAAgB,CAAC,CAe3B"}