@elaraai/e3-core 0.0.2-beta.4 → 0.0.2-beta.40

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (208) hide show
  1. package/README.md +25 -22
  2. package/dist/src/dataflow/api-compat.d.ts +90 -0
  3. package/dist/src/dataflow/api-compat.d.ts.map +1 -0
  4. package/dist/src/dataflow/api-compat.js +139 -0
  5. package/dist/src/dataflow/api-compat.js.map +1 -0
  6. package/dist/src/dataflow/index.d.ts +18 -0
  7. package/dist/src/dataflow/index.d.ts.map +1 -0
  8. package/dist/src/dataflow/index.js +23 -0
  9. package/dist/src/dataflow/index.js.map +1 -0
  10. package/dist/src/dataflow/orchestrator/LocalOrchestrator.d.ts +76 -0
  11. package/dist/src/dataflow/orchestrator/LocalOrchestrator.d.ts.map +1 -0
  12. package/dist/src/dataflow/orchestrator/LocalOrchestrator.js +695 -0
  13. package/dist/src/dataflow/orchestrator/LocalOrchestrator.js.map +1 -0
  14. package/dist/src/dataflow/orchestrator/index.d.ts +12 -0
  15. package/dist/src/dataflow/orchestrator/index.d.ts.map +1 -0
  16. package/dist/src/dataflow/orchestrator/index.js +12 -0
  17. package/dist/src/dataflow/orchestrator/index.js.map +1 -0
  18. package/dist/src/dataflow/orchestrator/interfaces.d.ts +163 -0
  19. package/dist/src/dataflow/orchestrator/interfaces.d.ts.map +1 -0
  20. package/dist/src/dataflow/orchestrator/interfaces.js +52 -0
  21. package/dist/src/dataflow/orchestrator/interfaces.js.map +1 -0
  22. package/dist/src/dataflow/state-store/FileStateStore.d.ts +67 -0
  23. package/dist/src/dataflow/state-store/FileStateStore.d.ts.map +1 -0
  24. package/dist/src/dataflow/state-store/FileStateStore.js +300 -0
  25. package/dist/src/dataflow/state-store/FileStateStore.js.map +1 -0
  26. package/dist/src/dataflow/state-store/InMemoryStateStore.d.ts +42 -0
  27. package/dist/src/dataflow/state-store/InMemoryStateStore.d.ts.map +1 -0
  28. package/dist/src/dataflow/state-store/InMemoryStateStore.js +229 -0
  29. package/dist/src/dataflow/state-store/InMemoryStateStore.js.map +1 -0
  30. package/dist/src/dataflow/state-store/index.d.ts +13 -0
  31. package/dist/src/dataflow/state-store/index.d.ts.map +1 -0
  32. package/dist/src/dataflow/state-store/index.js +13 -0
  33. package/dist/src/dataflow/state-store/index.js.map +1 -0
  34. package/dist/src/dataflow/state-store/interfaces.d.ts +159 -0
  35. package/dist/src/dataflow/state-store/interfaces.d.ts.map +1 -0
  36. package/dist/src/dataflow/state-store/interfaces.js +6 -0
  37. package/dist/src/dataflow/state-store/interfaces.js.map +1 -0
  38. package/dist/src/dataflow/steps.d.ts +222 -0
  39. package/dist/src/dataflow/steps.d.ts.map +1 -0
  40. package/dist/src/dataflow/steps.js +707 -0
  41. package/dist/src/dataflow/steps.js.map +1 -0
  42. package/dist/src/dataflow/types.d.ts +127 -0
  43. package/dist/src/dataflow/types.d.ts.map +1 -0
  44. package/dist/src/dataflow/types.js +7 -0
  45. package/dist/src/dataflow/types.js.map +1 -0
  46. package/dist/src/dataflow.d.ts +113 -38
  47. package/dist/src/dataflow.d.ts.map +1 -1
  48. package/dist/src/dataflow.js +269 -416
  49. package/dist/src/dataflow.js.map +1 -1
  50. package/dist/src/dataset-refs.d.ts +124 -0
  51. package/dist/src/dataset-refs.d.ts.map +1 -0
  52. package/dist/src/dataset-refs.js +319 -0
  53. package/dist/src/dataset-refs.js.map +1 -0
  54. package/dist/src/errors.d.ts +39 -9
  55. package/dist/src/errors.d.ts.map +1 -1
  56. package/dist/src/errors.js +51 -8
  57. package/dist/src/errors.js.map +1 -1
  58. package/dist/src/execution/LocalTaskRunner.d.ts +73 -0
  59. package/dist/src/execution/LocalTaskRunner.d.ts.map +1 -0
  60. package/dist/src/execution/LocalTaskRunner.js +399 -0
  61. package/dist/src/execution/LocalTaskRunner.js.map +1 -0
  62. package/dist/src/execution/MockTaskRunner.d.ts +49 -0
  63. package/dist/src/execution/MockTaskRunner.d.ts.map +1 -0
  64. package/dist/src/execution/MockTaskRunner.js +54 -0
  65. package/dist/src/execution/MockTaskRunner.js.map +1 -0
  66. package/dist/src/execution/index.d.ts +16 -0
  67. package/dist/src/execution/index.d.ts.map +1 -0
  68. package/dist/src/execution/index.js +8 -0
  69. package/dist/src/execution/index.js.map +1 -0
  70. package/dist/src/execution/interfaces.d.ts +246 -0
  71. package/dist/src/execution/interfaces.d.ts.map +1 -0
  72. package/dist/src/execution/interfaces.js +6 -0
  73. package/dist/src/execution/interfaces.js.map +1 -0
  74. package/dist/src/execution/processHelpers.d.ts +20 -0
  75. package/dist/src/execution/processHelpers.d.ts.map +1 -0
  76. package/dist/src/execution/processHelpers.js +62 -0
  77. package/dist/src/execution/processHelpers.js.map +1 -0
  78. package/dist/src/executions.d.ts +71 -104
  79. package/dist/src/executions.d.ts.map +1 -1
  80. package/dist/src/executions.js +110 -476
  81. package/dist/src/executions.js.map +1 -1
  82. package/dist/src/index.d.ts +19 -9
  83. package/dist/src/index.d.ts.map +1 -1
  84. package/dist/src/index.js +48 -18
  85. package/dist/src/index.js.map +1 -1
  86. package/dist/src/objects.d.ts +8 -51
  87. package/dist/src/objects.d.ts.map +1 -1
  88. package/dist/src/objects.js +13 -230
  89. package/dist/src/objects.js.map +1 -1
  90. package/dist/src/packages.d.ts +22 -14
  91. package/dist/src/packages.d.ts.map +1 -1
  92. package/dist/src/packages.js +134 -88
  93. package/dist/src/packages.js.map +1 -1
  94. package/dist/src/storage/in-memory/InMemoryRepoStore.d.ts +35 -0
  95. package/dist/src/storage/in-memory/InMemoryRepoStore.d.ts.map +1 -0
  96. package/dist/src/storage/in-memory/InMemoryRepoStore.js +107 -0
  97. package/dist/src/storage/in-memory/InMemoryRepoStore.js.map +1 -0
  98. package/dist/src/storage/in-memory/InMemoryStorage.d.ts +139 -0
  99. package/dist/src/storage/in-memory/InMemoryStorage.d.ts.map +1 -0
  100. package/dist/src/storage/in-memory/InMemoryStorage.js +439 -0
  101. package/dist/src/storage/in-memory/InMemoryStorage.js.map +1 -0
  102. package/dist/src/storage/in-memory/index.d.ts +12 -0
  103. package/dist/src/storage/in-memory/index.d.ts.map +1 -0
  104. package/dist/src/storage/in-memory/index.js +12 -0
  105. package/dist/src/storage/in-memory/index.js.map +1 -0
  106. package/dist/src/storage/index.d.ts +18 -0
  107. package/dist/src/storage/index.d.ts.map +1 -0
  108. package/dist/src/storage/index.js +10 -0
  109. package/dist/src/storage/index.js.map +1 -0
  110. package/dist/src/storage/interfaces.d.ts +581 -0
  111. package/dist/src/storage/interfaces.d.ts.map +1 -0
  112. package/dist/src/storage/interfaces.js +6 -0
  113. package/dist/src/storage/interfaces.js.map +1 -0
  114. package/dist/src/storage/local/LocalBackend.d.ts +56 -0
  115. package/dist/src/storage/local/LocalBackend.d.ts.map +1 -0
  116. package/dist/src/storage/local/LocalBackend.js +145 -0
  117. package/dist/src/storage/local/LocalBackend.js.map +1 -0
  118. package/dist/src/storage/local/LocalDatasetRefStore.d.ts +22 -0
  119. package/dist/src/storage/local/LocalDatasetRefStore.d.ts.map +1 -0
  120. package/dist/src/storage/local/LocalDatasetRefStore.js +118 -0
  121. package/dist/src/storage/local/LocalDatasetRefStore.js.map +1 -0
  122. package/dist/src/storage/local/LocalLockService.d.ts +111 -0
  123. package/dist/src/storage/local/LocalLockService.d.ts.map +1 -0
  124. package/dist/src/storage/local/LocalLockService.js +355 -0
  125. package/dist/src/storage/local/LocalLockService.js.map +1 -0
  126. package/dist/src/storage/local/LocalLogStore.d.ts +23 -0
  127. package/dist/src/storage/local/LocalLogStore.d.ts.map +1 -0
  128. package/dist/src/storage/local/LocalLogStore.js +66 -0
  129. package/dist/src/storage/local/LocalLogStore.js.map +1 -0
  130. package/dist/src/storage/local/LocalObjectStore.d.ts +55 -0
  131. package/dist/src/storage/local/LocalObjectStore.d.ts.map +1 -0
  132. package/dist/src/storage/local/LocalObjectStore.js +300 -0
  133. package/dist/src/storage/local/LocalObjectStore.js.map +1 -0
  134. package/dist/src/storage/local/LocalRefStore.d.ts +50 -0
  135. package/dist/src/storage/local/LocalRefStore.d.ts.map +1 -0
  136. package/dist/src/storage/local/LocalRefStore.js +337 -0
  137. package/dist/src/storage/local/LocalRefStore.js.map +1 -0
  138. package/dist/src/storage/local/LocalRepoStore.d.ts +55 -0
  139. package/dist/src/storage/local/LocalRepoStore.d.ts.map +1 -0
  140. package/dist/src/storage/local/LocalRepoStore.js +365 -0
  141. package/dist/src/storage/local/LocalRepoStore.js.map +1 -0
  142. package/dist/src/storage/local/gc.d.ts +92 -0
  143. package/dist/src/storage/local/gc.d.ts.map +1 -0
  144. package/dist/src/storage/local/gc.js +377 -0
  145. package/dist/src/storage/local/gc.js.map +1 -0
  146. package/dist/src/storage/local/index.d.ts +18 -0
  147. package/dist/src/storage/local/index.d.ts.map +1 -0
  148. package/dist/src/storage/local/index.js +18 -0
  149. package/dist/src/storage/local/index.js.map +1 -0
  150. package/dist/src/storage/local/localHelpers.d.ts +25 -0
  151. package/dist/src/storage/local/localHelpers.d.ts.map +1 -0
  152. package/dist/src/storage/local/localHelpers.js +69 -0
  153. package/dist/src/storage/local/localHelpers.js.map +1 -0
  154. package/dist/src/{repository.d.ts → storage/local/repository.d.ts} +8 -4
  155. package/dist/src/storage/local/repository.d.ts.map +1 -0
  156. package/dist/src/{repository.js → storage/local/repository.js} +31 -29
  157. package/dist/src/storage/local/repository.js.map +1 -0
  158. package/dist/src/tasks.d.ts +16 -10
  159. package/dist/src/tasks.d.ts.map +1 -1
  160. package/dist/src/tasks.js +35 -41
  161. package/dist/src/tasks.js.map +1 -1
  162. package/dist/src/test-helpers.d.ts +5 -4
  163. package/dist/src/test-helpers.d.ts.map +1 -1
  164. package/dist/src/test-helpers.js +9 -21
  165. package/dist/src/test-helpers.js.map +1 -1
  166. package/dist/src/transfer/InMemoryTransferBackend.d.ts +66 -0
  167. package/dist/src/transfer/InMemoryTransferBackend.d.ts.map +1 -0
  168. package/dist/src/transfer/InMemoryTransferBackend.js +166 -0
  169. package/dist/src/transfer/InMemoryTransferBackend.js.map +1 -0
  170. package/dist/src/transfer/index.d.ts +8 -0
  171. package/dist/src/transfer/index.d.ts.map +1 -0
  172. package/dist/src/transfer/index.js +9 -0
  173. package/dist/src/transfer/index.js.map +1 -0
  174. package/dist/src/transfer/interfaces.d.ts +103 -0
  175. package/dist/src/transfer/interfaces.d.ts.map +1 -0
  176. package/dist/src/transfer/interfaces.js +6 -0
  177. package/dist/src/transfer/interfaces.js.map +1 -0
  178. package/dist/src/transfer/types.d.ts +79 -0
  179. package/dist/src/transfer/types.d.ts.map +1 -0
  180. package/dist/src/transfer/types.js +58 -0
  181. package/dist/src/transfer/types.js.map +1 -0
  182. package/dist/src/trees.d.ts +147 -59
  183. package/dist/src/trees.d.ts.map +1 -1
  184. package/dist/src/trees.js +372 -419
  185. package/dist/src/trees.js.map +1 -1
  186. package/dist/src/uuid.d.ts +26 -0
  187. package/dist/src/uuid.d.ts.map +1 -0
  188. package/dist/src/uuid.js +80 -0
  189. package/dist/src/uuid.js.map +1 -0
  190. package/dist/src/workspaceStatus.d.ts +6 -4
  191. package/dist/src/workspaceStatus.d.ts.map +1 -1
  192. package/dist/src/workspaceStatus.js +43 -49
  193. package/dist/src/workspaceStatus.js.map +1 -1
  194. package/dist/src/workspaces.d.ts +35 -47
  195. package/dist/src/workspaces.d.ts.map +1 -1
  196. package/dist/src/workspaces.js +194 -156
  197. package/dist/src/workspaces.js.map +1 -1
  198. package/package.json +4 -4
  199. package/dist/src/gc.d.ts +0 -54
  200. package/dist/src/gc.d.ts.map +0 -1
  201. package/dist/src/gc.js +0 -233
  202. package/dist/src/gc.js.map +0 -1
  203. package/dist/src/repository.d.ts.map +0 -1
  204. package/dist/src/repository.js.map +0 -1
  205. package/dist/src/workspaceLock.d.ts +0 -67
  206. package/dist/src/workspaceLock.d.ts.map +0 -1
  207. package/dist/src/workspaceLock.js +0 -217
  208. package/dist/src/workspaceLock.js.map +0 -1
@@ -0,0 +1,54 @@
1
+ /**
2
+ * Copyright (c) 2025 Elara AI Pty Ltd
3
+ * Licensed under BSL 1.1. See LICENSE for details.
4
+ */
5
+ /**
6
+ * TaskRunner mock for testing dataflow orchestration without spawning processes.
7
+ *
8
+ * Allows configuring responses per task and records all calls for assertions.
9
+ */
10
+ export class MockTaskRunner {
11
+ results = new Map();
12
+ calls = [];
13
+ defaultResult = { state: 'success', cached: false, outputHash: 'mock-hash' };
14
+ /**
15
+ * Set result for a specific task hash.
16
+ *
17
+ * @param taskHash - The task hash to configure
18
+ * @param result - Either a static TaskResult or a function that computes result from inputHashes
19
+ */
20
+ setResult(taskHash, result) {
21
+ this.results.set(taskHash, result);
22
+ }
23
+ /**
24
+ * Set default result for tasks without specific results configured.
25
+ *
26
+ * @param result - The default TaskResult to return
27
+ */
28
+ setDefaultResult(result) {
29
+ this.defaultResult = result;
30
+ }
31
+ /**
32
+ * Get all recorded calls.
33
+ *
34
+ * @returns Readonly array of all execute() calls
35
+ */
36
+ getCalls() {
37
+ return this.calls;
38
+ }
39
+ /**
40
+ * Clear recorded calls.
41
+ */
42
+ clearCalls() {
43
+ this.calls = [];
44
+ }
45
+ async execute(_storage, taskHash, inputHashes, options) {
46
+ this.calls.push({ taskHash, inputHashes, options });
47
+ const configured = this.results.get(taskHash);
48
+ if (configured) {
49
+ return typeof configured === 'function' ? await configured(inputHashes) : configured;
50
+ }
51
+ return this.defaultResult;
52
+ }
53
+ }
54
+ //# sourceMappingURL=MockTaskRunner.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"MockTaskRunner.js","sourceRoot":"","sources":["../../../src/execution/MockTaskRunner.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAcH;;;;GAIG;AACH,MAAM,OAAO,cAAc;IACjB,OAAO,GAAG,IAAI,GAAG,EAAsF,CAAC;IACxG,KAAK,GAAmB,EAAE,CAAC;IAC3B,aAAa,GAAe,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,EAAE,KAAK,EAAE,UAAU,EAAE,WAAW,EAAE,CAAC;IAEjG;;;;;OAKG;IACH,SAAS,CAAC,QAAgB,EAAE,MAAkF;QAC5G,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;IACrC,CAAC;IAED;;;;OAIG;IACH,gBAAgB,CAAC,MAAkB;QACjC,IAAI,CAAC,aAAa,GAAG,MAAM,CAAC;IAC9B,CAAC;IAED;;;;OAIG;IACH,QAAQ;QACN,OAAO,IAAI,CAAC,KAAK,CAAC;IACpB,CAAC;IAED;;OAEG;IACH,UAAU;QACR,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC;IAClB,CAAC;IAED,KAAK,CAAC,OAAO,CACX,QAAwB,EACxB,QAAgB,EAChB,WAAqB,EACrB,OAA4B;QAE5B,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,WAAW,EAAE,OAAO,EAAE,CAAC,CAAC;QAEpD,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAC9C,IAAI,UAAU,EAAE,CAAC;YACf,OAAO,OAAO,UAAU,KAAK,UAAU,CAAC,CAAC,CAAC,MAAM,UAAU,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC;QACvF,CAAC;QACD,OAAO,IAAI,CAAC,aAAa,CAAC;IAC5B,CAAC;CACF"}
@@ -0,0 +1,16 @@
1
+ /**
2
+ * Copyright (c) 2025 Elara AI Pty Ltd
3
+ * Licensed under BSL 1.1. See LICENSE for details.
4
+ */
5
+ /**
6
+ * Execution abstraction layer for e3 dataflow.
7
+ *
8
+ * This module provides interfaces that separate orchestration from business
9
+ * logic, enabling different execution strategies:
10
+ * - LocalDataflowExecutor: In-process execution with AsyncMutex (CLI, local dev)
11
+ * - StepFunctionsDataflowExecutor: AWS Step Functions orchestration (cloud)
12
+ */
13
+ export { type TaskExecuteOptions, type TaskResult, type TaskRunner, type ExecutionHandle, type DataflowStatus, type DataflowExecuteOptions, type DataflowExecuteResult, type DataflowExecutor, type TaskGraph, type DataflowGetGraphFn, type DataflowCheckCacheFn, type DataflowWriteOutputFn, type DataflowGetReadyTasksFn, } from './interfaces.js';
14
+ export { LocalTaskRunner } from './LocalTaskRunner.js';
15
+ export { MockTaskRunner, type MockTaskCall } from './MockTaskRunner.js';
16
+ //# sourceMappingURL=index.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/execution/index.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH;;;;;;;GAOG;AAEH,OAAO,EAEL,KAAK,kBAAkB,EACvB,KAAK,UAAU,EACf,KAAK,UAAU,EAEf,KAAK,eAAe,EACpB,KAAK,cAAc,EACnB,KAAK,sBAAsB,EAC3B,KAAK,qBAAqB,EAC1B,KAAK,gBAAgB,EAErB,KAAK,SAAS,EAEd,KAAK,kBAAkB,EACvB,KAAK,oBAAoB,EACzB,KAAK,qBAAqB,EAC1B,KAAK,uBAAuB,GAC7B,MAAM,iBAAiB,CAAC;AAGzB,OAAO,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AACvD,OAAO,EAAE,cAAc,EAAE,KAAK,YAAY,EAAE,MAAM,qBAAqB,CAAC"}
@@ -0,0 +1,8 @@
1
+ /**
2
+ * Copyright (c) 2025 Elara AI Pty Ltd
3
+ * Licensed under BSL 1.1. See LICENSE for details.
4
+ */
5
+ // TaskRunner implementations
6
+ export { LocalTaskRunner } from './LocalTaskRunner.js';
7
+ export { MockTaskRunner } from './MockTaskRunner.js';
8
+ //# sourceMappingURL=index.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/execution/index.ts"],"names":[],"mappings":"AAAA;;;GAGG;AA+BH,6BAA6B;AAC7B,OAAO,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AACvD,OAAO,EAAE,cAAc,EAAqB,MAAM,qBAAqB,CAAC"}
@@ -0,0 +1,246 @@
1
+ /**
2
+ * Copyright (c) 2025 Elara AI Pty Ltd
3
+ * Licensed under BSL 1.1. See LICENSE for details.
4
+ */
5
+ /**
6
+ * Execution abstraction interfaces for e3 dataflow.
7
+ *
8
+ * These interfaces separate orchestration from business logic, enabling:
9
+ * - LocalDataflowExecutor: In-process execution with AsyncMutex (CLI, local dev)
10
+ * - StepFunctionsDataflowExecutor: AWS Step Functions orchestration (cloud)
11
+ *
12
+ * The core insight: Everything in dataflowExecute() except the processQueue()
13
+ * loop is pure business logic that both local and cloud execution share.
14
+ * By extracting these as functions and abstracting the orchestration,
15
+ * Step Functions can replace the local loop while reusing all e3-core logic.
16
+ */
17
+ import type { StorageBackend, LockHandle } from '../storage/interfaces.js';
18
+ /**
19
+ * Options for task execution.
20
+ */
21
+ export interface TaskExecuteOptions {
22
+ /** Force execution even if cached */
23
+ force?: boolean;
24
+ /** AbortSignal for cancellation */
25
+ signal?: AbortSignal;
26
+ /** Callback for stdout data */
27
+ onStdout?: (data: string) => void;
28
+ /** Callback for stderr data */
29
+ onStderr?: (data: string) => void;
30
+ }
31
+ /**
32
+ * Result of a single task execution.
33
+ */
34
+ export interface TaskResult {
35
+ /** Final state */
36
+ state: 'success' | 'failed' | 'error';
37
+ /** Whether the result was served from cache */
38
+ cached: boolean;
39
+ /** Execution ID (UUIDv7) */
40
+ executionId?: string;
41
+ /** Output hash (if state is 'success') */
42
+ outputHash?: string;
43
+ /** Exit code (if state is 'failed') */
44
+ exitCode?: number;
45
+ /** Error message (if state is 'error') */
46
+ error?: string;
47
+ }
48
+ /**
49
+ * Task execution abstraction.
50
+ *
51
+ * Implementations:
52
+ * - LocalTaskRunner: Spawns east-node/east-py/julia processes locally
53
+ * - LambdaTaskRunner: Dispatches to AWS Lambda
54
+ * - FargateTaskRunner: Dispatches to AWS Fargate
55
+ */
56
+ export interface TaskRunner {
57
+ /**
58
+ * Execute a task.
59
+ *
60
+ * @param storage - Storage backend
61
+ * @param taskHash - Hash of the TaskObject
62
+ * @param inputHashes - Hashes of input datasets
63
+ * @param options - Execution options
64
+ * @returns Task result
65
+ */
66
+ execute(storage: StorageBackend, taskHash: string, inputHashes: string[], options?: TaskExecuteOptions): Promise<TaskResult>;
67
+ }
68
+ /**
69
+ * Handle to a running dataflow execution.
70
+ */
71
+ export interface ExecutionHandle {
72
+ /** Unique execution ID (Local: UUID, Cloud: Step Functions ARN) */
73
+ readonly id: string;
74
+ }
75
+ /**
76
+ * Status of a dataflow execution.
77
+ */
78
+ export interface DataflowStatus {
79
+ /** Current state */
80
+ state: 'running' | 'completed' | 'failed' | 'cancelled';
81
+ /** Tasks that have completed successfully */
82
+ completed: string[];
83
+ /** Tasks currently running */
84
+ running: string[];
85
+ /** Tasks waiting to run */
86
+ pending: string[];
87
+ /** Tasks that failed */
88
+ failed: string[];
89
+ /** Tasks skipped due to upstream failure */
90
+ skipped: string[];
91
+ /** Error message if state is 'failed' */
92
+ error?: string;
93
+ }
94
+ /**
95
+ * Options for dataflow execution.
96
+ */
97
+ export interface DataflowExecuteOptions {
98
+ /** Maximum concurrent task executions (default: 4) */
99
+ concurrency?: number;
100
+ /** Force re-execution even if cached (default: false) */
101
+ force?: boolean;
102
+ /** Filter to run only specific task(s) by exact name */
103
+ filter?: string;
104
+ /** AbortSignal for cancellation */
105
+ signal?: AbortSignal;
106
+ /** External lock handle (if caller manages locking) */
107
+ lock?: LockHandle;
108
+ /** Callback when a task starts */
109
+ onTaskStart?: (taskName: string) => void;
110
+ /** Callback when a task completes */
111
+ onTaskComplete?: (taskName: string, result: TaskResult) => void;
112
+ /** Callback for task stdout */
113
+ onStdout?: (taskName: string, data: string) => void;
114
+ /** Callback for task stderr */
115
+ onStderr?: (taskName: string, data: string) => void;
116
+ }
117
+ /**
118
+ * Result of a dataflow execution.
119
+ */
120
+ export interface DataflowExecuteResult {
121
+ /** Overall success - true if all tasks completed successfully */
122
+ success: boolean;
123
+ /** Number of tasks executed (not from cache) */
124
+ executed: number;
125
+ /** Number of tasks served from cache */
126
+ cached: number;
127
+ /** Number of tasks that failed */
128
+ failed: number;
129
+ /** Number of tasks skipped due to upstream failure */
130
+ skipped: number;
131
+ /** Total duration in milliseconds */
132
+ duration: number;
133
+ }
134
+ /**
135
+ * Dataflow orchestration abstraction.
136
+ *
137
+ * Implementations:
138
+ * - LocalDataflowExecutor: In-process loop with AsyncMutex
139
+ * - StepFunctionsDataflowExecutor: AWS Step Functions state machine
140
+ */
141
+ export interface DataflowExecutor {
142
+ /**
143
+ * Start a dataflow execution.
144
+ *
145
+ * Returns immediately with a handle. Use getStatus() to poll for completion.
146
+ *
147
+ * @param storage - Storage backend
148
+ * @param workspace - Workspace name
149
+ * @param options - Execution options
150
+ * @returns Execution handle
151
+ */
152
+ start(storage: StorageBackend, workspace: string, options?: DataflowExecuteOptions): Promise<ExecutionHandle>;
153
+ /**
154
+ * Get the status of a dataflow execution.
155
+ *
156
+ * @param handle - Execution handle from start()
157
+ * @returns Current status
158
+ */
159
+ getStatus(handle: ExecutionHandle): Promise<DataflowStatus>;
160
+ /**
161
+ * Cancel a running dataflow execution.
162
+ *
163
+ * Running tasks will be terminated. The execution will transition to
164
+ * 'cancelled' state.
165
+ *
166
+ * @param handle - Execution handle from start()
167
+ */
168
+ cancel(handle: ExecutionHandle): Promise<void>;
169
+ /**
170
+ * Wait for a dataflow execution to complete.
171
+ *
172
+ * @param handle - Execution handle from start()
173
+ * @returns Final result
174
+ */
175
+ wait(handle: ExecutionHandle): Promise<DataflowExecuteResult>;
176
+ }
177
+ /**
178
+ * Task dependency graph.
179
+ */
180
+ export interface TaskGraph {
181
+ /** Map of task name -> task info */
182
+ tasks: Map<string, {
183
+ hash: string;
184
+ inputPaths: string[];
185
+ outputPath: string;
186
+ dependencies: string[];
187
+ }>;
188
+ /** Map of output path -> producing task name */
189
+ outputToTask: Map<string, string>;
190
+ }
191
+ /**
192
+ * These functions are the shared business logic that both local and cloud
193
+ * execution use. They are called by the orchestrator (LocalDataflowExecutor
194
+ * or Step Functions via Lambda handlers).
195
+ *
196
+ * Note: These are defined as function signatures here for documentation.
197
+ * The actual implementations are in dataflow.ts and will be exported
198
+ * as standalone functions.
199
+ */
200
+ /**
201
+ * Get the task dependency graph for a workspace.
202
+ *
203
+ * Pure function - reads workspace state and package to build DAG.
204
+ *
205
+ * @param storage - Storage backend
206
+ * @param workspace - Workspace name
207
+ * @returns Task dependency graph
208
+ */
209
+ export type DataflowGetGraphFn = (storage: StorageBackend, workspace: string) => Promise<TaskGraph>;
210
+ /**
211
+ * Check if a task's output is cached.
212
+ *
213
+ * Returns the cached output hash if available and valid.
214
+ *
215
+ * @param storage - Storage backend
216
+ * @param workspace - Workspace name
217
+ * @param taskHash - Task object hash
218
+ * @param inputHashes - Input dataset hashes
219
+ * @returns Cached output hash, or null if not cached or invalid
220
+ */
221
+ export type DataflowCheckCacheFn = (storage: StorageBackend, workspace: string, taskHash: string, inputHashes: string[]) => Promise<string | null>;
222
+ /**
223
+ * Write task output to workspace tree.
224
+ *
225
+ * Called after successful task execution to update the workspace.
226
+ *
227
+ * @param storage - Storage backend
228
+ * @param workspace - Workspace name
229
+ * @param taskName - Task name
230
+ * @param outputPath - Output dataset path
231
+ * @param outputHash - Output object hash
232
+ */
233
+ export type DataflowWriteOutputFn = (storage: StorageBackend, workspace: string, taskName: string, outputPath: string, outputHash: string) => Promise<void>;
234
+ /**
235
+ * Get tasks that are ready to execute.
236
+ *
237
+ * A task is ready when all its input dependencies have values assigned.
238
+ *
239
+ * @param storage - Storage backend
240
+ * @param workspace - Workspace name
241
+ * @param graph - Task dependency graph
242
+ * @param completed - Set of completed task names
243
+ * @returns Array of task names ready to execute
244
+ */
245
+ export type DataflowGetReadyTasksFn = (storage: StorageBackend, workspace: string, graph: TaskGraph, completed: Set<string>) => Promise<string[]>;
246
+ //# sourceMappingURL=interfaces.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"interfaces.d.ts","sourceRoot":"","sources":["../../../src/execution/interfaces.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH;;;;;;;;;;;GAWG;AAEH,OAAO,KAAK,EAAE,cAAc,EAAE,UAAU,EAAE,MAAM,0BAA0B,CAAC;AAM3E;;GAEG;AACH,MAAM,WAAW,kBAAkB;IACjC,qCAAqC;IACrC,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,mCAAmC;IACnC,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,+BAA+B;IAC/B,QAAQ,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,CAAC;IAClC,+BAA+B;IAC/B,QAAQ,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,CAAC;CACnC;AAED;;GAEG;AACH,MAAM,WAAW,UAAU;IACzB,kBAAkB;IAClB,KAAK,EAAE,SAAS,GAAG,QAAQ,GAAG,OAAO,CAAC;IACtC,+CAA+C;IAC/C,MAAM,EAAE,OAAO,CAAC;IAChB,4BAA4B;IAC5B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,0CAA0C;IAC1C,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,uCAAuC;IACvC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,0CAA0C;IAC1C,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED;;;;;;;GAOG;AACH,MAAM,WAAW,UAAU;IACzB;;;;;;;;OAQG;IACH,OAAO,CACL,OAAO,EAAE,cAAc,EACvB,QAAQ,EAAE,MAAM,EAChB,WAAW,EAAE,MAAM,EAAE,EACrB,OAAO,CAAC,EAAE,kBAAkB,GAC3B,OAAO,CAAC,UAAU,CAAC,CAAC;CACxB;AAMD;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,mEAAmE;IACnE,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,oBAAoB;IACpB,KAAK,EAAE,SAAS,GAAG,WAAW,GAAG,QAAQ,GAAG,WAAW,CAAC;IACxD,6CAA6C;IAC7C,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,8BAA8B;IAC9B,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,2BAA2B;IAC3B,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,wBAAwB;IACxB,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,4CAA4C;IAC5C,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,yCAAyC;IACzC,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED;;GAEG;AACH,MAAM,WAAW,sBAAsB;IACrC,sDAAsD;IACtD,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,yDAAyD;IACzD,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,wDAAwD;IACxD,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,mCAAmC;IACnC,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,uDAAuD;IACvD,IAAI,CAAC,EAAE,UAAU,CAAC;IAClB,kCAAkC;IAClC,WAAW,CAAC,EAAE,CAAC,QAAQ,EAAE,MAAM,KAAK,IAAI,CAAC;IACzC,qCAAqC;IACrC,cAAc,CAAC,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,KAAK,IAAI,CAAC;IAChE,+BAA+B;IAC/B,QAAQ,CAAC,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,KAAK,IAAI,CAAC;IACpD,+BAA+B;IAC/B,QAAQ,CAAC,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,KAAK,IAAI,CAAC;CACrD;AAED;;GAEG;AACH,MAAM,WAAW,qBAAqB;IACpC,iEAAiE;IACjE,OAAO,EAAE,OAAO,CAAC;IACjB,gDAAgD;IAChD,QAAQ,EAAE,MAAM,CAAC;IACjB,wCAAwC;IACxC,MAAM,EAAE,MAAM,CAAC;IACf,kCAAkC;IAClC,MAAM,EAAE,MAAM,CAAC;IACf,sDAAsD;IACtD,OAAO,EAAE,MAAM,CAAC;IAChB,qCAAqC;IACrC,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED;;;;;;GAMG;AACH,MAAM,WAAW,gBAAgB;IAC/B;;;;;;;;;OASG;IACH,KAAK,CACH,OAAO,EAAE,cAAc,EACvB,SAAS,EAAE,MAAM,EACjB,OAAO,CAAC,EAAE,sBAAsB,GAC/B,OAAO,CAAC,eAAe,CAAC,CAAC;IAE5B;;;;;OAKG;IACH,SAAS,CAAC,MAAM,EAAE,eAAe,GAAG,OAAO,CAAC,cAAc,CAAC,CAAC;IAE5D;;;;;;;OAOG;IACH,MAAM,CAAC,MAAM,EAAE,eAAe,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAE/C;;;;;OAKG;IACH,IAAI,CAAC,MAAM,EAAE,eAAe,GAAG,OAAO,CAAC,qBAAqB,CAAC,CAAC;CAC/D;AAMD;;GAEG;AACH,MAAM,WAAW,SAAS;IACxB,oCAAoC;IACpC,KAAK,EAAE,GAAG,CAAC,MAAM,EAAE;QACjB,IAAI,EAAE,MAAM,CAAC;QACb,UAAU,EAAE,MAAM,EAAE,CAAC;QACrB,UAAU,EAAE,MAAM,CAAC;QACnB,YAAY,EAAE,MAAM,EAAE,CAAC;KACxB,CAAC,CAAC;IACH,gDAAgD;IAChD,YAAY,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CACnC;AAED;;;;;;;;GAQG;AAEH;;;;;;;;GAQG;AACH,MAAM,MAAM,kBAAkB,GAAG,CAC/B,OAAO,EAAE,cAAc,EACvB,SAAS,EAAE,MAAM,KACd,OAAO,CAAC,SAAS,CAAC,CAAC;AAExB;;;;;;;;;;GAUG;AACH,MAAM,MAAM,oBAAoB,GAAG,CACjC,OAAO,EAAE,cAAc,EACvB,SAAS,EAAE,MAAM,EACjB,QAAQ,EAAE,MAAM,EAChB,WAAW,EAAE,MAAM,EAAE,KAClB,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC;AAE5B;;;;;;;;;;GAUG;AACH,MAAM,MAAM,qBAAqB,GAAG,CAClC,OAAO,EAAE,cAAc,EACvB,SAAS,EAAE,MAAM,EACjB,QAAQ,EAAE,MAAM,EAChB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,KACf,OAAO,CAAC,IAAI,CAAC,CAAC;AAEnB;;;;;;;;;;GAUG;AACH,MAAM,MAAM,uBAAuB,GAAG,CACpC,OAAO,EAAE,cAAc,EACvB,SAAS,EAAE,MAAM,EACjB,KAAK,EAAE,SAAS,EAChB,SAAS,EAAE,GAAG,CAAC,MAAM,CAAC,KACnB,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC"}
@@ -0,0 +1,6 @@
1
+ /**
2
+ * Copyright (c) 2025 Elara AI Pty Ltd
3
+ * Licensed under BSL 1.1. See LICENSE for details.
4
+ */
5
+ export {};
6
+ //# sourceMappingURL=interfaces.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"interfaces.js","sourceRoot":"","sources":["../../../src/execution/interfaces.ts"],"names":[],"mappings":"AAAA;;;GAGG"}
@@ -0,0 +1,20 @@
1
+ /**
2
+ * Copyright (c) 2025 Elara AI Pty Ltd
3
+ * Licensed under BSL 1.1. See LICENSE for details.
4
+ */
5
+ /**
6
+ * Get the current system boot ID.
7
+ * Used for detecting stale locks/processes after system reboot.
8
+ */
9
+ export declare function getBootId(): Promise<string>;
10
+ /**
11
+ * Get process start time from /proc/<pid>/stat.
12
+ * Returns the starttime field (field 22) which is jiffies since boot.
13
+ * Used together with boot ID to uniquely identify a process (handles PID reuse).
14
+ */
15
+ export declare function getPidStartTime(pid: number): Promise<number>;
16
+ /**
17
+ * Check if a process is still alive based on stored identification
18
+ */
19
+ export declare function isProcessAlive(pid: number, pidStartTime: number, bootId: string): Promise<boolean>;
20
+ //# sourceMappingURL=processHelpers.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"processHelpers.d.ts","sourceRoot":"","sources":["../../../src/execution/processHelpers.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAWH;;;GAGG;AACH,wBAAsB,SAAS,IAAI,OAAO,CAAC,MAAM,CAAC,CAQjD;AAED;;;;GAIG;AACH,wBAAsB,eAAe,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAalE;AAED;;GAEG;AACH,wBAAsB,cAAc,CAClC,GAAG,EAAE,MAAM,EACX,YAAY,EAAE,MAAM,EACpB,MAAM,EAAE,MAAM,GACb,OAAO,CAAC,OAAO,CAAC,CAWlB"}
@@ -0,0 +1,62 @@
1
+ /**
2
+ * Copyright (c) 2025 Elara AI Pty Ltd
3
+ * Licensed under BSL 1.1. See LICENSE for details.
4
+ */
5
+ /**
6
+ * Process identification helpers for crash detection.
7
+ *
8
+ * These functions are used by LocalTaskRunner and LocalLockService
9
+ * to detect stale processes and locks after crashes or reboots.
10
+ */
11
+ import * as fs from 'fs/promises';
12
+ /**
13
+ * Get the current system boot ID.
14
+ * Used for detecting stale locks/processes after system reboot.
15
+ */
16
+ export async function getBootId() {
17
+ try {
18
+ const data = await fs.readFile('/proc/sys/kernel/random/boot_id', 'utf-8');
19
+ return data.trim();
20
+ }
21
+ catch {
22
+ // Not on Linux, use a placeholder
23
+ return 'unknown-boot-id';
24
+ }
25
+ }
26
+ /**
27
+ * Get process start time from /proc/<pid>/stat.
28
+ * Returns the starttime field (field 22) which is jiffies since boot.
29
+ * Used together with boot ID to uniquely identify a process (handles PID reuse).
30
+ */
31
+ export async function getPidStartTime(pid) {
32
+ try {
33
+ const data = await fs.readFile(`/proc/${pid}/stat`, 'utf-8');
34
+ // Fields are space-separated, but comm (field 2) can contain spaces and is in parens
35
+ // Find the closing paren, then split the rest
36
+ const closeParen = data.lastIndexOf(')');
37
+ const fields = data.slice(closeParen + 2).split(' ');
38
+ // After the closing paren, field index 0 is state (field 3), so starttime is at index 19
39
+ // (field 22 - 3 = 19)
40
+ return parseInt(fields[19], 10);
41
+ }
42
+ catch {
43
+ return 0;
44
+ }
45
+ }
46
+ /**
47
+ * Check if a process is still alive based on stored identification
48
+ */
49
+ export async function isProcessAlive(pid, pidStartTime, bootId) {
50
+ // Different boot? Process is dead
51
+ const currentBootId = await getBootId();
52
+ if (currentBootId !== bootId)
53
+ return false;
54
+ // Check if PID exists and has same start time
55
+ const currentStartTime = await getPidStartTime(pid);
56
+ if (currentStartTime === 0)
57
+ return false; // PID doesn't exist
58
+ if (currentStartTime !== pidStartTime)
59
+ return false; // PID reused
60
+ return true;
61
+ }
62
+ //# sourceMappingURL=processHelpers.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"processHelpers.js","sourceRoot":"","sources":["../../../src/execution/processHelpers.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH;;;;;GAKG;AAEH,OAAO,KAAK,EAAE,MAAM,aAAa,CAAC;AAElC;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,SAAS;IAC7B,IAAI,CAAC;QACH,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,iCAAiC,EAAE,OAAO,CAAC,CAAC;QAC3E,OAAO,IAAI,CAAC,IAAI,EAAE,CAAC;IACrB,CAAC;IAAC,MAAM,CAAC;QACP,kCAAkC;QAClC,OAAO,iBAAiB,CAAC;IAC3B,CAAC;AACH,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,GAAW;IAC/C,IAAI,CAAC;QACH,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,SAAS,GAAG,OAAO,EAAE,OAAO,CAAC,CAAC;QAC7D,qFAAqF;QACrF,8CAA8C;QAC9C,MAAM,UAAU,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;QACzC,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QACrD,yFAAyF;QACzF,sBAAsB;QACtB,OAAO,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;IAClC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,CAAC,CAAC;IACX,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,GAAW,EACX,YAAoB,EACpB,MAAc;IAEd,kCAAkC;IAClC,MAAM,aAAa,GAAG,MAAM,SAAS,EAAE,CAAC;IACxC,IAAI,aAAa,KAAK,MAAM;QAAE,OAAO,KAAK,CAAC;IAE3C,8CAA8C;IAC9C,MAAM,gBAAgB,GAAG,MAAM,eAAe,CAAC,GAAG,CAAC,CAAC;IACpD,IAAI,gBAAgB,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC,CAAC,oBAAoB;IAC9D,IAAI,gBAAgB,KAAK,YAAY;QAAE,OAAO,KAAK,CAAC,CAAC,aAAa;IAElE,OAAO,IAAI,CAAC;AACd,CAAC"}