@elaraai/e3-core 0.0.2-beta.2 → 0.0.2-beta.21

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (179) hide show
  1. package/README.md +25 -22
  2. package/dist/src/dataflow/api-compat.d.ts +90 -0
  3. package/dist/src/dataflow/api-compat.d.ts.map +1 -0
  4. package/dist/src/dataflow/api-compat.js +134 -0
  5. package/dist/src/dataflow/api-compat.js.map +1 -0
  6. package/dist/src/dataflow/index.d.ts +18 -0
  7. package/dist/src/dataflow/index.d.ts.map +1 -0
  8. package/dist/src/dataflow/index.js +23 -0
  9. package/dist/src/dataflow/index.js.map +1 -0
  10. package/dist/src/dataflow/orchestrator/LocalOrchestrator.d.ts +53 -0
  11. package/dist/src/dataflow/orchestrator/LocalOrchestrator.d.ts.map +1 -0
  12. package/dist/src/dataflow/orchestrator/LocalOrchestrator.js +416 -0
  13. package/dist/src/dataflow/orchestrator/LocalOrchestrator.js.map +1 -0
  14. package/dist/src/dataflow/orchestrator/index.d.ts +12 -0
  15. package/dist/src/dataflow/orchestrator/index.d.ts.map +1 -0
  16. package/dist/src/dataflow/orchestrator/index.js +12 -0
  17. package/dist/src/dataflow/orchestrator/index.js.map +1 -0
  18. package/dist/src/dataflow/orchestrator/interfaces.d.ts +157 -0
  19. package/dist/src/dataflow/orchestrator/interfaces.d.ts.map +1 -0
  20. package/dist/src/dataflow/orchestrator/interfaces.js +51 -0
  21. package/dist/src/dataflow/orchestrator/interfaces.js.map +1 -0
  22. package/dist/src/dataflow/state-store/FileStateStore.d.ts +67 -0
  23. package/dist/src/dataflow/state-store/FileStateStore.d.ts.map +1 -0
  24. package/dist/src/dataflow/state-store/FileStateStore.js +286 -0
  25. package/dist/src/dataflow/state-store/FileStateStore.js.map +1 -0
  26. package/dist/src/dataflow/state-store/InMemoryStateStore.d.ts +42 -0
  27. package/dist/src/dataflow/state-store/InMemoryStateStore.d.ts.map +1 -0
  28. package/dist/src/dataflow/state-store/InMemoryStateStore.js +214 -0
  29. package/dist/src/dataflow/state-store/InMemoryStateStore.js.map +1 -0
  30. package/dist/src/dataflow/state-store/index.d.ts +13 -0
  31. package/dist/src/dataflow/state-store/index.d.ts.map +1 -0
  32. package/dist/src/dataflow/state-store/index.js +13 -0
  33. package/dist/src/dataflow/state-store/index.js.map +1 -0
  34. package/dist/src/dataflow/state-store/interfaces.d.ts +159 -0
  35. package/dist/src/dataflow/state-store/interfaces.d.ts.map +1 -0
  36. package/dist/src/dataflow/state-store/interfaces.js +6 -0
  37. package/dist/src/dataflow/state-store/interfaces.js.map +1 -0
  38. package/dist/src/dataflow/steps.d.ts +176 -0
  39. package/dist/src/dataflow/steps.d.ts.map +1 -0
  40. package/dist/src/dataflow/steps.js +528 -0
  41. package/dist/src/dataflow/steps.js.map +1 -0
  42. package/dist/src/dataflow/types.d.ts +116 -0
  43. package/dist/src/dataflow/types.d.ts.map +1 -0
  44. package/dist/src/dataflow/types.js +7 -0
  45. package/dist/src/dataflow/types.js.map +1 -0
  46. package/dist/src/dataflow.d.ts +138 -9
  47. package/dist/src/dataflow.d.ts.map +1 -1
  48. package/dist/src/dataflow.js +293 -59
  49. package/dist/src/dataflow.js.map +1 -1
  50. package/dist/src/errors.d.ts +36 -6
  51. package/dist/src/errors.d.ts.map +1 -1
  52. package/dist/src/errors.js +46 -3
  53. package/dist/src/errors.js.map +1 -1
  54. package/dist/src/execution/LocalTaskRunner.d.ts +71 -0
  55. package/dist/src/execution/LocalTaskRunner.d.ts.map +1 -0
  56. package/dist/src/execution/LocalTaskRunner.js +360 -0
  57. package/dist/src/execution/LocalTaskRunner.js.map +1 -0
  58. package/dist/src/execution/MockTaskRunner.d.ts +49 -0
  59. package/dist/src/execution/MockTaskRunner.d.ts.map +1 -0
  60. package/dist/src/execution/MockTaskRunner.js +55 -0
  61. package/dist/src/execution/MockTaskRunner.js.map +1 -0
  62. package/dist/src/execution/index.d.ts +16 -0
  63. package/dist/src/execution/index.d.ts.map +1 -0
  64. package/dist/src/execution/index.js +8 -0
  65. package/dist/src/execution/index.js.map +1 -0
  66. package/dist/src/execution/interfaces.d.ts +244 -0
  67. package/dist/src/execution/interfaces.d.ts.map +1 -0
  68. package/dist/src/execution/interfaces.js +6 -0
  69. package/dist/src/execution/interfaces.js.map +1 -0
  70. package/dist/src/execution/processHelpers.d.ts +20 -0
  71. package/dist/src/execution/processHelpers.d.ts.map +1 -0
  72. package/dist/src/execution/processHelpers.js +62 -0
  73. package/dist/src/execution/processHelpers.js.map +1 -0
  74. package/dist/src/executions.d.ts +46 -104
  75. package/dist/src/executions.d.ts.map +1 -1
  76. package/dist/src/executions.js +74 -478
  77. package/dist/src/executions.js.map +1 -1
  78. package/dist/src/index.d.ts +16 -9
  79. package/dist/src/index.d.ts.map +1 -1
  80. package/dist/src/index.js +41 -17
  81. package/dist/src/index.js.map +1 -1
  82. package/dist/src/objects.d.ts +6 -53
  83. package/dist/src/objects.d.ts.map +1 -1
  84. package/dist/src/objects.js +11 -232
  85. package/dist/src/objects.js.map +1 -1
  86. package/dist/src/packages.d.ts +22 -14
  87. package/dist/src/packages.d.ts.map +1 -1
  88. package/dist/src/packages.js +45 -79
  89. package/dist/src/packages.js.map +1 -1
  90. package/dist/src/storage/in-memory/InMemoryRepoStore.d.ts +36 -0
  91. package/dist/src/storage/in-memory/InMemoryRepoStore.d.ts.map +1 -0
  92. package/dist/src/storage/in-memory/InMemoryRepoStore.js +103 -0
  93. package/dist/src/storage/in-memory/InMemoryRepoStore.js.map +1 -0
  94. package/dist/src/storage/in-memory/InMemoryStorage.d.ts +105 -0
  95. package/dist/src/storage/in-memory/InMemoryStorage.d.ts.map +1 -0
  96. package/dist/src/storage/in-memory/InMemoryStorage.js +282 -0
  97. package/dist/src/storage/in-memory/InMemoryStorage.js.map +1 -0
  98. package/dist/src/storage/in-memory/index.d.ts +12 -0
  99. package/dist/src/storage/in-memory/index.d.ts.map +1 -0
  100. package/dist/src/storage/in-memory/index.js +12 -0
  101. package/dist/src/storage/in-memory/index.js.map +1 -0
  102. package/dist/src/storage/index.d.ts +18 -0
  103. package/dist/src/storage/index.d.ts.map +1 -0
  104. package/dist/src/storage/index.js +10 -0
  105. package/dist/src/storage/index.js.map +1 -0
  106. package/dist/src/storage/interfaces.d.ts +450 -0
  107. package/dist/src/storage/interfaces.d.ts.map +1 -0
  108. package/dist/src/storage/interfaces.js +6 -0
  109. package/dist/src/storage/interfaces.js.map +1 -0
  110. package/dist/src/storage/local/LocalBackend.d.ts +54 -0
  111. package/dist/src/storage/local/LocalBackend.d.ts.map +1 -0
  112. package/dist/src/storage/local/LocalBackend.js +124 -0
  113. package/dist/src/storage/local/LocalBackend.js.map +1 -0
  114. package/dist/src/storage/local/LocalLockService.d.ts +105 -0
  115. package/dist/src/storage/local/LocalLockService.d.ts.map +1 -0
  116. package/dist/src/storage/local/LocalLockService.js +342 -0
  117. package/dist/src/storage/local/LocalLockService.js.map +1 -0
  118. package/dist/src/storage/local/LocalLogStore.d.ts +23 -0
  119. package/dist/src/storage/local/LocalLogStore.d.ts.map +1 -0
  120. package/dist/src/storage/local/LocalLogStore.js +66 -0
  121. package/dist/src/storage/local/LocalLogStore.js.map +1 -0
  122. package/dist/src/storage/local/LocalObjectStore.d.ts +51 -0
  123. package/dist/src/storage/local/LocalObjectStore.d.ts.map +1 -0
  124. package/dist/src/storage/local/LocalObjectStore.js +263 -0
  125. package/dist/src/storage/local/LocalObjectStore.js.map +1 -0
  126. package/dist/src/storage/local/LocalRefStore.d.ts +35 -0
  127. package/dist/src/storage/local/LocalRefStore.d.ts.map +1 -0
  128. package/dist/src/storage/local/LocalRefStore.js +233 -0
  129. package/dist/src/storage/local/LocalRefStore.js.map +1 -0
  130. package/dist/src/storage/local/LocalRepoStore.d.ts +81 -0
  131. package/dist/src/storage/local/LocalRepoStore.d.ts.map +1 -0
  132. package/dist/src/storage/local/LocalRepoStore.js +454 -0
  133. package/dist/src/storage/local/LocalRepoStore.js.map +1 -0
  134. package/dist/src/{gc.d.ts → storage/local/gc.d.ts} +9 -2
  135. package/dist/src/storage/local/gc.d.ts.map +1 -0
  136. package/dist/src/storage/local/gc.js +152 -0
  137. package/dist/src/storage/local/gc.js.map +1 -0
  138. package/dist/src/storage/local/index.d.ts +17 -0
  139. package/dist/src/storage/local/index.d.ts.map +1 -0
  140. package/dist/src/storage/local/index.js +17 -0
  141. package/dist/src/storage/local/index.js.map +1 -0
  142. package/dist/src/storage/local/localHelpers.d.ts +25 -0
  143. package/dist/src/storage/local/localHelpers.d.ts.map +1 -0
  144. package/dist/src/storage/local/localHelpers.js +69 -0
  145. package/dist/src/storage/local/localHelpers.js.map +1 -0
  146. package/dist/src/{repository.d.ts → storage/local/repository.d.ts} +8 -4
  147. package/dist/src/storage/local/repository.d.ts.map +1 -0
  148. package/dist/src/{repository.js → storage/local/repository.js} +31 -29
  149. package/dist/src/storage/local/repository.js.map +1 -0
  150. package/dist/src/tasks.d.ts +16 -10
  151. package/dist/src/tasks.d.ts.map +1 -1
  152. package/dist/src/tasks.js +35 -41
  153. package/dist/src/tasks.js.map +1 -1
  154. package/dist/src/test-helpers.d.ts +4 -4
  155. package/dist/src/test-helpers.d.ts.map +1 -1
  156. package/dist/src/test-helpers.js +7 -21
  157. package/dist/src/test-helpers.js.map +1 -1
  158. package/dist/src/trees.d.ts +89 -27
  159. package/dist/src/trees.d.ts.map +1 -1
  160. package/dist/src/trees.js +218 -100
  161. package/dist/src/trees.js.map +1 -1
  162. package/dist/src/workspaceStatus.d.ts +6 -4
  163. package/dist/src/workspaceStatus.d.ts.map +1 -1
  164. package/dist/src/workspaceStatus.js +40 -47
  165. package/dist/src/workspaceStatus.js.map +1 -1
  166. package/dist/src/workspaces.d.ts +35 -26
  167. package/dist/src/workspaces.d.ts.map +1 -1
  168. package/dist/src/workspaces.js +93 -116
  169. package/dist/src/workspaces.js.map +1 -1
  170. package/package.json +4 -4
  171. package/dist/src/gc.d.ts.map +0 -1
  172. package/dist/src/gc.js +0 -233
  173. package/dist/src/gc.js.map +0 -1
  174. package/dist/src/repository.d.ts.map +0 -1
  175. package/dist/src/repository.js.map +0 -1
  176. package/dist/src/workspaceLock.d.ts +0 -67
  177. package/dist/src/workspaceLock.d.ts.map +0 -1
  178. package/dist/src/workspaceLock.js +0 -217
  179. package/dist/src/workspaceLock.js.map +0 -1
package/README.md CHANGED
@@ -17,48 +17,51 @@ Pure business logic with no UI dependencies. Use this to build custom tools, int
17
17
  ### Repository
18
18
 
19
19
  ```typescript
20
- import { initRepository, findRepository, getRepository } from '@elaraai/e3-core';
20
+ import { repoInit, repoFind, repoGet } from '@elaraai/e3-core';
21
21
 
22
- initRepository('/path/to/project');
23
- const repoPath = findRepository(); // Searches cwd and parents
22
+ await repoInit('/path/to/project');
23
+ const repoPath = repoFind(); // Searches cwd and parents
24
24
  ```
25
25
 
26
- ### Objects
26
+ ### Packages
27
27
 
28
28
  ```typescript
29
- import { storeObject, loadObject, computeTaskId } from '@elaraai/e3-core';
29
+ import { packageImport, packageList, packageResolve } from '@elaraai/e3-core';
30
30
 
31
- const hash = await storeObject(repoPath, data, '.beast2');
32
- const data = await loadObject(repoPath, hash, '.beast2');
33
- const taskId = computeTaskId(irHash, argsHashes);
31
+ await packageImport(repoPath, zipPath);
32
+ const packages = await packageList(repoPath);
33
+ const pkg = await packageResolve(repoPath, 'my-pkg@1.0.0');
34
34
  ```
35
35
 
36
- ### Commits
36
+ ### Workspaces
37
37
 
38
38
  ```typescript
39
- import { createNewTaskCommit, createTaskDoneCommit, loadCommit } from '@elaraai/e3-core';
39
+ import { workspaceCreate, workspaceList, workspaceDeploy, workspaceStatus } from '@elaraai/e3-core';
40
40
 
41
- const commitHash = await createNewTaskCommit(repoPath, taskId, irHash, argsHashes, 'node', null);
42
- const commit = await loadCommit(repoPath, commitHash);
41
+ await workspaceCreate(repoPath, 'dev');
42
+ await workspaceDeploy(repoPath, 'dev', 'my-pkg@1.0.0');
43
+ const status = await workspaceStatus(storage, repoPath, 'dev');
43
44
  ```
44
45
 
45
- ### Tasks
46
+ ### Datasets
46
47
 
47
48
  ```typescript
48
- import { updateTaskState, getTaskState, listTasks } from '@elaraai/e3-core';
49
+ import { workspaceGetDataset, workspaceSetDataset } from '@elaraai/e3-core';
49
50
 
50
- await updateTaskState(repoPath, taskId, commitHash);
51
- const commit = await getTaskState(repoPath, taskId);
52
- const tasks = await listTasks(repoPath);
51
+ const value = await workspaceGetDataset(storage, repoPath, 'dev', ['inputs', 'config']);
52
+ await workspaceSetDataset(storage, repoPath, 'dev', ['inputs', 'config'], newValue);
53
53
  ```
54
54
 
55
- ### Refs
55
+ ### Dataflow Execution
56
56
 
57
57
  ```typescript
58
- import { setTaskRef, deleteTaskRef, listTaskRefs, resolveToTaskId } from '@elaraai/e3-core';
58
+ import { dataflowExecute, dataflowStart } from '@elaraai/e3-core';
59
59
 
60
- await setTaskRef(repoPath, 'my-task', taskId);
61
- const taskId = await resolveToTaskId(repoPath, 'my-task');
60
+ // Blocking execution
61
+ const result = await dataflowExecute(storage, repoPath, 'dev', { force: true });
62
+
63
+ // Non-blocking execution
64
+ const handle = await dataflowStart(storage, repoPath, 'dev');
62
65
  ```
63
66
  ## License
64
67
 
@@ -82,7 +85,7 @@ BSL 1.1. See [LICENSE.md](./LICENSE.md).
82
85
  - [@elaraai/e3](https://www.npmjs.com/package/@elaraai/e3): SDK for authoring e3 packages with typed tasks and pipelines
83
86
  - [@elaraai/e3-core](https://www.npmjs.com/package/@elaraai/e3-core): Git-like object store, task queue, result caching
84
87
  - [@elaraai/e3-types](https://www.npmjs.com/package/@elaraai/e3-types): Shared type definitions for e3 packages
85
- - [@elaraai/e3-cli](https://www.npmjs.com/package/@elaraai/e3-cli): `e3 init`, `e3 run`, `e3 logs` commands for managing and monitoring tasks
88
+ - [@elaraai/e3-cli](https://www.npmjs.com/package/@elaraai/e3-cli): `e3 repo`, `e3 workspace`, `e3 start`, `e3 logs` commands for managing repositories, workspaces, and tasks
86
89
  - [@elaraai/e3-api-client](https://www.npmjs.com/package/@elaraai/e3-api-client): HTTP client for remote e3 servers
87
90
  - [@elaraai/e3-api-server](https://www.npmjs.com/package/@elaraai/e3-api-server): REST API server for e3 repositories
88
91
 
@@ -0,0 +1,90 @@
1
+ /**
2
+ * Copyright (c) 2025 Elara AI Pty Ltd
3
+ * Licensed under BSL 1.1. See LICENSE for details.
4
+ */
5
+ /**
6
+ * API compatibility layer for dataflow execution.
7
+ *
8
+ * Maps e3-core internal event/status types to API-compatible types
9
+ * used by e3-api-server and e3-api-client. This ensures consumers
10
+ * of the API see a stable interface regardless of internal changes.
11
+ *
12
+ * Vocabulary mappings:
13
+ * - e3-core 'cancelled' -> API 'aborted'
14
+ * - e3-core 'task_started' -> API 'start'
15
+ * - e3-core 'task_completed' (cached: false) -> API 'complete'
16
+ * - e3-core 'task_completed' (cached: true) -> API 'cached'
17
+ * - e3-core 'task_failed' -> API 'failed' or 'error'
18
+ * - e3-core 'task_skipped' -> API 'input_unavailable'
19
+ */
20
+ import type { ExecutionEvent, DataflowExecutionStatus, DataflowExecutionState } from './types.js';
21
+ /**
22
+ * API-compatible event types for dataflow execution polling.
23
+ * These match the DataflowEventType from e3-api-server/types.ts.
24
+ */
25
+ export type ApiDataflowEventType = 'start' | 'complete' | 'cached' | 'failed' | 'error' | 'input_unavailable';
26
+ /**
27
+ * API-compatible event structure.
28
+ */
29
+ export interface ApiDataflowEvent {
30
+ type: ApiDataflowEventType;
31
+ task: string;
32
+ timestamp: string;
33
+ duration?: number;
34
+ exitCode?: bigint;
35
+ message?: string;
36
+ reason?: string;
37
+ }
38
+ /**
39
+ * API-compatible execution status.
40
+ */
41
+ export type ApiExecutionStatus = 'running' | 'completed' | 'failed' | 'aborted';
42
+ /**
43
+ * API-compatible execution summary.
44
+ */
45
+ export interface ApiExecutionSummary {
46
+ executed: bigint;
47
+ cached: bigint;
48
+ failed: bigint;
49
+ skipped: bigint;
50
+ duration: number;
51
+ }
52
+ /**
53
+ * API-compatible execution state (matches DataflowExecutionStateType).
54
+ */
55
+ export interface ApiExecutionState {
56
+ status: ApiExecutionStatus;
57
+ startedAt: string;
58
+ completedAt: string | null;
59
+ summary: ApiExecutionSummary | null;
60
+ events: ApiDataflowEvent[];
61
+ totalEvents: bigint;
62
+ }
63
+ /**
64
+ * Convert e3-core execution event to API-compatible event.
65
+ *
66
+ * Returns null for events that don't have an API equivalent
67
+ * (e.g., execution_started, task_ready, execution_completed, execution_cancelled).
68
+ *
69
+ * @param event - The e3-core execution event (variant format)
70
+ * @returns API-compatible event or null if no mapping exists
71
+ */
72
+ export declare function coreEventToApiEvent(event: ExecutionEvent): ApiDataflowEvent | null;
73
+ /**
74
+ * Convert e3-core execution status to API status.
75
+ *
76
+ * @param status - The e3-core execution status
77
+ * @returns API-compatible status string
78
+ */
79
+ export declare function coreStatusToApiStatus(status: DataflowExecutionStatus): ApiExecutionStatus;
80
+ /**
81
+ * Convert e3-core execution state to API-compatible state.
82
+ *
83
+ * @param state - The e3-core execution state
84
+ * @param events - Events to include (already filtered by offset/limit)
85
+ * @param totalEvents - Total number of events for pagination
86
+ * @param duration - Total execution duration in milliseconds
87
+ * @returns API-compatible execution state
88
+ */
89
+ export declare function coreStateToApiState(state: DataflowExecutionState, events: ExecutionEvent[], totalEvents: number, duration: number): ApiExecutionState;
90
+ //# sourceMappingURL=api-compat.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"api-compat.d.ts","sourceRoot":"","sources":["../../../src/dataflow/api-compat.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH;;;;;;;;;;;;;;GAcG;AAEH,OAAO,KAAK,EACV,cAAc,EACd,uBAAuB,EACvB,sBAAsB,EACvB,MAAM,YAAY,CAAC;AAMpB;;;GAGG;AACH,MAAM,MAAM,oBAAoB,GAC5B,OAAO,GACP,UAAU,GACV,QAAQ,GACR,QAAQ,GACR,OAAO,GACP,mBAAmB,CAAC;AAExB;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,IAAI,EAAE,oBAAoB,CAAC;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED;;GAEG;AACH,MAAM,MAAM,kBAAkB,GAAG,SAAS,GAAG,WAAW,GAAG,QAAQ,GAAG,SAAS,CAAC;AAEhF;;GAEG;AACH,MAAM,WAAW,mBAAmB;IAClC,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,MAAM,EAAE,kBAAkB,CAAC;IAC3B,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,OAAO,EAAE,mBAAmB,GAAG,IAAI,CAAC;IACpC,MAAM,EAAE,gBAAgB,EAAE,CAAC;IAC3B,WAAW,EAAE,MAAM,CAAC;CACrB;AAMD;;;;;;;;GAQG;AACH,wBAAgB,mBAAmB,CAAC,KAAK,EAAE,cAAc,GAAG,gBAAgB,GAAG,IAAI,CA6DlF;AAED;;;;;GAKG;AACH,wBAAgB,qBAAqB,CAAC,MAAM,EAAE,uBAAuB,GAAG,kBAAkB,CAWzF;AAED;;;;;;;;GAQG;AACH,wBAAgB,mBAAmB,CACjC,KAAK,EAAE,sBAAsB,EAC7B,MAAM,EAAE,cAAc,EAAE,EACxB,WAAW,EAAE,MAAM,EACnB,QAAQ,EAAE,MAAM,GACf,iBAAiB,CAmCnB"}
@@ -0,0 +1,134 @@
1
+ /**
2
+ * Copyright (c) 2025 Elara AI Pty Ltd
3
+ * Licensed under BSL 1.1. See LICENSE for details.
4
+ */
5
+ // =============================================================================
6
+ // Conversion Functions
7
+ // =============================================================================
8
+ /**
9
+ * Convert e3-core execution event to API-compatible event.
10
+ *
11
+ * Returns null for events that don't have an API equivalent
12
+ * (e.g., execution_started, task_ready, execution_completed, execution_cancelled).
13
+ *
14
+ * @param event - The e3-core execution event (variant format)
15
+ * @returns API-compatible event or null if no mapping exists
16
+ */
17
+ export function coreEventToApiEvent(event) {
18
+ switch (event.type) {
19
+ case 'task_started':
20
+ return {
21
+ type: 'start',
22
+ task: event.value.task,
23
+ timestamp: event.value.timestamp.toISOString(),
24
+ };
25
+ case 'task_completed':
26
+ if (event.value.cached) {
27
+ return {
28
+ type: 'cached',
29
+ task: event.value.task,
30
+ timestamp: event.value.timestamp.toISOString(),
31
+ };
32
+ }
33
+ return {
34
+ type: 'complete',
35
+ task: event.value.task,
36
+ timestamp: event.value.timestamp.toISOString(),
37
+ duration: Number(event.value.duration),
38
+ };
39
+ case 'task_failed': {
40
+ // If there's an exit code, treat as 'failed'; otherwise as 'error'
41
+ const exitCode = event.value.exitCode.type === 'some' ? event.value.exitCode.value : undefined;
42
+ const error = event.value.error.type === 'some' ? event.value.error.value : undefined;
43
+ if (exitCode !== undefined) {
44
+ return {
45
+ type: 'failed',
46
+ task: event.value.task,
47
+ timestamp: event.value.timestamp.toISOString(),
48
+ duration: Number(event.value.duration),
49
+ exitCode,
50
+ };
51
+ }
52
+ return {
53
+ type: 'error',
54
+ task: event.value.task,
55
+ timestamp: event.value.timestamp.toISOString(),
56
+ message: error ?? 'Unknown error',
57
+ };
58
+ }
59
+ case 'task_skipped':
60
+ return {
61
+ type: 'input_unavailable',
62
+ task: event.value.task,
63
+ timestamp: event.value.timestamp.toISOString(),
64
+ reason: `Upstream task '${event.value.cause}' failed`,
65
+ };
66
+ // Events without API equivalents
67
+ case 'execution_started':
68
+ case 'task_ready':
69
+ case 'execution_completed':
70
+ case 'execution_cancelled':
71
+ return null;
72
+ }
73
+ }
74
+ /**
75
+ * Convert e3-core execution status to API status.
76
+ *
77
+ * @param status - The e3-core execution status
78
+ * @returns API-compatible status string
79
+ */
80
+ export function coreStatusToApiStatus(status) {
81
+ switch (status) {
82
+ case 'running':
83
+ return 'running';
84
+ case 'completed':
85
+ return 'completed';
86
+ case 'failed':
87
+ return 'failed';
88
+ case 'cancelled':
89
+ return 'aborted';
90
+ }
91
+ }
92
+ /**
93
+ * Convert e3-core execution state to API-compatible state.
94
+ *
95
+ * @param state - The e3-core execution state
96
+ * @param events - Events to include (already filtered by offset/limit)
97
+ * @param totalEvents - Total number of events for pagination
98
+ * @param duration - Total execution duration in milliseconds
99
+ * @returns API-compatible execution state
100
+ */
101
+ export function coreStateToApiState(state, events, totalEvents, duration) {
102
+ // Convert events, filtering out those without API equivalents
103
+ const apiEvents = [];
104
+ for (const event of events) {
105
+ const apiEvent = coreEventToApiEvent(event);
106
+ if (apiEvent !== null) {
107
+ apiEvents.push(apiEvent);
108
+ }
109
+ }
110
+ // Build summary if execution is complete
111
+ let summary = null;
112
+ if (state.status !== 'running') {
113
+ summary = {
114
+ executed: state.executed,
115
+ cached: state.cached,
116
+ failed: state.failed,
117
+ skipped: state.skipped,
118
+ duration,
119
+ };
120
+ }
121
+ // Get completedAt value (handle Option type)
122
+ const completedAtValue = state.completedAt.type === 'some'
123
+ ? state.completedAt.value.toISOString()
124
+ : null;
125
+ return {
126
+ status: coreStatusToApiStatus(state.status),
127
+ startedAt: state.startedAt.toISOString(),
128
+ completedAt: completedAtValue,
129
+ summary,
130
+ events: apiEvents,
131
+ totalEvents: BigInt(totalEvents),
132
+ };
133
+ }
134
+ //# sourceMappingURL=api-compat.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"api-compat.js","sourceRoot":"","sources":["../../../src/dataflow/api-compat.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAiFH,gFAAgF;AAChF,uBAAuB;AACvB,gFAAgF;AAEhF;;;;;;;;GAQG;AACH,MAAM,UAAU,mBAAmB,CAAC,KAAqB;IACvD,QAAQ,KAAK,CAAC,IAAI,EAAE,CAAC;QACnB,KAAK,cAAc;YACjB,OAAO;gBACL,IAAI,EAAE,OAAO;gBACb,IAAI,EAAE,KAAK,CAAC,KAAK,CAAC,IAAI;gBACtB,SAAS,EAAE,KAAK,CAAC,KAAK,CAAC,SAAS,CAAC,WAAW,EAAE;aAC/C,CAAC;QAEJ,KAAK,gBAAgB;YACnB,IAAI,KAAK,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC;gBACvB,OAAO;oBACL,IAAI,EAAE,QAAQ;oBACd,IAAI,EAAE,KAAK,CAAC,KAAK,CAAC,IAAI;oBACtB,SAAS,EAAE,KAAK,CAAC,KAAK,CAAC,SAAS,CAAC,WAAW,EAAE;iBAC/C,CAAC;YACJ,CAAC;YACD,OAAO;gBACL,IAAI,EAAE,UAAU;gBAChB,IAAI,EAAE,KAAK,CAAC,KAAK,CAAC,IAAI;gBACtB,SAAS,EAAE,KAAK,CAAC,KAAK,CAAC,SAAS,CAAC,WAAW,EAAE;gBAC9C,QAAQ,EAAE,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,QAAQ,CAAC;aACvC,CAAC;QAEJ,KAAK,aAAa,CAAC,CAAC,CAAC;YACnB,mEAAmE;YACnE,MAAM,QAAQ,GAAG,KAAK,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC;YAC/F,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC;YAEtF,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;gBAC3B,OAAO;oBACL,IAAI,EAAE,QAAQ;oBACd,IAAI,EAAE,KAAK,CAAC,KAAK,CAAC,IAAI;oBACtB,SAAS,EAAE,KAAK,CAAC,KAAK,CAAC,SAAS,CAAC,WAAW,EAAE;oBAC9C,QAAQ,EAAE,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,QAAQ,CAAC;oBACtC,QAAQ;iBACT,CAAC;YACJ,CAAC;YACD,OAAO;gBACL,IAAI,EAAE,OAAO;gBACb,IAAI,EAAE,KAAK,CAAC,KAAK,CAAC,IAAI;gBACtB,SAAS,EAAE,KAAK,CAAC,KAAK,CAAC,SAAS,CAAC,WAAW,EAAE;gBAC9C,OAAO,EAAE,KAAK,IAAI,eAAe;aAClC,CAAC;QACJ,CAAC;QAED,KAAK,cAAc;YACjB,OAAO;gBACL,IAAI,EAAE,mBAAmB;gBACzB,IAAI,EAAE,KAAK,CAAC,KAAK,CAAC,IAAI;gBACtB,SAAS,EAAE,KAAK,CAAC,KAAK,CAAC,SAAS,CAAC,WAAW,EAAE;gBAC9C,MAAM,EAAE,kBAAkB,KAAK,CAAC,KAAK,CAAC,KAAK,UAAU;aACtD,CAAC;QAEJ,iCAAiC;QACjC,KAAK,mBAAmB,CAAC;QACzB,KAAK,YAAY,CAAC;QAClB,KAAK,qBAAqB,CAAC;QAC3B,KAAK,qBAAqB;YACxB,OAAO,IAAI,CAAC;IAChB,CAAC;AACH,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,qBAAqB,CAAC,MAA+B;IACnE,QAAQ,MAAM,EAAE,CAAC;QACf,KAAK,SAAS;YACZ,OAAO,SAAS,CAAC;QACnB,KAAK,WAAW;YACd,OAAO,WAAW,CAAC;QACrB,KAAK,QAAQ;YACX,OAAO,QAAQ,CAAC;QAClB,KAAK,WAAW;YACd,OAAO,SAAS,CAAC;IACrB,CAAC;AACH,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,mBAAmB,CACjC,KAA6B,EAC7B,MAAwB,EACxB,WAAmB,EACnB,QAAgB;IAEhB,8DAA8D;IAC9D,MAAM,SAAS,GAAuB,EAAE,CAAC;IACzC,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,MAAM,QAAQ,GAAG,mBAAmB,CAAC,KAAK,CAAC,CAAC;QAC5C,IAAI,QAAQ,KAAK,IAAI,EAAE,CAAC;YACtB,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC3B,CAAC;IACH,CAAC;IAED,yCAAyC;IACzC,IAAI,OAAO,GAA+B,IAAI,CAAC;IAC/C,IAAI,KAAK,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;QAC/B,OAAO,GAAG;YACR,QAAQ,EAAE,KAAK,CAAC,QAAQ;YACxB,MAAM,EAAE,KAAK,CAAC,MAAM;YACpB,MAAM,EAAE,KAAK,CAAC,MAAM;YACpB,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,QAAQ;SACT,CAAC;IACJ,CAAC;IAED,6CAA6C;IAC7C,MAAM,gBAAgB,GAAG,KAAK,CAAC,WAAW,CAAC,IAAI,KAAK,MAAM;QACxD,CAAC,CAAC,KAAK,CAAC,WAAW,CAAC,KAAK,CAAC,WAAW,EAAE;QACvC,CAAC,CAAC,IAAI,CAAC;IAET,OAAO;QACL,MAAM,EAAE,qBAAqB,CAAC,KAAK,CAAC,MAAiC,CAAC;QACtE,SAAS,EAAE,KAAK,CAAC,SAAS,CAAC,WAAW,EAAE;QACxC,WAAW,EAAE,gBAAgB;QAC7B,OAAO;QACP,MAAM,EAAE,SAAS;QACjB,WAAW,EAAE,MAAM,CAAC,WAAW,CAAC;KACjC,CAAC;AACJ,CAAC"}
@@ -0,0 +1,18 @@
1
+ /**
2
+ * Copyright (c) 2025 Elara AI Pty Ltd
3
+ * Licensed under BSL 1.1. See LICENSE for details.
4
+ */
5
+ /**
6
+ * Resumable dataflow execution module.
7
+ *
8
+ * This module provides interfaces and implementations for resumable,
9
+ * portable dataflow execution that supports both local and cloud environments.
10
+ *
11
+ * @packageDocumentation
12
+ */
13
+ export * from './types.js';
14
+ export * from './steps.js';
15
+ export * from './state-store/index.js';
16
+ export * from './orchestrator/index.js';
17
+ export * from './api-compat.js';
18
+ //# sourceMappingURL=index.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/dataflow/index.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH;;;;;;;GAOG;AAGH,cAAc,YAAY,CAAC;AAG3B,cAAc,YAAY,CAAC;AAG3B,cAAc,wBAAwB,CAAC;AAGvC,cAAc,yBAAyB,CAAC;AAGxC,cAAc,iBAAiB,CAAC"}
@@ -0,0 +1,23 @@
1
+ /**
2
+ * Copyright (c) 2025 Elara AI Pty Ltd
3
+ * Licensed under BSL 1.1. See LICENSE for details.
4
+ */
5
+ /**
6
+ * Resumable dataflow execution module.
7
+ *
8
+ * This module provides interfaces and implementations for resumable,
9
+ * portable dataflow execution that supports both local and cloud environments.
10
+ *
11
+ * @packageDocumentation
12
+ */
13
+ // Types
14
+ export * from './types.js';
15
+ // Step functions
16
+ export * from './steps.js';
17
+ // State store
18
+ export * from './state-store/index.js';
19
+ // Orchestrator
20
+ export * from './orchestrator/index.js';
21
+ // API compatibility layer
22
+ export * from './api-compat.js';
23
+ //# sourceMappingURL=index.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/dataflow/index.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH;;;;;;;GAOG;AAEH,QAAQ;AACR,cAAc,YAAY,CAAC;AAE3B,iBAAiB;AACjB,cAAc,YAAY,CAAC;AAE3B,cAAc;AACd,cAAc,wBAAwB,CAAC;AAEvC,eAAe;AACf,cAAc,yBAAyB,CAAC;AAExC,0BAA0B;AAC1B,cAAc,iBAAiB,CAAC"}
@@ -0,0 +1,53 @@
1
+ /**
2
+ * Copyright (c) 2025 Elara AI Pty Ltd
3
+ * Licensed under BSL 1.1. See LICENSE for details.
4
+ */
5
+ import type { StorageBackend } from '../../storage/interfaces.js';
6
+ import type { DataflowOrchestrator, ExecutionHandle, ExecutionStatus, OrchestratorStartOptions } from './interfaces.js';
7
+ import type { ExecutionStateStore } from '../state-store/interfaces.js';
8
+ import type { ExecutionEvent, FinalizeResult } from '../types.js';
9
+ /**
10
+ * Local orchestrator for in-process dataflow execution.
11
+ *
12
+ * @remarks
13
+ * - Uses step functions for each operation
14
+ * - Serializes workspace writes with AsyncMutex
15
+ * - Supports AbortSignal for cancellation
16
+ * - Persists state through the provided state store
17
+ */
18
+ export declare class LocalOrchestrator implements DataflowOrchestrator {
19
+ private readonly stateStore?;
20
+ private executions;
21
+ /**
22
+ * Create a new LocalOrchestrator.
23
+ *
24
+ * @param stateStore - Optional state store for persistence.
25
+ * If not provided, state is only kept in memory.
26
+ */
27
+ constructor(stateStore?: ExecutionStateStore | undefined);
28
+ start(storage: StorageBackend, repo: string, workspace: string, options?: OrchestratorStartOptions): Promise<ExecutionHandle>;
29
+ wait(handle: ExecutionHandle): Promise<FinalizeResult>;
30
+ getStatus(handle: ExecutionHandle): Promise<ExecutionStatus>;
31
+ cancel(handle: ExecutionHandle): Promise<void>;
32
+ getEvents(handle: ExecutionHandle, sinceSeq: number): Promise<ExecutionEvent[]>;
33
+ /**
34
+ * Main execution loop.
35
+ *
36
+ * Uses step functions to execute tasks, managing concurrency and
37
+ * workspace state updates.
38
+ */
39
+ private runExecutionLoop;
40
+ /**
41
+ * Execute a single task.
42
+ */
43
+ private executeTask;
44
+ /**
45
+ * Build partial results for abort error.
46
+ */
47
+ private buildPartialResults;
48
+ /**
49
+ * Generate unique key for an execution.
50
+ */
51
+ private executionKey;
52
+ }
53
+ //# sourceMappingURL=LocalOrchestrator.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"LocalOrchestrator.d.ts","sourceRoot":"","sources":["../../../../src/dataflow/orchestrator/LocalOrchestrator.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAUH,OAAO,KAAK,EAAE,cAAc,EAAc,MAAM,6BAA6B,CAAC;AAO9E,OAAO,KAAK,EACV,oBAAoB,EACpB,eAAe,EACf,eAAe,EACf,wBAAwB,EACzB,MAAM,iBAAiB,CAAC;AAEzB,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,8BAA8B,CAAC;AACxE,OAAO,KAAK,EAEV,cAAc,EACd,cAAc,EACf,MAAM,aAAa,CAAC;AA2ErB;;;;;;;;GAQG;AACH,qBAAa,iBAAkB,YAAW,oBAAoB;IAShD,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAC;IARxC,OAAO,CAAC,UAAU,CAAuC;IAEzD;;;;;OAKG;gBAC0B,UAAU,CAAC,EAAE,mBAAmB,YAAA;IAEvD,KAAK,CACT,OAAO,EAAE,cAAc,EACvB,IAAI,EAAE,MAAM,EACZ,SAAS,EAAE,MAAM,EACjB,OAAO,GAAE,wBAA6B,GACrC,OAAO,CAAC,eAAe,CAAC;IA6FrB,IAAI,CAAC,MAAM,EAAE,eAAe,GAAG,OAAO,CAAC,cAAc,CAAC;IAWtD,SAAS,CAAC,MAAM,EAAE,eAAe,GAAG,OAAO,CAAC,eAAe,CAAC;IAkB5D,MAAM,CAAC,MAAM,EAAE,eAAe,GAAG,OAAO,CAAC,IAAI,CAAC;IAuB9C,SAAS,CAAC,MAAM,EAAE,eAAe,EAAE,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,cAAc,EAAE,CAAC;IAOrF;;;;;OAKG;YACW,gBAAgB;IA+M9B;;OAEG;YACW,WAAW;IAgDzB;;OAEG;IACH,OAAO,CAAC,mBAAmB;IAyB3B;;OAEG;IACH,OAAO,CAAC,YAAY;CAGrB"}