@elaraai/e3-core 0.0.2-beta.2 → 0.0.2-beta.21
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +25 -22
- package/dist/src/dataflow/api-compat.d.ts +90 -0
- package/dist/src/dataflow/api-compat.d.ts.map +1 -0
- package/dist/src/dataflow/api-compat.js +134 -0
- package/dist/src/dataflow/api-compat.js.map +1 -0
- package/dist/src/dataflow/index.d.ts +18 -0
- package/dist/src/dataflow/index.d.ts.map +1 -0
- package/dist/src/dataflow/index.js +23 -0
- package/dist/src/dataflow/index.js.map +1 -0
- package/dist/src/dataflow/orchestrator/LocalOrchestrator.d.ts +53 -0
- package/dist/src/dataflow/orchestrator/LocalOrchestrator.d.ts.map +1 -0
- package/dist/src/dataflow/orchestrator/LocalOrchestrator.js +416 -0
- package/dist/src/dataflow/orchestrator/LocalOrchestrator.js.map +1 -0
- package/dist/src/dataflow/orchestrator/index.d.ts +12 -0
- package/dist/src/dataflow/orchestrator/index.d.ts.map +1 -0
- package/dist/src/dataflow/orchestrator/index.js +12 -0
- package/dist/src/dataflow/orchestrator/index.js.map +1 -0
- package/dist/src/dataflow/orchestrator/interfaces.d.ts +157 -0
- package/dist/src/dataflow/orchestrator/interfaces.d.ts.map +1 -0
- package/dist/src/dataflow/orchestrator/interfaces.js +51 -0
- package/dist/src/dataflow/orchestrator/interfaces.js.map +1 -0
- package/dist/src/dataflow/state-store/FileStateStore.d.ts +67 -0
- package/dist/src/dataflow/state-store/FileStateStore.d.ts.map +1 -0
- package/dist/src/dataflow/state-store/FileStateStore.js +286 -0
- package/dist/src/dataflow/state-store/FileStateStore.js.map +1 -0
- package/dist/src/dataflow/state-store/InMemoryStateStore.d.ts +42 -0
- package/dist/src/dataflow/state-store/InMemoryStateStore.d.ts.map +1 -0
- package/dist/src/dataflow/state-store/InMemoryStateStore.js +214 -0
- package/dist/src/dataflow/state-store/InMemoryStateStore.js.map +1 -0
- package/dist/src/dataflow/state-store/index.d.ts +13 -0
- package/dist/src/dataflow/state-store/index.d.ts.map +1 -0
- package/dist/src/dataflow/state-store/index.js +13 -0
- package/dist/src/dataflow/state-store/index.js.map +1 -0
- package/dist/src/dataflow/state-store/interfaces.d.ts +159 -0
- package/dist/src/dataflow/state-store/interfaces.d.ts.map +1 -0
- package/dist/src/dataflow/state-store/interfaces.js +6 -0
- package/dist/src/dataflow/state-store/interfaces.js.map +1 -0
- package/dist/src/dataflow/steps.d.ts +176 -0
- package/dist/src/dataflow/steps.d.ts.map +1 -0
- package/dist/src/dataflow/steps.js +528 -0
- package/dist/src/dataflow/steps.js.map +1 -0
- package/dist/src/dataflow/types.d.ts +116 -0
- package/dist/src/dataflow/types.d.ts.map +1 -0
- package/dist/src/dataflow/types.js +7 -0
- package/dist/src/dataflow/types.js.map +1 -0
- package/dist/src/dataflow.d.ts +138 -9
- package/dist/src/dataflow.d.ts.map +1 -1
- package/dist/src/dataflow.js +293 -59
- package/dist/src/dataflow.js.map +1 -1
- package/dist/src/errors.d.ts +36 -6
- package/dist/src/errors.d.ts.map +1 -1
- package/dist/src/errors.js +46 -3
- package/dist/src/errors.js.map +1 -1
- package/dist/src/execution/LocalTaskRunner.d.ts +71 -0
- package/dist/src/execution/LocalTaskRunner.d.ts.map +1 -0
- package/dist/src/execution/LocalTaskRunner.js +360 -0
- package/dist/src/execution/LocalTaskRunner.js.map +1 -0
- package/dist/src/execution/MockTaskRunner.d.ts +49 -0
- package/dist/src/execution/MockTaskRunner.d.ts.map +1 -0
- package/dist/src/execution/MockTaskRunner.js +55 -0
- package/dist/src/execution/MockTaskRunner.js.map +1 -0
- package/dist/src/execution/index.d.ts +16 -0
- package/dist/src/execution/index.d.ts.map +1 -0
- package/dist/src/execution/index.js +8 -0
- package/dist/src/execution/index.js.map +1 -0
- package/dist/src/execution/interfaces.d.ts +244 -0
- package/dist/src/execution/interfaces.d.ts.map +1 -0
- package/dist/src/execution/interfaces.js +6 -0
- package/dist/src/execution/interfaces.js.map +1 -0
- package/dist/src/execution/processHelpers.d.ts +20 -0
- package/dist/src/execution/processHelpers.d.ts.map +1 -0
- package/dist/src/execution/processHelpers.js +62 -0
- package/dist/src/execution/processHelpers.js.map +1 -0
- package/dist/src/executions.d.ts +46 -104
- package/dist/src/executions.d.ts.map +1 -1
- package/dist/src/executions.js +74 -478
- package/dist/src/executions.js.map +1 -1
- package/dist/src/index.d.ts +16 -9
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +41 -17
- package/dist/src/index.js.map +1 -1
- package/dist/src/objects.d.ts +6 -53
- package/dist/src/objects.d.ts.map +1 -1
- package/dist/src/objects.js +11 -232
- package/dist/src/objects.js.map +1 -1
- package/dist/src/packages.d.ts +22 -14
- package/dist/src/packages.d.ts.map +1 -1
- package/dist/src/packages.js +45 -79
- package/dist/src/packages.js.map +1 -1
- package/dist/src/storage/in-memory/InMemoryRepoStore.d.ts +36 -0
- package/dist/src/storage/in-memory/InMemoryRepoStore.d.ts.map +1 -0
- package/dist/src/storage/in-memory/InMemoryRepoStore.js +103 -0
- package/dist/src/storage/in-memory/InMemoryRepoStore.js.map +1 -0
- package/dist/src/storage/in-memory/InMemoryStorage.d.ts +105 -0
- package/dist/src/storage/in-memory/InMemoryStorage.d.ts.map +1 -0
- package/dist/src/storage/in-memory/InMemoryStorage.js +282 -0
- package/dist/src/storage/in-memory/InMemoryStorage.js.map +1 -0
- package/dist/src/storage/in-memory/index.d.ts +12 -0
- package/dist/src/storage/in-memory/index.d.ts.map +1 -0
- package/dist/src/storage/in-memory/index.js +12 -0
- package/dist/src/storage/in-memory/index.js.map +1 -0
- package/dist/src/storage/index.d.ts +18 -0
- package/dist/src/storage/index.d.ts.map +1 -0
- package/dist/src/storage/index.js +10 -0
- package/dist/src/storage/index.js.map +1 -0
- package/dist/src/storage/interfaces.d.ts +450 -0
- package/dist/src/storage/interfaces.d.ts.map +1 -0
- package/dist/src/storage/interfaces.js +6 -0
- package/dist/src/storage/interfaces.js.map +1 -0
- package/dist/src/storage/local/LocalBackend.d.ts +54 -0
- package/dist/src/storage/local/LocalBackend.d.ts.map +1 -0
- package/dist/src/storage/local/LocalBackend.js +124 -0
- package/dist/src/storage/local/LocalBackend.js.map +1 -0
- package/dist/src/storage/local/LocalLockService.d.ts +105 -0
- package/dist/src/storage/local/LocalLockService.d.ts.map +1 -0
- package/dist/src/storage/local/LocalLockService.js +342 -0
- package/dist/src/storage/local/LocalLockService.js.map +1 -0
- package/dist/src/storage/local/LocalLogStore.d.ts +23 -0
- package/dist/src/storage/local/LocalLogStore.d.ts.map +1 -0
- package/dist/src/storage/local/LocalLogStore.js +66 -0
- package/dist/src/storage/local/LocalLogStore.js.map +1 -0
- package/dist/src/storage/local/LocalObjectStore.d.ts +51 -0
- package/dist/src/storage/local/LocalObjectStore.d.ts.map +1 -0
- package/dist/src/storage/local/LocalObjectStore.js +263 -0
- package/dist/src/storage/local/LocalObjectStore.js.map +1 -0
- package/dist/src/storage/local/LocalRefStore.d.ts +35 -0
- package/dist/src/storage/local/LocalRefStore.d.ts.map +1 -0
- package/dist/src/storage/local/LocalRefStore.js +233 -0
- package/dist/src/storage/local/LocalRefStore.js.map +1 -0
- package/dist/src/storage/local/LocalRepoStore.d.ts +81 -0
- package/dist/src/storage/local/LocalRepoStore.d.ts.map +1 -0
- package/dist/src/storage/local/LocalRepoStore.js +454 -0
- package/dist/src/storage/local/LocalRepoStore.js.map +1 -0
- package/dist/src/{gc.d.ts → storage/local/gc.d.ts} +9 -2
- package/dist/src/storage/local/gc.d.ts.map +1 -0
- package/dist/src/storage/local/gc.js +152 -0
- package/dist/src/storage/local/gc.js.map +1 -0
- package/dist/src/storage/local/index.d.ts +17 -0
- package/dist/src/storage/local/index.d.ts.map +1 -0
- package/dist/src/storage/local/index.js +17 -0
- package/dist/src/storage/local/index.js.map +1 -0
- package/dist/src/storage/local/localHelpers.d.ts +25 -0
- package/dist/src/storage/local/localHelpers.d.ts.map +1 -0
- package/dist/src/storage/local/localHelpers.js +69 -0
- package/dist/src/storage/local/localHelpers.js.map +1 -0
- package/dist/src/{repository.d.ts → storage/local/repository.d.ts} +8 -4
- package/dist/src/storage/local/repository.d.ts.map +1 -0
- package/dist/src/{repository.js → storage/local/repository.js} +31 -29
- package/dist/src/storage/local/repository.js.map +1 -0
- package/dist/src/tasks.d.ts +16 -10
- package/dist/src/tasks.d.ts.map +1 -1
- package/dist/src/tasks.js +35 -41
- package/dist/src/tasks.js.map +1 -1
- package/dist/src/test-helpers.d.ts +4 -4
- package/dist/src/test-helpers.d.ts.map +1 -1
- package/dist/src/test-helpers.js +7 -21
- package/dist/src/test-helpers.js.map +1 -1
- package/dist/src/trees.d.ts +89 -27
- package/dist/src/trees.d.ts.map +1 -1
- package/dist/src/trees.js +218 -100
- package/dist/src/trees.js.map +1 -1
- package/dist/src/workspaceStatus.d.ts +6 -4
- package/dist/src/workspaceStatus.d.ts.map +1 -1
- package/dist/src/workspaceStatus.js +40 -47
- package/dist/src/workspaceStatus.js.map +1 -1
- package/dist/src/workspaces.d.ts +35 -26
- package/dist/src/workspaces.d.ts.map +1 -1
- package/dist/src/workspaces.js +93 -116
- package/dist/src/workspaces.js.map +1 -1
- package/package.json +4 -4
- package/dist/src/gc.d.ts.map +0 -1
- package/dist/src/gc.js +0 -233
- package/dist/src/gc.js.map +0 -1
- package/dist/src/repository.d.ts.map +0 -1
- package/dist/src/repository.js.map +0 -1
- package/dist/src/workspaceLock.d.ts +0 -67
- package/dist/src/workspaceLock.d.ts.map +0 -1
- package/dist/src/workspaceLock.js +0 -217
- package/dist/src/workspaceLock.js.map +0 -1
package/README.md
CHANGED
|
@@ -17,48 +17,51 @@ Pure business logic with no UI dependencies. Use this to build custom tools, int
|
|
|
17
17
|
### Repository
|
|
18
18
|
|
|
19
19
|
```typescript
|
|
20
|
-
import {
|
|
20
|
+
import { repoInit, repoFind, repoGet } from '@elaraai/e3-core';
|
|
21
21
|
|
|
22
|
-
|
|
23
|
-
const repoPath =
|
|
22
|
+
await repoInit('/path/to/project');
|
|
23
|
+
const repoPath = repoFind(); // Searches cwd and parents
|
|
24
24
|
```
|
|
25
25
|
|
|
26
|
-
###
|
|
26
|
+
### Packages
|
|
27
27
|
|
|
28
28
|
```typescript
|
|
29
|
-
import {
|
|
29
|
+
import { packageImport, packageList, packageResolve } from '@elaraai/e3-core';
|
|
30
30
|
|
|
31
|
-
|
|
32
|
-
const
|
|
33
|
-
const
|
|
31
|
+
await packageImport(repoPath, zipPath);
|
|
32
|
+
const packages = await packageList(repoPath);
|
|
33
|
+
const pkg = await packageResolve(repoPath, 'my-pkg@1.0.0');
|
|
34
34
|
```
|
|
35
35
|
|
|
36
|
-
###
|
|
36
|
+
### Workspaces
|
|
37
37
|
|
|
38
38
|
```typescript
|
|
39
|
-
import {
|
|
39
|
+
import { workspaceCreate, workspaceList, workspaceDeploy, workspaceStatus } from '@elaraai/e3-core';
|
|
40
40
|
|
|
41
|
-
|
|
42
|
-
|
|
41
|
+
await workspaceCreate(repoPath, 'dev');
|
|
42
|
+
await workspaceDeploy(repoPath, 'dev', 'my-pkg@1.0.0');
|
|
43
|
+
const status = await workspaceStatus(storage, repoPath, 'dev');
|
|
43
44
|
```
|
|
44
45
|
|
|
45
|
-
###
|
|
46
|
+
### Datasets
|
|
46
47
|
|
|
47
48
|
```typescript
|
|
48
|
-
import {
|
|
49
|
+
import { workspaceGetDataset, workspaceSetDataset } from '@elaraai/e3-core';
|
|
49
50
|
|
|
50
|
-
await
|
|
51
|
-
|
|
52
|
-
const tasks = await listTasks(repoPath);
|
|
51
|
+
const value = await workspaceGetDataset(storage, repoPath, 'dev', ['inputs', 'config']);
|
|
52
|
+
await workspaceSetDataset(storage, repoPath, 'dev', ['inputs', 'config'], newValue);
|
|
53
53
|
```
|
|
54
54
|
|
|
55
|
-
###
|
|
55
|
+
### Dataflow Execution
|
|
56
56
|
|
|
57
57
|
```typescript
|
|
58
|
-
import {
|
|
58
|
+
import { dataflowExecute, dataflowStart } from '@elaraai/e3-core';
|
|
59
59
|
|
|
60
|
-
|
|
61
|
-
const
|
|
60
|
+
// Blocking execution
|
|
61
|
+
const result = await dataflowExecute(storage, repoPath, 'dev', { force: true });
|
|
62
|
+
|
|
63
|
+
// Non-blocking execution
|
|
64
|
+
const handle = await dataflowStart(storage, repoPath, 'dev');
|
|
62
65
|
```
|
|
63
66
|
## License
|
|
64
67
|
|
|
@@ -82,7 +85,7 @@ BSL 1.1. See [LICENSE.md](./LICENSE.md).
|
|
|
82
85
|
- [@elaraai/e3](https://www.npmjs.com/package/@elaraai/e3): SDK for authoring e3 packages with typed tasks and pipelines
|
|
83
86
|
- [@elaraai/e3-core](https://www.npmjs.com/package/@elaraai/e3-core): Git-like object store, task queue, result caching
|
|
84
87
|
- [@elaraai/e3-types](https://www.npmjs.com/package/@elaraai/e3-types): Shared type definitions for e3 packages
|
|
85
|
-
- [@elaraai/e3-cli](https://www.npmjs.com/package/@elaraai/e3-cli): `e3
|
|
88
|
+
- [@elaraai/e3-cli](https://www.npmjs.com/package/@elaraai/e3-cli): `e3 repo`, `e3 workspace`, `e3 start`, `e3 logs` commands for managing repositories, workspaces, and tasks
|
|
86
89
|
- [@elaraai/e3-api-client](https://www.npmjs.com/package/@elaraai/e3-api-client): HTTP client for remote e3 servers
|
|
87
90
|
- [@elaraai/e3-api-server](https://www.npmjs.com/package/@elaraai/e3-api-server): REST API server for e3 repositories
|
|
88
91
|
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Copyright (c) 2025 Elara AI Pty Ltd
|
|
3
|
+
* Licensed under BSL 1.1. See LICENSE for details.
|
|
4
|
+
*/
|
|
5
|
+
/**
|
|
6
|
+
* API compatibility layer for dataflow execution.
|
|
7
|
+
*
|
|
8
|
+
* Maps e3-core internal event/status types to API-compatible types
|
|
9
|
+
* used by e3-api-server and e3-api-client. This ensures consumers
|
|
10
|
+
* of the API see a stable interface regardless of internal changes.
|
|
11
|
+
*
|
|
12
|
+
* Vocabulary mappings:
|
|
13
|
+
* - e3-core 'cancelled' -> API 'aborted'
|
|
14
|
+
* - e3-core 'task_started' -> API 'start'
|
|
15
|
+
* - e3-core 'task_completed' (cached: false) -> API 'complete'
|
|
16
|
+
* - e3-core 'task_completed' (cached: true) -> API 'cached'
|
|
17
|
+
* - e3-core 'task_failed' -> API 'failed' or 'error'
|
|
18
|
+
* - e3-core 'task_skipped' -> API 'input_unavailable'
|
|
19
|
+
*/
|
|
20
|
+
import type { ExecutionEvent, DataflowExecutionStatus, DataflowExecutionState } from './types.js';
|
|
21
|
+
/**
|
|
22
|
+
* API-compatible event types for dataflow execution polling.
|
|
23
|
+
* These match the DataflowEventType from e3-api-server/types.ts.
|
|
24
|
+
*/
|
|
25
|
+
export type ApiDataflowEventType = 'start' | 'complete' | 'cached' | 'failed' | 'error' | 'input_unavailable';
|
|
26
|
+
/**
|
|
27
|
+
* API-compatible event structure.
|
|
28
|
+
*/
|
|
29
|
+
export interface ApiDataflowEvent {
|
|
30
|
+
type: ApiDataflowEventType;
|
|
31
|
+
task: string;
|
|
32
|
+
timestamp: string;
|
|
33
|
+
duration?: number;
|
|
34
|
+
exitCode?: bigint;
|
|
35
|
+
message?: string;
|
|
36
|
+
reason?: string;
|
|
37
|
+
}
|
|
38
|
+
/**
|
|
39
|
+
* API-compatible execution status.
|
|
40
|
+
*/
|
|
41
|
+
export type ApiExecutionStatus = 'running' | 'completed' | 'failed' | 'aborted';
|
|
42
|
+
/**
|
|
43
|
+
* API-compatible execution summary.
|
|
44
|
+
*/
|
|
45
|
+
export interface ApiExecutionSummary {
|
|
46
|
+
executed: bigint;
|
|
47
|
+
cached: bigint;
|
|
48
|
+
failed: bigint;
|
|
49
|
+
skipped: bigint;
|
|
50
|
+
duration: number;
|
|
51
|
+
}
|
|
52
|
+
/**
|
|
53
|
+
* API-compatible execution state (matches DataflowExecutionStateType).
|
|
54
|
+
*/
|
|
55
|
+
export interface ApiExecutionState {
|
|
56
|
+
status: ApiExecutionStatus;
|
|
57
|
+
startedAt: string;
|
|
58
|
+
completedAt: string | null;
|
|
59
|
+
summary: ApiExecutionSummary | null;
|
|
60
|
+
events: ApiDataflowEvent[];
|
|
61
|
+
totalEvents: bigint;
|
|
62
|
+
}
|
|
63
|
+
/**
|
|
64
|
+
* Convert e3-core execution event to API-compatible event.
|
|
65
|
+
*
|
|
66
|
+
* Returns null for events that don't have an API equivalent
|
|
67
|
+
* (e.g., execution_started, task_ready, execution_completed, execution_cancelled).
|
|
68
|
+
*
|
|
69
|
+
* @param event - The e3-core execution event (variant format)
|
|
70
|
+
* @returns API-compatible event or null if no mapping exists
|
|
71
|
+
*/
|
|
72
|
+
export declare function coreEventToApiEvent(event: ExecutionEvent): ApiDataflowEvent | null;
|
|
73
|
+
/**
|
|
74
|
+
* Convert e3-core execution status to API status.
|
|
75
|
+
*
|
|
76
|
+
* @param status - The e3-core execution status
|
|
77
|
+
* @returns API-compatible status string
|
|
78
|
+
*/
|
|
79
|
+
export declare function coreStatusToApiStatus(status: DataflowExecutionStatus): ApiExecutionStatus;
|
|
80
|
+
/**
|
|
81
|
+
* Convert e3-core execution state to API-compatible state.
|
|
82
|
+
*
|
|
83
|
+
* @param state - The e3-core execution state
|
|
84
|
+
* @param events - Events to include (already filtered by offset/limit)
|
|
85
|
+
* @param totalEvents - Total number of events for pagination
|
|
86
|
+
* @param duration - Total execution duration in milliseconds
|
|
87
|
+
* @returns API-compatible execution state
|
|
88
|
+
*/
|
|
89
|
+
export declare function coreStateToApiState(state: DataflowExecutionState, events: ExecutionEvent[], totalEvents: number, duration: number): ApiExecutionState;
|
|
90
|
+
//# sourceMappingURL=api-compat.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"api-compat.d.ts","sourceRoot":"","sources":["../../../src/dataflow/api-compat.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH;;;;;;;;;;;;;;GAcG;AAEH,OAAO,KAAK,EACV,cAAc,EACd,uBAAuB,EACvB,sBAAsB,EACvB,MAAM,YAAY,CAAC;AAMpB;;;GAGG;AACH,MAAM,MAAM,oBAAoB,GAC5B,OAAO,GACP,UAAU,GACV,QAAQ,GACR,QAAQ,GACR,OAAO,GACP,mBAAmB,CAAC;AAExB;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,IAAI,EAAE,oBAAoB,CAAC;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED;;GAEG;AACH,MAAM,MAAM,kBAAkB,GAAG,SAAS,GAAG,WAAW,GAAG,QAAQ,GAAG,SAAS,CAAC;AAEhF;;GAEG;AACH,MAAM,WAAW,mBAAmB;IAClC,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,MAAM,EAAE,kBAAkB,CAAC;IAC3B,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,OAAO,EAAE,mBAAmB,GAAG,IAAI,CAAC;IACpC,MAAM,EAAE,gBAAgB,EAAE,CAAC;IAC3B,WAAW,EAAE,MAAM,CAAC;CACrB;AAMD;;;;;;;;GAQG;AACH,wBAAgB,mBAAmB,CAAC,KAAK,EAAE,cAAc,GAAG,gBAAgB,GAAG,IAAI,CA6DlF;AAED;;;;;GAKG;AACH,wBAAgB,qBAAqB,CAAC,MAAM,EAAE,uBAAuB,GAAG,kBAAkB,CAWzF;AAED;;;;;;;;GAQG;AACH,wBAAgB,mBAAmB,CACjC,KAAK,EAAE,sBAAsB,EAC7B,MAAM,EAAE,cAAc,EAAE,EACxB,WAAW,EAAE,MAAM,EACnB,QAAQ,EAAE,MAAM,GACf,iBAAiB,CAmCnB"}
|
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Copyright (c) 2025 Elara AI Pty Ltd
|
|
3
|
+
* Licensed under BSL 1.1. See LICENSE for details.
|
|
4
|
+
*/
|
|
5
|
+
// =============================================================================
|
|
6
|
+
// Conversion Functions
|
|
7
|
+
// =============================================================================
|
|
8
|
+
/**
|
|
9
|
+
* Convert e3-core execution event to API-compatible event.
|
|
10
|
+
*
|
|
11
|
+
* Returns null for events that don't have an API equivalent
|
|
12
|
+
* (e.g., execution_started, task_ready, execution_completed, execution_cancelled).
|
|
13
|
+
*
|
|
14
|
+
* @param event - The e3-core execution event (variant format)
|
|
15
|
+
* @returns API-compatible event or null if no mapping exists
|
|
16
|
+
*/
|
|
17
|
+
export function coreEventToApiEvent(event) {
|
|
18
|
+
switch (event.type) {
|
|
19
|
+
case 'task_started':
|
|
20
|
+
return {
|
|
21
|
+
type: 'start',
|
|
22
|
+
task: event.value.task,
|
|
23
|
+
timestamp: event.value.timestamp.toISOString(),
|
|
24
|
+
};
|
|
25
|
+
case 'task_completed':
|
|
26
|
+
if (event.value.cached) {
|
|
27
|
+
return {
|
|
28
|
+
type: 'cached',
|
|
29
|
+
task: event.value.task,
|
|
30
|
+
timestamp: event.value.timestamp.toISOString(),
|
|
31
|
+
};
|
|
32
|
+
}
|
|
33
|
+
return {
|
|
34
|
+
type: 'complete',
|
|
35
|
+
task: event.value.task,
|
|
36
|
+
timestamp: event.value.timestamp.toISOString(),
|
|
37
|
+
duration: Number(event.value.duration),
|
|
38
|
+
};
|
|
39
|
+
case 'task_failed': {
|
|
40
|
+
// If there's an exit code, treat as 'failed'; otherwise as 'error'
|
|
41
|
+
const exitCode = event.value.exitCode.type === 'some' ? event.value.exitCode.value : undefined;
|
|
42
|
+
const error = event.value.error.type === 'some' ? event.value.error.value : undefined;
|
|
43
|
+
if (exitCode !== undefined) {
|
|
44
|
+
return {
|
|
45
|
+
type: 'failed',
|
|
46
|
+
task: event.value.task,
|
|
47
|
+
timestamp: event.value.timestamp.toISOString(),
|
|
48
|
+
duration: Number(event.value.duration),
|
|
49
|
+
exitCode,
|
|
50
|
+
};
|
|
51
|
+
}
|
|
52
|
+
return {
|
|
53
|
+
type: 'error',
|
|
54
|
+
task: event.value.task,
|
|
55
|
+
timestamp: event.value.timestamp.toISOString(),
|
|
56
|
+
message: error ?? 'Unknown error',
|
|
57
|
+
};
|
|
58
|
+
}
|
|
59
|
+
case 'task_skipped':
|
|
60
|
+
return {
|
|
61
|
+
type: 'input_unavailable',
|
|
62
|
+
task: event.value.task,
|
|
63
|
+
timestamp: event.value.timestamp.toISOString(),
|
|
64
|
+
reason: `Upstream task '${event.value.cause}' failed`,
|
|
65
|
+
};
|
|
66
|
+
// Events without API equivalents
|
|
67
|
+
case 'execution_started':
|
|
68
|
+
case 'task_ready':
|
|
69
|
+
case 'execution_completed':
|
|
70
|
+
case 'execution_cancelled':
|
|
71
|
+
return null;
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
/**
|
|
75
|
+
* Convert e3-core execution status to API status.
|
|
76
|
+
*
|
|
77
|
+
* @param status - The e3-core execution status
|
|
78
|
+
* @returns API-compatible status string
|
|
79
|
+
*/
|
|
80
|
+
export function coreStatusToApiStatus(status) {
|
|
81
|
+
switch (status) {
|
|
82
|
+
case 'running':
|
|
83
|
+
return 'running';
|
|
84
|
+
case 'completed':
|
|
85
|
+
return 'completed';
|
|
86
|
+
case 'failed':
|
|
87
|
+
return 'failed';
|
|
88
|
+
case 'cancelled':
|
|
89
|
+
return 'aborted';
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
/**
|
|
93
|
+
* Convert e3-core execution state to API-compatible state.
|
|
94
|
+
*
|
|
95
|
+
* @param state - The e3-core execution state
|
|
96
|
+
* @param events - Events to include (already filtered by offset/limit)
|
|
97
|
+
* @param totalEvents - Total number of events for pagination
|
|
98
|
+
* @param duration - Total execution duration in milliseconds
|
|
99
|
+
* @returns API-compatible execution state
|
|
100
|
+
*/
|
|
101
|
+
export function coreStateToApiState(state, events, totalEvents, duration) {
|
|
102
|
+
// Convert events, filtering out those without API equivalents
|
|
103
|
+
const apiEvents = [];
|
|
104
|
+
for (const event of events) {
|
|
105
|
+
const apiEvent = coreEventToApiEvent(event);
|
|
106
|
+
if (apiEvent !== null) {
|
|
107
|
+
apiEvents.push(apiEvent);
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
// Build summary if execution is complete
|
|
111
|
+
let summary = null;
|
|
112
|
+
if (state.status !== 'running') {
|
|
113
|
+
summary = {
|
|
114
|
+
executed: state.executed,
|
|
115
|
+
cached: state.cached,
|
|
116
|
+
failed: state.failed,
|
|
117
|
+
skipped: state.skipped,
|
|
118
|
+
duration,
|
|
119
|
+
};
|
|
120
|
+
}
|
|
121
|
+
// Get completedAt value (handle Option type)
|
|
122
|
+
const completedAtValue = state.completedAt.type === 'some'
|
|
123
|
+
? state.completedAt.value.toISOString()
|
|
124
|
+
: null;
|
|
125
|
+
return {
|
|
126
|
+
status: coreStatusToApiStatus(state.status),
|
|
127
|
+
startedAt: state.startedAt.toISOString(),
|
|
128
|
+
completedAt: completedAtValue,
|
|
129
|
+
summary,
|
|
130
|
+
events: apiEvents,
|
|
131
|
+
totalEvents: BigInt(totalEvents),
|
|
132
|
+
};
|
|
133
|
+
}
|
|
134
|
+
//# sourceMappingURL=api-compat.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"api-compat.js","sourceRoot":"","sources":["../../../src/dataflow/api-compat.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAiFH,gFAAgF;AAChF,uBAAuB;AACvB,gFAAgF;AAEhF;;;;;;;;GAQG;AACH,MAAM,UAAU,mBAAmB,CAAC,KAAqB;IACvD,QAAQ,KAAK,CAAC,IAAI,EAAE,CAAC;QACnB,KAAK,cAAc;YACjB,OAAO;gBACL,IAAI,EAAE,OAAO;gBACb,IAAI,EAAE,KAAK,CAAC,KAAK,CAAC,IAAI;gBACtB,SAAS,EAAE,KAAK,CAAC,KAAK,CAAC,SAAS,CAAC,WAAW,EAAE;aAC/C,CAAC;QAEJ,KAAK,gBAAgB;YACnB,IAAI,KAAK,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC;gBACvB,OAAO;oBACL,IAAI,EAAE,QAAQ;oBACd,IAAI,EAAE,KAAK,CAAC,KAAK,CAAC,IAAI;oBACtB,SAAS,EAAE,KAAK,CAAC,KAAK,CAAC,SAAS,CAAC,WAAW,EAAE;iBAC/C,CAAC;YACJ,CAAC;YACD,OAAO;gBACL,IAAI,EAAE,UAAU;gBAChB,IAAI,EAAE,KAAK,CAAC,KAAK,CAAC,IAAI;gBACtB,SAAS,EAAE,KAAK,CAAC,KAAK,CAAC,SAAS,CAAC,WAAW,EAAE;gBAC9C,QAAQ,EAAE,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,QAAQ,CAAC;aACvC,CAAC;QAEJ,KAAK,aAAa,CAAC,CAAC,CAAC;YACnB,mEAAmE;YACnE,MAAM,QAAQ,GAAG,KAAK,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC;YAC/F,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC;YAEtF,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;gBAC3B,OAAO;oBACL,IAAI,EAAE,QAAQ;oBACd,IAAI,EAAE,KAAK,CAAC,KAAK,CAAC,IAAI;oBACtB,SAAS,EAAE,KAAK,CAAC,KAAK,CAAC,SAAS,CAAC,WAAW,EAAE;oBAC9C,QAAQ,EAAE,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,QAAQ,CAAC;oBACtC,QAAQ;iBACT,CAAC;YACJ,CAAC;YACD,OAAO;gBACL,IAAI,EAAE,OAAO;gBACb,IAAI,EAAE,KAAK,CAAC,KAAK,CAAC,IAAI;gBACtB,SAAS,EAAE,KAAK,CAAC,KAAK,CAAC,SAAS,CAAC,WAAW,EAAE;gBAC9C,OAAO,EAAE,KAAK,IAAI,eAAe;aAClC,CAAC;QACJ,CAAC;QAED,KAAK,cAAc;YACjB,OAAO;gBACL,IAAI,EAAE,mBAAmB;gBACzB,IAAI,EAAE,KAAK,CAAC,KAAK,CAAC,IAAI;gBACtB,SAAS,EAAE,KAAK,CAAC,KAAK,CAAC,SAAS,CAAC,WAAW,EAAE;gBAC9C,MAAM,EAAE,kBAAkB,KAAK,CAAC,KAAK,CAAC,KAAK,UAAU;aACtD,CAAC;QAEJ,iCAAiC;QACjC,KAAK,mBAAmB,CAAC;QACzB,KAAK,YAAY,CAAC;QAClB,KAAK,qBAAqB,CAAC;QAC3B,KAAK,qBAAqB;YACxB,OAAO,IAAI,CAAC;IAChB,CAAC;AACH,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,qBAAqB,CAAC,MAA+B;IACnE,QAAQ,MAAM,EAAE,CAAC;QACf,KAAK,SAAS;YACZ,OAAO,SAAS,CAAC;QACnB,KAAK,WAAW;YACd,OAAO,WAAW,CAAC;QACrB,KAAK,QAAQ;YACX,OAAO,QAAQ,CAAC;QAClB,KAAK,WAAW;YACd,OAAO,SAAS,CAAC;IACrB,CAAC;AACH,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,mBAAmB,CACjC,KAA6B,EAC7B,MAAwB,EACxB,WAAmB,EACnB,QAAgB;IAEhB,8DAA8D;IAC9D,MAAM,SAAS,GAAuB,EAAE,CAAC;IACzC,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,MAAM,QAAQ,GAAG,mBAAmB,CAAC,KAAK,CAAC,CAAC;QAC5C,IAAI,QAAQ,KAAK,IAAI,EAAE,CAAC;YACtB,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC3B,CAAC;IACH,CAAC;IAED,yCAAyC;IACzC,IAAI,OAAO,GAA+B,IAAI,CAAC;IAC/C,IAAI,KAAK,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;QAC/B,OAAO,GAAG;YACR,QAAQ,EAAE,KAAK,CAAC,QAAQ;YACxB,MAAM,EAAE,KAAK,CAAC,MAAM;YACpB,MAAM,EAAE,KAAK,CAAC,MAAM;YACpB,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,QAAQ;SACT,CAAC;IACJ,CAAC;IAED,6CAA6C;IAC7C,MAAM,gBAAgB,GAAG,KAAK,CAAC,WAAW,CAAC,IAAI,KAAK,MAAM;QACxD,CAAC,CAAC,KAAK,CAAC,WAAW,CAAC,KAAK,CAAC,WAAW,EAAE;QACvC,CAAC,CAAC,IAAI,CAAC;IAET,OAAO;QACL,MAAM,EAAE,qBAAqB,CAAC,KAAK,CAAC,MAAiC,CAAC;QACtE,SAAS,EAAE,KAAK,CAAC,SAAS,CAAC,WAAW,EAAE;QACxC,WAAW,EAAE,gBAAgB;QAC7B,OAAO;QACP,MAAM,EAAE,SAAS;QACjB,WAAW,EAAE,MAAM,CAAC,WAAW,CAAC;KACjC,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Copyright (c) 2025 Elara AI Pty Ltd
|
|
3
|
+
* Licensed under BSL 1.1. See LICENSE for details.
|
|
4
|
+
*/
|
|
5
|
+
/**
|
|
6
|
+
* Resumable dataflow execution module.
|
|
7
|
+
*
|
|
8
|
+
* This module provides interfaces and implementations for resumable,
|
|
9
|
+
* portable dataflow execution that supports both local and cloud environments.
|
|
10
|
+
*
|
|
11
|
+
* @packageDocumentation
|
|
12
|
+
*/
|
|
13
|
+
export * from './types.js';
|
|
14
|
+
export * from './steps.js';
|
|
15
|
+
export * from './state-store/index.js';
|
|
16
|
+
export * from './orchestrator/index.js';
|
|
17
|
+
export * from './api-compat.js';
|
|
18
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/dataflow/index.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH;;;;;;;GAOG;AAGH,cAAc,YAAY,CAAC;AAG3B,cAAc,YAAY,CAAC;AAG3B,cAAc,wBAAwB,CAAC;AAGvC,cAAc,yBAAyB,CAAC;AAGxC,cAAc,iBAAiB,CAAC"}
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Copyright (c) 2025 Elara AI Pty Ltd
|
|
3
|
+
* Licensed under BSL 1.1. See LICENSE for details.
|
|
4
|
+
*/
|
|
5
|
+
/**
|
|
6
|
+
* Resumable dataflow execution module.
|
|
7
|
+
*
|
|
8
|
+
* This module provides interfaces and implementations for resumable,
|
|
9
|
+
* portable dataflow execution that supports both local and cloud environments.
|
|
10
|
+
*
|
|
11
|
+
* @packageDocumentation
|
|
12
|
+
*/
|
|
13
|
+
// Types
|
|
14
|
+
export * from './types.js';
|
|
15
|
+
// Step functions
|
|
16
|
+
export * from './steps.js';
|
|
17
|
+
// State store
|
|
18
|
+
export * from './state-store/index.js';
|
|
19
|
+
// Orchestrator
|
|
20
|
+
export * from './orchestrator/index.js';
|
|
21
|
+
// API compatibility layer
|
|
22
|
+
export * from './api-compat.js';
|
|
23
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/dataflow/index.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH;;;;;;;GAOG;AAEH,QAAQ;AACR,cAAc,YAAY,CAAC;AAE3B,iBAAiB;AACjB,cAAc,YAAY,CAAC;AAE3B,cAAc;AACd,cAAc,wBAAwB,CAAC;AAEvC,eAAe;AACf,cAAc,yBAAyB,CAAC;AAExC,0BAA0B;AAC1B,cAAc,iBAAiB,CAAC"}
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Copyright (c) 2025 Elara AI Pty Ltd
|
|
3
|
+
* Licensed under BSL 1.1. See LICENSE for details.
|
|
4
|
+
*/
|
|
5
|
+
import type { StorageBackend } from '../../storage/interfaces.js';
|
|
6
|
+
import type { DataflowOrchestrator, ExecutionHandle, ExecutionStatus, OrchestratorStartOptions } from './interfaces.js';
|
|
7
|
+
import type { ExecutionStateStore } from '../state-store/interfaces.js';
|
|
8
|
+
import type { ExecutionEvent, FinalizeResult } from '../types.js';
|
|
9
|
+
/**
|
|
10
|
+
* Local orchestrator for in-process dataflow execution.
|
|
11
|
+
*
|
|
12
|
+
* @remarks
|
|
13
|
+
* - Uses step functions for each operation
|
|
14
|
+
* - Serializes workspace writes with AsyncMutex
|
|
15
|
+
* - Supports AbortSignal for cancellation
|
|
16
|
+
* - Persists state through the provided state store
|
|
17
|
+
*/
|
|
18
|
+
export declare class LocalOrchestrator implements DataflowOrchestrator {
|
|
19
|
+
private readonly stateStore?;
|
|
20
|
+
private executions;
|
|
21
|
+
/**
|
|
22
|
+
* Create a new LocalOrchestrator.
|
|
23
|
+
*
|
|
24
|
+
* @param stateStore - Optional state store for persistence.
|
|
25
|
+
* If not provided, state is only kept in memory.
|
|
26
|
+
*/
|
|
27
|
+
constructor(stateStore?: ExecutionStateStore | undefined);
|
|
28
|
+
start(storage: StorageBackend, repo: string, workspace: string, options?: OrchestratorStartOptions): Promise<ExecutionHandle>;
|
|
29
|
+
wait(handle: ExecutionHandle): Promise<FinalizeResult>;
|
|
30
|
+
getStatus(handle: ExecutionHandle): Promise<ExecutionStatus>;
|
|
31
|
+
cancel(handle: ExecutionHandle): Promise<void>;
|
|
32
|
+
getEvents(handle: ExecutionHandle, sinceSeq: number): Promise<ExecutionEvent[]>;
|
|
33
|
+
/**
|
|
34
|
+
* Main execution loop.
|
|
35
|
+
*
|
|
36
|
+
* Uses step functions to execute tasks, managing concurrency and
|
|
37
|
+
* workspace state updates.
|
|
38
|
+
*/
|
|
39
|
+
private runExecutionLoop;
|
|
40
|
+
/**
|
|
41
|
+
* Execute a single task.
|
|
42
|
+
*/
|
|
43
|
+
private executeTask;
|
|
44
|
+
/**
|
|
45
|
+
* Build partial results for abort error.
|
|
46
|
+
*/
|
|
47
|
+
private buildPartialResults;
|
|
48
|
+
/**
|
|
49
|
+
* Generate unique key for an execution.
|
|
50
|
+
*/
|
|
51
|
+
private executionKey;
|
|
52
|
+
}
|
|
53
|
+
//# sourceMappingURL=LocalOrchestrator.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"LocalOrchestrator.d.ts","sourceRoot":"","sources":["../../../../src/dataflow/orchestrator/LocalOrchestrator.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAUH,OAAO,KAAK,EAAE,cAAc,EAAc,MAAM,6BAA6B,CAAC;AAO9E,OAAO,KAAK,EACV,oBAAoB,EACpB,eAAe,EACf,eAAe,EACf,wBAAwB,EACzB,MAAM,iBAAiB,CAAC;AAEzB,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,8BAA8B,CAAC;AACxE,OAAO,KAAK,EAEV,cAAc,EACd,cAAc,EACf,MAAM,aAAa,CAAC;AA2ErB;;;;;;;;GAQG;AACH,qBAAa,iBAAkB,YAAW,oBAAoB;IAShD,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAC;IARxC,OAAO,CAAC,UAAU,CAAuC;IAEzD;;;;;OAKG;gBAC0B,UAAU,CAAC,EAAE,mBAAmB,YAAA;IAEvD,KAAK,CACT,OAAO,EAAE,cAAc,EACvB,IAAI,EAAE,MAAM,EACZ,SAAS,EAAE,MAAM,EACjB,OAAO,GAAE,wBAA6B,GACrC,OAAO,CAAC,eAAe,CAAC;IA6FrB,IAAI,CAAC,MAAM,EAAE,eAAe,GAAG,OAAO,CAAC,cAAc,CAAC;IAWtD,SAAS,CAAC,MAAM,EAAE,eAAe,GAAG,OAAO,CAAC,eAAe,CAAC;IAkB5D,MAAM,CAAC,MAAM,EAAE,eAAe,GAAG,OAAO,CAAC,IAAI,CAAC;IAuB9C,SAAS,CAAC,MAAM,EAAE,eAAe,EAAE,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,cAAc,EAAE,CAAC;IAOrF;;;;;OAKG;YACW,gBAAgB;IA+M9B;;OAEG;YACW,WAAW;IAgDzB;;OAEG;IACH,OAAO,CAAC,mBAAmB;IAyB3B;;OAEG;IACH,OAAO,CAAC,YAAY;CAGrB"}
|