@elaraai/e3-core 0.0.2-beta.12 → 0.0.2-beta.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/dataflow.d.ts +118 -9
- package/dist/src/dataflow.d.ts.map +1 -1
- package/dist/src/dataflow.js +283 -54
- package/dist/src/dataflow.js.map +1 -1
- package/dist/src/errors.d.ts +11 -6
- package/dist/src/errors.d.ts.map +1 -1
- package/dist/src/errors.js +10 -3
- package/dist/src/errors.js.map +1 -1
- package/dist/src/execution/index.d.ts +14 -0
- package/dist/src/execution/index.d.ts.map +1 -0
- package/dist/src/execution/index.js +6 -0
- package/dist/src/execution/index.js.map +1 -0
- package/dist/src/execution/interfaces.d.ts +244 -0
- package/dist/src/execution/interfaces.d.ts.map +1 -0
- package/dist/src/execution/interfaces.js +6 -0
- package/dist/src/execution/interfaces.js.map +1 -0
- package/dist/src/executions.d.ts +48 -38
- package/dist/src/executions.d.ts.map +1 -1
- package/dist/src/executions.js +117 -162
- package/dist/src/executions.js.map +1 -1
- package/dist/src/gc.d.ts +9 -2
- package/dist/src/gc.d.ts.map +1 -1
- package/dist/src/gc.js +19 -9
- package/dist/src/gc.js.map +1 -1
- package/dist/src/index.d.ts +8 -6
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +22 -5
- package/dist/src/index.js.map +1 -1
- package/dist/src/objects.d.ts +6 -6
- package/dist/src/objects.js +6 -6
- package/dist/src/packages.d.ts +22 -14
- package/dist/src/packages.d.ts.map +1 -1
- package/dist/src/packages.js +45 -79
- package/dist/src/packages.js.map +1 -1
- package/dist/src/repository.d.ts +8 -4
- package/dist/src/repository.d.ts.map +1 -1
- package/dist/src/repository.js +25 -29
- package/dist/src/repository.js.map +1 -1
- package/dist/src/storage/index.d.ts +17 -0
- package/dist/src/storage/index.d.ts.map +1 -0
- package/dist/src/storage/index.js +8 -0
- package/dist/src/storage/index.js.map +1 -0
- package/dist/src/storage/interfaces.d.ts +299 -0
- package/dist/src/storage/interfaces.d.ts.map +1 -0
- package/dist/src/storage/interfaces.js +6 -0
- package/dist/src/storage/interfaces.js.map +1 -0
- package/dist/src/storage/local/LocalBackend.d.ts +51 -0
- package/dist/src/storage/local/LocalBackend.d.ts.map +1 -0
- package/dist/src/storage/local/LocalBackend.js +73 -0
- package/dist/src/storage/local/LocalBackend.js.map +1 -0
- package/dist/src/storage/local/LocalLockService.d.ts +22 -0
- package/dist/src/storage/local/LocalLockService.d.ts.map +1 -0
- package/dist/src/storage/local/LocalLockService.js +38 -0
- package/dist/src/storage/local/LocalLockService.js.map +1 -0
- package/dist/src/storage/local/LocalLogStore.d.ts +23 -0
- package/dist/src/storage/local/LocalLogStore.d.ts.map +1 -0
- package/dist/src/storage/local/LocalLogStore.js +66 -0
- package/dist/src/storage/local/LocalLogStore.js.map +1 -0
- package/dist/src/storage/local/LocalObjectStore.d.ts +19 -0
- package/dist/src/storage/local/LocalObjectStore.d.ts.map +1 -0
- package/dist/src/storage/local/LocalObjectStore.js +68 -0
- package/dist/src/storage/local/LocalObjectStore.js.map +1 -0
- package/dist/src/storage/local/LocalRefStore.d.ts +35 -0
- package/dist/src/storage/local/LocalRefStore.d.ts.map +1 -0
- package/dist/src/storage/local/LocalRefStore.js +233 -0
- package/dist/src/storage/local/LocalRefStore.js.map +1 -0
- package/dist/src/storage/local/index.d.ts +16 -0
- package/dist/src/storage/local/index.d.ts.map +1 -0
- package/dist/src/storage/local/index.js +16 -0
- package/dist/src/storage/local/index.js.map +1 -0
- package/dist/src/tasks.d.ts +16 -10
- package/dist/src/tasks.d.ts.map +1 -1
- package/dist/src/tasks.js +35 -41
- package/dist/src/tasks.js.map +1 -1
- package/dist/src/test-helpers.d.ts +4 -4
- package/dist/src/test-helpers.d.ts.map +1 -1
- package/dist/src/test-helpers.js +6 -20
- package/dist/src/test-helpers.js.map +1 -1
- package/dist/src/trees.d.ts +41 -29
- package/dist/src/trees.d.ts.map +1 -1
- package/dist/src/trees.js +112 -109
- package/dist/src/trees.js.map +1 -1
- package/dist/src/workspaceLock.d.ts +29 -7
- package/dist/src/workspaceLock.d.ts.map +1 -1
- package/dist/src/workspaceLock.js +130 -40
- package/dist/src/workspaceLock.js.map +1 -1
- package/dist/src/workspaceStatus.d.ts +6 -4
- package/dist/src/workspaceStatus.d.ts.map +1 -1
- package/dist/src/workspaceStatus.js +42 -58
- package/dist/src/workspaceStatus.js.map +1 -1
- package/dist/src/workspaces.d.ts +35 -26
- package/dist/src/workspaces.d.ts.map +1 -1
- package/dist/src/workspaces.js +93 -116
- package/dist/src/workspaces.js.map +1 -1
- package/package.json +3 -3
package/dist/src/trees.js
CHANGED
|
@@ -15,14 +15,10 @@
|
|
|
15
15
|
* Low-level operations work with hashes directly (by-hash).
|
|
16
16
|
* High-level operations traverse paths from a root (by-path).
|
|
17
17
|
*/
|
|
18
|
-
import { decodeBeast2, decodeBeast2For, encodeBeast2For, StructType, } from '@elaraai/east';
|
|
18
|
+
import { decodeBeast2, decodeBeast2For, encodeBeast2For, StructType, variant, } from '@elaraai/east';
|
|
19
19
|
import { DataRefType, PackageObjectType, WorkspaceStateType } from '@elaraai/e3-types';
|
|
20
|
-
import { objectRead, objectWrite } from './objects.js';
|
|
21
20
|
import { packageRead } from './packages.js';
|
|
22
|
-
import { WorkspaceNotFoundError, WorkspaceNotDeployedError,
|
|
23
|
-
import { acquireWorkspaceLock } from './workspaceLock.js';
|
|
24
|
-
import * as fs from 'fs/promises';
|
|
25
|
-
import * as path from 'path';
|
|
21
|
+
import { WorkspaceNotFoundError, WorkspaceNotDeployedError, WorkspaceLockError, } from './errors.js';
|
|
26
22
|
/**
|
|
27
23
|
* Build the EastType for a tree object based on its structure.
|
|
28
24
|
*
|
|
@@ -50,32 +46,34 @@ function treeTypeFromStructure(structure) {
|
|
|
50
46
|
/**
|
|
51
47
|
* Read and decode a tree object from the object store.
|
|
52
48
|
*
|
|
53
|
-
* @param
|
|
49
|
+
* @param storage - Storage backend
|
|
50
|
+
* @param repo - Repository identifier
|
|
54
51
|
* @param hash - Hash of the tree object
|
|
55
52
|
* @param structure - The structure describing this tree node's shape
|
|
56
53
|
* @returns The decoded tree object (field name -> DataRef)
|
|
57
54
|
* @throws If object not found, structure is not a tree, or decoding fails
|
|
58
55
|
*/
|
|
59
|
-
export async function treeRead(
|
|
56
|
+
export async function treeRead(storage, repo, hash, structure) {
|
|
60
57
|
const treeType = treeTypeFromStructure(structure);
|
|
61
|
-
const data = await
|
|
58
|
+
const data = await storage.objects.read(repo, hash);
|
|
62
59
|
const decoder = decodeBeast2For(treeType);
|
|
63
60
|
return decoder(Buffer.from(data));
|
|
64
61
|
}
|
|
65
62
|
/**
|
|
66
63
|
* Encode and write a tree object to the object store.
|
|
67
64
|
*
|
|
68
|
-
* @param
|
|
65
|
+
* @param storage - Storage backend
|
|
66
|
+
* @param repo - Repository identifier
|
|
69
67
|
* @param fields - Object mapping field names to DataRefs
|
|
70
68
|
* @param structure - The structure describing this tree node's shape
|
|
71
69
|
* @returns Hash of the written tree object
|
|
72
70
|
* @throws If structure is not a tree or encoding fails
|
|
73
71
|
*/
|
|
74
|
-
export async function treeWrite(
|
|
72
|
+
export async function treeWrite(storage, repo, fields, structure) {
|
|
75
73
|
const treeType = treeTypeFromStructure(structure);
|
|
76
74
|
const encoder = encodeBeast2For(treeType);
|
|
77
75
|
const data = encoder(fields);
|
|
78
|
-
return
|
|
76
|
+
return storage.objects.write(repo, data);
|
|
79
77
|
}
|
|
80
78
|
/**
|
|
81
79
|
* Read and decode a dataset value from the object store.
|
|
@@ -83,43 +81,46 @@ export async function treeWrite(repoPath, fields, structure) {
|
|
|
83
81
|
* The .beast2 format includes type information in the header, so values
|
|
84
82
|
* can be decoded without knowing the schema in advance.
|
|
85
83
|
*
|
|
86
|
-
* @param
|
|
84
|
+
* @param storage - Storage backend
|
|
85
|
+
* @param repo - Repository identifier
|
|
87
86
|
* @param hash - Hash of the dataset value
|
|
88
87
|
* @returns The decoded value and its type
|
|
89
88
|
* @throws If object not found or not a valid beast2 object
|
|
90
89
|
*/
|
|
91
|
-
export async function datasetRead(
|
|
92
|
-
const data = await
|
|
90
|
+
export async function datasetRead(storage, repo, hash) {
|
|
91
|
+
const data = await storage.objects.read(repo, hash);
|
|
93
92
|
const result = decodeBeast2(Buffer.from(data));
|
|
94
93
|
return { type: result.type, value: result.value };
|
|
95
94
|
}
|
|
96
95
|
/**
|
|
97
96
|
* Encode and write a dataset value to the object store.
|
|
98
97
|
*
|
|
99
|
-
* @param
|
|
98
|
+
* @param storage - Storage backend
|
|
99
|
+
* @param repo - Repository identifier
|
|
100
100
|
* @param value - The value to encode
|
|
101
101
|
* @param type - The East type for encoding (EastType or EastTypeValue)
|
|
102
102
|
* @returns Hash of the written dataset value
|
|
103
103
|
*/
|
|
104
|
-
export async function datasetWrite(
|
|
104
|
+
export async function datasetWrite(storage, repo, value, type) {
|
|
105
105
|
// encodeBeast2For accepts both EastType and EastTypeValue, but TypeScript
|
|
106
106
|
// overloads don't support union types directly. Cast to EastTypeValue since
|
|
107
107
|
// that's the more general case and the runtime handles both.
|
|
108
108
|
const encoder = encodeBeast2For(type);
|
|
109
109
|
const data = encoder(value);
|
|
110
|
-
return
|
|
110
|
+
return storage.objects.write(repo, data);
|
|
111
111
|
}
|
|
112
112
|
/**
|
|
113
113
|
* Traverse a tree from root to a path, co-walking structure and data.
|
|
114
114
|
*
|
|
115
|
-
* @param
|
|
115
|
+
* @param storage - Storage backend
|
|
116
|
+
* @param repo - Repository identifier
|
|
116
117
|
* @param rootHash - Hash of the root tree object
|
|
117
118
|
* @param rootStructure - Structure of the root tree
|
|
118
119
|
* @param path - Path to traverse
|
|
119
120
|
* @returns The structure and DataRef at the path location
|
|
120
121
|
* @throws If path is invalid or traversal fails
|
|
121
122
|
*/
|
|
122
|
-
async function traverse(
|
|
123
|
+
async function traverse(storage, repo, rootHash, rootStructure, path) {
|
|
123
124
|
let currentStructure = rootStructure;
|
|
124
125
|
let currentHash = rootHash;
|
|
125
126
|
for (let i = 0; i < path.length; i++) {
|
|
@@ -134,7 +135,7 @@ async function traverse(repoPath, rootHash, rootStructure, path) {
|
|
|
134
135
|
throw new Error(`Cannot descend into non-struct at path '${pathSoFar}'`);
|
|
135
136
|
}
|
|
136
137
|
// Read the current tree object
|
|
137
|
-
const treeObject = await treeRead(
|
|
138
|
+
const treeObject = await treeRead(storage, repo, currentHash, currentStructure);
|
|
138
139
|
// Look up the child ref
|
|
139
140
|
const childRef = treeObject[fieldName];
|
|
140
141
|
if (!childRef) {
|
|
@@ -168,16 +169,17 @@ async function traverse(repoPath, rootHash, rootStructure, path) {
|
|
|
168
169
|
/**
|
|
169
170
|
* List field names at a tree path within a package's data tree.
|
|
170
171
|
*
|
|
171
|
-
* @param
|
|
172
|
+
* @param storage - Storage backend
|
|
173
|
+
* @param repo - Repository identifier
|
|
172
174
|
* @param name - Package name
|
|
173
175
|
* @param version - Package version
|
|
174
176
|
* @param path - Path to the tree node
|
|
175
177
|
* @returns Array of field names at the path
|
|
176
178
|
* @throws If package not found, path invalid, or path points to a dataset
|
|
177
179
|
*/
|
|
178
|
-
export async function packageListTree(
|
|
180
|
+
export async function packageListTree(storage, repo, name, version, path) {
|
|
179
181
|
// Read the package to get root structure and hash
|
|
180
|
-
const pkg = await packageRead(
|
|
182
|
+
const pkg = await packageRead(storage, repo, name, version);
|
|
181
183
|
const rootStructure = pkg.data.structure;
|
|
182
184
|
const rootHash = pkg.data.value;
|
|
183
185
|
if (path.length === 0) {
|
|
@@ -185,11 +187,11 @@ export async function packageListTree(repoPath, name, version, path) {
|
|
|
185
187
|
if (rootStructure.type !== 'struct') {
|
|
186
188
|
throw new Error('Root is not a tree');
|
|
187
189
|
}
|
|
188
|
-
const treeObject = await treeRead(
|
|
190
|
+
const treeObject = await treeRead(storage, repo, rootHash, rootStructure);
|
|
189
191
|
return Object.keys(treeObject);
|
|
190
192
|
}
|
|
191
193
|
// Traverse to the path
|
|
192
|
-
const { structure, ref } = await traverse(
|
|
194
|
+
const { structure, ref } = await traverse(storage, repo, rootHash, rootStructure, path);
|
|
193
195
|
// Must be a tree structure
|
|
194
196
|
if (structure.type !== 'struct') {
|
|
195
197
|
const pathStr = path.map(s => s.value).join('.');
|
|
@@ -201,29 +203,30 @@ export async function packageListTree(repoPath, name, version, path) {
|
|
|
201
203
|
throw new Error(`Path '${pathStr}' has ref type '${ref.type}', expected 'tree'`);
|
|
202
204
|
}
|
|
203
205
|
// Read the tree and return field names
|
|
204
|
-
const treeObject = await treeRead(
|
|
206
|
+
const treeObject = await treeRead(storage, repo, ref.value, structure);
|
|
205
207
|
return Object.keys(treeObject);
|
|
206
208
|
}
|
|
207
209
|
/**
|
|
208
210
|
* Read and decode a dataset value at a path within a package's data tree.
|
|
209
211
|
*
|
|
210
|
-
* @param
|
|
212
|
+
* @param storage - Storage backend
|
|
213
|
+
* @param repo - Repository identifier
|
|
211
214
|
* @param name - Package name
|
|
212
215
|
* @param version - Package version
|
|
213
216
|
* @param path - Path to the dataset
|
|
214
217
|
* @returns The decoded dataset value
|
|
215
218
|
* @throws If package not found, path invalid, or path points to a tree
|
|
216
219
|
*/
|
|
217
|
-
export async function packageGetDataset(
|
|
220
|
+
export async function packageGetDataset(storage, repo, name, version, path) {
|
|
218
221
|
// Read the package to get root structure and hash
|
|
219
|
-
const pkg = await packageRead(
|
|
222
|
+
const pkg = await packageRead(storage, repo, name, version);
|
|
220
223
|
const rootStructure = pkg.data.structure;
|
|
221
224
|
const rootHash = pkg.data.value;
|
|
222
225
|
if (path.length === 0) {
|
|
223
226
|
throw new Error('Cannot get dataset at root path - root is always a tree');
|
|
224
227
|
}
|
|
225
228
|
// Traverse to the path
|
|
226
|
-
const { structure, ref } = await traverse(
|
|
229
|
+
const { structure, ref } = await traverse(storage, repo, rootHash, rootStructure, path);
|
|
227
230
|
// Must be a value structure
|
|
228
231
|
if (structure.type !== 'value') {
|
|
229
232
|
const pathStr = path.map(s => s.value).join('.');
|
|
@@ -241,7 +244,7 @@ export async function packageGetDataset(repoPath, name, version, path) {
|
|
|
241
244
|
throw new Error(`Path '${pathStr}' structure says value but ref is tree`);
|
|
242
245
|
}
|
|
243
246
|
// Read and return the dataset value
|
|
244
|
-
const result = await datasetRead(
|
|
247
|
+
const result = await datasetRead(storage, repo, ref.value);
|
|
245
248
|
return result.value;
|
|
246
249
|
}
|
|
247
250
|
/**
|
|
@@ -253,7 +256,8 @@ export async function packageGetDataset(repoPath, name, version, path) {
|
|
|
253
256
|
* Acquires an exclusive lock on the workspace for the duration of the write
|
|
254
257
|
* to prevent concurrent modifications.
|
|
255
258
|
*
|
|
256
|
-
* @param
|
|
259
|
+
* @param storage - Storage backend
|
|
260
|
+
* @param repo - Repository identifier
|
|
257
261
|
* @param ws - Workspace name
|
|
258
262
|
* @param treePath - Path to the dataset
|
|
259
263
|
* @param value - The new value to write
|
|
@@ -262,15 +266,25 @@ export async function packageGetDataset(repoPath, name, version, path) {
|
|
|
262
266
|
* @throws {WorkspaceLockError} If workspace is locked by another process
|
|
263
267
|
* @throws If workspace not deployed, path invalid, or path points to a tree
|
|
264
268
|
*/
|
|
265
|
-
export async function workspaceSetDataset(
|
|
269
|
+
export async function workspaceSetDataset(storage, repo, ws, treePath, value, type, options = {}) {
|
|
266
270
|
if (treePath.length === 0) {
|
|
267
271
|
throw new Error('Cannot set dataset at root path - root is always a tree');
|
|
268
272
|
}
|
|
269
273
|
// Acquire lock if not provided externally
|
|
270
274
|
const externalLock = options.lock;
|
|
271
|
-
|
|
275
|
+
let lock = externalLock ?? null;
|
|
276
|
+
if (!lock) {
|
|
277
|
+
lock = await storage.locks.acquire(repo, ws, variant('dataset_write', null));
|
|
278
|
+
if (!lock) {
|
|
279
|
+
const state = await storage.locks.getState(repo, ws);
|
|
280
|
+
throw new WorkspaceLockError(ws, state ? {
|
|
281
|
+
acquiredAt: state.acquiredAt.toISOString(),
|
|
282
|
+
operation: state.operation.type,
|
|
283
|
+
} : undefined);
|
|
284
|
+
}
|
|
285
|
+
}
|
|
272
286
|
try {
|
|
273
|
-
await workspaceSetDatasetUnlocked(
|
|
287
|
+
await workspaceSetDatasetUnlocked(storage, repo, ws, treePath, value, type);
|
|
274
288
|
}
|
|
275
289
|
finally {
|
|
276
290
|
// Only release the lock if we acquired it internally
|
|
@@ -283,10 +297,10 @@ export async function workspaceSetDataset(repoPath, ws, treePath, value, type, o
|
|
|
283
297
|
* Internal: Update a dataset without acquiring a lock.
|
|
284
298
|
* Caller must hold the workspace lock.
|
|
285
299
|
*/
|
|
286
|
-
async function workspaceSetDatasetUnlocked(
|
|
287
|
-
const state = await readWorkspaceState(
|
|
300
|
+
async function workspaceSetDatasetUnlocked(storage, repo, ws, treePath, value, type) {
|
|
301
|
+
const state = await readWorkspaceState(storage, repo, ws);
|
|
288
302
|
// Read the deployed package object to get the structure
|
|
289
|
-
const pkgData = await
|
|
303
|
+
const pkgData = await storage.objects.read(repo, state.packageHash);
|
|
290
304
|
const decoder = decodeBeast2For(PackageObjectType);
|
|
291
305
|
const pkgObject = decoder(Buffer.from(pkgData));
|
|
292
306
|
const rootStructure = pkgObject.data.structure;
|
|
@@ -315,7 +329,7 @@ async function workspaceSetDatasetUnlocked(repoPath, ws, treePath, value, type)
|
|
|
315
329
|
throw new Error(`Path '${pathStr}' points to a tree, not a dataset`);
|
|
316
330
|
}
|
|
317
331
|
// Write the new dataset value
|
|
318
|
-
const newValueHash = await datasetWrite(
|
|
332
|
+
const newValueHash = await datasetWrite(storage, repo, value, type);
|
|
319
333
|
// Now rebuild the tree path from leaf to root (structural sharing)
|
|
320
334
|
// We need to read each tree along the path, modify it, and write a new version
|
|
321
335
|
// Collect all tree hashes and structures along the path
|
|
@@ -326,7 +340,7 @@ async function workspaceSetDatasetUnlocked(repoPath, ws, treePath, value, type)
|
|
|
326
340
|
for (let i = 0; i < treePath.length - 1; i++) {
|
|
327
341
|
treeInfos.push({ hash: currentHash, structure: currentStructure });
|
|
328
342
|
const segment = treePath[i];
|
|
329
|
-
const treeObject = await treeRead(
|
|
343
|
+
const treeObject = await treeRead(storage, repo, currentHash, currentStructure);
|
|
330
344
|
const childRef = treeObject[segment.value];
|
|
331
345
|
if (!childRef || childRef.type !== 'tree') {
|
|
332
346
|
throw new Error(`Expected tree ref at path segment ${i}`);
|
|
@@ -343,14 +357,14 @@ async function workspaceSetDatasetUnlocked(repoPath, ws, treePath, value, type)
|
|
|
343
357
|
const { hash, structure } = treeInfos[i];
|
|
344
358
|
const fieldName = treePath[i].value;
|
|
345
359
|
// Read the current tree
|
|
346
|
-
const treeObject = await treeRead(
|
|
360
|
+
const treeObject = await treeRead(storage, repo, hash, structure);
|
|
347
361
|
// Create modified tree with the new ref
|
|
348
362
|
const newTreeObject = {
|
|
349
363
|
...treeObject,
|
|
350
364
|
[fieldName]: newRef,
|
|
351
365
|
};
|
|
352
366
|
// Write the new tree
|
|
353
|
-
const newTreeHash = await treeWrite(
|
|
367
|
+
const newTreeHash = await treeWrite(storage, repo, newTreeObject, structure);
|
|
354
368
|
// This becomes the new ref for the parent
|
|
355
369
|
newRef = { type: 'tree', value: newTreeHash };
|
|
356
370
|
}
|
|
@@ -361,7 +375,7 @@ async function workspaceSetDatasetUnlocked(repoPath, ws, treePath, value, type)
|
|
|
361
375
|
}
|
|
362
376
|
const newRootHash = newRef.value;
|
|
363
377
|
// Update workspace state atomically
|
|
364
|
-
await writeWorkspaceState(
|
|
378
|
+
await writeWorkspaceState(storage, repo, ws, {
|
|
365
379
|
...state,
|
|
366
380
|
rootHash: newRootHash,
|
|
367
381
|
rootUpdatedAt: new Date(),
|
|
@@ -373,51 +387,35 @@ async function workspaceSetDatasetUnlocked(repoPath, ws, treePath, value, type)
|
|
|
373
387
|
/**
|
|
374
388
|
* Write workspace state to file atomically.
|
|
375
389
|
*/
|
|
376
|
-
async function writeWorkspaceState(
|
|
377
|
-
const wsDir = path.join(repoPath, 'workspaces');
|
|
378
|
-
const stateFile = path.join(wsDir, `${ws}.beast2`);
|
|
379
|
-
// Ensure workspaces directory exists
|
|
380
|
-
await fs.mkdir(wsDir, { recursive: true });
|
|
390
|
+
async function writeWorkspaceState(storage, repo, ws, state) {
|
|
381
391
|
const encoder = encodeBeast2For(WorkspaceStateType);
|
|
382
392
|
const data = encoder(state);
|
|
383
|
-
|
|
384
|
-
const randomSuffix = Math.random().toString(36).slice(2, 10);
|
|
385
|
-
const tempPath = path.join(wsDir, `.${ws}.${Date.now()}.${randomSuffix}.tmp`);
|
|
386
|
-
await fs.writeFile(tempPath, data);
|
|
387
|
-
await fs.rename(tempPath, stateFile);
|
|
393
|
+
await storage.refs.workspaceWrite(repo, ws, data);
|
|
388
394
|
}
|
|
389
395
|
/**
|
|
390
396
|
* Read workspace state from file.
|
|
391
397
|
* @throws {WorkspaceNotFoundError} If workspace doesn't exist
|
|
392
398
|
* @throws {WorkspaceNotDeployedError} If workspace exists but not deployed
|
|
393
399
|
*/
|
|
394
|
-
async function readWorkspaceState(
|
|
395
|
-
const
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
const decoder = decodeBeast2For(WorkspaceStateType);
|
|
402
|
-
return decoder(data);
|
|
403
|
-
}
|
|
404
|
-
catch (err) {
|
|
405
|
-
if (err instanceof WorkspaceNotDeployedError)
|
|
406
|
-
throw err;
|
|
407
|
-
if (isNotFoundError(err)) {
|
|
408
|
-
throw new WorkspaceNotFoundError(ws);
|
|
409
|
-
}
|
|
410
|
-
throw err;
|
|
400
|
+
async function readWorkspaceState(storage, repo, ws) {
|
|
401
|
+
const data = await storage.refs.workspaceRead(repo, ws);
|
|
402
|
+
if (data === null) {
|
|
403
|
+
throw new WorkspaceNotFoundError(ws);
|
|
404
|
+
}
|
|
405
|
+
if (data.length === 0) {
|
|
406
|
+
throw new WorkspaceNotDeployedError(ws);
|
|
411
407
|
}
|
|
408
|
+
const decoder = decodeBeast2For(WorkspaceStateType);
|
|
409
|
+
return decoder(data);
|
|
412
410
|
}
|
|
413
411
|
/**
|
|
414
412
|
* Get root structure and hash for a workspace.
|
|
415
413
|
* Reads the deployed package object to get the structure.
|
|
416
414
|
*/
|
|
417
|
-
async function getWorkspaceRootInfo(
|
|
418
|
-
const state = await readWorkspaceState(
|
|
415
|
+
async function getWorkspaceRootInfo(storage, repo, ws) {
|
|
416
|
+
const state = await readWorkspaceState(storage, repo, ws);
|
|
419
417
|
// Read the deployed package object using the stored hash
|
|
420
|
-
const pkgData = await
|
|
418
|
+
const pkgData = await storage.objects.read(repo, state.packageHash);
|
|
421
419
|
const decoder = decodeBeast2For(PackageObjectType);
|
|
422
420
|
const pkgObject = decoder(Buffer.from(pkgData));
|
|
423
421
|
return {
|
|
@@ -431,24 +429,25 @@ async function getWorkspaceRootInfo(repoPath, ws) {
|
|
|
431
429
|
/**
|
|
432
430
|
* List field names at a tree path within a workspace's data tree.
|
|
433
431
|
*
|
|
434
|
-
* @param
|
|
432
|
+
* @param storage - Storage backend
|
|
433
|
+
* @param repo - Repository identifier
|
|
435
434
|
* @param ws - Workspace name
|
|
436
435
|
* @param path - Path to the tree node
|
|
437
436
|
* @returns Array of field names at the path
|
|
438
437
|
* @throws If workspace not deployed, path invalid, or path points to a dataset
|
|
439
438
|
*/
|
|
440
|
-
export async function workspaceListTree(
|
|
441
|
-
const { rootHash, rootStructure } = await getWorkspaceRootInfo(
|
|
439
|
+
export async function workspaceListTree(storage, repo, ws, treePath) {
|
|
440
|
+
const { rootHash, rootStructure } = await getWorkspaceRootInfo(storage, repo, ws);
|
|
442
441
|
if (treePath.length === 0) {
|
|
443
442
|
// Empty path - list root tree fields
|
|
444
443
|
if (rootStructure.type !== 'struct') {
|
|
445
444
|
throw new Error('Root is not a tree');
|
|
446
445
|
}
|
|
447
|
-
const treeObject = await treeRead(
|
|
446
|
+
const treeObject = await treeRead(storage, repo, rootHash, rootStructure);
|
|
448
447
|
return Object.keys(treeObject);
|
|
449
448
|
}
|
|
450
449
|
// Traverse to the path
|
|
451
|
-
const { structure, ref } = await traverse(
|
|
450
|
+
const { structure, ref } = await traverse(storage, repo, rootHash, rootStructure, treePath);
|
|
452
451
|
// Must be a tree structure
|
|
453
452
|
if (structure.type !== 'struct') {
|
|
454
453
|
const pathStr = treePath.map(s => s.value).join('.');
|
|
@@ -460,25 +459,26 @@ export async function workspaceListTree(repoPath, ws, treePath) {
|
|
|
460
459
|
throw new Error(`Path '${pathStr}' has ref type '${ref.type}', expected 'tree'`);
|
|
461
460
|
}
|
|
462
461
|
// Read the tree and return field names
|
|
463
|
-
const treeObject = await treeRead(
|
|
462
|
+
const treeObject = await treeRead(storage, repo, ref.value, structure);
|
|
464
463
|
return Object.keys(treeObject);
|
|
465
464
|
}
|
|
466
465
|
/**
|
|
467
466
|
* Read and decode a dataset value at a path within a workspace's data tree.
|
|
468
467
|
*
|
|
469
|
-
* @param
|
|
468
|
+
* @param storage - Storage backend
|
|
469
|
+
* @param repo - Repository identifier
|
|
470
470
|
* @param ws - Workspace name
|
|
471
471
|
* @param path - Path to the dataset
|
|
472
472
|
* @returns The decoded dataset value
|
|
473
473
|
* @throws If workspace not deployed, path invalid, or path points to a tree
|
|
474
474
|
*/
|
|
475
|
-
export async function workspaceGetDataset(
|
|
476
|
-
const { rootHash, rootStructure } = await getWorkspaceRootInfo(
|
|
475
|
+
export async function workspaceGetDataset(storage, repo, ws, treePath) {
|
|
476
|
+
const { rootHash, rootStructure } = await getWorkspaceRootInfo(storage, repo, ws);
|
|
477
477
|
if (treePath.length === 0) {
|
|
478
478
|
throw new Error('Cannot get dataset at root path - root is always a tree');
|
|
479
479
|
}
|
|
480
480
|
// Traverse to the path
|
|
481
|
-
const { structure, ref } = await traverse(
|
|
481
|
+
const { structure, ref } = await traverse(storage, repo, rootHash, rootStructure, treePath);
|
|
482
482
|
// Must be a value structure
|
|
483
483
|
if (structure.type !== 'value') {
|
|
484
484
|
const pathStr = treePath.map(s => s.value).join('.');
|
|
@@ -496,7 +496,7 @@ export async function workspaceGetDataset(repoPath, ws, treePath) {
|
|
|
496
496
|
throw new Error(`Path '${pathStr}' structure says value but ref is tree`);
|
|
497
497
|
}
|
|
498
498
|
// Read and return the dataset value
|
|
499
|
-
const result = await datasetRead(
|
|
499
|
+
const result = await datasetRead(storage, repo, ref.value);
|
|
500
500
|
return result.value;
|
|
501
501
|
}
|
|
502
502
|
/**
|
|
@@ -505,19 +505,20 @@ export async function workspaceGetDataset(repoPath, ws, treePath) {
|
|
|
505
505
|
* Unlike workspaceGetDataset which decodes the value, this returns the raw
|
|
506
506
|
* hash reference. Useful for dataflow execution which operates on hashes.
|
|
507
507
|
*
|
|
508
|
-
* @param
|
|
508
|
+
* @param storage - Storage backend
|
|
509
|
+
* @param repo - Repository identifier
|
|
509
510
|
* @param ws - Workspace name
|
|
510
511
|
* @param treePath - Path to the dataset
|
|
511
512
|
* @returns Object with ref type and hash (null for unassigned/null refs)
|
|
512
513
|
* @throws If workspace not deployed, path invalid, or path points to a tree
|
|
513
514
|
*/
|
|
514
|
-
export async function workspaceGetDatasetHash(
|
|
515
|
-
const { rootHash, rootStructure } = await getWorkspaceRootInfo(
|
|
515
|
+
export async function workspaceGetDatasetHash(storage, repo, ws, treePath) {
|
|
516
|
+
const { rootHash, rootStructure } = await getWorkspaceRootInfo(storage, repo, ws);
|
|
516
517
|
if (treePath.length === 0) {
|
|
517
518
|
throw new Error('Cannot get dataset at root path - root is always a tree');
|
|
518
519
|
}
|
|
519
520
|
// Traverse to the path
|
|
520
|
-
const { structure, ref } = await traverse(
|
|
521
|
+
const { structure, ref } = await traverse(storage, repo, rootHash, rootStructure, treePath);
|
|
521
522
|
// Must be a value structure
|
|
522
523
|
if (structure.type !== 'value') {
|
|
523
524
|
const pathStr = treePath.map(s => s.value).join('.');
|
|
@@ -540,23 +541,24 @@ export async function workspaceGetDatasetHash(repoPath, ws, treePath) {
|
|
|
540
541
|
* directly. Useful for dataflow execution which already has the output hash.
|
|
541
542
|
*
|
|
542
543
|
* IMPORTANT: This function does NOT acquire a workspace lock. The caller must
|
|
543
|
-
* hold an exclusive lock on the workspace
|
|
544
|
-
*
|
|
545
|
-
*
|
|
544
|
+
* hold an exclusive lock on the workspace before calling this function. This
|
|
545
|
+
* is typically used by dataflowExecute which holds the lock for the entire
|
|
546
|
+
* execution.
|
|
546
547
|
*
|
|
547
|
-
* @param
|
|
548
|
+
* @param storage - Storage backend
|
|
549
|
+
* @param repo - Repository identifier
|
|
548
550
|
* @param ws - Workspace name
|
|
549
551
|
* @param treePath - Path to the dataset
|
|
550
552
|
* @param valueHash - Hash of the dataset value already in the object store
|
|
551
553
|
* @throws If workspace not deployed, path invalid, or path points to a tree
|
|
552
554
|
*/
|
|
553
|
-
export async function workspaceSetDatasetByHash(
|
|
555
|
+
export async function workspaceSetDatasetByHash(storage, repo, ws, treePath, valueHash) {
|
|
554
556
|
if (treePath.length === 0) {
|
|
555
557
|
throw new Error('Cannot set dataset at root path - root is always a tree');
|
|
556
558
|
}
|
|
557
|
-
const state = await readWorkspaceState(
|
|
559
|
+
const state = await readWorkspaceState(storage, repo, ws);
|
|
558
560
|
// Read the deployed package object to get the structure
|
|
559
|
-
const pkgData = await
|
|
561
|
+
const pkgData = await storage.objects.read(repo, state.packageHash);
|
|
560
562
|
const decoder = decodeBeast2For(PackageObjectType);
|
|
561
563
|
const pkgObject = decoder(Buffer.from(pkgData));
|
|
562
564
|
const rootStructure = pkgObject.data.structure;
|
|
@@ -593,7 +595,7 @@ export async function workspaceSetDatasetByHash(repoPath, ws, treePath, valueHas
|
|
|
593
595
|
for (let i = 0; i < treePath.length - 1; i++) {
|
|
594
596
|
treeInfos.push({ hash: currentHash, structure: currentStructure });
|
|
595
597
|
const segment = treePath[i];
|
|
596
|
-
const treeObject = await treeRead(
|
|
598
|
+
const treeObject = await treeRead(storage, repo, currentHash, currentStructure);
|
|
597
599
|
const childRef = treeObject[segment.value];
|
|
598
600
|
if (!childRef || childRef.type !== 'tree') {
|
|
599
601
|
throw new Error(`Expected tree ref at path segment ${i}`);
|
|
@@ -610,14 +612,14 @@ export async function workspaceSetDatasetByHash(repoPath, ws, treePath, valueHas
|
|
|
610
612
|
const { hash, structure } = treeInfos[i];
|
|
611
613
|
const fieldName = treePath[i].value;
|
|
612
614
|
// Read the current tree
|
|
613
|
-
const treeObject = await treeRead(
|
|
615
|
+
const treeObject = await treeRead(storage, repo, hash, structure);
|
|
614
616
|
// Create modified tree with the new ref
|
|
615
617
|
const newTreeObject = {
|
|
616
618
|
...treeObject,
|
|
617
619
|
[fieldName]: newRef,
|
|
618
620
|
};
|
|
619
621
|
// Write the new tree
|
|
620
|
-
const newTreeHash = await treeWrite(
|
|
622
|
+
const newTreeHash = await treeWrite(storage, repo, newTreeObject, structure);
|
|
621
623
|
// This becomes the new ref for the parent
|
|
622
624
|
newRef = { type: 'tree', value: newTreeHash };
|
|
623
625
|
}
|
|
@@ -627,7 +629,7 @@ export async function workspaceSetDatasetByHash(repoPath, ws, treePath, valueHas
|
|
|
627
629
|
}
|
|
628
630
|
const newRootHash = newRef.value;
|
|
629
631
|
// Update workspace state atomically
|
|
630
|
-
await writeWorkspaceState(
|
|
632
|
+
await writeWorkspaceState(storage, repo, ws, {
|
|
631
633
|
...state,
|
|
632
634
|
rootHash: newRootHash,
|
|
633
635
|
rootUpdatedAt: new Date(),
|
|
@@ -660,25 +662,26 @@ function getTaskOutputTypeFromStructure(structure) {
|
|
|
660
662
|
* Recursively walks the tree and returns a hierarchical structure
|
|
661
663
|
* suitable for display. Tasks are shown as leaves with their output type.
|
|
662
664
|
*
|
|
663
|
-
* @param
|
|
665
|
+
* @param storage - Storage backend
|
|
666
|
+
* @param repo - Repository identifier
|
|
664
667
|
* @param ws - Workspace name
|
|
665
668
|
* @param treePath - Path to start from (empty for root)
|
|
666
669
|
* @param options - Optional settings for depth limit and type inclusion
|
|
667
670
|
* @returns Array of tree nodes at the path
|
|
668
671
|
* @throws If workspace not deployed or path invalid
|
|
669
672
|
*/
|
|
670
|
-
export async function workspaceGetTree(
|
|
671
|
-
const { rootHash, rootStructure } = await getWorkspaceRootInfo(
|
|
673
|
+
export async function workspaceGetTree(storage, repo, ws, treePath, options = {}) {
|
|
674
|
+
const { rootHash, rootStructure } = await getWorkspaceRootInfo(storage, repo, ws);
|
|
672
675
|
const { maxDepth, includeTypes } = options;
|
|
673
676
|
// If path is empty, start from root
|
|
674
677
|
if (treePath.length === 0) {
|
|
675
678
|
if (rootStructure.type !== 'struct') {
|
|
676
679
|
throw new Error('Root is not a tree');
|
|
677
680
|
}
|
|
678
|
-
return walkTree(
|
|
681
|
+
return walkTree(storage, repo, rootHash, rootStructure, 0, maxDepth, includeTypes);
|
|
679
682
|
}
|
|
680
683
|
// Traverse to the path first
|
|
681
|
-
const { structure, ref } = await traverse(
|
|
684
|
+
const { structure, ref } = await traverse(storage, repo, rootHash, rootStructure, treePath);
|
|
682
685
|
// Must be a tree structure
|
|
683
686
|
if (structure.type !== 'struct') {
|
|
684
687
|
const pathStr = treePath.map(s => s.value).join('.');
|
|
@@ -689,16 +692,16 @@ export async function workspaceGetTree(repoPath, ws, treePath, options = {}) {
|
|
|
689
692
|
const pathStr = treePath.map(s => s.value).join('.');
|
|
690
693
|
throw new Error(`Path '${pathStr}' has ref type '${ref.type}', expected 'tree'`);
|
|
691
694
|
}
|
|
692
|
-
return walkTree(
|
|
695
|
+
return walkTree(storage, repo, ref.value, structure, 0, maxDepth, includeTypes);
|
|
693
696
|
}
|
|
694
697
|
/**
|
|
695
698
|
* Recursively walk a tree and build TreeNode array.
|
|
696
699
|
*/
|
|
697
|
-
async function walkTree(
|
|
700
|
+
async function walkTree(storage, repo, treeHash, structure, currentDepth, maxDepth, includeTypes) {
|
|
698
701
|
if (structure.type !== 'struct') {
|
|
699
702
|
throw new Error('Expected struct structure for tree walk');
|
|
700
703
|
}
|
|
701
|
-
const treeObject = await treeRead(
|
|
704
|
+
const treeObject = await treeRead(storage, repo, treeHash, structure);
|
|
702
705
|
const nodes = [];
|
|
703
706
|
for (const [fieldName, childRef] of Object.entries(treeObject)) {
|
|
704
707
|
const childStructure = structure.value.get(fieldName);
|
|
@@ -731,7 +734,7 @@ async function walkTree(repoPath, treeHash, structure, currentDepth, maxDepth, i
|
|
|
731
734
|
// Recurse if we haven't hit max depth
|
|
732
735
|
if (maxDepth === undefined || currentDepth < maxDepth) {
|
|
733
736
|
if (childRef.type === 'tree') {
|
|
734
|
-
children = await walkTree(
|
|
737
|
+
children = await walkTree(storage, repo, childRef.value, childStructure, currentDepth + 1, maxDepth, includeTypes);
|
|
735
738
|
}
|
|
736
739
|
}
|
|
737
740
|
const node = {
|