@elaraai/e3-core 0.0.2-beta.20 → 0.0.2-beta.21
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +25 -22
- package/dist/src/dataflow/api-compat.d.ts +90 -0
- package/dist/src/dataflow/api-compat.d.ts.map +1 -0
- package/dist/src/dataflow/api-compat.js +134 -0
- package/dist/src/dataflow/api-compat.js.map +1 -0
- package/dist/src/dataflow/index.d.ts +18 -0
- package/dist/src/dataflow/index.d.ts.map +1 -0
- package/dist/src/dataflow/index.js +23 -0
- package/dist/src/dataflow/index.js.map +1 -0
- package/dist/src/dataflow/orchestrator/LocalOrchestrator.d.ts +53 -0
- package/dist/src/dataflow/orchestrator/LocalOrchestrator.d.ts.map +1 -0
- package/dist/src/dataflow/orchestrator/LocalOrchestrator.js +416 -0
- package/dist/src/dataflow/orchestrator/LocalOrchestrator.js.map +1 -0
- package/dist/src/dataflow/orchestrator/index.d.ts +12 -0
- package/dist/src/dataflow/orchestrator/index.d.ts.map +1 -0
- package/dist/src/dataflow/orchestrator/index.js +12 -0
- package/dist/src/dataflow/orchestrator/index.js.map +1 -0
- package/dist/src/dataflow/orchestrator/interfaces.d.ts +157 -0
- package/dist/src/dataflow/orchestrator/interfaces.d.ts.map +1 -0
- package/dist/src/dataflow/orchestrator/interfaces.js +51 -0
- package/dist/src/dataflow/orchestrator/interfaces.js.map +1 -0
- package/dist/src/dataflow/state-store/FileStateStore.d.ts +67 -0
- package/dist/src/dataflow/state-store/FileStateStore.d.ts.map +1 -0
- package/dist/src/dataflow/state-store/FileStateStore.js +286 -0
- package/dist/src/dataflow/state-store/FileStateStore.js.map +1 -0
- package/dist/src/dataflow/state-store/InMemoryStateStore.d.ts +42 -0
- package/dist/src/dataflow/state-store/InMemoryStateStore.d.ts.map +1 -0
- package/dist/src/dataflow/state-store/InMemoryStateStore.js +214 -0
- package/dist/src/dataflow/state-store/InMemoryStateStore.js.map +1 -0
- package/dist/src/dataflow/state-store/index.d.ts +13 -0
- package/dist/src/dataflow/state-store/index.d.ts.map +1 -0
- package/dist/src/dataflow/state-store/index.js +13 -0
- package/dist/src/dataflow/state-store/index.js.map +1 -0
- package/dist/src/dataflow/state-store/interfaces.d.ts +159 -0
- package/dist/src/dataflow/state-store/interfaces.d.ts.map +1 -0
- package/dist/src/dataflow/state-store/interfaces.js +6 -0
- package/dist/src/dataflow/state-store/interfaces.js.map +1 -0
- package/dist/src/dataflow/steps.d.ts +176 -0
- package/dist/src/dataflow/steps.d.ts.map +1 -0
- package/dist/src/dataflow/steps.js +528 -0
- package/dist/src/dataflow/steps.js.map +1 -0
- package/dist/src/dataflow/types.d.ts +116 -0
- package/dist/src/dataflow/types.d.ts.map +1 -0
- package/dist/src/dataflow/types.js +7 -0
- package/dist/src/dataflow/types.js.map +1 -0
- package/dist/src/dataflow.d.ts +9 -0
- package/dist/src/dataflow.d.ts.map +1 -1
- package/dist/src/dataflow.js +11 -6
- package/dist/src/dataflow.js.map +1 -1
- package/dist/src/execution/LocalTaskRunner.d.ts +71 -0
- package/dist/src/execution/LocalTaskRunner.d.ts.map +1 -0
- package/dist/src/execution/LocalTaskRunner.js +360 -0
- package/dist/src/execution/LocalTaskRunner.js.map +1 -0
- package/dist/src/execution/MockTaskRunner.d.ts +49 -0
- package/dist/src/execution/MockTaskRunner.d.ts.map +1 -0
- package/dist/src/execution/MockTaskRunner.js +55 -0
- package/dist/src/execution/MockTaskRunner.js.map +1 -0
- package/dist/src/execution/index.d.ts +2 -0
- package/dist/src/execution/index.d.ts.map +1 -1
- package/dist/src/execution/index.js +3 -1
- package/dist/src/execution/index.js.map +1 -1
- package/dist/src/execution/processHelpers.d.ts +20 -0
- package/dist/src/execution/processHelpers.d.ts.map +1 -0
- package/dist/src/execution/processHelpers.js +62 -0
- package/dist/src/execution/processHelpers.js.map +1 -0
- package/dist/src/executions.d.ts +1 -69
- package/dist/src/executions.d.ts.map +1 -1
- package/dist/src/executions.js +6 -365
- package/dist/src/executions.js.map +1 -1
- package/dist/src/index.d.ts +10 -5
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +19 -12
- package/dist/src/index.js.map +1 -1
- package/dist/src/objects.d.ts +6 -53
- package/dist/src/objects.d.ts.map +1 -1
- package/dist/src/objects.js +11 -232
- package/dist/src/objects.js.map +1 -1
- package/dist/src/storage/local/LocalLockService.d.ts +84 -1
- package/dist/src/storage/local/LocalLockService.d.ts.map +1 -1
- package/dist/src/storage/local/LocalLockService.js +305 -1
- package/dist/src/storage/local/LocalLockService.js.map +1 -1
- package/dist/src/storage/local/LocalObjectStore.d.ts +33 -1
- package/dist/src/storage/local/LocalObjectStore.d.ts.map +1 -1
- package/dist/src/storage/local/LocalObjectStore.js +198 -3
- package/dist/src/storage/local/LocalObjectStore.js.map +1 -1
- package/dist/src/{gc.d.ts → storage/local/gc.d.ts} +1 -1
- package/dist/src/storage/local/gc.d.ts.map +1 -0
- package/dist/src/{gc.js → storage/local/gc.js} +5 -2
- package/dist/src/storage/local/gc.js.map +1 -0
- package/dist/src/storage/local/localHelpers.d.ts +25 -0
- package/dist/src/storage/local/localHelpers.d.ts.map +1 -0
- package/dist/src/storage/local/localHelpers.js +69 -0
- package/dist/src/storage/local/localHelpers.js.map +1 -0
- package/dist/src/storage/local/repository.d.ts.map +1 -0
- package/dist/src/{repository.js → storage/local/repository.js} +6 -0
- package/dist/src/storage/local/repository.js.map +1 -0
- package/dist/src/test-helpers.js +1 -1
- package/dist/src/test-helpers.js.map +1 -1
- package/dist/src/trees.d.ts +2 -1
- package/dist/src/trees.d.ts.map +1 -1
- package/dist/src/trees.js +2 -0
- package/dist/src/trees.js.map +1 -1
- package/dist/src/workspaceStatus.js +3 -2
- package/dist/src/workspaceStatus.js.map +1 -1
- package/package.json +3 -3
- package/dist/src/gc.d.ts.map +0 -1
- package/dist/src/gc.js.map +0 -1
- package/dist/src/repository.d.ts.map +0 -1
- package/dist/src/repository.js.map +0 -1
- package/dist/src/workspaceLock.d.ts +0 -89
- package/dist/src/workspaceLock.d.ts.map +0 -1
- package/dist/src/workspaceLock.js +0 -307
- package/dist/src/workspaceLock.js.map +0 -1
- /package/dist/src/{repository.d.ts → storage/local/repository.d.ts} +0 -0
package/dist/src/objects.js
CHANGED
|
@@ -2,244 +2,23 @@
|
|
|
2
2
|
* Copyright (c) 2025 Elara AI Pty Ltd
|
|
3
3
|
* Licensed under BSL 1.1. See LICENSE for details.
|
|
4
4
|
*/
|
|
5
|
-
import * as fs from 'fs/promises';
|
|
6
|
-
import * as path from 'path';
|
|
7
|
-
import * as crypto from 'crypto';
|
|
8
|
-
import { Readable } from 'stream';
|
|
9
|
-
import { pipeline } from 'stream/promises';
|
|
10
|
-
import { createWriteStream } from 'fs';
|
|
11
|
-
import { ObjectNotFoundError, isNotFoundError } from './errors.js';
|
|
12
|
-
/**
|
|
13
|
-
* Calculate SHA256 hash of data
|
|
14
|
-
*/
|
|
15
|
-
export function computeHash(data) {
|
|
16
|
-
return crypto.createHash('sha256').update(data).digest('hex');
|
|
17
|
-
}
|
|
18
5
|
/**
|
|
19
|
-
*
|
|
20
|
-
* @internal
|
|
21
|
-
*/
|
|
22
|
-
async function computeHashFromStream(stream) {
|
|
23
|
-
const hash = crypto.createHash('sha256');
|
|
24
|
-
const chunks = [];
|
|
25
|
-
const reader = stream.getReader();
|
|
26
|
-
while (true) {
|
|
27
|
-
const { done, value } = await reader.read();
|
|
28
|
-
if (done)
|
|
29
|
-
break;
|
|
30
|
-
hash.update(value);
|
|
31
|
-
chunks.push(value);
|
|
32
|
-
}
|
|
33
|
-
return {
|
|
34
|
-
hash: hash.digest('hex'),
|
|
35
|
-
data: chunks,
|
|
36
|
-
};
|
|
37
|
-
}
|
|
38
|
-
/**
|
|
39
|
-
* Atomically write an object to the repository
|
|
6
|
+
* Generic object utilities for e3.
|
|
40
7
|
*
|
|
41
|
-
*
|
|
42
|
-
*
|
|
43
|
-
* @returns SHA256 hash of the data
|
|
8
|
+
* This module contains only storage-agnostic utilities.
|
|
9
|
+
* Local filesystem operations are in storage/local/LocalObjectStore.ts
|
|
44
10
|
*/
|
|
45
|
-
|
|
46
|
-
const extension = '.beast2';
|
|
47
|
-
const hash = computeHash(data);
|
|
48
|
-
// Split hash: first 2 chars as directory
|
|
49
|
-
const dirName = hash.slice(0, 2);
|
|
50
|
-
const fileName = hash.slice(2) + extension;
|
|
51
|
-
const dirPath = path.join(repoPath, 'objects', dirName);
|
|
52
|
-
const filePath = path.join(dirPath, fileName);
|
|
53
|
-
// Check if already exists
|
|
54
|
-
try {
|
|
55
|
-
await fs.access(filePath);
|
|
56
|
-
return hash; // Already exists
|
|
57
|
-
}
|
|
58
|
-
catch {
|
|
59
|
-
// Doesn't exist, continue
|
|
60
|
-
}
|
|
61
|
-
// Create directory if needed
|
|
62
|
-
await fs.mkdir(dirPath, { recursive: true });
|
|
63
|
-
// Write atomically: stage in same directory (same filesystem) + rename
|
|
64
|
-
// Staging files use .partial extension; gc can clean up any orphaned ones
|
|
65
|
-
// Use random suffix to avoid collisions with concurrent writes
|
|
66
|
-
const randomSuffix = Math.random().toString(36).slice(2, 10);
|
|
67
|
-
const stagingPath = path.join(dirPath, `${fileName}.${Date.now()}.${randomSuffix}.partial`);
|
|
68
|
-
await fs.writeFile(stagingPath, data);
|
|
69
|
-
try {
|
|
70
|
-
await fs.rename(stagingPath, filePath);
|
|
71
|
-
}
|
|
72
|
-
catch (err) {
|
|
73
|
-
// If rename fails because target exists (concurrent write won), that's fine
|
|
74
|
-
// Clean up our staging file
|
|
75
|
-
try {
|
|
76
|
-
await fs.unlink(stagingPath);
|
|
77
|
-
}
|
|
78
|
-
catch {
|
|
79
|
-
// Ignore cleanup errors
|
|
80
|
-
}
|
|
81
|
-
// Verify the file exists (another writer should have created it)
|
|
82
|
-
try {
|
|
83
|
-
await fs.access(filePath);
|
|
84
|
-
}
|
|
85
|
-
catch {
|
|
86
|
-
// File doesn't exist and rename failed - re-throw original error
|
|
87
|
-
throw err;
|
|
88
|
-
}
|
|
89
|
-
}
|
|
90
|
-
return hash;
|
|
91
|
-
}
|
|
92
|
-
/**
|
|
93
|
-
* Atomically write a stream to the repository
|
|
94
|
-
*
|
|
95
|
-
* @param repoPath - Path to e3 repository
|
|
96
|
-
* @param stream - Stream to store
|
|
97
|
-
* @returns SHA256 hash of the data
|
|
98
|
-
*/
|
|
99
|
-
export async function objectWriteStream(repoPath, stream) {
|
|
100
|
-
const extension = '.beast2';
|
|
101
|
-
// First pass: compute hash while collecting data
|
|
102
|
-
const { hash, data } = await computeHashFromStream(stream);
|
|
103
|
-
// Split hash: first 2 chars as directory
|
|
104
|
-
const dirName = hash.slice(0, 2);
|
|
105
|
-
const fileName = hash.slice(2) + extension;
|
|
106
|
-
const dirPath = path.join(repoPath, 'objects', dirName);
|
|
107
|
-
const filePath = path.join(dirPath, fileName);
|
|
108
|
-
// Check if already exists
|
|
109
|
-
try {
|
|
110
|
-
await fs.access(filePath);
|
|
111
|
-
return hash; // Already exists
|
|
112
|
-
}
|
|
113
|
-
catch {
|
|
114
|
-
// Doesn't exist, continue
|
|
115
|
-
}
|
|
116
|
-
// Create directory if needed
|
|
117
|
-
await fs.mkdir(dirPath, { recursive: true });
|
|
118
|
-
// Write atomically: stage in same directory (same filesystem) + rename
|
|
119
|
-
// Staging files use .partial extension; gc can clean up any orphaned ones
|
|
120
|
-
// Use random suffix to avoid collisions with concurrent writes
|
|
121
|
-
const randomSuffix = Math.random().toString(36).slice(2, 10);
|
|
122
|
-
const stagingPath = path.join(dirPath, `${fileName}.${Date.now()}.${randomSuffix}.partial`);
|
|
123
|
-
// Reconstruct stream from collected chunks
|
|
124
|
-
const nodeStream = Readable.from(data);
|
|
125
|
-
const writeStream = createWriteStream(stagingPath);
|
|
126
|
-
await pipeline(nodeStream, writeStream);
|
|
127
|
-
try {
|
|
128
|
-
await fs.rename(stagingPath, filePath);
|
|
129
|
-
}
|
|
130
|
-
catch (err) {
|
|
131
|
-
// If rename fails because target exists (concurrent write won), that's fine
|
|
132
|
-
// Clean up our staging file
|
|
133
|
-
try {
|
|
134
|
-
await fs.unlink(stagingPath);
|
|
135
|
-
}
|
|
136
|
-
catch {
|
|
137
|
-
// Ignore cleanup errors
|
|
138
|
-
}
|
|
139
|
-
// Verify the file exists (another writer should have created it)
|
|
140
|
-
try {
|
|
141
|
-
await fs.access(filePath);
|
|
142
|
-
}
|
|
143
|
-
catch {
|
|
144
|
-
// File doesn't exist and rename failed - re-throw original error
|
|
145
|
-
throw err;
|
|
146
|
-
}
|
|
147
|
-
}
|
|
148
|
-
return hash;
|
|
149
|
-
}
|
|
150
|
-
/**
|
|
151
|
-
* Read an object from the repository
|
|
152
|
-
*
|
|
153
|
-
* @param repoPath - Path to e3 repository
|
|
154
|
-
* @param hash - SHA256 hash of the object
|
|
155
|
-
* @returns Object data
|
|
156
|
-
* @throws {ObjectNotFoundError} If object not found
|
|
157
|
-
*/
|
|
158
|
-
export async function objectRead(repoPath, hash) {
|
|
159
|
-
const extension = '.beast2';
|
|
160
|
-
const dirName = hash.slice(0, 2);
|
|
161
|
-
const fileName = hash.slice(2) + extension;
|
|
162
|
-
const filePath = path.join(repoPath, 'objects', dirName, fileName);
|
|
163
|
-
try {
|
|
164
|
-
return await fs.readFile(filePath);
|
|
165
|
-
}
|
|
166
|
-
catch (err) {
|
|
167
|
-
if (isNotFoundError(err)) {
|
|
168
|
-
throw new ObjectNotFoundError(hash);
|
|
169
|
-
}
|
|
170
|
-
throw err;
|
|
171
|
-
}
|
|
172
|
-
}
|
|
173
|
-
/**
|
|
174
|
-
* Check if an object exists in the repository
|
|
175
|
-
*
|
|
176
|
-
* @param repoPath - Path to e3 repository
|
|
177
|
-
* @param hash - SHA256 hash of the object
|
|
178
|
-
* @returns true if object exists
|
|
179
|
-
*/
|
|
180
|
-
export async function objectExists(repoPath, hash) {
|
|
181
|
-
const filePath = objectPath(repoPath, hash);
|
|
182
|
-
try {
|
|
183
|
-
await fs.access(filePath);
|
|
184
|
-
return true;
|
|
185
|
-
}
|
|
186
|
-
catch {
|
|
187
|
-
return false;
|
|
188
|
-
}
|
|
189
|
-
}
|
|
190
|
-
/**
|
|
191
|
-
* Get the filesystem path for an object
|
|
192
|
-
*
|
|
193
|
-
* @param repoPath - Path to e3 repository
|
|
194
|
-
* @param hash - SHA256 hash of the object
|
|
195
|
-
* @returns Filesystem path: objects/<hash[0..2]>/<hash[2..]>.beast2
|
|
196
|
-
*/
|
|
197
|
-
export function objectPath(repoPath, hash) {
|
|
198
|
-
const dirName = hash.slice(0, 2);
|
|
199
|
-
const fileName = hash.slice(2) + '.beast2';
|
|
200
|
-
return path.join(repoPath, 'objects', dirName, fileName);
|
|
201
|
-
}
|
|
11
|
+
import * as crypto from 'crypto';
|
|
202
12
|
/**
|
|
203
|
-
*
|
|
13
|
+
* Calculate SHA256 hash of data.
|
|
204
14
|
*
|
|
205
|
-
*
|
|
206
|
-
*
|
|
15
|
+
* This is the core hashing function used throughout e3 for content addressing.
|
|
16
|
+
* It's storage-agnostic and can be used with any backend.
|
|
207
17
|
*
|
|
208
|
-
* @param
|
|
209
|
-
* @
|
|
210
|
-
* @param minLength - Minimum prefix length to return (default: 4)
|
|
211
|
-
* @returns Minimum unambiguous prefix length
|
|
18
|
+
* @param data - Data to hash
|
|
19
|
+
* @returns SHA256 hash as a hex string
|
|
212
20
|
*/
|
|
213
|
-
export
|
|
214
|
-
|
|
215
|
-
const targetPrefix = hash.slice(0, 2);
|
|
216
|
-
// Collect all hashes that share the same 2-char prefix directory
|
|
217
|
-
const hashes = [];
|
|
218
|
-
try {
|
|
219
|
-
const dirPath = path.join(objectsDir, targetPrefix);
|
|
220
|
-
const entries = await fs.readdir(dirPath);
|
|
221
|
-
for (const entry of entries) {
|
|
222
|
-
if (entry.endsWith('.beast2') && !entry.includes('.partial')) {
|
|
223
|
-
// Reconstruct full hash: dir prefix + filename without extension
|
|
224
|
-
const fullHash = targetPrefix + entry.slice(0, -7); // remove '.beast2'
|
|
225
|
-
hashes.push(fullHash);
|
|
226
|
-
}
|
|
227
|
-
}
|
|
228
|
-
}
|
|
229
|
-
catch {
|
|
230
|
-
// Directory doesn't exist - hash is unique at minimum length
|
|
231
|
-
return minLength;
|
|
232
|
-
}
|
|
233
|
-
// Find minimum length that disambiguates from all other hashes
|
|
234
|
-
let length = minLength;
|
|
235
|
-
while (length < hash.length) {
|
|
236
|
-
const prefix = hash.slice(0, length);
|
|
237
|
-
const conflicts = hashes.filter((h) => h !== hash && h.startsWith(prefix));
|
|
238
|
-
if (conflicts.length === 0) {
|
|
239
|
-
return length;
|
|
240
|
-
}
|
|
241
|
-
length++;
|
|
242
|
-
}
|
|
243
|
-
return hash.length;
|
|
21
|
+
export function computeHash(data) {
|
|
22
|
+
return crypto.createHash('sha256').update(data).digest('hex');
|
|
244
23
|
}
|
|
245
24
|
//# sourceMappingURL=objects.js.map
|
package/dist/src/objects.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"objects.js","sourceRoot":"","sources":["../../src/objects.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH
|
|
1
|
+
{"version":3,"file":"objects.js","sourceRoot":"","sources":["../../src/objects.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH;;;;;GAKG;AAEH,OAAO,KAAK,MAAM,MAAM,QAAQ,CAAC;AAEjC;;;;;;;;GAQG;AACH,MAAM,UAAU,WAAW,CAAC,IAAgB;IAC1C,OAAO,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AAChE,CAAC"}
|
|
@@ -2,8 +2,91 @@
|
|
|
2
2
|
* Copyright (c) 2025 Elara AI Pty Ltd
|
|
3
3
|
* Licensed under BSL 1.1. See LICENSE for details.
|
|
4
4
|
*/
|
|
5
|
-
import type
|
|
5
|
+
import { type LockState, type LockOperation } from '@elaraai/e3-types';
|
|
6
|
+
import { type LockHolderInfo } from '../../errors.js';
|
|
6
7
|
import type { LockHandle, LockService } from '../interfaces.js';
|
|
8
|
+
/**
|
|
9
|
+
* Handle to a held workspace lock.
|
|
10
|
+
* Call release() when done to free the lock.
|
|
11
|
+
*/
|
|
12
|
+
export interface WorkspaceLockHandle {
|
|
13
|
+
/** The resource (workspace name) this lock is for - compatible with LockHandle */
|
|
14
|
+
readonly resource: string;
|
|
15
|
+
/** The workspace name this lock is for */
|
|
16
|
+
readonly workspace: string;
|
|
17
|
+
/** Path to the lock file */
|
|
18
|
+
readonly lockPath: string;
|
|
19
|
+
/** Release the lock. Safe to call multiple times. */
|
|
20
|
+
release(): Promise<void>;
|
|
21
|
+
}
|
|
22
|
+
/**
|
|
23
|
+
* Options for acquiring a workspace lock.
|
|
24
|
+
*/
|
|
25
|
+
export interface AcquireLockOptions {
|
|
26
|
+
/**
|
|
27
|
+
* If true, wait for the lock to become available instead of failing immediately.
|
|
28
|
+
* Default: false (fail fast if locked)
|
|
29
|
+
*/
|
|
30
|
+
wait?: boolean;
|
|
31
|
+
/**
|
|
32
|
+
* Timeout in milliseconds when wait=true. Default: 30000 (30 seconds)
|
|
33
|
+
*/
|
|
34
|
+
timeout?: number;
|
|
35
|
+
}
|
|
36
|
+
/**
|
|
37
|
+
* Get the lock file path for a workspace.
|
|
38
|
+
*/
|
|
39
|
+
export declare function workspaceLockPath(repoPath: string, workspace: string): string;
|
|
40
|
+
/**
|
|
41
|
+
* Convert LockState to LockHolderInfo for error display.
|
|
42
|
+
*/
|
|
43
|
+
export declare function lockStateToHolderInfo(state: LockState): LockHolderInfo;
|
|
44
|
+
/**
|
|
45
|
+
* Check if a lock holder is still alive.
|
|
46
|
+
* @param holderStr - East text-encoded holder string
|
|
47
|
+
*/
|
|
48
|
+
export declare function isLockHolderAlive(holderStr: string): Promise<boolean>;
|
|
49
|
+
/**
|
|
50
|
+
* Acquire an exclusive lock on a workspace.
|
|
51
|
+
*
|
|
52
|
+
* Uses Linux flock() for kernel-managed locking. The lock is automatically
|
|
53
|
+
* released when the process exits (even on crash/kill).
|
|
54
|
+
*
|
|
55
|
+
* @param repoPath - Path to e3 repository
|
|
56
|
+
* @param workspace - Workspace name to lock
|
|
57
|
+
* @param operation - What operation is acquiring the lock
|
|
58
|
+
* @param options - Lock acquisition options
|
|
59
|
+
* @returns Lock handle - call release() when done
|
|
60
|
+
* @throws {WorkspaceLockError} If workspace is locked by another process
|
|
61
|
+
*
|
|
62
|
+
* @example
|
|
63
|
+
* ```typescript
|
|
64
|
+
* const lock = await acquireWorkspaceLock(repoPath, 'production', { type: 'dataflow', value: null });
|
|
65
|
+
* try {
|
|
66
|
+
* await dataflowExecute(repoPath, 'production', { lock });
|
|
67
|
+
* } finally {
|
|
68
|
+
* await lock.release();
|
|
69
|
+
* }
|
|
70
|
+
* ```
|
|
71
|
+
*/
|
|
72
|
+
export declare function acquireWorkspaceLock(repoPath: string, workspace: string, operation: LockOperation, options?: AcquireLockOptions): Promise<WorkspaceLockHandle>;
|
|
73
|
+
/**
|
|
74
|
+
* Get the lock state for a workspace.
|
|
75
|
+
*
|
|
76
|
+
* @param repoPath - Path to e3 repository
|
|
77
|
+
* @param workspace - Workspace name to check
|
|
78
|
+
* @returns Lock state if locked, null if not locked
|
|
79
|
+
*/
|
|
80
|
+
export declare function getWorkspaceLockState(repoPath: string, workspace: string): Promise<LockState | null>;
|
|
81
|
+
/**
|
|
82
|
+
* Get lock holder info for a workspace (for backwards compatibility).
|
|
83
|
+
*
|
|
84
|
+
* @param repoPath - Path to e3 repository
|
|
85
|
+
* @param workspace - Workspace name to check
|
|
86
|
+
* @returns Lock holder info if locked, null if not locked
|
|
87
|
+
* @deprecated Use getWorkspaceLockState for full lock information
|
|
88
|
+
*/
|
|
89
|
+
export declare function getWorkspaceLockHolder(repoPath: string, workspace: string): Promise<LockHolderInfo | null>;
|
|
7
90
|
/**
|
|
8
91
|
* Local filesystem implementation of LockService.
|
|
9
92
|
*
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"LocalLockService.d.ts","sourceRoot":"","sources":["../../../../src/storage/local/LocalLockService.ts"],"names":[],"mappings":"AAAA;;;GAGG;
|
|
1
|
+
{"version":3,"file":"LocalLockService.d.ts","sourceRoot":"","sources":["../../../../src/storage/local/LocalLockService.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAqBH,OAAO,EAAoC,KAAK,SAAS,EAAE,KAAK,aAAa,EAAE,MAAM,mBAAmB,CAAC;AACzG,OAAO,EAAsB,KAAK,cAAc,EAAE,MAAM,iBAAiB,CAAC;AAE1E,OAAO,KAAK,EAAE,UAAU,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAkChE;;;GAGG;AACH,MAAM,WAAW,mBAAmB;IAClC,kFAAkF;IAClF,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,0CAA0C;IAC1C,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,4BAA4B;IAC5B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,qDAAqD;IACrD,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CAC1B;AAED;;GAEG;AACH,MAAM,WAAW,kBAAkB;IACjC;;;OAGG;IACH,IAAI,CAAC,EAAE,OAAO,CAAC;IACf;;OAEG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAMD;;GAEG;AACH,wBAAgB,iBAAiB,CAAC,QAAQ,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,MAAM,CAE7E;AA0BD;;GAEG;AACH,wBAAgB,qBAAqB,CAAC,KAAK,EAAE,SAAS,GAAG,cAAc,CAgBtE;AAED;;;GAGG;AACH,wBAAsB,iBAAiB,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAc3E;AAMD;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,wBAAsB,oBAAoB,CACxC,QAAQ,EAAE,MAAM,EAChB,SAAS,EAAE,MAAM,EACjB,SAAS,EAAE,aAAa,EACxB,OAAO,GAAE,kBAAuB,GAC/B,OAAO,CAAC,mBAAmB,CAAC,CAgE9B;AAuFD;;;;;;GAMG;AACH,wBAAsB,qBAAqB,CACzC,QAAQ,EAAE,MAAM,EAChB,SAAS,EAAE,MAAM,GAChB,OAAO,CAAC,SAAS,GAAG,IAAI,CAAC,CAoB3B;AAED;;;;;;;GAOG;AACH,wBAAsB,sBAAsB,CAC1C,QAAQ,EAAE,MAAM,EAChB,SAAS,EAAE,MAAM,GAChB,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC,CAGhC;AAMD;;;;;;GAMG;AACH,qBAAa,gBAAiB,YAAW,WAAW;IAC5C,OAAO,CACX,IAAI,EAAE,MAAM,EACZ,QAAQ,EAAE,MAAM,EAChB,SAAS,EAAE,aAAa,EACxB,OAAO,CAAC,EAAE;QAAE,IAAI,CAAC,EAAE,OAAO,CAAC;QAAC,OAAO,CAAC,EAAE,MAAM,CAAA;KAAE,GAC7C,OAAO,CAAC,UAAU,GAAG,IAAI,CAAC;IAkB7B,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,SAAS,GAAG,IAAI,CAAC;IAInE,aAAa,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;CAGhD"}
|
|
@@ -2,7 +2,311 @@
|
|
|
2
2
|
* Copyright (c) 2025 Elara AI Pty Ltd
|
|
3
3
|
* Licensed under BSL 1.1. See LICENSE for details.
|
|
4
4
|
*/
|
|
5
|
-
|
|
5
|
+
/**
|
|
6
|
+
* Local filesystem implementation of workspace locking.
|
|
7
|
+
*
|
|
8
|
+
* Provides exclusive locks on workspaces to prevent concurrent dataflow
|
|
9
|
+
* executions or writes that could corrupt workspace state. Uses Linux
|
|
10
|
+
* flock() for automatic lock release on process death.
|
|
11
|
+
*
|
|
12
|
+
* Lock mechanism:
|
|
13
|
+
* - Uses flock(LOCK_EX | LOCK_NB) via the `flock` command for kernel-managed locking
|
|
14
|
+
* - Lock is automatically released when the process dies (kernel handles this)
|
|
15
|
+
* - Lock state stored in beast2 format using LockStateType from e3-types
|
|
16
|
+
* - Holder stored as East text string (e.g., `.process (pid=1234, ...)`)
|
|
17
|
+
* - Stale lock detection via bootId comparison (handles system restarts)
|
|
18
|
+
*/
|
|
19
|
+
import * as fs from 'fs/promises';
|
|
20
|
+
import * as path from 'path';
|
|
21
|
+
import { spawn } from 'child_process';
|
|
22
|
+
import { encodeBeast2For, decodeBeast2For, printFor, parseInferred, variant, none, VariantType } from '@elaraai/east';
|
|
23
|
+
import { LockStateType, ProcessHolderType } from '@elaraai/e3-types';
|
|
24
|
+
import { WorkspaceLockError } from '../../errors.js';
|
|
25
|
+
import { getBootId, getPidStartTime, isProcessAlive } from '../../execution/processHelpers.js';
|
|
26
|
+
// =============================================================================
|
|
27
|
+
// Holder Encoding
|
|
28
|
+
// =============================================================================
|
|
29
|
+
/**
|
|
30
|
+
* Variant type for encoding holder as East text.
|
|
31
|
+
* The holder string stores `.process (...)` or other backend-specific variants.
|
|
32
|
+
*/
|
|
33
|
+
const HolderVariantType = VariantType({
|
|
34
|
+
process: ProcessHolderType,
|
|
35
|
+
});
|
|
36
|
+
/** Print a process holder to East text format */
|
|
37
|
+
const printProcessHolder = printFor(HolderVariantType);
|
|
38
|
+
/**
|
|
39
|
+
* Parse an East text holder string.
|
|
40
|
+
* Returns the parsed variant or null if parsing fails.
|
|
41
|
+
*/
|
|
42
|
+
function parseHolder(holderStr) {
|
|
43
|
+
try {
|
|
44
|
+
const [_type, value] = parseInferred(holderStr);
|
|
45
|
+
return value;
|
|
46
|
+
}
|
|
47
|
+
catch {
|
|
48
|
+
return null;
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
// =============================================================================
|
|
52
|
+
// Lock File Helpers
|
|
53
|
+
// =============================================================================
|
|
54
|
+
/**
|
|
55
|
+
* Get the lock file path for a workspace.
|
|
56
|
+
*/
|
|
57
|
+
export function workspaceLockPath(repoPath, workspace) {
|
|
58
|
+
return path.join(repoPath, 'workspaces', `${workspace}.lock`);
|
|
59
|
+
}
|
|
60
|
+
/**
|
|
61
|
+
* Read lock state from a lock file.
|
|
62
|
+
* Returns null if file doesn't exist or is invalid.
|
|
63
|
+
*/
|
|
64
|
+
async function readLockState(lockPath) {
|
|
65
|
+
try {
|
|
66
|
+
const data = await fs.readFile(lockPath);
|
|
67
|
+
if (data.length === 0)
|
|
68
|
+
return null;
|
|
69
|
+
const decoder = decodeBeast2For(LockStateType);
|
|
70
|
+
return decoder(data);
|
|
71
|
+
}
|
|
72
|
+
catch {
|
|
73
|
+
return null;
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
/**
|
|
77
|
+
* Write lock state to a lock file in beast2 format.
|
|
78
|
+
*/
|
|
79
|
+
async function writeLockState(lockPath, state) {
|
|
80
|
+
const encoder = encodeBeast2For(LockStateType);
|
|
81
|
+
const data = encoder(state);
|
|
82
|
+
await fs.writeFile(lockPath, data);
|
|
83
|
+
}
|
|
84
|
+
/**
|
|
85
|
+
* Convert LockState to LockHolderInfo for error display.
|
|
86
|
+
*/
|
|
87
|
+
export function lockStateToHolderInfo(state) {
|
|
88
|
+
const info = {
|
|
89
|
+
acquiredAt: state.acquiredAt.toISOString(),
|
|
90
|
+
operation: state.operation.type,
|
|
91
|
+
};
|
|
92
|
+
// Parse the holder string to extract process-specific fields
|
|
93
|
+
const holder = parseHolder(state.holder);
|
|
94
|
+
if (holder?.type === 'process') {
|
|
95
|
+
info.pid = Number(holder.value.pid);
|
|
96
|
+
info.bootId = holder.value.bootId;
|
|
97
|
+
info.startTime = Number(holder.value.startTime);
|
|
98
|
+
info.command = holder.value.command;
|
|
99
|
+
}
|
|
100
|
+
return info;
|
|
101
|
+
}
|
|
102
|
+
/**
|
|
103
|
+
* Check if a lock holder is still alive.
|
|
104
|
+
* @param holderStr - East text-encoded holder string
|
|
105
|
+
*/
|
|
106
|
+
export async function isLockHolderAlive(holderStr) {
|
|
107
|
+
const holder = parseHolder(holderStr);
|
|
108
|
+
if (!holder)
|
|
109
|
+
return true; // Can't parse - assume alive (safer)
|
|
110
|
+
if (holder.type === 'process') {
|
|
111
|
+
return isProcessAlive(Number(holder.value.pid), Number(holder.value.startTime), holder.value.bootId);
|
|
112
|
+
}
|
|
113
|
+
// Unknown holder type - assume alive (safer default)
|
|
114
|
+
return true;
|
|
115
|
+
}
|
|
116
|
+
// =============================================================================
|
|
117
|
+
// Lock Acquisition
|
|
118
|
+
// =============================================================================
|
|
119
|
+
/**
|
|
120
|
+
* Acquire an exclusive lock on a workspace.
|
|
121
|
+
*
|
|
122
|
+
* Uses Linux flock() for kernel-managed locking. The lock is automatically
|
|
123
|
+
* released when the process exits (even on crash/kill).
|
|
124
|
+
*
|
|
125
|
+
* @param repoPath - Path to e3 repository
|
|
126
|
+
* @param workspace - Workspace name to lock
|
|
127
|
+
* @param operation - What operation is acquiring the lock
|
|
128
|
+
* @param options - Lock acquisition options
|
|
129
|
+
* @returns Lock handle - call release() when done
|
|
130
|
+
* @throws {WorkspaceLockError} If workspace is locked by another process
|
|
131
|
+
*
|
|
132
|
+
* @example
|
|
133
|
+
* ```typescript
|
|
134
|
+
* const lock = await acquireWorkspaceLock(repoPath, 'production', { type: 'dataflow', value: null });
|
|
135
|
+
* try {
|
|
136
|
+
* await dataflowExecute(repoPath, 'production', { lock });
|
|
137
|
+
* } finally {
|
|
138
|
+
* await lock.release();
|
|
139
|
+
* }
|
|
140
|
+
* ```
|
|
141
|
+
*/
|
|
142
|
+
export async function acquireWorkspaceLock(repoPath, workspace, operation, options = {}) {
|
|
143
|
+
const lockPath = workspaceLockPath(repoPath, workspace);
|
|
144
|
+
// Ensure workspaces directory exists
|
|
145
|
+
await fs.mkdir(path.dirname(lockPath), { recursive: true });
|
|
146
|
+
// Gather our process identification
|
|
147
|
+
const pid = process.pid;
|
|
148
|
+
const bootId = await getBootId();
|
|
149
|
+
const startTime = await getPidStartTime(pid);
|
|
150
|
+
const command = process.argv.join(' ');
|
|
151
|
+
const acquiredAt = new Date();
|
|
152
|
+
// Encode holder as East text: .process (pid=..., bootId="...", ...)
|
|
153
|
+
const holderVariant = variant('process', {
|
|
154
|
+
pid: BigInt(pid),
|
|
155
|
+
bootId,
|
|
156
|
+
startTime: BigInt(startTime),
|
|
157
|
+
command,
|
|
158
|
+
});
|
|
159
|
+
const holder = printProcessHolder(holderVariant);
|
|
160
|
+
const lockState = {
|
|
161
|
+
operation,
|
|
162
|
+
holder,
|
|
163
|
+
acquiredAt,
|
|
164
|
+
expiresAt: none,
|
|
165
|
+
};
|
|
166
|
+
// Try to acquire flock via subprocess
|
|
167
|
+
// The subprocess holds the lock and we communicate with it via stdin/signals
|
|
168
|
+
const flockProcess = await tryAcquireFlock(lockPath, lockState, options);
|
|
169
|
+
if (!flockProcess) {
|
|
170
|
+
// Failed to acquire - read lock state to report who has it
|
|
171
|
+
const existingState = await readLockState(lockPath);
|
|
172
|
+
const holderInfo = existingState ? lockStateToHolderInfo(existingState) : undefined;
|
|
173
|
+
throw new WorkspaceLockError(workspace, holderInfo);
|
|
174
|
+
}
|
|
175
|
+
// Lock acquired! Create handle
|
|
176
|
+
let released = false;
|
|
177
|
+
const handle = {
|
|
178
|
+
resource: workspace,
|
|
179
|
+
workspace,
|
|
180
|
+
lockPath,
|
|
181
|
+
async release() {
|
|
182
|
+
if (released)
|
|
183
|
+
return;
|
|
184
|
+
released = true;
|
|
185
|
+
// Kill the flock subprocess to release the lock
|
|
186
|
+
flockProcess.kill('SIGTERM');
|
|
187
|
+
// Clean up lock file (best effort)
|
|
188
|
+
try {
|
|
189
|
+
await fs.unlink(lockPath);
|
|
190
|
+
}
|
|
191
|
+
catch {
|
|
192
|
+
// Ignore - file might already be gone
|
|
193
|
+
}
|
|
194
|
+
},
|
|
195
|
+
};
|
|
196
|
+
return handle;
|
|
197
|
+
}
|
|
198
|
+
/**
|
|
199
|
+
* Try to acquire flock using a subprocess.
|
|
200
|
+
*
|
|
201
|
+
* We spawn `flock --nonblock <lockfile> cat` which:
|
|
202
|
+
* 1. Tries to acquire exclusive lock (non-blocking)
|
|
203
|
+
* 2. If successful, runs `cat` which blocks reading stdin forever
|
|
204
|
+
* 3. We keep the subprocess alive to hold the lock
|
|
205
|
+
* 4. When we kill the subprocess, the lock is released
|
|
206
|
+
*
|
|
207
|
+
* Returns the subprocess if lock acquired, null if lock is held by another.
|
|
208
|
+
*/
|
|
209
|
+
async function tryAcquireFlock(lockPath, lockState, options) {
|
|
210
|
+
// First, check if there's a stale lock we can clean up
|
|
211
|
+
await checkAndCleanStaleLock(lockPath);
|
|
212
|
+
const args = options.wait
|
|
213
|
+
? ['--timeout', String((options.timeout ?? 30000) / 1000), lockPath, 'cat']
|
|
214
|
+
: ['--nonblock', lockPath, 'cat'];
|
|
215
|
+
const child = spawn('flock', args, {
|
|
216
|
+
stdio: ['pipe', 'pipe', 'pipe'],
|
|
217
|
+
detached: false,
|
|
218
|
+
});
|
|
219
|
+
return new Promise((resolve) => {
|
|
220
|
+
let resolved = false;
|
|
221
|
+
// If flock fails to acquire, it exits with code 1
|
|
222
|
+
child.on('error', () => {
|
|
223
|
+
if (!resolved) {
|
|
224
|
+
resolved = true;
|
|
225
|
+
resolve(null);
|
|
226
|
+
}
|
|
227
|
+
});
|
|
228
|
+
child.on('exit', () => {
|
|
229
|
+
if (!resolved) {
|
|
230
|
+
resolved = true;
|
|
231
|
+
// Exit code 1 means lock is held by another
|
|
232
|
+
resolve(null);
|
|
233
|
+
}
|
|
234
|
+
});
|
|
235
|
+
// Give flock a moment to either acquire or fail
|
|
236
|
+
// If it's still running after 100ms, we have the lock
|
|
237
|
+
setTimeout(() => {
|
|
238
|
+
if (!resolved && !child.killed && child.exitCode === null) {
|
|
239
|
+
resolved = true;
|
|
240
|
+
// Write lock state to lock file now that we have the lock
|
|
241
|
+
void writeLockState(lockPath, lockState).catch((err) => {
|
|
242
|
+
console.warn(`Failed to write lock state: ${err instanceof Error ? err.message : String(err)}`);
|
|
243
|
+
});
|
|
244
|
+
resolve(child);
|
|
245
|
+
}
|
|
246
|
+
}, 100);
|
|
247
|
+
});
|
|
248
|
+
}
|
|
249
|
+
/**
|
|
250
|
+
* Check if a lock file exists with stale lock state and clean it up.
|
|
251
|
+
* A lock is stale if the holder process no longer exists.
|
|
252
|
+
*/
|
|
253
|
+
async function checkAndCleanStaleLock(lockPath) {
|
|
254
|
+
const state = await readLockState(lockPath);
|
|
255
|
+
if (!state)
|
|
256
|
+
return;
|
|
257
|
+
// Check if the holder is still alive
|
|
258
|
+
const alive = await isLockHolderAlive(state.holder);
|
|
259
|
+
if (!alive) {
|
|
260
|
+
// Stale lock - try to remove it
|
|
261
|
+
try {
|
|
262
|
+
await fs.unlink(lockPath);
|
|
263
|
+
}
|
|
264
|
+
catch {
|
|
265
|
+
// Ignore - another process might have cleaned it up
|
|
266
|
+
}
|
|
267
|
+
}
|
|
268
|
+
}
|
|
269
|
+
/**
|
|
270
|
+
* Get the lock state for a workspace.
|
|
271
|
+
*
|
|
272
|
+
* @param repoPath - Path to e3 repository
|
|
273
|
+
* @param workspace - Workspace name to check
|
|
274
|
+
* @returns Lock state if locked, null if not locked
|
|
275
|
+
*/
|
|
276
|
+
export async function getWorkspaceLockState(repoPath, workspace) {
|
|
277
|
+
const lockPath = workspaceLockPath(repoPath, workspace);
|
|
278
|
+
const state = await readLockState(lockPath);
|
|
279
|
+
if (!state)
|
|
280
|
+
return null;
|
|
281
|
+
// Check if the holder is still alive
|
|
282
|
+
const alive = await isLockHolderAlive(state.holder);
|
|
283
|
+
if (!alive) {
|
|
284
|
+
// Stale lock - clean it up and report as not locked
|
|
285
|
+
try {
|
|
286
|
+
await fs.unlink(lockPath);
|
|
287
|
+
}
|
|
288
|
+
catch {
|
|
289
|
+
// Ignore
|
|
290
|
+
}
|
|
291
|
+
return null;
|
|
292
|
+
}
|
|
293
|
+
return state;
|
|
294
|
+
}
|
|
295
|
+
/**
|
|
296
|
+
* Get lock holder info for a workspace (for backwards compatibility).
|
|
297
|
+
*
|
|
298
|
+
* @param repoPath - Path to e3 repository
|
|
299
|
+
* @param workspace - Workspace name to check
|
|
300
|
+
* @returns Lock holder info if locked, null if not locked
|
|
301
|
+
* @deprecated Use getWorkspaceLockState for full lock information
|
|
302
|
+
*/
|
|
303
|
+
export async function getWorkspaceLockHolder(repoPath, workspace) {
|
|
304
|
+
const state = await getWorkspaceLockState(repoPath, workspace);
|
|
305
|
+
return state ? lockStateToHolderInfo(state) : null;
|
|
306
|
+
}
|
|
307
|
+
// =============================================================================
|
|
308
|
+
// LockService Interface Implementation
|
|
309
|
+
// =============================================================================
|
|
6
310
|
/**
|
|
7
311
|
* Local filesystem implementation of LockService.
|
|
8
312
|
*
|