@the-open-engine/zeroshot 6.25.0 → 6.26.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/cli/index.js +102 -74
- package/package.json +4 -2
- package/src/agent/provider-session.js +18 -4
- package/src/hosted-target/bounds.ts +6 -0
- package/src/hosted-target/errors.ts +90 -0
- package/src/hosted-target/index.ts +44 -0
- package/src/hosted-target/response-validation.ts +86 -0
- package/src/hosted-target/retry.ts +46 -0
- package/src/hosted-target/target-adapter.ts +10 -0
- package/src/hosted-target/types.ts +58 -0
- package/src/hosted-target/zero-cloud-v1-adapter.ts +386 -0
- package/src/omp-session-limits.js +25 -1
- package/src/omp-session-partition.js +159 -32
- package/src/omp-session-verifier.js +251 -87
- package/task-lib/commands/clean.js +80 -46
- package/task-lib/commands/kill.js +21 -0
- package/task-lib/commands/resume.js +20 -0
- package/task-lib/commands/run.js +17 -2
- package/task-lib/omp-session-cleanup.js +46 -9
- package/task-lib/omp-session-ownership-schema.js +21 -15
- package/task-lib/omp-session-ownership.js +66 -31
- package/task-lib/rpc-watcher.js +51 -15
- package/task-lib/runner.js +51 -4
- package/task-lib/store.js +67 -72
|
@@ -0,0 +1,386 @@
|
|
|
1
|
+
import { MAX_ERROR_BODY_BYTES, MAX_RESPONSE_BYTES, MAX_RETRY_ELAPSED_MS, IDEMPOTENCY_KEY_PATTERN } from './bounds.ts';
|
|
2
|
+
import {
|
|
3
|
+
TargetAuthError,
|
|
4
|
+
TargetConflictError,
|
|
5
|
+
TargetNotFoundError,
|
|
6
|
+
TargetProtocolError,
|
|
7
|
+
TargetRateLimitError,
|
|
8
|
+
TargetTransportError,
|
|
9
|
+
} from './errors.ts';
|
|
10
|
+
import { TargetAdapterError } from './errors.ts';
|
|
11
|
+
import { DefaultRetryPolicy, parseRetryAfter } from './retry.ts';
|
|
12
|
+
import {
|
|
13
|
+
assertCapsule,
|
|
14
|
+
assertCapsuleAccess,
|
|
15
|
+
assertCapsuleLimits,
|
|
16
|
+
assertCapsuleListPage,
|
|
17
|
+
} from './response-validation.ts';
|
|
18
|
+
import type { TargetAdapter } from './target-adapter.ts';
|
|
19
|
+
import type {
|
|
20
|
+
AllocateRequest,
|
|
21
|
+
Capsule,
|
|
22
|
+
CapsuleAccess,
|
|
23
|
+
CapsuleLimits,
|
|
24
|
+
CapsuleListPage,
|
|
25
|
+
Clock,
|
|
26
|
+
HttpTransport,
|
|
27
|
+
RetryPolicy,
|
|
28
|
+
TargetAccessTokenProvider,
|
|
29
|
+
TargetDiscovery,
|
|
30
|
+
} from './types.ts';
|
|
31
|
+
|
|
32
|
+
interface ZeroCloudV1Options {
|
|
33
|
+
readonly discovery: TargetDiscovery;
|
|
34
|
+
readonly organization: string;
|
|
35
|
+
readonly tokenProvider: TargetAccessTokenProvider;
|
|
36
|
+
readonly transport?: HttpTransport;
|
|
37
|
+
readonly clock?: Clock;
|
|
38
|
+
readonly retryPolicy?: RetryPolicy;
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
const DEFAULT_TRANSPORT: HttpTransport = {
|
|
42
|
+
fetch(url: string, init: RequestInit & { redirect: 'error' }): Promise<Response> {
|
|
43
|
+
return globalThis.fetch(url, init);
|
|
44
|
+
},
|
|
45
|
+
};
|
|
46
|
+
|
|
47
|
+
const DEFAULT_CLOCK: Clock = { now: () => Date.now() };
|
|
48
|
+
|
|
49
|
+
function throwIfAborted(signal?: AbortSignal): void {
|
|
50
|
+
if (signal?.aborted) {
|
|
51
|
+
throw signal.reason === undefined
|
|
52
|
+
? new DOMException('The operation was aborted', 'AbortError')
|
|
53
|
+
: signal.reason;
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
async function waitForRetryDelay(delayMs: number, signal?: AbortSignal): Promise<void> {
|
|
58
|
+
throwIfAborted(signal);
|
|
59
|
+
if (delayMs <= 0) return;
|
|
60
|
+
|
|
61
|
+
await new Promise<void>((resolve, reject) => {
|
|
62
|
+
const onAbort = (): void => {
|
|
63
|
+
clearTimeout(timer);
|
|
64
|
+
reject(
|
|
65
|
+
signal!.reason === undefined
|
|
66
|
+
? new DOMException('The operation was aborted', 'AbortError')
|
|
67
|
+
: signal!.reason,
|
|
68
|
+
);
|
|
69
|
+
};
|
|
70
|
+
const timer = setTimeout(() => {
|
|
71
|
+
signal?.removeEventListener('abort', onAbort);
|
|
72
|
+
resolve();
|
|
73
|
+
}, delayMs);
|
|
74
|
+
signal?.addEventListener('abort', onAbort, { once: true });
|
|
75
|
+
});
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
function originOf(url: string): string {
|
|
79
|
+
try {
|
|
80
|
+
const u = new URL(url);
|
|
81
|
+
return u.origin;
|
|
82
|
+
} catch {
|
|
83
|
+
throw new TargetProtocolError(`Invalid URL: ${url}`);
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
export class ZeroCloudV1TargetAdapter implements TargetAdapter {
|
|
88
|
+
private readonly discovery: TargetDiscovery;
|
|
89
|
+
private readonly organization: string;
|
|
90
|
+
private readonly tokenProvider: TargetAccessTokenProvider;
|
|
91
|
+
private readonly transport: HttpTransport;
|
|
92
|
+
private readonly clock: Clock;
|
|
93
|
+
private readonly retryPolicy: RetryPolicy;
|
|
94
|
+
private readonly expectedOrigin: string;
|
|
95
|
+
|
|
96
|
+
constructor(opts: ZeroCloudV1Options) {
|
|
97
|
+
this.discovery = opts.discovery;
|
|
98
|
+
this.organization = opts.organization;
|
|
99
|
+
this.tokenProvider = opts.tokenProvider;
|
|
100
|
+
this.transport = opts.transport ?? DEFAULT_TRANSPORT;
|
|
101
|
+
this.clock = opts.clock ?? DEFAULT_CLOCK;
|
|
102
|
+
this.retryPolicy = opts.retryPolicy ?? new DefaultRetryPolicy();
|
|
103
|
+
this.expectedOrigin = originOf(opts.discovery.capsuleV1);
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
async allocate(req: AllocateRequest, signal?: AbortSignal): Promise<Capsule> {
|
|
107
|
+
if (!IDEMPOTENCY_KEY_PATTERN.test(req.idempotencyKey)) {
|
|
108
|
+
throw new TargetProtocolError(
|
|
109
|
+
`Invalid idempotency key: must match ${IDEMPOTENCY_KEY_PATTERN}`,
|
|
110
|
+
);
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
const body = JSON.stringify({ profile: req.profile, organization: this.organization });
|
|
114
|
+
|
|
115
|
+
return this._withRetry(async () => {
|
|
116
|
+
const resp = await this._request('POST', '/capsules', {
|
|
117
|
+
signal,
|
|
118
|
+
body,
|
|
119
|
+
headers: { 'Idempotency-Key': req.idempotencyKey },
|
|
120
|
+
});
|
|
121
|
+
const json = await this._readJson(resp);
|
|
122
|
+
return assertCapsule(json);
|
|
123
|
+
}, signal);
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
async list(cursor?: string, signal?: AbortSignal): Promise<CapsuleListPage> {
|
|
127
|
+
return this._withRetry(async () => {
|
|
128
|
+
const params = new URLSearchParams({ organization: this.organization });
|
|
129
|
+
if (cursor) params.set('cursor', cursor);
|
|
130
|
+
const resp = await this._request('GET', `/capsules?${params.toString()}`, { signal });
|
|
131
|
+
const json = await this._readJson(resp);
|
|
132
|
+
const page = assertCapsuleListPage(json);
|
|
133
|
+
if (page.cursor !== undefined && page.cursor === cursor) {
|
|
134
|
+
throw new TargetProtocolError(
|
|
135
|
+
`Pagination loop detected: server returned the same cursor "${cursor}"`,
|
|
136
|
+
);
|
|
137
|
+
}
|
|
138
|
+
return page;
|
|
139
|
+
}, signal);
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
async inspect(capsuleId: string, signal?: AbortSignal): Promise<Capsule> {
|
|
143
|
+
return this._withRetry(async () => {
|
|
144
|
+
const resp = await this._request('GET', `/capsules/${encodeURIComponent(capsuleId)}`, {
|
|
145
|
+
signal,
|
|
146
|
+
});
|
|
147
|
+
const json = await this._readJson(resp);
|
|
148
|
+
return assertCapsule(json);
|
|
149
|
+
}, signal);
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
async terminate(capsuleId: string, signal?: AbortSignal): Promise<void> {
|
|
153
|
+
await this._withRetry(async () => {
|
|
154
|
+
const resp = await this._request(
|
|
155
|
+
'DELETE',
|
|
156
|
+
`/capsules/${encodeURIComponent(capsuleId)}`,
|
|
157
|
+
{ signal },
|
|
158
|
+
);
|
|
159
|
+
if (resp.status !== 204) {
|
|
160
|
+
const json = await this._readJson(resp);
|
|
161
|
+
throw new TargetProtocolError(`Unexpected terminate response: ${JSON.stringify(json)}`);
|
|
162
|
+
}
|
|
163
|
+
}, signal);
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
async limits(signal?: AbortSignal): Promise<CapsuleLimits> {
|
|
167
|
+
return this._withRetry(async () => {
|
|
168
|
+
const params = new URLSearchParams({ organization: this.organization });
|
|
169
|
+
const resp = await this._request('GET', `/limits?${params.toString()}`, { signal });
|
|
170
|
+
const json = await this._readJson(resp);
|
|
171
|
+
return assertCapsuleLimits(json);
|
|
172
|
+
}, signal);
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
async access(capsuleId: string, signal?: AbortSignal): Promise<CapsuleAccess> {
|
|
176
|
+
return this._withRetry(async () => {
|
|
177
|
+
const resp = await this._request(
|
|
178
|
+
'POST',
|
|
179
|
+
`/capsules/${encodeURIComponent(capsuleId)}/access`,
|
|
180
|
+
{ signal },
|
|
181
|
+
);
|
|
182
|
+
const json = await this._readJson(resp);
|
|
183
|
+
return assertCapsuleAccess(json);
|
|
184
|
+
}, signal);
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
private async _request(
|
|
188
|
+
method: string,
|
|
189
|
+
path: string,
|
|
190
|
+
opts: { signal?: AbortSignal | undefined; body?: string | undefined; headers?: Record<string, string> | undefined },
|
|
191
|
+
): Promise<Response> {
|
|
192
|
+
const url = `${this.discovery.capsuleV1}${path}`;
|
|
193
|
+
|
|
194
|
+
const responseOrigin = originOf(url);
|
|
195
|
+
if (responseOrigin !== this.expectedOrigin) {
|
|
196
|
+
throw new TargetProtocolError(
|
|
197
|
+
`Origin mismatch: expected ${this.expectedOrigin}, got ${responseOrigin}`,
|
|
198
|
+
);
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
let token: string;
|
|
202
|
+
try {
|
|
203
|
+
token = await this.tokenProvider.getAccessToken(opts.signal);
|
|
204
|
+
} catch (err) {
|
|
205
|
+
throw new TargetTransportError('Failed to acquire access token', err);
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
const headers: Record<string, string> = {
|
|
209
|
+
'Authorization': `Bearer ${token}`,
|
|
210
|
+
'Content-Type': 'application/json',
|
|
211
|
+
'Accept': 'application/json',
|
|
212
|
+
...opts.headers,
|
|
213
|
+
};
|
|
214
|
+
|
|
215
|
+
let response: Response;
|
|
216
|
+
const fetchInit: RequestInit & { redirect: 'error' } = {
|
|
217
|
+
method,
|
|
218
|
+
headers,
|
|
219
|
+
redirect: 'error',
|
|
220
|
+
};
|
|
221
|
+
if (opts.body !== undefined) fetchInit.body = opts.body;
|
|
222
|
+
if (opts.signal !== undefined) fetchInit.signal = opts.signal;
|
|
223
|
+
|
|
224
|
+
try {
|
|
225
|
+
response = await this.transport.fetch(url, fetchInit);
|
|
226
|
+
} catch (err) {
|
|
227
|
+
if (err instanceof TargetAdapterError) throw err;
|
|
228
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
229
|
+
if (msg.includes('redirect')) {
|
|
230
|
+
throw new TargetProtocolError(`Redirect rejected for ${method} ${path}`, err);
|
|
231
|
+
}
|
|
232
|
+
throw new TargetTransportError(`Network error during ${method} ${path}`, err);
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
if (response.status < 200 || response.status >= 300) {
|
|
236
|
+
await this._mapStatusError(response, method, path, opts.headers);
|
|
237
|
+
}
|
|
238
|
+
|
|
239
|
+
return response;
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
private async _mapStatusError(
|
|
243
|
+
response: Response,
|
|
244
|
+
method: string,
|
|
245
|
+
path: string,
|
|
246
|
+
headers?: Record<string, string>,
|
|
247
|
+
): Promise<never> {
|
|
248
|
+
const status = response.status;
|
|
249
|
+
const errorBody = await this._readErrorBody(response);
|
|
250
|
+
const context = `${status} ${method} ${path}: ${errorBody}`;
|
|
251
|
+
|
|
252
|
+
if (status === 401 || status === 403) throw new TargetAuthError(context);
|
|
253
|
+
if (status === 404) throw new TargetNotFoundError(context);
|
|
254
|
+
if (status === 409) {
|
|
255
|
+
const idempotencyKey = headers?.['Idempotency-Key'] ?? 'unknown';
|
|
256
|
+
throw new TargetConflictError(idempotencyKey, context);
|
|
257
|
+
}
|
|
258
|
+
if (status === 429) {
|
|
259
|
+
const retryAfterMs = parseRetryAfter(response.headers.get('Retry-After'), this.clock);
|
|
260
|
+
throw new TargetRateLimitError(context, retryAfterMs ?? undefined);
|
|
261
|
+
}
|
|
262
|
+
if (status >= 500) throw new TargetTransportError(context);
|
|
263
|
+
throw new TargetProtocolError(`Unexpected status ${context}`);
|
|
264
|
+
}
|
|
265
|
+
|
|
266
|
+
private async _readJson(response: Response): Promise<unknown> {
|
|
267
|
+
const contentLength = response.headers.get('Content-Length');
|
|
268
|
+
if (contentLength !== null) {
|
|
269
|
+
const len = parseInt(contentLength, 10);
|
|
270
|
+
if (Number.isFinite(len) && len > MAX_RESPONSE_BYTES) {
|
|
271
|
+
throw new TargetProtocolError(
|
|
272
|
+
`Response too large: ${len} bytes exceeds limit of ${MAX_RESPONSE_BYTES}`,
|
|
273
|
+
);
|
|
274
|
+
}
|
|
275
|
+
}
|
|
276
|
+
|
|
277
|
+
let text: string;
|
|
278
|
+
try {
|
|
279
|
+
const reader = response.body?.getReader();
|
|
280
|
+
if (!reader) {
|
|
281
|
+
text = await response.text();
|
|
282
|
+
} else {
|
|
283
|
+
const chunks: Uint8Array[] = [];
|
|
284
|
+
let totalBytes = 0;
|
|
285
|
+
while (true) {
|
|
286
|
+
const { done, value } = await reader.read();
|
|
287
|
+
if (done) break;
|
|
288
|
+
totalBytes += value.byteLength;
|
|
289
|
+
if (totalBytes > MAX_RESPONSE_BYTES) {
|
|
290
|
+
reader.cancel();
|
|
291
|
+
throw new TargetProtocolError(
|
|
292
|
+
`Response body exceeds limit of ${MAX_RESPONSE_BYTES} bytes`,
|
|
293
|
+
);
|
|
294
|
+
}
|
|
295
|
+
chunks.push(value);
|
|
296
|
+
}
|
|
297
|
+
const combined = new Uint8Array(totalBytes);
|
|
298
|
+
let offset = 0;
|
|
299
|
+
for (const chunk of chunks) {
|
|
300
|
+
combined.set(chunk, offset);
|
|
301
|
+
offset += chunk.byteLength;
|
|
302
|
+
}
|
|
303
|
+
text = new TextDecoder().decode(combined);
|
|
304
|
+
}
|
|
305
|
+
} catch (err) {
|
|
306
|
+
if (err instanceof TargetProtocolError) throw err;
|
|
307
|
+
throw new TargetProtocolError('Failed to read response body', err);
|
|
308
|
+
}
|
|
309
|
+
|
|
310
|
+
try {
|
|
311
|
+
return JSON.parse(text);
|
|
312
|
+
} catch (err) {
|
|
313
|
+
throw new TargetProtocolError('Invalid JSON in response body', err);
|
|
314
|
+
}
|
|
315
|
+
}
|
|
316
|
+
|
|
317
|
+
private async _readErrorBody(response: Response): Promise<string> {
|
|
318
|
+
const reader = response.body?.getReader();
|
|
319
|
+
if (!reader) return '';
|
|
320
|
+
|
|
321
|
+
const chunks: Uint8Array[] = [];
|
|
322
|
+
let totalBytes = 0;
|
|
323
|
+
try {
|
|
324
|
+
while (totalBytes < MAX_ERROR_BODY_BYTES) {
|
|
325
|
+
const { done, value } = await reader.read();
|
|
326
|
+
if (done) break;
|
|
327
|
+
|
|
328
|
+
const remaining = MAX_ERROR_BODY_BYTES - totalBytes;
|
|
329
|
+
const retained = value.byteLength > remaining ? value.slice(0, remaining) : value;
|
|
330
|
+
chunks.push(retained);
|
|
331
|
+
totalBytes += retained.byteLength;
|
|
332
|
+
if (retained.byteLength !== value.byteLength || totalBytes === MAX_ERROR_BODY_BYTES) {
|
|
333
|
+
void reader.cancel().catch(() => undefined);
|
|
334
|
+
break;
|
|
335
|
+
}
|
|
336
|
+
}
|
|
337
|
+
} catch {
|
|
338
|
+
void reader.cancel().catch(() => undefined);
|
|
339
|
+
return '<unreadable>';
|
|
340
|
+
}
|
|
341
|
+
|
|
342
|
+
const combined = new Uint8Array(totalBytes);
|
|
343
|
+
let offset = 0;
|
|
344
|
+
for (const chunk of chunks) {
|
|
345
|
+
combined.set(chunk, offset);
|
|
346
|
+
offset += chunk.byteLength;
|
|
347
|
+
}
|
|
348
|
+
return new TextDecoder().decode(combined);
|
|
349
|
+
}
|
|
350
|
+
|
|
351
|
+
private async _withRetry<T>(
|
|
352
|
+
fn: () => Promise<T>,
|
|
353
|
+
signal?: AbortSignal,
|
|
354
|
+
): Promise<T> {
|
|
355
|
+
const startTime = this.clock.now();
|
|
356
|
+
let attempt = 0;
|
|
357
|
+
|
|
358
|
+
while (true) {
|
|
359
|
+
throwIfAborted(signal);
|
|
360
|
+
try {
|
|
361
|
+
return await fn();
|
|
362
|
+
} catch (err) {
|
|
363
|
+
throwIfAborted(signal);
|
|
364
|
+
if (!(err instanceof TargetAdapterError)) throw err;
|
|
365
|
+
if (err instanceof TargetAuthError) throw err;
|
|
366
|
+
|
|
367
|
+
attempt++;
|
|
368
|
+
const elapsed = this.clock.now() - startTime;
|
|
369
|
+
const decision = this.retryPolicy.shouldRetry(attempt, elapsed, err);
|
|
370
|
+
const remaining = MAX_RETRY_ELAPSED_MS - elapsed;
|
|
371
|
+
|
|
372
|
+
if (
|
|
373
|
+
!decision.retry ||
|
|
374
|
+
!Number.isFinite(decision.delayMs) ||
|
|
375
|
+
decision.delayMs < 0 ||
|
|
376
|
+
decision.delayMs >= remaining
|
|
377
|
+
) {
|
|
378
|
+
throw err;
|
|
379
|
+
}
|
|
380
|
+
|
|
381
|
+
await waitForRetryDelay(decision.delayMs, signal);
|
|
382
|
+
if (this.clock.now() - startTime >= MAX_RETRY_ELAPSED_MS) throw err;
|
|
383
|
+
}
|
|
384
|
+
}
|
|
385
|
+
}
|
|
386
|
+
}
|
|
@@ -14,4 +14,28 @@ const OMP_SESSION_LIMITS = Object.freeze({
|
|
|
14
14
|
maxReferencedBlobBytes: 67108864,
|
|
15
15
|
});
|
|
16
16
|
|
|
17
|
-
|
|
17
|
+
/**
|
|
18
|
+
* The largest single JSONL record the verifier will buffer, DERIVED from the constants above
|
|
19
|
+
* rather than chosen — it is `maxReferencedBlobBytes`, and it is not a new knob (there is nothing
|
|
20
|
+
* to configure and no caller may override it).
|
|
21
|
+
*
|
|
22
|
+
* Why a per-record bound is needed at all: `maxSessionBytes` bounds the *file*, not a line within
|
|
23
|
+
* it. A hostile 256 MiB session with no newline in it is one record, and buffering it would cost
|
|
24
|
+
* the raw bytes, a concatenated copy, a UTF-16 string for JSON.parse, and the parsed value — a
|
|
25
|
+
* multi-hundred-megabyte spike driven entirely by the attacker's choice of where to put newlines.
|
|
26
|
+
*
|
|
27
|
+
* Why this value: `maxReferencedBlobBytes` is the issue's own answer to "how large may one
|
|
28
|
+
* addressable unit of session content be". OMP externalizes anything bigger than a message to the
|
|
29
|
+
* shared CAS store (blob-store.ts) and leaves only a 76-byte `blob:sha256:<hex>` reference in the
|
|
30
|
+
* record, so a legitimate record is orders of magnitude smaller than this; the bound exists to cap
|
|
31
|
+
* the pathological case, not to constrain real transcripts.
|
|
32
|
+
*
|
|
33
|
+
* Remaining allocation, exactly: verification buffers at most MAX_SESSION_RECORD_BYTES of raw
|
|
34
|
+
* record bytes, and `JSON.parse` necessarily materializes that record as one UTF-16 string plus its
|
|
35
|
+
* parsed value. Peak per-record cost is therefore O(MAX_SESSION_RECORD_BYTES) and independent of
|
|
36
|
+
* `maxSessionBytes`, the record count, and the file's newline placement. Nothing else in the
|
|
37
|
+
* verifier accumulates session, artifact, or blob bytes.
|
|
38
|
+
*/
|
|
39
|
+
const MAX_SESSION_RECORD_BYTES = OMP_SESSION_LIMITS.maxReferencedBlobBytes;
|
|
40
|
+
|
|
41
|
+
module.exports = { OMP_SESSION_LIMITS, MAX_SESSION_RECORD_BYTES };
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
const fs = require('fs');
|
|
2
2
|
const path = require('path');
|
|
3
|
-
const { randomUUID } = require('crypto');
|
|
3
|
+
const { createHash, randomUUID } = require('crypto');
|
|
4
4
|
const { isInsideOmpBlobsDir } = require('./omp-blob-root');
|
|
5
5
|
|
|
6
6
|
// Every OMP session partition lives under <storageRoot>/omp-sessions/<uuid>/. storageRoot is the
|
|
@@ -83,16 +83,34 @@ function currentUid() {
|
|
|
83
83
|
return typeof process.getuid === 'function' ? String(process.getuid()) : '0';
|
|
84
84
|
}
|
|
85
85
|
|
|
86
|
+
function stagingPathForOwnership(root, ownership) {
|
|
87
|
+
const owner = ownership.owner ?? {};
|
|
88
|
+
const persistedOwnerIdentity = JSON.stringify([
|
|
89
|
+
ownership.partitionId,
|
|
90
|
+
ownership.ownerUid ?? null,
|
|
91
|
+
ownership.storageRootIdentity?.device ?? null,
|
|
92
|
+
ownership.storageRootIdentity?.inode ?? null,
|
|
93
|
+
ownership.partitionIdentity?.device ?? null,
|
|
94
|
+
ownership.partitionIdentity?.inode ?? null,
|
|
95
|
+
owner.kind ?? null,
|
|
96
|
+
owner.clusterId ?? null,
|
|
97
|
+
owner.agentId ?? null,
|
|
98
|
+
owner.taskId ?? null,
|
|
99
|
+
]);
|
|
100
|
+
const digest = createHash('sha256').update(persistedOwnerIdentity).digest('hex');
|
|
101
|
+
return path.join(root, `${DELETING_PREFIX}${ownership.partitionId}-${digest}`);
|
|
102
|
+
}
|
|
103
|
+
|
|
86
104
|
/**
|
|
87
105
|
* Phase 1 of deletion: validate the owner record against what is actually on disk and move the
|
|
88
106
|
* partition out of its canonical name.
|
|
89
107
|
*
|
|
90
108
|
* The check/use race (CodeQL js/file-system-race) is closed by *moving before deleting*: the
|
|
91
|
-
* partition is renamed, within its own parent, to
|
|
92
|
-
*
|
|
93
|
-
*
|
|
94
|
-
* proves it is still the same directory that passed
|
|
95
|
-
*
|
|
109
|
+
* partition is renamed, within its own parent, to a deterministic staging name bound to its
|
|
110
|
+
* partition id and exact persisted owner identity, and only then re-pinned. A retry can therefore
|
|
111
|
+
* recover the staged directory after a crash or failed recursive removal. `rename(2)` is atomic,
|
|
112
|
+
* and the post-rename identity comparison proves it is still the same directory that passed
|
|
113
|
+
* validation. A canonical/staged conflict or identity mismatch leaves both names untouched.
|
|
96
114
|
*
|
|
97
115
|
* Splitting the rename from the recursive removal is what lets a caller hold a *task-store* write
|
|
98
116
|
* fence across "no other row claims this partition" -> "the partition no longer answers to its
|
|
@@ -178,28 +196,100 @@ function stageOmpSessionPartitionForDeletion(ownership) {
|
|
|
178
196
|
};
|
|
179
197
|
}
|
|
180
198
|
if (String(storagePin.uid) !== currentUid()) {
|
|
181
|
-
return {
|
|
199
|
+
return {
|
|
200
|
+
staged: false,
|
|
201
|
+
deleted: false,
|
|
202
|
+
reason: `${storageRoot} is not owned by the current user`,
|
|
203
|
+
};
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
let stagingPath;
|
|
207
|
+
try {
|
|
208
|
+
stagingPath = stagingPathForOwnership(root, ownership);
|
|
209
|
+
} catch (error) {
|
|
210
|
+
return {
|
|
211
|
+
staged: false,
|
|
212
|
+
deleted: false,
|
|
213
|
+
reason: `could not derive a staging name from the persisted owner identity: ${error.message}`,
|
|
214
|
+
};
|
|
182
215
|
}
|
|
183
216
|
|
|
184
217
|
let before;
|
|
185
218
|
try {
|
|
186
219
|
before = pinDirectoryIdentity(expectedPartitionPath);
|
|
187
220
|
} catch (error) {
|
|
188
|
-
if (error.code
|
|
189
|
-
|
|
190
|
-
|
|
221
|
+
if (error.code !== 'ENOENT') {
|
|
222
|
+
if (error.code === 'ELOOP' || error.code === 'EMLINK') {
|
|
223
|
+
return {
|
|
224
|
+
staged: false,
|
|
225
|
+
deleted: false,
|
|
226
|
+
reason: `${expectedPartitionPath} is a symlink; refusing to delete`,
|
|
227
|
+
};
|
|
228
|
+
}
|
|
229
|
+
if (error.code === 'ENOTDIR') {
|
|
230
|
+
return {
|
|
231
|
+
staged: false,
|
|
232
|
+
deleted: false,
|
|
233
|
+
reason: `${expectedPartitionPath} is not a real directory; refusing to delete`,
|
|
234
|
+
};
|
|
235
|
+
}
|
|
236
|
+
return { staged: false, deleted: false, reason: error.message };
|
|
191
237
|
}
|
|
192
|
-
|
|
238
|
+
|
|
239
|
+
let recovered;
|
|
240
|
+
try {
|
|
241
|
+
recovered = pinDirectoryIdentity(stagingPath);
|
|
242
|
+
} catch (stagedError) {
|
|
243
|
+
if (stagedError.code === 'ENOENT') {
|
|
244
|
+
return { staged: false, deleted: true, reason: 'already absent' };
|
|
245
|
+
}
|
|
193
246
|
return {
|
|
194
247
|
staged: false,
|
|
195
248
|
deleted: false,
|
|
196
|
-
reason:
|
|
249
|
+
reason: `canonical partition is absent, but staged ${stagingPath} is unsafe: ${stagedError.message}`,
|
|
250
|
+
};
|
|
251
|
+
}
|
|
252
|
+
if (String(recovered.uid) !== currentUid()) {
|
|
253
|
+
return {
|
|
254
|
+
staged: false,
|
|
255
|
+
deleted: false,
|
|
256
|
+
reason: `staged ${stagingPath} is not owned by the current user`,
|
|
257
|
+
};
|
|
258
|
+
}
|
|
259
|
+
if (
|
|
260
|
+
ownership.partitionIdentity &&
|
|
261
|
+
!sameIdentity(recovered.identity, ownership.partitionIdentity)
|
|
262
|
+
) {
|
|
263
|
+
return {
|
|
264
|
+
staged: false,
|
|
265
|
+
deleted: false,
|
|
266
|
+
reason: `staged ${stagingPath} identity ${recovered.identity.device}:${recovered.identity.inode} does not match the recorded ${ownership.partitionIdentity.device}:${ownership.partitionIdentity.inode}`,
|
|
267
|
+
};
|
|
268
|
+
}
|
|
269
|
+
return { staged: true, stagingPath };
|
|
270
|
+
}
|
|
271
|
+
try {
|
|
272
|
+
pinDirectoryIdentity(stagingPath);
|
|
273
|
+
return {
|
|
274
|
+
staged: false,
|
|
275
|
+
deleted: false,
|
|
276
|
+
reason: `both canonical ${expectedPartitionPath} and staged ${stagingPath} exist; refusing to choose one`,
|
|
277
|
+
};
|
|
278
|
+
} catch (error) {
|
|
279
|
+
if (error.code !== 'ENOENT') {
|
|
280
|
+
return {
|
|
281
|
+
staged: false,
|
|
282
|
+
deleted: false,
|
|
283
|
+
reason: `staging conflict at ${stagingPath}: ${error.message}`,
|
|
197
284
|
};
|
|
198
285
|
}
|
|
199
|
-
return { staged: false, deleted: false, reason: error.message };
|
|
200
286
|
}
|
|
201
287
|
if (String(before.uid) !== currentUid()) {
|
|
202
|
-
return {
|
|
288
|
+
return {
|
|
289
|
+
staged: false,
|
|
290
|
+
deleted: false,
|
|
291
|
+
reason: `${expectedPartitionPath} is not owned by the current user`,
|
|
292
|
+
};
|
|
203
293
|
}
|
|
204
294
|
if (ownership.partitionIdentity && !sameIdentity(before.identity, ownership.partitionIdentity)) {
|
|
205
295
|
return {
|
|
@@ -209,12 +299,14 @@ function stageOmpSessionPartitionForDeletion(ownership) {
|
|
|
209
299
|
};
|
|
210
300
|
}
|
|
211
301
|
|
|
212
|
-
const stagingPath = path.join(root, `${DELETING_PREFIX}${randomUUID()}`);
|
|
213
302
|
try {
|
|
214
303
|
fs.renameSync(expectedPartitionPath, stagingPath);
|
|
215
304
|
} catch (error) {
|
|
216
|
-
|
|
217
|
-
|
|
305
|
+
return {
|
|
306
|
+
staged: false,
|
|
307
|
+
deleted: false,
|
|
308
|
+
reason: `could not stage ${expectedPartitionPath}: ${error.message}`,
|
|
309
|
+
};
|
|
218
310
|
}
|
|
219
311
|
|
|
220
312
|
let after;
|
|
@@ -224,7 +316,7 @@ function stageOmpSessionPartitionForDeletion(ownership) {
|
|
|
224
316
|
return {
|
|
225
317
|
staged: false,
|
|
226
318
|
deleted: false,
|
|
227
|
-
reason: `staged ${stagingPath} could not be pinned (${error.message}); left in place for inspection`,
|
|
319
|
+
reason: `staged ${stagingPath} could not be pinned (${error.message}); left in place for retry or inspection`,
|
|
228
320
|
};
|
|
229
321
|
}
|
|
230
322
|
if (!sameIdentity(after.identity, before.identity)) {
|
|
@@ -239,26 +331,61 @@ function stageOmpSessionPartitionForDeletion(ownership) {
|
|
|
239
331
|
}
|
|
240
332
|
|
|
241
333
|
/**
|
|
242
|
-
* Phase 2: remove a directory that {@link stageOmpSessionPartitionForDeletion}
|
|
243
|
-
*
|
|
244
|
-
*
|
|
334
|
+
* Phase 2: remove a directory that {@link stageOmpSessionPartitionForDeletion} parked under its
|
|
335
|
+
* deterministic, owner-bound staging name. Safe to run outside the task-store lock after
|
|
336
|
+
* revalidating that exact staged name and its persisted partition identity.
|
|
245
337
|
*/
|
|
246
|
-
function removeStagedOmpSessionPartition(stagingPath) {
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
// rather than trusting the caller to have got it from stageOmpSessionPartitionForDeletion.
|
|
250
|
-
if (typeof stagingPath !== 'string' || !path.isAbsolute(stagingPath)) {
|
|
251
|
-
return { deleted: false, reason: `${stagingPath} is not an absolute staged partition path` };
|
|
338
|
+
function removeStagedOmpSessionPartition(stagingPath, ownership) {
|
|
339
|
+
if (!ownership || typeof ownership !== 'object') {
|
|
340
|
+
return { deleted: false, reason: 'no ownership record for staged partition removal' };
|
|
252
341
|
}
|
|
253
|
-
|
|
254
|
-
|
|
342
|
+
|
|
343
|
+
let expectedPartitionPath;
|
|
344
|
+
try {
|
|
345
|
+
expectedPartitionPath = partitionPathFor(ownership.storageRoot, ownership.partitionId);
|
|
346
|
+
} catch (error) {
|
|
347
|
+
return { deleted: false, reason: error.message };
|
|
255
348
|
}
|
|
256
|
-
if (
|
|
349
|
+
if (expectedPartitionPath !== ownership.partitionPath) {
|
|
257
350
|
return {
|
|
258
351
|
deleted: false,
|
|
259
|
-
reason: `${
|
|
352
|
+
reason: `${ownership.partitionPath} is not the canonical partition path for ${ownership.partitionId}`,
|
|
260
353
|
};
|
|
261
354
|
}
|
|
355
|
+
const root = ompSessionsRoot(ownership.storageRoot);
|
|
356
|
+
let expectedStagingPath;
|
|
357
|
+
try {
|
|
358
|
+
expectedStagingPath = stagingPathForOwnership(root, ownership);
|
|
359
|
+
} catch (error) {
|
|
360
|
+
return {
|
|
361
|
+
deleted: false,
|
|
362
|
+
reason: `could not derive a staging name from the persisted owner identity: ${error.message}`,
|
|
363
|
+
};
|
|
364
|
+
}
|
|
365
|
+
if (stagingPath !== expectedStagingPath) {
|
|
366
|
+
return {
|
|
367
|
+
deleted: false,
|
|
368
|
+
reason: `${stagingPath} is not the staged path bound to partition ${ownership.partitionId} and its persisted owner identity`,
|
|
369
|
+
};
|
|
370
|
+
}
|
|
371
|
+
|
|
372
|
+
let staged;
|
|
373
|
+
try {
|
|
374
|
+
staged = pinDirectoryIdentity(stagingPath);
|
|
375
|
+
} catch (error) {
|
|
376
|
+
if (error.code === 'ENOENT') return { deleted: true, reason: 'already absent' };
|
|
377
|
+
return { deleted: false, reason: `${stagingPath}: ${error.message}` };
|
|
378
|
+
}
|
|
379
|
+
if (String(staged.uid) !== currentUid()) {
|
|
380
|
+
return { deleted: false, reason: `${stagingPath} is not owned by the current user` };
|
|
381
|
+
}
|
|
382
|
+
if (ownership.partitionIdentity && !sameIdentity(staged.identity, ownership.partitionIdentity)) {
|
|
383
|
+
return {
|
|
384
|
+
deleted: false,
|
|
385
|
+
reason: `${stagingPath} identity ${staged.identity.device}:${staged.identity.inode} does not match the recorded ${ownership.partitionIdentity.device}:${ownership.partitionIdentity.inode}`,
|
|
386
|
+
};
|
|
387
|
+
}
|
|
388
|
+
|
|
262
389
|
try {
|
|
263
390
|
fs.rmSync(stagingPath, { recursive: true, force: true });
|
|
264
391
|
} catch (error) {
|
|
@@ -279,7 +406,7 @@ function removeStagedOmpSessionPartition(stagingPath) {
|
|
|
279
406
|
function deleteOmpSessionPartition(ownership) {
|
|
280
407
|
const staged = stageOmpSessionPartitionForDeletion(ownership);
|
|
281
408
|
if (!staged.staged) return { deleted: staged.deleted === true, reason: staged.reason };
|
|
282
|
-
return removeStagedOmpSessionPartition(staged.stagingPath);
|
|
409
|
+
return removeStagedOmpSessionPartition(staged.stagingPath, ownership);
|
|
283
410
|
}
|
|
284
411
|
|
|
285
412
|
module.exports = {
|