@gea-ai/cli-darwin-arm64 0.1.260825-alpha.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +183 -0
- package/gea-darwin-arm64 +0 -0
- package/gea-worker-runtime +0 -0
- package/package.json +36 -0
- package/system-workers/filesystem.md +85 -0
- package/system-workers/filesystem.mjs +2038 -0
|
@@ -0,0 +1,2038 @@
|
|
|
1
|
+
import { DurableObject } from "cloudflare:workers";
|
|
2
|
+
|
|
3
|
+
const MAX_ALLOCATION_ID_BYTES = 512;
|
|
4
|
+
const MAX_MUTATION_ID_BYTES = 256;
|
|
5
|
+
const MAX_OPERATIONS_PER_BATCH = 256;
|
|
6
|
+
const MAX_INLINE_FILE_BYTES = 1024 * 1024;
|
|
7
|
+
const MAX_INLINE_BYTES_PER_OBJECT = 16 * 1024 * 1024;
|
|
8
|
+
const MAX_ENTRIES_PER_OBJECT = 100_000;
|
|
9
|
+
const MAX_ENTRIES_PER_DIRECTORY = 4_096;
|
|
10
|
+
const MAX_CHECKPOINT_IMAGE_BYTES = 4 * 1024 * 1024 * 1024;
|
|
11
|
+
const MAX_CHECKPOINT_MANIFEST_BYTES = 16 * 1024 * 1024;
|
|
12
|
+
const MAX_CHECKPOINT_TRANSFER_CHUNK_BYTES = 8 * 1024 * 1024;
|
|
13
|
+
const MAX_SQL_BLOB_BYTES = 1024 * 1024;
|
|
14
|
+
const MAX_PENDING_CHECKPOINT_UPLOADS = 16;
|
|
15
|
+
const CHECKPOINT_UPLOAD_TTL_MS = 24 * 60 * 60 * 1000;
|
|
16
|
+
const RETAINED_CHECKPOINTS = 2;
|
|
17
|
+
const MAX_MOUNT_BYTES = 20 * 1024 * 1024;
|
|
18
|
+
const MAX_MOUNT_FILES = 10_000;
|
|
19
|
+
const MAX_MOUNTS = 16;
|
|
20
|
+
const PROTOCOL_VERSION = 1;
|
|
21
|
+
const UUID_PATTERN =
|
|
22
|
+
/^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/;
|
|
23
|
+
|
|
24
|
+
class FilesystemRequestError extends Error {
|
|
25
|
+
constructor(status, message, code = "INVALID_REQUEST", retryable = false) {
|
|
26
|
+
super(message);
|
|
27
|
+
this.status = status;
|
|
28
|
+
this.code = code;
|
|
29
|
+
this.retryable = retryable;
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
export class FilesystemObject extends DurableObject {
|
|
34
|
+
constructor(ctx, env) {
|
|
35
|
+
super(ctx, env);
|
|
36
|
+
this.ctx.storage.sql.exec(`
|
|
37
|
+
CREATE TABLE IF NOT EXISTS filesystem_meta (
|
|
38
|
+
singleton INTEGER PRIMARY KEY CHECK (singleton = 1),
|
|
39
|
+
revision INTEGER NOT NULL,
|
|
40
|
+
logical_bytes INTEGER NOT NULL,
|
|
41
|
+
entry_count INTEGER NOT NULL
|
|
42
|
+
);
|
|
43
|
+
CREATE TABLE IF NOT EXISTS filesystem_entries (
|
|
44
|
+
path TEXT PRIMARY KEY,
|
|
45
|
+
parent_path TEXT,
|
|
46
|
+
name TEXT NOT NULL,
|
|
47
|
+
kind TEXT NOT NULL CHECK (kind IN ('directory', 'file')),
|
|
48
|
+
content BLOB,
|
|
49
|
+
sha256 TEXT,
|
|
50
|
+
size_bytes INTEGER NOT NULL,
|
|
51
|
+
created_at_ms INTEGER NOT NULL,
|
|
52
|
+
modified_at_ms INTEGER NOT NULL,
|
|
53
|
+
revision INTEGER NOT NULL
|
|
54
|
+
);
|
|
55
|
+
CREATE INDEX IF NOT EXISTS filesystem_entries_parent
|
|
56
|
+
ON filesystem_entries(parent_path, name);
|
|
57
|
+
CREATE TABLE IF NOT EXISTS filesystem_mutations (
|
|
58
|
+
mutation_id TEXT PRIMARY KEY,
|
|
59
|
+
request_hash TEXT NOT NULL,
|
|
60
|
+
applied_revision INTEGER NOT NULL,
|
|
61
|
+
result_json TEXT NOT NULL
|
|
62
|
+
);
|
|
63
|
+
CREATE TABLE IF NOT EXISTS filesystem_changes (
|
|
64
|
+
revision INTEGER NOT NULL,
|
|
65
|
+
sequence INTEGER NOT NULL,
|
|
66
|
+
operation TEXT NOT NULL,
|
|
67
|
+
path TEXT NOT NULL,
|
|
68
|
+
kind TEXT NOT NULL,
|
|
69
|
+
PRIMARY KEY (revision, sequence)
|
|
70
|
+
);
|
|
71
|
+
CREATE TABLE IF NOT EXISTS filesystem_checkpoint_meta (
|
|
72
|
+
singleton INTEGER PRIMARY KEY CHECK (singleton = 1),
|
|
73
|
+
checkpoint_revision INTEGER NOT NULL
|
|
74
|
+
);
|
|
75
|
+
CREATE TABLE IF NOT EXISTS filesystem_checkpoints (
|
|
76
|
+
checkpoint_revision INTEGER PRIMARY KEY,
|
|
77
|
+
parent_checkpoint_revision INTEGER NOT NULL,
|
|
78
|
+
checkpoint_id TEXT NOT NULL UNIQUE,
|
|
79
|
+
image_blob_id TEXT NOT NULL,
|
|
80
|
+
image_sha256 TEXT NOT NULL,
|
|
81
|
+
image_size_bytes INTEGER NOT NULL,
|
|
82
|
+
manifest_blob_id TEXT NOT NULL,
|
|
83
|
+
manifest_sha256 TEXT NOT NULL,
|
|
84
|
+
manifest_size_bytes INTEGER NOT NULL,
|
|
85
|
+
created_at_ms INTEGER NOT NULL
|
|
86
|
+
);
|
|
87
|
+
CREATE TABLE IF NOT EXISTS filesystem_checkpoint_blob_chunks (
|
|
88
|
+
blob_id TEXT NOT NULL,
|
|
89
|
+
offset INTEGER NOT NULL,
|
|
90
|
+
content BLOB NOT NULL,
|
|
91
|
+
PRIMARY KEY (blob_id, offset)
|
|
92
|
+
);
|
|
93
|
+
CREATE TABLE IF NOT EXISTS filesystem_checkpoint_uploads (
|
|
94
|
+
upload_id TEXT PRIMARY KEY,
|
|
95
|
+
kind TEXT NOT NULL CHECK (kind IN ('image', 'manifest')),
|
|
96
|
+
sha256 TEXT NOT NULL,
|
|
97
|
+
size_bytes INTEGER NOT NULL,
|
|
98
|
+
received_bytes INTEGER NOT NULL,
|
|
99
|
+
sha256_state_json TEXT NOT NULL,
|
|
100
|
+
committed INTEGER NOT NULL CHECK (committed IN (0, 1)),
|
|
101
|
+
created_at_ms INTEGER NOT NULL
|
|
102
|
+
);
|
|
103
|
+
CREATE TABLE IF NOT EXISTS filesystem_mounts (
|
|
104
|
+
mount TEXT PRIMARY KEY,
|
|
105
|
+
content_id TEXT NOT NULL,
|
|
106
|
+
mode TEXT NOT NULL CHECK (mode = 'read-only'),
|
|
107
|
+
file_count INTEGER NOT NULL,
|
|
108
|
+
size_bytes INTEGER NOT NULL,
|
|
109
|
+
created_at_ms INTEGER NOT NULL
|
|
110
|
+
);
|
|
111
|
+
CREATE TABLE IF NOT EXISTS filesystem_mount_files (
|
|
112
|
+
mount TEXT NOT NULL,
|
|
113
|
+
path TEXT NOT NULL,
|
|
114
|
+
size_bytes INTEGER NOT NULL,
|
|
115
|
+
PRIMARY KEY (mount, path)
|
|
116
|
+
);
|
|
117
|
+
CREATE TABLE IF NOT EXISTS filesystem_mount_file_chunks (
|
|
118
|
+
mount TEXT NOT NULL,
|
|
119
|
+
path TEXT NOT NULL,
|
|
120
|
+
offset INTEGER NOT NULL,
|
|
121
|
+
content BLOB NOT NULL,
|
|
122
|
+
PRIMARY KEY (mount, path, offset)
|
|
123
|
+
);
|
|
124
|
+
`);
|
|
125
|
+
this.#initializeFilesystem();
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
async fetch(request) {
|
|
129
|
+
try {
|
|
130
|
+
const url = new URL(request.url);
|
|
131
|
+
if (request.method === "GET" && url.pathname === "/") {
|
|
132
|
+
return Response.json(this.#inspect());
|
|
133
|
+
}
|
|
134
|
+
if (request.method === "POST" && url.pathname === "/batches") {
|
|
135
|
+
return Response.json(await this.#applyBatch(await readJson(request)));
|
|
136
|
+
}
|
|
137
|
+
if (request.method === "POST" && url.pathname === "/invoke") {
|
|
138
|
+
return Response.json(await this.#invoke(await readJson(request)));
|
|
139
|
+
}
|
|
140
|
+
if (request.method === "GET" && url.pathname === "/entries") {
|
|
141
|
+
return Response.json(this.#listEntries(url.searchParams.get("path")));
|
|
142
|
+
}
|
|
143
|
+
if (request.method === "GET" && url.pathname === "/files") {
|
|
144
|
+
return Response.json(this.#readFile(url.searchParams.get("path")));
|
|
145
|
+
}
|
|
146
|
+
return jsonError(404, "Filesystem operation was not found");
|
|
147
|
+
} catch (error) {
|
|
148
|
+
if (error instanceof FilesystemRequestError) {
|
|
149
|
+
return jsonError(
|
|
150
|
+
error.status,
|
|
151
|
+
error.message,
|
|
152
|
+
error.code,
|
|
153
|
+
error.retryable,
|
|
154
|
+
);
|
|
155
|
+
}
|
|
156
|
+
console.error("FilesystemObject request failed", error);
|
|
157
|
+
return jsonError(500, "Filesystem operation failed");
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
#initializeFilesystem() {
|
|
162
|
+
this.ctx.storage.sql.exec(
|
|
163
|
+
`INSERT OR IGNORE INTO filesystem_checkpoint_meta
|
|
164
|
+
(singleton, checkpoint_revision)
|
|
165
|
+
VALUES (1, 0)`,
|
|
166
|
+
);
|
|
167
|
+
const meta = this.ctx.storage.sql
|
|
168
|
+
.exec("SELECT revision FROM filesystem_meta WHERE singleton = 1")
|
|
169
|
+
.toArray();
|
|
170
|
+
if (meta.length !== 0) return;
|
|
171
|
+
|
|
172
|
+
const now = Date.now();
|
|
173
|
+
this.ctx.storage.transactionSync((storage) => {
|
|
174
|
+
storage.sql.exec(
|
|
175
|
+
`INSERT INTO filesystem_meta
|
|
176
|
+
(singleton, revision, logical_bytes, entry_count)
|
|
177
|
+
VALUES (1, 0, 0, 3)`,
|
|
178
|
+
);
|
|
179
|
+
for (const [path, parentPath, name] of [
|
|
180
|
+
["/", null, ""],
|
|
181
|
+
["/workspace", "/", "workspace"],
|
|
182
|
+
["/.gea", "/", ".gea"],
|
|
183
|
+
]) {
|
|
184
|
+
storage.sql.exec(
|
|
185
|
+
`INSERT INTO filesystem_entries
|
|
186
|
+
(path, parent_path, name, kind, content, sha256, size_bytes,
|
|
187
|
+
created_at_ms, modified_at_ms, revision)
|
|
188
|
+
VALUES (?, ?, ?, 'directory', NULL, NULL, 0, ?, ?, 0)`,
|
|
189
|
+
path,
|
|
190
|
+
parentPath,
|
|
191
|
+
name,
|
|
192
|
+
now,
|
|
193
|
+
now,
|
|
194
|
+
);
|
|
195
|
+
}
|
|
196
|
+
});
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
#inspect() {
|
|
200
|
+
const meta = readMeta(this.ctx.storage);
|
|
201
|
+
return {
|
|
202
|
+
revision: meta.revision,
|
|
203
|
+
logicalBytes: meta.logical_bytes,
|
|
204
|
+
entryCount: meta.entry_count,
|
|
205
|
+
};
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
async #applyBatch(input) {
|
|
209
|
+
const batch = parseBatch(input);
|
|
210
|
+
const preparedOperations = [];
|
|
211
|
+
let preparedInlineBytes = 0;
|
|
212
|
+
for (const operation of batch.operations) {
|
|
213
|
+
if (operation.type === "createDirectory") {
|
|
214
|
+
preparedOperations.push(operation);
|
|
215
|
+
continue;
|
|
216
|
+
}
|
|
217
|
+
const content = decodeBase64(operation.contentBase64);
|
|
218
|
+
if (content.byteLength > MAX_INLINE_FILE_BYTES) {
|
|
219
|
+
throw new FilesystemRequestError(
|
|
220
|
+
413,
|
|
221
|
+
`Inline file content exceeds ${MAX_INLINE_FILE_BYTES} bytes`,
|
|
222
|
+
);
|
|
223
|
+
}
|
|
224
|
+
preparedInlineBytes += content.byteLength;
|
|
225
|
+
if (preparedInlineBytes > MAX_INLINE_BYTES_PER_OBJECT) {
|
|
226
|
+
throw new FilesystemRequestError(
|
|
227
|
+
413,
|
|
228
|
+
`Batch inline content exceeds ${MAX_INLINE_BYTES_PER_OBJECT} bytes`,
|
|
229
|
+
);
|
|
230
|
+
}
|
|
231
|
+
preparedOperations.push({
|
|
232
|
+
...operation,
|
|
233
|
+
content,
|
|
234
|
+
sha256: await sha256(content),
|
|
235
|
+
});
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
const requestHash = await sha256(
|
|
239
|
+
new TextEncoder().encode(JSON.stringify(batch)),
|
|
240
|
+
);
|
|
241
|
+
const prior = this.ctx.storage.sql
|
|
242
|
+
.exec(
|
|
243
|
+
`SELECT request_hash, result_json
|
|
244
|
+
FROM filesystem_mutations
|
|
245
|
+
WHERE mutation_id = ?`,
|
|
246
|
+
batch.mutationId,
|
|
247
|
+
)
|
|
248
|
+
.toArray();
|
|
249
|
+
if (prior.length !== 0) return replayMutation(prior[0], requestHash);
|
|
250
|
+
|
|
251
|
+
return this.ctx.storage.transactionSync((storage) => {
|
|
252
|
+
const replay = storage.sql
|
|
253
|
+
.exec(
|
|
254
|
+
`SELECT request_hash, result_json
|
|
255
|
+
FROM filesystem_mutations
|
|
256
|
+
WHERE mutation_id = ?`,
|
|
257
|
+
batch.mutationId,
|
|
258
|
+
)
|
|
259
|
+
.toArray();
|
|
260
|
+
if (replay.length !== 0) return replayMutation(replay[0], requestHash);
|
|
261
|
+
|
|
262
|
+
const meta = readMeta(storage);
|
|
263
|
+
if (meta.revision !== batch.expectedRevision) {
|
|
264
|
+
throw new FilesystemRequestError(
|
|
265
|
+
409,
|
|
266
|
+
`Expected filesystem revision ${batch.expectedRevision}, current revision is ${meta.revision}`,
|
|
267
|
+
);
|
|
268
|
+
}
|
|
269
|
+
|
|
270
|
+
const nextRevision = meta.revision + 1;
|
|
271
|
+
const changes = [];
|
|
272
|
+
let logicalBytes = meta.logical_bytes;
|
|
273
|
+
let entryCount = meta.entry_count;
|
|
274
|
+
const now = Date.now();
|
|
275
|
+
|
|
276
|
+
for (const [sequence, operation] of preparedOperations.entries()) {
|
|
277
|
+
const parentPath = parentOf(operation.path);
|
|
278
|
+
requireDirectory(storage, parentPath);
|
|
279
|
+
|
|
280
|
+
if (operation.type === "createDirectory") {
|
|
281
|
+
requireNewPath(storage, operation.path);
|
|
282
|
+
requireDirectoryCapacity(storage, parentPath);
|
|
283
|
+
if (entryCount >= MAX_ENTRIES_PER_OBJECT) {
|
|
284
|
+
throw new FilesystemRequestError(
|
|
285
|
+
413,
|
|
286
|
+
`Filesystem contains the maximum ${MAX_ENTRIES_PER_OBJECT} entries`,
|
|
287
|
+
);
|
|
288
|
+
}
|
|
289
|
+
storage.sql.exec(
|
|
290
|
+
`INSERT INTO filesystem_entries
|
|
291
|
+
(path, parent_path, name, kind, content, sha256, size_bytes,
|
|
292
|
+
created_at_ms, modified_at_ms, revision)
|
|
293
|
+
VALUES (?, ?, ?, 'directory', NULL, NULL, 0, ?, ?, ?)`,
|
|
294
|
+
operation.path,
|
|
295
|
+
parentPath,
|
|
296
|
+
basename(operation.path),
|
|
297
|
+
now,
|
|
298
|
+
now,
|
|
299
|
+
nextRevision,
|
|
300
|
+
);
|
|
301
|
+
entryCount += 1;
|
|
302
|
+
} else {
|
|
303
|
+
const existing = storage.sql
|
|
304
|
+
.exec(
|
|
305
|
+
`SELECT kind, size_bytes
|
|
306
|
+
FROM filesystem_entries
|
|
307
|
+
WHERE path = ?`,
|
|
308
|
+
operation.path,
|
|
309
|
+
)
|
|
310
|
+
.toArray();
|
|
311
|
+
if (existing.length !== 0 && existing[0].kind !== "file") {
|
|
312
|
+
throw new FilesystemRequestError(
|
|
313
|
+
409,
|
|
314
|
+
`A directory already exists at ${operation.path}`,
|
|
315
|
+
);
|
|
316
|
+
}
|
|
317
|
+
if (existing.length === 0) {
|
|
318
|
+
requireDirectoryCapacity(storage, parentPath);
|
|
319
|
+
if (entryCount >= MAX_ENTRIES_PER_OBJECT) {
|
|
320
|
+
throw new FilesystemRequestError(
|
|
321
|
+
413,
|
|
322
|
+
`Filesystem contains the maximum ${MAX_ENTRIES_PER_OBJECT} entries`,
|
|
323
|
+
);
|
|
324
|
+
}
|
|
325
|
+
}
|
|
326
|
+
const previousSize = existing[0]?.size_bytes ?? 0;
|
|
327
|
+
const nextLogicalBytes =
|
|
328
|
+
logicalBytes - previousSize + operation.content.byteLength;
|
|
329
|
+
if (nextLogicalBytes > MAX_INLINE_BYTES_PER_OBJECT) {
|
|
330
|
+
throw new FilesystemRequestError(
|
|
331
|
+
413,
|
|
332
|
+
`Filesystem inline content exceeds ${MAX_INLINE_BYTES_PER_OBJECT} bytes`,
|
|
333
|
+
);
|
|
334
|
+
}
|
|
335
|
+
if (existing.length === 0) {
|
|
336
|
+
storage.sql.exec(
|
|
337
|
+
`INSERT INTO filesystem_entries
|
|
338
|
+
(path, parent_path, name, kind, content, sha256, size_bytes,
|
|
339
|
+
created_at_ms, modified_at_ms, revision)
|
|
340
|
+
VALUES (?, ?, ?, 'file', ?, ?, ?, ?, ?, ?)`,
|
|
341
|
+
operation.path,
|
|
342
|
+
parentPath,
|
|
343
|
+
basename(operation.path),
|
|
344
|
+
operation.content,
|
|
345
|
+
operation.sha256,
|
|
346
|
+
operation.content.byteLength,
|
|
347
|
+
now,
|
|
348
|
+
now,
|
|
349
|
+
nextRevision,
|
|
350
|
+
);
|
|
351
|
+
entryCount += 1;
|
|
352
|
+
} else {
|
|
353
|
+
storage.sql.exec(
|
|
354
|
+
`UPDATE filesystem_entries
|
|
355
|
+
SET content = ?, sha256 = ?, size_bytes = ?, modified_at_ms = ?,
|
|
356
|
+
revision = ?
|
|
357
|
+
WHERE path = ?`,
|
|
358
|
+
operation.content,
|
|
359
|
+
operation.sha256,
|
|
360
|
+
operation.content.byteLength,
|
|
361
|
+
now,
|
|
362
|
+
nextRevision,
|
|
363
|
+
operation.path,
|
|
364
|
+
);
|
|
365
|
+
}
|
|
366
|
+
logicalBytes = nextLogicalBytes;
|
|
367
|
+
}
|
|
368
|
+
|
|
369
|
+
const kind =
|
|
370
|
+
operation.type === "createDirectory" ? "directory" : "file";
|
|
371
|
+
const change = {
|
|
372
|
+
operation: operation.type,
|
|
373
|
+
path: operation.path,
|
|
374
|
+
kind,
|
|
375
|
+
};
|
|
376
|
+
changes.push(change);
|
|
377
|
+
storage.sql.exec(
|
|
378
|
+
`INSERT INTO filesystem_changes
|
|
379
|
+
(revision, sequence, operation, path, kind)
|
|
380
|
+
VALUES (?, ?, ?, ?, ?)`,
|
|
381
|
+
nextRevision,
|
|
382
|
+
sequence,
|
|
383
|
+
change.operation,
|
|
384
|
+
change.path,
|
|
385
|
+
change.kind,
|
|
386
|
+
);
|
|
387
|
+
}
|
|
388
|
+
|
|
389
|
+
storage.sql.exec(
|
|
390
|
+
`UPDATE filesystem_meta
|
|
391
|
+
SET revision = ?, logical_bytes = ?, entry_count = ?
|
|
392
|
+
WHERE singleton = 1`,
|
|
393
|
+
nextRevision,
|
|
394
|
+
logicalBytes,
|
|
395
|
+
entryCount,
|
|
396
|
+
);
|
|
397
|
+
const result = {
|
|
398
|
+
revision: nextRevision,
|
|
399
|
+
logicalBytes,
|
|
400
|
+
changes,
|
|
401
|
+
};
|
|
402
|
+
storage.sql.exec(
|
|
403
|
+
`INSERT INTO filesystem_mutations
|
|
404
|
+
(mutation_id, request_hash, applied_revision, result_json)
|
|
405
|
+
VALUES (?, ?, ?, ?)`,
|
|
406
|
+
batch.mutationId,
|
|
407
|
+
requestHash,
|
|
408
|
+
nextRevision,
|
|
409
|
+
JSON.stringify(result),
|
|
410
|
+
);
|
|
411
|
+
return result;
|
|
412
|
+
});
|
|
413
|
+
}
|
|
414
|
+
|
|
415
|
+
async #invoke(input) {
|
|
416
|
+
const invocation = parseInvocation(input);
|
|
417
|
+
let result;
|
|
418
|
+
switch (invocation.operation) {
|
|
419
|
+
case "inspect":
|
|
420
|
+
result = this.#inspect();
|
|
421
|
+
break;
|
|
422
|
+
case "inspectCheckpoints":
|
|
423
|
+
result = this.#inspectCheckpoints();
|
|
424
|
+
break;
|
|
425
|
+
case "initializeMount":
|
|
426
|
+
result = await this.#initializeMount(invocation);
|
|
427
|
+
break;
|
|
428
|
+
case "listMounts":
|
|
429
|
+
result = this.#listMounts();
|
|
430
|
+
break;
|
|
431
|
+
case "getMount":
|
|
432
|
+
result = this.#getMount(invocation.input);
|
|
433
|
+
break;
|
|
434
|
+
case "beginCheckpointUpload":
|
|
435
|
+
result = this.#beginCheckpointUpload(invocation.input);
|
|
436
|
+
break;
|
|
437
|
+
case "appendCheckpointUpload":
|
|
438
|
+
result = this.#appendCheckpointUpload(invocation.input);
|
|
439
|
+
break;
|
|
440
|
+
case "commitCheckpoint":
|
|
441
|
+
result = await this.#commitCheckpoint(invocation);
|
|
442
|
+
break;
|
|
443
|
+
case "getCheckpoint":
|
|
444
|
+
result = this.#getCheckpoint(invocation.input);
|
|
445
|
+
break;
|
|
446
|
+
case "readCheckpointBlobRange":
|
|
447
|
+
result = this.#readCheckpointBlobRange(invocation.input);
|
|
448
|
+
break;
|
|
449
|
+
case "beginDirectCheckpointUpload":
|
|
450
|
+
case "abortDirectCheckpointUpload":
|
|
451
|
+
case "getDirectCheckpointDownload":
|
|
452
|
+
throw new FilesystemRequestError(
|
|
453
|
+
501,
|
|
454
|
+
"Direct checkpoint transfer is not supported by the Filesystem System Worker",
|
|
455
|
+
"UNSUPPORTED",
|
|
456
|
+
);
|
|
457
|
+
default:
|
|
458
|
+
throw new FilesystemRequestError(
|
|
459
|
+
404,
|
|
460
|
+
`Filesystem operation was not found: ${invocation.operation}`,
|
|
461
|
+
"UNSUPPORTED",
|
|
462
|
+
);
|
|
463
|
+
}
|
|
464
|
+
return {
|
|
465
|
+
protocolVersion: PROTOCOL_VERSION,
|
|
466
|
+
objectId: invocation.objectId,
|
|
467
|
+
epoch: 1,
|
|
468
|
+
result,
|
|
469
|
+
};
|
|
470
|
+
}
|
|
471
|
+
|
|
472
|
+
#inspectCheckpoints() {
|
|
473
|
+
const revision = readCheckpointRevision(this.ctx.storage);
|
|
474
|
+
return {
|
|
475
|
+
checkpointRevision: revision,
|
|
476
|
+
latestCheckpoint:
|
|
477
|
+
revision === 0 ? null : readCheckpoint(this.ctx.storage, revision),
|
|
478
|
+
maxCheckpointImageBytes: MAX_CHECKPOINT_IMAGE_BYTES,
|
|
479
|
+
maxCheckpointManifestBytes: MAX_CHECKPOINT_MANIFEST_BYTES,
|
|
480
|
+
maxCheckpointRecords: RETAINED_CHECKPOINTS,
|
|
481
|
+
};
|
|
482
|
+
}
|
|
483
|
+
|
|
484
|
+
#beginCheckpointUpload(input) {
|
|
485
|
+
const upload = parseCheckpointUpload(input);
|
|
486
|
+
this.ctx.storage.transactionSync((storage) => {
|
|
487
|
+
const staleUploads = storage.sql
|
|
488
|
+
.exec(
|
|
489
|
+
`SELECT upload_id
|
|
490
|
+
FROM filesystem_checkpoint_uploads
|
|
491
|
+
WHERE committed = 0 AND created_at_ms < ?`,
|
|
492
|
+
Date.now() - CHECKPOINT_UPLOAD_TTL_MS,
|
|
493
|
+
)
|
|
494
|
+
.toArray();
|
|
495
|
+
for (const stale of staleUploads) {
|
|
496
|
+
storage.sql.exec(
|
|
497
|
+
"DELETE FROM filesystem_checkpoint_blob_chunks WHERE blob_id = ?",
|
|
498
|
+
stale.upload_id,
|
|
499
|
+
);
|
|
500
|
+
storage.sql.exec(
|
|
501
|
+
"DELETE FROM filesystem_checkpoint_uploads WHERE upload_id = ?",
|
|
502
|
+
stale.upload_id,
|
|
503
|
+
);
|
|
504
|
+
}
|
|
505
|
+
});
|
|
506
|
+
const existing = this.ctx.storage.sql
|
|
507
|
+
.exec(
|
|
508
|
+
`SELECT kind, sha256, size_bytes
|
|
509
|
+
FROM filesystem_checkpoint_uploads
|
|
510
|
+
WHERE upload_id = ?`,
|
|
511
|
+
upload.uploadId,
|
|
512
|
+
)
|
|
513
|
+
.toArray();
|
|
514
|
+
if (existing.length !== 0) {
|
|
515
|
+
const row = existing[0];
|
|
516
|
+
if (
|
|
517
|
+
row.kind !== upload.kind ||
|
|
518
|
+
row.sha256 !== upload.sha256 ||
|
|
519
|
+
row.size_bytes !== upload.sizeBytes
|
|
520
|
+
) {
|
|
521
|
+
throw new FilesystemRequestError(
|
|
522
|
+
409,
|
|
523
|
+
"Checkpoint upload ID already exists with different metadata",
|
|
524
|
+
);
|
|
525
|
+
}
|
|
526
|
+
return {
|
|
527
|
+
maxChunkBytes: MAX_CHECKPOINT_TRANSFER_CHUNK_BYTES,
|
|
528
|
+
sizeBytes: upload.sizeBytes,
|
|
529
|
+
uploadId: upload.uploadId,
|
|
530
|
+
};
|
|
531
|
+
}
|
|
532
|
+
const pending = this.ctx.storage.sql
|
|
533
|
+
.exec(
|
|
534
|
+
"SELECT COUNT(*) AS count FROM filesystem_checkpoint_uploads WHERE committed = 0",
|
|
535
|
+
)
|
|
536
|
+
.one();
|
|
537
|
+
if (pending.count >= MAX_PENDING_CHECKPOINT_UPLOADS) {
|
|
538
|
+
throw new FilesystemRequestError(
|
|
539
|
+
409,
|
|
540
|
+
`Filesystem has reached the ${MAX_PENDING_CHECKPOINT_UPLOADS}-upload limit`,
|
|
541
|
+
);
|
|
542
|
+
}
|
|
543
|
+
this.ctx.storage.sql.exec(
|
|
544
|
+
`INSERT INTO filesystem_checkpoint_uploads
|
|
545
|
+
(upload_id, kind, sha256, size_bytes, received_bytes,
|
|
546
|
+
sha256_state_json, committed, created_at_ms)
|
|
547
|
+
VALUES (?, ?, ?, ?, 0, ?, 0, ?)`,
|
|
548
|
+
upload.uploadId,
|
|
549
|
+
upload.kind,
|
|
550
|
+
upload.sha256,
|
|
551
|
+
upload.sizeBytes,
|
|
552
|
+
JSON.stringify(new Sha256State().toJSON()),
|
|
553
|
+
Date.now(),
|
|
554
|
+
);
|
|
555
|
+
return {
|
|
556
|
+
maxChunkBytes: MAX_CHECKPOINT_TRANSFER_CHUNK_BYTES,
|
|
557
|
+
sizeBytes: upload.sizeBytes,
|
|
558
|
+
uploadId: upload.uploadId,
|
|
559
|
+
};
|
|
560
|
+
}
|
|
561
|
+
|
|
562
|
+
#appendCheckpointUpload(input) {
|
|
563
|
+
const append = parseCheckpointAppend(input);
|
|
564
|
+
return this.ctx.storage.transactionSync((storage) => {
|
|
565
|
+
const rows = storage.sql
|
|
566
|
+
.exec(
|
|
567
|
+
`SELECT size_bytes, received_bytes, sha256_state_json, committed
|
|
568
|
+
FROM filesystem_checkpoint_uploads
|
|
569
|
+
WHERE upload_id = ?`,
|
|
570
|
+
append.uploadId,
|
|
571
|
+
)
|
|
572
|
+
.toArray();
|
|
573
|
+
if (rows.length === 0) {
|
|
574
|
+
throw new FilesystemRequestError(
|
|
575
|
+
404,
|
|
576
|
+
"Checkpoint upload was not found",
|
|
577
|
+
);
|
|
578
|
+
}
|
|
579
|
+
const upload = rows[0];
|
|
580
|
+
const end = append.offset + append.bytes.byteLength;
|
|
581
|
+
if (!Number.isSafeInteger(end) || end > upload.size_bytes) {
|
|
582
|
+
throw new FilesystemRequestError(
|
|
583
|
+
409,
|
|
584
|
+
"Checkpoint upload chunk exceeds its declared size",
|
|
585
|
+
);
|
|
586
|
+
}
|
|
587
|
+
if (upload.committed === 1) {
|
|
588
|
+
const replay = readUploadRange(
|
|
589
|
+
storage,
|
|
590
|
+
append.uploadId,
|
|
591
|
+
append.offset,
|
|
592
|
+
append.bytes.byteLength,
|
|
593
|
+
);
|
|
594
|
+
if (!equalBytes(replay, append.bytes)) {
|
|
595
|
+
throw new FilesystemRequestError(
|
|
596
|
+
409,
|
|
597
|
+
"Committed checkpoint upload bytes cannot be replaced",
|
|
598
|
+
);
|
|
599
|
+
}
|
|
600
|
+
return { receivedBytes: end, uploadId: append.uploadId };
|
|
601
|
+
}
|
|
602
|
+
if (upload.received_bytes !== append.offset) {
|
|
603
|
+
if (upload.received_bytes === end) {
|
|
604
|
+
const replay = readUploadRange(
|
|
605
|
+
storage,
|
|
606
|
+
append.uploadId,
|
|
607
|
+
append.offset,
|
|
608
|
+
append.bytes.byteLength,
|
|
609
|
+
);
|
|
610
|
+
if (equalBytes(replay, append.bytes)) {
|
|
611
|
+
return {
|
|
612
|
+
receivedBytes: end,
|
|
613
|
+
uploadId: append.uploadId,
|
|
614
|
+
};
|
|
615
|
+
}
|
|
616
|
+
}
|
|
617
|
+
throw new FilesystemRequestError(
|
|
618
|
+
409,
|
|
619
|
+
"Checkpoint upload chunks must be appended sequentially",
|
|
620
|
+
);
|
|
621
|
+
}
|
|
622
|
+
for (let offset = 0; offset < append.bytes.byteLength; ) {
|
|
623
|
+
const chunk = append.bytes.slice(offset, offset + MAX_SQL_BLOB_BYTES);
|
|
624
|
+
storage.sql.exec(
|
|
625
|
+
`INSERT INTO filesystem_checkpoint_blob_chunks
|
|
626
|
+
(blob_id, offset, content)
|
|
627
|
+
VALUES (?, ?, ?)`,
|
|
628
|
+
append.uploadId,
|
|
629
|
+
append.offset + offset,
|
|
630
|
+
chunk,
|
|
631
|
+
);
|
|
632
|
+
offset += chunk.byteLength;
|
|
633
|
+
}
|
|
634
|
+
const digest = Sha256State.fromJSON(JSON.parse(upload.sha256_state_json));
|
|
635
|
+
digest.update(append.bytes);
|
|
636
|
+
storage.sql.exec(
|
|
637
|
+
`UPDATE filesystem_checkpoint_uploads
|
|
638
|
+
SET received_bytes = ?, sha256_state_json = ?
|
|
639
|
+
WHERE upload_id = ?`,
|
|
640
|
+
end,
|
|
641
|
+
JSON.stringify(digest.toJSON()),
|
|
642
|
+
append.uploadId,
|
|
643
|
+
);
|
|
644
|
+
return { receivedBytes: end, uploadId: append.uploadId };
|
|
645
|
+
});
|
|
646
|
+
}
|
|
647
|
+
|
|
648
|
+
async #commitCheckpoint(invocation) {
|
|
649
|
+
const commit = parseCheckpointCommit(invocation);
|
|
650
|
+
const requestHash = await mutationHash(invocation);
|
|
651
|
+
const prior = this.ctx.storage.sql
|
|
652
|
+
.exec(
|
|
653
|
+
`SELECT request_hash, result_json
|
|
654
|
+
FROM filesystem_mutations
|
|
655
|
+
WHERE mutation_id = ?`,
|
|
656
|
+
invocation.mutationId,
|
|
657
|
+
)
|
|
658
|
+
.toArray();
|
|
659
|
+
if (prior.length !== 0) return replayMutation(prior[0], requestHash);
|
|
660
|
+
|
|
661
|
+
return this.ctx.storage.transactionSync((storage) => {
|
|
662
|
+
const replay = storage.sql
|
|
663
|
+
.exec(
|
|
664
|
+
`SELECT request_hash, result_json
|
|
665
|
+
FROM filesystem_mutations
|
|
666
|
+
WHERE mutation_id = ?`,
|
|
667
|
+
invocation.mutationId,
|
|
668
|
+
)
|
|
669
|
+
.toArray();
|
|
670
|
+
if (replay.length !== 0) return replayMutation(replay[0], requestHash);
|
|
671
|
+
const actualRevision = readCheckpointRevision(storage);
|
|
672
|
+
if (actualRevision !== commit.expectedCheckpointRevision) {
|
|
673
|
+
throw new FilesystemRequestError(
|
|
674
|
+
409,
|
|
675
|
+
`Workspace checkpoint revision conflict: expected ${commit.expectedCheckpointRevision}, actual ${actualRevision}`,
|
|
676
|
+
);
|
|
677
|
+
}
|
|
678
|
+
const duplicateId = storage.sql
|
|
679
|
+
.exec(
|
|
680
|
+
"SELECT 1 AS present FROM filesystem_checkpoints WHERE checkpoint_id = ?",
|
|
681
|
+
commit.checkpointId,
|
|
682
|
+
)
|
|
683
|
+
.toArray();
|
|
684
|
+
if (duplicateId.length !== 0) {
|
|
685
|
+
throw new FilesystemRequestError(
|
|
686
|
+
409,
|
|
687
|
+
"Workspace checkpoint ID already exists",
|
|
688
|
+
);
|
|
689
|
+
}
|
|
690
|
+
const image = requireCompleteUpload(storage, commit.image, "image");
|
|
691
|
+
const manifest = requireCompleteUpload(
|
|
692
|
+
storage,
|
|
693
|
+
commit.manifest,
|
|
694
|
+
"manifest",
|
|
695
|
+
);
|
|
696
|
+
const nextRevision = actualRevision + 1;
|
|
697
|
+
const createdAtMs = Date.now();
|
|
698
|
+
storage.sql.exec(
|
|
699
|
+
`INSERT INTO filesystem_checkpoints
|
|
700
|
+
(checkpoint_revision, parent_checkpoint_revision, checkpoint_id,
|
|
701
|
+
image_blob_id, image_sha256, image_size_bytes, manifest_blob_id,
|
|
702
|
+
manifest_sha256, manifest_size_bytes, created_at_ms)
|
|
703
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
|
704
|
+
nextRevision,
|
|
705
|
+
actualRevision,
|
|
706
|
+
commit.checkpointId,
|
|
707
|
+
image.upload_id,
|
|
708
|
+
image.sha256,
|
|
709
|
+
image.size_bytes,
|
|
710
|
+
manifest.upload_id,
|
|
711
|
+
manifest.sha256,
|
|
712
|
+
manifest.size_bytes,
|
|
713
|
+
createdAtMs,
|
|
714
|
+
);
|
|
715
|
+
storage.sql.exec(
|
|
716
|
+
`UPDATE filesystem_checkpoint_meta
|
|
717
|
+
SET checkpoint_revision = ?
|
|
718
|
+
WHERE singleton = 1`,
|
|
719
|
+
nextRevision,
|
|
720
|
+
);
|
|
721
|
+
for (const upload of [image, manifest]) {
|
|
722
|
+
storage.sql.exec(
|
|
723
|
+
`UPDATE filesystem_checkpoint_uploads
|
|
724
|
+
SET committed = 1
|
|
725
|
+
WHERE upload_id = ?`,
|
|
726
|
+
upload.upload_id,
|
|
727
|
+
);
|
|
728
|
+
}
|
|
729
|
+
const retained = storage.sql
|
|
730
|
+
.exec(
|
|
731
|
+
`SELECT checkpoint_revision, image_blob_id, manifest_blob_id
|
|
732
|
+
FROM filesystem_checkpoints
|
|
733
|
+
ORDER BY checkpoint_revision DESC`,
|
|
734
|
+
)
|
|
735
|
+
.toArray();
|
|
736
|
+
for (const stale of retained.slice(RETAINED_CHECKPOINTS)) {
|
|
737
|
+
storage.sql.exec(
|
|
738
|
+
"DELETE FROM filesystem_checkpoint_blob_chunks WHERE blob_id IN (?, ?)",
|
|
739
|
+
stale.image_blob_id,
|
|
740
|
+
stale.manifest_blob_id,
|
|
741
|
+
);
|
|
742
|
+
storage.sql.exec(
|
|
743
|
+
"DELETE FROM filesystem_checkpoint_uploads WHERE upload_id IN (?, ?)",
|
|
744
|
+
stale.image_blob_id,
|
|
745
|
+
stale.manifest_blob_id,
|
|
746
|
+
);
|
|
747
|
+
storage.sql.exec(
|
|
748
|
+
"DELETE FROM filesystem_checkpoints WHERE checkpoint_revision = ?",
|
|
749
|
+
stale.checkpoint_revision,
|
|
750
|
+
);
|
|
751
|
+
}
|
|
752
|
+
const checkpoint = checkpointDescriptor({
|
|
753
|
+
checkpoint_revision: nextRevision,
|
|
754
|
+
parent_checkpoint_revision: actualRevision,
|
|
755
|
+
checkpoint_id: commit.checkpointId,
|
|
756
|
+
image_blob_id: image.upload_id,
|
|
757
|
+
image_sha256: image.sha256,
|
|
758
|
+
image_size_bytes: image.size_bytes,
|
|
759
|
+
manifest_blob_id: manifest.upload_id,
|
|
760
|
+
manifest_sha256: manifest.sha256,
|
|
761
|
+
manifest_size_bytes: manifest.size_bytes,
|
|
762
|
+
created_at_ms: createdAtMs,
|
|
763
|
+
});
|
|
764
|
+
const result = { checkpoint, checkpointRevision: nextRevision };
|
|
765
|
+
storage.sql.exec(
|
|
766
|
+
`INSERT INTO filesystem_mutations
|
|
767
|
+
(mutation_id, request_hash, applied_revision, result_json)
|
|
768
|
+
VALUES (?, ?, ?, ?)`,
|
|
769
|
+
invocation.mutationId,
|
|
770
|
+
requestHash,
|
|
771
|
+
nextRevision,
|
|
772
|
+
JSON.stringify(result),
|
|
773
|
+
);
|
|
774
|
+
return result;
|
|
775
|
+
});
|
|
776
|
+
}
|
|
777
|
+
|
|
778
|
+
#getCheckpoint(input) {
|
|
779
|
+
const requested = optionalNonNegativeInteger(
|
|
780
|
+
input?.checkpointRevision,
|
|
781
|
+
"checkpointRevision",
|
|
782
|
+
);
|
|
783
|
+
const current = readCheckpointRevision(this.ctx.storage);
|
|
784
|
+
const revision = requested ?? current;
|
|
785
|
+
if (revision === 0) {
|
|
786
|
+
throw new FilesystemRequestError(
|
|
787
|
+
404,
|
|
788
|
+
"Workspace checkpoint was not found",
|
|
789
|
+
);
|
|
790
|
+
}
|
|
791
|
+
return {
|
|
792
|
+
checkpoint: readCheckpoint(this.ctx.storage, revision),
|
|
793
|
+
checkpointRevision: current,
|
|
794
|
+
};
|
|
795
|
+
}
|
|
796
|
+
|
|
797
|
+
#readCheckpointBlobRange(input) {
|
|
798
|
+
const range = parseCheckpointRange(input);
|
|
799
|
+
const checkpoint = readCheckpointRow(
|
|
800
|
+
this.ctx.storage,
|
|
801
|
+
range.checkpointRevision,
|
|
802
|
+
);
|
|
803
|
+
const sha256 = checkpoint[`${range.kind}_sha256`];
|
|
804
|
+
const sizeBytes = checkpoint[`${range.kind}_size_bytes`];
|
|
805
|
+
const blobId = checkpoint[`${range.kind}_blob_id`];
|
|
806
|
+
if (range.offset >= sizeBytes) {
|
|
807
|
+
throw new FilesystemRequestError(
|
|
808
|
+
409,
|
|
809
|
+
"Workspace checkpoint range starts beyond EOF",
|
|
810
|
+
);
|
|
811
|
+
}
|
|
812
|
+
const length = Math.min(range.length, sizeBytes - range.offset);
|
|
813
|
+
const bytes = readCheckpointRange(
|
|
814
|
+
this.ctx.storage,
|
|
815
|
+
blobId,
|
|
816
|
+
range.offset,
|
|
817
|
+
length,
|
|
818
|
+
);
|
|
819
|
+
return {
|
|
820
|
+
checkpointRevision: range.checkpointRevision,
|
|
821
|
+
dataBase64: encodeBase64(bytes),
|
|
822
|
+
eof: range.offset + bytes.byteLength === sizeBytes,
|
|
823
|
+
kind: range.kind,
|
|
824
|
+
offset: range.offset,
|
|
825
|
+
sha256,
|
|
826
|
+
sizeBytes,
|
|
827
|
+
};
|
|
828
|
+
}
|
|
829
|
+
|
|
830
|
+
async #initializeMount(invocation) {
|
|
831
|
+
const mount = parseMount(invocation.input);
|
|
832
|
+
const requestHash = await mutationHash(invocation);
|
|
833
|
+
const prior = this.ctx.storage.sql
|
|
834
|
+
.exec(
|
|
835
|
+
`SELECT request_hash, result_json
|
|
836
|
+
FROM filesystem_mutations
|
|
837
|
+
WHERE mutation_id = ?`,
|
|
838
|
+
invocation.mutationId,
|
|
839
|
+
)
|
|
840
|
+
.toArray();
|
|
841
|
+
if (prior.length !== 0) return replayMutation(prior[0], requestHash);
|
|
842
|
+
return this.ctx.storage.transactionSync((storage) => {
|
|
843
|
+
const existingMutation = storage.sql
|
|
844
|
+
.exec(
|
|
845
|
+
`SELECT request_hash, result_json
|
|
846
|
+
FROM filesystem_mutations
|
|
847
|
+
WHERE mutation_id = ?`,
|
|
848
|
+
invocation.mutationId,
|
|
849
|
+
)
|
|
850
|
+
.toArray();
|
|
851
|
+
if (existingMutation.length !== 0) {
|
|
852
|
+
return replayMutation(existingMutation[0], requestHash);
|
|
853
|
+
}
|
|
854
|
+
const existing = storage.sql
|
|
855
|
+
.exec(
|
|
856
|
+
"SELECT content_id FROM filesystem_mounts WHERE mount = ?",
|
|
857
|
+
mount.mount,
|
|
858
|
+
)
|
|
859
|
+
.toArray();
|
|
860
|
+
const reused = existing[0]?.content_id === mount.contentId;
|
|
861
|
+
if (!reused) {
|
|
862
|
+
if (existing.length === 0) {
|
|
863
|
+
const count = storage.sql
|
|
864
|
+
.exec("SELECT COUNT(*) AS count FROM filesystem_mounts")
|
|
865
|
+
.one();
|
|
866
|
+
if (count.count >= MAX_MOUNTS) {
|
|
867
|
+
throw new FilesystemRequestError(
|
|
868
|
+
409,
|
|
869
|
+
`Filesystem has reached the ${MAX_MOUNTS}-mount limit`,
|
|
870
|
+
);
|
|
871
|
+
}
|
|
872
|
+
}
|
|
873
|
+
storage.sql.exec(
|
|
874
|
+
"DELETE FROM filesystem_mount_file_chunks WHERE mount = ?",
|
|
875
|
+
mount.mount,
|
|
876
|
+
);
|
|
877
|
+
storage.sql.exec(
|
|
878
|
+
"DELETE FROM filesystem_mount_files WHERE mount = ?",
|
|
879
|
+
mount.mount,
|
|
880
|
+
);
|
|
881
|
+
storage.sql.exec(
|
|
882
|
+
"DELETE FROM filesystem_mounts WHERE mount = ?",
|
|
883
|
+
mount.mount,
|
|
884
|
+
);
|
|
885
|
+
storage.sql.exec(
|
|
886
|
+
`INSERT INTO filesystem_mounts
|
|
887
|
+
(mount, content_id, mode, file_count, size_bytes, created_at_ms)
|
|
888
|
+
VALUES (?, ?, 'read-only', ?, ?, ?)`,
|
|
889
|
+
mount.mount,
|
|
890
|
+
mount.contentId,
|
|
891
|
+
mount.files.length,
|
|
892
|
+
mount.sizeBytes,
|
|
893
|
+
Date.now(),
|
|
894
|
+
);
|
|
895
|
+
for (const file of mount.files) {
|
|
896
|
+
storage.sql.exec(
|
|
897
|
+
`INSERT INTO filesystem_mount_files
|
|
898
|
+
(mount, path, size_bytes)
|
|
899
|
+
VALUES (?, ?, ?)`,
|
|
900
|
+
mount.mount,
|
|
901
|
+
file.path,
|
|
902
|
+
file.bytes.byteLength,
|
|
903
|
+
);
|
|
904
|
+
for (let offset = 0; offset < file.bytes.byteLength; ) {
|
|
905
|
+
const chunk = file.bytes.slice(offset, offset + MAX_SQL_BLOB_BYTES);
|
|
906
|
+
storage.sql.exec(
|
|
907
|
+
`INSERT INTO filesystem_mount_file_chunks
|
|
908
|
+
(mount, path, offset, content)
|
|
909
|
+
VALUES (?, ?, ?, ?)`,
|
|
910
|
+
mount.mount,
|
|
911
|
+
file.path,
|
|
912
|
+
offset,
|
|
913
|
+
chunk,
|
|
914
|
+
);
|
|
915
|
+
offset += chunk.byteLength;
|
|
916
|
+
}
|
|
917
|
+
}
|
|
918
|
+
}
|
|
919
|
+
const result = {
|
|
920
|
+
contentId: mount.contentId,
|
|
921
|
+
mode: "read-only",
|
|
922
|
+
mount: mount.mount,
|
|
923
|
+
reused,
|
|
924
|
+
};
|
|
925
|
+
storage.sql.exec(
|
|
926
|
+
`INSERT INTO filesystem_mutations
|
|
927
|
+
(mutation_id, request_hash, applied_revision, result_json)
|
|
928
|
+
VALUES (?, ?, ?, ?)`,
|
|
929
|
+
invocation.mutationId,
|
|
930
|
+
requestHash,
|
|
931
|
+
readMeta(storage).revision,
|
|
932
|
+
JSON.stringify(result),
|
|
933
|
+
);
|
|
934
|
+
return result;
|
|
935
|
+
});
|
|
936
|
+
}
|
|
937
|
+
|
|
938
|
+
#listMounts() {
|
|
939
|
+
const mounts = this.ctx.storage.sql
|
|
940
|
+
.exec(
|
|
941
|
+
`SELECT mount, content_id, mode, file_count, size_bytes
|
|
942
|
+
FROM filesystem_mounts
|
|
943
|
+
ORDER BY mount`,
|
|
944
|
+
)
|
|
945
|
+
.toArray()
|
|
946
|
+
.map((mount) => ({
|
|
947
|
+
contentId: mount.content_id,
|
|
948
|
+
fileCount: mount.file_count,
|
|
949
|
+
mode: mount.mode,
|
|
950
|
+
mount: mount.mount,
|
|
951
|
+
sizeBytes: mount.size_bytes,
|
|
952
|
+
}));
|
|
953
|
+
return { mounts };
|
|
954
|
+
}
|
|
955
|
+
|
|
956
|
+
#getMount(input) {
|
|
957
|
+
const mount = parseMountName(input?.mount);
|
|
958
|
+
const rows = this.ctx.storage.sql
|
|
959
|
+
.exec(
|
|
960
|
+
`SELECT content_id, mode
|
|
961
|
+
FROM filesystem_mounts
|
|
962
|
+
WHERE mount = ?`,
|
|
963
|
+
mount,
|
|
964
|
+
)
|
|
965
|
+
.toArray();
|
|
966
|
+
if (rows.length === 0) {
|
|
967
|
+
throw new FilesystemRequestError(404, `Mount does not exist: ${mount}`);
|
|
968
|
+
}
|
|
969
|
+
const files = this.ctx.storage.sql
|
|
970
|
+
.exec(
|
|
971
|
+
`SELECT path, size_bytes
|
|
972
|
+
FROM filesystem_mount_files
|
|
973
|
+
WHERE mount = ?
|
|
974
|
+
ORDER BY path`,
|
|
975
|
+
mount,
|
|
976
|
+
)
|
|
977
|
+
.toArray()
|
|
978
|
+
.map((file) => ({
|
|
979
|
+
dataBase64: encodeBase64(
|
|
980
|
+
readMountFile(this.ctx.storage, mount, file.path, file.size_bytes),
|
|
981
|
+
),
|
|
982
|
+
path: file.path,
|
|
983
|
+
}));
|
|
984
|
+
return {
|
|
985
|
+
contentId: rows[0].content_id,
|
|
986
|
+
files,
|
|
987
|
+
mode: rows[0].mode,
|
|
988
|
+
mount,
|
|
989
|
+
};
|
|
990
|
+
}
|
|
991
|
+
|
|
992
|
+
#listEntries(rawPath) {
|
|
993
|
+
const path = parsePath(rawPath, "path");
|
|
994
|
+
requireDirectory(this.ctx.storage, path);
|
|
995
|
+
const entries = this.ctx.storage.sql
|
|
996
|
+
.exec(
|
|
997
|
+
`SELECT path, name, kind, size_bytes, sha256, modified_at_ms, revision
|
|
998
|
+
FROM filesystem_entries
|
|
999
|
+
WHERE parent_path = ?
|
|
1000
|
+
ORDER BY name ASC`,
|
|
1001
|
+
path,
|
|
1002
|
+
)
|
|
1003
|
+
.toArray()
|
|
1004
|
+
.map((entry) => ({
|
|
1005
|
+
path: entry.path,
|
|
1006
|
+
name: entry.name,
|
|
1007
|
+
kind: entry.kind,
|
|
1008
|
+
sizeBytes: entry.size_bytes,
|
|
1009
|
+
sha256: entry.sha256,
|
|
1010
|
+
modifiedAtMs: entry.modified_at_ms,
|
|
1011
|
+
revision: entry.revision,
|
|
1012
|
+
}));
|
|
1013
|
+
return { revision: readMeta(this.ctx.storage).revision, entries };
|
|
1014
|
+
}
|
|
1015
|
+
|
|
1016
|
+
#readFile(rawPath) {
|
|
1017
|
+
const path = parsePath(rawPath, "path");
|
|
1018
|
+
const rows = this.ctx.storage.sql
|
|
1019
|
+
.exec(
|
|
1020
|
+
`SELECT content, sha256, size_bytes, modified_at_ms, revision
|
|
1021
|
+
FROM filesystem_entries
|
|
1022
|
+
WHERE path = ? AND kind = 'file'`,
|
|
1023
|
+
path,
|
|
1024
|
+
)
|
|
1025
|
+
.toArray();
|
|
1026
|
+
if (rows.length === 0) {
|
|
1027
|
+
throw new FilesystemRequestError(404, `File does not exist: ${path}`);
|
|
1028
|
+
}
|
|
1029
|
+
const file = rows[0];
|
|
1030
|
+
return {
|
|
1031
|
+
path,
|
|
1032
|
+
contentBase64: encodeBase64(new Uint8Array(file.content)),
|
|
1033
|
+
sha256: file.sha256,
|
|
1034
|
+
sizeBytes: file.size_bytes,
|
|
1035
|
+
modifiedAtMs: file.modified_at_ms,
|
|
1036
|
+
revision: file.revision,
|
|
1037
|
+
};
|
|
1038
|
+
}
|
|
1039
|
+
}
|
|
1040
|
+
|
|
1041
|
+
export default {
|
|
1042
|
+
async fetch(request, env) {
|
|
1043
|
+
try {
|
|
1044
|
+
const url = new URL(request.url);
|
|
1045
|
+
const match =
|
|
1046
|
+
/^\/v1\/objects\/([^/]+)(\/(?:batches|entries|files|invoke))?$/.exec(
|
|
1047
|
+
url.pathname,
|
|
1048
|
+
);
|
|
1049
|
+
if (!match) return jsonError(404, "Filesystem route was not found");
|
|
1050
|
+
|
|
1051
|
+
const allocationId = parseAllocationId(match[1]);
|
|
1052
|
+
const objectPath = match[2] ?? "/";
|
|
1053
|
+
const body = ["GET", "HEAD"].includes(request.method)
|
|
1054
|
+
? undefined
|
|
1055
|
+
: await request.arrayBuffer();
|
|
1056
|
+
if (objectPath === "/invoke") {
|
|
1057
|
+
let invocation;
|
|
1058
|
+
try {
|
|
1059
|
+
invocation = JSON.parse(new TextDecoder().decode(body));
|
|
1060
|
+
} catch {
|
|
1061
|
+
throw new FilesystemRequestError(
|
|
1062
|
+
400,
|
|
1063
|
+
"Request body must be valid JSON",
|
|
1064
|
+
);
|
|
1065
|
+
}
|
|
1066
|
+
if (invocation?.objectId !== allocationId) {
|
|
1067
|
+
throw new FilesystemRequestError(
|
|
1068
|
+
409,
|
|
1069
|
+
"Invocation objectId does not match the routed allocation",
|
|
1070
|
+
);
|
|
1071
|
+
}
|
|
1072
|
+
}
|
|
1073
|
+
const objectUrl = new URL(
|
|
1074
|
+
objectPath,
|
|
1075
|
+
"https://filesystem.object.internal",
|
|
1076
|
+
);
|
|
1077
|
+
objectUrl.search = url.search;
|
|
1078
|
+
return env.FILESYSTEM.getByName(allocationId).fetch(objectUrl, {
|
|
1079
|
+
method: request.method,
|
|
1080
|
+
headers: request.headers,
|
|
1081
|
+
body,
|
|
1082
|
+
signal: request.signal,
|
|
1083
|
+
});
|
|
1084
|
+
} catch (error) {
|
|
1085
|
+
if (error instanceof FilesystemRequestError) {
|
|
1086
|
+
return jsonError(
|
|
1087
|
+
error.status,
|
|
1088
|
+
error.message,
|
|
1089
|
+
error.code,
|
|
1090
|
+
error.retryable,
|
|
1091
|
+
);
|
|
1092
|
+
}
|
|
1093
|
+
console.error("Filesystem System Worker request failed", error);
|
|
1094
|
+
return jsonError(500, "Filesystem request failed");
|
|
1095
|
+
}
|
|
1096
|
+
},
|
|
1097
|
+
};
|
|
1098
|
+
|
|
1099
|
+
function parseInvocation(input) {
|
|
1100
|
+
if (input === null || typeof input !== "object" || Array.isArray(input)) {
|
|
1101
|
+
throw new FilesystemRequestError(400, "Invocation body must be an object");
|
|
1102
|
+
}
|
|
1103
|
+
if (input.protocolVersion !== PROTOCOL_VERSION) {
|
|
1104
|
+
throw new FilesystemRequestError(
|
|
1105
|
+
400,
|
|
1106
|
+
`Unsupported filesystem protocol version: ${String(input.protocolVersion)}`,
|
|
1107
|
+
);
|
|
1108
|
+
}
|
|
1109
|
+
if (input.class !== "filesystem") {
|
|
1110
|
+
throw new FilesystemRequestError(
|
|
1111
|
+
400,
|
|
1112
|
+
"Invocation class must be filesystem",
|
|
1113
|
+
);
|
|
1114
|
+
}
|
|
1115
|
+
const objectId = requireBoundedString(input.objectId, "objectId", 512);
|
|
1116
|
+
if (!UUID_PATTERN.test(objectId)) {
|
|
1117
|
+
throw new FilesystemRequestError(400, "objectId must be a UUID");
|
|
1118
|
+
}
|
|
1119
|
+
const operation = requireBoundedString(input.operation, "operation", 128);
|
|
1120
|
+
requireBoundedString(input.requestId, "requestId", 256);
|
|
1121
|
+
if (
|
|
1122
|
+
!Number.isSafeInteger(input.deadlineMs) ||
|
|
1123
|
+
input.deadlineMs <= 0 ||
|
|
1124
|
+
input.deadlineMs > 15 * 60 * 1000
|
|
1125
|
+
) {
|
|
1126
|
+
throw new FilesystemRequestError(
|
|
1127
|
+
400,
|
|
1128
|
+
"deadlineMs must be a positive safe integer no greater than 900000",
|
|
1129
|
+
);
|
|
1130
|
+
}
|
|
1131
|
+
if (
|
|
1132
|
+
input.input === null ||
|
|
1133
|
+
typeof input.input !== "object" ||
|
|
1134
|
+
Array.isArray(input.input)
|
|
1135
|
+
) {
|
|
1136
|
+
throw new FilesystemRequestError(400, "Invocation input must be an object");
|
|
1137
|
+
}
|
|
1138
|
+
const mutationId =
|
|
1139
|
+
input.mutationId === null || input.mutationId === undefined
|
|
1140
|
+
? null
|
|
1141
|
+
: requireBoundedString(
|
|
1142
|
+
input.mutationId,
|
|
1143
|
+
"mutationId",
|
|
1144
|
+
MAX_MUTATION_ID_BYTES,
|
|
1145
|
+
);
|
|
1146
|
+
if (
|
|
1147
|
+
["initializeMount", "commitCheckpoint"].includes(operation) &&
|
|
1148
|
+
!mutationId
|
|
1149
|
+
) {
|
|
1150
|
+
throw new FilesystemRequestError(400, `${operation} requires a mutationId`);
|
|
1151
|
+
}
|
|
1152
|
+
return {
|
|
1153
|
+
deadlineMs: input.deadlineMs,
|
|
1154
|
+
input: input.input,
|
|
1155
|
+
mutationId,
|
|
1156
|
+
objectId,
|
|
1157
|
+
operation,
|
|
1158
|
+
protocolVersion: input.protocolVersion,
|
|
1159
|
+
requestId: input.requestId,
|
|
1160
|
+
};
|
|
1161
|
+
}
|
|
1162
|
+
|
|
1163
|
+
function parseCheckpointUpload(input) {
|
|
1164
|
+
const kind = parseCheckpointKind(input?.kind);
|
|
1165
|
+
const sizeBytes = positiveSafeInteger(input?.sizeBytes, "sizeBytes");
|
|
1166
|
+
const maximum =
|
|
1167
|
+
kind === "image"
|
|
1168
|
+
? MAX_CHECKPOINT_IMAGE_BYTES
|
|
1169
|
+
: MAX_CHECKPOINT_MANIFEST_BYTES;
|
|
1170
|
+
if (sizeBytes > maximum) {
|
|
1171
|
+
throw new FilesystemRequestError(
|
|
1172
|
+
413,
|
|
1173
|
+
`${kind} checkpoint exceeds ${maximum} bytes`,
|
|
1174
|
+
);
|
|
1175
|
+
}
|
|
1176
|
+
return {
|
|
1177
|
+
kind,
|
|
1178
|
+
sha256: parseSha256(input?.sha256, "sha256"),
|
|
1179
|
+
sizeBytes,
|
|
1180
|
+
uploadId: requireBoundedString(input?.uploadId, "uploadId", 256),
|
|
1181
|
+
};
|
|
1182
|
+
}
|
|
1183
|
+
|
|
1184
|
+
function parseCheckpointAppend(input) {
|
|
1185
|
+
const bytes = decodeBase64(input?.dataBase64);
|
|
1186
|
+
if (
|
|
1187
|
+
bytes.byteLength === 0 ||
|
|
1188
|
+
bytes.byteLength > MAX_CHECKPOINT_TRANSFER_CHUNK_BYTES
|
|
1189
|
+
) {
|
|
1190
|
+
throw new FilesystemRequestError(
|
|
1191
|
+
413,
|
|
1192
|
+
`Checkpoint chunk must contain between 1 and ${MAX_CHECKPOINT_TRANSFER_CHUNK_BYTES} bytes`,
|
|
1193
|
+
);
|
|
1194
|
+
}
|
|
1195
|
+
return {
|
|
1196
|
+
bytes,
|
|
1197
|
+
offset: nonNegativeSafeInteger(input?.offset, "offset"),
|
|
1198
|
+
uploadId: requireBoundedString(input?.uploadId, "uploadId", 256),
|
|
1199
|
+
};
|
|
1200
|
+
}
|
|
1201
|
+
|
|
1202
|
+
function parseCheckpointCommit(invocation) {
|
|
1203
|
+
const input = invocation.input;
|
|
1204
|
+
if (
|
|
1205
|
+
input.checkpoint === null ||
|
|
1206
|
+
typeof input.checkpoint !== "object" ||
|
|
1207
|
+
Array.isArray(input.checkpoint)
|
|
1208
|
+
) {
|
|
1209
|
+
throw new FilesystemRequestError(400, "checkpoint must be an object");
|
|
1210
|
+
}
|
|
1211
|
+
const blob = (kind) => {
|
|
1212
|
+
const value = input.checkpoint[kind];
|
|
1213
|
+
if (value === null || typeof value !== "object" || Array.isArray(value)) {
|
|
1214
|
+
throw new FilesystemRequestError(400, `${kind} must be an object`);
|
|
1215
|
+
}
|
|
1216
|
+
if (value.directUpload !== null && value.directUpload !== undefined) {
|
|
1217
|
+
throw new FilesystemRequestError(
|
|
1218
|
+
501,
|
|
1219
|
+
"Direct checkpoint transfer is not supported by the Filesystem System Worker",
|
|
1220
|
+
"UNSUPPORTED",
|
|
1221
|
+
);
|
|
1222
|
+
}
|
|
1223
|
+
const sizeBytes = positiveSafeInteger(value.sizeBytes, `${kind}.sizeBytes`);
|
|
1224
|
+
const maximum =
|
|
1225
|
+
kind === "image"
|
|
1226
|
+
? MAX_CHECKPOINT_IMAGE_BYTES
|
|
1227
|
+
: MAX_CHECKPOINT_MANIFEST_BYTES;
|
|
1228
|
+
if (sizeBytes > maximum) {
|
|
1229
|
+
throw new FilesystemRequestError(
|
|
1230
|
+
413,
|
|
1231
|
+
`${kind} checkpoint exceeds ${maximum} bytes`,
|
|
1232
|
+
);
|
|
1233
|
+
}
|
|
1234
|
+
return {
|
|
1235
|
+
sha256: parseSha256(value.sha256, `${kind}.sha256`),
|
|
1236
|
+
sizeBytes,
|
|
1237
|
+
uploadId: requireBoundedString(value.uploadId, `${kind}.uploadId`, 256),
|
|
1238
|
+
};
|
|
1239
|
+
};
|
|
1240
|
+
return {
|
|
1241
|
+
checkpointId: requireBoundedString(
|
|
1242
|
+
input.checkpoint.checkpointId,
|
|
1243
|
+
"checkpointId",
|
|
1244
|
+
256,
|
|
1245
|
+
),
|
|
1246
|
+
expectedCheckpointRevision: nonNegativeSafeInteger(
|
|
1247
|
+
input.expectedCheckpointRevision,
|
|
1248
|
+
"expectedCheckpointRevision",
|
|
1249
|
+
),
|
|
1250
|
+
image: blob("image"),
|
|
1251
|
+
manifest: blob("manifest"),
|
|
1252
|
+
};
|
|
1253
|
+
}
|
|
1254
|
+
|
|
1255
|
+
function parseCheckpointRange(input) {
|
|
1256
|
+
const length = positiveSafeInteger(input?.length, "length");
|
|
1257
|
+
if (length > MAX_CHECKPOINT_TRANSFER_CHUNK_BYTES) {
|
|
1258
|
+
throw new FilesystemRequestError(
|
|
1259
|
+
413,
|
|
1260
|
+
`Checkpoint range exceeds ${MAX_CHECKPOINT_TRANSFER_CHUNK_BYTES} bytes`,
|
|
1261
|
+
);
|
|
1262
|
+
}
|
|
1263
|
+
return {
|
|
1264
|
+
checkpointRevision: positiveSafeInteger(
|
|
1265
|
+
input?.checkpointRevision,
|
|
1266
|
+
"checkpointRevision",
|
|
1267
|
+
),
|
|
1268
|
+
kind: parseCheckpointKind(input?.kind),
|
|
1269
|
+
length,
|
|
1270
|
+
offset: nonNegativeSafeInteger(input?.offset, "offset"),
|
|
1271
|
+
};
|
|
1272
|
+
}
|
|
1273
|
+
|
|
1274
|
+
function parseCheckpointKind(value) {
|
|
1275
|
+
if (value !== "image" && value !== "manifest") {
|
|
1276
|
+
throw new FilesystemRequestError(
|
|
1277
|
+
400,
|
|
1278
|
+
"Checkpoint kind must be image or manifest",
|
|
1279
|
+
);
|
|
1280
|
+
}
|
|
1281
|
+
return value;
|
|
1282
|
+
}
|
|
1283
|
+
|
|
1284
|
+
function parseMount(input) {
|
|
1285
|
+
const mount = parseMountName(input?.mount);
|
|
1286
|
+
if (input?.mode !== "read-only") {
|
|
1287
|
+
throw new FilesystemRequestError(400, "Mount mode must be read-only");
|
|
1288
|
+
}
|
|
1289
|
+
const contentId = parseSha256(input?.contentId, "contentId");
|
|
1290
|
+
if (
|
|
1291
|
+
!Array.isArray(input?.files) ||
|
|
1292
|
+
input.files.length === 0 ||
|
|
1293
|
+
input.files.length > MAX_MOUNT_FILES
|
|
1294
|
+
) {
|
|
1295
|
+
throw new FilesystemRequestError(
|
|
1296
|
+
400,
|
|
1297
|
+
`Mount must contain between 1 and ${MAX_MOUNT_FILES} files`,
|
|
1298
|
+
);
|
|
1299
|
+
}
|
|
1300
|
+
let sizeBytes = 0;
|
|
1301
|
+
const files = input.files
|
|
1302
|
+
.map((file) => {
|
|
1303
|
+
if (file === null || typeof file !== "object" || Array.isArray(file)) {
|
|
1304
|
+
throw new FilesystemRequestError(400, "Mount file must be an object");
|
|
1305
|
+
}
|
|
1306
|
+
const path = parseRelativePath(file.path, "mount file path");
|
|
1307
|
+
const bytes = decodeBase64(file.dataBase64);
|
|
1308
|
+
sizeBytes +=
|
|
1309
|
+
16 + new TextEncoder().encode(path).byteLength + bytes.byteLength;
|
|
1310
|
+
if (sizeBytes > MAX_MOUNT_BYTES) {
|
|
1311
|
+
throw new FilesystemRequestError(
|
|
1312
|
+
413,
|
|
1313
|
+
`Mount content exceeds ${MAX_MOUNT_BYTES} bytes`,
|
|
1314
|
+
);
|
|
1315
|
+
}
|
|
1316
|
+
return { bytes, path };
|
|
1317
|
+
})
|
|
1318
|
+
.sort((left, right) =>
|
|
1319
|
+
compareBytes(
|
|
1320
|
+
new TextEncoder().encode(left.path),
|
|
1321
|
+
new TextEncoder().encode(right.path),
|
|
1322
|
+
),
|
|
1323
|
+
);
|
|
1324
|
+
if (new Set(files.map((file) => file.path)).size !== files.length) {
|
|
1325
|
+
throw new FilesystemRequestError(409, "Mount file paths must be unique");
|
|
1326
|
+
}
|
|
1327
|
+
const digest = new Sha256State();
|
|
1328
|
+
for (const file of files) {
|
|
1329
|
+
const pathBytes = new TextEncoder().encode(file.path);
|
|
1330
|
+
digest.update(unsigned64Bytes(pathBytes.byteLength));
|
|
1331
|
+
digest.update(pathBytes);
|
|
1332
|
+
digest.update(unsigned64Bytes(file.bytes.byteLength));
|
|
1333
|
+
digest.update(file.bytes);
|
|
1334
|
+
}
|
|
1335
|
+
if (`sha256:${digest.digestHex()}` !== contentId) {
|
|
1336
|
+
throw new FilesystemRequestError(
|
|
1337
|
+
409,
|
|
1338
|
+
"Mount contentId does not match the supplied files",
|
|
1339
|
+
);
|
|
1340
|
+
}
|
|
1341
|
+
return { contentId, files, mount, sizeBytes };
|
|
1342
|
+
}
|
|
1343
|
+
|
|
1344
|
+
function parseMountName(value) {
|
|
1345
|
+
const mount = requireBoundedString(value, "mount", 256);
|
|
1346
|
+
if (
|
|
1347
|
+
!/^\/[A-Za-z0-9._-]+$/.test(mount) ||
|
|
1348
|
+
["/workspace", "/.gea", "/tmp"].includes(mount)
|
|
1349
|
+
) {
|
|
1350
|
+
throw new FilesystemRequestError(
|
|
1351
|
+
400,
|
|
1352
|
+
"Mount must be one non-reserved absolute path segment",
|
|
1353
|
+
);
|
|
1354
|
+
}
|
|
1355
|
+
return mount;
|
|
1356
|
+
}
|
|
1357
|
+
|
|
1358
|
+
function parseRelativePath(value, field) {
|
|
1359
|
+
const path = requireBoundedString(value, field, 4096);
|
|
1360
|
+
if (path.startsWith("/") || path.includes("\\") || path.includes("\0")) {
|
|
1361
|
+
throw new FilesystemRequestError(
|
|
1362
|
+
400,
|
|
1363
|
+
`${field} must be a relative POSIX path`,
|
|
1364
|
+
);
|
|
1365
|
+
}
|
|
1366
|
+
if (
|
|
1367
|
+
path
|
|
1368
|
+
.split("/")
|
|
1369
|
+
.some((segment) => segment === "" || segment === "." || segment === "..")
|
|
1370
|
+
) {
|
|
1371
|
+
throw new FilesystemRequestError(400, `${field} must be canonical`);
|
|
1372
|
+
}
|
|
1373
|
+
return path;
|
|
1374
|
+
}
|
|
1375
|
+
|
|
1376
|
+
function parseSha256(value, field) {
|
|
1377
|
+
if (typeof value !== "string" || !/^sha256:[0-9a-f]{64}$/.test(value)) {
|
|
1378
|
+
throw new FilesystemRequestError(
|
|
1379
|
+
400,
|
|
1380
|
+
`${field} must be a lowercase sha256 digest`,
|
|
1381
|
+
);
|
|
1382
|
+
}
|
|
1383
|
+
return value;
|
|
1384
|
+
}
|
|
1385
|
+
|
|
1386
|
+
function positiveSafeInteger(value, field) {
|
|
1387
|
+
if (!Number.isSafeInteger(value) || value <= 0) {
|
|
1388
|
+
throw new FilesystemRequestError(
|
|
1389
|
+
400,
|
|
1390
|
+
`${field} must be a positive safe integer`,
|
|
1391
|
+
);
|
|
1392
|
+
}
|
|
1393
|
+
return value;
|
|
1394
|
+
}
|
|
1395
|
+
|
|
1396
|
+
function nonNegativeSafeInteger(value, field) {
|
|
1397
|
+
if (!Number.isSafeInteger(value) || value < 0) {
|
|
1398
|
+
throw new FilesystemRequestError(
|
|
1399
|
+
400,
|
|
1400
|
+
`${field} must be a non-negative safe integer`,
|
|
1401
|
+
);
|
|
1402
|
+
}
|
|
1403
|
+
return value;
|
|
1404
|
+
}
|
|
1405
|
+
|
|
1406
|
+
function optionalNonNegativeInteger(value, field) {
|
|
1407
|
+
return value === undefined || value === null
|
|
1408
|
+
? null
|
|
1409
|
+
: nonNegativeSafeInteger(value, field);
|
|
1410
|
+
}
|
|
1411
|
+
|
|
1412
|
+
function readCheckpointRevision(storage) {
|
|
1413
|
+
return storage.sql
|
|
1414
|
+
.exec(
|
|
1415
|
+
`SELECT checkpoint_revision
|
|
1416
|
+
FROM filesystem_checkpoint_meta
|
|
1417
|
+
WHERE singleton = 1`,
|
|
1418
|
+
)
|
|
1419
|
+
.one().checkpoint_revision;
|
|
1420
|
+
}
|
|
1421
|
+
|
|
1422
|
+
function readCheckpoint(storage, revision) {
|
|
1423
|
+
return checkpointDescriptor(readCheckpointRow(storage, revision));
|
|
1424
|
+
}
|
|
1425
|
+
|
|
1426
|
+
function readCheckpointRow(storage, revision) {
|
|
1427
|
+
const rows = storage.sql
|
|
1428
|
+
.exec(
|
|
1429
|
+
`SELECT checkpoint_revision, parent_checkpoint_revision, checkpoint_id,
|
|
1430
|
+
image_blob_id, image_sha256, image_size_bytes,
|
|
1431
|
+
manifest_blob_id, manifest_sha256, manifest_size_bytes,
|
|
1432
|
+
created_at_ms
|
|
1433
|
+
FROM filesystem_checkpoints
|
|
1434
|
+
WHERE checkpoint_revision = ?`,
|
|
1435
|
+
revision,
|
|
1436
|
+
)
|
|
1437
|
+
.toArray();
|
|
1438
|
+
if (rows.length === 0) {
|
|
1439
|
+
throw new FilesystemRequestError(
|
|
1440
|
+
404,
|
|
1441
|
+
`Workspace checkpoint revision was not found: ${revision}`,
|
|
1442
|
+
);
|
|
1443
|
+
}
|
|
1444
|
+
return rows[0];
|
|
1445
|
+
}
|
|
1446
|
+
|
|
1447
|
+
function checkpointDescriptor(row) {
|
|
1448
|
+
return {
|
|
1449
|
+
checkpointId: row.checkpoint_id,
|
|
1450
|
+
checkpointRevision: row.checkpoint_revision,
|
|
1451
|
+
createdAtMs: row.created_at_ms,
|
|
1452
|
+
image: {
|
|
1453
|
+
sha256: row.image_sha256,
|
|
1454
|
+
sizeBytes: row.image_size_bytes,
|
|
1455
|
+
},
|
|
1456
|
+
manifest: {
|
|
1457
|
+
sha256: row.manifest_sha256,
|
|
1458
|
+
sizeBytes: row.manifest_size_bytes,
|
|
1459
|
+
},
|
|
1460
|
+
parentCheckpointRevision: row.parent_checkpoint_revision,
|
|
1461
|
+
};
|
|
1462
|
+
}
|
|
1463
|
+
|
|
1464
|
+
function requireCompleteUpload(storage, requested, kind) {
|
|
1465
|
+
const rows = storage.sql
|
|
1466
|
+
.exec(
|
|
1467
|
+
`SELECT upload_id, kind, sha256, size_bytes, received_bytes,
|
|
1468
|
+
sha256_state_json, committed
|
|
1469
|
+
FROM filesystem_checkpoint_uploads
|
|
1470
|
+
WHERE upload_id = ?`,
|
|
1471
|
+
requested.uploadId,
|
|
1472
|
+
)
|
|
1473
|
+
.toArray();
|
|
1474
|
+
if (rows.length === 0) {
|
|
1475
|
+
throw new FilesystemRequestError(
|
|
1476
|
+
409,
|
|
1477
|
+
`${kind} checkpoint upload was not found`,
|
|
1478
|
+
);
|
|
1479
|
+
}
|
|
1480
|
+
const upload = rows[0];
|
|
1481
|
+
if (
|
|
1482
|
+
upload.kind !== kind ||
|
|
1483
|
+
upload.sha256 !== requested.sha256 ||
|
|
1484
|
+
upload.size_bytes !== requested.sizeBytes ||
|
|
1485
|
+
upload.received_bytes !== upload.size_bytes ||
|
|
1486
|
+
upload.committed !== 0
|
|
1487
|
+
) {
|
|
1488
|
+
throw new FilesystemRequestError(
|
|
1489
|
+
409,
|
|
1490
|
+
`${kind} checkpoint upload metadata is incomplete or inconsistent`,
|
|
1491
|
+
);
|
|
1492
|
+
}
|
|
1493
|
+
const digest = Sha256State.fromJSON(
|
|
1494
|
+
JSON.parse(upload.sha256_state_json),
|
|
1495
|
+
).digestHex();
|
|
1496
|
+
if (`sha256:${digest}` !== upload.sha256) {
|
|
1497
|
+
throw new FilesystemRequestError(
|
|
1498
|
+
409,
|
|
1499
|
+
`${kind} checkpoint upload digest does not match its declaration`,
|
|
1500
|
+
);
|
|
1501
|
+
}
|
|
1502
|
+
return upload;
|
|
1503
|
+
}
|
|
1504
|
+
|
|
1505
|
+
function readUploadRange(storage, uploadId, offset, length) {
|
|
1506
|
+
return readChunkRange(
|
|
1507
|
+
storage.sql
|
|
1508
|
+
.exec(
|
|
1509
|
+
`SELECT offset, content
|
|
1510
|
+
FROM filesystem_checkpoint_blob_chunks
|
|
1511
|
+
WHERE blob_id = ?
|
|
1512
|
+
AND offset < ?
|
|
1513
|
+
AND offset + length(content) > ?
|
|
1514
|
+
ORDER BY offset`,
|
|
1515
|
+
uploadId,
|
|
1516
|
+
offset + length,
|
|
1517
|
+
offset,
|
|
1518
|
+
)
|
|
1519
|
+
.toArray(),
|
|
1520
|
+
offset,
|
|
1521
|
+
length,
|
|
1522
|
+
);
|
|
1523
|
+
}
|
|
1524
|
+
|
|
1525
|
+
function readCheckpointRange(storage, blobId, offset, length) {
|
|
1526
|
+
return readChunkRange(
|
|
1527
|
+
storage.sql
|
|
1528
|
+
.exec(
|
|
1529
|
+
`SELECT offset, content
|
|
1530
|
+
FROM filesystem_checkpoint_blob_chunks
|
|
1531
|
+
WHERE blob_id = ?
|
|
1532
|
+
AND offset < ?
|
|
1533
|
+
AND offset + length(content) > ?
|
|
1534
|
+
ORDER BY offset`,
|
|
1535
|
+
blobId,
|
|
1536
|
+
offset + length,
|
|
1537
|
+
offset,
|
|
1538
|
+
)
|
|
1539
|
+
.toArray(),
|
|
1540
|
+
offset,
|
|
1541
|
+
length,
|
|
1542
|
+
);
|
|
1543
|
+
}
|
|
1544
|
+
|
|
1545
|
+
function readMountFile(storage, mount, path, sizeBytes) {
|
|
1546
|
+
return readChunkRange(
|
|
1547
|
+
storage.sql
|
|
1548
|
+
.exec(
|
|
1549
|
+
`SELECT offset, content
|
|
1550
|
+
FROM filesystem_mount_file_chunks
|
|
1551
|
+
WHERE mount = ? AND path = ?
|
|
1552
|
+
ORDER BY offset`,
|
|
1553
|
+
mount,
|
|
1554
|
+
path,
|
|
1555
|
+
)
|
|
1556
|
+
.toArray(),
|
|
1557
|
+
0,
|
|
1558
|
+
sizeBytes,
|
|
1559
|
+
);
|
|
1560
|
+
}
|
|
1561
|
+
|
|
1562
|
+
function readChunkRange(rows, offset, length) {
|
|
1563
|
+
const bytes = new Uint8Array(length);
|
|
1564
|
+
let copied = 0;
|
|
1565
|
+
const end = offset + length;
|
|
1566
|
+
for (const row of rows) {
|
|
1567
|
+
const content = new Uint8Array(row.content);
|
|
1568
|
+
const rowEnd = row.offset + content.byteLength;
|
|
1569
|
+
const start = Math.max(offset, row.offset);
|
|
1570
|
+
const partEnd = Math.min(end, rowEnd);
|
|
1571
|
+
if (partEnd <= start) continue;
|
|
1572
|
+
const sourceStart = start - row.offset;
|
|
1573
|
+
const destinationStart = start - offset;
|
|
1574
|
+
bytes.set(
|
|
1575
|
+
content.subarray(sourceStart, sourceStart + partEnd - start),
|
|
1576
|
+
destinationStart,
|
|
1577
|
+
);
|
|
1578
|
+
copied += partEnd - start;
|
|
1579
|
+
}
|
|
1580
|
+
if (copied !== length) {
|
|
1581
|
+
throw new FilesystemRequestError(
|
|
1582
|
+
500,
|
|
1583
|
+
"Filesystem blob chunks are incomplete",
|
|
1584
|
+
"STORAGE_FAILED",
|
|
1585
|
+
true,
|
|
1586
|
+
);
|
|
1587
|
+
}
|
|
1588
|
+
return bytes;
|
|
1589
|
+
}
|
|
1590
|
+
|
|
1591
|
+
function equalBytes(left, right) {
|
|
1592
|
+
if (left.byteLength !== right.byteLength) return false;
|
|
1593
|
+
return left.every((byte, index) => byte === right[index]);
|
|
1594
|
+
}
|
|
1595
|
+
|
|
1596
|
+
function compareBytes(left, right) {
|
|
1597
|
+
const length = Math.min(left.byteLength, right.byteLength);
|
|
1598
|
+
for (let index = 0; index < length; index += 1) {
|
|
1599
|
+
if (left[index] !== right[index]) return left[index] - right[index];
|
|
1600
|
+
}
|
|
1601
|
+
return left.byteLength - right.byteLength;
|
|
1602
|
+
}
|
|
1603
|
+
|
|
1604
|
+
function unsigned64Bytes(value) {
|
|
1605
|
+
const bytes = new Uint8Array(8);
|
|
1606
|
+
const view = new DataView(bytes.buffer);
|
|
1607
|
+
view.setUint32(0, Math.floor(value / 0x1_0000_0000), false);
|
|
1608
|
+
view.setUint32(4, value >>> 0, false);
|
|
1609
|
+
return bytes;
|
|
1610
|
+
}
|
|
1611
|
+
|
|
1612
|
+
function parseBatch(input) {
|
|
1613
|
+
if (input === null || typeof input !== "object" || Array.isArray(input)) {
|
|
1614
|
+
throw new FilesystemRequestError(400, "Batch body must be an object");
|
|
1615
|
+
}
|
|
1616
|
+
const expectedRevision = input.expectedRevision;
|
|
1617
|
+
if (!Number.isSafeInteger(expectedRevision) || expectedRevision < 0) {
|
|
1618
|
+
throw new FilesystemRequestError(
|
|
1619
|
+
400,
|
|
1620
|
+
"expectedRevision must be a non-negative safe integer",
|
|
1621
|
+
);
|
|
1622
|
+
}
|
|
1623
|
+
const mutationId = requireBoundedString(
|
|
1624
|
+
input.mutationId,
|
|
1625
|
+
"mutationId",
|
|
1626
|
+
MAX_MUTATION_ID_BYTES,
|
|
1627
|
+
);
|
|
1628
|
+
if (!Array.isArray(input.operations) || input.operations.length === 0) {
|
|
1629
|
+
throw new FilesystemRequestError(400, "operations must not be empty");
|
|
1630
|
+
}
|
|
1631
|
+
if (input.operations.length > MAX_OPERATIONS_PER_BATCH) {
|
|
1632
|
+
throw new FilesystemRequestError(
|
|
1633
|
+
413,
|
|
1634
|
+
`A batch may contain at most ${MAX_OPERATIONS_PER_BATCH} operations`,
|
|
1635
|
+
);
|
|
1636
|
+
}
|
|
1637
|
+
const operations = input.operations.map((operation) => {
|
|
1638
|
+
if (
|
|
1639
|
+
operation === null ||
|
|
1640
|
+
typeof operation !== "object" ||
|
|
1641
|
+
Array.isArray(operation)
|
|
1642
|
+
) {
|
|
1643
|
+
throw new FilesystemRequestError(400, "Each operation must be an object");
|
|
1644
|
+
}
|
|
1645
|
+
const path = parsePath(operation.path, "operation path");
|
|
1646
|
+
if (path === "/" || path === "/workspace" || path === "/.gea") {
|
|
1647
|
+
throw new FilesystemRequestError(
|
|
1648
|
+
409,
|
|
1649
|
+
`Reserved path cannot be changed: ${path}`,
|
|
1650
|
+
);
|
|
1651
|
+
}
|
|
1652
|
+
if (operation.type === "createDirectory") {
|
|
1653
|
+
return { type: "createDirectory", path };
|
|
1654
|
+
}
|
|
1655
|
+
if (operation.type === "writeFile") {
|
|
1656
|
+
if (typeof operation.contentBase64 !== "string") {
|
|
1657
|
+
throw new FilesystemRequestError(
|
|
1658
|
+
400,
|
|
1659
|
+
"writeFile contentBase64 must be a string",
|
|
1660
|
+
);
|
|
1661
|
+
}
|
|
1662
|
+
return {
|
|
1663
|
+
type: "writeFile",
|
|
1664
|
+
path,
|
|
1665
|
+
contentBase64: operation.contentBase64,
|
|
1666
|
+
};
|
|
1667
|
+
}
|
|
1668
|
+
throw new FilesystemRequestError(
|
|
1669
|
+
400,
|
|
1670
|
+
`Unsupported filesystem operation: ${String(operation.type)}`,
|
|
1671
|
+
);
|
|
1672
|
+
});
|
|
1673
|
+
return { expectedRevision, mutationId, operations };
|
|
1674
|
+
}
|
|
1675
|
+
|
|
1676
|
+
function parseAllocationId(encoded) {
|
|
1677
|
+
let allocationId;
|
|
1678
|
+
try {
|
|
1679
|
+
allocationId = decodeURIComponent(encoded);
|
|
1680
|
+
} catch {
|
|
1681
|
+
throw new FilesystemRequestError(
|
|
1682
|
+
400,
|
|
1683
|
+
"allocation id must be valid URL text",
|
|
1684
|
+
);
|
|
1685
|
+
}
|
|
1686
|
+
requireBoundedString(allocationId, "allocation id", MAX_ALLOCATION_ID_BYTES);
|
|
1687
|
+
if (allocationId.includes("/") || allocationId.includes("\0")) {
|
|
1688
|
+
throw new FilesystemRequestError(
|
|
1689
|
+
400,
|
|
1690
|
+
"allocation id must not contain separators or NUL bytes",
|
|
1691
|
+
);
|
|
1692
|
+
}
|
|
1693
|
+
return allocationId;
|
|
1694
|
+
}
|
|
1695
|
+
|
|
1696
|
+
async function readJson(request) {
|
|
1697
|
+
try {
|
|
1698
|
+
return await request.json();
|
|
1699
|
+
} catch {
|
|
1700
|
+
throw new FilesystemRequestError(400, "Request body must be valid JSON");
|
|
1701
|
+
}
|
|
1702
|
+
}
|
|
1703
|
+
|
|
1704
|
+
function readMeta(storage) {
|
|
1705
|
+
return storage.sql
|
|
1706
|
+
.exec(
|
|
1707
|
+
`SELECT revision, logical_bytes, entry_count
|
|
1708
|
+
FROM filesystem_meta
|
|
1709
|
+
WHERE singleton = 1`,
|
|
1710
|
+
)
|
|
1711
|
+
.one();
|
|
1712
|
+
}
|
|
1713
|
+
|
|
1714
|
+
function requireDirectory(storage, path) {
|
|
1715
|
+
const rows = storage.sql
|
|
1716
|
+
.exec("SELECT kind FROM filesystem_entries WHERE path = ?", path)
|
|
1717
|
+
.toArray();
|
|
1718
|
+
if (rows.length === 0) {
|
|
1719
|
+
throw new FilesystemRequestError(404, `Directory does not exist: ${path}`);
|
|
1720
|
+
}
|
|
1721
|
+
if (rows[0].kind !== "directory") {
|
|
1722
|
+
throw new FilesystemRequestError(409, `Path is not a directory: ${path}`);
|
|
1723
|
+
}
|
|
1724
|
+
}
|
|
1725
|
+
|
|
1726
|
+
function requireNewPath(storage, path) {
|
|
1727
|
+
const rows = storage.sql
|
|
1728
|
+
.exec("SELECT 1 AS present FROM filesystem_entries WHERE path = ?", path)
|
|
1729
|
+
.toArray();
|
|
1730
|
+
if (rows.length !== 0) {
|
|
1731
|
+
throw new FilesystemRequestError(409, `Path already exists: ${path}`);
|
|
1732
|
+
}
|
|
1733
|
+
}
|
|
1734
|
+
|
|
1735
|
+
function requireDirectoryCapacity(storage, path) {
|
|
1736
|
+
const row = storage.sql
|
|
1737
|
+
.exec(
|
|
1738
|
+
"SELECT COUNT(*) AS entry_count FROM filesystem_entries WHERE parent_path = ?",
|
|
1739
|
+
path,
|
|
1740
|
+
)
|
|
1741
|
+
.one();
|
|
1742
|
+
if (row.entry_count >= MAX_ENTRIES_PER_DIRECTORY) {
|
|
1743
|
+
throw new FilesystemRequestError(
|
|
1744
|
+
413,
|
|
1745
|
+
`Directory contains the maximum ${MAX_ENTRIES_PER_DIRECTORY} entries: ${path}`,
|
|
1746
|
+
);
|
|
1747
|
+
}
|
|
1748
|
+
}
|
|
1749
|
+
|
|
1750
|
+
function replayMutation(row, requestHash) {
|
|
1751
|
+
if (row.request_hash !== requestHash) {
|
|
1752
|
+
throw new FilesystemRequestError(
|
|
1753
|
+
409,
|
|
1754
|
+
"mutationId was already used for a different batch",
|
|
1755
|
+
);
|
|
1756
|
+
}
|
|
1757
|
+
return JSON.parse(row.result_json);
|
|
1758
|
+
}
|
|
1759
|
+
|
|
1760
|
+
function parsePath(value, field) {
|
|
1761
|
+
if (typeof value !== "string") {
|
|
1762
|
+
throw new FilesystemRequestError(400, `${field} must be a string`);
|
|
1763
|
+
}
|
|
1764
|
+
if (value.length === 0 || !value.startsWith("/") || value.includes("\\")) {
|
|
1765
|
+
throw new FilesystemRequestError(
|
|
1766
|
+
400,
|
|
1767
|
+
`${field} must be an absolute POSIX path`,
|
|
1768
|
+
);
|
|
1769
|
+
}
|
|
1770
|
+
if (value.includes("\0")) {
|
|
1771
|
+
throw new FilesystemRequestError(
|
|
1772
|
+
400,
|
|
1773
|
+
`${field} must not contain NUL bytes`,
|
|
1774
|
+
);
|
|
1775
|
+
}
|
|
1776
|
+
const segments = value.split("/");
|
|
1777
|
+
if (
|
|
1778
|
+
value !== "/" &&
|
|
1779
|
+
segments.some((segment, index) => {
|
|
1780
|
+
return (
|
|
1781
|
+
index !== 0 && (segment === "" || segment === "." || segment === "..")
|
|
1782
|
+
);
|
|
1783
|
+
})
|
|
1784
|
+
) {
|
|
1785
|
+
throw new FilesystemRequestError(400, `${field} must be canonical`);
|
|
1786
|
+
}
|
|
1787
|
+
const encoded = new TextEncoder().encode(value);
|
|
1788
|
+
if (encoded.byteLength > 4096) {
|
|
1789
|
+
throw new FilesystemRequestError(413, `${field} exceeds 4096 bytes`);
|
|
1790
|
+
}
|
|
1791
|
+
return value;
|
|
1792
|
+
}
|
|
1793
|
+
|
|
1794
|
+
function parentOf(path) {
|
|
1795
|
+
const index = path.lastIndexOf("/");
|
|
1796
|
+
return index === 0 ? "/" : path.slice(0, index);
|
|
1797
|
+
}
|
|
1798
|
+
|
|
1799
|
+
function basename(path) {
|
|
1800
|
+
return path.slice(path.lastIndexOf("/") + 1);
|
|
1801
|
+
}
|
|
1802
|
+
|
|
1803
|
+
function requireBoundedString(value, field, maxBytes) {
|
|
1804
|
+
if (typeof value !== "string" || value.length === 0) {
|
|
1805
|
+
throw new FilesystemRequestError(
|
|
1806
|
+
400,
|
|
1807
|
+
`${field} must be a non-empty string`,
|
|
1808
|
+
);
|
|
1809
|
+
}
|
|
1810
|
+
if (new TextEncoder().encode(value).byteLength > maxBytes) {
|
|
1811
|
+
throw new FilesystemRequestError(413, `${field} exceeds ${maxBytes} bytes`);
|
|
1812
|
+
}
|
|
1813
|
+
return value;
|
|
1814
|
+
}
|
|
1815
|
+
|
|
1816
|
+
function decodeBase64(value) {
|
|
1817
|
+
if (
|
|
1818
|
+
!/^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/.test(
|
|
1819
|
+
value,
|
|
1820
|
+
)
|
|
1821
|
+
) {
|
|
1822
|
+
throw new FilesystemRequestError(
|
|
1823
|
+
400,
|
|
1824
|
+
"contentBase64 must be canonical base64",
|
|
1825
|
+
);
|
|
1826
|
+
}
|
|
1827
|
+
try {
|
|
1828
|
+
const binary = atob(value);
|
|
1829
|
+
const bytes = Uint8Array.from(binary, (character) =>
|
|
1830
|
+
character.charCodeAt(0),
|
|
1831
|
+
);
|
|
1832
|
+
if (encodeBase64(bytes) !== value) {
|
|
1833
|
+
throw new FilesystemRequestError(
|
|
1834
|
+
400,
|
|
1835
|
+
"contentBase64 must be canonical base64",
|
|
1836
|
+
);
|
|
1837
|
+
}
|
|
1838
|
+
return bytes;
|
|
1839
|
+
} catch {
|
|
1840
|
+
throw new FilesystemRequestError(400, "contentBase64 must be valid base64");
|
|
1841
|
+
}
|
|
1842
|
+
}
|
|
1843
|
+
|
|
1844
|
+
function encodeBase64(bytes) {
|
|
1845
|
+
let binary = "";
|
|
1846
|
+
for (let offset = 0; offset < bytes.length; offset += 0x8000) {
|
|
1847
|
+
binary += String.fromCharCode(...bytes.subarray(offset, offset + 0x8000));
|
|
1848
|
+
}
|
|
1849
|
+
return btoa(binary);
|
|
1850
|
+
}
|
|
1851
|
+
|
|
1852
|
+
async function sha256(bytes) {
|
|
1853
|
+
const digest = new Uint8Array(await crypto.subtle.digest("SHA-256", bytes));
|
|
1854
|
+
return Array.from(digest, (byte) => byte.toString(16).padStart(2, "0")).join(
|
|
1855
|
+
"",
|
|
1856
|
+
);
|
|
1857
|
+
}
|
|
1858
|
+
|
|
1859
|
+
async function mutationHash(invocation) {
|
|
1860
|
+
const canonical = canonicalJson({
|
|
1861
|
+
input: invocation.input,
|
|
1862
|
+
operation: invocation.operation,
|
|
1863
|
+
});
|
|
1864
|
+
return sha256(new TextEncoder().encode(JSON.stringify(canonical)));
|
|
1865
|
+
}
|
|
1866
|
+
|
|
1867
|
+
function canonicalJson(value) {
|
|
1868
|
+
if (Array.isArray(value)) return value.map(canonicalJson);
|
|
1869
|
+
if (value === null || typeof value !== "object") return value;
|
|
1870
|
+
return Object.fromEntries(
|
|
1871
|
+
Object.keys(value)
|
|
1872
|
+
.sort()
|
|
1873
|
+
.map((key) => [key, canonicalJson(value[key])]),
|
|
1874
|
+
);
|
|
1875
|
+
}
|
|
1876
|
+
|
|
1877
|
+
class Sha256State {
|
|
1878
|
+
static #constants = new Uint32Array([
|
|
1879
|
+
0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, 0x3956c25b, 0x59f111f1,
|
|
1880
|
+
0x923f82a4, 0xab1c5ed5, 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3,
|
|
1881
|
+
0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174, 0xe49b69c1, 0xefbe4786,
|
|
1882
|
+
0x0fc19dc6, 0x240ca1cc, 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da,
|
|
1883
|
+
0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7, 0xc6e00bf3, 0xd5a79147,
|
|
1884
|
+
0x06ca6351, 0x14292967, 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13,
|
|
1885
|
+
0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85, 0xa2bfe8a1, 0xa81a664b,
|
|
1886
|
+
0xc24b8b70, 0xc76c51a3, 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070,
|
|
1887
|
+
0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5, 0x391c0cb3, 0x4ed8aa4a,
|
|
1888
|
+
0x5b9cca4f, 0x682e6ff3, 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208,
|
|
1889
|
+
0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2,
|
|
1890
|
+
]);
|
|
1891
|
+
|
|
1892
|
+
constructor(
|
|
1893
|
+
words = [
|
|
1894
|
+
0x6a09e667, 0xbb67ae85, 0x3c6ef372, 0xa54ff53a, 0x510e527f, 0x9b05688c,
|
|
1895
|
+
0x1f83d9ab, 0x5be0cd19,
|
|
1896
|
+
],
|
|
1897
|
+
buffer = new Uint8Array(),
|
|
1898
|
+
bytesHashed = 0,
|
|
1899
|
+
) {
|
|
1900
|
+
this.words = new Uint32Array(words);
|
|
1901
|
+
this.buffer = new Uint8Array(buffer);
|
|
1902
|
+
this.bytesHashed = bytesHashed;
|
|
1903
|
+
}
|
|
1904
|
+
|
|
1905
|
+
static fromJSON(value) {
|
|
1906
|
+
if (
|
|
1907
|
+
value === null ||
|
|
1908
|
+
typeof value !== "object" ||
|
|
1909
|
+
!Array.isArray(value.words) ||
|
|
1910
|
+
value.words.length !== 8 ||
|
|
1911
|
+
!Array.isArray(value.buffer) ||
|
|
1912
|
+
value.buffer.length >= 64 ||
|
|
1913
|
+
!Number.isSafeInteger(value.bytesHashed) ||
|
|
1914
|
+
value.bytesHashed < 0
|
|
1915
|
+
) {
|
|
1916
|
+
throw new FilesystemRequestError(
|
|
1917
|
+
500,
|
|
1918
|
+
"Checkpoint digest state is invalid",
|
|
1919
|
+
"STORAGE_FAILED",
|
|
1920
|
+
);
|
|
1921
|
+
}
|
|
1922
|
+
return new Sha256State(value.words, value.buffer, value.bytesHashed);
|
|
1923
|
+
}
|
|
1924
|
+
|
|
1925
|
+
toJSON() {
|
|
1926
|
+
return {
|
|
1927
|
+
buffer: Array.from(this.buffer),
|
|
1928
|
+
bytesHashed: this.bytesHashed,
|
|
1929
|
+
words: Array.from(this.words),
|
|
1930
|
+
};
|
|
1931
|
+
}
|
|
1932
|
+
|
|
1933
|
+
update(bytes) {
|
|
1934
|
+
const input = new Uint8Array(bytes);
|
|
1935
|
+
const combined = new Uint8Array(this.buffer.byteLength + input.byteLength);
|
|
1936
|
+
combined.set(this.buffer);
|
|
1937
|
+
combined.set(input, this.buffer.byteLength);
|
|
1938
|
+
let offset = 0;
|
|
1939
|
+
while (offset + 64 <= combined.byteLength) {
|
|
1940
|
+
this.#compress(combined.subarray(offset, offset + 64));
|
|
1941
|
+
offset += 64;
|
|
1942
|
+
}
|
|
1943
|
+
this.buffer = combined.slice(offset);
|
|
1944
|
+
this.bytesHashed += input.byteLength;
|
|
1945
|
+
if (!Number.isSafeInteger(this.bytesHashed)) {
|
|
1946
|
+
throw new FilesystemRequestError(413, "Digest input is too large");
|
|
1947
|
+
}
|
|
1948
|
+
return this;
|
|
1949
|
+
}
|
|
1950
|
+
|
|
1951
|
+
digestHex() {
|
|
1952
|
+
const clone = new Sha256State(this.words, this.buffer, this.bytesHashed);
|
|
1953
|
+
const paddingBytes = (64 + 56 - ((clone.bytesHashed + 1) % 64)) % 64;
|
|
1954
|
+
const trailer = new Uint8Array(1 + paddingBytes + 8);
|
|
1955
|
+
trailer[0] = 0x80;
|
|
1956
|
+
const bitLength = clone.bytesHashed * 8;
|
|
1957
|
+
const view = new DataView(trailer.buffer);
|
|
1958
|
+
view.setUint32(
|
|
1959
|
+
trailer.byteLength - 8,
|
|
1960
|
+
Math.floor(bitLength / 0x1_0000_0000),
|
|
1961
|
+
false,
|
|
1962
|
+
);
|
|
1963
|
+
view.setUint32(trailer.byteLength - 4, bitLength >>> 0, false);
|
|
1964
|
+
clone.update(trailer);
|
|
1965
|
+
if (clone.buffer.byteLength !== 0) {
|
|
1966
|
+
throw new FilesystemRequestError(
|
|
1967
|
+
500,
|
|
1968
|
+
"Checkpoint digest finalization failed",
|
|
1969
|
+
"STORAGE_FAILED",
|
|
1970
|
+
);
|
|
1971
|
+
}
|
|
1972
|
+
return Array.from(clone.words, (word) =>
|
|
1973
|
+
word.toString(16).padStart(8, "0"),
|
|
1974
|
+
).join("");
|
|
1975
|
+
}
|
|
1976
|
+
|
|
1977
|
+
#compress(block) {
|
|
1978
|
+
const schedule = new Uint32Array(64);
|
|
1979
|
+
const view = new DataView(block.buffer, block.byteOffset, block.byteLength);
|
|
1980
|
+
for (let index = 0; index < 16; index += 1) {
|
|
1981
|
+
schedule[index] = view.getUint32(index * 4, false);
|
|
1982
|
+
}
|
|
1983
|
+
for (let index = 16; index < 64; index += 1) {
|
|
1984
|
+
const earlier = schedule[index - 15];
|
|
1985
|
+
const later = schedule[index - 2];
|
|
1986
|
+
const sigma0 =
|
|
1987
|
+
rotateRight(earlier, 7) ^ rotateRight(earlier, 18) ^ (earlier >>> 3);
|
|
1988
|
+
const sigma1 =
|
|
1989
|
+
rotateRight(later, 17) ^ rotateRight(later, 19) ^ (later >>> 10);
|
|
1990
|
+
schedule[index] =
|
|
1991
|
+
(schedule[index - 16] + sigma0 + schedule[index - 7] + sigma1) >>> 0;
|
|
1992
|
+
}
|
|
1993
|
+
let [a, b, c, d, e, f, g, h] = this.words;
|
|
1994
|
+
for (let index = 0; index < 64; index += 1) {
|
|
1995
|
+
const sum1 = rotateRight(e, 6) ^ rotateRight(e, 11) ^ rotateRight(e, 25);
|
|
1996
|
+
const choice = (e & f) ^ (~e & g);
|
|
1997
|
+
const first =
|
|
1998
|
+
(h +
|
|
1999
|
+
sum1 +
|
|
2000
|
+
choice +
|
|
2001
|
+
Sha256State.#constants[index] +
|
|
2002
|
+
schedule[index]) >>>
|
|
2003
|
+
0;
|
|
2004
|
+
const sum0 = rotateRight(a, 2) ^ rotateRight(a, 13) ^ rotateRight(a, 22);
|
|
2005
|
+
const majority = (a & b) ^ (a & c) ^ (b & c);
|
|
2006
|
+
const second = (sum0 + majority) >>> 0;
|
|
2007
|
+
h = g;
|
|
2008
|
+
g = f;
|
|
2009
|
+
f = e;
|
|
2010
|
+
e = (d + first) >>> 0;
|
|
2011
|
+
d = c;
|
|
2012
|
+
c = b;
|
|
2013
|
+
b = a;
|
|
2014
|
+
a = (first + second) >>> 0;
|
|
2015
|
+
}
|
|
2016
|
+
this.words[0] = (this.words[0] + a) >>> 0;
|
|
2017
|
+
this.words[1] = (this.words[1] + b) >>> 0;
|
|
2018
|
+
this.words[2] = (this.words[2] + c) >>> 0;
|
|
2019
|
+
this.words[3] = (this.words[3] + d) >>> 0;
|
|
2020
|
+
this.words[4] = (this.words[4] + e) >>> 0;
|
|
2021
|
+
this.words[5] = (this.words[5] + f) >>> 0;
|
|
2022
|
+
this.words[6] = (this.words[6] + g) >>> 0;
|
|
2023
|
+
this.words[7] = (this.words[7] + h) >>> 0;
|
|
2024
|
+
}
|
|
2025
|
+
}
|
|
2026
|
+
|
|
2027
|
+
function rotateRight(value, count) {
|
|
2028
|
+
return (value >>> count) | (value << (32 - count));
|
|
2029
|
+
}
|
|
2030
|
+
|
|
2031
|
+
function jsonError(
|
|
2032
|
+
status,
|
|
2033
|
+
message,
|
|
2034
|
+
code = status >= 500 ? "INTERNAL" : "INVALID_REQUEST",
|
|
2035
|
+
retryable = status >= 500,
|
|
2036
|
+
) {
|
|
2037
|
+
return Response.json({ error: { code, message, retryable } }, { status });
|
|
2038
|
+
}
|