@minnowdb/core 0.7.0 → 0.7.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/block-format/block.js +254 -325
- package/dist/block-format/checksum.js +30 -52
- package/dist/block-format/codecs.js +112 -125
- package/dist/block-format/column.js +187 -205
- package/dist/block-format/index.js +5 -1
- package/dist/block-format/physical.js +371 -387
- package/dist/block-format/types.js +4 -1
- package/dist/block-format/unicode.js +30 -39
- package/dist/date-value.js +67 -78
- package/dist/engine/artifact-cache.js +106 -116
- package/dist/engine/batch.js +36 -44
- package/dist/engine/buffered-writer.js +156 -158
- package/dist/engine/byte-estimates.js +20 -22
- package/dist/engine/cache-limits.js +24 -25
- package/dist/engine/cancellation.js +5 -3
- package/dist/engine/catalog.js +64 -73
- package/dist/engine/client.js +715 -780
- package/dist/engine/database.js +15941 -19313
- package/dist/engine/defaults.js +104 -111
- package/dist/engine/errors.js +113 -128
- package/dist/engine/fts.js +233 -280
- package/dist/engine/group-index.js +287 -321
- package/dist/engine/index.js +23 -12
- package/dist/engine/join-index.js +177 -186
- package/dist/engine/keyed-live.js +168 -173
- package/dist/engine/live-api.js +0 -1
- package/dist/engine/live-equal.js +32 -37
- package/dist/engine/live.js +603 -647
- package/dist/engine/memory.js +109 -111
- package/dist/engine/optimizer.d.ts +19 -1
- package/dist/engine/optimizer.js +2633 -2934
- package/dist/engine/point-read.js +122 -182
- package/dist/engine/query-api.js +12 -3
- package/dist/engine/query-cache.js +61 -69
- package/dist/engine/query.d.ts +6 -0
- package/dist/engine/query.js +8503 -9980
- package/dist/engine/result-wire.js +221 -252
- package/dist/engine/schema-wire.js +97 -112
- package/dist/engine/schema.js +1099 -1254
- package/dist/engine/sort-keys.js +315 -389
- package/dist/engine/sql-domains.js +646 -769
- package/dist/engine/sql-driver.js +0 -1
- package/dist/engine/sql-functions.js +1031 -1125
- package/dist/engine/sql-json.js +199 -218
- package/dist/engine/sql-semantics.js +572 -639
- package/dist/engine/typed-live.js +271 -293
- package/dist/engine/vector.js +4484 -5145
- package/dist/engine/worker-host.d.ts +8 -63
- package/dist/engine/worker-host.js +28 -1130
- package/dist/engine/worker-indexeddb.d.ts +1 -0
- package/dist/engine/worker-indexeddb.js +3 -0
- package/dist/engine/worker-memory.d.ts +1 -0
- package/dist/engine/worker-memory.js +3 -0
- package/dist/engine/worker-opfs.d.ts +1 -0
- package/dist/engine/worker-opfs.js +3 -0
- package/dist/engine/worker-server.d.ts +93 -0
- package/dist/engine/worker-server.js +999 -0
- package/dist/engine/worker-store-indexeddb.d.ts +2 -0
- package/dist/engine/worker-store-indexeddb.js +11 -0
- package/dist/engine/worker-store-memory.d.ts +2 -0
- package/dist/engine/worker-store-memory.js +6 -0
- package/dist/engine/worker-store-opfs.d.ts +2 -0
- package/dist/engine/worker-store-opfs.js +9 -0
- package/dist/engine/worker.js +0 -12
- package/dist/engine/write-block-planner.js +93 -100
- package/dist/plan/index.js +15 -16
- package/dist/plan/model.d.ts +6 -0
- package/dist/plan/model.js +20 -27
- package/dist/storage/index.js +0 -12
- package/dist/storage/indexeddb.js +11776 -13171
- package/dist/storage/memory.js +1198 -1241
- package/dist/storage/opfs/files.js +238 -295
- package/dist/storage/opfs/index.js +9 -3
- package/dist/storage/opfs/leader.js +4386 -4856
- package/dist/storage/opfs/rpc.js +223 -259
- package/dist/storage/opfs/snapshot-ledger.js +219 -260
- package/dist/storage/opfs/store.js +884 -1003
- package/dist/storage/persistence.js +59 -74
- package/dist/storage/snapshot-stream.js +788 -826
- package/dist/storage/snapshot.js +36 -37
- package/dist/storage/toolkit/extents.js +444 -520
- package/dist/storage/toolkit/index.js +28 -29
- package/dist/storage/toolkit/record-core.js +5792 -6377
- package/dist/storage/toolkit/sync-file.js +34 -42
- package/dist/storage/toolkit/wal.js +92 -127
- package/dist/storage/toolkit/wire.js +232 -256
- package/dist/storage/types.js +2948 -3250
- package/dist/testing/block-store-conformance.js +1556 -1629
- package/dist/testing/index.js +310 -291
- package/dist/testing/opfs-shim.js +280 -312
- package/dist/testing/simulator.js +496 -534
- package/dist/testing/sqllogictest.js +425 -365
- package/dist/transactions/index.js +1464 -1712
- package/dist/worker-protocol/index.js +70 -69
- package/package.json +20 -2
|
@@ -1,538 +1,462 @@
|
|
|
1
1
|
import { readFully, writeFully } from "./sync-file.js";
|
|
2
2
|
import { crc32 } from "../../block-format/index.js";
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
this.tailOffset = tailOffset;
|
|
14
|
-
}
|
|
3
|
+
class ExtentBatchMark {
|
|
4
|
+
tailExtentId;
|
|
5
|
+
nextExtentId;
|
|
6
|
+
tailOffset;
|
|
7
|
+
_liveBytesBefore = /* @__PURE__ */ new Map();
|
|
8
|
+
constructor(tailExtentId, nextExtentId, tailOffset) {
|
|
9
|
+
this.tailExtentId = tailExtentId;
|
|
10
|
+
this.nextExtentId = nextExtentId;
|
|
11
|
+
this.tailOffset = tailOffset;
|
|
12
|
+
}
|
|
15
13
|
}
|
|
16
14
|
const SEAL_BYTES = 8 * 1024 * 1024;
|
|
17
15
|
const READ_HANDLE_CACHE = 12;
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
}
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
}
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
}
|
|
414
|
-
/** Removes a drained extent's file and forgets it. Idempotent. */
|
|
415
|
-
async deleteExtent(id) {
|
|
416
|
-
if (this.#activeBatch !== undefined) {
|
|
417
|
-
throw new Error("Cannot delete an extent while an unpublished batch is active");
|
|
418
|
-
}
|
|
419
|
-
if (id === this.#tailId)
|
|
420
|
-
return;
|
|
421
|
-
this.#deletingExtents.add(id);
|
|
422
|
-
try {
|
|
423
|
-
const opening = this.#openingReadHandles.get(id);
|
|
424
|
-
if (opening !== undefined)
|
|
425
|
-
await opening.promise.catch(() => undefined);
|
|
426
|
-
const cached = this.#readHandles.get(id);
|
|
427
|
-
if (cached !== undefined && cached.readers > 0) {
|
|
428
|
-
await new Promise((resolve) => cached.drained.push(resolve));
|
|
429
|
-
}
|
|
430
|
-
this.#readHandles.get(id)?.handle.close();
|
|
431
|
-
this.#readHandles.delete(id);
|
|
432
|
-
await this.#tree.deleteFile(extentPath(id));
|
|
433
|
-
// Preserve the zero-live entry when the physical delete is refused. The caller can then
|
|
434
|
-
// retry it and account the cleanup debt instead of silently orphaning an untracked file.
|
|
435
|
-
this.#liveBytes.delete(id);
|
|
436
|
-
}
|
|
437
|
-
finally {
|
|
438
|
-
this.#deletingExtents.delete(id);
|
|
439
|
-
}
|
|
440
|
-
}
|
|
441
|
-
meta() {
|
|
442
|
-
return {
|
|
443
|
-
nextExtentId: this.#nextId,
|
|
444
|
-
tailExtentId: this.#tailId,
|
|
445
|
-
tailOffset: this.#tailOffset,
|
|
446
|
-
liveBytes: [...this.#liveBytes.entries()],
|
|
447
|
-
};
|
|
448
|
-
}
|
|
449
|
-
flush() {
|
|
450
|
-
if (!this.#tailDirty)
|
|
451
|
-
return;
|
|
452
|
-
this.#tailHandle.flush();
|
|
453
|
-
this.#tailDirty = false;
|
|
454
|
-
}
|
|
455
|
-
close() {
|
|
456
|
-
this.#closed = true;
|
|
457
|
-
this.#tailHandle.close();
|
|
458
|
-
for (const { handle } of this.#readHandles.values())
|
|
459
|
-
handle.close();
|
|
460
|
-
this.#readHandles.clear();
|
|
461
|
-
}
|
|
462
|
-
#rememberLiveBytes(id) {
|
|
463
|
-
const batch = this.#activeBatch;
|
|
464
|
-
if (batch === undefined || batch._liveBytesBefore.has(id))
|
|
465
|
-
return;
|
|
466
|
-
batch._liveBytesBefore.set(id, this.#liveBytes.get(id));
|
|
467
|
-
}
|
|
16
|
+
class ExtentPool {
|
|
17
|
+
#tree;
|
|
18
|
+
#checksum;
|
|
19
|
+
#tailId;
|
|
20
|
+
#nextId;
|
|
21
|
+
#tailOffset;
|
|
22
|
+
#tailHandle;
|
|
23
|
+
#tailDirty = false;
|
|
24
|
+
#liveBytes = /* @__PURE__ */ new Map();
|
|
25
|
+
#readHandles = /* @__PURE__ */ new Map();
|
|
26
|
+
#openingReadHandles = /* @__PURE__ */ new Map();
|
|
27
|
+
#deletingExtents = /* @__PURE__ */ new Set();
|
|
28
|
+
#activeBatch;
|
|
29
|
+
#closed = false;
|
|
30
|
+
constructor(tree, meta, tailHandle, checksum) {
|
|
31
|
+
this.#tree = tree;
|
|
32
|
+
this.#checksum = checksum;
|
|
33
|
+
this.#tailId = meta.tailId;
|
|
34
|
+
this.#nextId = meta.nextId;
|
|
35
|
+
this.#tailOffset = meta.tailOffset;
|
|
36
|
+
this.#tailHandle = tailHandle;
|
|
37
|
+
}
|
|
38
|
+
static async open(tree, meta, options = {}) {
|
|
39
|
+
if (meta !== void 0)
|
|
40
|
+
assertValidExtentMeta(meta);
|
|
41
|
+
const tailId = meta?.tailExtentId ?? 0;
|
|
42
|
+
const nextId = meta?.nextExtentId ?? 1;
|
|
43
|
+
const tailHandle = await tree.openHandle(extentPath(tailId), { create: true });
|
|
44
|
+
const pool = new ExtentPool(tree, { tailId, nextId, tailOffset: meta?.tailOffset ?? 0 }, tailHandle, options._checksumForTests ?? crc32);
|
|
45
|
+
for (const [id, bytes] of meta?.liveBytes ?? [])
|
|
46
|
+
pool.#liveBytes.set(id, bytes);
|
|
47
|
+
if (tailHandle.getSize() < pool.#tailOffset) {
|
|
48
|
+
pool.close();
|
|
49
|
+
throw new Error(`OPFS tail extent ${String(tailId)} is shorter than its checkpointed offset: ${String(tailHandle.getSize())} < ${String(pool.#tailOffset)}`);
|
|
50
|
+
}
|
|
51
|
+
return pool;
|
|
52
|
+
}
|
|
53
|
+
markBatch() {
|
|
54
|
+
if (this.#activeBatch !== void 0)
|
|
55
|
+
throw new Error("An extent append batch is already active");
|
|
56
|
+
const mark = new ExtentBatchMark(this.#tailId, this.#nextId, this.#tailOffset);
|
|
57
|
+
this.#activeBatch = mark;
|
|
58
|
+
return mark;
|
|
59
|
+
}
|
|
60
|
+
commitBatch(mark) {
|
|
61
|
+
if (this.#activeBatch !== mark)
|
|
62
|
+
throw new Error("Extent batch mark is not active");
|
|
63
|
+
this.#activeBatch = void 0;
|
|
64
|
+
}
|
|
65
|
+
async rollbackBatch(mark) {
|
|
66
|
+
if (this.#activeBatch !== mark)
|
|
67
|
+
throw new Error("Extent batch mark is not active");
|
|
68
|
+
if (mark.nextExtentId > this.#nextId) {
|
|
69
|
+
throw new Error("Cannot roll an extent pool forward to a future batch mark");
|
|
70
|
+
}
|
|
71
|
+
this.#tailHandle.close();
|
|
72
|
+
for (let id = mark.nextExtentId; id < this.#nextId; id += 1) {
|
|
73
|
+
await this.#closeCachedReadHandle(id);
|
|
74
|
+
await this.#tree.deleteFile(extentPath(id));
|
|
75
|
+
}
|
|
76
|
+
await this.#closeCachedReadHandle(mark.tailExtentId);
|
|
77
|
+
this.#tailHandle = await this.#tree.openHandle(extentPath(mark.tailExtentId), { create: true });
|
|
78
|
+
this.#tailHandle.truncate(mark.tailOffset);
|
|
79
|
+
this.#tailId = mark.tailExtentId;
|
|
80
|
+
this.#nextId = mark.nextExtentId;
|
|
81
|
+
this.#tailOffset = mark.tailOffset;
|
|
82
|
+
this.#tailDirty = false;
|
|
83
|
+
for (const [id, bytes] of mark._liveBytesBefore) {
|
|
84
|
+
if (bytes === void 0)
|
|
85
|
+
this.#liveBytes.delete(id);
|
|
86
|
+
else
|
|
87
|
+
this.#liveBytes.set(id, bytes);
|
|
88
|
+
}
|
|
89
|
+
this.#activeBatch = void 0;
|
|
90
|
+
}
|
|
91
|
+
async #closeCachedReadHandle(id) {
|
|
92
|
+
const opening = this.#openingReadHandles.get(id);
|
|
93
|
+
if (opening !== void 0)
|
|
94
|
+
await opening.promise.catch(() => void 0);
|
|
95
|
+
const cached = this.#readHandles.get(id);
|
|
96
|
+
if (cached !== void 0 && cached.readers > 0) {
|
|
97
|
+
await new Promise((resolve) => cached.drained.push(resolve));
|
|
98
|
+
}
|
|
99
|
+
this.#readHandles.get(id)?.handle.close();
|
|
100
|
+
this.#readHandles.delete(id);
|
|
101
|
+
}
|
|
102
|
+
truncateTailToPublishedOffset() {
|
|
103
|
+
const size = this.#tailHandle.getSize();
|
|
104
|
+
if (size < this.#tailOffset) {
|
|
105
|
+
throw new Error(`OPFS tail extent ${String(this.#tailId)} is shorter than its published offset: ${String(size)} < ${String(this.#tailOffset)}`);
|
|
106
|
+
}
|
|
107
|
+
if (size > this.#tailOffset)
|
|
108
|
+
this.#tailHandle.truncate(this.#tailOffset);
|
|
109
|
+
}
|
|
110
|
+
restorePlacement(placement) {
|
|
111
|
+
assertValidPlacement(placement);
|
|
112
|
+
const liveBytes = safeExtentSum(this.#liveBytes.get(placement.extent) ?? 0, placement.length, "Extent live bytes");
|
|
113
|
+
this.#liveBytes.set(placement.extent, liveBytes);
|
|
114
|
+
if (placement.extent === this.#tailId) {
|
|
115
|
+
this.#tailOffset = Math.max(this.#tailOffset, safeExtentSum(placement.offset, placement.length, "Extent tail offset"));
|
|
116
|
+
}
|
|
117
|
+
if (placement.extent >= this.#nextId) {
|
|
118
|
+
this.#nextId = safeExtentSuccessor(placement.extent, "Extent ID");
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
async adoptTail(extentId, offset) {
|
|
122
|
+
assertValidExtentCoordinate(extentId, "Extent ID");
|
|
123
|
+
assertValidExtentCoordinate(offset, "Extent tail offset");
|
|
124
|
+
const nextId = safeExtentSuccessor(extentId, "Extent ID");
|
|
125
|
+
if (extentId === this.#tailId) {
|
|
126
|
+
this.#tailOffset = Math.max(this.#tailOffset, offset);
|
|
127
|
+
return;
|
|
128
|
+
}
|
|
129
|
+
this.#tailHandle.close();
|
|
130
|
+
this.#readHandles.get(extentId)?.handle.close();
|
|
131
|
+
this.#readHandles.delete(extentId);
|
|
132
|
+
this.#tailHandle = await this.#tree.openHandle(extentPath(extentId), { create: true });
|
|
133
|
+
this.#tailId = extentId;
|
|
134
|
+
this.#tailOffset = offset;
|
|
135
|
+
this.#nextId = Math.max(this.#nextId, nextId);
|
|
136
|
+
}
|
|
137
|
+
get tailExtentId() {
|
|
138
|
+
return this.#tailId;
|
|
139
|
+
}
|
|
140
|
+
async append(bytes, flush) {
|
|
141
|
+
const initialEnd = safeExtentSum(this.#tailOffset, bytes.byteLength, "Extent tail offset");
|
|
142
|
+
if (this.#tailOffset > 0 && initialEnd > SEAL_BYTES) {
|
|
143
|
+
await this.#seal();
|
|
144
|
+
}
|
|
145
|
+
const nextOffset = safeExtentSum(this.#tailOffset, bytes.byteLength, "Extent tail offset");
|
|
146
|
+
const nextLiveBytes = safeExtentSum(this.#liveBytes.get(this.#tailId) ?? 0, bytes.byteLength, "Extent live bytes");
|
|
147
|
+
const placement = {
|
|
148
|
+
extent: this.#tailId,
|
|
149
|
+
offset: this.#tailOffset,
|
|
150
|
+
length: bytes.byteLength,
|
|
151
|
+
checksum: this.#checksum(bytes)
|
|
152
|
+
};
|
|
153
|
+
writeFully(this.#tailHandle, bytes, placement.offset, `appending extent ${String(placement.extent)}`);
|
|
154
|
+
this.#tailDirty = true;
|
|
155
|
+
if (flush)
|
|
156
|
+
this.flush();
|
|
157
|
+
this.#tailOffset = nextOffset;
|
|
158
|
+
this.#rememberLiveBytes(this.#tailId);
|
|
159
|
+
this.#liveBytes.set(this.#tailId, nextLiveBytes);
|
|
160
|
+
return placement;
|
|
161
|
+
}
|
|
162
|
+
async #seal() {
|
|
163
|
+
const nextId = safeExtentSuccessor(this.#nextId, "Extent ID");
|
|
164
|
+
this.flush();
|
|
165
|
+
this.#tailHandle.close();
|
|
166
|
+
this.#tailId = this.#nextId;
|
|
167
|
+
this.#nextId = nextId;
|
|
168
|
+
this.#tailOffset = 0;
|
|
169
|
+
this.#tailHandle = await this.#tree.openHandle(extentPath(this.#tailId), { create: true });
|
|
170
|
+
}
|
|
171
|
+
async read(placement) {
|
|
172
|
+
assertValidPlacement(placement);
|
|
173
|
+
if (placement.extent === this.#tailId)
|
|
174
|
+
return this.#readFromHandle(placement, this.#tailHandle);
|
|
175
|
+
const acquired = this.#acquireReadHandle(placement.extent);
|
|
176
|
+
const handle = isHandlePromise(acquired) ? await acquired : acquired;
|
|
177
|
+
try {
|
|
178
|
+
return this.#readFromHandle(placement, handle);
|
|
179
|
+
} finally {
|
|
180
|
+
this.#releaseReadHandle(placement.extent);
|
|
181
|
+
}
|
|
182
|
+
}
|
|
183
|
+
async readVerified(placement) {
|
|
184
|
+
const bytes = await this.read(placement);
|
|
185
|
+
if (this.#checksum(bytes) !== placement.checksum) {
|
|
186
|
+
throw new Error(`Extent placement checksum mismatch: ${String(placement.extent)}:${String(placement.offset)}+${String(placement.length)}`);
|
|
187
|
+
}
|
|
188
|
+
return bytes;
|
|
189
|
+
}
|
|
190
|
+
async contains(placement) {
|
|
191
|
+
if (!validPlacement(placement))
|
|
192
|
+
return false;
|
|
193
|
+
try {
|
|
194
|
+
if (placement.extent === this.#tailId) {
|
|
195
|
+
return placement.offset + placement.length <= this.#tailHandle.getSize();
|
|
196
|
+
}
|
|
197
|
+
const acquired = this.#acquireReadHandle(placement.extent);
|
|
198
|
+
const handle = isHandlePromise(acquired) ? await acquired : acquired;
|
|
199
|
+
try {
|
|
200
|
+
return placement.offset + placement.length <= handle.getSize();
|
|
201
|
+
} finally {
|
|
202
|
+
this.#releaseReadHandle(placement.extent);
|
|
203
|
+
}
|
|
204
|
+
} catch (error) {
|
|
205
|
+
if (error instanceof DOMException && error.name === "NotFoundError")
|
|
206
|
+
return false;
|
|
207
|
+
throw error;
|
|
208
|
+
}
|
|
209
|
+
}
|
|
210
|
+
async fragmentedExtentId() {
|
|
211
|
+
let candidate;
|
|
212
|
+
for (const [id, liveBytes] of this.#liveBytes) {
|
|
213
|
+
if (id === this.#tailId || liveBytes === 0)
|
|
214
|
+
continue;
|
|
215
|
+
const acquired = this.#acquireReadHandle(id);
|
|
216
|
+
const handle = isHandlePromise(acquired) ? await acquired : acquired;
|
|
217
|
+
let size;
|
|
218
|
+
try {
|
|
219
|
+
size = handle.getSize();
|
|
220
|
+
} finally {
|
|
221
|
+
this.#releaseReadHandle(id);
|
|
222
|
+
}
|
|
223
|
+
if (size === 0 || liveBytes * 2 >= size)
|
|
224
|
+
continue;
|
|
225
|
+
const wasteRatio = (size - liveBytes) / size;
|
|
226
|
+
if (candidate === void 0 || wasteRatio > candidate.wasteRatio) {
|
|
227
|
+
candidate = { id, wasteRatio };
|
|
228
|
+
}
|
|
229
|
+
}
|
|
230
|
+
return candidate?.id;
|
|
231
|
+
}
|
|
232
|
+
async physicalByteLengths() {
|
|
233
|
+
const sizes = /* @__PURE__ */ new Map();
|
|
234
|
+
const ids = /* @__PURE__ */ new Set([...this.#liveBytes.keys(), this.#tailId]);
|
|
235
|
+
for (const id of ids) {
|
|
236
|
+
if (id === this.#tailId) {
|
|
237
|
+
sizes.set(id, this.#tailHandle.getSize());
|
|
238
|
+
continue;
|
|
239
|
+
}
|
|
240
|
+
const acquired = this.#acquireReadHandle(id);
|
|
241
|
+
const handle = isHandlePromise(acquired) ? await acquired : acquired;
|
|
242
|
+
try {
|
|
243
|
+
sizes.set(id, handle.getSize());
|
|
244
|
+
} finally {
|
|
245
|
+
this.#releaseReadHandle(id);
|
|
246
|
+
}
|
|
247
|
+
}
|
|
248
|
+
return sizes;
|
|
249
|
+
}
|
|
250
|
+
#acquireReadHandle(extent) {
|
|
251
|
+
if (this.#closed)
|
|
252
|
+
throw new Error("This extent pool is closed");
|
|
253
|
+
if (this.#deletingExtents.has(extent)) {
|
|
254
|
+
throw new Error(`Extent ${String(extent)} is being deleted`);
|
|
255
|
+
}
|
|
256
|
+
const cached = this.#readHandles.get(extent);
|
|
257
|
+
if (cached !== void 0) {
|
|
258
|
+
this.#readHandles.delete(extent);
|
|
259
|
+
this.#readHandles.set(extent, cached);
|
|
260
|
+
cached.readers += 1;
|
|
261
|
+
return cached.handle;
|
|
262
|
+
}
|
|
263
|
+
const opening = this.#openingReadHandles.get(extent);
|
|
264
|
+
if (opening !== void 0) {
|
|
265
|
+
opening.reservations += 1;
|
|
266
|
+
return opening.promise;
|
|
267
|
+
}
|
|
268
|
+
const pending = { reservations: 1 };
|
|
269
|
+
pending.promise = this.#tree.openHandle(extentPath(extent), { create: false }).then((handle) => {
|
|
270
|
+
this.#openingReadHandles.delete(extent);
|
|
271
|
+
if (this.#closed) {
|
|
272
|
+
handle.close();
|
|
273
|
+
throw new Error(`Extent ${String(extent)} became unavailable while opening`);
|
|
274
|
+
}
|
|
275
|
+
this.#readHandles.set(extent, {
|
|
276
|
+
handle,
|
|
277
|
+
readers: pending.reservations,
|
|
278
|
+
drained: []
|
|
279
|
+
});
|
|
280
|
+
this.#evictReadHandles();
|
|
281
|
+
return handle;
|
|
282
|
+
}, (error) => {
|
|
283
|
+
this.#openingReadHandles.delete(extent);
|
|
284
|
+
throw error;
|
|
285
|
+
});
|
|
286
|
+
this.#openingReadHandles.set(extent, pending);
|
|
287
|
+
return pending.promise;
|
|
288
|
+
}
|
|
289
|
+
#releaseReadHandle(extent) {
|
|
290
|
+
const cached = this.#readHandles.get(extent);
|
|
291
|
+
if (cached === void 0)
|
|
292
|
+
return;
|
|
293
|
+
cached.readers -= 1;
|
|
294
|
+
if (cached.readers < 0)
|
|
295
|
+
throw new Error(`Extent ${String(extent)} read lease underflow`);
|
|
296
|
+
if (cached.readers === 0) {
|
|
297
|
+
for (const resolve of cached.drained.splice(0))
|
|
298
|
+
resolve();
|
|
299
|
+
this.#evictReadHandles();
|
|
300
|
+
}
|
|
301
|
+
}
|
|
302
|
+
#evictReadHandles() {
|
|
303
|
+
while (this.#readHandles.size > READ_HANDLE_CACHE) {
|
|
304
|
+
const idle = [...this.#readHandles].find(([, cached2]) => cached2.readers === 0);
|
|
305
|
+
if (idle === void 0)
|
|
306
|
+
return;
|
|
307
|
+
const [extent, cached] = idle;
|
|
308
|
+
cached.handle.close();
|
|
309
|
+
this.#readHandles.delete(extent);
|
|
310
|
+
}
|
|
311
|
+
}
|
|
312
|
+
#readFromHandle(placement, handle) {
|
|
313
|
+
const end = placement.offset + placement.length;
|
|
314
|
+
const size = handle.getSize();
|
|
315
|
+
if (end > size) {
|
|
316
|
+
throw new Error(`Extent placement exceeds its file: ${String(placement.extent)}:${String(placement.offset)}+${String(placement.length)} > ${String(size)}`);
|
|
317
|
+
}
|
|
318
|
+
const bytes = new Uint8Array(placement.length);
|
|
319
|
+
readFully(handle, bytes, placement.offset, `reading extent ${String(placement.extent)} placement`);
|
|
320
|
+
return bytes;
|
|
321
|
+
}
|
|
322
|
+
release(placements) {
|
|
323
|
+
const decrements = /* @__PURE__ */ new Map();
|
|
324
|
+
const ranges = /* @__PURE__ */ new Map();
|
|
325
|
+
for (const placement of placements) {
|
|
326
|
+
assertValidPlacement(placement);
|
|
327
|
+
decrements.set(placement.extent, (decrements.get(placement.extent) ?? 0) + placement.length);
|
|
328
|
+
if (placement.length > 0) {
|
|
329
|
+
const extentRanges = ranges.get(placement.extent) ?? [];
|
|
330
|
+
extentRanges.push({ start: placement.offset, end: placement.offset + placement.length });
|
|
331
|
+
ranges.set(placement.extent, extentRanges);
|
|
332
|
+
}
|
|
333
|
+
}
|
|
334
|
+
for (const [id, extentRanges] of ranges) {
|
|
335
|
+
extentRanges.sort((left, right) => left.start - right.start || left.end - right.end);
|
|
336
|
+
for (let index = 1; index < extentRanges.length; index += 1) {
|
|
337
|
+
const previous = extentRanges[index - 1];
|
|
338
|
+
const current = extentRanges[index];
|
|
339
|
+
if (previous !== void 0 && current !== void 0 && current.start < previous.end) {
|
|
340
|
+
throw new Error(`Extent ${String(id)} release contains overlapping or duplicate placements: ${String(previous.start)}..${String(previous.end)} and ${String(current.start)}..${String(current.end)}`);
|
|
341
|
+
}
|
|
342
|
+
}
|
|
343
|
+
}
|
|
344
|
+
for (const [id, decrement] of decrements) {
|
|
345
|
+
const live = this.#liveBytes.get(id);
|
|
346
|
+
if (live === void 0 || decrement > live) {
|
|
347
|
+
throw new Error(`Extent ${String(id)} live-byte accounting underflow: release ${String(decrement)} from ${live === void 0 ? "an unknown extent" : String(live)}`);
|
|
348
|
+
}
|
|
349
|
+
}
|
|
350
|
+
for (const [id, decrement] of decrements) {
|
|
351
|
+
this.#rememberLiveBytes(id);
|
|
352
|
+
this.#liveBytes.set(id, (this.#liveBytes.get(id) ?? 0) - decrement);
|
|
353
|
+
}
|
|
354
|
+
const deletable = [];
|
|
355
|
+
for (const [id, bytes] of this.#liveBytes) {
|
|
356
|
+
if (bytes === 0 && id !== this.#tailId)
|
|
357
|
+
deletable.push(id);
|
|
358
|
+
}
|
|
359
|
+
return deletable;
|
|
360
|
+
}
|
|
361
|
+
async deleteExtent(id) {
|
|
362
|
+
if (this.#activeBatch !== void 0) {
|
|
363
|
+
throw new Error("Cannot delete an extent while an unpublished batch is active");
|
|
364
|
+
}
|
|
365
|
+
if (id === this.#tailId)
|
|
366
|
+
return;
|
|
367
|
+
this.#deletingExtents.add(id);
|
|
368
|
+
try {
|
|
369
|
+
const opening = this.#openingReadHandles.get(id);
|
|
370
|
+
if (opening !== void 0)
|
|
371
|
+
await opening.promise.catch(() => void 0);
|
|
372
|
+
const cached = this.#readHandles.get(id);
|
|
373
|
+
if (cached !== void 0 && cached.readers > 0) {
|
|
374
|
+
await new Promise((resolve) => cached.drained.push(resolve));
|
|
375
|
+
}
|
|
376
|
+
this.#readHandles.get(id)?.handle.close();
|
|
377
|
+
this.#readHandles.delete(id);
|
|
378
|
+
await this.#tree.deleteFile(extentPath(id));
|
|
379
|
+
this.#liveBytes.delete(id);
|
|
380
|
+
} finally {
|
|
381
|
+
this.#deletingExtents.delete(id);
|
|
382
|
+
}
|
|
383
|
+
}
|
|
384
|
+
meta() {
|
|
385
|
+
return {
|
|
386
|
+
nextExtentId: this.#nextId,
|
|
387
|
+
tailExtentId: this.#tailId,
|
|
388
|
+
tailOffset: this.#tailOffset,
|
|
389
|
+
liveBytes: [...this.#liveBytes.entries()]
|
|
390
|
+
};
|
|
391
|
+
}
|
|
392
|
+
flush() {
|
|
393
|
+
if (!this.#tailDirty)
|
|
394
|
+
return;
|
|
395
|
+
this.#tailHandle.flush();
|
|
396
|
+
this.#tailDirty = false;
|
|
397
|
+
}
|
|
398
|
+
close() {
|
|
399
|
+
this.#closed = true;
|
|
400
|
+
this.#tailHandle.close();
|
|
401
|
+
for (const { handle } of this.#readHandles.values())
|
|
402
|
+
handle.close();
|
|
403
|
+
this.#readHandles.clear();
|
|
404
|
+
}
|
|
405
|
+
#rememberLiveBytes(id) {
|
|
406
|
+
const batch = this.#activeBatch;
|
|
407
|
+
if (batch === void 0 || batch._liveBytesBefore.has(id))
|
|
408
|
+
return;
|
|
409
|
+
batch._liveBytesBefore.set(id, this.#liveBytes.get(id));
|
|
410
|
+
}
|
|
468
411
|
}
|
|
469
|
-
|
|
470
|
-
|
|
412
|
+
function extentPath(id) {
|
|
413
|
+
return ["extents", String(id).padStart(6, "0")];
|
|
471
414
|
}
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
placement.extent >= 0 &&
|
|
475
|
-
placement.extent < Number.MAX_SAFE_INTEGER &&
|
|
476
|
-
Number.isSafeInteger(placement.offset) &&
|
|
477
|
-
placement.offset >= 0 &&
|
|
478
|
-
Number.isSafeInteger(placement.length) &&
|
|
479
|
-
placement.length >= 0 &&
|
|
480
|
-
Number.isSafeInteger(placement.checksum) &&
|
|
481
|
-
placement.checksum >= 0 &&
|
|
482
|
-
placement.checksum <= 0xffff_ffff &&
|
|
483
|
-
placement.offset + placement.length <= Number.MAX_SAFE_INTEGER);
|
|
415
|
+
function validPlacement(placement) {
|
|
416
|
+
return Number.isSafeInteger(placement.extent) && placement.extent >= 0 && placement.extent < Number.MAX_SAFE_INTEGER && Number.isSafeInteger(placement.offset) && placement.offset >= 0 && Number.isSafeInteger(placement.length) && placement.length >= 0 && Number.isSafeInteger(placement.checksum) && placement.checksum >= 0 && placement.checksum <= 4294967295 && placement.offset + placement.length <= Number.MAX_SAFE_INTEGER;
|
|
484
417
|
}
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
}
|
|
418
|
+
function assertValidPlacement(placement) {
|
|
419
|
+
if (!validPlacement(placement)) {
|
|
420
|
+
throw new TypeError(`Invalid extent placement: ${String(placement.extent)}:${String(placement.offset)}+${String(placement.length)} checksum ${String(placement.checksum)}`);
|
|
421
|
+
}
|
|
490
422
|
}
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
for (const entry of meta.liveBytes) {
|
|
504
|
-
const [id, bytes] = entry;
|
|
505
|
-
if (!Number.isSafeInteger(id) ||
|
|
506
|
-
id < 0 ||
|
|
507
|
-
id >= meta.nextExtentId ||
|
|
508
|
-
ids.has(id) ||
|
|
509
|
-
!Number.isSafeInteger(bytes) ||
|
|
510
|
-
bytes < 0) {
|
|
511
|
-
throw new TypeError(`Invalid OPFS extent live-byte entry: ${String(id)} => ${String(bytes)}`);
|
|
512
|
-
}
|
|
513
|
-
ids.add(id);
|
|
514
|
-
}
|
|
423
|
+
function assertValidExtentMeta(meta) {
|
|
424
|
+
if (!Number.isSafeInteger(meta.nextExtentId) || meta.nextExtentId < 1 || !Number.isSafeInteger(meta.tailExtentId) || meta.tailExtentId < 0 || meta.tailExtentId >= meta.nextExtentId || !Number.isSafeInteger(meta.tailOffset) || meta.tailOffset < 0 || !Array.isArray(meta.liveBytes)) {
|
|
425
|
+
throw new TypeError("Invalid OPFS extent checkpoint metadata");
|
|
426
|
+
}
|
|
427
|
+
const ids = /* @__PURE__ */ new Set();
|
|
428
|
+
for (const entry of meta.liveBytes) {
|
|
429
|
+
const [id, bytes] = entry;
|
|
430
|
+
if (!Number.isSafeInteger(id) || id < 0 || id >= meta.nextExtentId || ids.has(id) || !Number.isSafeInteger(bytes) || bytes < 0) {
|
|
431
|
+
throw new TypeError(`Invalid OPFS extent live-byte entry: ${String(id)} => ${String(bytes)}`);
|
|
432
|
+
}
|
|
433
|
+
ids.add(id);
|
|
434
|
+
}
|
|
515
435
|
}
|
|
516
436
|
function isHandlePromise(value) {
|
|
517
|
-
|
|
437
|
+
return typeof value.then === "function";
|
|
518
438
|
}
|
|
519
439
|
function assertValidExtentCoordinate(value, label) {
|
|
520
|
-
|
|
521
|
-
|
|
440
|
+
if (!Number.isSafeInteger(value) || value < 0)
|
|
441
|
+
throw new RangeError(`${label} is out of range`);
|
|
522
442
|
}
|
|
523
443
|
function safeExtentSuccessor(value, label) {
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
444
|
+
if (!Number.isSafeInteger(value) || value < 0 || value >= Number.MAX_SAFE_INTEGER) {
|
|
445
|
+
throw new RangeError(`${label} cannot exceed the safe integer range`);
|
|
446
|
+
}
|
|
447
|
+
return value + 1;
|
|
528
448
|
}
|
|
529
449
|
function safeExtentSum(left, right, label) {
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
left > Number.MAX_SAFE_INTEGER - right) {
|
|
535
|
-
throw new RangeError(`${label} cannot exceed the safe integer range`);
|
|
536
|
-
}
|
|
537
|
-
return left + right;
|
|
450
|
+
if (!Number.isSafeInteger(left) || left < 0 || !Number.isSafeInteger(right) || right < 0 || left > Number.MAX_SAFE_INTEGER - right) {
|
|
451
|
+
throw new RangeError(`${label} cannot exceed the safe integer range`);
|
|
452
|
+
}
|
|
453
|
+
return left + right;
|
|
538
454
|
}
|
|
455
|
+
export {
|
|
456
|
+
ExtentBatchMark,
|
|
457
|
+
ExtentPool,
|
|
458
|
+
assertValidExtentMeta,
|
|
459
|
+
assertValidPlacement,
|
|
460
|
+
extentPath,
|
|
461
|
+
validPlacement
|
|
462
|
+
};
|