@minnowdb/core 0.9.0 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/engine/auto-store.d.ts +40 -0
- package/dist/engine/auto-store.js +115 -0
- package/dist/engine/buffered-writer.d.ts +2 -0
- package/dist/engine/buffered-writer.js +15 -2
- package/dist/engine/client.d.ts +55 -6
- package/dist/engine/client.js +155 -40
- package/dist/engine/database.d.ts +15 -1
- package/dist/engine/database.js +1085 -227
- package/dist/engine/errors.d.ts +61 -2
- package/dist/engine/errors.js +116 -3
- package/dist/engine/index.d.ts +1 -0
- package/dist/engine/index.js +2 -0
- package/dist/engine/live.d.ts +24 -1
- package/dist/engine/live.js +33 -9
- package/dist/engine/scope-write-set.js +36 -0
- package/dist/engine/worker-auto.d.ts +1 -0
- package/dist/engine/worker-auto.js +3 -0
- package/dist/engine/worker-host.d.ts +2 -1
- package/dist/engine/worker-host.js +17 -1
- package/dist/engine/worker-server.d.ts +53 -1
- package/dist/engine/worker-server.js +118 -14
- package/dist/engine/worker-store-auto.js +36 -0
- package/dist/engine/worker-store-opfs.js +3 -2
- package/dist/engine/write-coordinator.js +24 -2
- package/dist/storage/indexeddb.js +494 -207
- package/dist/storage/opfs/leader.js +201 -14
- package/dist/storage/opfs/rpc.js +23 -43
- package/dist/storage/opfs/store.d.ts +20 -0
- package/dist/storage/opfs/store.js +501 -70
- package/dist/storage/toolkit/record-core.js +67 -38
- package/dist/storage/toolkit/wire.d.ts +1 -1
- package/dist/storage/toolkit/wire.js +1 -1
- package/dist/storage/types.d.ts +29 -8
- package/dist/storage/types.js +27 -16
- package/dist/testing/opfs-shim.js +14 -6
- package/dist/transactions/index.d.ts +12 -0
- package/dist/transactions/index.js +83 -23
- package/dist/worker-protocol/index.d.ts +50 -2
- package/dist/worker-protocol/index.js +106 -4
- package/package.json +7 -2
|
@@ -3,6 +3,7 @@ import { dateIsoString } from "../../date-value.js";
|
|
|
3
3
|
import { decodeSnapshotMetadataItems, encodeSnapshotMetadataPage, extendSnapshotFrameStreamChecksum, prepareSnapshotFrameStreamHeader, snapshotFrameEnvelopeParts, snapshotFrameStreamHeaderIdentity } from "../snapshot-stream.js";
|
|
4
4
|
import { RecordCore, validateBeginTransactionInput, validateBlockWriteBytes, validateFtsBaseInput, validateFtsPostingChunks, validateId, validateTempRunPage, validateTempRunPageIdentity } from "../toolkit/record-core.js";
|
|
5
5
|
import { encodeSegment, decodeSegment } from "./files.js";
|
|
6
|
+
import { estimateRpcValueBytes } from "./rpc.js";
|
|
6
7
|
import { SnapshotFrameLedger, snapshotLedgerPath } from "./snapshot-ledger.js";
|
|
7
8
|
import { decodePostingChunk, decodeSyncCheckpoint, encodePostingChunk, encodeSyncCheckpoint } from "../toolkit/wire.js";
|
|
8
9
|
import { WalWriter, iterateWalFrames } from "../toolkit/wal.js";
|
|
@@ -20,6 +21,10 @@ const FTS_CHUNK_CACHE_SIZE = 64;
|
|
|
20
21
|
const MAX_RELOCATION_PAYLOADS = 256;
|
|
21
22
|
const SNAPSHOT_POSTING_TERMS_PER_CHUNK = 128;
|
|
22
23
|
const TEMP_FILE_CLEANUP_DEBT_FLOOR_BYTES = 64 * 1024;
|
|
24
|
+
const MAX_SERVED_LEDGER_ENTRIES = 65536;
|
|
25
|
+
const MAX_SERVED_LEDGER_AGE_MS = 10 * 60 * 1e3;
|
|
26
|
+
const MAX_SERVED_LEDGER_RESULT_BYTES = 64 * 1024;
|
|
27
|
+
const MAX_SERVED_LEDGER_BYTES = 8 * 1024 * 1024;
|
|
23
28
|
const CORE_READ_METHODS = [
|
|
24
29
|
"getTempOwner",
|
|
25
30
|
"getTable",
|
|
@@ -136,7 +141,20 @@ class OpfsLeader {
|
|
|
136
141
|
#lastCleanupError;
|
|
137
142
|
#cleanupRetryScheduled = false;
|
|
138
143
|
#cleanupDebtBytes = 0;
|
|
139
|
-
|
|
144
|
+
#onDiagnostic;
|
|
145
|
+
servingRequest;
|
|
146
|
+
#servedLedger = /* @__PURE__ */ new Map();
|
|
147
|
+
#servedResultBytes = /* @__PURE__ */ new Map();
|
|
148
|
+
#servedLedgerBytes = 0;
|
|
149
|
+
#servedLedgerResultBytes;
|
|
150
|
+
#servedCoverageSince = 0;
|
|
151
|
+
#servedLedgerAgeMs;
|
|
152
|
+
onBeforeCheckpoint;
|
|
153
|
+
#lastCheckpointMs = 0;
|
|
154
|
+
constructor(tree, strict, walHandle, slots, checkpointEntries, cleanupLimitBytes, onDiagnostic, servedLedgerAgeMs, servedLedgerResultBytes) {
|
|
155
|
+
this.#onDiagnostic = onDiagnostic;
|
|
156
|
+
this.#servedLedgerAgeMs = servedLedgerAgeMs ?? MAX_SERVED_LEDGER_AGE_MS;
|
|
157
|
+
this.#servedLedgerResultBytes = servedLedgerResultBytes ?? MAX_SERVED_LEDGER_RESULT_BYTES;
|
|
140
158
|
this.#checkpointEntries = checkpointEntries ?? CHECKPOINT_ENTRIES;
|
|
141
159
|
this.#cleanupLimitBytes = cleanupLimitBytes ?? MAX_OPFS_CLEANUP_DEBT_BYTES;
|
|
142
160
|
if (!Number.isSafeInteger(this.#cleanupLimitBytes) || this.#cleanupLimitBytes < 1) {
|
|
@@ -153,8 +171,8 @@ class OpfsLeader {
|
|
|
153
171
|
blockChecksum: (id) => this.#blockIndex.get(id)?.checksum ?? (this.#permissivePhysical ? 0 : void 0)
|
|
154
172
|
});
|
|
155
173
|
}
|
|
156
|
-
static async recover(tree, strict, handles, checkpointEntries, cleanupLimitBytes) {
|
|
157
|
-
const leader = new OpfsLeader(tree, strict, handles.wal, [handles.slotA, handles.slotB], checkpointEntries, cleanupLimitBytes);
|
|
174
|
+
static async recover(tree, strict, handles, checkpointEntries, cleanupLimitBytes, onDiagnostic, servedLedgerAgeMs, servedLedgerResultBytes) {
|
|
175
|
+
const leader = new OpfsLeader(tree, strict, handles.wal, [handles.slotA, handles.slotB], checkpointEntries, cleanupLimitBytes, onDiagnostic, servedLedgerAgeMs, servedLedgerResultBytes);
|
|
158
176
|
try {
|
|
159
177
|
await leader.#loadFromDisk();
|
|
160
178
|
} catch (error) {
|
|
@@ -270,6 +288,16 @@ class OpfsLeader {
|
|
|
270
288
|
this.#applyTempPageUpdates(checkpoint.tempPages);
|
|
271
289
|
this.#seq = checkpoint.lastSeq;
|
|
272
290
|
this.#checkpointGeneration = checkpoint.generation;
|
|
291
|
+
this.#servedLedger.clear();
|
|
292
|
+
this.#servedResultBytes.clear();
|
|
293
|
+
this.#servedLedgerBytes = 0;
|
|
294
|
+
this.#servedCoverageSince = checkpoint.servedCoverageSince;
|
|
295
|
+
for (const outcome of checkpoint.servedRequests) {
|
|
296
|
+
this.#servedLedger.set(outcome.key, { ...outcome });
|
|
297
|
+
if (outcome.settled && outcome.withheld !== true) {
|
|
298
|
+
this.#retainServedResult(outcome.key, estimateRpcValueBytes(outcome.result));
|
|
299
|
+
}
|
|
300
|
+
}
|
|
273
301
|
}
|
|
274
302
|
const newest = this.#slots[this.#newestSlot];
|
|
275
303
|
this.#lastCheckpointBytes = newest === void 0 ? 0 : newest.getSize();
|
|
@@ -328,6 +356,21 @@ class OpfsLeader {
|
|
|
328
356
|
}
|
|
329
357
|
applied += 1;
|
|
330
358
|
this.#applyReplayed(entry);
|
|
359
|
+
if (entry.request !== void 0) {
|
|
360
|
+
this.#recordServed({ ...entry.request, seq: entry.seq, settled: false });
|
|
361
|
+
}
|
|
362
|
+
if (entry.op === "servedResult") {
|
|
363
|
+
const served = this.#servedLedger.get(entry.key);
|
|
364
|
+
if (served !== void 0) {
|
|
365
|
+
served.settled = true;
|
|
366
|
+
if (entry.withheld === true)
|
|
367
|
+
served.withheld = true;
|
|
368
|
+
else {
|
|
369
|
+
served.result = entry.result;
|
|
370
|
+
this.#retainServedResult(entry.key, estimateRpcValueBytes(entry.result));
|
|
371
|
+
}
|
|
372
|
+
}
|
|
373
|
+
}
|
|
331
374
|
for (const placement of placementsOf(entry)) {
|
|
332
375
|
this.#pool.restorePlacement(placement);
|
|
333
376
|
}
|
|
@@ -395,6 +438,65 @@ class OpfsLeader {
|
|
|
395
438
|
return;
|
|
396
439
|
await this.#run(() => void 0);
|
|
397
440
|
}
|
|
441
|
+
#recordServed(outcome) {
|
|
442
|
+
this.#forgetServedResult(outcome.key);
|
|
443
|
+
this.#servedLedger.delete(outcome.key);
|
|
444
|
+
this.#servedLedger.set(outcome.key, outcome);
|
|
445
|
+
this.#trimServedLedger(outcome.sentAt - this.#servedLedgerAgeMs);
|
|
446
|
+
}
|
|
447
|
+
#trimServedLedger(horizon) {
|
|
448
|
+
for (const [key, entry] of this.#servedLedger) {
|
|
449
|
+
if (this.#servedLedger.size <= MAX_SERVED_LEDGER_ENTRIES && this.#servedLedgerBytes <= MAX_SERVED_LEDGER_BYTES && entry.sentAt >= horizon) {
|
|
450
|
+
break;
|
|
451
|
+
}
|
|
452
|
+
this.#forgetServedResult(key);
|
|
453
|
+
this.#servedLedger.delete(key);
|
|
454
|
+
this.#servedCoverageSince = Math.max(this.#servedCoverageSince, entry.sentAt + 1);
|
|
455
|
+
}
|
|
456
|
+
}
|
|
457
|
+
#retainServedResult(key, bytes) {
|
|
458
|
+
this.#forgetServedResult(key);
|
|
459
|
+
this.#servedResultBytes.set(key, bytes);
|
|
460
|
+
this.#servedLedgerBytes += bytes;
|
|
461
|
+
}
|
|
462
|
+
#forgetServedResult(key) {
|
|
463
|
+
const bytes = this.#servedResultBytes.get(key);
|
|
464
|
+
if (bytes === void 0)
|
|
465
|
+
return;
|
|
466
|
+
this.#servedResultBytes.delete(key);
|
|
467
|
+
this.#servedLedgerBytes -= bytes;
|
|
468
|
+
}
|
|
469
|
+
servedOutcome(key) {
|
|
470
|
+
return this.#servedLedger.get(key);
|
|
471
|
+
}
|
|
472
|
+
async completeServed(key, result) {
|
|
473
|
+
const entry = this.#servedLedger.get(key);
|
|
474
|
+
if (entry === void 0 || entry.settled)
|
|
475
|
+
return;
|
|
476
|
+
const bytes = estimateRpcValueBytes(result);
|
|
477
|
+
await this.#run(() => {
|
|
478
|
+
if (bytes > this.#servedLedgerResultBytes) {
|
|
479
|
+
this.#appendFrame({ op: "servedResult", key, withheld: true }, false);
|
|
480
|
+
entry.settled = true;
|
|
481
|
+
entry.withheld = true;
|
|
482
|
+
return;
|
|
483
|
+
}
|
|
484
|
+
this.#appendFrame({ op: "servedResult", key, result }, false);
|
|
485
|
+
entry.result = result;
|
|
486
|
+
entry.settled = true;
|
|
487
|
+
this.#retainServedResult(key, bytes);
|
|
488
|
+
this.#trimServedLedger(Number.NEGATIVE_INFINITY);
|
|
489
|
+
});
|
|
490
|
+
}
|
|
491
|
+
get servedCoverageSince() {
|
|
492
|
+
return this.#servedCoverageSince;
|
|
493
|
+
}
|
|
494
|
+
#diagnostic(error, context) {
|
|
495
|
+
try {
|
|
496
|
+
this.#onDiagnostic?.(error, context);
|
|
497
|
+
} catch {
|
|
498
|
+
}
|
|
499
|
+
}
|
|
398
500
|
#logged(body) {
|
|
399
501
|
if (this.#wal.byteLength >= MAX_OPFS_WAL_BYTES - 64 * 1024 * 1024) {
|
|
400
502
|
this.checkpointNow();
|
|
@@ -414,16 +516,21 @@ class OpfsLeader {
|
|
|
414
516
|
this.#appendFrame(body);
|
|
415
517
|
return result;
|
|
416
518
|
}
|
|
417
|
-
#appendFrame(body) {
|
|
519
|
+
#appendFrame(body, flush = this.#strict) {
|
|
520
|
+
const request = this.servingRequest;
|
|
521
|
+
this.servingRequest = void 0;
|
|
522
|
+
let nextSeq;
|
|
418
523
|
try {
|
|
419
|
-
|
|
420
|
-
this.#wal.append({ seq: nextSeq, ...body },
|
|
524
|
+
nextSeq = safeSuccessor(this.#seq, "OPFS WAL sequence");
|
|
525
|
+
this.#wal.append({ seq: nextSeq, ...body, ...request === void 0 ? {} : { request } }, flush);
|
|
421
526
|
this.#seq = nextSeq;
|
|
422
527
|
this.#entriesSinceCheckpoint = safeSuccessor(this.#entriesSinceCheckpoint, "OPFS checkpoint entry count");
|
|
423
528
|
} catch (error) {
|
|
424
529
|
this.#poisoned = true;
|
|
425
530
|
throw error;
|
|
426
531
|
}
|
|
532
|
+
if (request !== void 0)
|
|
533
|
+
this.#recordServed({ ...request, seq: nextSeq, settled: false });
|
|
427
534
|
if (this.#checkpointDue() && this.#entriesSinceCheckpoint >= this.#checkpointRetryAtEntries && !this.#checkpointScheduled) {
|
|
428
535
|
this.#checkpointScheduled = true;
|
|
429
536
|
void this.#run(() => {
|
|
@@ -431,7 +538,9 @@ class OpfsLeader {
|
|
|
431
538
|
if (this.#checkpointDue()) {
|
|
432
539
|
this.checkpointNow();
|
|
433
540
|
}
|
|
434
|
-
}).catch(() =>
|
|
541
|
+
}).catch((error) => {
|
|
542
|
+
this.#diagnostic(error, "opfs checkpoint");
|
|
543
|
+
});
|
|
435
544
|
}
|
|
436
545
|
}
|
|
437
546
|
#checkpointDue() {
|
|
@@ -937,6 +1046,8 @@ class OpfsLeader {
|
|
|
937
1046
|
this.#snapshotFrameImport = void 0;
|
|
938
1047
|
return void 0;
|
|
939
1048
|
}
|
|
1049
|
+
case "servedResult":
|
|
1050
|
+
return void 0;
|
|
940
1051
|
case "cancelSnapshotFrameImport": {
|
|
941
1052
|
const session = this.#snapshotFrameImport;
|
|
942
1053
|
if (session?.identity !== body.input.identity) {
|
|
@@ -1085,8 +1196,9 @@ class OpfsLeader {
|
|
|
1085
1196
|
#applyReplayed(entry) {
|
|
1086
1197
|
this.#permissivePhysical = true;
|
|
1087
1198
|
try {
|
|
1088
|
-
const { seq: _seq, ...body } = entry;
|
|
1199
|
+
const { seq: _seq, request: _request, ...body } = entry;
|
|
1089
1200
|
void _seq;
|
|
1201
|
+
void _request;
|
|
1090
1202
|
this.#applyBody(body);
|
|
1091
1203
|
this.#clearCompletedSnapshotImportIfAdvanced();
|
|
1092
1204
|
} finally {
|
|
@@ -1416,7 +1528,10 @@ class OpfsLeader {
|
|
|
1416
1528
|
}
|
|
1417
1529
|
checkpointNow() {
|
|
1418
1530
|
try {
|
|
1531
|
+
this.onBeforeCheckpoint?.(this.#lastCheckpointMs);
|
|
1532
|
+
const started = Date.now();
|
|
1419
1533
|
this.#checkpointNowUnchecked();
|
|
1534
|
+
this.#lastCheckpointMs = Date.now() - started;
|
|
1420
1535
|
this.#checkpointFailures = 0;
|
|
1421
1536
|
this.#lastCheckpointError = void 0;
|
|
1422
1537
|
this.#checkpointRetryAtEntries = 0;
|
|
@@ -1448,7 +1563,9 @@ class OpfsLeader {
|
|
|
1448
1563
|
...this.#completedSnapshotFrameImport === void 0 ? {} : {
|
|
1449
1564
|
completedSnapshotFrameImport: structuredClone(this.#completedSnapshotFrameImport)
|
|
1450
1565
|
},
|
|
1451
|
-
extents: this.#pool.meta()
|
|
1566
|
+
extents: this.#pool.meta(),
|
|
1567
|
+
servedRequests: [...this.#servedLedger.values()],
|
|
1568
|
+
servedCoverageSince: this.#servedCoverageSince
|
|
1452
1569
|
};
|
|
1453
1570
|
const bytes = encodeSyncCheckpoint(state);
|
|
1454
1571
|
if (bytes.byteLength > MAX_OPFS_CHECKPOINT_BYTES) {
|
|
@@ -1532,6 +1649,7 @@ class OpfsLeader {
|
|
|
1532
1649
|
} catch (error) {
|
|
1533
1650
|
this.#cleanupFailures += 1;
|
|
1534
1651
|
this.#lastCleanupError = error;
|
|
1652
|
+
this.#diagnostic(error, "opfs cleanup");
|
|
1535
1653
|
await this.#refreshCleanupDebtAfterFailure();
|
|
1536
1654
|
}
|
|
1537
1655
|
if (this.#cleanupRetryScheduled || this.#closed)
|
|
@@ -1547,9 +1665,12 @@ class OpfsLeader {
|
|
|
1547
1665
|
} catch (error) {
|
|
1548
1666
|
this.#cleanupFailures += 1;
|
|
1549
1667
|
this.#lastCleanupError = error;
|
|
1668
|
+
this.#diagnostic(error, "opfs cleanup retry");
|
|
1550
1669
|
await this.#refreshCleanupDebtAfterFailure();
|
|
1551
1670
|
}
|
|
1552
|
-
}).catch(() =>
|
|
1671
|
+
}).catch((error) => {
|
|
1672
|
+
this.#diagnostic(error, "opfs cleanup retry");
|
|
1673
|
+
});
|
|
1553
1674
|
}
|
|
1554
1675
|
async #refreshCleanupDebtAfterFailure() {
|
|
1555
1676
|
try {
|
|
@@ -3228,11 +3349,15 @@ class OpfsLeader {
|
|
|
3228
3349
|
}
|
|
3229
3350
|
};
|
|
3230
3351
|
}
|
|
3352
|
+
isClosed() {
|
|
3353
|
+
return this.#closed;
|
|
3354
|
+
}
|
|
3231
3355
|
async shutdown() {
|
|
3232
3356
|
await this.#run(() => {
|
|
3233
3357
|
this.checkpointNow();
|
|
3234
3358
|
this.#closed = true;
|
|
3235
|
-
}).catch(() => {
|
|
3359
|
+
}).catch((error) => {
|
|
3360
|
+
this.#diagnostic(error, "opfs shutdown checkpoint");
|
|
3236
3361
|
this.#closed = true;
|
|
3237
3362
|
});
|
|
3238
3363
|
this.#walHandle.close();
|
|
@@ -3347,7 +3472,8 @@ const WAL_BODY_KEYS = {
|
|
|
3347
3472
|
renewSnapshotFrameImport: ["input"],
|
|
3348
3473
|
appendSnapshotImportFrames: ["input", "state", "blockPlacements", "replay"],
|
|
3349
3474
|
finishSnapshotFrameImport: ["input"],
|
|
3350
|
-
cancelSnapshotFrameImport: ["input"]
|
|
3475
|
+
cancelSnapshotFrameImport: ["input"],
|
|
3476
|
+
servedResult: ["key", "result", "withheld"]
|
|
3351
3477
|
};
|
|
3352
3478
|
function validateWalEntry(value) {
|
|
3353
3479
|
if (!isRecord(value))
|
|
@@ -3543,6 +3669,14 @@ function validateWalEntry(value) {
|
|
|
3543
3669
|
requireTimestamp(entry.input.expiresAtCutoff, "finishSnapshotFrameImport cutoff");
|
|
3544
3670
|
requireRecord(entry.input.footer, "finishSnapshotFrameImport footer");
|
|
3545
3671
|
break;
|
|
3672
|
+
case "servedResult":
|
|
3673
|
+
if (typeof entry.key !== "string" || entry.key.length === 0 || entry.key.length > 4096) {
|
|
3674
|
+
throw new Error("servedResult key is invalid");
|
|
3675
|
+
}
|
|
3676
|
+
if (Object.hasOwn(entry, "withheld") && (entry.withheld !== true || Object.hasOwn(entry, "result"))) {
|
|
3677
|
+
throw new Error("servedResult withholding is invalid");
|
|
3678
|
+
}
|
|
3679
|
+
break;
|
|
3546
3680
|
case "cancelSnapshotFrameImport":
|
|
3547
3681
|
requireRecord(entry.input, "cancelSnapshotFrameImport input");
|
|
3548
3682
|
assertExactRecordKeys(entry.input, ["identity", "ownerId"], "cancelSnapshotFrameImport input");
|
|
@@ -3570,11 +3704,46 @@ function validateIdPlacements(value, label) {
|
|
|
3570
3704
|
}
|
|
3571
3705
|
}
|
|
3572
3706
|
function assertWalKeys(value, bodyKeys) {
|
|
3573
|
-
const allowed = /* @__PURE__ */ new Set(["seq", "op", ...bodyKeys]);
|
|
3707
|
+
const allowed = /* @__PURE__ */ new Set(["seq", "op", ...bodyKeys, "request"]);
|
|
3574
3708
|
for (const key of Object.keys(value)) {
|
|
3575
3709
|
if (!allowed.has(key))
|
|
3576
3710
|
throw new Error(`Unexpected ${String(value.op)} WAL field: ${key}`);
|
|
3577
3711
|
}
|
|
3712
|
+
if (value.request !== void 0)
|
|
3713
|
+
validateServedRequest(value.request);
|
|
3714
|
+
}
|
|
3715
|
+
function validateServedRequest(value) {
|
|
3716
|
+
if (!isRecord(value))
|
|
3717
|
+
throw new Error("OPFS WAL served request is not an object");
|
|
3718
|
+
assertExactRecordKeys(value, ["key", "method", "signature", "requestBytes", "sentAt"], "OPFS WAL served request");
|
|
3719
|
+
if (typeof value.key !== "string" || value.key.length === 0 || value.key.length > 4096) {
|
|
3720
|
+
throw new Error("OPFS WAL served request has an invalid key");
|
|
3721
|
+
}
|
|
3722
|
+
if (typeof value.method !== "string" || value.method.length === 0) {
|
|
3723
|
+
throw new Error("OPFS WAL served request has an invalid method");
|
|
3724
|
+
}
|
|
3725
|
+
if (typeof value.signature !== "string" || value.signature.length === 0) {
|
|
3726
|
+
throw new Error("OPFS WAL served request has an invalid signature");
|
|
3727
|
+
}
|
|
3728
|
+
if (!isNonNegativeSafeInteger(value.requestBytes) || !isNonNegativeSafeInteger(value.sentAt)) {
|
|
3729
|
+
throw new Error("OPFS WAL served request has invalid numbers");
|
|
3730
|
+
}
|
|
3731
|
+
}
|
|
3732
|
+
function validateServedOutcome(value) {
|
|
3733
|
+
if (!isRecord(value))
|
|
3734
|
+
throw new Error("OPFS served outcome is not an object");
|
|
3735
|
+
const { seq, settled, withheld, result: _result, ...request } = value;
|
|
3736
|
+
void _result;
|
|
3737
|
+
validateServedRequest(request);
|
|
3738
|
+
if (!Number.isSafeInteger(seq) || seq < 1) {
|
|
3739
|
+
throw new Error("OPFS served outcome has an invalid sequence");
|
|
3740
|
+
}
|
|
3741
|
+
if (typeof settled !== "boolean" || !settled && Object.hasOwn(value, "result")) {
|
|
3742
|
+
throw new Error("OPFS served outcome has an invalid settlement");
|
|
3743
|
+
}
|
|
3744
|
+
if (withheld !== void 0 && (withheld !== true || !settled || Object.hasOwn(value, "result"))) {
|
|
3745
|
+
throw new Error("OPFS served outcome has an invalid withholding");
|
|
3746
|
+
}
|
|
3578
3747
|
}
|
|
3579
3748
|
function assertExactRecordKeys(value, keys, label) {
|
|
3580
3749
|
const expected = new Set(keys);
|
|
@@ -3757,6 +3926,20 @@ function validateCheckpointState(value) {
|
|
|
3757
3926
|
if (!Array.isArray(value.tempPages)) {
|
|
3758
3927
|
throw new Error("Invalid OPFS checkpoint temp-page ledger");
|
|
3759
3928
|
}
|
|
3929
|
+
if (!Array.isArray(value.servedRequests)) {
|
|
3930
|
+
throw new Error("Invalid OPFS checkpoint served-request ledger");
|
|
3931
|
+
}
|
|
3932
|
+
if (!isNonNegativeSafeInteger(value.servedCoverageSince)) {
|
|
3933
|
+
throw new Error("Invalid OPFS checkpoint served-request coverage");
|
|
3934
|
+
}
|
|
3935
|
+
const seenServed = /* @__PURE__ */ new Set();
|
|
3936
|
+
for (const entry of value.servedRequests) {
|
|
3937
|
+
validateServedOutcome(entry);
|
|
3938
|
+
if (seenServed.has(entry.key)) {
|
|
3939
|
+
throw new Error(`Duplicate OPFS checkpoint served request: ${entry.key}`);
|
|
3940
|
+
}
|
|
3941
|
+
seenServed.add(entry.key);
|
|
3942
|
+
}
|
|
3760
3943
|
assertExactRecordKeys(value, [
|
|
3761
3944
|
"formatVersion",
|
|
3762
3945
|
"generation",
|
|
@@ -3769,7 +3952,9 @@ function validateCheckpointState(value) {
|
|
|
3769
3952
|
...value.snapshotFrameExport === void 0 ? [] : ["snapshotFrameExport"],
|
|
3770
3953
|
...value.snapshotFrameImport === void 0 ? [] : ["snapshotFrameImport"],
|
|
3771
3954
|
...value.completedSnapshotFrameImport === void 0 ? [] : ["completedSnapshotFrameImport"],
|
|
3772
|
-
"extents"
|
|
3955
|
+
"extents",
|
|
3956
|
+
"servedRequests",
|
|
3957
|
+
"servedCoverageSince"
|
|
3773
3958
|
], "OPFS checkpoint");
|
|
3774
3959
|
assertValidExtentMeta(value.extents);
|
|
3775
3960
|
const blockIds = /* @__PURE__ */ new Set();
|
|
@@ -4542,6 +4727,8 @@ function parseTempPageFilePath(path) {
|
|
|
4542
4727
|
}
|
|
4543
4728
|
function placementsOf(entry) {
|
|
4544
4729
|
switch (entry.op) {
|
|
4730
|
+
case "servedResult":
|
|
4731
|
+
return [];
|
|
4545
4732
|
case "stageTransactionArtifacts":
|
|
4546
4733
|
case "writeTransaction":
|
|
4547
4734
|
return entry.blocks.map(({ placement }) => placement);
|
package/dist/storage/opfs/rpc.js
CHANGED
|
@@ -1,6 +1,9 @@
|
|
|
1
|
-
import { BlockReadBatchTooLargeError, CompactionBacklogError, CompactionJobConflictError, GarbageCollectionJobConflictError, IndexedDbSchemaUpgradeBlockedError, LeaseConflictError, LeaseExpiredError, LeaseOwnerConflictError, OpfsCoordinationError, OpfsDatabaseInUseError, OpfsUncertainOutcomeError, PostingBuildConflictError, SnapshotImportConflictError, SnapshotManifestMissingError, StorageCorruptionError, StorageFormatVersionError, StorageResourceLimitError, TableInUseError, TableRecordConflictError, TempOwnerConflictError, TransactionRecordConflictError, UniqueKeyBuildConflictError, UniqueKeyConflictError,
|
|
1
|
+
import { BlockReadBatchTooLargeError, CompactionBacklogError, CompactionJobConflictError, ConnectionLostError, GarbageCollectionJobConflictError, IndexedDbSchemaUpgradeBlockedError, LeaseConflictError, LeaseExpiredError, LeaseOwnerConflictError, OpfsCoordinationError, OpfsDatabaseInUseError, OpfsUncertainOutcomeError, PostingBuildConflictError, SchemaConflictError, SnapshotImportConflictError, SnapshotManifestMissingError, StorageCorruptionError, StorageFormatVersionError, StorageResourceLimitError, TableInUseError, TableRecordConflictError, TempOwnerConflictError, TransactionRecordConflictError, UniqueIndexCoverageError, UniqueKeyBuildConflictError, UniqueKeyConflictError, UnknownOutcomeError, WriteConflictError } from "../types.js";
|
|
2
2
|
import { dateIsoString } from "../../date-value.js";
|
|
3
|
+
import { MAX_SERIALIZED_CAUSE_DEPTH, rehydrateError, serializeError } from "../../worker-protocol/index.js";
|
|
4
|
+
const MAX_OPFS_RPC_STACK_CHARACTERS = 64 * 1024;
|
|
3
5
|
const MAX_OPFS_RPC_MESSAGE_BYTES = 66 * 1024 * 1024;
|
|
6
|
+
const MAX_OPFS_RPC_HOLD_MS = 6e4;
|
|
4
7
|
const MAX_OPFS_RPC_IDENTIFIER_CHARACTERS = 1024;
|
|
5
8
|
const MAX_OPFS_RPC_DEPTH = 64;
|
|
6
9
|
const MAX_OPFS_RPC_NODES = 11e5;
|
|
@@ -66,12 +69,13 @@ function estimateRpcValueBytes(value) {
|
|
|
66
69
|
}
|
|
67
70
|
return bytes;
|
|
68
71
|
}
|
|
69
|
-
function validSerializedError(value) {
|
|
72
|
+
function validSerializedError(value, depth = 0) {
|
|
70
73
|
if (typeof value !== "object" || value === null || Array.isArray(value))
|
|
71
74
|
return false;
|
|
72
75
|
const record = value;
|
|
73
|
-
const
|
|
74
|
-
|
|
76
|
+
const optional = ["stack", "domException", "props", "cause"];
|
|
77
|
+
const allowed = ["name", "message", ...optional.filter((key) => record[key] !== void 0)];
|
|
78
|
+
return exactKeys(record, allowed) && boundedRpcString(record.name) && typeof record.message === "string" && record.message.length <= MAX_OPFS_RPC_IDENTIFIER_CHARACTERS * 4 && (record.stack === void 0 || typeof record.stack === "string" && record.stack.length <= MAX_OPFS_RPC_STACK_CHARACTERS) && (record.domException === void 0 || record.domException === true) && (record.props === void 0 || typeof record.props === "object" && record.props !== null && !Array.isArray(record.props)) && (record.cause === void 0 || depth < MAX_SERIALIZED_CAUSE_DEPTH && validSerializedError(record.cause, depth + 1));
|
|
75
79
|
}
|
|
76
80
|
function parseStoreRpcMessage(value, knownMethods) {
|
|
77
81
|
try {
|
|
@@ -81,7 +85,7 @@ function parseStoreRpcMessage(value, knownMethods) {
|
|
|
81
85
|
estimateRpcValueBytes(record);
|
|
82
86
|
switch (record.kind) {
|
|
83
87
|
case "op":
|
|
84
|
-
if (!exactKeys(record, ["kind", "requestId", "from", "method", "args"]) || !boundedRpcString(record.requestId) || !boundedRpcString(record.from) || !boundedRpcString(record.method) || !knownMethods.has(record.method) || !Array.isArray(record.args) || estimateRpcValueBytes(record.args) > MAX_OPFS_RPC_MESSAGE_BYTES)
|
|
88
|
+
if (!exactKeys(record, ["kind", "requestId", "from", "method", "args", "sentAt"]) || !boundedRpcString(record.requestId) || !boundedRpcString(record.from) || !boundedRpcString(record.method) || !knownMethods.has(record.method) || !Array.isArray(record.args) || !Number.isSafeInteger(record.sentAt) || record.sentAt < 0 || estimateRpcValueBytes(record.args) > MAX_OPFS_RPC_MESSAGE_BYTES)
|
|
85
89
|
return void 0;
|
|
86
90
|
return record;
|
|
87
91
|
case "result":
|
|
@@ -96,6 +100,14 @@ function parseStoreRpcMessage(value, knownMethods) {
|
|
|
96
100
|
return record;
|
|
97
101
|
case "busy":
|
|
98
102
|
return exactKeys(record, ["kind", "requestId"]) && boundedRpcString(record.requestId) ? record : void 0;
|
|
103
|
+
case "hold":
|
|
104
|
+
return exactKeys(record, ["kind", "requestId", "ms"]) && boundedRpcString(record.requestId) && typeof record.ms === "number" && Number.isSafeInteger(record.ms) && record.ms >= 0 && record.ms <= MAX_OPFS_RPC_HOLD_MS ? record : void 0;
|
|
105
|
+
case "uncertain":
|
|
106
|
+
return exactKeys(record, ["kind", "requestId"]) && boundedRpcString(record.requestId) ? record : void 0;
|
|
107
|
+
case "declined":
|
|
108
|
+
return exactKeys(record, ["kind", "requestId", "reason"]) && boundedRpcString(record.requestId) && record.reason === "not-leader" ? record : void 0;
|
|
109
|
+
case "state":
|
|
110
|
+
return exactKeys(record, ["kind", "leaderId", "foreground"]) && boundedRpcString(record.leaderId) && typeof record.foreground === "boolean" ? record : void 0;
|
|
99
111
|
case "leader":
|
|
100
112
|
case "released":
|
|
101
113
|
return exactKeys(record, ["kind", "leaderId"]) && boundedRpcString(record.leaderId) ? record : void 0;
|
|
@@ -192,50 +204,18 @@ const errorRegistry = new Map([
|
|
|
192
204
|
StorageFormatVersionError,
|
|
193
205
|
OpfsCoordinationError,
|
|
194
206
|
OpfsDatabaseInUseError,
|
|
195
|
-
OpfsUncertainOutcomeError
|
|
207
|
+
OpfsUncertainOutcomeError,
|
|
208
|
+
UnknownOutcomeError,
|
|
209
|
+
ConnectionLostError
|
|
196
210
|
].map((constructor) => [constructor.name, constructor]));
|
|
197
211
|
function serializeStoreError(error) {
|
|
198
|
-
|
|
199
|
-
return { name: error.name, message: error.message, domException: true };
|
|
200
|
-
}
|
|
201
|
-
if (!(error instanceof Error))
|
|
202
|
-
return { name: "Error", message: String(error) };
|
|
203
|
-
const props = {};
|
|
204
|
-
for (const key of Object.keys(error)) {
|
|
205
|
-
const value = error[key];
|
|
206
|
-
try {
|
|
207
|
-
structuredClone(value);
|
|
208
|
-
props[key] = value;
|
|
209
|
-
} catch {
|
|
210
|
-
}
|
|
211
|
-
}
|
|
212
|
-
return {
|
|
213
|
-
name: error.name,
|
|
214
|
-
message: error.message,
|
|
215
|
-
...Object.keys(props).length === 0 ? {} : { props }
|
|
216
|
-
};
|
|
212
|
+
return serializeError(error);
|
|
217
213
|
}
|
|
218
214
|
function rehydrateStoreError(serialized) {
|
|
219
|
-
|
|
220
|
-
return new DOMException(serialized.message, serialized.name);
|
|
221
|
-
}
|
|
222
|
-
const constructor = errorRegistry.get(serialized.name);
|
|
223
|
-
const error = constructor === void 0 ? new Error(serialized.message) : Object.create(constructor.prototype);
|
|
224
|
-
Object.defineProperty(error, "message", {
|
|
225
|
-
value: serialized.message,
|
|
226
|
-
writable: true,
|
|
227
|
-
configurable: true
|
|
228
|
-
});
|
|
229
|
-
Object.defineProperty(error, "name", {
|
|
230
|
-
value: serialized.name,
|
|
231
|
-
writable: true,
|
|
232
|
-
configurable: true
|
|
233
|
-
});
|
|
234
|
-
if (serialized.props !== void 0)
|
|
235
|
-
Object.assign(error, serialized.props);
|
|
236
|
-
return error;
|
|
215
|
+
return rehydrateError(serialized, errorRegistry);
|
|
237
216
|
}
|
|
238
217
|
export {
|
|
218
|
+
MAX_OPFS_RPC_HOLD_MS,
|
|
239
219
|
MAX_OPFS_RPC_IDENTIFIER_CHARACTERS,
|
|
240
220
|
MAX_OPFS_RPC_MESSAGE_BYTES,
|
|
241
221
|
estimateRpcValueBytes,
|
|
@@ -15,8 +15,24 @@ export interface OpfsBlockStoreOptions {
|
|
|
15
15
|
checkpointEntries?: number;
|
|
16
16
|
/** @internal Test seam: cleanup-debt backpressure limit (default 64 MiB). */
|
|
17
17
|
cleanupLimitBytes?: number;
|
|
18
|
+
/** @internal Test seam: how long served requests stay answerable (default 10 minutes). */
|
|
19
|
+
servedLedgerAgeMs?: number;
|
|
20
|
+
/** @internal Test seam: the largest served result the ledger retains (default 64 KiB). */
|
|
21
|
+
servedLedgerResultBytes?: number;
|
|
18
22
|
/** @internal Test seam: how long a follower waits for the leader (default 1000ms). */
|
|
19
23
|
rpcTimeoutMs?: number;
|
|
24
|
+
/** @internal Test seam: minimum spacing between foreground yields (default 3000ms). */
|
|
25
|
+
yieldCooldownMs?: number;
|
|
26
|
+
/** @internal Test seam: idle time before a hidden leader releases (default 15000ms). */
|
|
27
|
+
hiddenIdleReleaseMs?: number;
|
|
28
|
+
/** @internal Test seam: how long an ex-leader stays out of elections (default 1500ms). */
|
|
29
|
+
handoverGraceMs?: number;
|
|
30
|
+
/**
|
|
31
|
+
* Hears failures no operation reports: a background checkpoint or cleanup that failed, an
|
|
32
|
+
* election or handover that threw, a served request that could not be answered. The worker
|
|
33
|
+
* host wires it to the client's `onWorkerError`; a direct caller may log it.
|
|
34
|
+
*/
|
|
35
|
+
onDiagnostic?: (error: unknown, context: string) => void;
|
|
20
36
|
}
|
|
21
37
|
/**
|
|
22
38
|
* The OPFS block store: one leader per database holds every file handle and does all storage
|
|
@@ -50,6 +66,8 @@ export declare class OpfsBlockStore {
|
|
|
50
66
|
readonly liveQueryChannelName: string;
|
|
51
67
|
private constructor();
|
|
52
68
|
static open(options: OpfsBlockStoreOptions): Promise<OpfsBlockStore>;
|
|
69
|
+
/** @internal Simulates a leader from a release without keepalives. */
|
|
70
|
+
_suppressKeepaliveForTests(): void;
|
|
53
71
|
/** Marks this connection as the one the user is looking at — a leadership preference. */
|
|
54
72
|
setForeground(foreground: boolean): void;
|
|
55
73
|
putTempRunPage(page: TempRunPage): Promise<void>;
|
|
@@ -69,6 +87,8 @@ export declare class OpfsBlockStore {
|
|
|
69
87
|
_dropNextRpcResultForTests(): void;
|
|
70
88
|
/** Test-only: holds newly admitted served mutations until the returned release is called. */
|
|
71
89
|
_holdServedMutationsForTests(): () => void;
|
|
90
|
+
/** Test-only: the request id of the oldest pending request, for hand-posted answers. */
|
|
91
|
+
_oldestPendingRequestIdForTests(): string | undefined;
|
|
72
92
|
/** Test-only: retransmits the oldest request with its stable deduplication identity. */
|
|
73
93
|
_resendOldestPendingForTests(): void;
|
|
74
94
|
/** Test-only counters that pin the connection's bounded RPC state and close-time cleanup. */
|