@minnowdb/core 0.9.1 → 0.10.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/engine/auto-store.d.ts +52 -0
- package/dist/engine/auto-store.js +157 -0
- package/dist/engine/buffered-writer.d.ts +2 -0
- package/dist/engine/buffered-writer.js +15 -2
- package/dist/engine/client-audit-harness.js +123 -0
- package/dist/engine/client.d.ts +55 -6
- package/dist/engine/client.js +176 -46
- package/dist/engine/database.d.ts +15 -1
- package/dist/engine/database.js +1276 -251
- package/dist/engine/errors.d.ts +61 -2
- package/dist/engine/errors.js +116 -3
- package/dist/engine/index.d.ts +1 -0
- package/dist/engine/index.js +2 -0
- package/dist/engine/live.d.ts +24 -1
- package/dist/engine/live.js +33 -9
- package/dist/engine/scope-write-set.js +36 -0
- package/dist/engine/worker-auto.d.ts +1 -0
- package/dist/engine/worker-auto.js +3 -0
- package/dist/engine/worker-host.d.ts +2 -1
- package/dist/engine/worker-host.js +19 -1
- package/dist/engine/worker-server.d.ts +53 -1
- package/dist/engine/worker-server.js +122 -19
- package/dist/engine/worker-store-auto.js +36 -0
- package/dist/engine/worker-store-opfs.js +3 -2
- package/dist/engine/write-coordinator.js +44 -2
- package/dist/storage/indexeddb-audit-helpers.js +269 -0
- package/dist/storage/indexeddb.js +599 -374
- package/dist/storage/opfs/coordination-helpers.js +54 -0
- package/dist/storage/opfs/index.d.ts +1 -1
- package/dist/storage/opfs/index.js +3 -2
- package/dist/storage/opfs/leader.js +243 -17
- package/dist/storage/opfs/power-loss-model.js +62 -0
- package/dist/storage/opfs/rpc.js +24 -43
- package/dist/storage/opfs/store.d.ts +32 -0
- package/dist/storage/opfs/store.js +531 -65
- package/dist/storage/toolkit/record-core.js +67 -38
- package/dist/storage/toolkit/wal.js +16 -0
- package/dist/storage/toolkit/wire.d.ts +1 -1
- package/dist/storage/toolkit/wire.js +4 -4
- package/dist/storage/types.d.ts +31 -10
- package/dist/storage/types.js +27 -16
- package/dist/testing/opfs-shim.js +14 -6
- package/dist/transactions/index.d.ts +19 -0
- package/dist/transactions/index.js +99 -25
- package/dist/worker-protocol/index.d.ts +50 -2
- package/dist/worker-protocol/index.js +106 -4
- package/package.json +7 -2
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
import { OpfsBlockStore } from "./store.js";
|
|
2
|
+
function table(name) {
|
|
3
|
+
return {
|
|
4
|
+
id: `table-${name}`,
|
|
5
|
+
name,
|
|
6
|
+
columns: [{ id: "c1", name: "id", type: "number", nullable: false }],
|
|
7
|
+
managed: false,
|
|
8
|
+
revision: 0,
|
|
9
|
+
createdAt: "2026-08-19T00:00:00.000Z"
|
|
10
|
+
};
|
|
11
|
+
}
|
|
12
|
+
function sleep(ms) {
|
|
13
|
+
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
14
|
+
}
|
|
15
|
+
async function waitFor(condition, what, attempts = 800) {
|
|
16
|
+
for (let attempt = 0; attempt < attempts; attempt += 1) {
|
|
17
|
+
if (await condition())
|
|
18
|
+
return;
|
|
19
|
+
await sleep(5);
|
|
20
|
+
}
|
|
21
|
+
throw new Error(`Timed out waiting for ${what}`);
|
|
22
|
+
}
|
|
23
|
+
function opener(shim, name, overrides = {}) {
|
|
24
|
+
return () => OpfsBlockStore.open({ name, root: shim.root, rpcTimeoutMs: 200, ...overrides });
|
|
25
|
+
}
|
|
26
|
+
async function outcome(work) {
|
|
27
|
+
return work.then(() => "ok", (error) => error instanceof Error ? error.name : String(error));
|
|
28
|
+
}
|
|
29
|
+
async function outcomeMessage(work) {
|
|
30
|
+
return work.then(() => "ok", (error) => error instanceof Error ? `${error.name}: ${error.message}` : String(error));
|
|
31
|
+
}
|
|
32
|
+
async function delaySyncHandleOpens(shim, matches, delayMs) {
|
|
33
|
+
const probe = await shim.root.getFileHandle("__delay-probe__", { create: true });
|
|
34
|
+
const prototype = Object.getPrototypeOf(probe);
|
|
35
|
+
const original = prototype.createSyncAccessHandle;
|
|
36
|
+
prototype.createSyncAccessHandle = async function patched() {
|
|
37
|
+
if (matches(this.name))
|
|
38
|
+
await sleep(delayMs);
|
|
39
|
+
return original.call(this);
|
|
40
|
+
};
|
|
41
|
+
await shim.root.removeEntry("__delay-probe__");
|
|
42
|
+
return () => {
|
|
43
|
+
prototype.createSyncAccessHandle = original;
|
|
44
|
+
};
|
|
45
|
+
}
|
|
46
|
+
export {
|
|
47
|
+
delaySyncHandleOpens,
|
|
48
|
+
opener,
|
|
49
|
+
outcome,
|
|
50
|
+
outcomeMessage,
|
|
51
|
+
sleep,
|
|
52
|
+
table,
|
|
53
|
+
waitFor
|
|
54
|
+
};
|
|
@@ -1,3 +1,3 @@
|
|
|
1
|
-
export { OpfsBlockStore, deleteOpfsDatabase, type OpfsBlockStoreOptions } from "./store.js";
|
|
1
|
+
export { OpfsBlockStore, deleteOpfsDatabase, opfsDatabaseExists, type OpfsBlockStoreOptions, } from "./store.js";
|
|
2
2
|
export { OpfsCoordinationError, OpfsDatabaseInUseError, OpfsUncertainOutcomeError, } from "../types.js";
|
|
3
3
|
export { OpfsTree, type ReadFileOptions, type WriteFileOptions } from "./files.js";
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { OpfsBlockStore, deleteOpfsDatabase } from "./store.js";
|
|
1
|
+
import { OpfsBlockStore, deleteOpfsDatabase, opfsDatabaseExists } from "./store.js";
|
|
2
2
|
import { OpfsCoordinationError, OpfsDatabaseInUseError, OpfsUncertainOutcomeError } from "../types.js";
|
|
3
3
|
import { OpfsTree } from "./files.js";
|
|
4
4
|
export {
|
|
@@ -7,5 +7,6 @@ export {
|
|
|
7
7
|
OpfsDatabaseInUseError,
|
|
8
8
|
OpfsTree,
|
|
9
9
|
OpfsUncertainOutcomeError,
|
|
10
|
-
deleteOpfsDatabase
|
|
10
|
+
deleteOpfsDatabase,
|
|
11
|
+
opfsDatabaseExists
|
|
11
12
|
};
|
|
@@ -3,6 +3,7 @@ import { dateIsoString } from "../../date-value.js";
|
|
|
3
3
|
import { decodeSnapshotMetadataItems, encodeSnapshotMetadataPage, extendSnapshotFrameStreamChecksum, prepareSnapshotFrameStreamHeader, snapshotFrameEnvelopeParts, snapshotFrameStreamHeaderIdentity } from "../snapshot-stream.js";
|
|
4
4
|
import { RecordCore, validateBeginTransactionInput, validateBlockWriteBytes, validateFtsBaseInput, validateFtsPostingChunks, validateId, validateTempRunPage, validateTempRunPageIdentity } from "../toolkit/record-core.js";
|
|
5
5
|
import { encodeSegment, decodeSegment } from "./files.js";
|
|
6
|
+
import { estimateRpcValueBytes } from "./rpc.js";
|
|
6
7
|
import { SnapshotFrameLedger, snapshotLedgerPath } from "./snapshot-ledger.js";
|
|
7
8
|
import { decodePostingChunk, decodeSyncCheckpoint, encodePostingChunk, encodeSyncCheckpoint } from "../toolkit/wire.js";
|
|
8
9
|
import { WalWriter, iterateWalFrames } from "../toolkit/wal.js";
|
|
@@ -20,6 +21,10 @@ const FTS_CHUNK_CACHE_SIZE = 64;
|
|
|
20
21
|
const MAX_RELOCATION_PAYLOADS = 256;
|
|
21
22
|
const SNAPSHOT_POSTING_TERMS_PER_CHUNK = 128;
|
|
22
23
|
const TEMP_FILE_CLEANUP_DEBT_FLOOR_BYTES = 64 * 1024;
|
|
24
|
+
const MAX_SERVED_LEDGER_ENTRIES = 65536;
|
|
25
|
+
const MAX_SERVED_LEDGER_AGE_MS = 10 * 60 * 1e3;
|
|
26
|
+
const MAX_SERVED_LEDGER_RESULT_BYTES = 64 * 1024;
|
|
27
|
+
const MAX_SERVED_LEDGER_BYTES = 8 * 1024 * 1024;
|
|
23
28
|
const CORE_READ_METHODS = [
|
|
24
29
|
"getTempOwner",
|
|
25
30
|
"getTable",
|
|
@@ -80,6 +85,36 @@ const LOGGED_BODY_BUILDERS = {
|
|
|
80
85
|
removeGarbageCollectionJob: ([id]) => ({ op: "removeGarbageCollectionJob", id }),
|
|
81
86
|
removePrunedManifestRecords: ([maxItems]) => ({ op: "removePrunedManifestRecords", maxItems })
|
|
82
87
|
};
|
|
88
|
+
const STATE_SHRINKING_OPS = /* @__PURE__ */ new Set([
|
|
89
|
+
"rollbackTransactionArtifacts",
|
|
90
|
+
"abortTransactionIfExpired",
|
|
91
|
+
"removeLease",
|
|
92
|
+
"removeLeaseIfExpired",
|
|
93
|
+
"removeTempRun",
|
|
94
|
+
"removeTempOwner",
|
|
95
|
+
"removeTempOwnerIfExpired",
|
|
96
|
+
"removeCompactionJob",
|
|
97
|
+
"cancelCompactionJob",
|
|
98
|
+
"removeGarbageCollectionJob",
|
|
99
|
+
"removePrunedManifestRecords",
|
|
100
|
+
"removeAbortedSegment",
|
|
101
|
+
"abortFtsBaseBuild",
|
|
102
|
+
"abortUniqueKeyBuild",
|
|
103
|
+
"cancelSnapshotFrameImport",
|
|
104
|
+
"closeSnapshotFrameExport",
|
|
105
|
+
"garbageCollectionStep"
|
|
106
|
+
]);
|
|
107
|
+
function shrinksLoggedState(body) {
|
|
108
|
+
if (body.op === "updateTransaction")
|
|
109
|
+
return body.update.status === "aborted";
|
|
110
|
+
return STATE_SHRINKING_OPS.has(body.op);
|
|
111
|
+
}
|
|
112
|
+
class OpfsLeaderClosedError extends Error {
|
|
113
|
+
name = "OpfsLeaderClosedError";
|
|
114
|
+
constructor() {
|
|
115
|
+
super("This OPFS store connection is closed");
|
|
116
|
+
}
|
|
117
|
+
}
|
|
83
118
|
class OpfsLeader {
|
|
84
119
|
#tree;
|
|
85
120
|
#strict;
|
|
@@ -136,7 +171,20 @@ class OpfsLeader {
|
|
|
136
171
|
#lastCleanupError;
|
|
137
172
|
#cleanupRetryScheduled = false;
|
|
138
173
|
#cleanupDebtBytes = 0;
|
|
139
|
-
|
|
174
|
+
#onDiagnostic;
|
|
175
|
+
servingRequest;
|
|
176
|
+
#servedLedger = /* @__PURE__ */ new Map();
|
|
177
|
+
#servedResultBytes = /* @__PURE__ */ new Map();
|
|
178
|
+
#servedLedgerBytes = 0;
|
|
179
|
+
#servedLedgerResultBytes;
|
|
180
|
+
#servedCoverageSince = 0;
|
|
181
|
+
#servedLedgerAgeMs;
|
|
182
|
+
onBeforeCheckpoint;
|
|
183
|
+
#lastCheckpointMs = 0;
|
|
184
|
+
constructor(tree, strict, walHandle, slots, checkpointEntries, cleanupLimitBytes, onDiagnostic, servedLedgerAgeMs, servedLedgerResultBytes) {
|
|
185
|
+
this.#onDiagnostic = onDiagnostic;
|
|
186
|
+
this.#servedLedgerAgeMs = servedLedgerAgeMs ?? MAX_SERVED_LEDGER_AGE_MS;
|
|
187
|
+
this.#servedLedgerResultBytes = servedLedgerResultBytes ?? MAX_SERVED_LEDGER_RESULT_BYTES;
|
|
140
188
|
this.#checkpointEntries = checkpointEntries ?? CHECKPOINT_ENTRIES;
|
|
141
189
|
this.#cleanupLimitBytes = cleanupLimitBytes ?? MAX_OPFS_CLEANUP_DEBT_BYTES;
|
|
142
190
|
if (!Number.isSafeInteger(this.#cleanupLimitBytes) || this.#cleanupLimitBytes < 1) {
|
|
@@ -153,8 +201,8 @@ class OpfsLeader {
|
|
|
153
201
|
blockChecksum: (id) => this.#blockIndex.get(id)?.checksum ?? (this.#permissivePhysical ? 0 : void 0)
|
|
154
202
|
});
|
|
155
203
|
}
|
|
156
|
-
static async recover(tree, strict, handles, checkpointEntries, cleanupLimitBytes) {
|
|
157
|
-
const leader = new OpfsLeader(tree, strict, handles.wal, [handles.slotA, handles.slotB], checkpointEntries, cleanupLimitBytes);
|
|
204
|
+
static async recover(tree, strict, handles, checkpointEntries, cleanupLimitBytes, onDiagnostic, servedLedgerAgeMs, servedLedgerResultBytes) {
|
|
205
|
+
const leader = new OpfsLeader(tree, strict, handles.wal, [handles.slotA, handles.slotB], checkpointEntries, cleanupLimitBytes, onDiagnostic, servedLedgerAgeMs, servedLedgerResultBytes);
|
|
158
206
|
try {
|
|
159
207
|
await leader.#loadFromDisk();
|
|
160
208
|
} catch (error) {
|
|
@@ -270,6 +318,16 @@ class OpfsLeader {
|
|
|
270
318
|
this.#applyTempPageUpdates(checkpoint.tempPages);
|
|
271
319
|
this.#seq = checkpoint.lastSeq;
|
|
272
320
|
this.#checkpointGeneration = checkpoint.generation;
|
|
321
|
+
this.#servedLedger.clear();
|
|
322
|
+
this.#servedResultBytes.clear();
|
|
323
|
+
this.#servedLedgerBytes = 0;
|
|
324
|
+
this.#servedCoverageSince = checkpoint.servedCoverageSince;
|
|
325
|
+
for (const outcome of checkpoint.servedRequests) {
|
|
326
|
+
this.#servedLedger.set(outcome.key, { ...outcome });
|
|
327
|
+
if (outcome.settled && outcome.withheld !== true) {
|
|
328
|
+
this.#retainServedResult(outcome.key, estimateRpcValueBytes(outcome.result));
|
|
329
|
+
}
|
|
330
|
+
}
|
|
273
331
|
}
|
|
274
332
|
const newest = this.#slots[this.#newestSlot];
|
|
275
333
|
this.#lastCheckpointBytes = newest === void 0 ? 0 : newest.getSize();
|
|
@@ -328,6 +386,21 @@ class OpfsLeader {
|
|
|
328
386
|
}
|
|
329
387
|
applied += 1;
|
|
330
388
|
this.#applyReplayed(entry);
|
|
389
|
+
if (entry.request !== void 0) {
|
|
390
|
+
this.#recordServed({ ...entry.request, seq: entry.seq, settled: false });
|
|
391
|
+
}
|
|
392
|
+
if (entry.op === "servedResult") {
|
|
393
|
+
const served = this.#servedLedger.get(entry.key);
|
|
394
|
+
if (served !== void 0) {
|
|
395
|
+
served.settled = true;
|
|
396
|
+
if (entry.withheld === true)
|
|
397
|
+
served.withheld = true;
|
|
398
|
+
else {
|
|
399
|
+
served.result = entry.result;
|
|
400
|
+
this.#retainServedResult(entry.key, estimateRpcValueBytes(entry.result));
|
|
401
|
+
}
|
|
402
|
+
}
|
|
403
|
+
}
|
|
331
404
|
for (const placement of placementsOf(entry)) {
|
|
332
405
|
this.#pool.restorePlacement(placement);
|
|
333
406
|
}
|
|
@@ -382,7 +455,7 @@ class OpfsLeader {
|
|
|
382
455
|
#run(work) {
|
|
383
456
|
const result = this.#chain.then(async () => {
|
|
384
457
|
if (this.#closed)
|
|
385
|
-
throw new
|
|
458
|
+
throw new OpfsLeaderClosedError();
|
|
386
459
|
if (this.#poisoned)
|
|
387
460
|
await this.#loadFromDisk();
|
|
388
461
|
return work();
|
|
@@ -395,9 +468,76 @@ class OpfsLeader {
|
|
|
395
468
|
return;
|
|
396
469
|
await this.#run(() => void 0);
|
|
397
470
|
}
|
|
471
|
+
#recordServed(outcome) {
|
|
472
|
+
this.#forgetServedResult(outcome.key);
|
|
473
|
+
this.#servedLedger.delete(outcome.key);
|
|
474
|
+
this.#servedLedger.set(outcome.key, outcome);
|
|
475
|
+
this.#trimServedLedger(outcome.sentAt - this.#servedLedgerAgeMs);
|
|
476
|
+
}
|
|
477
|
+
#trimServedLedger(horizon) {
|
|
478
|
+
for (const [key, entry] of this.#servedLedger) {
|
|
479
|
+
if (this.#servedLedger.size <= MAX_SERVED_LEDGER_ENTRIES && this.#servedLedgerBytes <= MAX_SERVED_LEDGER_BYTES && entry.sentAt >= horizon) {
|
|
480
|
+
break;
|
|
481
|
+
}
|
|
482
|
+
this.#forgetServedResult(key);
|
|
483
|
+
this.#servedLedger.delete(key);
|
|
484
|
+
this.#servedCoverageSince = Math.max(this.#servedCoverageSince, entry.sentAt + 1);
|
|
485
|
+
}
|
|
486
|
+
}
|
|
487
|
+
#retainServedResult(key, bytes) {
|
|
488
|
+
this.#forgetServedResult(key);
|
|
489
|
+
this.#servedResultBytes.set(key, bytes);
|
|
490
|
+
this.#servedLedgerBytes += bytes;
|
|
491
|
+
}
|
|
492
|
+
#forgetServedResult(key) {
|
|
493
|
+
const bytes = this.#servedResultBytes.get(key);
|
|
494
|
+
if (bytes === void 0)
|
|
495
|
+
return;
|
|
496
|
+
this.#servedResultBytes.delete(key);
|
|
497
|
+
this.#servedLedgerBytes -= bytes;
|
|
498
|
+
}
|
|
499
|
+
servedOutcome(key) {
|
|
500
|
+
return this.#servedLedger.get(key);
|
|
501
|
+
}
|
|
502
|
+
async completeServed(key, result) {
|
|
503
|
+
if (!this.#servedLedger.has(key))
|
|
504
|
+
return;
|
|
505
|
+
const bytes = estimateRpcValueBytes(result);
|
|
506
|
+
await this.#run(() => {
|
|
507
|
+
const entry = this.#servedLedger.get(key);
|
|
508
|
+
if (entry === void 0 || entry.settled)
|
|
509
|
+
return;
|
|
510
|
+
if (bytes > this.#servedLedgerResultBytes) {
|
|
511
|
+
this.#appendFrame({ op: "servedResult", key, withheld: true }, false);
|
|
512
|
+
entry.settled = true;
|
|
513
|
+
entry.withheld = true;
|
|
514
|
+
return;
|
|
515
|
+
}
|
|
516
|
+
this.#appendFrame({ op: "servedResult", key, result }, false);
|
|
517
|
+
entry.result = result;
|
|
518
|
+
entry.settled = true;
|
|
519
|
+
this.#retainServedResult(key, bytes);
|
|
520
|
+
this.#trimServedLedger(Number.NEGATIVE_INFINITY);
|
|
521
|
+
});
|
|
522
|
+
}
|
|
523
|
+
get servedCoverageSince() {
|
|
524
|
+
return this.#servedCoverageSince;
|
|
525
|
+
}
|
|
526
|
+
#diagnostic(error, context) {
|
|
527
|
+
try {
|
|
528
|
+
this.#onDiagnostic?.(error, context);
|
|
529
|
+
} catch {
|
|
530
|
+
}
|
|
531
|
+
}
|
|
398
532
|
#logged(body) {
|
|
399
533
|
if (this.#wal.byteLength >= MAX_OPFS_WAL_BYTES - 64 * 1024 * 1024) {
|
|
400
|
-
|
|
534
|
+
try {
|
|
535
|
+
this.checkpointNow();
|
|
536
|
+
} catch (error) {
|
|
537
|
+
if (!shrinksLoggedState(body) || this.#wal.byteLength >= MAX_OPFS_WAL_BYTES - 16 * 1024 * 1024) {
|
|
538
|
+
throw error;
|
|
539
|
+
}
|
|
540
|
+
}
|
|
401
541
|
}
|
|
402
542
|
if (this.#strict) {
|
|
403
543
|
try {
|
|
@@ -414,16 +554,21 @@ class OpfsLeader {
|
|
|
414
554
|
this.#appendFrame(body);
|
|
415
555
|
return result;
|
|
416
556
|
}
|
|
417
|
-
#appendFrame(body) {
|
|
557
|
+
#appendFrame(body, flush = this.#strict) {
|
|
558
|
+
const request = this.servingRequest;
|
|
559
|
+
this.servingRequest = void 0;
|
|
560
|
+
let nextSeq;
|
|
418
561
|
try {
|
|
419
|
-
|
|
420
|
-
this.#wal.append({ seq: nextSeq, ...body },
|
|
562
|
+
nextSeq = safeSuccessor(this.#seq, "OPFS WAL sequence");
|
|
563
|
+
this.#wal.append({ seq: nextSeq, ...body, ...request === void 0 ? {} : { request } }, flush);
|
|
421
564
|
this.#seq = nextSeq;
|
|
422
565
|
this.#entriesSinceCheckpoint = safeSuccessor(this.#entriesSinceCheckpoint, "OPFS checkpoint entry count");
|
|
423
566
|
} catch (error) {
|
|
424
567
|
this.#poisoned = true;
|
|
425
568
|
throw error;
|
|
426
569
|
}
|
|
570
|
+
if (request !== void 0)
|
|
571
|
+
this.#recordServed({ ...request, seq: nextSeq, settled: false });
|
|
427
572
|
if (this.#checkpointDue() && this.#entriesSinceCheckpoint >= this.#checkpointRetryAtEntries && !this.#checkpointScheduled) {
|
|
428
573
|
this.#checkpointScheduled = true;
|
|
429
574
|
void this.#run(() => {
|
|
@@ -431,7 +576,9 @@ class OpfsLeader {
|
|
|
431
576
|
if (this.#checkpointDue()) {
|
|
432
577
|
this.checkpointNow();
|
|
433
578
|
}
|
|
434
|
-
}).catch(() =>
|
|
579
|
+
}).catch((error) => {
|
|
580
|
+
this.#diagnostic(error, "opfs checkpoint");
|
|
581
|
+
});
|
|
435
582
|
}
|
|
436
583
|
}
|
|
437
584
|
#checkpointDue() {
|
|
@@ -937,6 +1084,8 @@ class OpfsLeader {
|
|
|
937
1084
|
this.#snapshotFrameImport = void 0;
|
|
938
1085
|
return void 0;
|
|
939
1086
|
}
|
|
1087
|
+
case "servedResult":
|
|
1088
|
+
return void 0;
|
|
940
1089
|
case "cancelSnapshotFrameImport": {
|
|
941
1090
|
const session = this.#snapshotFrameImport;
|
|
942
1091
|
if (session?.identity !== body.input.identity) {
|
|
@@ -1085,8 +1234,9 @@ class OpfsLeader {
|
|
|
1085
1234
|
#applyReplayed(entry) {
|
|
1086
1235
|
this.#permissivePhysical = true;
|
|
1087
1236
|
try {
|
|
1088
|
-
const { seq: _seq, ...body } = entry;
|
|
1237
|
+
const { seq: _seq, request: _request, ...body } = entry;
|
|
1089
1238
|
void _seq;
|
|
1239
|
+
void _request;
|
|
1090
1240
|
this.#applyBody(body);
|
|
1091
1241
|
this.#clearCompletedSnapshotImportIfAdvanced();
|
|
1092
1242
|
} finally {
|
|
@@ -1416,7 +1566,10 @@ class OpfsLeader {
|
|
|
1416
1566
|
}
|
|
1417
1567
|
checkpointNow() {
|
|
1418
1568
|
try {
|
|
1569
|
+
this.onBeforeCheckpoint?.(this.#lastCheckpointMs);
|
|
1570
|
+
const started = Date.now();
|
|
1419
1571
|
this.#checkpointNowUnchecked();
|
|
1572
|
+
this.#lastCheckpointMs = Date.now() - started;
|
|
1420
1573
|
this.#checkpointFailures = 0;
|
|
1421
1574
|
this.#lastCheckpointError = void 0;
|
|
1422
1575
|
this.#checkpointRetryAtEntries = 0;
|
|
@@ -1448,11 +1601,13 @@ class OpfsLeader {
|
|
|
1448
1601
|
...this.#completedSnapshotFrameImport === void 0 ? {} : {
|
|
1449
1602
|
completedSnapshotFrameImport: structuredClone(this.#completedSnapshotFrameImport)
|
|
1450
1603
|
},
|
|
1451
|
-
extents: this.#pool.meta()
|
|
1604
|
+
extents: this.#pool.meta(),
|
|
1605
|
+
servedRequests: [...this.#servedLedger.values()],
|
|
1606
|
+
servedCoverageSince: this.#servedCoverageSince
|
|
1452
1607
|
};
|
|
1453
1608
|
const bytes = encodeSyncCheckpoint(state);
|
|
1454
1609
|
if (bytes.byteLength > MAX_OPFS_CHECKPOINT_BYTES) {
|
|
1455
|
-
throw new
|
|
1610
|
+
throw new StorageResourceLimitError("checkpoint byte", bytes.byteLength, MAX_OPFS_CHECKPOINT_BYTES);
|
|
1456
1611
|
}
|
|
1457
1612
|
const slotIndex = this.#newestSlot === 0 ? 1 : 0;
|
|
1458
1613
|
const mirrorIndex = slotIndex === 0 ? 1 : 0;
|
|
@@ -1532,6 +1687,7 @@ class OpfsLeader {
|
|
|
1532
1687
|
} catch (error) {
|
|
1533
1688
|
this.#cleanupFailures += 1;
|
|
1534
1689
|
this.#lastCleanupError = error;
|
|
1690
|
+
this.#diagnostic(error, "opfs cleanup");
|
|
1535
1691
|
await this.#refreshCleanupDebtAfterFailure();
|
|
1536
1692
|
}
|
|
1537
1693
|
if (this.#cleanupRetryScheduled || this.#closed)
|
|
@@ -1547,9 +1703,12 @@ class OpfsLeader {
|
|
|
1547
1703
|
} catch (error) {
|
|
1548
1704
|
this.#cleanupFailures += 1;
|
|
1549
1705
|
this.#lastCleanupError = error;
|
|
1706
|
+
this.#diagnostic(error, "opfs cleanup retry");
|
|
1550
1707
|
await this.#refreshCleanupDebtAfterFailure();
|
|
1551
1708
|
}
|
|
1552
|
-
}).catch(() =>
|
|
1709
|
+
}).catch((error) => {
|
|
1710
|
+
this.#diagnostic(error, "opfs cleanup retry");
|
|
1711
|
+
});
|
|
1553
1712
|
}
|
|
1554
1713
|
async #refreshCleanupDebtAfterFailure() {
|
|
1555
1714
|
try {
|
|
@@ -3228,11 +3387,15 @@ class OpfsLeader {
|
|
|
3228
3387
|
}
|
|
3229
3388
|
};
|
|
3230
3389
|
}
|
|
3390
|
+
isClosed() {
|
|
3391
|
+
return this.#closed;
|
|
3392
|
+
}
|
|
3231
3393
|
async shutdown() {
|
|
3232
3394
|
await this.#run(() => {
|
|
3233
3395
|
this.checkpointNow();
|
|
3234
3396
|
this.#closed = true;
|
|
3235
|
-
}).catch(() => {
|
|
3397
|
+
}).catch((error) => {
|
|
3398
|
+
this.#diagnostic(error, "opfs shutdown checkpoint");
|
|
3236
3399
|
this.#closed = true;
|
|
3237
3400
|
});
|
|
3238
3401
|
this.#walHandle.close();
|
|
@@ -3347,7 +3510,8 @@ const WAL_BODY_KEYS = {
|
|
|
3347
3510
|
renewSnapshotFrameImport: ["input"],
|
|
3348
3511
|
appendSnapshotImportFrames: ["input", "state", "blockPlacements", "replay"],
|
|
3349
3512
|
finishSnapshotFrameImport: ["input"],
|
|
3350
|
-
cancelSnapshotFrameImport: ["input"]
|
|
3513
|
+
cancelSnapshotFrameImport: ["input"],
|
|
3514
|
+
servedResult: ["key", "result", "withheld"]
|
|
3351
3515
|
};
|
|
3352
3516
|
function validateWalEntry(value) {
|
|
3353
3517
|
if (!isRecord(value))
|
|
@@ -3543,6 +3707,14 @@ function validateWalEntry(value) {
|
|
|
3543
3707
|
requireTimestamp(entry.input.expiresAtCutoff, "finishSnapshotFrameImport cutoff");
|
|
3544
3708
|
requireRecord(entry.input.footer, "finishSnapshotFrameImport footer");
|
|
3545
3709
|
break;
|
|
3710
|
+
case "servedResult":
|
|
3711
|
+
if (typeof entry.key !== "string" || entry.key.length === 0 || entry.key.length > 4096) {
|
|
3712
|
+
throw new Error("servedResult key is invalid");
|
|
3713
|
+
}
|
|
3714
|
+
if (Object.hasOwn(entry, "withheld") && (entry.withheld !== true || Object.hasOwn(entry, "result"))) {
|
|
3715
|
+
throw new Error("servedResult withholding is invalid");
|
|
3716
|
+
}
|
|
3717
|
+
break;
|
|
3546
3718
|
case "cancelSnapshotFrameImport":
|
|
3547
3719
|
requireRecord(entry.input, "cancelSnapshotFrameImport input");
|
|
3548
3720
|
assertExactRecordKeys(entry.input, ["identity", "ownerId"], "cancelSnapshotFrameImport input");
|
|
@@ -3570,11 +3742,46 @@ function validateIdPlacements(value, label) {
|
|
|
3570
3742
|
}
|
|
3571
3743
|
}
|
|
3572
3744
|
function assertWalKeys(value, bodyKeys) {
|
|
3573
|
-
const allowed = /* @__PURE__ */ new Set(["seq", "op", ...bodyKeys]);
|
|
3745
|
+
const allowed = /* @__PURE__ */ new Set(["seq", "op", ...bodyKeys, "request"]);
|
|
3574
3746
|
for (const key of Object.keys(value)) {
|
|
3575
3747
|
if (!allowed.has(key))
|
|
3576
3748
|
throw new Error(`Unexpected ${String(value.op)} WAL field: ${key}`);
|
|
3577
3749
|
}
|
|
3750
|
+
if (value.request !== void 0)
|
|
3751
|
+
validateServedRequest(value.request);
|
|
3752
|
+
}
|
|
3753
|
+
function validateServedRequest(value) {
|
|
3754
|
+
if (!isRecord(value))
|
|
3755
|
+
throw new Error("OPFS WAL served request is not an object");
|
|
3756
|
+
assertExactRecordKeys(value, ["key", "method", "signature", "requestBytes", "sentAt"], "OPFS WAL served request");
|
|
3757
|
+
if (typeof value.key !== "string" || value.key.length === 0 || value.key.length > 4096) {
|
|
3758
|
+
throw new Error("OPFS WAL served request has an invalid key");
|
|
3759
|
+
}
|
|
3760
|
+
if (typeof value.method !== "string" || value.method.length === 0) {
|
|
3761
|
+
throw new Error("OPFS WAL served request has an invalid method");
|
|
3762
|
+
}
|
|
3763
|
+
if (typeof value.signature !== "string" || value.signature.length === 0) {
|
|
3764
|
+
throw new Error("OPFS WAL served request has an invalid signature");
|
|
3765
|
+
}
|
|
3766
|
+
if (!isNonNegativeSafeInteger(value.requestBytes) || !isNonNegativeSafeInteger(value.sentAt)) {
|
|
3767
|
+
throw new Error("OPFS WAL served request has invalid numbers");
|
|
3768
|
+
}
|
|
3769
|
+
}
|
|
3770
|
+
function validateServedOutcome(value) {
|
|
3771
|
+
if (!isRecord(value))
|
|
3772
|
+
throw new Error("OPFS served outcome is not an object");
|
|
3773
|
+
const { seq, settled, withheld, result: _result, ...request } = value;
|
|
3774
|
+
void _result;
|
|
3775
|
+
validateServedRequest(request);
|
|
3776
|
+
if (!Number.isSafeInteger(seq) || seq < 1) {
|
|
3777
|
+
throw new Error("OPFS served outcome has an invalid sequence");
|
|
3778
|
+
}
|
|
3779
|
+
if (typeof settled !== "boolean" || !settled && Object.hasOwn(value, "result")) {
|
|
3780
|
+
throw new Error("OPFS served outcome has an invalid settlement");
|
|
3781
|
+
}
|
|
3782
|
+
if (withheld !== void 0 && (withheld !== true || !settled || Object.hasOwn(value, "result"))) {
|
|
3783
|
+
throw new Error("OPFS served outcome has an invalid withholding");
|
|
3784
|
+
}
|
|
3578
3785
|
}
|
|
3579
3786
|
function assertExactRecordKeys(value, keys, label) {
|
|
3580
3787
|
const expected = new Set(keys);
|
|
@@ -3757,6 +3964,20 @@ function validateCheckpointState(value) {
|
|
|
3757
3964
|
if (!Array.isArray(value.tempPages)) {
|
|
3758
3965
|
throw new Error("Invalid OPFS checkpoint temp-page ledger");
|
|
3759
3966
|
}
|
|
3967
|
+
if (!Array.isArray(value.servedRequests)) {
|
|
3968
|
+
throw new Error("Invalid OPFS checkpoint served-request ledger");
|
|
3969
|
+
}
|
|
3970
|
+
if (!isNonNegativeSafeInteger(value.servedCoverageSince)) {
|
|
3971
|
+
throw new Error("Invalid OPFS checkpoint served-request coverage");
|
|
3972
|
+
}
|
|
3973
|
+
const seenServed = /* @__PURE__ */ new Set();
|
|
3974
|
+
for (const entry of value.servedRequests) {
|
|
3975
|
+
validateServedOutcome(entry);
|
|
3976
|
+
if (seenServed.has(entry.key)) {
|
|
3977
|
+
throw new Error(`Duplicate OPFS checkpoint served request: ${entry.key}`);
|
|
3978
|
+
}
|
|
3979
|
+
seenServed.add(entry.key);
|
|
3980
|
+
}
|
|
3760
3981
|
assertExactRecordKeys(value, [
|
|
3761
3982
|
"formatVersion",
|
|
3762
3983
|
"generation",
|
|
@@ -3769,7 +3990,9 @@ function validateCheckpointState(value) {
|
|
|
3769
3990
|
...value.snapshotFrameExport === void 0 ? [] : ["snapshotFrameExport"],
|
|
3770
3991
|
...value.snapshotFrameImport === void 0 ? [] : ["snapshotFrameImport"],
|
|
3771
3992
|
...value.completedSnapshotFrameImport === void 0 ? [] : ["completedSnapshotFrameImport"],
|
|
3772
|
-
"extents"
|
|
3993
|
+
"extents",
|
|
3994
|
+
"servedRequests",
|
|
3995
|
+
"servedCoverageSince"
|
|
3773
3996
|
], "OPFS checkpoint");
|
|
3774
3997
|
assertValidExtentMeta(value.extents);
|
|
3775
3998
|
const blockIds = /* @__PURE__ */ new Set();
|
|
@@ -4542,6 +4765,8 @@ function parseTempPageFilePath(path) {
|
|
|
4542
4765
|
}
|
|
4543
4766
|
function placementsOf(entry) {
|
|
4544
4767
|
switch (entry.op) {
|
|
4768
|
+
case "servedResult":
|
|
4769
|
+
return [];
|
|
4545
4770
|
case "stageTransactionArtifacts":
|
|
4546
4771
|
case "writeTransaction":
|
|
4547
4772
|
return entry.blocks.map(({ placement }) => placement);
|
|
@@ -4615,5 +4840,6 @@ export {
|
|
|
4615
4840
|
MAX_OPFS_CHECKPOINT_BYTES,
|
|
4616
4841
|
MAX_OPFS_WAL_BYTES,
|
|
4617
4842
|
OpfsLeader,
|
|
4843
|
+
OpfsLeaderClosedError,
|
|
4618
4844
|
assertBlockReadBatchByteLimit
|
|
4619
4845
|
};
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
class PowerLossModel {
|
|
2
|
+
#durable = /* @__PURE__ */ new Map();
|
|
3
|
+
#touched = /* @__PURE__ */ new Set();
|
|
4
|
+
#shim;
|
|
5
|
+
constructor(shim) {
|
|
6
|
+
this.#shim = shim;
|
|
7
|
+
shim.setWriteFault((path, phase) => {
|
|
8
|
+
if (phase === "flush") {
|
|
9
|
+
this.#durable.set(path, shim.readFileBytes(path) ?? new Uint8Array());
|
|
10
|
+
} else {
|
|
11
|
+
this.#touched.add(path);
|
|
12
|
+
}
|
|
13
|
+
});
|
|
14
|
+
}
|
|
15
|
+
durableBytes(path) {
|
|
16
|
+
return this.#durable.get(path);
|
|
17
|
+
}
|
|
18
|
+
powerLoss(keepPrefix) {
|
|
19
|
+
const reverted = [];
|
|
20
|
+
for (const path of this.#touched) {
|
|
21
|
+
const current = this.#shim.readFileBytes(path);
|
|
22
|
+
if (current === void 0)
|
|
23
|
+
continue;
|
|
24
|
+
const durable = this.#durable.get(path) ?? new Uint8Array();
|
|
25
|
+
let kept = durable;
|
|
26
|
+
if (keepPrefix !== void 0 && current.byteLength > durable.byteLength && startsWith(current, durable)) {
|
|
27
|
+
const unflushed = current.byteLength - durable.byteLength;
|
|
28
|
+
const extra = keepPrefix(unflushed, path);
|
|
29
|
+
kept = current.slice(0, durable.byteLength + Math.max(0, Math.min(extra, unflushed)));
|
|
30
|
+
}
|
|
31
|
+
if (!bytesEqual(kept, current)) {
|
|
32
|
+
this.#shim.writeFileBytes(path, kept);
|
|
33
|
+
reverted.push(path);
|
|
34
|
+
}
|
|
35
|
+
this.#durable.set(path, kept.slice());
|
|
36
|
+
}
|
|
37
|
+
this.#touched.clear();
|
|
38
|
+
return reverted;
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
function startsWith(bytes, prefix) {
|
|
42
|
+
if (prefix.byteLength > bytes.byteLength)
|
|
43
|
+
return false;
|
|
44
|
+
for (let index = 0; index < prefix.byteLength; index += 1) {
|
|
45
|
+
if (bytes[index] !== prefix[index])
|
|
46
|
+
return false;
|
|
47
|
+
}
|
|
48
|
+
return true;
|
|
49
|
+
}
|
|
50
|
+
function bytesEqual(left, right) {
|
|
51
|
+
if (left.byteLength !== right.byteLength)
|
|
52
|
+
return false;
|
|
53
|
+
for (let index = 0; index < left.byteLength; index += 1) {
|
|
54
|
+
if (left[index] !== right[index])
|
|
55
|
+
return false;
|
|
56
|
+
}
|
|
57
|
+
return true;
|
|
58
|
+
}
|
|
59
|
+
export {
|
|
60
|
+
PowerLossModel,
|
|
61
|
+
bytesEqual
|
|
62
|
+
};
|