@minnowdb/core 0.10.0 → 0.10.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/engine/auto-store.d.ts +15 -3
- package/dist/engine/auto-store.js +48 -6
- package/dist/engine/client-audit-harness.js +123 -0
- package/dist/engine/client.js +21 -6
- package/dist/engine/database.js +223 -56
- package/dist/engine/worker-host.js +2 -0
- package/dist/engine/worker-server.js +4 -5
- package/dist/engine/worker-store-auto.js +2 -2
- package/dist/engine/write-coordinator.js +21 -1
- package/dist/storage/indexeddb-audit-helpers.js +269 -0
- package/dist/storage/indexeddb.js +107 -169
- package/dist/storage/opfs/coordination-helpers.js +54 -0
- package/dist/storage/opfs/index.d.ts +1 -1
- package/dist/storage/opfs/index.js +3 -2
- package/dist/storage/opfs/leader.js +44 -5
- package/dist/storage/opfs/power-loss-model.js +62 -0
- package/dist/storage/opfs/rpc.js +1 -0
- package/dist/storage/opfs/store.d.ts +12 -0
- package/dist/storage/opfs/store.js +59 -12
- package/dist/storage/toolkit/wal.js +16 -0
- package/dist/storage/toolkit/wire.js +3 -3
- package/dist/storage/types.d.ts +2 -2
- package/dist/transactions/index.d.ts +7 -0
- package/dist/transactions/index.js +17 -3
- package/package.json +1 -1
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
import { OpfsBlockStore } from "./store.js";
|
|
2
|
+
function table(name) {
|
|
3
|
+
return {
|
|
4
|
+
id: `table-${name}`,
|
|
5
|
+
name,
|
|
6
|
+
columns: [{ id: "c1", name: "id", type: "number", nullable: false }],
|
|
7
|
+
managed: false,
|
|
8
|
+
revision: 0,
|
|
9
|
+
createdAt: "2026-08-19T00:00:00.000Z"
|
|
10
|
+
};
|
|
11
|
+
}
|
|
12
|
+
function sleep(ms) {
|
|
13
|
+
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
14
|
+
}
|
|
15
|
+
async function waitFor(condition, what, attempts = 800) {
|
|
16
|
+
for (let attempt = 0; attempt < attempts; attempt += 1) {
|
|
17
|
+
if (await condition())
|
|
18
|
+
return;
|
|
19
|
+
await sleep(5);
|
|
20
|
+
}
|
|
21
|
+
throw new Error(`Timed out waiting for ${what}`);
|
|
22
|
+
}
|
|
23
|
+
function opener(shim, name, overrides = {}) {
|
|
24
|
+
return () => OpfsBlockStore.open({ name, root: shim.root, rpcTimeoutMs: 200, ...overrides });
|
|
25
|
+
}
|
|
26
|
+
async function outcome(work) {
|
|
27
|
+
return work.then(() => "ok", (error) => error instanceof Error ? error.name : String(error));
|
|
28
|
+
}
|
|
29
|
+
async function outcomeMessage(work) {
|
|
30
|
+
return work.then(() => "ok", (error) => error instanceof Error ? `${error.name}: ${error.message}` : String(error));
|
|
31
|
+
}
|
|
32
|
+
async function delaySyncHandleOpens(shim, matches, delayMs) {
|
|
33
|
+
const probe = await shim.root.getFileHandle("__delay-probe__", { create: true });
|
|
34
|
+
const prototype = Object.getPrototypeOf(probe);
|
|
35
|
+
const original = prototype.createSyncAccessHandle;
|
|
36
|
+
prototype.createSyncAccessHandle = async function patched() {
|
|
37
|
+
if (matches(this.name))
|
|
38
|
+
await sleep(delayMs);
|
|
39
|
+
return original.call(this);
|
|
40
|
+
};
|
|
41
|
+
await shim.root.removeEntry("__delay-probe__");
|
|
42
|
+
return () => {
|
|
43
|
+
prototype.createSyncAccessHandle = original;
|
|
44
|
+
};
|
|
45
|
+
}
|
|
46
|
+
export {
|
|
47
|
+
delaySyncHandleOpens,
|
|
48
|
+
opener,
|
|
49
|
+
outcome,
|
|
50
|
+
outcomeMessage,
|
|
51
|
+
sleep,
|
|
52
|
+
table,
|
|
53
|
+
waitFor
|
|
54
|
+
};
|
|
@@ -1,3 +1,3 @@
|
|
|
1
|
-
export { OpfsBlockStore, deleteOpfsDatabase, type OpfsBlockStoreOptions } from "./store.js";
|
|
1
|
+
export { OpfsBlockStore, deleteOpfsDatabase, opfsDatabaseExists, type OpfsBlockStoreOptions, } from "./store.js";
|
|
2
2
|
export { OpfsCoordinationError, OpfsDatabaseInUseError, OpfsUncertainOutcomeError, } from "../types.js";
|
|
3
3
|
export { OpfsTree, type ReadFileOptions, type WriteFileOptions } from "./files.js";
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { OpfsBlockStore, deleteOpfsDatabase } from "./store.js";
|
|
1
|
+
import { OpfsBlockStore, deleteOpfsDatabase, opfsDatabaseExists } from "./store.js";
|
|
2
2
|
import { OpfsCoordinationError, OpfsDatabaseInUseError, OpfsUncertainOutcomeError } from "../types.js";
|
|
3
3
|
import { OpfsTree } from "./files.js";
|
|
4
4
|
export {
|
|
@@ -7,5 +7,6 @@ export {
|
|
|
7
7
|
OpfsDatabaseInUseError,
|
|
8
8
|
OpfsTree,
|
|
9
9
|
OpfsUncertainOutcomeError,
|
|
10
|
-
deleteOpfsDatabase
|
|
10
|
+
deleteOpfsDatabase,
|
|
11
|
+
opfsDatabaseExists
|
|
11
12
|
};
|
|
@@ -85,6 +85,36 @@ const LOGGED_BODY_BUILDERS = {
|
|
|
85
85
|
removeGarbageCollectionJob: ([id]) => ({ op: "removeGarbageCollectionJob", id }),
|
|
86
86
|
removePrunedManifestRecords: ([maxItems]) => ({ op: "removePrunedManifestRecords", maxItems })
|
|
87
87
|
};
|
|
88
|
+
const STATE_SHRINKING_OPS = /* @__PURE__ */ new Set([
|
|
89
|
+
"rollbackTransactionArtifacts",
|
|
90
|
+
"abortTransactionIfExpired",
|
|
91
|
+
"removeLease",
|
|
92
|
+
"removeLeaseIfExpired",
|
|
93
|
+
"removeTempRun",
|
|
94
|
+
"removeTempOwner",
|
|
95
|
+
"removeTempOwnerIfExpired",
|
|
96
|
+
"removeCompactionJob",
|
|
97
|
+
"cancelCompactionJob",
|
|
98
|
+
"removeGarbageCollectionJob",
|
|
99
|
+
"removePrunedManifestRecords",
|
|
100
|
+
"removeAbortedSegment",
|
|
101
|
+
"abortFtsBaseBuild",
|
|
102
|
+
"abortUniqueKeyBuild",
|
|
103
|
+
"cancelSnapshotFrameImport",
|
|
104
|
+
"closeSnapshotFrameExport",
|
|
105
|
+
"garbageCollectionStep"
|
|
106
|
+
]);
|
|
107
|
+
function shrinksLoggedState(body) {
|
|
108
|
+
if (body.op === "updateTransaction")
|
|
109
|
+
return body.update.status === "aborted";
|
|
110
|
+
return STATE_SHRINKING_OPS.has(body.op);
|
|
111
|
+
}
|
|
112
|
+
class OpfsLeaderClosedError extends Error {
|
|
113
|
+
name = "OpfsLeaderClosedError";
|
|
114
|
+
constructor() {
|
|
115
|
+
super("This OPFS store connection is closed");
|
|
116
|
+
}
|
|
117
|
+
}
|
|
88
118
|
class OpfsLeader {
|
|
89
119
|
#tree;
|
|
90
120
|
#strict;
|
|
@@ -425,7 +455,7 @@ class OpfsLeader {
|
|
|
425
455
|
#run(work) {
|
|
426
456
|
const result = this.#chain.then(async () => {
|
|
427
457
|
if (this.#closed)
|
|
428
|
-
throw new
|
|
458
|
+
throw new OpfsLeaderClosedError();
|
|
429
459
|
if (this.#poisoned)
|
|
430
460
|
await this.#loadFromDisk();
|
|
431
461
|
return work();
|
|
@@ -470,11 +500,13 @@ class OpfsLeader {
|
|
|
470
500
|
return this.#servedLedger.get(key);
|
|
471
501
|
}
|
|
472
502
|
async completeServed(key, result) {
|
|
473
|
-
|
|
474
|
-
if (entry === void 0 || entry.settled)
|
|
503
|
+
if (!this.#servedLedger.has(key))
|
|
475
504
|
return;
|
|
476
505
|
const bytes = estimateRpcValueBytes(result);
|
|
477
506
|
await this.#run(() => {
|
|
507
|
+
const entry = this.#servedLedger.get(key);
|
|
508
|
+
if (entry === void 0 || entry.settled)
|
|
509
|
+
return;
|
|
478
510
|
if (bytes > this.#servedLedgerResultBytes) {
|
|
479
511
|
this.#appendFrame({ op: "servedResult", key, withheld: true }, false);
|
|
480
512
|
entry.settled = true;
|
|
@@ -499,7 +531,13 @@ class OpfsLeader {
|
|
|
499
531
|
}
|
|
500
532
|
#logged(body) {
|
|
501
533
|
if (this.#wal.byteLength >= MAX_OPFS_WAL_BYTES - 64 * 1024 * 1024) {
|
|
502
|
-
|
|
534
|
+
try {
|
|
535
|
+
this.checkpointNow();
|
|
536
|
+
} catch (error) {
|
|
537
|
+
if (!shrinksLoggedState(body) || this.#wal.byteLength >= MAX_OPFS_WAL_BYTES - 16 * 1024 * 1024) {
|
|
538
|
+
throw error;
|
|
539
|
+
}
|
|
540
|
+
}
|
|
503
541
|
}
|
|
504
542
|
if (this.#strict) {
|
|
505
543
|
try {
|
|
@@ -1569,7 +1607,7 @@ class OpfsLeader {
|
|
|
1569
1607
|
};
|
|
1570
1608
|
const bytes = encodeSyncCheckpoint(state);
|
|
1571
1609
|
if (bytes.byteLength > MAX_OPFS_CHECKPOINT_BYTES) {
|
|
1572
|
-
throw new
|
|
1610
|
+
throw new StorageResourceLimitError("checkpoint byte", bytes.byteLength, MAX_OPFS_CHECKPOINT_BYTES);
|
|
1573
1611
|
}
|
|
1574
1612
|
const slotIndex = this.#newestSlot === 0 ? 1 : 0;
|
|
1575
1613
|
const mirrorIndex = slotIndex === 0 ? 1 : 0;
|
|
@@ -4802,5 +4840,6 @@ export {
|
|
|
4802
4840
|
MAX_OPFS_CHECKPOINT_BYTES,
|
|
4803
4841
|
MAX_OPFS_WAL_BYTES,
|
|
4804
4842
|
OpfsLeader,
|
|
4843
|
+
OpfsLeaderClosedError,
|
|
4805
4844
|
assertBlockReadBatchByteLimit
|
|
4806
4845
|
};
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
class PowerLossModel {
|
|
2
|
+
#durable = /* @__PURE__ */ new Map();
|
|
3
|
+
#touched = /* @__PURE__ */ new Set();
|
|
4
|
+
#shim;
|
|
5
|
+
constructor(shim) {
|
|
6
|
+
this.#shim = shim;
|
|
7
|
+
shim.setWriteFault((path, phase) => {
|
|
8
|
+
if (phase === "flush") {
|
|
9
|
+
this.#durable.set(path, shim.readFileBytes(path) ?? new Uint8Array());
|
|
10
|
+
} else {
|
|
11
|
+
this.#touched.add(path);
|
|
12
|
+
}
|
|
13
|
+
});
|
|
14
|
+
}
|
|
15
|
+
durableBytes(path) {
|
|
16
|
+
return this.#durable.get(path);
|
|
17
|
+
}
|
|
18
|
+
powerLoss(keepPrefix) {
|
|
19
|
+
const reverted = [];
|
|
20
|
+
for (const path of this.#touched) {
|
|
21
|
+
const current = this.#shim.readFileBytes(path);
|
|
22
|
+
if (current === void 0)
|
|
23
|
+
continue;
|
|
24
|
+
const durable = this.#durable.get(path) ?? new Uint8Array();
|
|
25
|
+
let kept = durable;
|
|
26
|
+
if (keepPrefix !== void 0 && current.byteLength > durable.byteLength && startsWith(current, durable)) {
|
|
27
|
+
const unflushed = current.byteLength - durable.byteLength;
|
|
28
|
+
const extra = keepPrefix(unflushed, path);
|
|
29
|
+
kept = current.slice(0, durable.byteLength + Math.max(0, Math.min(extra, unflushed)));
|
|
30
|
+
}
|
|
31
|
+
if (!bytesEqual(kept, current)) {
|
|
32
|
+
this.#shim.writeFileBytes(path, kept);
|
|
33
|
+
reverted.push(path);
|
|
34
|
+
}
|
|
35
|
+
this.#durable.set(path, kept.slice());
|
|
36
|
+
}
|
|
37
|
+
this.#touched.clear();
|
|
38
|
+
return reverted;
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
function startsWith(bytes, prefix) {
|
|
42
|
+
if (prefix.byteLength > bytes.byteLength)
|
|
43
|
+
return false;
|
|
44
|
+
for (let index = 0; index < prefix.byteLength; index += 1) {
|
|
45
|
+
if (bytes[index] !== prefix[index])
|
|
46
|
+
return false;
|
|
47
|
+
}
|
|
48
|
+
return true;
|
|
49
|
+
}
|
|
50
|
+
function bytesEqual(left, right) {
|
|
51
|
+
if (left.byteLength !== right.byteLength)
|
|
52
|
+
return false;
|
|
53
|
+
for (let index = 0; index < left.byteLength; index += 1) {
|
|
54
|
+
if (left[index] !== right[index])
|
|
55
|
+
return false;
|
|
56
|
+
}
|
|
57
|
+
return true;
|
|
58
|
+
}
|
|
59
|
+
export {
|
|
60
|
+
PowerLossModel,
|
|
61
|
+
bytesEqual
|
|
62
|
+
};
|
package/dist/storage/opfs/rpc.js
CHANGED
|
@@ -109,6 +109,7 @@ function parseStoreRpcMessage(value, knownMethods) {
|
|
|
109
109
|
case "state":
|
|
110
110
|
return exactKeys(record, ["kind", "leaderId", "foreground"]) && boundedRpcString(record.leaderId) && typeof record.foreground === "boolean" ? record : void 0;
|
|
111
111
|
case "leader":
|
|
112
|
+
case "wait":
|
|
112
113
|
case "released":
|
|
113
114
|
return exactKeys(record, ["kind", "leaderId"]) && boundedRpcString(record.leaderId) ? record : void 0;
|
|
114
115
|
case "ping":
|
|
@@ -27,6 +27,8 @@ export interface OpfsBlockStoreOptions {
|
|
|
27
27
|
hiddenIdleReleaseMs?: number;
|
|
28
28
|
/** @internal Test seam: how long an ex-leader stays out of elections (default 1500ms). */
|
|
29
29
|
handoverGraceMs?: number;
|
|
30
|
+
/** @internal Test seam: how long an operation looks for a leader (default 10000ms). */
|
|
31
|
+
dispatchBudgetMs?: number;
|
|
30
32
|
/**
|
|
31
33
|
* Hears failures no operation reports: a background checkpoint or cleanup that failed, an
|
|
32
34
|
* election or handover that threw, a served request that could not be answered. The worker
|
|
@@ -108,3 +110,13 @@ export declare function deleteOpfsDatabase(options: {
|
|
|
108
110
|
name: string;
|
|
109
111
|
root?: FileSystemDirectoryHandle;
|
|
110
112
|
}): Promise<void>;
|
|
113
|
+
/**
|
|
114
|
+
* Whether a database directory of this name exists in the origin's private file system (or
|
|
115
|
+
* under `root`). Nothing is created: an `auto` descriptor asks this before choosing a store for
|
|
116
|
+
* a name it has no record of, so a database an explicit `opfs` descriptor created is found
|
|
117
|
+
* rather than shadowed by an empty one.
|
|
118
|
+
*/
|
|
119
|
+
export declare function opfsDatabaseExists(options: {
|
|
120
|
+
name: string;
|
|
121
|
+
root?: FileSystemDirectoryHandle;
|
|
122
|
+
}): Promise<boolean>;
|
|
@@ -2,11 +2,11 @@ import { assertStorageBulkReadItems, assertTempRunPageBatchLimits, OpfsCoordinat
|
|
|
2
2
|
import { validateTempRunPage, validateTempRunPageIdentity } from "../toolkit/record-core.js";
|
|
3
3
|
import { OpfsTree, encodeSegment, isDomError } from "./files.js";
|
|
4
4
|
import { LOG_FORMAT_VERSION } from "../toolkit/wire.js";
|
|
5
|
-
import { OpfsLeader } from "./leader.js";
|
|
5
|
+
import { OpfsLeader, OpfsLeaderClosedError } from "./leader.js";
|
|
6
6
|
import { rehydrateStoreError, estimateRpcValueBytes, fingerprintStoreRequest, MAX_OPFS_RPC_HOLD_MS, MAX_OPFS_RPC_MESSAGE_BYTES, parseStoreRpcMessage, serializeStoreError } from "./rpc.js";
|
|
7
7
|
const RPC_TIMEOUT_MS = 1e3;
|
|
8
8
|
const DISCOVERY_WAIT_MS = 150;
|
|
9
|
-
const
|
|
9
|
+
const DISPATCH_BUDGET_MS = 1e4;
|
|
10
10
|
const YIELD_COOLDOWN_MS = 3e3;
|
|
11
11
|
const HANDOVER_GRACE_MS = 1500;
|
|
12
12
|
const HIDDEN_IDLE_RELEASE_MS = 15e3;
|
|
@@ -185,6 +185,7 @@ class OpfsBlockStore {
|
|
|
185
185
|
#yieldCooldownMs;
|
|
186
186
|
#hiddenIdleReleaseMs;
|
|
187
187
|
#handoverGraceMs;
|
|
188
|
+
#dispatchBudgetMs;
|
|
188
189
|
#onDiagnostic;
|
|
189
190
|
#instanceId = crypto.randomUUID();
|
|
190
191
|
#channelName;
|
|
@@ -197,6 +198,8 @@ class OpfsBlockStore {
|
|
|
197
198
|
#closed = false;
|
|
198
199
|
#lastYieldAt = 0;
|
|
199
200
|
#electing;
|
|
201
|
+
#recovering = false;
|
|
202
|
+
#waitHeardAt = 0;
|
|
200
203
|
#pending = /* @__PURE__ */ new Map();
|
|
201
204
|
#inFlightMutations = /* @__PURE__ */ new Map();
|
|
202
205
|
#settledMutations = /* @__PURE__ */ new Map();
|
|
@@ -230,6 +233,7 @@ class OpfsBlockStore {
|
|
|
230
233
|
this.#yieldCooldownMs = options.yieldCooldownMs ?? YIELD_COOLDOWN_MS;
|
|
231
234
|
this.#hiddenIdleReleaseMs = options.hiddenIdleReleaseMs ?? HIDDEN_IDLE_RELEASE_MS;
|
|
232
235
|
this.#handoverGraceMs = options.handoverGraceMs ?? HANDOVER_GRACE_MS;
|
|
236
|
+
this.#dispatchBudgetMs = options.dispatchBudgetMs ?? DISPATCH_BUDGET_MS;
|
|
233
237
|
this.#onDiagnostic = options.onDiagnostic;
|
|
234
238
|
this.#channelName = `minnowdb-store:${options.name}`;
|
|
235
239
|
this.liveQueryChannelName = `minnowdb-live:opfs:${options.name}`;
|
|
@@ -311,7 +315,9 @@ class OpfsBlockStore {
|
|
|
311
315
|
}
|
|
312
316
|
let slotA;
|
|
313
317
|
let slotB;
|
|
318
|
+
this.#recovering = true;
|
|
314
319
|
try {
|
|
320
|
+
this.#post({ kind: "wait", leaderId: this.#instanceId });
|
|
315
321
|
slotA = await this.#openWithRetry(["checkpoint-a"]);
|
|
316
322
|
slotB = await this.#openWithRetry(["checkpoint-b"]);
|
|
317
323
|
this.#leader = await OpfsLeader.recover(this.#tree, this.#durability === "strict", { wal, slotA, slotB }, this.#checkpointEntries, this.#cleanupLimitBytes, this.#onDiagnostic, this.#servedLedgerAgeMs, this.#servedLedgerResultBytes);
|
|
@@ -322,6 +328,8 @@ class OpfsBlockStore {
|
|
|
322
328
|
if (isLockContention(error))
|
|
323
329
|
return false;
|
|
324
330
|
throw error;
|
|
331
|
+
} finally {
|
|
332
|
+
this.#recovering = false;
|
|
325
333
|
}
|
|
326
334
|
if (this.#closed) {
|
|
327
335
|
const leader = this.#leader;
|
|
@@ -409,7 +417,7 @@ class OpfsBlockStore {
|
|
|
409
417
|
return;
|
|
410
418
|
this.#leader = void 0;
|
|
411
419
|
this.#knownLeader = void 0;
|
|
412
|
-
const shutdown = leader
|
|
420
|
+
const shutdown = this.#shutdownAfterMutations(leader).catch((error) => {
|
|
413
421
|
this.#diagnostic(error, "opfs idle release shutdown");
|
|
414
422
|
leader.crash();
|
|
415
423
|
});
|
|
@@ -494,7 +502,7 @@ class OpfsBlockStore {
|
|
|
494
502
|
this.#leader = void 0;
|
|
495
503
|
this.#knownLeader = void 0;
|
|
496
504
|
this.#lastYieldAt = Date.now();
|
|
497
|
-
const shutdown = leader
|
|
505
|
+
const shutdown = this.#shutdownAfterMutations(leader).catch((error) => {
|
|
498
506
|
this.#diagnostic(error, "opfs yield shutdown");
|
|
499
507
|
leader.crash();
|
|
500
508
|
});
|
|
@@ -516,6 +524,14 @@ class OpfsBlockStore {
|
|
|
516
524
|
}, this.#handoverGraceMs);
|
|
517
525
|
}
|
|
518
526
|
#yielding;
|
|
527
|
+
#shutdownAfterMutations(leader) {
|
|
528
|
+
return this.#withMutationTurn(() => leader.shutdown());
|
|
529
|
+
}
|
|
530
|
+
async #servedDrained() {
|
|
531
|
+
const deadline = Date.now() + DECLINE_AFTER_CLOSE_MS;
|
|
532
|
+
while (this.#served.size > 0 && Date.now() < deadline)
|
|
533
|
+
await sleep(5);
|
|
534
|
+
}
|
|
519
535
|
#onMessage(message) {
|
|
520
536
|
if (this.#closed) {
|
|
521
537
|
if (message.kind === "op")
|
|
@@ -645,9 +661,16 @@ class OpfsBlockStore {
|
|
|
645
661
|
case "ping": {
|
|
646
662
|
if (this.#leader !== void 0) {
|
|
647
663
|
this.#post({ kind: "leader", leaderId: this.#instanceId });
|
|
664
|
+
} else if (this.#recovering || this.#yielding !== void 0) {
|
|
665
|
+
this.#post({ kind: "wait", leaderId: this.#instanceId });
|
|
648
666
|
}
|
|
649
667
|
return;
|
|
650
668
|
}
|
|
669
|
+
case "wait": {
|
|
670
|
+
if (message.leaderId !== this.#instanceId)
|
|
671
|
+
this.#waitHeardAt = Date.now();
|
|
672
|
+
return;
|
|
673
|
+
}
|
|
651
674
|
case "bid": {
|
|
652
675
|
if (message.foreground && message.bidderId !== this.#instanceId) {
|
|
653
676
|
this.#considerBid(message.bidderId);
|
|
@@ -1036,7 +1059,7 @@ class OpfsBlockStore {
|
|
|
1036
1059
|
let sentRemotely = false;
|
|
1037
1060
|
let mayHaveRun = false;
|
|
1038
1061
|
this.#lastActivityAt = sentAt;
|
|
1039
|
-
|
|
1062
|
+
while (Date.now() - Math.max(sentAt, this.#waitHeardAt) < this.#dispatchBudgetMs) {
|
|
1040
1063
|
this.#assertOpen();
|
|
1041
1064
|
const leader = this.#leader;
|
|
1042
1065
|
if (leader !== void 0) {
|
|
@@ -1058,7 +1081,16 @@ class OpfsBlockStore {
|
|
|
1058
1081
|
if (sentAt < leader.servedCoverageSince)
|
|
1059
1082
|
throw new OpfsUncertainOutcomeError(method);
|
|
1060
1083
|
}
|
|
1061
|
-
return await this.#withMutationTurn(() =>
|
|
1084
|
+
return await this.#withMutationTurn(() => {
|
|
1085
|
+
if (leader.isClosed() || !this.#leads(leader))
|
|
1086
|
+
throw new OpfsLeaderClosedError();
|
|
1087
|
+
return bound.apply(leader, args);
|
|
1088
|
+
});
|
|
1089
|
+
} catch (error) {
|
|
1090
|
+
if (!this.#leads(leader) && (isRead || error instanceof OpfsLeaderClosedError)) {
|
|
1091
|
+
continue;
|
|
1092
|
+
}
|
|
1093
|
+
throw error;
|
|
1062
1094
|
} finally {
|
|
1063
1095
|
this.#lastActivityAt = Date.now();
|
|
1064
1096
|
}
|
|
@@ -1187,10 +1219,11 @@ class OpfsBlockStore {
|
|
|
1187
1219
|
const leader = this.#leader;
|
|
1188
1220
|
this.#leader = void 0;
|
|
1189
1221
|
if (leader !== void 0) {
|
|
1190
|
-
void leader
|
|
1222
|
+
void this.#shutdownAfterMutations(leader).catch((error) => {
|
|
1191
1223
|
this.#diagnostic(error, "opfs close shutdown");
|
|
1192
1224
|
leader.crash();
|
|
1193
|
-
}).then(() => {
|
|
1225
|
+
}).then(() => this.#servedDrained()).then(() => {
|
|
1226
|
+
this.#stopKeepalive();
|
|
1194
1227
|
this.#post({ kind: "released", leaderId: this.#instanceId });
|
|
1195
1228
|
this.#closeChannels();
|
|
1196
1229
|
this.#releaseWhenHandlesClose();
|
|
@@ -1198,13 +1231,15 @@ class OpfsBlockStore {
|
|
|
1198
1231
|
return;
|
|
1199
1232
|
}
|
|
1200
1233
|
if (this.#yielding !== void 0) {
|
|
1201
|
-
void this.#yielding.then(() => {
|
|
1234
|
+
void this.#yielding.then(() => this.#servedDrained()).then(() => {
|
|
1235
|
+
this.#stopKeepalive();
|
|
1202
1236
|
this.#post({ kind: "released", leaderId: this.#instanceId });
|
|
1203
1237
|
this.#closeChannels();
|
|
1204
1238
|
this.#releaseWhenHandlesClose();
|
|
1205
1239
|
});
|
|
1206
1240
|
return;
|
|
1207
1241
|
}
|
|
1242
|
+
this.#stopKeepalive();
|
|
1208
1243
|
this.#closeChannels();
|
|
1209
1244
|
this.#releaseWhenHandlesClose();
|
|
1210
1245
|
}
|
|
@@ -1212,10 +1247,8 @@ class OpfsBlockStore {
|
|
|
1212
1247
|
if (this.#reacquireTimer !== void 0)
|
|
1213
1248
|
clearTimeout(this.#reacquireTimer);
|
|
1214
1249
|
this.#reacquireTimer = void 0;
|
|
1215
|
-
this.#stopKeepalive();
|
|
1216
1250
|
this.#clearDeferredBid();
|
|
1217
1251
|
this.#clearHiddenIdleTimer();
|
|
1218
|
-
this.#served.clear();
|
|
1219
1252
|
}
|
|
1220
1253
|
#releaseWhenHandlesClose() {
|
|
1221
1254
|
const release = this.#releaseConnectionLock;
|
|
@@ -1275,6 +1308,8 @@ class OpfsBlockStore {
|
|
|
1275
1308
|
_crashForTests() {
|
|
1276
1309
|
this.#closed = true;
|
|
1277
1310
|
this.#clearCoordinationTimers();
|
|
1311
|
+
this.#stopKeepalive();
|
|
1312
|
+
this.#served.clear();
|
|
1278
1313
|
for (const pending of this.#pending.values())
|
|
1279
1314
|
clearTimeout(pending.timer);
|
|
1280
1315
|
this.#pending.clear();
|
|
@@ -1423,6 +1458,17 @@ async function resolveDatabaseRoot(options) {
|
|
|
1423
1458
|
const namespace = await root.getDirectoryHandle("minnowdb", { create: true });
|
|
1424
1459
|
return namespace.getDirectoryHandle(encodedName, { create: true });
|
|
1425
1460
|
}
|
|
1461
|
+
async function opfsDatabaseExists(options) {
|
|
1462
|
+
const encodedName = encodeSegment(validateStorageDatabaseName(options.name));
|
|
1463
|
+
try {
|
|
1464
|
+
const root = options.root ?? await navigator.storage.getDirectory();
|
|
1465
|
+
const namespace = await root.getDirectoryHandle("minnowdb");
|
|
1466
|
+
await namespace.getDirectoryHandle(encodedName);
|
|
1467
|
+
return true;
|
|
1468
|
+
} catch {
|
|
1469
|
+
return false;
|
|
1470
|
+
}
|
|
1471
|
+
}
|
|
1426
1472
|
function isLockContention(error) {
|
|
1427
1473
|
return isDomError(error, "NoModificationAllowedError") || isDomError(error, "InvalidStateError");
|
|
1428
1474
|
}
|
|
@@ -1431,5 +1477,6 @@ function sleep(ms) {
|
|
|
1431
1477
|
}
|
|
1432
1478
|
export {
|
|
1433
1479
|
OpfsBlockStore,
|
|
1434
|
-
deleteOpfsDatabase
|
|
1480
|
+
deleteOpfsDatabase,
|
|
1481
|
+
opfsDatabaseExists
|
|
1435
1482
|
};
|
|
@@ -79,6 +79,8 @@ function* iterateWalFrames(handle) {
|
|
|
79
79
|
while (offset + FRAME_HEADER_BYTES <= size) {
|
|
80
80
|
readFully(handle, header, offset, "reading a WAL frame header for recovery");
|
|
81
81
|
if (headerView.getUint32(0, true) !== FRAME_MAGIC) {
|
|
82
|
+
if (isZeroFilled(handle, offset, size))
|
|
83
|
+
break;
|
|
82
84
|
throw new Error(`WAL frame marker mismatch at offset ${String(offset)}`);
|
|
83
85
|
}
|
|
84
86
|
const length = headerView.getUint32(4, true);
|
|
@@ -92,12 +94,26 @@ function* iterateWalFrames(handle) {
|
|
|
92
94
|
const payloadBytes = new Uint8Array(length);
|
|
93
95
|
readFully(handle, payloadBytes, offset + FRAME_HEADER_BYTES, "reading a WAL frame payload for recovery");
|
|
94
96
|
if (crc32(payloadBytes) !== checksum) {
|
|
97
|
+
if (payloadBytes.every((byte) => byte === 0) && isZeroFilled(handle, end, size))
|
|
98
|
+
break;
|
|
95
99
|
throw new Error(`WAL frame checksum mismatch at offset ${String(offset)}`);
|
|
96
100
|
}
|
|
97
101
|
yield { payload: decodeRecordJson(payloadBytes), frameEnd: end };
|
|
98
102
|
offset = end;
|
|
99
103
|
}
|
|
100
104
|
}
|
|
105
|
+
function isZeroFilled(handle, offset, size) {
|
|
106
|
+
const chunk = new Uint8Array(Math.min(64 * 1024, Math.max(0, size - offset)));
|
|
107
|
+
for (let at = offset; at < size; at += chunk.byteLength) {
|
|
108
|
+
const length = Math.min(chunk.byteLength, size - at);
|
|
109
|
+
const window = length === chunk.byteLength ? chunk : chunk.subarray(0, length);
|
|
110
|
+
readFully(handle, window, at, "reading a WAL tail for recovery");
|
|
111
|
+
for (let index = 0; index < length; index += 1)
|
|
112
|
+
if (window[index] !== 0)
|
|
113
|
+
return false;
|
|
114
|
+
}
|
|
115
|
+
return true;
|
|
116
|
+
}
|
|
101
117
|
export {
|
|
102
118
|
MAX_WAL_FRAME_BYTES,
|
|
103
119
|
WalWriter,
|
|
@@ -58,15 +58,15 @@ function decodeEnvelope(magic, bytes) {
|
|
|
58
58
|
}
|
|
59
59
|
const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
|
|
60
60
|
const version = view.getUint32(8, true);
|
|
61
|
-
if (version !== LOG_FORMAT_VERSION) {
|
|
62
|
-
throw new StorageFormatVersionError("opfs", `envelope/${magic}`, version, LOG_FORMAT_VERSION, version < LOG_FORMAT_VERSION ? "older" : "newer");
|
|
63
|
-
}
|
|
64
61
|
const payloadLength = view.getUint32(12, true);
|
|
65
62
|
if (bytes.byteLength !== ENVELOPE_HEADER_BYTES + payloadLength)
|
|
66
63
|
return void 0;
|
|
67
64
|
const payload = bytes.subarray(ENVELOPE_HEADER_BYTES, ENVELOPE_HEADER_BYTES + payloadLength);
|
|
68
65
|
if (crc32(payload) !== view.getUint32(16, true))
|
|
69
66
|
return void 0;
|
|
67
|
+
if (version !== LOG_FORMAT_VERSION) {
|
|
68
|
+
throw new StorageFormatVersionError("opfs", `envelope/${magic}`, version, LOG_FORMAT_VERSION, version < LOG_FORMAT_VERSION ? "older" : "newer");
|
|
69
|
+
}
|
|
70
70
|
return payload;
|
|
71
71
|
}
|
|
72
72
|
const CHUNK_MAGIC = "MNWCHNK1";
|
package/dist/storage/types.d.ts
CHANGED
|
@@ -502,11 +502,11 @@ export declare const MAX_PINNED_RETIRED_BYTES: number;
|
|
|
502
502
|
export declare const MAX_RETIRED_HISTORY_BYTES: number;
|
|
503
503
|
/** A durable resource family reached its fixed corruption/growth safety ceiling. */
|
|
504
504
|
export declare class StorageResourceLimitError extends Error {
|
|
505
|
-
readonly resource: "lease" | "temp owner" | "temp run" | "temp page" | "temporary run total" | "temporary page total" | "temp owner byte" | "temporary byte" | "compaction job" | "garbage collection job" | "unique-key build" | "full-text build" | "secondary-index build" | "accelerator build byte" | "accelerator build entry" | "transaction" | "staged artifact byte" | "staged block" | "staged segment" | "catalog record" | "catalog byte" | "manifest record" | "manifest byte" | "segment record" | "segment byte" | "terminal transaction" | "terminal compaction job" | "completed garbage collection job" | "pinned manifest version lag" | "pinned retired block" | "pinned retired byte" | "snapshot accelerator byte" | "snapshot accelerator entry" | "retired history byte";
|
|
505
|
+
readonly resource: "lease" | "temp owner" | "temp run" | "temp page" | "temporary run total" | "temporary page total" | "temp owner byte" | "temporary byte" | "compaction job" | "garbage collection job" | "unique-key build" | "full-text build" | "secondary-index build" | "accelerator build byte" | "accelerator build entry" | "transaction" | "staged artifact byte" | "staged block" | "staged segment" | "catalog record" | "catalog byte" | "manifest record" | "manifest byte" | "segment record" | "segment byte" | "terminal transaction" | "terminal compaction job" | "completed garbage collection job" | "pinned manifest version lag" | "pinned retired block" | "pinned retired byte" | "snapshot accelerator byte" | "snapshot accelerator entry" | "retired history byte" | "checkpoint byte";
|
|
506
506
|
readonly count: number;
|
|
507
507
|
readonly limit: number;
|
|
508
508
|
readonly name = "StorageResourceLimitError";
|
|
509
|
-
constructor(resource: "lease" | "temp owner" | "temp run" | "temp page" | "temporary run total" | "temporary page total" | "temp owner byte" | "temporary byte" | "compaction job" | "garbage collection job" | "unique-key build" | "full-text build" | "secondary-index build" | "accelerator build byte" | "accelerator build entry" | "transaction" | "staged artifact byte" | "staged block" | "staged segment" | "catalog record" | "catalog byte" | "manifest record" | "manifest byte" | "segment record" | "segment byte" | "terminal transaction" | "terminal compaction job" | "completed garbage collection job" | "pinned manifest version lag" | "pinned retired block" | "pinned retired byte" | "snapshot accelerator byte" | "snapshot accelerator entry" | "retired history byte", count: number, limit: number);
|
|
509
|
+
constructor(resource: "lease" | "temp owner" | "temp run" | "temp page" | "temporary run total" | "temporary page total" | "temp owner byte" | "temporary byte" | "compaction job" | "garbage collection job" | "unique-key build" | "full-text build" | "secondary-index build" | "accelerator build byte" | "accelerator build entry" | "transaction" | "staged artifact byte" | "staged block" | "staged segment" | "catalog record" | "catalog byte" | "manifest record" | "manifest byte" | "segment record" | "segment byte" | "terminal transaction" | "terminal compaction job" | "completed garbage collection job" | "pinned manifest version lag" | "pinned retired block" | "pinned retired byte" | "snapshot accelerator byte" | "snapshot accelerator entry" | "retired history byte" | "checkpoint byte", count: number, limit: number);
|
|
510
510
|
}
|
|
511
511
|
export interface LeaseRecord {
|
|
512
512
|
id: string;
|
|
@@ -145,6 +145,13 @@ export declare class DatabaseTransaction {
|
|
|
145
145
|
get compactionSourceBlockIds(): string[];
|
|
146
146
|
/** Extends durable ownership; scope owners call this while user code is legitimately waiting. */
|
|
147
147
|
renew(): Promise<void>;
|
|
148
|
+
/**
|
|
149
|
+
* Renews the lease and record only when a third of their lifetime has gone: an O(1) deadline
|
|
150
|
+
* check otherwise. A scope that buffers statement after statement stages nothing, so nothing
|
|
151
|
+
* renews inline, and a loop that never yields to the event loop never lets the heartbeat
|
|
152
|
+
* timer fire either — the buffered path calls this per statement instead.
|
|
153
|
+
*/
|
|
154
|
+
renewIfDue(): Promise<void>;
|
|
148
155
|
/**
|
|
149
156
|
* Counts every registration this transaction carries: staged blocks and segments, unique-key
|
|
150
157
|
* entries, full-text entries, and supersessions. A caller that fails part-way through a
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { dateIsoString, dateMilliseconds } from "../date-value.js";
|
|
2
|
-
import { assertTransactionArtifactBatchLimits, GarbageCollectionJobConflictError, LeaseConflictError, LeaseExpiredError, MAX_LEVEL_ZERO_SEGMENTS, MAX_MANIFEST_BLOCK_PRESENCE_IDS, MAX_LEASE_TTL_MS, MAX_TRANSACTION_STAGE_BLOCKS, MAX_TRANSACTION_STAGE_BYTES, MAX_TRANSACTION_STAGE_SEGMENTS, MAX_TRANSACTION_COMMIT_DELTA_BYTES, MAX_TRANSACTION_COMMIT_DELTA_ENTRIES, transactionCommitDeltaRetainedBytes, SnapshotManifestMissingError, TransactionRecordConflictError, WriteConflictError } from "../storage/types.js";
|
|
2
|
+
import { assertTransactionArtifactBatchLimits, GarbageCollectionJobConflictError, LeaseConflictError, LeaseExpiredError, MAX_LEVEL_ZERO_SEGMENTS, MAX_MANIFEST_BLOCK_PRESENCE_IDS, MAX_LEASE_TTL_MS, MAX_TRANSACTION_STAGE_BLOCKS, MAX_TRANSACTION_STAGE_BYTES, MAX_TRANSACTION_STAGE_SEGMENTS, MAX_TRANSACTION_COMMIT_DELTA_BYTES, MAX_TRANSACTION_COMMIT_DELTA_ENTRIES, transactionCommitDeltaRetainedBytes, SnapshotManifestMissingError, TransactionRecordConflictError, UnknownOutcomeError, WriteConflictError } from "../storage/types.js";
|
|
3
3
|
const DEFAULT_TRANSACTION_TTL_MS = 3e4;
|
|
4
4
|
class TransactionClosedError extends Error {
|
|
5
5
|
transactionId;
|
|
@@ -233,6 +233,11 @@ class DatabaseTransaction {
|
|
|
233
233
|
this.#assertActive();
|
|
234
234
|
await this.#renewOwnership(true);
|
|
235
235
|
}
|
|
236
|
+
async renewIfDue() {
|
|
237
|
+
if (this.#record.status !== "active")
|
|
238
|
+
return;
|
|
239
|
+
await this.#renewOwnership(false);
|
|
240
|
+
}
|
|
236
241
|
get stagedWorkCount() {
|
|
237
242
|
return this.pendingBlockCount + this.pendingSegmentCount + this.#uniqueKeyChanges.length + this.#ftsChanges.size + this.#compactionSourceBlockIds.size;
|
|
238
243
|
}
|
|
@@ -406,7 +411,7 @@ class DatabaseTransaction {
|
|
|
406
411
|
updatedAt: dateIsoString(this.now())
|
|
407
412
|
});
|
|
408
413
|
} catch (error) {
|
|
409
|
-
await this.#recoverStagedAcknowledgement(error, blocks, segments);
|
|
414
|
+
this.#record = await this.#recoverStagedAcknowledgement(error, blocks, segments);
|
|
410
415
|
}
|
|
411
416
|
this.#journalAppended(previous, blocks.map((block) => block.id));
|
|
412
417
|
}
|
|
@@ -912,7 +917,14 @@ class DatabaseTransaction {
|
|
|
912
917
|
if (error instanceof SnapshotManifestMissingError) {
|
|
913
918
|
throw new WriteConflictError(this.#record.snapshotVersion, await this.store.getCurrentManifestVersion());
|
|
914
919
|
}
|
|
915
|
-
|
|
920
|
+
const persisted = await this.store.getTransaction(this.id).catch(() => void 0);
|
|
921
|
+
if (persisted?.status !== "active" || persisted.ownerId !== this.#record.ownerId || persisted.revision !== this.#record.revision || persisted.snapshotVersion !== this.#record.snapshotVersion || persisted.pendingBlockIds.length !== 0 || persisted.pendingSegmentIds.length !== 0) {
|
|
922
|
+
throw error;
|
|
923
|
+
}
|
|
924
|
+
this.#record = persisted;
|
|
925
|
+
this.#persisted = true;
|
|
926
|
+
if (!(error instanceof UnknownOutcomeError))
|
|
927
|
+
throw error;
|
|
916
928
|
}
|
|
917
929
|
this.#persisted = true;
|
|
918
930
|
void this.#releaseSnapshotLease().catch(() => void 0);
|
|
@@ -1070,6 +1082,8 @@ class DatabaseTransaction {
|
|
|
1070
1082
|
})) {
|
|
1071
1083
|
this.#record = persisted;
|
|
1072
1084
|
this.#persisted = true;
|
|
1085
|
+
if (error instanceof UnknownOutcomeError)
|
|
1086
|
+
return persisted;
|
|
1073
1087
|
}
|
|
1074
1088
|
}
|
|
1075
1089
|
} catch {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@minnowdb/core",
|
|
3
|
-
"version": "0.10.
|
|
3
|
+
"version": "0.10.1",
|
|
4
4
|
"description": "A columnar SQL database for the browser: PostgreSQL-style SQL over durable IndexedDB or OPFS data, with no server or WebAssembly module.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"author": "Eric Wilhite",
|