@minnowdb/core 0.8.0 → 0.9.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/engine/artifact-cache.js +3 -2
- package/dist/engine/buffered-writer.js +12 -2
- package/dist/engine/client.d.ts +8 -2
- package/dist/engine/client.js +106 -22
- package/dist/engine/database.d.ts +5 -3
- package/dist/engine/database.js +2044 -1674
- package/dist/engine/errors.d.ts +19 -0
- package/dist/engine/errors.js +29 -0
- package/dist/engine/index.d.ts +1 -0
- package/dist/engine/index.js +4 -0
- package/dist/engine/live-accept.js +13 -0
- package/dist/engine/live-aggregate.js +54 -16
- package/dist/engine/live.d.ts +2 -0
- package/dist/engine/live.js +109 -108
- package/dist/engine/query-cache.js +17 -2
- package/dist/engine/query.d.ts +1 -2
- package/dist/engine/query.js +4 -7
- package/dist/engine/result-state.d.ts +7 -0
- package/dist/engine/result-state.js +15 -0
- package/dist/engine/vector.d.ts +4 -0
- package/dist/engine/vector.js +14 -14
- package/dist/engine/windows.js +15 -14
- package/dist/engine/worker-server.d.ts +1 -1
- package/dist/engine/worker-server.js +28 -15
- package/dist/engine/write-coordinator.js +54 -0
- package/dist/storage/indexeddb.js +134 -89
- package/dist/storage/opfs/index.d.ts +1 -1
- package/dist/storage/opfs/index.js +3 -1
- package/dist/storage/opfs/leader.js +20 -9
- package/dist/storage/opfs/rpc.js +3 -1
- package/dist/storage/opfs/store.d.ts +2 -0
- package/dist/storage/opfs/store.js +128 -40
- package/dist/storage/toolkit/record-core.js +2 -1
- package/dist/storage/types.d.ts +14 -0
- package/dist/storage/types.js +21 -0
- package/dist/transactions/index.d.ts +3 -0
- package/dist/transactions/index.js +4 -1
- package/dist/worker-protocol/index.d.ts +1 -1
- package/dist/worker-protocol/index.js +1 -1
- package/package.json +2 -2
- package/dist/date-value.d.ts +0 -20
- package/dist/engine/artifact-cache.d.ts +0 -29
- package/dist/engine/byte-estimates.d.ts +0 -11
- package/dist/engine/cancellation.d.ts +0 -2
- package/dist/engine/defaults.d.ts +0 -29
- package/dist/engine/group-index.d.ts +0 -33
- package/dist/engine/join-index.d.ts +0 -10
- package/dist/engine/live-aggregate.d.ts +0 -12
- package/dist/engine/live-equal.d.ts +0 -7
- package/dist/engine/point-read.d.ts +0 -59
- package/dist/engine/query-cache.d.ts +0 -21
- package/dist/engine/query-generations.d.ts +0 -8
- package/dist/engine/query-identity.d.ts +0 -3
- package/dist/engine/result-wire.d.ts +0 -70
- package/dist/engine/sort-keys.d.ts +0 -73
- package/dist/engine/sql-domains.d.ts +0 -97
- package/dist/engine/sql-functions.d.ts +0 -11
- package/dist/engine/sql-json.d.ts +0 -40
- package/dist/engine/sql-semantics.d.ts +0 -66
- package/dist/engine/worker-store-indexeddb.d.ts +0 -2
- package/dist/engine/worker-store-memory.d.ts +0 -2
- package/dist/engine/worker-store-opfs.d.ts +0 -2
- package/dist/engine/write-block-planner.d.ts +0 -19
- package/dist/storage/opfs/leader.d.ts +0 -460
- package/dist/storage/opfs/rpc.d.ts +0 -82
- package/dist/storage/opfs/snapshot-ledger.d.ts +0 -41
|
@@ -1774,7 +1774,8 @@ class OpfsLeader {
|
|
|
1774
1774
|
}
|
|
1775
1775
|
async getBlock(id) {
|
|
1776
1776
|
validateId(id);
|
|
1777
|
-
|
|
1777
|
+
while (this.#poisoned)
|
|
1778
|
+
await this.#healthy();
|
|
1778
1779
|
const placement = this.#blockIndex.get(id);
|
|
1779
1780
|
if (placement === void 0)
|
|
1780
1781
|
return void 0;
|
|
@@ -1784,13 +1785,15 @@ class OpfsLeader {
|
|
|
1784
1785
|
assertStorageBulkReadItems(ids, "Block read");
|
|
1785
1786
|
for (const id of ids)
|
|
1786
1787
|
validateId(id);
|
|
1787
|
-
|
|
1788
|
+
while (this.#poisoned)
|
|
1789
|
+
await this.#healthy();
|
|
1788
1790
|
const placements = ids.map((id) => this.#blockIndex.get(id));
|
|
1789
1791
|
assertBlockReadBatchByteLimit(placements);
|
|
1790
1792
|
return Promise.all(placements.map(async (placement) => placement === void 0 ? void 0 : await this.#pool.read(placement)));
|
|
1791
1793
|
}
|
|
1792
1794
|
async readManifestBlock(version, id) {
|
|
1793
|
-
|
|
1795
|
+
while (this.#poisoned)
|
|
1796
|
+
await this.#healthy();
|
|
1794
1797
|
if (this.#core.hasManifestBlocks(version, [id])[0] !== true)
|
|
1795
1798
|
return void 0;
|
|
1796
1799
|
const placement = this.#blockIndex.get(id);
|
|
@@ -1804,18 +1807,21 @@ class OpfsLeader {
|
|
|
1804
1807
|
}
|
|
1805
1808
|
}
|
|
1806
1809
|
async hasManifestBlocks(version, ids) {
|
|
1807
|
-
|
|
1810
|
+
while (this.#poisoned)
|
|
1811
|
+
await this.#healthy();
|
|
1808
1812
|
if (ids.length > MAX_MANIFEST_BLOCK_PRESENCE_IDS) {
|
|
1809
1813
|
throw new RangeError(`Manifest block presence accepts at most ${String(MAX_MANIFEST_BLOCK_PRESENCE_IDS)} ids`);
|
|
1810
1814
|
}
|
|
1811
1815
|
return this.#core.hasManifestBlocks(version, ids);
|
|
1812
1816
|
}
|
|
1813
1817
|
async listManifestBlockPage(input) {
|
|
1814
|
-
|
|
1818
|
+
while (this.#poisoned)
|
|
1819
|
+
await this.#healthy();
|
|
1815
1820
|
return this.#core.listManifestBlockPage(input);
|
|
1816
1821
|
}
|
|
1817
1822
|
async listRetiredManifestBlockPage(input) {
|
|
1818
|
-
|
|
1823
|
+
while (this.#poisoned)
|
|
1824
|
+
await this.#healthy();
|
|
1819
1825
|
return this.#core.listRetiredManifestBlockPage(input);
|
|
1820
1826
|
}
|
|
1821
1827
|
async putTempRunPage(page) {
|
|
@@ -1894,7 +1900,8 @@ class OpfsLeader {
|
|
|
1894
1900
|
});
|
|
1895
1901
|
}
|
|
1896
1902
|
async _readCoreGenerated(method, args) {
|
|
1897
|
-
|
|
1903
|
+
while (this.#poisoned)
|
|
1904
|
+
await this.#healthy();
|
|
1898
1905
|
const read = Reflect.get(this.#core, method);
|
|
1899
1906
|
return Reflect.apply(read, this.#core, args);
|
|
1900
1907
|
}
|
|
@@ -1909,7 +1916,8 @@ class OpfsLeader {
|
|
|
1909
1916
|
return removed;
|
|
1910
1917
|
}
|
|
1911
1918
|
async listTempOwnerIdsPage(afterOwnerId, limit) {
|
|
1912
|
-
|
|
1919
|
+
while (this.#poisoned)
|
|
1920
|
+
await this.#healthy();
|
|
1913
1921
|
this.#core.listTempOwnerIdsPage(afterOwnerId, limit, []);
|
|
1914
1922
|
const physicalOwnerIds = [];
|
|
1915
1923
|
for await (const name of this.#tree.iterateNames(["temp"])) {
|
|
@@ -1923,10 +1931,13 @@ class OpfsLeader {
|
|
|
1923
1931
|
if (physicalOwnerIds.length > limit)
|
|
1924
1932
|
physicalOwnerIds.pop();
|
|
1925
1933
|
}
|
|
1934
|
+
while (this.#poisoned)
|
|
1935
|
+
await this.#healthy();
|
|
1926
1936
|
return this.#core.listTempOwnerIdsPage(afterOwnerId, limit, physicalOwnerIds);
|
|
1927
1937
|
}
|
|
1928
1938
|
async listExpiredTempOwnerPage(expiresAtCutoff, afterCursor, limit) {
|
|
1929
|
-
|
|
1939
|
+
while (this.#poisoned)
|
|
1940
|
+
await this.#healthy();
|
|
1930
1941
|
return this.#core.listExpiredTempOwnerPage(expiresAtCutoff, afterCursor, limit);
|
|
1931
1942
|
}
|
|
1932
1943
|
async addTable(record, options = {}) {
|
package/dist/storage/opfs/rpc.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { BlockReadBatchTooLargeError, CompactionBacklogError, CompactionJobConflictError, GarbageCollectionJobConflictError, IndexedDbSchemaUpgradeBlockedError, LeaseConflictError, LeaseExpiredError, LeaseOwnerConflictError, OpfsUncertainOutcomeError, PostingBuildConflictError, SnapshotImportConflictError, SnapshotManifestMissingError, StorageCorruptionError, StorageFormatVersionError, StorageResourceLimitError, TableInUseError, TableRecordConflictError, TempOwnerConflictError, TransactionRecordConflictError, UniqueKeyBuildConflictError, UniqueKeyConflictError, UniqueIndexCoverageError, WriteConflictError, SchemaConflictError } from "../types.js";
|
|
1
|
+
import { BlockReadBatchTooLargeError, CompactionBacklogError, CompactionJobConflictError, GarbageCollectionJobConflictError, IndexedDbSchemaUpgradeBlockedError, LeaseConflictError, LeaseExpiredError, LeaseOwnerConflictError, OpfsCoordinationError, OpfsDatabaseInUseError, OpfsUncertainOutcomeError, PostingBuildConflictError, SnapshotImportConflictError, SnapshotManifestMissingError, StorageCorruptionError, StorageFormatVersionError, StorageResourceLimitError, TableInUseError, TableRecordConflictError, TempOwnerConflictError, TransactionRecordConflictError, UniqueKeyBuildConflictError, UniqueKeyConflictError, UniqueIndexCoverageError, WriteConflictError, SchemaConflictError } from "../types.js";
|
|
2
2
|
import { dateIsoString } from "../../date-value.js";
|
|
3
3
|
const MAX_OPFS_RPC_MESSAGE_BYTES = 66 * 1024 * 1024;
|
|
4
4
|
const MAX_OPFS_RPC_IDENTIFIER_CHARACTERS = 1024;
|
|
@@ -190,6 +190,8 @@ const errorRegistry = new Map([
|
|
|
190
190
|
PostingBuildConflictError,
|
|
191
191
|
StorageCorruptionError,
|
|
192
192
|
StorageFormatVersionError,
|
|
193
|
+
OpfsCoordinationError,
|
|
194
|
+
OpfsDatabaseInUseError,
|
|
193
195
|
OpfsUncertainOutcomeError
|
|
194
196
|
].map((constructor) => [constructor.name, constructor]));
|
|
195
197
|
function serializeStoreError(error) {
|
|
@@ -61,6 +61,8 @@ export declare class OpfsBlockStore {
|
|
|
61
61
|
close(): void;
|
|
62
62
|
/** Test-only: whether this connection currently holds the database's handles. */
|
|
63
63
|
_isLeaderForTests(): boolean;
|
|
64
|
+
/** @internal Simulates a suspended leader without releasing its file handles. */
|
|
65
|
+
_pauseCoordinationForTests(): () => void;
|
|
64
66
|
/** Test-only: the id that names this connection's inbox channel. */
|
|
65
67
|
_instanceIdForTests(): string;
|
|
66
68
|
/** Test-only: simulates an acknowledgement lost after the served operation settles. */
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { assertStorageBulkReadItems, assertTempRunPageBatchLimits, OpfsUncertainOutcomeError, StorageCorruptionError, StorageFormatVersionError, validateStorageDatabaseName } from "../types.js";
|
|
1
|
+
import { assertStorageBulkReadItems, assertTempRunPageBatchLimits, OpfsCoordinationError, OpfsDatabaseInUseError, OpfsUncertainOutcomeError, StorageCorruptionError, StorageFormatVersionError, validateStorageDatabaseName } from "../types.js";
|
|
2
2
|
import { validateTempRunPage, validateTempRunPageIdentity } from "../toolkit/record-core.js";
|
|
3
3
|
import { OpfsTree, encodeSegment, isDomError } from "./files.js";
|
|
4
4
|
import { LOG_FORMAT_VERSION } from "../toolkit/wire.js";
|
|
@@ -193,6 +193,7 @@ class OpfsBlockStore {
|
|
|
193
193
|
#servedRequestLocks = /* @__PURE__ */ new Map();
|
|
194
194
|
#inFlightMutationBytes = 0;
|
|
195
195
|
#inFlightReadBytes = 0;
|
|
196
|
+
#readCapacityChanged;
|
|
196
197
|
#settledMutationBytes = 0;
|
|
197
198
|
#pendingRpcBytes = 0;
|
|
198
199
|
#servedRequestCount = 0;
|
|
@@ -209,25 +210,41 @@ class OpfsBlockStore {
|
|
|
209
210
|
this.#channelName = `minnowdb-store:${options.name}`;
|
|
210
211
|
this.liveQueryChannelName = `minnowdb-live:opfs:${options.name}`;
|
|
211
212
|
}
|
|
213
|
+
#releaseConnectionLock;
|
|
212
214
|
static async open(options) {
|
|
213
|
-
const
|
|
214
|
-
|
|
215
|
-
await
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
store.#
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
215
|
+
const ownedOptions = { ...options };
|
|
216
|
+
validateStorageDatabaseName(ownedOptions.name);
|
|
217
|
+
const release = await holdConnectionLock(ownedOptions.name);
|
|
218
|
+
let store;
|
|
219
|
+
try {
|
|
220
|
+
const tree = new OpfsTree(await resolveDatabaseRoot(ownedOptions));
|
|
221
|
+
store = new OpfsBlockStore(tree, ownedOptions);
|
|
222
|
+
} catch (error) {
|
|
223
|
+
release?.();
|
|
224
|
+
throw error;
|
|
225
|
+
}
|
|
226
|
+
store.#releaseConnectionLock = release;
|
|
227
|
+
try {
|
|
228
|
+
await store.#ensureFormatMarker();
|
|
229
|
+
if (typeof BroadcastChannel === "function") {
|
|
230
|
+
const onMessage = (event) => {
|
|
231
|
+
const message = parseStoreRpcMessage(event.data, RPC_METHODS);
|
|
232
|
+
if (message !== void 0)
|
|
233
|
+
store.#onMessage(message);
|
|
234
|
+
};
|
|
235
|
+
const channel = new BroadcastChannel(store.#channelName);
|
|
236
|
+
channel.onmessage = onMessage;
|
|
237
|
+
store.#channel = channel;
|
|
238
|
+
const inbox = new BroadcastChannel(store.#inboxName(store.#instanceId));
|
|
239
|
+
inbox.onmessage = onMessage;
|
|
240
|
+
store.#inbox = inbox;
|
|
241
|
+
}
|
|
242
|
+
await store.#tryBecomeLeader();
|
|
243
|
+
return store;
|
|
244
|
+
} catch (error) {
|
|
245
|
+
store.close();
|
|
246
|
+
throw error;
|
|
247
|
+
}
|
|
231
248
|
}
|
|
232
249
|
async #tryBecomeLeader() {
|
|
233
250
|
if (this.#closed)
|
|
@@ -305,11 +322,15 @@ class OpfsBlockStore {
|
|
|
305
322
|
this.#leader = void 0;
|
|
306
323
|
this.#knownLeader = void 0;
|
|
307
324
|
this.#lastYieldAt = Date.now();
|
|
308
|
-
|
|
309
|
-
await leader.shutdown();
|
|
310
|
-
} catch {
|
|
325
|
+
const shutdown = leader.shutdown().catch(() => {
|
|
311
326
|
leader.crash();
|
|
312
|
-
}
|
|
327
|
+
});
|
|
328
|
+
this.#yielding = shutdown;
|
|
329
|
+
await shutdown;
|
|
330
|
+
if (this.#yielding === shutdown)
|
|
331
|
+
this.#yielding = void 0;
|
|
332
|
+
if (this.#closed)
|
|
333
|
+
return;
|
|
313
334
|
this.#closeAnswerChannels();
|
|
314
335
|
this.#post({ kind: "yield", to });
|
|
315
336
|
if (this.#reacquireTimer !== void 0)
|
|
@@ -319,9 +340,12 @@ class OpfsBlockStore {
|
|
|
319
340
|
void this.#tryBecomeLeader();
|
|
320
341
|
}, 1500);
|
|
321
342
|
}
|
|
343
|
+
#yielding;
|
|
322
344
|
#onMessage(message) {
|
|
323
345
|
if (this.#closed)
|
|
324
346
|
return;
|
|
347
|
+
if (this.#coordinationPausedForTests)
|
|
348
|
+
return;
|
|
325
349
|
switch (message.kind) {
|
|
326
350
|
case "op": {
|
|
327
351
|
if (this.#leader !== void 0) {
|
|
@@ -331,7 +355,7 @@ class OpfsBlockStore {
|
|
|
331
355
|
kind: "result",
|
|
332
356
|
requestId: message.requestId,
|
|
333
357
|
ok: false,
|
|
334
|
-
error:
|
|
358
|
+
error: serializeStoreError(new OpfsCoordinationError("leader-queue-full", message.method))
|
|
335
359
|
});
|
|
336
360
|
return;
|
|
337
361
|
}
|
|
@@ -481,26 +505,33 @@ class OpfsBlockStore {
|
|
|
481
505
|
kind: "result",
|
|
482
506
|
requestId: message.requestId,
|
|
483
507
|
ok: false,
|
|
484
|
-
error:
|
|
508
|
+
error: serializeStoreError(new OpfsCoordinationError("mutation-queue-full", message.method))
|
|
485
509
|
});
|
|
486
510
|
return;
|
|
487
511
|
}
|
|
488
512
|
if (isRead) {
|
|
489
513
|
const reservation = Math.max(fingerprint.retainedBytes, MAX_OPFS_RPC_MESSAGE_BYTES);
|
|
490
|
-
|
|
491
|
-
this.#answer(message.from, {
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
514
|
+
while (this.#inFlightReadBytes + reservation > RPC_IN_FLIGHT_READ_BYTES) {
|
|
515
|
+
this.#answer(message.from, { kind: "busy", requestId: message.requestId });
|
|
516
|
+
if (this.#readCapacityChanged === void 0) {
|
|
517
|
+
let resolve;
|
|
518
|
+
const promise = new Promise((done) => {
|
|
519
|
+
resolve = done;
|
|
520
|
+
});
|
|
521
|
+
this.#readCapacityChanged = { promise, resolve };
|
|
522
|
+
}
|
|
523
|
+
await this.#readCapacityChanged.promise;
|
|
524
|
+
if (this.#closed || this.#leader !== leader)
|
|
525
|
+
return;
|
|
498
526
|
}
|
|
527
|
+
if (this.#closed || this.#leader !== leader)
|
|
528
|
+
return;
|
|
499
529
|
this.#inFlightReadBytes += reservation;
|
|
500
530
|
try {
|
|
501
531
|
settled = await this.#executeServedOpAfterGate(leader, message, true);
|
|
502
532
|
} finally {
|
|
503
533
|
this.#inFlightReadBytes = Math.max(0, this.#inFlightReadBytes - reservation);
|
|
534
|
+
this.#wakeReadWaiters();
|
|
504
535
|
}
|
|
505
536
|
} else {
|
|
506
537
|
this.#inFlightMutationBytes += fingerprint.retainedBytes;
|
|
@@ -565,6 +596,11 @@ class OpfsBlockStore {
|
|
|
565
596
|
}
|
|
566
597
|
});
|
|
567
598
|
}
|
|
599
|
+
#wakeReadWaiters() {
|
|
600
|
+
const waiting = this.#readCapacityChanged;
|
|
601
|
+
this.#readCapacityChanged = void 0;
|
|
602
|
+
waiting?.resolve();
|
|
603
|
+
}
|
|
568
604
|
async #executeServedOpAfterGate(leader, message, isRead) {
|
|
569
605
|
const gate = this.#servedMutationGateForTests;
|
|
570
606
|
if (!isRead && gate !== void 0)
|
|
@@ -674,7 +710,7 @@ class OpfsBlockStore {
|
|
|
674
710
|
throw error;
|
|
675
711
|
}
|
|
676
712
|
}
|
|
677
|
-
throw new
|
|
713
|
+
throw new OpfsCoordinationError("leader-unavailable", method);
|
|
678
714
|
}
|
|
679
715
|
#leaderKnown() {
|
|
680
716
|
return this.#knownLeader !== void 0;
|
|
@@ -687,7 +723,7 @@ class OpfsBlockStore {
|
|
|
687
723
|
return Promise.reject(error instanceof Error ? error : new Error(String(error)));
|
|
688
724
|
}
|
|
689
725
|
if (this.#pending.size >= RPC_IN_FLIGHT_LIMIT || this.#pendingRpcBytes + retainedBytes > RPC_IN_FLIGHT_MUTATION_BYTES) {
|
|
690
|
-
return Promise.reject(new
|
|
726
|
+
return Promise.reject(new OpfsCoordinationError("follower-queue-full", method));
|
|
691
727
|
}
|
|
692
728
|
return new Promise((resolve, reject) => {
|
|
693
729
|
const message = { kind: "op", requestId, from: this.#instanceId, method, args };
|
|
@@ -754,6 +790,7 @@ class OpfsBlockStore {
|
|
|
754
790
|
this.#pendingRpcBytes = 0;
|
|
755
791
|
this.#inFlightMutationBytes = 0;
|
|
756
792
|
this.#inFlightReadBytes = 0;
|
|
793
|
+
this.#wakeReadWaiters();
|
|
757
794
|
this.#settledMutationBytes = 0;
|
|
758
795
|
this.#servedRequestCount = 0;
|
|
759
796
|
this.#servedRequestBytes = 0;
|
|
@@ -765,14 +802,31 @@ class OpfsBlockStore {
|
|
|
765
802
|
}).then(() => {
|
|
766
803
|
this.#post({ kind: "released", leaderId: this.#instanceId });
|
|
767
804
|
this.#closeChannels();
|
|
805
|
+
this.#releaseWhenHandlesClose();
|
|
768
806
|
});
|
|
769
807
|
return;
|
|
770
808
|
}
|
|
771
809
|
this.#closeChannels();
|
|
810
|
+
this.#releaseWhenHandlesClose();
|
|
811
|
+
}
|
|
812
|
+
#releaseWhenHandlesClose() {
|
|
813
|
+
const release = this.#releaseConnectionLock;
|
|
814
|
+
this.#releaseConnectionLock = void 0;
|
|
815
|
+
if (this.#electing === void 0 && this.#yielding === void 0)
|
|
816
|
+
release?.();
|
|
817
|
+
else
|
|
818
|
+
void Promise.allSettled([this.#electing, this.#yielding]).then(() => release?.());
|
|
772
819
|
}
|
|
773
820
|
_isLeaderForTests() {
|
|
774
821
|
return this.#leader !== void 0;
|
|
775
822
|
}
|
|
823
|
+
#coordinationPausedForTests = false;
|
|
824
|
+
_pauseCoordinationForTests() {
|
|
825
|
+
this.#coordinationPausedForTests = true;
|
|
826
|
+
return () => {
|
|
827
|
+
this.#coordinationPausedForTests = false;
|
|
828
|
+
};
|
|
829
|
+
}
|
|
776
830
|
_instanceIdForTests() {
|
|
777
831
|
return this.#instanceId;
|
|
778
832
|
}
|
|
@@ -820,12 +874,14 @@ class OpfsBlockStore {
|
|
|
820
874
|
this.#pendingRpcBytes = 0;
|
|
821
875
|
this.#inFlightMutationBytes = 0;
|
|
822
876
|
this.#inFlightReadBytes = 0;
|
|
877
|
+
this.#wakeReadWaiters();
|
|
823
878
|
this.#settledMutationBytes = 0;
|
|
824
879
|
this.#servedRequestCount = 0;
|
|
825
880
|
this.#servedRequestBytes = 0;
|
|
826
881
|
this.#leader?.crash();
|
|
827
882
|
this.#leader = void 0;
|
|
828
883
|
this.#closeChannels();
|
|
884
|
+
this.#releaseWhenHandlesClose();
|
|
829
885
|
}
|
|
830
886
|
async #ensureFormatMarker() {
|
|
831
887
|
const existing = await this.#tree.readFile(["format.json"], {
|
|
@@ -898,17 +954,49 @@ for (const method of RPC_METHODS) {
|
|
|
898
954
|
}
|
|
899
955
|
});
|
|
900
956
|
}
|
|
957
|
+
const DELETE_LOCK_WAIT_MS = 1e3;
|
|
901
958
|
async function deleteOpfsDatabase(options) {
|
|
902
959
|
const encodedName = encodeSegment(validateStorageDatabaseName(options.name));
|
|
903
|
-
const
|
|
960
|
+
const locks = globalThis.navigator?.locks;
|
|
961
|
+
if (locks === void 0)
|
|
962
|
+
throw new Error("Deleting an OPFS database requires Web Locks");
|
|
963
|
+
const abort = new AbortController();
|
|
964
|
+
const timer = setTimeout(() => {
|
|
965
|
+
abort.abort();
|
|
966
|
+
}, DELETE_LOCK_WAIT_MS);
|
|
904
967
|
try {
|
|
905
|
-
|
|
906
|
-
|
|
968
|
+
await locks.request(connectionLockName(options.name), { mode: "exclusive", signal: abort.signal }, async () => {
|
|
969
|
+
const root = options.root ?? await navigator.storage.getDirectory();
|
|
970
|
+
try {
|
|
971
|
+
const namespace = await root.getDirectoryHandle("minnowdb");
|
|
972
|
+
await namespace.removeEntry(encodedName, { recursive: true });
|
|
973
|
+
} catch (error) {
|
|
974
|
+
if (!isDomError(error, "NotFoundError"))
|
|
975
|
+
throw error;
|
|
976
|
+
}
|
|
977
|
+
});
|
|
907
978
|
} catch (error) {
|
|
908
|
-
if (
|
|
909
|
-
throw
|
|
979
|
+
if (abort.signal.aborted && isDomError(error, "AbortError")) {
|
|
980
|
+
throw new OpfsDatabaseInUseError(options.name);
|
|
981
|
+
}
|
|
982
|
+
throw error;
|
|
983
|
+
} finally {
|
|
984
|
+
clearTimeout(timer);
|
|
910
985
|
}
|
|
911
986
|
}
|
|
987
|
+
function connectionLockName(name) {
|
|
988
|
+
return `minnowdb-opfs-connections:${name}`;
|
|
989
|
+
}
|
|
990
|
+
async function holdConnectionLock(name) {
|
|
991
|
+
const locks = globalThis.navigator?.locks;
|
|
992
|
+
if (locks === void 0)
|
|
993
|
+
return void 0;
|
|
994
|
+
return new Promise((resolve, reject) => {
|
|
995
|
+
void locks.request(connectionLockName(name), { mode: "shared" }, () => new Promise((release) => {
|
|
996
|
+
resolve(release);
|
|
997
|
+
})).catch(reject);
|
|
998
|
+
});
|
|
999
|
+
}
|
|
912
1000
|
const RPC_TIMED_OUT = new Error("The leader did not answer in time");
|
|
913
1001
|
async function resolveDatabaseRoot(options) {
|
|
914
1002
|
const encodedName = encodeSegment(validateStorageDatabaseName(options.name));
|
|
@@ -5711,6 +5711,7 @@ function collectIndexedPhysicalRoots(candidateSegmentIds, candidateBlockIds, man
|
|
|
5711
5711
|
};
|
|
5712
5712
|
const rootedBlockIds = /* @__PURE__ */ new Set();
|
|
5713
5713
|
const rootedSegmentIds = /* @__PURE__ */ new Set();
|
|
5714
|
+
const candidateSegments = new Set(candidateSegmentIds);
|
|
5714
5715
|
for (const id of candidateSegmentIds) {
|
|
5715
5716
|
const segment = segments.get(id);
|
|
5716
5717
|
if (segment !== void 0 && isSegmentRoot(segment))
|
|
@@ -5723,7 +5724,7 @@ function collectIndexedPhysicalRoots(candidateSegmentIds, candidateBlockIds, man
|
|
|
5723
5724
|
}
|
|
5724
5725
|
for (const segmentId of segments.segmentIdsForBlock(id)) {
|
|
5725
5726
|
const segment = segments.get(segmentId);
|
|
5726
|
-
if (segment !== void 0 &&
|
|
5727
|
+
if (segment !== void 0 && (!candidateSegments.has(segmentId) || rootedSegmentIds.has(segmentId))) {
|
|
5727
5728
|
rootedBlockIds.add(id);
|
|
5728
5729
|
break;
|
|
5729
5730
|
}
|
package/dist/storage/types.d.ts
CHANGED
|
@@ -1360,6 +1360,20 @@ export declare class OpfsUncertainOutcomeError extends Error {
|
|
|
1360
1360
|
readonly name = "OpfsUncertainOutcomeError";
|
|
1361
1361
|
constructor(method: string);
|
|
1362
1362
|
}
|
|
1363
|
+
/** Temporary OPFS coordination failure. No operation was admitted, or a read exhausted retries. */
|
|
1364
|
+
export declare class OpfsCoordinationError extends Error {
|
|
1365
|
+
readonly reason: "leader-unavailable" | "leader-queue-full" | "mutation-queue-full" | "follower-queue-full";
|
|
1366
|
+
readonly method: string | null;
|
|
1367
|
+
readonly name = "OpfsCoordinationError";
|
|
1368
|
+
readonly backend = "opfs";
|
|
1369
|
+
constructor(reason: "leader-unavailable" | "leader-queue-full" | "mutation-queue-full" | "follower-queue-full", method?: string | null);
|
|
1370
|
+
}
|
|
1371
|
+
/** Deletion was refused before removing files because another connection still owns the store. */
|
|
1372
|
+
export declare class OpfsDatabaseInUseError extends Error {
|
|
1373
|
+
readonly databaseName: string;
|
|
1374
|
+
readonly name = "OpfsDatabaseInUseError";
|
|
1375
|
+
constructor(databaseName: string);
|
|
1376
|
+
}
|
|
1363
1377
|
/**
|
|
1364
1378
|
* The commit input carries only the change: added blocks are the transaction's journaled pending
|
|
1365
1379
|
* blocks, removals are the superseded ids. The store derives the published manifest from its
|
package/dist/storage/types.js
CHANGED
|
@@ -1005,6 +1005,25 @@ class OpfsUncertainOutcomeError extends Error {
|
|
|
1005
1005
|
this.method = method;
|
|
1006
1006
|
}
|
|
1007
1007
|
}
|
|
1008
|
+
class OpfsCoordinationError extends Error {
|
|
1009
|
+
reason;
|
|
1010
|
+
method;
|
|
1011
|
+
name = "OpfsCoordinationError";
|
|
1012
|
+
backend = "opfs";
|
|
1013
|
+
constructor(reason, method = null) {
|
|
1014
|
+
super(reason === "leader-queue-full" ? "The OPFS leader RPC queue is full" : reason === "mutation-queue-full" ? "The OPFS leader mutation queue is full" : reason === "follower-queue-full" ? "The OPFS follower request queue is full" : "The OPFS store could not reach or become a leader");
|
|
1015
|
+
this.reason = reason;
|
|
1016
|
+
this.method = method;
|
|
1017
|
+
}
|
|
1018
|
+
}
|
|
1019
|
+
class OpfsDatabaseInUseError extends Error {
|
|
1020
|
+
databaseName;
|
|
1021
|
+
name = "OpfsDatabaseInUseError";
|
|
1022
|
+
constructor(databaseName) {
|
|
1023
|
+
super(`The OPFS database ${databaseName} is open; close all connections before deleting it`);
|
|
1024
|
+
this.databaseName = databaseName;
|
|
1025
|
+
}
|
|
1026
|
+
}
|
|
1008
1027
|
const MAX_UNIQUE_KEY_BUILD_TOKENS_PER_CHUNK = 4096;
|
|
1009
1028
|
const MAX_UNIQUE_KEY_BUILD_CHUNK_BYTES = 4 * 1024 * 1024;
|
|
1010
1029
|
const MAX_UNIQUE_KEY_BUILD_TTL_MS = 60 * 60 * 1e3;
|
|
@@ -3025,6 +3044,8 @@ export {
|
|
|
3025
3044
|
MAX_UNIQUE_KEY_BUILD_STAGED_BYTES_TOTAL,
|
|
3026
3045
|
MAX_UNIQUE_KEY_BUILD_TOKENS_PER_CHUNK,
|
|
3027
3046
|
MAX_UNIQUE_KEY_BUILD_TTL_MS,
|
|
3047
|
+
OpfsCoordinationError,
|
|
3048
|
+
OpfsDatabaseInUseError,
|
|
3028
3049
|
OpfsUncertainOutcomeError,
|
|
3029
3050
|
PostingBuildConflictError,
|
|
3030
3051
|
SNAPSHOT_FRAME_KINDS,
|
|
@@ -32,6 +32,8 @@ export interface OpenLeasedSnapshotOptions {
|
|
|
32
32
|
version?: number | null;
|
|
33
33
|
}
|
|
34
34
|
export interface BeginDeferredOptions {
|
|
35
|
+
/** Keep successive stages local while their combined artifacts fit one storage batch. */
|
|
36
|
+
coalesceArtifacts?: boolean;
|
|
35
37
|
/**
|
|
36
38
|
* Pins the captured snapshot with a renewable durable reader lease until this transaction
|
|
37
39
|
* either persists its own durable owner or finishes. Enabled by default; tightly bounded
|
|
@@ -97,6 +99,7 @@ export declare class DatabaseTransaction {
|
|
|
97
99
|
deref(): DatabaseTransaction | undefined;
|
|
98
100
|
}, options?: {
|
|
99
101
|
persisted?: boolean;
|
|
102
|
+
coalesceArtifacts?: boolean;
|
|
100
103
|
initialCatalogProbe?: {
|
|
101
104
|
catalogEpoch: number;
|
|
102
105
|
manifestVersion: number | null;
|
|
@@ -119,6 +119,7 @@ class DatabaseTransaction {
|
|
|
119
119
|
#deferredSegments = [];
|
|
120
120
|
#knownSegments = /* @__PURE__ */ new Map();
|
|
121
121
|
#persisted;
|
|
122
|
+
#coalesceArtifacts;
|
|
122
123
|
#snapshotLease;
|
|
123
124
|
#snapshotLeaseOperation = Promise.resolve();
|
|
124
125
|
#snapshotRenewal;
|
|
@@ -131,6 +132,7 @@ class DatabaseTransaction {
|
|
|
131
132
|
this.ttlMs = ttlMs;
|
|
132
133
|
this.#record = structuredClone(record);
|
|
133
134
|
this.#persisted = options.persisted ?? true;
|
|
135
|
+
this.#coalesceArtifacts = options.coalesceArtifacts ?? false;
|
|
134
136
|
this.#initialCatalogProbe = options.initialCatalogProbe;
|
|
135
137
|
this.#snapshotLease = options.snapshotLease;
|
|
136
138
|
const reference = createWeakRef(this);
|
|
@@ -384,7 +386,7 @@ class DatabaseTransaction {
|
|
|
384
386
|
if (blocks.length === 0 && ordered.length === 0)
|
|
385
387
|
return;
|
|
386
388
|
const batches = transactionArtifactBatches(blocks, ordered);
|
|
387
|
-
if (!this.#persisted && this.#deferredBlocks.length === 0 && this.#deferredSegments.length === 0 && batches.length === 1) {
|
|
389
|
+
if (!this.#persisted && (this.#coalesceArtifacts || this.#deferredBlocks.length === 0 && this.#deferredSegments.length === 0) && this.#deferredBlocks.length + blocks.length <= MAX_TRANSACTION_STAGE_BLOCKS && this.#deferredSegments.length + ordered.length <= MAX_TRANSACTION_STAGE_SEGMENTS && [...this.#deferredBlocks, ...blocks].reduce((bytes, block) => bytes + block.bytes.byteLength, 0) <= MAX_TRANSACTION_STAGE_BYTES && batches.length === 1) {
|
|
388
390
|
assertTransactionArtifactBatchLimits(blocks, ordered);
|
|
389
391
|
this.#deferredBlocks.push(...blocks.map((block) => ({ id: block.id, bytes: new Uint8Array(block.bytes) })));
|
|
390
392
|
this.#deferredSegments.push(...structuredClone(ordered));
|
|
@@ -1120,6 +1122,7 @@ class TransactionManager {
|
|
|
1120
1122
|
committedVersion: null
|
|
1121
1123
|
}, this.#now, this.#transactionTtlMs, this.#createWeakRef, {
|
|
1122
1124
|
persisted: false,
|
|
1125
|
+
coalesceArtifacts: options.coalesceArtifacts ?? false,
|
|
1123
1126
|
initialCatalogProbe: probe,
|
|
1124
1127
|
...snapshotLease === void 0 ? {} : { snapshotLease }
|
|
1125
1128
|
});
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@minnowdb/core",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.9.1",
|
|
4
4
|
"description": "A columnar SQL database for the browser: PostgreSQL-style SQL over durable IndexedDB or OPFS data, with no server or WebAssembly module.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"author": "Eric Wilhite",
|
|
@@ -32,7 +32,7 @@
|
|
|
32
32
|
"./dist/engine/worker-memory.js"
|
|
33
33
|
],
|
|
34
34
|
"scripts": {
|
|
35
|
-
"prepack": "node ../../scripts/
|
|
35
|
+
"prepack": "node ../../scripts/prepare-package.mjs --strip-comments"
|
|
36
36
|
},
|
|
37
37
|
"exports": {
|
|
38
38
|
".": {
|
package/dist/date-value.d.ts
DELETED
|
@@ -1,20 +0,0 @@
|
|
|
1
|
-
/** Returns a Date's internal millisecond value without invoking an instance override. */
|
|
2
|
-
export declare function dateMilliseconds(value: Date): number;
|
|
3
|
-
/** Copies a Date by its internal value without invoking instance methods. */
|
|
4
|
-
export declare function copyDate(value: Date): Date;
|
|
5
|
-
/** Formats a Date's internal value without invoking an instance override. */
|
|
6
|
-
export declare function dateIsoString(value: Date): string;
|
|
7
|
-
export declare function dateUtcFullYear(value: Date): number;
|
|
8
|
-
export declare function dateUtcMonth(value: Date): number;
|
|
9
|
-
export declare function dateUtcDate(value: Date): number;
|
|
10
|
-
export declare function dateUtcDay(value: Date): number;
|
|
11
|
-
export declare function dateUtcHours(value: Date): number;
|
|
12
|
-
export declare function dateUtcMinutes(value: Date): number;
|
|
13
|
-
export declare function dateUtcSeconds(value: Date): number;
|
|
14
|
-
export declare function setDateUtcDate(value: Date, day: number): number;
|
|
15
|
-
export declare function setDateUtcMonth(value: Date, month: number): number;
|
|
16
|
-
/** Whole days since 1970-01-01 for an epoch-millisecond value (floored, so negatives work). */
|
|
17
|
-
export declare function epochDays(milliseconds: number): number;
|
|
18
|
-
export declare function civilFromDays(days: number): readonly [number, number, number];
|
|
19
|
-
/** Days since 1970-01-01 for a civil date; month is 0-11 as in Date.UTC. */
|
|
20
|
-
export declare function daysFromCivil(year: number, month: number, day: number): number;
|
|
@@ -1,29 +0,0 @@
|
|
|
1
|
-
/** Snapshot of the byte-bounded artifact cache's lifetime counters. */
|
|
2
|
-
interface ArtifactCacheStats {
|
|
3
|
-
limitBytes: number;
|
|
4
|
-
usedBytes: number;
|
|
5
|
-
entries: number;
|
|
6
|
-
hits: number;
|
|
7
|
-
misses: number;
|
|
8
|
-
evictions: number;
|
|
9
|
-
}
|
|
10
|
-
/**
|
|
11
|
-
* Byte-bounded LRU for decoded blocks, vectors, derived results, and memoized query results.
|
|
12
|
-
* Payloads are immutable by convention; callers own the byte estimates for their artifact type.
|
|
13
|
-
*
|
|
14
|
-
* Recency lives on an intrusive doubly-linked list rather than Map insertion order: a hit
|
|
15
|
-
* relinks four pointers instead of deleting and re-inserting the key, which re-hashed the
|
|
16
|
-
* long block-identity keys twice per touch and was the single hottest frame of a cached point
|
|
17
|
-
* lookup.
|
|
18
|
-
*/
|
|
19
|
-
export declare class ArtifactCache {
|
|
20
|
-
#private;
|
|
21
|
-
constructor(limitBytes: number);
|
|
22
|
-
get enabled(): boolean;
|
|
23
|
-
get(key: string): unknown;
|
|
24
|
-
put(key: string, payload: unknown, bytes: number): void;
|
|
25
|
-
/** Releases every retained payload while preserving lifetime counters for final diagnostics. */
|
|
26
|
-
clear(): void;
|
|
27
|
-
stats(): ArtifactCacheStats;
|
|
28
|
-
}
|
|
29
|
-
export {};
|
|
@@ -1,11 +0,0 @@
|
|
|
1
|
-
import type { ColumnarBatch } from "./batch.js";
|
|
2
|
-
/**
|
|
3
|
-
* The one modeled per-value size used for flush thresholds, cache accounting, and write
|
|
4
|
-
* metrics. One byte per UTF-16 code unit approximates the UTF-8 payload (exact for ASCII)
|
|
5
|
-
* without encoding the string just to measure it — this estimate feeds metrics and flush
|
|
6
|
-
* thresholds, not the physical format. Keeping a single implementation keeps those
|
|
7
|
-
* accountings from drifting apart.
|
|
8
|
-
*/
|
|
9
|
-
export declare function estimateValuesBytes(values: readonly unknown[]): number;
|
|
10
|
-
export declare function estimateRowBytes(row: Readonly<Record<string, unknown>>): number;
|
|
11
|
-
export declare function estimateBatchBytes(input: ColumnarBatch): number;
|