@minnowdb/core 0.1.1 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +12 -9
- package/dist/engine/client.d.ts +9 -1
- package/dist/engine/client.d.ts.map +1 -1
- package/dist/engine/client.js +29 -5
- package/dist/engine/client.js.map +1 -1
- package/dist/engine/database.d.ts +51 -6
- package/dist/engine/database.d.ts.map +1 -1
- package/dist/engine/database.js +2597 -865
- package/dist/engine/database.js.map +1 -1
- package/dist/engine/live.d.ts.map +1 -1
- package/dist/engine/live.js +35 -11
- package/dist/engine/live.js.map +1 -1
- package/dist/engine/query-cache.d.ts +13 -1
- package/dist/engine/query-cache.d.ts.map +1 -1
- package/dist/engine/query-cache.js +34 -5
- package/dist/engine/query-cache.js.map +1 -1
- package/dist/engine/query.d.ts +11 -1
- package/dist/engine/query.d.ts.map +1 -1
- package/dist/engine/query.js +49 -41
- package/dist/engine/query.js.map +1 -1
- package/dist/engine/result-wire.d.ts +68 -0
- package/dist/engine/result-wire.d.ts.map +1 -0
- package/dist/engine/result-wire.js +264 -0
- package/dist/engine/result-wire.js.map +1 -0
- package/dist/engine/sort-keys.d.ts +50 -12
- package/dist/engine/sort-keys.d.ts.map +1 -1
- package/dist/engine/sort-keys.js +371 -25
- package/dist/engine/sort-keys.js.map +1 -1
- package/dist/engine/vector.d.ts +11 -0
- package/dist/engine/vector.d.ts.map +1 -1
- package/dist/engine/vector.js +99 -203
- package/dist/engine/vector.js.map +1 -1
- package/dist/engine/worker-host.d.ts +17 -1
- package/dist/engine/worker-host.d.ts.map +1 -1
- package/dist/engine/worker-host.js +83 -13
- package/dist/engine/worker-host.js.map +1 -1
- package/dist/storage/index.d.ts +13 -0
- package/dist/storage/index.d.ts.map +1 -1
- package/dist/storage/index.js +13 -0
- package/dist/storage/index.js.map +1 -1
- package/dist/storage/indexeddb.d.ts +5 -1
- package/dist/storage/indexeddb.d.ts.map +1 -1
- package/dist/storage/indexeddb.js +484 -402
- package/dist/storage/indexeddb.js.map +1 -1
- package/dist/storage/memory.d.ts +24 -21
- package/dist/storage/memory.d.ts.map +1 -1
- package/dist/storage/memory.js +124 -1199
- package/dist/storage/memory.js.map +1 -1
- package/dist/storage/opfs/files.d.ts +64 -0
- package/dist/storage/opfs/files.d.ts.map +1 -0
- package/dist/storage/opfs/files.js +243 -0
- package/dist/storage/opfs/files.js.map +1 -0
- package/dist/storage/opfs/index.d.ts +3 -0
- package/dist/storage/opfs/index.d.ts.map +1 -0
- package/dist/storage/opfs/index.js +3 -0
- package/dist/storage/opfs/index.js.map +1 -0
- package/dist/storage/opfs/leader.d.ts +130 -0
- package/dist/storage/opfs/leader.d.ts.map +1 -0
- package/dist/storage/opfs/leader.js +1104 -0
- package/dist/storage/opfs/leader.js.map +1 -0
- package/dist/storage/opfs/rpc.d.ts +70 -0
- package/dist/storage/opfs/rpc.d.ts.map +1 -0
- package/dist/storage/opfs/rpc.js +58 -0
- package/dist/storage/opfs/rpc.js.map +1 -0
- package/dist/storage/opfs/store.d.ts +154 -0
- package/dist/storage/opfs/store.d.ts.map +1 -0
- package/dist/storage/opfs/store.js +958 -0
- package/dist/storage/opfs/store.js.map +1 -0
- package/dist/storage/toolkit/extents.d.ts +69 -0
- package/dist/storage/toolkit/extents.d.ts.map +1 -0
- package/dist/storage/toolkit/extents.js +175 -0
- package/dist/storage/toolkit/extents.js.map +1 -0
- package/dist/storage/toolkit/index.d.ts +30 -0
- package/dist/storage/toolkit/index.d.ts.map +1 -0
- package/dist/storage/toolkit/index.js +5 -0
- package/dist/storage/toolkit/index.js.map +1 -0
- package/dist/storage/toolkit/record-core.d.ts +252 -0
- package/dist/storage/toolkit/record-core.d.ts.map +1 -0
- package/dist/storage/toolkit/record-core.js +1670 -0
- package/dist/storage/toolkit/record-core.js.map +1 -0
- package/dist/storage/toolkit/sync-file.d.ts +29 -0
- package/dist/storage/toolkit/sync-file.d.ts.map +1 -0
- package/dist/storage/toolkit/sync-file.js +2 -0
- package/dist/storage/toolkit/sync-file.js.map +1 -0
- package/dist/storage/toolkit/wal.d.ts +22 -0
- package/dist/storage/toolkit/wal.d.ts.map +1 -0
- package/dist/storage/toolkit/wal.js +79 -0
- package/dist/storage/toolkit/wal.js.map +1 -0
- package/dist/storage/toolkit/wire.d.ts +25 -0
- package/dist/storage/toolkit/wire.d.ts.map +1 -0
- package/dist/storage/toolkit/wire.js +100 -0
- package/dist/storage/toolkit/wire.js.map +1 -0
- package/dist/storage/types.d.ts +343 -59
- package/dist/storage/types.d.ts.map +1 -1
- package/dist/storage/types.js +112 -11
- package/dist/storage/types.js.map +1 -1
- package/dist/testing/block-store-conformance.d.ts +51 -0
- package/dist/testing/block-store-conformance.d.ts.map +1 -0
- package/dist/testing/block-store-conformance.js +745 -0
- package/dist/testing/block-store-conformance.js.map +1 -0
- package/dist/testing/index.d.ts +3 -0
- package/dist/testing/index.d.ts.map +1 -1
- package/dist/testing/index.js +5 -0
- package/dist/testing/index.js.map +1 -1
- package/dist/testing/opfs-shim.d.ts +35 -0
- package/dist/testing/opfs-shim.d.ts.map +1 -0
- package/dist/testing/opfs-shim.js +295 -0
- package/dist/testing/opfs-shim.js.map +1 -0
- package/dist/transactions/index.d.ts +42 -1
- package/dist/transactions/index.d.ts.map +1 -1
- package/dist/transactions/index.js +219 -13
- package/dist/transactions/index.js.map +1 -1
- package/dist/worker-protocol/index.d.ts +2 -2
- package/dist/worker-protocol/index.d.ts.map +1 -1
- package/dist/worker-protocol/index.js +2 -1
- package/dist/worker-protocol/index.js.map +1 -1
- package/package.json +10 -2
|
@@ -10,17 +10,16 @@ const ROW_ID_PREFIX = "row-id/";
|
|
|
10
10
|
const AUTO_INCREMENT_PREFIX = "auto-increment/";
|
|
11
11
|
const UNIQUE_KEY_CHUNK_INDEX = "unique-key-chunk-index";
|
|
12
12
|
const UNIQUE_KEY_CHUNK = "unique-key-chunk";
|
|
13
|
-
const UNIQUE_KEY_BASE = "unique-key-base";
|
|
14
13
|
/**
|
|
15
|
-
* Tail length at which a commit folds the chunk tail into
|
|
14
|
+
* Tail length at which a commit folds the chunk tail into the partitioned base. Appending one
|
|
16
15
|
* chunk per commit keeps commits O(1), but every lookup must deserialize the whole tail; the
|
|
17
|
-
* fold bounds that at this many chunk records plus point
|
|
16
|
+
* fold bounds that at this many chunk records plus point reads against the base partitions,
|
|
18
17
|
* instead of a scan whose cost grows with every key ever written.
|
|
19
18
|
*/
|
|
20
19
|
const UNIQUE_KEY_TAIL_CHUNK_LIMIT = 16;
|
|
21
20
|
const UNIQUE_KEY_BASE_PART = "unique-key-base-part";
|
|
22
21
|
/**
|
|
23
|
-
* Tokens per
|
|
22
|
+
* Tokens per folded base partition. Small enough that a keyed point mutation deserializes
|
|
24
23
|
* one partition in about a millisecond; large enough that a multi-million-key table folds into
|
|
25
24
|
* hundreds of records instead of millions.
|
|
26
25
|
*/
|
|
@@ -138,6 +137,18 @@ export class IndexedDbBlockStore {
|
|
|
138
137
|
transaction.objectStore("temp").put(page.bytes.slice(), tempRunPageKey(page));
|
|
139
138
|
await transactionDone(transaction);
|
|
140
139
|
}
|
|
140
|
+
async putTempRunPages(pages) {
|
|
141
|
+
for (const page of pages)
|
|
142
|
+
validateTempRunPage(page);
|
|
143
|
+
if (pages.length === 0)
|
|
144
|
+
return;
|
|
145
|
+
// One transaction for the whole batch — the per-page path pays one commit each.
|
|
146
|
+
const transaction = this.#transaction("temp", "readwrite");
|
|
147
|
+
const store = transaction.objectStore("temp");
|
|
148
|
+
for (const page of pages)
|
|
149
|
+
store.put(page.bytes.slice(), tempRunPageKey(page));
|
|
150
|
+
await transactionDone(transaction);
|
|
151
|
+
}
|
|
141
152
|
async getTempRunPage(ownerId, runId, pageIndex) {
|
|
142
153
|
validateTempRunPageIdentity(ownerId, runId, pageIndex);
|
|
143
154
|
const transaction = this.#transaction("temp", "readonly");
|
|
@@ -328,10 +339,8 @@ export class IndexedDbBlockStore {
|
|
|
328
339
|
`${FTS_CHUNK_PREFIX}${id}/`,
|
|
329
340
|
];
|
|
330
341
|
const ownedArrayKinds = new Set([
|
|
331
|
-
"unique-key",
|
|
332
342
|
UNIQUE_KEY_CHUNK_INDEX,
|
|
333
343
|
UNIQUE_KEY_CHUNK,
|
|
334
|
-
UNIQUE_KEY_BASE,
|
|
335
344
|
UNIQUE_KEY_BASE_PART,
|
|
336
345
|
]);
|
|
337
346
|
const doomed = [];
|
|
@@ -355,6 +364,10 @@ export class IndexedDbBlockStore {
|
|
|
355
364
|
segments.delete(key);
|
|
356
365
|
await bumpCatalogEpoch(catalog);
|
|
357
366
|
await transactionDone(transaction);
|
|
367
|
+
// The membership cache is keyed by manifest version, which a table drop does not move;
|
|
368
|
+
// without this, a lookup after the drop would answer from the dead table's keys.
|
|
369
|
+
if (this.#uniqueKeyCache?.tableId === id)
|
|
370
|
+
this.#uniqueKeyCache = undefined;
|
|
358
371
|
}
|
|
359
372
|
async writeFtsBase(tableId, columnId, input) {
|
|
360
373
|
const transaction = this.#transaction("catalog", "readwrite");
|
|
@@ -532,25 +545,15 @@ export class IndexedDbBlockStore {
|
|
|
532
545
|
const tokens = [...new Set(keyTokens)];
|
|
533
546
|
const transaction = this.#transaction("catalog", "readonly");
|
|
534
547
|
const store = transaction.objectStore("catalog");
|
|
535
|
-
const
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
]);
|
|
539
|
-
const table = tableValue === undefined ? undefined : asTableRecord(tableValue);
|
|
540
|
-
const mode = table?.uniqueKeyStorage;
|
|
541
|
-
if (mode === "chunks-v1" || mode === "chunks-v2") {
|
|
542
|
-
const cache = this.#uniqueKeyCache;
|
|
543
|
-
if (cache?.tableId === tableId && cache.mode === mode && cache.version === versionValue) {
|
|
544
|
-
await transactionDone(transaction);
|
|
545
|
-
return tokens.filter((token) => cache.present.has(token)).sort();
|
|
546
|
-
}
|
|
547
|
-
const { existing } = await readChunkedUniqueKeys(store, tableId, tokens, mode);
|
|
548
|
+
const versionValue = await requestResult(store.get(CURRENT_MANIFEST_KEY));
|
|
549
|
+
const cache = this.#uniqueKeyCache;
|
|
550
|
+
if (cache?.tableId === tableId && cache.version === versionValue) {
|
|
548
551
|
await transactionDone(transaction);
|
|
549
|
-
return
|
|
552
|
+
return tokens.filter((token) => cache.present.has(token)).sort();
|
|
550
553
|
}
|
|
551
|
-
const
|
|
554
|
+
const { existing } = await readChunkedUniqueKeys(store, tableId, tokens);
|
|
552
555
|
await transactionDone(transaction);
|
|
553
|
-
return
|
|
556
|
+
return [...existing].sort();
|
|
554
557
|
}
|
|
555
558
|
async getCurrentManifestVersion() {
|
|
556
559
|
const transaction = this.#transaction("catalog", "readonly");
|
|
@@ -884,25 +887,105 @@ export class IndexedDbBlockStore {
|
|
|
884
887
|
}
|
|
885
888
|
async commitTransaction(input) {
|
|
886
889
|
const transaction = this.#transaction(["blocks", "catalog", "manifests", "transactions", "segments"], "readwrite");
|
|
887
|
-
|
|
888
|
-
|
|
889
|
-
|
|
890
|
-
|
|
891
|
-
|
|
890
|
+
try {
|
|
891
|
+
const record = await readActiveTransactionRecord(transaction, input.transactionId, input.expectedTransactionRevision);
|
|
892
|
+
const outcome = await this.#commitInTransaction(transaction, record, input);
|
|
893
|
+
await transactionDone(transaction);
|
|
894
|
+
outcome.settle();
|
|
895
|
+
return outcome.manifest;
|
|
896
|
+
}
|
|
897
|
+
catch (error) {
|
|
898
|
+
abortIfActive(transaction);
|
|
892
899
|
await ignoreAbort(transaction);
|
|
893
|
-
throw
|
|
900
|
+
throw error;
|
|
901
|
+
}
|
|
902
|
+
}
|
|
903
|
+
async writeTransaction(input) {
|
|
904
|
+
const blockIds = new Set();
|
|
905
|
+
for (const block of input.blocks) {
|
|
906
|
+
if (block.id.length === 0)
|
|
907
|
+
throw new TypeError("Block ID cannot be empty");
|
|
908
|
+
if (blockIds.has(block.id))
|
|
909
|
+
throw new Error(`Block already exists: ${block.id}`);
|
|
910
|
+
blockIds.add(block.id);
|
|
894
911
|
}
|
|
912
|
+
if ("record" in input.transaction &&
|
|
913
|
+
(input.transaction.record.pendingBlockIds.length > 0 ||
|
|
914
|
+
input.transaction.record.pendingSegmentIds.length > 0)) {
|
|
915
|
+
throw new TypeError("A fresh transaction cannot begin with pending artifacts");
|
|
916
|
+
}
|
|
917
|
+
const transaction = this.#transaction(["blocks", "catalog", "manifests", "transactions", "segments"], "readwrite");
|
|
918
|
+
try {
|
|
919
|
+
const transactionStore = transaction.objectStore("transactions");
|
|
920
|
+
let base;
|
|
921
|
+
if ("record" in input.transaction) {
|
|
922
|
+
const id = input.transaction.record.id;
|
|
923
|
+
if ((await requestResult(transactionStore.getKey(id))) !== undefined) {
|
|
924
|
+
throw new Error(`Transaction already exists: ${id}`);
|
|
925
|
+
}
|
|
926
|
+
// The fresh record pins the version the caller prepared against, so the commit below
|
|
927
|
+
// is the same compare-and-swap a begun transaction would make; a moved manifest is a
|
|
928
|
+
// WriteConflictError with nothing written, record included.
|
|
929
|
+
const current = (await requestResult(transaction.objectStore("catalog").get(CURRENT_MANIFEST_KEY)));
|
|
930
|
+
const actualVersion = current ?? null;
|
|
931
|
+
if (actualVersion !== input.expectedManifestVersion) {
|
|
932
|
+
throw new WriteConflictError(input.expectedManifestVersion, actualVersion);
|
|
933
|
+
}
|
|
934
|
+
base = { ...structuredClone(input.transaction.record), snapshotVersion: actualVersion };
|
|
935
|
+
}
|
|
936
|
+
else {
|
|
937
|
+
base = await readActiveTransactionRecord(transaction, input.transaction.id, input.transaction.expectedRevision);
|
|
938
|
+
}
|
|
939
|
+
const staged = updateTransactionRecord(base, {
|
|
940
|
+
pendingBlockIds: [...base.pendingBlockIds, ...blockIds],
|
|
941
|
+
pendingSegmentIds: [
|
|
942
|
+
...base.pendingSegmentIds,
|
|
943
|
+
...input.segments.map((segment) => segment.id),
|
|
944
|
+
],
|
|
945
|
+
updatedAt: input.committedAt,
|
|
946
|
+
});
|
|
947
|
+
// Only previously journaled artifacts need existence probes; the ones added below commit
|
|
948
|
+
// or fail atomically with everything else.
|
|
949
|
+
await assertPendingArtifactsAvailableInTransaction(transaction, base);
|
|
950
|
+
const blockStore = transaction.objectStore("blocks");
|
|
951
|
+
for (const block of input.blocks) {
|
|
952
|
+
const bytes = block.bytes.byteOffset === 0 && block.bytes.byteLength === block.bytes.buffer.byteLength
|
|
953
|
+
? block.bytes
|
|
954
|
+
: block.bytes.slice();
|
|
955
|
+
blockStore.add(bytes, block.id);
|
|
956
|
+
}
|
|
957
|
+
const segmentStore = transaction.objectStore("segments");
|
|
958
|
+
for (const segment of input.segments) {
|
|
959
|
+
const normalized = normalizeSegmentRecord(segment);
|
|
960
|
+
segmentStore.add(normalized, normalized.id);
|
|
961
|
+
}
|
|
962
|
+
const outcome = await this.#commitInTransaction(transaction, staged, input);
|
|
963
|
+
await transactionDone(transaction);
|
|
964
|
+
outcome.settle();
|
|
965
|
+
return outcome.manifest;
|
|
966
|
+
}
|
|
967
|
+
catch (error) {
|
|
968
|
+
abortIfActive(transaction);
|
|
969
|
+
await ignoreAbort(transaction);
|
|
970
|
+
throw error;
|
|
971
|
+
}
|
|
972
|
+
}
|
|
973
|
+
/**
|
|
974
|
+
* The commit proper, inside an open readwrite transaction over every commit store, against a
|
|
975
|
+
* record the caller already resolved as active at the expected revision — the journaled
|
|
976
|
+
* record for `commitTransaction`, the freshly staged one for `writeTransaction`. Throws to
|
|
977
|
+
* refuse (the caller aborts the storage transaction); returns the summary plus the cache
|
|
978
|
+
* advances to apply once the transaction has proven durable.
|
|
979
|
+
*/
|
|
980
|
+
async #commitInTransaction(transaction, record, input) {
|
|
981
|
+
const transactionStore = transaction.objectStore("transactions");
|
|
895
982
|
const catalog = transaction.objectStore("catalog");
|
|
896
983
|
const current = (await requestResult(catalog.get(CURRENT_MANIFEST_KEY)));
|
|
897
984
|
const actualVersion = current ?? null;
|
|
898
985
|
if (actualVersion !== input.expectedManifestVersion) {
|
|
899
|
-
transaction.abort();
|
|
900
|
-
await ignoreAbort(transaction);
|
|
901
986
|
throw new WriteConflictError(input.expectedManifestVersion, actualVersion);
|
|
902
987
|
}
|
|
903
988
|
if (record.snapshotVersion !== input.expectedManifestVersion) {
|
|
904
|
-
transaction.abort();
|
|
905
|
-
await ignoreAbort(transaction);
|
|
906
989
|
throw new Error("Transaction snapshot does not match the expected manifest");
|
|
907
990
|
}
|
|
908
991
|
const manifestStore = transaction.objectStore("manifests");
|
|
@@ -913,15 +996,11 @@ export class IndexedDbBlockStore {
|
|
|
913
996
|
return value === undefined ? undefined : asStoredManifestRecord(value);
|
|
914
997
|
})();
|
|
915
998
|
if (input.expectedManifestVersion !== null && baseRecord === undefined) {
|
|
916
|
-
transaction.abort();
|
|
917
|
-
await ignoreAbort(transaction);
|
|
918
999
|
throw new Error(`Snapshot manifest is missing: ${String(input.expectedManifestVersion)}`);
|
|
919
1000
|
}
|
|
920
1001
|
const removedBlockIds = [...new Set(input.removedBlockIds ?? [])].sort();
|
|
921
1002
|
const pendingRemovedBlock = removedBlockIds.find((id) => record.pendingBlockIds.includes(id));
|
|
922
1003
|
if (pendingRemovedBlock !== undefined) {
|
|
923
|
-
transaction.abort();
|
|
924
|
-
await ignoreAbort(transaction);
|
|
925
1004
|
throw new Error(`Cannot supersede a pending block: ${pendingRemovedBlock}`);
|
|
926
1005
|
}
|
|
927
1006
|
// The resolved base block set is needed only to validate removals and to write checkpoints;
|
|
@@ -946,8 +1025,6 @@ export class IndexedDbBlockStore {
|
|
|
946
1025
|
}
|
|
947
1026
|
const invalidRemovedBlock = removedBlockIds.find((id) => !baseBlockIdSet?.has(id));
|
|
948
1027
|
if (invalidRemovedBlock !== undefined) {
|
|
949
|
-
transaction.abort();
|
|
950
|
-
await ignoreAbort(transaction);
|
|
951
1028
|
throw new Error(`Cannot supersede a block outside the transaction snapshot: ${invalidRemovedBlock}`);
|
|
952
1029
|
}
|
|
953
1030
|
}
|
|
@@ -955,23 +1032,17 @@ export class IndexedDbBlockStore {
|
|
|
955
1032
|
const storedBlockKeys = await Promise.all(record.pendingBlockIds.map((id) => requestResult(blockStore.getKey(id))));
|
|
956
1033
|
const missingBlockIndex = storedBlockKeys.findIndex((key) => key === undefined);
|
|
957
1034
|
if (missingBlockIndex >= 0) {
|
|
958
|
-
transaction.abort();
|
|
959
|
-
await ignoreAbort(transaction);
|
|
960
1035
|
throw new Error(`Manifest references missing block: ${record.pendingBlockIds[missingBlockIndex] ?? ""}`);
|
|
961
1036
|
}
|
|
962
1037
|
const segmentStore = transaction.objectStore("segments");
|
|
963
1038
|
const pendingSegmentValues = await Promise.all(record.pendingSegmentIds.map((id) => requestResult(segmentStore.get(id))));
|
|
964
1039
|
const missingSegmentIndex = pendingSegmentValues.findIndex((value) => value === undefined);
|
|
965
1040
|
if (missingSegmentIndex >= 0) {
|
|
966
|
-
transaction.abort();
|
|
967
|
-
await ignoreAbort(transaction);
|
|
968
1041
|
throw new Error(`Transaction references missing segment: ${record.pendingSegmentIds[missingSegmentIndex] ?? ""}`);
|
|
969
1042
|
}
|
|
970
1043
|
const pendingSegments = pendingSegmentValues.map(asSegmentRecord);
|
|
971
1044
|
const foreignSegment = pendingSegments.find((segment) => segment.transactionId !== record.id);
|
|
972
1045
|
if (foreignSegment !== undefined) {
|
|
973
|
-
transaction.abort();
|
|
974
|
-
await ignoreAbort(transaction);
|
|
975
1046
|
throw new Error(`Segment ${foreignSegment.id} belongs to another transaction`);
|
|
976
1047
|
}
|
|
977
1048
|
// The single-entry path below is the hot bulk-load path and keeps the membership cache;
|
|
@@ -979,50 +1050,34 @@ export class IndexedDbBlockStore {
|
|
|
979
1050
|
// drop the cache.
|
|
980
1051
|
const uniqueKeyEntries = input.uniqueKeyChanges ?? [];
|
|
981
1052
|
const uniqueKeyChanges = uniqueKeyEntries.length === 1 ? uniqueKeyEntries[0] : undefined;
|
|
982
|
-
const keyMode = uniqueKeyChanges?.storageMode;
|
|
983
|
-
const chunkedKeys = keyMode === "chunks-v1" || keyMode === "chunks-v2";
|
|
984
1053
|
const keyCache = this.#uniqueKeyCache;
|
|
985
|
-
const keyCacheValid =
|
|
986
|
-
keyCache !== undefined &&
|
|
1054
|
+
const keyCacheValid = keyCache !== undefined &&
|
|
987
1055
|
keyCache.tableId === uniqueKeyChanges?.tableId &&
|
|
988
|
-
keyCache.mode === keyMode &&
|
|
989
1056
|
keyCache.version === input.expectedManifestVersion;
|
|
990
1057
|
let keyState;
|
|
991
1058
|
if (uniqueKeyChanges !== undefined) {
|
|
992
1059
|
let conflictToken;
|
|
993
|
-
if (
|
|
994
|
-
|
|
995
|
-
|
|
996
|
-
|
|
997
|
-
|
|
998
|
-
|
|
999
|
-
existing.add(token);
|
|
1000
|
-
}
|
|
1001
|
-
keyState = {
|
|
1002
|
-
existing,
|
|
1003
|
-
index: keyCache.index,
|
|
1004
|
-
chunks: keyCache.chunks,
|
|
1005
|
-
fullPresent: keyCache.present,
|
|
1006
|
-
};
|
|
1007
|
-
}
|
|
1008
|
-
else {
|
|
1009
|
-
keyState = await readChunkedUniqueKeys(catalog, uniqueKeyChanges.tableId, uniqueKeyChanges.keyTokens, keyMode);
|
|
1010
|
-
}
|
|
1011
|
-
if (uniqueKeyChanges.requireAbsent) {
|
|
1012
|
-
conflictToken = uniqueKeyChanges.keyTokens.find((token) => keyState?.existing.has(token));
|
|
1060
|
+
if (keyCacheValid) {
|
|
1061
|
+
// Steady-state bulk load: membership answered entirely from memory, zero reads.
|
|
1062
|
+
const existing = new Set();
|
|
1063
|
+
for (const token of uniqueKeyChanges.keyTokens) {
|
|
1064
|
+
if (keyCache.present.has(token))
|
|
1065
|
+
existing.add(token);
|
|
1013
1066
|
}
|
|
1067
|
+
keyState = {
|
|
1068
|
+
existing,
|
|
1069
|
+
index: keyCache.index,
|
|
1070
|
+
chunks: keyCache.chunks,
|
|
1071
|
+
fullPresent: keyCache.present,
|
|
1072
|
+
};
|
|
1014
1073
|
}
|
|
1015
1074
|
else {
|
|
1016
|
-
|
|
1017
|
-
|
|
1018
|
-
|
|
1019
|
-
|
|
1020
|
-
conflictIndex >= 0 ? (uniqueKeyChanges.keyTokens[conflictIndex] ?? "") : undefined;
|
|
1021
|
-
}
|
|
1075
|
+
keyState = await readChunkedUniqueKeys(catalog, uniqueKeyChanges.tableId, uniqueKeyChanges.keyTokens);
|
|
1076
|
+
}
|
|
1077
|
+
if (uniqueKeyChanges.requireAbsent) {
|
|
1078
|
+
conflictToken = uniqueKeyChanges.keyTokens.find((token) => keyState?.existing.has(token));
|
|
1022
1079
|
}
|
|
1023
1080
|
if (conflictToken !== undefined) {
|
|
1024
|
-
transaction.abort();
|
|
1025
|
-
await ignoreAbort(transaction);
|
|
1026
1081
|
throw new UniqueKeyConflictError(uniqueKeyChanges.tableId, conflictToken);
|
|
1027
1082
|
}
|
|
1028
1083
|
}
|
|
@@ -1041,18 +1096,9 @@ export class IndexedDbBlockStore {
|
|
|
1041
1096
|
byTable.set(entry.tableId, list);
|
|
1042
1097
|
}
|
|
1043
1098
|
for (const [tableId, entries] of byTable) {
|
|
1044
|
-
const mode = entries[0]?.storageMode;
|
|
1045
|
-
const chunked = mode === "chunks-v1" || mode === "chunks-v2";
|
|
1046
1099
|
const unionTokens = [...new Set(entries.flatMap((entry) => entry.keyTokens))];
|
|
1047
|
-
|
|
1048
|
-
|
|
1049
|
-
const state = await readChunkedUniqueKeys(catalog, tableId, unionTokens, mode);
|
|
1050
|
-
present = state.existing;
|
|
1051
|
-
}
|
|
1052
|
-
else {
|
|
1053
|
-
const stored = await Promise.all(unionTokens.map((token) => requestResult(catalog.getKey(uniqueKeyKey(tableId, token)))));
|
|
1054
|
-
present = new Set(unionTokens.filter((_, index) => stored[index] !== undefined));
|
|
1055
|
-
}
|
|
1100
|
+
const state = await readChunkedUniqueKeys(catalog, tableId, unionTokens);
|
|
1101
|
+
const present = state.existing;
|
|
1056
1102
|
const original = new Set(present);
|
|
1057
1103
|
for (const entry of entries) {
|
|
1058
1104
|
for (const token of entry.keyTokens) {
|
|
@@ -1061,8 +1107,6 @@ export class IndexedDbBlockStore {
|
|
|
1061
1107
|
continue;
|
|
1062
1108
|
}
|
|
1063
1109
|
if (entry.requireAbsent && present.has(token)) {
|
|
1064
|
-
transaction.abort();
|
|
1065
|
-
await ignoreAbort(transaction);
|
|
1066
1110
|
throw new UniqueKeyConflictError(tableId, token);
|
|
1067
1111
|
}
|
|
1068
1112
|
present.add(token);
|
|
@@ -1070,7 +1114,6 @@ export class IndexedDbBlockStore {
|
|
|
1070
1114
|
}
|
|
1071
1115
|
multiKeyWork.push({
|
|
1072
1116
|
tableId,
|
|
1073
|
-
...(mode === undefined ? {} : { mode }),
|
|
1074
1117
|
addedTokens: unionTokens.filter((token) => present.has(token) && !original.has(token)),
|
|
1075
1118
|
removedTokens: unionTokens.filter((token) => !present.has(token) && original.has(token)),
|
|
1076
1119
|
});
|
|
@@ -1124,7 +1167,7 @@ export class IndexedDbBlockStore {
|
|
|
1124
1167
|
// full knowledge; `drop` forgets a same-table cache we could not keep truthful.
|
|
1125
1168
|
let keyCachePlan;
|
|
1126
1169
|
if (uniqueKeyChanges !== undefined) {
|
|
1127
|
-
|
|
1170
|
+
{
|
|
1128
1171
|
const existing = keyState?.existing ?? new Set();
|
|
1129
1172
|
const chunk = {
|
|
1130
1173
|
addedTokens: uniqueKeyChanges.remove === true
|
|
@@ -1144,147 +1187,116 @@ export class IndexedDbBlockStore {
|
|
|
1144
1187
|
(index.hasBase ? undefined : replayChunks(keyState?.chunks ?? []));
|
|
1145
1188
|
if (index.versions.length + 1 > UNIQUE_KEY_TAIL_CHUNK_LIMIT) {
|
|
1146
1189
|
const tailChunks = [...(keyState?.chunks ?? []), chunk];
|
|
1147
|
-
|
|
1148
|
-
|
|
1149
|
-
|
|
1150
|
-
|
|
1151
|
-
|
|
1152
|
-
|
|
1153
|
-
|
|
1154
|
-
|
|
1155
|
-
|
|
1156
|
-
|
|
1157
|
-
|
|
1158
|
-
|
|
1159
|
-
|
|
1160
|
-
|
|
1161
|
-
|
|
1162
|
-
|
|
1163
|
-
|
|
1164
|
-
|
|
1165
|
-
|
|
1166
|
-
|
|
1167
|
-
|
|
1168
|
-
|
|
1169
|
-
catalog.delete(uniqueKeyChunkKey(tableId, version));
|
|
1170
|
-
}
|
|
1171
|
-
catalog.put(merged, uniqueKeyChunkKey(tableId, manifest.version));
|
|
1172
|
-
const nextIndex = {
|
|
1173
|
-
versions: [manifest.version],
|
|
1174
|
-
hasBase: true,
|
|
1175
|
-
...(index.partitions === undefined ? {} : { partitions: index.partitions }),
|
|
1176
|
-
...(index.tokenCount === undefined ? {} : { tokenCount: index.tokenCount }),
|
|
1177
|
-
};
|
|
1178
|
-
catalog.put(nextIndex, uniqueKeyChunkIndexKey(tableId));
|
|
1179
|
-
let cachePresent = priorPresent;
|
|
1180
|
-
if (cachePresent !== undefined) {
|
|
1181
|
-
if (cachePresent === keyState?.fullPresent)
|
|
1182
|
-
cachePresent = new Set(cachePresent);
|
|
1183
|
-
applyChunk(cachePresent, chunk);
|
|
1184
|
-
}
|
|
1185
|
-
keyCachePlan =
|
|
1186
|
-
cachePresent === undefined
|
|
1187
|
-
? { action: "drop" }
|
|
1188
|
-
: {
|
|
1189
|
-
action: "replace",
|
|
1190
|
-
present: cachePresent,
|
|
1191
|
-
chunks: [merged],
|
|
1192
|
-
index: nextIndex,
|
|
1193
|
-
};
|
|
1190
|
+
// Fold the tail into hash-partitioned base chunks, atomically with the manifest
|
|
1191
|
+
// publication: hundreds of records regardless of key count, never one per key.
|
|
1192
|
+
//
|
|
1193
|
+
// A tail of point writes is worth almost nothing next to the base it would be
|
|
1194
|
+
// folded into, and folding it would rewrite whole partitions to record a handful
|
|
1195
|
+
// of tokens — a cost that grows with the table while the write does not. Such a
|
|
1196
|
+
// tail collapses into a single equivalent chunk instead, leaving the base alone.
|
|
1197
|
+
// The base is only rewritten once the tail is big enough to be worth it.
|
|
1198
|
+
const merged = index.hasBase && countTailTokens(tailChunks) < UNIQUE_KEY_PARTITION_TARGET
|
|
1199
|
+
? mergeTailChunks(tailChunks)
|
|
1200
|
+
: undefined;
|
|
1201
|
+
// A token's partition is fixed by its hash and the partition count, so a tail
|
|
1202
|
+
// that leaves the count unchanged only disturbs the partitions its own tokens
|
|
1203
|
+
// land in. That path reads and rewrites those, and nothing else. Everything else
|
|
1204
|
+
// (no recorded size, a partition count that has to change, a base written before
|
|
1205
|
+
// this existed) takes the full rewrite below.
|
|
1206
|
+
const incremental = merged !== undefined
|
|
1207
|
+
? undefined
|
|
1208
|
+
: await foldTailIntoTouchedPartitions(catalog, tableId, index, tailChunks);
|
|
1209
|
+
if (merged !== undefined) {
|
|
1210
|
+
for (const version of index.versions) {
|
|
1211
|
+
catalog.delete(uniqueKeyChunkKey(tableId, version));
|
|
1194
1212
|
}
|
|
1195
|
-
|
|
1196
|
-
|
|
1197
|
-
|
|
1198
|
-
|
|
1199
|
-
|
|
1200
|
-
|
|
1201
|
-
|
|
1202
|
-
|
|
1203
|
-
|
|
1204
|
-
|
|
1205
|
-
|
|
1206
|
-
|
|
1207
|
-
|
|
1208
|
-
let cachePresent = priorPresent;
|
|
1209
|
-
if (cachePresent !== undefined) {
|
|
1210
|
-
if (cachePresent === keyState?.fullPresent)
|
|
1211
|
-
cachePresent = new Set(cachePresent);
|
|
1212
|
-
applyChunk(cachePresent, chunk);
|
|
1213
|
-
}
|
|
1214
|
-
keyCachePlan =
|
|
1215
|
-
cachePresent === undefined
|
|
1216
|
-
? { action: "drop" }
|
|
1217
|
-
: { action: "replace", present: cachePresent, chunks: [], index: nextIndex };
|
|
1213
|
+
catalog.put(merged, uniqueKeyChunkKey(tableId, manifest.version));
|
|
1214
|
+
const nextIndex = {
|
|
1215
|
+
versions: [manifest.version],
|
|
1216
|
+
hasBase: true,
|
|
1217
|
+
...(index.partitions === undefined ? {} : { partitions: index.partitions }),
|
|
1218
|
+
...(index.tokenCount === undefined ? {} : { tokenCount: index.tokenCount }),
|
|
1219
|
+
};
|
|
1220
|
+
catalog.put(nextIndex, uniqueKeyChunkIndexKey(tableId));
|
|
1221
|
+
let cachePresent = priorPresent;
|
|
1222
|
+
if (cachePresent !== undefined) {
|
|
1223
|
+
if (cachePresent === keyState?.fullPresent)
|
|
1224
|
+
cachePresent = new Set(cachePresent);
|
|
1225
|
+
applyChunk(cachePresent, chunk);
|
|
1218
1226
|
}
|
|
1219
|
-
|
|
1220
|
-
|
|
1221
|
-
|
|
1222
|
-
|
|
1223
|
-
|
|
1224
|
-
|
|
1225
|
-
|
|
1226
|
-
|
|
1227
|
-
|
|
1228
|
-
|
|
1229
|
-
|
|
1230
|
-
|
|
1231
|
-
|
|
1232
|
-
const parts = Array.from({ length: partitions }, () => []);
|
|
1233
|
-
for (const token of nextPresent) {
|
|
1234
|
-
parts[fnv1a(token) % partitions]?.push(token);
|
|
1235
|
-
}
|
|
1236
|
-
parts.forEach((tokens, ordinal) => {
|
|
1237
|
-
catalog.put(tokens, uniqueKeyBasePartKey(tableId, ordinal));
|
|
1238
|
-
});
|
|
1239
|
-
for (let ordinal = partitions; ordinal < (index.partitions ?? 0); ordinal += 1) {
|
|
1240
|
-
catalog.delete(uniqueKeyBasePartKey(tableId, ordinal));
|
|
1241
|
-
}
|
|
1242
|
-
for (const version of index.versions) {
|
|
1243
|
-
catalog.delete(uniqueKeyChunkKey(tableId, version));
|
|
1244
|
-
}
|
|
1245
|
-
const nextIndex = {
|
|
1246
|
-
versions: [],
|
|
1247
|
-
hasBase: true,
|
|
1248
|
-
partitions,
|
|
1249
|
-
tokenCount: nextPresent.size,
|
|
1250
|
-
};
|
|
1251
|
-
catalog.put(nextIndex, uniqueKeyChunkIndexKey(tableId));
|
|
1252
|
-
keyCachePlan = {
|
|
1253
|
-
action: "replace",
|
|
1254
|
-
present: nextPresent,
|
|
1255
|
-
chunks: [],
|
|
1256
|
-
index: nextIndex,
|
|
1257
|
-
};
|
|
1227
|
+
keyCachePlan =
|
|
1228
|
+
cachePresent === undefined
|
|
1229
|
+
? { action: "drop" }
|
|
1230
|
+
: {
|
|
1231
|
+
action: "replace",
|
|
1232
|
+
present: cachePresent,
|
|
1233
|
+
chunks: [merged],
|
|
1234
|
+
index: nextIndex,
|
|
1235
|
+
};
|
|
1236
|
+
}
|
|
1237
|
+
else if (incremental !== undefined) {
|
|
1238
|
+
for (const version of index.versions) {
|
|
1239
|
+
catalog.delete(uniqueKeyChunkKey(tableId, version));
|
|
1258
1240
|
}
|
|
1241
|
+
const nextIndex = {
|
|
1242
|
+
versions: [],
|
|
1243
|
+
hasBase: true,
|
|
1244
|
+
partitions: incremental.partitions,
|
|
1245
|
+
tokenCount: incremental.tokenCount,
|
|
1246
|
+
};
|
|
1247
|
+
catalog.put(nextIndex, uniqueKeyChunkIndexKey(tableId));
|
|
1248
|
+
// The membership cache only survives when it was already complete; rebuilding
|
|
1249
|
+
// it here would re-read every partition and undo the saving.
|
|
1250
|
+
let cachePresent = priorPresent;
|
|
1251
|
+
if (cachePresent !== undefined) {
|
|
1252
|
+
if (cachePresent === keyState?.fullPresent)
|
|
1253
|
+
cachePresent = new Set(cachePresent);
|
|
1254
|
+
applyChunk(cachePresent, chunk);
|
|
1255
|
+
}
|
|
1256
|
+
keyCachePlan =
|
|
1257
|
+
cachePresent === undefined
|
|
1258
|
+
? { action: "drop" }
|
|
1259
|
+
: { action: "replace", present: cachePresent, chunks: [], index: nextIndex };
|
|
1259
1260
|
}
|
|
1260
1261
|
else {
|
|
1261
|
-
|
|
1262
|
-
|
|
1263
|
-
|
|
1264
|
-
|
|
1265
|
-
|
|
1266
|
-
|
|
1267
|
-
|
|
1268
|
-
|
|
1262
|
+
let nextPresent = priorPresent;
|
|
1263
|
+
if (nextPresent === undefined) {
|
|
1264
|
+
nextPresent = await readAllV2BaseTokens(catalog, tableId, index);
|
|
1265
|
+
for (const tail of keyState?.chunks ?? [])
|
|
1266
|
+
applyChunk(nextPresent, tail);
|
|
1267
|
+
}
|
|
1268
|
+
else if (nextPresent === keyState?.fullPresent) {
|
|
1269
|
+
// The cache's own set must not be mutated before the commit is durable.
|
|
1270
|
+
nextPresent = new Set(nextPresent);
|
|
1271
|
+
}
|
|
1272
|
+
applyChunk(nextPresent, chunk);
|
|
1273
|
+
const partitions = Math.max(1, Math.ceil(nextPresent.size / UNIQUE_KEY_PARTITION_TARGET));
|
|
1274
|
+
const parts = Array.from({ length: partitions }, () => []);
|
|
1275
|
+
for (const token of nextPresent) {
|
|
1276
|
+
parts[fnv1a(token) % partitions]?.push(token);
|
|
1277
|
+
}
|
|
1278
|
+
parts.forEach((tokens, ordinal) => {
|
|
1279
|
+
catalog.put(tokens, uniqueKeyBasePartKey(tableId, ordinal));
|
|
1280
|
+
});
|
|
1281
|
+
for (let ordinal = partitions; ordinal < (index.partitions ?? 0); ordinal += 1) {
|
|
1282
|
+
catalog.delete(uniqueKeyBasePartKey(tableId, ordinal));
|
|
1269
1283
|
}
|
|
1270
1284
|
for (const version of index.versions) {
|
|
1271
1285
|
catalog.delete(uniqueKeyChunkKey(tableId, version));
|
|
1272
1286
|
}
|
|
1273
|
-
const nextIndex = {
|
|
1287
|
+
const nextIndex = {
|
|
1288
|
+
versions: [],
|
|
1289
|
+
hasBase: true,
|
|
1290
|
+
partitions,
|
|
1291
|
+
tokenCount: nextPresent.size,
|
|
1292
|
+
};
|
|
1274
1293
|
catalog.put(nextIndex, uniqueKeyChunkIndexKey(tableId));
|
|
1275
|
-
|
|
1276
|
-
|
|
1277
|
-
|
|
1278
|
-
|
|
1279
|
-
|
|
1280
|
-
|
|
1281
|
-
chunks: [],
|
|
1282
|
-
index: nextIndex,
|
|
1283
|
-
};
|
|
1284
|
-
}
|
|
1285
|
-
else {
|
|
1286
|
-
keyCachePlan = { action: "drop" };
|
|
1287
|
-
}
|
|
1294
|
+
keyCachePlan = {
|
|
1295
|
+
action: "replace",
|
|
1296
|
+
present: nextPresent,
|
|
1297
|
+
chunks: [],
|
|
1298
|
+
index: nextIndex,
|
|
1299
|
+
};
|
|
1288
1300
|
}
|
|
1289
1301
|
}
|
|
1290
1302
|
else {
|
|
@@ -1326,18 +1338,9 @@ export class IndexedDbBlockStore {
|
|
|
1326
1338
|
}
|
|
1327
1339
|
}
|
|
1328
1340
|
}
|
|
1329
|
-
else {
|
|
1330
|
-
uniqueKeyChanges.keyTokens.forEach((token) => {
|
|
1331
|
-
const key = uniqueKeyKey(uniqueKeyChanges.tableId, token);
|
|
1332
|
-
if (uniqueKeyChanges.remove === true)
|
|
1333
|
-
catalog.delete(key);
|
|
1334
|
-
else
|
|
1335
|
-
catalog.put(manifest.version, key);
|
|
1336
|
-
});
|
|
1337
|
-
}
|
|
1338
1341
|
}
|
|
1339
1342
|
for (const work of multiKeyWork) {
|
|
1340
|
-
|
|
1343
|
+
{
|
|
1341
1344
|
if (work.addedTokens.length > 0 || work.removedTokens.length > 0) {
|
|
1342
1345
|
const indexValue = (await requestResult(catalog.get(uniqueKeyChunkIndexKey(work.tableId))));
|
|
1343
1346
|
const index = indexValue ?? { versions: [], hasBase: false };
|
|
@@ -1351,14 +1354,6 @@ export class IndexedDbBlockStore {
|
|
|
1351
1354
|
}, uniqueKeyChunkIndexKey(work.tableId));
|
|
1352
1355
|
}
|
|
1353
1356
|
}
|
|
1354
|
-
else {
|
|
1355
|
-
for (const token of work.addedTokens) {
|
|
1356
|
-
catalog.put(manifest.version, uniqueKeyKey(work.tableId, token));
|
|
1357
|
-
}
|
|
1358
|
-
for (const token of work.removedTokens) {
|
|
1359
|
-
catalog.delete(uniqueKeyKey(work.tableId, token));
|
|
1360
|
-
}
|
|
1361
|
-
}
|
|
1362
1357
|
}
|
|
1363
1358
|
// Full-text deltas apply atomically with the publish; a stale writer (one that committed
|
|
1364
1359
|
// segments to an indexed table without deltas) flips the affected columns to "invalid"
|
|
@@ -1387,71 +1382,68 @@ export class IndexedDbBlockStore {
|
|
|
1387
1382
|
}
|
|
1388
1383
|
}
|
|
1389
1384
|
transactionStore.put(committed, committed.id);
|
|
1390
|
-
|
|
1391
|
-
|
|
1392
|
-
|
|
1393
|
-
|
|
1394
|
-
this.#uniqueKeyCache = undefined;
|
|
1395
|
-
}
|
|
1396
|
-
// The unique-key cache follows the same rule as the manifest cache below: advance it only
|
|
1397
|
-
// after the durable commit. Commits from this instance cannot silently change another
|
|
1398
|
-
// table's key records, so a cache for a different table just moves to the new version.
|
|
1399
|
-
if (uniqueKeyChanges !== undefined && chunkedKeys && keyCachePlan !== undefined) {
|
|
1400
|
-
if (keyCachePlan.action === "keep" && keyCacheValid) {
|
|
1401
|
-
// In-place delta on the live cache: no copies of a multi-million-token set per batch.
|
|
1402
|
-
applyChunk(keyCache.present, keyCachePlan.chunk);
|
|
1403
|
-
keyCache.chunks.push(keyCachePlan.chunk);
|
|
1404
|
-
keyCache.index = keyCachePlan.index;
|
|
1405
|
-
keyCache.version = manifest.version;
|
|
1406
|
-
}
|
|
1407
|
-
else if (keyCachePlan.action === "replace") {
|
|
1408
|
-
this.#uniqueKeyCache = {
|
|
1409
|
-
version: manifest.version,
|
|
1410
|
-
tableId: uniqueKeyChanges.tableId,
|
|
1411
|
-
mode: keyMode,
|
|
1412
|
-
present: keyCachePlan.present,
|
|
1413
|
-
chunks: keyCachePlan.chunks,
|
|
1414
|
-
index: keyCachePlan.index,
|
|
1415
|
-
};
|
|
1416
|
-
}
|
|
1417
|
-
else if (this.#uniqueKeyCache?.tableId === uniqueKeyChanges.tableId) {
|
|
1385
|
+
const settle = () => {
|
|
1386
|
+
if (uniqueKeyEntries.length > 1) {
|
|
1387
|
+
// Multi-entry commits take the merged path above; the single-table membership cache
|
|
1388
|
+
// cannot describe them, so it drops and rebuilds on the next bulk load.
|
|
1418
1389
|
this.#uniqueKeyCache = undefined;
|
|
1419
1390
|
}
|
|
1420
|
-
|
|
1391
|
+
// The unique-key cache follows the same rule as the manifest cache below: advance it only
|
|
1392
|
+
// after the durable commit. Commits from this instance cannot silently change another
|
|
1393
|
+
// table's key records, so a cache for a different table just moves to the new version.
|
|
1394
|
+
if (uniqueKeyChanges !== undefined && keyCachePlan !== undefined) {
|
|
1395
|
+
if (keyCachePlan.action === "keep" && keyCacheValid) {
|
|
1396
|
+
// In-place delta on the live cache: no copies of a multi-million-token set per batch.
|
|
1397
|
+
applyChunk(keyCache.present, keyCachePlan.chunk);
|
|
1398
|
+
keyCache.chunks.push(keyCachePlan.chunk);
|
|
1399
|
+
keyCache.index = keyCachePlan.index;
|
|
1400
|
+
keyCache.version = manifest.version;
|
|
1401
|
+
}
|
|
1402
|
+
else if (keyCachePlan.action === "replace") {
|
|
1403
|
+
this.#uniqueKeyCache = {
|
|
1404
|
+
version: manifest.version,
|
|
1405
|
+
tableId: uniqueKeyChanges.tableId,
|
|
1406
|
+
present: keyCachePlan.present,
|
|
1407
|
+
chunks: keyCachePlan.chunks,
|
|
1408
|
+
index: keyCachePlan.index,
|
|
1409
|
+
};
|
|
1410
|
+
}
|
|
1411
|
+
else if (this.#uniqueKeyCache?.tableId === uniqueKeyChanges.tableId) {
|
|
1412
|
+
this.#uniqueKeyCache = undefined;
|
|
1413
|
+
}
|
|
1414
|
+
else if (this.#uniqueKeyCache !== undefined) {
|
|
1415
|
+
this.#uniqueKeyCache.version = manifest.version;
|
|
1416
|
+
}
|
|
1417
|
+
}
|
|
1418
|
+
else if (uniqueKeyChanges !== undefined &&
|
|
1419
|
+
this.#uniqueKeyCache?.tableId === uniqueKeyChanges.tableId) {
|
|
1420
|
+
// A no-op key write against the cached table: nothing changed on disk, so the cache
|
|
1421
|
+
// just moves to the new version.
|
|
1421
1422
|
this.#uniqueKeyCache.version = manifest.version;
|
|
1422
1423
|
}
|
|
1423
|
-
|
|
1424
|
-
else if (uniqueKeyChanges !== undefined &&
|
|
1425
|
-
this.#uniqueKeyCache?.tableId === uniqueKeyChanges.tableId) {
|
|
1426
|
-
// A no-op or legacy-mode write against the cached table: nothing changed on disk for a
|
|
1427
|
-
// no-op, but dropping is the safe answer for the legacy path.
|
|
1428
|
-
if (chunkedKeys)
|
|
1424
|
+
else if (this.#uniqueKeyCache !== undefined) {
|
|
1429
1425
|
this.#uniqueKeyCache.version = manifest.version;
|
|
1430
|
-
|
|
1431
|
-
|
|
1432
|
-
|
|
1433
|
-
|
|
1434
|
-
|
|
1435
|
-
|
|
1436
|
-
|
|
1437
|
-
|
|
1438
|
-
|
|
1439
|
-
|
|
1440
|
-
|
|
1441
|
-
|
|
1442
|
-
|
|
1443
|
-
|
|
1444
|
-
|
|
1445
|
-
|
|
1446
|
-
|
|
1447
|
-
|
|
1448
|
-
|
|
1449
|
-
}
|
|
1450
|
-
|
|
1451
|
-
// No resolved base was materialized; a later removal or checkpoint commit will resolve.
|
|
1452
|
-
this.#manifestCache = undefined;
|
|
1453
|
-
}
|
|
1454
|
-
return manifest;
|
|
1426
|
+
}
|
|
1427
|
+
// Advance the resolved-set cache only after the durable commit succeeded. A checkpoint
|
|
1428
|
+
// commit already built the new set; a delta commit with a resolved base applies the delta
|
|
1429
|
+
// in place (the cache owns that set from here on).
|
|
1430
|
+
if (checkpointBlockIds !== undefined) {
|
|
1431
|
+
this.#manifestCache = { version: manifest.version, blockIds: checkpointBlockIds };
|
|
1432
|
+
}
|
|
1433
|
+
else if (baseBlockIdSet !== undefined || cacheOwnsBaseSet) {
|
|
1434
|
+
const blockIds = baseBlockIdSet ?? new Set();
|
|
1435
|
+
for (const id of removedBlockIds)
|
|
1436
|
+
blockIds.delete(id);
|
|
1437
|
+
for (const id of addedBlockIds)
|
|
1438
|
+
blockIds.add(id);
|
|
1439
|
+
this.#manifestCache = { version: manifest.version, blockIds };
|
|
1440
|
+
}
|
|
1441
|
+
else {
|
|
1442
|
+
// No resolved base was materialized; a later removal or checkpoint commit will resolve.
|
|
1443
|
+
this.#manifestCache = undefined;
|
|
1444
|
+
}
|
|
1445
|
+
};
|
|
1446
|
+
return { manifest, settle };
|
|
1455
1447
|
}
|
|
1456
1448
|
async createLease(record) {
|
|
1457
1449
|
validateLeaseExpiration(record.expiresAt);
|
|
@@ -1503,6 +1495,30 @@ export class IndexedDbBlockStore {
|
|
|
1503
1495
|
throw error;
|
|
1504
1496
|
}
|
|
1505
1497
|
}
|
|
1498
|
+
async moveLease(id, expectedRevision, manifestVersion, expiresAt) {
|
|
1499
|
+
validateLeaseExpiration(expiresAt);
|
|
1500
|
+
const transaction = this.#transaction(["leases", "manifests", "blocks"], "readwrite");
|
|
1501
|
+
const store = transaction.objectStore("leases");
|
|
1502
|
+
const value = await requestResult(store.get(id));
|
|
1503
|
+
const record = value === undefined ? undefined : asLeaseRecord(value);
|
|
1504
|
+
if (record?.revision !== expectedRevision) {
|
|
1505
|
+
transaction.abort();
|
|
1506
|
+
await ignoreAbort(transaction);
|
|
1507
|
+
throw new LeaseConflictError(id, expectedRevision, record?.revision ?? null);
|
|
1508
|
+
}
|
|
1509
|
+
try {
|
|
1510
|
+
await assertSnapshotAvailableInTransaction(transaction, manifestVersion);
|
|
1511
|
+
const moved = { ...record, manifestVersion, expiresAt, revision: record.revision + 1 };
|
|
1512
|
+
store.put(moved, id);
|
|
1513
|
+
await transactionDone(transaction);
|
|
1514
|
+
return structuredClone(moved);
|
|
1515
|
+
}
|
|
1516
|
+
catch (error) {
|
|
1517
|
+
abortIfActive(transaction);
|
|
1518
|
+
await ignoreAbort(transaction);
|
|
1519
|
+
throw error;
|
|
1520
|
+
}
|
|
1521
|
+
}
|
|
1506
1522
|
async removeLeaseIfExpired(id, expectedRevision, expiresAtCutoff) {
|
|
1507
1523
|
const cutoff = Date.parse(expiresAtCutoff);
|
|
1508
1524
|
if (!Number.isFinite(cutoff))
|
|
@@ -1703,6 +1719,7 @@ export class IndexedDbBlockStore {
|
|
|
1703
1719
|
const manifestStore = transaction.objectStore("manifests");
|
|
1704
1720
|
const segmentStore = transaction.objectStore("segments");
|
|
1705
1721
|
const blockStore = transaction.objectStore("blocks");
|
|
1722
|
+
const transactionStore = transaction.objectStore("transactions");
|
|
1706
1723
|
const currentVersionValue = await requestResult(catalog.get(CURRENT_MANIFEST_KEY));
|
|
1707
1724
|
if (currentVersionValue !== undefined &&
|
|
1708
1725
|
(typeof currentVersionValue !== "number" ||
|
|
@@ -1723,6 +1740,9 @@ export class IndexedDbBlockStore {
|
|
|
1723
1740
|
const reclaimedBlockIds = [];
|
|
1724
1741
|
const retainedBlockIds = [];
|
|
1725
1742
|
const missingBlockIds = [];
|
|
1743
|
+
const reclaimedTransactionIds = [];
|
|
1744
|
+
const retainedTransactionIds = [];
|
|
1745
|
+
const missingTransactionIds = [];
|
|
1726
1746
|
let reclaimedBlockBytes = 0;
|
|
1727
1747
|
let remaining = input.maxItems;
|
|
1728
1748
|
let manifestIndex = current.cursor.manifestIndex;
|
|
@@ -1778,6 +1798,59 @@ export class IndexedDbBlockStore {
|
|
|
1778
1798
|
remaining -= 1;
|
|
1779
1799
|
}
|
|
1780
1800
|
let blockIndex = current.cursor.blockIndex;
|
|
1801
|
+
let transactionIndex = current.cursor.transactionIndex;
|
|
1802
|
+
if (remaining > 0 &&
|
|
1803
|
+
manifestIndex === current.candidateManifestVersions.length &&
|
|
1804
|
+
segmentIndex === current.candidateSegmentIds.length &&
|
|
1805
|
+
blockIndex === current.candidateBlockIds.length) {
|
|
1806
|
+
const candidates = new Set(current.candidateTransactionIds.slice(transactionIndex));
|
|
1807
|
+
const segmentOwners = new Set();
|
|
1808
|
+
await visitObjectStoreSequentially(segmentStore, (value) => {
|
|
1809
|
+
const owner = asSegmentRecord(value).transactionId;
|
|
1810
|
+
if (candidates.has(owner))
|
|
1811
|
+
segmentOwners.add(owner);
|
|
1812
|
+
});
|
|
1813
|
+
const unfinishedCompactionOwners = new Set();
|
|
1814
|
+
await visitObjectStoreSequentially(gcStore, (value) => {
|
|
1815
|
+
if (!isCompactionJobEnvelope(value))
|
|
1816
|
+
return;
|
|
1817
|
+
const job = asCompactionJobEnvelope(value);
|
|
1818
|
+
if (!isTerminalCompactionJob(job) &&
|
|
1819
|
+
job.transactionId !== null &&
|
|
1820
|
+
candidates.has(job.transactionId)) {
|
|
1821
|
+
unfinishedCompactionOwners.add(job.transactionId);
|
|
1822
|
+
}
|
|
1823
|
+
});
|
|
1824
|
+
while (remaining > 0 && transactionIndex < current.candidateTransactionIds.length) {
|
|
1825
|
+
const id = current.candidateTransactionIds[transactionIndex];
|
|
1826
|
+
if (id === undefined) {
|
|
1827
|
+
throw new Error("Garbage collection transaction cursor is invalid");
|
|
1828
|
+
}
|
|
1829
|
+
const transactionValue = await requestResult(transactionStore.get(id));
|
|
1830
|
+
const record = transactionValue === undefined ? undefined : asTransactionRecord(transactionValue);
|
|
1831
|
+
const manifestValue = record?.committedVersion === null || record?.committedVersion === undefined
|
|
1832
|
+
? undefined
|
|
1833
|
+
: await requestResult(manifestStore.get(record.committedVersion));
|
|
1834
|
+
const manifest = manifestValue === undefined ? undefined : asStoredManifestRecord(manifestValue);
|
|
1835
|
+
if (record === undefined)
|
|
1836
|
+
missingTransactionIds.push(id);
|
|
1837
|
+
else if (unfinishedCompactionOwners.has(id) ||
|
|
1838
|
+
(record.status === "committed"
|
|
1839
|
+
? (manifest !== undefined && manifest.prunedAt === undefined) || segmentOwners.has(id)
|
|
1840
|
+
: record.status === "aborted"
|
|
1841
|
+
? (await anyObjectStoreKeyExists(blockStore, record.pendingBlockIds)) ||
|
|
1842
|
+
(await anyObjectStoreKeyExists(segmentStore, record.pendingSegmentIds))
|
|
1843
|
+
: true)) {
|
|
1844
|
+
retainedTransactionIds.push(id);
|
|
1845
|
+
}
|
|
1846
|
+
else {
|
|
1847
|
+
transactionStore.delete(id);
|
|
1848
|
+
reclaimedTransactionIds.push(id);
|
|
1849
|
+
}
|
|
1850
|
+
transactionIndex += 1;
|
|
1851
|
+
remaining -= 1;
|
|
1852
|
+
}
|
|
1853
|
+
}
|
|
1781
1854
|
while (remaining > 0 &&
|
|
1782
1855
|
manifestIndex === current.candidateManifestVersions.length &&
|
|
1783
1856
|
segmentIndex === current.candidateSegmentIds.length &&
|
|
@@ -1817,6 +1890,12 @@ export class IndexedDbBlockStore {
|
|
|
1817
1890
|
retainedBlockCount: retainedBlockIds.length,
|
|
1818
1891
|
missingBlockCount: missingBlockIds.length,
|
|
1819
1892
|
reclaimedBlockBytes,
|
|
1893
|
+
examinedTransactionCount: reclaimedTransactionIds.length +
|
|
1894
|
+
retainedTransactionIds.length +
|
|
1895
|
+
missingTransactionIds.length,
|
|
1896
|
+
reclaimedTransactionCount: reclaimedTransactionIds.length,
|
|
1897
|
+
retainedTransactionCount: retainedTransactionIds.length,
|
|
1898
|
+
missingTransactionCount: missingTransactionIds.length,
|
|
1820
1899
|
updatedAt: input.updatedAt,
|
|
1821
1900
|
});
|
|
1822
1901
|
gcStore.put(garbageCollectionJobEnvelope(updated), key);
|
|
@@ -1834,6 +1913,9 @@ export class IndexedDbBlockStore {
|
|
|
1834
1913
|
retainedBlockIds,
|
|
1835
1914
|
missingBlockIds,
|
|
1836
1915
|
reclaimedBlockBytes,
|
|
1916
|
+
reclaimedTransactionIds,
|
|
1917
|
+
retainedTransactionIds,
|
|
1918
|
+
missingTransactionIds,
|
|
1837
1919
|
};
|
|
1838
1920
|
}
|
|
1839
1921
|
catch (error) {
|
|
@@ -1842,6 +1924,53 @@ export class IndexedDbBlockStore {
|
|
|
1842
1924
|
throw error;
|
|
1843
1925
|
}
|
|
1844
1926
|
}
|
|
1927
|
+
async removePrunedManifestRecords() {
|
|
1928
|
+
const transaction = this.#transaction(["catalog", "manifests", "blocks"], "readwrite");
|
|
1929
|
+
const store = transaction.objectStore("manifests");
|
|
1930
|
+
const blockStore = transaction.objectStore("blocks");
|
|
1931
|
+
try {
|
|
1932
|
+
const values = await requestResult(store.getAll());
|
|
1933
|
+
const records = values.map(asStoredManifestRecord);
|
|
1934
|
+
const earliestReadable = records
|
|
1935
|
+
.filter((record) => record.prunedAt === undefined)
|
|
1936
|
+
.sort((left, right) => left.version - right.version)[0];
|
|
1937
|
+
const safeBelow = earliestReadable === undefined
|
|
1938
|
+
? Number.POSITIVE_INFINITY
|
|
1939
|
+
: earliestReadable.version - (earliestReadable.deltaDepth ?? 0);
|
|
1940
|
+
const currentVersionValue = await requestResult(transaction.objectStore("catalog").get(CURRENT_MANIFEST_KEY));
|
|
1941
|
+
let currentBlockIds = new Set();
|
|
1942
|
+
if (typeof currentVersionValue === "number") {
|
|
1943
|
+
const currentValue = await requestResult(store.get(currentVersionValue));
|
|
1944
|
+
if (currentValue === undefined)
|
|
1945
|
+
throw new Error("Current manifest is missing");
|
|
1946
|
+
currentBlockIds = await resolveManifestBlockSetInTransaction(store, asStoredManifestRecord(currentValue));
|
|
1947
|
+
}
|
|
1948
|
+
const removable = [];
|
|
1949
|
+
// Resolve every candidate before issuing deletes: a later candidate may still use an
|
|
1950
|
+
// earlier checkpoint even though both sit below the readable chain's safe boundary.
|
|
1951
|
+
for (const record of records) {
|
|
1952
|
+
if (record.prunedAt === undefined || record.version >= safeBelow)
|
|
1953
|
+
continue;
|
|
1954
|
+
const blockIds = await resolveManifestBlockSetInTransaction(store, record);
|
|
1955
|
+
const garbageIds = [...blockIds].filter((id) => !currentBlockIds.has(id));
|
|
1956
|
+
if (await anyObjectStoreKeyExists(blockStore, garbageIds))
|
|
1957
|
+
continue;
|
|
1958
|
+
removable.push(record.version);
|
|
1959
|
+
}
|
|
1960
|
+
let removed = 0;
|
|
1961
|
+
for (const version of removable) {
|
|
1962
|
+
store.delete(version);
|
|
1963
|
+
removed += 1;
|
|
1964
|
+
}
|
|
1965
|
+
await transactionDone(transaction);
|
|
1966
|
+
return removed;
|
|
1967
|
+
}
|
|
1968
|
+
catch (error) {
|
|
1969
|
+
transaction.abort();
|
|
1970
|
+
await ignoreAbort(transaction);
|
|
1971
|
+
throw error;
|
|
1972
|
+
}
|
|
1973
|
+
}
|
|
1845
1974
|
async removeGarbageCollectionJob(id) {
|
|
1846
1975
|
const transaction = this.#transaction("gc", "readwrite");
|
|
1847
1976
|
transaction.objectStore("gc").delete(garbageCollectionJobKey(id));
|
|
@@ -1897,7 +2026,7 @@ export class IndexedDbBlockStore {
|
|
|
1897
2026
|
]);
|
|
1898
2027
|
const uniqueKeyTokens = record.uniqueKeyColumnId === undefined
|
|
1899
2028
|
? []
|
|
1900
|
-
: [...(await readAllUniqueKeyTokens(catalog, record.id
|
|
2029
|
+
: [...(await readAllUniqueKeyTokens(catalog, record.id))];
|
|
1901
2030
|
const fts = [];
|
|
1902
2031
|
const ftsColumns = { ...(record.ftsColumns ?? {}) };
|
|
1903
2032
|
for (const [columnId, state] of Object.entries(ftsColumns)) {
|
|
@@ -2005,7 +2134,7 @@ export class IndexedDbBlockStore {
|
|
|
2005
2134
|
// as commit chunks instead would leave a tail long enough to make the first write crawl.
|
|
2006
2135
|
const record = {
|
|
2007
2136
|
...table.record,
|
|
2008
|
-
...(keyed ? { uniqueKeyLookupReady: true
|
|
2137
|
+
...(keyed ? { uniqueKeyLookupReady: true } : {}),
|
|
2009
2138
|
};
|
|
2010
2139
|
const transaction = this.#transaction("catalog", "readwrite");
|
|
2011
2140
|
const catalog = transaction.objectStore("catalog");
|
|
@@ -2163,6 +2292,13 @@ function asBytes(value) {
|
|
|
2163
2292
|
function asStoredManifestRecord(value) {
|
|
2164
2293
|
return structuredClone(value);
|
|
2165
2294
|
}
|
|
2295
|
+
async function anyObjectStoreKeyExists(store, ids) {
|
|
2296
|
+
for (const id of ids) {
|
|
2297
|
+
if ((await requestResult(store.getKey(id))) !== undefined)
|
|
2298
|
+
return true;
|
|
2299
|
+
}
|
|
2300
|
+
return false;
|
|
2301
|
+
}
|
|
2166
2302
|
/** Builds the resolved public view of one stored record from its resolved block set. */
|
|
2167
2303
|
function manifestView(record, blockIds) {
|
|
2168
2304
|
return {
|
|
@@ -2232,6 +2368,19 @@ function asLeaseRecord(value) {
|
|
|
2232
2368
|
* deletes blocks unreachable from every non-pruned manifest — so this stays O(chain) instead of
|
|
2233
2369
|
* probing every live block on each transaction and lease creation.
|
|
2234
2370
|
*/
|
|
2371
|
+
/**
|
|
2372
|
+
* The transaction record a commit needs: present, active, and at the expected revision —
|
|
2373
|
+
* otherwise a `TransactionRecordConflictError`, the caller's storage transaction still open for
|
|
2374
|
+
* it to abort.
|
|
2375
|
+
*/
|
|
2376
|
+
async function readActiveTransactionRecord(transaction, id, expectedRevision) {
|
|
2377
|
+
const value = await requestResult(transaction.objectStore("transactions").get(id));
|
|
2378
|
+
const record = value === undefined ? undefined : asTransactionRecord(value);
|
|
2379
|
+
if (record?.revision !== expectedRevision || record.status !== "active") {
|
|
2380
|
+
throw new TransactionRecordConflictError(id, expectedRevision, record?.revision ?? null);
|
|
2381
|
+
}
|
|
2382
|
+
return record;
|
|
2383
|
+
}
|
|
2235
2384
|
async function assertSnapshotAvailableInTransaction(transaction, version) {
|
|
2236
2385
|
if (version === null)
|
|
2237
2386
|
return;
|
|
@@ -2392,6 +2541,15 @@ async function assertGarbageCollectionCandidateProvenanceInTransaction(transacti
|
|
|
2392
2541
|
}
|
|
2393
2542
|
candidateManifestBlockIds.push(await resolveManifestBlockSetInTransaction(manifestStore, asStoredManifestRecord(value)));
|
|
2394
2543
|
}
|
|
2544
|
+
for (const id of job.candidateTransactionIds) {
|
|
2545
|
+
const value = await requestResult(transaction.objectStore("transactions").get(id));
|
|
2546
|
+
const record = value === undefined ? undefined : asTransactionRecord(value);
|
|
2547
|
+
if (record === undefined ||
|
|
2548
|
+
(record.status !== "aborted" &&
|
|
2549
|
+
(record.status !== "committed" || record.committedVersion === null))) {
|
|
2550
|
+
throw new Error(`Garbage collection transaction candidate is not terminal: ${id}`);
|
|
2551
|
+
}
|
|
2552
|
+
}
|
|
2395
2553
|
const blockHasProvenance = async (id) => {
|
|
2396
2554
|
if (candidateManifestBlockIds.some((blockIds) => blockIds.has(id)))
|
|
2397
2555
|
return true;
|
|
@@ -2630,6 +2788,9 @@ function emptyGarbageCollectionStep(job) {
|
|
|
2630
2788
|
retainedBlockIds: [],
|
|
2631
2789
|
missingBlockIds: [],
|
|
2632
2790
|
reclaimedBlockBytes: 0,
|
|
2791
|
+
reclaimedTransactionIds: [],
|
|
2792
|
+
retainedTransactionIds: [],
|
|
2793
|
+
missingTransactionIds: [],
|
|
2633
2794
|
};
|
|
2634
2795
|
}
|
|
2635
2796
|
function safeStorageSum(left, right) {
|
|
@@ -2777,18 +2938,12 @@ function readTempPageOwnerIds(store, afterOwnerId, max) {
|
|
|
2777
2938
|
};
|
|
2778
2939
|
});
|
|
2779
2940
|
}
|
|
2780
|
-
function uniqueKeyKey(tableId, keyToken) {
|
|
2781
|
-
return ["unique-key", tableId, keyToken];
|
|
2782
|
-
}
|
|
2783
2941
|
function uniqueKeyChunkIndexKey(tableId) {
|
|
2784
2942
|
return [UNIQUE_KEY_CHUNK_INDEX, tableId];
|
|
2785
2943
|
}
|
|
2786
2944
|
function uniqueKeyChunkKey(tableId, version) {
|
|
2787
2945
|
return [UNIQUE_KEY_CHUNK, tableId, version];
|
|
2788
2946
|
}
|
|
2789
|
-
function uniqueKeyBaseKey(tableId, keyToken) {
|
|
2790
|
-
return [UNIQUE_KEY_BASE, tableId, keyToken];
|
|
2791
|
-
}
|
|
2792
2947
|
function uniqueKeyBasePartKey(tableId, ordinal) {
|
|
2793
2948
|
return [UNIQUE_KEY_BASE_PART, tableId, ordinal];
|
|
2794
2949
|
}
|
|
@@ -2847,7 +3002,6 @@ function asBasePartition(value) {
|
|
|
2847
3002
|
throw new Error("Unique-key base partition is invalid");
|
|
2848
3003
|
return value;
|
|
2849
3004
|
}
|
|
2850
|
-
/** Reads every chunks-v2 base partition into one membership set. */
|
|
2851
3005
|
/** How many distinct tokens a tail carries, counting a token touched twice once. */
|
|
2852
3006
|
function countTailTokens(tailChunks) {
|
|
2853
3007
|
const seen = new Set();
|
|
@@ -2952,61 +3106,45 @@ async function readAllV2BaseTokens(store, tableId, index) {
|
|
|
2952
3106
|
return present;
|
|
2953
3107
|
}
|
|
2954
3108
|
/**
|
|
2955
|
-
* A table's complete unique-key membership: the folded base
|
|
2956
|
-
*
|
|
2957
|
-
*
|
|
3109
|
+
* A table's complete unique-key membership: the folded base plus every unfolded tail chunk
|
|
3110
|
+
* replayed in commit order. Snapshots need the whole set, where the commit path only ever
|
|
3111
|
+
* needs the tokens it is about to write.
|
|
2958
3112
|
*/
|
|
2959
|
-
async function readAllUniqueKeyTokens(store, tableId
|
|
2960
|
-
if (mode !== "chunks-v1" && mode !== "chunks-v2") {
|
|
2961
|
-
// The pre-chunk layout stored one record per key and predates every shipped writer.
|
|
2962
|
-
return scanLegacyUniqueKeyTokens(store, tableId);
|
|
2963
|
-
}
|
|
3113
|
+
async function readAllUniqueKeyTokens(store, tableId) {
|
|
2964
3114
|
const index = asUniqueKeyChunkIndex(await requestResult(store.get(uniqueKeyChunkIndexKey(tableId))));
|
|
2965
3115
|
const present = index.hasBase
|
|
2966
|
-
?
|
|
2967
|
-
? await readAllV2BaseTokens(store, tableId, index)
|
|
2968
|
-
: await scanUniqueKeyBaseTokens(store, tableId)
|
|
3116
|
+
? await readAllV2BaseTokens(store, tableId, index)
|
|
2969
3117
|
: new Set();
|
|
2970
3118
|
const chunks = await Promise.all(index.versions.map((version) => requestResult(store.get(uniqueKeyChunkKey(tableId, version)))));
|
|
2971
3119
|
for (const chunk of chunks)
|
|
2972
3120
|
applyChunk(present, asUniqueKeyChunk(chunk));
|
|
2973
3121
|
return present;
|
|
2974
3122
|
}
|
|
2975
|
-
async function readChunkedUniqueKeys(store, tableId, requestedTokens
|
|
3123
|
+
async function readChunkedUniqueKeys(store, tableId, requestedTokens) {
|
|
2976
3124
|
const rawIndex = await requestResult(store.get(uniqueKeyChunkIndexKey(tableId)));
|
|
2977
3125
|
const index = asUniqueKeyChunkIndex(rawIndex);
|
|
2978
3126
|
const bulk = requestedTokens.length >= UNIQUE_KEY_BASE_SCAN_THRESHOLD;
|
|
2979
3127
|
const chunkReads = Promise.all(index.versions.map((version) => requestResult(store.get(uniqueKeyChunkKey(tableId, version)))));
|
|
2980
3128
|
let baseMembership;
|
|
2981
3129
|
let fullBase;
|
|
2982
|
-
let probedKeys = [];
|
|
2983
3130
|
if (index.hasBase) {
|
|
2984
|
-
if (
|
|
2985
|
-
|
|
2986
|
-
fullBase = await readAllV2BaseTokens(store, tableId, index);
|
|
2987
|
-
baseMembership = fullBase;
|
|
2988
|
-
}
|
|
2989
|
-
else {
|
|
2990
|
-
// Point reads touch only the partitions the requested tokens hash to.
|
|
2991
|
-
const partitions = index.partitions ?? 1;
|
|
2992
|
-
const ordinals = [...new Set(requestedTokens.map((token) => fnv1a(token) % partitions))];
|
|
2993
|
-
const parts = await Promise.all(ordinals.map((ordinal) => requestResult(store.get(uniqueKeyBasePartKey(tableId, ordinal)))));
|
|
2994
|
-
const membership = new Set();
|
|
2995
|
-
for (const part of parts) {
|
|
2996
|
-
if (part === undefined)
|
|
2997
|
-
continue;
|
|
2998
|
-
for (const token of asBasePartition(part))
|
|
2999
|
-
membership.add(token);
|
|
3000
|
-
}
|
|
3001
|
-
baseMembership = membership;
|
|
3002
|
-
}
|
|
3003
|
-
}
|
|
3004
|
-
else if (bulk) {
|
|
3005
|
-
fullBase = await scanUniqueKeyBaseTokens(store, tableId);
|
|
3131
|
+
if (bulk) {
|
|
3132
|
+
fullBase = await readAllV2BaseTokens(store, tableId, index);
|
|
3006
3133
|
baseMembership = fullBase;
|
|
3007
3134
|
}
|
|
3008
3135
|
else {
|
|
3009
|
-
|
|
3136
|
+
// Point reads touch only the partitions the requested tokens hash to.
|
|
3137
|
+
const partitions = index.partitions ?? 1;
|
|
3138
|
+
const ordinals = [...new Set(requestedTokens.map((token) => fnv1a(token) % partitions))];
|
|
3139
|
+
const parts = await Promise.all(ordinals.map((ordinal) => requestResult(store.get(uniqueKeyBasePartKey(tableId, ordinal)))));
|
|
3140
|
+
const membership = new Set();
|
|
3141
|
+
for (const part of parts) {
|
|
3142
|
+
if (part === undefined)
|
|
3143
|
+
continue;
|
|
3144
|
+
for (const token of asBasePartition(part))
|
|
3145
|
+
membership.add(token);
|
|
3146
|
+
}
|
|
3147
|
+
baseMembership = membership;
|
|
3010
3148
|
}
|
|
3011
3149
|
}
|
|
3012
3150
|
const chunks = (await chunkReads).map(asUniqueKeyChunk);
|
|
@@ -3018,12 +3156,6 @@ async function readChunkedUniqueKeys(store, tableId, requestedTokens, mode) {
|
|
|
3018
3156
|
existing.add(token);
|
|
3019
3157
|
}
|
|
3020
3158
|
}
|
|
3021
|
-
else {
|
|
3022
|
-
requestedTokens.forEach((token, position) => {
|
|
3023
|
-
if (probedKeys[position] !== undefined)
|
|
3024
|
-
existing.add(token);
|
|
3025
|
-
});
|
|
3026
|
-
}
|
|
3027
3159
|
for (const chunk of chunks) {
|
|
3028
3160
|
for (const token of chunk.addedTokens) {
|
|
3029
3161
|
if (requested.has(token))
|
|
@@ -3045,56 +3177,6 @@ async function readChunkedUniqueKeys(store, tableId, requestedTokens, mode) {
|
|
|
3045
3177
|
...(fullPresent === undefined ? {} : { fullPresent }),
|
|
3046
3178
|
};
|
|
3047
3179
|
}
|
|
3048
|
-
/**
|
|
3049
|
-
* One key-cursor pass over a table's folded base records. Seeks with cursor.continue(key)
|
|
3050
|
-
* rather than IDBKeyRange, which the injected test factory lacks; a seek that lands past the
|
|
3051
|
-
* range (or throws because the cursor is already beyond it) means the walk is complete.
|
|
3052
|
-
*/
|
|
3053
|
-
function scanUniqueKeyBaseTokens(store, tableId) {
|
|
3054
|
-
return scanUniqueKeyTokenRange(store, UNIQUE_KEY_BASE, tableId);
|
|
3055
|
-
}
|
|
3056
|
-
/** The same walk over the pre-chunk layout, where each key was its own record. */
|
|
3057
|
-
function scanLegacyUniqueKeyTokens(store, tableId) {
|
|
3058
|
-
return scanUniqueKeyTokenRange(store, "unique-key", tableId);
|
|
3059
|
-
}
|
|
3060
|
-
function scanUniqueKeyTokenRange(store, prefix, tableId) {
|
|
3061
|
-
return new Promise((resolve, reject) => {
|
|
3062
|
-
const tokens = new Set();
|
|
3063
|
-
const rangeStart = [prefix, tableId, ""];
|
|
3064
|
-
const request = store.openKeyCursor();
|
|
3065
|
-
let seeked = false;
|
|
3066
|
-
request.onerror = () => reject(request.error ?? new Error("IndexedDB cursor failed"));
|
|
3067
|
-
request.onsuccess = () => {
|
|
3068
|
-
const cursor = request.result;
|
|
3069
|
-
if (cursor === null) {
|
|
3070
|
-
resolve(tokens);
|
|
3071
|
-
return;
|
|
3072
|
-
}
|
|
3073
|
-
const key = cursor.key;
|
|
3074
|
-
if (Array.isArray(key) &&
|
|
3075
|
-
key[0] === prefix &&
|
|
3076
|
-
key[1] === tableId &&
|
|
3077
|
-
typeof key[2] === "string") {
|
|
3078
|
-
tokens.add(key[2]);
|
|
3079
|
-
cursor.continue();
|
|
3080
|
-
return;
|
|
3081
|
-
}
|
|
3082
|
-
if (!seeked) {
|
|
3083
|
-
seeked = true;
|
|
3084
|
-
try {
|
|
3085
|
-
cursor.continue(rangeStart);
|
|
3086
|
-
}
|
|
3087
|
-
catch {
|
|
3088
|
-
// The first visited key already sorts at or past the range start: nothing to walk.
|
|
3089
|
-
resolve(tokens);
|
|
3090
|
-
}
|
|
3091
|
-
return;
|
|
3092
|
-
}
|
|
3093
|
-
// Keys arrive in order, so the first non-matching key after the seek ends the range.
|
|
3094
|
-
resolve(tokens);
|
|
3095
|
-
};
|
|
3096
|
-
});
|
|
3097
|
-
}
|
|
3098
3180
|
function asUniqueKeyChunk(value) {
|
|
3099
3181
|
if (typeof value !== "object" || value === null) {
|
|
3100
3182
|
throw new Error("Unique-key chunk is missing or invalid");
|