@minnowdb/core 0.0.1 → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +27 -1
- package/dist/block-format/block.d.ts +46 -0
- package/dist/block-format/block.d.ts.map +1 -0
- package/dist/block-format/block.js +244 -0
- package/dist/block-format/block.js.map +1 -0
- package/dist/block-format/checksum.d.ts +7 -0
- package/dist/block-format/checksum.d.ts.map +1 -0
- package/dist/block-format/checksum.js +56 -0
- package/dist/block-format/checksum.js.map +1 -0
- package/dist/block-format/codecs.d.ts +24 -0
- package/dist/block-format/codecs.d.ts.map +1 -0
- package/dist/block-format/codecs.js +102 -0
- package/dist/block-format/codecs.js.map +1 -0
- package/dist/block-format/column.d.ts +8 -0
- package/dist/block-format/column.d.ts.map +1 -0
- package/dist/block-format/column.js +166 -0
- package/dist/block-format/column.js.map +1 -0
- package/dist/block-format/index.d.ts +7 -0
- package/dist/block-format/index.d.ts.map +1 -0
- package/dist/block-format/index.js +7 -0
- package/dist/block-format/index.js.map +1 -0
- package/dist/block-format/physical.d.ts +20 -0
- package/dist/block-format/physical.d.ts.map +1 -0
- package/dist/block-format/physical.js +439 -0
- package/dist/block-format/physical.js.map +1 -0
- package/dist/block-format/types.d.ts +93 -0
- package/dist/block-format/types.d.ts.map +1 -0
- package/dist/block-format/types.js +2 -0
- package/dist/block-format/types.js.map +1 -0
- package/dist/engine/artifact-cache.d.ts +22 -0
- package/dist/engine/artifact-cache.d.ts.map +1 -0
- package/dist/engine/artifact-cache.js +68 -0
- package/dist/engine/artifact-cache.js.map +1 -0
- package/dist/engine/batch.d.ts +28 -0
- package/dist/engine/batch.d.ts.map +1 -0
- package/dist/engine/batch.js +38 -0
- package/dist/engine/batch.js.map +1 -0
- package/dist/engine/buffered-writer.d.ts +43 -0
- package/dist/engine/buffered-writer.d.ts.map +1 -0
- package/dist/engine/buffered-writer.js +158 -0
- package/dist/engine/buffered-writer.js.map +1 -0
- package/dist/engine/catalog.d.ts +82 -0
- package/dist/engine/catalog.d.ts.map +1 -0
- package/dist/engine/catalog.js +47 -0
- package/dist/engine/catalog.js.map +1 -0
- package/dist/engine/client.d.ts +211 -0
- package/dist/engine/client.d.ts.map +1 -0
- package/dist/engine/client.js +538 -0
- package/dist/engine/client.js.map +1 -0
- package/dist/engine/coordinator.d.ts +17 -0
- package/dist/engine/coordinator.d.ts.map +1 -0
- package/dist/engine/coordinator.js +60 -0
- package/dist/engine/coordinator.js.map +1 -0
- package/dist/engine/database.d.ts +661 -0
- package/dist/engine/database.d.ts.map +1 -0
- package/dist/engine/database.js +10313 -0
- package/dist/engine/database.js.map +1 -0
- package/dist/engine/defaults.d.ts +33 -0
- package/dist/engine/defaults.d.ts.map +1 -0
- package/dist/engine/defaults.js +87 -0
- package/dist/engine/defaults.js.map +1 -0
- package/dist/engine/dsl/db.d.ts +137 -0
- package/dist/engine/dsl/db.d.ts.map +1 -0
- package/dist/engine/dsl/db.js +161 -0
- package/dist/engine/dsl/db.js.map +1 -0
- package/dist/engine/dsl/expression.d.ts +172 -0
- package/dist/engine/dsl/expression.d.ts.map +1 -0
- package/dist/engine/dsl/expression.js +397 -0
- package/dist/engine/dsl/expression.js.map +1 -0
- package/dist/engine/dsl/index.d.ts +8 -0
- package/dist/engine/dsl/index.d.ts.map +1 -0
- package/dist/engine/dsl/index.js +7 -0
- package/dist/engine/dsl/index.js.map +1 -0
- package/dist/engine/dsl/live-query.d.ts +45 -0
- package/dist/engine/dsl/live-query.d.ts.map +1 -0
- package/dist/engine/dsl/live-query.js +116 -0
- package/dist/engine/dsl/live-query.js.map +1 -0
- package/dist/engine/dsl/mutations.d.ts +123 -0
- package/dist/engine/dsl/mutations.d.ts.map +1 -0
- package/dist/engine/dsl/mutations.js +241 -0
- package/dist/engine/dsl/mutations.js.map +1 -0
- package/dist/engine/dsl/select-query-builder.d.ts +148 -0
- package/dist/engine/dsl/select-query-builder.d.ts.map +1 -0
- package/dist/engine/dsl/select-query-builder.js +377 -0
- package/dist/engine/dsl/select-query-builder.js.map +1 -0
- package/dist/engine/dsl/sql-tag.d.ts +46 -0
- package/dist/engine/dsl/sql-tag.d.ts.map +1 -0
- package/dist/engine/dsl/sql-tag.js +85 -0
- package/dist/engine/dsl/sql-tag.js.map +1 -0
- package/dist/engine/dsl/types.d.ts +133 -0
- package/dist/engine/dsl/types.d.ts.map +1 -0
- package/dist/engine/dsl/types.js +9 -0
- package/dist/engine/dsl/types.js.map +1 -0
- package/dist/engine/errors.d.ts +55 -0
- package/dist/engine/errors.d.ts.map +1 -0
- package/dist/engine/errors.js +81 -0
- package/dist/engine/errors.js.map +1 -0
- package/dist/engine/fts.d.ts +93 -0
- package/dist/engine/fts.d.ts.map +1 -0
- package/dist/engine/fts.js +273 -0
- package/dist/engine/fts.js.map +1 -0
- package/dist/engine/group-index.d.ts +32 -0
- package/dist/engine/group-index.d.ts.map +1 -0
- package/dist/engine/group-index.js +366 -0
- package/dist/engine/group-index.js.map +1 -0
- package/dist/engine/index.d.ts +13 -0
- package/dist/engine/index.d.ts.map +1 -0
- package/dist/engine/index.js +12 -0
- package/dist/engine/index.js.map +1 -0
- package/dist/engine/join-index.d.ts +12 -0
- package/dist/engine/join-index.d.ts.map +1 -0
- package/dist/engine/join-index.js +217 -0
- package/dist/engine/join-index.js.map +1 -0
- package/dist/engine/live.d.ts +86 -0
- package/dist/engine/live.d.ts.map +1 -0
- package/dist/engine/live.js +289 -0
- package/dist/engine/live.js.map +1 -0
- package/dist/engine/memory.d.ts +37 -0
- package/dist/engine/memory.d.ts.map +1 -0
- package/dist/engine/memory.js +119 -0
- package/dist/engine/memory.js.map +1 -0
- package/dist/engine/optimizer.d.ts +20 -0
- package/dist/engine/optimizer.d.ts.map +1 -0
- package/dist/engine/optimizer.js +1170 -0
- package/dist/engine/optimizer.js.map +1 -0
- package/dist/engine/query-cache.d.ts +10 -0
- package/dist/engine/query-cache.d.ts.map +1 -0
- package/dist/engine/query-cache.js +58 -0
- package/dist/engine/query-cache.js.map +1 -0
- package/dist/engine/query.d.ts +844 -0
- package/dist/engine/query.d.ts.map +1 -0
- package/dist/engine/query.js +7450 -0
- package/dist/engine/query.js.map +1 -0
- package/dist/engine/schema-wire.d.ts +92 -0
- package/dist/engine/schema-wire.d.ts.map +1 -0
- package/dist/engine/schema-wire.js +115 -0
- package/dist/engine/schema-wire.js.map +1 -0
- package/dist/engine/schema.d.ts +394 -0
- package/dist/engine/schema.d.ts.map +1 -0
- package/dist/engine/schema.js +709 -0
- package/dist/engine/schema.js.map +1 -0
- package/dist/engine/sort-keys.d.ts +36 -0
- package/dist/engine/sort-keys.d.ts.map +1 -0
- package/dist/engine/sort-keys.js +60 -0
- package/dist/engine/sort-keys.js.map +1 -0
- package/dist/engine/sql-json.d.ts +30 -0
- package/dist/engine/sql-json.d.ts.map +1 -0
- package/dist/engine/sql-json.js +120 -0
- package/dist/engine/sql-json.js.map +1 -0
- package/dist/engine/sql-semantics.d.ts +31 -0
- package/dist/engine/sql-semantics.d.ts.map +1 -0
- package/dist/engine/sql-semantics.js +121 -0
- package/dist/engine/sql-semantics.js.map +1 -0
- package/dist/engine/vector.d.ts +87 -0
- package/dist/engine/vector.d.ts.map +1 -0
- package/dist/engine/vector.js +4227 -0
- package/dist/engine/vector.js.map +1 -0
- package/dist/engine/worker-host.d.ts +48 -0
- package/dist/engine/worker-host.d.ts.map +1 -0
- package/dist/engine/worker-host.js +501 -0
- package/dist/engine/worker-host.js.map +1 -0
- package/dist/engine/worker.d.ts +2 -0
- package/dist/engine/worker.d.ts.map +1 -0
- package/dist/engine/worker.js +14 -0
- package/dist/engine/worker.js.map +1 -0
- package/dist/index.d.ts +2 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +2 -0
- package/dist/index.js.map +1 -0
- package/dist/plan/index.d.ts +17 -0
- package/dist/plan/index.d.ts.map +1 -0
- package/dist/plan/index.js +17 -0
- package/dist/plan/index.js.map +1 -0
- package/dist/storage/fixture-shape.d.ts +42 -0
- package/dist/storage/fixture-shape.d.ts.map +1 -0
- package/dist/storage/fixture-shape.js +146 -0
- package/dist/storage/fixture-shape.js.map +1 -0
- package/dist/storage/index.d.ts +5 -0
- package/dist/storage/index.d.ts.map +1 -0
- package/dist/storage/index.js +5 -0
- package/dist/storage/index.js.map +1 -0
- package/dist/storage/indexeddb.d.ts +139 -0
- package/dist/storage/indexeddb.d.ts.map +1 -0
- package/dist/storage/indexeddb.js +3161 -0
- package/dist/storage/indexeddb.js.map +1 -0
- package/dist/storage/memory.d.ts +121 -0
- package/dist/storage/memory.d.ts.map +1 -0
- package/dist/storage/memory.js +1455 -0
- package/dist/storage/memory.js.map +1 -0
- package/dist/storage/snapshot.d.ts +108 -0
- package/dist/storage/snapshot.d.ts.map +1 -0
- package/dist/storage/snapshot.js +276 -0
- package/dist/storage/snapshot.js.map +1 -0
- package/dist/storage/types.d.ts +895 -0
- package/dist/storage/types.d.ts.map +1 -0
- package/dist/storage/types.js +1549 -0
- package/dist/storage/types.js.map +1 -0
- package/dist/testing/index.d.ts +76 -0
- package/dist/testing/index.d.ts.map +1 -0
- package/dist/testing/index.js +233 -0
- package/dist/testing/index.js.map +1 -0
- package/dist/testing/seeds.d.ts +11 -0
- package/dist/testing/seeds.d.ts.map +1 -0
- package/dist/testing/seeds.js +50 -0
- package/dist/testing/seeds.js.map +1 -0
- package/dist/transactions/index.d.ts +148 -0
- package/dist/transactions/index.d.ts.map +1 -0
- package/dist/transactions/index.js +662 -0
- package/dist/transactions/index.js.map +1 -0
- package/dist/worker-protocol/index.d.ts +84 -0
- package/dist/worker-protocol/index.d.ts.map +1 -0
- package/dist/worker-protocol/index.js +117 -0
- package/dist/worker-protocol/index.js.map +1 -0
- package/package.json +66 -16
- package/sql-feature-matrix.json +1275 -0
- package/index.d.ts +0 -1
- package/index.js +0 -1
|
@@ -0,0 +1,3161 @@
|
|
|
1
|
+
import { CompactionJobConflictError, createManifest, createGarbageCollectionJobRecord, storeNames, advanceGarbageCollectionJobRecord, collectFtsCandidates, invalidateUncoveredFtsColumns, GarbageCollectionJobConflictError, LeaseConflictError, MANIFEST_CHECKPOINT_INTERVAL, applyManifestRecord, SnapshotManifestMissingError, TableRecordConflictError, TempOwnerConflictError, TransactionRecordConflictError, UniqueKeyConflictError, normalizeCompactionJobRecord, normalizeGarbageCollectionJobRecord, normalizeSegmentRecord, updateCompactionJobRecord, updateTransactionRecord, WriteConflictError, } from "./types.js";
|
|
2
|
+
import { selectLiveRecords, } from "./snapshot.js";
|
|
3
|
+
const SCHEMA_VERSION = 2;
|
|
4
|
+
const CURRENT_MANIFEST_KEY = "manifest/current";
|
|
5
|
+
const CATALOG_EPOCH_KEY = "catalog/epoch";
|
|
6
|
+
const SEGMENT_TABLE_INDEX = "byTable";
|
|
7
|
+
const TABLE_ID_PREFIX = "table/id/";
|
|
8
|
+
const TABLE_NAME_PREFIX = "table/name/";
|
|
9
|
+
const ROW_ID_PREFIX = "row-id/";
|
|
10
|
+
const AUTO_INCREMENT_PREFIX = "auto-increment/";
|
|
11
|
+
const UNIQUE_KEY_CHUNK_INDEX = "unique-key-chunk-index";
|
|
12
|
+
const UNIQUE_KEY_CHUNK = "unique-key-chunk";
|
|
13
|
+
const UNIQUE_KEY_BASE = "unique-key-base";
|
|
14
|
+
/**
|
|
15
|
+
* Tail length at which a commit folds the chunk tail into per-key base records. Appending one
|
|
16
|
+
* chunk per commit keeps commits O(1), but every lookup must deserialize the whole tail; the
|
|
17
|
+
* fold bounds that at this many chunk records plus point `getKey` probes against the base,
|
|
18
|
+
* instead of a scan whose cost grows with every key ever written.
|
|
19
|
+
*/
|
|
20
|
+
const UNIQUE_KEY_TAIL_CHUNK_LIMIT = 16;
|
|
21
|
+
const UNIQUE_KEY_BASE_PART = "unique-key-base-part";
|
|
22
|
+
/**
|
|
23
|
+
* Tokens per chunks-v2 base partition. Small enough that a keyed point mutation deserializes
|
|
24
|
+
* one partition in about a millisecond; large enough that a multi-million-key table folds into
|
|
25
|
+
* hundreds of records instead of millions.
|
|
26
|
+
*/
|
|
27
|
+
const UNIQUE_KEY_PARTITION_TARGET = 16_384;
|
|
28
|
+
const FTS_BASE_INDEX_PREFIX = "fts-base-index/";
|
|
29
|
+
const FTS_BASE_PREFIX = "fts-base/";
|
|
30
|
+
const FTS_CHUNK_PREFIX = "fts-chunk/";
|
|
31
|
+
const COMPACTION_JOB_KEY_PREFIX = "compaction-job/";
|
|
32
|
+
const GARBAGE_COLLECTION_JOB_KEY_PREFIX = "garbage-collection-job/";
|
|
33
|
+
const storageTextEncoder = new TextEncoder();
|
|
34
|
+
const storageTextBuffer = new Uint8Array(1_024);
|
|
35
|
+
/** Blocks per read when copying a database out; keeps one export request list bounded. */
|
|
36
|
+
const SNAPSHOT_BLOCK_BATCH = 512;
|
|
37
|
+
/** Bytes per write transaction when loading a snapshot in. */
|
|
38
|
+
const SNAPSHOT_BATCH_BYTES = 8 * 1024 * 1024;
|
|
39
|
+
export class IndexedDbBlockStore {
|
|
40
|
+
#db;
|
|
41
|
+
#durability;
|
|
42
|
+
/** Remembered name-to-id mappings; see getTableByName for why they need no invalidation. */
|
|
43
|
+
#tableIdsByName = new Map();
|
|
44
|
+
/**
|
|
45
|
+
* Memoized unique-key state for the most recently written keyed table, mirroring
|
|
46
|
+
* #manifestCache's validity rule: usable only while the next commit's expected manifest
|
|
47
|
+
* version matches, so any commit from another instance or tab invalidates it. `present` is
|
|
48
|
+
* the complete membership (folded base plus tail replay) and `chunks` are the unfolded tail
|
|
49
|
+
* in commit order, so a valid cache lets a commit skip every unique-key read and lets a
|
|
50
|
+
* fold rewrite the base without re-reading it. Bulk loads hit this on every batch.
|
|
51
|
+
*/
|
|
52
|
+
#uniqueKeyCache;
|
|
53
|
+
/**
|
|
54
|
+
* The most recently resolved manifest block set, advanced in place as this instance commits.
|
|
55
|
+
* Purely a memoization of immutable records: a miss (fresh instance, or another tab moved the
|
|
56
|
+
* version) re-resolves from storage inside the committing transaction.
|
|
57
|
+
*/
|
|
58
|
+
#manifestCache;
|
|
59
|
+
constructor(db, durability) {
|
|
60
|
+
this.#db = db;
|
|
61
|
+
this.#durability = durability;
|
|
62
|
+
}
|
|
63
|
+
static async open(options) {
|
|
64
|
+
const factory = options.indexedDB ?? getGlobalIndexedDb();
|
|
65
|
+
if (factory === undefined)
|
|
66
|
+
throw new Error("IndexedDB is unavailable");
|
|
67
|
+
const request = factory.open(options.name, SCHEMA_VERSION);
|
|
68
|
+
request.addEventListener("upgradeneeded", () => {
|
|
69
|
+
for (const storeName of storeNames) {
|
|
70
|
+
if (!request.result.objectStoreNames.contains(storeName)) {
|
|
71
|
+
request.result.createObjectStore(storeName);
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
// Version 2: segments become range-readable by table. createIndex back-fills from
|
|
75
|
+
// existing records, so version-1 databases upgrade in place with no data rewrite.
|
|
76
|
+
const upgrade = request.transaction;
|
|
77
|
+
if (upgrade !== null) {
|
|
78
|
+
const segments = upgrade.objectStore("segments");
|
|
79
|
+
if (!segments.indexNames.contains(SEGMENT_TABLE_INDEX)) {
|
|
80
|
+
segments.createIndex(SEGMENT_TABLE_INDEX, "tableId");
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
});
|
|
84
|
+
const db = await requestResult(request);
|
|
85
|
+
return new IndexedDbBlockStore(db, options.durability ?? "relaxed");
|
|
86
|
+
}
|
|
87
|
+
async addBlock(id, bytes) {
|
|
88
|
+
return this.addBlocks([{ id, bytes }]);
|
|
89
|
+
}
|
|
90
|
+
async addBlocks(blocks) {
|
|
91
|
+
const ids = new Set();
|
|
92
|
+
for (const block of blocks) {
|
|
93
|
+
if (block.id.length === 0)
|
|
94
|
+
throw new TypeError("Block ID cannot be empty");
|
|
95
|
+
if (ids.has(block.id))
|
|
96
|
+
throw new Error(`Block already exists: ${block.id}`);
|
|
97
|
+
ids.add(block.id);
|
|
98
|
+
}
|
|
99
|
+
const transaction = this.#transaction("blocks", "readwrite");
|
|
100
|
+
const store = transaction.objectStore("blocks");
|
|
101
|
+
for (const block of blocks) {
|
|
102
|
+
// Structured clone serializes a view's entire underlying buffer, so only a partial view
|
|
103
|
+
// needs compacting first; a whole-buffer view clones exactly once inside add().
|
|
104
|
+
const bytes = block.bytes.byteOffset === 0 && block.bytes.byteLength === block.bytes.buffer.byteLength
|
|
105
|
+
? block.bytes
|
|
106
|
+
: block.bytes.slice();
|
|
107
|
+
store.add(bytes, block.id);
|
|
108
|
+
}
|
|
109
|
+
await transactionDone(transaction);
|
|
110
|
+
}
|
|
111
|
+
async getBlock(id) {
|
|
112
|
+
const transaction = this.#transaction("blocks", "readonly");
|
|
113
|
+
const value = await requestResult(transaction.objectStore("blocks").get(id));
|
|
114
|
+
await transactionDone(transaction);
|
|
115
|
+
return value === undefined ? undefined : asBytes(value);
|
|
116
|
+
}
|
|
117
|
+
async getBlocks(ids) {
|
|
118
|
+
const transaction = this.#transaction("blocks", "readonly");
|
|
119
|
+
const store = transaction.objectStore("blocks");
|
|
120
|
+
const values = await Promise.all(ids.map((id) => requestResult(store.get(id))));
|
|
121
|
+
await transactionDone(transaction);
|
|
122
|
+
return values.map((value) => (value === undefined ? undefined : asBytes(value)));
|
|
123
|
+
}
|
|
124
|
+
async removeBlock(id) {
|
|
125
|
+
const transaction = this.#transaction("blocks", "readwrite");
|
|
126
|
+
transaction.objectStore("blocks").delete(id);
|
|
127
|
+
await transactionDone(transaction);
|
|
128
|
+
}
|
|
129
|
+
async listBlockIds() {
|
|
130
|
+
const transaction = this.#transaction("blocks", "readonly");
|
|
131
|
+
const keys = await requestResult(transaction.objectStore("blocks").getAllKeys());
|
|
132
|
+
await transactionDone(transaction);
|
|
133
|
+
return keys.map(String).sort();
|
|
134
|
+
}
|
|
135
|
+
async putTempRunPage(page) {
|
|
136
|
+
validateTempRunPage(page);
|
|
137
|
+
const transaction = this.#transaction("temp", "readwrite");
|
|
138
|
+
transaction.objectStore("temp").put(page.bytes.slice(), tempRunPageKey(page));
|
|
139
|
+
await transactionDone(transaction);
|
|
140
|
+
}
|
|
141
|
+
async getTempRunPage(ownerId, runId, pageIndex) {
|
|
142
|
+
validateTempRunPageIdentity(ownerId, runId, pageIndex);
|
|
143
|
+
const transaction = this.#transaction("temp", "readonly");
|
|
144
|
+
const value = await requestResult(transaction.objectStore("temp").get(["run", ownerId, runId, pageIndex]));
|
|
145
|
+
await transactionDone(transaction);
|
|
146
|
+
return value === undefined ? undefined : asBytes(value).slice();
|
|
147
|
+
}
|
|
148
|
+
async removeTempRun(ownerId, runId) {
|
|
149
|
+
validateTempRunPageIdentity(ownerId, runId, 0);
|
|
150
|
+
const transaction = this.#transaction("temp", "readwrite");
|
|
151
|
+
const store = transaction.objectStore("temp");
|
|
152
|
+
await visitObjectStoreSequentially(store, (_value, key) => {
|
|
153
|
+
if (isTempRunPageKey(key, ownerId, runId))
|
|
154
|
+
store.delete(key);
|
|
155
|
+
});
|
|
156
|
+
await transactionDone(transaction);
|
|
157
|
+
}
|
|
158
|
+
async removeTempOwner(ownerId) {
|
|
159
|
+
validateTempId(ownerId, "Temp run owner ID");
|
|
160
|
+
const transaction = this.#transaction("temp", "readwrite");
|
|
161
|
+
const store = transaction.objectStore("temp");
|
|
162
|
+
await visitObjectStoreSequentially(store, (_value, key) => {
|
|
163
|
+
if (isTempRunPageKey(key, ownerId))
|
|
164
|
+
store.delete(key);
|
|
165
|
+
});
|
|
166
|
+
store.delete(tempOwnerKey(ownerId));
|
|
167
|
+
await transactionDone(transaction);
|
|
168
|
+
}
|
|
169
|
+
async createTempOwner(record) {
|
|
170
|
+
validateTempOwnerRecord(record);
|
|
171
|
+
const transaction = this.#transaction("temp", "readwrite");
|
|
172
|
+
const store = transaction.objectStore("temp");
|
|
173
|
+
const existing = await requestResult(store.getKey(tempOwnerKey(record.ownerId)));
|
|
174
|
+
if (existing !== undefined) {
|
|
175
|
+
transaction.abort();
|
|
176
|
+
await ignoreAbort(transaction);
|
|
177
|
+
throw new Error(`Temp owner already exists: ${record.ownerId}`);
|
|
178
|
+
}
|
|
179
|
+
store.put(structuredClone(record), tempOwnerKey(record.ownerId));
|
|
180
|
+
await transactionDone(transaction);
|
|
181
|
+
}
|
|
182
|
+
async getTempOwner(ownerId) {
|
|
183
|
+
validateTempId(ownerId, "Temp run owner ID");
|
|
184
|
+
const transaction = this.#transaction("temp", "readonly");
|
|
185
|
+
const value = await requestResult(transaction.objectStore("temp").get(tempOwnerKey(ownerId)));
|
|
186
|
+
await transactionDone(transaction);
|
|
187
|
+
return value === undefined ? undefined : asTempOwnerRecord(value);
|
|
188
|
+
}
|
|
189
|
+
async renewTempOwner(ownerId, expectedRevision, expiresAt) {
|
|
190
|
+
validateTempId(ownerId, "Temp run owner ID");
|
|
191
|
+
validateLeaseExpiration(expiresAt);
|
|
192
|
+
const transaction = this.#transaction("temp", "readwrite");
|
|
193
|
+
const store = transaction.objectStore("temp");
|
|
194
|
+
const value = await requestResult(store.get(tempOwnerKey(ownerId)));
|
|
195
|
+
const record = value === undefined ? undefined : asTempOwnerRecord(value);
|
|
196
|
+
if (record?.revision !== expectedRevision) {
|
|
197
|
+
transaction.abort();
|
|
198
|
+
await ignoreAbort(transaction);
|
|
199
|
+
throw new TempOwnerConflictError(ownerId, expectedRevision, record?.revision ?? null);
|
|
200
|
+
}
|
|
201
|
+
const renewed = { ...record, expiresAt, revision: record.revision + 1 };
|
|
202
|
+
store.put(renewed, tempOwnerKey(ownerId));
|
|
203
|
+
await transactionDone(transaction);
|
|
204
|
+
return structuredClone(renewed);
|
|
205
|
+
}
|
|
206
|
+
async removeTempOwnerIfExpired(ownerId, expiresAtCutoff) {
|
|
207
|
+
validateTempId(ownerId, "Temp run owner ID");
|
|
208
|
+
const cutoff = Date.parse(expiresAtCutoff);
|
|
209
|
+
if (!Number.isFinite(cutoff))
|
|
210
|
+
throw new TypeError("Temp owner expiry cutoff must be valid");
|
|
211
|
+
const transaction = this.#transaction("temp", "readwrite");
|
|
212
|
+
const store = transaction.objectStore("temp");
|
|
213
|
+
const value = await requestResult(store.get(tempOwnerKey(ownerId)));
|
|
214
|
+
if (value !== undefined) {
|
|
215
|
+
const record = asTempOwnerRecord(value);
|
|
216
|
+
const expiresAt = Date.parse(record.expiresAt);
|
|
217
|
+
if (Number.isFinite(expiresAt) && expiresAt > cutoff) {
|
|
218
|
+
await transactionDone(transaction);
|
|
219
|
+
return false;
|
|
220
|
+
}
|
|
221
|
+
}
|
|
222
|
+
await visitObjectStoreSequentially(store, (_value, key) => {
|
|
223
|
+
if (isTempRunPageKey(key, ownerId))
|
|
224
|
+
store.delete(key);
|
|
225
|
+
});
|
|
226
|
+
store.delete(tempOwnerKey(ownerId));
|
|
227
|
+
await transactionDone(transaction);
|
|
228
|
+
return true;
|
|
229
|
+
}
|
|
230
|
+
async listTempOwnerIdsPage(afterOwnerId, limit) {
|
|
231
|
+
validatePageLimit(limit);
|
|
232
|
+
const transaction = this.#transaction("temp", "readonly");
|
|
233
|
+
const store = transaction.objectStore("temp");
|
|
234
|
+
const [recordOwnerIds, pageOwnerIds] = await Promise.all([
|
|
235
|
+
readTempOwnerRecordIds(store, afterOwnerId, limit + 1),
|
|
236
|
+
readTempPageOwnerIds(store, afterOwnerId, limit + 1),
|
|
237
|
+
]);
|
|
238
|
+
await transactionDone(transaction);
|
|
239
|
+
const union = [...new Set([...recordOwnerIds, ...pageOwnerIds])].sort();
|
|
240
|
+
const records = union.slice(0, limit);
|
|
241
|
+
return {
|
|
242
|
+
records,
|
|
243
|
+
nextCursor: union.length > limit ? (records.at(-1) ?? null) : null,
|
|
244
|
+
};
|
|
245
|
+
}
|
|
246
|
+
async addTable(record) {
|
|
247
|
+
const transaction = this.#transaction("catalog", "readwrite");
|
|
248
|
+
const store = transaction.objectStore("catalog");
|
|
249
|
+
const idKey = `${TABLE_ID_PREFIX}${record.id}`;
|
|
250
|
+
const nameKey = `${TABLE_NAME_PREFIX}${record.name}`;
|
|
251
|
+
const [existingId, existingName] = await Promise.all([
|
|
252
|
+
requestResult(store.getKey(idKey)),
|
|
253
|
+
requestResult(store.getKey(nameKey)),
|
|
254
|
+
]);
|
|
255
|
+
if (existingId !== undefined || existingName !== undefined) {
|
|
256
|
+
transaction.abort();
|
|
257
|
+
await ignoreAbort(transaction);
|
|
258
|
+
throw new Error(existingId !== undefined
|
|
259
|
+
? `Table already exists: ${record.id}`
|
|
260
|
+
: `Table name already exists: ${record.name}`);
|
|
261
|
+
}
|
|
262
|
+
store.add(structuredClone(record), idKey);
|
|
263
|
+
store.add(record.id, nameKey);
|
|
264
|
+
await bumpCatalogEpoch(store);
|
|
265
|
+
await transactionDone(transaction);
|
|
266
|
+
}
|
|
267
|
+
async getTable(id) {
|
|
268
|
+
const transaction = this.#transaction("catalog", "readonly");
|
|
269
|
+
const value = await requestResult(transaction.objectStore("catalog").get(`${TABLE_ID_PREFIX}${id}`));
|
|
270
|
+
await transactionDone(transaction);
|
|
271
|
+
return value === undefined ? undefined : asTableRecord(value);
|
|
272
|
+
}
|
|
273
|
+
async updateTable(id, expectedRevision, update) {
|
|
274
|
+
if (update.columns !== undefined)
|
|
275
|
+
validateTableColumns(update.columns);
|
|
276
|
+
const transaction = this.#transaction("catalog", "readwrite");
|
|
277
|
+
const store = transaction.objectStore("catalog");
|
|
278
|
+
const idKey = `${TABLE_ID_PREFIX}${id}`;
|
|
279
|
+
const value = await requestResult(store.get(idKey));
|
|
280
|
+
const record = value === undefined ? undefined : structuredClone(value);
|
|
281
|
+
const actualRevision = record === undefined ? null : (record.revision ?? 0);
|
|
282
|
+
if (record === undefined || actualRevision !== expectedRevision) {
|
|
283
|
+
transaction.abort();
|
|
284
|
+
await ignoreAbort(transaction);
|
|
285
|
+
throw new TableRecordConflictError(id, expectedRevision, actualRevision);
|
|
286
|
+
}
|
|
287
|
+
const { ftsColumns: previousFts, triggers: previousTriggers, ...base } = record;
|
|
288
|
+
const nextFts = update.ftsColumns === undefined ? previousFts : update.ftsColumns;
|
|
289
|
+
const nextTriggers = update.triggers === undefined ? previousTriggers : update.triggers;
|
|
290
|
+
const updated = {
|
|
291
|
+
...base,
|
|
292
|
+
columns: update.columns === undefined ? record.columns : structuredClone(update.columns),
|
|
293
|
+
...(nextFts === null || nextFts === undefined
|
|
294
|
+
? {}
|
|
295
|
+
: { ftsColumns: structuredClone(nextFts) }),
|
|
296
|
+
...(nextTriggers === null || nextTriggers === undefined
|
|
297
|
+
? {}
|
|
298
|
+
: { triggers: structuredClone(nextTriggers) }),
|
|
299
|
+
revision: expectedRevision + 1,
|
|
300
|
+
};
|
|
301
|
+
store.put(structuredClone(updated), idKey);
|
|
302
|
+
await bumpCatalogEpoch(store);
|
|
303
|
+
await transactionDone(transaction);
|
|
304
|
+
return updated;
|
|
305
|
+
}
|
|
306
|
+
async removeTable(id, expectedRevision) {
|
|
307
|
+
const transaction = this.#transaction(["catalog", "segments"], "readwrite");
|
|
308
|
+
const catalog = transaction.objectStore("catalog");
|
|
309
|
+
const value = await requestResult(catalog.get(`${TABLE_ID_PREFIX}${id}`));
|
|
310
|
+
const record = value === undefined ? undefined : structuredClone(value);
|
|
311
|
+
const actualRevision = record === undefined ? null : (record.revision ?? 0);
|
|
312
|
+
if (record === undefined || actualRevision !== expectedRevision) {
|
|
313
|
+
transaction.abort();
|
|
314
|
+
await ignoreAbort(transaction);
|
|
315
|
+
throw new TableRecordConflictError(id, expectedRevision, actualRevision);
|
|
316
|
+
}
|
|
317
|
+
catalog.delete(`${TABLE_ID_PREFIX}${id}`);
|
|
318
|
+
catalog.delete(`${TABLE_NAME_PREFIX}${record.name}`);
|
|
319
|
+
catalog.delete(`${ROW_ID_PREFIX}${id}`);
|
|
320
|
+
// Everything else this table owns is keyed by its id under one of a handful of prefixes,
|
|
321
|
+
// string-keyed or array-keyed. One sequential pass collects them all: a dropped table is
|
|
322
|
+
// rare and the alternative is a range read per prefix, several of which the injected test
|
|
323
|
+
// factory cannot express.
|
|
324
|
+
const ownedStringPrefixes = [
|
|
325
|
+
`${AUTO_INCREMENT_PREFIX}${id}/`,
|
|
326
|
+
`${FTS_BASE_INDEX_PREFIX}${id}/`,
|
|
327
|
+
`${FTS_BASE_PREFIX}${id}/`,
|
|
328
|
+
`${FTS_CHUNK_PREFIX}${id}/`,
|
|
329
|
+
];
|
|
330
|
+
const ownedArrayKinds = new Set([
|
|
331
|
+
"unique-key",
|
|
332
|
+
UNIQUE_KEY_CHUNK_INDEX,
|
|
333
|
+
UNIQUE_KEY_CHUNK,
|
|
334
|
+
UNIQUE_KEY_BASE,
|
|
335
|
+
UNIQUE_KEY_BASE_PART,
|
|
336
|
+
]);
|
|
337
|
+
const doomed = [];
|
|
338
|
+
await visitObjectStoreSequentially(catalog, (_value, key) => {
|
|
339
|
+
if (typeof key === "string") {
|
|
340
|
+
if (ownedStringPrefixes.some((prefix) => key.startsWith(prefix)))
|
|
341
|
+
doomed.push(key);
|
|
342
|
+
return;
|
|
343
|
+
}
|
|
344
|
+
if (!Array.isArray(key))
|
|
345
|
+
return;
|
|
346
|
+
const [kind, owner] = key;
|
|
347
|
+
if (typeof kind === "string" && ownedArrayKinds.has(kind) && owner === id)
|
|
348
|
+
doomed.push(key);
|
|
349
|
+
});
|
|
350
|
+
for (const key of doomed)
|
|
351
|
+
catalog.delete(key);
|
|
352
|
+
const segments = transaction.objectStore("segments");
|
|
353
|
+
const segmentKeys = await requestResult(segments.index(SEGMENT_TABLE_INDEX).getAllKeys(id));
|
|
354
|
+
for (const key of segmentKeys)
|
|
355
|
+
segments.delete(key);
|
|
356
|
+
await bumpCatalogEpoch(catalog);
|
|
357
|
+
await transactionDone(transaction);
|
|
358
|
+
}
|
|
359
|
+
async writeFtsBase(tableId, columnId, input) {
|
|
360
|
+
const transaction = this.#transaction("catalog", "readwrite");
|
|
361
|
+
const store = transaction.objectStore("catalog");
|
|
362
|
+
const tocKey = `${FTS_BASE_INDEX_PREFIX}${tableId}/${columnId}`;
|
|
363
|
+
const previous = (await requestResult(store.get(tocKey)));
|
|
364
|
+
const chunkPrefix = `${FTS_BASE_PREFIX}${tableId}/${columnId}/`;
|
|
365
|
+
const previousCount = previous?.boundaries.length ?? 0;
|
|
366
|
+
for (let ordinal = input.chunks.length; ordinal < previousCount; ordinal += 1) {
|
|
367
|
+
store.delete(`${chunkPrefix}${String(ordinal).padStart(6, "0")}`);
|
|
368
|
+
}
|
|
369
|
+
const boundaries = [];
|
|
370
|
+
input.chunks.forEach((chunk, ordinal) => {
|
|
371
|
+
boundaries.push({
|
|
372
|
+
first: chunk[0]?.term ?? "",
|
|
373
|
+
last: chunk[chunk.length - 1]?.term ?? "",
|
|
374
|
+
});
|
|
375
|
+
store.put(structuredClone(chunk), `${chunkPrefix}${String(ordinal).padStart(6, "0")}`);
|
|
376
|
+
});
|
|
377
|
+
store.put({ coversVersion: input.coversVersion, boundaries, totalTokens: input.totalTokens }, tocKey);
|
|
378
|
+
// Commit deltas the base now covers are dead; drop their chunks and shrink the version list
|
|
379
|
+
// (mirroring the unique-key chunk index — no key-range scans in this environment).
|
|
380
|
+
const deltaIndexKey = ftsChunkIndexKey(tableId, columnId);
|
|
381
|
+
const deltaIndex = (await requestResult(store.get(deltaIndexKey)));
|
|
382
|
+
const surviving = [];
|
|
383
|
+
for (const version of deltaIndex?.versions ?? []) {
|
|
384
|
+
if (version <= input.coversVersion) {
|
|
385
|
+
store.delete(ftsChunkKey(tableId, columnId, version));
|
|
386
|
+
}
|
|
387
|
+
else {
|
|
388
|
+
surviving.push(version);
|
|
389
|
+
}
|
|
390
|
+
}
|
|
391
|
+
store.put({ versions: surviving }, deltaIndexKey);
|
|
392
|
+
await transactionDone(transaction);
|
|
393
|
+
}
|
|
394
|
+
async readFtsCandidates(tableId, columnId, terms, upToVersion) {
|
|
395
|
+
const transaction = this.#transaction("catalog", "readonly");
|
|
396
|
+
const store = transaction.objectStore("catalog");
|
|
397
|
+
const [toc, deltaIndex] = (await Promise.all([
|
|
398
|
+
requestResult(store.get(`${FTS_BASE_INDEX_PREFIX}${tableId}/${columnId}`)),
|
|
399
|
+
requestResult(store.get(ftsChunkIndexKey(tableId, columnId))),
|
|
400
|
+
]));
|
|
401
|
+
const chunkPrefix = `${FTS_BASE_PREFIX}${tableId}/${columnId}/`;
|
|
402
|
+
const wantedOrdinals = (toc?.boundaries ?? []).flatMap((boundary, ordinal) => terms.some((term) => {
|
|
403
|
+
const upper = term.prefix ? `${term.term}` : term.term;
|
|
404
|
+
return term.term <= boundary.last && upper >= boundary.first;
|
|
405
|
+
})
|
|
406
|
+
? [ordinal]
|
|
407
|
+
: []);
|
|
408
|
+
const coversVersion = toc?.coversVersion ?? -1;
|
|
409
|
+
const wantedVersions = (deltaIndex?.versions ?? []).filter((version) => version > coversVersion && version <= upToVersion);
|
|
410
|
+
// One readonly transaction pipelines all chunk reads concurrently instead of paying an
|
|
411
|
+
// event-loop round trip per chunk.
|
|
412
|
+
const [baseChunks, deltaChunks] = await Promise.all([
|
|
413
|
+
Promise.all(wantedOrdinals.map((ordinal) => requestResult(store.get(`${chunkPrefix}${String(ordinal).padStart(6, "0")}`)))),
|
|
414
|
+
Promise.all(wantedVersions.map((version) => requestResult(store.get(ftsChunkKey(tableId, columnId, version))))),
|
|
415
|
+
]);
|
|
416
|
+
await transactionDone(transaction);
|
|
417
|
+
const present = deltaChunks.filter((chunk) => chunk !== undefined);
|
|
418
|
+
const chunkLists = [
|
|
419
|
+
...baseChunks.filter((chunk) => chunk !== undefined),
|
|
420
|
+
...present.map((chunk) => chunk.postings),
|
|
421
|
+
];
|
|
422
|
+
const totalTokens = (toc?.totalTokens ?? 0) + present.reduce((total, chunk) => total + chunk.totalTokens, 0);
|
|
423
|
+
return {
|
|
424
|
+
...collectFtsCandidates(chunkLists, terms),
|
|
425
|
+
deltaChunkCount: present.length,
|
|
426
|
+
totalTokens,
|
|
427
|
+
coversVersion,
|
|
428
|
+
};
|
|
429
|
+
}
|
|
430
|
+
/**
|
|
431
|
+
* Resolving a name means two reads — name to id, then id to record — and the second cannot
|
|
432
|
+
* start until the first lands, so the write path pays two serialized round trips for one
|
|
433
|
+
* lookup. A remembered id collapses that to one read, and needs no invalidation: the record
|
|
434
|
+
* carries its own name, so a mapping that has gone stale fails the check below and falls
|
|
435
|
+
* back to the full resolution. Table ids are unique per creation, so a dropped and recreated
|
|
436
|
+
* table cannot be mistaken for its predecessor.
|
|
437
|
+
*/
|
|
438
|
+
async getTableByName(name) {
|
|
439
|
+
const transaction = this.#transaction("catalog", "readonly");
|
|
440
|
+
const store = transaction.objectStore("catalog");
|
|
441
|
+
const rememberedId = this.#tableIdsByName.get(name);
|
|
442
|
+
if (rememberedId !== undefined) {
|
|
443
|
+
const cached = await requestResult(store.get(`${TABLE_ID_PREFIX}${rememberedId}`));
|
|
444
|
+
if (cached !== undefined) {
|
|
445
|
+
const record = asTableRecord(cached);
|
|
446
|
+
if (record.name === name) {
|
|
447
|
+
await transactionDone(transaction);
|
|
448
|
+
return record;
|
|
449
|
+
}
|
|
450
|
+
}
|
|
451
|
+
this.#tableIdsByName.delete(name);
|
|
452
|
+
}
|
|
453
|
+
const id = (await requestResult(store.get(`${TABLE_NAME_PREFIX}${name}`)));
|
|
454
|
+
const value = id === undefined ? undefined : await requestResult(store.get(`${TABLE_ID_PREFIX}${id}`));
|
|
455
|
+
await transactionDone(transaction);
|
|
456
|
+
if (value === undefined)
|
|
457
|
+
return undefined;
|
|
458
|
+
const record = asTableRecord(value);
|
|
459
|
+
if (record.name === name)
|
|
460
|
+
this.#tableIdsByName.set(name, record.id);
|
|
461
|
+
return record;
|
|
462
|
+
}
|
|
463
|
+
async listTables() {
|
|
464
|
+
const transaction = this.#transaction("catalog", "readonly");
|
|
465
|
+
const store = transaction.objectStore("catalog");
|
|
466
|
+
const values = await requestResult(store.getAll());
|
|
467
|
+
await transactionDone(transaction);
|
|
468
|
+
return values
|
|
469
|
+
.filter(isTableRecord)
|
|
470
|
+
.map(asTableRecord)
|
|
471
|
+
.sort((left, right) => left.name.localeCompare(right.name));
|
|
472
|
+
}
|
|
473
|
+
async addSegment(record) {
|
|
474
|
+
const normalized = normalizeSegmentRecord(record);
|
|
475
|
+
const transaction = this.#transaction("segments", "readwrite");
|
|
476
|
+
transaction.objectStore("segments").add(normalized, normalized.id);
|
|
477
|
+
await transactionDone(transaction);
|
|
478
|
+
}
|
|
479
|
+
async getSegment(id) {
|
|
480
|
+
const transaction = this.#transaction("segments", "readonly");
|
|
481
|
+
const value = await requestResult(transaction.objectStore("segments").get(id));
|
|
482
|
+
await transactionDone(transaction);
|
|
483
|
+
return value === undefined ? undefined : asSegmentRecord(value);
|
|
484
|
+
}
|
|
485
|
+
async listSegments(tableId) {
|
|
486
|
+
const transaction = this.#transaction("segments", "readonly");
|
|
487
|
+
if (tableId !== undefined) {
|
|
488
|
+
// Index range read: cost scales with this table's segments, not the store's total.
|
|
489
|
+
const values = await requestResult(transaction.objectStore("segments").index(SEGMENT_TABLE_INDEX).getAll(tableId));
|
|
490
|
+
await transactionDone(transaction);
|
|
491
|
+
return values
|
|
492
|
+
.map((value) => asSegmentRecord(value))
|
|
493
|
+
.sort((left, right) => left.id.localeCompare(right.id));
|
|
494
|
+
}
|
|
495
|
+
const records = [];
|
|
496
|
+
await visitObjectStoreSequentially(transaction.objectStore("segments"), (value) => {
|
|
497
|
+
records.push(asSegmentRecord(value));
|
|
498
|
+
});
|
|
499
|
+
await transactionDone(transaction);
|
|
500
|
+
return records.sort((left, right) => left.id.localeCompare(right.id));
|
|
501
|
+
}
|
|
502
|
+
async removeSegment(id) {
|
|
503
|
+
const transaction = this.#transaction("segments", "readwrite");
|
|
504
|
+
transaction.objectStore("segments").delete(id);
|
|
505
|
+
await transactionDone(transaction);
|
|
506
|
+
}
|
|
507
|
+
async reserveRowIds(tableId, count) {
|
|
508
|
+
validateCount(count);
|
|
509
|
+
const transaction = this.#transaction("catalog", "readwrite");
|
|
510
|
+
const store = transaction.objectStore("catalog");
|
|
511
|
+
const key = `${ROW_ID_PREFIX}${tableId}`;
|
|
512
|
+
const current = (await requestResult(store.get(key)));
|
|
513
|
+
const start = current ?? 1n;
|
|
514
|
+
const endExclusive = start + BigInt(count);
|
|
515
|
+
store.put(endExclusive, key);
|
|
516
|
+
await transactionDone(transaction);
|
|
517
|
+
return { start, endExclusive };
|
|
518
|
+
}
|
|
519
|
+
async reserveAutoIncrement(tableId, columnId, count, atLeast) {
|
|
520
|
+
validateAutoIncrementReservation(count, atLeast);
|
|
521
|
+
const transaction = this.#transaction("catalog", "readwrite");
|
|
522
|
+
const store = transaction.objectStore("catalog");
|
|
523
|
+
const key = `${AUTO_INCREMENT_PREFIX}${tableId}/${columnId}`;
|
|
524
|
+
const current = (await requestResult(store.get(key)));
|
|
525
|
+
const start = maxBigInt(current ?? 1n, atLeast ?? 1n);
|
|
526
|
+
const endExclusive = start + BigInt(count);
|
|
527
|
+
store.put(endExclusive, key);
|
|
528
|
+
await transactionDone(transaction);
|
|
529
|
+
return { start, endExclusive };
|
|
530
|
+
}
|
|
531
|
+
async getExistingUniqueKeys(tableId, keyTokens) {
|
|
532
|
+
const tokens = [...new Set(keyTokens)];
|
|
533
|
+
const transaction = this.#transaction("catalog", "readonly");
|
|
534
|
+
const store = transaction.objectStore("catalog");
|
|
535
|
+
const [tableValue, versionValue] = await Promise.all([
|
|
536
|
+
requestResult(store.get(`${TABLE_ID_PREFIX}${tableId}`)),
|
|
537
|
+
requestResult(store.get(CURRENT_MANIFEST_KEY)),
|
|
538
|
+
]);
|
|
539
|
+
const table = tableValue === undefined ? undefined : asTableRecord(tableValue);
|
|
540
|
+
const mode = table?.uniqueKeyStorage;
|
|
541
|
+
if (mode === "chunks-v1" || mode === "chunks-v2") {
|
|
542
|
+
const cache = this.#uniqueKeyCache;
|
|
543
|
+
if (cache?.tableId === tableId && cache.mode === mode && cache.version === versionValue) {
|
|
544
|
+
await transactionDone(transaction);
|
|
545
|
+
return tokens.filter((token) => cache.present.has(token)).sort();
|
|
546
|
+
}
|
|
547
|
+
const { existing } = await readChunkedUniqueKeys(store, tableId, tokens, mode);
|
|
548
|
+
await transactionDone(transaction);
|
|
549
|
+
return [...existing].sort();
|
|
550
|
+
}
|
|
551
|
+
const keys = await Promise.all(tokens.map((token) => requestResult(store.getKey(uniqueKeyKey(tableId, token)))));
|
|
552
|
+
await transactionDone(transaction);
|
|
553
|
+
return tokens.filter((_, index) => keys[index] !== undefined).sort();
|
|
554
|
+
}
|
|
555
|
+
async getCurrentManifestVersion() {
|
|
556
|
+
const transaction = this.#transaction("catalog", "readonly");
|
|
557
|
+
const value = await requestResult(transaction.objectStore("catalog").get(CURRENT_MANIFEST_KEY));
|
|
558
|
+
await transactionDone(transaction);
|
|
559
|
+
return typeof value === "number" ? value : null;
|
|
560
|
+
}
|
|
561
|
+
/**
|
|
562
|
+
* The freshness probe runs once per query, so its latency is a floor under every read the
|
|
563
|
+
* database serves. Both values are final the moment their reads land: a read-only
|
|
564
|
+
* transaction has nothing left to commit, and waiting for its `complete` event only costs
|
|
565
|
+
* another turn of the event loop — measured at roughly 40% of the probe in Chromium — while
|
|
566
|
+
* changing nothing about what was read. The reads still race the transaction's failure
|
|
567
|
+
* events, so an abort surfaces as a rejection here instead of hanging.
|
|
568
|
+
*/
|
|
569
|
+
async getCatalogProbe() {
|
|
570
|
+
const transaction = this.#transaction("catalog", "readonly");
|
|
571
|
+
const catalog = transaction.objectStore("catalog");
|
|
572
|
+
const [versionValue, epochValue] = await Promise.race([
|
|
573
|
+
Promise.all([
|
|
574
|
+
requestResult(catalog.get(CURRENT_MANIFEST_KEY)),
|
|
575
|
+
requestResult(catalog.get(CATALOG_EPOCH_KEY)),
|
|
576
|
+
]),
|
|
577
|
+
transactionFailure(transaction),
|
|
578
|
+
]);
|
|
579
|
+
return {
|
|
580
|
+
manifestVersion: typeof versionValue === "number" ? versionValue : null,
|
|
581
|
+
catalogEpoch: typeof epochValue === "number" ? epochValue : 0,
|
|
582
|
+
};
|
|
583
|
+
}
|
|
584
|
+
async getQueryCatalogState(tableNames) {
|
|
585
|
+
const transaction = this.#transaction(["catalog", "segments", "transactions"], "readonly");
|
|
586
|
+
const catalog = transaction.objectStore("catalog");
|
|
587
|
+
const [versionValue, epochValue, tableIds] = await Promise.all([
|
|
588
|
+
requestResult(catalog.get(CURRENT_MANIFEST_KEY)),
|
|
589
|
+
requestResult(catalog.get(CATALOG_EPOCH_KEY)),
|
|
590
|
+
Promise.all(tableNames.map((name) => requestResult(catalog.get(`${TABLE_NAME_PREFIX}${name}`)))),
|
|
591
|
+
]);
|
|
592
|
+
const tables = await Promise.all(tableIds.map(async (id) => {
|
|
593
|
+
if (typeof id !== "string")
|
|
594
|
+
return undefined;
|
|
595
|
+
const value = await requestResult(catalog.get(`${TABLE_ID_PREFIX}${id}`));
|
|
596
|
+
return value === undefined ? undefined : asTableRecord(value);
|
|
597
|
+
}));
|
|
598
|
+
const foundTableIds = [
|
|
599
|
+
...new Set(tables
|
|
600
|
+
.filter((table) => table !== undefined)
|
|
601
|
+
.map((table) => table.id)),
|
|
602
|
+
];
|
|
603
|
+
// Range-read each found table's segments through the byTable index: cost scales with the
|
|
604
|
+
// queried tables' segments, not the store's total segment count.
|
|
605
|
+
const segmentIndex = transaction.objectStore("segments").index(SEGMENT_TABLE_INDEX);
|
|
606
|
+
const segmentValues = await Promise.all(foundTableIds.map((tableId) => requestResult(segmentIndex.getAll(tableId))));
|
|
607
|
+
const segments = segmentValues.flat().map((value) => asSegmentRecord(value));
|
|
608
|
+
segments.sort((left, right) => left.id.localeCompare(right.id));
|
|
609
|
+
const transactionIds = [...new Set(segments.map((segment) => segment.transactionId))];
|
|
610
|
+
const transactionStore = transaction.objectStore("transactions");
|
|
611
|
+
const transactionValues = await Promise.all(transactionIds.map((id) => requestResult(transactionStore.get(id))));
|
|
612
|
+
await transactionDone(transaction);
|
|
613
|
+
return {
|
|
614
|
+
manifestVersion: typeof versionValue === "number" ? versionValue : null,
|
|
615
|
+
tables,
|
|
616
|
+
segments,
|
|
617
|
+
transactions: transactionValues
|
|
618
|
+
.filter((value) => value !== undefined)
|
|
619
|
+
.map((value) => asTransactionRecord(value)),
|
|
620
|
+
catalogEpoch: typeof epochValue === "number" ? epochValue : 0,
|
|
621
|
+
};
|
|
622
|
+
}
|
|
623
|
+
async getCurrentManifest() {
|
|
624
|
+
const transaction = this.#transaction(["catalog", "manifests"], "readonly");
|
|
625
|
+
const manifestStore = transaction.objectStore("manifests");
|
|
626
|
+
const version = (await requestResult(transaction.objectStore("catalog").get(CURRENT_MANIFEST_KEY)));
|
|
627
|
+
const value = version === undefined ? undefined : await requestResult(manifestStore.get(version));
|
|
628
|
+
const manifest = value === undefined
|
|
629
|
+
? undefined
|
|
630
|
+
: await resolveManifestInTransaction(manifestStore, asStoredManifestRecord(value));
|
|
631
|
+
await transactionDone(transaction);
|
|
632
|
+
if (version !== undefined && manifest === undefined)
|
|
633
|
+
throw new Error("Current manifest is missing");
|
|
634
|
+
return manifest;
|
|
635
|
+
}
|
|
636
|
+
async getManifest(version) {
|
|
637
|
+
const transaction = this.#transaction("manifests", "readonly");
|
|
638
|
+
const manifestStore = transaction.objectStore("manifests");
|
|
639
|
+
const value = await requestResult(manifestStore.get(version));
|
|
640
|
+
const manifest = value === undefined
|
|
641
|
+
? undefined
|
|
642
|
+
: await resolveManifestInTransaction(manifestStore, asStoredManifestRecord(value));
|
|
643
|
+
await transactionDone(transaction);
|
|
644
|
+
return manifest;
|
|
645
|
+
}
|
|
646
|
+
async listManifests() {
|
|
647
|
+
const transaction = this.#transaction("manifests", "readonly");
|
|
648
|
+
const values = await requestResult(transaction.objectStore("manifests").getAll());
|
|
649
|
+
await transactionDone(transaction);
|
|
650
|
+
const records = values
|
|
651
|
+
.map(asStoredManifestRecord)
|
|
652
|
+
.sort((left, right) => left.version - right.version);
|
|
653
|
+
// Versions are dense in ascending key order, so one running set resolves every record.
|
|
654
|
+
const blockIds = new Set();
|
|
655
|
+
return records.map((record) => {
|
|
656
|
+
applyManifestRecord(blockIds, record);
|
|
657
|
+
return manifestView(record, blockIds);
|
|
658
|
+
});
|
|
659
|
+
}
|
|
660
|
+
async listManifestPage(afterVersion, limit) {
|
|
661
|
+
validatePageLimit(limit);
|
|
662
|
+
const transaction = this.#transaction("manifests", "readonly");
|
|
663
|
+
const manifestStore = transaction.objectStore("manifests");
|
|
664
|
+
const stored = await readCursorPage(manifestStore, limit, asStoredManifestRecord, (key) => typeof key === "number" && (afterVersion === null || key > afterVersion));
|
|
665
|
+
// Resolve the first record by chain walk, then advance the same set across the page.
|
|
666
|
+
const records = [];
|
|
667
|
+
let blockIds;
|
|
668
|
+
let resolvedVersion = Number.NaN;
|
|
669
|
+
for (const record of stored) {
|
|
670
|
+
if (blockIds === undefined || record.previousVersion !== resolvedVersion) {
|
|
671
|
+
blockIds = await resolveManifestBlockSetInTransaction(manifestStore, record);
|
|
672
|
+
}
|
|
673
|
+
else {
|
|
674
|
+
applyManifestRecord(blockIds, record);
|
|
675
|
+
}
|
|
676
|
+
resolvedVersion = record.version;
|
|
677
|
+
records.push(manifestView(record, blockIds));
|
|
678
|
+
}
|
|
679
|
+
await transactionDone(transaction);
|
|
680
|
+
return {
|
|
681
|
+
records,
|
|
682
|
+
nextCursor: records.length === limit ? (records.at(-1)?.version ?? null) : null,
|
|
683
|
+
};
|
|
684
|
+
}
|
|
685
|
+
async publishManifest(input) {
|
|
686
|
+
const transaction = this.#transaction(["blocks", "catalog", "manifests"], "readwrite");
|
|
687
|
+
const catalog = transaction.objectStore("catalog");
|
|
688
|
+
const current = (await requestResult(catalog.get(CURRENT_MANIFEST_KEY)));
|
|
689
|
+
const actualVersion = current ?? null;
|
|
690
|
+
if (actualVersion !== input.expectedVersion) {
|
|
691
|
+
transaction.abort();
|
|
692
|
+
await ignoreAbort(transaction);
|
|
693
|
+
throw new WriteConflictError(input.expectedVersion, actualVersion);
|
|
694
|
+
}
|
|
695
|
+
const blocks = transaction.objectStore("blocks");
|
|
696
|
+
for (const id of input.blockIds) {
|
|
697
|
+
const key = await requestResult(blocks.getKey(id));
|
|
698
|
+
if (key === undefined) {
|
|
699
|
+
transaction.abort();
|
|
700
|
+
await ignoreAbort(transaction);
|
|
701
|
+
throw new Error(`Manifest references missing block: ${id}`);
|
|
702
|
+
}
|
|
703
|
+
}
|
|
704
|
+
// A published manifest carries its complete list, so it stores as a checkpoint and any
|
|
705
|
+
// delta chain above it starts fresh.
|
|
706
|
+
const manifest = createManifest(input);
|
|
707
|
+
transaction.objectStore("manifests").add(manifest, manifest.version);
|
|
708
|
+
catalog.put(manifest.version, CURRENT_MANIFEST_KEY);
|
|
709
|
+
await bumpCatalogEpoch(catalog);
|
|
710
|
+
await transactionDone(transaction);
|
|
711
|
+
this.#manifestCache = { version: manifest.version, blockIds: new Set(manifest.blockIds) };
|
|
712
|
+
return manifest;
|
|
713
|
+
}
|
|
714
|
+
async beginTransaction(input) {
|
|
715
|
+
if (input.record.pendingBlockIds.length > 0 || input.record.pendingSegmentIds.length > 0) {
|
|
716
|
+
throw new TypeError("A fresh transaction cannot begin with pending artifacts");
|
|
717
|
+
}
|
|
718
|
+
const transaction = this.#transaction(["transactions", "manifests", "catalog"], "readwrite");
|
|
719
|
+
try {
|
|
720
|
+
const catalog = transaction.objectStore("catalog");
|
|
721
|
+
const current = (await requestResult(catalog.get(CURRENT_MANIFEST_KEY)));
|
|
722
|
+
const record = {
|
|
723
|
+
...structuredClone(input.record),
|
|
724
|
+
snapshotVersion: current ?? null,
|
|
725
|
+
};
|
|
726
|
+
await assertSnapshotAvailableInTransaction(transaction, record.snapshotVersion);
|
|
727
|
+
transaction.objectStore("transactions").add(record, record.id);
|
|
728
|
+
let rowIds;
|
|
729
|
+
if (input.reserveRowIds !== undefined) {
|
|
730
|
+
validateCount(input.reserveRowIds.count);
|
|
731
|
+
const key = `${ROW_ID_PREFIX}${input.reserveRowIds.tableId}`;
|
|
732
|
+
const currentRowId = (await requestResult(catalog.get(key)));
|
|
733
|
+
const start = currentRowId ?? 1n;
|
|
734
|
+
const endExclusive = start + BigInt(input.reserveRowIds.count);
|
|
735
|
+
catalog.put(endExclusive, key);
|
|
736
|
+
rowIds = { start, endExclusive };
|
|
737
|
+
}
|
|
738
|
+
let autoIncrementValues;
|
|
739
|
+
if (input.reserveAutoIncrement !== undefined) {
|
|
740
|
+
const { tableId, columnId, count, atLeast } = input.reserveAutoIncrement;
|
|
741
|
+
validateAutoIncrementReservation(count, atLeast);
|
|
742
|
+
const key = `${AUTO_INCREMENT_PREFIX}${tableId}/${columnId}`;
|
|
743
|
+
const current = (await requestResult(catalog.get(key)));
|
|
744
|
+
const start = maxBigInt(current ?? 1n, atLeast ?? 1n);
|
|
745
|
+
const endExclusive = start + BigInt(count);
|
|
746
|
+
catalog.put(endExclusive, key);
|
|
747
|
+
autoIncrementValues = { start, endExclusive };
|
|
748
|
+
}
|
|
749
|
+
await transactionDone(transaction);
|
|
750
|
+
return {
|
|
751
|
+
record: structuredClone(record),
|
|
752
|
+
...(rowIds === undefined ? {} : { rowIds }),
|
|
753
|
+
...(autoIncrementValues === undefined ? {} : { autoIncrementValues }),
|
|
754
|
+
};
|
|
755
|
+
}
|
|
756
|
+
catch (error) {
|
|
757
|
+
abortIfActive(transaction);
|
|
758
|
+
await ignoreAbort(transaction);
|
|
759
|
+
throw error;
|
|
760
|
+
}
|
|
761
|
+
}
|
|
762
|
+
async createTransaction(record) {
|
|
763
|
+
const transaction = this.#transaction(["transactions", "manifests", "blocks", "segments"], "readwrite");
|
|
764
|
+
try {
|
|
765
|
+
await assertSnapshotAvailableInTransaction(transaction, record.snapshotVersion);
|
|
766
|
+
await assertPendingArtifactsAvailableInTransaction(transaction, record);
|
|
767
|
+
transaction.objectStore("transactions").add(structuredClone(record), record.id);
|
|
768
|
+
await transactionDone(transaction);
|
|
769
|
+
}
|
|
770
|
+
catch (error) {
|
|
771
|
+
abortIfActive(transaction);
|
|
772
|
+
await ignoreAbort(transaction);
|
|
773
|
+
throw error;
|
|
774
|
+
}
|
|
775
|
+
}
|
|
776
|
+
async getTransaction(id) {
|
|
777
|
+
const transaction = this.#transaction("transactions", "readonly");
|
|
778
|
+
const value = await requestResult(transaction.objectStore("transactions").get(id));
|
|
779
|
+
await transactionDone(transaction);
|
|
780
|
+
return value === undefined ? undefined : asTransactionRecord(value);
|
|
781
|
+
}
|
|
782
|
+
async getTransactions(ids) {
|
|
783
|
+
const transaction = this.#transaction("transactions", "readonly");
|
|
784
|
+
const store = transaction.objectStore("transactions");
|
|
785
|
+
const values = await Promise.all(ids.map((id) => requestResult(store.get(id))));
|
|
786
|
+
await transactionDone(transaction);
|
|
787
|
+
return values.map((value) => (value === undefined ? undefined : asTransactionRecord(value)));
|
|
788
|
+
}
|
|
789
|
+
async listTransactions() {
|
|
790
|
+
const transaction = this.#transaction("transactions", "readonly");
|
|
791
|
+
const values = await requestResult(transaction.objectStore("transactions").getAll());
|
|
792
|
+
await transactionDone(transaction);
|
|
793
|
+
return values
|
|
794
|
+
.map(asTransactionRecord)
|
|
795
|
+
.sort((left, right) => left.startedAt.localeCompare(right.startedAt) || left.id.localeCompare(right.id));
|
|
796
|
+
}
|
|
797
|
+
async listTransactionPage(afterId, limit) {
|
|
798
|
+
validatePageLimit(limit);
|
|
799
|
+
const transaction = this.#transaction("transactions", "readonly");
|
|
800
|
+
const records = await readCursorPage(transaction.objectStore("transactions"), limit, asTransactionRecord, (key) => typeof key === "string" && (afterId === null || key > afterId));
|
|
801
|
+
await transactionDone(transaction);
|
|
802
|
+
return { records, nextCursor: records.length === limit ? (records.at(-1)?.id ?? null) : null };
|
|
803
|
+
}
|
|
804
|
+
async updateTransaction(id, expectedRevision, update) {
|
|
805
|
+
const transaction = this.#transaction(["transactions", "manifests", "blocks", "segments"], "readwrite");
|
|
806
|
+
const store = transaction.objectStore("transactions");
|
|
807
|
+
const value = await requestResult(store.get(id));
|
|
808
|
+
const current = value === undefined ? undefined : asTransactionRecord(value);
|
|
809
|
+
if (current?.revision !== expectedRevision) {
|
|
810
|
+
transaction.abort();
|
|
811
|
+
await ignoreAbort(transaction);
|
|
812
|
+
throw new TransactionRecordConflictError(id, expectedRevision, current?.revision ?? null);
|
|
813
|
+
}
|
|
814
|
+
try {
|
|
815
|
+
assertGenericTransactionUpdateAllowed(current, update);
|
|
816
|
+
const updated = updateTransactionRecord(current, update);
|
|
817
|
+
if (update.snapshotVersion !== undefined) {
|
|
818
|
+
await assertSnapshotAvailableInTransaction(transaction, updated.snapshotVersion);
|
|
819
|
+
}
|
|
820
|
+
await assertPendingArtifactsAvailableInTransaction(transaction, updated, update.pendingBlockIds !== undefined, update.pendingSegmentIds !== undefined);
|
|
821
|
+
store.put(updated, id);
|
|
822
|
+
await transactionDone(transaction);
|
|
823
|
+
return structuredClone(updated);
|
|
824
|
+
}
|
|
825
|
+
catch (error) {
|
|
826
|
+
abortIfActive(transaction);
|
|
827
|
+
await ignoreAbort(transaction);
|
|
828
|
+
throw error;
|
|
829
|
+
}
|
|
830
|
+
}
|
|
831
|
+
async stageTransactionArtifacts(input) {
|
|
832
|
+
const ids = new Set();
|
|
833
|
+
for (const block of input.blocks) {
|
|
834
|
+
if (block.id.length === 0)
|
|
835
|
+
throw new TypeError("Block ID cannot be empty");
|
|
836
|
+
if (ids.has(block.id))
|
|
837
|
+
throw new Error(`Block already exists: ${block.id}`);
|
|
838
|
+
ids.add(block.id);
|
|
839
|
+
}
|
|
840
|
+
const transaction = this.#transaction(["blocks", "segments", "transactions"], "readwrite");
|
|
841
|
+
try {
|
|
842
|
+
const transactionStore = transaction.objectStore("transactions");
|
|
843
|
+
const value = await requestResult(transactionStore.get(input.transactionId));
|
|
844
|
+
const current = value === undefined ? undefined : asTransactionRecord(value);
|
|
845
|
+
if (current?.revision !== input.expectedRevision) {
|
|
846
|
+
transaction.abort();
|
|
847
|
+
await ignoreAbort(transaction);
|
|
848
|
+
throw new TransactionRecordConflictError(input.transactionId, input.expectedRevision, current?.revision ?? null);
|
|
849
|
+
}
|
|
850
|
+
const update = {
|
|
851
|
+
pendingBlockIds: [...current.pendingBlockIds, ...input.blocks.map((block) => block.id)],
|
|
852
|
+
pendingSegmentIds: [
|
|
853
|
+
...current.pendingSegmentIds,
|
|
854
|
+
...input.segments.map((segment) => segment.id),
|
|
855
|
+
],
|
|
856
|
+
updatedAt: input.updatedAt,
|
|
857
|
+
};
|
|
858
|
+
assertGenericTransactionUpdateAllowed(current, update);
|
|
859
|
+
const updated = updateTransactionRecord(current, update);
|
|
860
|
+
// Only previously journaled artifacts need existence probes: the ones added below commit
|
|
861
|
+
// or fail atomically with the journal update itself.
|
|
862
|
+
await assertPendingArtifactsAvailableInTransaction(transaction, current);
|
|
863
|
+
const blockStore = transaction.objectStore("blocks");
|
|
864
|
+
for (const block of input.blocks) {
|
|
865
|
+
const bytes = block.bytes.byteOffset === 0 && block.bytes.byteLength === block.bytes.buffer.byteLength
|
|
866
|
+
? block.bytes
|
|
867
|
+
: block.bytes.slice();
|
|
868
|
+
blockStore.add(bytes, block.id);
|
|
869
|
+
}
|
|
870
|
+
const segmentStore = transaction.objectStore("segments");
|
|
871
|
+
for (const segment of input.segments) {
|
|
872
|
+
const normalized = normalizeSegmentRecord(segment);
|
|
873
|
+
segmentStore.add(normalized, normalized.id);
|
|
874
|
+
}
|
|
875
|
+
transactionStore.put(updated, input.transactionId);
|
|
876
|
+
await transactionDone(transaction);
|
|
877
|
+
return structuredClone(updated);
|
|
878
|
+
}
|
|
879
|
+
catch (error) {
|
|
880
|
+
abortIfActive(transaction);
|
|
881
|
+
await ignoreAbort(transaction);
|
|
882
|
+
throw error;
|
|
883
|
+
}
|
|
884
|
+
}
|
|
885
|
+
async commitTransaction(input) {
|
|
886
|
+
const transaction = this.#transaction(["blocks", "catalog", "manifests", "transactions", "segments"], "readwrite");
|
|
887
|
+
const transactionStore = transaction.objectStore("transactions");
|
|
888
|
+
const transactionValue = await requestResult(transactionStore.get(input.transactionId));
|
|
889
|
+
const record = transactionValue === undefined ? undefined : asTransactionRecord(transactionValue);
|
|
890
|
+
if (record?.revision !== input.expectedTransactionRevision || record.status !== "active") {
|
|
891
|
+
transaction.abort();
|
|
892
|
+
await ignoreAbort(transaction);
|
|
893
|
+
throw new TransactionRecordConflictError(input.transactionId, input.expectedTransactionRevision, record?.revision ?? null);
|
|
894
|
+
}
|
|
895
|
+
const catalog = transaction.objectStore("catalog");
|
|
896
|
+
const current = (await requestResult(catalog.get(CURRENT_MANIFEST_KEY)));
|
|
897
|
+
const actualVersion = current ?? null;
|
|
898
|
+
if (actualVersion !== input.expectedManifestVersion) {
|
|
899
|
+
transaction.abort();
|
|
900
|
+
await ignoreAbort(transaction);
|
|
901
|
+
throw new WriteConflictError(input.expectedManifestVersion, actualVersion);
|
|
902
|
+
}
|
|
903
|
+
if (record.snapshotVersion !== input.expectedManifestVersion) {
|
|
904
|
+
transaction.abort();
|
|
905
|
+
await ignoreAbort(transaction);
|
|
906
|
+
throw new Error("Transaction snapshot does not match the expected manifest");
|
|
907
|
+
}
|
|
908
|
+
const manifestStore = transaction.objectStore("manifests");
|
|
909
|
+
const baseRecord = input.expectedManifestVersion === null
|
|
910
|
+
? undefined
|
|
911
|
+
: await (async () => {
|
|
912
|
+
const value = await requestResult(manifestStore.get(input.expectedManifestVersion ?? -1));
|
|
913
|
+
return value === undefined ? undefined : asStoredManifestRecord(value);
|
|
914
|
+
})();
|
|
915
|
+
if (input.expectedManifestVersion !== null && baseRecord === undefined) {
|
|
916
|
+
transaction.abort();
|
|
917
|
+
await ignoreAbort(transaction);
|
|
918
|
+
throw new Error(`Snapshot manifest is missing: ${String(input.expectedManifestVersion)}`);
|
|
919
|
+
}
|
|
920
|
+
const removedBlockIds = [...new Set(input.removedBlockIds ?? [])].sort();
|
|
921
|
+
const pendingRemovedBlock = removedBlockIds.find((id) => record.pendingBlockIds.includes(id));
|
|
922
|
+
if (pendingRemovedBlock !== undefined) {
|
|
923
|
+
transaction.abort();
|
|
924
|
+
await ignoreAbort(transaction);
|
|
925
|
+
throw new Error(`Cannot supersede a pending block: ${pendingRemovedBlock}`);
|
|
926
|
+
}
|
|
927
|
+
// The resolved base block set is needed only to validate removals and to write checkpoints;
|
|
928
|
+
// an ordinary delta commit with no removals never materializes it. The instance cache makes
|
|
929
|
+
// the removal/checkpoint path O(delta) when this store published the base version.
|
|
930
|
+
const baseDepth = baseRecord === undefined || baseRecord.blockIds !== undefined
|
|
931
|
+
? 0
|
|
932
|
+
: (baseRecord.deltaDepth ?? 0);
|
|
933
|
+
const writesCheckpoint = baseRecord === undefined || baseDepth + 1 >= MANIFEST_CHECKPOINT_INTERVAL;
|
|
934
|
+
let baseBlockIdSet;
|
|
935
|
+
let cacheOwnsBaseSet = false;
|
|
936
|
+
if (removedBlockIds.length > 0 || writesCheckpoint) {
|
|
937
|
+
if (this.#manifestCache?.version === input.expectedManifestVersion) {
|
|
938
|
+
baseBlockIdSet = this.#manifestCache.blockIds;
|
|
939
|
+
cacheOwnsBaseSet = true;
|
|
940
|
+
}
|
|
941
|
+
else if (baseRecord === undefined) {
|
|
942
|
+
baseBlockIdSet = new Set();
|
|
943
|
+
}
|
|
944
|
+
else {
|
|
945
|
+
baseBlockIdSet = await resolveManifestBlockSetInTransaction(manifestStore, baseRecord);
|
|
946
|
+
}
|
|
947
|
+
const invalidRemovedBlock = removedBlockIds.find((id) => !baseBlockIdSet?.has(id));
|
|
948
|
+
if (invalidRemovedBlock !== undefined) {
|
|
949
|
+
transaction.abort();
|
|
950
|
+
await ignoreAbort(transaction);
|
|
951
|
+
throw new Error(`Cannot supersede a block outside the transaction snapshot: ${invalidRemovedBlock}`);
|
|
952
|
+
}
|
|
953
|
+
}
|
|
954
|
+
const blockStore = transaction.objectStore("blocks");
|
|
955
|
+
const storedBlockKeys = await Promise.all(record.pendingBlockIds.map((id) => requestResult(blockStore.getKey(id))));
|
|
956
|
+
const missingBlockIndex = storedBlockKeys.findIndex((key) => key === undefined);
|
|
957
|
+
if (missingBlockIndex >= 0) {
|
|
958
|
+
transaction.abort();
|
|
959
|
+
await ignoreAbort(transaction);
|
|
960
|
+
throw new Error(`Manifest references missing block: ${record.pendingBlockIds[missingBlockIndex] ?? ""}`);
|
|
961
|
+
}
|
|
962
|
+
const segmentStore = transaction.objectStore("segments");
|
|
963
|
+
const pendingSegmentValues = await Promise.all(record.pendingSegmentIds.map((id) => requestResult(segmentStore.get(id))));
|
|
964
|
+
const missingSegmentIndex = pendingSegmentValues.findIndex((value) => value === undefined);
|
|
965
|
+
if (missingSegmentIndex >= 0) {
|
|
966
|
+
transaction.abort();
|
|
967
|
+
await ignoreAbort(transaction);
|
|
968
|
+
throw new Error(`Transaction references missing segment: ${record.pendingSegmentIds[missingSegmentIndex] ?? ""}`);
|
|
969
|
+
}
|
|
970
|
+
const pendingSegments = pendingSegmentValues.map(asSegmentRecord);
|
|
971
|
+
const foreignSegment = pendingSegments.find((segment) => segment.transactionId !== record.id);
|
|
972
|
+
if (foreignSegment !== undefined) {
|
|
973
|
+
transaction.abort();
|
|
974
|
+
await ignoreAbort(transaction);
|
|
975
|
+
throw new Error(`Segment ${foreignSegment.id} belongs to another transaction`);
|
|
976
|
+
}
|
|
977
|
+
// The single-entry path below is the hot bulk-load path and keeps the membership cache;
|
|
978
|
+
// multi-entry commits (atomic write scopes) take the simpler merged path further down and
|
|
979
|
+
// drop the cache.
|
|
980
|
+
const uniqueKeyEntries = input.uniqueKeyChanges ?? [];
|
|
981
|
+
const uniqueKeyChanges = uniqueKeyEntries.length === 1 ? uniqueKeyEntries[0] : undefined;
|
|
982
|
+
const keyMode = uniqueKeyChanges?.storageMode;
|
|
983
|
+
const chunkedKeys = keyMode === "chunks-v1" || keyMode === "chunks-v2";
|
|
984
|
+
const keyCache = this.#uniqueKeyCache;
|
|
985
|
+
const keyCacheValid = chunkedKeys &&
|
|
986
|
+
keyCache !== undefined &&
|
|
987
|
+
keyCache.tableId === uniqueKeyChanges?.tableId &&
|
|
988
|
+
keyCache.mode === keyMode &&
|
|
989
|
+
keyCache.version === input.expectedManifestVersion;
|
|
990
|
+
let keyState;
|
|
991
|
+
if (uniqueKeyChanges !== undefined) {
|
|
992
|
+
let conflictToken;
|
|
993
|
+
if (chunkedKeys) {
|
|
994
|
+
if (keyCacheValid) {
|
|
995
|
+
// Steady-state bulk load: membership answered entirely from memory, zero reads.
|
|
996
|
+
const existing = new Set();
|
|
997
|
+
for (const token of uniqueKeyChanges.keyTokens) {
|
|
998
|
+
if (keyCache.present.has(token))
|
|
999
|
+
existing.add(token);
|
|
1000
|
+
}
|
|
1001
|
+
keyState = {
|
|
1002
|
+
existing,
|
|
1003
|
+
index: keyCache.index,
|
|
1004
|
+
chunks: keyCache.chunks,
|
|
1005
|
+
fullPresent: keyCache.present,
|
|
1006
|
+
};
|
|
1007
|
+
}
|
|
1008
|
+
else {
|
|
1009
|
+
keyState = await readChunkedUniqueKeys(catalog, uniqueKeyChanges.tableId, uniqueKeyChanges.keyTokens, keyMode);
|
|
1010
|
+
}
|
|
1011
|
+
if (uniqueKeyChanges.requireAbsent) {
|
|
1012
|
+
conflictToken = uniqueKeyChanges.keyTokens.find((token) => keyState?.existing.has(token));
|
|
1013
|
+
}
|
|
1014
|
+
}
|
|
1015
|
+
else {
|
|
1016
|
+
const storedKeys = await Promise.all(uniqueKeyChanges.keyTokens.map((token) => requestResult(catalog.getKey(uniqueKeyKey(uniqueKeyChanges.tableId, token)))));
|
|
1017
|
+
if (uniqueKeyChanges.requireAbsent) {
|
|
1018
|
+
const conflictIndex = storedKeys.findIndex((key) => key !== undefined);
|
|
1019
|
+
conflictToken =
|
|
1020
|
+
conflictIndex >= 0 ? (uniqueKeyChanges.keyTokens[conflictIndex] ?? "") : undefined;
|
|
1021
|
+
}
|
|
1022
|
+
}
|
|
1023
|
+
if (conflictToken !== undefined) {
|
|
1024
|
+
transaction.abort();
|
|
1025
|
+
await ignoreAbort(transaction);
|
|
1026
|
+
throw new UniqueKeyConflictError(uniqueKeyChanges.tableId, conflictToken);
|
|
1027
|
+
}
|
|
1028
|
+
}
|
|
1029
|
+
/**
|
|
1030
|
+
* Multi-entry key work, resolved before the manifest writes: per table, read the union
|
|
1031
|
+
* of touched tokens once, then replay the entries in operation order over a working set
|
|
1032
|
+
* so in-scope conflicts fail exactly like cross-commit conflicts. The net add/remove
|
|
1033
|
+
* per table applies after the manifest version is known.
|
|
1034
|
+
*/
|
|
1035
|
+
const multiKeyWork = [];
|
|
1036
|
+
if (uniqueKeyEntries.length > 1) {
|
|
1037
|
+
const byTable = new Map();
|
|
1038
|
+
for (const entry of uniqueKeyEntries) {
|
|
1039
|
+
const list = byTable.get(entry.tableId) ?? [];
|
|
1040
|
+
list.push(entry);
|
|
1041
|
+
byTable.set(entry.tableId, list);
|
|
1042
|
+
}
|
|
1043
|
+
for (const [tableId, entries] of byTable) {
|
|
1044
|
+
const mode = entries[0]?.storageMode;
|
|
1045
|
+
const chunked = mode === "chunks-v1" || mode === "chunks-v2";
|
|
1046
|
+
const unionTokens = [...new Set(entries.flatMap((entry) => entry.keyTokens))];
|
|
1047
|
+
let present;
|
|
1048
|
+
if (chunked) {
|
|
1049
|
+
const state = await readChunkedUniqueKeys(catalog, tableId, unionTokens, mode);
|
|
1050
|
+
present = state.existing;
|
|
1051
|
+
}
|
|
1052
|
+
else {
|
|
1053
|
+
const stored = await Promise.all(unionTokens.map((token) => requestResult(catalog.getKey(uniqueKeyKey(tableId, token)))));
|
|
1054
|
+
present = new Set(unionTokens.filter((_, index) => stored[index] !== undefined));
|
|
1055
|
+
}
|
|
1056
|
+
const original = new Set(present);
|
|
1057
|
+
for (const entry of entries) {
|
|
1058
|
+
for (const token of entry.keyTokens) {
|
|
1059
|
+
if (entry.remove === true) {
|
|
1060
|
+
present.delete(token);
|
|
1061
|
+
continue;
|
|
1062
|
+
}
|
|
1063
|
+
if (entry.requireAbsent && present.has(token)) {
|
|
1064
|
+
transaction.abort();
|
|
1065
|
+
await ignoreAbort(transaction);
|
|
1066
|
+
throw new UniqueKeyConflictError(tableId, token);
|
|
1067
|
+
}
|
|
1068
|
+
present.add(token);
|
|
1069
|
+
}
|
|
1070
|
+
}
|
|
1071
|
+
multiKeyWork.push({
|
|
1072
|
+
tableId,
|
|
1073
|
+
...(mode === undefined ? {} : { mode }),
|
|
1074
|
+
addedTokens: unionTokens.filter((token) => present.has(token) && !original.has(token)),
|
|
1075
|
+
removedTokens: unionTokens.filter((token) => !present.has(token) && original.has(token)),
|
|
1076
|
+
});
|
|
1077
|
+
}
|
|
1078
|
+
}
|
|
1079
|
+
const addedBlockIds = [...new Set(record.pendingBlockIds)].sort();
|
|
1080
|
+
const summary = {
|
|
1081
|
+
version: input.expectedManifestVersion === null ? 0 : input.expectedManifestVersion + 1,
|
|
1082
|
+
previousVersion: input.expectedManifestVersion,
|
|
1083
|
+
createdAt: input.committedAt,
|
|
1084
|
+
...(input.changedTableIds === undefined
|
|
1085
|
+
? {}
|
|
1086
|
+
: { changedTableIds: [...input.changedTableIds] }),
|
|
1087
|
+
};
|
|
1088
|
+
let checkpointBlockIds;
|
|
1089
|
+
let manifestRecord;
|
|
1090
|
+
if (writesCheckpoint) {
|
|
1091
|
+
checkpointBlockIds = new Set(baseBlockIdSet);
|
|
1092
|
+
for (const id of removedBlockIds)
|
|
1093
|
+
checkpointBlockIds.delete(id);
|
|
1094
|
+
for (const id of addedBlockIds)
|
|
1095
|
+
checkpointBlockIds.add(id);
|
|
1096
|
+
manifestRecord = { ...summary, blockIds: [...checkpointBlockIds].sort() };
|
|
1097
|
+
}
|
|
1098
|
+
else {
|
|
1099
|
+
manifestRecord = {
|
|
1100
|
+
...summary,
|
|
1101
|
+
addedBlockIds,
|
|
1102
|
+
removedBlockIds,
|
|
1103
|
+
deltaDepth: baseDepth + 1,
|
|
1104
|
+
};
|
|
1105
|
+
}
|
|
1106
|
+
const manifest = summary;
|
|
1107
|
+
const committed = updateTransactionRecord(record, {
|
|
1108
|
+
status: "committed",
|
|
1109
|
+
committedVersion: manifest.version,
|
|
1110
|
+
updatedAt: input.committedAt,
|
|
1111
|
+
});
|
|
1112
|
+
manifestStore.add(manifestRecord, manifest.version);
|
|
1113
|
+
catalog.put(manifest.version, CURRENT_MANIFEST_KEY);
|
|
1114
|
+
await bumpCatalogEpoch(catalog);
|
|
1115
|
+
for (const segment of pendingSegments) {
|
|
1116
|
+
segmentStore.put({
|
|
1117
|
+
...segment,
|
|
1118
|
+
level: segment.level ?? 0,
|
|
1119
|
+
logicalOrder: segment.logicalOrder ?? manifest.version,
|
|
1120
|
+
}, segment.id);
|
|
1121
|
+
}
|
|
1122
|
+
// How the unique-key cache advances once this commit proves durable: `keep` carries the
|
|
1123
|
+
// valid cache through with its membership delta applied; `replace` installs newly resolved
|
|
1124
|
+
// full knowledge; `drop` forgets a same-table cache we could not keep truthful.
|
|
1125
|
+
let keyCachePlan;
|
|
1126
|
+
if (uniqueKeyChanges !== undefined) {
|
|
1127
|
+
if (chunkedKeys) {
|
|
1128
|
+
const existing = keyState?.existing ?? new Set();
|
|
1129
|
+
const chunk = {
|
|
1130
|
+
addedTokens: uniqueKeyChanges.remove === true
|
|
1131
|
+
? []
|
|
1132
|
+
: uniqueKeyChanges.keyTokens.filter((token) => !existing.has(token)),
|
|
1133
|
+
removedTokens: uniqueKeyChanges.remove === true
|
|
1134
|
+
? uniqueKeyChanges.keyTokens.filter((token) => existing.has(token))
|
|
1135
|
+
: [],
|
|
1136
|
+
};
|
|
1137
|
+
if (chunk.addedTokens.length > 0 || chunk.removedTokens.length > 0) {
|
|
1138
|
+
const tableId = uniqueKeyChanges.tableId;
|
|
1139
|
+
const index = keyState?.index ?? { versions: [], hasBase: false };
|
|
1140
|
+
// Complete membership before this commit, when known: from the live cache, from a
|
|
1141
|
+
// bulk read that resolved every partition, or trivially empty for a base-less table
|
|
1142
|
+
// whose whole tail is in hand.
|
|
1143
|
+
const priorPresent = keyState?.fullPresent ??
|
|
1144
|
+
(index.hasBase ? undefined : replayChunks(keyState?.chunks ?? []));
|
|
1145
|
+
if (index.versions.length + 1 > UNIQUE_KEY_TAIL_CHUNK_LIMIT) {
|
|
1146
|
+
const tailChunks = [...(keyState?.chunks ?? []), chunk];
|
|
1147
|
+
if (keyMode === "chunks-v2") {
|
|
1148
|
+
// Fold the tail into hash-partitioned base chunks, atomically with the manifest
|
|
1149
|
+
// publication: hundreds of records regardless of key count, never one per key.
|
|
1150
|
+
//
|
|
1151
|
+
// A tail of point writes is worth almost nothing next to the base it would be
|
|
1152
|
+
// folded into, and folding it would rewrite whole partitions to record a handful
|
|
1153
|
+
// of tokens — a cost that grows with the table while the write does not. Such a
|
|
1154
|
+
// tail collapses into a single equivalent chunk instead, leaving the base alone.
|
|
1155
|
+
// The base is only rewritten once the tail is big enough to be worth it.
|
|
1156
|
+
const merged = index.hasBase && countTailTokens(tailChunks) < UNIQUE_KEY_PARTITION_TARGET
|
|
1157
|
+
? mergeTailChunks(tailChunks)
|
|
1158
|
+
: undefined;
|
|
1159
|
+
// A token's partition is fixed by its hash and the partition count, so a tail
|
|
1160
|
+
// that leaves the count unchanged only disturbs the partitions its own tokens
|
|
1161
|
+
// land in. That path reads and rewrites those, and nothing else. Everything else
|
|
1162
|
+
// (no recorded size, a partition count that has to change, a base written before
|
|
1163
|
+
// this existed) takes the full rewrite below.
|
|
1164
|
+
const incremental = merged !== undefined
|
|
1165
|
+
? undefined
|
|
1166
|
+
: await foldTailIntoTouchedPartitions(catalog, tableId, index, tailChunks);
|
|
1167
|
+
if (merged !== undefined) {
|
|
1168
|
+
for (const version of index.versions) {
|
|
1169
|
+
catalog.delete(uniqueKeyChunkKey(tableId, version));
|
|
1170
|
+
}
|
|
1171
|
+
catalog.put(merged, uniqueKeyChunkKey(tableId, manifest.version));
|
|
1172
|
+
const nextIndex = {
|
|
1173
|
+
versions: [manifest.version],
|
|
1174
|
+
hasBase: true,
|
|
1175
|
+
...(index.partitions === undefined ? {} : { partitions: index.partitions }),
|
|
1176
|
+
...(index.tokenCount === undefined ? {} : { tokenCount: index.tokenCount }),
|
|
1177
|
+
};
|
|
1178
|
+
catalog.put(nextIndex, uniqueKeyChunkIndexKey(tableId));
|
|
1179
|
+
let cachePresent = priorPresent;
|
|
1180
|
+
if (cachePresent !== undefined) {
|
|
1181
|
+
if (cachePresent === keyState?.fullPresent)
|
|
1182
|
+
cachePresent = new Set(cachePresent);
|
|
1183
|
+
applyChunk(cachePresent, chunk);
|
|
1184
|
+
}
|
|
1185
|
+
keyCachePlan =
|
|
1186
|
+
cachePresent === undefined
|
|
1187
|
+
? { action: "drop" }
|
|
1188
|
+
: {
|
|
1189
|
+
action: "replace",
|
|
1190
|
+
present: cachePresent,
|
|
1191
|
+
chunks: [merged],
|
|
1192
|
+
index: nextIndex,
|
|
1193
|
+
};
|
|
1194
|
+
}
|
|
1195
|
+
else if (incremental !== undefined) {
|
|
1196
|
+
for (const version of index.versions) {
|
|
1197
|
+
catalog.delete(uniqueKeyChunkKey(tableId, version));
|
|
1198
|
+
}
|
|
1199
|
+
const nextIndex = {
|
|
1200
|
+
versions: [],
|
|
1201
|
+
hasBase: true,
|
|
1202
|
+
partitions: incremental.partitions,
|
|
1203
|
+
tokenCount: incremental.tokenCount,
|
|
1204
|
+
};
|
|
1205
|
+
catalog.put(nextIndex, uniqueKeyChunkIndexKey(tableId));
|
|
1206
|
+
// The membership cache only survives when it was already complete; rebuilding
|
|
1207
|
+
// it here would re-read every partition and undo the saving.
|
|
1208
|
+
let cachePresent = priorPresent;
|
|
1209
|
+
if (cachePresent !== undefined) {
|
|
1210
|
+
if (cachePresent === keyState?.fullPresent)
|
|
1211
|
+
cachePresent = new Set(cachePresent);
|
|
1212
|
+
applyChunk(cachePresent, chunk);
|
|
1213
|
+
}
|
|
1214
|
+
keyCachePlan =
|
|
1215
|
+
cachePresent === undefined
|
|
1216
|
+
? { action: "drop" }
|
|
1217
|
+
: { action: "replace", present: cachePresent, chunks: [], index: nextIndex };
|
|
1218
|
+
}
|
|
1219
|
+
else {
|
|
1220
|
+
let nextPresent = priorPresent;
|
|
1221
|
+
if (nextPresent === undefined) {
|
|
1222
|
+
nextPresent = await readAllV2BaseTokens(catalog, tableId, index);
|
|
1223
|
+
for (const tail of keyState?.chunks ?? [])
|
|
1224
|
+
applyChunk(nextPresent, tail);
|
|
1225
|
+
}
|
|
1226
|
+
else if (nextPresent === keyState?.fullPresent) {
|
|
1227
|
+
// The cache's own set must not be mutated before the commit is durable.
|
|
1228
|
+
nextPresent = new Set(nextPresent);
|
|
1229
|
+
}
|
|
1230
|
+
applyChunk(nextPresent, chunk);
|
|
1231
|
+
const partitions = Math.max(1, Math.ceil(nextPresent.size / UNIQUE_KEY_PARTITION_TARGET));
|
|
1232
|
+
const parts = Array.from({ length: partitions }, () => []);
|
|
1233
|
+
for (const token of nextPresent) {
|
|
1234
|
+
parts[fnv1a(token) % partitions]?.push(token);
|
|
1235
|
+
}
|
|
1236
|
+
parts.forEach((tokens, ordinal) => {
|
|
1237
|
+
catalog.put(tokens, uniqueKeyBasePartKey(tableId, ordinal));
|
|
1238
|
+
});
|
|
1239
|
+
for (let ordinal = partitions; ordinal < (index.partitions ?? 0); ordinal += 1) {
|
|
1240
|
+
catalog.delete(uniqueKeyBasePartKey(tableId, ordinal));
|
|
1241
|
+
}
|
|
1242
|
+
for (const version of index.versions) {
|
|
1243
|
+
catalog.delete(uniqueKeyChunkKey(tableId, version));
|
|
1244
|
+
}
|
|
1245
|
+
const nextIndex = {
|
|
1246
|
+
versions: [],
|
|
1247
|
+
hasBase: true,
|
|
1248
|
+
partitions,
|
|
1249
|
+
tokenCount: nextPresent.size,
|
|
1250
|
+
};
|
|
1251
|
+
catalog.put(nextIndex, uniqueKeyChunkIndexKey(tableId));
|
|
1252
|
+
keyCachePlan = {
|
|
1253
|
+
action: "replace",
|
|
1254
|
+
present: nextPresent,
|
|
1255
|
+
chunks: [],
|
|
1256
|
+
index: nextIndex,
|
|
1257
|
+
};
|
|
1258
|
+
}
|
|
1259
|
+
}
|
|
1260
|
+
else {
|
|
1261
|
+
// chunks-v1 folds into per-key base records, incrementally over the tail.
|
|
1262
|
+
for (const tail of tailChunks) {
|
|
1263
|
+
for (const token of tail.addedTokens) {
|
|
1264
|
+
catalog.put(manifest.version, uniqueKeyBaseKey(tableId, token));
|
|
1265
|
+
}
|
|
1266
|
+
for (const token of tail.removedTokens) {
|
|
1267
|
+
catalog.delete(uniqueKeyBaseKey(tableId, token));
|
|
1268
|
+
}
|
|
1269
|
+
}
|
|
1270
|
+
for (const version of index.versions) {
|
|
1271
|
+
catalog.delete(uniqueKeyChunkKey(tableId, version));
|
|
1272
|
+
}
|
|
1273
|
+
const nextIndex = { versions: [], hasBase: true };
|
|
1274
|
+
catalog.put(nextIndex, uniqueKeyChunkIndexKey(tableId));
|
|
1275
|
+
if (priorPresent !== undefined) {
|
|
1276
|
+
const nextPresent = priorPresent === keyState?.fullPresent ? new Set(priorPresent) : priorPresent;
|
|
1277
|
+
applyChunk(nextPresent, chunk);
|
|
1278
|
+
keyCachePlan = {
|
|
1279
|
+
action: "replace",
|
|
1280
|
+
present: nextPresent,
|
|
1281
|
+
chunks: [],
|
|
1282
|
+
index: nextIndex,
|
|
1283
|
+
};
|
|
1284
|
+
}
|
|
1285
|
+
else {
|
|
1286
|
+
keyCachePlan = { action: "drop" };
|
|
1287
|
+
}
|
|
1288
|
+
}
|
|
1289
|
+
}
|
|
1290
|
+
else {
|
|
1291
|
+
catalog.put(chunk, uniqueKeyChunkKey(tableId, manifest.version));
|
|
1292
|
+
const nextIndex = {
|
|
1293
|
+
versions: [...index.versions, manifest.version],
|
|
1294
|
+
hasBase: index.hasBase,
|
|
1295
|
+
...(index.partitions === undefined ? {} : { partitions: index.partitions }),
|
|
1296
|
+
// The base is untouched here, so its recorded size still describes it. Dropping
|
|
1297
|
+
// it would cost the next fold its incremental path.
|
|
1298
|
+
...(index.tokenCount === undefined ? {} : { tokenCount: index.tokenCount }),
|
|
1299
|
+
};
|
|
1300
|
+
catalog.put(nextIndex, uniqueKeyChunkIndexKey(tableId));
|
|
1301
|
+
if (keyCacheValid) {
|
|
1302
|
+
keyCachePlan = { action: "keep", chunk, index: nextIndex };
|
|
1303
|
+
}
|
|
1304
|
+
else if (keyState?.fullPresent !== undefined) {
|
|
1305
|
+
const nextPresent = new Set(keyState.fullPresent);
|
|
1306
|
+
applyChunk(nextPresent, chunk);
|
|
1307
|
+
keyCachePlan = {
|
|
1308
|
+
action: "replace",
|
|
1309
|
+
present: nextPresent,
|
|
1310
|
+
chunks: [...keyState.chunks, chunk],
|
|
1311
|
+
index: nextIndex,
|
|
1312
|
+
};
|
|
1313
|
+
}
|
|
1314
|
+
else if (priorPresent !== undefined) {
|
|
1315
|
+
applyChunk(priorPresent, chunk);
|
|
1316
|
+
keyCachePlan = {
|
|
1317
|
+
action: "replace",
|
|
1318
|
+
present: priorPresent,
|
|
1319
|
+
chunks: [...(keyState?.chunks ?? []), chunk],
|
|
1320
|
+
index: nextIndex,
|
|
1321
|
+
};
|
|
1322
|
+
}
|
|
1323
|
+
else {
|
|
1324
|
+
keyCachePlan = { action: "drop" };
|
|
1325
|
+
}
|
|
1326
|
+
}
|
|
1327
|
+
}
|
|
1328
|
+
}
|
|
1329
|
+
else {
|
|
1330
|
+
uniqueKeyChanges.keyTokens.forEach((token) => {
|
|
1331
|
+
const key = uniqueKeyKey(uniqueKeyChanges.tableId, token);
|
|
1332
|
+
if (uniqueKeyChanges.remove === true)
|
|
1333
|
+
catalog.delete(key);
|
|
1334
|
+
else
|
|
1335
|
+
catalog.put(manifest.version, key);
|
|
1336
|
+
});
|
|
1337
|
+
}
|
|
1338
|
+
}
|
|
1339
|
+
for (const work of multiKeyWork) {
|
|
1340
|
+
if (work.mode === "chunks-v1" || work.mode === "chunks-v2") {
|
|
1341
|
+
if (work.addedTokens.length > 0 || work.removedTokens.length > 0) {
|
|
1342
|
+
const indexValue = (await requestResult(catalog.get(uniqueKeyChunkIndexKey(work.tableId))));
|
|
1343
|
+
const index = indexValue ?? { versions: [], hasBase: false };
|
|
1344
|
+
catalog.put({ addedTokens: work.addedTokens, removedTokens: work.removedTokens }, uniqueKeyChunkKey(work.tableId, manifest.version));
|
|
1345
|
+
catalog.put({
|
|
1346
|
+
versions: [...index.versions, manifest.version],
|
|
1347
|
+
hasBase: index.hasBase,
|
|
1348
|
+
...(index.partitions === undefined ? {} : { partitions: index.partitions }),
|
|
1349
|
+
// Appending a tail chunk leaves the base alone, so its recorded size still holds.
|
|
1350
|
+
...(index.tokenCount === undefined ? {} : { tokenCount: index.tokenCount }),
|
|
1351
|
+
}, uniqueKeyChunkIndexKey(work.tableId));
|
|
1352
|
+
}
|
|
1353
|
+
}
|
|
1354
|
+
else {
|
|
1355
|
+
for (const token of work.addedTokens) {
|
|
1356
|
+
catalog.put(manifest.version, uniqueKeyKey(work.tableId, token));
|
|
1357
|
+
}
|
|
1358
|
+
for (const token of work.removedTokens) {
|
|
1359
|
+
catalog.delete(uniqueKeyKey(work.tableId, token));
|
|
1360
|
+
}
|
|
1361
|
+
}
|
|
1362
|
+
}
|
|
1363
|
+
// Full-text deltas apply atomically with the publish; a stale writer (one that committed
|
|
1364
|
+
// segments to an indexed table without deltas) flips the affected columns to "invalid"
|
|
1365
|
+
// instead of failing the data commit — the index self-heals through a rebuild.
|
|
1366
|
+
const changedFtsTableIds = new Set(pendingSegments.map((segment) => segment.tableId));
|
|
1367
|
+
for (const tableId of changedFtsTableIds) {
|
|
1368
|
+
const tableValue = await requestResult(catalog.get(`${TABLE_ID_PREFIX}${tableId}`));
|
|
1369
|
+
if (tableValue === undefined)
|
|
1370
|
+
continue;
|
|
1371
|
+
const forTable = (input.ftsChanges ?? []).find((entry) => entry.tableId === tableId);
|
|
1372
|
+
const covered = new Set(forTable?.columns.map((column) => column.columnId) ?? []);
|
|
1373
|
+
const invalidated = invalidateUncoveredFtsColumns(structuredClone(tableValue), covered);
|
|
1374
|
+
if (invalidated !== undefined) {
|
|
1375
|
+
catalog.put(structuredClone(invalidated), `${TABLE_ID_PREFIX}${tableId}`);
|
|
1376
|
+
}
|
|
1377
|
+
}
|
|
1378
|
+
for (const ftsEntry of input.ftsChanges ?? []) {
|
|
1379
|
+
for (const column of ftsEntry.columns) {
|
|
1380
|
+
catalog.put(structuredClone({
|
|
1381
|
+
postings: column.postings,
|
|
1382
|
+
totalTokens: column.totalTokens,
|
|
1383
|
+
}), ftsChunkKey(ftsEntry.tableId, column.columnId, manifest.version));
|
|
1384
|
+
const indexKey = ftsChunkIndexKey(ftsEntry.tableId, column.columnId);
|
|
1385
|
+
const chunkIndex = (await requestResult(catalog.get(indexKey)));
|
|
1386
|
+
catalog.put({ versions: [...(chunkIndex?.versions ?? []), manifest.version] }, indexKey);
|
|
1387
|
+
}
|
|
1388
|
+
}
|
|
1389
|
+
transactionStore.put(committed, committed.id);
|
|
1390
|
+
await transactionDone(transaction);
|
|
1391
|
+
if (uniqueKeyEntries.length > 1) {
|
|
1392
|
+
// Multi-entry commits take the merged path above; the single-table membership cache
|
|
1393
|
+
// cannot describe them, so it drops and rebuilds on the next bulk load.
|
|
1394
|
+
this.#uniqueKeyCache = undefined;
|
|
1395
|
+
}
|
|
1396
|
+
// The unique-key cache follows the same rule as the manifest cache below: advance it only
|
|
1397
|
+
// after the durable commit. Commits from this instance cannot silently change another
|
|
1398
|
+
// table's key records, so a cache for a different table just moves to the new version.
|
|
1399
|
+
if (uniqueKeyChanges !== undefined && chunkedKeys && keyCachePlan !== undefined) {
|
|
1400
|
+
if (keyCachePlan.action === "keep" && keyCacheValid) {
|
|
1401
|
+
// In-place delta on the live cache: no copies of a multi-million-token set per batch.
|
|
1402
|
+
applyChunk(keyCache.present, keyCachePlan.chunk);
|
|
1403
|
+
keyCache.chunks.push(keyCachePlan.chunk);
|
|
1404
|
+
keyCache.index = keyCachePlan.index;
|
|
1405
|
+
keyCache.version = manifest.version;
|
|
1406
|
+
}
|
|
1407
|
+
else if (keyCachePlan.action === "replace") {
|
|
1408
|
+
this.#uniqueKeyCache = {
|
|
1409
|
+
version: manifest.version,
|
|
1410
|
+
tableId: uniqueKeyChanges.tableId,
|
|
1411
|
+
mode: keyMode,
|
|
1412
|
+
present: keyCachePlan.present,
|
|
1413
|
+
chunks: keyCachePlan.chunks,
|
|
1414
|
+
index: keyCachePlan.index,
|
|
1415
|
+
};
|
|
1416
|
+
}
|
|
1417
|
+
else if (this.#uniqueKeyCache?.tableId === uniqueKeyChanges.tableId) {
|
|
1418
|
+
this.#uniqueKeyCache = undefined;
|
|
1419
|
+
}
|
|
1420
|
+
else if (this.#uniqueKeyCache !== undefined) {
|
|
1421
|
+
this.#uniqueKeyCache.version = manifest.version;
|
|
1422
|
+
}
|
|
1423
|
+
}
|
|
1424
|
+
else if (uniqueKeyChanges !== undefined &&
|
|
1425
|
+
this.#uniqueKeyCache?.tableId === uniqueKeyChanges.tableId) {
|
|
1426
|
+
// A no-op or legacy-mode write against the cached table: nothing changed on disk for a
|
|
1427
|
+
// no-op, but dropping is the safe answer for the legacy path.
|
|
1428
|
+
if (chunkedKeys)
|
|
1429
|
+
this.#uniqueKeyCache.version = manifest.version;
|
|
1430
|
+
else
|
|
1431
|
+
this.#uniqueKeyCache = undefined;
|
|
1432
|
+
}
|
|
1433
|
+
else if (this.#uniqueKeyCache !== undefined) {
|
|
1434
|
+
this.#uniqueKeyCache.version = manifest.version;
|
|
1435
|
+
}
|
|
1436
|
+
// Advance the resolved-set cache only after the durable commit succeeded. A checkpoint
|
|
1437
|
+
// commit already built the new set; a delta commit with a resolved base applies the delta
|
|
1438
|
+
// in place (the cache owns that set from here on).
|
|
1439
|
+
if (checkpointBlockIds !== undefined) {
|
|
1440
|
+
this.#manifestCache = { version: manifest.version, blockIds: checkpointBlockIds };
|
|
1441
|
+
}
|
|
1442
|
+
else if (baseBlockIdSet !== undefined || cacheOwnsBaseSet) {
|
|
1443
|
+
const blockIds = baseBlockIdSet ?? new Set();
|
|
1444
|
+
for (const id of removedBlockIds)
|
|
1445
|
+
blockIds.delete(id);
|
|
1446
|
+
for (const id of addedBlockIds)
|
|
1447
|
+
blockIds.add(id);
|
|
1448
|
+
this.#manifestCache = { version: manifest.version, blockIds };
|
|
1449
|
+
}
|
|
1450
|
+
else {
|
|
1451
|
+
// No resolved base was materialized; a later removal or checkpoint commit will resolve.
|
|
1452
|
+
this.#manifestCache = undefined;
|
|
1453
|
+
}
|
|
1454
|
+
return manifest;
|
|
1455
|
+
}
|
|
1456
|
+
async createLease(record) {
|
|
1457
|
+
validateLeaseExpiration(record.expiresAt);
|
|
1458
|
+
const transaction = this.#transaction(["leases", "manifests", "blocks"], "readwrite");
|
|
1459
|
+
try {
|
|
1460
|
+
await assertSnapshotAvailableInTransaction(transaction, record.manifestVersion);
|
|
1461
|
+
transaction.objectStore("leases").add(structuredClone(record), record.id);
|
|
1462
|
+
await transactionDone(transaction);
|
|
1463
|
+
}
|
|
1464
|
+
catch (error) {
|
|
1465
|
+
abortIfActive(transaction);
|
|
1466
|
+
await ignoreAbort(transaction);
|
|
1467
|
+
throw error;
|
|
1468
|
+
}
|
|
1469
|
+
}
|
|
1470
|
+
async getLease(id) {
|
|
1471
|
+
const transaction = this.#transaction("leases", "readonly");
|
|
1472
|
+
const value = await requestResult(transaction.objectStore("leases").get(id));
|
|
1473
|
+
await transactionDone(transaction);
|
|
1474
|
+
return value === undefined ? undefined : asLeaseRecord(value);
|
|
1475
|
+
}
|
|
1476
|
+
async listLeases() {
|
|
1477
|
+
const transaction = this.#transaction("leases", "readonly");
|
|
1478
|
+
const values = await requestResult(transaction.objectStore("leases").getAll());
|
|
1479
|
+
await transactionDone(transaction);
|
|
1480
|
+
return values.map(asLeaseRecord).sort((left, right) => left.id.localeCompare(right.id));
|
|
1481
|
+
}
|
|
1482
|
+
async renewLease(id, expectedRevision, expiresAt) {
|
|
1483
|
+
validateLeaseExpiration(expiresAt);
|
|
1484
|
+
const transaction = this.#transaction(["leases", "manifests", "blocks"], "readwrite");
|
|
1485
|
+
const store = transaction.objectStore("leases");
|
|
1486
|
+
const value = await requestResult(store.get(id));
|
|
1487
|
+
const record = value === undefined ? undefined : asLeaseRecord(value);
|
|
1488
|
+
if (record?.revision !== expectedRevision) {
|
|
1489
|
+
transaction.abort();
|
|
1490
|
+
await ignoreAbort(transaction);
|
|
1491
|
+
throw new LeaseConflictError(id, expectedRevision, record?.revision ?? null);
|
|
1492
|
+
}
|
|
1493
|
+
try {
|
|
1494
|
+
await assertSnapshotAvailableInTransaction(transaction, record.manifestVersion);
|
|
1495
|
+
const renewed = { ...record, expiresAt, revision: record.revision + 1 };
|
|
1496
|
+
store.put(renewed, id);
|
|
1497
|
+
await transactionDone(transaction);
|
|
1498
|
+
return structuredClone(renewed);
|
|
1499
|
+
}
|
|
1500
|
+
catch (error) {
|
|
1501
|
+
abortIfActive(transaction);
|
|
1502
|
+
await ignoreAbort(transaction);
|
|
1503
|
+
throw error;
|
|
1504
|
+
}
|
|
1505
|
+
}
|
|
1506
|
+
async removeLeaseIfExpired(id, expectedRevision, expiresAtCutoff) {
|
|
1507
|
+
const cutoff = Date.parse(expiresAtCutoff);
|
|
1508
|
+
if (!Number.isFinite(cutoff))
|
|
1509
|
+
throw new TypeError("Lease expiry cutoff must be valid");
|
|
1510
|
+
const transaction = this.#transaction("leases", "readwrite");
|
|
1511
|
+
const store = transaction.objectStore("leases");
|
|
1512
|
+
const value = await requestResult(store.get(id));
|
|
1513
|
+
const record = value === undefined ? undefined : asLeaseRecord(value);
|
|
1514
|
+
if (record?.revision !== expectedRevision) {
|
|
1515
|
+
transaction.abort();
|
|
1516
|
+
await ignoreAbort(transaction);
|
|
1517
|
+
throw new LeaseConflictError(id, expectedRevision, record?.revision ?? null);
|
|
1518
|
+
}
|
|
1519
|
+
const expiresAt = Date.parse(record.expiresAt);
|
|
1520
|
+
if (!Number.isFinite(expiresAt) || expiresAt > cutoff) {
|
|
1521
|
+
await transactionDone(transaction);
|
|
1522
|
+
return false;
|
|
1523
|
+
}
|
|
1524
|
+
store.delete(id);
|
|
1525
|
+
await transactionDone(transaction);
|
|
1526
|
+
return true;
|
|
1527
|
+
}
|
|
1528
|
+
async removeLease(id) {
|
|
1529
|
+
const transaction = this.#transaction("leases", "readwrite");
|
|
1530
|
+
transaction.objectStore("leases").delete(id);
|
|
1531
|
+
await transactionDone(transaction);
|
|
1532
|
+
}
|
|
1533
|
+
async createCompactionJob(record) {
|
|
1534
|
+
const normalized = normalizeCompactionJobRecord(record);
|
|
1535
|
+
const transaction = this.#transaction("gc", "readwrite");
|
|
1536
|
+
const store = transaction.objectStore("gc");
|
|
1537
|
+
const key = compactionJobKey(normalized.id);
|
|
1538
|
+
if ((await requestResult(store.getKey(key))) !== undefined) {
|
|
1539
|
+
transaction.abort();
|
|
1540
|
+
await ignoreAbort(transaction);
|
|
1541
|
+
throw new Error(`Compaction job already exists: ${normalized.id}`);
|
|
1542
|
+
}
|
|
1543
|
+
store.add(compactionJobEnvelope(normalized), key);
|
|
1544
|
+
await transactionDone(transaction);
|
|
1545
|
+
}
|
|
1546
|
+
async getCompactionJob(id) {
|
|
1547
|
+
const transaction = this.#transaction("gc", "readonly");
|
|
1548
|
+
const value = await requestResult(transaction.objectStore("gc").get(compactionJobKey(id)));
|
|
1549
|
+
await transactionDone(transaction);
|
|
1550
|
+
return value === undefined ? undefined : asCompactionJobEnvelope(value);
|
|
1551
|
+
}
|
|
1552
|
+
async listCompactionJobs(tableId) {
|
|
1553
|
+
const transaction = this.#transaction("gc", "readonly");
|
|
1554
|
+
const values = await requestResult(transaction.objectStore("gc").getAll());
|
|
1555
|
+
await transactionDone(transaction);
|
|
1556
|
+
return values
|
|
1557
|
+
.filter(isCompactionJobEnvelope)
|
|
1558
|
+
.map(asCompactionJobEnvelope)
|
|
1559
|
+
.filter((record) => tableId === undefined || record.tableId === tableId)
|
|
1560
|
+
.sort((left, right) => left.createdAt.localeCompare(right.createdAt) || left.id.localeCompare(right.id));
|
|
1561
|
+
}
|
|
1562
|
+
async listCompactionJobPage(afterId, limit) {
|
|
1563
|
+
validatePageLimit(limit);
|
|
1564
|
+
const transaction = this.#transaction("gc", "readonly");
|
|
1565
|
+
const records = await readCursorPage(transaction.objectStore("gc"), limit, (value) => asCompactionJobEnvelope(value), (key) => typeof key === "string" &&
|
|
1566
|
+
key.startsWith(COMPACTION_JOB_KEY_PREFIX) &&
|
|
1567
|
+
(afterId === null || key > compactionJobKey(afterId)));
|
|
1568
|
+
await transactionDone(transaction);
|
|
1569
|
+
return { records, nextCursor: records.length === limit ? (records.at(-1)?.id ?? null) : null };
|
|
1570
|
+
}
|
|
1571
|
+
async updateCompactionJob(id, expectedRevision, update) {
|
|
1572
|
+
if (update.state === "cancelled") {
|
|
1573
|
+
throw new TypeError("Use cancelCompactionJob to cancel a compaction job");
|
|
1574
|
+
}
|
|
1575
|
+
const transaction = this.#transaction("gc", "readwrite");
|
|
1576
|
+
const store = transaction.objectStore("gc");
|
|
1577
|
+
const key = compactionJobKey(id);
|
|
1578
|
+
const value = await requestResult(store.get(key));
|
|
1579
|
+
const current = value === undefined ? undefined : asCompactionJobEnvelope(value);
|
|
1580
|
+
if (current?.revision !== expectedRevision) {
|
|
1581
|
+
transaction.abort();
|
|
1582
|
+
await ignoreAbort(transaction);
|
|
1583
|
+
throw new CompactionJobConflictError(id, expectedRevision, current?.revision ?? null);
|
|
1584
|
+
}
|
|
1585
|
+
const updated = updateCompactionJobRecord(current, update);
|
|
1586
|
+
store.put(compactionJobEnvelope(updated), key);
|
|
1587
|
+
await transactionDone(transaction);
|
|
1588
|
+
return structuredClone(updated);
|
|
1589
|
+
}
|
|
1590
|
+
async cancelCompactionJob(id, expectedRevision, cancelledAt) {
|
|
1591
|
+
const transaction = this.#transaction(["gc", "transactions"], "readwrite");
|
|
1592
|
+
const jobStore = transaction.objectStore("gc");
|
|
1593
|
+
const transactionStore = transaction.objectStore("transactions");
|
|
1594
|
+
const key = compactionJobKey(id);
|
|
1595
|
+
const value = await requestResult(jobStore.get(key));
|
|
1596
|
+
const current = value === undefined ? undefined : asCompactionJobEnvelope(value);
|
|
1597
|
+
if (current?.revision !== expectedRevision) {
|
|
1598
|
+
transaction.abort();
|
|
1599
|
+
await ignoreAbort(transaction);
|
|
1600
|
+
throw new CompactionJobConflictError(id, expectedRevision, current?.revision ?? null);
|
|
1601
|
+
}
|
|
1602
|
+
if (isTerminalCompactionJob(current)) {
|
|
1603
|
+
await transactionDone(transaction);
|
|
1604
|
+
return structuredClone(current);
|
|
1605
|
+
}
|
|
1606
|
+
const transactionValue = current.transactionId === null
|
|
1607
|
+
? undefined
|
|
1608
|
+
: await requestResult(transactionStore.get(current.transactionId));
|
|
1609
|
+
const linkedTransaction = transactionValue === undefined ? undefined : asTransactionRecord(transactionValue);
|
|
1610
|
+
let updated;
|
|
1611
|
+
try {
|
|
1612
|
+
if (linkedTransaction?.status === "committed") {
|
|
1613
|
+
if (linkedTransaction.committedVersion === null) {
|
|
1614
|
+
throw new Error(`Committed transaction has no manifest version: ${linkedTransaction.id}`);
|
|
1615
|
+
}
|
|
1616
|
+
updated = updateCompactionJobRecord(current, {
|
|
1617
|
+
state: "published",
|
|
1618
|
+
publishedVersion: linkedTransaction.committedVersion,
|
|
1619
|
+
updatedAt: cancelledAt,
|
|
1620
|
+
error: null,
|
|
1621
|
+
});
|
|
1622
|
+
}
|
|
1623
|
+
else {
|
|
1624
|
+
updated = updateCompactionJobRecord(current, {
|
|
1625
|
+
state: "cancelled",
|
|
1626
|
+
updatedAt: cancelledAt,
|
|
1627
|
+
error: null,
|
|
1628
|
+
});
|
|
1629
|
+
if (linkedTransaction?.status === "active") {
|
|
1630
|
+
transactionStore.put(updateTransactionRecord(linkedTransaction, {
|
|
1631
|
+
status: "aborted",
|
|
1632
|
+
updatedAt: cancelledAt,
|
|
1633
|
+
committedVersion: null,
|
|
1634
|
+
}), linkedTransaction.id);
|
|
1635
|
+
}
|
|
1636
|
+
}
|
|
1637
|
+
}
|
|
1638
|
+
catch (error) {
|
|
1639
|
+
transaction.abort();
|
|
1640
|
+
await ignoreAbort(transaction);
|
|
1641
|
+
throw error;
|
|
1642
|
+
}
|
|
1643
|
+
jobStore.put(compactionJobEnvelope(updated), key);
|
|
1644
|
+
await transactionDone(transaction);
|
|
1645
|
+
return structuredClone(updated);
|
|
1646
|
+
}
|
|
1647
|
+
async removeCompactionJob(id) {
|
|
1648
|
+
const transaction = this.#transaction("gc", "readwrite");
|
|
1649
|
+
transaction.objectStore("gc").delete(compactionJobKey(id));
|
|
1650
|
+
await transactionDone(transaction);
|
|
1651
|
+
}
|
|
1652
|
+
async createGarbageCollectionJob(input) {
|
|
1653
|
+
const record = createGarbageCollectionJobRecord(input);
|
|
1654
|
+
const transaction = this.#transaction(["gc", "manifests", "segments", "transactions"], "readwrite");
|
|
1655
|
+
const gcStore = transaction.objectStore("gc");
|
|
1656
|
+
const key = garbageCollectionJobKey(record.id);
|
|
1657
|
+
try {
|
|
1658
|
+
if ((await requestResult(gcStore.getKey(key))) !== undefined) {
|
|
1659
|
+
throw new Error(`Garbage collection job already exists: ${record.id}`);
|
|
1660
|
+
}
|
|
1661
|
+
await assertGarbageCollectionCandidateProvenanceInTransaction(transaction, record);
|
|
1662
|
+
gcStore.add(garbageCollectionJobEnvelope(record), key);
|
|
1663
|
+
await transactionDone(transaction);
|
|
1664
|
+
return structuredClone(record);
|
|
1665
|
+
}
|
|
1666
|
+
catch (error) {
|
|
1667
|
+
abortIfActive(transaction);
|
|
1668
|
+
await ignoreAbort(transaction);
|
|
1669
|
+
throw error;
|
|
1670
|
+
}
|
|
1671
|
+
}
|
|
1672
|
+
async getGarbageCollectionJob(id) {
|
|
1673
|
+
const transaction = this.#transaction("gc", "readonly");
|
|
1674
|
+
const value = await requestResult(transaction.objectStore("gc").get(garbageCollectionJobKey(id)));
|
|
1675
|
+
await transactionDone(transaction);
|
|
1676
|
+
return value === undefined ? undefined : asGarbageCollectionJobEnvelope(value);
|
|
1677
|
+
}
|
|
1678
|
+
async listGarbageCollectionJobs() {
|
|
1679
|
+
const transaction = this.#transaction("gc", "readonly");
|
|
1680
|
+
const values = await requestResult(transaction.objectStore("gc").getAll());
|
|
1681
|
+
await transactionDone(transaction);
|
|
1682
|
+
return values
|
|
1683
|
+
.filter(isGarbageCollectionJobEnvelope)
|
|
1684
|
+
.map(asGarbageCollectionJobEnvelope)
|
|
1685
|
+
.sort((left, right) => left.createdAt.localeCompare(right.createdAt) || left.id.localeCompare(right.id));
|
|
1686
|
+
}
|
|
1687
|
+
async runGarbageCollectionStep(input) {
|
|
1688
|
+
validateGarbageCollectionStepInput(input);
|
|
1689
|
+
const transaction = this.#transaction(["gc", "blocks", "segments", "catalog", "manifests", "transactions", "leases"], "readwrite");
|
|
1690
|
+
const gcStore = transaction.objectStore("gc");
|
|
1691
|
+
const key = garbageCollectionJobKey(input.jobId);
|
|
1692
|
+
try {
|
|
1693
|
+
const value = await requestResult(gcStore.get(key));
|
|
1694
|
+
const current = value === undefined ? undefined : asGarbageCollectionJobEnvelope(value);
|
|
1695
|
+
if (current?.revision !== input.expectedRevision) {
|
|
1696
|
+
throw new GarbageCollectionJobConflictError(input.jobId, input.expectedRevision, current?.revision ?? null);
|
|
1697
|
+
}
|
|
1698
|
+
if (current.state === "completed") {
|
|
1699
|
+
await transactionDone(transaction);
|
|
1700
|
+
return emptyGarbageCollectionStep(current);
|
|
1701
|
+
}
|
|
1702
|
+
const catalog = transaction.objectStore("catalog");
|
|
1703
|
+
const manifestStore = transaction.objectStore("manifests");
|
|
1704
|
+
const segmentStore = transaction.objectStore("segments");
|
|
1705
|
+
const blockStore = transaction.objectStore("blocks");
|
|
1706
|
+
const currentVersionValue = await requestResult(catalog.get(CURRENT_MANIFEST_KEY));
|
|
1707
|
+
if (currentVersionValue !== undefined &&
|
|
1708
|
+
(typeof currentVersionValue !== "number" ||
|
|
1709
|
+
!Number.isSafeInteger(currentVersionValue) ||
|
|
1710
|
+
currentVersionValue < 0)) {
|
|
1711
|
+
throw new Error("Current manifest version is invalid");
|
|
1712
|
+
}
|
|
1713
|
+
const currentVersion = currentVersionValue ?? null;
|
|
1714
|
+
const leaseCutoff = Date.parse(current.leaseCutoff);
|
|
1715
|
+
await assertGarbageCollectionPinsAvailableInTransaction(transaction, currentVersion, leaseCutoff);
|
|
1716
|
+
const prunedManifestVersions = [];
|
|
1717
|
+
const alreadyPrunedManifestVersions = [];
|
|
1718
|
+
const retainedManifestVersions = [];
|
|
1719
|
+
const missingManifestVersions = [];
|
|
1720
|
+
const reclaimedSegmentIds = [];
|
|
1721
|
+
const retainedSegmentIds = [];
|
|
1722
|
+
const missingSegmentIds = [];
|
|
1723
|
+
const reclaimedBlockIds = [];
|
|
1724
|
+
const retainedBlockIds = [];
|
|
1725
|
+
const missingBlockIds = [];
|
|
1726
|
+
let reclaimedBlockBytes = 0;
|
|
1727
|
+
let remaining = input.maxItems;
|
|
1728
|
+
let manifestIndex = current.cursor.manifestIndex;
|
|
1729
|
+
while (remaining > 0 && manifestIndex < current.candidateManifestVersions.length) {
|
|
1730
|
+
const version = current.candidateManifestVersions[manifestIndex];
|
|
1731
|
+
if (version === undefined)
|
|
1732
|
+
throw new Error("Garbage collection manifest cursor is invalid");
|
|
1733
|
+
const manifestValue = await requestResult(manifestStore.get(version));
|
|
1734
|
+
if (manifestValue === undefined)
|
|
1735
|
+
missingManifestVersions.push(version);
|
|
1736
|
+
else if (asStoredManifestRecord(manifestValue).prunedAt !== undefined) {
|
|
1737
|
+
alreadyPrunedManifestVersions.push(version);
|
|
1738
|
+
}
|
|
1739
|
+
else if (await isManifestVersionPinnedInTransaction(transaction, version, currentVersion, leaseCutoff))
|
|
1740
|
+
retainedManifestVersions.push(version);
|
|
1741
|
+
else {
|
|
1742
|
+
// The tombstone keeps the record's full content (checkpoint list or delta) so chains
|
|
1743
|
+
// above it keep resolving; it only stops counting as a reachability root.
|
|
1744
|
+
manifestStore.put({ ...asStoredManifestRecord(manifestValue), prunedAt: input.updatedAt }, version);
|
|
1745
|
+
prunedManifestVersions.push(version);
|
|
1746
|
+
}
|
|
1747
|
+
manifestIndex += 1;
|
|
1748
|
+
remaining -= 1;
|
|
1749
|
+
}
|
|
1750
|
+
const prunedManifestVersionSet = new Set(prunedManifestVersions);
|
|
1751
|
+
await assertRemainingManifestRecordsAvailable(transaction, prunedManifestVersionSet);
|
|
1752
|
+
let segmentIndex = current.cursor.segmentIndex;
|
|
1753
|
+
const segmentIdsToExamine = manifestIndex === current.candidateManifestVersions.length
|
|
1754
|
+
? current.candidateSegmentIds.slice(segmentIndex, segmentIndex + remaining)
|
|
1755
|
+
: [];
|
|
1756
|
+
const blockCapacity = Math.max(0, remaining - segmentIdsToExamine.length);
|
|
1757
|
+
const blockIdsToExamine = manifestIndex === current.candidateManifestVersions.length &&
|
|
1758
|
+
segmentIndex + segmentIdsToExamine.length === current.candidateSegmentIds.length
|
|
1759
|
+
? current.candidateBlockIds.slice(current.cursor.blockIndex, current.cursor.blockIndex + blockCapacity)
|
|
1760
|
+
: [];
|
|
1761
|
+
const roots = await collectBoundedPhysicalRootsInTransaction(transaction, segmentIdsToExamine, blockIdsToExamine, prunedManifestVersionSet);
|
|
1762
|
+
while (remaining > 0 &&
|
|
1763
|
+
manifestIndex === current.candidateManifestVersions.length &&
|
|
1764
|
+
segmentIndex < current.candidateSegmentIds.length) {
|
|
1765
|
+
const id = current.candidateSegmentIds[segmentIndex];
|
|
1766
|
+
if (id === undefined)
|
|
1767
|
+
throw new Error("Garbage collection segment cursor is invalid");
|
|
1768
|
+
const segmentValue = await requestResult(segmentStore.get(id));
|
|
1769
|
+
if (segmentValue === undefined)
|
|
1770
|
+
missingSegmentIds.push(id);
|
|
1771
|
+
else if (roots.segmentIds.has(id))
|
|
1772
|
+
retainedSegmentIds.push(id);
|
|
1773
|
+
else {
|
|
1774
|
+
segmentStore.delete(id);
|
|
1775
|
+
reclaimedSegmentIds.push(id);
|
|
1776
|
+
}
|
|
1777
|
+
segmentIndex += 1;
|
|
1778
|
+
remaining -= 1;
|
|
1779
|
+
}
|
|
1780
|
+
let blockIndex = current.cursor.blockIndex;
|
|
1781
|
+
while (remaining > 0 &&
|
|
1782
|
+
manifestIndex === current.candidateManifestVersions.length &&
|
|
1783
|
+
segmentIndex === current.candidateSegmentIds.length &&
|
|
1784
|
+
blockIndex < current.candidateBlockIds.length) {
|
|
1785
|
+
const id = current.candidateBlockIds[blockIndex];
|
|
1786
|
+
if (id === undefined)
|
|
1787
|
+
throw new Error("Garbage collection block cursor is invalid");
|
|
1788
|
+
const blockValue = await requestResult(blockStore.get(id));
|
|
1789
|
+
if (blockValue === undefined)
|
|
1790
|
+
missingBlockIds.push(id);
|
|
1791
|
+
else if (roots.blockIds.has(id))
|
|
1792
|
+
retainedBlockIds.push(id);
|
|
1793
|
+
else {
|
|
1794
|
+
const bytes = asBytes(blockValue);
|
|
1795
|
+
blockStore.delete(id);
|
|
1796
|
+
reclaimedBlockIds.push(id);
|
|
1797
|
+
reclaimedBlockBytes = safeStorageSum(reclaimedBlockBytes, bytes.byteLength);
|
|
1798
|
+
}
|
|
1799
|
+
blockIndex += 1;
|
|
1800
|
+
remaining -= 1;
|
|
1801
|
+
}
|
|
1802
|
+
const updated = advanceGarbageCollectionJobRecord(current, {
|
|
1803
|
+
examinedManifestCount: prunedManifestVersions.length +
|
|
1804
|
+
alreadyPrunedManifestVersions.length +
|
|
1805
|
+
retainedManifestVersions.length +
|
|
1806
|
+
missingManifestVersions.length,
|
|
1807
|
+
prunedManifestCount: prunedManifestVersions.length,
|
|
1808
|
+
alreadyPrunedManifestCount: alreadyPrunedManifestVersions.length,
|
|
1809
|
+
retainedManifestCount: retainedManifestVersions.length,
|
|
1810
|
+
missingManifestCount: missingManifestVersions.length,
|
|
1811
|
+
examinedSegmentCount: reclaimedSegmentIds.length + retainedSegmentIds.length + missingSegmentIds.length,
|
|
1812
|
+
reclaimedSegmentCount: reclaimedSegmentIds.length,
|
|
1813
|
+
retainedSegmentCount: retainedSegmentIds.length,
|
|
1814
|
+
missingSegmentCount: missingSegmentIds.length,
|
|
1815
|
+
examinedBlockCount: reclaimedBlockIds.length + retainedBlockIds.length + missingBlockIds.length,
|
|
1816
|
+
reclaimedBlockCount: reclaimedBlockIds.length,
|
|
1817
|
+
retainedBlockCount: retainedBlockIds.length,
|
|
1818
|
+
missingBlockCount: missingBlockIds.length,
|
|
1819
|
+
reclaimedBlockBytes,
|
|
1820
|
+
updatedAt: input.updatedAt,
|
|
1821
|
+
});
|
|
1822
|
+
gcStore.put(garbageCollectionJobEnvelope(updated), key);
|
|
1823
|
+
await transactionDone(transaction);
|
|
1824
|
+
return {
|
|
1825
|
+
job: structuredClone(updated),
|
|
1826
|
+
prunedManifestVersions,
|
|
1827
|
+
alreadyPrunedManifestVersions,
|
|
1828
|
+
retainedManifestVersions,
|
|
1829
|
+
missingManifestVersions,
|
|
1830
|
+
reclaimedSegmentIds,
|
|
1831
|
+
retainedSegmentIds,
|
|
1832
|
+
missingSegmentIds,
|
|
1833
|
+
reclaimedBlockIds,
|
|
1834
|
+
retainedBlockIds,
|
|
1835
|
+
missingBlockIds,
|
|
1836
|
+
reclaimedBlockBytes,
|
|
1837
|
+
};
|
|
1838
|
+
}
|
|
1839
|
+
catch (error) {
|
|
1840
|
+
abortIfActive(transaction);
|
|
1841
|
+
await ignoreAbort(transaction);
|
|
1842
|
+
throw error;
|
|
1843
|
+
}
|
|
1844
|
+
}
|
|
1845
|
+
async removeGarbageCollectionJob(id) {
|
|
1846
|
+
const transaction = this.#transaction("gc", "readwrite");
|
|
1847
|
+
transaction.objectStore("gc").delete(garbageCollectionJobKey(id));
|
|
1848
|
+
await transactionDone(transaction);
|
|
1849
|
+
}
|
|
1850
|
+
async getLogicalStorageBytes() {
|
|
1851
|
+
const transaction = this.#transaction([...storeNames], "readonly");
|
|
1852
|
+
let total = 0;
|
|
1853
|
+
for (const storeName of storeNames) {
|
|
1854
|
+
total += await logicalObjectStoreBytes(transaction.objectStore(storeName));
|
|
1855
|
+
}
|
|
1856
|
+
await transactionDone(transaction);
|
|
1857
|
+
return total;
|
|
1858
|
+
}
|
|
1859
|
+
/**
|
|
1860
|
+
* Copies the current version out as a portable snapshot. See `./snapshot.ts` for what a
|
|
1861
|
+
* snapshot carries and what it deliberately leaves behind.
|
|
1862
|
+
*
|
|
1863
|
+
* This reads across several transactions rather than one, so a concurrent commit could move
|
|
1864
|
+
* the version underneath it. Hold a `backup` lease (see `../transactions/index.ts`) around
|
|
1865
|
+
* the call when that is possible; the offline generators that produce published datasets are
|
|
1866
|
+
* the only writer in their process and do not need one.
|
|
1867
|
+
*/
|
|
1868
|
+
async exportSnapshot() {
|
|
1869
|
+
const manifest = await this.getCurrentManifest();
|
|
1870
|
+
if (manifest === undefined)
|
|
1871
|
+
throw new Error("There is no committed version to snapshot");
|
|
1872
|
+
const version = manifest.version;
|
|
1873
|
+
const blocks = [];
|
|
1874
|
+
for (let start = 0; start < manifest.blockIds.length; start += SNAPSHOT_BLOCK_BATCH) {
|
|
1875
|
+
const ids = manifest.blockIds.slice(start, start + SNAPSHOT_BLOCK_BATCH);
|
|
1876
|
+
const values = await this.getBlocks(ids);
|
|
1877
|
+
values.forEach((bytes, index) => {
|
|
1878
|
+
const id = ids[index] ?? "";
|
|
1879
|
+
if (bytes === undefined)
|
|
1880
|
+
throw new Error(`Manifest references missing block: ${id}`);
|
|
1881
|
+
blocks.push({ id, bytes });
|
|
1882
|
+
});
|
|
1883
|
+
}
|
|
1884
|
+
const { segments, transactions } = selectLiveRecords({
|
|
1885
|
+
liveBlockIds: new Set(manifest.blockIds),
|
|
1886
|
+
segments: await this.listSegments(),
|
|
1887
|
+
transactions: await this.listTransactions(),
|
|
1888
|
+
version,
|
|
1889
|
+
});
|
|
1890
|
+
const tables = [];
|
|
1891
|
+
for (const record of await this.listTables()) {
|
|
1892
|
+
const transaction = this.#transaction("catalog", "readonly");
|
|
1893
|
+
const catalog = transaction.objectStore("catalog");
|
|
1894
|
+
const [rowIdValue, ...autoIncrementValues] = await Promise.all([
|
|
1895
|
+
requestResult(catalog.get(`${ROW_ID_PREFIX}${record.id}`)),
|
|
1896
|
+
...record.columns.map((column) => requestResult(catalog.get(`${AUTO_INCREMENT_PREFIX}${record.id}/${column.id}`))),
|
|
1897
|
+
]);
|
|
1898
|
+
const uniqueKeyTokens = record.uniqueKeyColumnId === undefined
|
|
1899
|
+
? []
|
|
1900
|
+
: [...(await readAllUniqueKeyTokens(catalog, record.id, record.uniqueKeyStorage))];
|
|
1901
|
+
const fts = [];
|
|
1902
|
+
const ftsColumns = { ...(record.ftsColumns ?? {}) };
|
|
1903
|
+
for (const [columnId, state] of Object.entries(ftsColumns)) {
|
|
1904
|
+
const toc = (await requestResult(catalog.get(`${FTS_BASE_INDEX_PREFIX}${record.id}/${columnId}`)));
|
|
1905
|
+
if (state.state !== "ready" || toc?.coversVersion !== version) {
|
|
1906
|
+
// Anything not already covering this exact version restores as a rebuild. The index
|
|
1907
|
+
// is a pruning accelerator that the scan re-verifies, so that costs speed, not truth.
|
|
1908
|
+
ftsColumns[columnId] = { ...state, state: "invalid" };
|
|
1909
|
+
continue;
|
|
1910
|
+
}
|
|
1911
|
+
const prefix = `${FTS_BASE_PREFIX}${record.id}/${columnId}/`;
|
|
1912
|
+
const chunks = (await Promise.all(toc.boundaries.map((_, ordinal) => requestResult(catalog.get(`${prefix}${String(ordinal).padStart(6, "0")}`)))));
|
|
1913
|
+
fts.push({
|
|
1914
|
+
columnId,
|
|
1915
|
+
coversVersion: toc.coversVersion,
|
|
1916
|
+
totalTokens: toc.totalTokens,
|
|
1917
|
+
chunks: chunks.map((chunk) => chunk ?? []),
|
|
1918
|
+
});
|
|
1919
|
+
}
|
|
1920
|
+
await transactionDone(transaction);
|
|
1921
|
+
tables.push({
|
|
1922
|
+
record: {
|
|
1923
|
+
...record,
|
|
1924
|
+
...(Object.keys(ftsColumns).length === 0 ? {} : { ftsColumns }),
|
|
1925
|
+
},
|
|
1926
|
+
nextRowId: rowIdValue ?? 1n,
|
|
1927
|
+
autoIncrement: record.columns.flatMap((column, index) => {
|
|
1928
|
+
const next = autoIncrementValues[index];
|
|
1929
|
+
return next === undefined ? [] : [{ columnId: column.id, next }];
|
|
1930
|
+
}),
|
|
1931
|
+
uniqueKeyTokens,
|
|
1932
|
+
fts,
|
|
1933
|
+
});
|
|
1934
|
+
}
|
|
1935
|
+
return { version, createdAt: new Date().toISOString(), tables, segments, transactions, blocks };
|
|
1936
|
+
}
|
|
1937
|
+
/**
|
|
1938
|
+
* Loads a snapshot into this store, which must be empty. Block payloads go in batched
|
|
1939
|
+
* transactions so a large dataset does not build one enormous write, and the current-version
|
|
1940
|
+
* pointer is written last: until it lands, an interrupted load leaves a store that still reads
|
|
1941
|
+
* as empty rather than as half a database.
|
|
1942
|
+
*/
|
|
1943
|
+
async importSnapshot(snapshot, options = {}) {
|
|
1944
|
+
const existing = await this.getCurrentManifestVersion();
|
|
1945
|
+
if (existing !== null)
|
|
1946
|
+
throw new Error("This store already holds a database");
|
|
1947
|
+
if ((await this.listTables()).length > 0) {
|
|
1948
|
+
throw new Error("This store already holds a catalog");
|
|
1949
|
+
}
|
|
1950
|
+
const totalBytes = snapshot.blocks.reduce((total, block) => total + block.bytes.byteLength, 0);
|
|
1951
|
+
let writtenBytes = 0;
|
|
1952
|
+
options.onProgress?.({ phase: "blocks", writtenBytes, totalBytes });
|
|
1953
|
+
let batch = [];
|
|
1954
|
+
let batchBytes = 0;
|
|
1955
|
+
const flush = async () => {
|
|
1956
|
+
if (batch.length === 0)
|
|
1957
|
+
return;
|
|
1958
|
+
await this.addBlocks(batch);
|
|
1959
|
+
writtenBytes += batchBytes;
|
|
1960
|
+
options.onProgress?.({ phase: "blocks", writtenBytes, totalBytes });
|
|
1961
|
+
batch = [];
|
|
1962
|
+
batchBytes = 0;
|
|
1963
|
+
};
|
|
1964
|
+
for (const block of snapshot.blocks) {
|
|
1965
|
+
batch.push(block);
|
|
1966
|
+
batchBytes += block.bytes.byteLength;
|
|
1967
|
+
if (batchBytes >= SNAPSHOT_BATCH_BYTES)
|
|
1968
|
+
await flush();
|
|
1969
|
+
}
|
|
1970
|
+
await flush();
|
|
1971
|
+
options.onProgress?.({ phase: "catalog", writtenBytes, totalBytes });
|
|
1972
|
+
for (const table of snapshot.tables)
|
|
1973
|
+
await this.#importSnapshotTable(table, snapshot.version);
|
|
1974
|
+
const transaction = this.#transaction(["manifests", "segments", "transactions"], "readwrite");
|
|
1975
|
+
const manifests = transaction.objectStore("manifests");
|
|
1976
|
+
const segments = transaction.objectStore("segments");
|
|
1977
|
+
const transactions = transaction.objectStore("transactions");
|
|
1978
|
+
// One checkpoint carrying the complete block list, so nothing resolves through a delta
|
|
1979
|
+
// chain that no longer exists.
|
|
1980
|
+
const record = {
|
|
1981
|
+
version: snapshot.version,
|
|
1982
|
+
previousVersion: null,
|
|
1983
|
+
blockIds: snapshot.blocks.map((block) => block.id),
|
|
1984
|
+
createdAt: snapshot.createdAt,
|
|
1985
|
+
deltaDepth: 0,
|
|
1986
|
+
};
|
|
1987
|
+
manifests.put(record, snapshot.version);
|
|
1988
|
+
for (const segment of snapshot.segments) {
|
|
1989
|
+
segments.put(normalizeSegmentRecord(segment), segment.id);
|
|
1990
|
+
}
|
|
1991
|
+
for (const entry of snapshot.transactions)
|
|
1992
|
+
transactions.put(structuredClone(entry), entry.id);
|
|
1993
|
+
await transactionDone(transaction);
|
|
1994
|
+
const pointer = this.#transaction("catalog", "readwrite");
|
|
1995
|
+
const catalog = pointer.objectStore("catalog");
|
|
1996
|
+
catalog.put(snapshot.version, CURRENT_MANIFEST_KEY);
|
|
1997
|
+
await bumpCatalogEpoch(catalog);
|
|
1998
|
+
await transactionDone(pointer);
|
|
1999
|
+
options.onProgress?.({ phase: "done", writtenBytes, totalBytes });
|
|
2000
|
+
}
|
|
2001
|
+
async #importSnapshotTable(table, version) {
|
|
2002
|
+
const keyed = table.record.uniqueKeyColumnId !== undefined;
|
|
2003
|
+
// The snapshot carries logical membership, not a storage layout, so it always loads as the
|
|
2004
|
+
// current one. Writing the base pre-folded is what keeps it that way: replaying the tokens
|
|
2005
|
+
// as commit chunks instead would leave a tail long enough to make the first write crawl.
|
|
2006
|
+
const record = {
|
|
2007
|
+
...table.record,
|
|
2008
|
+
...(keyed ? { uniqueKeyLookupReady: true, uniqueKeyStorage: "chunks-v2" } : {}),
|
|
2009
|
+
};
|
|
2010
|
+
const transaction = this.#transaction("catalog", "readwrite");
|
|
2011
|
+
const catalog = transaction.objectStore("catalog");
|
|
2012
|
+
catalog.put(structuredClone(record), `${TABLE_ID_PREFIX}${record.id}`);
|
|
2013
|
+
catalog.put(record.id, `${TABLE_NAME_PREFIX}${record.name}`);
|
|
2014
|
+
catalog.put(table.nextRowId, `${ROW_ID_PREFIX}${record.id}`);
|
|
2015
|
+
for (const entry of table.autoIncrement) {
|
|
2016
|
+
catalog.put(entry.next, `${AUTO_INCREMENT_PREFIX}${record.id}/${entry.columnId}`);
|
|
2017
|
+
}
|
|
2018
|
+
if (keyed) {
|
|
2019
|
+
const partitions = Math.max(1, Math.ceil(table.uniqueKeyTokens.length / UNIQUE_KEY_PARTITION_TARGET));
|
|
2020
|
+
const parts = Array.from({ length: partitions }, () => []);
|
|
2021
|
+
for (const token of table.uniqueKeyTokens)
|
|
2022
|
+
parts[fnv1a(token) % partitions]?.push(token);
|
|
2023
|
+
parts.forEach((tokens, ordinal) => {
|
|
2024
|
+
catalog.put(tokens, uniqueKeyBasePartKey(record.id, ordinal));
|
|
2025
|
+
});
|
|
2026
|
+
const index = {
|
|
2027
|
+
versions: [],
|
|
2028
|
+
hasBase: true,
|
|
2029
|
+
partitions,
|
|
2030
|
+
tokenCount: table.uniqueKeyTokens.length,
|
|
2031
|
+
};
|
|
2032
|
+
catalog.put(index, uniqueKeyChunkIndexKey(record.id));
|
|
2033
|
+
}
|
|
2034
|
+
for (const entry of table.fts) {
|
|
2035
|
+
const prefix = `${FTS_BASE_PREFIX}${record.id}/${entry.columnId}/`;
|
|
2036
|
+
const boundaries = entry.chunks.map((chunk) => ({
|
|
2037
|
+
first: chunk[0]?.term ?? "",
|
|
2038
|
+
last: chunk[chunk.length - 1]?.term ?? "",
|
|
2039
|
+
}));
|
|
2040
|
+
entry.chunks.forEach((chunk, ordinal) => {
|
|
2041
|
+
catalog.put(structuredClone(chunk), `${prefix}${String(ordinal).padStart(6, "0")}`);
|
|
2042
|
+
});
|
|
2043
|
+
catalog.put({ coversVersion: version, boundaries, totalTokens: entry.totalTokens }, `${FTS_BASE_INDEX_PREFIX}${record.id}/${entry.columnId}`);
|
|
2044
|
+
catalog.put({ versions: [] }, ftsChunkIndexKey(record.id, entry.columnId));
|
|
2045
|
+
}
|
|
2046
|
+
await transactionDone(transaction);
|
|
2047
|
+
}
|
|
2048
|
+
close() {
|
|
2049
|
+
this.#db.close();
|
|
2050
|
+
}
|
|
2051
|
+
#transaction(stores, mode) {
|
|
2052
|
+
return this.#db.transaction(stores, mode, { durability: this.#durability });
|
|
2053
|
+
}
|
|
2054
|
+
}
|
|
2055
|
+
function requestResult(request) {
|
|
2056
|
+
return new Promise((resolve, reject) => {
|
|
2057
|
+
request.addEventListener("success", () => resolve(request.result), { once: true });
|
|
2058
|
+
request.addEventListener("error", () => reject(request.error ?? new Error("IndexedDB request failed")), {
|
|
2059
|
+
once: true,
|
|
2060
|
+
});
|
|
2061
|
+
});
|
|
2062
|
+
}
|
|
2063
|
+
function readCursorPage(store, limit, decode, acceptKey) {
|
|
2064
|
+
return new Promise((resolve, reject) => {
|
|
2065
|
+
const records = [];
|
|
2066
|
+
const request = store.openCursor();
|
|
2067
|
+
request.onerror = () => reject(request.error ?? new Error("IndexedDB cursor failed"));
|
|
2068
|
+
request.onsuccess = () => {
|
|
2069
|
+
const cursor = request.result;
|
|
2070
|
+
if (cursor === null || records.length === limit) {
|
|
2071
|
+
resolve(records);
|
|
2072
|
+
return;
|
|
2073
|
+
}
|
|
2074
|
+
try {
|
|
2075
|
+
if (acceptKey(cursor.key))
|
|
2076
|
+
records.push(decode(cursor.value));
|
|
2077
|
+
if (records.length === limit)
|
|
2078
|
+
resolve(records);
|
|
2079
|
+
else
|
|
2080
|
+
cursor.continue();
|
|
2081
|
+
}
|
|
2082
|
+
catch (error) {
|
|
2083
|
+
reject(error instanceof Error ? error : new Error(String(error)));
|
|
2084
|
+
}
|
|
2085
|
+
};
|
|
2086
|
+
});
|
|
2087
|
+
}
|
|
2088
|
+
/**
|
|
2089
|
+
* Advances the catalog epoch inside an already-open readwrite transaction on the catalog
|
|
2090
|
+
* store. Every catalog mutation calls this in its own transaction, so the epoch read by
|
|
2091
|
+
* `getCatalogProbe` is an atomic, monotonic proof of catalog identity: an unchanged epoch
|
|
2092
|
+
* means no table record, manifest publish, or commit has landed since the epoch was read.
|
|
2093
|
+
*/
|
|
2094
|
+
async function bumpCatalogEpoch(catalog) {
|
|
2095
|
+
const value = (await requestResult(catalog.get(CATALOG_EPOCH_KEY)));
|
|
2096
|
+
catalog.put((value ?? 0) + 1, CATALOG_EPOCH_KEY);
|
|
2097
|
+
}
|
|
2098
|
+
function visitObjectStoreSequentially(store, visit) {
|
|
2099
|
+
return new Promise((resolve, reject) => {
|
|
2100
|
+
const request = store.openCursor();
|
|
2101
|
+
request.onerror = () => reject(request.error ?? new Error("IndexedDB cursor failed"));
|
|
2102
|
+
request.onsuccess = () => {
|
|
2103
|
+
const cursor = request.result;
|
|
2104
|
+
if (cursor === null) {
|
|
2105
|
+
resolve(false);
|
|
2106
|
+
return;
|
|
2107
|
+
}
|
|
2108
|
+
let result;
|
|
2109
|
+
try {
|
|
2110
|
+
result = visit(cursor.value, cursor.key);
|
|
2111
|
+
}
|
|
2112
|
+
catch (error) {
|
|
2113
|
+
reject(error instanceof Error ? error : new Error(String(error)));
|
|
2114
|
+
return;
|
|
2115
|
+
}
|
|
2116
|
+
Promise.resolve(result).then((stop) => {
|
|
2117
|
+
if (stop === true)
|
|
2118
|
+
resolve(true);
|
|
2119
|
+
else
|
|
2120
|
+
cursor.continue();
|
|
2121
|
+
}, (error) => reject(error instanceof Error ? error : new Error(String(error))));
|
|
2122
|
+
};
|
|
2123
|
+
});
|
|
2124
|
+
}
|
|
2125
|
+
/**
|
|
2126
|
+
* Rejects if a transaction aborts or fails, and never resolves otherwise. Lets a caller that
|
|
2127
|
+
* does not wait for `complete` still surface the failure it would otherwise have missed.
|
|
2128
|
+
*/
|
|
2129
|
+
function transactionFailure(transaction) {
|
|
2130
|
+
return new Promise((_resolve, reject) => {
|
|
2131
|
+
transaction.addEventListener("abort", () => reject(transaction.error ?? new Error("Transaction aborted")), { once: true });
|
|
2132
|
+
transaction.addEventListener("error", () => reject(transaction.error ?? new Error("Transaction failed")), { once: true });
|
|
2133
|
+
});
|
|
2134
|
+
}
|
|
2135
|
+
function transactionDone(transaction) {
|
|
2136
|
+
return new Promise((resolve, reject) => {
|
|
2137
|
+
transaction.addEventListener("complete", () => resolve(), { once: true });
|
|
2138
|
+
transaction.addEventListener("abort", () => reject(transaction.error ?? new Error("Transaction aborted")), {
|
|
2139
|
+
once: true,
|
|
2140
|
+
});
|
|
2141
|
+
transaction.addEventListener("error", () => reject(transaction.error ?? new Error("Transaction failed")), {
|
|
2142
|
+
once: true,
|
|
2143
|
+
});
|
|
2144
|
+
});
|
|
2145
|
+
}
|
|
2146
|
+
async function ignoreAbort(transaction) {
|
|
2147
|
+
try {
|
|
2148
|
+
await transactionDone(transaction);
|
|
2149
|
+
}
|
|
2150
|
+
catch {
|
|
2151
|
+
// The caller reports the more useful domain error.
|
|
2152
|
+
}
|
|
2153
|
+
}
|
|
2154
|
+
function asBytes(value) {
|
|
2155
|
+
// Every IndexedDB get deserializes a fresh, unshared value, so the bytes return without a
|
|
2156
|
+
// defensive copy; wrapping an ArrayBuffer in a view is also zero-copy.
|
|
2157
|
+
if (value instanceof Uint8Array)
|
|
2158
|
+
return value;
|
|
2159
|
+
if (value instanceof ArrayBuffer)
|
|
2160
|
+
return new Uint8Array(value);
|
|
2161
|
+
throw new Error("Stored block is not binary data");
|
|
2162
|
+
}
|
|
2163
|
+
function asStoredManifestRecord(value) {
|
|
2164
|
+
return structuredClone(value);
|
|
2165
|
+
}
|
|
2166
|
+
/** Builds the resolved public view of one stored record from its resolved block set. */
|
|
2167
|
+
function manifestView(record, blockIds) {
|
|
2168
|
+
return {
|
|
2169
|
+
version: record.version,
|
|
2170
|
+
previousVersion: record.previousVersion,
|
|
2171
|
+
blockIds: [...blockIds].sort(),
|
|
2172
|
+
createdAt: record.createdAt,
|
|
2173
|
+
...(record.changedTableIds === undefined
|
|
2174
|
+
? {}
|
|
2175
|
+
: { changedTableIds: [...record.changedTableIds] }),
|
|
2176
|
+
...(record.prunedAt === undefined ? {} : { prunedAt: record.prunedAt }),
|
|
2177
|
+
};
|
|
2178
|
+
}
|
|
2179
|
+
/**
|
|
2180
|
+
* Resolves one stored record's complete block set by walking `previousVersion` links down to
|
|
2181
|
+
* the nearest checkpoint and replaying the deltas forward. Pruned records keep their content,
|
|
2182
|
+
* so the chain below any readable version always exists; a missing link is corruption.
|
|
2183
|
+
*/
|
|
2184
|
+
async function resolveManifestBlockSetInTransaction(manifestStore, record) {
|
|
2185
|
+
const chain = [record];
|
|
2186
|
+
let cursor = record;
|
|
2187
|
+
while (cursor.blockIds === undefined) {
|
|
2188
|
+
if (cursor.previousVersion === null) {
|
|
2189
|
+
throw new Error(`Manifest delta chain has no checkpoint below version ${String(record.version)}`);
|
|
2190
|
+
}
|
|
2191
|
+
const value = await requestResult(manifestStore.get(cursor.previousVersion));
|
|
2192
|
+
if (value === undefined) {
|
|
2193
|
+
throw new Error(`Manifest delta chain is broken at version ${String(cursor.previousVersion)}`);
|
|
2194
|
+
}
|
|
2195
|
+
cursor = asStoredManifestRecord(value);
|
|
2196
|
+
chain.push(cursor);
|
|
2197
|
+
}
|
|
2198
|
+
const blockIds = new Set();
|
|
2199
|
+
for (let index = chain.length - 1; index >= 0; index -= 1) {
|
|
2200
|
+
const link = chain[index];
|
|
2201
|
+
if (link !== undefined)
|
|
2202
|
+
applyManifestRecord(blockIds, link);
|
|
2203
|
+
}
|
|
2204
|
+
return blockIds;
|
|
2205
|
+
}
|
|
2206
|
+
async function resolveManifestInTransaction(manifestStore, record) {
|
|
2207
|
+
return manifestView(record, await resolveManifestBlockSetInTransaction(manifestStore, record));
|
|
2208
|
+
}
|
|
2209
|
+
function asTransactionRecord(value) {
|
|
2210
|
+
return structuredClone(value);
|
|
2211
|
+
}
|
|
2212
|
+
function asTableRecord(value) {
|
|
2213
|
+
return structuredClone(value);
|
|
2214
|
+
}
|
|
2215
|
+
function isTableRecord(value) {
|
|
2216
|
+
return (typeof value === "object" &&
|
|
2217
|
+
value !== null &&
|
|
2218
|
+
typeof Reflect.get(value, "id") === "string" &&
|
|
2219
|
+
typeof Reflect.get(value, "name") === "string" &&
|
|
2220
|
+
Array.isArray(Reflect.get(value, "columns")));
|
|
2221
|
+
}
|
|
2222
|
+
function asSegmentRecord(value) {
|
|
2223
|
+
return normalizeSegmentRecord(value);
|
|
2224
|
+
}
|
|
2225
|
+
function asLeaseRecord(value) {
|
|
2226
|
+
return structuredClone(value);
|
|
2227
|
+
}
|
|
2228
|
+
/**
|
|
2229
|
+
* Verifies the snapshot at `version` is readable: the record exists, is not pruned, and its
|
|
2230
|
+
* delta chain down to a checkpoint is intact. Block existence is verified inductively — every
|
|
2231
|
+
* commit proves its own added blocks exist before publishing, and garbage collection only
|
|
2232
|
+
* deletes blocks unreachable from every non-pruned manifest — so this stays O(chain) instead of
|
|
2233
|
+
* probing every live block on each transaction and lease creation.
|
|
2234
|
+
*/
|
|
2235
|
+
async function assertSnapshotAvailableInTransaction(transaction, version) {
|
|
2236
|
+
if (version === null)
|
|
2237
|
+
return;
|
|
2238
|
+
const manifestStore = transaction.objectStore("manifests");
|
|
2239
|
+
const value = await requestResult(manifestStore.get(version));
|
|
2240
|
+
if (value === undefined)
|
|
2241
|
+
throw new SnapshotManifestMissingError(version);
|
|
2242
|
+
let cursor = asStoredManifestRecord(value);
|
|
2243
|
+
if (cursor.prunedAt !== undefined)
|
|
2244
|
+
throw new SnapshotManifestMissingError(version);
|
|
2245
|
+
while (cursor.blockIds === undefined) {
|
|
2246
|
+
if (cursor.previousVersion === null)
|
|
2247
|
+
throw new SnapshotManifestMissingError(version);
|
|
2248
|
+
const linkValue = await requestResult(manifestStore.get(cursor.previousVersion));
|
|
2249
|
+
if (linkValue === undefined)
|
|
2250
|
+
throw new SnapshotManifestMissingError(version);
|
|
2251
|
+
cursor = asStoredManifestRecord(linkValue);
|
|
2252
|
+
}
|
|
2253
|
+
}
|
|
2254
|
+
async function assertGarbageCollectionPinsAvailableInTransaction(transaction, currentVersion, leaseCutoff) {
|
|
2255
|
+
await assertSnapshotAvailableInTransaction(transaction, currentVersion);
|
|
2256
|
+
await visitObjectStoreSequentially(transaction.objectStore("transactions"), async (value) => {
|
|
2257
|
+
const record = asTransactionRecord(value);
|
|
2258
|
+
if (record.status === "active") {
|
|
2259
|
+
await assertSnapshotAvailableInTransaction(transaction, record.snapshotVersion);
|
|
2260
|
+
}
|
|
2261
|
+
});
|
|
2262
|
+
await visitObjectStoreSequentially(transaction.objectStore("leases"), async (value) => {
|
|
2263
|
+
const lease = asLeaseRecord(value);
|
|
2264
|
+
const expiresAt = Date.parse(lease.expiresAt);
|
|
2265
|
+
if (!Number.isFinite(expiresAt) || expiresAt > leaseCutoff) {
|
|
2266
|
+
await assertSnapshotAvailableInTransaction(transaction, lease.manifestVersion);
|
|
2267
|
+
}
|
|
2268
|
+
});
|
|
2269
|
+
await visitObjectStoreSequentially(transaction.objectStore("gc"), async (value) => {
|
|
2270
|
+
if (!isCompactionJobEnvelope(value))
|
|
2271
|
+
return;
|
|
2272
|
+
const job = asCompactionJobEnvelope(value);
|
|
2273
|
+
if (isTerminalCompactionJob(job))
|
|
2274
|
+
return;
|
|
2275
|
+
await assertSnapshotAvailableInTransaction(transaction, job.sourceManifestVersion);
|
|
2276
|
+
if (job.transactionId === null)
|
|
2277
|
+
return;
|
|
2278
|
+
const transactionValue = await requestResult(transaction.objectStore("transactions").get(job.transactionId));
|
|
2279
|
+
if (transactionValue === undefined)
|
|
2280
|
+
return;
|
|
2281
|
+
const linkedTransaction = asTransactionRecord(transactionValue);
|
|
2282
|
+
if (linkedTransaction.status !== "committed")
|
|
2283
|
+
return;
|
|
2284
|
+
if (linkedTransaction.committedVersion === null) {
|
|
2285
|
+
throw new Error(`Committed transaction has no manifest version: ${linkedTransaction.id}`);
|
|
2286
|
+
}
|
|
2287
|
+
await assertSnapshotAvailableInTransaction(transaction, linkedTransaction.committedVersion);
|
|
2288
|
+
});
|
|
2289
|
+
}
|
|
2290
|
+
async function assertPendingArtifactsAvailableInTransaction(transaction, record, validateBlocks = true, validateSegments = true) {
|
|
2291
|
+
const [blockKeys, segmentKeys] = await Promise.all([
|
|
2292
|
+
validateBlocks
|
|
2293
|
+
? Promise.all(record.pendingBlockIds.map((id) => requestResult(transaction.objectStore("blocks").getKey(id))))
|
|
2294
|
+
: Promise.resolve([]),
|
|
2295
|
+
validateSegments
|
|
2296
|
+
? Promise.all(record.pendingSegmentIds.map((id) => requestResult(transaction.objectStore("segments").getKey(id))))
|
|
2297
|
+
: Promise.resolve([]),
|
|
2298
|
+
]);
|
|
2299
|
+
const missingBlockIndex = blockKeys.findIndex((key) => key === undefined);
|
|
2300
|
+
if (missingBlockIndex >= 0) {
|
|
2301
|
+
throw new Error(`Transaction references missing pending block: ${record.pendingBlockIds[missingBlockIndex] ?? ""}`);
|
|
2302
|
+
}
|
|
2303
|
+
const missingSegmentIndex = segmentKeys.findIndex((key) => key === undefined);
|
|
2304
|
+
if (missingSegmentIndex >= 0) {
|
|
2305
|
+
throw new Error(`Transaction references missing pending segment: ${record.pendingSegmentIds[missingSegmentIndex] ?? ""}`);
|
|
2306
|
+
}
|
|
2307
|
+
}
|
|
2308
|
+
/**
|
|
2309
|
+
* After pruning, every remaining manifest must still be fully readable. One ascending pass
|
|
2310
|
+
* resolves the whole chain with a single running set (pruned records still contribute their
|
|
2311
|
+
* deltas), unions the blocks referenced by any remaining manifest, and verifies each referenced
|
|
2312
|
+
* block once instead of once per manifest.
|
|
2313
|
+
*/
|
|
2314
|
+
async function assertRemainingManifestRecordsAvailable(transaction, newlyPrunedVersions) {
|
|
2315
|
+
const running = new Set();
|
|
2316
|
+
const referenced = new Map();
|
|
2317
|
+
await visitObjectStoreSequentially(transaction.objectStore("manifests"), (value) => {
|
|
2318
|
+
const record = asStoredManifestRecord(value);
|
|
2319
|
+
applyManifestRecord(running, record);
|
|
2320
|
+
if (record.prunedAt !== undefined || newlyPrunedVersions.has(record.version))
|
|
2321
|
+
return;
|
|
2322
|
+
for (const id of running) {
|
|
2323
|
+
if (!referenced.has(id))
|
|
2324
|
+
referenced.set(id, record.version);
|
|
2325
|
+
}
|
|
2326
|
+
});
|
|
2327
|
+
const blockStore = transaction.objectStore("blocks");
|
|
2328
|
+
for (const [id, version] of referenced) {
|
|
2329
|
+
if ((await requestResult(blockStore.getKey(id))) === undefined) {
|
|
2330
|
+
throw new SnapshotManifestMissingError(version);
|
|
2331
|
+
}
|
|
2332
|
+
}
|
|
2333
|
+
}
|
|
2334
|
+
function compactionJobKey(id) {
|
|
2335
|
+
return `${COMPACTION_JOB_KEY_PREFIX}${id}`;
|
|
2336
|
+
}
|
|
2337
|
+
function garbageCollectionJobKey(id) {
|
|
2338
|
+
return `${GARBAGE_COLLECTION_JOB_KEY_PREFIX}${id}`;
|
|
2339
|
+
}
|
|
2340
|
+
function compactionJobEnvelope(record) {
|
|
2341
|
+
return { kind: "compaction-job", record: structuredClone(record) };
|
|
2342
|
+
}
|
|
2343
|
+
function isCompactionJobEnvelope(value) {
|
|
2344
|
+
return (typeof value === "object" &&
|
|
2345
|
+
value !== null &&
|
|
2346
|
+
Reflect.get(value, "kind") === "compaction-job" &&
|
|
2347
|
+
typeof Reflect.get(value, "record") === "object" &&
|
|
2348
|
+
Reflect.get(value, "record") !== null);
|
|
2349
|
+
}
|
|
2350
|
+
function asCompactionJobEnvelope(value) {
|
|
2351
|
+
if (!isCompactionJobEnvelope(value))
|
|
2352
|
+
throw new Error("Stored compaction job is invalid");
|
|
2353
|
+
return normalizeCompactionJobRecord(structuredClone(value.record));
|
|
2354
|
+
}
|
|
2355
|
+
function garbageCollectionJobEnvelope(record) {
|
|
2356
|
+
return { kind: "garbage-collection-job", record: structuredClone(record) };
|
|
2357
|
+
}
|
|
2358
|
+
function isGarbageCollectionJobEnvelope(value) {
|
|
2359
|
+
return (typeof value === "object" &&
|
|
2360
|
+
value !== null &&
|
|
2361
|
+
Reflect.get(value, "kind") === "garbage-collection-job" &&
|
|
2362
|
+
typeof Reflect.get(value, "record") === "object" &&
|
|
2363
|
+
Reflect.get(value, "record") !== null);
|
|
2364
|
+
}
|
|
2365
|
+
function asGarbageCollectionJobEnvelope(value) {
|
|
2366
|
+
if (!isGarbageCollectionJobEnvelope(value)) {
|
|
2367
|
+
throw new Error("Stored garbage collection job is invalid");
|
|
2368
|
+
}
|
|
2369
|
+
return normalizeGarbageCollectionJobRecord(structuredClone(value.record));
|
|
2370
|
+
}
|
|
2371
|
+
function isTerminalCompactionJob(record) {
|
|
2372
|
+
return record.state === "published" || record.state === "cancelled" || record.state === "aborted";
|
|
2373
|
+
}
|
|
2374
|
+
function assertGenericTransactionUpdateAllowed(record, update) {
|
|
2375
|
+
if (record.status !== "active") {
|
|
2376
|
+
throw new TypeError(`Only active transactions can be updated; found ${record.status}`);
|
|
2377
|
+
}
|
|
2378
|
+
if (update.status === "committed") {
|
|
2379
|
+
throw new TypeError("Use commitTransaction to commit a transaction");
|
|
2380
|
+
}
|
|
2381
|
+
if (Reflect.has(update, "committedVersion")) {
|
|
2382
|
+
throw new TypeError("Only commitTransaction can set a committed transaction version");
|
|
2383
|
+
}
|
|
2384
|
+
}
|
|
2385
|
+
async function assertGarbageCollectionCandidateProvenanceInTransaction(transaction, job) {
|
|
2386
|
+
const manifestStore = transaction.objectStore("manifests");
|
|
2387
|
+
const candidateManifestBlockIds = [];
|
|
2388
|
+
for (const version of job.candidateManifestVersions) {
|
|
2389
|
+
const value = await requestResult(manifestStore.get(version));
|
|
2390
|
+
if (value === undefined) {
|
|
2391
|
+
throw new Error(`Garbage collection candidate manifest is missing: ${String(version)}`);
|
|
2392
|
+
}
|
|
2393
|
+
candidateManifestBlockIds.push(await resolveManifestBlockSetInTransaction(manifestStore, asStoredManifestRecord(value)));
|
|
2394
|
+
}
|
|
2395
|
+
const blockHasProvenance = async (id) => {
|
|
2396
|
+
if (candidateManifestBlockIds.some((blockIds) => blockIds.has(id)))
|
|
2397
|
+
return true;
|
|
2398
|
+
const transactionProven = await visitObjectStoreSequentially(transaction.objectStore("transactions"), (value) => {
|
|
2399
|
+
const record = asTransactionRecord(value);
|
|
2400
|
+
return record.status === "aborted" && record.pendingBlockIds.includes(id);
|
|
2401
|
+
});
|
|
2402
|
+
if (transactionProven)
|
|
2403
|
+
return true;
|
|
2404
|
+
return visitObjectStoreSequentially(transaction.objectStore("gc"), (value) => {
|
|
2405
|
+
if (!isCompactionJobEnvelope(value))
|
|
2406
|
+
return false;
|
|
2407
|
+
const record = asCompactionJobEnvelope(value);
|
|
2408
|
+
return (isTerminalCompactionJob(record) &&
|
|
2409
|
+
(record.sourceBlockIds.includes(id) || record.outputBlockIds.includes(id)));
|
|
2410
|
+
});
|
|
2411
|
+
};
|
|
2412
|
+
for (const id of job.candidateBlockIds) {
|
|
2413
|
+
if (await blockHasProvenance(id))
|
|
2414
|
+
continue;
|
|
2415
|
+
throw new Error(`Garbage collection block candidate has no persisted provenance: ${id}`);
|
|
2416
|
+
}
|
|
2417
|
+
for (const id of job.candidateSegmentIds) {
|
|
2418
|
+
let proven = await visitObjectStoreSequentially(transaction.objectStore("transactions"), (value) => {
|
|
2419
|
+
const record = asTransactionRecord(value);
|
|
2420
|
+
return record.status === "aborted" && record.pendingSegmentIds.includes(id);
|
|
2421
|
+
});
|
|
2422
|
+
if (!proven) {
|
|
2423
|
+
proven = await visitObjectStoreSequentially(transaction.objectStore("gc"), (value) => {
|
|
2424
|
+
if (!isCompactionJobEnvelope(value))
|
|
2425
|
+
return false;
|
|
2426
|
+
const record = asCompactionJobEnvelope(value);
|
|
2427
|
+
return (isTerminalCompactionJob(record) &&
|
|
2428
|
+
(record.sourceSegmentIds.includes(id) || record.outputSegmentId === id));
|
|
2429
|
+
});
|
|
2430
|
+
}
|
|
2431
|
+
if (!proven) {
|
|
2432
|
+
const segmentValue = await requestResult(transaction.objectStore("segments").get(id));
|
|
2433
|
+
if (segmentValue !== undefined) {
|
|
2434
|
+
const blockIds = segmentBlockIds(asSegmentRecord(segmentValue));
|
|
2435
|
+
proven = blockIds.length > 0;
|
|
2436
|
+
for (const blockId of blockIds) {
|
|
2437
|
+
if (!(await blockHasProvenance(blockId))) {
|
|
2438
|
+
proven = false;
|
|
2439
|
+
break;
|
|
2440
|
+
}
|
|
2441
|
+
}
|
|
2442
|
+
}
|
|
2443
|
+
}
|
|
2444
|
+
if (proven)
|
|
2445
|
+
continue;
|
|
2446
|
+
throw new Error(`Garbage collection segment candidate has no persisted provenance: ${id}`);
|
|
2447
|
+
}
|
|
2448
|
+
}
|
|
2449
|
+
async function isManifestVersionPinnedInTransaction(transaction, version, currentVersion, leaseCutoff) {
|
|
2450
|
+
if (currentVersion === version)
|
|
2451
|
+
return true;
|
|
2452
|
+
const transactionPinned = await visitObjectStoreSequentially(transaction.objectStore("transactions"), (value) => {
|
|
2453
|
+
const record = asTransactionRecord(value);
|
|
2454
|
+
return record.status === "active" && record.snapshotVersion === version;
|
|
2455
|
+
});
|
|
2456
|
+
if (transactionPinned)
|
|
2457
|
+
return true;
|
|
2458
|
+
const leasePinned = await visitObjectStoreSequentially(transaction.objectStore("leases"), (value) => {
|
|
2459
|
+
const lease = asLeaseRecord(value);
|
|
2460
|
+
const expiresAt = Date.parse(lease.expiresAt);
|
|
2461
|
+
return (lease.manifestVersion === version &&
|
|
2462
|
+
(!Number.isFinite(expiresAt) || expiresAt > leaseCutoff));
|
|
2463
|
+
});
|
|
2464
|
+
if (leasePinned)
|
|
2465
|
+
return true;
|
|
2466
|
+
return visitObjectStoreSequentially(transaction.objectStore("gc"), async (value) => {
|
|
2467
|
+
if (!isCompactionJobEnvelope(value))
|
|
2468
|
+
return false;
|
|
2469
|
+
const job = asCompactionJobEnvelope(value);
|
|
2470
|
+
if (isTerminalCompactionJob(job))
|
|
2471
|
+
return false;
|
|
2472
|
+
if (job.sourceManifestVersion === version)
|
|
2473
|
+
return true;
|
|
2474
|
+
if (job.transactionId === null)
|
|
2475
|
+
return false;
|
|
2476
|
+
const transactionValue = await requestResult(transaction.objectStore("transactions").get(job.transactionId));
|
|
2477
|
+
if (transactionValue === undefined)
|
|
2478
|
+
return false;
|
|
2479
|
+
const linkedTransaction = asTransactionRecord(transactionValue);
|
|
2480
|
+
if (linkedTransaction.status !== "committed")
|
|
2481
|
+
return false;
|
|
2482
|
+
if (linkedTransaction.committedVersion === null) {
|
|
2483
|
+
throw new Error(`Committed transaction has no manifest version: ${linkedTransaction.id}`);
|
|
2484
|
+
}
|
|
2485
|
+
return linkedTransaction.committedVersion === version;
|
|
2486
|
+
});
|
|
2487
|
+
}
|
|
2488
|
+
const MAX_GARBAGE_COLLECTION_ROOT_DEPENDENCIES = 4_096;
|
|
2489
|
+
async function collectBoundedPhysicalRootsInTransaction(transaction, candidateSegmentIds, candidateBlockIds, newlyPrunedVersions) {
|
|
2490
|
+
const candidateSegments = new Set(candidateSegmentIds);
|
|
2491
|
+
const candidateBlocks = new Set(candidateBlockIds);
|
|
2492
|
+
const probeBlockIds = new Set(candidateBlockIds);
|
|
2493
|
+
const probeSegmentIds = new Set(candidateSegmentIds);
|
|
2494
|
+
const relatedSegments = new Map();
|
|
2495
|
+
const dependencyOverflow = await visitObjectStoreSequentially(transaction.objectStore("segments"), (value) => {
|
|
2496
|
+
const segment = asSegmentRecord(value);
|
|
2497
|
+
const ids = segmentBlockIds(segment);
|
|
2498
|
+
if (!candidateSegments.has(segment.id) && !ids.some((id) => candidateBlocks.has(id)))
|
|
2499
|
+
return;
|
|
2500
|
+
const newBlockIds = new Set(ids.filter((id) => !probeBlockIds.has(id)));
|
|
2501
|
+
if (relatedSegments.size >= MAX_GARBAGE_COLLECTION_ROOT_DEPENDENCIES ||
|
|
2502
|
+
probeBlockIds.size + newBlockIds.size > MAX_GARBAGE_COLLECTION_ROOT_DEPENDENCIES) {
|
|
2503
|
+
return true;
|
|
2504
|
+
}
|
|
2505
|
+
relatedSegments.set(segment.id, segment);
|
|
2506
|
+
probeSegmentIds.add(segment.id);
|
|
2507
|
+
ids.forEach((id) => probeBlockIds.add(id));
|
|
2508
|
+
return false;
|
|
2509
|
+
});
|
|
2510
|
+
if (dependencyOverflow) {
|
|
2511
|
+
return { blockIds: new Set(candidateBlockIds), segmentIds: new Set(candidateSegmentIds) };
|
|
2512
|
+
}
|
|
2513
|
+
const directBlockRoots = new Set();
|
|
2514
|
+
const directSegmentRoots = new Set();
|
|
2515
|
+
// One ascending pass resolves every manifest with a running set (pruned records still apply
|
|
2516
|
+
// their deltas so the chain stays coherent); a probed block roots when it is a member at any
|
|
2517
|
+
// version that still counts.
|
|
2518
|
+
{
|
|
2519
|
+
const running = new Set();
|
|
2520
|
+
const unrooted = new Set(probeBlockIds);
|
|
2521
|
+
await visitObjectStoreSequentially(transaction.objectStore("manifests"), (value) => {
|
|
2522
|
+
const record = asStoredManifestRecord(value);
|
|
2523
|
+
applyManifestRecord(running, record);
|
|
2524
|
+
if (record.prunedAt !== undefined || newlyPrunedVersions.has(record.version))
|
|
2525
|
+
return;
|
|
2526
|
+
if (unrooted.size === 0)
|
|
2527
|
+
return true;
|
|
2528
|
+
for (const id of unrooted) {
|
|
2529
|
+
if (running.has(id)) {
|
|
2530
|
+
directBlockRoots.add(id);
|
|
2531
|
+
unrooted.delete(id);
|
|
2532
|
+
}
|
|
2533
|
+
}
|
|
2534
|
+
return false;
|
|
2535
|
+
});
|
|
2536
|
+
}
|
|
2537
|
+
const ownerTransactionIds = new Set([...relatedSegments.values()].map((segment) => segment.transactionId));
|
|
2538
|
+
const activeOwnerTransactionIds = new Set();
|
|
2539
|
+
await visitObjectStoreSequentially(transaction.objectStore("transactions"), (value) => {
|
|
2540
|
+
const record = asTransactionRecord(value);
|
|
2541
|
+
if (record.status !== "active")
|
|
2542
|
+
return;
|
|
2543
|
+
if (ownerTransactionIds.has(record.id))
|
|
2544
|
+
activeOwnerTransactionIds.add(record.id);
|
|
2545
|
+
for (const id of record.pendingBlockIds)
|
|
2546
|
+
if (probeBlockIds.has(id))
|
|
2547
|
+
directBlockRoots.add(id);
|
|
2548
|
+
for (const id of record.pendingSegmentIds)
|
|
2549
|
+
if (probeSegmentIds.has(id))
|
|
2550
|
+
directSegmentRoots.add(id);
|
|
2551
|
+
});
|
|
2552
|
+
await visitObjectStoreSequentially(transaction.objectStore("gc"), (value) => {
|
|
2553
|
+
if (!isCompactionJobEnvelope(value))
|
|
2554
|
+
return;
|
|
2555
|
+
const job = asCompactionJobEnvelope(value);
|
|
2556
|
+
if (isTerminalCompactionJob(job))
|
|
2557
|
+
return;
|
|
2558
|
+
for (const id of job.sourceBlockIds)
|
|
2559
|
+
if (probeBlockIds.has(id))
|
|
2560
|
+
directBlockRoots.add(id);
|
|
2561
|
+
for (const id of job.outputBlockIds)
|
|
2562
|
+
if (probeBlockIds.has(id))
|
|
2563
|
+
directBlockRoots.add(id);
|
|
2564
|
+
for (const id of job.sourceSegmentIds)
|
|
2565
|
+
if (probeSegmentIds.has(id))
|
|
2566
|
+
directSegmentRoots.add(id);
|
|
2567
|
+
if (job.outputSegmentId !== null && probeSegmentIds.has(job.outputSegmentId)) {
|
|
2568
|
+
directSegmentRoots.add(job.outputSegmentId);
|
|
2569
|
+
}
|
|
2570
|
+
});
|
|
2571
|
+
const rootedBlockIds = new Set();
|
|
2572
|
+
const rootedSegmentIds = new Set();
|
|
2573
|
+
for (const segment of relatedSegments.values()) {
|
|
2574
|
+
const ids = segmentBlockIds(segment);
|
|
2575
|
+
if (directSegmentRoots.has(segment.id) ||
|
|
2576
|
+
activeOwnerTransactionIds.has(segment.transactionId) ||
|
|
2577
|
+
(ids.length > 0 && ids.every((id) => directBlockRoots.has(id)))) {
|
|
2578
|
+
if (candidateSegments.has(segment.id))
|
|
2579
|
+
rootedSegmentIds.add(segment.id);
|
|
2580
|
+
for (const id of ids)
|
|
2581
|
+
if (candidateBlocks.has(id))
|
|
2582
|
+
rootedBlockIds.add(id);
|
|
2583
|
+
}
|
|
2584
|
+
}
|
|
2585
|
+
for (const id of candidateBlockIds)
|
|
2586
|
+
if (directBlockRoots.has(id))
|
|
2587
|
+
rootedBlockIds.add(id);
|
|
2588
|
+
for (const id of candidateSegmentIds)
|
|
2589
|
+
if (directSegmentRoots.has(id))
|
|
2590
|
+
rootedSegmentIds.add(id);
|
|
2591
|
+
return { blockIds: rootedBlockIds, segmentIds: rootedSegmentIds };
|
|
2592
|
+
}
|
|
2593
|
+
function segmentBlockIds(segment) {
|
|
2594
|
+
return [...new Set(Object.values(segment.columnBlockIds).flat())];
|
|
2595
|
+
}
|
|
2596
|
+
function validateGarbageCollectionStepInput(input) {
|
|
2597
|
+
if (input.jobId.length === 0)
|
|
2598
|
+
throw new TypeError("Garbage collection job ID cannot be empty");
|
|
2599
|
+
if (!Number.isSafeInteger(input.expectedRevision) || input.expectedRevision < 0) {
|
|
2600
|
+
throw new RangeError("Garbage collection expected revision must be a non-negative whole number");
|
|
2601
|
+
}
|
|
2602
|
+
if (!Number.isSafeInteger(input.maxItems) || input.maxItems <= 0) {
|
|
2603
|
+
throw new RangeError("Garbage collection item limit must be a positive whole number");
|
|
2604
|
+
}
|
|
2605
|
+
if (input.updatedAt.length === 0 || !Number.isFinite(Date.parse(input.updatedAt))) {
|
|
2606
|
+
throw new TypeError("Garbage collection update timestamp must be valid");
|
|
2607
|
+
}
|
|
2608
|
+
}
|
|
2609
|
+
function validatePageLimit(limit) {
|
|
2610
|
+
if (!Number.isSafeInteger(limit) || limit <= 0) {
|
|
2611
|
+
throw new RangeError("Storage page limit must be a positive whole number");
|
|
2612
|
+
}
|
|
2613
|
+
}
|
|
2614
|
+
function validateLeaseExpiration(expiresAt) {
|
|
2615
|
+
if (expiresAt.length === 0 || !Number.isFinite(Date.parse(expiresAt))) {
|
|
2616
|
+
throw new TypeError("Lease expiration must be valid");
|
|
2617
|
+
}
|
|
2618
|
+
}
|
|
2619
|
+
function emptyGarbageCollectionStep(job) {
|
|
2620
|
+
return {
|
|
2621
|
+
job: structuredClone(job),
|
|
2622
|
+
prunedManifestVersions: [],
|
|
2623
|
+
alreadyPrunedManifestVersions: [],
|
|
2624
|
+
retainedManifestVersions: [],
|
|
2625
|
+
missingManifestVersions: [],
|
|
2626
|
+
reclaimedSegmentIds: [],
|
|
2627
|
+
retainedSegmentIds: [],
|
|
2628
|
+
missingSegmentIds: [],
|
|
2629
|
+
reclaimedBlockIds: [],
|
|
2630
|
+
retainedBlockIds: [],
|
|
2631
|
+
missingBlockIds: [],
|
|
2632
|
+
reclaimedBlockBytes: 0,
|
|
2633
|
+
};
|
|
2634
|
+
}
|
|
2635
|
+
function safeStorageSum(left, right) {
|
|
2636
|
+
const total = left + right;
|
|
2637
|
+
if (!Number.isSafeInteger(total)) {
|
|
2638
|
+
throw new RangeError("Garbage collection reclaimed block bytes exceed the safe range");
|
|
2639
|
+
}
|
|
2640
|
+
return total;
|
|
2641
|
+
}
|
|
2642
|
+
function abortIfActive(transaction) {
|
|
2643
|
+
try {
|
|
2644
|
+
transaction.abort();
|
|
2645
|
+
}
|
|
2646
|
+
catch {
|
|
2647
|
+
// An already completed or aborted transaction needs no further action.
|
|
2648
|
+
}
|
|
2649
|
+
}
|
|
2650
|
+
function validateCount(count) {
|
|
2651
|
+
if (!Number.isSafeInteger(count) || count <= 0) {
|
|
2652
|
+
throw new RangeError("Row ID reservation count must be a positive whole number");
|
|
2653
|
+
}
|
|
2654
|
+
}
|
|
2655
|
+
function ftsChunkKey(tableId, columnId, version) {
|
|
2656
|
+
return `${FTS_CHUNK_PREFIX}${tableId}/${columnId}/${String(version)}`;
|
|
2657
|
+
}
|
|
2658
|
+
function ftsChunkIndexKey(tableId, columnId) {
|
|
2659
|
+
return `${FTS_CHUNK_PREFIX}index/${tableId}/${columnId}`;
|
|
2660
|
+
}
|
|
2661
|
+
function validateAutoIncrementReservation(count, atLeast) {
|
|
2662
|
+
if (!Number.isSafeInteger(count) || count < 0) {
|
|
2663
|
+
throw new RangeError("Auto-increment reservation count must be a non-negative whole number");
|
|
2664
|
+
}
|
|
2665
|
+
if (atLeast !== undefined && atLeast < 1n) {
|
|
2666
|
+
throw new RangeError("Auto-increment bump target must be at least 1");
|
|
2667
|
+
}
|
|
2668
|
+
}
|
|
2669
|
+
function maxBigInt(a, b) {
|
|
2670
|
+
return a > b ? a : b;
|
|
2671
|
+
}
|
|
2672
|
+
function validateTempRunPage(page) {
|
|
2673
|
+
validateTempRunPageIdentity(page.ownerId, page.runId, page.pageIndex);
|
|
2674
|
+
if (!(page.bytes instanceof Uint8Array))
|
|
2675
|
+
throw new TypeError("Temp run page bytes are invalid");
|
|
2676
|
+
}
|
|
2677
|
+
function validateTempRunPageIdentity(ownerId, runId, pageIndex) {
|
|
2678
|
+
validateTempId(ownerId, "Temp run owner ID");
|
|
2679
|
+
validateTempId(runId, "Temp run ID");
|
|
2680
|
+
if (!Number.isSafeInteger(pageIndex) || pageIndex < 0) {
|
|
2681
|
+
throw new RangeError("Temp run page index must be a non-negative whole number");
|
|
2682
|
+
}
|
|
2683
|
+
}
|
|
2684
|
+
function validateTempId(id, label) {
|
|
2685
|
+
if (id.length === 0)
|
|
2686
|
+
throw new TypeError(`${label} cannot be empty`);
|
|
2687
|
+
}
|
|
2688
|
+
function tempRunPageKey(page) {
|
|
2689
|
+
return ["run", page.ownerId, page.runId, page.pageIndex];
|
|
2690
|
+
}
|
|
2691
|
+
function isTempRunPageKey(key, ownerId, runId) {
|
|
2692
|
+
return (Array.isArray(key) &&
|
|
2693
|
+
key[0] === "run" &&
|
|
2694
|
+
key[1] === ownerId &&
|
|
2695
|
+
(runId === undefined || key[2] === runId));
|
|
2696
|
+
}
|
|
2697
|
+
function tempOwnerKey(ownerId) {
|
|
2698
|
+
return ["owner", ownerId];
|
|
2699
|
+
}
|
|
2700
|
+
function validateTableColumns(columns) {
|
|
2701
|
+
if (columns.length === 0)
|
|
2702
|
+
throw new TypeError("A table needs at least one column");
|
|
2703
|
+
const ids = new Set(columns.map(({ id }) => id));
|
|
2704
|
+
const names = new Set(columns.map(({ name }) => name));
|
|
2705
|
+
if (ids.size !== columns.length || names.size !== columns.length) {
|
|
2706
|
+
throw new TypeError("Table columns must have unique IDs and names");
|
|
2707
|
+
}
|
|
2708
|
+
}
|
|
2709
|
+
function validateTempOwnerRecord(record) {
|
|
2710
|
+
validateTempId(record.ownerId, "Temp run owner ID");
|
|
2711
|
+
validateLeaseExpiration(record.expiresAt);
|
|
2712
|
+
if (record.revision !== 0) {
|
|
2713
|
+
throw new RangeError("Temp owner record must be created at revision zero");
|
|
2714
|
+
}
|
|
2715
|
+
}
|
|
2716
|
+
function asTempOwnerRecord(value) {
|
|
2717
|
+
return structuredClone(value);
|
|
2718
|
+
}
|
|
2719
|
+
// Both scans run without IDBKeyRange (absent from the injected test factory environment): owner
|
|
2720
|
+
// keys ["owner", id] sort before page keys ["run", id, runId, index], and an empty array sorts
|
|
2721
|
+
// after every string, so ["run", id, []] jumps past one owner's remaining pages.
|
|
2722
|
+
function readTempOwnerRecordIds(store, afterOwnerId, max) {
|
|
2723
|
+
return new Promise((resolve, reject) => {
|
|
2724
|
+
const ownerIds = [];
|
|
2725
|
+
const request = store.openKeyCursor();
|
|
2726
|
+
request.onerror = () => reject(request.error ?? new Error("IndexedDB cursor failed"));
|
|
2727
|
+
request.onsuccess = () => {
|
|
2728
|
+
const cursor = request.result;
|
|
2729
|
+
if (cursor === null || ownerIds.length >= max) {
|
|
2730
|
+
resolve(ownerIds);
|
|
2731
|
+
return;
|
|
2732
|
+
}
|
|
2733
|
+
const key = cursor.key;
|
|
2734
|
+
if (!Array.isArray(key) || key[0] !== "owner" || typeof key[1] !== "string") {
|
|
2735
|
+
resolve(ownerIds);
|
|
2736
|
+
return;
|
|
2737
|
+
}
|
|
2738
|
+
if (afterOwnerId === null || key[1] > afterOwnerId)
|
|
2739
|
+
ownerIds.push(key[1]);
|
|
2740
|
+
if (ownerIds.length >= max)
|
|
2741
|
+
resolve(ownerIds);
|
|
2742
|
+
else
|
|
2743
|
+
cursor.continue();
|
|
2744
|
+
};
|
|
2745
|
+
});
|
|
2746
|
+
}
|
|
2747
|
+
function readTempPageOwnerIds(store, afterOwnerId, max) {
|
|
2748
|
+
return new Promise((resolve, reject) => {
|
|
2749
|
+
const ownerIds = [];
|
|
2750
|
+
const request = store.openKeyCursor();
|
|
2751
|
+
request.onerror = () => reject(request.error ?? new Error("IndexedDB cursor failed"));
|
|
2752
|
+
request.onsuccess = () => {
|
|
2753
|
+
const cursor = request.result;
|
|
2754
|
+
if (cursor === null || ownerIds.length >= max) {
|
|
2755
|
+
resolve(ownerIds);
|
|
2756
|
+
return;
|
|
2757
|
+
}
|
|
2758
|
+
const key = cursor.key;
|
|
2759
|
+
if (Array.isArray(key) && key[0] === "owner") {
|
|
2760
|
+
cursor.continue(["run"]);
|
|
2761
|
+
return;
|
|
2762
|
+
}
|
|
2763
|
+
if (!Array.isArray(key) || key[0] !== "run" || typeof key[1] !== "string") {
|
|
2764
|
+
cursor.continue();
|
|
2765
|
+
return;
|
|
2766
|
+
}
|
|
2767
|
+
if (afterOwnerId !== null && key[1] <= afterOwnerId) {
|
|
2768
|
+
cursor.continue(["run", afterOwnerId, []]);
|
|
2769
|
+
return;
|
|
2770
|
+
}
|
|
2771
|
+
ownerIds.push(key[1]);
|
|
2772
|
+
if (ownerIds.length >= max) {
|
|
2773
|
+
resolve(ownerIds);
|
|
2774
|
+
return;
|
|
2775
|
+
}
|
|
2776
|
+
cursor.continue(["run", key[1], []]);
|
|
2777
|
+
};
|
|
2778
|
+
});
|
|
2779
|
+
}
|
|
2780
|
+
function uniqueKeyKey(tableId, keyToken) {
|
|
2781
|
+
return ["unique-key", tableId, keyToken];
|
|
2782
|
+
}
|
|
2783
|
+
function uniqueKeyChunkIndexKey(tableId) {
|
|
2784
|
+
return [UNIQUE_KEY_CHUNK_INDEX, tableId];
|
|
2785
|
+
}
|
|
2786
|
+
function uniqueKeyChunkKey(tableId, version) {
|
|
2787
|
+
return [UNIQUE_KEY_CHUNK, tableId, version];
|
|
2788
|
+
}
|
|
2789
|
+
function uniqueKeyBaseKey(tableId, keyToken) {
|
|
2790
|
+
return [UNIQUE_KEY_BASE, tableId, keyToken];
|
|
2791
|
+
}
|
|
2792
|
+
function uniqueKeyBasePartKey(tableId, ordinal) {
|
|
2793
|
+
return [UNIQUE_KEY_BASE_PART, tableId, ordinal];
|
|
2794
|
+
}
|
|
2795
|
+
/** Accepts the legacy bare-array shape (tail only, no base) and the current object shape. */
|
|
2796
|
+
function asUniqueKeyChunkIndex(value) {
|
|
2797
|
+
const versionsOf = (candidate) => Array.isArray(candidate)
|
|
2798
|
+
? candidate.filter((entry) => Number.isSafeInteger(entry))
|
|
2799
|
+
: [];
|
|
2800
|
+
if (Array.isArray(value))
|
|
2801
|
+
return { versions: versionsOf(value), hasBase: false };
|
|
2802
|
+
if (typeof value === "object" && value !== null) {
|
|
2803
|
+
const record = value;
|
|
2804
|
+
return {
|
|
2805
|
+
versions: versionsOf(record.versions),
|
|
2806
|
+
hasBase: record.hasBase === true,
|
|
2807
|
+
...(Number.isSafeInteger(record.partitions) && record.partitions > 0
|
|
2808
|
+
? { partitions: record.partitions }
|
|
2809
|
+
: {}),
|
|
2810
|
+
...(Number.isSafeInteger(record.tokenCount) && record.tokenCount >= 0
|
|
2811
|
+
? { tokenCount: record.tokenCount }
|
|
2812
|
+
: {}),
|
|
2813
|
+
};
|
|
2814
|
+
}
|
|
2815
|
+
return { versions: [], hasBase: false };
|
|
2816
|
+
}
|
|
2817
|
+
/**
|
|
2818
|
+
* Above this many tokens, probing the folded base per token costs more than one cursor pass
|
|
2819
|
+
* over it; below, point lookups win. Bulk loads sit far above, keyed point writes far below.
|
|
2820
|
+
*/
|
|
2821
|
+
const UNIQUE_KEY_BASE_SCAN_THRESHOLD = 2_048;
|
|
2822
|
+
/** FNV-1a over UTF-16 code units: the stable hash that assigns a token to a base partition. */
|
|
2823
|
+
function fnv1a(token) {
|
|
2824
|
+
let hash = 0x811c9dc5;
|
|
2825
|
+
for (let index = 0; index < token.length; index += 1) {
|
|
2826
|
+
hash ^= token.charCodeAt(index);
|
|
2827
|
+
hash = Math.imul(hash, 0x01000193);
|
|
2828
|
+
}
|
|
2829
|
+
return hash >>> 0;
|
|
2830
|
+
}
|
|
2831
|
+
/** Applies one tail chunk's membership delta in place, in the order commits recorded it. */
|
|
2832
|
+
function applyChunk(present, chunk) {
|
|
2833
|
+
for (const token of chunk.addedTokens)
|
|
2834
|
+
present.add(token);
|
|
2835
|
+
for (const token of chunk.removedTokens)
|
|
2836
|
+
present.delete(token);
|
|
2837
|
+
}
|
|
2838
|
+
/** Replays tail chunks over an empty base into a fresh membership set. */
|
|
2839
|
+
function replayChunks(chunks) {
|
|
2840
|
+
const present = new Set();
|
|
2841
|
+
for (const chunk of chunks)
|
|
2842
|
+
applyChunk(present, chunk);
|
|
2843
|
+
return present;
|
|
2844
|
+
}
|
|
2845
|
+
function asBasePartition(value) {
|
|
2846
|
+
if (!isStringArray(value))
|
|
2847
|
+
throw new Error("Unique-key base partition is invalid");
|
|
2848
|
+
return value;
|
|
2849
|
+
}
|
|
2850
|
+
/** Reads every chunks-v2 base partition into one membership set. */
|
|
2851
|
+
/** How many distinct tokens a tail carries, counting a token touched twice once. */
|
|
2852
|
+
function countTailTokens(tailChunks) {
|
|
2853
|
+
const seen = new Set();
|
|
2854
|
+
for (const tail of tailChunks) {
|
|
2855
|
+
for (const token of tail.addedTokens)
|
|
2856
|
+
seen.add(token);
|
|
2857
|
+
for (const token of tail.removedTokens)
|
|
2858
|
+
seen.add(token);
|
|
2859
|
+
}
|
|
2860
|
+
return seen.size;
|
|
2861
|
+
}
|
|
2862
|
+
/**
|
|
2863
|
+
* Collapses a run of tail chunks into one chunk with the same effect. Replaying in commit
|
|
2864
|
+
* order and keeping the two sets disjoint is what makes it equivalent: a token added and then
|
|
2865
|
+
* removed ends up only in `removedTokens`, and the reverse ends up only in `addedTokens`, so
|
|
2866
|
+
* applying the result once lands exactly where applying the run in order would.
|
|
2867
|
+
*/
|
|
2868
|
+
function mergeTailChunks(tailChunks) {
|
|
2869
|
+
const added = new Set();
|
|
2870
|
+
const removed = new Set();
|
|
2871
|
+
for (const tail of tailChunks) {
|
|
2872
|
+
for (const token of tail.addedTokens) {
|
|
2873
|
+
removed.delete(token);
|
|
2874
|
+
added.add(token);
|
|
2875
|
+
}
|
|
2876
|
+
for (const token of tail.removedTokens) {
|
|
2877
|
+
added.delete(token);
|
|
2878
|
+
removed.add(token);
|
|
2879
|
+
}
|
|
2880
|
+
}
|
|
2881
|
+
return { addedTokens: [...added], removedTokens: [...removed] };
|
|
2882
|
+
}
|
|
2883
|
+
/**
|
|
2884
|
+
* Folds a tail into the folded base by touching only the partitions its tokens hash into.
|
|
2885
|
+
* Returns undefined when that cannot be done correctly and the caller must rewrite the whole
|
|
2886
|
+
* base: no folded base yet, no recorded token count, or a resulting size that needs a
|
|
2887
|
+
* different partition count (which moves every token, since the partition is `hash % count`).
|
|
2888
|
+
*
|
|
2889
|
+
* The touched partitions are read, replayed in commit order, and written back. Their sizes
|
|
2890
|
+
* before and after give the new total exactly, so the count stays true without a full read.
|
|
2891
|
+
*/
|
|
2892
|
+
async function foldTailIntoTouchedPartitions(store, tableId, index, tailChunks) {
|
|
2893
|
+
const partitions = index.partitions;
|
|
2894
|
+
const priorCount = index.tokenCount;
|
|
2895
|
+
if (!index.hasBase || partitions === undefined || priorCount === undefined)
|
|
2896
|
+
return undefined;
|
|
2897
|
+
const touchedOrdinals = new Set();
|
|
2898
|
+
for (const tail of tailChunks) {
|
|
2899
|
+
for (const token of tail.addedTokens)
|
|
2900
|
+
touchedOrdinals.add(fnv1a(token) % partitions);
|
|
2901
|
+
for (const token of tail.removedTokens)
|
|
2902
|
+
touchedOrdinals.add(fnv1a(token) % partitions);
|
|
2903
|
+
}
|
|
2904
|
+
if (touchedOrdinals.size === 0)
|
|
2905
|
+
return { partitions, tokenCount: priorCount };
|
|
2906
|
+
// No threshold on how many partitions are touched. Even a tail that reaches most of them
|
|
2907
|
+
// still hashes only its own tokens, where the rewrite hashes every token in the table —
|
|
2908
|
+
// and that hashing, not the record I/O, is what a fold spends its time on.
|
|
2909
|
+
const ordinals = [...touchedOrdinals];
|
|
2910
|
+
const stored = await Promise.all(ordinals.map((ordinal) => requestResult(store.get(uniqueKeyBasePartKey(tableId, ordinal)))));
|
|
2911
|
+
const buckets = new Map();
|
|
2912
|
+
let sizeBefore = 0;
|
|
2913
|
+
for (let index = 0; index < ordinals.length; index += 1) {
|
|
2914
|
+
const value = stored[index];
|
|
2915
|
+
const tokens = new Set(value === undefined ? [] : asBasePartition(value));
|
|
2916
|
+
sizeBefore += tokens.size;
|
|
2917
|
+
buckets.set(ordinals[index] ?? 0, tokens);
|
|
2918
|
+
}
|
|
2919
|
+
// Replay in commit order, exactly as a full replay would: a token added then removed by a
|
|
2920
|
+
// later chunk must end up absent, and the reverse must end up present.
|
|
2921
|
+
for (const tail of tailChunks) {
|
|
2922
|
+
for (const token of tail.addedTokens)
|
|
2923
|
+
buckets.get(fnv1a(token) % partitions)?.add(token);
|
|
2924
|
+
for (const token of tail.removedTokens)
|
|
2925
|
+
buckets.get(fnv1a(token) % partitions)?.delete(token);
|
|
2926
|
+
}
|
|
2927
|
+
let sizeAfter = 0;
|
|
2928
|
+
for (const tokens of buckets.values())
|
|
2929
|
+
sizeAfter += tokens.size;
|
|
2930
|
+
const tokenCount = priorCount - sizeBefore + sizeAfter;
|
|
2931
|
+
// A size that now wants a different partition count has to be rehashed in full.
|
|
2932
|
+
if (Math.max(1, Math.ceil(tokenCount / UNIQUE_KEY_PARTITION_TARGET)) !== partitions) {
|
|
2933
|
+
return undefined;
|
|
2934
|
+
}
|
|
2935
|
+
for (const [ordinal, tokens] of buckets) {
|
|
2936
|
+
store.put([...tokens], uniqueKeyBasePartKey(tableId, ordinal));
|
|
2937
|
+
}
|
|
2938
|
+
return { partitions, tokenCount };
|
|
2939
|
+
}
|
|
2940
|
+
async function readAllV2BaseTokens(store, tableId, index) {
|
|
2941
|
+
if (!index.hasBase)
|
|
2942
|
+
return new Set();
|
|
2943
|
+
const partitions = index.partitions ?? 1;
|
|
2944
|
+
const parts = await Promise.all(Array.from({ length: partitions }, (_, ordinal) => requestResult(store.get(uniqueKeyBasePartKey(tableId, ordinal)))));
|
|
2945
|
+
const present = new Set();
|
|
2946
|
+
for (const part of parts) {
|
|
2947
|
+
if (part === undefined)
|
|
2948
|
+
continue;
|
|
2949
|
+
for (const token of asBasePartition(part))
|
|
2950
|
+
present.add(token);
|
|
2951
|
+
}
|
|
2952
|
+
return present;
|
|
2953
|
+
}
|
|
2954
|
+
/**
|
|
2955
|
+
* A table's complete unique-key membership: the folded base for whichever layout it uses, plus
|
|
2956
|
+
* every unfolded tail chunk replayed in commit order. Snapshots need the whole set, where the
|
|
2957
|
+
* commit path only ever needs the tokens it is about to write.
|
|
2958
|
+
*/
|
|
2959
|
+
async function readAllUniqueKeyTokens(store, tableId, mode) {
|
|
2960
|
+
if (mode !== "chunks-v1" && mode !== "chunks-v2") {
|
|
2961
|
+
// The pre-chunk layout stored one record per key and predates every shipped writer.
|
|
2962
|
+
return scanLegacyUniqueKeyTokens(store, tableId);
|
|
2963
|
+
}
|
|
2964
|
+
const index = asUniqueKeyChunkIndex(await requestResult(store.get(uniqueKeyChunkIndexKey(tableId))));
|
|
2965
|
+
const present = index.hasBase
|
|
2966
|
+
? mode === "chunks-v2"
|
|
2967
|
+
? await readAllV2BaseTokens(store, tableId, index)
|
|
2968
|
+
: await scanUniqueKeyBaseTokens(store, tableId)
|
|
2969
|
+
: new Set();
|
|
2970
|
+
const chunks = await Promise.all(index.versions.map((version) => requestResult(store.get(uniqueKeyChunkKey(tableId, version)))));
|
|
2971
|
+
for (const chunk of chunks)
|
|
2972
|
+
applyChunk(present, asUniqueKeyChunk(chunk));
|
|
2973
|
+
return present;
|
|
2974
|
+
}
|
|
2975
|
+
async function readChunkedUniqueKeys(store, tableId, requestedTokens, mode) {
|
|
2976
|
+
const rawIndex = await requestResult(store.get(uniqueKeyChunkIndexKey(tableId)));
|
|
2977
|
+
const index = asUniqueKeyChunkIndex(rawIndex);
|
|
2978
|
+
const bulk = requestedTokens.length >= UNIQUE_KEY_BASE_SCAN_THRESHOLD;
|
|
2979
|
+
const chunkReads = Promise.all(index.versions.map((version) => requestResult(store.get(uniqueKeyChunkKey(tableId, version)))));
|
|
2980
|
+
let baseMembership;
|
|
2981
|
+
let fullBase;
|
|
2982
|
+
let probedKeys = [];
|
|
2983
|
+
if (index.hasBase) {
|
|
2984
|
+
if (mode === "chunks-v2") {
|
|
2985
|
+
if (bulk) {
|
|
2986
|
+
fullBase = await readAllV2BaseTokens(store, tableId, index);
|
|
2987
|
+
baseMembership = fullBase;
|
|
2988
|
+
}
|
|
2989
|
+
else {
|
|
2990
|
+
// Point reads touch only the partitions the requested tokens hash to.
|
|
2991
|
+
const partitions = index.partitions ?? 1;
|
|
2992
|
+
const ordinals = [...new Set(requestedTokens.map((token) => fnv1a(token) % partitions))];
|
|
2993
|
+
const parts = await Promise.all(ordinals.map((ordinal) => requestResult(store.get(uniqueKeyBasePartKey(tableId, ordinal)))));
|
|
2994
|
+
const membership = new Set();
|
|
2995
|
+
for (const part of parts) {
|
|
2996
|
+
if (part === undefined)
|
|
2997
|
+
continue;
|
|
2998
|
+
for (const token of asBasePartition(part))
|
|
2999
|
+
membership.add(token);
|
|
3000
|
+
}
|
|
3001
|
+
baseMembership = membership;
|
|
3002
|
+
}
|
|
3003
|
+
}
|
|
3004
|
+
else if (bulk) {
|
|
3005
|
+
fullBase = await scanUniqueKeyBaseTokens(store, tableId);
|
|
3006
|
+
baseMembership = fullBase;
|
|
3007
|
+
}
|
|
3008
|
+
else {
|
|
3009
|
+
probedKeys = await Promise.all(requestedTokens.map((token) => requestResult(store.getKey(uniqueKeyBaseKey(tableId, token)))));
|
|
3010
|
+
}
|
|
3011
|
+
}
|
|
3012
|
+
const chunks = (await chunkReads).map(asUniqueKeyChunk);
|
|
3013
|
+
const requested = new Set(requestedTokens);
|
|
3014
|
+
const existing = new Set();
|
|
3015
|
+
if (baseMembership !== undefined) {
|
|
3016
|
+
for (const token of requestedTokens) {
|
|
3017
|
+
if (baseMembership.has(token))
|
|
3018
|
+
existing.add(token);
|
|
3019
|
+
}
|
|
3020
|
+
}
|
|
3021
|
+
else {
|
|
3022
|
+
requestedTokens.forEach((token, position) => {
|
|
3023
|
+
if (probedKeys[position] !== undefined)
|
|
3024
|
+
existing.add(token);
|
|
3025
|
+
});
|
|
3026
|
+
}
|
|
3027
|
+
for (const chunk of chunks) {
|
|
3028
|
+
for (const token of chunk.addedTokens) {
|
|
3029
|
+
if (requested.has(token))
|
|
3030
|
+
existing.add(token);
|
|
3031
|
+
}
|
|
3032
|
+
for (const token of chunk.removedTokens)
|
|
3033
|
+
existing.delete(token);
|
|
3034
|
+
}
|
|
3035
|
+
let fullPresent;
|
|
3036
|
+
if (fullBase !== undefined || !index.hasBase) {
|
|
3037
|
+
fullPresent = fullBase ?? new Set();
|
|
3038
|
+
for (const chunk of chunks)
|
|
3039
|
+
applyChunk(fullPresent, chunk);
|
|
3040
|
+
}
|
|
3041
|
+
return {
|
|
3042
|
+
existing,
|
|
3043
|
+
index,
|
|
3044
|
+
chunks,
|
|
3045
|
+
...(fullPresent === undefined ? {} : { fullPresent }),
|
|
3046
|
+
};
|
|
3047
|
+
}
|
|
3048
|
+
/**
|
|
3049
|
+
* One key-cursor pass over a table's folded base records. Seeks with cursor.continue(key)
|
|
3050
|
+
* rather than IDBKeyRange, which the injected test factory lacks; a seek that lands past the
|
|
3051
|
+
* range (or throws because the cursor is already beyond it) means the walk is complete.
|
|
3052
|
+
*/
|
|
3053
|
+
function scanUniqueKeyBaseTokens(store, tableId) {
|
|
3054
|
+
return scanUniqueKeyTokenRange(store, UNIQUE_KEY_BASE, tableId);
|
|
3055
|
+
}
|
|
3056
|
+
/** The same walk over the pre-chunk layout, where each key was its own record. */
|
|
3057
|
+
function scanLegacyUniqueKeyTokens(store, tableId) {
|
|
3058
|
+
return scanUniqueKeyTokenRange(store, "unique-key", tableId);
|
|
3059
|
+
}
|
|
3060
|
+
function scanUniqueKeyTokenRange(store, prefix, tableId) {
|
|
3061
|
+
return new Promise((resolve, reject) => {
|
|
3062
|
+
const tokens = new Set();
|
|
3063
|
+
const rangeStart = [prefix, tableId, ""];
|
|
3064
|
+
const request = store.openKeyCursor();
|
|
3065
|
+
let seeked = false;
|
|
3066
|
+
request.onerror = () => reject(request.error ?? new Error("IndexedDB cursor failed"));
|
|
3067
|
+
request.onsuccess = () => {
|
|
3068
|
+
const cursor = request.result;
|
|
3069
|
+
if (cursor === null) {
|
|
3070
|
+
resolve(tokens);
|
|
3071
|
+
return;
|
|
3072
|
+
}
|
|
3073
|
+
const key = cursor.key;
|
|
3074
|
+
if (Array.isArray(key) &&
|
|
3075
|
+
key[0] === prefix &&
|
|
3076
|
+
key[1] === tableId &&
|
|
3077
|
+
typeof key[2] === "string") {
|
|
3078
|
+
tokens.add(key[2]);
|
|
3079
|
+
cursor.continue();
|
|
3080
|
+
return;
|
|
3081
|
+
}
|
|
3082
|
+
if (!seeked) {
|
|
3083
|
+
seeked = true;
|
|
3084
|
+
try {
|
|
3085
|
+
cursor.continue(rangeStart);
|
|
3086
|
+
}
|
|
3087
|
+
catch {
|
|
3088
|
+
// The first visited key already sorts at or past the range start: nothing to walk.
|
|
3089
|
+
resolve(tokens);
|
|
3090
|
+
}
|
|
3091
|
+
return;
|
|
3092
|
+
}
|
|
3093
|
+
// Keys arrive in order, so the first non-matching key after the seek ends the range.
|
|
3094
|
+
resolve(tokens);
|
|
3095
|
+
};
|
|
3096
|
+
});
|
|
3097
|
+
}
|
|
3098
|
+
function asUniqueKeyChunk(value) {
|
|
3099
|
+
if (typeof value !== "object" || value === null) {
|
|
3100
|
+
throw new Error("Unique-key chunk is missing or invalid");
|
|
3101
|
+
}
|
|
3102
|
+
const record = value;
|
|
3103
|
+
const addedTokens = record.addedTokens;
|
|
3104
|
+
const removedTokens = record.removedTokens;
|
|
3105
|
+
if (!isStringArray(addedTokens) || !isStringArray(removedTokens)) {
|
|
3106
|
+
throw new Error("Unique-key chunk is invalid");
|
|
3107
|
+
}
|
|
3108
|
+
return { addedTokens, removedTokens };
|
|
3109
|
+
}
|
|
3110
|
+
function getGlobalIndexedDb() {
|
|
3111
|
+
return "indexedDB" in globalThis ? globalThis.indexedDB : undefined;
|
|
3112
|
+
}
|
|
3113
|
+
function logicalStoredBytes(value, seen = new Set()) {
|
|
3114
|
+
if (value === null || value === undefined)
|
|
3115
|
+
return 0;
|
|
3116
|
+
if (typeof value === "string")
|
|
3117
|
+
return utf8Bytes(value);
|
|
3118
|
+
if (typeof value === "number" || typeof value === "bigint")
|
|
3119
|
+
return 8;
|
|
3120
|
+
if (typeof value === "boolean")
|
|
3121
|
+
return 1;
|
|
3122
|
+
if (value instanceof Date)
|
|
3123
|
+
return 8;
|
|
3124
|
+
if (value instanceof ArrayBuffer)
|
|
3125
|
+
return value.byteLength;
|
|
3126
|
+
if (ArrayBuffer.isView(value))
|
|
3127
|
+
return value.byteLength;
|
|
3128
|
+
if (typeof value !== "object" || seen.has(value))
|
|
3129
|
+
return 0;
|
|
3130
|
+
seen.add(value);
|
|
3131
|
+
if (Array.isArray(value)) {
|
|
3132
|
+
return value.reduce((bytes, entry) => bytes + logicalStoredBytes(entry, seen), 0);
|
|
3133
|
+
}
|
|
3134
|
+
return Object.entries(value).reduce((bytes, [key, entry]) => bytes + logicalStoredBytes(key, seen) + logicalStoredBytes(entry, seen), 0);
|
|
3135
|
+
}
|
|
3136
|
+
function isStringArray(value) {
|
|
3137
|
+
return Array.isArray(value) && value.every((entry) => typeof entry === "string");
|
|
3138
|
+
}
|
|
3139
|
+
function logicalObjectStoreBytes(store) {
|
|
3140
|
+
return new Promise((resolve, reject) => {
|
|
3141
|
+
let total = 0;
|
|
3142
|
+
const request = store.openCursor();
|
|
3143
|
+
request.addEventListener("success", () => {
|
|
3144
|
+
const cursor = request.result;
|
|
3145
|
+
if (cursor === null) {
|
|
3146
|
+
resolve(total);
|
|
3147
|
+
return;
|
|
3148
|
+
}
|
|
3149
|
+
total += logicalStoredBytes(cursor.key) + logicalStoredBytes(cursor.value);
|
|
3150
|
+
cursor.continue();
|
|
3151
|
+
});
|
|
3152
|
+
request.addEventListener("error", () => reject(request.error ?? new Error("IndexedDB size cursor failed")), { once: true });
|
|
3153
|
+
});
|
|
3154
|
+
}
|
|
3155
|
+
function utf8Bytes(value) {
|
|
3156
|
+
if (value.length * 3 <= storageTextBuffer.byteLength) {
|
|
3157
|
+
return storageTextEncoder.encodeInto(value, storageTextBuffer).written;
|
|
3158
|
+
}
|
|
3159
|
+
return storageTextEncoder.encode(value).byteLength;
|
|
3160
|
+
}
|
|
3161
|
+
//# sourceMappingURL=indexeddb.js.map
|