@minnowdb/core 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +32 -18
- package/dist/engine/client.d.ts +9 -1
- package/dist/engine/client.d.ts.map +1 -1
- package/dist/engine/client.js +29 -5
- package/dist/engine/client.js.map +1 -1
- package/dist/engine/database.d.ts +51 -6
- package/dist/engine/database.d.ts.map +1 -1
- package/dist/engine/database.js +2597 -865
- package/dist/engine/database.js.map +1 -1
- package/dist/engine/live.d.ts.map +1 -1
- package/dist/engine/live.js +35 -11
- package/dist/engine/live.js.map +1 -1
- package/dist/engine/query-cache.d.ts +13 -1
- package/dist/engine/query-cache.d.ts.map +1 -1
- package/dist/engine/query-cache.js +34 -5
- package/dist/engine/query-cache.js.map +1 -1
- package/dist/engine/query.d.ts +11 -1
- package/dist/engine/query.d.ts.map +1 -1
- package/dist/engine/query.js +49 -41
- package/dist/engine/query.js.map +1 -1
- package/dist/engine/result-wire.d.ts +68 -0
- package/dist/engine/result-wire.d.ts.map +1 -0
- package/dist/engine/result-wire.js +264 -0
- package/dist/engine/result-wire.js.map +1 -0
- package/dist/engine/sort-keys.d.ts +50 -12
- package/dist/engine/sort-keys.d.ts.map +1 -1
- package/dist/engine/sort-keys.js +371 -25
- package/dist/engine/sort-keys.js.map +1 -1
- package/dist/engine/vector.d.ts +11 -0
- package/dist/engine/vector.d.ts.map +1 -1
- package/dist/engine/vector.js +99 -203
- package/dist/engine/vector.js.map +1 -1
- package/dist/engine/worker-host.d.ts +17 -1
- package/dist/engine/worker-host.d.ts.map +1 -1
- package/dist/engine/worker-host.js +83 -13
- package/dist/engine/worker-host.js.map +1 -1
- package/dist/storage/index.d.ts +13 -0
- package/dist/storage/index.d.ts.map +1 -1
- package/dist/storage/index.js +13 -0
- package/dist/storage/index.js.map +1 -1
- package/dist/storage/indexeddb.d.ts +5 -1
- package/dist/storage/indexeddb.d.ts.map +1 -1
- package/dist/storage/indexeddb.js +484 -402
- package/dist/storage/indexeddb.js.map +1 -1
- package/dist/storage/memory.d.ts +24 -21
- package/dist/storage/memory.d.ts.map +1 -1
- package/dist/storage/memory.js +124 -1199
- package/dist/storage/memory.js.map +1 -1
- package/dist/storage/opfs/files.d.ts +64 -0
- package/dist/storage/opfs/files.d.ts.map +1 -0
- package/dist/storage/opfs/files.js +243 -0
- package/dist/storage/opfs/files.js.map +1 -0
- package/dist/storage/opfs/index.d.ts +3 -0
- package/dist/storage/opfs/index.d.ts.map +1 -0
- package/dist/storage/opfs/index.js +3 -0
- package/dist/storage/opfs/index.js.map +1 -0
- package/dist/storage/opfs/leader.d.ts +130 -0
- package/dist/storage/opfs/leader.d.ts.map +1 -0
- package/dist/storage/opfs/leader.js +1104 -0
- package/dist/storage/opfs/leader.js.map +1 -0
- package/dist/storage/opfs/rpc.d.ts +70 -0
- package/dist/storage/opfs/rpc.d.ts.map +1 -0
- package/dist/storage/opfs/rpc.js +58 -0
- package/dist/storage/opfs/rpc.js.map +1 -0
- package/dist/storage/opfs/store.d.ts +154 -0
- package/dist/storage/opfs/store.d.ts.map +1 -0
- package/dist/storage/opfs/store.js +958 -0
- package/dist/storage/opfs/store.js.map +1 -0
- package/dist/storage/toolkit/extents.d.ts +69 -0
- package/dist/storage/toolkit/extents.d.ts.map +1 -0
- package/dist/storage/toolkit/extents.js +175 -0
- package/dist/storage/toolkit/extents.js.map +1 -0
- package/dist/storage/toolkit/index.d.ts +30 -0
- package/dist/storage/toolkit/index.d.ts.map +1 -0
- package/dist/storage/toolkit/index.js +5 -0
- package/dist/storage/toolkit/index.js.map +1 -0
- package/dist/storage/toolkit/record-core.d.ts +252 -0
- package/dist/storage/toolkit/record-core.d.ts.map +1 -0
- package/dist/storage/toolkit/record-core.js +1670 -0
- package/dist/storage/toolkit/record-core.js.map +1 -0
- package/dist/storage/toolkit/sync-file.d.ts +29 -0
- package/dist/storage/toolkit/sync-file.d.ts.map +1 -0
- package/dist/storage/toolkit/sync-file.js +2 -0
- package/dist/storage/toolkit/sync-file.js.map +1 -0
- package/dist/storage/toolkit/wal.d.ts +22 -0
- package/dist/storage/toolkit/wal.d.ts.map +1 -0
- package/dist/storage/toolkit/wal.js +79 -0
- package/dist/storage/toolkit/wal.js.map +1 -0
- package/dist/storage/toolkit/wire.d.ts +25 -0
- package/dist/storage/toolkit/wire.d.ts.map +1 -0
- package/dist/storage/toolkit/wire.js +100 -0
- package/dist/storage/toolkit/wire.js.map +1 -0
- package/dist/storage/types.d.ts +343 -59
- package/dist/storage/types.d.ts.map +1 -1
- package/dist/storage/types.js +112 -11
- package/dist/storage/types.js.map +1 -1
- package/dist/testing/block-store-conformance.d.ts +51 -0
- package/dist/testing/block-store-conformance.d.ts.map +1 -0
- package/dist/testing/block-store-conformance.js +745 -0
- package/dist/testing/block-store-conformance.js.map +1 -0
- package/dist/testing/index.d.ts +3 -0
- package/dist/testing/index.d.ts.map +1 -1
- package/dist/testing/index.js +5 -0
- package/dist/testing/index.js.map +1 -1
- package/dist/testing/opfs-shim.d.ts +35 -0
- package/dist/testing/opfs-shim.d.ts.map +1 -0
- package/dist/testing/opfs-shim.js +295 -0
- package/dist/testing/opfs-shim.js.map +1 -0
- package/dist/transactions/index.d.ts +42 -1
- package/dist/transactions/index.d.ts.map +1 -1
- package/dist/transactions/index.js +219 -13
- package/dist/transactions/index.js.map +1 -1
- package/dist/worker-protocol/index.d.ts +2 -2
- package/dist/worker-protocol/index.d.ts.map +1 -1
- package/dist/worker-protocol/index.js +2 -1
- package/dist/worker-protocol/index.js.map +1 -1
- package/package.json +10 -2
- package/dist/engine/dsl/db.d.ts +0 -137
- package/dist/engine/dsl/db.d.ts.map +0 -1
- package/dist/engine/dsl/db.js +0 -161
- package/dist/engine/dsl/db.js.map +0 -1
- package/dist/engine/dsl/expression.d.ts +0 -172
- package/dist/engine/dsl/expression.d.ts.map +0 -1
- package/dist/engine/dsl/expression.js +0 -397
- package/dist/engine/dsl/expression.js.map +0 -1
- package/dist/engine/dsl/index.d.ts +0 -8
- package/dist/engine/dsl/index.d.ts.map +0 -1
- package/dist/engine/dsl/index.js +0 -7
- package/dist/engine/dsl/index.js.map +0 -1
- package/dist/engine/dsl/live-query.d.ts +0 -45
- package/dist/engine/dsl/live-query.d.ts.map +0 -1
- package/dist/engine/dsl/live-query.js +0 -116
- package/dist/engine/dsl/live-query.js.map +0 -1
- package/dist/engine/dsl/mutations.d.ts +0 -123
- package/dist/engine/dsl/mutations.d.ts.map +0 -1
- package/dist/engine/dsl/mutations.js +0 -241
- package/dist/engine/dsl/mutations.js.map +0 -1
- package/dist/engine/dsl/select-query-builder.d.ts +0 -148
- package/dist/engine/dsl/select-query-builder.d.ts.map +0 -1
- package/dist/engine/dsl/select-query-builder.js +0 -377
- package/dist/engine/dsl/select-query-builder.js.map +0 -1
- package/dist/engine/dsl/sql-tag.d.ts +0 -46
- package/dist/engine/dsl/sql-tag.d.ts.map +0 -1
- package/dist/engine/dsl/sql-tag.js +0 -85
- package/dist/engine/dsl/sql-tag.js.map +0 -1
- package/dist/engine/dsl/types.d.ts +0 -133
- package/dist/engine/dsl/types.d.ts.map +0 -1
- package/dist/engine/dsl/types.js +0 -9
- package/dist/engine/dsl/types.js.map +0 -1
|
@@ -0,0 +1,1670 @@
|
|
|
1
|
+
import { CompactionJobConflictError, createManifest, createGarbageCollectionJobRecord, advanceGarbageCollectionJobRecord, collectFtsCandidates, invalidateUncoveredFtsColumns, GarbageCollectionJobConflictError, LeaseConflictError, SnapshotManifestMissingError, TableRecordConflictError, TempOwnerConflictError, TransactionRecordConflictError, UniqueKeyConflictError, normalizeCompactionJobRecord, normalizeSegmentRecord, updateCompactionJobRecord, updateTransactionRecord, WriteConflictError, } from "../types.js";
|
|
2
|
+
import { selectLiveRecords } from "../snapshot.js";
|
|
3
|
+
/**
|
|
4
|
+
* The synchronous record-state machine behind `MemoryBlockStore` and `OpfsBlockStore`, offered
|
|
5
|
+
* to adapter authors as the semantics half of a `BlockStore`: every record family the contract
|
|
6
|
+
* names, every validation order, every typed conflict error, every defensive copy — one method
|
|
7
|
+
* body per contract operation, so an adapter that wraps it inherits the behaviour the
|
|
8
|
+
* conformance kit checks and spends its own code purely on persistence.
|
|
9
|
+
*
|
|
10
|
+
* What wrapping it requires of the adapter:
|
|
11
|
+
*
|
|
12
|
+
* - **One writer at a time.** Methods validate then mutate synchronously; they are only safe
|
|
13
|
+
* when calls cannot interleave. Serialize access — a promise-chain queue (the memory store),
|
|
14
|
+
* a leader that is provably the sole writer (the OPFS store), a process-wide lock.
|
|
15
|
+
* - **Durability is yours.** The core is memory. Record each mutation (a write-ahead-log frame
|
|
16
|
+
* of the method and its input), checkpoint with `dump()`/`load()`, and replay on open.
|
|
17
|
+
* - **Block bytes are yours.** The core tracks records; the `PhysicalBlocks` seam asks you
|
|
18
|
+
* whether payload bytes exist and how large they are.
|
|
19
|
+
*
|
|
20
|
+
* Changing behaviour here changes every store built on it — which is the point.
|
|
21
|
+
*/
|
|
22
|
+
export class RecordCore {
|
|
23
|
+
#physical;
|
|
24
|
+
#manifests = new Map();
|
|
25
|
+
#transactions = new Map();
|
|
26
|
+
#tables = new Map();
|
|
27
|
+
#tableIdsByName = new Map();
|
|
28
|
+
#segments = new Map();
|
|
29
|
+
#leases = new Map();
|
|
30
|
+
#compactionJobs = new Map();
|
|
31
|
+
#garbageCollectionJobs = new Map();
|
|
32
|
+
#nextRowIds = new Map();
|
|
33
|
+
#nextAutoIncrement = new Map();
|
|
34
|
+
#ftsBases = new Map();
|
|
35
|
+
#ftsDeltas = new Map();
|
|
36
|
+
#uniqueKeys = new Map();
|
|
37
|
+
#tempOwners = new Map();
|
|
38
|
+
#currentVersion = null;
|
|
39
|
+
/** Advances on every catalog mutation; see `CatalogProbe` for the freshness contract. */
|
|
40
|
+
#catalogEpoch = 0;
|
|
41
|
+
constructor(physical) {
|
|
42
|
+
this.#physical = physical;
|
|
43
|
+
}
|
|
44
|
+
createTempOwner(record) {
|
|
45
|
+
validateTempOwnerRecord(record);
|
|
46
|
+
if (this.#tempOwners.has(record.ownerId)) {
|
|
47
|
+
throw new Error(`Temp owner already exists: ${record.ownerId}`);
|
|
48
|
+
}
|
|
49
|
+
this.#tempOwners.set(record.ownerId, structuredClone(record));
|
|
50
|
+
}
|
|
51
|
+
getTempOwner(ownerId) {
|
|
52
|
+
validateId(ownerId);
|
|
53
|
+
const record = this.#tempOwners.get(ownerId);
|
|
54
|
+
return record === undefined ? undefined : structuredClone(record);
|
|
55
|
+
}
|
|
56
|
+
renewTempOwner(ownerId, expectedRevision, expiresAt) {
|
|
57
|
+
validateLeaseExpiration(expiresAt);
|
|
58
|
+
const record = this.#tempOwners.get(ownerId);
|
|
59
|
+
if (record?.revision !== expectedRevision) {
|
|
60
|
+
throw new TempOwnerConflictError(ownerId, expectedRevision, record?.revision ?? null);
|
|
61
|
+
}
|
|
62
|
+
const renewed = { ...record, expiresAt, revision: record.revision + 1 };
|
|
63
|
+
this.#tempOwners.set(ownerId, renewed);
|
|
64
|
+
return structuredClone(renewed);
|
|
65
|
+
}
|
|
66
|
+
/**
|
|
67
|
+
* Removes the owner record when it is absent or expired, and reports whether the caller
|
|
68
|
+
* should also remove the owner's physical pages. Page bytes are the enclosing store's.
|
|
69
|
+
*/
|
|
70
|
+
removeTempOwnerIfExpired(ownerId, expiresAtCutoff) {
|
|
71
|
+
validateId(ownerId);
|
|
72
|
+
const cutoff = Date.parse(expiresAtCutoff);
|
|
73
|
+
if (!Number.isFinite(cutoff))
|
|
74
|
+
throw new TypeError("Temp owner expiry cutoff must be valid");
|
|
75
|
+
const record = this.#tempOwners.get(ownerId);
|
|
76
|
+
if (record !== undefined) {
|
|
77
|
+
const expiresAt = Date.parse(record.expiresAt);
|
|
78
|
+
if (Number.isFinite(expiresAt) && expiresAt > cutoff)
|
|
79
|
+
return false;
|
|
80
|
+
}
|
|
81
|
+
this.#tempOwners.delete(ownerId);
|
|
82
|
+
return true;
|
|
83
|
+
}
|
|
84
|
+
removeTempOwner(ownerId) {
|
|
85
|
+
validateId(ownerId);
|
|
86
|
+
this.#tempOwners.delete(ownerId);
|
|
87
|
+
}
|
|
88
|
+
/**
|
|
89
|
+
* Pages through owner IDs. `extraOwnerIds` carries owners the enclosing store discovered from
|
|
90
|
+
* physical pages that have no record — orphaned spill runs still need to be listed so the
|
|
91
|
+
* expiry sweep can reclaim them.
|
|
92
|
+
*/
|
|
93
|
+
listTempOwnerIdsPage(afterOwnerId, limit, extraOwnerIds) {
|
|
94
|
+
validatePageLimit(limit);
|
|
95
|
+
const ownerIds = new Set(this.#tempOwners.keys());
|
|
96
|
+
for (const ownerId of extraOwnerIds)
|
|
97
|
+
ownerIds.add(ownerId);
|
|
98
|
+
const sorted = [...ownerIds]
|
|
99
|
+
.filter((ownerId) => afterOwnerId === null || ownerId > afterOwnerId)
|
|
100
|
+
.sort();
|
|
101
|
+
const records = sorted.slice(0, limit);
|
|
102
|
+
return {
|
|
103
|
+
records,
|
|
104
|
+
nextCursor: sorted.length > limit ? (records[records.length - 1] ?? null) : null,
|
|
105
|
+
};
|
|
106
|
+
}
|
|
107
|
+
addTable(record) {
|
|
108
|
+
if (this.#tables.has(record.id))
|
|
109
|
+
throw new Error(`Table already exists: ${record.id}`);
|
|
110
|
+
if (this.#tableIdsByName.has(record.name))
|
|
111
|
+
throw new Error(`Table name already exists: ${record.name}`);
|
|
112
|
+
this.#tables.set(record.id, structuredClone(record));
|
|
113
|
+
this.#tableIdsByName.set(record.name, record.id);
|
|
114
|
+
this.#catalogEpoch += 1;
|
|
115
|
+
}
|
|
116
|
+
getTable(id) {
|
|
117
|
+
const record = this.#tables.get(id);
|
|
118
|
+
return record === undefined ? undefined : structuredClone(record);
|
|
119
|
+
}
|
|
120
|
+
updateTable(id, expectedRevision, update) {
|
|
121
|
+
const record = this.#tables.get(id);
|
|
122
|
+
const actualRevision = record === undefined ? null : (record.revision ?? 0);
|
|
123
|
+
if (record === undefined || actualRevision !== expectedRevision) {
|
|
124
|
+
throw new TableRecordConflictError(id, expectedRevision, actualRevision);
|
|
125
|
+
}
|
|
126
|
+
if (update.columns !== undefined)
|
|
127
|
+
validateTableColumns(update.columns);
|
|
128
|
+
const { ftsColumns: previousFts, triggers: previousTriggers, ...base } = record;
|
|
129
|
+
const nextFts = update.ftsColumns === undefined ? previousFts : update.ftsColumns;
|
|
130
|
+
const nextTriggers = update.triggers === undefined ? previousTriggers : update.triggers;
|
|
131
|
+
const updated = {
|
|
132
|
+
...base,
|
|
133
|
+
columns: update.columns === undefined ? record.columns : structuredClone(update.columns),
|
|
134
|
+
...(nextFts === null || nextFts === undefined
|
|
135
|
+
? {}
|
|
136
|
+
: { ftsColumns: structuredClone(nextFts) }),
|
|
137
|
+
...(nextTriggers === null || nextTriggers === undefined
|
|
138
|
+
? {}
|
|
139
|
+
: { triggers: structuredClone(nextTriggers) }),
|
|
140
|
+
revision: expectedRevision + 1,
|
|
141
|
+
};
|
|
142
|
+
this.#tables.set(id, updated);
|
|
143
|
+
this.#catalogEpoch += 1;
|
|
144
|
+
return structuredClone(updated);
|
|
145
|
+
}
|
|
146
|
+
removeTable(id, expectedRevision) {
|
|
147
|
+
const record = this.#tables.get(id);
|
|
148
|
+
const actualRevision = record === undefined ? null : (record.revision ?? 0);
|
|
149
|
+
if (record === undefined || actualRevision !== expectedRevision) {
|
|
150
|
+
throw new TableRecordConflictError(id, expectedRevision, actualRevision);
|
|
151
|
+
}
|
|
152
|
+
for (const [segmentId, segment] of this.#segments) {
|
|
153
|
+
if (segment.tableId === id)
|
|
154
|
+
this.#segments.delete(segmentId);
|
|
155
|
+
}
|
|
156
|
+
const owned = `${id}/`;
|
|
157
|
+
for (const key of [...this.#ftsBases.keys()]) {
|
|
158
|
+
if (key.startsWith(owned))
|
|
159
|
+
this.#ftsBases.delete(key);
|
|
160
|
+
}
|
|
161
|
+
for (const key of [...this.#ftsDeltas.keys()]) {
|
|
162
|
+
if (key.startsWith(owned))
|
|
163
|
+
this.#ftsDeltas.delete(key);
|
|
164
|
+
}
|
|
165
|
+
for (const key of [...this.#nextAutoIncrement.keys()]) {
|
|
166
|
+
if (key.startsWith(owned))
|
|
167
|
+
this.#nextAutoIncrement.delete(key);
|
|
168
|
+
}
|
|
169
|
+
this.#uniqueKeys.delete(id);
|
|
170
|
+
this.#nextRowIds.delete(id);
|
|
171
|
+
this.#tableIdsByName.delete(record.name);
|
|
172
|
+
this.#tables.delete(id);
|
|
173
|
+
this.#catalogEpoch += 1;
|
|
174
|
+
}
|
|
175
|
+
writeFtsBase(tableId, columnId, input) {
|
|
176
|
+
const key = `${tableId}/${columnId}`;
|
|
177
|
+
this.#ftsBases.set(key, structuredClone(input));
|
|
178
|
+
const deltas = this.#ftsDeltas.get(key);
|
|
179
|
+
if (deltas !== undefined) {
|
|
180
|
+
for (const version of [...deltas.keys()]) {
|
|
181
|
+
if (version <= input.coversVersion)
|
|
182
|
+
deltas.delete(version);
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
readFtsCandidates(tableId, columnId, terms, upToVersion) {
|
|
187
|
+
const key = `${tableId}/${columnId}`;
|
|
188
|
+
const base = this.#ftsBases.get(key);
|
|
189
|
+
const deltas = this.#ftsDeltas.get(key) ??
|
|
190
|
+
new Map();
|
|
191
|
+
const chunkLists = [...(base?.chunks ?? [])];
|
|
192
|
+
let deltaChunkCount = 0;
|
|
193
|
+
let totalTokens = base?.totalTokens ?? 0;
|
|
194
|
+
for (const [version, delta] of deltas) {
|
|
195
|
+
if (version <= (base?.coversVersion ?? -1) || version > upToVersion)
|
|
196
|
+
continue;
|
|
197
|
+
deltaChunkCount += 1;
|
|
198
|
+
totalTokens += delta.totalTokens;
|
|
199
|
+
chunkLists.push(delta.postings);
|
|
200
|
+
}
|
|
201
|
+
return {
|
|
202
|
+
...collectFtsCandidates(chunkLists, terms),
|
|
203
|
+
deltaChunkCount,
|
|
204
|
+
totalTokens,
|
|
205
|
+
coversVersion: base?.coversVersion ?? -1,
|
|
206
|
+
};
|
|
207
|
+
}
|
|
208
|
+
/**
|
|
209
|
+
* Applies a commit's full-text deltas, and closes the stale-writer race: a commit adding
|
|
210
|
+
* segments to a table with an active index but no delta for one of its columns flips that
|
|
211
|
+
* column to "invalid" so a rebuild self-heals instead of the data commit failing.
|
|
212
|
+
*/
|
|
213
|
+
#applyFtsChanges(pendingSegments, changeList, version) {
|
|
214
|
+
const changedTableIds = new Set(pendingSegments.map((segment) => segment.tableId));
|
|
215
|
+
for (const tableId of changedTableIds) {
|
|
216
|
+
const record = this.#tables.get(tableId);
|
|
217
|
+
if (record === undefined)
|
|
218
|
+
continue;
|
|
219
|
+
const forTable = (changeList ?? []).find((entry) => entry.tableId === tableId);
|
|
220
|
+
const covered = new Set(forTable?.columns.map((column) => column.columnId) ?? []);
|
|
221
|
+
const invalidated = invalidateUncoveredFtsColumns(record, covered);
|
|
222
|
+
if (invalidated !== undefined)
|
|
223
|
+
this.#tables.set(record.id, invalidated);
|
|
224
|
+
}
|
|
225
|
+
for (const changes of changeList ?? [])
|
|
226
|
+
this.#applyFtsEntry(changes, version);
|
|
227
|
+
}
|
|
228
|
+
#applyFtsEntry(changes, version) {
|
|
229
|
+
for (const column of changes.columns) {
|
|
230
|
+
const key = `${changes.tableId}/${column.columnId}`;
|
|
231
|
+
const deltas = this.#ftsDeltas.get(key) ??
|
|
232
|
+
new Map();
|
|
233
|
+
deltas.set(version, {
|
|
234
|
+
postings: structuredClone(column.postings),
|
|
235
|
+
totalTokens: column.totalTokens,
|
|
236
|
+
});
|
|
237
|
+
this.#ftsDeltas.set(key, deltas);
|
|
238
|
+
}
|
|
239
|
+
}
|
|
240
|
+
/**
|
|
241
|
+
* The delta half of a full-text read, for stores that keep base chunks outside the core
|
|
242
|
+
* (the OPFS store keeps them as immutable files). Mirrors `readFtsCandidates`' delta
|
|
243
|
+
* selection exactly: versions above the base's coverage, up to the read's snapshot.
|
|
244
|
+
*/
|
|
245
|
+
readFtsDeltas(tableId, columnId, coversVersion, upToVersion) {
|
|
246
|
+
const deltas = this.#ftsDeltas.get(`${tableId}/${columnId}`);
|
|
247
|
+
const chunkLists = [];
|
|
248
|
+
let deltaChunkCount = 0;
|
|
249
|
+
let deltaTokens = 0;
|
|
250
|
+
for (const [version, delta] of deltas ?? []) {
|
|
251
|
+
if (version <= coversVersion || version > upToVersion)
|
|
252
|
+
continue;
|
|
253
|
+
deltaChunkCount += 1;
|
|
254
|
+
deltaTokens += delta.totalTokens;
|
|
255
|
+
chunkLists.push(structuredClone(delta.postings));
|
|
256
|
+
}
|
|
257
|
+
return { chunkLists, deltaChunkCount, deltaTokens };
|
|
258
|
+
}
|
|
259
|
+
/** The delta-pruning half of `writeFtsBase`, for stores that keep the base itself outside. */
|
|
260
|
+
pruneFtsDeltas(tableId, columnId, coversVersion) {
|
|
261
|
+
const deltas = this.#ftsDeltas.get(`${tableId}/${columnId}`);
|
|
262
|
+
if (deltas === undefined)
|
|
263
|
+
return;
|
|
264
|
+
for (const version of [...deltas.keys()]) {
|
|
265
|
+
if (version <= coversVersion)
|
|
266
|
+
deltas.delete(version);
|
|
267
|
+
}
|
|
268
|
+
}
|
|
269
|
+
/**
|
|
270
|
+
* Applies a garbage-collection step's already-resolved record effect: the advanced job
|
|
271
|
+
* record, manifest tombstones, and segment removals — exactly the mutations
|
|
272
|
+
* `runGarbageCollectionStep` makes, reproduced from its outcome. The OPFS store logs the
|
|
273
|
+
* outcome rather than the request so replaying is deterministic without the executor's
|
|
274
|
+
* physical file knowledge.
|
|
275
|
+
*/
|
|
276
|
+
applyGarbageCollectionEffect(effect) {
|
|
277
|
+
for (const version of effect.prunedManifestVersions) {
|
|
278
|
+
const manifest = this.#manifests.get(version);
|
|
279
|
+
if (manifest !== undefined) {
|
|
280
|
+
this.#manifests.set(version, { ...manifest, prunedAt: effect.updatedAt });
|
|
281
|
+
}
|
|
282
|
+
}
|
|
283
|
+
for (const id of effect.reclaimedSegmentIds)
|
|
284
|
+
this.#segments.delete(id);
|
|
285
|
+
for (const id of effect.reclaimedTransactionIds ?? [])
|
|
286
|
+
this.#transactions.delete(id);
|
|
287
|
+
this.#garbageCollectionJobs.set(effect.job.id, structuredClone(effect.job));
|
|
288
|
+
}
|
|
289
|
+
getTableByName(name) {
|
|
290
|
+
const id = this.#tableIdsByName.get(name);
|
|
291
|
+
return id === undefined ? undefined : this.getTable(id);
|
|
292
|
+
}
|
|
293
|
+
/** For stores that compose lookups over their own overridable methods. */
|
|
294
|
+
getTableIdByName(name) {
|
|
295
|
+
return this.#tableIdsByName.get(name);
|
|
296
|
+
}
|
|
297
|
+
listTables() {
|
|
298
|
+
return [...this.#tables.values()]
|
|
299
|
+
.sort((left, right) => left.name.localeCompare(right.name))
|
|
300
|
+
.map((record) => structuredClone(record));
|
|
301
|
+
}
|
|
302
|
+
addSegment(record) {
|
|
303
|
+
const normalized = normalizeSegmentRecord(record);
|
|
304
|
+
if (this.#segments.has(normalized.id)) {
|
|
305
|
+
throw new Error(`Segment already exists: ${normalized.id}`);
|
|
306
|
+
}
|
|
307
|
+
this.#segments.set(normalized.id, normalized);
|
|
308
|
+
}
|
|
309
|
+
getSegment(id) {
|
|
310
|
+
const record = this.#segments.get(id);
|
|
311
|
+
return record === undefined ? undefined : normalizeSegmentRecord(record);
|
|
312
|
+
}
|
|
313
|
+
listSegments(tableId) {
|
|
314
|
+
return [...this.#segments.values()]
|
|
315
|
+
.filter((record) => tableId === undefined || record.tableId === tableId)
|
|
316
|
+
.sort((left, right) => left.id.localeCompare(right.id))
|
|
317
|
+
.map((record) => normalizeSegmentRecord(record));
|
|
318
|
+
}
|
|
319
|
+
removeSegment(id) {
|
|
320
|
+
this.#segments.delete(id);
|
|
321
|
+
}
|
|
322
|
+
reserveRowIds(tableId, count) {
|
|
323
|
+
validateCount(count);
|
|
324
|
+
const start = this.#nextRowIds.get(tableId) ?? 1n;
|
|
325
|
+
const endExclusive = start + BigInt(count);
|
|
326
|
+
this.#nextRowIds.set(tableId, endExclusive);
|
|
327
|
+
return { start, endExclusive };
|
|
328
|
+
}
|
|
329
|
+
reserveAutoIncrement(tableId, columnId, count, atLeast) {
|
|
330
|
+
validateAutoIncrementReservation(count, atLeast);
|
|
331
|
+
return this.#reserveAutoIncrement(tableId, columnId, count, atLeast);
|
|
332
|
+
}
|
|
333
|
+
#reserveAutoIncrement(tableId, columnId, count, atLeast) {
|
|
334
|
+
const key = `${tableId}/${columnId}`;
|
|
335
|
+
const current = this.#nextAutoIncrement.get(key) ?? 1n;
|
|
336
|
+
const floor = atLeast ?? 1n;
|
|
337
|
+
const start = current > floor ? current : floor;
|
|
338
|
+
const endExclusive = start + BigInt(count);
|
|
339
|
+
this.#nextAutoIncrement.set(key, endExclusive);
|
|
340
|
+
return { start, endExclusive };
|
|
341
|
+
}
|
|
342
|
+
getExistingUniqueKeys(tableId, keyTokens) {
|
|
343
|
+
const existing = this.#uniqueKeys.get(tableId);
|
|
344
|
+
if (existing === undefined)
|
|
345
|
+
return [];
|
|
346
|
+
return [...new Set(keyTokens)].filter((token) => existing.has(token)).sort();
|
|
347
|
+
}
|
|
348
|
+
getCurrentManifestVersion() {
|
|
349
|
+
return this.#currentVersion;
|
|
350
|
+
}
|
|
351
|
+
getCatalogProbe() {
|
|
352
|
+
return { manifestVersion: this.#currentVersion, catalogEpoch: this.#catalogEpoch };
|
|
353
|
+
}
|
|
354
|
+
getQueryCatalogState(tableNames) {
|
|
355
|
+
const tables = tableNames.map((name) => this.getTableByName(name));
|
|
356
|
+
const foundTableIds = new Set(tables.filter((table) => table !== undefined).map((table) => table.id));
|
|
357
|
+
// Filter raw records first: normalizing and sorting every segment in the database to
|
|
358
|
+
// keep a handful is the wrong order of operations on a hot path.
|
|
359
|
+
const segments = [...this.#segments.values()]
|
|
360
|
+
.filter((record) => foundTableIds.has(record.tableId))
|
|
361
|
+
.sort((left, right) => left.id.localeCompare(right.id))
|
|
362
|
+
.map((record) => normalizeSegmentRecord(record));
|
|
363
|
+
const transactionIds = [...new Set(segments.map((segment) => segment.transactionId))];
|
|
364
|
+
const transactions = this.getTransactions(transactionIds).filter((record) => record !== undefined);
|
|
365
|
+
return {
|
|
366
|
+
manifestVersion: this.#currentVersion,
|
|
367
|
+
tables,
|
|
368
|
+
segments,
|
|
369
|
+
transactions,
|
|
370
|
+
catalogEpoch: this.#catalogEpoch,
|
|
371
|
+
};
|
|
372
|
+
}
|
|
373
|
+
getCurrentManifest() {
|
|
374
|
+
const manifest = this.#currentVersion === null ? undefined : this.#manifests.get(this.#currentVersion);
|
|
375
|
+
return manifest === undefined ? undefined : structuredClone(manifest);
|
|
376
|
+
}
|
|
377
|
+
getManifest(version) {
|
|
378
|
+
const manifest = this.#manifests.get(version);
|
|
379
|
+
return manifest === undefined ? undefined : structuredClone(manifest);
|
|
380
|
+
}
|
|
381
|
+
listManifests() {
|
|
382
|
+
return [...this.#manifests.values()]
|
|
383
|
+
.sort((left, right) => left.version - right.version)
|
|
384
|
+
.map((manifest) => structuredClone(manifest));
|
|
385
|
+
}
|
|
386
|
+
listManifestPage(afterVersion, limit) {
|
|
387
|
+
validatePageLimit(limit);
|
|
388
|
+
const records = [...this.#manifests.values()]
|
|
389
|
+
.filter((manifest) => afterVersion === null || manifest.version > afterVersion)
|
|
390
|
+
.sort((left, right) => left.version - right.version)
|
|
391
|
+
.slice(0, limit)
|
|
392
|
+
.map((manifest) => structuredClone(manifest));
|
|
393
|
+
return {
|
|
394
|
+
records,
|
|
395
|
+
nextCursor: records.length === limit ? (records.at(-1)?.version ?? null) : null,
|
|
396
|
+
};
|
|
397
|
+
}
|
|
398
|
+
/**
|
|
399
|
+
* RecordCore keeps resolved manifests. A tombstone below the first readable manifest is no
|
|
400
|
+
* longer needed for delta resolution, but its narrowed block list remains the collector's
|
|
401
|
+
* durable discovery record until every named payload has actually gone. Deleting it sooner
|
|
402
|
+
* loses garbage left behind by a bounded planning pass.
|
|
403
|
+
*/
|
|
404
|
+
removePrunedManifestRecords() {
|
|
405
|
+
const earliestReadable = [...this.#manifests.values()]
|
|
406
|
+
.filter((manifest) => manifest.prunedAt === undefined)
|
|
407
|
+
.reduce((earliest, manifest) => Math.min(earliest, manifest.version), Number.POSITIVE_INFINITY);
|
|
408
|
+
const safeBelow = earliestReadable;
|
|
409
|
+
const removed = [];
|
|
410
|
+
for (const [version, manifest] of this.#manifests) {
|
|
411
|
+
if (manifest.prunedAt === undefined || version >= safeBelow)
|
|
412
|
+
continue;
|
|
413
|
+
if (manifest.blockIds.some((id) => this.#physical.hasBlock(id)))
|
|
414
|
+
continue;
|
|
415
|
+
this.#manifests.delete(version);
|
|
416
|
+
removed.push(version);
|
|
417
|
+
}
|
|
418
|
+
return removed.sort((left, right) => left - right);
|
|
419
|
+
}
|
|
420
|
+
publishManifest(input) {
|
|
421
|
+
if (this.#currentVersion !== input.expectedVersion) {
|
|
422
|
+
throw new WriteConflictError(input.expectedVersion, this.#currentVersion);
|
|
423
|
+
}
|
|
424
|
+
for (const id of input.blockIds) {
|
|
425
|
+
if (!this.#physical.hasBlock(id))
|
|
426
|
+
throw new Error(`Manifest references missing block: ${id}`);
|
|
427
|
+
}
|
|
428
|
+
const manifest = createManifest(input);
|
|
429
|
+
this.#manifests.set(manifest.version, manifest);
|
|
430
|
+
this.#currentVersion = manifest.version;
|
|
431
|
+
this.#catalogEpoch += 1;
|
|
432
|
+
return structuredClone(manifest);
|
|
433
|
+
}
|
|
434
|
+
beginTransaction(input) {
|
|
435
|
+
validateBeginTransactionInput(input);
|
|
436
|
+
const record = {
|
|
437
|
+
...structuredClone(input.record),
|
|
438
|
+
snapshotVersion: this.#currentVersion,
|
|
439
|
+
};
|
|
440
|
+
this.#assertSnapshotAvailable(record.snapshotVersion);
|
|
441
|
+
if (this.#transactions.has(record.id)) {
|
|
442
|
+
throw new Error(`Transaction already exists: ${record.id}`);
|
|
443
|
+
}
|
|
444
|
+
this.#transactions.set(record.id, record);
|
|
445
|
+
let rowIds;
|
|
446
|
+
if (input.reserveRowIds !== undefined) {
|
|
447
|
+
validateCount(input.reserveRowIds.count);
|
|
448
|
+
const current = this.#nextRowIds.get(input.reserveRowIds.tableId) ?? 1n;
|
|
449
|
+
const endExclusive = current + BigInt(input.reserveRowIds.count);
|
|
450
|
+
this.#nextRowIds.set(input.reserveRowIds.tableId, endExclusive);
|
|
451
|
+
rowIds = { start: current, endExclusive };
|
|
452
|
+
}
|
|
453
|
+
let autoIncrementValues;
|
|
454
|
+
if (input.reserveAutoIncrement !== undefined) {
|
|
455
|
+
const { tableId, columnId, count, atLeast } = input.reserveAutoIncrement;
|
|
456
|
+
validateAutoIncrementReservation(count, atLeast);
|
|
457
|
+
autoIncrementValues = this.#reserveAutoIncrement(tableId, columnId, count, atLeast);
|
|
458
|
+
}
|
|
459
|
+
return {
|
|
460
|
+
record: structuredClone(record),
|
|
461
|
+
...(rowIds === undefined ? {} : { rowIds }),
|
|
462
|
+
...(autoIncrementValues === undefined ? {} : { autoIncrementValues }),
|
|
463
|
+
};
|
|
464
|
+
}
|
|
465
|
+
createTransaction(record) {
|
|
466
|
+
if (this.#transactions.has(record.id)) {
|
|
467
|
+
throw new Error(`Transaction already exists: ${record.id}`);
|
|
468
|
+
}
|
|
469
|
+
this.#assertSnapshotAvailable(record.snapshotVersion);
|
|
470
|
+
this.#assertPendingArtifactsAvailable(record);
|
|
471
|
+
this.#transactions.set(record.id, structuredClone(record));
|
|
472
|
+
}
|
|
473
|
+
getTransaction(id) {
|
|
474
|
+
const record = this.#transactions.get(id);
|
|
475
|
+
return record === undefined ? undefined : structuredClone(record);
|
|
476
|
+
}
|
|
477
|
+
getTransactions(ids) {
|
|
478
|
+
return ids.map((id) => {
|
|
479
|
+
const record = this.#transactions.get(id);
|
|
480
|
+
return record === undefined ? undefined : structuredClone(record);
|
|
481
|
+
});
|
|
482
|
+
}
|
|
483
|
+
listTransactions() {
|
|
484
|
+
return [...this.#transactions.values()]
|
|
485
|
+
.sort((left, right) => left.startedAt.localeCompare(right.startedAt) || left.id.localeCompare(right.id))
|
|
486
|
+
.map((record) => structuredClone(record));
|
|
487
|
+
}
|
|
488
|
+
listTransactionPage(afterId, limit) {
|
|
489
|
+
validatePageLimit(limit);
|
|
490
|
+
const records = [...this.#transactions.values()]
|
|
491
|
+
.filter((record) => afterId === null || record.id > afterId)
|
|
492
|
+
.sort((left, right) => left.id.localeCompare(right.id))
|
|
493
|
+
.slice(0, limit)
|
|
494
|
+
.map((record) => structuredClone(record));
|
|
495
|
+
return { records, nextCursor: records.length === limit ? (records.at(-1)?.id ?? null) : null };
|
|
496
|
+
}
|
|
497
|
+
updateTransaction(id, expectedRevision, update) {
|
|
498
|
+
const current = this.#transactions.get(id);
|
|
499
|
+
if (current?.revision !== expectedRevision) {
|
|
500
|
+
throw new TransactionRecordConflictError(id, expectedRevision, current?.revision ?? null);
|
|
501
|
+
}
|
|
502
|
+
assertGenericTransactionUpdateAllowed(current, update);
|
|
503
|
+
const updated = updateTransactionRecord(current, update);
|
|
504
|
+
if (update.snapshotVersion !== undefined) {
|
|
505
|
+
this.#assertSnapshotAvailable(updated.snapshotVersion);
|
|
506
|
+
}
|
|
507
|
+
this.#assertPendingArtifactsAvailable(updated, update.pendingBlockIds !== undefined, update.pendingSegmentIds !== undefined);
|
|
508
|
+
this.#transactions.set(id, updated);
|
|
509
|
+
return structuredClone(updated);
|
|
510
|
+
}
|
|
511
|
+
/**
|
|
512
|
+
* Validates and applies the record half of staging: the journal update and the segment
|
|
513
|
+
* records. Block bytes are the enclosing store's to write — after this returns, in the same
|
|
514
|
+
* atomic step.
|
|
515
|
+
*/
|
|
516
|
+
stageTransactionArtifacts(input, options = {}) {
|
|
517
|
+
if (options.blocksPrevalidated !== true) {
|
|
518
|
+
const ids = new Set();
|
|
519
|
+
for (const block of input.blocks) {
|
|
520
|
+
validateId(block.id);
|
|
521
|
+
if (ids.has(block.id) || this.#physical.hasBlock(block.id)) {
|
|
522
|
+
throw new Error(`Block already exists: ${block.id}`);
|
|
523
|
+
}
|
|
524
|
+
ids.add(block.id);
|
|
525
|
+
}
|
|
526
|
+
}
|
|
527
|
+
for (const segment of input.segments) {
|
|
528
|
+
if (this.#segments.has(segment.id)) {
|
|
529
|
+
throw new Error(`Segment already exists: ${segment.id}`);
|
|
530
|
+
}
|
|
531
|
+
}
|
|
532
|
+
const current = this.#transactions.get(input.transactionId);
|
|
533
|
+
if (current?.revision !== input.expectedRevision) {
|
|
534
|
+
throw new TransactionRecordConflictError(input.transactionId, input.expectedRevision, current?.revision ?? null);
|
|
535
|
+
}
|
|
536
|
+
const update = {
|
|
537
|
+
pendingBlockIds: [...current.pendingBlockIds, ...input.blocks.map((block) => block.id)],
|
|
538
|
+
pendingSegmentIds: [
|
|
539
|
+
...current.pendingSegmentIds,
|
|
540
|
+
...input.segments.map((segment) => segment.id),
|
|
541
|
+
],
|
|
542
|
+
updatedAt: input.updatedAt,
|
|
543
|
+
};
|
|
544
|
+
assertGenericTransactionUpdateAllowed(current, update);
|
|
545
|
+
const updated = updateTransactionRecord(current, update);
|
|
546
|
+
// Only previously journaled artifacts need existence checks; the new ones land with the
|
|
547
|
+
// journal update in this same atomic step.
|
|
548
|
+
this.#assertPendingArtifactsAvailable(current, true, true);
|
|
549
|
+
for (const segment of input.segments) {
|
|
550
|
+
this.#segments.set(segment.id, normalizeSegmentRecord(segment));
|
|
551
|
+
}
|
|
552
|
+
this.#transactions.set(input.transactionId, updated);
|
|
553
|
+
return structuredClone(updated);
|
|
554
|
+
}
|
|
555
|
+
commitTransaction(input) {
|
|
556
|
+
const transaction = this.#transactions.get(input.transactionId);
|
|
557
|
+
if (transaction?.revision !== input.expectedTransactionRevision ||
|
|
558
|
+
transaction.status !== "active") {
|
|
559
|
+
throw new TransactionRecordConflictError(input.transactionId, input.expectedTransactionRevision, transaction?.revision ?? null);
|
|
560
|
+
}
|
|
561
|
+
const plan = this.#planCommit(transaction, input, (id) => this.#physical.hasBlock(id), (id) => this.#segments.get(id));
|
|
562
|
+
return this.#applyCommit(plan);
|
|
563
|
+
}
|
|
564
|
+
/**
|
|
565
|
+
* The single-shot write: begin or continue a transaction, journal the artifacts, and commit
|
|
566
|
+
* — validated in full before anything mutates, so a refusal anywhere (a stale revision, a
|
|
567
|
+
* moved manifest, a duplicate key) leaves the core exactly as it was, fresh record included.
|
|
568
|
+
* Block bytes are the enclosing store's to write afterwards, in the same atomic step.
|
|
569
|
+
*/
|
|
570
|
+
writeTransaction(input, options = {}) {
|
|
571
|
+
const blockIds = new Set();
|
|
572
|
+
for (const block of input.blocks) {
|
|
573
|
+
if (options.blocksPrevalidated !== true) {
|
|
574
|
+
validateId(block.id);
|
|
575
|
+
if (blockIds.has(block.id) || this.#physical.hasBlock(block.id)) {
|
|
576
|
+
throw new Error(`Block already exists: ${block.id}`);
|
|
577
|
+
}
|
|
578
|
+
}
|
|
579
|
+
blockIds.add(block.id);
|
|
580
|
+
}
|
|
581
|
+
const segments = new Map();
|
|
582
|
+
for (const segment of input.segments) {
|
|
583
|
+
if (segments.has(segment.id) || this.#segments.has(segment.id)) {
|
|
584
|
+
throw new Error(`Segment already exists: ${segment.id}`);
|
|
585
|
+
}
|
|
586
|
+
segments.set(segment.id, normalizeSegmentRecord(segment));
|
|
587
|
+
}
|
|
588
|
+
let base;
|
|
589
|
+
if ("record" in input.transaction) {
|
|
590
|
+
validateBeginTransactionInput({ record: input.transaction.record });
|
|
591
|
+
if (this.#transactions.has(input.transaction.record.id)) {
|
|
592
|
+
throw new Error(`Transaction already exists: ${input.transaction.record.id}`);
|
|
593
|
+
}
|
|
594
|
+
if (this.#currentVersion !== input.expectedManifestVersion) {
|
|
595
|
+
throw new WriteConflictError(input.expectedManifestVersion, this.#currentVersion);
|
|
596
|
+
}
|
|
597
|
+
base = {
|
|
598
|
+
...structuredClone(input.transaction.record),
|
|
599
|
+
snapshotVersion: input.expectedManifestVersion,
|
|
600
|
+
};
|
|
601
|
+
this.#assertSnapshotAvailable(base.snapshotVersion);
|
|
602
|
+
}
|
|
603
|
+
else {
|
|
604
|
+
const current = this.#transactions.get(input.transaction.id);
|
|
605
|
+
if (current?.revision !== input.transaction.expectedRevision) {
|
|
606
|
+
throw new TransactionRecordConflictError(input.transaction.id, input.transaction.expectedRevision, current?.revision ?? null);
|
|
607
|
+
}
|
|
608
|
+
base = current;
|
|
609
|
+
}
|
|
610
|
+
const update = {
|
|
611
|
+
pendingBlockIds: [...base.pendingBlockIds, ...blockIds],
|
|
612
|
+
pendingSegmentIds: [...base.pendingSegmentIds, ...segments.keys()],
|
|
613
|
+
updatedAt: input.committedAt,
|
|
614
|
+
};
|
|
615
|
+
assertGenericTransactionUpdateAllowed(base, update);
|
|
616
|
+
const staged = updateTransactionRecord(base, update);
|
|
617
|
+
// Previously journaled artifacts must exist; the ones staged here land with the commit.
|
|
618
|
+
this.#assertPendingArtifactsAvailable(base, true, true);
|
|
619
|
+
const plan = this.#planCommit(staged, input, (id) => blockIds.has(id) || this.#physical.hasBlock(id), (id) => segments.get(id) ?? this.#segments.get(id));
|
|
620
|
+
for (const [id, segment] of segments)
|
|
621
|
+
this.#segments.set(id, segment);
|
|
622
|
+
this.#transactions.set(staged.id, staged);
|
|
623
|
+
return this.#applyCommit(plan);
|
|
624
|
+
}
|
|
625
|
+
/**
|
|
626
|
+
* Everything a commit checks, against a transaction record the caller already resolved, with
|
|
627
|
+
* the artifact lookups injected so the single-shot write can count its not-yet-stored blocks
|
|
628
|
+
* and segments as present. Pure: mutations happen in `#applyCommit`.
|
|
629
|
+
*/
|
|
630
|
+
#planCommit(transaction, input, hasBlock, getSegment) {
|
|
631
|
+
if (this.#currentVersion !== input.expectedManifestVersion) {
|
|
632
|
+
throw new WriteConflictError(input.expectedManifestVersion, this.#currentVersion);
|
|
633
|
+
}
|
|
634
|
+
if (transaction.snapshotVersion !== input.expectedManifestVersion) {
|
|
635
|
+
throw new Error("Transaction snapshot does not match the expected manifest");
|
|
636
|
+
}
|
|
637
|
+
const baseManifest = input.expectedManifestVersion === null
|
|
638
|
+
? undefined
|
|
639
|
+
: this.#manifests.get(input.expectedManifestVersion);
|
|
640
|
+
if (input.expectedManifestVersion !== null && baseManifest === undefined) {
|
|
641
|
+
throw new Error(`Snapshot manifest is missing: ${String(input.expectedManifestVersion)}`);
|
|
642
|
+
}
|
|
643
|
+
const baseBlockIds = baseManifest?.blockIds ?? [];
|
|
644
|
+
const removedBlockIds = [...new Set(input.removedBlockIds ?? [])];
|
|
645
|
+
const baseBlockIdSet = new Set(baseBlockIds);
|
|
646
|
+
for (const id of removedBlockIds) {
|
|
647
|
+
if (!baseBlockIdSet.has(id)) {
|
|
648
|
+
throw new Error(`Cannot supersede a block outside the transaction snapshot: ${id}`);
|
|
649
|
+
}
|
|
650
|
+
if (transaction.pendingBlockIds.includes(id)) {
|
|
651
|
+
throw new Error(`Cannot supersede a pending block: ${id}`);
|
|
652
|
+
}
|
|
653
|
+
}
|
|
654
|
+
const removedBlockIdSet = new Set(removedBlockIds);
|
|
655
|
+
// The published list derives from the stored base plus this commit's delta; the record
|
|
656
|
+
// core keeps full manifests internally since cloning in memory is cheap.
|
|
657
|
+
const nextBlockIds = [
|
|
658
|
+
...baseBlockIds.filter((id) => !removedBlockIdSet.has(id)),
|
|
659
|
+
...transaction.pendingBlockIds,
|
|
660
|
+
];
|
|
661
|
+
for (const id of transaction.pendingBlockIds) {
|
|
662
|
+
if (!hasBlock(id))
|
|
663
|
+
throw new Error(`Manifest references missing block: ${id}`);
|
|
664
|
+
}
|
|
665
|
+
const pendingSegments = transaction.pendingSegmentIds.map((id) => {
|
|
666
|
+
const segment = getSegment(id);
|
|
667
|
+
if (segment === undefined)
|
|
668
|
+
throw new Error(`Transaction references missing segment: ${id}`);
|
|
669
|
+
if (segment.transactionId !== transaction.id) {
|
|
670
|
+
throw new Error(`Segment ${id} belongs to another transaction`);
|
|
671
|
+
}
|
|
672
|
+
return segment;
|
|
673
|
+
});
|
|
674
|
+
const uniqueKeyEntries = input.uniqueKeyChanges ?? [];
|
|
675
|
+
// Entries apply in operation order over per-table deltas against the live membership, so
|
|
676
|
+
// a scope that inserts the same key twice conflicts exactly like two separate commits
|
|
677
|
+
// would. A delta rather than a working copy of the table's whole key set: the copy made
|
|
678
|
+
// every keyed commit cost O(table), and it is only ever adopted after validation passes,
|
|
679
|
+
// which the delta achieves by being applied at the mutation step below.
|
|
680
|
+
const uniqueKeyDeltas = new Map();
|
|
681
|
+
for (const entry of uniqueKeyEntries) {
|
|
682
|
+
let delta = uniqueKeyDeltas.get(entry.tableId);
|
|
683
|
+
if (delta === undefined) {
|
|
684
|
+
delta = { added: new Set(), removed: new Set() };
|
|
685
|
+
uniqueKeyDeltas.set(entry.tableId, delta);
|
|
686
|
+
}
|
|
687
|
+
const existing = this.#uniqueKeys.get(entry.tableId);
|
|
688
|
+
for (const token of entry.keyTokens) {
|
|
689
|
+
if (entry.remove === true) {
|
|
690
|
+
delta.added.delete(token);
|
|
691
|
+
delta.removed.add(token);
|
|
692
|
+
continue;
|
|
693
|
+
}
|
|
694
|
+
const present = delta.added.has(token) || (existing?.has(token) === true && !delta.removed.has(token));
|
|
695
|
+
if (entry.requireAbsent && present) {
|
|
696
|
+
throw new UniqueKeyConflictError(entry.tableId, token);
|
|
697
|
+
}
|
|
698
|
+
delta.removed.delete(token);
|
|
699
|
+
delta.added.add(token);
|
|
700
|
+
}
|
|
701
|
+
}
|
|
702
|
+
const manifest = createManifest({
|
|
703
|
+
expectedVersion: input.expectedManifestVersion,
|
|
704
|
+
blockIds: nextBlockIds,
|
|
705
|
+
createdAt: input.committedAt,
|
|
706
|
+
...(input.changedTableIds === undefined ? {} : { changedTableIds: input.changedTableIds }),
|
|
707
|
+
});
|
|
708
|
+
const committed = updateTransactionRecord(transaction, {
|
|
709
|
+
status: "committed",
|
|
710
|
+
committedVersion: manifest.version,
|
|
711
|
+
updatedAt: input.committedAt,
|
|
712
|
+
});
|
|
713
|
+
return {
|
|
714
|
+
manifest,
|
|
715
|
+
committed,
|
|
716
|
+
pendingSegments,
|
|
717
|
+
uniqueKeyDeltas,
|
|
718
|
+
ftsChanges: input.ftsChanges,
|
|
719
|
+
};
|
|
720
|
+
}
|
|
721
|
+
#applyCommit(plan) {
|
|
722
|
+
const { manifest, committed, pendingSegments } = plan;
|
|
723
|
+
this.#manifests.set(manifest.version, manifest);
|
|
724
|
+
this.#currentVersion = manifest.version;
|
|
725
|
+
this.#transactions.set(committed.id, committed);
|
|
726
|
+
for (const segment of pendingSegments) {
|
|
727
|
+
this.#segments.set(segment.id, {
|
|
728
|
+
...segment,
|
|
729
|
+
level: segment.level ?? 0,
|
|
730
|
+
logicalOrder: segment.logicalOrder ?? manifest.version,
|
|
731
|
+
});
|
|
732
|
+
}
|
|
733
|
+
for (const [tableId, delta] of plan.uniqueKeyDeltas) {
|
|
734
|
+
let tokens = this.#uniqueKeys.get(tableId);
|
|
735
|
+
if (tokens === undefined) {
|
|
736
|
+
tokens = new Set();
|
|
737
|
+
this.#uniqueKeys.set(tableId, tokens);
|
|
738
|
+
}
|
|
739
|
+
for (const token of delta.removed)
|
|
740
|
+
tokens.delete(token);
|
|
741
|
+
for (const token of delta.added)
|
|
742
|
+
tokens.add(token);
|
|
743
|
+
}
|
|
744
|
+
this.#applyFtsChanges(pendingSegments, plan.ftsChanges, manifest.version);
|
|
745
|
+
this.#catalogEpoch += 1;
|
|
746
|
+
// Match the IndexedDB store's observable commit shape: the summary without blockIds.
|
|
747
|
+
const { blockIds: _resolved, ...summary } = manifest;
|
|
748
|
+
void _resolved;
|
|
749
|
+
return structuredClone(summary);
|
|
750
|
+
}
|
|
751
|
+
createLease(record) {
|
|
752
|
+
validateLeaseExpiration(record.expiresAt);
|
|
753
|
+
if (this.#leases.has(record.id))
|
|
754
|
+
throw new Error(`Lease already exists: ${record.id}`);
|
|
755
|
+
this.#assertSnapshotAvailable(record.manifestVersion);
|
|
756
|
+
this.#leases.set(record.id, structuredClone(record));
|
|
757
|
+
}
|
|
758
|
+
getLease(id) {
|
|
759
|
+
const record = this.#leases.get(id);
|
|
760
|
+
return record === undefined ? undefined : structuredClone(record);
|
|
761
|
+
}
|
|
762
|
+
listLeases() {
|
|
763
|
+
return [...this.#leases.values()]
|
|
764
|
+
.sort((left, right) => left.id.localeCompare(right.id))
|
|
765
|
+
.map((record) => structuredClone(record));
|
|
766
|
+
}
|
|
767
|
+
renewLease(id, expectedRevision, expiresAt) {
|
|
768
|
+
validateLeaseExpiration(expiresAt);
|
|
769
|
+
const record = this.#leases.get(id);
|
|
770
|
+
if (record?.revision !== expectedRevision) {
|
|
771
|
+
throw new LeaseConflictError(id, expectedRevision, record?.revision ?? null);
|
|
772
|
+
}
|
|
773
|
+
this.#assertSnapshotAvailable(record.manifestVersion);
|
|
774
|
+
const renewed = { ...record, expiresAt, revision: record.revision + 1 };
|
|
775
|
+
this.#leases.set(id, renewed);
|
|
776
|
+
return structuredClone(renewed);
|
|
777
|
+
}
|
|
778
|
+
moveLease(id, expectedRevision, manifestVersion, expiresAt) {
|
|
779
|
+
validateLeaseExpiration(expiresAt);
|
|
780
|
+
const record = this.#leases.get(id);
|
|
781
|
+
if (record?.revision !== expectedRevision) {
|
|
782
|
+
throw new LeaseConflictError(id, expectedRevision, record?.revision ?? null);
|
|
783
|
+
}
|
|
784
|
+
this.#assertSnapshotAvailable(manifestVersion);
|
|
785
|
+
const moved = { ...record, manifestVersion, expiresAt, revision: record.revision + 1 };
|
|
786
|
+
this.#leases.set(id, moved);
|
|
787
|
+
return structuredClone(moved);
|
|
788
|
+
}
|
|
789
|
+
removeLeaseIfExpired(id, expectedRevision, expiresAtCutoff) {
|
|
790
|
+
const cutoff = Date.parse(expiresAtCutoff);
|
|
791
|
+
if (!Number.isFinite(cutoff))
|
|
792
|
+
throw new TypeError("Lease expiry cutoff must be valid");
|
|
793
|
+
const record = this.#leases.get(id);
|
|
794
|
+
if (record?.revision !== expectedRevision) {
|
|
795
|
+
throw new LeaseConflictError(id, expectedRevision, record?.revision ?? null);
|
|
796
|
+
}
|
|
797
|
+
const expiresAt = Date.parse(record.expiresAt);
|
|
798
|
+
if (!Number.isFinite(expiresAt) || expiresAt > cutoff)
|
|
799
|
+
return false;
|
|
800
|
+
this.#leases.delete(id);
|
|
801
|
+
return true;
|
|
802
|
+
}
|
|
803
|
+
removeLease(id) {
|
|
804
|
+
this.#leases.delete(id);
|
|
805
|
+
}
|
|
806
|
+
createCompactionJob(record) {
|
|
807
|
+
const normalized = normalizeCompactionJobRecord(record);
|
|
808
|
+
if (this.#compactionJobs.has(normalized.id)) {
|
|
809
|
+
throw new Error(`Compaction job already exists: ${normalized.id}`);
|
|
810
|
+
}
|
|
811
|
+
this.#compactionJobs.set(normalized.id, normalized);
|
|
812
|
+
}
|
|
813
|
+
getCompactionJob(id) {
|
|
814
|
+
const record = this.#compactionJobs.get(id);
|
|
815
|
+
return record === undefined ? undefined : structuredClone(record);
|
|
816
|
+
}
|
|
817
|
+
listCompactionJobs(tableId) {
|
|
818
|
+
return [...this.#compactionJobs.values()]
|
|
819
|
+
.filter((record) => tableId === undefined || record.tableId === tableId)
|
|
820
|
+
.sort((left, right) => left.createdAt.localeCompare(right.createdAt) || left.id.localeCompare(right.id))
|
|
821
|
+
.map((record) => structuredClone(record));
|
|
822
|
+
}
|
|
823
|
+
listCompactionJobPage(afterId, limit) {
|
|
824
|
+
validatePageLimit(limit);
|
|
825
|
+
const records = [...this.#compactionJobs.values()]
|
|
826
|
+
.filter((record) => afterId === null || record.id > afterId)
|
|
827
|
+
.sort((left, right) => left.id.localeCompare(right.id))
|
|
828
|
+
.slice(0, limit)
|
|
829
|
+
.map((record) => structuredClone(record));
|
|
830
|
+
return { records, nextCursor: records.length === limit ? (records.at(-1)?.id ?? null) : null };
|
|
831
|
+
}
|
|
832
|
+
updateCompactionJob(id, expectedRevision, update) {
|
|
833
|
+
if (update.state === "cancelled") {
|
|
834
|
+
throw new TypeError("Use cancelCompactionJob to cancel a compaction job");
|
|
835
|
+
}
|
|
836
|
+
const current = this.#compactionJobs.get(id);
|
|
837
|
+
if (current?.revision !== expectedRevision) {
|
|
838
|
+
throw new CompactionJobConflictError(id, expectedRevision, current?.revision ?? null);
|
|
839
|
+
}
|
|
840
|
+
const updated = updateCompactionJobRecord(current, update);
|
|
841
|
+
this.#compactionJobs.set(id, updated);
|
|
842
|
+
return structuredClone(updated);
|
|
843
|
+
}
|
|
844
|
+
cancelCompactionJob(id, expectedRevision, cancelledAt) {
|
|
845
|
+
const current = this.#compactionJobs.get(id);
|
|
846
|
+
if (current?.revision !== expectedRevision) {
|
|
847
|
+
throw new CompactionJobConflictError(id, expectedRevision, current?.revision ?? null);
|
|
848
|
+
}
|
|
849
|
+
if (isTerminalCompactionJob(current)) {
|
|
850
|
+
return structuredClone(current);
|
|
851
|
+
}
|
|
852
|
+
const transaction = current.transactionId === null ? undefined : this.#transactions.get(current.transactionId);
|
|
853
|
+
if (transaction?.status === "committed") {
|
|
854
|
+
if (transaction.committedVersion === null) {
|
|
855
|
+
throw new Error(`Committed transaction has no manifest version: ${transaction.id}`);
|
|
856
|
+
}
|
|
857
|
+
const published = updateCompactionJobRecord(current, {
|
|
858
|
+
state: "published",
|
|
859
|
+
publishedVersion: transaction.committedVersion,
|
|
860
|
+
updatedAt: cancelledAt,
|
|
861
|
+
error: null,
|
|
862
|
+
});
|
|
863
|
+
this.#compactionJobs.set(id, published);
|
|
864
|
+
return structuredClone(published);
|
|
865
|
+
}
|
|
866
|
+
const cancelled = updateCompactionJobRecord(current, {
|
|
867
|
+
state: "cancelled",
|
|
868
|
+
updatedAt: cancelledAt,
|
|
869
|
+
error: null,
|
|
870
|
+
});
|
|
871
|
+
const abortedTransaction = transaction?.status === "active"
|
|
872
|
+
? updateTransactionRecord(transaction, {
|
|
873
|
+
status: "aborted",
|
|
874
|
+
updatedAt: cancelledAt,
|
|
875
|
+
committedVersion: null,
|
|
876
|
+
})
|
|
877
|
+
: undefined;
|
|
878
|
+
if (abortedTransaction !== undefined) {
|
|
879
|
+
this.#transactions.set(abortedTransaction.id, abortedTransaction);
|
|
880
|
+
}
|
|
881
|
+
this.#compactionJobs.set(id, cancelled);
|
|
882
|
+
return structuredClone(cancelled);
|
|
883
|
+
}
|
|
884
|
+
removeCompactionJob(id) {
|
|
885
|
+
this.#compactionJobs.delete(id);
|
|
886
|
+
}
|
|
887
|
+
createGarbageCollectionJob(input) {
|
|
888
|
+
const record = createGarbageCollectionJobRecord(input);
|
|
889
|
+
if (this.#garbageCollectionJobs.has(record.id)) {
|
|
890
|
+
throw new Error(`Garbage collection job already exists: ${record.id}`);
|
|
891
|
+
}
|
|
892
|
+
assertGarbageCollectionCandidateProvenance(record, this.#manifests, this.#segments, this.#transactions, this.#compactionJobs);
|
|
893
|
+
this.#garbageCollectionJobs.set(record.id, record);
|
|
894
|
+
return structuredClone(record);
|
|
895
|
+
}
|
|
896
|
+
getGarbageCollectionJob(id) {
|
|
897
|
+
const record = this.#garbageCollectionJobs.get(id);
|
|
898
|
+
return record === undefined ? undefined : structuredClone(record);
|
|
899
|
+
}
|
|
900
|
+
listGarbageCollectionJobs() {
|
|
901
|
+
return [...this.#garbageCollectionJobs.values()]
|
|
902
|
+
.sort((left, right) => left.createdAt.localeCompare(right.createdAt) || left.id.localeCompare(right.id))
|
|
903
|
+
.map((record) => structuredClone(record));
|
|
904
|
+
}
|
|
905
|
+
/**
|
|
906
|
+
* Advances the job and mutates every record it reclaims. Physical block bytes are the
|
|
907
|
+
* caller's to delete afterwards, from the result's `reclaimedBlockIds` — in the same atomic
|
|
908
|
+
* step for the memory store, or after the logged entry for a store whose bytes are files.
|
|
909
|
+
*/
|
|
910
|
+
runGarbageCollectionStep(input) {
|
|
911
|
+
validateGarbageCollectionStepInput(input);
|
|
912
|
+
const current = this.#garbageCollectionJobs.get(input.jobId);
|
|
913
|
+
if (current?.revision !== input.expectedRevision) {
|
|
914
|
+
throw new GarbageCollectionJobConflictError(input.jobId, input.expectedRevision, current?.revision ?? null);
|
|
915
|
+
}
|
|
916
|
+
if (current.state === "completed")
|
|
917
|
+
return emptyGarbageCollectionStep(current);
|
|
918
|
+
const prunedManifestVersions = [];
|
|
919
|
+
const alreadyPrunedManifestVersions = [];
|
|
920
|
+
const retainedManifestVersions = [];
|
|
921
|
+
const missingManifestVersions = [];
|
|
922
|
+
const reclaimedSegmentIds = [];
|
|
923
|
+
const retainedSegmentIds = [];
|
|
924
|
+
const missingSegmentIds = [];
|
|
925
|
+
const reclaimedBlockIds = [];
|
|
926
|
+
const retainedBlockIds = [];
|
|
927
|
+
const missingBlockIds = [];
|
|
928
|
+
const reclaimedTransactionIds = [];
|
|
929
|
+
const retainedTransactionIds = [];
|
|
930
|
+
const missingTransactionIds = [];
|
|
931
|
+
let reclaimedBlockBytes = 0;
|
|
932
|
+
let remaining = input.maxItems;
|
|
933
|
+
const leaseCutoff = Date.parse(current.leaseCutoff);
|
|
934
|
+
assertGarbageCollectionPinsAvailable(this.#currentVersion, this.#transactions, this.#leases.values(), this.#compactionJobs.values(), leaseCutoff, this.#manifests, this.#physical);
|
|
935
|
+
let manifestIndex = current.cursor.manifestIndex;
|
|
936
|
+
while (remaining > 0 && manifestIndex < current.candidateManifestVersions.length) {
|
|
937
|
+
const version = current.candidateManifestVersions[manifestIndex];
|
|
938
|
+
if (version === undefined)
|
|
939
|
+
throw new Error("Garbage collection manifest cursor is invalid");
|
|
940
|
+
const manifest = this.#manifests.get(version);
|
|
941
|
+
if (manifest === undefined)
|
|
942
|
+
missingManifestVersions.push(version);
|
|
943
|
+
else if (manifest.prunedAt !== undefined)
|
|
944
|
+
alreadyPrunedManifestVersions.push(version);
|
|
945
|
+
else if (isManifestVersionPinned(version, this.#currentVersion, this.#transactions, this.#leases.values(), this.#compactionJobs.values(), leaseCutoff))
|
|
946
|
+
retainedManifestVersions.push(version);
|
|
947
|
+
else
|
|
948
|
+
prunedManifestVersions.push(version);
|
|
949
|
+
manifestIndex += 1;
|
|
950
|
+
remaining -= 1;
|
|
951
|
+
}
|
|
952
|
+
const prunedManifestVersionSet = new Set(prunedManifestVersions);
|
|
953
|
+
assertRemainingManifestBlocksAvailable(this.#manifests.values(), prunedManifestVersionSet, this.#physical);
|
|
954
|
+
let segmentIndex = current.cursor.segmentIndex;
|
|
955
|
+
const segmentIdsToExamine = manifestIndex === current.candidateManifestVersions.length
|
|
956
|
+
? current.candidateSegmentIds.slice(segmentIndex, segmentIndex + remaining)
|
|
957
|
+
: [];
|
|
958
|
+
const blockCapacity = Math.max(0, remaining - segmentIdsToExamine.length);
|
|
959
|
+
const blockIdsToExamine = manifestIndex === current.candidateManifestVersions.length &&
|
|
960
|
+
segmentIndex + segmentIdsToExamine.length === current.candidateSegmentIds.length
|
|
961
|
+
? current.candidateBlockIds.slice(current.cursor.blockIndex, current.cursor.blockIndex + blockCapacity)
|
|
962
|
+
: [];
|
|
963
|
+
const roots = collectBoundedPhysicalRoots(segmentIdsToExamine, blockIdsToExamine, this.#manifests.values(), prunedManifestVersionSet, this.#segments, this.#transactions, this.#compactionJobs.values());
|
|
964
|
+
while (remaining > 0 &&
|
|
965
|
+
manifestIndex === current.candidateManifestVersions.length &&
|
|
966
|
+
segmentIndex < current.candidateSegmentIds.length) {
|
|
967
|
+
const id = current.candidateSegmentIds[segmentIndex];
|
|
968
|
+
if (id === undefined)
|
|
969
|
+
throw new Error("Garbage collection segment cursor is invalid");
|
|
970
|
+
if (!this.#segments.has(id))
|
|
971
|
+
missingSegmentIds.push(id);
|
|
972
|
+
else if (roots.segmentIds.has(id))
|
|
973
|
+
retainedSegmentIds.push(id);
|
|
974
|
+
else
|
|
975
|
+
reclaimedSegmentIds.push(id);
|
|
976
|
+
segmentIndex += 1;
|
|
977
|
+
remaining -= 1;
|
|
978
|
+
}
|
|
979
|
+
let blockIndex = current.cursor.blockIndex;
|
|
980
|
+
// Transaction records are last. A committed record remains an
|
|
981
|
+
// idempotency/reconciliation witness until its manifest has actually been pruned, and it
|
|
982
|
+
// remains structural metadata while any segment or unfinished fold still names it.
|
|
983
|
+
let transactionIndex = current.cursor.transactionIndex;
|
|
984
|
+
if (remaining > 0 &&
|
|
985
|
+
manifestIndex === current.candidateManifestVersions.length &&
|
|
986
|
+
segmentIndex === current.candidateSegmentIds.length &&
|
|
987
|
+
blockIndex === current.candidateBlockIds.length) {
|
|
988
|
+
const segmentOwners = new Set([...this.#segments.values()]
|
|
989
|
+
.filter((segment) => !reclaimedSegmentIds.includes(segment.id))
|
|
990
|
+
.map((segment) => segment.transactionId));
|
|
991
|
+
const unfinishedCompactionOwners = new Set([...this.#compactionJobs.values()].flatMap((job) => !isTerminalCompactionJob(job) && job.transactionId !== null ? [job.transactionId] : []));
|
|
992
|
+
while (remaining > 0 && transactionIndex < current.candidateTransactionIds.length) {
|
|
993
|
+
const id = current.candidateTransactionIds[transactionIndex];
|
|
994
|
+
if (id === undefined)
|
|
995
|
+
throw new Error("Garbage collection transaction cursor is invalid");
|
|
996
|
+
const record = this.#transactions.get(id);
|
|
997
|
+
const manifest = record?.committedVersion === null || record?.committedVersion === undefined
|
|
998
|
+
? undefined
|
|
999
|
+
: this.#manifests.get(record.committedVersion);
|
|
1000
|
+
if (record === undefined)
|
|
1001
|
+
missingTransactionIds.push(id);
|
|
1002
|
+
else if (unfinishedCompactionOwners.has(id) ||
|
|
1003
|
+
(record.status === "committed"
|
|
1004
|
+
? (manifest !== undefined &&
|
|
1005
|
+
manifest.prunedAt === undefined &&
|
|
1006
|
+
!prunedManifestVersionSet.has(record.committedVersion ?? -1)) ||
|
|
1007
|
+
segmentOwners.has(id)
|
|
1008
|
+
: record.status === "aborted"
|
|
1009
|
+
? record.pendingBlockIds.some((blockId) => this.#physical.hasBlock(blockId)) ||
|
|
1010
|
+
record.pendingSegmentIds.some((segmentId) => this.#segments.has(segmentId))
|
|
1011
|
+
: true)) {
|
|
1012
|
+
retainedTransactionIds.push(id);
|
|
1013
|
+
}
|
|
1014
|
+
else {
|
|
1015
|
+
reclaimedTransactionIds.push(id);
|
|
1016
|
+
}
|
|
1017
|
+
transactionIndex += 1;
|
|
1018
|
+
remaining -= 1;
|
|
1019
|
+
}
|
|
1020
|
+
}
|
|
1021
|
+
while (remaining > 0 &&
|
|
1022
|
+
manifestIndex === current.candidateManifestVersions.length &&
|
|
1023
|
+
segmentIndex === current.candidateSegmentIds.length &&
|
|
1024
|
+
blockIndex < current.candidateBlockIds.length) {
|
|
1025
|
+
const id = current.candidateBlockIds[blockIndex];
|
|
1026
|
+
if (id === undefined)
|
|
1027
|
+
throw new Error("Garbage collection block cursor is invalid");
|
|
1028
|
+
const byteLength = this.#physical.blockByteLength(id);
|
|
1029
|
+
if (byteLength === undefined)
|
|
1030
|
+
missingBlockIds.push(id);
|
|
1031
|
+
else if (roots.blockIds.has(id))
|
|
1032
|
+
retainedBlockIds.push(id);
|
|
1033
|
+
else {
|
|
1034
|
+
reclaimedBlockIds.push(id);
|
|
1035
|
+
reclaimedBlockBytes = safeStorageSum(reclaimedBlockBytes, byteLength);
|
|
1036
|
+
}
|
|
1037
|
+
blockIndex += 1;
|
|
1038
|
+
remaining -= 1;
|
|
1039
|
+
}
|
|
1040
|
+
const updated = advanceGarbageCollectionJobRecord(current, {
|
|
1041
|
+
examinedManifestCount: prunedManifestVersions.length +
|
|
1042
|
+
alreadyPrunedManifestVersions.length +
|
|
1043
|
+
retainedManifestVersions.length +
|
|
1044
|
+
missingManifestVersions.length,
|
|
1045
|
+
prunedManifestCount: prunedManifestVersions.length,
|
|
1046
|
+
alreadyPrunedManifestCount: alreadyPrunedManifestVersions.length,
|
|
1047
|
+
retainedManifestCount: retainedManifestVersions.length,
|
|
1048
|
+
missingManifestCount: missingManifestVersions.length,
|
|
1049
|
+
examinedSegmentCount: reclaimedSegmentIds.length + retainedSegmentIds.length + missingSegmentIds.length,
|
|
1050
|
+
reclaimedSegmentCount: reclaimedSegmentIds.length,
|
|
1051
|
+
retainedSegmentCount: retainedSegmentIds.length,
|
|
1052
|
+
missingSegmentCount: missingSegmentIds.length,
|
|
1053
|
+
examinedBlockCount: reclaimedBlockIds.length + retainedBlockIds.length + missingBlockIds.length,
|
|
1054
|
+
reclaimedBlockCount: reclaimedBlockIds.length,
|
|
1055
|
+
retainedBlockCount: retainedBlockIds.length,
|
|
1056
|
+
missingBlockCount: missingBlockIds.length,
|
|
1057
|
+
reclaimedBlockBytes,
|
|
1058
|
+
examinedTransactionCount: reclaimedTransactionIds.length +
|
|
1059
|
+
retainedTransactionIds.length +
|
|
1060
|
+
missingTransactionIds.length,
|
|
1061
|
+
reclaimedTransactionCount: reclaimedTransactionIds.length,
|
|
1062
|
+
retainedTransactionCount: retainedTransactionIds.length,
|
|
1063
|
+
missingTransactionCount: missingTransactionIds.length,
|
|
1064
|
+
updatedAt: input.updatedAt,
|
|
1065
|
+
});
|
|
1066
|
+
// A pruned manifest is a tombstone: it cannot be pinned or read, and it roots nothing. The
|
|
1067
|
+
// only use left for its block list is finding the blocks that became garbage when it was
|
|
1068
|
+
// pruned — the ones the current manifest no longer carries — so that is all it keeps.
|
|
1069
|
+
// Keeping the whole list made every commit's manifest cost O(blocks) of memory forever,
|
|
1070
|
+
// and every OPFS checkpoint re-serialize all of them.
|
|
1071
|
+
const currentBlockIds = new Set(this.#currentVersion === null
|
|
1072
|
+
? []
|
|
1073
|
+
: (this.#manifests.get(this.#currentVersion)?.blockIds ?? []));
|
|
1074
|
+
prunedManifestVersions.forEach((version) => {
|
|
1075
|
+
const manifest = this.#manifests.get(version);
|
|
1076
|
+
if (manifest !== undefined) {
|
|
1077
|
+
this.#manifests.set(version, {
|
|
1078
|
+
...manifest,
|
|
1079
|
+
prunedAt: input.updatedAt,
|
|
1080
|
+
blockIds: manifest.blockIds.filter((id) => !currentBlockIds.has(id)),
|
|
1081
|
+
});
|
|
1082
|
+
}
|
|
1083
|
+
});
|
|
1084
|
+
reclaimedSegmentIds.forEach((id) => this.#segments.delete(id));
|
|
1085
|
+
reclaimedTransactionIds.forEach((id) => this.#transactions.delete(id));
|
|
1086
|
+
this.#garbageCollectionJobs.set(updated.id, updated);
|
|
1087
|
+
return {
|
|
1088
|
+
job: structuredClone(updated),
|
|
1089
|
+
prunedManifestVersions,
|
|
1090
|
+
alreadyPrunedManifestVersions,
|
|
1091
|
+
retainedManifestVersions,
|
|
1092
|
+
missingManifestVersions,
|
|
1093
|
+
reclaimedSegmentIds,
|
|
1094
|
+
retainedSegmentIds,
|
|
1095
|
+
missingSegmentIds,
|
|
1096
|
+
reclaimedBlockIds,
|
|
1097
|
+
retainedBlockIds,
|
|
1098
|
+
missingBlockIds,
|
|
1099
|
+
reclaimedBlockBytes,
|
|
1100
|
+
reclaimedTransactionIds,
|
|
1101
|
+
retainedTransactionIds,
|
|
1102
|
+
missingTransactionIds,
|
|
1103
|
+
};
|
|
1104
|
+
}
|
|
1105
|
+
removeGarbageCollectionJob(id) {
|
|
1106
|
+
this.#garbageCollectionJobs.delete(id);
|
|
1107
|
+
}
|
|
1108
|
+
/**
|
|
1109
|
+
* Copies the current version out as a portable snapshot. The caller provides the block bytes
|
|
1110
|
+
* and the atomicity — run this where no commit can interleave.
|
|
1111
|
+
*
|
|
1112
|
+
* Only blocks the current manifest still points at are copied, which drops every superseded
|
|
1113
|
+
* block a compaction left behind, and only full-text bases that already cover this exact
|
|
1114
|
+
* version are carried; any other indexed column is exported as `invalid` so the loaded
|
|
1115
|
+
* database rebuilds it. That costs a rebuild, never a wrong answer — the index is a pruning
|
|
1116
|
+
* accelerator and the scan re-verifies every candidate.
|
|
1117
|
+
*/
|
|
1118
|
+
exportSnapshot(readBlockBytes) {
|
|
1119
|
+
const { records, blockIds } = this.exportSnapshotRecords();
|
|
1120
|
+
const blocks = blockIds.map((id) => {
|
|
1121
|
+
const bytes = readBlockBytes(id);
|
|
1122
|
+
if (bytes === undefined)
|
|
1123
|
+
throw new Error(`Manifest references missing block: ${id}`);
|
|
1124
|
+
return { id, bytes: bytes.slice() };
|
|
1125
|
+
});
|
|
1126
|
+
return { ...records, blocks };
|
|
1127
|
+
}
|
|
1128
|
+
/**
|
|
1129
|
+
* The record half of `exportSnapshot`, for stores whose block bytes need asynchronous reads:
|
|
1130
|
+
* capture the records synchronously at one version, then read the immutable (and pinned)
|
|
1131
|
+
* block files at leisure.
|
|
1132
|
+
*/
|
|
1133
|
+
exportSnapshotRecords() {
|
|
1134
|
+
const version = this.#currentVersion;
|
|
1135
|
+
if (version === null)
|
|
1136
|
+
throw new Error("There is no committed version to snapshot");
|
|
1137
|
+
const manifest = this.#manifests.get(version);
|
|
1138
|
+
if (manifest === undefined)
|
|
1139
|
+
throw new SnapshotManifestMissingError(version);
|
|
1140
|
+
const liveBlockIds = new Set(manifest.blockIds);
|
|
1141
|
+
const { segments, transactions } = selectLiveRecords({
|
|
1142
|
+
liveBlockIds,
|
|
1143
|
+
segments: [...this.#segments.values()].map((record) => normalizeSegmentRecord(record)),
|
|
1144
|
+
transactions: [...this.#transactions.values()],
|
|
1145
|
+
version,
|
|
1146
|
+
});
|
|
1147
|
+
const tables = [...this.#tables.values()].map((table) => {
|
|
1148
|
+
const fts = [];
|
|
1149
|
+
const ftsColumns = { ...(table.ftsColumns ?? {}) };
|
|
1150
|
+
for (const [columnId, state] of Object.entries(ftsColumns)) {
|
|
1151
|
+
const base = this.#ftsBases.get(`${table.id}/${columnId}`);
|
|
1152
|
+
if (state.state === "ready" && base?.coversVersion === version) {
|
|
1153
|
+
fts.push({ columnId, ...structuredClone(base) });
|
|
1154
|
+
}
|
|
1155
|
+
else {
|
|
1156
|
+
ftsColumns[columnId] = { ...state, state: "invalid" };
|
|
1157
|
+
}
|
|
1158
|
+
}
|
|
1159
|
+
return {
|
|
1160
|
+
record: {
|
|
1161
|
+
...structuredClone(table),
|
|
1162
|
+
...(Object.keys(ftsColumns).length === 0 ? {} : { ftsColumns }),
|
|
1163
|
+
},
|
|
1164
|
+
nextRowId: this.#nextRowIds.get(table.id) ?? 1n,
|
|
1165
|
+
autoIncrement: table.columns.flatMap((column) => {
|
|
1166
|
+
const next = this.#nextAutoIncrement.get(`${table.id}/${column.id}`);
|
|
1167
|
+
return next === undefined ? [] : [{ columnId: column.id, next }];
|
|
1168
|
+
}),
|
|
1169
|
+
uniqueKeyTokens: [...(this.#uniqueKeys.get(table.id) ?? [])],
|
|
1170
|
+
fts,
|
|
1171
|
+
};
|
|
1172
|
+
});
|
|
1173
|
+
return {
|
|
1174
|
+
records: {
|
|
1175
|
+
version,
|
|
1176
|
+
createdAt: new Date().toISOString(),
|
|
1177
|
+
tables,
|
|
1178
|
+
segments,
|
|
1179
|
+
transactions,
|
|
1180
|
+
},
|
|
1181
|
+
blockIds: [...manifest.blockIds],
|
|
1182
|
+
};
|
|
1183
|
+
}
|
|
1184
|
+
/** Refuses to load over an existing database; see `loadSnapshot` for the load itself. */
|
|
1185
|
+
importSnapshot(snapshot, storeBlockBytes) {
|
|
1186
|
+
if (this.#currentVersion !== null)
|
|
1187
|
+
throw new Error("This store already holds a database");
|
|
1188
|
+
if (this.#tables.size > 0)
|
|
1189
|
+
throw new Error("This store already holds a catalog");
|
|
1190
|
+
this.loadSnapshot(snapshot, storeBlockBytes);
|
|
1191
|
+
}
|
|
1192
|
+
loadSnapshot(snapshot, storeBlockBytes) {
|
|
1193
|
+
for (const block of snapshot.blocks)
|
|
1194
|
+
storeBlockBytes(block.id, block.bytes);
|
|
1195
|
+
this.loadSnapshotRecords({ ...snapshot, blocks: undefined }, snapshot.blocks.map((block) => block.id));
|
|
1196
|
+
}
|
|
1197
|
+
/** The record half of a snapshot load; block bytes are the caller's, ids theirs to name. */
|
|
1198
|
+
loadSnapshotRecords(snapshot, blockIds) {
|
|
1199
|
+
for (const segment of snapshot.segments) {
|
|
1200
|
+
this.#segments.set(segment.id, normalizeSegmentRecord(segment));
|
|
1201
|
+
}
|
|
1202
|
+
for (const record of snapshot.transactions) {
|
|
1203
|
+
this.#transactions.set(record.id, structuredClone(record));
|
|
1204
|
+
}
|
|
1205
|
+
for (const table of snapshot.tables) {
|
|
1206
|
+
const record = structuredClone(table.record);
|
|
1207
|
+
this.#tables.set(record.id, record);
|
|
1208
|
+
this.#tableIdsByName.set(record.name, record.id);
|
|
1209
|
+
this.#nextRowIds.set(record.id, table.nextRowId);
|
|
1210
|
+
for (const entry of table.autoIncrement) {
|
|
1211
|
+
this.#nextAutoIncrement.set(`${record.id}/${entry.columnId}`, entry.next);
|
|
1212
|
+
}
|
|
1213
|
+
if (table.uniqueKeyTokens.length > 0) {
|
|
1214
|
+
this.#uniqueKeys.set(record.id, new Set(table.uniqueKeyTokens));
|
|
1215
|
+
}
|
|
1216
|
+
for (const index of table.fts) {
|
|
1217
|
+
this.#ftsBases.set(`${record.id}/${index.columnId}`, {
|
|
1218
|
+
coversVersion: index.coversVersion,
|
|
1219
|
+
chunks: structuredClone(index.chunks),
|
|
1220
|
+
totalTokens: index.totalTokens,
|
|
1221
|
+
});
|
|
1222
|
+
}
|
|
1223
|
+
}
|
|
1224
|
+
// One checkpoint, not the captured history: a snapshot restores as a single readable
|
|
1225
|
+
// version, and the version number itself is preserved so committed transactions still
|
|
1226
|
+
// sort and compare against it.
|
|
1227
|
+
this.#manifests.set(snapshot.version, {
|
|
1228
|
+
version: snapshot.version,
|
|
1229
|
+
previousVersion: null,
|
|
1230
|
+
blockIds: [...blockIds],
|
|
1231
|
+
createdAt: snapshot.createdAt,
|
|
1232
|
+
});
|
|
1233
|
+
this.#currentVersion = snapshot.version;
|
|
1234
|
+
this.#catalogEpoch += 1;
|
|
1235
|
+
}
|
|
1236
|
+
/**
|
|
1237
|
+
* The complete record state as one value — the checkpoint payload of a log-structured store.
|
|
1238
|
+
* Everything `load` needs to reproduce this core exactly, including operational state the
|
|
1239
|
+
* portable snapshot format deliberately drops (transactions, leases, jobs, counters). The
|
|
1240
|
+
* shape is JSON plus bigints; `encodeRecordJson` in this module serializes it verbatim.
|
|
1241
|
+
*
|
|
1242
|
+
* The arrays are fresh but their records alias live state: serialize (or clone) the result
|
|
1243
|
+
* before the next mutation. The one caller serializes synchronously in the same run, and
|
|
1244
|
+
* `load` clones on the way in — skipping a deep clone here saves tens of milliseconds per
|
|
1245
|
+
* checkpoint at scale.
|
|
1246
|
+
*/
|
|
1247
|
+
dump() {
|
|
1248
|
+
return {
|
|
1249
|
+
currentVersion: this.#currentVersion,
|
|
1250
|
+
catalogEpoch: this.#catalogEpoch,
|
|
1251
|
+
manifests: [...this.#manifests.values()],
|
|
1252
|
+
transactions: [...this.#transactions.values()],
|
|
1253
|
+
tables: [...this.#tables.values()],
|
|
1254
|
+
segments: [...this.#segments.values()],
|
|
1255
|
+
leases: [...this.#leases.values()],
|
|
1256
|
+
compactionJobs: [...this.#compactionJobs.values()],
|
|
1257
|
+
garbageCollectionJobs: [...this.#garbageCollectionJobs.values()],
|
|
1258
|
+
nextRowIds: [...this.#nextRowIds.entries()],
|
|
1259
|
+
nextAutoIncrement: [...this.#nextAutoIncrement.entries()],
|
|
1260
|
+
ftsBases: [...this.#ftsBases.entries()],
|
|
1261
|
+
ftsDeltas: [...this.#ftsDeltas.entries()].map(([key, deltas]) => [key, [...deltas.entries()]]),
|
|
1262
|
+
uniqueKeys: [...this.#uniqueKeys.entries()].map(([tableId, tokens]) => [tableId, [...tokens]]),
|
|
1263
|
+
tempOwners: [...this.#tempOwners.values()],
|
|
1264
|
+
};
|
|
1265
|
+
}
|
|
1266
|
+
/** Replaces the whole record state with a dump's content. */
|
|
1267
|
+
load(state) {
|
|
1268
|
+
const cloned = structuredClone(state);
|
|
1269
|
+
for (const map of [
|
|
1270
|
+
this.#manifests,
|
|
1271
|
+
this.#transactions,
|
|
1272
|
+
this.#tables,
|
|
1273
|
+
this.#tableIdsByName,
|
|
1274
|
+
this.#segments,
|
|
1275
|
+
this.#leases,
|
|
1276
|
+
this.#compactionJobs,
|
|
1277
|
+
this.#garbageCollectionJobs,
|
|
1278
|
+
this.#nextRowIds,
|
|
1279
|
+
this.#nextAutoIncrement,
|
|
1280
|
+
this.#ftsBases,
|
|
1281
|
+
this.#ftsDeltas,
|
|
1282
|
+
this.#uniqueKeys,
|
|
1283
|
+
this.#tempOwners,
|
|
1284
|
+
]) {
|
|
1285
|
+
map.clear();
|
|
1286
|
+
}
|
|
1287
|
+
this.#currentVersion = cloned.currentVersion;
|
|
1288
|
+
this.#catalogEpoch = cloned.catalogEpoch;
|
|
1289
|
+
for (const manifest of cloned.manifests)
|
|
1290
|
+
this.#manifests.set(manifest.version, manifest);
|
|
1291
|
+
for (const record of cloned.transactions)
|
|
1292
|
+
this.#transactions.set(record.id, record);
|
|
1293
|
+
for (const record of cloned.tables) {
|
|
1294
|
+
this.#tables.set(record.id, record);
|
|
1295
|
+
this.#tableIdsByName.set(record.name, record.id);
|
|
1296
|
+
}
|
|
1297
|
+
for (const record of cloned.segments)
|
|
1298
|
+
this.#segments.set(record.id, record);
|
|
1299
|
+
for (const record of cloned.leases)
|
|
1300
|
+
this.#leases.set(record.id, record);
|
|
1301
|
+
for (const record of cloned.compactionJobs)
|
|
1302
|
+
this.#compactionJobs.set(record.id, record);
|
|
1303
|
+
for (const record of cloned.garbageCollectionJobs) {
|
|
1304
|
+
this.#garbageCollectionJobs.set(record.id, record);
|
|
1305
|
+
}
|
|
1306
|
+
for (const [tableId, next] of cloned.nextRowIds)
|
|
1307
|
+
this.#nextRowIds.set(tableId, next);
|
|
1308
|
+
for (const [key, next] of cloned.nextAutoIncrement)
|
|
1309
|
+
this.#nextAutoIncrement.set(key, next);
|
|
1310
|
+
for (const [key, base] of cloned.ftsBases)
|
|
1311
|
+
this.#ftsBases.set(key, base);
|
|
1312
|
+
for (const [key, deltas] of cloned.ftsDeltas)
|
|
1313
|
+
this.#ftsDeltas.set(key, new Map(deltas));
|
|
1314
|
+
for (const [tableId, tokens] of cloned.uniqueKeys) {
|
|
1315
|
+
this.#uniqueKeys.set(tableId, new Set(tokens));
|
|
1316
|
+
}
|
|
1317
|
+
for (const record of cloned.tempOwners)
|
|
1318
|
+
this.#tempOwners.set(record.ownerId, record);
|
|
1319
|
+
}
|
|
1320
|
+
#assertSnapshotAvailable(version) {
|
|
1321
|
+
assertSnapshotAvailable(version, this.#manifests, this.#physical);
|
|
1322
|
+
}
|
|
1323
|
+
#assertPendingArtifactsAvailable(transaction, validateBlocks = true, validateSegments = true) {
|
|
1324
|
+
assertPendingArtifactsAvailable(transaction, this.#physical, this.#segments, validateBlocks, validateSegments);
|
|
1325
|
+
}
|
|
1326
|
+
}
|
|
1327
|
+
export function validateId(id) {
|
|
1328
|
+
if (id.length === 0)
|
|
1329
|
+
throw new TypeError("Block ID cannot be empty");
|
|
1330
|
+
}
|
|
1331
|
+
function validateCount(count) {
|
|
1332
|
+
if (!Number.isSafeInteger(count) || count <= 0) {
|
|
1333
|
+
throw new RangeError("Row ID reservation count must be a positive whole number");
|
|
1334
|
+
}
|
|
1335
|
+
}
|
|
1336
|
+
export function validateAutoIncrementReservation(count, atLeast) {
|
|
1337
|
+
if (!Number.isSafeInteger(count) || count < 0) {
|
|
1338
|
+
throw new RangeError("Auto-increment reservation count must be a non-negative whole number");
|
|
1339
|
+
}
|
|
1340
|
+
if (atLeast !== undefined && atLeast < 1n) {
|
|
1341
|
+
throw new RangeError("Auto-increment bump target must be at least 1");
|
|
1342
|
+
}
|
|
1343
|
+
}
|
|
1344
|
+
export function validateBeginTransactionInput(input) {
|
|
1345
|
+
if (input.record.pendingBlockIds.length > 0 || input.record.pendingSegmentIds.length > 0) {
|
|
1346
|
+
throw new TypeError("A fresh transaction cannot begin with pending artifacts");
|
|
1347
|
+
}
|
|
1348
|
+
}
|
|
1349
|
+
function isTerminalCompactionJob(record) {
|
|
1350
|
+
return record.state === "published" || record.state === "cancelled" || record.state === "aborted";
|
|
1351
|
+
}
|
|
1352
|
+
function assertGenericTransactionUpdateAllowed(record, update) {
|
|
1353
|
+
if (record.status !== "active") {
|
|
1354
|
+
throw new TypeError(`Only active transactions can be updated; found ${record.status}`);
|
|
1355
|
+
}
|
|
1356
|
+
if (update.status === "committed") {
|
|
1357
|
+
throw new TypeError("Use commitTransaction to commit a transaction");
|
|
1358
|
+
}
|
|
1359
|
+
if (Reflect.has(update, "committedVersion")) {
|
|
1360
|
+
throw new TypeError("Only commitTransaction can set a committed transaction version");
|
|
1361
|
+
}
|
|
1362
|
+
}
|
|
1363
|
+
function assertSnapshotAvailable(version, manifests, physical) {
|
|
1364
|
+
if (version === null)
|
|
1365
|
+
return;
|
|
1366
|
+
const manifest = manifests.get(version);
|
|
1367
|
+
if (manifest === undefined ||
|
|
1368
|
+
manifest.prunedAt !== undefined ||
|
|
1369
|
+
manifest.blockIds.some((id) => !physical.hasBlock(id))) {
|
|
1370
|
+
throw new SnapshotManifestMissingError(version);
|
|
1371
|
+
}
|
|
1372
|
+
}
|
|
1373
|
+
function assertGarbageCollectionPinsAvailable(currentVersion, transactions, leases, compactionJobs, leaseCutoff, manifests, physical) {
|
|
1374
|
+
if (currentVersion !== null)
|
|
1375
|
+
assertSnapshotAvailable(currentVersion, manifests, physical);
|
|
1376
|
+
for (const transaction of transactions.values()) {
|
|
1377
|
+
if (transaction.status === "active") {
|
|
1378
|
+
assertSnapshotAvailable(transaction.snapshotVersion, manifests, physical);
|
|
1379
|
+
}
|
|
1380
|
+
}
|
|
1381
|
+
for (const lease of leases) {
|
|
1382
|
+
const expiresAt = Date.parse(lease.expiresAt);
|
|
1383
|
+
if (!Number.isFinite(expiresAt) || expiresAt > leaseCutoff) {
|
|
1384
|
+
assertSnapshotAvailable(lease.manifestVersion, manifests, physical);
|
|
1385
|
+
}
|
|
1386
|
+
}
|
|
1387
|
+
for (const job of compactionJobs) {
|
|
1388
|
+
if (isTerminalCompactionJob(job))
|
|
1389
|
+
continue;
|
|
1390
|
+
assertSnapshotAvailable(job.sourceManifestVersion, manifests, physical);
|
|
1391
|
+
const linkedTransaction = job.transactionId === null ? undefined : transactions.get(job.transactionId);
|
|
1392
|
+
if (linkedTransaction?.status === "committed") {
|
|
1393
|
+
if (linkedTransaction.committedVersion === null) {
|
|
1394
|
+
throw new Error(`Committed transaction has no manifest version: ${linkedTransaction.id}`);
|
|
1395
|
+
}
|
|
1396
|
+
assertSnapshotAvailable(linkedTransaction.committedVersion, manifests, physical);
|
|
1397
|
+
}
|
|
1398
|
+
}
|
|
1399
|
+
}
|
|
1400
|
+
function assertPendingArtifactsAvailable(transaction, physical, segments, validateBlocks = true, validateSegments = true) {
|
|
1401
|
+
const missingBlockId = validateBlocks
|
|
1402
|
+
? transaction.pendingBlockIds.find((id) => !physical.hasBlock(id))
|
|
1403
|
+
: undefined;
|
|
1404
|
+
if (missingBlockId !== undefined) {
|
|
1405
|
+
throw new Error(`Transaction references missing pending block: ${missingBlockId}`);
|
|
1406
|
+
}
|
|
1407
|
+
const missingSegmentId = validateSegments
|
|
1408
|
+
? transaction.pendingSegmentIds.find((id) => !segments.has(id))
|
|
1409
|
+
: undefined;
|
|
1410
|
+
if (missingSegmentId !== undefined) {
|
|
1411
|
+
throw new Error(`Transaction references missing pending segment: ${missingSegmentId}`);
|
|
1412
|
+
}
|
|
1413
|
+
}
|
|
1414
|
+
function assertRemainingManifestBlocksAvailable(manifests, newlyPrunedVersions, physical) {
|
|
1415
|
+
for (const manifest of manifests) {
|
|
1416
|
+
if (manifest.prunedAt !== undefined || newlyPrunedVersions.has(manifest.version))
|
|
1417
|
+
continue;
|
|
1418
|
+
if (manifest.blockIds.some((id) => !physical.hasBlock(id))) {
|
|
1419
|
+
throw new SnapshotManifestMissingError(manifest.version);
|
|
1420
|
+
}
|
|
1421
|
+
}
|
|
1422
|
+
}
|
|
1423
|
+
function assertGarbageCollectionCandidateProvenance(job, manifests, segments, transactions, compactionJobs) {
|
|
1424
|
+
for (const version of job.candidateManifestVersions) {
|
|
1425
|
+
if (!manifests.has(version)) {
|
|
1426
|
+
throw new Error(`Garbage collection candidate manifest is missing: ${String(version)}`);
|
|
1427
|
+
}
|
|
1428
|
+
}
|
|
1429
|
+
for (const id of job.candidateTransactionIds) {
|
|
1430
|
+
const transaction = transactions.get(id);
|
|
1431
|
+
if (transaction === undefined ||
|
|
1432
|
+
(transaction.status !== "aborted" &&
|
|
1433
|
+
(transaction.status !== "committed" || transaction.committedVersion === null))) {
|
|
1434
|
+
throw new Error(`Garbage collection transaction candidate is not terminal: ${id}`);
|
|
1435
|
+
}
|
|
1436
|
+
}
|
|
1437
|
+
const blockHasProvenance = (id) => {
|
|
1438
|
+
for (const version of job.candidateManifestVersions) {
|
|
1439
|
+
if (manifests.get(version)?.blockIds.includes(id))
|
|
1440
|
+
return true;
|
|
1441
|
+
}
|
|
1442
|
+
for (const transaction of transactions.values()) {
|
|
1443
|
+
if (transaction.status === "aborted" && transaction.pendingBlockIds.includes(id))
|
|
1444
|
+
return true;
|
|
1445
|
+
}
|
|
1446
|
+
for (const compaction of compactionJobs.values()) {
|
|
1447
|
+
if (isTerminalCompactionJob(compaction) &&
|
|
1448
|
+
(compaction.sourceBlockIds.includes(id) || compaction.outputBlockIds.includes(id))) {
|
|
1449
|
+
return true;
|
|
1450
|
+
}
|
|
1451
|
+
}
|
|
1452
|
+
return false;
|
|
1453
|
+
};
|
|
1454
|
+
const unprovenBlockId = job.candidateBlockIds.find((id) => !blockHasProvenance(id));
|
|
1455
|
+
if (unprovenBlockId !== undefined) {
|
|
1456
|
+
throw new Error(`Garbage collection block candidate has no persisted provenance: ${unprovenBlockId}`);
|
|
1457
|
+
}
|
|
1458
|
+
const unprovenSegmentId = job.candidateSegmentIds.find((id) => {
|
|
1459
|
+
for (const transaction of transactions.values()) {
|
|
1460
|
+
if (transaction.status === "aborted" && transaction.pendingSegmentIds.includes(id)) {
|
|
1461
|
+
return false;
|
|
1462
|
+
}
|
|
1463
|
+
}
|
|
1464
|
+
for (const compaction of compactionJobs.values()) {
|
|
1465
|
+
if (isTerminalCompactionJob(compaction) &&
|
|
1466
|
+
(compaction.sourceSegmentIds.includes(id) || compaction.outputSegmentId === id)) {
|
|
1467
|
+
return false;
|
|
1468
|
+
}
|
|
1469
|
+
}
|
|
1470
|
+
const segment = segments.get(id);
|
|
1471
|
+
const blockIds = segment === undefined ? [] : segmentBlockIds(segment);
|
|
1472
|
+
return blockIds.length === 0 || !blockIds.every(blockHasProvenance);
|
|
1473
|
+
});
|
|
1474
|
+
if (unprovenSegmentId !== undefined) {
|
|
1475
|
+
throw new Error(`Garbage collection segment candidate has no persisted provenance: ${unprovenSegmentId}`);
|
|
1476
|
+
}
|
|
1477
|
+
}
|
|
1478
|
+
function isManifestVersionPinned(version, currentVersion, transactions, leases, compactionJobs, leaseCutoff) {
|
|
1479
|
+
if (currentVersion === version)
|
|
1480
|
+
return true;
|
|
1481
|
+
for (const transaction of transactions.values()) {
|
|
1482
|
+
if (transaction.status === "active" && transaction.snapshotVersion === version)
|
|
1483
|
+
return true;
|
|
1484
|
+
}
|
|
1485
|
+
for (const lease of leases) {
|
|
1486
|
+
const expiresAt = Date.parse(lease.expiresAt);
|
|
1487
|
+
if (lease.manifestVersion === version &&
|
|
1488
|
+
(!Number.isFinite(expiresAt) || expiresAt > leaseCutoff)) {
|
|
1489
|
+
return true;
|
|
1490
|
+
}
|
|
1491
|
+
}
|
|
1492
|
+
for (const job of compactionJobs) {
|
|
1493
|
+
if (isTerminalCompactionJob(job))
|
|
1494
|
+
continue;
|
|
1495
|
+
if (job.sourceManifestVersion === version)
|
|
1496
|
+
return true;
|
|
1497
|
+
const linkedTransaction = job.transactionId === null ? undefined : transactions.get(job.transactionId);
|
|
1498
|
+
if (linkedTransaction?.status === "committed") {
|
|
1499
|
+
if (linkedTransaction.committedVersion === null) {
|
|
1500
|
+
throw new Error(`Committed transaction has no manifest version: ${linkedTransaction.id}`);
|
|
1501
|
+
}
|
|
1502
|
+
if (linkedTransaction.committedVersion === version)
|
|
1503
|
+
return true;
|
|
1504
|
+
}
|
|
1505
|
+
}
|
|
1506
|
+
return false;
|
|
1507
|
+
}
|
|
1508
|
+
const MAX_GARBAGE_COLLECTION_ROOT_DEPENDENCIES = 4_096;
|
|
1509
|
+
function collectBoundedPhysicalRoots(candidateSegmentIds, candidateBlockIds, manifests, newlyPrunedVersions, segments, transactions, compactionJobs) {
|
|
1510
|
+
const candidateBlocks = new Set(candidateBlockIds);
|
|
1511
|
+
const probeBlockIds = new Set(candidateBlockIds);
|
|
1512
|
+
const probeSegmentIds = new Set(candidateSegmentIds);
|
|
1513
|
+
const relatedSegments = new Map();
|
|
1514
|
+
let dependencyOverflow = false;
|
|
1515
|
+
for (const segment of segments.values()) {
|
|
1516
|
+
const ids = segmentBlockIds(segment);
|
|
1517
|
+
if (!candidateSegmentIds.includes(segment.id) && !ids.some((id) => candidateBlocks.has(id))) {
|
|
1518
|
+
continue;
|
|
1519
|
+
}
|
|
1520
|
+
const newBlockIds = new Set(ids.filter((id) => !probeBlockIds.has(id)));
|
|
1521
|
+
if (relatedSegments.size >= MAX_GARBAGE_COLLECTION_ROOT_DEPENDENCIES ||
|
|
1522
|
+
probeBlockIds.size + newBlockIds.size > MAX_GARBAGE_COLLECTION_ROOT_DEPENDENCIES) {
|
|
1523
|
+
dependencyOverflow = true;
|
|
1524
|
+
break;
|
|
1525
|
+
}
|
|
1526
|
+
relatedSegments.set(segment.id, segment);
|
|
1527
|
+
probeSegmentIds.add(segment.id);
|
|
1528
|
+
ids.forEach((id) => probeBlockIds.add(id));
|
|
1529
|
+
}
|
|
1530
|
+
if (dependencyOverflow) {
|
|
1531
|
+
return { blockIds: new Set(candidateBlockIds), segmentIds: new Set(candidateSegmentIds) };
|
|
1532
|
+
}
|
|
1533
|
+
const directBlockRoots = new Set();
|
|
1534
|
+
const directSegmentRoots = new Set();
|
|
1535
|
+
for (const manifest of manifests) {
|
|
1536
|
+
if (manifest.prunedAt !== undefined || newlyPrunedVersions.has(manifest.version))
|
|
1537
|
+
continue;
|
|
1538
|
+
for (const id of manifest.blockIds)
|
|
1539
|
+
if (probeBlockIds.has(id))
|
|
1540
|
+
directBlockRoots.add(id);
|
|
1541
|
+
}
|
|
1542
|
+
for (const transaction of transactions.values()) {
|
|
1543
|
+
if (transaction.status !== "active")
|
|
1544
|
+
continue;
|
|
1545
|
+
for (const id of transaction.pendingBlockIds)
|
|
1546
|
+
if (probeBlockIds.has(id))
|
|
1547
|
+
directBlockRoots.add(id);
|
|
1548
|
+
for (const id of transaction.pendingSegmentIds)
|
|
1549
|
+
if (probeSegmentIds.has(id))
|
|
1550
|
+
directSegmentRoots.add(id);
|
|
1551
|
+
}
|
|
1552
|
+
for (const job of compactionJobs) {
|
|
1553
|
+
if (isTerminalCompactionJob(job))
|
|
1554
|
+
continue;
|
|
1555
|
+
for (const id of job.sourceBlockIds)
|
|
1556
|
+
if (probeBlockIds.has(id))
|
|
1557
|
+
directBlockRoots.add(id);
|
|
1558
|
+
for (const id of job.outputBlockIds)
|
|
1559
|
+
if (probeBlockIds.has(id))
|
|
1560
|
+
directBlockRoots.add(id);
|
|
1561
|
+
for (const id of job.sourceSegmentIds)
|
|
1562
|
+
if (probeSegmentIds.has(id))
|
|
1563
|
+
directSegmentRoots.add(id);
|
|
1564
|
+
if (job.outputSegmentId !== null && probeSegmentIds.has(job.outputSegmentId)) {
|
|
1565
|
+
directSegmentRoots.add(job.outputSegmentId);
|
|
1566
|
+
}
|
|
1567
|
+
}
|
|
1568
|
+
const rootedBlockIds = new Set();
|
|
1569
|
+
const rootedSegmentIds = new Set();
|
|
1570
|
+
for (const segment of relatedSegments.values()) {
|
|
1571
|
+
const ids = segmentBlockIds(segment);
|
|
1572
|
+
if (directSegmentRoots.has(segment.id) ||
|
|
1573
|
+
transactions.get(segment.transactionId)?.status === "active" ||
|
|
1574
|
+
(ids.length > 0 && ids.every((id) => directBlockRoots.has(id)))) {
|
|
1575
|
+
if (candidateSegmentIds.includes(segment.id))
|
|
1576
|
+
rootedSegmentIds.add(segment.id);
|
|
1577
|
+
for (const id of ids)
|
|
1578
|
+
if (candidateBlocks.has(id))
|
|
1579
|
+
rootedBlockIds.add(id);
|
|
1580
|
+
}
|
|
1581
|
+
}
|
|
1582
|
+
for (const id of candidateBlockIds)
|
|
1583
|
+
if (directBlockRoots.has(id))
|
|
1584
|
+
rootedBlockIds.add(id);
|
|
1585
|
+
for (const id of candidateSegmentIds)
|
|
1586
|
+
if (directSegmentRoots.has(id))
|
|
1587
|
+
rootedSegmentIds.add(id);
|
|
1588
|
+
return { blockIds: rootedBlockIds, segmentIds: rootedSegmentIds };
|
|
1589
|
+
}
|
|
1590
|
+
function segmentBlockIds(segment) {
|
|
1591
|
+
return [...new Set(Object.values(segment.columnBlockIds).flat())];
|
|
1592
|
+
}
|
|
1593
|
+
function validateGarbageCollectionStepInput(input) {
|
|
1594
|
+
if (input.jobId.length === 0)
|
|
1595
|
+
throw new TypeError("Garbage collection job ID cannot be empty");
|
|
1596
|
+
if (!Number.isSafeInteger(input.expectedRevision) || input.expectedRevision < 0) {
|
|
1597
|
+
throw new RangeError("Garbage collection expected revision must be a non-negative whole number");
|
|
1598
|
+
}
|
|
1599
|
+
if (!Number.isSafeInteger(input.maxItems) || input.maxItems <= 0) {
|
|
1600
|
+
throw new RangeError("Garbage collection item limit must be a positive whole number");
|
|
1601
|
+
}
|
|
1602
|
+
if (input.updatedAt.length === 0 || !Number.isFinite(Date.parse(input.updatedAt))) {
|
|
1603
|
+
throw new TypeError("Garbage collection update timestamp must be valid");
|
|
1604
|
+
}
|
|
1605
|
+
}
|
|
1606
|
+
function validatePageLimit(limit) {
|
|
1607
|
+
if (!Number.isSafeInteger(limit) || limit <= 0) {
|
|
1608
|
+
throw new RangeError("Storage page limit must be a positive whole number");
|
|
1609
|
+
}
|
|
1610
|
+
}
|
|
1611
|
+
function validateTableColumns(columns) {
|
|
1612
|
+
if (columns.length === 0)
|
|
1613
|
+
throw new TypeError("A table needs at least one column");
|
|
1614
|
+
const ids = new Set(columns.map(({ id }) => id));
|
|
1615
|
+
const names = new Set(columns.map(({ name }) => name));
|
|
1616
|
+
if (ids.size !== columns.length || names.size !== columns.length) {
|
|
1617
|
+
throw new TypeError("Table columns must have unique IDs and names");
|
|
1618
|
+
}
|
|
1619
|
+
}
|
|
1620
|
+
function validateTempOwnerRecord(record) {
|
|
1621
|
+
validateId(record.ownerId);
|
|
1622
|
+
validateLeaseExpiration(record.expiresAt);
|
|
1623
|
+
if (record.revision !== 0) {
|
|
1624
|
+
throw new RangeError("Temp owner record must be created at revision zero");
|
|
1625
|
+
}
|
|
1626
|
+
}
|
|
1627
|
+
export function validateLeaseExpiration(expiresAt) {
|
|
1628
|
+
if (expiresAt.length === 0 || !Number.isFinite(Date.parse(expiresAt))) {
|
|
1629
|
+
throw new TypeError("Lease expiration must be valid");
|
|
1630
|
+
}
|
|
1631
|
+
}
|
|
1632
|
+
export function validateTempRunPage(page) {
|
|
1633
|
+
validateTempRunPageIdentity(page.ownerId, page.runId, page.pageIndex);
|
|
1634
|
+
if (!(page.bytes instanceof Uint8Array))
|
|
1635
|
+
throw new TypeError("Temp run page bytes are invalid");
|
|
1636
|
+
}
|
|
1637
|
+
export function validateTempRunPageIdentity(ownerId, runId, pageIndex) {
|
|
1638
|
+
validateId(ownerId);
|
|
1639
|
+
validateId(runId);
|
|
1640
|
+
if (!Number.isSafeInteger(pageIndex) || pageIndex < 0) {
|
|
1641
|
+
throw new RangeError("Temp run page index must be a non-negative whole number");
|
|
1642
|
+
}
|
|
1643
|
+
}
|
|
1644
|
+
function emptyGarbageCollectionStep(job) {
|
|
1645
|
+
return {
|
|
1646
|
+
job: structuredClone(job),
|
|
1647
|
+
prunedManifestVersions: [],
|
|
1648
|
+
alreadyPrunedManifestVersions: [],
|
|
1649
|
+
retainedManifestVersions: [],
|
|
1650
|
+
missingManifestVersions: [],
|
|
1651
|
+
reclaimedSegmentIds: [],
|
|
1652
|
+
retainedSegmentIds: [],
|
|
1653
|
+
missingSegmentIds: [],
|
|
1654
|
+
reclaimedBlockIds: [],
|
|
1655
|
+
retainedBlockIds: [],
|
|
1656
|
+
missingBlockIds: [],
|
|
1657
|
+
reclaimedBlockBytes: 0,
|
|
1658
|
+
reclaimedTransactionIds: [],
|
|
1659
|
+
retainedTransactionIds: [],
|
|
1660
|
+
missingTransactionIds: [],
|
|
1661
|
+
};
|
|
1662
|
+
}
|
|
1663
|
+
function safeStorageSum(left, right) {
|
|
1664
|
+
const total = left + right;
|
|
1665
|
+
if (!Number.isSafeInteger(total)) {
|
|
1666
|
+
throw new RangeError("Garbage collection reclaimed block bytes exceed the safe range");
|
|
1667
|
+
}
|
|
1668
|
+
return total;
|
|
1669
|
+
}
|
|
1670
|
+
//# sourceMappingURL=record-core.js.map
|