@minnowdb/core 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +32 -18
- package/dist/engine/client.d.ts +9 -1
- package/dist/engine/client.d.ts.map +1 -1
- package/dist/engine/client.js +29 -5
- package/dist/engine/client.js.map +1 -1
- package/dist/engine/database.d.ts +51 -6
- package/dist/engine/database.d.ts.map +1 -1
- package/dist/engine/database.js +2597 -865
- package/dist/engine/database.js.map +1 -1
- package/dist/engine/live.d.ts.map +1 -1
- package/dist/engine/live.js +35 -11
- package/dist/engine/live.js.map +1 -1
- package/dist/engine/query-cache.d.ts +13 -1
- package/dist/engine/query-cache.d.ts.map +1 -1
- package/dist/engine/query-cache.js +34 -5
- package/dist/engine/query-cache.js.map +1 -1
- package/dist/engine/query.d.ts +11 -1
- package/dist/engine/query.d.ts.map +1 -1
- package/dist/engine/query.js +49 -41
- package/dist/engine/query.js.map +1 -1
- package/dist/engine/result-wire.d.ts +68 -0
- package/dist/engine/result-wire.d.ts.map +1 -0
- package/dist/engine/result-wire.js +264 -0
- package/dist/engine/result-wire.js.map +1 -0
- package/dist/engine/sort-keys.d.ts +50 -12
- package/dist/engine/sort-keys.d.ts.map +1 -1
- package/dist/engine/sort-keys.js +371 -25
- package/dist/engine/sort-keys.js.map +1 -1
- package/dist/engine/vector.d.ts +11 -0
- package/dist/engine/vector.d.ts.map +1 -1
- package/dist/engine/vector.js +99 -203
- package/dist/engine/vector.js.map +1 -1
- package/dist/engine/worker-host.d.ts +17 -1
- package/dist/engine/worker-host.d.ts.map +1 -1
- package/dist/engine/worker-host.js +83 -13
- package/dist/engine/worker-host.js.map +1 -1
- package/dist/storage/index.d.ts +13 -0
- package/dist/storage/index.d.ts.map +1 -1
- package/dist/storage/index.js +13 -0
- package/dist/storage/index.js.map +1 -1
- package/dist/storage/indexeddb.d.ts +5 -1
- package/dist/storage/indexeddb.d.ts.map +1 -1
- package/dist/storage/indexeddb.js +484 -402
- package/dist/storage/indexeddb.js.map +1 -1
- package/dist/storage/memory.d.ts +24 -21
- package/dist/storage/memory.d.ts.map +1 -1
- package/dist/storage/memory.js +124 -1199
- package/dist/storage/memory.js.map +1 -1
- package/dist/storage/opfs/files.d.ts +64 -0
- package/dist/storage/opfs/files.d.ts.map +1 -0
- package/dist/storage/opfs/files.js +243 -0
- package/dist/storage/opfs/files.js.map +1 -0
- package/dist/storage/opfs/index.d.ts +3 -0
- package/dist/storage/opfs/index.d.ts.map +1 -0
- package/dist/storage/opfs/index.js +3 -0
- package/dist/storage/opfs/index.js.map +1 -0
- package/dist/storage/opfs/leader.d.ts +130 -0
- package/dist/storage/opfs/leader.d.ts.map +1 -0
- package/dist/storage/opfs/leader.js +1104 -0
- package/dist/storage/opfs/leader.js.map +1 -0
- package/dist/storage/opfs/rpc.d.ts +70 -0
- package/dist/storage/opfs/rpc.d.ts.map +1 -0
- package/dist/storage/opfs/rpc.js +58 -0
- package/dist/storage/opfs/rpc.js.map +1 -0
- package/dist/storage/opfs/store.d.ts +154 -0
- package/dist/storage/opfs/store.d.ts.map +1 -0
- package/dist/storage/opfs/store.js +958 -0
- package/dist/storage/opfs/store.js.map +1 -0
- package/dist/storage/toolkit/extents.d.ts +69 -0
- package/dist/storage/toolkit/extents.d.ts.map +1 -0
- package/dist/storage/toolkit/extents.js +175 -0
- package/dist/storage/toolkit/extents.js.map +1 -0
- package/dist/storage/toolkit/index.d.ts +30 -0
- package/dist/storage/toolkit/index.d.ts.map +1 -0
- package/dist/storage/toolkit/index.js +5 -0
- package/dist/storage/toolkit/index.js.map +1 -0
- package/dist/storage/toolkit/record-core.d.ts +252 -0
- package/dist/storage/toolkit/record-core.d.ts.map +1 -0
- package/dist/storage/toolkit/record-core.js +1670 -0
- package/dist/storage/toolkit/record-core.js.map +1 -0
- package/dist/storage/toolkit/sync-file.d.ts +29 -0
- package/dist/storage/toolkit/sync-file.d.ts.map +1 -0
- package/dist/storage/toolkit/sync-file.js +2 -0
- package/dist/storage/toolkit/sync-file.js.map +1 -0
- package/dist/storage/toolkit/wal.d.ts +22 -0
- package/dist/storage/toolkit/wal.d.ts.map +1 -0
- package/dist/storage/toolkit/wal.js +79 -0
- package/dist/storage/toolkit/wal.js.map +1 -0
- package/dist/storage/toolkit/wire.d.ts +25 -0
- package/dist/storage/toolkit/wire.d.ts.map +1 -0
- package/dist/storage/toolkit/wire.js +100 -0
- package/dist/storage/toolkit/wire.js.map +1 -0
- package/dist/storage/types.d.ts +343 -59
- package/dist/storage/types.d.ts.map +1 -1
- package/dist/storage/types.js +112 -11
- package/dist/storage/types.js.map +1 -1
- package/dist/testing/block-store-conformance.d.ts +51 -0
- package/dist/testing/block-store-conformance.d.ts.map +1 -0
- package/dist/testing/block-store-conformance.js +745 -0
- package/dist/testing/block-store-conformance.js.map +1 -0
- package/dist/testing/index.d.ts +3 -0
- package/dist/testing/index.d.ts.map +1 -1
- package/dist/testing/index.js +5 -0
- package/dist/testing/index.js.map +1 -1
- package/dist/testing/opfs-shim.d.ts +35 -0
- package/dist/testing/opfs-shim.d.ts.map +1 -0
- package/dist/testing/opfs-shim.js +295 -0
- package/dist/testing/opfs-shim.js.map +1 -0
- package/dist/transactions/index.d.ts +42 -1
- package/dist/transactions/index.d.ts.map +1 -1
- package/dist/transactions/index.js +219 -13
- package/dist/transactions/index.js.map +1 -1
- package/dist/worker-protocol/index.d.ts +2 -2
- package/dist/worker-protocol/index.d.ts.map +1 -1
- package/dist/worker-protocol/index.js +2 -1
- package/dist/worker-protocol/index.js.map +1 -1
- package/package.json +10 -2
- package/dist/engine/dsl/db.d.ts +0 -137
- package/dist/engine/dsl/db.d.ts.map +0 -1
- package/dist/engine/dsl/db.js +0 -161
- package/dist/engine/dsl/db.js.map +0 -1
- package/dist/engine/dsl/expression.d.ts +0 -172
- package/dist/engine/dsl/expression.d.ts.map +0 -1
- package/dist/engine/dsl/expression.js +0 -397
- package/dist/engine/dsl/expression.js.map +0 -1
- package/dist/engine/dsl/index.d.ts +0 -8
- package/dist/engine/dsl/index.d.ts.map +0 -1
- package/dist/engine/dsl/index.js +0 -7
- package/dist/engine/dsl/index.js.map +0 -1
- package/dist/engine/dsl/live-query.d.ts +0 -45
- package/dist/engine/dsl/live-query.d.ts.map +0 -1
- package/dist/engine/dsl/live-query.js +0 -116
- package/dist/engine/dsl/live-query.js.map +0 -1
- package/dist/engine/dsl/mutations.d.ts +0 -123
- package/dist/engine/dsl/mutations.d.ts.map +0 -1
- package/dist/engine/dsl/mutations.js +0 -241
- package/dist/engine/dsl/mutations.js.map +0 -1
- package/dist/engine/dsl/select-query-builder.d.ts +0 -148
- package/dist/engine/dsl/select-query-builder.d.ts.map +0 -1
- package/dist/engine/dsl/select-query-builder.js +0 -377
- package/dist/engine/dsl/select-query-builder.js.map +0 -1
- package/dist/engine/dsl/sql-tag.d.ts +0 -46
- package/dist/engine/dsl/sql-tag.d.ts.map +0 -1
- package/dist/engine/dsl/sql-tag.js +0 -85
- package/dist/engine/dsl/sql-tag.js.map +0 -1
- package/dist/engine/dsl/types.d.ts +0 -133
- package/dist/engine/dsl/types.d.ts.map +0 -1
- package/dist/engine/dsl/types.js +0 -9
- package/dist/engine/dsl/types.js.map +0 -1
package/dist/engine/database.js
CHANGED
|
@@ -9,7 +9,7 @@ import { cachedQueryTerms, FTS_TOKENIZER_VERSION, renderDocumentValue, tokenize
|
|
|
9
9
|
import { simpleDataTypes, floorWholeNumberProduct, validateColumnDefault, validateEnumValues, CompactionJobConflictError, GarbageCollectionJobConflictError, decodeSnapshot, encodeSnapshot, SnapshotManifestMissingError, TableRecordConflictError, TransactionRecordConflictError, UniqueKeyConflictError, WriteConflictError, } from "../storage/index.js";
|
|
10
10
|
import { Snapshot, TransactionManager, } from "../transactions/index.js";
|
|
11
11
|
import { applyWindowFunctions, bindPlanParameters, bindStatementParameters, DUAL_TABLE, dualTableRows, blockHasSubqueries, combineUnionResults, compileCheckExpression, compileQuery, hasAggregate, createRecursiveCteState, compileStatement, createPreparedColumnarQuery, evaluateJoinedRowExpression, evaluateRowExpression, expressionColumnNames, inferBlockSchema, referencedColumns, childExpressions, expandFtsColumns, expandNaturalJoins, expandSourceColumnAliases, expandViewSources, forEachBlockExpression, planContainsFts, planHasNaturalJoins, planHasSourceColumnAliases, planReadsViews, planReadsBeyondSingleScan, projectResultColumns, subqueryResolutionSteps, topLevelFtsMatchConjuncts, transparentProjectionSource, windowOutputType, } from "./query.js";
|
|
12
|
-
import { copyQueryResult, queryResultMemoKey, queryResultRetainedBytes, RESULT_MEMO_MAX_BYTES, } from "./query-cache.js";
|
|
12
|
+
import { copyQueryResult, planMemoKey, queryResultMemoKey, queryResultRetainedBytes, RESULT_MEMO_MAX_BYTES, } from "./query-cache.js";
|
|
13
13
|
import { QueryMemoryBudgetError, QueryMemoryContext, } from "./memory.js";
|
|
14
14
|
import { LiveQuerySet } from "./live.js";
|
|
15
15
|
import { chooseJoinOrder, renderPlan } from "./optimizer.js";
|
|
@@ -25,20 +25,60 @@ const FTS_FOLD_DELTA_CHUNKS = 16;
|
|
|
25
25
|
const DEFAULT_COMPACTION_TARGET_BLOCK_BYTES = 2 * 1024 * 1024;
|
|
26
26
|
const DEFAULT_COMPACTION_MEMORY_BUDGET_BYTES = 32 * 1024 * 1024;
|
|
27
27
|
const DEFAULT_COMPACTION_MINIMUM_LEVEL_ZERO_SEGMENTS = 2;
|
|
28
|
-
const DEFAULT_COMPACTION_MAXIMUM_LEVEL_ZERO_SEGMENTS =
|
|
28
|
+
const DEFAULT_COMPACTION_MAXIMUM_LEVEL_ZERO_SEGMENTS = 64;
|
|
29
29
|
const DEFAULT_COMPACTION_MAXIMUM_LEVEL_ZERO_STORED_BYTES = 64 * 1024 * 1024;
|
|
30
|
+
/**
|
|
31
|
+
* Rows a keyed fold aims to keep in one level-one partition. A fold rewrites only the
|
|
32
|
+
* partitions its deltas touch, so this bounds how much one touched key costs to absorb; the
|
|
33
|
+
* table's partition count, and with it the per-query block count, grows as rows divided by it.
|
|
34
|
+
*/
|
|
35
|
+
const DEFAULT_COMPACTION_PARTITION_ROWS = 16_384;
|
|
30
36
|
const DEFAULT_LEVEL_TWO_MAX_WRITE_AMPLIFICATION = 16;
|
|
31
37
|
const MAX_COMPACTION_TARGET_BLOCK_BYTES = 64 * 1024 * 1024;
|
|
32
38
|
const MAX_BLOCK_ENVELOPE_BYTES = 1024;
|
|
33
39
|
const INTERNAL_READ_LEASE_TTL_MS = 60_000;
|
|
40
|
+
/** Live proof windows kept resident; a sweep uses one, concurrent sets a few. */
|
|
41
|
+
const LIVE_PROOF_CONTEXT_LIMIT = 4;
|
|
34
42
|
/** Distinct table-name sets whose catalog state stays resident; entries are tiny (records only). */
|
|
35
43
|
const CATALOG_STATE_CACHE_LIMIT = 64;
|
|
36
44
|
/** Blocks fetched per round trip when a streamed scan window needs more data. */
|
|
37
45
|
const STREAMED_SCAN_LOOKAHEAD_BLOCKS = 8;
|
|
38
|
-
/** Visible segments per table at which a
|
|
46
|
+
/** Visible segments per table at which a scan or a commit schedules a compaction step. */
|
|
39
47
|
const AUTO_COMPACT_SCAN_SEGMENTS = 48;
|
|
40
|
-
/** Visible delete/update segments at which a
|
|
41
|
-
const AUTO_COMPACT_DELTA_SEGMENTS =
|
|
48
|
+
/** Visible delete/update segments at which a scan or a commit schedules a compaction step. */
|
|
49
|
+
const AUTO_COMPACT_DELTA_SEGMENTS = 32;
|
|
50
|
+
/** Commits to one table between auto-compaction checks on the write path. */
|
|
51
|
+
const AUTO_COMPACT_COMMIT_CHECK_INTERVAL = 8;
|
|
52
|
+
/** Quiet time after a write burst before checking its final, sub-interval tail. */
|
|
53
|
+
const AUTO_COMPACT_IDLE_CHECK_MS = 25;
|
|
54
|
+
/** Commits between background collection passes; each prunes the manifests they wrote. */
|
|
55
|
+
const AUTO_COLLECT_COMMIT_INTERVAL = 64;
|
|
56
|
+
/**
|
|
57
|
+
* Manifest versions background collection leaves readable behind the current one, and how
|
|
58
|
+
* old one may be before it is collected regardless. A version is kept only while both hold: the
|
|
59
|
+
* count serves a reader that names a version it was just handed, the age keeps a burst of
|
|
60
|
+
* commits from pinning everything it superseded until the next burst — an idle tab reclaims
|
|
61
|
+
* within a minute.
|
|
62
|
+
*/
|
|
63
|
+
const AUTO_COLLECT_RETAINED_VERSIONS = 64;
|
|
64
|
+
const AUTO_COLLECT_RETAINED_VERSION_MS = 60_000;
|
|
65
|
+
/** A commit this long after the last collection pass starts one, whatever the commit count. */
|
|
66
|
+
const AUTO_COLLECT_QUIET_MS = 60_000;
|
|
67
|
+
/** Candidates one background collection step examines before yielding to the event loop. */
|
|
68
|
+
const AUTO_COLLECT_STEP_ITEMS = 64;
|
|
69
|
+
/** Passes one background collection run makes before handing the rest to the next trigger. */
|
|
70
|
+
const AUTO_COLLECT_MAX_PASSES = 32;
|
|
71
|
+
/** Finished job records of each kind a background run leaves for inspection. */
|
|
72
|
+
const AUTO_COLLECT_RETAINED_JOB_RECORDS = 8;
|
|
73
|
+
/** Output blocks one background compaction step writes before yielding to the event loop. */
|
|
74
|
+
const AUTO_COMPACT_STEP_BLOCKS = 4;
|
|
75
|
+
/**
|
|
76
|
+
* Level-zero segments one background fold may absorb. A fold rewrites every partition its
|
|
77
|
+
* deltas touch, and a partition touched by several deltas is rewritten once, so absorbing
|
|
78
|
+
* everything pending in one pass costs one rewrite of those partitions where the default would
|
|
79
|
+
* cost several; the stored-bytes ceiling still bounds the pass.
|
|
80
|
+
*/
|
|
81
|
+
const AUTO_COMPACT_MAX_LEVEL_ZERO_SEGMENTS = 256;
|
|
42
82
|
/** Modeled retained bytes for one cached block description (header metadata, no payload). */
|
|
43
83
|
const ZONE_DESCRIPTION_CACHE_BYTES = 160;
|
|
44
84
|
/** Overlay logical order for a write scope's staged segments: after all committed data. */
|
|
@@ -70,8 +110,15 @@ const GZIP_WORTHWHILE_RATIO = 1.2;
|
|
|
70
110
|
* bounds how long a wrong observation can persist if the data changes underneath it.
|
|
71
111
|
*/
|
|
72
112
|
const GZIP_REPROBE_BLOCKS = 32;
|
|
73
|
-
/**
|
|
74
|
-
const
|
|
113
|
+
/** Failed per-column probes retained in one database session. */
|
|
114
|
+
const GZIP_VERDICT_CACHE_LIMIT = 256;
|
|
115
|
+
/**
|
|
116
|
+
* Below this many logical bytes a block is written raw: the compression pass on the write and
|
|
117
|
+
* the decompression pass on every read would cost more than the bytes they save, and a point
|
|
118
|
+
* update's or delete's one-row block is the common case — it used to pay a CompressionStream
|
|
119
|
+
* round trip to shrink a few dozen bytes.
|
|
120
|
+
*/
|
|
121
|
+
const GZIP_MINIMUM_INPUT_BYTES = 4 * 1024;
|
|
75
122
|
/** Bounds concurrent compression work without serializing independent column blocks. */
|
|
76
123
|
const WRITE_ENCODE_CONCURRENCY = 6;
|
|
77
124
|
export { CompactionJobCancelledError, CompactionMemoryBudgetError, CompactionWriteAmplificationError, MissingKeyError, SqlCompileError, UniqueConstraintError, };
|
|
@@ -109,19 +156,24 @@ function boundInsertValue(value) {
|
|
|
109
156
|
function quoteSqlIdentifier(identifier) {
|
|
110
157
|
return `"${identifier.replaceAll('"', '""')}"`;
|
|
111
158
|
}
|
|
159
|
+
/**
|
|
160
|
+
* Snapshots are optional members of `BlockStore` — a store can be a complete database backend
|
|
161
|
+
* without being able to copy itself out — so the database checks at the call and says plainly
|
|
162
|
+
* when the capability is absent rather than failing as a missing property.
|
|
163
|
+
*/
|
|
112
164
|
function exportingStore(store) {
|
|
113
|
-
const
|
|
114
|
-
if (
|
|
165
|
+
const exportSnapshot = store.exportSnapshot?.bind(store);
|
|
166
|
+
if (exportSnapshot === undefined) {
|
|
115
167
|
throw new Error("This database's block store cannot export snapshots");
|
|
116
168
|
}
|
|
117
|
-
return
|
|
169
|
+
return { exportSnapshot };
|
|
118
170
|
}
|
|
119
171
|
function importingStore(store) {
|
|
120
|
-
const
|
|
121
|
-
if (
|
|
172
|
+
const importSnapshot = store.importSnapshot?.bind(store);
|
|
173
|
+
if (importSnapshot === undefined) {
|
|
122
174
|
throw new Error("This database's block store cannot load snapshots");
|
|
123
175
|
}
|
|
124
|
-
return
|
|
176
|
+
return { importSnapshot };
|
|
125
177
|
}
|
|
126
178
|
export class MinnowDatabase {
|
|
127
179
|
store;
|
|
@@ -137,7 +189,7 @@ export class MinnowDatabase {
|
|
|
137
189
|
/** The scope a statement-level BEGIN opened, held until COMMIT, ROLLBACK, or the idle sweep. */
|
|
138
190
|
#openTransaction;
|
|
139
191
|
#compression;
|
|
140
|
-
/** Per-column
|
|
192
|
+
/** Per-column count since gzip last failed to repay itself; successful probes need no entry. */
|
|
141
193
|
#gzipVerdicts = new Map();
|
|
142
194
|
#rowsPerBlock;
|
|
143
195
|
#maxCommitRetries;
|
|
@@ -146,16 +198,51 @@ export class MinnowDatabase {
|
|
|
146
198
|
#createId;
|
|
147
199
|
#internalLeaseOwnerId = `minnow/${crypto.randomUUID()}`;
|
|
148
200
|
#liveSets = new Set();
|
|
201
|
+
/** Live proof inputs per commit window, keyed `after:until`; see #liveProofContext. */
|
|
202
|
+
#liveProofContexts = new Map();
|
|
149
203
|
#internalLeaseSequence = 0;
|
|
150
204
|
#artifactCache;
|
|
151
205
|
#ftsAutoIndexRows;
|
|
152
206
|
#autoCompact;
|
|
207
|
+
#compactionPartitionRows;
|
|
208
|
+
#autoCollect;
|
|
209
|
+
/** Data commits since the last background collection pass. */
|
|
210
|
+
#commitsSinceCollection = 0;
|
|
211
|
+
/**
|
|
212
|
+
* The highest manifest version below which everything is known to be collected — pruned,
|
|
213
|
+
* with no block left behind; a collection plan starts its walk there. In memory only: a
|
|
214
|
+
* fresh instance walks the whole history once and learns it again.
|
|
215
|
+
*/
|
|
216
|
+
#collectionWatermark = null;
|
|
217
|
+
#autoCollectionInFlight = false;
|
|
218
|
+
/** A trigger that arrived while a run was in flight; honoured when the run ends. */
|
|
219
|
+
#autoCollectionRequested = false;
|
|
220
|
+
#autoCollectionBackoffUntilCommit = 0;
|
|
221
|
+
/** When the last background collection pass started, by the database clock. */
|
|
222
|
+
#lastCollectionAt;
|
|
223
|
+
/** The idle pass scheduled after the last commit; reset by the next commit. */
|
|
224
|
+
#idleCollectionTimer;
|
|
225
|
+
/** The garbage-collection step in flight, so steps run one at a time: see #serializedCollectionStep. */
|
|
226
|
+
#collectionSteps = Promise.resolve();
|
|
153
227
|
/** One background build attempt per (table, column) per session; misses just stay scans. */
|
|
154
228
|
#ftsBuildsInFlight = new Set();
|
|
155
229
|
/** Tables with a fire-and-forget compaction step already running. */
|
|
156
230
|
#autoCompactionsInFlight = new Set();
|
|
231
|
+
/** Tables whose maintenance threshold was observed again while their fold was still running. */
|
|
232
|
+
#autoCompactionsRequested = new Set();
|
|
233
|
+
/** Changed tables awaiting the debounced check that closes a write burst. */
|
|
234
|
+
#idleCompactionTableIds = new Set();
|
|
235
|
+
#idleCompactionTimer;
|
|
236
|
+
/** Tables whose drop is retiring data; prevents a new background fold from starting. */
|
|
237
|
+
#droppingTables = new Set();
|
|
157
238
|
/** Per table: the visible segment count a failed auto-compaction must see before retrying. */
|
|
158
239
|
#autoCompactionBackoff = new Map();
|
|
240
|
+
/** Data commits per table since its last write-path auto-compaction check. */
|
|
241
|
+
#commitsSinceCompactionCheck = new Map();
|
|
242
|
+
/** The compaction step in flight per table, so steps on one table run one at a time. */
|
|
243
|
+
#compactionSteps = new Map();
|
|
244
|
+
/** The simple writes in flight, chained so they commit one after another: see #runWrite. */
|
|
245
|
+
#writeChain = Promise.resolve();
|
|
159
246
|
/**
|
|
160
247
|
* SQL text to optimized plan, LRU by insertion order. Compiled plans are never mutated after
|
|
161
248
|
* optimization — subquery resolution and CTE expansion clone before rewriting and join
|
|
@@ -169,6 +256,8 @@ export class MinnowDatabase {
|
|
|
169
256
|
#ftsCandidatesMemo = new WeakMap();
|
|
170
257
|
#sharedLease;
|
|
171
258
|
#sharedLeaseRenewal;
|
|
259
|
+
/** An in-flight re-pin of the shared lease; acquirers wait for it, never join it. */
|
|
260
|
+
#sharedLeaseMove;
|
|
172
261
|
/**
|
|
173
262
|
* Catalog states keyed by requested table-name set, valid only at #catalogStateEpoch.
|
|
174
263
|
* The (version, epoch) probe is the sole validity signal: a matched epoch proves a cached
|
|
@@ -200,6 +289,8 @@ export class MinnowDatabase {
|
|
|
200
289
|
this.#artifactCache = new ArtifactCache(options.bufferPoolBytes ?? 64 * 1024 * 1024);
|
|
201
290
|
this.#ftsAutoIndexRows = options.ftsAutoIndexRows ?? 4096;
|
|
202
291
|
this.#autoCompact = options.autoCompact ?? true;
|
|
292
|
+
this.#autoCollect = options.autoCollect ?? this.#autoCompact;
|
|
293
|
+
this.#compactionPartitionRows = positiveWholeNumber(options.compaction?.partitionRows ?? DEFAULT_COMPACTION_PARTITION_ROWS, "Compaction partition rows");
|
|
203
294
|
if (!Number.isSafeInteger(this.#ftsAutoIndexRows) || this.#ftsAutoIndexRows < 0) {
|
|
204
295
|
throw new RangeError("Full-text auto-index row threshold must be a non-negative whole number");
|
|
205
296
|
}
|
|
@@ -300,7 +391,6 @@ export class MinnowDatabase {
|
|
|
300
391
|
...(input.managed === true ? { managed: true } : {}),
|
|
301
392
|
...(uniqueKeyColumn === undefined ? {} : { uniqueKeyColumnId: uniqueKeyColumn.id }),
|
|
302
393
|
...(uniqueKeyColumn === undefined ? {} : { uniqueKeyLookupReady: true }),
|
|
303
|
-
...(uniqueKeyColumn === undefined ? {} : { uniqueKeyStorage: "chunks-v2" }),
|
|
304
394
|
createdAt: this.#now().toISOString(),
|
|
305
395
|
});
|
|
306
396
|
}
|
|
@@ -333,17 +423,42 @@ export class MinnowDatabase {
|
|
|
333
423
|
* conflict: unlike the plain rebase-and-retry, a restart re-reads pre-images and re-runs
|
|
334
424
|
* trigger bodies at the fresh state, so derivations can never publish stale values.
|
|
335
425
|
*/
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
426
|
+
/**
|
|
427
|
+
* Runs one simple write — insert, upsert, update, or delete — after every simple write this
|
|
428
|
+
* database already has in flight, restarting it when its trigger derivations went stale.
|
|
429
|
+
*
|
|
430
|
+
* Commits are optimistic: a writer reads the manifest version, stages, and publishes only if
|
|
431
|
+
* the version has not moved, rebasing and retrying otherwise up to `maxCommitRetries`.
|
|
432
|
+
* Writers issued concurrently from one database used to all read the same version and spend
|
|
433
|
+
* a retry per rival that landed first, so past `maxCommitRetries + 1` of them the rest failed
|
|
434
|
+
* for nothing — contention this database need not create, and the queue does not. Writers in
|
|
435
|
+
* other instances and other tabs still contend, and the retry loop is still what resolves
|
|
436
|
+
* them. Write scopes are not queued: a scope's callback may issue a plain write of its own,
|
|
437
|
+
* which must not wait on the scope that contains it.
|
|
438
|
+
*/
|
|
439
|
+
async #runWrite(run) {
|
|
440
|
+
const restarting = async () => {
|
|
441
|
+
for (let attempt = 0;; attempt += 1) {
|
|
442
|
+
try {
|
|
443
|
+
return await run();
|
|
444
|
+
}
|
|
445
|
+
catch (error) {
|
|
446
|
+
if (!(error instanceof StaleTriggerDerivationsError))
|
|
447
|
+
throw error;
|
|
448
|
+
if (attempt >= this.#maxCommitRetries)
|
|
449
|
+
throw error.conflict;
|
|
450
|
+
}
|
|
346
451
|
}
|
|
452
|
+
};
|
|
453
|
+
const previous = this.#writeChain;
|
|
454
|
+
const current = previous.then(restarting, restarting);
|
|
455
|
+
this.#writeChain = current;
|
|
456
|
+
try {
|
|
457
|
+
return await current;
|
|
458
|
+
}
|
|
459
|
+
finally {
|
|
460
|
+
if (this.#writeChain === current)
|
|
461
|
+
this.#writeChain = Promise.resolve();
|
|
347
462
|
}
|
|
348
463
|
}
|
|
349
464
|
/**
|
|
@@ -449,39 +564,57 @@ export class MinnowDatabase {
|
|
|
449
564
|
}
|
|
450
565
|
}
|
|
451
566
|
}
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
const
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
567
|
+
this.#droppingTables.add(table.id);
|
|
568
|
+
try {
|
|
569
|
+
// Stop every fold already attached to the table before its catalog record disappears.
|
|
570
|
+
// Otherwise an unpublished job can no longer resume or be cancelled by table name, and
|
|
571
|
+
// its transaction and staged output become permanent roots.
|
|
572
|
+
await this.#cancelTableCompactions(table.id);
|
|
573
|
+
// Retiring the blocks is a commit like any other, and background compaction publishes
|
|
574
|
+
// underneath it: a block this table owned a moment ago can already have been rewritten. The
|
|
575
|
+
// list is therefore taken from the transaction's own snapshot — the manifest its commit will
|
|
576
|
+
// be validated against — and a scope that loses the race simply runs again.
|
|
577
|
+
for (let attempt = 0;; attempt += 1) {
|
|
578
|
+
const transaction = await this.#transactions.begin();
|
|
579
|
+
try {
|
|
580
|
+
const segments = await this.store.listSegments(table.id);
|
|
581
|
+
const snapshot = transaction.snapshotVersion === null
|
|
582
|
+
? undefined
|
|
583
|
+
: await this.store.getManifest(transaction.snapshotVersion);
|
|
584
|
+
const live = new Set(snapshot?.blockIds ?? []);
|
|
585
|
+
const blockIds = [
|
|
586
|
+
...new Set(segments.flatMap((segment) => Object.values(segment.columnBlockIds).flat())),
|
|
587
|
+
].filter((id) => live.has(id));
|
|
588
|
+
transaction.markTableChanged(table.id);
|
|
589
|
+
if (blockIds.length > 0)
|
|
590
|
+
transaction.supersedeBlocks(blockIds);
|
|
591
|
+
await transaction.commit();
|
|
592
|
+
break;
|
|
593
|
+
}
|
|
594
|
+
catch (error) {
|
|
595
|
+
await transaction.abort();
|
|
596
|
+
if (!(error instanceof WriteConflictError) || attempt >= this.#maxCommitRetries)
|
|
597
|
+
throw error;
|
|
598
|
+
}
|
|
477
599
|
}
|
|
600
|
+
// The catalog goes last: until it does, the table is merely empty of live blocks, and a
|
|
601
|
+
// crash in between leaves a table whose rows are gone rather than a segment pointing at a
|
|
602
|
+
// table that is not there.
|
|
603
|
+
// Catch a fold that was already between its scheduling check and job creation when the
|
|
604
|
+
// drop began. The dropping marker prevents another one from starting after this point.
|
|
605
|
+
await this.#cancelTableCompactions(table.id);
|
|
606
|
+
await this.store.removeTable(table.id, table.revision ?? 0);
|
|
607
|
+
for (const column of table.columns)
|
|
608
|
+
this.#gzipVerdicts.delete(column.id);
|
|
609
|
+
this.#autoCompactionBackoff.delete(table.id);
|
|
610
|
+
this.#commitsSinceCompactionCheck.delete(table.id);
|
|
611
|
+
this.#idleCompactionTableIds.delete(table.id);
|
|
612
|
+
this.#planCache.clear();
|
|
613
|
+
return true;
|
|
614
|
+
}
|
|
615
|
+
finally {
|
|
616
|
+
this.#droppingTables.delete(table.id);
|
|
478
617
|
}
|
|
479
|
-
// The catalog goes last: until it does, the table is merely empty of live blocks, and a
|
|
480
|
-
// crash in between leaves a table whose rows are gone rather than a segment pointing at a
|
|
481
|
-
// table that is not there.
|
|
482
|
-
await this.store.removeTable(table.id, table.revision ?? 0);
|
|
483
|
-
this.#planCache.clear();
|
|
484
|
-
return true;
|
|
485
618
|
}
|
|
486
619
|
async insertBatch(tableName, input) {
|
|
487
620
|
const table = await this.#findTable(tableName);
|
|
@@ -490,7 +623,7 @@ export class MinnowDatabase {
|
|
|
490
623
|
const keys = autoIncrement === undefined || autoIncrement.missingIndexes.length === 0
|
|
491
624
|
? batchKeys(table, batch)
|
|
492
625
|
: undefined;
|
|
493
|
-
const result = await this.#
|
|
626
|
+
const result = await this.#runWrite(() => this.#writeBatch(table, batch, "insert", keys, autoIncrement));
|
|
494
627
|
collectAutoIncrementGenerated(batch, generated, autoIncrement);
|
|
495
628
|
return {
|
|
496
629
|
tableName: result.tableName,
|
|
@@ -525,7 +658,7 @@ export class MinnowDatabase {
|
|
|
525
658
|
await this.#assertForeignKeysPresent(table, (column) => batch.columns[column] ?? [], (sql, params) => this.query(sql, { params, memoize: false }));
|
|
526
659
|
const deferred = autoIncrement !== undefined && autoIncrement.missingIndexes.length > 0;
|
|
527
660
|
const keys = deferred ? undefined : batchKeys(table, batch);
|
|
528
|
-
const result = await this.#
|
|
661
|
+
const result = await this.#runWrite(() => this.#writeBatch(table, batch, "upsert", keys, autoIncrement));
|
|
529
662
|
collectAutoIncrementGenerated(batch, generated, autoIncrement);
|
|
530
663
|
return {
|
|
531
664
|
...result,
|
|
@@ -544,7 +677,7 @@ export class MinnowDatabase {
|
|
|
544
677
|
}
|
|
545
678
|
const keys = validateUpdateBatch(table, keyColumn, input);
|
|
546
679
|
await this.#assertForeignKeysPresent(table, (column) => input.changes[column] ?? [], (sql, params) => this.query(sql, { params, memoize: false }));
|
|
547
|
-
return this.#
|
|
680
|
+
return this.#runWrite(() => this.#writeUpdateBatch(table, keyColumn, input, keys));
|
|
548
681
|
}
|
|
549
682
|
async update(tableName, key, changes) {
|
|
550
683
|
return this.updateBatch(tableName, {
|
|
@@ -555,7 +688,7 @@ export class MinnowDatabase {
|
|
|
555
688
|
async deleteBatch(tableName, input) {
|
|
556
689
|
const dependents = await this.#childForeignKeys(tableName);
|
|
557
690
|
if (dependents.length === 0) {
|
|
558
|
-
return this.#
|
|
691
|
+
return this.#runWrite(() => this.#deleteBatchOnce(tableName, input));
|
|
559
692
|
}
|
|
560
693
|
// E141-04: the referential actions and the delete itself publish as one commit, so no tab
|
|
561
694
|
// can observe a parent gone while its children still point at it.
|
|
@@ -606,14 +739,15 @@ export class MinnowDatabase {
|
|
|
606
739
|
keys.set(token, value);
|
|
607
740
|
});
|
|
608
741
|
const logicalBytes = estimateValuesBytes(input.keys);
|
|
609
|
-
|
|
742
|
+
// Deferred: the record is written only if something stages in two steps (trigger rows),
|
|
743
|
+
// and otherwise rides the single-shot commit below — or never exists, for a no-op delete.
|
|
744
|
+
const transaction = await this.#transactions.beginDeferred();
|
|
610
745
|
const segmentId = this.#createId();
|
|
611
746
|
transaction.setUniqueKeyChanges({
|
|
612
747
|
tableId: table.id,
|
|
613
748
|
keyTokens: [...keys.keys()],
|
|
614
749
|
requireAbsent: false,
|
|
615
750
|
remove: true,
|
|
616
|
-
...(table.uniqueKeyStorage === undefined ? {} : { storageMode: table.uniqueKeyStorage }),
|
|
617
751
|
});
|
|
618
752
|
let deletedRowCount;
|
|
619
753
|
let storedBytes = 0;
|
|
@@ -674,11 +808,7 @@ export class MinnowDatabase {
|
|
|
674
808
|
const deletePreImages = (await this.#triggerPreImages(table, keyColumn, [...keys.values()], "delete")).filter((row) => row !== undefined);
|
|
675
809
|
const deleteValueAt = (source, column, rowIndex) => (source === "old" ? (deletePreImages[rowIndex]?.[column] ?? null) : null);
|
|
676
810
|
await this.#stageTriggerDerivedInserts(transaction, table, "delete", deletePreImages.length, deleteValueAt, "before");
|
|
677
|
-
|
|
678
|
-
await transaction.stageBlocks(blockWrites);
|
|
679
|
-
stageMs += performance.now() - stageStarted;
|
|
680
|
-
stageStarted = performance.now();
|
|
681
|
-
await transaction.stageSegment({
|
|
811
|
+
const segment = {
|
|
682
812
|
id: segmentId,
|
|
683
813
|
tableId: table.id,
|
|
684
814
|
transactionId: transaction.id,
|
|
@@ -690,15 +820,24 @@ export class MinnowDatabase {
|
|
|
690
820
|
keyColumnId: keyColumn.id,
|
|
691
821
|
level: 0,
|
|
692
822
|
createdAt: this.#now().toISOString(),
|
|
693
|
-
}
|
|
694
|
-
|
|
823
|
+
};
|
|
824
|
+
// AFTER triggers stage derived rows between the segment and the commit, which keeps the
|
|
825
|
+
// two apart; without them the stage and the commit collapse into one storage write.
|
|
826
|
+
const stagesAfter = firesAfterTriggers(table, "delete");
|
|
827
|
+
const stageStarted = performance.now();
|
|
828
|
+
if (stagesAfter) {
|
|
829
|
+
await transaction.stageArtifacts(blockWrites, [segment]);
|
|
830
|
+
await this.#stageTriggerDerivedInserts(transaction, table, "delete", deletePreImages.length, deleteValueAt, "after");
|
|
831
|
+
}
|
|
695
832
|
stageMs += performance.now() - stageStarted;
|
|
696
833
|
for (let attempt = 0; attempt <= this.#maxCommitRetries; attempt += 1) {
|
|
697
834
|
const commitStarted = performance.now();
|
|
698
835
|
try {
|
|
699
|
-
const manifest =
|
|
836
|
+
const manifest = stagesAfter
|
|
837
|
+
? await transaction.commit()
|
|
838
|
+
: await transaction.stageArtifactsAndCommit(blockWrites, [segment]);
|
|
700
839
|
commitMs += performance.now() - commitStarted;
|
|
701
|
-
this.#
|
|
840
|
+
this.#afterCommit(manifest);
|
|
702
841
|
return {
|
|
703
842
|
tableName: table.name,
|
|
704
843
|
segmentId,
|
|
@@ -745,7 +884,8 @@ export class MinnowDatabase {
|
|
|
745
884
|
const started = performance.now();
|
|
746
885
|
const logicalBytes = estimateValuesBytes(input.keys) +
|
|
747
886
|
Object.values(input.changes).reduce((total, values) => total + estimateValuesBytes(values), 0);
|
|
748
|
-
|
|
887
|
+
// Deferred: the record rides the single-shot commit below unless trigger rows stage first.
|
|
888
|
+
const transaction = await this.#transactions.beginDeferred();
|
|
749
889
|
const segmentId = this.#createId();
|
|
750
890
|
const columnBlockIds = {};
|
|
751
891
|
const changedColumns = Object.keys(input.changes).sort();
|
|
@@ -810,30 +950,36 @@ export class MinnowDatabase {
|
|
|
810
950
|
}
|
|
811
951
|
}
|
|
812
952
|
await this.#stageTriggerDerivedInserts(transaction, table, "update", input.keys.length, updateValueAt, "before");
|
|
953
|
+
const segment = {
|
|
954
|
+
id: segmentId,
|
|
955
|
+
tableId: table.id,
|
|
956
|
+
transactionId: transaction.id,
|
|
957
|
+
rowCount: input.keys.length,
|
|
958
|
+
rowIdStart: 0n,
|
|
959
|
+
rowIdEndExclusive: 0n,
|
|
960
|
+
columnBlockIds,
|
|
961
|
+
kind: "update",
|
|
962
|
+
keyColumnId: keyColumn.id,
|
|
963
|
+
level: 0,
|
|
964
|
+
createdAt: this.#now().toISOString(),
|
|
965
|
+
};
|
|
966
|
+
// AFTER triggers stage derived rows between the segment and the commit, which keeps the
|
|
967
|
+
// two apart; without them the stage and the commit collapse into one storage write.
|
|
968
|
+
const stagesAfter = firesAfterTriggers(table, "update");
|
|
813
969
|
const stageStarted = performance.now();
|
|
814
|
-
|
|
815
|
-
|
|
816
|
-
|
|
817
|
-
|
|
818
|
-
transactionId: transaction.id,
|
|
819
|
-
rowCount: input.keys.length,
|
|
820
|
-
rowIdStart: 0n,
|
|
821
|
-
rowIdEndExclusive: 0n,
|
|
822
|
-
columnBlockIds,
|
|
823
|
-
kind: "update",
|
|
824
|
-
keyColumnId: keyColumn.id,
|
|
825
|
-
level: 0,
|
|
826
|
-
createdAt: this.#now().toISOString(),
|
|
827
|
-
},
|
|
828
|
-
]);
|
|
829
|
-
await this.#stageTriggerDerivedInserts(transaction, table, "update", input.keys.length, updateValueAt, "after");
|
|
970
|
+
if (stagesAfter) {
|
|
971
|
+
await transaction.stageArtifacts(batchBlockWrites, [segment]);
|
|
972
|
+
await this.#stageTriggerDerivedInserts(transaction, table, "update", input.keys.length, updateValueAt, "after");
|
|
973
|
+
}
|
|
830
974
|
stageMs += performance.now() - stageStarted;
|
|
831
975
|
for (let attempt = 0; attempt <= this.#maxCommitRetries; attempt += 1) {
|
|
832
976
|
const commitStarted = performance.now();
|
|
833
977
|
try {
|
|
834
|
-
const manifest =
|
|
978
|
+
const manifest = stagesAfter
|
|
979
|
+
? await transaction.commit()
|
|
980
|
+
: await transaction.stageArtifactsAndCommit(batchBlockWrites, [segment]);
|
|
835
981
|
commitMs += performance.now() - commitStarted;
|
|
836
|
-
this.#
|
|
982
|
+
this.#afterCommit(manifest);
|
|
837
983
|
return {
|
|
838
984
|
tableName: table.name,
|
|
839
985
|
segmentId,
|
|
@@ -927,7 +1073,6 @@ export class MinnowDatabase {
|
|
|
927
1073
|
tableId: table.id,
|
|
928
1074
|
keyTokens: [...resolvedKeys.keys()],
|
|
929
1075
|
requireAbsent: kind === "insert",
|
|
930
|
-
...(table.uniqueKeyStorage === undefined ? {} : { storageMode: table.uniqueKeyStorage }),
|
|
931
1076
|
});
|
|
932
1077
|
}
|
|
933
1078
|
counts =
|
|
@@ -1008,20 +1153,29 @@ export class MinnowDatabase {
|
|
|
1008
1153
|
else if (upsertFirings !== undefined) {
|
|
1009
1154
|
await this.#stageUpsertTriggerFirings(transaction, table, input, upsertFirings, "before");
|
|
1010
1155
|
}
|
|
1011
|
-
|
|
1012
|
-
|
|
1013
|
-
|
|
1014
|
-
|
|
1015
|
-
|
|
1016
|
-
|
|
1156
|
+
// AFTER triggers stage derived rows between the segment and the commit, which keeps the
|
|
1157
|
+
// two apart; without them the stage and the commit collapse into one storage write.
|
|
1158
|
+
const stagesAfter = kind === "insert"
|
|
1159
|
+
? firesAfterTriggers(table, "insert")
|
|
1160
|
+
: upsertFirings !== undefined && firesAfterTriggers(table, "insert", "update");
|
|
1161
|
+
if (stagesAfter) {
|
|
1162
|
+
await transaction.stageArtifacts(batchBlockWrites, [segment]);
|
|
1163
|
+
if (kind === "insert") {
|
|
1164
|
+
await this.#stageTriggerDerivedInserts(transaction, table, "insert", rowCount, insertValueAt, "after");
|
|
1165
|
+
}
|
|
1166
|
+
else if (upsertFirings !== undefined) {
|
|
1167
|
+
await this.#stageUpsertTriggerFirings(transaction, table, input, upsertFirings, "after");
|
|
1168
|
+
}
|
|
1017
1169
|
}
|
|
1018
1170
|
stageMs += performance.now() - stageStarted;
|
|
1019
1171
|
for (let attempt = 0; attempt <= this.#maxCommitRetries; attempt += 1) {
|
|
1020
1172
|
const commitStarted = performance.now();
|
|
1021
1173
|
try {
|
|
1022
|
-
const manifest =
|
|
1174
|
+
const manifest = stagesAfter
|
|
1175
|
+
? await transaction.commit()
|
|
1176
|
+
: await transaction.stageArtifactsAndCommit(batchBlockWrites, [segment]);
|
|
1023
1177
|
commitMs += performance.now() - commitStarted;
|
|
1024
|
-
this.#
|
|
1178
|
+
this.#afterCommit(manifest);
|
|
1025
1179
|
return {
|
|
1026
1180
|
tableName: table.name,
|
|
1027
1181
|
segmentId,
|
|
@@ -1125,13 +1279,13 @@ export class MinnowDatabase {
|
|
|
1125
1279
|
}
|
|
1126
1280
|
return plan;
|
|
1127
1281
|
}
|
|
1128
|
-
async #prepareCompiledPlan(plan, options = {}) {
|
|
1282
|
+
async #prepareCompiledPlan(plan, options = {}, probe) {
|
|
1129
1283
|
// The ORDER-BY-expression desugar's wrapper is projection-only: prepare the inner block
|
|
1130
1284
|
// directly (no derived materialization) and project each result to the visible aliases,
|
|
1131
1285
|
// so `.search()` costs the same whether or not the caller also selects the score.
|
|
1132
1286
|
const wrapper = transparentProjectionSource(plan);
|
|
1133
1287
|
if (wrapper !== undefined) {
|
|
1134
|
-
const prepared = await this.#prepareCompiledPlan(wrapper.inner, options);
|
|
1288
|
+
const prepared = await this.#prepareCompiledPlan(wrapper.inner, options, probe);
|
|
1135
1289
|
return {
|
|
1136
1290
|
sql: prepared.sql,
|
|
1137
1291
|
tables: prepared.tables,
|
|
@@ -1179,7 +1333,7 @@ export class MinnowDatabase {
|
|
|
1179
1333
|
});
|
|
1180
1334
|
}
|
|
1181
1335
|
else {
|
|
1182
|
-
await this.#withSharedCatalogSnapshot(collectRealTableNames(plan), prepareAtSnapshot);
|
|
1336
|
+
await this.#withSharedCatalogSnapshot(collectRealTableNames(plan), prepareAtSnapshot, probe);
|
|
1183
1337
|
}
|
|
1184
1338
|
return createPreparedColumnarQuery(chooseJoinOrder(resolvedPlan, columnarTables), columnarTables, memory, ftsStats === undefined ? {} : { ftsStats });
|
|
1185
1339
|
}
|
|
@@ -1196,9 +1350,9 @@ export class MinnowDatabase {
|
|
|
1196
1350
|
* write transactions. If the manifest is pruned between the read and the lease, the
|
|
1197
1351
|
* state is re-read.
|
|
1198
1352
|
*/
|
|
1199
|
-
async #withSharedCatalogSnapshot(names, action) {
|
|
1353
|
+
async #withSharedCatalogSnapshot(names, action, probe) {
|
|
1200
1354
|
for (;;) {
|
|
1201
|
-
const state = await this.#cachedCatalogState(names);
|
|
1355
|
+
const state = await this.#cachedCatalogState(names, probe);
|
|
1202
1356
|
const realTables = new Map();
|
|
1203
1357
|
names.forEach((name, index) => {
|
|
1204
1358
|
const table = state.tables[index];
|
|
@@ -1239,11 +1393,13 @@ export class MinnowDatabase {
|
|
|
1239
1393
|
* cached state was read, so reuse is exact, not heuristic. Stores without a probe are never
|
|
1240
1394
|
* cached. Entries key on the requested table-name set; a changed epoch clears them all.
|
|
1241
1395
|
*/
|
|
1242
|
-
async #cachedCatalogState(names) {
|
|
1243
|
-
|
|
1244
|
-
|
|
1396
|
+
async #cachedCatalogState(names, probe) {
|
|
1397
|
+
// A probe the caller read moments earlier in the same statement serves: a state read under
|
|
1398
|
+
// it is at least as fresh, and the cache is only consulted under its epoch.
|
|
1399
|
+
const read = this.store.getCatalogProbe?.bind(this.store);
|
|
1400
|
+
if (read === undefined)
|
|
1245
1401
|
return this.#queryCatalogState(names);
|
|
1246
|
-
const { catalogEpoch } = await
|
|
1402
|
+
const { catalogEpoch } = probe ?? (await read());
|
|
1247
1403
|
// Table names are only trimmed, never charset-restricted, so no join separator is
|
|
1248
1404
|
// collision-free; JSON encoding is.
|
|
1249
1405
|
const key = JSON.stringify(names);
|
|
@@ -1292,45 +1448,83 @@ export class MinnowDatabase {
|
|
|
1292
1448
|
}
|
|
1293
1449
|
/**
|
|
1294
1450
|
* Reuses the shared internal reader lease when it targets the requested version and has
|
|
1295
|
-
* not expired
|
|
1296
|
-
* one
|
|
1297
|
-
*
|
|
1451
|
+
* not expired. Otherwise the pin has to move: with no reader left on the old version the
|
|
1452
|
+
* one lease record is re-pinned in place (one storage write, instead of a create now and a
|
|
1453
|
+
* remove once the old one drains); while readers remain, a fresh lease opens at the exact
|
|
1454
|
+
* version and the old one retires as they finish. Returns undefined when the version's
|
|
1455
|
+
* manifest disappeared between the catalog read and the lease, so the caller can re-read.
|
|
1298
1456
|
*/
|
|
1299
1457
|
async #acquireSharedLease(version) {
|
|
1300
|
-
|
|
1301
|
-
|
|
1302
|
-
|
|
1303
|
-
|
|
1304
|
-
|
|
1305
|
-
|
|
1306
|
-
return current;
|
|
1458
|
+
for (;;) {
|
|
1459
|
+
// A move in flight is closing the shared snapshot it re-pins; wait for it rather than
|
|
1460
|
+
// hand that snapshot out, then look again.
|
|
1461
|
+
if (this.#sharedLeaseMove !== undefined) {
|
|
1462
|
+
await this.#sharedLeaseMove;
|
|
1463
|
+
continue;
|
|
1307
1464
|
}
|
|
1308
|
-
|
|
1309
|
-
|
|
1310
|
-
|
|
1465
|
+
const current = this.#sharedLease;
|
|
1466
|
+
if (current?.version === version &&
|
|
1467
|
+
current.lease.expiresAt.getTime() - this.#now().getTime() > 0) {
|
|
1468
|
+
current.refCount += 1;
|
|
1469
|
+
try {
|
|
1470
|
+
await this.#renewInternalLeaseIfNeeded(current.lease);
|
|
1471
|
+
return current;
|
|
1472
|
+
}
|
|
1473
|
+
catch (error) {
|
|
1474
|
+
this.#releaseSharedLease(current);
|
|
1475
|
+
throw error;
|
|
1476
|
+
}
|
|
1311
1477
|
}
|
|
1312
|
-
|
|
1313
|
-
let lease;
|
|
1314
|
-
try {
|
|
1315
|
-
lease = await this.#transactions.openLeasedSnapshot({
|
|
1478
|
+
const options = {
|
|
1316
1479
|
id: `${this.#internalLeaseOwnerId}/${String(this.#internalLeaseSequence++)}`,
|
|
1317
1480
|
ownerId: this.#internalLeaseOwnerId,
|
|
1318
1481
|
ttlMs: INTERNAL_READ_LEASE_TTL_MS,
|
|
1319
1482
|
version,
|
|
1320
|
-
}
|
|
1321
|
-
|
|
1322
|
-
|
|
1323
|
-
|
|
1324
|
-
|
|
1325
|
-
|
|
1326
|
-
|
|
1327
|
-
|
|
1328
|
-
|
|
1329
|
-
|
|
1330
|
-
|
|
1331
|
-
|
|
1483
|
+
};
|
|
1484
|
+
let lease;
|
|
1485
|
+
try {
|
|
1486
|
+
if (current?.refCount === 0) {
|
|
1487
|
+
const move = this.#transactions.moveLeasedSnapshot(current.lease, options);
|
|
1488
|
+
this.#sharedLeaseMove = move.then(() => undefined, () => undefined);
|
|
1489
|
+
try {
|
|
1490
|
+
lease = await move;
|
|
1491
|
+
}
|
|
1492
|
+
finally {
|
|
1493
|
+
this.#sharedLeaseMove = undefined;
|
|
1494
|
+
}
|
|
1495
|
+
}
|
|
1496
|
+
else {
|
|
1497
|
+
lease = await this.#transactions.openLeasedSnapshot(options);
|
|
1498
|
+
}
|
|
1499
|
+
}
|
|
1500
|
+
catch (error) {
|
|
1501
|
+
if (error instanceof SnapshotManifestMissingError)
|
|
1502
|
+
return undefined;
|
|
1503
|
+
throw error;
|
|
1504
|
+
}
|
|
1505
|
+
const entry = { lease, version, refCount: 1 };
|
|
1506
|
+
const previous = this.#sharedLease;
|
|
1507
|
+
this.#sharedLease = entry;
|
|
1508
|
+
if (previous?.refCount === 0) {
|
|
1509
|
+
// Already closed when it was the one just moved; a remove otherwise.
|
|
1510
|
+
void previous.lease.release().catch(() => undefined);
|
|
1511
|
+
}
|
|
1512
|
+
return entry;
|
|
1332
1513
|
}
|
|
1333
|
-
|
|
1514
|
+
}
|
|
1515
|
+
/**
|
|
1516
|
+
* Drops the shared reader lease when nothing holds it and it has fallen behind the current
|
|
1517
|
+
* version. The lease outlives the query that took it so the next query at the same version
|
|
1518
|
+
* reuses it — but after a burst of writes and a fold, an idle database's last lease sits at a
|
|
1519
|
+
* pre-fold version and roots every block that version referenced, which is exactly what a
|
|
1520
|
+
* collection pass is trying to reclaim. The next query simply takes a fresh lease.
|
|
1521
|
+
*/
|
|
1522
|
+
#releaseIdleSharedLease() {
|
|
1523
|
+
const current = this.#sharedLease;
|
|
1524
|
+
if (current?.refCount !== 0)
|
|
1525
|
+
return;
|
|
1526
|
+
this.#sharedLease = undefined;
|
|
1527
|
+
void current.lease.release().catch(() => undefined);
|
|
1334
1528
|
}
|
|
1335
1529
|
#releaseSharedLease(entry) {
|
|
1336
1530
|
entry.refCount -= 1;
|
|
@@ -1439,7 +1633,9 @@ export class MinnowDatabase {
|
|
|
1439
1633
|
}
|
|
1440
1634
|
}
|
|
1441
1635
|
const { result: inner, schema: innerSchema } = await this.#executeBlockWithSchemaCached(source.windowed.block, snapshot, visibility, memory, realTables, typedSchemas, cacheResults);
|
|
1442
|
-
const windowed = applyWindowFunctions(inner, source.windowed.windows
|
|
1636
|
+
const windowed = applyWindowFunctions(inner, source.windowed.windows, {
|
|
1637
|
+
copyRows: cacheResults,
|
|
1638
|
+
});
|
|
1443
1639
|
const schema = [
|
|
1444
1640
|
...innerSchema,
|
|
1445
1641
|
...source.windowed.windows.map((window) => ({
|
|
@@ -1522,12 +1718,20 @@ export class MinnowDatabase {
|
|
|
1522
1718
|
probe !== undefined;
|
|
1523
1719
|
if (probe === undefined || !memoizable)
|
|
1524
1720
|
return this.#queryCompiled(plan, options);
|
|
1525
|
-
|
|
1721
|
+
return this.#memoizedQuery(plan, `res ${queryResultMemoKey(sql, options.params ?? [])}`, options, probe);
|
|
1722
|
+
}
|
|
1723
|
+
/**
|
|
1724
|
+
* The result memo: a pure cache over the freshness probe, keyed by the statement and the
|
|
1725
|
+
* catalog epoch it was answered at. The probe read before execution is handed down to the
|
|
1726
|
+
* execution itself — the view lookup and the catalog state would otherwise each probe again,
|
|
1727
|
+
* and on IndexedDB every probe is a read transaction, a floor under every small query.
|
|
1728
|
+
*/
|
|
1729
|
+
async #memoizedQuery(plan, key, options, probe) {
|
|
1526
1730
|
const before = await probe();
|
|
1527
1731
|
const cached = this.#cacheGet(`${key}\u0001${String(before.catalogEpoch)}`);
|
|
1528
1732
|
if (cached !== undefined)
|
|
1529
1733
|
return copyQueryResult(cached);
|
|
1530
|
-
const result = await this.#queryCompiled(plan, options);
|
|
1734
|
+
const result = await this.#queryCompiled(plan, options, before);
|
|
1531
1735
|
const bytes = queryResultRetainedBytes(result);
|
|
1532
1736
|
if (bytes <= RESULT_MEMO_MAX_BYTES) {
|
|
1533
1737
|
// Cache only when the epoch did not move during execution: the result is then exactly
|
|
@@ -1545,14 +1749,14 @@ export class MinnowDatabase {
|
|
|
1545
1749
|
* (E051-09), and a NATURAL join becomes the equality over the columns its sides share
|
|
1546
1750
|
* (F401-01). Reads the catalog only for the statements that ask for one of them.
|
|
1547
1751
|
*/
|
|
1548
|
-
async #applyCatalogRewrites(plan) {
|
|
1752
|
+
async #applyCatalogRewrites(plan, probe) {
|
|
1549
1753
|
const aliased = planHasSourceColumnAliases(plan);
|
|
1550
1754
|
const natural = planHasNaturalJoins(plan);
|
|
1551
1755
|
// Whether a name is a view cannot be read off the statement, so this is the one thing every
|
|
1552
1756
|
// read has to ask the catalog. It asks by epoch — an O(1) probe the store already serves for
|
|
1553
1757
|
// result memoization — and only re-reads the view set when the catalog has actually moved.
|
|
1554
1758
|
// A database with no views therefore pays one probe, not a catalog scan per query.
|
|
1555
|
-
const { views } = await this.#catalogFacts();
|
|
1759
|
+
const { views } = await this.#catalogFacts(probe);
|
|
1556
1760
|
let rewritten = plan;
|
|
1557
1761
|
if (views.size > 0 && planReadsViews(plan, (name) => views.has(name))) {
|
|
1558
1762
|
const bodies = new Map();
|
|
@@ -1587,8 +1791,8 @@ export class MinnowDatabase {
|
|
|
1587
1791
|
* the steady state has to be one epoch probe and no allocation; the facts are rebuilt only
|
|
1588
1792
|
* when a catalog mutation — anywhere, including another tab — moves the epoch.
|
|
1589
1793
|
*/
|
|
1590
|
-
async #catalogFacts() {
|
|
1591
|
-
|
|
1794
|
+
async #catalogFacts(probe) {
|
|
1795
|
+
probe ??= await this.store.getCatalogProbe?.();
|
|
1592
1796
|
const epoch = probe?.catalogEpoch;
|
|
1593
1797
|
const cached = this.#catalogCache;
|
|
1594
1798
|
if (cached !== undefined && epoch !== undefined && cached.epoch === epoch)
|
|
@@ -1620,13 +1824,16 @@ export class MinnowDatabase {
|
|
|
1620
1824
|
* re-runs — routes through the same streaming-first execution, so builder/SQL parity holds
|
|
1621
1825
|
* for the execution path as well as the plan.
|
|
1622
1826
|
*/
|
|
1623
|
-
async #queryCompiled(plan, options = {}) {
|
|
1624
|
-
|
|
1827
|
+
async #queryCompiled(plan, options = {}, probe) {
|
|
1828
|
+
// One freshness probe per query: read here unless the caller already has one, and handed
|
|
1829
|
+
// to the view lookup and the catalog state below, which would otherwise probe again each.
|
|
1830
|
+
probe ??= await this.store.getCatalogProbe?.();
|
|
1831
|
+
plan = await this.#applyCatalogRewrites(plan, probe);
|
|
1625
1832
|
const spillPageRows = options.spillPageRows === undefined
|
|
1626
1833
|
? undefined
|
|
1627
1834
|
: positiveWholeNumber(options.spillPageRows, "Query spill page rows");
|
|
1628
1835
|
if (this.#canStreamPlanShape(plan, options)) {
|
|
1629
|
-
const streamed = await this.#queryStreamed(plan, options, spillPageRows);
|
|
1836
|
+
const streamed = await this.#queryStreamed(plan, options, spillPageRows, probe);
|
|
1630
1837
|
if (streamed !== undefined)
|
|
1631
1838
|
return streamed;
|
|
1632
1839
|
}
|
|
@@ -1636,12 +1843,12 @@ export class MinnowDatabase {
|
|
|
1636
1843
|
// query its streaming eligibility.
|
|
1637
1844
|
const wrapper = transparentProjectionSource(plan);
|
|
1638
1845
|
if (wrapper !== undefined && this.#canStreamPlanShape(wrapper.inner, options)) {
|
|
1639
|
-
const streamed = await this.#queryStreamed(wrapper.inner, options, spillPageRows);
|
|
1846
|
+
const streamed = await this.#queryStreamed(wrapper.inner, options, spillPageRows, probe);
|
|
1640
1847
|
if (streamed !== undefined)
|
|
1641
1848
|
return projectResultColumns(streamed, wrapper.aliases);
|
|
1642
1849
|
}
|
|
1643
1850
|
}
|
|
1644
|
-
const prepared = await this.#prepareCompiledPlan(plan, options);
|
|
1851
|
+
const prepared = await this.#prepareCompiledPlan(plan, options, probe);
|
|
1645
1852
|
// Read the peak before close(): closing releases the context and zeroes what it tracked.
|
|
1646
1853
|
const report = (result) => {
|
|
1647
1854
|
options.onStats?.({ peakMemoryBytes: prepared.memoryUsage.peakBytes });
|
|
@@ -1695,11 +1902,22 @@ export class MinnowDatabase {
|
|
|
1695
1902
|
const renewed = await this.store.renewTempOwner(ownerId, lease.revision, new Date(expiresAtMs).toISOString());
|
|
1696
1903
|
leases.set(ownerId, { revision: renewed.revision, expiresAtMs });
|
|
1697
1904
|
};
|
|
1905
|
+
const batched = this.store.putTempRunPages?.bind(this.store);
|
|
1698
1906
|
return {
|
|
1699
1907
|
putPage: async (ownerId, runId, pageIndex, bytes) => {
|
|
1700
1908
|
await ensureLease(ownerId);
|
|
1701
1909
|
await this.store.putTempRunPage({ ownerId, runId, pageIndex, bytes });
|
|
1702
1910
|
},
|
|
1911
|
+
...(batched === undefined
|
|
1912
|
+
? {}
|
|
1913
|
+
: {
|
|
1914
|
+
putPages: async (pages) => {
|
|
1915
|
+
for (const owner of new Set(pages.map((page) => page.ownerId))) {
|
|
1916
|
+
await ensureLease(owner);
|
|
1917
|
+
}
|
|
1918
|
+
await batched(pages);
|
|
1919
|
+
},
|
|
1920
|
+
}),
|
|
1703
1921
|
getPage: async (ownerId, runId, pageIndex) => {
|
|
1704
1922
|
await ensureLease(ownerId);
|
|
1705
1923
|
return this.store.getTempRunPage(ownerId, runId, pageIndex);
|
|
@@ -1774,12 +1992,13 @@ export class MinnowDatabase {
|
|
|
1774
1992
|
return set;
|
|
1775
1993
|
}
|
|
1776
1994
|
/**
|
|
1777
|
-
*
|
|
1778
|
-
* (after, until] to
|
|
1779
|
-
*
|
|
1780
|
-
* reject the
|
|
1781
|
-
*
|
|
1782
|
-
*
|
|
1995
|
+
* The data-layer proof behind a live sweep's zone skips: whether the commits in
|
|
1996
|
+
* (after, until] to these tables can change this query's result. False only on proof —
|
|
1997
|
+
* every segment the window introduced to the table is a compaction rewrite or an insert
|
|
1998
|
+
* whose zone maps reject the query's predicates, and every version that changed the table
|
|
1999
|
+
* left a segment to inspect.
|
|
2000
|
+
*
|
|
2001
|
+
* Its inputs come from `#liveProofContext`, shared by every subscription in the sweep.
|
|
1783
2002
|
*/
|
|
1784
2003
|
async #liveChangeCanAffect(query, tableIds, after, until) {
|
|
1785
2004
|
const plan = typeof query === "string" ? this.#compileCached(query) : query.plan;
|
|
@@ -1790,44 +2009,13 @@ export class MinnowDatabase {
|
|
|
1790
2009
|
// predicates reject (e.g. `value > (SELECT AVG(value) FROM t)`).
|
|
1791
2010
|
if (planReadsBeyondSingleScan(plan))
|
|
1792
2011
|
return true;
|
|
1793
|
-
|
|
1794
|
-
// one of them to be accounted for by a surviving, inspected segment: garbage collection
|
|
1795
|
-
// deletes reclaimed segments outright, so "no segment in the window" is absence of
|
|
1796
|
-
// evidence, not evidence of neutrality.
|
|
1797
|
-
const changedVersions = new Map();
|
|
1798
|
-
{
|
|
1799
|
-
let cursor = after;
|
|
1800
|
-
pages: for (;;) {
|
|
1801
|
-
const page = await this.store.listManifestPage(cursor, 64);
|
|
1802
|
-
for (const manifest of page.records) {
|
|
1803
|
-
if (manifest.version > until)
|
|
1804
|
-
break pages;
|
|
1805
|
-
for (const tableId of manifest.changedTableIds ?? []) {
|
|
1806
|
-
const versions = changedVersions.get(tableId) ?? new Set();
|
|
1807
|
-
versions.add(manifest.version);
|
|
1808
|
-
changedVersions.set(tableId, versions);
|
|
1809
|
-
}
|
|
1810
|
-
if (manifest.version === until)
|
|
1811
|
-
break pages;
|
|
1812
|
-
}
|
|
1813
|
-
if (page.nextCursor === null)
|
|
1814
|
-
break;
|
|
1815
|
-
cursor = page.nextCursor;
|
|
1816
|
-
}
|
|
1817
|
-
}
|
|
2012
|
+
const context = await this.#liveProofContext(after, until);
|
|
1818
2013
|
for (const tableId of tableIds) {
|
|
1819
|
-
const
|
|
1820
|
-
if (
|
|
2014
|
+
const entry = await this.#liveProofTable(context, tableId);
|
|
2015
|
+
if (entry === undefined)
|
|
1821
2016
|
return true;
|
|
1822
|
-
const predicates = zonePredicates(plan, table);
|
|
1823
|
-
const
|
|
1824
|
-
const transactions = new Map((await this.#transactionRecordsForSegments(segments)).map((record) => [record.id, record]));
|
|
1825
|
-
const coveredVersions = new Set();
|
|
1826
|
-
for (const segment of segments) {
|
|
1827
|
-
const committed = transactions.get(segment.transactionId)?.committedVersion ?? null;
|
|
1828
|
-
if (committed === null || committed <= (after ?? -1) || committed > until)
|
|
1829
|
-
continue;
|
|
1830
|
-
coveredVersions.add(committed);
|
|
2017
|
+
const predicates = zonePredicates(plan, entry.table);
|
|
2018
|
+
for (const segment of entry.windowSegments) {
|
|
1831
2019
|
const kind = segment.kind ?? "insert";
|
|
1832
2020
|
// Compaction rewrites are visible-data-neutral by construction.
|
|
1833
2021
|
if (kind === "base")
|
|
@@ -1884,13 +2072,85 @@ export class MinnowDatabase {
|
|
|
1884
2072
|
}
|
|
1885
2073
|
// A version that changed this table but left no surviving segment to inspect (its
|
|
1886
2074
|
// segments were compacted away and reclaimed) cannot be proven neutral.
|
|
1887
|
-
for (const version of changedVersions.get(tableId) ?? []) {
|
|
1888
|
-
if (!coveredVersions.has(version))
|
|
2075
|
+
for (const version of context.changedVersions.get(tableId) ?? []) {
|
|
2076
|
+
if (!entry.coveredVersions.has(version))
|
|
1889
2077
|
return true;
|
|
1890
2078
|
}
|
|
1891
2079
|
}
|
|
1892
2080
|
return false;
|
|
1893
2081
|
}
|
|
2082
|
+
/**
|
|
2083
|
+
* The inputs every live proof over one commit window shares: the versions in (after, until]
|
|
2084
|
+
* that recorded a change to each table, and — filled in per table as proofs ask — the table
|
|
2085
|
+
* record, its segments committed in the window, and the versions those segments account
|
|
2086
|
+
* for. A sweep proves each subscription separately, and twenty subscriptions on one table
|
|
2087
|
+
* used to list its segments and transactions twenty times, a readonly transaction each on
|
|
2088
|
+
* IndexedDB and a cross-tab round trip each on an OPFS follower. A few recent windows stay
|
|
2089
|
+
* resident so concurrent sets sweeping different windows do not evict each other.
|
|
2090
|
+
*/
|
|
2091
|
+
async #liveProofContext(after, until) {
|
|
2092
|
+
const key = `${String(after)}:${String(until)}`;
|
|
2093
|
+
const cached = this.#liveProofContexts.get(key);
|
|
2094
|
+
if (cached !== undefined)
|
|
2095
|
+
return cached;
|
|
2096
|
+
const context = (async () => {
|
|
2097
|
+
// Proof requires every version to be accounted for by a surviving, inspected segment:
|
|
2098
|
+
// garbage collection deletes reclaimed segments outright, so "no segment in the window"
|
|
2099
|
+
// is absence of evidence, not evidence of neutrality.
|
|
2100
|
+
const changedVersions = new Map();
|
|
2101
|
+
let cursor = after;
|
|
2102
|
+
pages: for (;;) {
|
|
2103
|
+
const page = await this.store.listManifestPage(cursor, 64);
|
|
2104
|
+
for (const manifest of page.records) {
|
|
2105
|
+
if (manifest.version > until)
|
|
2106
|
+
break pages;
|
|
2107
|
+
for (const tableId of manifest.changedTableIds ?? []) {
|
|
2108
|
+
const versions = changedVersions.get(tableId) ?? new Set();
|
|
2109
|
+
versions.add(manifest.version);
|
|
2110
|
+
changedVersions.set(tableId, versions);
|
|
2111
|
+
}
|
|
2112
|
+
if (manifest.version === until)
|
|
2113
|
+
break pages;
|
|
2114
|
+
}
|
|
2115
|
+
if (page.nextCursor === null)
|
|
2116
|
+
break;
|
|
2117
|
+
cursor = page.nextCursor;
|
|
2118
|
+
}
|
|
2119
|
+
return { after, until, changedVersions, tables: new Map() };
|
|
2120
|
+
})();
|
|
2121
|
+
this.#liveProofContexts.set(key, context);
|
|
2122
|
+
if (this.#liveProofContexts.size > LIVE_PROOF_CONTEXT_LIMIT) {
|
|
2123
|
+
const oldest = this.#liveProofContexts.keys().next().value;
|
|
2124
|
+
if (oldest !== undefined)
|
|
2125
|
+
this.#liveProofContexts.delete(oldest);
|
|
2126
|
+
}
|
|
2127
|
+
return context;
|
|
2128
|
+
}
|
|
2129
|
+
#liveProofTable(context, tableId) {
|
|
2130
|
+
const cached = context.tables.get(tableId);
|
|
2131
|
+
if (cached !== undefined)
|
|
2132
|
+
return cached;
|
|
2133
|
+
const entry = (async () => {
|
|
2134
|
+
const table = await this.store.getTable(tableId);
|
|
2135
|
+
if (table === undefined)
|
|
2136
|
+
return undefined;
|
|
2137
|
+
const segments = await this.store.listSegments(tableId);
|
|
2138
|
+
const transactions = new Map((await this.#transactionRecordsForSegments(segments)).map((record) => [record.id, record]));
|
|
2139
|
+
const windowSegments = [];
|
|
2140
|
+
const coveredVersions = new Set();
|
|
2141
|
+
for (const segment of segments) {
|
|
2142
|
+
const committed = transactions.get(segment.transactionId)?.committedVersion ?? null;
|
|
2143
|
+
if (committed === null || committed <= (context.after ?? -1) || committed > context.until) {
|
|
2144
|
+
continue;
|
|
2145
|
+
}
|
|
2146
|
+
coveredVersions.add(committed);
|
|
2147
|
+
windowSegments.push(segment);
|
|
2148
|
+
}
|
|
2149
|
+
return { table, windowSegments, coveredVersions };
|
|
2150
|
+
})();
|
|
2151
|
+
context.tables.set(tableId, entry);
|
|
2152
|
+
return entry;
|
|
2153
|
+
}
|
|
1894
2154
|
/**
|
|
1895
2155
|
* True when every manifest published in (after, until] changed no row in any table: the
|
|
1896
2156
|
* explicitly empty `changedTableIds` that compaction publishes through
|
|
@@ -1942,29 +2202,168 @@ export class MinnowDatabase {
|
|
|
1942
2202
|
#maybeScheduleAutoCompaction(table, segments) {
|
|
1943
2203
|
if (!this.#autoCompact)
|
|
1944
2204
|
return;
|
|
1945
|
-
|
|
1946
|
-
|
|
1947
|
-
|
|
1948
|
-
}).length;
|
|
1949
|
-
if (segments.length < AUTO_COMPACT_SCAN_SEGMENTS && deltas < AUTO_COMPACT_DELTA_SEGMENTS) {
|
|
2205
|
+
if (this.#droppingTables.has(table.id))
|
|
2206
|
+
return;
|
|
2207
|
+
if (!autoCompactionDue(segments))
|
|
1950
2208
|
return;
|
|
1951
|
-
}
|
|
1952
2209
|
if (segments.length < (this.#autoCompactionBackoff.get(table.id) ?? 0))
|
|
1953
2210
|
return;
|
|
1954
|
-
if (this.#autoCompactionsInFlight.has(table.id))
|
|
2211
|
+
if (this.#autoCompactionsInFlight.has(table.id)) {
|
|
2212
|
+
// A final burst can cross the threshold while the prior fold is still planning or
|
|
2213
|
+
// running. Remember it: otherwise no later commit or scan may arrive to trigger the fold
|
|
2214
|
+
// that the final state still needs.
|
|
2215
|
+
this.#autoCompactionsRequested.add(table.id);
|
|
1955
2216
|
return;
|
|
2217
|
+
}
|
|
1956
2218
|
this.#autoCompactionsInFlight.add(table.id);
|
|
1957
|
-
void this
|
|
1958
|
-
.then(() => {
|
|
1959
|
-
|
|
2219
|
+
void this.#runAutoCompaction(table)
|
|
2220
|
+
.then((folded) => {
|
|
2221
|
+
if (folded)
|
|
2222
|
+
this.#autoCompactionBackoff.delete(table.id);
|
|
2223
|
+
else {
|
|
2224
|
+
this.#autoCompactionBackoff.set(table.id, Math.min(AUTO_COMPACT_MAX_LEVEL_ZERO_SEGMENTS, Math.max(2, segments.length * 2)));
|
|
2225
|
+
}
|
|
1960
2226
|
})
|
|
1961
2227
|
.catch(() => {
|
|
1962
|
-
|
|
2228
|
+
// Back off deterministic failures, but never beyond the maximum L0 prefix a fold can
|
|
2229
|
+
// consume. A transient conflict near the end of a burst must not strand hundreds of
|
|
2230
|
+
// segments waiting for a segment count the idle database can never reach.
|
|
2231
|
+
this.#autoCompactionBackoff.set(table.id, Math.min(AUTO_COMPACT_MAX_LEVEL_ZERO_SEGMENTS, Math.max(2, segments.length * 2)));
|
|
1963
2232
|
})
|
|
1964
2233
|
.finally(() => {
|
|
1965
2234
|
this.#autoCompactionsInFlight.delete(table.id);
|
|
2235
|
+
if (this.#autoCompactionsRequested.delete(table.id)) {
|
|
2236
|
+
void yieldToEventLoop().then(() => this.#checkAutoCompaction(table.id));
|
|
2237
|
+
}
|
|
1966
2238
|
});
|
|
1967
2239
|
}
|
|
2240
|
+
/**
|
|
2241
|
+
* Plans a compaction job and drives it to publication in small steps, yielding to the event
|
|
2242
|
+
* loop between them so queries and writes interleave with the maintenance; then, while the
|
|
2243
|
+
* table is still due, plans the next. A job that only advanced when the next scan happened
|
|
2244
|
+
* to trigger it would sit half-written in an idle tab, its output staged and its sources
|
|
2245
|
+
* still read on every query, and the deltas that landed while it ran would wait for a scan
|
|
2246
|
+
* that may never come. Returns whether anything was folded: a table compaction cannot help
|
|
2247
|
+
* (an unsupported layout, keys living in published partitions) must not be re-planned on
|
|
2248
|
+
* every trigger, so the caller backs it off as it would a failure.
|
|
2249
|
+
*/
|
|
2250
|
+
async #runAutoCompaction(table) {
|
|
2251
|
+
const options = {
|
|
2252
|
+
maxBlocks: AUTO_COMPACT_STEP_BLOCKS,
|
|
2253
|
+
maxLevel0Segments: AUTO_COMPACT_MAX_LEVEL_ZERO_SEGMENTS,
|
|
2254
|
+
};
|
|
2255
|
+
let folded = false;
|
|
2256
|
+
for (;;) {
|
|
2257
|
+
if (this.#droppingTables.has(table.id))
|
|
2258
|
+
return folded;
|
|
2259
|
+
let progress = await this.compactTableStep(table.name, options);
|
|
2260
|
+
while (progress.result === null) {
|
|
2261
|
+
if (progress.jobId === null)
|
|
2262
|
+
throw new Error("Compaction progress lost its job ID");
|
|
2263
|
+
await yieldToEventLoop();
|
|
2264
|
+
progress = await this.resumeCompactionJob(progress.jobId, options);
|
|
2265
|
+
}
|
|
2266
|
+
if (!progress.result.compacted)
|
|
2267
|
+
return folded;
|
|
2268
|
+
folded = true;
|
|
2269
|
+
// The fold's sources are garbage now; collect before planning the next fold.
|
|
2270
|
+
this.#maybeScheduleAutoCollection();
|
|
2271
|
+
await yieldToEventLoop();
|
|
2272
|
+
const current = await this.store.getTable(table.id);
|
|
2273
|
+
if (current === undefined ||
|
|
2274
|
+
!autoCompactionDue(await this.#currentVisibleSegments(current))) {
|
|
2275
|
+
return folded;
|
|
2276
|
+
}
|
|
2277
|
+
}
|
|
2278
|
+
}
|
|
2279
|
+
/** The table's visible segments at the current manifest. */
|
|
2280
|
+
async #currentVisibleSegments(table) {
|
|
2281
|
+
// This is an optimistic metadata read, not a user snapshot: taking a durable reader lease
|
|
2282
|
+
// here would add a readwrite transaction to whichever foreground write happened to trigger
|
|
2283
|
+
// maintenance. Verify the manifest did not move while its segment records were loaded; the
|
|
2284
|
+
// current manifest itself cannot be pruned, so a matching version is the same stability proof
|
|
2285
|
+
// without persistent state. Do not chase a busy writer forever: this probe is only a hint,
|
|
2286
|
+
// periodic checks keep arriving during the burst, and the quiet-tail check gets a stable view
|
|
2287
|
+
// once it ends.
|
|
2288
|
+
let segments = [];
|
|
2289
|
+
for (let attempt = 0; attempt < 3; attempt += 1) {
|
|
2290
|
+
const manifest = await this.store.getCurrentManifest();
|
|
2291
|
+
segments = await this.#visibleSegmentRecords(table, new Snapshot(this.store, manifest?.version ?? null, manifest?.blockIds ?? []));
|
|
2292
|
+
if ((await this.store.getCurrentManifestVersion()) === (manifest?.version ?? null)) {
|
|
2293
|
+
return segments;
|
|
2294
|
+
}
|
|
2295
|
+
}
|
|
2296
|
+
return segments;
|
|
2297
|
+
}
|
|
2298
|
+
/**
|
|
2299
|
+
* What every data commit shares: live sets learn of it, and the tables it changed count
|
|
2300
|
+
* toward their next write-path auto-compaction check.
|
|
2301
|
+
*/
|
|
2302
|
+
#afterCommit(manifest) {
|
|
2303
|
+
this.#notifyLiveCommit();
|
|
2304
|
+
this.#commitsSinceCollection += 1;
|
|
2305
|
+
const now = this.#now().getTime();
|
|
2306
|
+
if (this.#commitsSinceCollection >= AUTO_COLLECT_COMMIT_INTERVAL ||
|
|
2307
|
+
(this.#lastCollectionAt !== undefined &&
|
|
2308
|
+
now - this.#lastCollectionAt >= AUTO_COLLECT_QUIET_MS)) {
|
|
2309
|
+
this.#maybeScheduleAutoCollection();
|
|
2310
|
+
}
|
|
2311
|
+
this.#armIdleCollection();
|
|
2312
|
+
if (!this.#autoCompact)
|
|
2313
|
+
return;
|
|
2314
|
+
for (const tableId of manifest.changedTableIds ?? []) {
|
|
2315
|
+
this.#idleCompactionTableIds.add(tableId);
|
|
2316
|
+
const commits = (this.#commitsSinceCompactionCheck.get(tableId) ?? 0) + 1;
|
|
2317
|
+
if (commits < AUTO_COMPACT_COMMIT_CHECK_INTERVAL) {
|
|
2318
|
+
this.#commitsSinceCompactionCheck.set(tableId, commits);
|
|
2319
|
+
continue;
|
|
2320
|
+
}
|
|
2321
|
+
this.#commitsSinceCompactionCheck.delete(tableId);
|
|
2322
|
+
void this.#checkAutoCompaction(tableId);
|
|
2323
|
+
}
|
|
2324
|
+
this.#armIdleCompactionCheck();
|
|
2325
|
+
}
|
|
2326
|
+
/**
|
|
2327
|
+
* Debounces the final write-path check for a burst. Sampling every few commits keeps the hot
|
|
2328
|
+
* path cheap, but the last one through seven commits can be the ones that cross a fold
|
|
2329
|
+
* threshold. Without this check an idle table can remain due forever because no later write or
|
|
2330
|
+
* scan arrives to notice it.
|
|
2331
|
+
*/
|
|
2332
|
+
#armIdleCompactionCheck() {
|
|
2333
|
+
if (this.#idleCompactionTableIds.size === 0)
|
|
2334
|
+
return;
|
|
2335
|
+
if (this.#idleCompactionTimer !== undefined)
|
|
2336
|
+
clearTimeout(this.#idleCompactionTimer);
|
|
2337
|
+
const timer = setTimeout(() => {
|
|
2338
|
+
this.#idleCompactionTimer = undefined;
|
|
2339
|
+
const tableIds = [...this.#idleCompactionTableIds];
|
|
2340
|
+
this.#idleCompactionTableIds.clear();
|
|
2341
|
+
for (const tableId of tableIds) {
|
|
2342
|
+
this.#commitsSinceCompactionCheck.delete(tableId);
|
|
2343
|
+
void this.#checkAutoCompaction(tableId);
|
|
2344
|
+
}
|
|
2345
|
+
}, AUTO_COMPACT_IDLE_CHECK_MS);
|
|
2346
|
+
timer.unref?.();
|
|
2347
|
+
this.#idleCompactionTimer = timer;
|
|
2348
|
+
}
|
|
2349
|
+
/**
|
|
2350
|
+
* The write-path auto-compaction check: the table's visible segments at the current manifest,
|
|
2351
|
+
* judged by the same thresholds a streamed scan applies. Without it a write-heavy phase with
|
|
2352
|
+
* no reads in between piles deltas up unfolded, and the next query pays for all of them at
|
|
2353
|
+
* once. Background maintenance never surfaces through a write; a failed check waits for the
|
|
2354
|
+
* next one.
|
|
2355
|
+
*/
|
|
2356
|
+
async #checkAutoCompaction(tableId) {
|
|
2357
|
+
try {
|
|
2358
|
+
const table = await this.store.getTable(tableId);
|
|
2359
|
+
if (table === undefined)
|
|
2360
|
+
return;
|
|
2361
|
+
this.#maybeScheduleAutoCompaction(table, await this.#currentVisibleSegments(table));
|
|
2362
|
+
}
|
|
2363
|
+
catch {
|
|
2364
|
+
// Deliberately silent: the next commit or scan checks again.
|
|
2365
|
+
}
|
|
2366
|
+
}
|
|
1968
2367
|
/**
|
|
1969
2368
|
* Persists one AFTER trigger on its table record (compare-and-swap with retry, like
|
|
1970
2369
|
* migration). Validation is CREATE-time so firing can trust the record: events bind only
|
|
@@ -2464,7 +2863,7 @@ export class MinnowDatabase {
|
|
|
2464
2863
|
for (let attempt = 0; attempt <= this.#maxCommitRetries; attempt += 1) {
|
|
2465
2864
|
try {
|
|
2466
2865
|
const manifest = await transaction.commit();
|
|
2467
|
-
this.#
|
|
2866
|
+
this.#afterCommit(manifest);
|
|
2468
2867
|
return { result, version: manifest.version };
|
|
2469
2868
|
}
|
|
2470
2869
|
catch (error) {
|
|
@@ -2616,7 +3015,6 @@ export class MinnowDatabase {
|
|
|
2616
3015
|
tableId: table.id,
|
|
2617
3016
|
keyTokens: [...keys.keys()],
|
|
2618
3017
|
requireAbsent: kind === "insert",
|
|
2619
|
-
...(table.uniqueKeyStorage === undefined ? {} : { storageMode: table.uniqueKeyStorage }),
|
|
2620
3018
|
});
|
|
2621
3019
|
}
|
|
2622
3020
|
const rowIds = await this.store.reserveRowIds(table.id, rowCount);
|
|
@@ -2765,7 +3163,6 @@ export class MinnowDatabase {
|
|
|
2765
3163
|
keyTokens: [...keys.keys()],
|
|
2766
3164
|
requireAbsent: false,
|
|
2767
3165
|
remove: true,
|
|
2768
|
-
...(table.uniqueKeyStorage === undefined ? {} : { storageMode: table.uniqueKeyStorage }),
|
|
2769
3166
|
});
|
|
2770
3167
|
// Fire only per existing row (session-visible state included): missing keys must not
|
|
2771
3168
|
// produce phantom all-null OLD images.
|
|
@@ -2815,7 +3212,14 @@ export class MinnowDatabase {
|
|
|
2815
3212
|
}
|
|
2816
3213
|
/** Executes a built ORM query through the same streaming-first pipeline as compiled SQL. */
|
|
2817
3214
|
async run(query) {
|
|
2818
|
-
|
|
3215
|
+
const probe = this.store.getCatalogProbe?.bind(this.store);
|
|
3216
|
+
// The same memo a SQL query gets, keyed by the plan: a typed query is compiled once by the
|
|
3217
|
+
// builder and run many times, and it used to re-execute on every run.
|
|
3218
|
+
if (probe === undefined || query.plan.usesStatementDatetime === true) {
|
|
3219
|
+
return (await this.#queryCompiled(query.plan)).rows;
|
|
3220
|
+
}
|
|
3221
|
+
return (await this.#memoizedQuery(query.plan, `typed ${planMemoKey(query.plan)}`, {}, probe))
|
|
3222
|
+
.rows;
|
|
2819
3223
|
}
|
|
2820
3224
|
/**
|
|
2821
3225
|
* Applies a schema definition to the catalog through metadata-only steps: creating missing
|
|
@@ -3554,17 +3958,12 @@ export class MinnowDatabase {
|
|
|
3554
3958
|
having: [],
|
|
3555
3959
|
orderBy: [],
|
|
3556
3960
|
};
|
|
3557
|
-
|
|
3558
|
-
|
|
3559
|
-
|
|
3560
|
-
|
|
3561
|
-
|
|
3562
|
-
|
|
3563
|
-
}));
|
|
3564
|
-
}
|
|
3565
|
-
finally {
|
|
3566
|
-
prepared.close();
|
|
3567
|
-
}
|
|
3961
|
+
// The streaming-first pipeline: a keyed IN list narrows to the blocks that can hold the
|
|
3962
|
+
// keys, where the prepared path materialized the table's columns first.
|
|
3963
|
+
const existing = new Set((await this.#queryCompiled(plan)).rows.map((row) => {
|
|
3964
|
+
const value = row.key ?? null;
|
|
3965
|
+
return value instanceof Date ? value.toISOString() : value;
|
|
3966
|
+
}));
|
|
3568
3967
|
return {
|
|
3569
3968
|
...statement,
|
|
3570
3969
|
rows: statement.rows.filter((row) => {
|
|
@@ -3907,13 +4306,10 @@ export class MinnowDatabase {
|
|
|
3907
4306
|
rows = (await options.writer.queryPlan(plan)).rows;
|
|
3908
4307
|
}
|
|
3909
4308
|
else {
|
|
3910
|
-
|
|
3911
|
-
|
|
3912
|
-
|
|
3913
|
-
|
|
3914
|
-
finally {
|
|
3915
|
-
prepared.close();
|
|
3916
|
-
}
|
|
4309
|
+
// The same streaming-first pipeline a SELECT takes: its zone pruning and ascending-range
|
|
4310
|
+
// narrowing find the rows to touch, where the prepared path materialized the table's
|
|
4311
|
+
// columns first — most of a bulk delete's cost, at 200k rows.
|
|
4312
|
+
rows = (await this.#queryCompiled(plan)).rows;
|
|
3917
4313
|
}
|
|
3918
4314
|
const keys = rows.map((row) => row[keyColumn.name]);
|
|
3919
4315
|
if (keys.some((key) => key === null || key === undefined)) {
|
|
@@ -4011,13 +4407,13 @@ export class MinnowDatabase {
|
|
|
4011
4407
|
* Returns undefined when the base table's visible shape is ineligible (keyed mutation replay),
|
|
4012
4408
|
* so the caller falls back to the materialized path.
|
|
4013
4409
|
*/
|
|
4014
|
-
async #queryStreamed(plan, options, spillPageRows) {
|
|
4410
|
+
async #queryStreamed(plan, options, spillPageRows, probe) {
|
|
4015
4411
|
const tableNames = [plan.base.table, ...plan.joins.map((join) => join.table)];
|
|
4016
4412
|
const uniqueTableNames = [...new Set(tableNames)];
|
|
4017
4413
|
if (options.version === undefined) {
|
|
4018
4414
|
// The common path shares the probe-gated catalog state and the shared reader lease
|
|
4019
4415
|
// with every other statement at the current version.
|
|
4020
|
-
return this.#withSharedCatalogSnapshot(uniqueTableNames, (snapshot, realTables, visibility) => this.#queryStreamedAtSnapshot(plan, options, spillPageRows, snapshot, [...realTables.values()], visibility));
|
|
4416
|
+
return this.#withSharedCatalogSnapshot(uniqueTableNames, (snapshot, realTables, visibility) => this.#queryStreamedAtSnapshot(plan, options, spillPageRows, snapshot, [...realTables.values()], visibility), probe);
|
|
4021
4417
|
}
|
|
4022
4418
|
// Explicit time travel keeps the per-call lease and version-anchored reads.
|
|
4023
4419
|
const tables = await Promise.all(uniqueTableNames.map((name) => this.#findTable(name)));
|
|
@@ -4620,19 +5016,6 @@ export class MinnowDatabase {
|
|
|
4620
5016
|
load,
|
|
4621
5017
|
};
|
|
4622
5018
|
}
|
|
4623
|
-
/**
|
|
4624
|
-
* Builds a streamed view of a keyed table whose visible history contains update and delete
|
|
4625
|
-
* segments (no upserts — those interleave new rows into slot order and keep the materialized
|
|
4626
|
-
* path). Mutation deltas are the small part of such a history, so they replay into resident
|
|
4627
|
-
* state — a dead-row bitmap over the base rows plus per-slot column patches referencing the
|
|
4628
|
-
* resident update vectors — while the base rows stream through the existing block-aligned
|
|
4629
|
-
* inner window. The outer view compacts dead rows and overlays patches per window, producing
|
|
4630
|
-
* exactly the materialized replay's rows in exactly its order.
|
|
4631
|
-
*
|
|
4632
|
-
* The replay tracks only mutation-touched key tokens, so its memory is bounded by the
|
|
4633
|
-
* mutation size, not the table; the duplicate-key corruption guard consequently only fires
|
|
4634
|
-
* for touched keys on this path.
|
|
4635
|
-
*/
|
|
4636
5019
|
/**
|
|
4637
5020
|
* Block header/metadata descriptions, cached and fetched in one round trip: no decompress and
|
|
4638
5021
|
* no payload validation. Descriptions are immutable per block id, so a repeated query pays
|
|
@@ -4664,24 +5047,232 @@ export class MinnowDatabase {
|
|
|
4664
5047
|
}
|
|
4665
5048
|
return descriptions;
|
|
4666
5049
|
}
|
|
5050
|
+
/**
|
|
5051
|
+
* Builds a streamed view of a keyed table whose visible history contains update and delete
|
|
5052
|
+
* segments (no upserts — those interleave new rows into slot order and keep the materialized
|
|
5053
|
+
* path). Mutation deltas are the small part of such a history, so they replay into resident
|
|
5054
|
+
* state — a dead-row bitmap over the base rows plus per-slot column patches referencing the
|
|
5055
|
+
* resident update vectors — while the base rows stream through the block-aligned inner
|
|
5056
|
+
* window. The outer view compacts dead rows and overlays patches per window, producing
|
|
5057
|
+
* exactly the materialized replay's rows in exactly its order.
|
|
5058
|
+
*
|
|
5059
|
+
* The replay is a pure function of the visible segment set, so it is built once per commit
|
|
5060
|
+
* and shared by every query until the next one (`#streamedOverlayState`). The outer loader
|
|
5061
|
+
* serves whole inner windows: one whose rows nothing touched is installed by reference, the
|
|
5062
|
+
* difference between a copy per window and none; one with a dead or patched row is compacted
|
|
5063
|
+
* once, in runs rather than cells.
|
|
5064
|
+
*
|
|
5065
|
+
* The replay tracks only mutation-touched keys, so its memory is bounded by the mutation
|
|
5066
|
+
* size, not the table; the duplicate-key corruption guard consequently only fires for
|
|
5067
|
+
* touched keys on this path.
|
|
5068
|
+
*/
|
|
4667
5069
|
async #createStreamedMutationTable(table, keyColumn, projectedColumns, baseSegments, snapshot, memory, zonePruned = false, storedBlocks) {
|
|
4668
5070
|
const scanSegments = baseSegments.filter((segment) => {
|
|
4669
5071
|
const kind = segment.kind ?? "insert";
|
|
4670
5072
|
return kind === "insert" || kind === "base";
|
|
4671
5073
|
});
|
|
4672
|
-
|
|
4673
|
-
|
|
5074
|
+
const overlay = await this.#streamedOverlayState(table, keyColumn, baseSegments, scanSegments, snapshot, memory, zonePruned);
|
|
5075
|
+
const { baseRows, dead, deadCount, patches, patchedSlots } = overlay;
|
|
5076
|
+
const hasPatches = patchedSlots.length > 0;
|
|
5077
|
+
const outputRows = baseRows - deadCount;
|
|
5078
|
+
const inner = this.#createStreamedTable(table, projectedColumns, scanSegments, snapshot, baseRows, memory, storedBlocks);
|
|
5079
|
+
// Deltas that touch no row this scan reads — every one of them eliminated with its row
|
|
5080
|
+
// group, or aimed at keys this table no longer holds — leave the scan exactly as it was.
|
|
5081
|
+
if (deadCount === 0 && !hasPatches)
|
|
5082
|
+
return inner;
|
|
5083
|
+
const states = projectedColumns.map((column) => ({
|
|
5084
|
+
column,
|
|
5085
|
+
vector: createStreamedColumnVector(column.type, outputRows),
|
|
5086
|
+
reservations: [],
|
|
5087
|
+
}));
|
|
5088
|
+
// Forward-only cursor: cursorOutput live rows exist strictly before base row cursorBase.
|
|
5089
|
+
let cursorOutput = 0;
|
|
5090
|
+
let cursorBase = 0;
|
|
5091
|
+
const load = async (start, length) => {
|
|
5092
|
+
const end = Math.min(start + length, outputRows);
|
|
5093
|
+
// COUNT(*) and friends project nothing: the replay already knows how many rows survive,
|
|
5094
|
+
// so there is no window to build and no reason to walk the base rows to build it.
|
|
5095
|
+
if (states.length === 0)
|
|
5096
|
+
return end;
|
|
5097
|
+
const window = states[0]?.vector.window;
|
|
5098
|
+
if (window !== undefined && start >= window.start && start < window.start + window.length) {
|
|
5099
|
+
return window.start + window.length;
|
|
5100
|
+
}
|
|
5101
|
+
if (window !== undefined && start < window.start) {
|
|
5102
|
+
throw new Error(`Streamed scan moved backward: ${table.name}`);
|
|
5103
|
+
}
|
|
5104
|
+
// The cursor stops at the end of the window it last served, which can sit past a start
|
|
5105
|
+
// that falls before it. Rewinding costs one pass over the dead-row bitmap.
|
|
5106
|
+
if (cursorOutput > start) {
|
|
5107
|
+
cursorOutput = 0;
|
|
5108
|
+
cursorBase = 0;
|
|
5109
|
+
}
|
|
5110
|
+
while (cursorOutput < start && cursorBase < baseRows) {
|
|
5111
|
+
if (!bitmapHasValue(dead, cursorBase))
|
|
5112
|
+
cursorOutput += 1;
|
|
5113
|
+
cursorBase += 1;
|
|
5114
|
+
}
|
|
5115
|
+
// Skip the dead rows in front of the first live one, so a window never starts dead.
|
|
5116
|
+
while (cursorBase < baseRows && bitmapHasValue(dead, cursorBase))
|
|
5117
|
+
cursorBase += 1;
|
|
5118
|
+
if (cursorOutput !== start || cursorBase >= baseRows) {
|
|
5119
|
+
throw new Error(`Column row count mismatch: ${table.name}`);
|
|
5120
|
+
}
|
|
5121
|
+
const baseStart = cursorBase;
|
|
5122
|
+
// The inner loader serves whole blocks; the outer window covers the suffix of the inner
|
|
5123
|
+
// window from baseStart, however long, and the caller clamps to what it asked for.
|
|
5124
|
+
const innerEnd = await inner.load(baseStart, baseRows - baseStart);
|
|
5125
|
+
const baseEnd = typeof innerEnd === "number" ? Math.min(innerEnd, baseRows) : baseRows;
|
|
5126
|
+
if (baseEnd <= baseStart)
|
|
5127
|
+
throw new Error(`Column row count mismatch: ${table.name}`);
|
|
5128
|
+
const deadInWindow = bitmapCountRange(dead, baseStart, baseEnd);
|
|
5129
|
+
const patchedInWindow = hasPatches ? sortedCountRange(patchedSlots, baseStart, baseEnd) : 0;
|
|
5130
|
+
const liveRows = baseEnd - baseStart - deadInWindow;
|
|
5131
|
+
const untouched = deadInWindow === 0 && patchedInWindow === 0;
|
|
5132
|
+
const runs = untouched
|
|
5133
|
+
? undefined
|
|
5134
|
+
: overlayWindowRuns(dead, patchedSlots, baseStart, baseEnd, patchedInWindow);
|
|
5135
|
+
const targets = [];
|
|
5136
|
+
try {
|
|
5137
|
+
for (const state of states) {
|
|
5138
|
+
const innerVector = inner.table.columns.get(state.column.name);
|
|
5139
|
+
const innerWindow = innerVector?.window;
|
|
5140
|
+
if (innerVector === undefined || innerWindow === undefined) {
|
|
5141
|
+
throw new Error(`Streamed column is missing: ${state.column.name}`);
|
|
5142
|
+
}
|
|
5143
|
+
const offset = baseStart - innerWindow.start;
|
|
5144
|
+
if (offset < 0 || offset + (baseEnd - baseStart) > innerWindow.length) {
|
|
5145
|
+
throw new Error(`Column row count mismatch: ${state.column.name}`);
|
|
5146
|
+
}
|
|
5147
|
+
const replacements = [];
|
|
5148
|
+
const fields = runs === undefined
|
|
5149
|
+
? overlayWindowView(innerVector, offset, liveRows, memory, state.column, replacements)
|
|
5150
|
+
: overlayWindowCompacted(innerVector, innerWindow.start, runs, liveRows, hasPatches ? patches : undefined, state.column, memory, replacements);
|
|
5151
|
+
fields.window = { start, length: liveRows };
|
|
5152
|
+
targets.push({ state, fields, replacements });
|
|
5153
|
+
}
|
|
5154
|
+
// Every fallible byte is reserved above; the installs below cannot throw, so a budget
|
|
5155
|
+
// overflow leaves every state's previous window and reservations intact.
|
|
5156
|
+
for (const { state, fields, replacements } of targets) {
|
|
5157
|
+
const mutable = state.vector;
|
|
5158
|
+
mutable.validity = fields.validity;
|
|
5159
|
+
if (fields.values !== undefined)
|
|
5160
|
+
mutable.values = fields.values;
|
|
5161
|
+
if (fields.codes !== undefined) {
|
|
5162
|
+
mutable.codes = fields.codes;
|
|
5163
|
+
mutable.dictionary = fields.dictionary ?? [];
|
|
5164
|
+
}
|
|
5165
|
+
mutable.window = fields.window;
|
|
5166
|
+
for (const previous of state.reservations)
|
|
5167
|
+
previous.release();
|
|
5168
|
+
state.reservations = replacements;
|
|
5169
|
+
}
|
|
5170
|
+
}
|
|
5171
|
+
catch (error) {
|
|
5172
|
+
for (const entry of targets) {
|
|
5173
|
+
for (const replacement of entry.replacements)
|
|
5174
|
+
replacement.release();
|
|
5175
|
+
}
|
|
5176
|
+
throw error;
|
|
5177
|
+
}
|
|
5178
|
+
cursorOutput = start + liveRows;
|
|
5179
|
+
cursorBase = baseEnd;
|
|
5180
|
+
return start + liveRows;
|
|
5181
|
+
};
|
|
5182
|
+
return {
|
|
5183
|
+
table: {
|
|
5184
|
+
name: table.name,
|
|
5185
|
+
rowCount: outputRows,
|
|
5186
|
+
columns: new Map(states.map((state) => [state.column.name, state.vector])),
|
|
5187
|
+
},
|
|
5188
|
+
load,
|
|
5189
|
+
};
|
|
5190
|
+
}
|
|
5191
|
+
/**
|
|
5192
|
+
* The replayed mutation state for one visible segment set: which base rows are dead, and
|
|
5193
|
+
* which columns of which slots an update replaced. Cached under the segment ids in the
|
|
5194
|
+
* artifact LRU, because nothing about it changes between commits — before this, every query
|
|
5195
|
+
* over a table with so much as one deleted row rebuilt it, which made COUNT(*) on such a
|
|
5196
|
+
* table cost twenty times what it costs on a clean one. A cache hit is charged to the
|
|
5197
|
+
* query's memory as a tally, the same bytes a build reserves.
|
|
5198
|
+
*/
|
|
5199
|
+
async #streamedOverlayState(table, keyColumn, baseSegments, scanSegments, snapshot, memory, zonePruned) {
|
|
5200
|
+
// A zone-pruned scan keeps a segment's id with a subset of its blocks, and the slots the
|
|
5201
|
+
// replay addresses are the key blocks' rows in order — so the key blocks, not the segment
|
|
5202
|
+
// ids alone, are what identify the state.
|
|
5203
|
+
const key = [
|
|
5204
|
+
"overlay",
|
|
5205
|
+
table.id,
|
|
5206
|
+
zonePruned ? "pruned" : "full",
|
|
5207
|
+
baseSegments
|
|
5208
|
+
.map((segment) => mutationSegmentKind(segment)
|
|
5209
|
+
? segment.id
|
|
5210
|
+
: `${segment.id}:${(segment.columnBlockIds[keyColumn.id] ?? []).join("+")}`)
|
|
5211
|
+
.join(","),
|
|
5212
|
+
].join(" ");
|
|
5213
|
+
const cached = this.#cacheGet(key);
|
|
5214
|
+
if (cached !== undefined) {
|
|
5215
|
+
memory.tally(cached.bytes, "Streamed mutation replay");
|
|
5216
|
+
return cached;
|
|
5217
|
+
}
|
|
5218
|
+
const state = await this.#buildStreamedOverlayState(table, keyColumn, baseSegments, scanSegments, snapshot, memory, zonePruned);
|
|
5219
|
+
this.#cachePut(key, state, state.bytes);
|
|
5220
|
+
return state;
|
|
5221
|
+
}
|
|
5222
|
+
async #buildStreamedOverlayState(table, keyColumn, baseSegments, scanSegments, snapshot, memory, zonePruned) {
|
|
5223
|
+
// Phase A: each mutation segment's key vector, and every column an update changed —
|
|
5224
|
+
// resident and reserved, bounded by the mutation history's size. All of an update's
|
|
5225
|
+
// columns, not only the ones this query projects: the state outlives the query. The
|
|
5226
|
+
// history's blocks come out of the buffer pool in one round trip, as block vectors: one
|
|
5227
|
+
// await per segment is what made a table with a few hundred deltas pay ten milliseconds
|
|
5228
|
+
// to rebuild this state after every commit.
|
|
5229
|
+
const deltaSegments = baseSegments.filter(mutationSegmentKind);
|
|
5230
|
+
const deltaBlockIds = new Set();
|
|
5231
|
+
for (const segment of deltaSegments) {
|
|
5232
|
+
for (const column of table.columns) {
|
|
5233
|
+
for (const blockId of segment.columnBlockIds[column.id] ?? [])
|
|
5234
|
+
deltaBlockIds.add(blockId);
|
|
5235
|
+
}
|
|
5236
|
+
}
|
|
5237
|
+
const decodedDeltaBlocks = new Map();
|
|
5238
|
+
if (deltaBlockIds.size > 0) {
|
|
5239
|
+
const ids = [...deltaBlockIds];
|
|
5240
|
+
const decoded = await this.#decodedBlocksThroughCache(ids, snapshot);
|
|
5241
|
+
ids.forEach((id, index) => {
|
|
5242
|
+
const block = decoded[index];
|
|
5243
|
+
if (block !== undefined)
|
|
5244
|
+
decodedDeltaBlocks.set(id, block);
|
|
5245
|
+
});
|
|
5246
|
+
}
|
|
5247
|
+
const deltaVector = async (column, segment) => {
|
|
5248
|
+
const blockIds = segment.columnBlockIds[column.id] ?? [];
|
|
5249
|
+
const blockId = blockIds[0];
|
|
5250
|
+
if (blockIds.length !== 1 || blockId === undefined) {
|
|
5251
|
+
// A delta written in more than one block — a bulk update past rowsPerBlock — concatenates.
|
|
5252
|
+
return this.#materializeAppendColumnVector(column, [segment], snapshot, segment.rowCount);
|
|
5253
|
+
}
|
|
5254
|
+
const decoded = decodedDeltaBlocks.get(blockId);
|
|
5255
|
+
if (decoded === undefined)
|
|
5256
|
+
throw new Error(`Visible block is missing: ${blockId}`);
|
|
5257
|
+
if (decoded.column.type !== column.type) {
|
|
5258
|
+
throw new Error(`Column type mismatch: ${column.name}`);
|
|
5259
|
+
}
|
|
5260
|
+
const vector = this.#blockColumnVector(blockId, decoded);
|
|
5261
|
+
if (vector.length !== segment.rowCount) {
|
|
5262
|
+
throw new Error(`Column row count mismatch: ${column.name}`);
|
|
5263
|
+
}
|
|
5264
|
+
return vector;
|
|
5265
|
+
};
|
|
4674
5266
|
const mutationKeyVectors = new Map();
|
|
4675
5267
|
const mutationChangedVectors = new Map();
|
|
4676
5268
|
// Keys as the primitives the vectors already hold. The replay used to build one string
|
|
4677
5269
|
// token per row on both sides, which made a single deleted row cost an allocation and a
|
|
4678
5270
|
// hash of every key in the table.
|
|
4679
5271
|
const touched = new Set();
|
|
4680
|
-
|
|
5272
|
+
let retainedBytes = 0;
|
|
5273
|
+
for (const segment of deltaSegments) {
|
|
4681
5274
|
const kind = segment.kind ?? "insert";
|
|
4682
|
-
|
|
4683
|
-
continue;
|
|
4684
|
-
const keyVector = await this.#materializeAppendColumnVector(keyColumn, [segment], snapshot, segment.rowCount);
|
|
5275
|
+
const keyVector = await deltaVector(keyColumn, segment);
|
|
4685
5276
|
memory.reserve(columnVectorRetainedBytes(keyVector), "Streamed mutation replay");
|
|
4686
5277
|
mutationKeyVectors.set(segment.id, keyVector);
|
|
4687
5278
|
const readMutationKey = requiredColumnVectorKeyReader(keyVector);
|
|
@@ -4690,20 +5281,22 @@ export class MinnowDatabase {
|
|
|
4690
5281
|
}
|
|
4691
5282
|
if (kind === "update") {
|
|
4692
5283
|
const changed = new Map();
|
|
4693
|
-
for (const column of
|
|
5284
|
+
for (const column of table.columns) {
|
|
4694
5285
|
if (column.id === keyColumn.id)
|
|
4695
5286
|
continue;
|
|
4696
5287
|
if ((segment.columnBlockIds[column.id]?.length ?? 0) === 0)
|
|
4697
5288
|
continue;
|
|
4698
|
-
const vector = await
|
|
4699
|
-
|
|
5289
|
+
const vector = await deltaVector(column, segment);
|
|
5290
|
+
const bytes = columnVectorRetainedBytes(vector);
|
|
5291
|
+
memory.reserve(bytes, "Streamed mutation replay");
|
|
5292
|
+
retainedBytes += bytes;
|
|
4700
5293
|
changed.set(column.id, vector);
|
|
4701
5294
|
}
|
|
4702
5295
|
mutationChangedVectors.set(segment.id, changed);
|
|
4703
5296
|
}
|
|
4704
5297
|
}
|
|
4705
5298
|
// Phase B: one bounded pass over the scan segments' key blocks — a single block resident
|
|
4706
|
-
// at a time — recording, per scan segment, the touched
|
|
5299
|
+
// at a time — recording, per scan segment, the touched keys and their absolute slots.
|
|
4707
5300
|
const touchedByScanSegment = new Map();
|
|
4708
5301
|
// A mutation history is small, and a unique key is usually written in order, so most key
|
|
4709
5302
|
// blocks cannot hold any touched key at all. Their zone maps say so from the header alone,
|
|
@@ -4734,26 +5327,10 @@ export class MinnowDatabase {
|
|
|
4734
5327
|
if (decoded.column.type !== keyColumn.type) {
|
|
4735
5328
|
throw new Error(`Column type mismatch: ${keyColumn.name}`);
|
|
4736
5329
|
}
|
|
4737
|
-
|
|
4738
|
-
|
|
4739
|
-
const
|
|
4740
|
-
|
|
4741
|
-
: keyColumn.type === "string"
|
|
4742
|
-
? undefined
|
|
4743
|
-
: new Float64Array(rows);
|
|
4744
|
-
const codes = keyColumn.type === "string" ? new Uint32Array(rows) : undefined;
|
|
4745
|
-
codes?.fill(NULL_STRING_VECTOR_CODE);
|
|
4746
|
-
const builder = new StringDictionaryBuilder();
|
|
4747
|
-
appendPhysicalColumnToVector(decoded.column, 0, validity, values, codes, builder);
|
|
4748
|
-
const blockVector = (codes !== undefined
|
|
4749
|
-
? {
|
|
4750
|
-
kind: "string",
|
|
4751
|
-
length: rows,
|
|
4752
|
-
validity,
|
|
4753
|
-
codes,
|
|
4754
|
-
dictionary: builder.dictionary,
|
|
4755
|
-
}
|
|
4756
|
-
: { kind: keyColumn.type, length: rows, validity, values });
|
|
5330
|
+
// The block's vector form is what a scan of this table reads anyway, so it comes from
|
|
5331
|
+
// (and stays in) the buffer pool rather than being rebuilt for the replay.
|
|
5332
|
+
const blockVector = this.#blockColumnVector(blockId, decoded);
|
|
5333
|
+
const rows = blockVector.length;
|
|
4757
5334
|
const readBlockKey = requiredColumnVectorKeyReader(blockVector);
|
|
4758
5335
|
for (let row = 0; row < rows; row += 1) {
|
|
4759
5336
|
const key = readBlockKey(row);
|
|
@@ -4825,195 +5402,22 @@ export class MinnowDatabase {
|
|
|
4825
5402
|
slotPatches.set(columnId, { vector, row });
|
|
4826
5403
|
}
|
|
4827
5404
|
}
|
|
4828
|
-
|
|
4829
|
-
const
|
|
4830
|
-
|
|
4831
|
-
|
|
4832
|
-
|
|
4833
|
-
// group, or aimed at keys this table no longer holds — leave the scan exactly as it was.
|
|
4834
|
-
// The plain streamed scan hands decoded blocks straight through, so skipping the overlay
|
|
4835
|
-
// here is the difference between a copy per window and none.
|
|
4836
|
-
if (deadCount === 0 && !hasPatches)
|
|
4837
|
-
return inner;
|
|
4838
|
-
const states = projectedColumns.map((column) => ({
|
|
4839
|
-
column,
|
|
4840
|
-
vector: createStreamedColumnVector(column.type, outputRows),
|
|
4841
|
-
reservations: [],
|
|
4842
|
-
}));
|
|
4843
|
-
// Forward-only cursor: cursorOutput live rows exist strictly before base row cursorBase.
|
|
4844
|
-
let cursorOutput = 0;
|
|
4845
|
-
let cursorBase = 0;
|
|
4846
|
-
const load = async (start, length) => {
|
|
4847
|
-
const end = Math.min(start + length, outputRows);
|
|
4848
|
-
// COUNT(*) and friends project nothing: the replay already knows how many rows survive,
|
|
4849
|
-
// so there is no window to build and no reason to walk the base rows to build it.
|
|
4850
|
-
if (states.length === 0)
|
|
4851
|
-
return end;
|
|
4852
|
-
const window = states[0]?.vector.window;
|
|
4853
|
-
if (window !== undefined && start >= window.start && end <= window.start + window.length) {
|
|
4854
|
-
return window.start + window.length;
|
|
4855
|
-
}
|
|
4856
|
-
if (window !== undefined && start < window.start) {
|
|
4857
|
-
throw new Error(`Streamed scan moved backward: ${table.name}`);
|
|
4858
|
-
}
|
|
4859
|
-
// The cursor stops at the end of the window it last built, which can sit past a start
|
|
4860
|
-
// that falls inside that window. Rewinding costs one pass over the dead-row bitmap and
|
|
4861
|
-
// keeps the window aligned with what was asked for, rather than with where the cursor
|
|
4862
|
-
// happened to stop.
|
|
4863
|
-
if (cursorOutput > start) {
|
|
4864
|
-
cursorOutput = 0;
|
|
4865
|
-
cursorBase = 0;
|
|
4866
|
-
}
|
|
4867
|
-
while (cursorOutput < start && cursorBase < baseRows) {
|
|
4868
|
-
if (!bitmapHasValue(dead, cursorBase))
|
|
4869
|
-
cursorOutput += 1;
|
|
4870
|
-
cursorBase += 1;
|
|
4871
|
-
}
|
|
4872
|
-
let scanBase = cursorBase;
|
|
4873
|
-
let scanOutput = cursorOutput;
|
|
4874
|
-
const liveBaseRows = [];
|
|
4875
|
-
while (scanOutput < end && scanBase < baseRows) {
|
|
4876
|
-
if (!bitmapHasValue(dead, scanBase)) {
|
|
4877
|
-
liveBaseRows.push(scanBase);
|
|
4878
|
-
scanOutput += 1;
|
|
4879
|
-
}
|
|
4880
|
-
scanBase += 1;
|
|
4881
|
-
}
|
|
4882
|
-
if (scanOutput < end)
|
|
4883
|
-
throw new Error(`Column row count mismatch: ${table.name}`);
|
|
4884
|
-
const windowRows = end - start;
|
|
4885
|
-
const targets = [];
|
|
4886
|
-
try {
|
|
4887
|
-
for (const state of states) {
|
|
4888
|
-
const innerVector = inner.table.columns.get(state.column.name);
|
|
4889
|
-
if (innerVector === undefined) {
|
|
4890
|
-
throw new Error(`Streamed column is missing: ${state.column.name}`);
|
|
4891
|
-
}
|
|
4892
|
-
const validityBytes = Math.ceil(windowRows / 8);
|
|
4893
|
-
const typedBytes = validityBytes +
|
|
4894
|
-
(state.column.type === "boolean"
|
|
4895
|
-
? windowRows
|
|
4896
|
-
: state.column.type === "string"
|
|
4897
|
-
? windowRows * Uint32Array.BYTES_PER_ELEMENT
|
|
4898
|
-
: windowRows * Float64Array.BYTES_PER_ELEMENT);
|
|
4899
|
-
const replacements = [];
|
|
4900
|
-
replacements.push(memory.reserve(typedBytes, `Streamed window ${state.column.name}`));
|
|
4901
|
-
const validity = new Uint8Array(validityBytes);
|
|
4902
|
-
const values = state.column.type === "boolean"
|
|
4903
|
-
? new Uint8Array(windowRows)
|
|
4904
|
-
: state.column.type === "string"
|
|
4905
|
-
? undefined
|
|
4906
|
-
: new Float64Array(windowRows);
|
|
4907
|
-
const codes = state.column.type === "string" ? new Uint32Array(windowRows) : undefined;
|
|
4908
|
-
codes?.fill(NULL_STRING_VECTOR_CODE);
|
|
4909
|
-
const dictionary = [];
|
|
4910
|
-
const target = (codes !== undefined
|
|
4911
|
-
? { kind: "string", length: windowRows, validity, codes, dictionary }
|
|
4912
|
-
: { kind: state.column.type, length: windowRows, validity, values });
|
|
4913
|
-
targets.push({
|
|
4914
|
-
state,
|
|
4915
|
-
innerVector,
|
|
4916
|
-
validity,
|
|
4917
|
-
...(values === undefined ? {} : { values }),
|
|
4918
|
-
...(codes === undefined ? {} : { codes }),
|
|
4919
|
-
dictionary,
|
|
4920
|
-
dictionaryIndex: new Map(),
|
|
4921
|
-
replacements,
|
|
4922
|
-
target,
|
|
4923
|
-
});
|
|
4924
|
-
}
|
|
4925
|
-
// The inner loader serves whole blocks, so the copy walks live rows one inner window
|
|
4926
|
-
// at a time: patched slots read the resident mutation vectors, everything else reads
|
|
4927
|
-
// the inner window at its own offset.
|
|
4928
|
-
let index = 0;
|
|
4929
|
-
while (index < liveBaseRows.length) {
|
|
4930
|
-
const chunkStart = liveBaseRows[index] ?? 0;
|
|
4931
|
-
const innerEnd = await inner.load(chunkStart, scanBase - chunkStart);
|
|
4932
|
-
const usable = typeof innerEnd === "number" ? Math.min(innerEnd, scanBase) : scanBase;
|
|
4933
|
-
if (usable <= chunkStart) {
|
|
4934
|
-
throw new Error(`Column row count mismatch: ${table.name}`);
|
|
4935
|
-
}
|
|
4936
|
-
let chunkEndIndex = index;
|
|
4937
|
-
while (chunkEndIndex < liveBaseRows.length &&
|
|
4938
|
-
(liveBaseRows[chunkEndIndex] ?? 0) < usable) {
|
|
4939
|
-
chunkEndIndex += 1;
|
|
4940
|
-
}
|
|
4941
|
-
for (const entry of targets) {
|
|
4942
|
-
const innerWindow = entry.innerVector.window;
|
|
4943
|
-
if (innerWindow === undefined) {
|
|
4944
|
-
throw new Error(`Streamed column is missing: ${entry.state.column.name}`);
|
|
4945
|
-
}
|
|
4946
|
-
// Live rows are consecutive except where a delete cut them, so the copy walks runs:
|
|
4947
|
-
// one typed-array slice each, with patched slots taken out individually. Copying
|
|
4948
|
-
// cell by cell here is what made a table with one deleted row scan like a replay.
|
|
4949
|
-
const columnId = entry.state.column.id;
|
|
4950
|
-
const remap = entry.innerVector.kind === "string"
|
|
4951
|
-
? remapDictionary(entry.innerVector.dictionary, entry.dictionary, entry.dictionaryIndex)
|
|
4952
|
-
: undefined;
|
|
4953
|
-
let live = index;
|
|
4954
|
-
while (live < chunkEndIndex) {
|
|
4955
|
-
const baseRow = liveBaseRows[live] ?? 0;
|
|
4956
|
-
const patch = hasPatches ? patches.get(baseRow)?.get(columnId) : undefined;
|
|
4957
|
-
if (patch !== undefined) {
|
|
4958
|
-
copyColumnVectorValue(patch.vector, patch.row, entry.target, live, entry.dictionaryIndex);
|
|
4959
|
-
live += 1;
|
|
4960
|
-
continue;
|
|
4961
|
-
}
|
|
4962
|
-
let runEnd = live + 1;
|
|
4963
|
-
while (runEnd < chunkEndIndex &&
|
|
4964
|
-
(liveBaseRows[runEnd] ?? 0) === baseRow + (runEnd - live) &&
|
|
4965
|
-
(!hasPatches || patches.get(liveBaseRows[runEnd] ?? 0)?.get(columnId) === undefined)) {
|
|
4966
|
-
runEnd += 1;
|
|
4967
|
-
}
|
|
4968
|
-
copyVectorSpan(entry.innerVector, baseRow - innerWindow.start, runEnd - live, entry.target, live, remap);
|
|
4969
|
-
live = runEnd;
|
|
4970
|
-
}
|
|
4971
|
-
}
|
|
4972
|
-
index = chunkEndIndex;
|
|
4973
|
-
}
|
|
4974
|
-
// Reserve every fallible byte first; the installs below cannot throw, so a budget
|
|
4975
|
-
// overflow here leaves every state's previous window and reservations intact.
|
|
4976
|
-
for (const entry of targets) {
|
|
4977
|
-
let dictionaryBytes = 0;
|
|
4978
|
-
for (const value of entry.dictionary)
|
|
4979
|
-
dictionaryBytes += value.length;
|
|
4980
|
-
if (dictionaryBytes > 0) {
|
|
4981
|
-
entry.replacements.push(memory.reserve(dictionaryBytes, `Streamed window ${entry.state.column.name}`));
|
|
4982
|
-
}
|
|
4983
|
-
}
|
|
4984
|
-
for (const entry of targets) {
|
|
4985
|
-
const mutable = entry.state.vector;
|
|
4986
|
-
mutable.validity = entry.validity;
|
|
4987
|
-
if (entry.values !== undefined)
|
|
4988
|
-
mutable.values = entry.values;
|
|
4989
|
-
if (entry.codes !== undefined) {
|
|
4990
|
-
mutable.codes = entry.codes;
|
|
4991
|
-
mutable.dictionary = entry.dictionary;
|
|
4992
|
-
}
|
|
4993
|
-
mutable.window = { start, length: windowRows };
|
|
4994
|
-
for (const previous of entry.state.reservations)
|
|
4995
|
-
previous.release();
|
|
4996
|
-
entry.state.reservations = entry.replacements;
|
|
4997
|
-
}
|
|
4998
|
-
}
|
|
4999
|
-
catch (error) {
|
|
5000
|
-
for (const entry of targets) {
|
|
5001
|
-
for (const replacement of entry.replacements)
|
|
5002
|
-
replacement.release();
|
|
5003
|
-
}
|
|
5004
|
-
throw error;
|
|
5005
|
-
}
|
|
5006
|
-
cursorOutput = end;
|
|
5007
|
-
cursorBase = scanBase;
|
|
5008
|
-
return end;
|
|
5009
|
-
};
|
|
5405
|
+
let patchCells = 0;
|
|
5406
|
+
for (const slotPatches of patches.values())
|
|
5407
|
+
patchCells += slotPatches.size;
|
|
5408
|
+
memory.tally(patches.size * 96 + patchCells * 48, "Streamed mutation replay");
|
|
5409
|
+
const patchedSlots = Uint32Array.from(patches.keys()).sort();
|
|
5010
5410
|
return {
|
|
5011
|
-
|
|
5012
|
-
|
|
5013
|
-
|
|
5014
|
-
|
|
5015
|
-
|
|
5016
|
-
|
|
5411
|
+
baseRows,
|
|
5412
|
+
deadCount,
|
|
5413
|
+
dead,
|
|
5414
|
+
patches,
|
|
5415
|
+
patchedSlots,
|
|
5416
|
+
bytes: dead.byteLength +
|
|
5417
|
+
retainedBytes +
|
|
5418
|
+
patchedSlots.byteLength +
|
|
5419
|
+
patches.size * 96 +
|
|
5420
|
+
patchCells * 48,
|
|
5017
5421
|
};
|
|
5018
5422
|
}
|
|
5019
5423
|
/** Prepares one block's inputs and executes it, returning the caller-owned result. */
|
|
@@ -5174,15 +5578,17 @@ export class MinnowDatabase {
|
|
|
5174
5578
|
/** Plans or advances one restart-safe physical compaction job. */
|
|
5175
5579
|
async compactTableStep(tableName, options = {}) {
|
|
5176
5580
|
const table = await this.#findTable(tableName);
|
|
5177
|
-
|
|
5178
|
-
|
|
5179
|
-
|
|
5180
|
-
|
|
5181
|
-
|
|
5182
|
-
|
|
5183
|
-
|
|
5184
|
-
|
|
5185
|
-
|
|
5581
|
+
return this.#serializedCompactionStep(table.id, async () => {
|
|
5582
|
+
const active = (await this.store.listCompactionJobs(table.id)).find((job) => isActiveCompactionState(job.state));
|
|
5583
|
+
let job = active;
|
|
5584
|
+
if (job === undefined) {
|
|
5585
|
+
const planned = await this.#planCompaction(table, options);
|
|
5586
|
+
if ("compacted" in planned)
|
|
5587
|
+
return compactionSkippedProgress(planned);
|
|
5588
|
+
job = planned;
|
|
5589
|
+
}
|
|
5590
|
+
return this.#runCompactionJob(table, job, positiveWholeNumber(options.maxBlocks ?? options.maxBlocksPerStep ?? 1, "Compaction step block limit"));
|
|
5591
|
+
});
|
|
5186
5592
|
}
|
|
5187
5593
|
/** Continues a persisted compaction job after a cooperative yield or restart. */
|
|
5188
5594
|
async resumeCompactionJob(jobId, options = {}) {
|
|
@@ -5192,7 +5598,26 @@ export class MinnowDatabase {
|
|
|
5192
5598
|
const table = await this.store.getTable(job.tableId);
|
|
5193
5599
|
if (table === undefined)
|
|
5194
5600
|
throw new Error(`Compaction table not found: ${job.tableId}`);
|
|
5195
|
-
return this.#runCompactionJob(table, job, positiveWholeNumber(options.maxBlocks ?? 1, "Compaction step block limit"));
|
|
5601
|
+
return this.#serializedCompactionStep(table.id, () => this.#runCompactionJob(table, job, positiveWholeNumber(options.maxBlocks ?? 1, "Compaction step block limit")));
|
|
5602
|
+
}
|
|
5603
|
+
/**
|
|
5604
|
+
* One compaction step at a time per table within this database: background compaction
|
|
5605
|
+
* drives a job in steps, and a caller stepping the same table explicitly must take turns with
|
|
5606
|
+
* it rather than advance the same job concurrently, which would write its output blocks
|
|
5607
|
+
* twice. Each step loads the job record fresh, so alternating drivers simply continue where
|
|
5608
|
+
* the other left off. Between instances and tabs the job's revision is the guard.
|
|
5609
|
+
*/
|
|
5610
|
+
async #serializedCompactionStep(tableId, step) {
|
|
5611
|
+
const previous = this.#compactionSteps.get(tableId) ?? Promise.resolve();
|
|
5612
|
+
const run = previous.then(step, step);
|
|
5613
|
+
this.#compactionSteps.set(tableId, run);
|
|
5614
|
+
try {
|
|
5615
|
+
return await run;
|
|
5616
|
+
}
|
|
5617
|
+
finally {
|
|
5618
|
+
if (this.#compactionSteps.get(tableId) === run)
|
|
5619
|
+
this.#compactionSteps.delete(tableId);
|
|
5620
|
+
}
|
|
5196
5621
|
}
|
|
5197
5622
|
async listCompactionJobs(tableName) {
|
|
5198
5623
|
if (tableName === undefined)
|
|
@@ -5219,6 +5644,13 @@ export class MinnowDatabase {
|
|
|
5219
5644
|
}
|
|
5220
5645
|
}
|
|
5221
5646
|
}
|
|
5647
|
+
async #cancelTableCompactions(tableId) {
|
|
5648
|
+
for (const job of await this.store.listCompactionJobs(tableId)) {
|
|
5649
|
+
if (job.state === "planned" || job.state === "running" || job.state === "ready") {
|
|
5650
|
+
await this.cancelCompactionJob(job.id);
|
|
5651
|
+
}
|
|
5652
|
+
}
|
|
5653
|
+
}
|
|
5222
5654
|
/** Runs restart-safe lease-aware reclamation to completion in bounded durable steps. */
|
|
5223
5655
|
async collectGarbage(options = {}) {
|
|
5224
5656
|
const maxItems = positiveWholeNumber(options.maxItemsPerStep ?? 64, "Garbage collection items per step");
|
|
@@ -5227,38 +5659,201 @@ export class MinnowDatabase {
|
|
|
5227
5659
|
...(options.maxPlanningItems === undefined
|
|
5228
5660
|
? {}
|
|
5229
5661
|
: { maxPlanningItems: options.maxPlanningItems }),
|
|
5662
|
+
...(options.retainRecentVersions === undefined
|
|
5663
|
+
? {}
|
|
5664
|
+
: { retainRecentVersions: options.retainRecentVersions }),
|
|
5230
5665
|
});
|
|
5231
5666
|
while (progress.result === null) {
|
|
5232
5667
|
progress = await this.resumeGarbageCollectionJob(progress.jobId, { maxItems });
|
|
5233
5668
|
}
|
|
5669
|
+
await this.#pruneFinishedJobRecords();
|
|
5234
5670
|
return progress.result;
|
|
5235
5671
|
}
|
|
5236
5672
|
/** Plans or advances one durable garbage-collection pass. */
|
|
5237
5673
|
async collectGarbageStep(options = {}) {
|
|
5238
|
-
|
|
5239
|
-
|
|
5240
|
-
|
|
5241
|
-
|
|
5674
|
+
return this.#collectGarbageStep(options);
|
|
5675
|
+
}
|
|
5676
|
+
/** `collectGarbageStep`, with the age bound background collection adds to its retention. */
|
|
5677
|
+
async #collectGarbageStep(options, retainedVersionMaxAgeMs = Number.POSITIVE_INFINITY) {
|
|
5678
|
+
return this.#serializedCollectionStep(async () => {
|
|
5679
|
+
const active = (await this.store.listGarbageCollectionJobs()).find((job) => job.state === "planned" || job.state === "running");
|
|
5680
|
+
const job = active ??
|
|
5681
|
+
(await this.#planGarbageCollection(positiveWholeNumber(options.maxPlanningItems ?? 1_024, "Garbage collection planning limit"), nonNegativeWholeNumber(options.retainRecentVersions ?? 0, "Garbage collection retained versions"), retainedVersionMaxAgeMs));
|
|
5682
|
+
return this.#runGarbageCollectionJob(job, positiveWholeNumber(options.maxItems ?? 1, "Garbage collection item limit"));
|
|
5683
|
+
});
|
|
5242
5684
|
}
|
|
5243
5685
|
/** Continues a persisted reclamation pass after a cooperative yield or restart. */
|
|
5244
5686
|
async resumeGarbageCollectionJob(jobId, options = {}) {
|
|
5245
5687
|
const job = await this.store.getGarbageCollectionJob(jobId);
|
|
5246
5688
|
if (job === undefined)
|
|
5247
5689
|
throw new Error(`Garbage collection job not found: ${jobId}`);
|
|
5248
|
-
return this.#runGarbageCollectionJob(job, positiveWholeNumber(options.maxItems ?? 1, "Garbage collection item limit"));
|
|
5690
|
+
return this.#serializedCollectionStep(() => this.#runGarbageCollectionJob(job, positiveWholeNumber(options.maxItems ?? 1, "Garbage collection item limit")));
|
|
5691
|
+
}
|
|
5692
|
+
/**
|
|
5693
|
+
* One garbage-collection step at a time within this database, for the same reason
|
|
5694
|
+
* compaction steps take turns (`#serializedCompactionStep`): background collection drives
|
|
5695
|
+
* a job in steps, and a caller stepping collection explicitly continues the same job rather
|
|
5696
|
+
* than racing it.
|
|
5697
|
+
*/
|
|
5698
|
+
async #serializedCollectionStep(step) {
|
|
5699
|
+
const previous = this.#collectionSteps;
|
|
5700
|
+
const run = previous.then(step, step);
|
|
5701
|
+
this.#collectionSteps = run;
|
|
5702
|
+
try {
|
|
5703
|
+
return await run;
|
|
5704
|
+
}
|
|
5705
|
+
finally {
|
|
5706
|
+
if (this.#collectionSteps === run)
|
|
5707
|
+
this.#collectionSteps = Promise.resolve();
|
|
5708
|
+
}
|
|
5709
|
+
}
|
|
5710
|
+
/**
|
|
5711
|
+
* Background collection: plans one pass and drives it to completion in yielding steps.
|
|
5712
|
+
* Runs after a background fold, whose superseded blocks are what a pass reclaims, and every
|
|
5713
|
+
* AUTO_COLLECT_COMMIT_INTERVAL commits, since every commit writes a manifest that stays on
|
|
5714
|
+
* disk until pruned. Keeps the most recent versions readable. Never surfaces through a
|
|
5715
|
+
* write or a scan; a failed pass backs off for an interval of commits.
|
|
5716
|
+
*/
|
|
5717
|
+
#maybeScheduleAutoCollection() {
|
|
5718
|
+
if (!this.#autoCollect)
|
|
5719
|
+
return;
|
|
5720
|
+
if (this.#autoCollectionInFlight) {
|
|
5721
|
+
// A fold finishing or a quiet minute passing while a run is under way is a reason for
|
|
5722
|
+
// one more run once this one ends — a dropped trigger after the last commit of a burst
|
|
5723
|
+
// would otherwise leave the burst's leftovers until the next one.
|
|
5724
|
+
this.#autoCollectionRequested = true;
|
|
5725
|
+
return;
|
|
5726
|
+
}
|
|
5727
|
+
if (this.#commitsSinceCollection < this.#autoCollectionBackoffUntilCommit)
|
|
5728
|
+
return;
|
|
5729
|
+
this.#autoCollectionInFlight = true;
|
|
5730
|
+
this.#autoCollectionRequested = false;
|
|
5731
|
+
this.#commitsSinceCollection = 0;
|
|
5732
|
+
this.#lastCollectionAt = this.#now().getTime();
|
|
5733
|
+
void this.#runAutoCollection()
|
|
5734
|
+
.then(() => {
|
|
5735
|
+
this.#autoCollectionBackoffUntilCommit = 0;
|
|
5736
|
+
})
|
|
5737
|
+
.catch(() => {
|
|
5738
|
+
this.#autoCollectionBackoffUntilCommit = AUTO_COLLECT_COMMIT_INTERVAL * 2;
|
|
5739
|
+
})
|
|
5740
|
+
.finally(() => {
|
|
5741
|
+
this.#autoCollectionInFlight = false;
|
|
5742
|
+
if (this.#autoCollectionRequested) {
|
|
5743
|
+
this.#autoCollectionRequested = false;
|
|
5744
|
+
void yieldToEventLoop().then(() => {
|
|
5745
|
+
this.#maybeScheduleAutoCollection();
|
|
5746
|
+
});
|
|
5747
|
+
}
|
|
5748
|
+
});
|
|
5749
|
+
}
|
|
5750
|
+
/**
|
|
5751
|
+
* A pass a quiet period after the last commit, for a tab that stops writing: the retained
|
|
5752
|
+
* window's age bound lets that pass reclaim what the last burst superseded, which no commit
|
|
5753
|
+
* would otherwise arrive to trigger. Re-armed by every commit; unreferenced, so it never
|
|
5754
|
+
* keeps a process alive.
|
|
5755
|
+
*/
|
|
5756
|
+
#armIdleCollection() {
|
|
5757
|
+
if (!this.#autoCollect)
|
|
5758
|
+
return;
|
|
5759
|
+
if (this.#idleCollectionTimer !== undefined)
|
|
5760
|
+
clearTimeout(this.#idleCollectionTimer);
|
|
5761
|
+
const timer = setTimeout(() => {
|
|
5762
|
+
this.#idleCollectionTimer = undefined;
|
|
5763
|
+
this.#maybeScheduleAutoCollection();
|
|
5764
|
+
}, AUTO_COLLECT_QUIET_MS);
|
|
5765
|
+
timer.unref?.();
|
|
5766
|
+
this.#idleCollectionTimer = timer;
|
|
5767
|
+
}
|
|
5768
|
+
async #runAutoCollection() {
|
|
5769
|
+
// One pass plans a bounded number of candidates, so a backlog — a burst of commits that
|
|
5770
|
+
// outran the passes between them — takes several. Keep passing while a pass still finds
|
|
5771
|
+
// something, up to a ceiling that keeps a pathological store from pinning the loop.
|
|
5772
|
+
for (let pass = 0; pass < AUTO_COLLECT_MAX_PASSES; pass += 1) {
|
|
5773
|
+
this.#releaseIdleSharedLease();
|
|
5774
|
+
let progress = await this.#collectGarbageStep({
|
|
5775
|
+
maxItems: AUTO_COLLECT_STEP_ITEMS,
|
|
5776
|
+
retainRecentVersions: AUTO_COLLECT_RETAINED_VERSIONS,
|
|
5777
|
+
}, AUTO_COLLECT_RETAINED_VERSION_MS);
|
|
5778
|
+
while (progress.result === null) {
|
|
5779
|
+
await yieldToEventLoop();
|
|
5780
|
+
progress = await this.resumeGarbageCollectionJob(progress.jobId, {
|
|
5781
|
+
maxItems: AUTO_COLLECT_STEP_ITEMS,
|
|
5782
|
+
});
|
|
5783
|
+
}
|
|
5784
|
+
const result = progress.result;
|
|
5785
|
+
if (result.prunedManifestCount === 0 &&
|
|
5786
|
+
result.reclaimedBlockCount === 0 &&
|
|
5787
|
+
result.reclaimedSegmentCount === 0 &&
|
|
5788
|
+
result.reclaimedTransactionCount === 0) {
|
|
5789
|
+
break;
|
|
5790
|
+
}
|
|
5791
|
+
await yieldToEventLoop();
|
|
5792
|
+
}
|
|
5793
|
+
await this.#pruneFinishedJobRecords();
|
|
5794
|
+
}
|
|
5795
|
+
/** Drops finished maintenance records while preserving the state needed for safe L2 retries. */
|
|
5796
|
+
async #pruneFinishedJobRecords() {
|
|
5797
|
+
const newestFirst = (jobs) => jobs.sort((left, right) => right.updatedAt.localeCompare(left.updatedAt));
|
|
5798
|
+
const terminalCompactions = newestFirst((await this.store.listCompactionJobs()).filter((job) => job.state === "published" || job.state === "cancelled" || job.state === "aborted"));
|
|
5799
|
+
const retainedCompactionIds = new Set(terminalCompactions.slice(0, AUTO_COLLECT_RETAINED_JOB_RECORDS).map((job) => job.id));
|
|
5800
|
+
// A retry persists the cumulative bytes written by earlier attempts. Keep only the newest
|
|
5801
|
+
// failure for each still-readable source snapshot; it carries the whole lifetime budget.
|
|
5802
|
+
// Once the source manifest is pruned, that exact retry can never be planned again.
|
|
5803
|
+
const retainedFailureBases = new Set();
|
|
5804
|
+
const manifestReadable = new Map();
|
|
5805
|
+
for (const job of terminalCompactions) {
|
|
5806
|
+
if (job.state !== "cancelled" && job.state !== "aborted")
|
|
5807
|
+
continue;
|
|
5808
|
+
const baseId = job.id.split("/retry/", 1)[0] ?? job.id;
|
|
5809
|
+
if (retainedFailureBases.has(baseId))
|
|
5810
|
+
continue;
|
|
5811
|
+
let readable = manifestReadable.get(job.sourceManifestVersion);
|
|
5812
|
+
if (readable === undefined) {
|
|
5813
|
+
const manifest = await this.store.getManifest(job.sourceManifestVersion);
|
|
5814
|
+
readable = manifest !== undefined && manifest.prunedAt === undefined;
|
|
5815
|
+
manifestReadable.set(job.sourceManifestVersion, readable);
|
|
5816
|
+
}
|
|
5817
|
+
if (!readable)
|
|
5818
|
+
continue;
|
|
5819
|
+
retainedFailureBases.add(baseId);
|
|
5820
|
+
retainedCompactionIds.add(job.id);
|
|
5821
|
+
}
|
|
5822
|
+
for (const job of terminalCompactions) {
|
|
5823
|
+
if (!retainedCompactionIds.has(job.id))
|
|
5824
|
+
await this.store.removeCompactionJob(job.id);
|
|
5825
|
+
}
|
|
5826
|
+
const collections = newestFirst((await this.store.listGarbageCollectionJobs()).filter((job) => job.state === "completed"));
|
|
5827
|
+
for (const job of collections.slice(AUTO_COLLECT_RETAINED_JOB_RECORDS)) {
|
|
5828
|
+
await this.store.removeGarbageCollectionJob(job.id);
|
|
5829
|
+
}
|
|
5249
5830
|
}
|
|
5250
5831
|
async listGarbageCollectionJobs() {
|
|
5251
5832
|
return this.store.listGarbageCollectionJobs();
|
|
5252
5833
|
}
|
|
5253
|
-
async #planGarbageCollection(maxPlanningItems) {
|
|
5834
|
+
async #planGarbageCollection(maxPlanningItems, retainRecentVersions, retainedVersionMaxAgeMs = Number.POSITIVE_INFINITY) {
|
|
5835
|
+
// Crashed readers and query spill owners are metadata roots too. Sweep their expired
|
|
5836
|
+
// records as part of every explicit or background collection so callers never need a
|
|
5837
|
+
// separate maintenance loop to keep either family bounded.
|
|
5838
|
+
await this.#transactions.removeExpiredLeases(this.#now());
|
|
5839
|
+
await this.cleanupQuerySpill();
|
|
5254
5840
|
const current = await this.store.getCurrentManifest();
|
|
5255
5841
|
const currentBlockIds = new Set(current?.blockIds ?? []);
|
|
5842
|
+
// Manifest versions are consecutive, so the retained window is a version floor; a version
|
|
5843
|
+
// inside it is still collected once it is older than the window's age.
|
|
5844
|
+
const retainAbove = (current?.version ?? 0) - retainRecentVersions;
|
|
5845
|
+
const retainAfter = this.#now().getTime() - retainedVersionMaxAgeMs;
|
|
5846
|
+
const retained = (manifest) => manifest.version > retainAbove && Date.parse(manifest.createdAt) > retainAfter;
|
|
5256
5847
|
const candidateManifestVersions = [];
|
|
5257
5848
|
const candidateBlockIds = [];
|
|
5258
5849
|
const candidateSegmentIds = [];
|
|
5850
|
+
const candidateTransactionIds = [];
|
|
5259
5851
|
const candidateBlockIdSet = new Set();
|
|
5260
5852
|
const candidateSegmentIdSet = new Set();
|
|
5261
|
-
const remaining = () => maxPlanningItems -
|
|
5853
|
+
const remaining = () => maxPlanningItems -
|
|
5854
|
+
candidateBlockIds.length -
|
|
5855
|
+
candidateSegmentIds.length -
|
|
5856
|
+
candidateTransactionIds.length;
|
|
5262
5857
|
const addBlocks = (ids) => {
|
|
5263
5858
|
for (const id of ids) {
|
|
5264
5859
|
if (remaining() <= 0)
|
|
@@ -5279,33 +5874,81 @@ export class MinnowDatabase {
|
|
|
5279
5874
|
candidateSegmentIds.push(id);
|
|
5280
5875
|
}
|
|
5281
5876
|
};
|
|
5282
|
-
|
|
5283
|
-
|
|
5877
|
+
// The walk starts past the prefix of history this database has already seen fully
|
|
5878
|
+
// collected — every manifest pruned and none of its blocks left — and extends that prefix
|
|
5879
|
+
// as it goes. A pruned manifest's blocks can only disappear, and a block it shares with a
|
|
5880
|
+
// later manifest is found through that manifest or the job that superseded it, so skipping
|
|
5881
|
+
// the dead prefix loses nothing; it is what keeps a pass proportional to the live history
|
|
5882
|
+
// rather than to everything the database ever committed.
|
|
5883
|
+
let manifestCursor = this.#collectionWatermark;
|
|
5884
|
+
let deadPrefixEnd = this.#collectionWatermark;
|
|
5885
|
+
let prefixContiguous = true;
|
|
5886
|
+
walk: do {
|
|
5284
5887
|
const page = await this.store.listManifestPage(manifestCursor, 64);
|
|
5285
5888
|
for (const manifest of page.records) {
|
|
5286
5889
|
if (manifest.version === current?.version)
|
|
5890
|
+
break walk;
|
|
5891
|
+
if (retained(manifest)) {
|
|
5892
|
+
prefixContiguous = false;
|
|
5287
5893
|
continue;
|
|
5894
|
+
}
|
|
5288
5895
|
const existing = await this.#existingGarbageBlockCandidates(manifest.blockIds, currentBlockIds, remaining());
|
|
5289
|
-
if (manifest.prunedAt === undefined
|
|
5290
|
-
|
|
5291
|
-
|
|
5896
|
+
if (manifest.prunedAt === undefined) {
|
|
5897
|
+
// Only an unpruned manifest is a pruning candidate: one already pruned would spend a
|
|
5898
|
+
// step's capacity confirming it, and with enough of them in front, the unpruned ones
|
|
5899
|
+
// behind them were never reached at all. Their leftover blocks still count.
|
|
5900
|
+
if (candidateManifestVersions.length < 64)
|
|
5901
|
+
candidateManifestVersions.push(manifest.version);
|
|
5902
|
+
prefixContiguous = false;
|
|
5292
5903
|
}
|
|
5293
|
-
if (
|
|
5294
|
-
|
|
5904
|
+
else if (prefixContiguous && existing.length === 0) {
|
|
5905
|
+
deadPrefixEnd = manifest.version;
|
|
5906
|
+
}
|
|
5907
|
+
else {
|
|
5908
|
+
prefixContiguous = false;
|
|
5909
|
+
}
|
|
5910
|
+
addBlocks(existing);
|
|
5911
|
+
if (remaining() <= 0)
|
|
5912
|
+
break walk;
|
|
5295
5913
|
}
|
|
5296
|
-
if (remaining() <= 0 || candidateManifestVersions.length === 64)
|
|
5297
|
-
break;
|
|
5298
5914
|
manifestCursor = page.nextCursor;
|
|
5299
5915
|
} while (manifestCursor !== null);
|
|
5916
|
+
this.#collectionWatermark = deadPrefixEnd;
|
|
5300
5917
|
if (remaining() > 0) {
|
|
5918
|
+
const segmentOwnerIds = new Set((await this.store.listSegments()).map((segment) => segment.transactionId));
|
|
5919
|
+
const manifestEligibility = new Map();
|
|
5920
|
+
const candidateManifestSet = new Set(candidateManifestVersions);
|
|
5301
5921
|
let transactionCursor = null;
|
|
5302
5922
|
do {
|
|
5303
5923
|
const page = await this.store.listTransactionPage(transactionCursor, 64);
|
|
5304
5924
|
for (const transaction of page.records) {
|
|
5305
|
-
if (transaction.status
|
|
5306
|
-
|
|
5307
|
-
|
|
5308
|
-
|
|
5925
|
+
if (transaction.status === "aborted") {
|
|
5926
|
+
const pendingBlocks = await this.#existingGarbageBlockCandidates(transaction.pendingBlockIds, currentBlockIds, remaining());
|
|
5927
|
+
addBlocks(pendingBlocks);
|
|
5928
|
+
const pendingSegments = await this.#existingGarbageSegmentCandidates(transaction.pendingSegmentIds, remaining());
|
|
5929
|
+
addSegments(pendingSegments);
|
|
5930
|
+
// The artifacts are deleted before transaction candidates within a job. A later
|
|
5931
|
+
// pass sees the empty journal and removes the aborted record itself.
|
|
5932
|
+
if (pendingBlocks.length === 0 && pendingSegments.length === 0 && remaining() > 0) {
|
|
5933
|
+
candidateTransactionIds.push(transaction.id);
|
|
5934
|
+
}
|
|
5935
|
+
}
|
|
5936
|
+
else if (transaction.status === "committed" &&
|
|
5937
|
+
transaction.committedVersion !== null &&
|
|
5938
|
+
!segmentOwnerIds.has(transaction.id) &&
|
|
5939
|
+
remaining() > 0) {
|
|
5940
|
+
let eligible = manifestEligibility.get(transaction.committedVersion);
|
|
5941
|
+
if (eligible === undefined) {
|
|
5942
|
+
const manifest = await this.store.getManifest(transaction.committedVersion);
|
|
5943
|
+
eligible =
|
|
5944
|
+
manifest === undefined ||
|
|
5945
|
+
manifest.prunedAt !== undefined ||
|
|
5946
|
+
candidateManifestSet.has(transaction.committedVersion);
|
|
5947
|
+
manifestEligibility.set(transaction.committedVersion, eligible);
|
|
5948
|
+
}
|
|
5949
|
+
if (eligible)
|
|
5950
|
+
candidateTransactionIds.push(transaction.id);
|
|
5951
|
+
}
|
|
5309
5952
|
if (remaining() <= 0)
|
|
5310
5953
|
break;
|
|
5311
5954
|
}
|
|
@@ -5323,10 +5966,7 @@ export class MinnowDatabase {
|
|
|
5323
5966
|
continue;
|
|
5324
5967
|
}
|
|
5325
5968
|
addBlocks(await this.#existingGarbageBlockCandidates([...job.sourceBlockIds, ...job.outputBlockIds], currentBlockIds, remaining()));
|
|
5326
|
-
addSegments(await this.#existingGarbageSegmentCandidates([
|
|
5327
|
-
...job.sourceSegmentIds,
|
|
5328
|
-
...(job.outputSegmentId === null ? [] : [job.outputSegmentId]),
|
|
5329
|
-
], remaining()));
|
|
5969
|
+
addSegments(await this.#existingGarbageSegmentCandidates([...job.sourceSegmentIds, ...compactionOutputSegmentIds(job)], remaining()));
|
|
5330
5970
|
if (remaining() <= 0)
|
|
5331
5971
|
break;
|
|
5332
5972
|
}
|
|
@@ -5341,22 +5981,35 @@ export class MinnowDatabase {
|
|
|
5341
5981
|
candidateManifestVersions,
|
|
5342
5982
|
candidateSegmentIds,
|
|
5343
5983
|
candidateBlockIds,
|
|
5984
|
+
candidateTransactionIds,
|
|
5344
5985
|
leaseCutoff: timestamp,
|
|
5345
5986
|
createdAt: timestamp,
|
|
5346
5987
|
});
|
|
5347
5988
|
}
|
|
5348
5989
|
async #existingGarbageBlockCandidates(ids, currentBlockIds, limit) {
|
|
5349
|
-
|
|
5990
|
+
if (limit <= 0)
|
|
5991
|
+
return [];
|
|
5992
|
+
// A block the current manifest still carries is not garbage, whatever else references it,
|
|
5993
|
+
// and an unpruned manifest shares nearly all of its blocks with the current one. Deciding
|
|
5994
|
+
// that from the set first leaves the store lookups to the few blocks that might be gone —
|
|
5995
|
+
// reading every block of every manifest to find them made a planning pass cost the table
|
|
5996
|
+
// times the history.
|
|
5997
|
+
const possible = [];
|
|
5350
5998
|
const seen = new Set();
|
|
5351
|
-
for (
|
|
5352
|
-
|
|
5999
|
+
for (const id of ids) {
|
|
6000
|
+
if (currentBlockIds.has(id) || seen.has(id))
|
|
6001
|
+
continue;
|
|
6002
|
+
seen.add(id);
|
|
6003
|
+
possible.push(id);
|
|
6004
|
+
}
|
|
6005
|
+
const candidates = [];
|
|
6006
|
+
for (let start = 0; start < possible.length && candidates.length < limit; start += 64) {
|
|
6007
|
+
const page = possible.slice(start, start + 64);
|
|
5353
6008
|
const blocks = await this.store.getBlocks(page);
|
|
5354
6009
|
for (let index = 0; index < page.length && candidates.length < limit; index += 1) {
|
|
5355
6010
|
const id = page[index] ?? "";
|
|
5356
|
-
if (blocks[index] !== undefined
|
|
5357
|
-
seen.add(id);
|
|
6011
|
+
if (blocks[index] !== undefined)
|
|
5358
6012
|
candidates.push(id);
|
|
5359
|
-
}
|
|
5360
6013
|
}
|
|
5361
6014
|
}
|
|
5362
6015
|
return candidates;
|
|
@@ -5377,8 +6030,10 @@ export class MinnowDatabase {
|
|
|
5377
6030
|
async #runGarbageCollectionJob(initialJob, maxItems) {
|
|
5378
6031
|
let job = initialJob;
|
|
5379
6032
|
for (;;) {
|
|
5380
|
-
if (job.state === "completed")
|
|
6033
|
+
if (job.state === "completed") {
|
|
6034
|
+
await this.store.removePrunedManifestRecords();
|
|
5381
6035
|
return garbageCollectionProgress(job);
|
|
6036
|
+
}
|
|
5382
6037
|
try {
|
|
5383
6038
|
const step = await this.store.runGarbageCollectionStep({
|
|
5384
6039
|
jobId: job.id,
|
|
@@ -5386,6 +6041,8 @@ export class MinnowDatabase {
|
|
|
5386
6041
|
maxItems,
|
|
5387
6042
|
updatedAt: this.#now().toISOString(),
|
|
5388
6043
|
});
|
|
6044
|
+
if (step.job.state === "completed")
|
|
6045
|
+
await this.store.removePrunedManifestRecords();
|
|
5389
6046
|
return garbageCollectionProgress(step.job);
|
|
5390
6047
|
}
|
|
5391
6048
|
catch (error) {
|
|
@@ -5440,31 +6097,52 @@ export class MinnowDatabase {
|
|
|
5440
6097
|
const levelTwoMaxWriteAmplification = targetLevel === 2
|
|
5441
6098
|
? positiveFiniteNumber(options.maxWriteAmplification ?? DEFAULT_LEVEL_TWO_MAX_WRITE_AMPLIFICATION, "Compaction maximum write amplification")
|
|
5442
6099
|
: undefined;
|
|
5443
|
-
|
|
6100
|
+
const targetBlockBytes = positiveWholeNumber(options.targetBlockBytes ?? DEFAULT_COMPACTION_TARGET_BLOCK_BYTES, "Compaction target block bytes");
|
|
6101
|
+
if (targetBlockBytes > MAX_COMPACTION_TARGET_BLOCK_BYTES) {
|
|
6102
|
+
throw new RangeError(`Compaction target block bytes cannot exceed ${String(MAX_COMPACTION_TARGET_BLOCK_BYTES)}`);
|
|
6103
|
+
}
|
|
6104
|
+
const outputCompression = validateCompression(options.outputCompression ?? "gzip", "Compaction output compression");
|
|
6105
|
+
if (getCompressionMemoryBound(outputCompression, targetBlockBytes).maximumOutputBytes >
|
|
6106
|
+
MAX_COMPACTION_TARGET_BLOCK_BYTES) {
|
|
6107
|
+
throw new RangeError(`Compaction target block bytes exceed the ${outputCompression} worst-case format limit`);
|
|
6108
|
+
}
|
|
6109
|
+
const memoryBudgetBytes = positiveWholeNumber(options.memoryBudgetBytes ?? DEFAULT_COMPACTION_MEMORY_BUDGET_BYTES, "Compaction memory budget");
|
|
6110
|
+
const partitionRows = positiveWholeNumber(options.partitionRows ?? this.#compactionPartitionRows, "Compaction partition rows");
|
|
6111
|
+
let anchors = [];
|
|
5444
6112
|
let level0Segments;
|
|
5445
6113
|
let effectiveMinimumLevel0Segments;
|
|
5446
6114
|
let outputPartitionOrdinal;
|
|
5447
6115
|
let keyedLevelTwo = false;
|
|
5448
|
-
|
|
5449
|
-
|
|
5450
|
-
|
|
5451
|
-
|
|
5452
|
-
|
|
5453
|
-
|
|
5454
|
-
|
|
6116
|
+
let keyedLevelOne;
|
|
6117
|
+
let keylessLevelOne;
|
|
6118
|
+
if (targetLevel === 1 && table.uniqueKeyColumnId !== undefined) {
|
|
6119
|
+
// Keyed L1: a prefix of level-one partitions, then level-zero history. A fold rewrites
|
|
6120
|
+
// only the partitions the selected deltas touch (and the tail partition new rows join),
|
|
6121
|
+
// so which partitions it sources is decided after the level-zero selection below.
|
|
6122
|
+
const layout = keyedLevelOneLayout(visibleSegments);
|
|
6123
|
+
if (layout === null) {
|
|
6124
|
+
return compactTableSkipped(table.name, "unsupported-level-layout", visibleSegments, visibleBlockIds, version);
|
|
6125
|
+
}
|
|
6126
|
+
keyedLevelOne = layout;
|
|
6127
|
+
level0Segments = layout.level0Segments;
|
|
6128
|
+
effectiveMinimumLevel0Segments =
|
|
6129
|
+
layout.partitions.length > 0 ? minimumLevel0Segments : Math.max(2, minimumLevel0Segments);
|
|
6130
|
+
}
|
|
6131
|
+
else if (targetLevel === 1) {
|
|
6132
|
+
const layout = keylessLevelOneLayout(visibleSegments);
|
|
6133
|
+
if (layout === null) {
|
|
5455
6134
|
return compactTableSkipped(table.name, "unsupported-level-layout", visibleSegments, visibleBlockIds, version);
|
|
5456
6135
|
}
|
|
5457
|
-
|
|
5458
|
-
level0Segments =
|
|
5459
|
-
effectiveMinimumLevel0Segments =
|
|
5460
|
-
? minimumLevel0Segments
|
|
5461
|
-
: Math.max(2, minimumLevel0Segments);
|
|
6136
|
+
keylessLevelOne = layout;
|
|
6137
|
+
level0Segments = layout.level0Segments;
|
|
6138
|
+
effectiveMinimumLevel0Segments =
|
|
6139
|
+
layout.partitions.length > 0 ? minimumLevel0Segments : Math.max(2, minimumLevel0Segments);
|
|
5462
6140
|
}
|
|
5463
6141
|
else if (table.uniqueKeyColumnId !== undefined ||
|
|
5464
6142
|
visibleSegments.some((segment) => (segment.kind ?? "insert") !== "insert" || segment.rowIdSpans !== undefined)) {
|
|
5465
|
-
// Keyed multi-range L2: merge (
|
|
5466
|
-
// span-carrying partition. Published partitions are never rewritten; mutation
|
|
5467
|
-
// without a unique key cannot merge and keep the materialized skip.
|
|
6143
|
+
// Keyed multi-range L2: merge (the level-one partitions + oldest level-zero prefix) into
|
|
6144
|
+
// a new span-carrying partition. Published partitions are never rewritten; mutation
|
|
6145
|
+
// kinds without a unique key cannot merge and keep the materialized skip.
|
|
5468
6146
|
if (table.uniqueKeyColumnId === undefined) {
|
|
5469
6147
|
return compactTableSkipped(table.name, "contains-mutation-segments", visibleSegments, visibleBlockIds, version);
|
|
5470
6148
|
}
|
|
@@ -5472,7 +6150,7 @@ export class MinnowDatabase {
|
|
|
5472
6150
|
if (layout === null) {
|
|
5473
6151
|
return compactTableSkipped(table.name, "unsupported-level-layout", visibleSegments, visibleBlockIds, version);
|
|
5474
6152
|
}
|
|
5475
|
-
|
|
6153
|
+
anchors = layout.anchors;
|
|
5476
6154
|
level0Segments = layout.level0Segments;
|
|
5477
6155
|
effectiveMinimumLevel0Segments = minimumLevel0Segments;
|
|
5478
6156
|
outputPartitionOrdinal = layout.levelTwoSegments.length;
|
|
@@ -5490,38 +6168,70 @@ export class MinnowDatabase {
|
|
|
5490
6168
|
if (level0Segments.length < effectiveMinimumLevel0Segments) {
|
|
5491
6169
|
return compactTableSkipped(table.name, "below-segment-threshold", visibleSegments, visibleBlockIds, version);
|
|
5492
6170
|
}
|
|
5493
|
-
|
|
6171
|
+
if (version === null)
|
|
6172
|
+
throw new Error("Visible compaction segments require a manifest");
|
|
6173
|
+
const level0Selection = await this.#selectLevelZeroSources(level0Segments, effectiveMinimumLevel0Segments, maxLevel0Segments, maxLevel0StoredBytes, snapshot);
|
|
6174
|
+
let partitioning;
|
|
6175
|
+
let rechunkPartitioning;
|
|
6176
|
+
if (keyedLevelOne !== undefined) {
|
|
6177
|
+
const keyColumn = getUniqueKeyColumn(table);
|
|
6178
|
+
if (keyColumn === undefined) {
|
|
6179
|
+
throw new Error(`Mutation compaction requires a unique key: ${table.name}`);
|
|
6180
|
+
}
|
|
6181
|
+
const touched = await this.#touchedPartitionIds(keyColumn, keyedLevelOne.partitions, level0Selection.segments, memoryBudgetBytes, snapshot);
|
|
6182
|
+
// New rows join the last partition while it is small, or when it is being rewritten
|
|
6183
|
+
// anyway; otherwise they open a new partition behind it and it stays untouched.
|
|
6184
|
+
const last = keyedLevelOne.partitions[keyedLevelOne.partitions.length - 1];
|
|
6185
|
+
const bearsNewRows = level0Selection.segments.some((segment) => mergeSourceBearsRows(segment.kind ?? "insert"));
|
|
6186
|
+
const absorbsTail = last !== undefined &&
|
|
6187
|
+
bearsNewRows &&
|
|
6188
|
+
(touched.has(last.id) || last.rowCount < partitionRows);
|
|
6189
|
+
anchors = keyedLevelOne.partitions.filter((partition) => partition.rowCount > partitionRows ||
|
|
6190
|
+
touched.has(partition.id) ||
|
|
6191
|
+
(absorbsTail && partition.id === last.id));
|
|
6192
|
+
partitioning = {
|
|
6193
|
+
partitions: keyedLevelOne.partitions,
|
|
6194
|
+
partitionRows,
|
|
6195
|
+
absorbsTail,
|
|
6196
|
+
nextLevelZeroOrder: level0Selection.nextLogicalOrder ?? version + 1,
|
|
6197
|
+
};
|
|
6198
|
+
}
|
|
6199
|
+
else if (keylessLevelOne !== undefined) {
|
|
6200
|
+
const last = keylessLevelOne.partitions.at(-1);
|
|
6201
|
+
// A partial tail is extended. An oversized legacy anchor is included once so this fold
|
|
6202
|
+
// heals it into bounded partitions; a full tail stays immutable and new rows start after it.
|
|
6203
|
+
const absorbsTail = last !== undefined && (last.rowCount < partitionRows || last.rowCount > partitionRows);
|
|
6204
|
+
anchors = absorbsTail ? [last] : [];
|
|
6205
|
+
rechunkPartitioning = {
|
|
6206
|
+
partitionRows,
|
|
6207
|
+
nextLevelZeroOrder: level0Selection.nextLogicalOrder ?? version + 1,
|
|
6208
|
+
};
|
|
6209
|
+
}
|
|
6210
|
+
const anchorMeasurement = await this.#measureCompactionSources(anchors, level0Selection.blockIds, snapshot);
|
|
6211
|
+
const selection = {
|
|
6212
|
+
sourceSegments: [...anchors, ...level0Selection.segments],
|
|
6213
|
+
level0SourceStoredBytes: level0Selection.storedBytes,
|
|
6214
|
+
anchorSourceStoredBytes: anchorMeasurement.storedBytes,
|
|
6215
|
+
};
|
|
5494
6216
|
const sourceSegments = selection.sourceSegments;
|
|
5495
6217
|
const sourceBlockIds = uniqueSegmentBlockIds(sourceSegments);
|
|
5496
6218
|
const hasContiguousSourceRowIds = hasContiguousRowIds(sourceSegments);
|
|
5497
6219
|
const hasPositiveSourceRowIds = (sourceSegments[0]?.rowIdStart ?? 0n) > 0n;
|
|
5498
|
-
// A keyed
|
|
5499
|
-
//
|
|
5500
|
-
|
|
6220
|
+
// A keyed fold always merges: one uniform partition shape (a full-row base with row-ID
|
|
6221
|
+
// spans, bounded by `partitionRows`) regardless of whether the selected prefix happens to
|
|
6222
|
+
// be pure inserts.
|
|
6223
|
+
const requiresMerge = keyedLevelTwo || keyedLevelOne !== undefined
|
|
5501
6224
|
? true
|
|
5502
6225
|
: targetLevel === 1 &&
|
|
5503
|
-
|
|
5504
|
-
(!hasContiguousSourceRowIds && table.uniqueKeyColumnId !== undefined));
|
|
6226
|
+
sourceSegments.some((segment) => (segment.kind ?? "insert") !== "insert" || segment.rowIdSpans !== undefined);
|
|
5505
6227
|
if (!requiresMerge &&
|
|
5506
6228
|
(!hasContiguousSourceRowIds || (targetLevel === 2 && !hasPositiveSourceRowIds))) {
|
|
5507
6229
|
return compactTableSkipped(table.name, "non-contiguous-row-ids", sourceSegments, sourceBlockIds, version);
|
|
5508
6230
|
}
|
|
5509
|
-
if (version === null)
|
|
5510
|
-
throw new Error("Visible compaction segments require a manifest");
|
|
5511
|
-
const targetBlockBytes = positiveWholeNumber(options.targetBlockBytes ?? DEFAULT_COMPACTION_TARGET_BLOCK_BYTES, "Compaction target block bytes");
|
|
5512
|
-
if (targetBlockBytes > MAX_COMPACTION_TARGET_BLOCK_BYTES) {
|
|
5513
|
-
throw new RangeError(`Compaction target block bytes cannot exceed ${String(MAX_COMPACTION_TARGET_BLOCK_BYTES)}`);
|
|
5514
|
-
}
|
|
5515
|
-
const outputCompression = validateCompression(options.outputCompression ?? "gzip", "Compaction output compression");
|
|
5516
|
-
if (getCompressionMemoryBound(outputCompression, targetBlockBytes).maximumOutputBytes >
|
|
5517
|
-
MAX_COMPACTION_TARGET_BLOCK_BYTES) {
|
|
5518
|
-
throw new RangeError(`Compaction target block bytes exceed the ${outputCompression} worst-case format limit`);
|
|
5519
|
-
}
|
|
5520
|
-
const memoryBudgetBytes = positiveWholeNumber(options.memoryBudgetBytes ?? DEFAULT_COMPACTION_MEMORY_BUDGET_BYTES, "Compaction memory budget");
|
|
5521
6231
|
let mergePlan;
|
|
5522
6232
|
if (requiresMerge) {
|
|
5523
6233
|
try {
|
|
5524
|
-
mergePlan = await this.#createMergeCompactionPlan(table, sourceSegments, targetBlockBytes, outputCompression, memoryBudgetBytes, snapshot);
|
|
6234
|
+
mergePlan = await this.#createMergeCompactionPlan(table, sourceSegments, targetBlockBytes, outputCompression, memoryBudgetBytes, snapshot, partitioning);
|
|
5525
6235
|
}
|
|
5526
6236
|
catch (error) {
|
|
5527
6237
|
// A keyed L2 prefix whose mutations reference keys living in already-published
|
|
@@ -5534,7 +6244,7 @@ export class MinnowDatabase {
|
|
|
5534
6244
|
}
|
|
5535
6245
|
}
|
|
5536
6246
|
const rewritePlan = mergePlan ??
|
|
5537
|
-
(await this.#createRechunkCompactionPlan(table, sourceSegments, targetBlockBytes, outputCompression, memoryBudgetBytes, snapshot));
|
|
6247
|
+
(await this.#createRechunkCompactionPlan(table, sourceSegments, targetBlockBytes, outputCompression, memoryBudgetBytes, snapshot, rechunkPartitioning));
|
|
5538
6248
|
const minimumMemoryBytes = compactionMinimumMemoryBytes(rewritePlan);
|
|
5539
6249
|
if (minimumMemoryBytes > memoryBudgetBytes) {
|
|
5540
6250
|
throw new CompactionMemoryBudgetError(memoryBudgetBytes, minimumMemoryBytes);
|
|
@@ -5560,7 +6270,7 @@ export class MinnowDatabase {
|
|
|
5560
6270
|
const priorAttemptOutputStoredBytes = (await this.store.listCompactionJobs(table.id))
|
|
5561
6271
|
.filter((candidate) => (candidate.state === "cancelled" || candidate.state === "aborted") &&
|
|
5562
6272
|
(candidate.id === baseJobId || candidate.id.startsWith(`${baseJobId}/retry/`)))
|
|
5563
|
-
.reduce((
|
|
6273
|
+
.reduce((largest, candidate) => Math.max(largest, safeWholeNumberSum([candidate.priorAttemptOutputStoredBytes ?? 0, candidate.outputStoredBytes], "Compaction prior-attempt output stored bytes")), 0);
|
|
5564
6274
|
const maximumOutputStoredBytes = Math.max(0, floorWholeNumberProduct(selection.level0SourceStoredBytes, maxWriteAmplification, "Compaction maximum output stored bytes") - priorAttemptOutputStoredBytes);
|
|
5565
6275
|
const plannedOutputStoredBytesUpperBound = await this.#plannedPhysicalOutputStoredBytesUpperBound(rewritePlan, snapshot);
|
|
5566
6276
|
levelTwoBudget = {
|
|
@@ -5646,11 +6356,40 @@ export class MinnowDatabase {
|
|
|
5646
6356
|
throw error;
|
|
5647
6357
|
}
|
|
5648
6358
|
}
|
|
5649
|
-
|
|
5650
|
-
|
|
5651
|
-
|
|
5652
|
-
|
|
5653
|
-
|
|
6359
|
+
/**
|
|
6360
|
+
* Sums the stored bytes of the given segments' blocks, refusing a block that appears twice
|
|
6361
|
+
* among them or in `seenBlockIds` — a source block may only be superseded once.
|
|
6362
|
+
*/
|
|
6363
|
+
async #measureCompactionSources(segments, seenBlockIds, snapshot) {
|
|
6364
|
+
let total = 0;
|
|
6365
|
+
const blockIds = [];
|
|
6366
|
+
const measuredBlockIds = new Set();
|
|
6367
|
+
for (const segment of segments) {
|
|
6368
|
+
for (const blockId of Object.values(segment.columnBlockIds).flat()) {
|
|
6369
|
+
if (seenBlockIds.has(blockId) || measuredBlockIds.has(blockId)) {
|
|
6370
|
+
throw new Error(`Compaction source block is referenced more than once: ${blockId}`);
|
|
6371
|
+
}
|
|
6372
|
+
measuredBlockIds.add(blockId);
|
|
6373
|
+
blockIds.push(blockId);
|
|
6374
|
+
await this.#renewInternalLeaseIfNeeded(snapshot);
|
|
6375
|
+
const bytes = await this.store.getBlock(blockId);
|
|
6376
|
+
if (bytes === undefined)
|
|
6377
|
+
throw new Error(`Compaction source block is missing: ${blockId}`);
|
|
6378
|
+
total = safeWholeNumberSum([total, bytes.byteLength], "Compaction selected stored bytes");
|
|
6379
|
+
}
|
|
6380
|
+
}
|
|
6381
|
+
return { storedBytes: total, blockIds };
|
|
6382
|
+
}
|
|
6383
|
+
/**
|
|
6384
|
+
* The oldest level-zero prefix one job promotes: whole equal-order groups, at least the
|
|
6385
|
+
* minimum, and past it no more than the segment and stored-byte ceilings allow. Also reports
|
|
6386
|
+
* the order of the first segment left behind, which bounds the orders a fold may publish.
|
|
6387
|
+
*/
|
|
6388
|
+
async #selectLevelZeroSources(level0Segments, minimumLevel0Segments, maxLevel0Segments, maxLevel0StoredBytes, snapshot) {
|
|
6389
|
+
const transactions = new Map((await this.#transactionRecordsForSegments(level0Segments)).map((record) => [
|
|
6390
|
+
record.id,
|
|
6391
|
+
record,
|
|
6392
|
+
]));
|
|
5654
6393
|
const logicalOrder = (segment) => {
|
|
5655
6394
|
const owner = transactions.get(segment.transactionId);
|
|
5656
6395
|
if (owner?.status !== "committed" || owner.committedVersion === null) {
|
|
@@ -5659,42 +6398,10 @@ export class MinnowDatabase {
|
|
|
5659
6398
|
return segment.logicalOrder ?? owner.committedVersion;
|
|
5660
6399
|
};
|
|
5661
6400
|
const seenBlockIds = new Set();
|
|
5662
|
-
const
|
|
5663
|
-
|
|
5664
|
-
|
|
5665
|
-
|
|
5666
|
-
let duplicateBlockId = null;
|
|
5667
|
-
for (const segment of segments) {
|
|
5668
|
-
for (const blockId of Object.values(segment.columnBlockIds).flat()) {
|
|
5669
|
-
if (seenBlockIds.has(blockId) || measuredBlockIds.has(blockId)) {
|
|
5670
|
-
duplicateBlockId ??= blockId;
|
|
5671
|
-
}
|
|
5672
|
-
measuredBlockIds.add(blockId);
|
|
5673
|
-
blockIds.push(blockId);
|
|
5674
|
-
await this.#renewInternalLeaseIfNeeded(snapshot);
|
|
5675
|
-
const bytes = await this.store.getBlock(blockId);
|
|
5676
|
-
if (bytes === undefined)
|
|
5677
|
-
throw new Error(`Compaction source block is missing: ${blockId}`);
|
|
5678
|
-
total = safeWholeNumberSum([total, bytes.byteLength], "Compaction selected stored bytes");
|
|
5679
|
-
}
|
|
5680
|
-
}
|
|
5681
|
-
return { storedBytes: total, blockIds, duplicateBlockId };
|
|
5682
|
-
};
|
|
5683
|
-
const acceptMeasurement = (measurement) => {
|
|
5684
|
-
if (measurement.duplicateBlockId !== null) {
|
|
5685
|
-
throw new Error(`Compaction source block is referenced more than once: ${measurement.duplicateBlockId}`);
|
|
5686
|
-
}
|
|
5687
|
-
measurement.blockIds.forEach((blockId) => seenBlockIds.add(blockId));
|
|
5688
|
-
};
|
|
5689
|
-
let anchorSourceStoredBytes = 0;
|
|
5690
|
-
if (anchor !== undefined) {
|
|
5691
|
-
const anchorMeasurement = await measureStoredBytes([anchor]);
|
|
5692
|
-
acceptMeasurement(anchorMeasurement);
|
|
5693
|
-
anchorSourceStoredBytes = anchorMeasurement.storedBytes;
|
|
5694
|
-
}
|
|
5695
|
-
const selectedLevel0 = [];
|
|
5696
|
-
let level0SourceStoredBytes = 0;
|
|
5697
|
-
for (let start = 0; start < level0Segments.length;) {
|
|
6401
|
+
const selected = [];
|
|
6402
|
+
let storedBytes = 0;
|
|
6403
|
+
let start = 0;
|
|
6404
|
+
while (start < level0Segments.length) {
|
|
5698
6405
|
const first = level0Segments[start];
|
|
5699
6406
|
if (first === undefined)
|
|
5700
6407
|
throw new Error("Compaction L0 source selection is unavailable");
|
|
@@ -5707,30 +6414,109 @@ export class MinnowDatabase {
|
|
|
5707
6414
|
end += 1;
|
|
5708
6415
|
}
|
|
5709
6416
|
const group = level0Segments.slice(start, end);
|
|
5710
|
-
if (
|
|
5711
|
-
|
|
6417
|
+
if (selected.length >= minimumLevel0Segments &&
|
|
6418
|
+
selected.length + group.length > maxLevel0Segments) {
|
|
5712
6419
|
break;
|
|
5713
6420
|
}
|
|
5714
|
-
const
|
|
5715
|
-
if (
|
|
5716
|
-
|
|
6421
|
+
const measurement = await this.#measureCompactionSources(group, seenBlockIds, snapshot);
|
|
6422
|
+
if (selected.length >= minimumLevel0Segments &&
|
|
6423
|
+
measurement.storedBytes > maxLevel0StoredBytes - storedBytes) {
|
|
5717
6424
|
break;
|
|
5718
6425
|
}
|
|
5719
|
-
|
|
5720
|
-
|
|
5721
|
-
|
|
6426
|
+
measurement.blockIds.forEach((blockId) => seenBlockIds.add(blockId));
|
|
6427
|
+
selected.push(...group);
|
|
6428
|
+
storedBytes = safeWholeNumberSum([storedBytes, measurement.storedBytes], "Compaction selected L0 stored bytes");
|
|
5722
6429
|
start = end;
|
|
5723
6430
|
}
|
|
5724
|
-
if (
|
|
6431
|
+
if (selected.length < minimumLevel0Segments) {
|
|
5725
6432
|
throw new Error("Compaction source selection did not satisfy its minimum L0 segment count");
|
|
5726
6433
|
}
|
|
6434
|
+
const next = level0Segments[start];
|
|
5727
6435
|
return {
|
|
5728
|
-
|
|
5729
|
-
|
|
5730
|
-
|
|
6436
|
+
segments: selected,
|
|
6437
|
+
storedBytes,
|
|
6438
|
+
blockIds: seenBlockIds,
|
|
6439
|
+
nextLogicalOrder: next === undefined ? null : logicalOrder(next),
|
|
5731
6440
|
};
|
|
5732
6441
|
}
|
|
5733
|
-
|
|
6442
|
+
/**
|
|
6443
|
+
* Which level-one partitions the selected deltas reach into: those holding a key that some
|
|
6444
|
+
* delete, update, or upsert among them names. Inserts and upserts of new keys touch nothing;
|
|
6445
|
+
* their rows join the tail. Key blocks whose zone map rules every referenced key out are
|
|
6446
|
+
* skipped from the header, so a sorted key costs one decoded block per partition at most and
|
|
6447
|
+
* usually none; the referenced-key set is the same size the merge planner's is.
|
|
6448
|
+
*/
|
|
6449
|
+
async #touchedPartitionIds(keyColumn, partitions, level0Segments, memoryBudgetBytes, snapshot) {
|
|
6450
|
+
const touched = new Set();
|
|
6451
|
+
if (partitions.length === 0)
|
|
6452
|
+
return touched;
|
|
6453
|
+
const deltas = level0Segments.filter((segment) => mergeSourceReferencesKeys(segment.kind ?? "insert"));
|
|
6454
|
+
if (deltas.length === 0)
|
|
6455
|
+
return touched;
|
|
6456
|
+
const referencedBytes = safeWholeNumberProduct(deltas.reduce((total, segment) => total + segment.rowCount, 0), MERGE_PLANNER_KEY_BYTES, "Compaction referenced keys");
|
|
6457
|
+
if (referencedBytes > memoryBudgetBytes) {
|
|
6458
|
+
throw new CompactionMemoryBudgetError(memoryBudgetBytes, referencedBytes);
|
|
6459
|
+
}
|
|
6460
|
+
const referenced = new Set();
|
|
6461
|
+
for (const segment of deltas) {
|
|
6462
|
+
await this.#forEachSegmentKey(segment, keyColumn, snapshot, (value) => {
|
|
6463
|
+
referenced.add(overlayKeyOf(keyColumn.type, value));
|
|
6464
|
+
});
|
|
6465
|
+
}
|
|
6466
|
+
const predicate = touchedKeyPredicate(keyColumn, referenced);
|
|
6467
|
+
const descriptions = predicate === undefined
|
|
6468
|
+
? new Map()
|
|
6469
|
+
: await this.#zoneDescriptions(partitions.flatMap((partition) => partition.columnBlockIds[keyColumn.id] ?? []), snapshot);
|
|
6470
|
+
for (const partition of partitions) {
|
|
6471
|
+
const blockIds = partition.columnBlockIds[keyColumn.id] ?? [];
|
|
6472
|
+
if (blockIds.length === 0) {
|
|
6473
|
+
throw new Error(`Partition has no key column blocks: ${partition.id}`);
|
|
6474
|
+
}
|
|
6475
|
+
for (const blockId of blockIds) {
|
|
6476
|
+
const description = descriptions.get(blockId);
|
|
6477
|
+
if (predicate !== undefined &&
|
|
6478
|
+
description !== undefined &&
|
|
6479
|
+
!zoneMapCanMatch(description, predicate)) {
|
|
6480
|
+
continue;
|
|
6481
|
+
}
|
|
6482
|
+
await this.#renewInternalLeaseIfNeeded(snapshot);
|
|
6483
|
+
const bytes = await this.store.getBlock(blockId);
|
|
6484
|
+
if (bytes === undefined)
|
|
6485
|
+
throw new Error(`Compaction source block is missing: ${blockId}`);
|
|
6486
|
+
const decoded = await decodeBlock(bytes);
|
|
6487
|
+
if (decoded.column.type !== keyColumn.type) {
|
|
6488
|
+
throw new Error(`Compaction source block differs from table schema: ${blockId}`);
|
|
6489
|
+
}
|
|
6490
|
+
if (decoded.column.values.some((value) => referenced.has(overlayKeyOf(keyColumn.type, value)))) {
|
|
6491
|
+
touched.add(partition.id);
|
|
6492
|
+
break;
|
|
6493
|
+
}
|
|
6494
|
+
}
|
|
6495
|
+
}
|
|
6496
|
+
return touched;
|
|
6497
|
+
}
|
|
6498
|
+
/** Decodes a segment's key column in row order, one block resident at a time. */
|
|
6499
|
+
async #forEachSegmentKey(segment, keyColumn, snapshot, action) {
|
|
6500
|
+
let rowIndex = 0;
|
|
6501
|
+
for (const blockId of segment.columnBlockIds[keyColumn.id] ?? []) {
|
|
6502
|
+
await this.#renewInternalLeaseIfNeeded(snapshot);
|
|
6503
|
+
const bytes = await this.store.getBlock(blockId);
|
|
6504
|
+
if (bytes === undefined)
|
|
6505
|
+
throw new Error(`Compaction source block is missing: ${blockId}`);
|
|
6506
|
+
const decoded = await decodeBlock(bytes);
|
|
6507
|
+
if (decoded.column.type !== keyColumn.type) {
|
|
6508
|
+
throw new Error(`Compaction source block differs from table schema: ${blockId}`);
|
|
6509
|
+
}
|
|
6510
|
+
for (const value of decoded.column.values) {
|
|
6511
|
+
action(value, rowIndex);
|
|
6512
|
+
rowIndex += 1;
|
|
6513
|
+
}
|
|
6514
|
+
}
|
|
6515
|
+
if (rowIndex !== segment.rowCount) {
|
|
6516
|
+
throw new Error(`Mutation segment key rows differ: ${segment.id}`);
|
|
6517
|
+
}
|
|
6518
|
+
}
|
|
6519
|
+
async #createRechunkCompactionPlan(table, sourceSegments, targetBlockBytes, outputCompression, memoryBudgetBytes, snapshot, partitioning) {
|
|
5734
6520
|
const first = sourceSegments[0];
|
|
5735
6521
|
const last = sourceSegments[sourceSegments.length - 1];
|
|
5736
6522
|
if (first === undefined || last === undefined) {
|
|
@@ -5784,12 +6570,20 @@ export class MinnowDatabase {
|
|
|
5784
6570
|
throw new Error("Compaction could not estimate an output block size");
|
|
5785
6571
|
}
|
|
5786
6572
|
const rowsPerOutput = Math.max(1, Math.min(0xffff_ffff, Math.floor(targetBlockBytes / maximumEncodedBytesPerRow)));
|
|
6573
|
+
const logicalOrder = await this.#firstLogicalOrder(sourceSegments);
|
|
6574
|
+
const partitions = partitioning === undefined
|
|
6575
|
+
? undefined
|
|
6576
|
+
: planLinearOutputPartitions(totalRows, partitioning.partitionRows, logicalOrder, partitioning.nextLevelZeroOrder);
|
|
5787
6577
|
const estimatedOutputs = [];
|
|
5788
|
-
|
|
5789
|
-
|
|
5790
|
-
|
|
5791
|
-
|
|
5792
|
-
|
|
6578
|
+
const outputRegions = partitions ?? [{ rowStart: 0, rowCount: totalRows }];
|
|
6579
|
+
for (const region of outputRegions) {
|
|
6580
|
+
const regionEnd = region.rowStart + region.rowCount;
|
|
6581
|
+
for (let rowStart = region.rowStart; rowStart < regionEnd; rowStart += rowsPerOutput) {
|
|
6582
|
+
estimatedOutputs.push({
|
|
6583
|
+
rowStart,
|
|
6584
|
+
rowCount: Math.min(rowsPerOutput, regionEnd - rowStart),
|
|
6585
|
+
});
|
|
6586
|
+
}
|
|
5793
6587
|
}
|
|
5794
6588
|
const outputs = await this.#refinePhysicalOutputWindows(rechunkPhysicalColumns(columns), estimatedOutputs, targetBlockBytes, outputCompression, memoryBudgetBytes, snapshot);
|
|
5795
6589
|
return {
|
|
@@ -5799,12 +6593,19 @@ export class MinnowDatabase {
|
|
|
5799
6593
|
totalRows,
|
|
5800
6594
|
rowIdStart: first.rowIdStart,
|
|
5801
6595
|
rowIdEndExclusive: last.rowIdEndExclusive,
|
|
5802
|
-
logicalOrder
|
|
6596
|
+
logicalOrder,
|
|
5803
6597
|
columns,
|
|
5804
6598
|
outputs,
|
|
6599
|
+
...(partitions === undefined ? {} : { partitions }),
|
|
5805
6600
|
};
|
|
5806
6601
|
}
|
|
5807
|
-
|
|
6602
|
+
/**
|
|
6603
|
+
* Plans a merge of the sources into one canonical output. With `partitioning`, the output
|
|
6604
|
+
* is also cut into level-one partitions: each rewritten source partition keeps its rows (and
|
|
6605
|
+
* its logical order) in place, new rows form the tail, and every run is chunked to at most
|
|
6606
|
+
* `partitionRows`, using fractional orders between unchanged neighbours.
|
|
6607
|
+
*/
|
|
6608
|
+
async #createMergeCompactionPlan(table, sourceSegments, targetBlockBytes, outputCompression, memoryBudgetBytes, snapshot, partitioning) {
|
|
5808
6609
|
const keyColumn = getUniqueKeyColumn(table);
|
|
5809
6610
|
if (keyColumn === undefined) {
|
|
5810
6611
|
throw new Error(`Mutation compaction requires a unique key: ${table.name}`);
|
|
@@ -5886,6 +6687,9 @@ export class MinnowDatabase {
|
|
|
5886
6687
|
const resolved = await this.#resolveMergeOutput(table, describedSegments, keyColumn, memoryBudgetBytes, snapshot);
|
|
5887
6688
|
const { columns, rowIdSpans, totalRows } = resolved;
|
|
5888
6689
|
const rowIdEnvelope = rowIdSpanEnvelope(rowIdSpans);
|
|
6690
|
+
const partitions = partitioning === undefined
|
|
6691
|
+
? undefined
|
|
6692
|
+
: planOutputPartitions(partitioning, describedSegments, resolved.sourceOutputRowStarts, totalRows);
|
|
5889
6693
|
let outputs = [];
|
|
5890
6694
|
if (totalRows > 0) {
|
|
5891
6695
|
let maximumEncodedBytesPerRow = 0;
|
|
@@ -5902,12 +6706,16 @@ export class MinnowDatabase {
|
|
|
5902
6706
|
throw new Error("Compaction could not estimate an output block size");
|
|
5903
6707
|
}
|
|
5904
6708
|
const rowsPerOutput = Math.max(1, Math.min(0xffff_ffff, Math.floor(targetBlockBytes / maximumEncodedBytesPerRow)));
|
|
6709
|
+
// Windows never straddle a partition: each partition's blocks are its own.
|
|
5905
6710
|
const estimatedOutputs = [];
|
|
5906
|
-
for (
|
|
5907
|
-
|
|
5908
|
-
|
|
5909
|
-
|
|
5910
|
-
|
|
6711
|
+
for (const region of partitions ?? [{ rowStart: 0, rowCount: totalRows }]) {
|
|
6712
|
+
const regionEnd = region.rowStart + region.rowCount;
|
|
6713
|
+
for (let rowStart = region.rowStart; rowStart < regionEnd; rowStart += rowsPerOutput) {
|
|
6714
|
+
estimatedOutputs.push({
|
|
6715
|
+
rowStart,
|
|
6716
|
+
rowCount: Math.min(rowsPerOutput, regionEnd - rowStart),
|
|
6717
|
+
});
|
|
6718
|
+
}
|
|
5911
6719
|
}
|
|
5912
6720
|
outputs = await this.#refinePhysicalOutputWindows(mergePhysicalColumns(columns, describedSegments), estimatedOutputs, targetBlockBytes, outputCompression, memoryBudgetBytes, snapshot);
|
|
5913
6721
|
}
|
|
@@ -5924,97 +6732,165 @@ export class MinnowDatabase {
|
|
|
5924
6732
|
sourceSegments: describedSegments,
|
|
5925
6733
|
columns,
|
|
5926
6734
|
outputs,
|
|
6735
|
+
...(partitions === undefined ? {} : { partitions }),
|
|
5927
6736
|
};
|
|
5928
6737
|
}
|
|
6738
|
+
/**
|
|
6739
|
+
* Replays the source segments' mutations into one canonical output order, in memory that
|
|
6740
|
+
* scales with the deltas rather than with the table.
|
|
6741
|
+
*
|
|
6742
|
+
* Every row of every row-bearing source (base, insert, upsert) gets a slot, numbered in
|
|
6743
|
+
* canonical source order, and the output is the live slots in slot order. A row can only be
|
|
6744
|
+
* referenced later through its key, and only delete, update, and upsert sources reference
|
|
6745
|
+
* keys, so the first pass collects those keys — the touched set — and the replay then tracks
|
|
6746
|
+
* slots for touched keys alone. An untouched row can never be deleted, patched, or replaced:
|
|
6747
|
+
* it passes through as part of a run, one output range per source block rather than one per
|
|
6748
|
+
* row. Memory is O(delta rows + touched rows) plus two bytes per slot.
|
|
6749
|
+
*
|
|
6750
|
+
* The semantics are those of a per-row replay:
|
|
6751
|
+
* - delete: the key's live slot dies; a later insert of the key takes a new slot.
|
|
6752
|
+
* - update: the named columns of the key's live slot come from the update row; the key must
|
|
6753
|
+
* be live.
|
|
6754
|
+
* - upsert: when the key is live, every column of that slot comes from the upsert row and the
|
|
6755
|
+
* upsert row's own slot dies, so the row keeps its position and row ID; otherwise the
|
|
6756
|
+
* upsert row is a new live row.
|
|
6757
|
+
* - insert/base: a new live row; a second live occurrence of a touched key is an error.
|
|
6758
|
+
*/
|
|
5929
6759
|
async #resolveMergeOutput(table, segments, keyColumn, memoryBudgetBytes, snapshot) {
|
|
5930
6760
|
const plannerMemoryBytes = mergePlannerMemoryBound(table, segments, keyColumn.id);
|
|
5931
6761
|
if (plannerMemoryBytes > memoryBudgetBytes) {
|
|
5932
6762
|
throw new CompactionMemoryBudgetError(memoryBudgetBytes, plannerMemoryBytes);
|
|
5933
6763
|
}
|
|
5934
6764
|
const columnIndexById = new Map(table.columns.map((column, index) => [column.id, index]));
|
|
5935
|
-
|
|
5936
|
-
const
|
|
6765
|
+
// Pass 1: the keys any delta references, and each delta's keys in row order.
|
|
6766
|
+
const touched = new Set();
|
|
6767
|
+
const deltaKeys = new Map();
|
|
6768
|
+
for (const segment of segments) {
|
|
6769
|
+
if (!mergeSourceReferencesKeys(segment.kind))
|
|
6770
|
+
continue;
|
|
6771
|
+
const keys = [];
|
|
6772
|
+
await this.#forEachMergeSourceKey(segment, keyColumn, snapshot, (value) => {
|
|
6773
|
+
const key = overlayKeyOf(keyColumn.type, value);
|
|
6774
|
+
keys.push(key);
|
|
6775
|
+
touched.add(key);
|
|
6776
|
+
});
|
|
6777
|
+
deltaKeys.set(segment.segmentId, keys);
|
|
6778
|
+
}
|
|
6779
|
+
// Pass 2: replay into slot state.
|
|
6780
|
+
let slotCount = 0;
|
|
6781
|
+
for (const segment of segments) {
|
|
6782
|
+
if (mergeSourceBearsRows(segment.kind))
|
|
6783
|
+
slotCount += segment.rowCount;
|
|
6784
|
+
}
|
|
6785
|
+
const dead = new Uint8Array(slotCount);
|
|
6786
|
+
const patched = new Uint8Array(slotCount);
|
|
6787
|
+
const patches = new Map();
|
|
6788
|
+
const liveSlotByKey = new Map();
|
|
6789
|
+
let slotBase = 0;
|
|
5937
6790
|
for (const segment of segments) {
|
|
5938
6791
|
if (segment.kind === "delete") {
|
|
5939
|
-
|
|
5940
|
-
const
|
|
5941
|
-
|
|
5942
|
-
|
|
5943
|
-
|
|
5944
|
-
|
|
5945
|
-
|
|
6792
|
+
for (const key of deltaKeys.get(segment.segmentId) ?? []) {
|
|
6793
|
+
const slot = liveSlotByKey.get(key);
|
|
6794
|
+
if (slot === undefined)
|
|
6795
|
+
continue;
|
|
6796
|
+
dead[slot] = 1;
|
|
6797
|
+
patched[slot] = 0;
|
|
6798
|
+
patches.delete(slot);
|
|
6799
|
+
liveSlotByKey.delete(key);
|
|
6800
|
+
}
|
|
5946
6801
|
continue;
|
|
5947
6802
|
}
|
|
5948
6803
|
if (segment.kind === "update") {
|
|
5949
|
-
const
|
|
6804
|
+
const changedColumns = segment.columns
|
|
5950
6805
|
.map((column) => column.columnId)
|
|
5951
|
-
.filter((columnId) => columnId !== keyColumn.id)
|
|
5952
|
-
|
|
5953
|
-
const
|
|
5954
|
-
|
|
5955
|
-
|
|
5956
|
-
|
|
6806
|
+
.filter((columnId) => columnId !== keyColumn.id)
|
|
6807
|
+
.map((columnId) => {
|
|
6808
|
+
const columnIndex = columnIndexById.get(columnId);
|
|
6809
|
+
if (columnIndex === undefined) {
|
|
6810
|
+
throw new Error(`Mutation compaction column is missing: ${columnId}`);
|
|
6811
|
+
}
|
|
6812
|
+
return { columnId, columnIndex };
|
|
6813
|
+
});
|
|
6814
|
+
const keys = deltaKeys.get(segment.segmentId) ?? [];
|
|
6815
|
+
for (let rowIndex = 0; rowIndex < keys.length; rowIndex += 1) {
|
|
6816
|
+
const key = keys[rowIndex];
|
|
6817
|
+
const slot = key === undefined ? undefined : liveSlotByKey.get(key);
|
|
6818
|
+
if (slot === undefined) {
|
|
5957
6819
|
throw new Error(`Update segment references a missing key: ${segment.segmentId}`);
|
|
5958
6820
|
}
|
|
5959
|
-
|
|
5960
|
-
|
|
5961
|
-
|
|
5962
|
-
|
|
5963
|
-
|
|
5964
|
-
existing.sources[columnIndex] = mergeSourceAt(segment, columnId, rowIndex);
|
|
6821
|
+
let patch = patches.get(slot);
|
|
6822
|
+
if (patch === undefined) {
|
|
6823
|
+
patch = new Array(table.columns.length).fill(undefined);
|
|
6824
|
+
patches.set(slot, patch);
|
|
6825
|
+
patched[slot] = 1;
|
|
5965
6826
|
}
|
|
5966
|
-
|
|
6827
|
+
for (const { columnId, columnIndex } of changedColumns) {
|
|
6828
|
+
patch[columnIndex] = mergeSourceAt(segment, columnId, rowIndex);
|
|
6829
|
+
}
|
|
6830
|
+
}
|
|
5967
6831
|
continue;
|
|
5968
6832
|
}
|
|
5969
|
-
|
|
5970
|
-
|
|
5971
|
-
|
|
5972
|
-
|
|
5973
|
-
|
|
5974
|
-
|
|
5975
|
-
|
|
5976
|
-
|
|
5977
|
-
|
|
5978
|
-
rows[existingIndex] = { rowId: existing.rowId, sources };
|
|
6833
|
+
// A row-bearing source: base, insert, or upsert.
|
|
6834
|
+
const base = slotBase;
|
|
6835
|
+
const visit = (key, rowIndex) => {
|
|
6836
|
+
if (!touched.has(key))
|
|
6837
|
+
return;
|
|
6838
|
+
const slot = base + rowIndex;
|
|
6839
|
+
const existing = liveSlotByKey.get(key);
|
|
6840
|
+
if (existing === undefined) {
|
|
6841
|
+
liveSlotByKey.set(key, slot);
|
|
5979
6842
|
return;
|
|
5980
6843
|
}
|
|
5981
|
-
if (
|
|
6844
|
+
if (segment.kind !== "upsert") {
|
|
5982
6845
|
throw new Error(`Insert segment contains a duplicate unique key: ${segment.segmentId}`);
|
|
5983
6846
|
}
|
|
5984
|
-
|
|
5985
|
-
|
|
5986
|
-
|
|
6847
|
+
patches.set(existing, table.columns.map((column) => mergeSourceAt(segment, column.id, rowIndex)));
|
|
6848
|
+
patched[existing] = 1;
|
|
6849
|
+
dead[slot] = 1;
|
|
6850
|
+
};
|
|
6851
|
+
const keys = deltaKeys.get(segment.segmentId);
|
|
6852
|
+
if (keys !== undefined) {
|
|
6853
|
+
keys.forEach(visit);
|
|
6854
|
+
}
|
|
6855
|
+
else if (touched.size > 0) {
|
|
6856
|
+
// With nothing referencing keys there is nothing to track: every row passes through.
|
|
6857
|
+
await this.#forEachMergeSourceKey(segment, keyColumn, snapshot, (value, rowIndex) => {
|
|
6858
|
+
visit(overlayKeyOf(keyColumn.type, value), rowIndex);
|
|
6859
|
+
});
|
|
6860
|
+
}
|
|
6861
|
+
slotBase += segment.rowCount;
|
|
5987
6862
|
}
|
|
5988
|
-
|
|
5989
|
-
|
|
5990
|
-
|
|
5991
|
-
|
|
5992
|
-
|
|
6863
|
+
touched.clear();
|
|
6864
|
+
liveSlotByKey.clear();
|
|
6865
|
+
deltaKeys.clear();
|
|
6866
|
+
// Pass 3: the live slots in slot order, as runs wherever nothing touched them.
|
|
6867
|
+
const output = new MergeOutputBuilder(table.columns);
|
|
6868
|
+
const sourceOutputRowStarts = new Map();
|
|
6869
|
+
slotBase = 0;
|
|
6870
|
+
for (const segment of segments) {
|
|
6871
|
+
if (!mergeSourceBearsRows(segment.kind))
|
|
5993
6872
|
continue;
|
|
5994
|
-
|
|
5995
|
-
|
|
5996
|
-
|
|
5997
|
-
const
|
|
5998
|
-
if (
|
|
5999
|
-
|
|
6000
|
-
|
|
6001
|
-
|
|
6002
|
-
|
|
6003
|
-
|
|
6004
|
-
|
|
6873
|
+
sourceOutputRowStarts.set(segment.segmentId, output.totalRows);
|
|
6874
|
+
let runStart = -1;
|
|
6875
|
+
for (let rowIndex = 0; rowIndex < segment.rowCount; rowIndex += 1) {
|
|
6876
|
+
const slot = slotBase + rowIndex;
|
|
6877
|
+
if (dead[slot] === 1 || patched[slot] === 1) {
|
|
6878
|
+
if (runStart >= 0) {
|
|
6879
|
+
output.appendRun(segment, runStart, rowIndex - runStart);
|
|
6880
|
+
runStart = -1;
|
|
6881
|
+
}
|
|
6882
|
+
if (patched[slot] === 1)
|
|
6883
|
+
output.appendPatchedRow(segment, rowIndex, patches.get(slot));
|
|
6884
|
+
continue;
|
|
6885
|
+
}
|
|
6886
|
+
if (runStart < 0)
|
|
6887
|
+
runStart = rowIndex;
|
|
6005
6888
|
}
|
|
6006
|
-
|
|
6889
|
+
if (runStart >= 0)
|
|
6890
|
+
output.appendRun(segment, runStart, segment.rowCount - runStart);
|
|
6891
|
+
slotBase += segment.rowCount;
|
|
6007
6892
|
}
|
|
6008
|
-
return {
|
|
6009
|
-
rowIdSpans,
|
|
6010
|
-
columns: table.columns.map((column, columnIndex) => {
|
|
6011
|
-
const sourceRanges = sourceRangesByColumn[columnIndex];
|
|
6012
|
-
if (sourceRanges === undefined)
|
|
6013
|
-
throw new Error("Mutation output column is missing");
|
|
6014
|
-
return { columnId: column.id, type: column.type, sourceRanges };
|
|
6015
|
-
}),
|
|
6016
|
-
totalRows,
|
|
6017
|
-
};
|
|
6893
|
+
return { ...output.finish(), sourceOutputRowStarts };
|
|
6018
6894
|
}
|
|
6019
6895
|
async #forEachMergeSourceKey(segment, column, snapshot, action) {
|
|
6020
6896
|
const planned = segment.columns.find((candidate) => candidate.columnId === column.id);
|
|
@@ -6264,56 +7140,32 @@ export class MinnowDatabase {
|
|
|
6264
7140
|
else {
|
|
6265
7141
|
if (outputSegmentId === null)
|
|
6266
7142
|
throw new Error("Compaction output segment ID is missing");
|
|
6267
|
-
const
|
|
6268
|
-
|
|
6269
|
-
|
|
6270
|
-
|
|
6271
|
-
|
|
6272
|
-
|
|
6273
|
-
|
|
6274
|
-
|
|
6275
|
-
|
|
6276
|
-
|
|
6277
|
-
|
|
6278
|
-
|
|
6279
|
-
|
|
6280
|
-
|
|
6281
|
-
|
|
6282
|
-
|
|
6283
|
-
|
|
6284
|
-
|
|
6285
|
-
|
|
6286
|
-
|
|
6287
|
-
|
|
6288
|
-
|
|
6289
|
-
|
|
6290
|
-
|
|
6291
|
-
|
|
6292
|
-
|
|
6293
|
-
createdAt: this.#now().toISOString(),
|
|
6294
|
-
};
|
|
6295
|
-
const outputSegment = await this.store.getSegment(outputSegmentId);
|
|
6296
|
-
if (outputSegment === undefined) {
|
|
6297
|
-
await transaction.stageSegment(desiredOutputSegment);
|
|
6298
|
-
}
|
|
6299
|
-
else if (outputSegment.transactionId === transaction.id) {
|
|
6300
|
-
if (!sameCompactionSegment(outputSegment, desiredOutputSegment)) {
|
|
6301
|
-
throw new Error(`A resumed compaction segment differs: ${outputSegmentId}`);
|
|
6302
|
-
}
|
|
6303
|
-
await transaction.stageExistingSegment(outputSegmentId);
|
|
6304
|
-
}
|
|
6305
|
-
else {
|
|
6306
|
-
const owner = await this.store.getTransaction(outputSegment.transactionId);
|
|
6307
|
-
if ((owner !== undefined && owner.status !== "aborted") ||
|
|
6308
|
-
!sameCompactionSegment(outputSegment, desiredOutputSegment)) {
|
|
6309
|
-
throw new Error(`Compaction output segment cannot be adopted: ${outputSegmentId}`);
|
|
6310
|
-
}
|
|
6311
|
-
const visible = await this.#unprunedManifestContainsAll(expectedOutputIds);
|
|
6312
|
-
if (visible) {
|
|
6313
|
-
throw new Error(`Compaction output segment is already visible: ${outputSegmentId}`);
|
|
6314
|
-
}
|
|
6315
|
-
await this.store.removeSegment(outputSegmentId);
|
|
6316
|
-
await transaction.stageSegment(desiredOutputSegment);
|
|
7143
|
+
const createdAt = this.#now().toISOString();
|
|
7144
|
+
const desiredOutputSegments = rewritePlan.kind === "copy-v1"
|
|
7145
|
+
? [
|
|
7146
|
+
{
|
|
7147
|
+
id: outputSegmentId,
|
|
7148
|
+
tableId: table.id,
|
|
7149
|
+
transactionId: transaction.id,
|
|
7150
|
+
rowCount: outputRowCount,
|
|
7151
|
+
rowIdStart: first?.rowIdStart ?? 0n,
|
|
7152
|
+
rowIdEndExclusive: last?.rowIdEndExclusive ?? 0n,
|
|
7153
|
+
columnBlockIds: compactionOutputColumns(table, sourceSegments, job.id),
|
|
7154
|
+
kind: "insert",
|
|
7155
|
+
...(table.uniqueKeyColumnId === undefined
|
|
7156
|
+
? {}
|
|
7157
|
+
: { keyColumnId: table.uniqueKeyColumnId }),
|
|
7158
|
+
level: job.targetLevel,
|
|
7159
|
+
...(job.outputPartitionOrdinal === undefined
|
|
7160
|
+
? {}
|
|
7161
|
+
: { partitionOrdinal: job.outputPartitionOrdinal }),
|
|
7162
|
+
logicalOrder: await this.#firstLogicalOrder(sourceSegments),
|
|
7163
|
+
createdAt,
|
|
7164
|
+
},
|
|
7165
|
+
]
|
|
7166
|
+
: compactionOutputSegments(table, job, rewritePlan, transaction.id, createdAt);
|
|
7167
|
+
for (const desiredOutputSegment of desiredOutputSegments) {
|
|
7168
|
+
await this.#stageCompactionOutputSegment(transaction, desiredOutputSegment, expectedOutputIds);
|
|
6317
7169
|
}
|
|
6318
7170
|
}
|
|
6319
7171
|
if (job.state !== "ready") {
|
|
@@ -6327,19 +7179,30 @@ export class MinnowDatabase {
|
|
|
6327
7179
|
transaction.supersedeBlocks(job.sourceBlockIds);
|
|
6328
7180
|
transaction.markLogicallyUnchanged();
|
|
6329
7181
|
let manifest;
|
|
6330
|
-
|
|
6331
|
-
|
|
6332
|
-
|
|
6333
|
-
|
|
6334
|
-
|
|
6335
|
-
|
|
7182
|
+
for (;;) {
|
|
7183
|
+
let publicationConflict;
|
|
7184
|
+
try {
|
|
7185
|
+
manifest = await transaction.commit();
|
|
7186
|
+
break;
|
|
7187
|
+
}
|
|
7188
|
+
catch (error) {
|
|
7189
|
+
if (!(error instanceof WriteConflictError))
|
|
7190
|
+
throw error;
|
|
7191
|
+
publicationConflict = error;
|
|
7192
|
+
}
|
|
7193
|
+
// Publication is logically neutral, so it may follow any number of concurrent data
|
|
7194
|
+
// commits while every source remains visible and in the same logical position. A single
|
|
7195
|
+
// retry is not sufficient: another tab (or this database's write queue) can win the
|
|
7196
|
+
// manifest CAS again between rebase and commit, leaving an otherwise complete job stuck
|
|
7197
|
+
// in `ready` after the last write. Keep rebasing until publication wins or a source
|
|
7198
|
+
// genuinely changes.
|
|
6336
7199
|
const current = await this.store.getCurrentManifest();
|
|
6337
7200
|
const currentIds = new Set(current?.blockIds ?? []);
|
|
6338
7201
|
if (job.sourceBlockIds.some((id) => !currentIds.has(id))) {
|
|
6339
7202
|
if (transaction.status === "active")
|
|
6340
7203
|
await transaction.abort();
|
|
6341
7204
|
job = await this.#abortCompactionJob(job, "Compaction sources changed before publication");
|
|
6342
|
-
throw new Error(job.error, { cause:
|
|
7205
|
+
throw new Error(job.error, { cause: publicationConflict });
|
|
6343
7206
|
}
|
|
6344
7207
|
const rebased = await transaction.rebase();
|
|
6345
7208
|
try {
|
|
@@ -6356,11 +7219,11 @@ export class MinnowDatabase {
|
|
|
6356
7219
|
if (transaction.status === "active")
|
|
6357
7220
|
await transaction.abort();
|
|
6358
7221
|
job = await this.#abortCompactionJob(job, `Compaction source is no longer visible: ${missingSourceId}`);
|
|
6359
|
-
throw new Error(job.error, { cause:
|
|
7222
|
+
throw new Error(job.error, { cause: publicationConflict });
|
|
6360
7223
|
}
|
|
6361
7224
|
transaction.supersedeBlocks(job.sourceBlockIds);
|
|
6362
7225
|
transaction.markLogicallyUnchanged();
|
|
6363
|
-
|
|
7226
|
+
await yieldToEventLoop();
|
|
6364
7227
|
}
|
|
6365
7228
|
job = await this.#markCompactionPublished(job, manifest.version);
|
|
6366
7229
|
return compactionProgress(table.name, job, this.#compactionResult(table, job, manifest.version));
|
|
@@ -6423,12 +7286,12 @@ export class MinnowDatabase {
|
|
|
6423
7286
|
const existing = await this.store.getBlock(outputBlockId);
|
|
6424
7287
|
let outputBytes;
|
|
6425
7288
|
if (existing === undefined) {
|
|
6426
|
-
outputBytes = await
|
|
7289
|
+
outputBytes = await this.#encodePreferredBlock(column.columnId, plan.outputCompression, built.physical.bytes.byteLength < GZIP_MINIMUM_INPUT_BYTES, (compression) => encodePhysicalBlock(built.physical, compression));
|
|
6427
7290
|
}
|
|
6428
7291
|
else {
|
|
6429
7292
|
const decoded = await decodePhysicalBlock(existing);
|
|
6430
7293
|
if (decoded.description.type !== column.type ||
|
|
6431
|
-
decoded.description.compression !==
|
|
7294
|
+
(plan.outputCompression === "raw" && decoded.description.compression !== "raw") ||
|
|
6432
7295
|
decoded.description.rowCount !== output.rowCount ||
|
|
6433
7296
|
!sameBytes(decoded.column.bytes, built.physical.bytes)) {
|
|
6434
7297
|
throw new Error(`A resumed compaction block differs: ${outputBlockId}`);
|
|
@@ -6542,6 +7405,16 @@ export class MinnowDatabase {
|
|
|
6542
7405
|
throw new Error(`Concurrent segment shares a compaction source block: ${segment.id}`);
|
|
6543
7406
|
}
|
|
6544
7407
|
}
|
|
7408
|
+
if (job.outputPartitionOrdinal === undefined &&
|
|
7409
|
+
job.targetLevel === 1 &&
|
|
7410
|
+
plan.kind !== "copy-v1" &&
|
|
7411
|
+
plan.partitions !== undefined) {
|
|
7412
|
+
const table = await this.store.getTable(job.tableId);
|
|
7413
|
+
if (table === undefined)
|
|
7414
|
+
throw new Error(`Compaction table is missing: ${job.tableId}`);
|
|
7415
|
+
await this.#assertPartitionedLevelOneSnapshotOrder(job, plan, table, visibleSegments, transactions);
|
|
7416
|
+
return;
|
|
7417
|
+
}
|
|
6545
7418
|
if (job.outputPartitionOrdinal !== undefined) {
|
|
6546
7419
|
if (plan.kind === "rechunk-v1") {
|
|
6547
7420
|
await this.#assertLevelTwoSnapshotOrder(job, plan, snapshot, transactions);
|
|
@@ -6559,8 +7432,11 @@ export class MinnowDatabase {
|
|
|
6559
7432
|
const visibleById = new Map(visibleSegments.map((segment) => [segment.id, segment]));
|
|
6560
7433
|
for (const planned of plan.sourceSegments) {
|
|
6561
7434
|
const actual = visibleById.get(planned.segmentId);
|
|
6562
|
-
|
|
6563
|
-
|
|
7435
|
+
if (actual === undefined) {
|
|
7436
|
+
throw new Error(`Compaction source is no longer visible: ${planned.segmentId}`);
|
|
7437
|
+
}
|
|
7438
|
+
const owner = transactions.get(actual.transactionId);
|
|
7439
|
+
if (!sameMergeSourceSegment(actual, owner, planned)) {
|
|
6564
7440
|
throw new Error(`Compaction source segment differs from its plan: ${planned.segmentId}`);
|
|
6565
7441
|
}
|
|
6566
7442
|
}
|
|
@@ -6623,6 +7499,93 @@ export class MinnowDatabase {
|
|
|
6623
7499
|
}
|
|
6624
7500
|
}
|
|
6625
7501
|
}
|
|
7502
|
+
/**
|
|
7503
|
+
* The partitioned level-one rebase rule, shared by keyed merges and keyless rechunks. The
|
|
7504
|
+
* sources must be exactly as planned. Every partition the plan left alone must still be visible
|
|
7505
|
+
* and unchanged — they are read back from the planning snapshot's manifest, which the job
|
|
7506
|
+
* roots until it ends, so the check needs no record of its own. Every other visible segment
|
|
7507
|
+
* must be level-zero history committed after the latest source and ordered after every
|
|
7508
|
+
* partition the job publishes, so the output slots into the same place relative to the deltas
|
|
7509
|
+
* it did not absorb.
|
|
7510
|
+
*/
|
|
7511
|
+
async #assertPartitionedLevelOneSnapshotOrder(job, plan, table, visibleSegments, transactions) {
|
|
7512
|
+
const sourceIds = new Set(job.sourceSegmentIds);
|
|
7513
|
+
const visibleById = new Map(visibleSegments.map((segment) => [segment.id, segment]));
|
|
7514
|
+
const sourceManifest = await this.store.getManifest(job.sourceManifestVersion);
|
|
7515
|
+
if (sourceManifest === undefined || sourceManifest.prunedAt !== undefined) {
|
|
7516
|
+
throw new Error(`Compaction source manifest is unavailable: ${String(job.sourceManifestVersion)}`);
|
|
7517
|
+
}
|
|
7518
|
+
const plannedVisible = await this.#visibleSegmentRecords(table, new Snapshot(this.store, sourceManifest.version, sourceManifest.blockIds));
|
|
7519
|
+
const plannedById = new Map(plannedVisible.map((segment) => [segment.id, segment]));
|
|
7520
|
+
const plannedLayout = table.uniqueKeyColumnId === undefined
|
|
7521
|
+
? keylessLevelOneLayout(plannedVisible)
|
|
7522
|
+
: keyedLevelOneLayout(plannedVisible);
|
|
7523
|
+
if (plannedLayout === null)
|
|
7524
|
+
throw new Error("Compaction planned layout is no longer valid");
|
|
7525
|
+
let latestSource = null;
|
|
7526
|
+
if (plan.kind === "merge-v1") {
|
|
7527
|
+
for (const planned of plan.sourceSegments) {
|
|
7528
|
+
const actual = visibleById.get(planned.segmentId);
|
|
7529
|
+
if (actual === undefined) {
|
|
7530
|
+
throw new Error(`Compaction source is no longer visible: ${planned.segmentId}`);
|
|
7531
|
+
}
|
|
7532
|
+
const owner = transactions.get(actual.transactionId);
|
|
7533
|
+
if (!sameMergeSourceSegment(actual, owner, planned)) {
|
|
7534
|
+
throw new Error(`Compaction source segment differs from its plan: ${planned.segmentId}`);
|
|
7535
|
+
}
|
|
7536
|
+
}
|
|
7537
|
+
latestSource = plan.sourceSegments[plan.sourceSegments.length - 1] ?? null;
|
|
7538
|
+
}
|
|
7539
|
+
else {
|
|
7540
|
+
for (const id of job.sourceSegmentIds) {
|
|
7541
|
+
const actual = visibleById.get(id);
|
|
7542
|
+
const planned = plannedById.get(id);
|
|
7543
|
+
if (actual === undefined)
|
|
7544
|
+
throw new Error(`Compaction source is no longer visible: ${id}`);
|
|
7545
|
+
if (actual.transactionId !== planned?.transactionId ||
|
|
7546
|
+
!sameCompactionSegment(actual, planned)) {
|
|
7547
|
+
throw new Error(`Compaction source segment differs from its plan: ${id}`);
|
|
7548
|
+
}
|
|
7549
|
+
const tuple = sourceOrderTuple(actual, transactions, "Compaction source");
|
|
7550
|
+
if (latestSource === null || compareMergeSourceOrder(latestSource, tuple) < 0) {
|
|
7551
|
+
latestSource = tuple;
|
|
7552
|
+
}
|
|
7553
|
+
}
|
|
7554
|
+
}
|
|
7555
|
+
if (latestSource === null)
|
|
7556
|
+
throw new Error("Compaction source order is unavailable");
|
|
7557
|
+
const maxOutputOrder = plan.partitions === undefined
|
|
7558
|
+
? plan.logicalOrder
|
|
7559
|
+
: Math.max(plan.logicalOrder, ...plan.partitions.map((partition) => partition.logicalOrder));
|
|
7560
|
+
const retained = new Map(plannedLayout.partitions
|
|
7561
|
+
.filter((partition) => !sourceIds.has(partition.id))
|
|
7562
|
+
.map((partition) => [partition.id, partition]));
|
|
7563
|
+
for (const segment of visibleSegments) {
|
|
7564
|
+
if (sourceIds.has(segment.id))
|
|
7565
|
+
continue;
|
|
7566
|
+
if ((segment.level ?? 0) === 1) {
|
|
7567
|
+
const planned = retained.get(segment.id);
|
|
7568
|
+
if (planned?.transactionId !== segment.transactionId ||
|
|
7569
|
+
!sameCompactionSegment(segment, planned)) {
|
|
7570
|
+
throw new Error(`Concurrent segment is not a retained partition: ${segment.id}`);
|
|
7571
|
+
}
|
|
7572
|
+
continue;
|
|
7573
|
+
}
|
|
7574
|
+
if ((segment.level ?? 0) !== 0) {
|
|
7575
|
+
throw new Error(`Concurrent segment has an unsupported compaction level: ${segment.id}`);
|
|
7576
|
+
}
|
|
7577
|
+
const tuple = sourceOrderTuple(segment, transactions, "Concurrent compaction segment");
|
|
7578
|
+
if (tuple.logicalOrder <= maxOutputOrder ||
|
|
7579
|
+
compareMergeSourceOrder(latestSource, tuple) >= 0) {
|
|
7580
|
+
throw new Error(`Concurrent segment would reorder compaction output: ${segment.id}`);
|
|
7581
|
+
}
|
|
7582
|
+
}
|
|
7583
|
+
for (const id of retained.keys()) {
|
|
7584
|
+
if (!visibleById.has(id)) {
|
|
7585
|
+
throw new Error(`Retained compaction partition is no longer visible: ${id}`);
|
|
7586
|
+
}
|
|
7587
|
+
}
|
|
7588
|
+
}
|
|
6626
7589
|
async #assertLevelTwoSnapshotOrder(job, plan, snapshot, transactions) {
|
|
6627
7590
|
const table = await this.store.getTable(job.tableId);
|
|
6628
7591
|
if (table === undefined)
|
|
@@ -6777,10 +7740,39 @@ export class MinnowDatabase {
|
|
|
6777
7740
|
throw error;
|
|
6778
7741
|
}
|
|
6779
7742
|
}
|
|
6780
|
-
|
|
6781
|
-
|
|
6782
|
-
|
|
6783
|
-
|
|
7743
|
+
/**
|
|
7744
|
+
* Stages one output segment, reconciling with what a previous attempt left: the same
|
|
7745
|
+
* segment staged by this transaction is reused, one left by an aborted transaction is
|
|
7746
|
+
* adopted when it matches and was never published, anything else is an error.
|
|
7747
|
+
*/
|
|
7748
|
+
async #stageCompactionOutputSegment(transaction, desired, expectedOutputIds) {
|
|
7749
|
+
const existing = await this.store.getSegment(desired.id);
|
|
7750
|
+
if (existing === undefined) {
|
|
7751
|
+
await transaction.stageSegment(desired);
|
|
7752
|
+
return;
|
|
7753
|
+
}
|
|
7754
|
+
if (existing.transactionId === transaction.id) {
|
|
7755
|
+
if (!sameCompactionSegment(existing, desired)) {
|
|
7756
|
+
throw new Error(`A resumed compaction segment differs: ${desired.id}`);
|
|
7757
|
+
}
|
|
7758
|
+
await transaction.stageExistingSegment(desired.id);
|
|
7759
|
+
return;
|
|
7760
|
+
}
|
|
7761
|
+
const owner = await this.store.getTransaction(existing.transactionId);
|
|
7762
|
+
if ((owner !== undefined && owner.status !== "aborted") ||
|
|
7763
|
+
!sameCompactionSegment(existing, desired)) {
|
|
7764
|
+
throw new Error(`Compaction output segment cannot be adopted: ${desired.id}`);
|
|
7765
|
+
}
|
|
7766
|
+
if (await this.#unprunedManifestContainsAll(expectedOutputIds)) {
|
|
7767
|
+
throw new Error(`Compaction output segment is already visible: ${desired.id}`);
|
|
7768
|
+
}
|
|
7769
|
+
await this.store.removeSegment(desired.id);
|
|
7770
|
+
await transaction.stageSegment(desired);
|
|
7771
|
+
}
|
|
7772
|
+
async #abortCompactionJob(job, error) {
|
|
7773
|
+
return this.store.updateCompactionJob(job.id, job.revision, {
|
|
7774
|
+
state: "aborted",
|
|
7775
|
+
updatedAt: this.#now().toISOString(),
|
|
6784
7776
|
error,
|
|
6785
7777
|
});
|
|
6786
7778
|
}
|
|
@@ -6803,6 +7795,7 @@ export class MinnowDatabase {
|
|
|
6803
7795
|
sourceSegmentCount: job.sourceSegmentIds.length,
|
|
6804
7796
|
sourceBlockCount: job.sourceBlockIds.length,
|
|
6805
7797
|
outputSegmentId: job.outputSegmentId,
|
|
7798
|
+
outputSegmentIds: compactionOutputSegmentIds(job),
|
|
6806
7799
|
outputBlockCount: job.outputBlockIds.length,
|
|
6807
7800
|
rowCount,
|
|
6808
7801
|
sourceStoredBytes: job.sourceStoredBytes,
|
|
@@ -8062,26 +9055,42 @@ export class MinnowDatabase {
|
|
|
8062
9055
|
* bytes and never correctness.
|
|
8063
9056
|
*/
|
|
8064
9057
|
async #encodeColumnBlock(columnId, input) {
|
|
8065
|
-
|
|
8066
|
-
|
|
9058
|
+
return this.#encodePreferredBlock(columnId, this.#compression, columnInputBytesBelow(input, GZIP_MINIMUM_INPUT_BYTES), (compression) => encodeBlock(input, compression));
|
|
9059
|
+
}
|
|
9060
|
+
/**
|
|
9061
|
+
* Applies the same adaptive gzip rule to ordinary writes and compaction output. `gzip` is a
|
|
9062
|
+
* preference, not a promise: tiny inputs and probes that save less than 20% stay raw. Only a
|
|
9063
|
+
* failed verdict is cached, so successful columns do not leave one map entry behind forever.
|
|
9064
|
+
*/
|
|
9065
|
+
async #encodePreferredBlock(columnId, preferred, belowMinimum, encode) {
|
|
9066
|
+
if (preferred !== "gzip")
|
|
9067
|
+
return encode(preferred);
|
|
9068
|
+
if (belowMinimum)
|
|
9069
|
+
return encode("raw");
|
|
8067
9070
|
const verdict = this.#gzipVerdicts.get(columnId);
|
|
8068
|
-
if (verdict !== undefined
|
|
8069
|
-
if (verdict
|
|
8070
|
-
|
|
8071
|
-
return
|
|
9071
|
+
if (verdict !== undefined) {
|
|
9072
|
+
if (verdict < GZIP_REPROBE_BLOCKS) {
|
|
9073
|
+
this.#gzipVerdicts.set(columnId, verdict + 1);
|
|
9074
|
+
return encode("raw");
|
|
8072
9075
|
}
|
|
8073
9076
|
this.#gzipVerdicts.delete(columnId);
|
|
8074
9077
|
}
|
|
8075
|
-
const bytes = await
|
|
9078
|
+
const bytes = await encode("gzip");
|
|
8076
9079
|
const description = inspectBlock(bytes);
|
|
8077
|
-
|
|
8078
|
-
|
|
8079
|
-
this.#gzipVerdicts.
|
|
9080
|
+
const worthwhile = description.encodedLength >= bytes.byteLength * GZIP_WORTHWHILE_RATIO;
|
|
9081
|
+
if (!worthwhile) {
|
|
9082
|
+
if (!this.#gzipVerdicts.has(columnId) &&
|
|
9083
|
+
this.#gzipVerdicts.size >= GZIP_VERDICT_CACHE_LIMIT) {
|
|
9084
|
+
const oldest = this.#gzipVerdicts.keys().next().value;
|
|
9085
|
+
if (oldest !== undefined)
|
|
9086
|
+
this.#gzipVerdicts.delete(oldest);
|
|
9087
|
+
}
|
|
9088
|
+
this.#gzipVerdicts.set(columnId, 0);
|
|
8080
9089
|
// Nothing was gained, so hand back the uncompressed form rather than make every read of
|
|
8081
9090
|
// this block pay to inflate it.
|
|
8082
|
-
|
|
8083
|
-
return encodeBlock(input, "raw");
|
|
9091
|
+
return encode("raw");
|
|
8084
9092
|
}
|
|
9093
|
+
this.#gzipVerdicts.delete(columnId);
|
|
8085
9094
|
return bytes;
|
|
8086
9095
|
}
|
|
8087
9096
|
async #findTable(name) {
|
|
@@ -8542,6 +9551,22 @@ function batchKeys(table, input) {
|
|
|
8542
9551
|
}
|
|
8543
9552
|
return keys;
|
|
8544
9553
|
}
|
|
9554
|
+
/**
|
|
9555
|
+
* Whether a column block's logical payload is under `limit` bytes: strings by length (two
|
|
9556
|
+
* bytes a code unit, stopping as soon as the limit is reached), everything else eight bytes a
|
|
9557
|
+
* value. An estimate, for the write path's codec choice — not an encoded size.
|
|
9558
|
+
*/
|
|
9559
|
+
function columnInputBytesBelow(input, limit) {
|
|
9560
|
+
if (input.type !== "string")
|
|
9561
|
+
return input.values.length * 8 < limit;
|
|
9562
|
+
let bytes = 0;
|
|
9563
|
+
for (const value of input.values) {
|
|
9564
|
+
bytes += 8 + (value === null ? 0 : value.length * 2);
|
|
9565
|
+
if (bytes >= limit)
|
|
9566
|
+
return false;
|
|
9567
|
+
}
|
|
9568
|
+
return true;
|
|
9569
|
+
}
|
|
8545
9570
|
/** `keyToken` for values that may not encode: undefined instead of a thrown encoding error. */
|
|
8546
9571
|
function tryKeyToken(type, value) {
|
|
8547
9572
|
try {
|
|
@@ -8551,6 +9576,34 @@ function tryKeyToken(type, value) {
|
|
|
8551
9576
|
return undefined;
|
|
8552
9577
|
}
|
|
8553
9578
|
}
|
|
9579
|
+
/**
|
|
9580
|
+
* Whether a table's visible segments warrant a background fold: enough segments for a scan to
|
|
9581
|
+
* pay per-segment overhead, or enough deltas that every query replays a history. Counted in
|
|
9582
|
+
* segments, not rows — a handful of deltas costs little however many rows they hold, and
|
|
9583
|
+
* folding rewrites the table's anchor, so it is reserved for when the count has built up.
|
|
9584
|
+
*/
|
|
9585
|
+
/**
|
|
9586
|
+
* Whether a table's visible history warrants a background fold: enough level-zero segments to
|
|
9587
|
+
* fragment a scan, or enough deltas to cost one. Partitions compaction itself published
|
|
9588
|
+
* (level one and above) are the folded state, not fragmentation, and do not count — a large
|
|
9589
|
+
* keyed table is many partitions by design.
|
|
9590
|
+
*/
|
|
9591
|
+
function autoCompactionDue(segments) {
|
|
9592
|
+
let levelZero = 0;
|
|
9593
|
+
let deltas = 0;
|
|
9594
|
+
for (const segment of segments) {
|
|
9595
|
+
if ((segment.level ?? 0) === 0)
|
|
9596
|
+
levelZero += 1;
|
|
9597
|
+
const kind = segment.kind ?? "insert";
|
|
9598
|
+
if (kind !== "insert" && kind !== "base")
|
|
9599
|
+
deltas += 1;
|
|
9600
|
+
}
|
|
9601
|
+
return levelZero >= AUTO_COMPACT_SCAN_SEGMENTS || deltas >= AUTO_COMPACT_DELTA_SEGMENTS;
|
|
9602
|
+
}
|
|
9603
|
+
/** A macrotask boundary, so background work lets queued queries and writes run between steps. */
|
|
9604
|
+
function yieldToEventLoop() {
|
|
9605
|
+
return new Promise((resolve) => setTimeout(resolve, 0));
|
|
9606
|
+
}
|
|
8554
9607
|
function keyToken(type, value) {
|
|
8555
9608
|
if (value === null)
|
|
8556
9609
|
throw new TypeError("Unique key cannot be null");
|
|
@@ -8578,6 +9631,12 @@ function keyToken(type, value) {
|
|
|
8578
9631
|
function formatValue(value) {
|
|
8579
9632
|
return value instanceof Date ? value.toISOString() : String(value);
|
|
8580
9633
|
}
|
|
9634
|
+
function nonNegativeWholeNumber(value, name) {
|
|
9635
|
+
if (!Number.isSafeInteger(value) || value < 0) {
|
|
9636
|
+
throw new RangeError(`${name} must be a non-negative whole number`);
|
|
9637
|
+
}
|
|
9638
|
+
return value;
|
|
9639
|
+
}
|
|
8581
9640
|
function positiveWholeNumber(value, name) {
|
|
8582
9641
|
if (!Number.isSafeInteger(value) || value <= 0) {
|
|
8583
9642
|
throw new RangeError(`${name} must be a positive whole number`);
|
|
@@ -9087,6 +10146,229 @@ function estimatedColumnarBytes(segments, columns) {
|
|
|
9087
10146
|
rowWidth += column.type === "string" ? 32 : 8;
|
|
9088
10147
|
return rows * Math.max(rowWidth, 1);
|
|
9089
10148
|
}
|
|
10149
|
+
/** Whether a visible segment is a delete or update delta rather than appended rows. */
|
|
10150
|
+
function mutationSegmentKind(segment) {
|
|
10151
|
+
const kind = segment.kind ?? "insert";
|
|
10152
|
+
return kind !== "insert" && kind !== "base";
|
|
10153
|
+
}
|
|
10154
|
+
const BYTE_POPCOUNT = new Uint8Array(256).map((_, byte) => {
|
|
10155
|
+
let count = 0;
|
|
10156
|
+
for (let value = byte; value !== 0; value &= value - 1)
|
|
10157
|
+
count += 1;
|
|
10158
|
+
return count;
|
|
10159
|
+
});
|
|
10160
|
+
/** Set bits in `bitmap` over bit indexes `[from, to)`. */
|
|
10161
|
+
function bitmapCountRange(bitmap, from, to) {
|
|
10162
|
+
let count = 0;
|
|
10163
|
+
let index = from;
|
|
10164
|
+
while (index < to && (index & 7) !== 0) {
|
|
10165
|
+
if (bitmapHasValue(bitmap, index))
|
|
10166
|
+
count += 1;
|
|
10167
|
+
index += 1;
|
|
10168
|
+
}
|
|
10169
|
+
while (index + 8 <= to) {
|
|
10170
|
+
count += BYTE_POPCOUNT[bitmap[index >>> 3] ?? 0] ?? 0;
|
|
10171
|
+
index += 8;
|
|
10172
|
+
}
|
|
10173
|
+
while (index < to) {
|
|
10174
|
+
if (bitmapHasValue(bitmap, index))
|
|
10175
|
+
count += 1;
|
|
10176
|
+
index += 1;
|
|
10177
|
+
}
|
|
10178
|
+
return count;
|
|
10179
|
+
}
|
|
10180
|
+
/** The first index in ascending `sorted` whose value is at least `value`. */
|
|
10181
|
+
function sortedLowerBound(sorted, value) {
|
|
10182
|
+
let low = 0;
|
|
10183
|
+
let high = sorted.length;
|
|
10184
|
+
while (low < high) {
|
|
10185
|
+
const middle = (low + high) >>> 1;
|
|
10186
|
+
if ((sorted[middle] ?? 0) < value)
|
|
10187
|
+
low = middle + 1;
|
|
10188
|
+
else
|
|
10189
|
+
high = middle;
|
|
10190
|
+
}
|
|
10191
|
+
return low;
|
|
10192
|
+
}
|
|
10193
|
+
/** Members of ascending `sorted` in `[from, to)`. */
|
|
10194
|
+
function sortedCountRange(sorted, from, to) {
|
|
10195
|
+
return sortedLowerBound(sorted, to) - sortedLowerBound(sorted, from);
|
|
10196
|
+
}
|
|
10197
|
+
function overlayWindowRuns(dead, patchedSlots, from, to, patchedInWindow) {
|
|
10198
|
+
const steps = [];
|
|
10199
|
+
let nextPatched = patchedInWindow > 0 ? sortedLowerBound(patchedSlots, from) : -1;
|
|
10200
|
+
let row = from;
|
|
10201
|
+
while (row < to) {
|
|
10202
|
+
// Dead rows, eight at a time where a whole byte is dead.
|
|
10203
|
+
if (bitmapHasValue(dead, row)) {
|
|
10204
|
+
row += 1;
|
|
10205
|
+
while (row < to && (row & 7) === 0 && dead[row >>> 3] === 0xff && row + 8 <= to)
|
|
10206
|
+
row += 8;
|
|
10207
|
+
while (row < to && bitmapHasValue(dead, row))
|
|
10208
|
+
row += 1;
|
|
10209
|
+
continue;
|
|
10210
|
+
}
|
|
10211
|
+
const patchedRow = nextPatched >= 0 ? (patchedSlots[nextPatched] ?? to) : to;
|
|
10212
|
+
if (row === patchedRow) {
|
|
10213
|
+
steps.push(row, 0);
|
|
10214
|
+
row += 1;
|
|
10215
|
+
nextPatched += 1;
|
|
10216
|
+
if (nextPatched >= patchedSlots.length)
|
|
10217
|
+
nextPatched = -1;
|
|
10218
|
+
continue;
|
|
10219
|
+
}
|
|
10220
|
+
// A live run: up to the next patched row, the window end, or the next dead row — live
|
|
10221
|
+
// rows are consecutive except where a delete cut them, and whole live bytes skip in one.
|
|
10222
|
+
const limit = Math.min(to, patchedRow);
|
|
10223
|
+
const runStart = row;
|
|
10224
|
+
row += 1;
|
|
10225
|
+
while (row < limit && (row & 7) === 0 && row + 8 <= limit && dead[row >>> 3] === 0)
|
|
10226
|
+
row += 8;
|
|
10227
|
+
while (row < limit && !bitmapHasValue(dead, row))
|
|
10228
|
+
row += 1;
|
|
10229
|
+
steps.push(runStart, row - runStart);
|
|
10230
|
+
}
|
|
10231
|
+
return steps;
|
|
10232
|
+
}
|
|
10233
|
+
/**
|
|
10234
|
+
* Copies `length` bits from `source` at bit `sourceStart` to `target` at bit `targetStart`,
|
|
10235
|
+
* whole bytes at a time once the target is byte-aligned: the target bytes it overwrites lie
|
|
10236
|
+
* entirely inside the copied range, so the target's other bits are left alone. This is what
|
|
10237
|
+
* makes a validity copy proportional to bytes rather than to cells.
|
|
10238
|
+
*/
|
|
10239
|
+
function copyBitRun(source, sourceStart, target, targetStart, length) {
|
|
10240
|
+
let remaining = length;
|
|
10241
|
+
let from = sourceStart;
|
|
10242
|
+
let to = targetStart;
|
|
10243
|
+
while (remaining > 0 && (to & 7) !== 0) {
|
|
10244
|
+
if (bitmapHasValue(source, from))
|
|
10245
|
+
setBitmapValue(target, to);
|
|
10246
|
+
from += 1;
|
|
10247
|
+
to += 1;
|
|
10248
|
+
remaining -= 1;
|
|
10249
|
+
}
|
|
10250
|
+
const shift = from & 7;
|
|
10251
|
+
if (shift === 0) {
|
|
10252
|
+
const bytes = remaining >>> 3;
|
|
10253
|
+
if (bytes > 0) {
|
|
10254
|
+
target.set(source.subarray(from >>> 3, (from >>> 3) + bytes), to >>> 3);
|
|
10255
|
+
from += bytes * 8;
|
|
10256
|
+
to += bytes * 8;
|
|
10257
|
+
remaining -= bytes * 8;
|
|
10258
|
+
}
|
|
10259
|
+
}
|
|
10260
|
+
else {
|
|
10261
|
+
while (remaining >= 8) {
|
|
10262
|
+
const sourceByte = from >>> 3;
|
|
10263
|
+
target[to >>> 3] =
|
|
10264
|
+
(((source[sourceByte] ?? 0) >>> shift) | ((source[sourceByte + 1] ?? 0) << (8 - shift))) &
|
|
10265
|
+
0xff;
|
|
10266
|
+
from += 8;
|
|
10267
|
+
to += 8;
|
|
10268
|
+
remaining -= 8;
|
|
10269
|
+
}
|
|
10270
|
+
}
|
|
10271
|
+
while (remaining > 0) {
|
|
10272
|
+
if (bitmapHasValue(source, from))
|
|
10273
|
+
setBitmapValue(target, to);
|
|
10274
|
+
from += 1;
|
|
10275
|
+
to += 1;
|
|
10276
|
+
remaining -= 1;
|
|
10277
|
+
}
|
|
10278
|
+
}
|
|
10279
|
+
/**
|
|
10280
|
+
* An outer window that is the inner window's rows from `offset` on, by reference: typed-array
|
|
10281
|
+
* views over the resident block, and its dictionary as-is. Validity is a view too when the
|
|
10282
|
+
* offset falls on a byte, and otherwise the one small copy a bit offset forces.
|
|
10283
|
+
*/
|
|
10284
|
+
function overlayWindowView(inner, offset, rows, memory, column, reservations) {
|
|
10285
|
+
let validity;
|
|
10286
|
+
if ((offset & 7) === 0) {
|
|
10287
|
+
validity = inner.validity.subarray(offset >>> 3, (offset >>> 3) + Math.ceil(rows / 8));
|
|
10288
|
+
}
|
|
10289
|
+
else {
|
|
10290
|
+
validity = new Uint8Array(Math.ceil(rows / 8));
|
|
10291
|
+
reservations.push(memory.reserve(validity.byteLength, `Streamed window ${column.name}`));
|
|
10292
|
+
copyBitRun(inner.validity, offset, validity, 0, rows);
|
|
10293
|
+
}
|
|
10294
|
+
const fields = { validity, window: { start: 0, length: rows } };
|
|
10295
|
+
if (inner.kind === "string") {
|
|
10296
|
+
fields.codes = inner.codes.subarray(offset, offset + rows);
|
|
10297
|
+
fields.dictionary = inner.dictionary;
|
|
10298
|
+
}
|
|
10299
|
+
else {
|
|
10300
|
+
fields.values = inner.values.subarray(offset, offset + rows);
|
|
10301
|
+
}
|
|
10302
|
+
return fields;
|
|
10303
|
+
}
|
|
10304
|
+
/**
|
|
10305
|
+
* An outer window compacted from an inner window: live runs copied as slices, patched rows
|
|
10306
|
+
* read from their update vectors. A string window shares the inner dictionary unless a patch
|
|
10307
|
+
* has to add to it, in which case it copies the dictionary first.
|
|
10308
|
+
*/
|
|
10309
|
+
function overlayWindowCompacted(inner, innerWindowStart, steps, rows, patches, column, memory, reservations) {
|
|
10310
|
+
const validityBytes = Math.ceil(rows / 8);
|
|
10311
|
+
const typedBytes = validityBytes +
|
|
10312
|
+
(inner.kind === "boolean"
|
|
10313
|
+
? rows
|
|
10314
|
+
: inner.kind === "string"
|
|
10315
|
+
? rows * Uint32Array.BYTES_PER_ELEMENT
|
|
10316
|
+
: rows * Float64Array.BYTES_PER_ELEMENT);
|
|
10317
|
+
reservations.push(memory.reserve(typedBytes, `Streamed window ${column.name}`));
|
|
10318
|
+
const validity = new Uint8Array(validityBytes);
|
|
10319
|
+
const values = inner.kind === "boolean"
|
|
10320
|
+
? new Uint8Array(rows)
|
|
10321
|
+
: inner.kind === "string"
|
|
10322
|
+
? undefined
|
|
10323
|
+
: new Float64Array(rows);
|
|
10324
|
+
const codes = inner.kind === "string" ? new Uint32Array(rows) : undefined;
|
|
10325
|
+
codes?.fill(NULL_STRING_VECTOR_CODE);
|
|
10326
|
+
let dictionary = inner.kind === "string" ? inner.dictionary : undefined;
|
|
10327
|
+
let dictionaryIndex;
|
|
10328
|
+
let dictionaryCopied = false;
|
|
10329
|
+
const target = (codes !== undefined
|
|
10330
|
+
? { kind: "string", length: rows, validity, codes, dictionary: dictionary ?? [] }
|
|
10331
|
+
: { kind: inner.kind, length: rows, validity, values });
|
|
10332
|
+
let out = 0;
|
|
10333
|
+
for (let index = 0; index < steps.length; index += 2) {
|
|
10334
|
+
const start = steps[index] ?? 0;
|
|
10335
|
+
const length = steps[index + 1] ?? 0;
|
|
10336
|
+
const patch = length === 0 ? patches?.get(start)?.get(column.id) : undefined;
|
|
10337
|
+
if (patch === undefined) {
|
|
10338
|
+
const count = Math.max(1, length);
|
|
10339
|
+
copyVectorSpan(inner, start - innerWindowStart, count, target, out);
|
|
10340
|
+
out += count;
|
|
10341
|
+
continue;
|
|
10342
|
+
}
|
|
10343
|
+
if (target.kind === "string" && !dictionaryCopied) {
|
|
10344
|
+
// A patch value may be new to this window's dictionary, and the inner's belongs to the
|
|
10345
|
+
// buffer pool: copy before the first append, and index the copy for the lookups.
|
|
10346
|
+
dictionary = [...(dictionary ?? [])];
|
|
10347
|
+
target.dictionary = dictionary;
|
|
10348
|
+
dictionaryIndex = new Map(dictionary.map((value, code) => [value, code]));
|
|
10349
|
+
dictionaryCopied = true;
|
|
10350
|
+
}
|
|
10351
|
+
copyColumnVectorValue(patch.vector, patch.row, target, out, dictionaryIndex);
|
|
10352
|
+
out += 1;
|
|
10353
|
+
}
|
|
10354
|
+
if (out !== rows)
|
|
10355
|
+
throw new Error(`Column row count mismatch: ${column.name}`);
|
|
10356
|
+
if (dictionaryCopied && dictionary !== undefined) {
|
|
10357
|
+
let dictionaryBytes = 0;
|
|
10358
|
+
for (const value of dictionary)
|
|
10359
|
+
dictionaryBytes += 16 + value.length * 2;
|
|
10360
|
+
reservations.push(memory.reserve(dictionaryBytes, `Streamed window ${column.name}`));
|
|
10361
|
+
}
|
|
10362
|
+
const fields = { validity, window: { start: 0, length: rows } };
|
|
10363
|
+
if (codes !== undefined) {
|
|
10364
|
+
fields.codes = codes;
|
|
10365
|
+
fields.dictionary = dictionary ?? [];
|
|
10366
|
+
}
|
|
10367
|
+
else if (values !== undefined) {
|
|
10368
|
+
fields.values = values;
|
|
10369
|
+
}
|
|
10370
|
+
return fields;
|
|
10371
|
+
}
|
|
9090
10372
|
/**
|
|
9091
10373
|
* Reads a key column's values as primitives. Dictionary-coded strings resolve through the
|
|
9092
10374
|
* dictionary the vector already holds, so a string key costs one array index and no encoding.
|
|
@@ -9140,9 +10422,10 @@ function touchedKeyPredicate(keyColumn, touched) {
|
|
|
9140
10422
|
return { column: keyColumn, operator: "IN", value: members[0] ?? 0, members };
|
|
9141
10423
|
}
|
|
9142
10424
|
/**
|
|
9143
|
-
* Copies a run of rows between vectors: values as one typed-array slice, validity
|
|
10425
|
+
* Copies a run of rows between vectors: values as one typed-array slice, validity as a bit run.
|
|
9144
10426
|
* A string run needs `remap` unless both sides share a dictionary — the codes mean nothing on
|
|
9145
|
-
* their own. This is what keeps a copy proportional to bytes rather than to cells.
|
|
10427
|
+
* their own. This is what keeps a copy proportional to bytes rather than to cells. The target
|
|
10428
|
+
* validity bits of the run must be clear beforehand, as a fresh window's are.
|
|
9146
10429
|
*/
|
|
9147
10430
|
function copyVectorSpan(source, sourceStart, length, target, targetStart, remap) {
|
|
9148
10431
|
if (source.kind === "string") {
|
|
@@ -9172,26 +10455,7 @@ function copyVectorSpan(source, sourceStart, length, target, targetStart, remap)
|
|
|
9172
10455
|
}
|
|
9173
10456
|
target.values.set(source.values.subarray(sourceStart, sourceStart + length), targetStart);
|
|
9174
10457
|
}
|
|
9175
|
-
|
|
9176
|
-
if (bitmapHasValue(source.validity, sourceStart + index)) {
|
|
9177
|
-
setBitmapValue(target.validity, targetStart + index);
|
|
9178
|
-
}
|
|
9179
|
-
}
|
|
9180
|
-
}
|
|
9181
|
-
/** Source dictionary code -> target dictionary code, built once per source window. */
|
|
9182
|
-
function remapDictionary(source, targetDictionary, targetIndex) {
|
|
9183
|
-
const remap = new Uint32Array(source.length);
|
|
9184
|
-
for (let code = 0; code < source.length; code += 1) {
|
|
9185
|
-
const value = source[code] ?? "";
|
|
9186
|
-
let mapped = targetIndex.get(value);
|
|
9187
|
-
if (mapped === undefined) {
|
|
9188
|
-
mapped = targetDictionary.length;
|
|
9189
|
-
targetDictionary.push(value);
|
|
9190
|
-
targetIndex.set(value, mapped);
|
|
9191
|
-
}
|
|
9192
|
-
remap[code] = mapped;
|
|
9193
|
-
}
|
|
9194
|
-
return remap;
|
|
10458
|
+
copyBitRun(source.validity, sourceStart, target.validity, targetStart, length);
|
|
9195
10459
|
}
|
|
9196
10460
|
/**
|
|
9197
10461
|
* The live rows of a vector, in order. Runs between deletions copy as typed-array slices and
|
|
@@ -9568,13 +10832,356 @@ function sourceOrderTuple(segment, transactions, label) {
|
|
|
9568
10832
|
segmentId: segment.id,
|
|
9569
10833
|
};
|
|
9570
10834
|
}
|
|
10835
|
+
/** Modeled bytes per referenced key the merge planner and the partition probe hold resident. */
|
|
10836
|
+
const MERGE_PLANNER_KEY_BYTES = 96;
|
|
10837
|
+
function planLinearOutputPartitions(totalRows, partitionRows, firstOrder, nextOrder) {
|
|
10838
|
+
const count = Math.max(1, Math.ceil(totalRows / partitionRows));
|
|
10839
|
+
const orders = fractionalLogicalOrders(firstOrder, nextOrder, count);
|
|
10840
|
+
const partitions = [];
|
|
10841
|
+
for (let index = 0, rowStart = 0; index < count; index += 1) {
|
|
10842
|
+
const rowCount = Math.min(partitionRows, totalRows - rowStart);
|
|
10843
|
+
const logicalOrder = orders[index];
|
|
10844
|
+
if (rowCount <= 0 || logicalOrder === undefined) {
|
|
10845
|
+
throw new Error("Rechunk partition layout is incomplete");
|
|
10846
|
+
}
|
|
10847
|
+
partitions.push({ rowStart, rowCount, logicalOrder });
|
|
10848
|
+
rowStart += rowCount;
|
|
10849
|
+
}
|
|
10850
|
+
return partitions;
|
|
10851
|
+
}
|
|
10852
|
+
/**
|
|
10853
|
+
* Cuts the canonical merged output into the partitions a keyed fold publishes.
|
|
10854
|
+
*
|
|
10855
|
+
* Each rewritten source partition's surviving rows form one region that keeps the partition's
|
|
10856
|
+
* logical order, and so its place among the partitions the fold leaves alone. The rows of the
|
|
10857
|
+
* level-zero sources — the tail — form a region behind every existing partition, or extend
|
|
10858
|
+
* the last partition's region when the fold absorbs them into it. A region is then chunked to
|
|
10859
|
+
* at most `partitionRows` rows per published partition. The first chunk keeps the source
|
|
10860
|
+
* partition's order and every further chunk takes an evenly spaced fractional order before
|
|
10861
|
+
* the unchanged successor;
|
|
10862
|
+
* a fresh tail starts at its earliest source's order. Fractional orders make room independent
|
|
10863
|
+
* of adjacent commit versions, so every output is bounded by `partitionRows`. The order is
|
|
10864
|
+
* stable: a published partition sorts strictly between its neighbours and below every
|
|
10865
|
+
* level-zero segment, so a later fold rewrites it alone without moving a row.
|
|
10866
|
+
*/
|
|
10867
|
+
function planOutputPartitions(partitioning, sources, sourceOutputRowStarts, totalRows) {
|
|
10868
|
+
const { partitions, partitionRows, absorbsTail, nextLevelZeroOrder } = partitioning;
|
|
10869
|
+
const startOf = (segmentId) => {
|
|
10870
|
+
const start = sourceOutputRowStarts.get(segmentId);
|
|
10871
|
+
if (start === undefined)
|
|
10872
|
+
throw new Error(`Merge source has no output position: ${segmentId}`);
|
|
10873
|
+
return start;
|
|
10874
|
+
};
|
|
10875
|
+
const levelZeroRowSources = sources.filter((source) => source.level === 0 && mergeSourceBearsRows(source.kind));
|
|
10876
|
+
const tailStart = levelZeroRowSources.length === 0 ? totalRows : startOf(levelZeroRowSources[0]?.segmentId ?? "");
|
|
10877
|
+
const sourceIds = new Set(sources.map((source) => source.segmentId));
|
|
10878
|
+
const sourcedPartitionIndexes = partitions.flatMap((partition, index) => sourceIds.has(partition.id) ? [index] : []);
|
|
10879
|
+
const regions = [];
|
|
10880
|
+
for (const [position, index] of sourcedPartitionIndexes.entries()) {
|
|
10881
|
+
const partition = partitions[index];
|
|
10882
|
+
const order = partition?.logicalOrder;
|
|
10883
|
+
if (partition === undefined || order === undefined) {
|
|
10884
|
+
throw new Error("Partitioned merge source is not a level-one partition");
|
|
10885
|
+
}
|
|
10886
|
+
const nextSourced = sourcedPartitionIndexes[position + 1];
|
|
10887
|
+
const isLast = index === partitions.length - 1;
|
|
10888
|
+
const rowStart = startOf(partition.id);
|
|
10889
|
+
const rowEnd = isLast && absorbsTail
|
|
10890
|
+
? totalRows
|
|
10891
|
+
: nextSourced === undefined
|
|
10892
|
+
? tailStart
|
|
10893
|
+
: startOf(partitions[nextSourced]?.id ?? "");
|
|
10894
|
+
const successorOrder = partitions[index + 1]?.logicalOrder ?? nextLevelZeroOrder;
|
|
10895
|
+
regions.push({
|
|
10896
|
+
rowStart,
|
|
10897
|
+
rowCount: rowEnd - rowStart,
|
|
10898
|
+
anchorOrder: order,
|
|
10899
|
+
roomStart: order,
|
|
10900
|
+
roomEnd: successorOrder,
|
|
10901
|
+
preferredOrder: order,
|
|
10902
|
+
});
|
|
10903
|
+
}
|
|
10904
|
+
if (!(absorbsTail && partitions.length > 0)) {
|
|
10905
|
+
const lastOrder = partitions[partitions.length - 1]?.logicalOrder ?? -1;
|
|
10906
|
+
regions.push({
|
|
10907
|
+
rowStart: tailStart,
|
|
10908
|
+
rowCount: totalRows - tailStart,
|
|
10909
|
+
anchorOrder: null,
|
|
10910
|
+
roomStart: lastOrder,
|
|
10911
|
+
roomEnd: nextLevelZeroOrder,
|
|
10912
|
+
preferredOrder: Math.min(...sources.filter((source) => source.level === 0).map((source) => source.logicalOrder)),
|
|
10913
|
+
});
|
|
10914
|
+
}
|
|
10915
|
+
const output = [];
|
|
10916
|
+
for (const region of regions) {
|
|
10917
|
+
if (region.rowCount <= 0)
|
|
10918
|
+
continue;
|
|
10919
|
+
const chunks = Math.max(1, Math.ceil(region.rowCount / partitionRows));
|
|
10920
|
+
const firstOrder = region.anchorOrder ?? region.preferredOrder;
|
|
10921
|
+
if (!validLogicalOrder(firstOrder) ||
|
|
10922
|
+
firstOrder >= region.roomEnd ||
|
|
10923
|
+
(region.anchorOrder === null && firstOrder <= region.roomStart)) {
|
|
10924
|
+
throw new Error("Partitioned merge has no logical-order interval for its output");
|
|
10925
|
+
}
|
|
10926
|
+
const logicalOrders = fractionalLogicalOrders(firstOrder, region.roomEnd, chunks);
|
|
10927
|
+
const baseRows = Math.floor(region.rowCount / chunks);
|
|
10928
|
+
const extraRows = region.rowCount % chunks;
|
|
10929
|
+
let rowStart = region.rowStart;
|
|
10930
|
+
for (let chunk = 0; chunk < chunks; chunk += 1) {
|
|
10931
|
+
const rowCount = baseRows + (chunk < extraRows ? 1 : 0);
|
|
10932
|
+
const logicalOrder = logicalOrders[chunk];
|
|
10933
|
+
if (logicalOrder === undefined)
|
|
10934
|
+
throw new Error("Partition logical order is unavailable");
|
|
10935
|
+
output.push({ rowStart, rowCount, logicalOrder });
|
|
10936
|
+
rowStart += rowCount;
|
|
10937
|
+
}
|
|
10938
|
+
}
|
|
10939
|
+
let coveredRows = 0;
|
|
10940
|
+
for (const [index, partition] of output.entries()) {
|
|
10941
|
+
const previous = output[index - 1];
|
|
10942
|
+
if (partition.rowStart !== coveredRows ||
|
|
10943
|
+
partition.rowCount <= 0 ||
|
|
10944
|
+
partition.logicalOrder >= nextLevelZeroOrder ||
|
|
10945
|
+
(previous !== undefined && previous.logicalOrder >= partition.logicalOrder)) {
|
|
10946
|
+
throw new Error("Partitioned merge produced an invalid partition layout");
|
|
10947
|
+
}
|
|
10948
|
+
coveredRows += partition.rowCount;
|
|
10949
|
+
}
|
|
10950
|
+
if (coveredRows !== totalRows) {
|
|
10951
|
+
throw new Error("Partitioned merge partitions do not cover the merged output");
|
|
10952
|
+
}
|
|
10953
|
+
return output;
|
|
10954
|
+
}
|
|
10955
|
+
/** `count` increasing doubles in [first, upper), retaining `first` exactly. */
|
|
10956
|
+
function fractionalLogicalOrders(first, upper, count) {
|
|
10957
|
+
if (!validLogicalOrder(first) || !Number.isFinite(upper) || upper <= first || count < 1) {
|
|
10958
|
+
throw new Error("Partition logical-order interval is invalid");
|
|
10959
|
+
}
|
|
10960
|
+
const orders = [];
|
|
10961
|
+
for (let index = 0; index < count; index += 1) {
|
|
10962
|
+
const order = index === 0 ? first : first + ((upper - first) * index) / count;
|
|
10963
|
+
const previous = orders[index - 1];
|
|
10964
|
+
if (!validLogicalOrder(order) ||
|
|
10965
|
+
order >= upper ||
|
|
10966
|
+
(previous !== undefined && order <= previous)) {
|
|
10967
|
+
throw new Error("Partition logical-order precision is exhausted");
|
|
10968
|
+
}
|
|
10969
|
+
orders.push(order);
|
|
10970
|
+
}
|
|
10971
|
+
return orders;
|
|
10972
|
+
}
|
|
10973
|
+
/**
|
|
10974
|
+
* The planner's working memory, as `#resolveMergeOutput` allocates it: two bytes per slot, the
|
|
10975
|
+
* touched-key set and live-slot map over the delta keys, one patch array per patched row, one
|
|
10976
|
+
* decoded key block at a time, and the output ranges themselves — which number the source
|
|
10977
|
+
* blocks plus one per patched cell, not one per row. Deliberately generous per element; this
|
|
10978
|
+
* bound is what a caller's `memoryBudgetBytes` is judged against, so it must not be optimistic.
|
|
10979
|
+
*/
|
|
9571
10980
|
function mergePlannerMemoryBound(table, segments, keyColumnId) {
|
|
9572
|
-
const
|
|
9573
|
-
|
|
9574
|
-
|
|
9575
|
-
const
|
|
9576
|
-
const
|
|
9577
|
-
|
|
10981
|
+
const SLOT_BYTES = 2;
|
|
10982
|
+
const KEY_BYTES = MERGE_PLANNER_KEY_BYTES;
|
|
10983
|
+
const PATCH_ROW_BYTES = 64;
|
|
10984
|
+
const PATCH_CELL_BYTES = 48;
|
|
10985
|
+
const RANGE_BYTES = 80;
|
|
10986
|
+
const DECODED_KEY_BLOCK_FACTOR = 4;
|
|
10987
|
+
let slotRows = 0;
|
|
10988
|
+
let deltaKeys = 0;
|
|
10989
|
+
let patchRows = 0;
|
|
10990
|
+
let sourceBlocks = 0;
|
|
10991
|
+
let largestKeyBlockBytes = 0;
|
|
10992
|
+
for (const segment of segments) {
|
|
10993
|
+
if (mergeSourceBearsRows(segment.kind))
|
|
10994
|
+
slotRows += segment.rowCount;
|
|
10995
|
+
if (mergeSourceReferencesKeys(segment.kind))
|
|
10996
|
+
deltaKeys += segment.rowCount;
|
|
10997
|
+
if (segment.kind === "update" || segment.kind === "upsert")
|
|
10998
|
+
patchRows += segment.rowCount;
|
|
10999
|
+
for (const column of segment.columns) {
|
|
11000
|
+
sourceBlocks += column.sourceBlocks.length;
|
|
11001
|
+
if (column.columnId !== keyColumnId)
|
|
11002
|
+
continue;
|
|
11003
|
+
for (const block of column.sourceBlocks) {
|
|
11004
|
+
largestKeyBlockBytes = Math.max(largestKeyBlockBytes, block.encodedBytes);
|
|
11005
|
+
}
|
|
11006
|
+
}
|
|
11007
|
+
}
|
|
11008
|
+
const columns = table.columns.length;
|
|
11009
|
+
return safeWholeNumberSum([
|
|
11010
|
+
safeWholeNumberProduct(slotRows, SLOT_BYTES, "Mutation compaction slots"),
|
|
11011
|
+
safeWholeNumberProduct(deltaKeys, KEY_BYTES, "Mutation compaction keys"),
|
|
11012
|
+
safeWholeNumberProduct(patchRows, safeWholeNumberSum([
|
|
11013
|
+
PATCH_ROW_BYTES,
|
|
11014
|
+
safeWholeNumberProduct(columns, PATCH_CELL_BYTES, "Mutation patch cells"),
|
|
11015
|
+
], "Mutation compaction patch row"), "Mutation compaction patches"),
|
|
11016
|
+
safeWholeNumberProduct(safeWholeNumberSum([sourceBlocks, safeWholeNumberProduct(patchRows, columns, "Mutation patched cells")], "Mutation compaction ranges"), RANGE_BYTES, "Mutation compaction range bytes"),
|
|
11017
|
+
safeWholeNumberProduct(largestKeyBlockBytes, DECODED_KEY_BLOCK_FACTOR, "Mutation compaction decoded key block"),
|
|
11018
|
+
], "Mutation compaction planner memory");
|
|
11019
|
+
}
|
|
11020
|
+
/** Whether a source of this kind contributes rows to the merged output. */
|
|
11021
|
+
function mergeSourceBearsRows(kind) {
|
|
11022
|
+
return kind === "insert" || kind === "upsert" || kind === "base";
|
|
11023
|
+
}
|
|
11024
|
+
/** Whether a source of this kind names existing rows by key. */
|
|
11025
|
+
function mergeSourceReferencesKeys(kind) {
|
|
11026
|
+
return kind === "delete" || kind === "update" || kind === "upsert";
|
|
11027
|
+
}
|
|
11028
|
+
/**
|
|
11029
|
+
* A decoded key value as the primitive the overlay replay keys on (`OverlayKey`): equal keys
|
|
11030
|
+
* are equal primitives, and one table's key has one type, so nothing can collide.
|
|
11031
|
+
*/
|
|
11032
|
+
function overlayKeyOf(type, value) {
|
|
11033
|
+
if (value === null)
|
|
11034
|
+
throw new TypeError("Unique key cannot be null");
|
|
11035
|
+
switch (type) {
|
|
11036
|
+
case "boolean":
|
|
11037
|
+
if (typeof value !== "boolean")
|
|
11038
|
+
throw new TypeError("Invalid boolean unique key");
|
|
11039
|
+
return value;
|
|
11040
|
+
case "number":
|
|
11041
|
+
if (typeof value !== "number" || !Number.isFinite(value)) {
|
|
11042
|
+
throw new TypeError("Invalid number unique key");
|
|
11043
|
+
}
|
|
11044
|
+
return value;
|
|
11045
|
+
case "string":
|
|
11046
|
+
if (typeof value !== "string")
|
|
11047
|
+
throw new TypeError("Invalid string unique key");
|
|
11048
|
+
return value;
|
|
11049
|
+
case "datetime":
|
|
11050
|
+
if (!(value instanceof Date) || !Number.isFinite(value.getTime())) {
|
|
11051
|
+
throw new TypeError("Invalid datetime unique key");
|
|
11052
|
+
}
|
|
11053
|
+
return value.getTime();
|
|
11054
|
+
}
|
|
11055
|
+
}
|
|
11056
|
+
/**
|
|
11057
|
+
* Accumulates the merged output as coalesced row-ID spans and per-column source ranges. A run
|
|
11058
|
+
* of untouched rows appends at most one range per source block it crosses, whatever its
|
|
11059
|
+
* length; a patched row appends one range per column. Adjacent ranges over the same block
|
|
11060
|
+
* merge in place, so the finished plan is proportional to blocks plus patched cells.
|
|
11061
|
+
*/
|
|
11062
|
+
class MergeOutputBuilder {
|
|
11063
|
+
#columns;
|
|
11064
|
+
#rowIdSpans = [];
|
|
11065
|
+
#rangesByColumn;
|
|
11066
|
+
#blocksBySegment = new Map();
|
|
11067
|
+
#totalRows = 0;
|
|
11068
|
+
constructor(columns) {
|
|
11069
|
+
this.#columns = columns;
|
|
11070
|
+
this.#rangesByColumn = columns.map(() => []);
|
|
11071
|
+
}
|
|
11072
|
+
/** Output rows appended so far. */
|
|
11073
|
+
get totalRows() {
|
|
11074
|
+
return this.#totalRows;
|
|
11075
|
+
}
|
|
11076
|
+
/** Rows `[rowStart, rowStart + rowCount)` of a row-bearing source, unchanged. */
|
|
11077
|
+
appendRun(segment, rowStart, rowCount) {
|
|
11078
|
+
if (rowCount <= 0)
|
|
11079
|
+
return;
|
|
11080
|
+
this.#appendRowIds(segment, rowStart, rowCount);
|
|
11081
|
+
const blocksByColumn = this.#sourceBlocks(segment);
|
|
11082
|
+
for (let columnIndex = 0; columnIndex < this.#columns.length; columnIndex += 1) {
|
|
11083
|
+
const blocks = blocksByColumn[columnIndex];
|
|
11084
|
+
const ranges = this.#rangesByColumn[columnIndex];
|
|
11085
|
+
if (blocks === undefined || ranges === undefined) {
|
|
11086
|
+
throw new Error("Mutation output column is missing");
|
|
11087
|
+
}
|
|
11088
|
+
let outputRow = this.#totalRows;
|
|
11089
|
+
let remaining = rowCount;
|
|
11090
|
+
let rowIndex = rowStart;
|
|
11091
|
+
while (remaining > 0) {
|
|
11092
|
+
const block = rowRangeAt(blocks, rowIndex);
|
|
11093
|
+
if (block === undefined) {
|
|
11094
|
+
throw new Error(`Mutation source row is missing: ${segment.segmentId}`);
|
|
11095
|
+
}
|
|
11096
|
+
const count = Math.min(remaining, block.rowStart + block.rowCount - rowIndex);
|
|
11097
|
+
appendMergeOutputRange(ranges, outputRow, block.blockId, rowIndex - block.rowStart, count);
|
|
11098
|
+
outputRow += count;
|
|
11099
|
+
rowIndex += count;
|
|
11100
|
+
remaining -= count;
|
|
11101
|
+
}
|
|
11102
|
+
}
|
|
11103
|
+
this.#totalRows += rowCount;
|
|
11104
|
+
}
|
|
11105
|
+
/** One row of a row-bearing source whose columns may come from later mutations. */
|
|
11106
|
+
appendPatchedRow(segment, rowIndex, patch) {
|
|
11107
|
+
this.#appendRowIds(segment, rowIndex, 1);
|
|
11108
|
+
for (let columnIndex = 0; columnIndex < this.#columns.length; columnIndex += 1) {
|
|
11109
|
+
const column = this.#columns[columnIndex];
|
|
11110
|
+
const ranges = this.#rangesByColumn[columnIndex];
|
|
11111
|
+
if (column === undefined || ranges === undefined) {
|
|
11112
|
+
throw new Error("Mutation output column is missing");
|
|
11113
|
+
}
|
|
11114
|
+
const source = patch?.[columnIndex] ?? mergeSourceAt(segment, column.id, rowIndex);
|
|
11115
|
+
appendMergeOutputRange(ranges, this.#totalRows, source.blockId, source.sourceRowIndex, 1);
|
|
11116
|
+
}
|
|
11117
|
+
this.#totalRows += 1;
|
|
11118
|
+
}
|
|
11119
|
+
finish() {
|
|
11120
|
+
return {
|
|
11121
|
+
rowIdSpans: this.#rowIdSpans,
|
|
11122
|
+
columns: this.#columns.map((column, columnIndex) => {
|
|
11123
|
+
const sourceRanges = this.#rangesByColumn[columnIndex];
|
|
11124
|
+
if (sourceRanges === undefined)
|
|
11125
|
+
throw new Error("Mutation output column is missing");
|
|
11126
|
+
return { columnId: column.id, type: column.type, sourceRanges };
|
|
11127
|
+
}),
|
|
11128
|
+
totalRows: this.#totalRows,
|
|
11129
|
+
};
|
|
11130
|
+
}
|
|
11131
|
+
#sourceBlocks(segment) {
|
|
11132
|
+
let blocks = this.#blocksBySegment.get(segment.segmentId);
|
|
11133
|
+
if (blocks === undefined) {
|
|
11134
|
+
blocks = this.#columns.map((column) => {
|
|
11135
|
+
const source = segment.columns.find((candidate) => candidate.columnId === column.id);
|
|
11136
|
+
if (source === undefined) {
|
|
11137
|
+
throw new Error(`Mutation source row is missing: ${segment.segmentId}:${column.id}`);
|
|
11138
|
+
}
|
|
11139
|
+
return source.sourceBlocks;
|
|
11140
|
+
});
|
|
11141
|
+
this.#blocksBySegment.set(segment.segmentId, blocks);
|
|
11142
|
+
}
|
|
11143
|
+
return blocks;
|
|
11144
|
+
}
|
|
11145
|
+
#appendRowIds(segment, rowStart, rowCount) {
|
|
11146
|
+
let outputRow = this.#totalRows;
|
|
11147
|
+
let remaining = rowCount;
|
|
11148
|
+
let rowIndex = rowStart;
|
|
11149
|
+
while (remaining > 0) {
|
|
11150
|
+
const span = rowRangeAt(segment.rowIdSpans, rowIndex);
|
|
11151
|
+
if (span === undefined) {
|
|
11152
|
+
throw new Error(`Mutation source row ID is missing: ${String(rowIndex)}`);
|
|
11153
|
+
}
|
|
11154
|
+
const count = Math.min(remaining, span.rowStart + span.rowCount - rowIndex);
|
|
11155
|
+
appendRowIdSpan(this.#rowIdSpans, outputRow, span.rowIdStart + BigInt(rowIndex - span.rowStart), count);
|
|
11156
|
+
outputRow += count;
|
|
11157
|
+
rowIndex += count;
|
|
11158
|
+
remaining -= count;
|
|
11159
|
+
}
|
|
11160
|
+
}
|
|
11161
|
+
}
|
|
11162
|
+
/** Appends `rowCount` consecutive row IDs from `rowId`, extending the last span when contiguous. */
|
|
11163
|
+
function appendRowIdSpan(spans, rowStart, rowId, rowCount) {
|
|
11164
|
+
const previous = spans[spans.length - 1];
|
|
11165
|
+
if (previous !== undefined &&
|
|
11166
|
+
previous.rowStart + previous.rowCount === rowStart &&
|
|
11167
|
+
previous.rowIdStart + BigInt(previous.rowCount) === rowId) {
|
|
11168
|
+
previous.rowCount += rowCount;
|
|
11169
|
+
}
|
|
11170
|
+
else {
|
|
11171
|
+
spans.push({ rowStart, rowCount, rowIdStart: rowId });
|
|
11172
|
+
}
|
|
11173
|
+
}
|
|
11174
|
+
/** Appends `rowCount` output rows read from one source block, extending the last range when contiguous. */
|
|
11175
|
+
function appendMergeOutputRange(ranges, outputRowStart, sourceBlockId, sourceRowStart, rowCount) {
|
|
11176
|
+
const previous = ranges[ranges.length - 1];
|
|
11177
|
+
if (previous?.sourceBlockId === sourceBlockId &&
|
|
11178
|
+
previous.outputRowStart + previous.rowCount === outputRowStart &&
|
|
11179
|
+
previous.sourceRowStart + previous.rowCount === sourceRowStart) {
|
|
11180
|
+
previous.rowCount += rowCount;
|
|
11181
|
+
}
|
|
11182
|
+
else {
|
|
11183
|
+
ranges.push({ outputRowStart, sourceBlockId, sourceRowStart, rowCount });
|
|
11184
|
+
}
|
|
9578
11185
|
}
|
|
9579
11186
|
function mergeSourceAt(segment, columnId, rowIndex) {
|
|
9580
11187
|
const column = segment.columns.find((candidate) => candidate.columnId === columnId);
|
|
@@ -9584,12 +11191,6 @@ function mergeSourceAt(segment, columnId, rowIndex) {
|
|
|
9584
11191
|
}
|
|
9585
11192
|
return { blockId: block.blockId, sourceRowIndex: rowIndex - block.rowStart };
|
|
9586
11193
|
}
|
|
9587
|
-
function rowIdAt(spans, rowIndex) {
|
|
9588
|
-
const span = rowRangeAt(spans, rowIndex);
|
|
9589
|
-
if (span === undefined)
|
|
9590
|
-
throw new Error(`Mutation source row ID is missing: ${String(rowIndex)}`);
|
|
9591
|
-
return span.rowIdStart + BigInt(rowIndex - span.rowStart);
|
|
9592
|
-
}
|
|
9593
11194
|
function rowRangeAt(ranges, rowIndex) {
|
|
9594
11195
|
let low = 0;
|
|
9595
11196
|
let high = ranges.length - 1;
|
|
@@ -9610,17 +11211,6 @@ function rowRangeAt(ranges, rowIndex) {
|
|
|
9610
11211
|
}
|
|
9611
11212
|
return undefined;
|
|
9612
11213
|
}
|
|
9613
|
-
function appendRowIdSpan(spans, rowStart, rowId) {
|
|
9614
|
-
const previous = spans[spans.length - 1];
|
|
9615
|
-
if (previous !== undefined &&
|
|
9616
|
-
previous.rowStart + previous.rowCount === rowStart &&
|
|
9617
|
-
previous.rowIdStart + BigInt(previous.rowCount) === rowId) {
|
|
9618
|
-
spans[spans.length - 1] = { ...previous, rowCount: previous.rowCount + 1 };
|
|
9619
|
-
}
|
|
9620
|
-
else {
|
|
9621
|
-
spans.push({ rowStart, rowCount: 1, rowIdStart: rowId });
|
|
9622
|
-
}
|
|
9623
|
-
}
|
|
9624
11214
|
function rowIdSpanEnvelope(spans) {
|
|
9625
11215
|
if (spans.length === 0)
|
|
9626
11216
|
return { start: 0n, endExclusive: 0n };
|
|
@@ -9635,22 +11225,6 @@ function rowIdSpanEnvelope(spans) {
|
|
|
9635
11225
|
}
|
|
9636
11226
|
return { start, endExclusive };
|
|
9637
11227
|
}
|
|
9638
|
-
function appendMergeOutputRange(ranges, outputRowStart, source) {
|
|
9639
|
-
const previous = ranges[ranges.length - 1];
|
|
9640
|
-
if (previous?.sourceBlockId === source.blockId &&
|
|
9641
|
-
previous.outputRowStart + previous.rowCount === outputRowStart &&
|
|
9642
|
-
previous.sourceRowStart + previous.rowCount === source.sourceRowIndex) {
|
|
9643
|
-
ranges[ranges.length - 1] = { ...previous, rowCount: previous.rowCount + 1 };
|
|
9644
|
-
}
|
|
9645
|
-
else {
|
|
9646
|
-
ranges.push({
|
|
9647
|
-
outputRowStart,
|
|
9648
|
-
sourceBlockId: source.blockId,
|
|
9649
|
-
sourceRowStart: source.sourceRowIndex,
|
|
9650
|
-
rowCount: 1,
|
|
9651
|
-
});
|
|
9652
|
-
}
|
|
9653
|
-
}
|
|
9654
11228
|
function validatePhysicalTablePlan(table, plan) {
|
|
9655
11229
|
if (table.columns.length !== plan.columns.length ||
|
|
9656
11230
|
table.columns.some((column, index) => {
|
|
@@ -9779,12 +11353,108 @@ function physicalOutputBlockId(jobId, outputIndex, columnIndex) {
|
|
|
9779
11353
|
function physicalOutputBlockIds(jobId, plan) {
|
|
9780
11354
|
return plan.outputs.flatMap((_output, outputIndex) => plan.columns.map((_column, columnIndex) => physicalOutputBlockId(jobId, outputIndex, columnIndex)));
|
|
9781
11355
|
}
|
|
9782
|
-
function physicalOutputColumns(jobId, plan
|
|
11356
|
+
function physicalOutputColumns(jobId, plan, window = {
|
|
11357
|
+
rowStart: 0,
|
|
11358
|
+
rowCount: plan.totalRows,
|
|
11359
|
+
}) {
|
|
11360
|
+
const windowEnd = window.rowStart + window.rowCount;
|
|
9783
11361
|
return Object.fromEntries(plan.columns.map((column, columnIndex) => [
|
|
9784
11362
|
column.columnId,
|
|
9785
|
-
plan.outputs.
|
|
11363
|
+
plan.outputs.flatMap((output, outputIndex) => output.rowStart >= window.rowStart && output.rowStart + output.rowCount <= windowEnd
|
|
11364
|
+
? [physicalOutputBlockId(jobId, outputIndex, columnIndex)]
|
|
11365
|
+
: []),
|
|
9786
11366
|
]));
|
|
9787
11367
|
}
|
|
11368
|
+
/** The segment ID partition `index` of a partitioned merge publishes under. */
|
|
11369
|
+
function partitionOutputSegmentId(outputSegmentId, index) {
|
|
11370
|
+
return index === 0 ? outputSegmentId : `${outputSegmentId}/${String(index)}`;
|
|
11371
|
+
}
|
|
11372
|
+
/** Every segment a job publishes: one per output partition, or the single output segment. */
|
|
11373
|
+
function compactionOutputSegmentIds(job) {
|
|
11374
|
+
if (job.outputSegmentId === null)
|
|
11375
|
+
return [];
|
|
11376
|
+
const plan = job.rewritePlan;
|
|
11377
|
+
if ((plan?.kind !== "merge-v1" && plan?.kind !== "rechunk-v1") || plan.partitions === undefined) {
|
|
11378
|
+
return [job.outputSegmentId];
|
|
11379
|
+
}
|
|
11380
|
+
const outputSegmentId = job.outputSegmentId;
|
|
11381
|
+
return plan.partitions.map((_partition, index) => partitionOutputSegmentId(outputSegmentId, index));
|
|
11382
|
+
}
|
|
11383
|
+
/**
|
|
11384
|
+
* The segments a physical compaction publishes, with the blocks of the windows each covers.
|
|
11385
|
+
* A partitioned rewrite publishes one level-one segment per planned partition. A merge carries
|
|
11386
|
+
* the slice of its row-ID spans; a rechunk carries the corresponding contiguous interval.
|
|
11387
|
+
*/
|
|
11388
|
+
function compactionOutputSegments(table, job, plan, transactionId, createdAt) {
|
|
11389
|
+
const outputSegmentId = job.outputSegmentId;
|
|
11390
|
+
if (outputSegmentId === null)
|
|
11391
|
+
throw new Error("Compaction output segment ID is missing");
|
|
11392
|
+
const keyColumn = table.uniqueKeyColumnId === undefined ? {} : { keyColumnId: table.uniqueKeyColumnId };
|
|
11393
|
+
const partitionOrdinal = job.outputPartitionOrdinal === undefined
|
|
11394
|
+
? {}
|
|
11395
|
+
: { partitionOrdinal: job.outputPartitionOrdinal };
|
|
11396
|
+
if (plan.partitions !== undefined) {
|
|
11397
|
+
return plan.partitions.map((partition, index) => {
|
|
11398
|
+
const rowIdSpans = plan.kind === "merge-v1"
|
|
11399
|
+
? sliceRowIdSpans(plan.rowIdSpans, partition.rowStart, partition.rowCount)
|
|
11400
|
+
: undefined;
|
|
11401
|
+
const envelope = rowIdSpans === undefined
|
|
11402
|
+
? {
|
|
11403
|
+
start: plan.rowIdStart + BigInt(partition.rowStart),
|
|
11404
|
+
endExclusive: plan.rowIdStart + BigInt(partition.rowStart + partition.rowCount),
|
|
11405
|
+
}
|
|
11406
|
+
: rowIdSpanEnvelope(rowIdSpans);
|
|
11407
|
+
return {
|
|
11408
|
+
id: partitionOutputSegmentId(outputSegmentId, index),
|
|
11409
|
+
tableId: table.id,
|
|
11410
|
+
transactionId,
|
|
11411
|
+
rowCount: partition.rowCount,
|
|
11412
|
+
rowIdStart: envelope.start,
|
|
11413
|
+
rowIdEndExclusive: envelope.endExclusive,
|
|
11414
|
+
columnBlockIds: physicalOutputColumns(job.id, plan, partition),
|
|
11415
|
+
kind: plan.kind === "merge-v1" ? "base" : "insert",
|
|
11416
|
+
...keyColumn,
|
|
11417
|
+
level: job.targetLevel,
|
|
11418
|
+
...partitionOrdinal,
|
|
11419
|
+
logicalOrder: partition.logicalOrder,
|
|
11420
|
+
...(rowIdSpans === undefined ? {} : { rowIdSpans }),
|
|
11421
|
+
createdAt,
|
|
11422
|
+
};
|
|
11423
|
+
});
|
|
11424
|
+
}
|
|
11425
|
+
return [
|
|
11426
|
+
{
|
|
11427
|
+
id: outputSegmentId,
|
|
11428
|
+
tableId: table.id,
|
|
11429
|
+
transactionId,
|
|
11430
|
+
rowCount: plan.totalRows,
|
|
11431
|
+
rowIdStart: plan.rowIdStart,
|
|
11432
|
+
rowIdEndExclusive: plan.rowIdEndExclusive,
|
|
11433
|
+
columnBlockIds: physicalOutputColumns(job.id, plan),
|
|
11434
|
+
kind: plan.kind === "merge-v1" ? "base" : "insert",
|
|
11435
|
+
...keyColumn,
|
|
11436
|
+
level: job.targetLevel,
|
|
11437
|
+
...partitionOrdinal,
|
|
11438
|
+
logicalOrder: plan.logicalOrder,
|
|
11439
|
+
...(plan.kind === "merge-v1" ? { rowIdSpans: structuredClone(plan.rowIdSpans) } : {}),
|
|
11440
|
+
createdAt,
|
|
11441
|
+
},
|
|
11442
|
+
];
|
|
11443
|
+
}
|
|
11444
|
+
/** The spans of output rows `[rowStart, rowStart + rowCount)`, rebased to start at row zero. */
|
|
11445
|
+
function sliceRowIdSpans(spans, rowStart, rowCount) {
|
|
11446
|
+
const sliced = [];
|
|
11447
|
+
const rowEnd = rowStart + rowCount;
|
|
11448
|
+
for (const span of spans) {
|
|
11449
|
+
const spanEnd = span.rowStart + span.rowCount;
|
|
11450
|
+
if (spanEnd <= rowStart || span.rowStart >= rowEnd)
|
|
11451
|
+
continue;
|
|
11452
|
+
const start = Math.max(span.rowStart, rowStart);
|
|
11453
|
+
const end = Math.min(spanEnd, rowEnd);
|
|
11454
|
+
appendRowIdSpan(sliced, start - rowStart, span.rowIdStart + BigInt(start - span.rowStart), end - start);
|
|
11455
|
+
}
|
|
11456
|
+
return sliced;
|
|
11457
|
+
}
|
|
9788
11458
|
function isActiveCompactionState(state) {
|
|
9789
11459
|
return state === "planned" || state === "running" || state === "ready";
|
|
9790
11460
|
}
|
|
@@ -9889,6 +11559,9 @@ function garbageCollectionProgress(job) {
|
|
|
9889
11559
|
reclaimedBlockCount: job.reclaimedBlockCount,
|
|
9890
11560
|
retainedBlockCount: job.retainedBlockCount,
|
|
9891
11561
|
missingBlockCount: job.missingBlockCount,
|
|
11562
|
+
reclaimedTransactionCount: job.reclaimedTransactionCount,
|
|
11563
|
+
retainedTransactionCount: job.retainedTransactionCount,
|
|
11564
|
+
missingTransactionCount: job.missingTransactionCount,
|
|
9892
11565
|
physicallyReclaimedBytes: job.reclaimedBlockBytes,
|
|
9893
11566
|
}
|
|
9894
11567
|
: null;
|
|
@@ -9898,9 +11571,14 @@ function garbageCollectionProgress(job) {
|
|
|
9898
11571
|
examinedManifestCount: job.cursor.manifestIndex,
|
|
9899
11572
|
examinedSegmentCount: job.cursor.segmentIndex,
|
|
9900
11573
|
examinedBlockCount: job.cursor.blockIndex,
|
|
11574
|
+
examinedTransactionCount: job.cursor.transactionIndex,
|
|
9901
11575
|
result,
|
|
9902
11576
|
};
|
|
9903
11577
|
}
|
|
11578
|
+
/** Whether any of the table's triggers for these events fire AFTER the write, staging rows. */
|
|
11579
|
+
function firesAfterTriggers(table, ...events) {
|
|
11580
|
+
return (table.triggers ?? []).some((trigger) => trigger.timing === "after" && events.includes(trigger.event));
|
|
11581
|
+
}
|
|
9904
11582
|
/** Collects every real table name referenced by a block, its derived sources, or its subqueries. */
|
|
9905
11583
|
function collectRealTableNames(plan) {
|
|
9906
11584
|
const names = new Set();
|
|
@@ -10090,22 +11768,19 @@ function hasContiguousRowIds(segments) {
|
|
|
10090
11768
|
return previous === undefined || previous.rowIdEndExclusive === segment.rowIdStart;
|
|
10091
11769
|
});
|
|
10092
11770
|
}
|
|
11771
|
+
function validLogicalOrder(value) {
|
|
11772
|
+
return value !== undefined && Number.isFinite(value) && value >= 0;
|
|
11773
|
+
}
|
|
10093
11774
|
function appendLevelTwoLayout(segments) {
|
|
10094
|
-
|
|
11775
|
+
const levelOneSegments = levelOnePartitionPrefix(segments);
|
|
11776
|
+
if (levelOneSegments === null ||
|
|
11777
|
+
levelOneSegments.some((segment) => (segment.kind ?? "insert") !== "insert" || segment.rowIdSpans !== undefined)) {
|
|
11778
|
+
return null;
|
|
11779
|
+
}
|
|
11780
|
+
let index = levelOneSegments.length;
|
|
10095
11781
|
const retainedPrefix = [];
|
|
10096
11782
|
const levelTwoSegments = [];
|
|
10097
|
-
|
|
10098
|
-
if (first !== undefined && (first.level ?? 0) === 1) {
|
|
10099
|
-
if ((first.kind ?? "insert") !== "insert" ||
|
|
10100
|
-
first.rowIdSpans !== undefined ||
|
|
10101
|
-
first.partitionOrdinal !== undefined ||
|
|
10102
|
-
(first.logicalOrder !== undefined &&
|
|
10103
|
-
(!Number.isSafeInteger(first.logicalOrder) || first.logicalOrder < 0))) {
|
|
10104
|
-
return null;
|
|
10105
|
-
}
|
|
10106
|
-
retainedPrefix.push(first);
|
|
10107
|
-
index += 1;
|
|
10108
|
-
}
|
|
11783
|
+
retainedPrefix.push(...levelOneSegments);
|
|
10109
11784
|
for (;;) {
|
|
10110
11785
|
const segment = segments[index];
|
|
10111
11786
|
if (segment === undefined || (segment.level ?? 0) !== 2)
|
|
@@ -10113,8 +11788,7 @@ function appendLevelTwoLayout(segments) {
|
|
|
10113
11788
|
if ((segment.kind ?? "insert") !== "insert" ||
|
|
10114
11789
|
segment.rowIdSpans !== undefined ||
|
|
10115
11790
|
segment.partitionOrdinal !== levelTwoSegments.length ||
|
|
10116
|
-
!
|
|
10117
|
-
(segment.logicalOrder ?? -1) < 0 ||
|
|
11791
|
+
!validLogicalOrder(segment.logicalOrder) ||
|
|
10118
11792
|
segment.rowIdEndExclusive - segment.rowIdStart !== BigInt(segment.rowCount)) {
|
|
10119
11793
|
return null;
|
|
10120
11794
|
}
|
|
@@ -10127,8 +11801,7 @@ function appendLevelTwoLayout(segments) {
|
|
|
10127
11801
|
segment.partitionOrdinal !== undefined ||
|
|
10128
11802
|
(segment.kind ?? "insert") !== "insert" ||
|
|
10129
11803
|
segment.rowIdSpans !== undefined ||
|
|
10130
|
-
(segment.logicalOrder !== undefined &&
|
|
10131
|
-
(!Number.isSafeInteger(segment.logicalOrder) || segment.logicalOrder < 0)))) {
|
|
11804
|
+
(segment.logicalOrder !== undefined && !validLogicalOrder(segment.logicalOrder)))) {
|
|
10132
11805
|
return null;
|
|
10133
11806
|
}
|
|
10134
11807
|
const rowIdIntervals = segments
|
|
@@ -10148,46 +11821,67 @@ function appendLevelTwoLayout(segments) {
|
|
|
10148
11821
|
return { retainedPrefix, levelTwoSegments, level0Segments };
|
|
10149
11822
|
}
|
|
10150
11823
|
/**
|
|
10151
|
-
* Validates a keyed table's visible history for
|
|
10152
|
-
*
|
|
10153
|
-
*
|
|
10154
|
-
*
|
|
10155
|
-
*
|
|
10156
|
-
* footprint. Returns null when the shape does not hold so the planner skips explicitly.
|
|
11824
|
+
* Validates a keyed table's visible history for a partitioned level-one fold: a prefix of
|
|
11825
|
+
* level-one partitions — merged bases carrying row-ID spans, or append-shaped inserts — each
|
|
11826
|
+
* with an explicit logical order, strictly increasing along the prefix; then level-zero
|
|
11827
|
+
* segments of any mutation kind. Every row footprint must be pairwise disjoint. Returns null
|
|
11828
|
+
* when the shape does not hold so the planner skips explicitly.
|
|
10157
11829
|
*/
|
|
10158
|
-
function
|
|
10159
|
-
|
|
10160
|
-
|
|
10161
|
-
|
|
10162
|
-
|
|
10163
|
-
|
|
11830
|
+
function keyedLevelOneLayout(segments) {
|
|
11831
|
+
const partitions = levelOnePartitionPrefix(segments);
|
|
11832
|
+
if (partitions === null)
|
|
11833
|
+
return null;
|
|
11834
|
+
const level0Segments = segments.slice(partitions.length);
|
|
11835
|
+
if (level0Segments.some((segment) => (segment.level ?? 0) !== 0 || segment.partitionOrdinal !== undefined)) {
|
|
11836
|
+
return null;
|
|
11837
|
+
}
|
|
11838
|
+
if (!disjointRowIdFootprints(segments))
|
|
11839
|
+
return null;
|
|
11840
|
+
return { partitions, level0Segments };
|
|
11841
|
+
}
|
|
11842
|
+
/** The append-only counterpart: bounded L1 partitions followed by contiguous insert deltas. */
|
|
11843
|
+
function keylessLevelOneLayout(segments) {
|
|
11844
|
+
const partitions = levelOnePartitionPrefix(segments);
|
|
11845
|
+
if (partitions === null ||
|
|
11846
|
+
partitions.some((segment) => (segment.kind ?? "insert") !== "insert")) {
|
|
11847
|
+
return null;
|
|
11848
|
+
}
|
|
11849
|
+
const level0Segments = segments.slice(partitions.length);
|
|
11850
|
+
if (level0Segments.some((segment) => (segment.level ?? 0) !== 0 ||
|
|
11851
|
+
segment.partitionOrdinal !== undefined ||
|
|
11852
|
+
(segment.kind ?? "insert") !== "insert" ||
|
|
11853
|
+
segment.rowIdSpans !== undefined) ||
|
|
11854
|
+
!hasContiguousRowIds(segments)) {
|
|
11855
|
+
return null;
|
|
11856
|
+
}
|
|
11857
|
+
return { partitions, level0Segments };
|
|
11858
|
+
}
|
|
11859
|
+
/**
|
|
11860
|
+
* The leading level-one segments, when they form a valid partition prefix: insert or base
|
|
11861
|
+
* kinds, no L2 ordinal, and explicit strictly increasing logical orders. Null otherwise.
|
|
11862
|
+
*/
|
|
11863
|
+
function levelOnePartitionPrefix(segments) {
|
|
11864
|
+
const partitions = [];
|
|
11865
|
+
for (const segment of segments) {
|
|
11866
|
+
if ((segment.level ?? 0) !== 1)
|
|
10164
11867
|
break;
|
|
10165
11868
|
const kind = segment.kind ?? "insert";
|
|
10166
|
-
|
|
10167
|
-
|
|
10168
|
-
|
|
10169
|
-
(
|
|
10170
|
-
(
|
|
10171
|
-
(kind === "base" && (segment.rowIdSpans?.length ?? 0) === 0)) {
|
|
11869
|
+
const previousOrder = partitions[partitions.length - 1]?.logicalOrder ?? -1;
|
|
11870
|
+
if ((kind !== "insert" && kind !== "base") ||
|
|
11871
|
+
segment.partitionOrdinal !== undefined ||
|
|
11872
|
+
!validLogicalOrder(segment.logicalOrder) ||
|
|
11873
|
+
(segment.logicalOrder ?? -1) <= previousOrder) {
|
|
10172
11874
|
return null;
|
|
10173
11875
|
}
|
|
10174
|
-
|
|
10175
|
-
index += 1;
|
|
10176
|
-
}
|
|
10177
|
-
let anchor;
|
|
10178
|
-
const maybeAnchor = segments[index];
|
|
10179
|
-
if (maybeAnchor !== undefined && (maybeAnchor.level ?? 0) === 1) {
|
|
10180
|
-
const kind = maybeAnchor.kind ?? "insert";
|
|
10181
|
-
if ((kind !== "insert" && kind !== "base") || maybeAnchor.partitionOrdinal !== undefined) {
|
|
10182
|
-
return null;
|
|
10183
|
-
}
|
|
10184
|
-
anchor = maybeAnchor;
|
|
10185
|
-
index += 1;
|
|
10186
|
-
}
|
|
10187
|
-
const level0Segments = segments.slice(index);
|
|
10188
|
-
if (level0Segments.some((segment) => (segment.level ?? 0) !== 0 || segment.partitionOrdinal !== undefined)) {
|
|
10189
|
-
return null;
|
|
11876
|
+
partitions.push(segment);
|
|
10190
11877
|
}
|
|
11878
|
+
return partitions;
|
|
11879
|
+
}
|
|
11880
|
+
/**
|
|
11881
|
+
* Whether the segments' row footprints — spans where present, otherwise the contiguous
|
|
11882
|
+
* interval — are positive and pairwise disjoint. Update and delete deltas carry no footprint.
|
|
11883
|
+
*/
|
|
11884
|
+
function disjointRowIdFootprints(segments) {
|
|
10191
11885
|
const intervals = [];
|
|
10192
11886
|
for (const segment of segments) {
|
|
10193
11887
|
if (segment.rowIdSpans !== undefined) {
|
|
@@ -10199,18 +11893,56 @@ function keyedLevelTwoLayout(segments) {
|
|
|
10199
11893
|
if (segment.rowIdEndExclusive <= segment.rowIdStart)
|
|
10200
11894
|
continue;
|
|
10201
11895
|
if (segment.rowIdEndExclusive - segment.rowIdStart !== BigInt(segment.rowCount))
|
|
10202
|
-
return
|
|
11896
|
+
return false;
|
|
10203
11897
|
intervals.push({ start: segment.rowIdStart, end: segment.rowIdEndExclusive });
|
|
10204
11898
|
}
|
|
10205
11899
|
intervals.sort((left, right) => left.start < right.start ? -1 : left.start > right.start ? 1 : 0);
|
|
10206
11900
|
for (const [intervalIndex, interval] of intervals.entries()) {
|
|
10207
11901
|
if (interval.start <= 0n)
|
|
10208
|
-
return
|
|
11902
|
+
return false;
|
|
10209
11903
|
const previous = intervals[intervalIndex - 1];
|
|
10210
11904
|
if (previous !== undefined && interval.start < previous.end)
|
|
11905
|
+
return false;
|
|
11906
|
+
}
|
|
11907
|
+
return true;
|
|
11908
|
+
}
|
|
11909
|
+
/**
|
|
11910
|
+
* Validates a keyed table's visible history for multi-range L2 promotion: existing partitions
|
|
11911
|
+
* (append-shaped inserts or merged bases carrying row-ID spans) with ordinals exactly 0..N-1,
|
|
11912
|
+
* then the level-one partitions, then level-zero segments of any mutation kind. Every row
|
|
11913
|
+
* footprint — a partition's spans or interval, the anchors', and each level-zero insert/upsert
|
|
11914
|
+
* interval — must be pairwise disjoint; update and delete deltas carry no footprint. Returns
|
|
11915
|
+
* null when the shape does not hold so the planner skips explicitly.
|
|
11916
|
+
*/
|
|
11917
|
+
function keyedLevelTwoLayout(segments) {
|
|
11918
|
+
let index = 0;
|
|
11919
|
+
const levelTwoSegments = [];
|
|
11920
|
+
for (;;) {
|
|
11921
|
+
const segment = segments[index];
|
|
11922
|
+
if (segment === undefined || (segment.level ?? 0) !== 2)
|
|
11923
|
+
break;
|
|
11924
|
+
const kind = segment.kind ?? "insert";
|
|
11925
|
+
if (segment.partitionOrdinal !== levelTwoSegments.length ||
|
|
11926
|
+
!validLogicalOrder(segment.logicalOrder) ||
|
|
11927
|
+
(kind !== "insert" && kind !== "base") ||
|
|
11928
|
+
(kind === "insert" && segment.rowIdSpans !== undefined) ||
|
|
11929
|
+
(kind === "base" && (segment.rowIdSpans?.length ?? 0) === 0)) {
|
|
10211
11930
|
return null;
|
|
11931
|
+
}
|
|
11932
|
+
levelTwoSegments.push(segment);
|
|
11933
|
+
index += 1;
|
|
10212
11934
|
}
|
|
10213
|
-
|
|
11935
|
+
const anchors = levelOnePartitionPrefix(segments.slice(index));
|
|
11936
|
+
if (anchors === null)
|
|
11937
|
+
return null;
|
|
11938
|
+
index += anchors.length;
|
|
11939
|
+
const level0Segments = segments.slice(index);
|
|
11940
|
+
if (level0Segments.some((segment) => (segment.level ?? 0) !== 0 || segment.partitionOrdinal !== undefined)) {
|
|
11941
|
+
return null;
|
|
11942
|
+
}
|
|
11943
|
+
if (!disjointRowIdFootprints(segments))
|
|
11944
|
+
return null;
|
|
11945
|
+
return { levelTwoSegments, anchors, level0Segments };
|
|
10214
11946
|
}
|
|
10215
11947
|
function compactionWriteAmplificationSkipped(input) {
|
|
10216
11948
|
return {
|