@minnowdb/core 0.0.1 → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +27 -1
- package/dist/block-format/block.d.ts +46 -0
- package/dist/block-format/block.d.ts.map +1 -0
- package/dist/block-format/block.js +244 -0
- package/dist/block-format/block.js.map +1 -0
- package/dist/block-format/checksum.d.ts +7 -0
- package/dist/block-format/checksum.d.ts.map +1 -0
- package/dist/block-format/checksum.js +56 -0
- package/dist/block-format/checksum.js.map +1 -0
- package/dist/block-format/codecs.d.ts +24 -0
- package/dist/block-format/codecs.d.ts.map +1 -0
- package/dist/block-format/codecs.js +102 -0
- package/dist/block-format/codecs.js.map +1 -0
- package/dist/block-format/column.d.ts +8 -0
- package/dist/block-format/column.d.ts.map +1 -0
- package/dist/block-format/column.js +166 -0
- package/dist/block-format/column.js.map +1 -0
- package/dist/block-format/index.d.ts +7 -0
- package/dist/block-format/index.d.ts.map +1 -0
- package/dist/block-format/index.js +7 -0
- package/dist/block-format/index.js.map +1 -0
- package/dist/block-format/physical.d.ts +20 -0
- package/dist/block-format/physical.d.ts.map +1 -0
- package/dist/block-format/physical.js +439 -0
- package/dist/block-format/physical.js.map +1 -0
- package/dist/block-format/types.d.ts +93 -0
- package/dist/block-format/types.d.ts.map +1 -0
- package/dist/block-format/types.js +2 -0
- package/dist/block-format/types.js.map +1 -0
- package/dist/engine/artifact-cache.d.ts +22 -0
- package/dist/engine/artifact-cache.d.ts.map +1 -0
- package/dist/engine/artifact-cache.js +68 -0
- package/dist/engine/artifact-cache.js.map +1 -0
- package/dist/engine/batch.d.ts +28 -0
- package/dist/engine/batch.d.ts.map +1 -0
- package/dist/engine/batch.js +38 -0
- package/dist/engine/batch.js.map +1 -0
- package/dist/engine/buffered-writer.d.ts +43 -0
- package/dist/engine/buffered-writer.d.ts.map +1 -0
- package/dist/engine/buffered-writer.js +158 -0
- package/dist/engine/buffered-writer.js.map +1 -0
- package/dist/engine/catalog.d.ts +82 -0
- package/dist/engine/catalog.d.ts.map +1 -0
- package/dist/engine/catalog.js +47 -0
- package/dist/engine/catalog.js.map +1 -0
- package/dist/engine/client.d.ts +211 -0
- package/dist/engine/client.d.ts.map +1 -0
- package/dist/engine/client.js +538 -0
- package/dist/engine/client.js.map +1 -0
- package/dist/engine/coordinator.d.ts +17 -0
- package/dist/engine/coordinator.d.ts.map +1 -0
- package/dist/engine/coordinator.js +60 -0
- package/dist/engine/coordinator.js.map +1 -0
- package/dist/engine/database.d.ts +661 -0
- package/dist/engine/database.d.ts.map +1 -0
- package/dist/engine/database.js +10313 -0
- package/dist/engine/database.js.map +1 -0
- package/dist/engine/defaults.d.ts +33 -0
- package/dist/engine/defaults.d.ts.map +1 -0
- package/dist/engine/defaults.js +87 -0
- package/dist/engine/defaults.js.map +1 -0
- package/dist/engine/dsl/db.d.ts +137 -0
- package/dist/engine/dsl/db.d.ts.map +1 -0
- package/dist/engine/dsl/db.js +161 -0
- package/dist/engine/dsl/db.js.map +1 -0
- package/dist/engine/dsl/expression.d.ts +172 -0
- package/dist/engine/dsl/expression.d.ts.map +1 -0
- package/dist/engine/dsl/expression.js +397 -0
- package/dist/engine/dsl/expression.js.map +1 -0
- package/dist/engine/dsl/index.d.ts +8 -0
- package/dist/engine/dsl/index.d.ts.map +1 -0
- package/dist/engine/dsl/index.js +7 -0
- package/dist/engine/dsl/index.js.map +1 -0
- package/dist/engine/dsl/live-query.d.ts +45 -0
- package/dist/engine/dsl/live-query.d.ts.map +1 -0
- package/dist/engine/dsl/live-query.js +116 -0
- package/dist/engine/dsl/live-query.js.map +1 -0
- package/dist/engine/dsl/mutations.d.ts +123 -0
- package/dist/engine/dsl/mutations.d.ts.map +1 -0
- package/dist/engine/dsl/mutations.js +241 -0
- package/dist/engine/dsl/mutations.js.map +1 -0
- package/dist/engine/dsl/select-query-builder.d.ts +148 -0
- package/dist/engine/dsl/select-query-builder.d.ts.map +1 -0
- package/dist/engine/dsl/select-query-builder.js +377 -0
- package/dist/engine/dsl/select-query-builder.js.map +1 -0
- package/dist/engine/dsl/sql-tag.d.ts +46 -0
- package/dist/engine/dsl/sql-tag.d.ts.map +1 -0
- package/dist/engine/dsl/sql-tag.js +85 -0
- package/dist/engine/dsl/sql-tag.js.map +1 -0
- package/dist/engine/dsl/types.d.ts +133 -0
- package/dist/engine/dsl/types.d.ts.map +1 -0
- package/dist/engine/dsl/types.js +9 -0
- package/dist/engine/dsl/types.js.map +1 -0
- package/dist/engine/errors.d.ts +55 -0
- package/dist/engine/errors.d.ts.map +1 -0
- package/dist/engine/errors.js +81 -0
- package/dist/engine/errors.js.map +1 -0
- package/dist/engine/fts.d.ts +93 -0
- package/dist/engine/fts.d.ts.map +1 -0
- package/dist/engine/fts.js +273 -0
- package/dist/engine/fts.js.map +1 -0
- package/dist/engine/group-index.d.ts +32 -0
- package/dist/engine/group-index.d.ts.map +1 -0
- package/dist/engine/group-index.js +366 -0
- package/dist/engine/group-index.js.map +1 -0
- package/dist/engine/index.d.ts +13 -0
- package/dist/engine/index.d.ts.map +1 -0
- package/dist/engine/index.js +12 -0
- package/dist/engine/index.js.map +1 -0
- package/dist/engine/join-index.d.ts +12 -0
- package/dist/engine/join-index.d.ts.map +1 -0
- package/dist/engine/join-index.js +217 -0
- package/dist/engine/join-index.js.map +1 -0
- package/dist/engine/live.d.ts +86 -0
- package/dist/engine/live.d.ts.map +1 -0
- package/dist/engine/live.js +289 -0
- package/dist/engine/live.js.map +1 -0
- package/dist/engine/memory.d.ts +37 -0
- package/dist/engine/memory.d.ts.map +1 -0
- package/dist/engine/memory.js +119 -0
- package/dist/engine/memory.js.map +1 -0
- package/dist/engine/optimizer.d.ts +20 -0
- package/dist/engine/optimizer.d.ts.map +1 -0
- package/dist/engine/optimizer.js +1170 -0
- package/dist/engine/optimizer.js.map +1 -0
- package/dist/engine/query-cache.d.ts +10 -0
- package/dist/engine/query-cache.d.ts.map +1 -0
- package/dist/engine/query-cache.js +58 -0
- package/dist/engine/query-cache.js.map +1 -0
- package/dist/engine/query.d.ts +844 -0
- package/dist/engine/query.d.ts.map +1 -0
- package/dist/engine/query.js +7450 -0
- package/dist/engine/query.js.map +1 -0
- package/dist/engine/schema-wire.d.ts +92 -0
- package/dist/engine/schema-wire.d.ts.map +1 -0
- package/dist/engine/schema-wire.js +115 -0
- package/dist/engine/schema-wire.js.map +1 -0
- package/dist/engine/schema.d.ts +394 -0
- package/dist/engine/schema.d.ts.map +1 -0
- package/dist/engine/schema.js +709 -0
- package/dist/engine/schema.js.map +1 -0
- package/dist/engine/sort-keys.d.ts +36 -0
- package/dist/engine/sort-keys.d.ts.map +1 -0
- package/dist/engine/sort-keys.js +60 -0
- package/dist/engine/sort-keys.js.map +1 -0
- package/dist/engine/sql-json.d.ts +30 -0
- package/dist/engine/sql-json.d.ts.map +1 -0
- package/dist/engine/sql-json.js +120 -0
- package/dist/engine/sql-json.js.map +1 -0
- package/dist/engine/sql-semantics.d.ts +31 -0
- package/dist/engine/sql-semantics.d.ts.map +1 -0
- package/dist/engine/sql-semantics.js +121 -0
- package/dist/engine/sql-semantics.js.map +1 -0
- package/dist/engine/vector.d.ts +87 -0
- package/dist/engine/vector.d.ts.map +1 -0
- package/dist/engine/vector.js +4227 -0
- package/dist/engine/vector.js.map +1 -0
- package/dist/engine/worker-host.d.ts +48 -0
- package/dist/engine/worker-host.d.ts.map +1 -0
- package/dist/engine/worker-host.js +501 -0
- package/dist/engine/worker-host.js.map +1 -0
- package/dist/engine/worker.d.ts +2 -0
- package/dist/engine/worker.d.ts.map +1 -0
- package/dist/engine/worker.js +14 -0
- package/dist/engine/worker.js.map +1 -0
- package/dist/index.d.ts +2 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +2 -0
- package/dist/index.js.map +1 -0
- package/dist/plan/index.d.ts +17 -0
- package/dist/plan/index.d.ts.map +1 -0
- package/dist/plan/index.js +17 -0
- package/dist/plan/index.js.map +1 -0
- package/dist/storage/fixture-shape.d.ts +42 -0
- package/dist/storage/fixture-shape.d.ts.map +1 -0
- package/dist/storage/fixture-shape.js +146 -0
- package/dist/storage/fixture-shape.js.map +1 -0
- package/dist/storage/index.d.ts +5 -0
- package/dist/storage/index.d.ts.map +1 -0
- package/dist/storage/index.js +5 -0
- package/dist/storage/index.js.map +1 -0
- package/dist/storage/indexeddb.d.ts +139 -0
- package/dist/storage/indexeddb.d.ts.map +1 -0
- package/dist/storage/indexeddb.js +3161 -0
- package/dist/storage/indexeddb.js.map +1 -0
- package/dist/storage/memory.d.ts +121 -0
- package/dist/storage/memory.d.ts.map +1 -0
- package/dist/storage/memory.js +1455 -0
- package/dist/storage/memory.js.map +1 -0
- package/dist/storage/snapshot.d.ts +108 -0
- package/dist/storage/snapshot.d.ts.map +1 -0
- package/dist/storage/snapshot.js +276 -0
- package/dist/storage/snapshot.js.map +1 -0
- package/dist/storage/types.d.ts +895 -0
- package/dist/storage/types.d.ts.map +1 -0
- package/dist/storage/types.js +1549 -0
- package/dist/storage/types.js.map +1 -0
- package/dist/testing/index.d.ts +76 -0
- package/dist/testing/index.d.ts.map +1 -0
- package/dist/testing/index.js +233 -0
- package/dist/testing/index.js.map +1 -0
- package/dist/testing/seeds.d.ts +11 -0
- package/dist/testing/seeds.d.ts.map +1 -0
- package/dist/testing/seeds.js +50 -0
- package/dist/testing/seeds.js.map +1 -0
- package/dist/transactions/index.d.ts +148 -0
- package/dist/transactions/index.d.ts.map +1 -0
- package/dist/transactions/index.js +662 -0
- package/dist/transactions/index.js.map +1 -0
- package/dist/worker-protocol/index.d.ts +84 -0
- package/dist/worker-protocol/index.d.ts.map +1 -0
- package/dist/worker-protocol/index.js +117 -0
- package/dist/worker-protocol/index.js.map +1 -0
- package/package.json +66 -16
- package/sql-feature-matrix.json +1275 -0
- package/index.d.ts +0 -1
- package/index.js +0 -1
|
@@ -0,0 +1,1549 @@
|
|
|
1
|
+
export const storeNames = [
|
|
2
|
+
"catalog",
|
|
3
|
+
"manifests",
|
|
4
|
+
"segments",
|
|
5
|
+
"blocks",
|
|
6
|
+
"transactions",
|
|
7
|
+
"leases",
|
|
8
|
+
"statistics",
|
|
9
|
+
"temp",
|
|
10
|
+
"gc",
|
|
11
|
+
];
|
|
12
|
+
/** Every this-many commits the store writes a full checkpoint instead of a delta. */
|
|
13
|
+
export const MANIFEST_CHECKPOINT_INTERVAL = 32;
|
|
14
|
+
/** Applies one stored record to a running block set (checkpoint replaces, delta mutates). */
|
|
15
|
+
export function applyManifestRecord(blockIds, record) {
|
|
16
|
+
if (record.blockIds !== undefined) {
|
|
17
|
+
blockIds.clear();
|
|
18
|
+
for (const id of record.blockIds)
|
|
19
|
+
blockIds.add(id);
|
|
20
|
+
return;
|
|
21
|
+
}
|
|
22
|
+
for (const id of record.removedBlockIds ?? [])
|
|
23
|
+
blockIds.delete(id);
|
|
24
|
+
for (const id of record.addedBlockIds ?? [])
|
|
25
|
+
blockIds.add(id);
|
|
26
|
+
}
|
|
27
|
+
export const simpleDataTypes = ["boolean", "number", "string", "datetime"];
|
|
28
|
+
/**
|
|
29
|
+
* The single authority on which enum declarations are legal, shared by the schema DSL's
|
|
30
|
+
* `column.enum()` and the engine's `createTable`: at least one value, every value a non-empty
|
|
31
|
+
* string, no duplicates. Returns a defensive copy.
|
|
32
|
+
*/
|
|
33
|
+
export function validateEnumValues(values, context) {
|
|
34
|
+
if (values.length === 0) {
|
|
35
|
+
throw new TypeError(`An enum needs at least one value: ${context}`);
|
|
36
|
+
}
|
|
37
|
+
const seen = new Set();
|
|
38
|
+
for (const value of values) {
|
|
39
|
+
if (typeof value !== "string" || value.length === 0) {
|
|
40
|
+
throw new TypeError(`Enum values must be non-empty strings: ${context}`);
|
|
41
|
+
}
|
|
42
|
+
if (seen.has(value)) {
|
|
43
|
+
throw new TypeError(`Duplicate enum value: ${context} has "${value}" twice`);
|
|
44
|
+
}
|
|
45
|
+
seen.add(value);
|
|
46
|
+
}
|
|
47
|
+
return [...values];
|
|
48
|
+
}
|
|
49
|
+
/**
|
|
50
|
+
* The single authority on which default declarations are legal, shared by the schema DSL's
|
|
51
|
+
* `table()` and the engine's `createTable` so the two entry points (and the wire path between
|
|
52
|
+
* them) can never drift: defaults require non-nullable columns, "now" is datetime-only,
|
|
53
|
+
* auto-increment is the number unique key, and the unique key never defaults to a constant.
|
|
54
|
+
*/
|
|
55
|
+
export function validateColumnDefault(column, defaultValue) {
|
|
56
|
+
if (column.nullable) {
|
|
57
|
+
throw new TypeError(`Defaults require a non-nullable column: ${column.name}`);
|
|
58
|
+
}
|
|
59
|
+
switch (defaultValue.kind) {
|
|
60
|
+
case "now":
|
|
61
|
+
if (column.type !== "datetime") {
|
|
62
|
+
throw new TypeError(`Default now requires a datetime column: ${column.name}`);
|
|
63
|
+
}
|
|
64
|
+
return;
|
|
65
|
+
case "autoincrement":
|
|
66
|
+
if (column.type !== "number") {
|
|
67
|
+
throw new TypeError(`Auto-increment requires a number column: ${column.name}`);
|
|
68
|
+
}
|
|
69
|
+
if (!column.isUniqueKey) {
|
|
70
|
+
throw new TypeError(`Auto-increment requires the unique key column: ${column.name}`);
|
|
71
|
+
}
|
|
72
|
+
return;
|
|
73
|
+
case "literal": {
|
|
74
|
+
const value = defaultValue.value;
|
|
75
|
+
if (column.type === "datetime") {
|
|
76
|
+
throw new TypeError(`Datetime columns default with now, not a literal: ${column.name}`);
|
|
77
|
+
}
|
|
78
|
+
if (typeof value !== column.type) {
|
|
79
|
+
throw new TypeError(`Default literal must be a ${column.type}: ${column.name}`);
|
|
80
|
+
}
|
|
81
|
+
if (typeof value === "number" && !Number.isFinite(value)) {
|
|
82
|
+
throw new TypeError(`Default literal must be finite: ${column.name}`);
|
|
83
|
+
}
|
|
84
|
+
if (column.enumValues !== undefined &&
|
|
85
|
+
typeof value === "string" &&
|
|
86
|
+
!column.enumValues.includes(value)) {
|
|
87
|
+
throw new TypeError(`Default must be one of the enum values: ${column.name}`);
|
|
88
|
+
}
|
|
89
|
+
if (column.isUniqueKey) {
|
|
90
|
+
throw new TypeError(`Unique key cannot default to a constant: ${column.name}`);
|
|
91
|
+
}
|
|
92
|
+
return;
|
|
93
|
+
}
|
|
94
|
+
// The wire path can hand this untyped data (including specs from removed generator kinds).
|
|
95
|
+
default:
|
|
96
|
+
throw new TypeError(`Unknown default kind: ${String(defaultValue.kind)}`);
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
export class TableRecordConflictError extends Error {
|
|
100
|
+
tableId;
|
|
101
|
+
expectedRevision;
|
|
102
|
+
actualRevision;
|
|
103
|
+
name = "TableRecordConflictError";
|
|
104
|
+
constructor(tableId, expectedRevision, actualRevision) {
|
|
105
|
+
super(`Table ${tableId} changed: expected revision ${String(expectedRevision)}, found ${String(actualRevision)}`);
|
|
106
|
+
this.tableId = tableId;
|
|
107
|
+
this.expectedRevision = expectedRevision;
|
|
108
|
+
this.actualRevision = actualRevision;
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
export const compactionJobStates = [
|
|
112
|
+
"planned",
|
|
113
|
+
"running",
|
|
114
|
+
"ready",
|
|
115
|
+
"published",
|
|
116
|
+
"cancelled",
|
|
117
|
+
"aborted",
|
|
118
|
+
];
|
|
119
|
+
export const compactionRewritePlanKinds = ["copy-v1", "rechunk-v1", "merge-v1"];
|
|
120
|
+
export const compactionOutputCompressions = ["raw", "gzip"];
|
|
121
|
+
export class CompactionJobConflictError extends Error {
|
|
122
|
+
jobId;
|
|
123
|
+
expectedRevision;
|
|
124
|
+
actualRevision;
|
|
125
|
+
name = "CompactionJobConflictError";
|
|
126
|
+
constructor(jobId, expectedRevision, actualRevision) {
|
|
127
|
+
super(`Compaction job ${jobId} changed: expected revision ${String(expectedRevision)}, found ${String(actualRevision)}`);
|
|
128
|
+
this.jobId = jobId;
|
|
129
|
+
this.expectedRevision = expectedRevision;
|
|
130
|
+
this.actualRevision = actualRevision;
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
export const garbageCollectionJobStates = ["planned", "running", "completed"];
|
|
134
|
+
export class GarbageCollectionJobConflictError extends Error {
|
|
135
|
+
jobId;
|
|
136
|
+
expectedRevision;
|
|
137
|
+
actualRevision;
|
|
138
|
+
name = "GarbageCollectionJobConflictError";
|
|
139
|
+
constructor(jobId, expectedRevision, actualRevision) {
|
|
140
|
+
super(`Garbage collection job ${jobId} changed: expected revision ${String(expectedRevision)}, found ${String(actualRevision)}`);
|
|
141
|
+
this.jobId = jobId;
|
|
142
|
+
this.expectedRevision = expectedRevision;
|
|
143
|
+
this.actualRevision = actualRevision;
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
export class SnapshotManifestMissingError extends Error {
|
|
147
|
+
version;
|
|
148
|
+
name = "SnapshotManifestMissingError";
|
|
149
|
+
constructor(version) {
|
|
150
|
+
super(`Snapshot manifest is unavailable: ${String(version)}`);
|
|
151
|
+
this.version = version;
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
export class LeaseConflictError extends Error {
|
|
155
|
+
leaseId;
|
|
156
|
+
expectedRevision;
|
|
157
|
+
actualRevision;
|
|
158
|
+
name = "LeaseConflictError";
|
|
159
|
+
constructor(leaseId, expectedRevision, actualRevision) {
|
|
160
|
+
super(`Lease ${leaseId} changed: expected revision ${String(expectedRevision)}, found ${String(actualRevision)}`);
|
|
161
|
+
this.leaseId = leaseId;
|
|
162
|
+
this.expectedRevision = expectedRevision;
|
|
163
|
+
this.actualRevision = actualRevision;
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
/**
|
|
167
|
+
* Shared candidate-merge core for both stores: fetching chunks is store-specific, but the
|
|
168
|
+
* term-match rule (exact, or prefix as a term range) and the sorted-unique row-id shape must
|
|
169
|
+
* never drift between backends — pruning would silently differ per store.
|
|
170
|
+
*/
|
|
171
|
+
export function collectFtsCandidates(chunkLists, terms) {
|
|
172
|
+
const sets = terms.map(() => new Set());
|
|
173
|
+
for (const postings of chunkLists) {
|
|
174
|
+
for (const posting of postings) {
|
|
175
|
+
for (let index = 0; index < terms.length; index += 1) {
|
|
176
|
+
const term = terms[index];
|
|
177
|
+
if (term === undefined)
|
|
178
|
+
continue;
|
|
179
|
+
const matches = term.prefix
|
|
180
|
+
? posting.term.startsWith(term.term)
|
|
181
|
+
: posting.term === term.term;
|
|
182
|
+
if (!matches)
|
|
183
|
+
continue;
|
|
184
|
+
const set = sets[index];
|
|
185
|
+
if (set !== undefined)
|
|
186
|
+
for (const rowId of posting.rowIds)
|
|
187
|
+
set.add(rowId);
|
|
188
|
+
}
|
|
189
|
+
}
|
|
190
|
+
}
|
|
191
|
+
return {
|
|
192
|
+
rowIdsByTerm: sets.map((set) => [...set].sort((left, right) => (left < right ? -1 : left > right ? 1 : 0))),
|
|
193
|
+
};
|
|
194
|
+
}
|
|
195
|
+
/**
|
|
196
|
+
* Shared stale-writer policy for both stores' commit steps: a commit that adds segments to a
|
|
197
|
+
* table with active full-text columns but no covering delta flips the uncovered columns to
|
|
198
|
+
* "invalid" (the index self-heals through a rebuild; the data commit itself always proceeds).
|
|
199
|
+
* Returns the updated record, or undefined when nothing changes.
|
|
200
|
+
*/
|
|
201
|
+
export function invalidateUncoveredFtsColumns(record, coveredColumnIds) {
|
|
202
|
+
const ftsColumns = record.ftsColumns;
|
|
203
|
+
if (ftsColumns === undefined)
|
|
204
|
+
return undefined;
|
|
205
|
+
let invalidated = false;
|
|
206
|
+
const next = {};
|
|
207
|
+
for (const [columnId, state] of Object.entries(ftsColumns)) {
|
|
208
|
+
if (state.state !== "invalid" && !coveredColumnIds.has(columnId)) {
|
|
209
|
+
next[columnId] = { ...state, state: "invalid" };
|
|
210
|
+
invalidated = true;
|
|
211
|
+
}
|
|
212
|
+
else {
|
|
213
|
+
next[columnId] = { ...state };
|
|
214
|
+
}
|
|
215
|
+
}
|
|
216
|
+
if (!invalidated)
|
|
217
|
+
return undefined;
|
|
218
|
+
return { ...record, ftsColumns: next, revision: (record.revision ?? 0) + 1 };
|
|
219
|
+
}
|
|
220
|
+
export class UniqueKeyConflictError extends Error {
|
|
221
|
+
tableId;
|
|
222
|
+
keyToken;
|
|
223
|
+
name = "UniqueKeyConflictError";
|
|
224
|
+
constructor(tableId, keyToken) {
|
|
225
|
+
super(`Unique key already exists in table ${tableId}`);
|
|
226
|
+
this.tableId = tableId;
|
|
227
|
+
this.keyToken = keyToken;
|
|
228
|
+
}
|
|
229
|
+
}
|
|
230
|
+
export class WriteConflictError extends Error {
|
|
231
|
+
expectedVersion;
|
|
232
|
+
actualVersion;
|
|
233
|
+
name = "WriteConflictError";
|
|
234
|
+
constructor(expectedVersion, actualVersion) {
|
|
235
|
+
super(`Manifest changed: expected ${String(expectedVersion)}, found ${String(actualVersion)}`);
|
|
236
|
+
this.expectedVersion = expectedVersion;
|
|
237
|
+
this.actualVersion = actualVersion;
|
|
238
|
+
}
|
|
239
|
+
}
|
|
240
|
+
export class TransactionRecordConflictError extends Error {
|
|
241
|
+
transactionId;
|
|
242
|
+
expectedRevision;
|
|
243
|
+
actualRevision;
|
|
244
|
+
name = "TransactionRecordConflictError";
|
|
245
|
+
constructor(transactionId, expectedRevision, actualRevision) {
|
|
246
|
+
super(`Transaction ${transactionId} changed: expected revision ${String(expectedRevision)}, found ${String(actualRevision)}`);
|
|
247
|
+
this.transactionId = transactionId;
|
|
248
|
+
this.expectedRevision = expectedRevision;
|
|
249
|
+
this.actualRevision = actualRevision;
|
|
250
|
+
}
|
|
251
|
+
}
|
|
252
|
+
export class TempOwnerConflictError extends Error {
|
|
253
|
+
ownerId;
|
|
254
|
+
expectedRevision;
|
|
255
|
+
actualRevision;
|
|
256
|
+
name = "TempOwnerConflictError";
|
|
257
|
+
constructor(ownerId, expectedRevision, actualRevision) {
|
|
258
|
+
super(`Temp owner ${ownerId} changed: expected revision ${String(expectedRevision)}, found ${String(actualRevision)}`);
|
|
259
|
+
this.ownerId = ownerId;
|
|
260
|
+
this.expectedRevision = expectedRevision;
|
|
261
|
+
this.actualRevision = actualRevision;
|
|
262
|
+
}
|
|
263
|
+
}
|
|
264
|
+
export function createManifest(input) {
|
|
265
|
+
return {
|
|
266
|
+
version: input.expectedVersion === null ? 0 : input.expectedVersion + 1,
|
|
267
|
+
previousVersion: input.expectedVersion,
|
|
268
|
+
blockIds: [...new Set(input.blockIds)].sort(),
|
|
269
|
+
createdAt: input.createdAt ?? new Date().toISOString(),
|
|
270
|
+
...(input.changedTableIds === undefined ? {} : { changedTableIds: [...input.changedTableIds] }),
|
|
271
|
+
};
|
|
272
|
+
}
|
|
273
|
+
/**
|
|
274
|
+
* Normalizes the additive L2 partition metadata while preserving legacy segment records verbatim.
|
|
275
|
+
*/
|
|
276
|
+
export function normalizeSegmentRecord(record) {
|
|
277
|
+
if (record.partitionOrdinal === undefined)
|
|
278
|
+
return structuredClone(record);
|
|
279
|
+
const partitionOrdinal = nonNegativeWholeNumber(record.partitionOrdinal, "Segment partition ordinal");
|
|
280
|
+
if (record.level !== 2) {
|
|
281
|
+
throw new TypeError("A partitioned segment must have explicit level two");
|
|
282
|
+
}
|
|
283
|
+
const kind = record.kind ?? "insert";
|
|
284
|
+
if (kind !== "insert" && kind !== "base") {
|
|
285
|
+
throw new TypeError("A partitioned segment must be an insert or a merged base");
|
|
286
|
+
}
|
|
287
|
+
if (record.logicalOrder === undefined) {
|
|
288
|
+
throw new TypeError("A partitioned segment requires an explicit logical order");
|
|
289
|
+
}
|
|
290
|
+
nonNegativeWholeNumber(record.logicalOrder, "Segment logical order");
|
|
291
|
+
const rowCount = positiveWholeNumber(record.rowCount, "Segment row count");
|
|
292
|
+
if (kind === "insert") {
|
|
293
|
+
// Append-row-range partition: one contiguous positive row-ID interval, no spans.
|
|
294
|
+
if (record.rowIdSpans !== undefined) {
|
|
295
|
+
throw new TypeError("A partitioned segment cannot contain row ID spans");
|
|
296
|
+
}
|
|
297
|
+
if (typeof record.rowIdStart !== "bigint" || record.rowIdStart <= 0n) {
|
|
298
|
+
throw new RangeError("Segment row ID start must be a positive bigint");
|
|
299
|
+
}
|
|
300
|
+
if (typeof record.rowIdEndExclusive !== "bigint" ||
|
|
301
|
+
record.rowIdEndExclusive !== record.rowIdStart + BigInt(rowCount)) {
|
|
302
|
+
throw new RangeError("A partitioned segment must have a contiguous positive row ID envelope");
|
|
303
|
+
}
|
|
304
|
+
return structuredClone({ ...record, partitionOrdinal });
|
|
305
|
+
}
|
|
306
|
+
// Keyed multi-range partition: a merged full-row base whose live rows keep their original
|
|
307
|
+
// ids, described by positive, sorted, non-overlapping spans that sum to the row count.
|
|
308
|
+
if (record.rowIdSpans === undefined || record.rowIdSpans.length === 0) {
|
|
309
|
+
throw new TypeError("A merged partitioned segment requires row ID spans");
|
|
310
|
+
}
|
|
311
|
+
let spanRows = 0;
|
|
312
|
+
let previousEnd = 0n;
|
|
313
|
+
for (const span of record.rowIdSpans) {
|
|
314
|
+
if (typeof span.rowIdStart !== "bigint" ||
|
|
315
|
+
span.rowIdStart <= 0n ||
|
|
316
|
+
!Number.isSafeInteger(span.rowCount) ||
|
|
317
|
+
span.rowCount <= 0) {
|
|
318
|
+
throw new RangeError("A partitioned segment span must be a positive non-empty interval");
|
|
319
|
+
}
|
|
320
|
+
if (span.rowIdStart < previousEnd) {
|
|
321
|
+
throw new RangeError("Partitioned segment spans must be sorted and non-overlapping");
|
|
322
|
+
}
|
|
323
|
+
previousEnd = span.rowIdStart + BigInt(span.rowCount);
|
|
324
|
+
spanRows += span.rowCount;
|
|
325
|
+
}
|
|
326
|
+
if (spanRows !== rowCount) {
|
|
327
|
+
throw new RangeError("Partitioned segment spans must cover exactly the row count");
|
|
328
|
+
}
|
|
329
|
+
return structuredClone({ ...record, partitionOrdinal });
|
|
330
|
+
}
|
|
331
|
+
export function updateTransactionRecord(record, update) {
|
|
332
|
+
return {
|
|
333
|
+
...record,
|
|
334
|
+
...(update.snapshotVersion === undefined ? {} : { snapshotVersion: update.snapshotVersion }),
|
|
335
|
+
...(update.pendingBlockIds === undefined
|
|
336
|
+
? {}
|
|
337
|
+
: { pendingBlockIds: [...new Set(update.pendingBlockIds)].sort() }),
|
|
338
|
+
...(update.pendingSegmentIds === undefined
|
|
339
|
+
? {}
|
|
340
|
+
: { pendingSegmentIds: [...new Set(update.pendingSegmentIds)].sort() }),
|
|
341
|
+
...(update.status === undefined ? {} : { status: update.status }),
|
|
342
|
+
...(update.committedVersion === undefined ? {} : { committedVersion: update.committedVersion }),
|
|
343
|
+
updatedAt: update.updatedAt,
|
|
344
|
+
revision: record.revision + 1,
|
|
345
|
+
};
|
|
346
|
+
}
|
|
347
|
+
export function createGarbageCollectionJobRecord(input) {
|
|
348
|
+
const candidateManifestVersions = uniqueWholeNumbers(input.candidateManifestVersions, "Garbage collection candidate manifest version");
|
|
349
|
+
const candidateSegmentIds = uniqueIds(input.candidateSegmentIds, "Garbage collection candidate segment ID", true);
|
|
350
|
+
const candidateBlockIds = uniqueIds(input.candidateBlockIds, "Garbage collection candidate block ID", true);
|
|
351
|
+
const createdAt = validTimestamp(input.createdAt, "Garbage collection creation timestamp");
|
|
352
|
+
const complete = candidateManifestVersions.length === 0 &&
|
|
353
|
+
candidateSegmentIds.length === 0 &&
|
|
354
|
+
candidateBlockIds.length === 0;
|
|
355
|
+
return {
|
|
356
|
+
id: nonEmptyString(input.id, "Garbage collection job ID"),
|
|
357
|
+
candidateManifestVersions,
|
|
358
|
+
candidateSegmentIds,
|
|
359
|
+
candidateBlockIds,
|
|
360
|
+
cursor: { manifestIndex: 0, segmentIndex: 0, blockIndex: 0 },
|
|
361
|
+
prunedManifestCount: 0,
|
|
362
|
+
alreadyPrunedManifestCount: 0,
|
|
363
|
+
retainedManifestCount: 0,
|
|
364
|
+
missingManifestCount: 0,
|
|
365
|
+
reclaimedSegmentCount: 0,
|
|
366
|
+
retainedSegmentCount: 0,
|
|
367
|
+
missingSegmentCount: 0,
|
|
368
|
+
reclaimedBlockCount: 0,
|
|
369
|
+
retainedBlockCount: 0,
|
|
370
|
+
missingBlockCount: 0,
|
|
371
|
+
reclaimedBlockBytes: 0,
|
|
372
|
+
state: complete ? "completed" : "planned",
|
|
373
|
+
revision: 0,
|
|
374
|
+
leaseCutoff: validTimestamp(input.leaseCutoff, "Garbage collection lease cutoff"),
|
|
375
|
+
createdAt,
|
|
376
|
+
updatedAt: createdAt,
|
|
377
|
+
};
|
|
378
|
+
}
|
|
379
|
+
export function normalizeGarbageCollectionJobRecord(record) {
|
|
380
|
+
const candidateManifestVersions = uniqueWholeNumbers(record.candidateManifestVersions, "Garbage collection candidate manifest version");
|
|
381
|
+
const candidateSegmentIds = uniqueIds(record.candidateSegmentIds, "Garbage collection candidate segment ID", true);
|
|
382
|
+
const candidateBlockIds = uniqueIds(record.candidateBlockIds, "Garbage collection candidate block ID", true);
|
|
383
|
+
const cursor = normalizeGarbageCollectionCursor(record.cursor);
|
|
384
|
+
const normalized = {
|
|
385
|
+
...record,
|
|
386
|
+
id: nonEmptyString(record.id, "Garbage collection job ID"),
|
|
387
|
+
candidateManifestVersions,
|
|
388
|
+
candidateSegmentIds,
|
|
389
|
+
candidateBlockIds,
|
|
390
|
+
cursor,
|
|
391
|
+
prunedManifestCount: nonNegativeWholeNumber(record.prunedManifestCount, "Garbage collection pruned manifest count"),
|
|
392
|
+
alreadyPrunedManifestCount: nonNegativeWholeNumber(record.alreadyPrunedManifestCount, "Garbage collection already-pruned manifest count"),
|
|
393
|
+
retainedManifestCount: nonNegativeWholeNumber(record.retainedManifestCount, "Garbage collection retained manifest count"),
|
|
394
|
+
missingManifestCount: nonNegativeWholeNumber(record.missingManifestCount, "Garbage collection missing manifest count"),
|
|
395
|
+
reclaimedSegmentCount: nonNegativeWholeNumber(record.reclaimedSegmentCount, "Garbage collection reclaimed segment count"),
|
|
396
|
+
retainedSegmentCount: nonNegativeWholeNumber(record.retainedSegmentCount, "Garbage collection retained segment count"),
|
|
397
|
+
missingSegmentCount: nonNegativeWholeNumber(record.missingSegmentCount, "Garbage collection missing segment count"),
|
|
398
|
+
reclaimedBlockCount: nonNegativeWholeNumber(record.reclaimedBlockCount, "Garbage collection reclaimed block count"),
|
|
399
|
+
retainedBlockCount: nonNegativeWholeNumber(record.retainedBlockCount, "Garbage collection retained block count"),
|
|
400
|
+
missingBlockCount: nonNegativeWholeNumber(record.missingBlockCount, "Garbage collection missing block count"),
|
|
401
|
+
reclaimedBlockBytes: nonNegativeWholeNumber(record.reclaimedBlockBytes, "Garbage collection reclaimed block bytes"),
|
|
402
|
+
state: garbageCollectionJobState(record.state),
|
|
403
|
+
revision: nonNegativeWholeNumber(record.revision, "Garbage collection job revision"),
|
|
404
|
+
leaseCutoff: validTimestamp(record.leaseCutoff, "Garbage collection lease cutoff"),
|
|
405
|
+
createdAt: validTimestamp(record.createdAt, "Garbage collection creation timestamp"),
|
|
406
|
+
updatedAt: validTimestamp(record.updatedAt, "Garbage collection update timestamp"),
|
|
407
|
+
};
|
|
408
|
+
if (safeSum([
|
|
409
|
+
normalized.prunedManifestCount,
|
|
410
|
+
normalized.alreadyPrunedManifestCount,
|
|
411
|
+
normalized.retainedManifestCount,
|
|
412
|
+
normalized.missingManifestCount,
|
|
413
|
+
], "Garbage collection examined manifest count") !== cursor.manifestIndex ||
|
|
414
|
+
safeSum([
|
|
415
|
+
normalized.reclaimedSegmentCount,
|
|
416
|
+
normalized.retainedSegmentCount,
|
|
417
|
+
normalized.missingSegmentCount,
|
|
418
|
+
], "Garbage collection examined segment count") !== cursor.segmentIndex ||
|
|
419
|
+
safeSum([normalized.reclaimedBlockCount, normalized.retainedBlockCount, normalized.missingBlockCount], "Garbage collection examined block count") !== cursor.blockIndex) {
|
|
420
|
+
throw new TypeError("Garbage collection cursor does not match its persisted accounting");
|
|
421
|
+
}
|
|
422
|
+
if (cursor.manifestIndex > candidateManifestVersions.length ||
|
|
423
|
+
cursor.segmentIndex > candidateSegmentIds.length ||
|
|
424
|
+
cursor.blockIndex > candidateBlockIds.length) {
|
|
425
|
+
throw new RangeError("Garbage collection cursor is outside its candidate selection");
|
|
426
|
+
}
|
|
427
|
+
const complete = garbageCollectionJobComplete(normalized);
|
|
428
|
+
if ((normalized.state === "completed") !== complete) {
|
|
429
|
+
throw new TypeError(complete
|
|
430
|
+
? "A finished garbage collection cursor requires completed state"
|
|
431
|
+
: "A completed garbage collection job requires a finished cursor");
|
|
432
|
+
}
|
|
433
|
+
if (normalized.state === "planned" &&
|
|
434
|
+
(cursor.manifestIndex !== 0 || cursor.segmentIndex !== 0 || cursor.blockIndex !== 0)) {
|
|
435
|
+
throw new TypeError("A planned garbage collection job cannot contain progress");
|
|
436
|
+
}
|
|
437
|
+
return structuredClone(normalized);
|
|
438
|
+
}
|
|
439
|
+
export function advanceGarbageCollectionJobRecord(record, accounting) {
|
|
440
|
+
const current = normalizeGarbageCollectionJobRecord(record);
|
|
441
|
+
if (current.state === "completed")
|
|
442
|
+
return current;
|
|
443
|
+
const increments = {
|
|
444
|
+
examinedManifestCount: nonNegativeWholeNumber(accounting.examinedManifestCount, "Garbage collection examined manifest increment"),
|
|
445
|
+
prunedManifestCount: nonNegativeWholeNumber(accounting.prunedManifestCount, "Garbage collection pruned manifest increment"),
|
|
446
|
+
alreadyPrunedManifestCount: nonNegativeWholeNumber(accounting.alreadyPrunedManifestCount, "Garbage collection already-pruned manifest increment"),
|
|
447
|
+
retainedManifestCount: nonNegativeWholeNumber(accounting.retainedManifestCount, "Garbage collection retained manifest increment"),
|
|
448
|
+
missingManifestCount: nonNegativeWholeNumber(accounting.missingManifestCount, "Garbage collection missing manifest increment"),
|
|
449
|
+
examinedSegmentCount: nonNegativeWholeNumber(accounting.examinedSegmentCount, "Garbage collection examined segment increment"),
|
|
450
|
+
reclaimedSegmentCount: nonNegativeWholeNumber(accounting.reclaimedSegmentCount, "Garbage collection reclaimed segment increment"),
|
|
451
|
+
retainedSegmentCount: nonNegativeWholeNumber(accounting.retainedSegmentCount, "Garbage collection retained segment increment"),
|
|
452
|
+
missingSegmentCount: nonNegativeWholeNumber(accounting.missingSegmentCount, "Garbage collection missing segment increment"),
|
|
453
|
+
examinedBlockCount: nonNegativeWholeNumber(accounting.examinedBlockCount, "Garbage collection examined block increment"),
|
|
454
|
+
reclaimedBlockCount: nonNegativeWholeNumber(accounting.reclaimedBlockCount, "Garbage collection reclaimed block increment"),
|
|
455
|
+
retainedBlockCount: nonNegativeWholeNumber(accounting.retainedBlockCount, "Garbage collection retained block increment"),
|
|
456
|
+
missingBlockCount: nonNegativeWholeNumber(accounting.missingBlockCount, "Garbage collection missing block increment"),
|
|
457
|
+
reclaimedBlockBytes: nonNegativeWholeNumber(accounting.reclaimedBlockBytes, "Garbage collection reclaimed block byte increment"),
|
|
458
|
+
};
|
|
459
|
+
if (increments.examinedManifestCount !==
|
|
460
|
+
safeSum([
|
|
461
|
+
increments.prunedManifestCount,
|
|
462
|
+
increments.alreadyPrunedManifestCount,
|
|
463
|
+
increments.retainedManifestCount,
|
|
464
|
+
increments.missingManifestCount,
|
|
465
|
+
], "Garbage collection manifest increment") ||
|
|
466
|
+
increments.examinedSegmentCount !==
|
|
467
|
+
safeSum([
|
|
468
|
+
increments.reclaimedSegmentCount,
|
|
469
|
+
increments.retainedSegmentCount,
|
|
470
|
+
increments.missingSegmentCount,
|
|
471
|
+
], "Garbage collection segment increment") ||
|
|
472
|
+
increments.examinedBlockCount !==
|
|
473
|
+
safeSum([
|
|
474
|
+
increments.reclaimedBlockCount,
|
|
475
|
+
increments.retainedBlockCount,
|
|
476
|
+
increments.missingBlockCount,
|
|
477
|
+
], "Garbage collection block increment")) {
|
|
478
|
+
throw new TypeError("Garbage collection step accounting is incomplete");
|
|
479
|
+
}
|
|
480
|
+
const cursor = {
|
|
481
|
+
manifestIndex: safeSum([current.cursor.manifestIndex, increments.examinedManifestCount], "Garbage collection manifest cursor"),
|
|
482
|
+
segmentIndex: safeSum([current.cursor.segmentIndex, increments.examinedSegmentCount], "Garbage collection segment cursor"),
|
|
483
|
+
blockIndex: safeSum([current.cursor.blockIndex, increments.examinedBlockCount], "Garbage collection block cursor"),
|
|
484
|
+
};
|
|
485
|
+
const updated = {
|
|
486
|
+
...current,
|
|
487
|
+
cursor,
|
|
488
|
+
prunedManifestCount: safeSum([current.prunedManifestCount, increments.prunedManifestCount], "Garbage collection pruned manifest count"),
|
|
489
|
+
alreadyPrunedManifestCount: safeSum([current.alreadyPrunedManifestCount, increments.alreadyPrunedManifestCount], "Garbage collection already-pruned manifest count"),
|
|
490
|
+
retainedManifestCount: safeSum([current.retainedManifestCount, increments.retainedManifestCount], "Garbage collection retained manifest count"),
|
|
491
|
+
missingManifestCount: safeSum([current.missingManifestCount, increments.missingManifestCount], "Garbage collection missing manifest count"),
|
|
492
|
+
reclaimedSegmentCount: safeSum([current.reclaimedSegmentCount, increments.reclaimedSegmentCount], "Garbage collection reclaimed segment count"),
|
|
493
|
+
retainedSegmentCount: safeSum([current.retainedSegmentCount, increments.retainedSegmentCount], "Garbage collection retained segment count"),
|
|
494
|
+
missingSegmentCount: safeSum([current.missingSegmentCount, increments.missingSegmentCount], "Garbage collection missing segment count"),
|
|
495
|
+
reclaimedBlockCount: safeSum([current.reclaimedBlockCount, increments.reclaimedBlockCount], "Garbage collection reclaimed block count"),
|
|
496
|
+
retainedBlockCount: safeSum([current.retainedBlockCount, increments.retainedBlockCount], "Garbage collection retained block count"),
|
|
497
|
+
missingBlockCount: safeSum([current.missingBlockCount, increments.missingBlockCount], "Garbage collection missing block count"),
|
|
498
|
+
reclaimedBlockBytes: safeSum([current.reclaimedBlockBytes, increments.reclaimedBlockBytes], "Garbage collection reclaimed block bytes"),
|
|
499
|
+
state: "running",
|
|
500
|
+
revision: current.revision + 1,
|
|
501
|
+
updatedAt: validTimestamp(accounting.updatedAt, "Garbage collection update timestamp"),
|
|
502
|
+
};
|
|
503
|
+
if (garbageCollectionJobComplete(updated))
|
|
504
|
+
updated.state = "completed";
|
|
505
|
+
return normalizeGarbageCollectionJobRecord(updated);
|
|
506
|
+
}
|
|
507
|
+
export function normalizeCompactionJobRecord(record) {
|
|
508
|
+
const error = record.error;
|
|
509
|
+
if (error !== undefined && typeof error !== "string") {
|
|
510
|
+
throw new TypeError("Compaction job error must be a string");
|
|
511
|
+
}
|
|
512
|
+
const rewritePlan = normalizeCompactionRewritePlan(record.rewritePlan);
|
|
513
|
+
const logicalBytes = nonNegativeWholeNumber(record.logicalBytes, "Compaction logical bytes");
|
|
514
|
+
const sourceStoredBytes = nonNegativeWholeNumber(record.sourceStoredBytes, "Compaction source stored bytes");
|
|
515
|
+
const outputStoredBytes = nonNegativeWholeNumber(record.outputStoredBytes, "Compaction output stored bytes");
|
|
516
|
+
const hasLevel0SourceStoredBytes = record.level0SourceStoredBytes !== undefined;
|
|
517
|
+
const hasAnchorSourceStoredBytes = record.anchorSourceStoredBytes !== undefined;
|
|
518
|
+
if (hasLevel0SourceStoredBytes !== hasAnchorSourceStoredBytes) {
|
|
519
|
+
throw new TypeError("Compaction source-level byte accounting requires both stored byte fields");
|
|
520
|
+
}
|
|
521
|
+
const sourceLevelStoredBytes = hasLevel0SourceStoredBytes
|
|
522
|
+
? {
|
|
523
|
+
level0SourceStoredBytes: positiveWholeNumber(record.level0SourceStoredBytes, "Compaction level-zero source stored bytes"),
|
|
524
|
+
anchorSourceStoredBytes: nonNegativeWholeNumber(record.anchorSourceStoredBytes, "Compaction anchor source stored bytes"),
|
|
525
|
+
}
|
|
526
|
+
: undefined;
|
|
527
|
+
if (sourceLevelStoredBytes !== undefined &&
|
|
528
|
+
safeSum([
|
|
529
|
+
sourceLevelStoredBytes.level0SourceStoredBytes,
|
|
530
|
+
sourceLevelStoredBytes.anchorSourceStoredBytes,
|
|
531
|
+
], "Compaction source-level stored bytes") !== sourceStoredBytes) {
|
|
532
|
+
throw new TypeError("Compaction source-level stored bytes must equal source stored bytes");
|
|
533
|
+
}
|
|
534
|
+
const level2PolicyValues = [
|
|
535
|
+
record.outputPartitionOrdinal,
|
|
536
|
+
record.maxWriteAmplification,
|
|
537
|
+
record.maximumOutputStoredBytes,
|
|
538
|
+
record.plannedOutputStoredBytesUpperBound,
|
|
539
|
+
];
|
|
540
|
+
const level2PolicyFieldCount = level2PolicyValues.filter((value) => value !== undefined).length;
|
|
541
|
+
if (level2PolicyFieldCount !== 0 && level2PolicyFieldCount !== level2PolicyValues.length) {
|
|
542
|
+
throw new TypeError("Append-row-range L2 compaction policy fields must be present together");
|
|
543
|
+
}
|
|
544
|
+
if (record.priorAttemptOutputStoredBytes !== undefined && level2PolicyFieldCount === 0) {
|
|
545
|
+
throw new TypeError("Compaction prior-attempt accounting requires the L2 compaction policy fields");
|
|
546
|
+
}
|
|
547
|
+
let level2Policy;
|
|
548
|
+
if (level2PolicyFieldCount !== 0) {
|
|
549
|
+
if (rewritePlan.kind !== "rechunk-v1" && rewritePlan.kind !== "merge-v1") {
|
|
550
|
+
throw new TypeError("L2 compaction requires a rechunk or merge plan");
|
|
551
|
+
}
|
|
552
|
+
if (record.targetLevel !== 2) {
|
|
553
|
+
throw new TypeError("Append-row-range L2 compaction must target level two");
|
|
554
|
+
}
|
|
555
|
+
// Append-row-range promotions consume pure level-zero prefixes; keyed merge promotions may
|
|
556
|
+
// also fold a retained level-one anchor, whose bytes never count toward the L0 ceiling.
|
|
557
|
+
if (rewritePlan.kind === "rechunk-v1" &&
|
|
558
|
+
(sourceLevelStoredBytes?.level0SourceStoredBytes !== sourceStoredBytes ||
|
|
559
|
+
sourceLevelStoredBytes.anchorSourceStoredBytes !== 0)) {
|
|
560
|
+
throw new TypeError("Append-row-range L2 compaction requires only level-zero source bytes");
|
|
561
|
+
}
|
|
562
|
+
if (rewritePlan.kind === "merge-v1" && sourceLevelStoredBytes === undefined) {
|
|
563
|
+
throw new TypeError("Keyed L2 compaction requires source-level byte accounting");
|
|
564
|
+
}
|
|
565
|
+
const outputPartitionOrdinal = nonNegativeWholeNumber(record.outputPartitionOrdinal, "Compaction output partition ordinal");
|
|
566
|
+
const maxWriteAmplification = positiveFiniteNumber(record.maxWriteAmplification, "Compaction maximum write amplification");
|
|
567
|
+
const maximumOutputStoredBytes = positiveWholeNumber(record.maximumOutputStoredBytes, "Compaction maximum output stored bytes");
|
|
568
|
+
const plannedOutputStoredBytesUpperBound = positiveWholeNumber(record.plannedOutputStoredBytesUpperBound, "Compaction planned output stored byte upper bound");
|
|
569
|
+
const priorAttemptOutputStoredBytes = record.priorAttemptOutputStoredBytes === undefined
|
|
570
|
+
? undefined
|
|
571
|
+
: nonNegativeWholeNumber(record.priorAttemptOutputStoredBytes, "Compaction prior-attempt output stored bytes");
|
|
572
|
+
const amplificationCeiling = floorWholeNumberProduct(sourceStoredBytes, maxWriteAmplification, "Compaction write amplification product");
|
|
573
|
+
// The persisted ceiling plus everything failed attempts already wrote must stay within
|
|
574
|
+
// the amplification limit — attempts share one lifetime budget.
|
|
575
|
+
if (maximumOutputStoredBytes + (priorAttemptOutputStoredBytes ?? 0) > amplificationCeiling) {
|
|
576
|
+
throw new RangeError("Compaction output stored byte ceiling exceeds its amplification limit");
|
|
577
|
+
}
|
|
578
|
+
if (plannedOutputStoredBytesUpperBound > maximumOutputStoredBytes) {
|
|
579
|
+
throw new RangeError("Compaction planned output exceeds its stored byte ceiling");
|
|
580
|
+
}
|
|
581
|
+
if (outputStoredBytes > plannedOutputStoredBytesUpperBound) {
|
|
582
|
+
throw new RangeError("Compaction output stored bytes exceed their planned upper bound");
|
|
583
|
+
}
|
|
584
|
+
level2Policy = {
|
|
585
|
+
outputPartitionOrdinal,
|
|
586
|
+
maxWriteAmplification,
|
|
587
|
+
maximumOutputStoredBytes,
|
|
588
|
+
plannedOutputStoredBytesUpperBound,
|
|
589
|
+
...(priorAttemptOutputStoredBytes === undefined ? {} : { priorAttemptOutputStoredBytes }),
|
|
590
|
+
};
|
|
591
|
+
}
|
|
592
|
+
const normalized = {
|
|
593
|
+
...record,
|
|
594
|
+
id: nonEmptyString(record.id, "Compaction job ID"),
|
|
595
|
+
tableId: nonEmptyString(record.tableId, "Compaction job table ID"),
|
|
596
|
+
sourceManifestVersion: nonNegativeWholeNumber(record.sourceManifestVersion, "Compaction source manifest version"),
|
|
597
|
+
sourceSegmentIds: rewritePlan.kind === "copy-v1"
|
|
598
|
+
? uniqueIds(record.sourceSegmentIds, "Compaction source segment ID", false)
|
|
599
|
+
: orderedUniqueIds(record.sourceSegmentIds, "Compaction source segment ID"),
|
|
600
|
+
sourceBlockIds: rewritePlan.kind === "copy-v1"
|
|
601
|
+
? uniqueIds(record.sourceBlockIds, "Compaction source block ID", true)
|
|
602
|
+
: orderedUniqueIds(record.sourceBlockIds, "Compaction source block ID").sort(),
|
|
603
|
+
outputBlockIds: rewritePlan.kind === "copy-v1"
|
|
604
|
+
? uniqueIds(record.outputBlockIds, "Compaction output block ID", true)
|
|
605
|
+
: orderedUniqueIds(record.outputBlockIds, "Compaction output block ID"),
|
|
606
|
+
cursor: normalizeCompactionJobCursor(record.cursor),
|
|
607
|
+
processedRows: nonNegativeWholeNumber(record.processedRows, "Compaction processed row count"),
|
|
608
|
+
sourceStoredBytes,
|
|
609
|
+
outputStoredBytes,
|
|
610
|
+
logicalBytes,
|
|
611
|
+
rewritePlan,
|
|
612
|
+
outputCursor: normalizeCompactionOutputCursor(record.outputCursor, rewritePlan),
|
|
613
|
+
memoryBudgetBytes: nonNegativeWholeNumber(record.memoryBudgetBytes ?? 0, "Compaction memory budget"),
|
|
614
|
+
minimumMemoryBytes: nonNegativeWholeNumber(record.minimumMemoryBytes ?? 0, "Compaction minimum memory"),
|
|
615
|
+
...(sourceLevelStoredBytes ?? {}),
|
|
616
|
+
...(level2Policy ?? {}),
|
|
617
|
+
peakWorkingBytes: nonNegativeWholeNumber(record.peakWorkingBytes ?? 0, "Compaction peak working bytes"),
|
|
618
|
+
outputLogicalBytes: nonNegativeWholeNumber(record.outputLogicalBytes ?? (rewritePlan.kind === "copy-v1" ? logicalBytes : 0), "Compaction output logical bytes"),
|
|
619
|
+
targetLevel: nonNegativeWholeNumber(record.targetLevel, "Compaction target level"),
|
|
620
|
+
state: compactionJobState(record.state),
|
|
621
|
+
transactionId: nullableId(record.transactionId, "Compaction transaction ID"),
|
|
622
|
+
outputSegmentId: nullableId(record.outputSegmentId, "Compaction output segment ID"),
|
|
623
|
+
publishedVersion: record.publishedVersion === null
|
|
624
|
+
? null
|
|
625
|
+
: nonNegativeWholeNumber(record.publishedVersion, "Compaction published version"),
|
|
626
|
+
revision: nonNegativeWholeNumber(record.revision, "Compaction job revision"),
|
|
627
|
+
createdAt: nonEmptyString(record.createdAt, "Compaction creation timestamp"),
|
|
628
|
+
updatedAt: nonEmptyString(record.updatedAt, "Compaction update timestamp"),
|
|
629
|
+
};
|
|
630
|
+
if (normalized.sourceSegmentIds.length === 0) {
|
|
631
|
+
throw new TypeError("Compaction requires at least one source segment");
|
|
632
|
+
}
|
|
633
|
+
if (normalized.cursor.sourceSegmentIndex > normalized.sourceSegmentIds.length) {
|
|
634
|
+
throw new RangeError("Compaction source segment cursor is outside the source selection");
|
|
635
|
+
}
|
|
636
|
+
if (normalized.cursor.sourceSegmentIndex === normalized.sourceSegmentIds.length &&
|
|
637
|
+
normalized.cursor.sourceBlockIndex !== 0) {
|
|
638
|
+
throw new RangeError("A completed compaction cursor must start at block zero");
|
|
639
|
+
}
|
|
640
|
+
validateCompactionRewrite(normalized);
|
|
641
|
+
validateCompactionJobState(normalized);
|
|
642
|
+
return structuredClone(normalized);
|
|
643
|
+
}
|
|
644
|
+
export function updateCompactionJobRecord(record, update) {
|
|
645
|
+
const current = normalizeCompactionJobRecord(record);
|
|
646
|
+
for (const field of [
|
|
647
|
+
"rewritePlan",
|
|
648
|
+
"memoryBudgetBytes",
|
|
649
|
+
"minimumMemoryBytes",
|
|
650
|
+
"level0SourceStoredBytes",
|
|
651
|
+
"anchorSourceStoredBytes",
|
|
652
|
+
"outputPartitionOrdinal",
|
|
653
|
+
"maxWriteAmplification",
|
|
654
|
+
"maximumOutputStoredBytes",
|
|
655
|
+
"plannedOutputStoredBytesUpperBound",
|
|
656
|
+
"priorAttemptOutputStoredBytes",
|
|
657
|
+
]) {
|
|
658
|
+
if (Reflect.has(update, field)) {
|
|
659
|
+
throw new TypeError(`Compaction ${field} is immutable`);
|
|
660
|
+
}
|
|
661
|
+
}
|
|
662
|
+
if (isOutputDrivenCompactionPlan(current.rewritePlan)) {
|
|
663
|
+
for (const field of ["cursor", "sourceStoredBytes", "logicalBytes"]) {
|
|
664
|
+
if (Reflect.has(update, field)) {
|
|
665
|
+
throw new TypeError(`Output-driven compaction ${field} is immutable`);
|
|
666
|
+
}
|
|
667
|
+
}
|
|
668
|
+
}
|
|
669
|
+
const mirrorCopyLogicalBytes = current.rewritePlan?.kind === "copy-v1" &&
|
|
670
|
+
update.logicalBytes !== undefined &&
|
|
671
|
+
update.outputLogicalBytes === undefined;
|
|
672
|
+
const updated = {
|
|
673
|
+
...current,
|
|
674
|
+
...(update.outputBlockIds === undefined ? {} : { outputBlockIds: [...update.outputBlockIds] }),
|
|
675
|
+
...(update.cursor === undefined ? {} : { cursor: update.cursor }),
|
|
676
|
+
...(update.processedRows === undefined ? {} : { processedRows: update.processedRows }),
|
|
677
|
+
...(update.sourceStoredBytes === undefined
|
|
678
|
+
? {}
|
|
679
|
+
: { sourceStoredBytes: update.sourceStoredBytes }),
|
|
680
|
+
...(update.outputStoredBytes === undefined
|
|
681
|
+
? {}
|
|
682
|
+
: { outputStoredBytes: update.outputStoredBytes }),
|
|
683
|
+
...(update.logicalBytes === undefined ? {} : { logicalBytes: update.logicalBytes }),
|
|
684
|
+
...(update.outputCursor === undefined ? {} : { outputCursor: update.outputCursor }),
|
|
685
|
+
...(update.peakWorkingBytes === undefined ? {} : { peakWorkingBytes: update.peakWorkingBytes }),
|
|
686
|
+
...(update.outputLogicalBytes === undefined
|
|
687
|
+
? mirrorCopyLogicalBytes
|
|
688
|
+
? { outputLogicalBytes: update.logicalBytes }
|
|
689
|
+
: {}
|
|
690
|
+
: { outputLogicalBytes: update.outputLogicalBytes }),
|
|
691
|
+
...(update.state === undefined ? {} : { state: update.state }),
|
|
692
|
+
...(update.transactionId === undefined ? {} : { transactionId: update.transactionId }),
|
|
693
|
+
...(update.outputSegmentId === undefined ? {} : { outputSegmentId: update.outputSegmentId }),
|
|
694
|
+
...(update.publishedVersion === undefined ? {} : { publishedVersion: update.publishedVersion }),
|
|
695
|
+
updatedAt: update.updatedAt,
|
|
696
|
+
revision: current.revision + 1,
|
|
697
|
+
};
|
|
698
|
+
if (update.error === null)
|
|
699
|
+
delete updated.error;
|
|
700
|
+
else if (update.error !== undefined)
|
|
701
|
+
updated.error = update.error;
|
|
702
|
+
const normalized = normalizeCompactionJobRecord(updated);
|
|
703
|
+
validateCompactionJobTransition(current.state, normalized.state);
|
|
704
|
+
validateCompactionJobProgress(current, normalized);
|
|
705
|
+
return normalized;
|
|
706
|
+
}
|
|
707
|
+
function validateCompactionJobState(record) {
|
|
708
|
+
const plan = record.rewritePlan ?? { kind: "copy-v1" };
|
|
709
|
+
if (plan.kind === "merge-v1") {
|
|
710
|
+
if (plan.totalRows === 0 && record.outputSegmentId !== null) {
|
|
711
|
+
throw new TypeError("An empty merge compaction cannot have an output segment");
|
|
712
|
+
}
|
|
713
|
+
if (plan.totalRows > 0 && record.outputSegmentId === null) {
|
|
714
|
+
throw new TypeError("A non-empty merge compaction requires an output segment ID");
|
|
715
|
+
}
|
|
716
|
+
}
|
|
717
|
+
if (record.state === "cancelled" && record.error !== undefined) {
|
|
718
|
+
throw new TypeError("A cancelled compaction cannot contain an error");
|
|
719
|
+
}
|
|
720
|
+
if (record.state === "planned") {
|
|
721
|
+
const isCopy = record.rewritePlan?.kind === "copy-v1";
|
|
722
|
+
const hasProgress = record.cursor.sourceSegmentIndex !== 0 ||
|
|
723
|
+
record.cursor.sourceBlockIndex !== 0 ||
|
|
724
|
+
record.outputBlockIds.length !== 0 ||
|
|
725
|
+
record.processedRows !== 0 ||
|
|
726
|
+
(isCopy && record.sourceStoredBytes !== 0) ||
|
|
727
|
+
record.outputStoredBytes !== 0 ||
|
|
728
|
+
(isCopy && record.logicalBytes !== 0) ||
|
|
729
|
+
record.outputLogicalBytes !== 0 ||
|
|
730
|
+
record.peakWorkingBytes !== 0;
|
|
731
|
+
if (hasProgress || record.transactionId !== null || !isInitialOutputCursor(record)) {
|
|
732
|
+
throw new TypeError("A planned compaction cannot contain transaction progress");
|
|
733
|
+
}
|
|
734
|
+
}
|
|
735
|
+
if (record.state === "running" && record.transactionId === null) {
|
|
736
|
+
throw new TypeError("A running compaction requires a transaction ID");
|
|
737
|
+
}
|
|
738
|
+
if (record.state === "ready" || record.state === "published") {
|
|
739
|
+
if (record.transactionId === null ||
|
|
740
|
+
(record.outputSegmentId === null && !(plan.kind === "merge-v1" && plan.totalRows === 0))) {
|
|
741
|
+
throw new TypeError(`${record.state} compaction requires its transaction and output segment`);
|
|
742
|
+
}
|
|
743
|
+
if (!hasCompletedCompactionCursor(record)) {
|
|
744
|
+
throw new TypeError(`${record.state} compaction requires a completed cursor`);
|
|
745
|
+
}
|
|
746
|
+
if (record.outputBlockIds.length !== expectedCompactionOutputCount(record)) {
|
|
747
|
+
throw new TypeError(`${record.state} compaction requires every output block`);
|
|
748
|
+
}
|
|
749
|
+
if (isOutputDrivenCompactionPlan(record.rewritePlan) &&
|
|
750
|
+
expectedCompactionOutputCount(record) > 0 &&
|
|
751
|
+
(record.peakWorkingBytes ?? 0) < (record.minimumMemoryBytes ?? 0)) {
|
|
752
|
+
throw new TypeError(`${record.state} compaction requires complete memory accounting`);
|
|
753
|
+
}
|
|
754
|
+
}
|
|
755
|
+
if (record.state === "published") {
|
|
756
|
+
if (record.publishedVersion === null) {
|
|
757
|
+
throw new TypeError("A published compaction requires a manifest version");
|
|
758
|
+
}
|
|
759
|
+
}
|
|
760
|
+
else if (record.publishedVersion !== null) {
|
|
761
|
+
throw new TypeError("Only a published compaction can have a manifest version");
|
|
762
|
+
}
|
|
763
|
+
}
|
|
764
|
+
function validateCompactionRewrite(record) {
|
|
765
|
+
const plan = record.rewritePlan ?? { kind: "copy-v1" };
|
|
766
|
+
const outputLogicalBytes = record.outputLogicalBytes ?? 0;
|
|
767
|
+
const memoryBudgetBytes = record.memoryBudgetBytes ?? 0;
|
|
768
|
+
const minimumMemoryBytes = record.minimumMemoryBytes ?? 0;
|
|
769
|
+
const peakWorkingBytes = record.peakWorkingBytes ?? 0;
|
|
770
|
+
if (plan.kind === "copy-v1") {
|
|
771
|
+
if (record.outputCursor !== null) {
|
|
772
|
+
throw new TypeError("A copy compaction cannot have an output cursor");
|
|
773
|
+
}
|
|
774
|
+
if (memoryBudgetBytes !== 0 || minimumMemoryBytes !== 0 || peakWorkingBytes !== 0) {
|
|
775
|
+
throw new TypeError("A copy compaction cannot have rechunk memory accounting");
|
|
776
|
+
}
|
|
777
|
+
if (outputLogicalBytes !== record.logicalBytes) {
|
|
778
|
+
throw new TypeError("A copy compaction must preserve its logical byte count");
|
|
779
|
+
}
|
|
780
|
+
if (record.outputBlockIds.length > record.sourceBlockIds.length) {
|
|
781
|
+
throw new RangeError("Compaction output cannot contain more blocks than its source selection");
|
|
782
|
+
}
|
|
783
|
+
return;
|
|
784
|
+
}
|
|
785
|
+
if (record.cursor.sourceSegmentIndex !== 0 || record.cursor.sourceBlockIndex !== 0) {
|
|
786
|
+
throw new TypeError("An output-driven compaction does not use the source cursor");
|
|
787
|
+
}
|
|
788
|
+
const permitsZeroMinimum = plan.kind === "merge-v1" && plan.totalRows === 0;
|
|
789
|
+
if (memoryBudgetBytes === 0 || (minimumMemoryBytes === 0 && !permitsZeroMinimum)) {
|
|
790
|
+
throw new RangeError("An output-driven compaction requires a memory budget and minimum memory");
|
|
791
|
+
}
|
|
792
|
+
if (minimumMemoryBytes > memoryBudgetBytes) {
|
|
793
|
+
throw new RangeError("Compaction minimum memory exceeds its memory budget");
|
|
794
|
+
}
|
|
795
|
+
if (peakWorkingBytes > memoryBudgetBytes) {
|
|
796
|
+
throw new RangeError("Compaction peak working bytes exceed its memory budget");
|
|
797
|
+
}
|
|
798
|
+
const plannedBlocks = plan.kind === "rechunk-v1"
|
|
799
|
+
? plan.columns.flatMap((column) => column.sourceBlocks)
|
|
800
|
+
: plan.sourceSegments.flatMap((segment) => segment.columns.flatMap((column) => column.sourceBlocks));
|
|
801
|
+
const plannedBlockIds = plannedBlocks.map((block) => block.blockId);
|
|
802
|
+
if (new Set(plannedBlockIds).size !== plannedBlockIds.length) {
|
|
803
|
+
throw new TypeError("A planned source block can only appear once in its source layout");
|
|
804
|
+
}
|
|
805
|
+
const sortedPlannedIds = [...plannedBlockIds].sort();
|
|
806
|
+
if (sortedPlannedIds.length !== record.sourceBlockIds.length ||
|
|
807
|
+
sortedPlannedIds.some((id, index) => id !== record.sourceBlockIds[index])) {
|
|
808
|
+
throw new TypeError("The rewrite source layout must describe every selected source block");
|
|
809
|
+
}
|
|
810
|
+
const plannedStoredBytes = safeSum(plannedBlocks.map((block) => block.storedBytes), "Rechunk source stored bytes");
|
|
811
|
+
const plannedEncodedBytes = safeSum(plannedBlocks.map((block) => block.encodedBytes), "Rechunk source encoded bytes");
|
|
812
|
+
if (record.sourceStoredBytes !== plannedStoredBytes) {
|
|
813
|
+
throw new TypeError("Source stored bytes must match the immutable rewrite layout");
|
|
814
|
+
}
|
|
815
|
+
if (record.logicalBytes !== plannedEncodedBytes) {
|
|
816
|
+
throw new TypeError("Logical bytes must match the immutable rewrite layout");
|
|
817
|
+
}
|
|
818
|
+
if (plan.kind === "merge-v1") {
|
|
819
|
+
if (plan.totalRows === 0 &&
|
|
820
|
+
(record.outputBlockIds.length !== 0 ||
|
|
821
|
+
record.outputStoredBytes !== 0 ||
|
|
822
|
+
outputLogicalBytes !== 0 ||
|
|
823
|
+
peakWorkingBytes !== 0 ||
|
|
824
|
+
minimumMemoryBytes !== 0)) {
|
|
825
|
+
throw new TypeError("An empty merge compaction cannot contain physical output progress");
|
|
826
|
+
}
|
|
827
|
+
const plannedSegmentIds = plan.sourceSegments.map((segment) => segment.segmentId);
|
|
828
|
+
if (plannedSegmentIds.length !== record.sourceSegmentIds.length ||
|
|
829
|
+
plannedSegmentIds.some((id, index) => id !== record.sourceSegmentIds[index])) {
|
|
830
|
+
throw new TypeError("Merge source layout must preserve every selected source segment in order");
|
|
831
|
+
}
|
|
832
|
+
}
|
|
833
|
+
const cursor = record.outputCursor;
|
|
834
|
+
if (cursor === null || cursor === undefined) {
|
|
835
|
+
throw new TypeError("An output-driven compaction requires an output cursor");
|
|
836
|
+
}
|
|
837
|
+
const completedOutputs = safeSum([
|
|
838
|
+
safeProduct(cursor.outputIndex, plan.columns.length, "Rechunk output cursor"),
|
|
839
|
+
cursor.columnIndex,
|
|
840
|
+
], "Rechunk output cursor");
|
|
841
|
+
if (record.outputBlockIds.length !== completedOutputs) {
|
|
842
|
+
throw new TypeError("Compaction output IDs must match the output cursor");
|
|
843
|
+
}
|
|
844
|
+
const expectedProcessedRows = cursor.outputIndex === plan.outputs.length
|
|
845
|
+
? plan.totalRows
|
|
846
|
+
: (plan.outputs[cursor.outputIndex]?.rowStart ?? 0);
|
|
847
|
+
if (record.processedRows !== expectedProcessedRows) {
|
|
848
|
+
throw new TypeError("Processed rows must match completed output windows");
|
|
849
|
+
}
|
|
850
|
+
}
|
|
851
|
+
function validateCompactionJobProgress(previous, next) {
|
|
852
|
+
for (const [label, previousValue, nextValue] of [
|
|
853
|
+
["processed rows", previous.processedRows, next.processedRows],
|
|
854
|
+
["source stored bytes", previous.sourceStoredBytes, next.sourceStoredBytes],
|
|
855
|
+
["output stored bytes", previous.outputStoredBytes, next.outputStoredBytes],
|
|
856
|
+
["logical bytes", previous.logicalBytes, next.logicalBytes],
|
|
857
|
+
["output logical bytes", previous.outputLogicalBytes ?? 0, next.outputLogicalBytes ?? 0],
|
|
858
|
+
["peak working bytes", previous.peakWorkingBytes ?? 0, next.peakWorkingBytes ?? 0],
|
|
859
|
+
]) {
|
|
860
|
+
if (nextValue < previousValue) {
|
|
861
|
+
throw new RangeError(`Compaction ${label} cannot decrease`);
|
|
862
|
+
}
|
|
863
|
+
}
|
|
864
|
+
if (next.outputBlockIds.length < previous.outputBlockIds.length) {
|
|
865
|
+
throw new TypeError("Compaction output block IDs cannot be removed");
|
|
866
|
+
}
|
|
867
|
+
if (isOutputDrivenCompactionPlan(previous.rewritePlan)) {
|
|
868
|
+
if (previous.outputBlockIds.some((id, index) => next.outputBlockIds[index] !== id)) {
|
|
869
|
+
throw new TypeError("Output block IDs are an append-only ordered checkpoint");
|
|
870
|
+
}
|
|
871
|
+
if (compactionOutputOrdinal(next) < compactionOutputOrdinal(previous)) {
|
|
872
|
+
throw new RangeError("Output cursor cannot move backwards");
|
|
873
|
+
}
|
|
874
|
+
if (previous.rewritePlan.kind === "merge-v1" &&
|
|
875
|
+
previous.outputSegmentId !== next.outputSegmentId) {
|
|
876
|
+
throw new TypeError("Merge output segment ID is immutable");
|
|
877
|
+
}
|
|
878
|
+
}
|
|
879
|
+
else {
|
|
880
|
+
const nextIds = new Set(next.outputBlockIds);
|
|
881
|
+
if (previous.outputBlockIds.some((id) => !nextIds.has(id))) {
|
|
882
|
+
throw new TypeError("Compaction output block IDs cannot be removed");
|
|
883
|
+
}
|
|
884
|
+
const previousCursor = previous.cursor;
|
|
885
|
+
const nextCursor = next.cursor;
|
|
886
|
+
if (nextCursor.sourceSegmentIndex < previousCursor.sourceSegmentIndex ||
|
|
887
|
+
(nextCursor.sourceSegmentIndex === previousCursor.sourceSegmentIndex &&
|
|
888
|
+
nextCursor.sourceBlockIndex < previousCursor.sourceBlockIndex)) {
|
|
889
|
+
throw new RangeError("Compaction source cursor cannot move backwards");
|
|
890
|
+
}
|
|
891
|
+
}
|
|
892
|
+
}
|
|
893
|
+
function isInitialOutputCursor(record) {
|
|
894
|
+
const plan = record.rewritePlan ?? { kind: "copy-v1" };
|
|
895
|
+
if (plan.kind === "copy-v1")
|
|
896
|
+
return record.outputCursor === null;
|
|
897
|
+
const cursor = record.outputCursor;
|
|
898
|
+
const initialRowStart = plan.outputs[0]?.rowStart ?? plan.totalRows;
|
|
899
|
+
return (cursor?.outputIndex === 0 && cursor.columnIndex === 0 && cursor.rowStart === initialRowStart);
|
|
900
|
+
}
|
|
901
|
+
function hasCompletedCompactionCursor(record) {
|
|
902
|
+
const plan = record.rewritePlan ?? { kind: "copy-v1" };
|
|
903
|
+
if (plan.kind === "copy-v1") {
|
|
904
|
+
return record.cursor.sourceSegmentIndex === record.sourceSegmentIds.length;
|
|
905
|
+
}
|
|
906
|
+
const cursor = record.outputCursor;
|
|
907
|
+
return (cursor?.outputIndex === plan.outputs.length &&
|
|
908
|
+
cursor.columnIndex === 0 &&
|
|
909
|
+
cursor.rowStart === plan.totalRows);
|
|
910
|
+
}
|
|
911
|
+
function expectedCompactionOutputCount(record) {
|
|
912
|
+
const plan = record.rewritePlan ?? { kind: "copy-v1" };
|
|
913
|
+
return plan.kind === "copy-v1"
|
|
914
|
+
? record.sourceBlockIds.length
|
|
915
|
+
: safeProduct(plan.outputs.length, plan.columns.length, "Rechunk output block count");
|
|
916
|
+
}
|
|
917
|
+
function compactionOutputOrdinal(record) {
|
|
918
|
+
const plan = record.rewritePlan;
|
|
919
|
+
const cursor = record.outputCursor;
|
|
920
|
+
if (!isOutputDrivenCompactionPlan(plan) || cursor === null || cursor === undefined)
|
|
921
|
+
return 0;
|
|
922
|
+
return safeSum([
|
|
923
|
+
safeProduct(cursor.outputIndex, plan.columns.length, "Rechunk output cursor"),
|
|
924
|
+
cursor.columnIndex,
|
|
925
|
+
], "Rechunk output cursor");
|
|
926
|
+
}
|
|
927
|
+
function isOutputDrivenCompactionPlan(plan) {
|
|
928
|
+
return plan?.kind === "rechunk-v1" || plan?.kind === "merge-v1";
|
|
929
|
+
}
|
|
930
|
+
function validateCompactionJobTransition(previous, next) {
|
|
931
|
+
const allowed = {
|
|
932
|
+
planned: ["planned", "running", "cancelled", "aborted"],
|
|
933
|
+
running: ["running", "ready", "published", "cancelled", "aborted"],
|
|
934
|
+
ready: ["ready", "running", "published", "cancelled", "aborted"],
|
|
935
|
+
published: ["published"],
|
|
936
|
+
cancelled: ["cancelled"],
|
|
937
|
+
aborted: ["aborted"],
|
|
938
|
+
};
|
|
939
|
+
if (!allowed[previous].includes(next)) {
|
|
940
|
+
throw new TypeError(`Invalid compaction state transition: ${previous} to ${next}`);
|
|
941
|
+
}
|
|
942
|
+
}
|
|
943
|
+
function normalizeCompactionJobCursor(value) {
|
|
944
|
+
if (typeof value !== "object" || value === null) {
|
|
945
|
+
throw new TypeError("Compaction cursor must be an object");
|
|
946
|
+
}
|
|
947
|
+
return {
|
|
948
|
+
sourceSegmentIndex: nonNegativeWholeNumber(Reflect.get(value, "sourceSegmentIndex"), "Compaction source segment cursor"),
|
|
949
|
+
sourceBlockIndex: nonNegativeWholeNumber(Reflect.get(value, "sourceBlockIndex"), "Compaction source block cursor"),
|
|
950
|
+
};
|
|
951
|
+
}
|
|
952
|
+
function normalizeCompactionRewritePlan(value) {
|
|
953
|
+
if (value === undefined)
|
|
954
|
+
return { kind: "copy-v1" };
|
|
955
|
+
if (typeof value !== "object" || value === null) {
|
|
956
|
+
throw new TypeError("Compaction rewrite plan must be an object");
|
|
957
|
+
}
|
|
958
|
+
const kind = Reflect.get(value, "kind");
|
|
959
|
+
if (kind === "copy-v1")
|
|
960
|
+
return { kind: "copy-v1" };
|
|
961
|
+
if (kind === "merge-v1")
|
|
962
|
+
return normalizeMergeCompactionRewritePlan(value);
|
|
963
|
+
if (kind !== "rechunk-v1") {
|
|
964
|
+
throw new TypeError(`Invalid compaction rewrite plan: ${String(kind)}`);
|
|
965
|
+
}
|
|
966
|
+
const totalRows = positiveWholeNumber(Reflect.get(value, "totalRows"), "Rechunk total row count");
|
|
967
|
+
const rowIdStart = nonNegativeBigInt(Reflect.get(value, "rowIdStart"), "Rechunk row ID start");
|
|
968
|
+
const rowIdEndExclusive = nonNegativeBigInt(Reflect.get(value, "rowIdEndExclusive"), "Rechunk row ID end");
|
|
969
|
+
if (rowIdEndExclusive - rowIdStart !== BigInt(totalRows)) {
|
|
970
|
+
throw new RangeError("Rechunk row ID range must match its total row count");
|
|
971
|
+
}
|
|
972
|
+
const columnsValue = Reflect.get(value, "columns");
|
|
973
|
+
if (!Array.isArray(columnsValue) || columnsValue.length === 0) {
|
|
974
|
+
throw new TypeError("A rechunk plan requires at least one source column");
|
|
975
|
+
}
|
|
976
|
+
const columns = columnsValue.map((column, index) => normalizeRechunkSourceColumn(column, totalRows, index));
|
|
977
|
+
const columnIds = columns.map((column) => column.columnId);
|
|
978
|
+
if (new Set(columnIds).size !== columnIds.length) {
|
|
979
|
+
throw new TypeError("A rechunk plan cannot contain duplicate source columns");
|
|
980
|
+
}
|
|
981
|
+
const outputsValue = Reflect.get(value, "outputs");
|
|
982
|
+
if (!Array.isArray(outputsValue) || outputsValue.length === 0) {
|
|
983
|
+
throw new TypeError("A rechunk plan requires at least one output window");
|
|
984
|
+
}
|
|
985
|
+
const outputs = outputsValue.map((output, index) => {
|
|
986
|
+
if (typeof output !== "object" || output === null) {
|
|
987
|
+
throw new TypeError(`Rechunk output window ${String(index)} must be an object`);
|
|
988
|
+
}
|
|
989
|
+
return {
|
|
990
|
+
rowStart: nonNegativeWholeNumber(Reflect.get(output, "rowStart"), `Rechunk output window ${String(index)} row start`),
|
|
991
|
+
rowCount: positiveUint32(Reflect.get(output, "rowCount"), `Rechunk output window ${String(index)} row count`),
|
|
992
|
+
};
|
|
993
|
+
});
|
|
994
|
+
validateContiguousRows(outputs, totalRows, "Rechunk output windows");
|
|
995
|
+
return {
|
|
996
|
+
kind: "rechunk-v1",
|
|
997
|
+
targetBlockBytes: positiveWholeNumber(Reflect.get(value, "targetBlockBytes"), "Rechunk target block bytes"),
|
|
998
|
+
outputCompression: compactionOutputCompression(Reflect.get(value, "outputCompression")),
|
|
999
|
+
totalRows,
|
|
1000
|
+
rowIdStart,
|
|
1001
|
+
rowIdEndExclusive,
|
|
1002
|
+
logicalOrder: nonNegativeWholeNumber(Reflect.get(value, "logicalOrder"), "Rechunk logical order"),
|
|
1003
|
+
columns,
|
|
1004
|
+
outputs,
|
|
1005
|
+
};
|
|
1006
|
+
}
|
|
1007
|
+
function normalizeRechunkSourceColumn(value, totalRows, columnIndex) {
|
|
1008
|
+
if (typeof value !== "object" || value === null) {
|
|
1009
|
+
throw new TypeError(`Rechunk source column ${String(columnIndex)} must be an object`);
|
|
1010
|
+
}
|
|
1011
|
+
const sourceBlocksValue = Reflect.get(value, "sourceBlocks");
|
|
1012
|
+
if (!Array.isArray(sourceBlocksValue) || sourceBlocksValue.length === 0) {
|
|
1013
|
+
throw new TypeError(`Rechunk source column ${String(columnIndex)} requires source blocks`);
|
|
1014
|
+
}
|
|
1015
|
+
const sourceBlocks = sourceBlocksValue.map((block, blockIndex) => {
|
|
1016
|
+
if (typeof block !== "object" || block === null) {
|
|
1017
|
+
throw new TypeError(`Rechunk source block ${String(columnIndex)}:${String(blockIndex)} must be an object`);
|
|
1018
|
+
}
|
|
1019
|
+
return {
|
|
1020
|
+
blockId: nonEmptyString(Reflect.get(block, "blockId"), `Rechunk source block ${String(columnIndex)}:${String(blockIndex)} ID`),
|
|
1021
|
+
rowStart: nonNegativeWholeNumber(Reflect.get(block, "rowStart"), `Rechunk source block ${String(columnIndex)}:${String(blockIndex)} row start`),
|
|
1022
|
+
rowCount: positiveUint32(Reflect.get(block, "rowCount"), `Rechunk source block ${String(columnIndex)}:${String(blockIndex)} row count`),
|
|
1023
|
+
storedBytes: positiveWholeNumber(Reflect.get(block, "storedBytes"), `Rechunk source block ${String(columnIndex)}:${String(blockIndex)} stored bytes`),
|
|
1024
|
+
encodedBytes: nonNegativeWholeNumber(Reflect.get(block, "encodedBytes"), `Rechunk source block ${String(columnIndex)}:${String(blockIndex)} encoded bytes`),
|
|
1025
|
+
checksum: uint32(Reflect.get(block, "checksum"), `Rechunk source block ${String(columnIndex)}:${String(blockIndex)} checksum`),
|
|
1026
|
+
};
|
|
1027
|
+
});
|
|
1028
|
+
validateContiguousRows(sourceBlocks, totalRows, `Rechunk source column ${String(columnIndex)} blocks`);
|
|
1029
|
+
return {
|
|
1030
|
+
columnId: nonEmptyString(Reflect.get(value, "columnId"), `Rechunk source column ${String(columnIndex)} ID`),
|
|
1031
|
+
type: simpleDataType(Reflect.get(value, "type")),
|
|
1032
|
+
sourceBlocks,
|
|
1033
|
+
};
|
|
1034
|
+
}
|
|
1035
|
+
function normalizeMergeCompactionRewritePlan(value) {
|
|
1036
|
+
const totalRows = nonNegativeWholeNumber(Reflect.get(value, "totalRows"), "Merge total row count");
|
|
1037
|
+
const rowIdStart = nonNegativeBigInt(Reflect.get(value, "rowIdStart"), "Merge row ID start");
|
|
1038
|
+
const rowIdEndExclusive = nonNegativeBigInt(Reflect.get(value, "rowIdEndExclusive"), "Merge row ID end");
|
|
1039
|
+
const rowIdSpans = normalizeRowIdSpans(Reflect.get(value, "rowIdSpans"), totalRows, rowIdStart, rowIdEndExclusive, "Merge output row ID spans");
|
|
1040
|
+
const keyColumnId = nonEmptyString(Reflect.get(value, "keyColumnId"), "Merge key column ID");
|
|
1041
|
+
const sourceSegmentsValue = Reflect.get(value, "sourceSegments");
|
|
1042
|
+
if (!Array.isArray(sourceSegmentsValue) || sourceSegmentsValue.length === 0) {
|
|
1043
|
+
throw new TypeError("A merge plan requires at least one source segment");
|
|
1044
|
+
}
|
|
1045
|
+
const sourceSegments = sourceSegmentsValue.map((segment, index) => normalizeMergeSourceSegment(segment, index));
|
|
1046
|
+
const sourceSegmentIds = sourceSegments.map((segment) => segment.segmentId);
|
|
1047
|
+
if (new Set(sourceSegmentIds).size !== sourceSegmentIds.length) {
|
|
1048
|
+
throw new TypeError("A merge source segment can only appear once");
|
|
1049
|
+
}
|
|
1050
|
+
for (let index = 1; index < sourceSegments.length; index += 1) {
|
|
1051
|
+
const previous = sourceSegments[index - 1];
|
|
1052
|
+
const current = sourceSegments[index];
|
|
1053
|
+
if (previous !== undefined &&
|
|
1054
|
+
current !== undefined &&
|
|
1055
|
+
compareMergeSourceSegments(previous, current) >= 0) {
|
|
1056
|
+
throw new TypeError("Merge source segments must use canonical logical order");
|
|
1057
|
+
}
|
|
1058
|
+
}
|
|
1059
|
+
const sourceBlocks = new Map();
|
|
1060
|
+
for (const segment of sourceSegments) {
|
|
1061
|
+
for (const column of segment.columns) {
|
|
1062
|
+
for (const block of column.sourceBlocks) {
|
|
1063
|
+
if (sourceBlocks.has(block.blockId)) {
|
|
1064
|
+
throw new TypeError("A merge source block can only appear once");
|
|
1065
|
+
}
|
|
1066
|
+
sourceBlocks.set(block.blockId, {
|
|
1067
|
+
columnId: column.columnId,
|
|
1068
|
+
type: column.type,
|
|
1069
|
+
rowCount: block.rowCount,
|
|
1070
|
+
});
|
|
1071
|
+
}
|
|
1072
|
+
}
|
|
1073
|
+
}
|
|
1074
|
+
const columnsValue = Reflect.get(value, "columns");
|
|
1075
|
+
if (!Array.isArray(columnsValue) || columnsValue.length === 0) {
|
|
1076
|
+
throw new TypeError("A merge plan requires at least one output column");
|
|
1077
|
+
}
|
|
1078
|
+
const columns = columnsValue.map((column, index) => normalizeMergeOutputColumn(column, totalRows, sourceBlocks, index));
|
|
1079
|
+
const columnIds = columns.map((column) => column.columnId);
|
|
1080
|
+
if (new Set(columnIds).size !== columnIds.length) {
|
|
1081
|
+
throw new TypeError("A merge plan cannot contain duplicate output columns");
|
|
1082
|
+
}
|
|
1083
|
+
if (!columnIds.includes(keyColumnId)) {
|
|
1084
|
+
throw new TypeError("A merge plan must output its key column");
|
|
1085
|
+
}
|
|
1086
|
+
validateMergeSourceShapes(sourceSegments, columns, keyColumnId);
|
|
1087
|
+
const outputsValue = Reflect.get(value, "outputs");
|
|
1088
|
+
if (!Array.isArray(outputsValue))
|
|
1089
|
+
throw new TypeError("Merge outputs must be an array");
|
|
1090
|
+
if ((totalRows === 0) !== (outputsValue.length === 0)) {
|
|
1091
|
+
throw new TypeError("Merge output windows must be empty exactly when no rows survive");
|
|
1092
|
+
}
|
|
1093
|
+
const outputs = outputsValue.map((output, index) => {
|
|
1094
|
+
if (typeof output !== "object" || output === null) {
|
|
1095
|
+
throw new TypeError(`Merge output window ${String(index)} must be an object`);
|
|
1096
|
+
}
|
|
1097
|
+
return {
|
|
1098
|
+
rowStart: nonNegativeWholeNumber(Reflect.get(output, "rowStart"), `Merge output window ${String(index)} row start`),
|
|
1099
|
+
rowCount: positiveUint32(Reflect.get(output, "rowCount"), `Merge output window ${String(index)} row count`),
|
|
1100
|
+
};
|
|
1101
|
+
});
|
|
1102
|
+
validateContiguousRows(outputs, totalRows, "Merge output windows");
|
|
1103
|
+
const logicalOrder = nonNegativeWholeNumber(Reflect.get(value, "logicalOrder"), "Merge logical order");
|
|
1104
|
+
if (logicalOrder !== sourceSegments[0]?.logicalOrder) {
|
|
1105
|
+
throw new TypeError("Merge logical order must match its earliest source segment");
|
|
1106
|
+
}
|
|
1107
|
+
return {
|
|
1108
|
+
kind: "merge-v1",
|
|
1109
|
+
targetBlockBytes: positiveWholeNumber(Reflect.get(value, "targetBlockBytes"), "Merge target block bytes"),
|
|
1110
|
+
outputCompression: compactionOutputCompression(Reflect.get(value, "outputCompression")),
|
|
1111
|
+
keyColumnId,
|
|
1112
|
+
totalRows,
|
|
1113
|
+
rowIdStart,
|
|
1114
|
+
rowIdEndExclusive,
|
|
1115
|
+
rowIdSpans,
|
|
1116
|
+
logicalOrder,
|
|
1117
|
+
sourceSegments,
|
|
1118
|
+
columns,
|
|
1119
|
+
outputs,
|
|
1120
|
+
};
|
|
1121
|
+
}
|
|
1122
|
+
function normalizeMergeSourceSegment(value, segmentIndex) {
|
|
1123
|
+
if (typeof value !== "object" || value === null) {
|
|
1124
|
+
throw new TypeError(`Merge source segment ${String(segmentIndex)} must be an object`);
|
|
1125
|
+
}
|
|
1126
|
+
const label = `Merge source segment ${String(segmentIndex)}`;
|
|
1127
|
+
const kind = segmentKind(Reflect.get(value, "kind"));
|
|
1128
|
+
const rowCount = positiveWholeNumber(Reflect.get(value, "rowCount"), `${label} row count`);
|
|
1129
|
+
const rowIdStart = nonNegativeBigInt(Reflect.get(value, "rowIdStart"), `${label} row ID start`);
|
|
1130
|
+
const rowIdEndExclusive = nonNegativeBigInt(Reflect.get(value, "rowIdEndExclusive"), `${label} row ID end`);
|
|
1131
|
+
let rowIdSpans;
|
|
1132
|
+
if (kind === "insert" || kind === "upsert" || kind === "base") {
|
|
1133
|
+
rowIdSpans = normalizeRowIdSpans(Reflect.get(value, "rowIdSpans"), rowCount, rowIdStart, rowIdEndExclusive, `${label} row ID spans`);
|
|
1134
|
+
}
|
|
1135
|
+
else {
|
|
1136
|
+
const spans = Reflect.get(value, "rowIdSpans");
|
|
1137
|
+
if (!Array.isArray(spans) || spans.length !== 0) {
|
|
1138
|
+
throw new TypeError(`${label} mutation markers cannot own row IDs`);
|
|
1139
|
+
}
|
|
1140
|
+
if (rowIdStart !== 0n || rowIdEndExclusive !== 0n) {
|
|
1141
|
+
throw new TypeError(`${label} mutation marker row ID envelope must be empty`);
|
|
1142
|
+
}
|
|
1143
|
+
rowIdSpans = [];
|
|
1144
|
+
}
|
|
1145
|
+
const columnsValue = Reflect.get(value, "columns");
|
|
1146
|
+
if (!Array.isArray(columnsValue) || columnsValue.length === 0) {
|
|
1147
|
+
throw new TypeError(`${label} requires at least one source column`);
|
|
1148
|
+
}
|
|
1149
|
+
const columns = columnsValue.map((column, columnIndex) => normalizeMergeSourceColumn(column, rowCount, segmentIndex, columnIndex));
|
|
1150
|
+
const columnIds = columns.map((column) => column.columnId);
|
|
1151
|
+
if (new Set(columnIds).size !== columnIds.length) {
|
|
1152
|
+
throw new TypeError(`${label} cannot contain duplicate source columns`);
|
|
1153
|
+
}
|
|
1154
|
+
return {
|
|
1155
|
+
segmentId: nonEmptyString(Reflect.get(value, "segmentId"), `${label} ID`),
|
|
1156
|
+
transactionId: nonEmptyString(Reflect.get(value, "transactionId"), `${label} transaction ID`),
|
|
1157
|
+
committedVersion: nonNegativeWholeNumber(Reflect.get(value, "committedVersion"), `${label} committed version`),
|
|
1158
|
+
kind,
|
|
1159
|
+
keyColumnId: nullableId(Reflect.get(value, "keyColumnId"), `${label} key column ID`),
|
|
1160
|
+
level: nonNegativeWholeNumber(Reflect.get(value, "level"), `${label} level`),
|
|
1161
|
+
logicalOrder: nonNegativeWholeNumber(Reflect.get(value, "logicalOrder"), `${label} logical order`),
|
|
1162
|
+
rowCount,
|
|
1163
|
+
rowIdStart,
|
|
1164
|
+
rowIdEndExclusive,
|
|
1165
|
+
rowIdSpans,
|
|
1166
|
+
columns,
|
|
1167
|
+
};
|
|
1168
|
+
}
|
|
1169
|
+
function normalizeMergeSourceColumn(value, segmentRowCount, segmentIndex, columnIndex) {
|
|
1170
|
+
const label = `Merge source column ${String(segmentIndex)}:${String(columnIndex)}`;
|
|
1171
|
+
if (typeof value !== "object" || value === null) {
|
|
1172
|
+
throw new TypeError(`${label} must be an object`);
|
|
1173
|
+
}
|
|
1174
|
+
const sourceBlocksValue = Reflect.get(value, "sourceBlocks");
|
|
1175
|
+
if (!Array.isArray(sourceBlocksValue) || sourceBlocksValue.length === 0) {
|
|
1176
|
+
throw new TypeError(`${label} requires source blocks`);
|
|
1177
|
+
}
|
|
1178
|
+
const sourceBlocks = sourceBlocksValue.map((block, blockIndex) => {
|
|
1179
|
+
if (typeof block !== "object" || block === null) {
|
|
1180
|
+
throw new TypeError(`${label} block ${String(blockIndex)} must be an object`);
|
|
1181
|
+
}
|
|
1182
|
+
return {
|
|
1183
|
+
blockId: nonEmptyString(Reflect.get(block, "blockId"), `${label} block ID`),
|
|
1184
|
+
rowStart: nonNegativeWholeNumber(Reflect.get(block, "rowStart"), `${label} block row start`),
|
|
1185
|
+
rowCount: positiveUint32(Reflect.get(block, "rowCount"), `${label} block row count`),
|
|
1186
|
+
storedBytes: positiveWholeNumber(Reflect.get(block, "storedBytes"), `${label} block stored bytes`),
|
|
1187
|
+
encodedBytes: nonNegativeWholeNumber(Reflect.get(block, "encodedBytes"), `${label} block encoded bytes`),
|
|
1188
|
+
checksum: uint32(Reflect.get(block, "checksum"), `${label} block checksum`),
|
|
1189
|
+
};
|
|
1190
|
+
});
|
|
1191
|
+
validateContiguousRows(sourceBlocks, segmentRowCount, `${label} blocks`);
|
|
1192
|
+
return {
|
|
1193
|
+
columnId: nonEmptyString(Reflect.get(value, "columnId"), `${label} ID`),
|
|
1194
|
+
type: simpleDataType(Reflect.get(value, "type")),
|
|
1195
|
+
sourceBlocks,
|
|
1196
|
+
};
|
|
1197
|
+
}
|
|
1198
|
+
function normalizeMergeOutputColumn(value, totalRows, sourceBlocks, columnIndex) {
|
|
1199
|
+
const label = `Merge output column ${String(columnIndex)}`;
|
|
1200
|
+
if (typeof value !== "object" || value === null) {
|
|
1201
|
+
throw new TypeError(`${label} must be an object`);
|
|
1202
|
+
}
|
|
1203
|
+
const columnId = nonEmptyString(Reflect.get(value, "columnId"), `${label} ID`);
|
|
1204
|
+
const type = simpleDataType(Reflect.get(value, "type"));
|
|
1205
|
+
const rangesValue = Reflect.get(value, "sourceRanges");
|
|
1206
|
+
if (!Array.isArray(rangesValue))
|
|
1207
|
+
throw new TypeError(`${label} source ranges must be an array`);
|
|
1208
|
+
const sourceRanges = rangesValue.map((range, rangeIndex) => {
|
|
1209
|
+
if (typeof range !== "object" || range === null) {
|
|
1210
|
+
throw new TypeError(`${label} source range ${String(rangeIndex)} must be an object`);
|
|
1211
|
+
}
|
|
1212
|
+
const normalized = {
|
|
1213
|
+
outputRowStart: nonNegativeWholeNumber(Reflect.get(range, "outputRowStart"), `${label} source range output row start`),
|
|
1214
|
+
sourceBlockId: nonEmptyString(Reflect.get(range, "sourceBlockId"), `${label} source range block ID`),
|
|
1215
|
+
sourceRowStart: nonNegativeWholeNumber(Reflect.get(range, "sourceRowStart"), `${label} source range block row start`),
|
|
1216
|
+
rowCount: positiveUint32(Reflect.get(range, "rowCount"), `${label} source range row count`),
|
|
1217
|
+
};
|
|
1218
|
+
const source = sourceBlocks.get(normalized.sourceBlockId);
|
|
1219
|
+
if (source === undefined)
|
|
1220
|
+
throw new TypeError(`${label} references an unknown source block`);
|
|
1221
|
+
if (source.columnId !== columnId || source.type !== type) {
|
|
1222
|
+
throw new TypeError(`${label} source range has the wrong column or type`);
|
|
1223
|
+
}
|
|
1224
|
+
if (safeSum([normalized.sourceRowStart, normalized.rowCount], `${label} source range rows`) >
|
|
1225
|
+
source.rowCount) {
|
|
1226
|
+
throw new RangeError(`${label} source range is outside its source block`);
|
|
1227
|
+
}
|
|
1228
|
+
return normalized;
|
|
1229
|
+
});
|
|
1230
|
+
let outputRowStart = 0;
|
|
1231
|
+
for (let index = 0; index < sourceRanges.length; index += 1) {
|
|
1232
|
+
const range = sourceRanges[index];
|
|
1233
|
+
if (range === undefined)
|
|
1234
|
+
continue;
|
|
1235
|
+
if (range.outputRowStart !== outputRowStart) {
|
|
1236
|
+
throw new RangeError(`${label} source ranges must cover output rows contiguously`);
|
|
1237
|
+
}
|
|
1238
|
+
const previous = sourceRanges[index - 1];
|
|
1239
|
+
if (previous?.sourceBlockId === range.sourceBlockId &&
|
|
1240
|
+
previous.sourceRowStart + previous.rowCount === range.sourceRowStart) {
|
|
1241
|
+
throw new TypeError(`${label} contains adjacent source ranges that must be coalesced`);
|
|
1242
|
+
}
|
|
1243
|
+
outputRowStart = safeSum([outputRowStart, range.rowCount], `${label} output row count`);
|
|
1244
|
+
}
|
|
1245
|
+
if (outputRowStart !== totalRows) {
|
|
1246
|
+
throw new RangeError(`${label} source ranges must cover every merged output row`);
|
|
1247
|
+
}
|
|
1248
|
+
return { columnId, type, sourceRanges };
|
|
1249
|
+
}
|
|
1250
|
+
function validateMergeSourceShapes(sourceSegments, outputColumns, keyColumnId) {
|
|
1251
|
+
const outputIds = outputColumns.map((column) => column.columnId);
|
|
1252
|
+
const outputTypes = new Map(outputColumns.map((column) => [column.columnId, column.type]));
|
|
1253
|
+
for (const segment of sourceSegments) {
|
|
1254
|
+
if (segment.keyColumnId !== keyColumnId) {
|
|
1255
|
+
throw new TypeError(`Merge source segment ${segment.segmentId} has the wrong key column`);
|
|
1256
|
+
}
|
|
1257
|
+
const sourceIds = segment.columns.map((column) => column.columnId);
|
|
1258
|
+
if (segment.columns.some((column) => outputTypes.get(column.columnId) !== column.type)) {
|
|
1259
|
+
throw new TypeError(`Merge source segment ${segment.segmentId} has an unknown column or type`);
|
|
1260
|
+
}
|
|
1261
|
+
const canonicalIds = outputIds.filter((id) => sourceIds.includes(id));
|
|
1262
|
+
if (canonicalIds.some((id, index) => sourceIds[index] !== id)) {
|
|
1263
|
+
throw new TypeError(`Merge source segment ${segment.segmentId} columns are not canonical`);
|
|
1264
|
+
}
|
|
1265
|
+
if (segment.kind === "insert" || segment.kind === "upsert" || segment.kind === "base") {
|
|
1266
|
+
if (sourceIds.length !== outputIds.length ||
|
|
1267
|
+
sourceIds.some((id, index) => outputIds[index] !== id)) {
|
|
1268
|
+
throw new TypeError(`Merge ${segment.kind} segment must contain every output column`);
|
|
1269
|
+
}
|
|
1270
|
+
}
|
|
1271
|
+
else if (segment.kind === "delete") {
|
|
1272
|
+
if (sourceIds.length !== 1 || sourceIds[0] !== keyColumnId) {
|
|
1273
|
+
throw new TypeError("A merge delete segment must contain only its key column");
|
|
1274
|
+
}
|
|
1275
|
+
}
|
|
1276
|
+
else if (sourceIds.length < 2 ||
|
|
1277
|
+
!sourceIds.includes(keyColumnId) ||
|
|
1278
|
+
sourceIds.every((id) => id === keyColumnId)) {
|
|
1279
|
+
throw new TypeError("A merge update segment requires its key and a changed column");
|
|
1280
|
+
}
|
|
1281
|
+
}
|
|
1282
|
+
}
|
|
1283
|
+
function normalizeRowIdSpans(value, totalRows, rowIdStart, rowIdEndExclusive, label) {
|
|
1284
|
+
if (!Array.isArray(value))
|
|
1285
|
+
throw new TypeError(`${label} must be an array`);
|
|
1286
|
+
const spans = value.map((span, index) => {
|
|
1287
|
+
if (typeof span !== "object" || span === null) {
|
|
1288
|
+
throw new TypeError(`${label} ${String(index)} must be an object`);
|
|
1289
|
+
}
|
|
1290
|
+
return {
|
|
1291
|
+
rowStart: nonNegativeWholeNumber(Reflect.get(span, "rowStart"), `${label} ${String(index)} row start`),
|
|
1292
|
+
rowCount: positiveWholeNumber(Reflect.get(span, "rowCount"), `${label} ${String(index)} row count`),
|
|
1293
|
+
rowIdStart: nonNegativeBigInt(Reflect.get(span, "rowIdStart"), `${label} ${String(index)} row ID start`),
|
|
1294
|
+
};
|
|
1295
|
+
});
|
|
1296
|
+
let rowStart = 0;
|
|
1297
|
+
for (let index = 0; index < spans.length; index += 1) {
|
|
1298
|
+
const span = spans[index];
|
|
1299
|
+
if (span === undefined)
|
|
1300
|
+
continue;
|
|
1301
|
+
if (span.rowStart !== rowStart)
|
|
1302
|
+
throw new RangeError(`${label} must cover rows contiguously`);
|
|
1303
|
+
const previous = spans[index - 1];
|
|
1304
|
+
if (previous !== undefined &&
|
|
1305
|
+
previous.rowIdStart + BigInt(previous.rowCount) === span.rowIdStart) {
|
|
1306
|
+
throw new TypeError(`${label} contains adjacent spans that must be coalesced`);
|
|
1307
|
+
}
|
|
1308
|
+
rowStart = safeSum([rowStart, span.rowCount], `${label} row count`);
|
|
1309
|
+
}
|
|
1310
|
+
if (rowStart !== totalRows)
|
|
1311
|
+
throw new RangeError(`${label} must cover every row`);
|
|
1312
|
+
if (spans.length === 0) {
|
|
1313
|
+
if (totalRows !== 0 || rowIdStart !== 0n || rowIdEndExclusive !== 0n) {
|
|
1314
|
+
throw new RangeError(`${label} has an invalid empty row ID envelope`);
|
|
1315
|
+
}
|
|
1316
|
+
return spans;
|
|
1317
|
+
}
|
|
1318
|
+
const intervals = spans
|
|
1319
|
+
.map((span) => ({ start: span.rowIdStart, end: span.rowIdStart + BigInt(span.rowCount) }))
|
|
1320
|
+
.sort((left, right) => (left.start < right.start ? -1 : left.start > right.start ? 1 : 0));
|
|
1321
|
+
for (let index = 1; index < intervals.length; index += 1) {
|
|
1322
|
+
const previous = intervals[index - 1];
|
|
1323
|
+
const current = intervals[index];
|
|
1324
|
+
if (previous !== undefined && current !== undefined && current.start < previous.end) {
|
|
1325
|
+
throw new RangeError(`${label} cannot contain overlapping row IDs`);
|
|
1326
|
+
}
|
|
1327
|
+
}
|
|
1328
|
+
const minimum = intervals[0]?.start;
|
|
1329
|
+
const maximum = intervals[intervals.length - 1]?.end;
|
|
1330
|
+
if (minimum !== rowIdStart || maximum !== rowIdEndExclusive) {
|
|
1331
|
+
throw new RangeError(`${label} must match its row ID envelope`);
|
|
1332
|
+
}
|
|
1333
|
+
return spans;
|
|
1334
|
+
}
|
|
1335
|
+
function compareMergeSourceSegments(left, right) {
|
|
1336
|
+
return (left.logicalOrder - right.logicalOrder ||
|
|
1337
|
+
left.committedVersion - right.committedVersion ||
|
|
1338
|
+
left.segmentId.localeCompare(right.segmentId));
|
|
1339
|
+
}
|
|
1340
|
+
function normalizeCompactionOutputCursor(value, plan) {
|
|
1341
|
+
if (plan.kind === "copy-v1") {
|
|
1342
|
+
if (value !== undefined && value !== null) {
|
|
1343
|
+
throw new TypeError("A copy compaction cannot have an output cursor");
|
|
1344
|
+
}
|
|
1345
|
+
return null;
|
|
1346
|
+
}
|
|
1347
|
+
if (value === undefined) {
|
|
1348
|
+
if (plan.kind === "merge-v1") {
|
|
1349
|
+
throw new TypeError("A merge compaction requires an explicit output cursor");
|
|
1350
|
+
}
|
|
1351
|
+
return { outputIndex: 0, columnIndex: 0, rowStart: plan.outputs[0]?.rowStart ?? 0 };
|
|
1352
|
+
}
|
|
1353
|
+
if (typeof value !== "object" || value === null) {
|
|
1354
|
+
throw new TypeError("Rechunk output cursor must be an object");
|
|
1355
|
+
}
|
|
1356
|
+
const cursor = {
|
|
1357
|
+
outputIndex: nonNegativeWholeNumber(Reflect.get(value, "outputIndex"), "Rechunk output cursor index"),
|
|
1358
|
+
columnIndex: nonNegativeWholeNumber(Reflect.get(value, "columnIndex"), "Rechunk output cursor column index"),
|
|
1359
|
+
rowStart: nonNegativeWholeNumber(Reflect.get(value, "rowStart"), "Rechunk output cursor row start"),
|
|
1360
|
+
};
|
|
1361
|
+
if (cursor.outputIndex === plan.outputs.length) {
|
|
1362
|
+
if (cursor.columnIndex !== 0 || cursor.rowStart !== plan.totalRows) {
|
|
1363
|
+
throw new RangeError("A completed rechunk output cursor is not canonical");
|
|
1364
|
+
}
|
|
1365
|
+
return cursor;
|
|
1366
|
+
}
|
|
1367
|
+
const output = plan.outputs[cursor.outputIndex];
|
|
1368
|
+
if (output === undefined || cursor.columnIndex >= plan.columns.length) {
|
|
1369
|
+
throw new RangeError("Rechunk output cursor is outside the output plan");
|
|
1370
|
+
}
|
|
1371
|
+
if (cursor.rowStart !== output.rowStart) {
|
|
1372
|
+
throw new RangeError("Rechunk output cursor row start does not match its output window");
|
|
1373
|
+
}
|
|
1374
|
+
return cursor;
|
|
1375
|
+
}
|
|
1376
|
+
function validateContiguousRows(ranges, totalRows, label) {
|
|
1377
|
+
let rowStart = 0;
|
|
1378
|
+
for (const range of ranges) {
|
|
1379
|
+
if (range.rowStart !== rowStart) {
|
|
1380
|
+
throw new RangeError(`${label} must cover rows contiguously from zero`);
|
|
1381
|
+
}
|
|
1382
|
+
rowStart = safeSum([rowStart, range.rowCount], `${label} row count`);
|
|
1383
|
+
}
|
|
1384
|
+
if (rowStart !== totalRows) {
|
|
1385
|
+
throw new RangeError(`${label} must cover every planned row`);
|
|
1386
|
+
}
|
|
1387
|
+
}
|
|
1388
|
+
function compactionOutputCompression(value) {
|
|
1389
|
+
if (typeof value !== "string" ||
|
|
1390
|
+
!compactionOutputCompressions.includes(value)) {
|
|
1391
|
+
throw new TypeError(`Invalid compaction output compression: ${String(value)}`);
|
|
1392
|
+
}
|
|
1393
|
+
return value;
|
|
1394
|
+
}
|
|
1395
|
+
function simpleDataType(value) {
|
|
1396
|
+
if (typeof value !== "string" || !simpleDataTypes.includes(value)) {
|
|
1397
|
+
throw new TypeError(`Invalid compaction column type: ${String(value)}`);
|
|
1398
|
+
}
|
|
1399
|
+
return value;
|
|
1400
|
+
}
|
|
1401
|
+
function segmentKind(value) {
|
|
1402
|
+
if (value !== "insert" &&
|
|
1403
|
+
value !== "upsert" &&
|
|
1404
|
+
value !== "update" &&
|
|
1405
|
+
value !== "delete" &&
|
|
1406
|
+
value !== "base") {
|
|
1407
|
+
throw new TypeError(`Invalid merge source segment kind: ${String(value)}`);
|
|
1408
|
+
}
|
|
1409
|
+
return value;
|
|
1410
|
+
}
|
|
1411
|
+
function compactionJobState(state) {
|
|
1412
|
+
if (typeof state !== "string" || !compactionJobStates.includes(state)) {
|
|
1413
|
+
throw new TypeError(`Invalid compaction job state: ${String(state)}`);
|
|
1414
|
+
}
|
|
1415
|
+
return state;
|
|
1416
|
+
}
|
|
1417
|
+
function garbageCollectionJobState(state) {
|
|
1418
|
+
if (typeof state !== "string" ||
|
|
1419
|
+
!garbageCollectionJobStates.includes(state)) {
|
|
1420
|
+
throw new TypeError(`Invalid garbage collection job state: ${String(state)}`);
|
|
1421
|
+
}
|
|
1422
|
+
return state;
|
|
1423
|
+
}
|
|
1424
|
+
function normalizeGarbageCollectionCursor(value) {
|
|
1425
|
+
if (typeof value !== "object" || value === null) {
|
|
1426
|
+
throw new TypeError("Garbage collection cursor must be an object");
|
|
1427
|
+
}
|
|
1428
|
+
return {
|
|
1429
|
+
manifestIndex: nonNegativeWholeNumber(Reflect.get(value, "manifestIndex"), "Garbage collection manifest cursor"),
|
|
1430
|
+
segmentIndex: nonNegativeWholeNumber(Reflect.get(value, "segmentIndex"), "Garbage collection segment cursor"),
|
|
1431
|
+
blockIndex: nonNegativeWholeNumber(Reflect.get(value, "blockIndex"), "Garbage collection block cursor"),
|
|
1432
|
+
};
|
|
1433
|
+
}
|
|
1434
|
+
function garbageCollectionJobComplete(record) {
|
|
1435
|
+
return (record.cursor.manifestIndex === record.candidateManifestVersions.length &&
|
|
1436
|
+
record.cursor.segmentIndex === record.candidateSegmentIds.length &&
|
|
1437
|
+
record.cursor.blockIndex === record.candidateBlockIds.length);
|
|
1438
|
+
}
|
|
1439
|
+
function uniqueIds(ids, label, sort) {
|
|
1440
|
+
if (!Array.isArray(ids))
|
|
1441
|
+
throw new TypeError(`${label}s must be an array`);
|
|
1442
|
+
const unique = [...new Set(ids.map((id) => nonEmptyString(id, label)))];
|
|
1443
|
+
return sort ? unique.sort() : unique;
|
|
1444
|
+
}
|
|
1445
|
+
function orderedUniqueIds(ids, label) {
|
|
1446
|
+
if (!Array.isArray(ids))
|
|
1447
|
+
throw new TypeError(`${label}s must be an array`);
|
|
1448
|
+
const normalized = ids.map((id) => nonEmptyString(id, label));
|
|
1449
|
+
if (new Set(normalized).size !== normalized.length) {
|
|
1450
|
+
throw new TypeError(`${label}s cannot contain duplicates`);
|
|
1451
|
+
}
|
|
1452
|
+
return normalized;
|
|
1453
|
+
}
|
|
1454
|
+
function uniqueWholeNumbers(values, label) {
|
|
1455
|
+
if (!Array.isArray(values))
|
|
1456
|
+
throw new TypeError(`${label}s must be an array`);
|
|
1457
|
+
return [...new Set(values.map((value) => nonNegativeWholeNumber(value, label)))].sort((left, right) => left - right);
|
|
1458
|
+
}
|
|
1459
|
+
function nullableId(id, label) {
|
|
1460
|
+
return id === null ? null : nonEmptyString(id, label);
|
|
1461
|
+
}
|
|
1462
|
+
function nonEmptyString(value, label) {
|
|
1463
|
+
if (typeof value !== "string" || value.length === 0) {
|
|
1464
|
+
throw new TypeError(`${label} cannot be empty`);
|
|
1465
|
+
}
|
|
1466
|
+
return value;
|
|
1467
|
+
}
|
|
1468
|
+
function validTimestamp(value, label) {
|
|
1469
|
+
const timestamp = nonEmptyString(value, label);
|
|
1470
|
+
if (!Number.isFinite(Date.parse(timestamp)))
|
|
1471
|
+
throw new TypeError(`${label} must be valid`);
|
|
1472
|
+
return timestamp;
|
|
1473
|
+
}
|
|
1474
|
+
function nonNegativeWholeNumber(value, label) {
|
|
1475
|
+
if (typeof value !== "number" || !Number.isSafeInteger(value) || value < 0) {
|
|
1476
|
+
throw new RangeError(`${label} must be a non-negative whole number`);
|
|
1477
|
+
}
|
|
1478
|
+
return value;
|
|
1479
|
+
}
|
|
1480
|
+
function nonNegativeBigInt(value, label) {
|
|
1481
|
+
if (typeof value !== "bigint" || value < 0n) {
|
|
1482
|
+
throw new RangeError(`${label} must be a non-negative bigint`);
|
|
1483
|
+
}
|
|
1484
|
+
return value;
|
|
1485
|
+
}
|
|
1486
|
+
function positiveWholeNumber(value, label) {
|
|
1487
|
+
const normalized = nonNegativeWholeNumber(value, label);
|
|
1488
|
+
if (normalized === 0)
|
|
1489
|
+
throw new RangeError(`${label} must be positive`);
|
|
1490
|
+
return normalized;
|
|
1491
|
+
}
|
|
1492
|
+
function positiveFiniteNumber(value, label) {
|
|
1493
|
+
if (typeof value !== "number" || !Number.isFinite(value) || value <= 0) {
|
|
1494
|
+
throw new RangeError(`${label} must be a positive finite number`);
|
|
1495
|
+
}
|
|
1496
|
+
return value;
|
|
1497
|
+
}
|
|
1498
|
+
function uint32(value, label) {
|
|
1499
|
+
const normalized = nonNegativeWholeNumber(value, label);
|
|
1500
|
+
if (normalized > 0xffff_ffff)
|
|
1501
|
+
throw new RangeError(`${label} must fit in 32 bits`);
|
|
1502
|
+
return normalized;
|
|
1503
|
+
}
|
|
1504
|
+
function positiveUint32(value, label) {
|
|
1505
|
+
const normalized = uint32(value, label);
|
|
1506
|
+
if (normalized === 0)
|
|
1507
|
+
throw new RangeError(`${label} must be positive`);
|
|
1508
|
+
return normalized;
|
|
1509
|
+
}
|
|
1510
|
+
function safeSum(values, label) {
|
|
1511
|
+
let total = 0;
|
|
1512
|
+
for (const value of values) {
|
|
1513
|
+
total += value;
|
|
1514
|
+
if (!Number.isSafeInteger(total))
|
|
1515
|
+
throw new RangeError(`${label} exceeds the safe range`);
|
|
1516
|
+
}
|
|
1517
|
+
return total;
|
|
1518
|
+
}
|
|
1519
|
+
function safeProduct(left, right, label) {
|
|
1520
|
+
const product = left * right;
|
|
1521
|
+
if (!Number.isSafeInteger(product))
|
|
1522
|
+
throw new RangeError(`${label} exceeds the safe range`);
|
|
1523
|
+
return product;
|
|
1524
|
+
}
|
|
1525
|
+
/** Floors an integer-times-double product without rounding the binary double upward. */
|
|
1526
|
+
export function floorWholeNumberProduct(left, right, label) {
|
|
1527
|
+
if (!Number.isSafeInteger(left) || left < 0 || !Number.isFinite(right) || right < 0) {
|
|
1528
|
+
throw new RangeError(`${label} exceeds the safe range`);
|
|
1529
|
+
}
|
|
1530
|
+
const bytes = new ArrayBuffer(8);
|
|
1531
|
+
const view = new DataView(bytes);
|
|
1532
|
+
view.setFloat64(0, right, false);
|
|
1533
|
+
const high = view.getUint32(0, false);
|
|
1534
|
+
const low = view.getUint32(4, false);
|
|
1535
|
+
const exponentBits = (high >>> 20) & 0x7ff;
|
|
1536
|
+
const fraction = (BigInt(high & 0x000f_ffff) << 32n) | BigInt(low);
|
|
1537
|
+
const significand = exponentBits === 0 ? fraction : (1n << 52n) | fraction;
|
|
1538
|
+
const binaryExponent = exponentBits === 0 ? -1074 : exponentBits - 1023 - 52;
|
|
1539
|
+
let product = BigInt(left) * significand;
|
|
1540
|
+
if (binaryExponent >= 0)
|
|
1541
|
+
product <<= BigInt(binaryExponent);
|
|
1542
|
+
else
|
|
1543
|
+
product >>= BigInt(-binaryExponent);
|
|
1544
|
+
if (product > BigInt(Number.MAX_SAFE_INTEGER)) {
|
|
1545
|
+
throw new RangeError(`${label} exceeds the safe range`);
|
|
1546
|
+
}
|
|
1547
|
+
return Number(product);
|
|
1548
|
+
}
|
|
1549
|
+
//# sourceMappingURL=types.js.map
|