@minnowdb/core 0.2.1 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +23 -32
- package/dist/block-format/block.d.ts +32 -13
- package/dist/block-format/block.js +179 -55
- package/dist/block-format/checksum.d.ts +2 -1
- package/dist/block-format/checksum.js +5 -2
- package/dist/block-format/codecs.d.ts +7 -2
- package/dist/block-format/codecs.js +53 -12
- package/dist/block-format/column.d.ts +3 -2
- package/dist/block-format/column.js +96 -30
- package/dist/block-format/index.d.ts +2 -2
- package/dist/block-format/index.js +2 -2
- package/dist/block-format/physical.d.ts +8 -2
- package/dist/block-format/physical.js +14 -6
- package/dist/block-format/types.d.ts +7 -3
- package/dist/block-format/types.js +0 -1
- package/dist/block-format/unicode.d.ts +11 -0
- package/dist/block-format/unicode.js +46 -0
- package/dist/date-value.d.ts +15 -0
- package/dist/date-value.js +64 -0
- package/dist/engine/artifact-cache.d.ts +2 -1
- package/dist/engine/artifact-cache.js +5 -1
- package/dist/engine/batch.d.ts +11 -3
- package/dist/engine/batch.js +17 -7
- package/dist/engine/buffered-writer.d.ts +2 -1
- package/dist/engine/buffered-writer.js +34 -9
- package/dist/engine/cache-limits.d.ts +25 -0
- package/dist/engine/cache-limits.js +25 -0
- package/dist/engine/catalog.d.ts +24 -5
- package/dist/engine/catalog.js +34 -7
- package/dist/engine/client.d.ts +29 -7
- package/dist/engine/client.js +216 -40
- package/dist/engine/database.d.ts +224 -39
- package/dist/engine/database.js +7475 -1539
- package/dist/engine/defaults.d.ts +4 -8
- package/dist/engine/defaults.js +46 -22
- package/dist/engine/errors.d.ts +19 -1
- package/dist/engine/errors.js +32 -2
- package/dist/engine/fts.d.ts +0 -6
- package/dist/engine/fts.js +41 -14
- package/dist/engine/group-index.d.ts +0 -1
- package/dist/engine/group-index.js +6 -11
- package/dist/engine/index.d.ts +16 -10
- package/dist/engine/index.js +12 -9
- package/dist/engine/join-index.d.ts +0 -1
- package/dist/engine/join-index.js +2 -2
- package/dist/engine/keyed-live.d.ts +57 -0
- package/dist/engine/keyed-live.js +226 -0
- package/dist/engine/live-api.d.ts +4 -0
- package/dist/engine/live-api.js +4 -0
- package/dist/engine/live.d.ts +43 -26
- package/dist/engine/live.js +504 -126
- package/dist/engine/memory.d.ts +2 -1
- package/dist/engine/memory.js +3 -2
- package/dist/engine/optimizer.d.ts +0 -1
- package/dist/engine/optimizer.js +262 -27
- package/dist/engine/query-api.d.ts +3 -0
- package/dist/engine/query-api.js +3 -0
- package/dist/engine/query-cache.d.ts +1 -2
- package/dist/engine/query-cache.js +7 -5
- package/dist/engine/query.d.ts +134 -29
- package/dist/engine/query.js +1668 -372
- package/dist/engine/result-wire.d.ts +0 -1
- package/dist/engine/result-wire.js +2 -2
- package/dist/engine/schema-wire.d.ts +6 -3
- package/dist/engine/schema-wire.js +40 -34
- package/dist/engine/schema.d.ts +156 -80
- package/dist/engine/schema.js +570 -108
- package/dist/engine/sort-keys.d.ts +4 -4
- package/dist/engine/sort-keys.js +44 -29
- package/dist/engine/sql-domains.d.ts +31 -0
- package/dist/engine/sql-domains.js +585 -0
- package/dist/engine/sql-driver.d.ts +19 -0
- package/dist/engine/sql-driver.js +1 -0
- package/dist/engine/sql-json.d.ts +7 -9
- package/dist/engine/sql-json.js +75 -15
- package/dist/engine/sql-semantics.d.ts +8 -3
- package/dist/engine/sql-semantics.js +458 -30
- package/dist/engine/typed-live.d.ts +67 -0
- package/dist/engine/typed-live.js +349 -0
- package/dist/engine/vector.d.ts +12 -2
- package/dist/engine/vector.js +635 -119
- package/dist/engine/worker-host.d.ts +9 -2
- package/dist/engine/worker-host.js +620 -152
- package/dist/engine/worker.d.ts +0 -1
- package/dist/engine/worker.js +3 -3
- package/dist/engine/write-block-planner.d.ts +18 -0
- package/dist/engine/write-block-planner.js +112 -0
- package/dist/index.d.ts +0 -1
- package/dist/index.js +0 -1
- package/dist/plan/index.d.ts +3 -4
- package/dist/plan/index.js +3 -4
- package/dist/storage/index.d.ts +1 -1
- package/dist/storage/index.js +1 -1
- package/dist/storage/indexeddb.d.ts +99 -51
- package/dist/storage/indexeddb.js +13448 -2660
- package/dist/storage/memory.d.ts +64 -57
- package/dist/storage/memory.js +1042 -128
- package/dist/storage/opfs/files.d.ts +18 -7
- package/dist/storage/opfs/files.js +115 -28
- package/dist/storage/opfs/index.d.ts +1 -1
- package/dist/storage/opfs/index.js +1 -1
- package/dist/storage/opfs/leader.d.ts +409 -75
- package/dist/storage/opfs/leader.js +4621 -650
- package/dist/storage/opfs/rpc.d.ts +15 -3
- package/dist/storage/opfs/rpc.js +224 -2
- package/dist/storage/opfs/snapshot-ledger.d.ts +40 -0
- package/dist/storage/opfs/snapshot-ledger.js +281 -0
- package/dist/storage/opfs/store.d.ts +33 -99
- package/dist/storage/opfs/store.js +453 -364
- package/dist/storage/persistence.d.ts +37 -0
- package/dist/storage/persistence.js +78 -0
- package/dist/storage/snapshot-stream.d.ts +24 -0
- package/dist/storage/snapshot-stream.js +897 -0
- package/dist/storage/snapshot.d.ts +9 -85
- package/dist/storage/snapshot.js +33 -265
- package/dist/storage/toolkit/extents.d.ts +41 -7
- package/dist/storage/toolkit/extents.js +402 -39
- package/dist/storage/toolkit/index.d.ts +4 -5
- package/dist/storage/toolkit/index.js +28 -4
- package/dist/storage/toolkit/record-core.d.ts +182 -55
- package/dist/storage/toolkit/record-core.js +6158 -1331
- package/dist/storage/toolkit/sync-file.d.ts +10 -1
- package/dist/storage/toolkit/sync-file.js +50 -2
- package/dist/storage/toolkit/wal.d.ts +23 -5
- package/dist/storage/toolkit/wal.js +89 -27
- package/dist/storage/toolkit/wire.d.ts +15 -2
- package/dist/storage/toolkit/wire.js +199 -10
- package/dist/storage/types.d.ts +1439 -204
- package/dist/storage/types.js +1745 -141
- package/dist/testing/block-store-conformance.d.ts +0 -1
- package/dist/testing/block-store-conformance.js +1009 -119
- package/dist/testing/index.d.ts +46 -26
- package/dist/testing/index.js +112 -57
- package/dist/testing/opfs-shim.d.ts +13 -3
- package/dist/testing/opfs-shim.js +40 -6
- package/dist/testing/simulator.d.ts +97 -0
- package/dist/testing/simulator.js +591 -0
- package/dist/testing/sqllogictest.d.ts +89 -0
- package/dist/testing/sqllogictest.js +434 -0
- package/dist/transactions/index.d.ts +92 -16
- package/dist/transactions/index.js +1074 -236
- package/dist/worker-protocol/index.d.ts +3 -2
- package/dist/worker-protocol/index.js +2 -5
- package/package.json +53 -3
- package/postgres-feature-profile.json +223 -0
- package/sql-feature-matrix.json +172 -252
- package/dist/block-format/block.d.ts.map +0 -1
- package/dist/block-format/block.js.map +0 -1
- package/dist/block-format/checksum.d.ts.map +0 -1
- package/dist/block-format/checksum.js.map +0 -1
- package/dist/block-format/codecs.d.ts.map +0 -1
- package/dist/block-format/codecs.js.map +0 -1
- package/dist/block-format/column.d.ts.map +0 -1
- package/dist/block-format/column.js.map +0 -1
- package/dist/block-format/index.d.ts.map +0 -1
- package/dist/block-format/index.js.map +0 -1
- package/dist/block-format/physical.d.ts.map +0 -1
- package/dist/block-format/physical.js.map +0 -1
- package/dist/block-format/types.d.ts.map +0 -1
- package/dist/block-format/types.js.map +0 -1
- package/dist/engine/artifact-cache.d.ts.map +0 -1
- package/dist/engine/artifact-cache.js.map +0 -1
- package/dist/engine/batch.d.ts.map +0 -1
- package/dist/engine/batch.js.map +0 -1
- package/dist/engine/buffered-writer.d.ts.map +0 -1
- package/dist/engine/buffered-writer.js.map +0 -1
- package/dist/engine/catalog.d.ts.map +0 -1
- package/dist/engine/catalog.js.map +0 -1
- package/dist/engine/client.d.ts.map +0 -1
- package/dist/engine/client.js.map +0 -1
- package/dist/engine/coordinator.d.ts +0 -17
- package/dist/engine/coordinator.d.ts.map +0 -1
- package/dist/engine/coordinator.js +0 -60
- package/dist/engine/coordinator.js.map +0 -1
- package/dist/engine/database.d.ts.map +0 -1
- package/dist/engine/database.js.map +0 -1
- package/dist/engine/defaults.d.ts.map +0 -1
- package/dist/engine/defaults.js.map +0 -1
- package/dist/engine/errors.d.ts.map +0 -1
- package/dist/engine/errors.js.map +0 -1
- package/dist/engine/fts.d.ts.map +0 -1
- package/dist/engine/fts.js.map +0 -1
- package/dist/engine/group-index.d.ts.map +0 -1
- package/dist/engine/group-index.js.map +0 -1
- package/dist/engine/index.d.ts.map +0 -1
- package/dist/engine/index.js.map +0 -1
- package/dist/engine/join-index.d.ts.map +0 -1
- package/dist/engine/join-index.js.map +0 -1
- package/dist/engine/live.d.ts.map +0 -1
- package/dist/engine/live.js.map +0 -1
- package/dist/engine/memory.d.ts.map +0 -1
- package/dist/engine/memory.js.map +0 -1
- package/dist/engine/optimizer.d.ts.map +0 -1
- package/dist/engine/optimizer.js.map +0 -1
- package/dist/engine/query-cache.d.ts.map +0 -1
- package/dist/engine/query-cache.js.map +0 -1
- package/dist/engine/query.d.ts.map +0 -1
- package/dist/engine/query.js.map +0 -1
- package/dist/engine/result-wire.d.ts.map +0 -1
- package/dist/engine/result-wire.js.map +0 -1
- package/dist/engine/schema-wire.d.ts.map +0 -1
- package/dist/engine/schema-wire.js.map +0 -1
- package/dist/engine/schema.d.ts.map +0 -1
- package/dist/engine/schema.js.map +0 -1
- package/dist/engine/sort-keys.d.ts.map +0 -1
- package/dist/engine/sort-keys.js.map +0 -1
- package/dist/engine/sql-json.d.ts.map +0 -1
- package/dist/engine/sql-json.js.map +0 -1
- package/dist/engine/sql-semantics.d.ts.map +0 -1
- package/dist/engine/sql-semantics.js.map +0 -1
- package/dist/engine/vector.d.ts.map +0 -1
- package/dist/engine/vector.js.map +0 -1
- package/dist/engine/worker-host.d.ts.map +0 -1
- package/dist/engine/worker-host.js.map +0 -1
- package/dist/engine/worker.d.ts.map +0 -1
- package/dist/engine/worker.js.map +0 -1
- package/dist/index.d.ts.map +0 -1
- package/dist/index.js.map +0 -1
- package/dist/plan/index.d.ts.map +0 -1
- package/dist/plan/index.js.map +0 -1
- package/dist/storage/fixture-shape.d.ts +0 -42
- package/dist/storage/fixture-shape.d.ts.map +0 -1
- package/dist/storage/fixture-shape.js +0 -146
- package/dist/storage/fixture-shape.js.map +0 -1
- package/dist/storage/index.d.ts.map +0 -1
- package/dist/storage/index.js.map +0 -1
- package/dist/storage/indexeddb.d.ts.map +0 -1
- package/dist/storage/indexeddb.js.map +0 -1
- package/dist/storage/memory.d.ts.map +0 -1
- package/dist/storage/memory.js.map +0 -1
- package/dist/storage/opfs/files.d.ts.map +0 -1
- package/dist/storage/opfs/files.js.map +0 -1
- package/dist/storage/opfs/index.d.ts.map +0 -1
- package/dist/storage/opfs/index.js.map +0 -1
- package/dist/storage/opfs/leader.d.ts.map +0 -1
- package/dist/storage/opfs/leader.js.map +0 -1
- package/dist/storage/opfs/rpc.d.ts.map +0 -1
- package/dist/storage/opfs/rpc.js.map +0 -1
- package/dist/storage/opfs/store.d.ts.map +0 -1
- package/dist/storage/opfs/store.js.map +0 -1
- package/dist/storage/snapshot.d.ts.map +0 -1
- package/dist/storage/snapshot.js.map +0 -1
- package/dist/storage/toolkit/extents.d.ts.map +0 -1
- package/dist/storage/toolkit/extents.js.map +0 -1
- package/dist/storage/toolkit/index.d.ts.map +0 -1
- package/dist/storage/toolkit/index.js.map +0 -1
- package/dist/storage/toolkit/record-core.d.ts.map +0 -1
- package/dist/storage/toolkit/record-core.js.map +0 -1
- package/dist/storage/toolkit/sync-file.d.ts.map +0 -1
- package/dist/storage/toolkit/sync-file.js.map +0 -1
- package/dist/storage/toolkit/wal.d.ts.map +0 -1
- package/dist/storage/toolkit/wal.js.map +0 -1
- package/dist/storage/toolkit/wire.d.ts.map +0 -1
- package/dist/storage/toolkit/wire.js.map +0 -1
- package/dist/storage/types.d.ts.map +0 -1
- package/dist/storage/types.js.map +0 -1
- package/dist/testing/block-store-conformance.d.ts.map +0 -1
- package/dist/testing/block-store-conformance.js.map +0 -1
- package/dist/testing/index.d.ts.map +0 -1
- package/dist/testing/index.js.map +0 -1
- package/dist/testing/opfs-shim.d.ts.map +0 -1
- package/dist/testing/opfs-shim.js.map +0 -1
- package/dist/testing/seeds.d.ts +0 -11
- package/dist/testing/seeds.d.ts.map +0 -1
- package/dist/testing/seeds.js +0 -50
- package/dist/testing/seeds.js.map +0 -1
- package/dist/transactions/index.d.ts.map +0 -1
- package/dist/transactions/index.js.map +0 -1
- package/dist/worker-protocol/index.d.ts.map +0 -1
- package/dist/worker-protocol/index.js.map +0 -1
package/dist/storage/types.js
CHANGED
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
import { dateIsoString, dateMilliseconds } from "../date-value.js";
|
|
2
|
+
import { assertWellFormedString, MAX_BLOCK_ROW_COUNT, MAX_STORED_BLOCK_BYTE_LENGTH, } from "../block-format/index.js";
|
|
1
3
|
export const storeNames = [
|
|
2
4
|
"catalog",
|
|
3
5
|
"manifests",
|
|
@@ -9,22 +11,59 @@ export const storeNames = [
|
|
|
9
11
|
"temp",
|
|
10
12
|
"gc",
|
|
11
13
|
];
|
|
12
|
-
|
|
13
|
-
export
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
return;
|
|
14
|
+
export const MAX_MANIFEST_CHANGED_TABLE_IDS = 1_024;
|
|
15
|
+
export function canonicalManifestChangedTableIds(ids) {
|
|
16
|
+
if (ids.length > MAX_MANIFEST_CHANGED_TABLE_IDS) {
|
|
17
|
+
throw new RangeError(`Manifest changed-table IDs cannot exceed ${String(MAX_MANIFEST_CHANGED_TABLE_IDS)}`);
|
|
18
|
+
}
|
|
19
|
+
const canonical = [...new Set(ids.map((id) => validateStorageId(id, "Changed table ID")))].sort();
|
|
20
|
+
if (canonical.length > MAX_MANIFEST_CHANGED_TABLE_IDS) {
|
|
21
|
+
throw new RangeError(`Manifest changed-table IDs cannot exceed ${String(MAX_MANIFEST_CHANGED_TABLE_IDS)}`);
|
|
21
22
|
}
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
23
|
+
return canonical;
|
|
24
|
+
}
|
|
25
|
+
export function validateCanonicalManifestChangedTableIds(ids) {
|
|
26
|
+
const canonical = canonicalManifestChangedTableIds(ids);
|
|
27
|
+
if (canonical.length !== ids.length || canonical.some((id, index) => id !== ids[index])) {
|
|
28
|
+
throw new TypeError("Manifest changed-table IDs must be unique and lexically sorted");
|
|
29
|
+
}
|
|
30
|
+
return canonical;
|
|
26
31
|
}
|
|
27
32
|
export const simpleDataTypes = ["boolean", "number", "string", "datetime"];
|
|
33
|
+
/** Validates the logical metadata shared by SQL DDL, the schema DSL, and restored catalogs. */
|
|
34
|
+
export function validateSqlDomain(domain, context) {
|
|
35
|
+
if (domain.kind === "numeric") {
|
|
36
|
+
const { precision, scale } = domain;
|
|
37
|
+
if ((precision !== undefined &&
|
|
38
|
+
(!Number.isSafeInteger(precision) || precision < 1 || precision > 100_000)) ||
|
|
39
|
+
(scale !== undefined && (!Number.isSafeInteger(scale) || scale < 0 || scale > 100_000)) ||
|
|
40
|
+
(precision !== undefined && scale !== undefined && scale > precision)) {
|
|
41
|
+
throw new TypeError(`Invalid NUMERIC domain metadata: ${context}`);
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
if (domain.kind === "array" &&
|
|
45
|
+
(domain.element.trim().length === 0 || domain.element.trim() !== domain.element)) {
|
|
46
|
+
throw new TypeError(`ARRAY element type must be a trimmed non-empty name: ${context}`);
|
|
47
|
+
}
|
|
48
|
+
if (domain.kind === "enum") {
|
|
49
|
+
if (domain.name.trim().length === 0 || domain.name.trim() !== domain.name) {
|
|
50
|
+
throw new TypeError(`Enum type name must be a trimmed non-empty name: ${context}`);
|
|
51
|
+
}
|
|
52
|
+
validateEnumValues(domain.values, domain.name);
|
|
53
|
+
}
|
|
54
|
+
return structuredClone(domain);
|
|
55
|
+
}
|
|
56
|
+
export const MAX_TABLE_COLUMNS = 1_024;
|
|
57
|
+
export const MAX_ENUM_VALUES = 4_096;
|
|
58
|
+
export const MAX_SECONDARY_INDEXES = 1_024;
|
|
59
|
+
export const MAX_TABLE_TRIGGERS = 256;
|
|
60
|
+
export const MAX_TRIGGER_STATEMENTS = 256;
|
|
61
|
+
const VALID_TRIGGER_EVENTS = ["insert", "update", "delete"];
|
|
62
|
+
const VALID_TRIGGER_TIMINGS = ["before", "after"];
|
|
63
|
+
const VALID_TRIGGER_BINDING_SOURCES = ["new", "old"];
|
|
64
|
+
export const MAX_TABLE_CONSTRAINTS = 1_024;
|
|
65
|
+
export const MAX_TABLE_RECORD_CHARACTERS = 1_048_576;
|
|
66
|
+
export const MAX_TABLE_RECORD_ENTRIES = 65_536;
|
|
28
67
|
/**
|
|
29
68
|
* The single authority on which enum declarations are legal, shared by the schema DSL's
|
|
30
69
|
* `column.enum()` and the engine's `createTable`: at least one value, every value a non-empty
|
|
@@ -34,6 +73,9 @@ export function validateEnumValues(values, context) {
|
|
|
34
73
|
if (values.length === 0) {
|
|
35
74
|
throw new TypeError(`An enum needs at least one value: ${context}`);
|
|
36
75
|
}
|
|
76
|
+
if (values.length > MAX_ENUM_VALUES) {
|
|
77
|
+
throw new RangeError(`An enum cannot exceed ${String(MAX_ENUM_VALUES)} values: ${context}`);
|
|
78
|
+
}
|
|
37
79
|
const seen = new Set();
|
|
38
80
|
for (const value of values) {
|
|
39
81
|
if (typeof value !== "string" || value.length === 0) {
|
|
@@ -46,22 +88,100 @@ export function validateEnumValues(values, context) {
|
|
|
46
88
|
}
|
|
47
89
|
return [...values];
|
|
48
90
|
}
|
|
91
|
+
/**
|
|
92
|
+
* Validates the catalog invariants owned by a table's column list. Storage adapters call this
|
|
93
|
+
* for ordinary catalog writes and before restoring snapshots or checkpoints, so malformed
|
|
94
|
+
* metadata cannot enter through a less common persistence path.
|
|
95
|
+
*/
|
|
96
|
+
export function validateTableColumns(columns) {
|
|
97
|
+
const runtimeColumns = columns;
|
|
98
|
+
if (!Array.isArray(runtimeColumns) || columns.length === 0) {
|
|
99
|
+
throw new TypeError("A table needs at least one column");
|
|
100
|
+
}
|
|
101
|
+
if (columns.length > MAX_TABLE_COLUMNS) {
|
|
102
|
+
throw new RangeError(`A table cannot exceed ${String(MAX_TABLE_COLUMNS)} columns`);
|
|
103
|
+
}
|
|
104
|
+
const runtimeEntries = runtimeColumns;
|
|
105
|
+
for (const entry of runtimeEntries) {
|
|
106
|
+
if (!hasValidColumnPrimitives(entry)) {
|
|
107
|
+
throw new TypeError("Table columns have invalid primitive metadata");
|
|
108
|
+
}
|
|
109
|
+
const column = entry;
|
|
110
|
+
validateStorageId(column.id, "Column ID");
|
|
111
|
+
validateCatalogName(column.name, "Column name");
|
|
112
|
+
const integer = column.integer;
|
|
113
|
+
if (integer !== undefined && (integer !== true || column.type !== "number")) {
|
|
114
|
+
throw new TypeError(`Integer domain requires a number column: ${column.name}`);
|
|
115
|
+
}
|
|
116
|
+
if (column.sqlDomain !== undefined && column.type !== "string") {
|
|
117
|
+
throw new TypeError(`SQL domain columns must use string storage: ${column.name}`);
|
|
118
|
+
}
|
|
119
|
+
if (column.sqlDomain !== undefined)
|
|
120
|
+
validateSqlDomain(column.sqlDomain, column.name);
|
|
121
|
+
if (column.integer === true && column.sqlDomain !== undefined) {
|
|
122
|
+
throw new TypeError(`A column cannot have both integer and SQL-domain metadata: ${column.name}`);
|
|
123
|
+
}
|
|
124
|
+
if (column.enumValues !== undefined && column.sqlDomain !== undefined) {
|
|
125
|
+
throw new TypeError(`A column cannot have both enum restrictions and a SQL domain: ${column.name}`);
|
|
126
|
+
}
|
|
127
|
+
if (column.enumValues !== undefined) {
|
|
128
|
+
if (column.type !== "string") {
|
|
129
|
+
throw new TypeError(`Enum restrictions require a string column: ${column.name}`);
|
|
130
|
+
}
|
|
131
|
+
validateEnumValues(column.enumValues, column.name);
|
|
132
|
+
}
|
|
133
|
+
if (column.defaultValue !== undefined) {
|
|
134
|
+
validateColumnDefault({
|
|
135
|
+
name: column.name,
|
|
136
|
+
type: column.type,
|
|
137
|
+
nullable: column.nullable,
|
|
138
|
+
isUniqueKey: true,
|
|
139
|
+
...(column.integer === undefined ? {} : { integer: column.integer }),
|
|
140
|
+
...(column.sqlDomain === undefined ? {} : { sqlDomain: column.sqlDomain }),
|
|
141
|
+
...(column.enumValues === undefined ? {} : { enumValues: column.enumValues }),
|
|
142
|
+
}, column.defaultValue);
|
|
143
|
+
}
|
|
144
|
+
if (column.backfill !== undefined) {
|
|
145
|
+
const value = column.backfill;
|
|
146
|
+
const validType = column.type === "datetime"
|
|
147
|
+
? value instanceof Date && Number.isFinite(dateMilliseconds(value))
|
|
148
|
+
: typeof value === column.type;
|
|
149
|
+
if (!validType ||
|
|
150
|
+
(typeof value === "number" && !Number.isFinite(value)) ||
|
|
151
|
+
(column.integer === true && !Number.isSafeInteger(value)) ||
|
|
152
|
+
(column.enumValues !== undefined &&
|
|
153
|
+
typeof value === "string" &&
|
|
154
|
+
!column.enumValues.includes(value))) {
|
|
155
|
+
throw new TypeError(`Invalid backfill value: ${column.name}`);
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
const ids = new Set(columns.map(({ id }) => id));
|
|
160
|
+
const names = new Set(columns.map(({ name }) => name));
|
|
161
|
+
if (ids.size !== columns.length || names.size !== columns.length) {
|
|
162
|
+
throw new TypeError("Table columns must have unique IDs and names");
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
function hasValidColumnPrimitives(value) {
|
|
166
|
+
if (value === null || typeof value !== "object")
|
|
167
|
+
return false;
|
|
168
|
+
const column = value;
|
|
169
|
+
return (typeof column.id === "string" &&
|
|
170
|
+
column.id.length > 0 &&
|
|
171
|
+
typeof column.name === "string" &&
|
|
172
|
+
column.name.length > 0 &&
|
|
173
|
+
simpleDataTypes.some((type) => type === column.type) &&
|
|
174
|
+
typeof column.nullable === "boolean" &&
|
|
175
|
+
(column.hidden === undefined || column.hidden === true));
|
|
176
|
+
}
|
|
49
177
|
/**
|
|
50
178
|
* The single authority on which default declarations are legal, shared by the schema DSL's
|
|
51
179
|
* `table()` and the engine's `createTable` so the two entry points (and the wire path between
|
|
52
|
-
* them) can never drift
|
|
53
|
-
*
|
|
180
|
+
* them) can never drift. Storage owns structural and literal validation; the engine additionally
|
|
181
|
+
* parses and type-checks SQL expressions before catalog mutation.
|
|
54
182
|
*/
|
|
55
183
|
export function validateColumnDefault(column, defaultValue) {
|
|
56
|
-
if (column.nullable) {
|
|
57
|
-
throw new TypeError(`Defaults require a non-nullable column: ${column.name}`);
|
|
58
|
-
}
|
|
59
184
|
switch (defaultValue.kind) {
|
|
60
|
-
case "now":
|
|
61
|
-
if (column.type !== "datetime") {
|
|
62
|
-
throw new TypeError(`Default now requires a datetime column: ${column.name}`);
|
|
63
|
-
}
|
|
64
|
-
return;
|
|
65
185
|
case "autoincrement":
|
|
66
186
|
if (column.type !== "number") {
|
|
67
187
|
throw new TypeError(`Auto-increment requires a number column: ${column.name}`);
|
|
@@ -70,32 +190,321 @@ export function validateColumnDefault(column, defaultValue) {
|
|
|
70
190
|
throw new TypeError(`Auto-increment requires the unique key column: ${column.name}`);
|
|
71
191
|
}
|
|
72
192
|
return;
|
|
193
|
+
case "expression":
|
|
194
|
+
if (typeof defaultValue.sql !== "string" ||
|
|
195
|
+
defaultValue.sql.length === 0 ||
|
|
196
|
+
defaultValue.sql.trim() !== defaultValue.sql) {
|
|
197
|
+
throw new TypeError(`Default SQL must be a trimmed non-empty expression: ${column.name}`);
|
|
198
|
+
}
|
|
199
|
+
return;
|
|
73
200
|
case "literal": {
|
|
74
201
|
const value = defaultValue.value;
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
202
|
+
const numericDomain = column.sqlDomain?.kind === "numeric";
|
|
203
|
+
const correctType = column.type === "datetime"
|
|
204
|
+
? value instanceof Date && Number.isFinite(dateMilliseconds(value))
|
|
205
|
+
: numericDomain
|
|
206
|
+
? typeof value === "number" || typeof value === "string"
|
|
207
|
+
: typeof value === column.type;
|
|
208
|
+
if (!correctType) {
|
|
79
209
|
throw new TypeError(`Default literal must be a ${column.type}: ${column.name}`);
|
|
80
210
|
}
|
|
81
211
|
if (typeof value === "number" && !Number.isFinite(value)) {
|
|
82
212
|
throw new TypeError(`Default literal must be finite: ${column.name}`);
|
|
83
213
|
}
|
|
214
|
+
if (column.integer === true && !Number.isSafeInteger(value)) {
|
|
215
|
+
throw new TypeError(`Default literal must be a safe integer: ${column.name}`);
|
|
216
|
+
}
|
|
84
217
|
if (column.enumValues !== undefined &&
|
|
85
218
|
typeof value === "string" &&
|
|
86
219
|
!column.enumValues.includes(value)) {
|
|
87
220
|
throw new TypeError(`Default must be one of the enum values: ${column.name}`);
|
|
88
221
|
}
|
|
89
|
-
if (column.
|
|
90
|
-
|
|
222
|
+
if (column.sqlDomain?.kind === "enum" &&
|
|
223
|
+
typeof value === "string" &&
|
|
224
|
+
!column.sqlDomain.values.includes(value)) {
|
|
225
|
+
throw new TypeError(`Default must be one of the enum values: ${column.name}`);
|
|
91
226
|
}
|
|
92
227
|
return;
|
|
93
228
|
}
|
|
94
|
-
//
|
|
229
|
+
// Storage restoration and untyped wire callers can still supply malformed catalog data.
|
|
95
230
|
default:
|
|
96
231
|
throw new TypeError(`Unknown default kind: ${String(defaultValue.kind)}`);
|
|
97
232
|
}
|
|
98
233
|
}
|
|
234
|
+
/**
|
|
235
|
+
* Whether a secondary-index replacement changes the contract a staged writer must honor.
|
|
236
|
+
* Physical posting-build state is deliberately excluded: postings are only a reverified pruning
|
|
237
|
+
* accelerator. Index identity, key shape, and UNIQUE enforcement are structural.
|
|
238
|
+
*/
|
|
239
|
+
export function secondaryIndexWriteContractChanged(previous, next) {
|
|
240
|
+
const previousEntries = Object.entries(previous ?? {}).sort(([left], [right]) => left.localeCompare(right));
|
|
241
|
+
const nextEntries = Object.entries(next ?? {}).sort(([left], [right]) => left.localeCompare(right));
|
|
242
|
+
if (previousEntries.length !== nextEntries.length)
|
|
243
|
+
return true;
|
|
244
|
+
return previousEntries.some(([id, left], position) => {
|
|
245
|
+
const rightEntry = nextEntries[position];
|
|
246
|
+
if (rightEntry?.[0] !== id)
|
|
247
|
+
return true;
|
|
248
|
+
const right = rightEntry[1];
|
|
249
|
+
return (left.name !== right.name ||
|
|
250
|
+
left.columnId !== right.columnId ||
|
|
251
|
+
left.columnIds.length !== right.columnIds.length ||
|
|
252
|
+
left.columnIds.some((columnId, index) => columnId !== right.columnIds[index]) ||
|
|
253
|
+
left.directions.length !== right.directions.length ||
|
|
254
|
+
left.directions.some((direction, index) => direction !== right.directions[index]) ||
|
|
255
|
+
left.unique !== right.unique ||
|
|
256
|
+
left.uniqueEnforced !== right.uniqueEnforced ||
|
|
257
|
+
left.storageColumnId !== right.storageColumnId ||
|
|
258
|
+
left.locator !== right.locator);
|
|
259
|
+
});
|
|
260
|
+
}
|
|
261
|
+
const catalogRecordTextEncoder = new TextEncoder();
|
|
262
|
+
function durableRecordRetainedBytes(record, label) {
|
|
263
|
+
// Keep this byte accounting identical to the canonical record-wire JSON codec without
|
|
264
|
+
// importing the toolkit codec back into this leaf contract module. Segment records always
|
|
265
|
+
// contain bigint row envelopes, so plain JSON.stringify is not sufficient.
|
|
266
|
+
let json;
|
|
267
|
+
try {
|
|
268
|
+
json = JSON.stringify(record);
|
|
269
|
+
}
|
|
270
|
+
catch {
|
|
271
|
+
json = JSON.stringify(record, (_key, entry) => {
|
|
272
|
+
if (typeof entry !== "bigint")
|
|
273
|
+
return entry;
|
|
274
|
+
if (entry < 0n || entry > MAX_ROW_ID_EXCLUSIVE_END) {
|
|
275
|
+
throw new RangeError("Record bigint exceeds the unsigned 64-bit persisted range");
|
|
276
|
+
}
|
|
277
|
+
return { $n: entry.toString() };
|
|
278
|
+
});
|
|
279
|
+
}
|
|
280
|
+
if (typeof json !== "string")
|
|
281
|
+
throw new TypeError(`${label} is not JSON-serializable`);
|
|
282
|
+
return catalogRecordTextEncoder.encode(json).byteLength;
|
|
283
|
+
}
|
|
284
|
+
/** Exact UTF-8 bytes used when the canonical record-wire JSON codec persists a table record. */
|
|
285
|
+
export function catalogRecordRetainedBytes(record) {
|
|
286
|
+
return durableRecordRetainedBytes(record, "Table record");
|
|
287
|
+
}
|
|
288
|
+
export function manifestRecordRetainedBytes(record) {
|
|
289
|
+
return durableRecordRetainedBytes(record, "Manifest record");
|
|
290
|
+
}
|
|
291
|
+
/**
|
|
292
|
+
* Admission charge for one manifest summary, including the exact canonical tombstone bytes that
|
|
293
|
+
* later reclamation may add. Reserving them up front prevents quota deadlock at the byte ceiling.
|
|
294
|
+
*/
|
|
295
|
+
export function manifestRecordRetainedReservationBytes(record) {
|
|
296
|
+
return manifestRecordRetainedBytes(record.prunedAt === undefined ? { ...record, prunedAt: "1970-01-01T00:00:00.000Z" } : record);
|
|
297
|
+
}
|
|
298
|
+
export function segmentRecordRetainedBytes(record) {
|
|
299
|
+
return durableRecordRetainedBytes(record, "Segment record");
|
|
300
|
+
}
|
|
301
|
+
/** Hard catalog cardinality/text bounds shared by engine, adapters, checkpoints, and snapshots. */
|
|
302
|
+
export function validateTableRecordBounds(record) {
|
|
303
|
+
validateStorageId(record.id, "Table ID");
|
|
304
|
+
validateCatalogName(record.name, "Table name");
|
|
305
|
+
if (typeof record.managed !== "boolean")
|
|
306
|
+
throw new TypeError("Table managed flag is required");
|
|
307
|
+
if (!Number.isSafeInteger(record.revision) || record.revision < 0) {
|
|
308
|
+
throw new RangeError("Table revision must be a non-negative safe integer");
|
|
309
|
+
}
|
|
310
|
+
if (record.view !== undefined && typeof record.view.managed !== "boolean") {
|
|
311
|
+
throw new TypeError("View managed flag is required");
|
|
312
|
+
}
|
|
313
|
+
if ((record.primaryKeyColumnIds?.length ?? 0) > MAX_TABLE_COLUMNS) {
|
|
314
|
+
throw new RangeError("A primary key names too many columns");
|
|
315
|
+
}
|
|
316
|
+
if (Object.keys(record.ftsColumns ?? {}).length > MAX_TABLE_COLUMNS) {
|
|
317
|
+
throw new RangeError("A table has too many full-text columns");
|
|
318
|
+
}
|
|
319
|
+
if (Object.keys(record.secondaryIndexes ?? {}).length > MAX_SECONDARY_INDEXES) {
|
|
320
|
+
throw new RangeError(`A table cannot exceed ${String(MAX_SECONDARY_INDEXES)} secondary indexes`);
|
|
321
|
+
}
|
|
322
|
+
if ((record.triggers?.length ?? 0) > MAX_TABLE_TRIGGERS) {
|
|
323
|
+
throw new RangeError(`A table cannot exceed ${String(MAX_TABLE_TRIGGERS)} triggers`);
|
|
324
|
+
}
|
|
325
|
+
const triggerIds = new Set();
|
|
326
|
+
const triggerNames = new Set();
|
|
327
|
+
const columnNames = new Set(record.columns.map((column) => column.name));
|
|
328
|
+
for (const trigger of record.triggers ?? []) {
|
|
329
|
+
validateStorageId(trigger.id, "Trigger ID");
|
|
330
|
+
validateCatalogName(trigger.name, "Trigger name");
|
|
331
|
+
if (!VALID_TRIGGER_EVENTS.includes(trigger.event)) {
|
|
332
|
+
throw new TypeError(`Trigger event is invalid: ${trigger.name}`);
|
|
333
|
+
}
|
|
334
|
+
if (!VALID_TRIGGER_TIMINGS.includes(trigger.timing)) {
|
|
335
|
+
throw new TypeError(`Trigger timing is invalid: ${trigger.name}`);
|
|
336
|
+
}
|
|
337
|
+
if (triggerIds.has(trigger.id)) {
|
|
338
|
+
throw new TypeError(`Trigger ID already exists: ${trigger.id}`);
|
|
339
|
+
}
|
|
340
|
+
if (triggerNames.has(trigger.name)) {
|
|
341
|
+
throw new TypeError(`Trigger already exists: ${trigger.name}`);
|
|
342
|
+
}
|
|
343
|
+
triggerIds.add(trigger.id);
|
|
344
|
+
triggerNames.add(trigger.name);
|
|
345
|
+
if (typeof trigger.createdAt !== "string") {
|
|
346
|
+
throw new TypeError(`Trigger creation timestamp is invalid: ${trigger.name}`);
|
|
347
|
+
}
|
|
348
|
+
const triggerCreatedAt = Date.parse(trigger.createdAt);
|
|
349
|
+
if (!Number.isFinite(triggerCreatedAt) ||
|
|
350
|
+
dateIsoString(new Date(triggerCreatedAt)) !== trigger.createdAt) {
|
|
351
|
+
throw new TypeError(`Trigger creation timestamp is invalid: ${trigger.name}`);
|
|
352
|
+
}
|
|
353
|
+
if (!Array.isArray(trigger.statements) || trigger.statements.length === 0) {
|
|
354
|
+
throw new TypeError(`A trigger needs at least one statement: ${trigger.name}`);
|
|
355
|
+
}
|
|
356
|
+
if (trigger.statements.length > MAX_TRIGGER_STATEMENTS) {
|
|
357
|
+
throw new RangeError(`A trigger cannot exceed ${String(MAX_TRIGGER_STATEMENTS)} statements: ${trigger.name}`);
|
|
358
|
+
}
|
|
359
|
+
for (const statement of trigger.statements) {
|
|
360
|
+
if (typeof statement.sql !== "string" || statement.sql.length === 0) {
|
|
361
|
+
throw new TypeError(`Trigger statement SQL is invalid: ${trigger.name}`);
|
|
362
|
+
}
|
|
363
|
+
if (!Array.isArray(statement.bindings)) {
|
|
364
|
+
throw new TypeError(`Trigger statement bindings are invalid: ${trigger.name}`);
|
|
365
|
+
}
|
|
366
|
+
for (const binding of statement.bindings) {
|
|
367
|
+
if (!VALID_TRIGGER_BINDING_SOURCES.includes(binding.source)) {
|
|
368
|
+
throw new TypeError(`Trigger binding source is invalid: ${trigger.name}`);
|
|
369
|
+
}
|
|
370
|
+
validateCatalogName(binding.column, "Trigger binding column");
|
|
371
|
+
if (!columnNames.has(binding.column)) {
|
|
372
|
+
throw new TypeError(`Trigger binding names an unknown column: ${binding.column}`);
|
|
373
|
+
}
|
|
374
|
+
if ((trigger.event === "insert" && binding.source === "old") ||
|
|
375
|
+
(trigger.event === "delete" && binding.source === "new")) {
|
|
376
|
+
throw new TypeError(`Trigger binding source is unavailable for ${trigger.event}`);
|
|
377
|
+
}
|
|
378
|
+
}
|
|
379
|
+
}
|
|
380
|
+
}
|
|
381
|
+
const namedConstraints = [
|
|
382
|
+
...(record.foreignKeys ?? []).map((constraint) => constraint.name),
|
|
383
|
+
...(record.checks ?? []).map((constraint) => constraint.name),
|
|
384
|
+
...Object.values(record.secondaryIndexes ?? {})
|
|
385
|
+
.filter((index) => index.unique === true)
|
|
386
|
+
.map((index) => index.name),
|
|
387
|
+
];
|
|
388
|
+
if (namedConstraints.length > MAX_TABLE_CONSTRAINTS) {
|
|
389
|
+
throw new RangeError(`A table cannot exceed ${String(MAX_TABLE_CONSTRAINTS)} named constraints`);
|
|
390
|
+
}
|
|
391
|
+
const constraintNames = new Set();
|
|
392
|
+
for (const constraintName of namedConstraints) {
|
|
393
|
+
validateCatalogName(constraintName, "Constraint name");
|
|
394
|
+
if (constraintNames.has(constraintName)) {
|
|
395
|
+
throw new TypeError(`Constraint already exists: ${constraintName}`);
|
|
396
|
+
}
|
|
397
|
+
constraintNames.add(constraintName);
|
|
398
|
+
}
|
|
399
|
+
if ((record.enumType?.values.length ?? 0) > MAX_ENUM_VALUES) {
|
|
400
|
+
throw new RangeError("A catalog enum has too many values");
|
|
401
|
+
}
|
|
402
|
+
const stack = [record];
|
|
403
|
+
const seen = new WeakSet();
|
|
404
|
+
let characters = 0;
|
|
405
|
+
let entries = 0;
|
|
406
|
+
while (stack.length > 0) {
|
|
407
|
+
const value = stack.pop();
|
|
408
|
+
if (typeof value === "string") {
|
|
409
|
+
assertWellFormedString(value, "Table record string");
|
|
410
|
+
characters += value.length;
|
|
411
|
+
}
|
|
412
|
+
else if (Array.isArray(value)) {
|
|
413
|
+
if (seen.has(value))
|
|
414
|
+
throw new TypeError("A table record cannot contain cycles or aliases");
|
|
415
|
+
seen.add(value);
|
|
416
|
+
entries += value.length;
|
|
417
|
+
for (const item of value)
|
|
418
|
+
stack.push(item);
|
|
419
|
+
}
|
|
420
|
+
else if (value instanceof Date) {
|
|
421
|
+
if (!Number.isFinite(dateMilliseconds(value))) {
|
|
422
|
+
throw new TypeError("A table record date is invalid");
|
|
423
|
+
}
|
|
424
|
+
}
|
|
425
|
+
else if (value !== null && typeof value === "object") {
|
|
426
|
+
if (seen.has(value))
|
|
427
|
+
throw new TypeError("A table record cannot contain cycles or aliases");
|
|
428
|
+
seen.add(value);
|
|
429
|
+
const fields = Object.entries(value);
|
|
430
|
+
entries += fields.length;
|
|
431
|
+
for (const [key, item] of fields) {
|
|
432
|
+
assertWellFormedString(key, "Table record field name");
|
|
433
|
+
characters += key.length;
|
|
434
|
+
stack.push(item);
|
|
435
|
+
}
|
|
436
|
+
}
|
|
437
|
+
if (!Number.isSafeInteger(entries) || entries > MAX_TABLE_RECORD_ENTRIES) {
|
|
438
|
+
throw new RangeError(`A table record cannot exceed ${String(MAX_TABLE_RECORD_ENTRIES)} aggregate entries`);
|
|
439
|
+
}
|
|
440
|
+
if (!Number.isSafeInteger(characters) || characters > MAX_TABLE_RECORD_CHARACTERS) {
|
|
441
|
+
throw new RangeError(`A table record cannot exceed ${String(MAX_TABLE_RECORD_CHARACTERS)} modeled characters`);
|
|
442
|
+
}
|
|
443
|
+
}
|
|
444
|
+
}
|
|
445
|
+
/** Validates the durable identities and ownership rules of one table's secondary indexes. */
|
|
446
|
+
export function validateSecondaryIndexes(record) {
|
|
447
|
+
validateTableRecordBounds(record);
|
|
448
|
+
const columnIds = new Set(record.columns.map((column) => column.id));
|
|
449
|
+
const names = new Set();
|
|
450
|
+
const storageIds = new Set(Object.keys(record.ftsColumns ?? {}));
|
|
451
|
+
for (const [indexId, index] of Object.entries(record.secondaryIndexes ?? {})) {
|
|
452
|
+
if (indexId.length === 0 || index.name.length === 0 || index.storageColumnId.length === 0) {
|
|
453
|
+
throw new TypeError("Secondary-index IDs and names must be non-empty");
|
|
454
|
+
}
|
|
455
|
+
validateStorageId(indexId, "Secondary-index ID");
|
|
456
|
+
validateCatalogName(index.name, "Secondary-index name");
|
|
457
|
+
validateStorageId(index.storageColumnId, "Secondary-index storage ID");
|
|
458
|
+
const indexedColumnIds = index.columnIds;
|
|
459
|
+
if (indexedColumnIds.length === 0 ||
|
|
460
|
+
indexedColumnIds[0] !== index.columnId ||
|
|
461
|
+
new Set(indexedColumnIds).size !== indexedColumnIds.length ||
|
|
462
|
+
indexedColumnIds.some((columnId) => !columnIds.has(columnId))) {
|
|
463
|
+
throw new TypeError(`Secondary index ${index.name} references an unknown column`);
|
|
464
|
+
}
|
|
465
|
+
const directions = index.directions;
|
|
466
|
+
const termEncoding = index.termEncoding;
|
|
467
|
+
if (directions.length !== indexedColumnIds.length ||
|
|
468
|
+
directions.some((direction) => direction !== "asc" && direction !== "desc") ||
|
|
469
|
+
termEncoding !== "tuple-v1") {
|
|
470
|
+
throw new TypeError(`Secondary index ${index.name} has invalid key metadata`);
|
|
471
|
+
}
|
|
472
|
+
if (index.uniqueEnforced === true && index.unique !== true) {
|
|
473
|
+
throw new TypeError(`Secondary index ${index.name} enforces uniqueness without UNIQUE`);
|
|
474
|
+
}
|
|
475
|
+
if (names.has(index.name))
|
|
476
|
+
throw new TypeError(`Index already exists: ${index.name}`);
|
|
477
|
+
names.add(index.name);
|
|
478
|
+
if (storageIds.has(index.storageColumnId)) {
|
|
479
|
+
throw new TypeError(`Secondary-index storage ID is already used: ${index.storageColumnId}`);
|
|
480
|
+
}
|
|
481
|
+
storageIds.add(index.storageColumnId);
|
|
482
|
+
const expectedLocator = record.uniqueKeyColumnId === undefined ? "row-id" : "key-hash-v1";
|
|
483
|
+
const storage = index.storage;
|
|
484
|
+
if (storage !== "postings-v1" || index.locator !== expectedLocator) {
|
|
485
|
+
throw new TypeError(`Secondary index ${index.name} has incompatible storage metadata`);
|
|
486
|
+
}
|
|
487
|
+
const state = index.state;
|
|
488
|
+
if ((state !== "building" && state !== "ready" && state !== "invalid") ||
|
|
489
|
+
!Number.isSafeInteger(index.buildFromVersion) ||
|
|
490
|
+
index.buildFromVersion < -1 ||
|
|
491
|
+
(index.state === "building") !== (index.buildId !== undefined)) {
|
|
492
|
+
throw new TypeError(`Secondary index ${index.name} has invalid build metadata`);
|
|
493
|
+
}
|
|
494
|
+
}
|
|
495
|
+
}
|
|
496
|
+
/** Ordered catalog column IDs for one canonical secondary-index record. */
|
|
497
|
+
export function secondaryIndexColumnIds(index) {
|
|
498
|
+
return index.columnIds;
|
|
499
|
+
}
|
|
500
|
+
/** Declared directions for one canonical secondary-index record. */
|
|
501
|
+
export function secondaryIndexDirections(index) {
|
|
502
|
+
return index.directions;
|
|
503
|
+
}
|
|
504
|
+
/** Unique-membership namespace owned by one physical secondary index. */
|
|
505
|
+
export function secondaryUniqueKeyNamespace(tableId, indexId) {
|
|
506
|
+
return `${tableId}\u0000secondary-index\u0000${indexId}`;
|
|
507
|
+
}
|
|
99
508
|
export class TableRecordConflictError extends Error {
|
|
100
509
|
tableId;
|
|
101
510
|
expectedRevision;
|
|
@@ -108,6 +517,94 @@ export class TableRecordConflictError extends Error {
|
|
|
108
517
|
this.actualRevision = actualRevision;
|
|
109
518
|
}
|
|
110
519
|
}
|
|
520
|
+
/** A catalog drop lost a race with durable work that still owns records for the table. */
|
|
521
|
+
export class TableInUseError extends Error {
|
|
522
|
+
tableId;
|
|
523
|
+
ownerKind;
|
|
524
|
+
ownerId;
|
|
525
|
+
name = "TableInUseError";
|
|
526
|
+
constructor(tableId, ownerKind, ownerId) {
|
|
527
|
+
super(`Cannot remove table ${tableId} while ${ownerKind} ${ownerId} is active`);
|
|
528
|
+
this.tableId = tableId;
|
|
529
|
+
this.ownerKind = ownerKind;
|
|
530
|
+
this.ownerId = ownerId;
|
|
531
|
+
}
|
|
532
|
+
}
|
|
533
|
+
/** Refuses a commit that would cross the persisted level-zero fragmentation safety ceiling. */
|
|
534
|
+
export class CompactionBacklogError extends Error {
|
|
535
|
+
tableName;
|
|
536
|
+
levelZeroSegments;
|
|
537
|
+
limit;
|
|
538
|
+
name = "CompactionBacklogError";
|
|
539
|
+
constructor(tableName, levelZeroSegments, limit) {
|
|
540
|
+
super(`Table ${tableName} has ${String(levelZeroSegments)} level-zero segments; compactTable() must reduce it below the ${String(limit)}-segment safety limit before more writes`);
|
|
541
|
+
this.tableName = tableName;
|
|
542
|
+
this.levelZeroSegments = levelZeroSegments;
|
|
543
|
+
this.limit = limit;
|
|
544
|
+
}
|
|
545
|
+
}
|
|
546
|
+
/** Absolute visible level-zero ceiling; every commit that adds L0 must enforce this or lower. */
|
|
547
|
+
export const MAX_LEVEL_ZERO_SEGMENTS = 4_096;
|
|
548
|
+
/** Largest persisted row ID/posting value; the wire format is canonical unsigned 64-bit. */
|
|
549
|
+
export const MAX_ROW_ID = (1n << 64n) - 1n;
|
|
550
|
+
/** One past `MAX_ROW_ID`, allowed only for exclusive range/counter ends. */
|
|
551
|
+
export const MAX_ROW_ID_EXCLUSIVE_END = 1n << 64n;
|
|
552
|
+
/** Auto-increment values are exposed as exact JavaScript numbers and therefore stop here. */
|
|
553
|
+
export const MAX_AUTO_INCREMENT_VALUE = BigInt(Number.MAX_SAFE_INTEGER);
|
|
554
|
+
export const MAX_AUTO_INCREMENT_EXCLUSIVE_END = MAX_AUTO_INCREMENT_VALUE + 1n;
|
|
555
|
+
/** A live owner renews; an abandoned durable pin expires within one hour. */
|
|
556
|
+
export const MAX_LEASE_TTL_MS = 60 * 60 * 1_000;
|
|
557
|
+
/** Durable resource ceilings; adapters sweep expired owners before atomically refusing creation. */
|
|
558
|
+
export const MAX_ACTIVE_LEASES = 4_096;
|
|
559
|
+
export const MAX_ACTIVE_TEMP_OWNERS = 1_024;
|
|
560
|
+
export const MAX_TEMP_RUNS_PER_OWNER = 1_024;
|
|
561
|
+
export const MAX_TEMP_PAGES_PER_OWNER = 16_384;
|
|
562
|
+
export const MAX_TEMP_RUNS_TOTAL = 65_536;
|
|
563
|
+
export const MAX_TEMP_PAGES_TOTAL = 262_144;
|
|
564
|
+
export const MAX_TEMP_BYTES_PER_OWNER = 512 * 1024 * 1024;
|
|
565
|
+
export const MAX_TEMP_BYTES_TOTAL = 1024 * 1024 * 1024;
|
|
566
|
+
export const MAX_ACTIVE_COMPACTION_JOBS = 1_024;
|
|
567
|
+
export const MAX_ACTIVE_GARBAGE_COLLECTION_JOBS = 1;
|
|
568
|
+
export const MAX_ACTIVE_UNIQUE_KEY_BUILDS = 1_024;
|
|
569
|
+
export const MAX_ACTIVE_FTS_BASE_BUILDS = 128;
|
|
570
|
+
export const MAX_ACTIVE_SECONDARY_INDEX_BUILDS = 128;
|
|
571
|
+
export const MAX_ACCELERATOR_BUILD_STAGED_BYTES_TOTAL = 1024 * 1024 * 1024;
|
|
572
|
+
export const MAX_ACCELERATOR_BUILD_STAGED_ENTRIES_TOTAL = 16_777_216;
|
|
573
|
+
export const MAX_ACTIVE_TRANSACTIONS = 4_096;
|
|
574
|
+
export const MAX_GLOBAL_STAGED_ARTIFACT_BYTES = 512 * 1024 * 1024;
|
|
575
|
+
export const MAX_GLOBAL_STAGED_BLOCKS = 65_536;
|
|
576
|
+
export const MAX_GLOBAL_STAGED_SEGMENTS = 65_536;
|
|
577
|
+
/** Catalog enumeration remains bounded even for cold public inspection APIs. */
|
|
578
|
+
export const MAX_CATALOG_RECORDS = 4_096;
|
|
579
|
+
/** Total canonical UTF-8 record-wire bytes retained by the durable catalog. */
|
|
580
|
+
export const MAX_CATALOG_RETAINED_BYTES = 64 * 1024 * 1024;
|
|
581
|
+
export const MAX_MANIFEST_RECORDS = 65_536;
|
|
582
|
+
export const MAX_MANIFEST_RETAINED_BYTES = 64 * 1024 * 1024;
|
|
583
|
+
export const MAX_SEGMENT_RECORDS = 1_048_576;
|
|
584
|
+
export const MAX_SEGMENT_RETAINED_BYTES = 512 * 1024 * 1024;
|
|
585
|
+
/** Durable diagnostic/provenance tails are pruned before admitting more terminal records. */
|
|
586
|
+
export const MAX_TERMINAL_TRANSACTION_RECORDS = 65_536;
|
|
587
|
+
export const MAX_TERMINAL_COMPACTION_JOB_RECORDS = 4_096;
|
|
588
|
+
export const MAX_COMPLETED_GARBAGE_COLLECTION_JOB_RECORDS = 1_024;
|
|
589
|
+
/** An old renewable pin may not force unbounded history retention while writes continue. */
|
|
590
|
+
export const MAX_PINNED_MANIFEST_VERSION_LAG = 4_096;
|
|
591
|
+
export const MAX_PINNED_RETIRED_BLOCKS = 65_536;
|
|
592
|
+
export const MAX_PINNED_RETIRED_BYTES = 512 * 1024 * 1024;
|
|
593
|
+
/** Total obsolete physical history is refused before it can consume an origin indefinitely. */
|
|
594
|
+
export const MAX_RETIRED_HISTORY_BYTES = 1024 * 1024 * 1024;
|
|
595
|
+
/** A durable resource family reached its fixed corruption/growth safety ceiling. */
|
|
596
|
+
export class StorageResourceLimitError extends Error {
|
|
597
|
+
resource;
|
|
598
|
+
count;
|
|
599
|
+
limit;
|
|
600
|
+
name = "StorageResourceLimitError";
|
|
601
|
+
constructor(resource, count, limit) {
|
|
602
|
+
super(`Storage resource ${resource} would reach ${String(count)}; the fixed safety limit is ${String(limit)}`);
|
|
603
|
+
this.resource = resource;
|
|
604
|
+
this.count = count;
|
|
605
|
+
this.limit = limit;
|
|
606
|
+
}
|
|
607
|
+
}
|
|
111
608
|
export const compactionJobStates = [
|
|
112
609
|
"planned",
|
|
113
610
|
"running",
|
|
@@ -118,6 +615,240 @@ export const compactionJobStates = [
|
|
|
118
615
|
];
|
|
119
616
|
export const compactionRewritePlanKinds = ["copy-v1", "rechunk-v1", "merge-v1"];
|
|
120
617
|
export const compactionOutputCompressions = ["raw", "gzip"];
|
|
618
|
+
/** Every segment ID owned by a compaction job, including merge partition outputs. */
|
|
619
|
+
export function compactionOutputSegmentIds(job) {
|
|
620
|
+
const outputSegmentId = job.outputSegmentId;
|
|
621
|
+
if (outputSegmentId === null)
|
|
622
|
+
return [];
|
|
623
|
+
const partitionCount = (job.rewritePlan.kind === "merge-v1" || job.rewritePlan.kind === "rechunk-v1") &&
|
|
624
|
+
job.rewritePlan.partitions !== undefined
|
|
625
|
+
? job.rewritePlan.partitions.length
|
|
626
|
+
: 1;
|
|
627
|
+
return Array.from({ length: partitionCount }, (_, index) => index === 0 ? outputSegmentId : `${outputSegmentId}/${String(index)}`);
|
|
628
|
+
}
|
|
629
|
+
/**
|
|
630
|
+
* Proves that a compaction transaction stages exactly the immutable job output. This is kept in
|
|
631
|
+
* the storage contract (rather than only in the engine) so adapters reject forged, reordered, or
|
|
632
|
+
* partially journaled output before publication and while loading durable state.
|
|
633
|
+
*/
|
|
634
|
+
export function assertCompactionOutputProvenance(job, table, transaction, sourceSegments, outputSegments, options = {}) {
|
|
635
|
+
if (job.transactionId !== transaction.id) {
|
|
636
|
+
throw new Error(`Compaction job ${job.id} belongs to another transaction`);
|
|
637
|
+
}
|
|
638
|
+
const expectedSegmentIds = compactionOutputSegmentIds(job);
|
|
639
|
+
const stagedExpectedSegmentIds = options.allowOutputPrefix
|
|
640
|
+
? expectedSegmentIds.slice(0, transaction.pendingSegmentIds.length)
|
|
641
|
+
: expectedSegmentIds;
|
|
642
|
+
assertExactOrderedIds(transaction.pendingSegmentIds, stagedExpectedSegmentIds, `Compaction job ${job.id} segment journal`);
|
|
643
|
+
assertExactOrderedIds(outputSegments.map((segment) => segment.id), stagedExpectedSegmentIds, `Compaction job ${job.id} output segments`);
|
|
644
|
+
assertExactOrderedIds(sourceSegments.map((segment) => segment.id), job.sourceSegmentIds, `Compaction job ${job.id} source segments`);
|
|
645
|
+
const plan = job.rewritePlan;
|
|
646
|
+
if (plan.kind === "copy-v1") {
|
|
647
|
+
const expectedOutputBlockIds = sourceSegments.flatMap((segment, segmentIndex) => table.columns.flatMap((column, columnIndex) => (segment.columnBlockIds[column.id] ?? []).map((_blockId, part) => copyCompactionOutputBlockId(job.id, segmentIndex, columnIndex, part))));
|
|
648
|
+
assertExactOrderedIds(job.outputBlockIds, expectedOutputBlockIds, `Compaction job ${job.id} output block plan`);
|
|
649
|
+
}
|
|
650
|
+
else {
|
|
651
|
+
if (table.columns.length !== plan.columns.length ||
|
|
652
|
+
table.columns.some((column, index) => {
|
|
653
|
+
const planned = plan.columns.at(index);
|
|
654
|
+
if (planned?.columnId !== column.id)
|
|
655
|
+
return true;
|
|
656
|
+
return planned.type !== column.type;
|
|
657
|
+
})) {
|
|
658
|
+
throw new Error(`Compaction job ${job.id} table schema differs from its rewrite plan`);
|
|
659
|
+
}
|
|
660
|
+
if (plan.kind === "merge-v1" && plan.keyColumnId !== table.uniqueKeyColumnId) {
|
|
661
|
+
throw new Error(`Compaction job ${job.id} table key differs from its rewrite plan`);
|
|
662
|
+
}
|
|
663
|
+
const expectedOutputBlockIds = plan.outputs.flatMap((_output, outputIndex) => plan.columns.map((_column, columnIndex) => physicalCompactionOutputBlockId(job.id, outputIndex, columnIndex)));
|
|
664
|
+
assertExactOrderedIds(job.outputBlockIds, expectedOutputBlockIds, `Compaction job ${job.id} output block plan`);
|
|
665
|
+
}
|
|
666
|
+
assertExactOrderedIds(transaction.pendingBlockIds, job.outputBlockIds, `Compaction job ${job.id} block journal`);
|
|
667
|
+
const expectedSegments = expectedCompactionOutputSegments(job, table, sourceSegments);
|
|
668
|
+
for (const [index, actual] of outputSegments.entries()) {
|
|
669
|
+
const expected = expectedSegments[index];
|
|
670
|
+
if (expected === undefined) {
|
|
671
|
+
throw new Error(`Compaction job ${job.id} stages an unexpected output segment`);
|
|
672
|
+
}
|
|
673
|
+
assertCompactionSegmentFields(actual, expected, job.id);
|
|
674
|
+
}
|
|
675
|
+
}
|
|
676
|
+
function expectedCompactionOutputSegments(job, table, sourceSegments) {
|
|
677
|
+
const outputSegmentId = job.outputSegmentId;
|
|
678
|
+
if (outputSegmentId === null)
|
|
679
|
+
return [];
|
|
680
|
+
const plan = job.rewritePlan;
|
|
681
|
+
const common = {
|
|
682
|
+
tableId: table.id,
|
|
683
|
+
transactionId: job.transactionId ?? "",
|
|
684
|
+
...(table.uniqueKeyColumnId === undefined ? {} : { keyColumnId: table.uniqueKeyColumnId }),
|
|
685
|
+
level: job.targetLevel,
|
|
686
|
+
...(job.outputPartitionOrdinal === undefined
|
|
687
|
+
? {}
|
|
688
|
+
: { partitionOrdinal: job.outputPartitionOrdinal }),
|
|
689
|
+
};
|
|
690
|
+
if (plan.kind === "copy-v1") {
|
|
691
|
+
const first = sourceSegments[0];
|
|
692
|
+
const last = sourceSegments[sourceSegments.length - 1];
|
|
693
|
+
if (first === undefined || last === undefined) {
|
|
694
|
+
throw new Error(`Compaction job ${job.id} has no source rows`);
|
|
695
|
+
}
|
|
696
|
+
return [
|
|
697
|
+
{
|
|
698
|
+
id: outputSegmentId,
|
|
699
|
+
...common,
|
|
700
|
+
rowCount: safeSum(sourceSegments.map((segment) => segment.rowCount), "Copy compaction output rows"),
|
|
701
|
+
rowIdStart: first.rowIdStart,
|
|
702
|
+
rowIdEndExclusive: last.rowIdEndExclusive,
|
|
703
|
+
columnBlockIds: Object.fromEntries(table.columns.map((column, columnIndex) => [
|
|
704
|
+
column.id,
|
|
705
|
+
sourceSegments.flatMap((segment, segmentIndex) => (segment.columnBlockIds[column.id] ?? []).map((_blockId, part) => copyCompactionOutputBlockId(job.id, segmentIndex, columnIndex, part))),
|
|
706
|
+
])),
|
|
707
|
+
kind: "insert",
|
|
708
|
+
logicalOrder: Math.min(...sourceSegments.map((segment) => segment.logicalOrder)),
|
|
709
|
+
commitOrdinal: 0,
|
|
710
|
+
rowIdSpans: [],
|
|
711
|
+
},
|
|
712
|
+
];
|
|
713
|
+
}
|
|
714
|
+
const partitions = plan.partitions ?? [
|
|
715
|
+
{ rowStart: 0, rowCount: plan.totalRows, logicalOrder: plan.logicalOrder },
|
|
716
|
+
];
|
|
717
|
+
return partitions.map((partition, index) => {
|
|
718
|
+
const rowIdSpans = plan.kind === "merge-v1"
|
|
719
|
+
? sliceCompactionRowIdSpans(plan.rowIdSpans, partition.rowStart, partition.rowCount)
|
|
720
|
+
: [];
|
|
721
|
+
const envelope = plan.kind === "merge-v1"
|
|
722
|
+
? compactionRowIdSpanEnvelope(rowIdSpans)
|
|
723
|
+
: {
|
|
724
|
+
start: plan.rowIdStart + BigInt(partition.rowStart),
|
|
725
|
+
endExclusive: plan.rowIdStart + BigInt(partition.rowStart + partition.rowCount),
|
|
726
|
+
};
|
|
727
|
+
return {
|
|
728
|
+
id: index === 0 ? outputSegmentId : `${outputSegmentId}/${String(index)}`,
|
|
729
|
+
...common,
|
|
730
|
+
rowCount: partition.rowCount,
|
|
731
|
+
rowIdStart: envelope.start,
|
|
732
|
+
rowIdEndExclusive: envelope.endExclusive,
|
|
733
|
+
columnBlockIds: Object.fromEntries(plan.columns.map((column, columnIndex) => [
|
|
734
|
+
column.columnId,
|
|
735
|
+
plan.outputs.flatMap((output, outputIndex) => output.rowStart >= partition.rowStart &&
|
|
736
|
+
output.rowStart + output.rowCount <= partition.rowStart + partition.rowCount
|
|
737
|
+
? [physicalCompactionOutputBlockId(job.id, outputIndex, columnIndex)]
|
|
738
|
+
: []),
|
|
739
|
+
])),
|
|
740
|
+
kind: plan.kind === "merge-v1" ? "base" : "insert",
|
|
741
|
+
logicalOrder: partition.logicalOrder,
|
|
742
|
+
commitOrdinal: index,
|
|
743
|
+
rowIdSpans,
|
|
744
|
+
};
|
|
745
|
+
});
|
|
746
|
+
}
|
|
747
|
+
function assertCompactionSegmentFields(actual, expected, jobId) {
|
|
748
|
+
for (const field of [
|
|
749
|
+
"id",
|
|
750
|
+
"tableId",
|
|
751
|
+
"transactionId",
|
|
752
|
+
"rowCount",
|
|
753
|
+
"rowIdStart",
|
|
754
|
+
"rowIdEndExclusive",
|
|
755
|
+
"kind",
|
|
756
|
+
"keyColumnId",
|
|
757
|
+
"level",
|
|
758
|
+
"logicalOrder",
|
|
759
|
+
"commitOrdinal",
|
|
760
|
+
"partitionOrdinal",
|
|
761
|
+
]) {
|
|
762
|
+
if (actual[field] !== expected[field]) {
|
|
763
|
+
throw new Error(`Compaction job ${jobId} output ${actual.id} has invalid ${field}`);
|
|
764
|
+
}
|
|
765
|
+
}
|
|
766
|
+
const expectedColumnIds = Object.keys(expected.columnBlockIds);
|
|
767
|
+
assertExactOrderedIds(Object.keys(actual.columnBlockIds).sort(), [...expectedColumnIds].sort(), `Compaction job ${jobId} output ${actual.id} columns`);
|
|
768
|
+
for (const columnId of expectedColumnIds) {
|
|
769
|
+
assertExactOrderedIds(actual.columnBlockIds[columnId] ?? [], expected.columnBlockIds[columnId] ?? [], `Compaction job ${jobId} output ${actual.id} column ${columnId}`);
|
|
770
|
+
}
|
|
771
|
+
if (actual.rowIdSpans.length !== expected.rowIdSpans.length ||
|
|
772
|
+
actual.rowIdSpans.some((span, index) => {
|
|
773
|
+
const planned = expected.rowIdSpans[index];
|
|
774
|
+
return (planned?.rowStart !== span.rowStart ||
|
|
775
|
+
planned.rowCount !== span.rowCount ||
|
|
776
|
+
planned.rowIdStart !== span.rowIdStart);
|
|
777
|
+
})) {
|
|
778
|
+
throw new Error(`Compaction job ${jobId} output ${actual.id} has invalid row-ID spans`);
|
|
779
|
+
}
|
|
780
|
+
}
|
|
781
|
+
function assertExactOrderedIds(actual, expected, label) {
|
|
782
|
+
if (actual.length !== expected.length ||
|
|
783
|
+
actual.some((value, index) => value !== expected[index])) {
|
|
784
|
+
throw new Error(`${label} is not the canonical ordered sequence`);
|
|
785
|
+
}
|
|
786
|
+
}
|
|
787
|
+
function physicalCompactionOutputBlockId(jobId, outputIndex, columnIndex) {
|
|
788
|
+
return [
|
|
789
|
+
jobId,
|
|
790
|
+
"rewrite",
|
|
791
|
+
"window",
|
|
792
|
+
String(outputIndex).padStart(8, "0"),
|
|
793
|
+
"column",
|
|
794
|
+
String(columnIndex).padStart(8, "0"),
|
|
795
|
+
].join("/");
|
|
796
|
+
}
|
|
797
|
+
function copyCompactionOutputBlockId(jobId, segmentIndex, columnIndex, part) {
|
|
798
|
+
return [
|
|
799
|
+
jobId,
|
|
800
|
+
"output",
|
|
801
|
+
"segment",
|
|
802
|
+
String(segmentIndex).padStart(6, "0"),
|
|
803
|
+
"column",
|
|
804
|
+
String(columnIndex).padStart(6, "0"),
|
|
805
|
+
"part",
|
|
806
|
+
String(part).padStart(6, "0"),
|
|
807
|
+
].join("/");
|
|
808
|
+
}
|
|
809
|
+
function sliceCompactionRowIdSpans(spans, rowStart, rowCount) {
|
|
810
|
+
const result = [];
|
|
811
|
+
const rowEnd = rowStart + rowCount;
|
|
812
|
+
for (const span of spans) {
|
|
813
|
+
const spanEnd = span.rowStart + span.rowCount;
|
|
814
|
+
if (spanEnd <= rowStart || span.rowStart >= rowEnd)
|
|
815
|
+
continue;
|
|
816
|
+
const start = Math.max(span.rowStart, rowStart);
|
|
817
|
+
const end = Math.min(spanEnd, rowEnd);
|
|
818
|
+
const next = {
|
|
819
|
+
rowStart: start - rowStart,
|
|
820
|
+
rowCount: end - start,
|
|
821
|
+
rowIdStart: span.rowIdStart + BigInt(start - span.rowStart),
|
|
822
|
+
};
|
|
823
|
+
const previous = result[result.length - 1];
|
|
824
|
+
if (previous !== undefined &&
|
|
825
|
+
previous.rowStart + previous.rowCount === next.rowStart &&
|
|
826
|
+
previous.rowIdStart + BigInt(previous.rowCount) === next.rowIdStart) {
|
|
827
|
+
result[result.length - 1] = {
|
|
828
|
+
...previous,
|
|
829
|
+
rowCount: previous.rowCount + next.rowCount,
|
|
830
|
+
};
|
|
831
|
+
}
|
|
832
|
+
else {
|
|
833
|
+
result.push(next);
|
|
834
|
+
}
|
|
835
|
+
}
|
|
836
|
+
return result;
|
|
837
|
+
}
|
|
838
|
+
function compactionRowIdSpanEnvelope(spans) {
|
|
839
|
+
if (spans.length === 0)
|
|
840
|
+
return { start: 0n, endExclusive: 0n };
|
|
841
|
+
let start = spans[0]?.rowIdStart ?? 0n;
|
|
842
|
+
let endExclusive = start + BigInt(spans[0]?.rowCount ?? 0);
|
|
843
|
+
for (const span of spans.slice(1)) {
|
|
844
|
+
if (span.rowIdStart < start)
|
|
845
|
+
start = span.rowIdStart;
|
|
846
|
+
const spanEnd = span.rowIdStart + BigInt(span.rowCount);
|
|
847
|
+
if (spanEnd > endExclusive)
|
|
848
|
+
endExclusive = spanEnd;
|
|
849
|
+
}
|
|
850
|
+
return { start, endExclusive };
|
|
851
|
+
}
|
|
121
852
|
export class CompactionJobConflictError extends Error {
|
|
122
853
|
jobId;
|
|
123
854
|
expectedRevision;
|
|
@@ -131,6 +862,15 @@ export class CompactionJobConflictError extends Error {
|
|
|
131
862
|
}
|
|
132
863
|
}
|
|
133
864
|
export const garbageCollectionJobStates = ["planned", "running", "completed"];
|
|
865
|
+
/** Absolute per-call bound for durable maintenance candidate arrays and storage transactions. */
|
|
866
|
+
export const MAX_MAINTENANCE_BATCH_ITEMS = 1_024;
|
|
867
|
+
export function boundedMaintenanceBatchItems(value, label) {
|
|
868
|
+
const normalized = positiveWholeNumber(value, label);
|
|
869
|
+
if (normalized > MAX_MAINTENANCE_BATCH_ITEMS) {
|
|
870
|
+
throw new RangeError(`${label} cannot exceed ${String(MAX_MAINTENANCE_BATCH_ITEMS)} items`);
|
|
871
|
+
}
|
|
872
|
+
return normalized;
|
|
873
|
+
}
|
|
134
874
|
export class GarbageCollectionJobConflictError extends Error {
|
|
135
875
|
jobId;
|
|
136
876
|
expectedRevision;
|
|
@@ -163,34 +903,541 @@ export class LeaseConflictError extends Error {
|
|
|
163
903
|
this.actualRevision = actualRevision;
|
|
164
904
|
}
|
|
165
905
|
}
|
|
906
|
+
/** A renewal/move cutoff reached the persisted expiry; expiry is irrevocable. */
|
|
907
|
+
export class LeaseExpiredError extends Error {
|
|
908
|
+
leaseId;
|
|
909
|
+
expiresAt;
|
|
910
|
+
expiresAtCutoff;
|
|
911
|
+
name = "LeaseExpiredError";
|
|
912
|
+
constructor(leaseId, expiresAt, expiresAtCutoff) {
|
|
913
|
+
super(`Lease ${leaseId} expired at ${expiresAt}`);
|
|
914
|
+
this.leaseId = leaseId;
|
|
915
|
+
this.expiresAt = expiresAt;
|
|
916
|
+
this.expiresAtCutoff = expiresAtCutoff;
|
|
917
|
+
}
|
|
918
|
+
}
|
|
919
|
+
/** Refuses a lease release from a different durable owner. */
|
|
920
|
+
export class LeaseOwnerConflictError extends Error {
|
|
921
|
+
leaseId;
|
|
922
|
+
expectedOwnerId;
|
|
923
|
+
actualOwnerId;
|
|
924
|
+
name = "LeaseOwnerConflictError";
|
|
925
|
+
constructor(leaseId, expectedOwnerId, actualOwnerId) {
|
|
926
|
+
super(`Lease ${leaseId} belongs to ${actualOwnerId}, not release owner ${expectedOwnerId}`);
|
|
927
|
+
this.leaseId = leaseId;
|
|
928
|
+
this.expectedOwnerId = expectedOwnerId;
|
|
929
|
+
this.actualOwnerId = actualOwnerId;
|
|
930
|
+
}
|
|
931
|
+
}
|
|
932
|
+
/** Maximum immutable payloads accepted by one atomic staging/WAL operation. */
|
|
933
|
+
export const MAX_TRANSACTION_STAGE_BLOCKS = 64;
|
|
934
|
+
/** Maximum segment records accepted by one atomic staging/WAL operation. */
|
|
935
|
+
export const MAX_TRANSACTION_STAGE_SEGMENTS = 64;
|
|
936
|
+
/**
|
|
937
|
+
* Maximum block bytes accepted by one atomic staging/WAL operation. The limit still admits one
|
|
938
|
+
* maximum-size physical block; callers split collections of smaller blocks into bounded calls.
|
|
939
|
+
*/
|
|
940
|
+
export const MAX_TRANSACTION_STAGE_BYTES = MAX_STORED_BLOCK_BYTE_LENGTH;
|
|
941
|
+
/** Durable journal ceilings prevent a forgotten or hostile transaction growing without bound. */
|
|
942
|
+
export const MAX_TRANSACTION_PENDING_BLOCKS = 4_096;
|
|
943
|
+
export const MAX_TRANSACTION_PENDING_SEGMENTS = 4_096;
|
|
944
|
+
/** Storage-boundary preflight shared by staging and single-shot write implementations. */
|
|
945
|
+
export function assertTransactionArtifactBatchLimits(blocks, segments) {
|
|
946
|
+
if (blocks.length > MAX_TRANSACTION_STAGE_BLOCKS) {
|
|
947
|
+
throw new RangeError(`Transaction artifact batch exceeds ${String(MAX_TRANSACTION_STAGE_BLOCKS)} blocks`);
|
|
948
|
+
}
|
|
949
|
+
if (segments.length > MAX_TRANSACTION_STAGE_SEGMENTS) {
|
|
950
|
+
throw new RangeError(`Transaction artifact batch exceeds ${String(MAX_TRANSACTION_STAGE_SEGMENTS)} segments`);
|
|
951
|
+
}
|
|
952
|
+
let byteLength = 0;
|
|
953
|
+
for (const block of blocks) {
|
|
954
|
+
if (!(block.bytes instanceof Uint8Array)) {
|
|
955
|
+
throw new TypeError("Transaction block bytes must be a Uint8Array");
|
|
956
|
+
}
|
|
957
|
+
byteLength += block.bytes.byteLength;
|
|
958
|
+
if (!Number.isSafeInteger(byteLength) || byteLength > MAX_TRANSACTION_STAGE_BYTES) {
|
|
959
|
+
throw new RangeError(`Transaction artifact batch exceeds ${String(MAX_TRANSACTION_STAGE_BYTES)} block bytes`);
|
|
960
|
+
}
|
|
961
|
+
}
|
|
962
|
+
}
|
|
963
|
+
/** Preflight for the complete persisted journal after a bounded staging operation. */
|
|
964
|
+
export function assertTransactionArtifactJournalLimits(pendingBlockIds, pendingSegmentIds) {
|
|
965
|
+
if (pendingBlockIds.length > MAX_TRANSACTION_PENDING_BLOCKS) {
|
|
966
|
+
throw new RangeError(`Transaction journal exceeds ${String(MAX_TRANSACTION_PENDING_BLOCKS)} pending blocks`);
|
|
967
|
+
}
|
|
968
|
+
if (pendingSegmentIds.length > MAX_TRANSACTION_PENDING_SEGMENTS) {
|
|
969
|
+
throw new RangeError(`Transaction journal exceeds ${String(MAX_TRANSACTION_PENDING_SEGMENTS)} pending segments`);
|
|
970
|
+
}
|
|
971
|
+
}
|
|
972
|
+
/** A snapshot session is renewable but can never pin storage for more than one hour unattended. */
|
|
973
|
+
export const MAX_SNAPSHOT_SESSION_TTL_MS = 60 * 60 * 1_000;
|
|
974
|
+
/** Snapshot v1 is an ordered bounded record stream; no database-sized collection is in header. */
|
|
975
|
+
export const SNAPSHOT_FRAME_KINDS = [
|
|
976
|
+
"catalog-page",
|
|
977
|
+
"segment-page",
|
|
978
|
+
"transaction-page",
|
|
979
|
+
"unique-page",
|
|
980
|
+
"posting-page",
|
|
981
|
+
"block",
|
|
982
|
+
];
|
|
983
|
+
export const MAX_SNAPSHOT_FRAME_ITEMS = 1_024;
|
|
984
|
+
export const MAX_SNAPSHOT_METADATA_FRAME_BYTES = 4 * 1024 * 1024;
|
|
985
|
+
export const MAX_SNAPSHOT_FRAME_BATCH_ITEMS = 64;
|
|
986
|
+
export const MAX_SNAPSHOT_METADATA_BATCH_BYTES = 16 * 1024 * 1024;
|
|
987
|
+
/** One block frame is permitted; metadata frames still use MAX_SNAPSHOT_METADATA_BATCH_BYTES. */
|
|
988
|
+
export const MAX_SNAPSHOT_FRAME_BATCH_BYTES = MAX_STORED_BLOCK_BYTE_LENGTH + 8 * 1024;
|
|
989
|
+
/**
|
|
990
|
+
* Framed restore builds accelerator generations in a disposable core before one atomic publish.
|
|
991
|
+
* These aggregate ceilings keep that validation heap bounded until generations become a native
|
|
992
|
+
* chunk-backed structure rather than one materialized in-memory collection.
|
|
993
|
+
*/
|
|
994
|
+
export const MAX_SNAPSHOT_IMPORT_ACCELERATOR_RETAINED_BYTES = MAX_ACCELERATOR_BUILD_STAGED_BYTES_TOTAL;
|
|
995
|
+
export const MAX_SNAPSHOT_IMPORT_ACCELERATOR_RETAINED_ENTRIES = MAX_ACCELERATOR_BUILD_STAGED_ENTRIES_TOTAL;
|
|
996
|
+
/** Modeled heap retained by one framed accelerator chunk during atomic snapshot validation. */
|
|
997
|
+
export function snapshotAcceleratorItemRetainedUsage(item) {
|
|
998
|
+
if (item.kind === "unique-chunk") {
|
|
999
|
+
return {
|
|
1000
|
+
bytes: uniqueKeyBuildChunkRetainedBytes(item.keyTokens),
|
|
1001
|
+
entries: item.keyTokens.length,
|
|
1002
|
+
};
|
|
1003
|
+
}
|
|
1004
|
+
let bytes = 0;
|
|
1005
|
+
let entries = 0;
|
|
1006
|
+
for (const posting of item.postings) {
|
|
1007
|
+
bytes = safeSum([bytes, 32 + posting.term.length * 2], "Snapshot posting retained bytes");
|
|
1008
|
+
entries = safeSum([entries, posting.rowIds.length], "Snapshot posting retained entries");
|
|
1009
|
+
// Bigint row IDs, term-frequency numbers, and the two array slots all remain live until the
|
|
1010
|
+
// generation is validated and promoted. The fixed model is intentionally conservative.
|
|
1011
|
+
bytes = safeSum([bytes, posting.rowIds.length * 48], "Snapshot posting retained bytes");
|
|
1012
|
+
}
|
|
1013
|
+
return { bytes, entries };
|
|
1014
|
+
}
|
|
1015
|
+
/** Shared arithmetic boundary used by streaming adapters and allocation-free cap tests. */
|
|
1016
|
+
export function assertSnapshotImportAcceleratorUsage(bytes, entries) {
|
|
1017
|
+
if (!Number.isSafeInteger(bytes) || bytes < 0) {
|
|
1018
|
+
throw new RangeError("Snapshot accelerator retained bytes are invalid");
|
|
1019
|
+
}
|
|
1020
|
+
if (!Number.isSafeInteger(entries) || entries < 0) {
|
|
1021
|
+
throw new RangeError("Snapshot accelerator retained entries are invalid");
|
|
1022
|
+
}
|
|
1023
|
+
if (bytes > MAX_SNAPSHOT_IMPORT_ACCELERATOR_RETAINED_BYTES) {
|
|
1024
|
+
throw new StorageResourceLimitError("snapshot accelerator byte", bytes, MAX_SNAPSHOT_IMPORT_ACCELERATOR_RETAINED_BYTES);
|
|
1025
|
+
}
|
|
1026
|
+
if (entries > MAX_SNAPSHOT_IMPORT_ACCELERATOR_RETAINED_ENTRIES) {
|
|
1027
|
+
throw new StorageResourceLimitError("snapshot accelerator entry", entries, MAX_SNAPSHOT_IMPORT_ACCELERATOR_RETAINED_ENTRIES);
|
|
1028
|
+
}
|
|
1029
|
+
}
|
|
1030
|
+
export class SnapshotImportConflictError extends Error {
|
|
1031
|
+
identity;
|
|
1032
|
+
ownerId;
|
|
1033
|
+
name = "SnapshotImportConflictError";
|
|
1034
|
+
constructor(identity, ownerId, message) {
|
|
1035
|
+
super(`Snapshot import ${identity} owned by ${ownerId}: ${message}`);
|
|
1036
|
+
this.identity = identity;
|
|
1037
|
+
this.ownerId = ownerId;
|
|
1038
|
+
}
|
|
1039
|
+
}
|
|
1040
|
+
/**
|
|
1041
|
+
* Persisted bytes belong to a recognized storage family, but not to the version this reader
|
|
1042
|
+
* supports. This is deliberately distinct from corruption: callers must not infer that repair,
|
|
1043
|
+
* recreation, or deletion is safe merely because versions differ.
|
|
1044
|
+
*/
|
|
1045
|
+
export class StorageFormatVersionError extends Error {
|
|
1046
|
+
backend;
|
|
1047
|
+
location;
|
|
1048
|
+
actualVersion;
|
|
1049
|
+
supportedVersion;
|
|
1050
|
+
relation;
|
|
1051
|
+
name = "StorageFormatVersionError";
|
|
1052
|
+
constructor(backend, location, actualVersion, supportedVersion, relation) {
|
|
1053
|
+
const actual = actualVersion === null ? "an unknown version" : `version ${String(actualVersion)}`;
|
|
1054
|
+
super(`${backend} storage format at ${location} uses ${actual}, which is ${relation} than ` +
|
|
1055
|
+
`the supported version ${String(supportedVersion)}`);
|
|
1056
|
+
this.backend = backend;
|
|
1057
|
+
this.location = location;
|
|
1058
|
+
this.actualVersion = actualVersion;
|
|
1059
|
+
this.supportedVersion = supportedVersion;
|
|
1060
|
+
this.relation = relation;
|
|
1061
|
+
}
|
|
1062
|
+
}
|
|
1063
|
+
/** A native IndexedDB schema upgrade is waiting for another connection to close. */
|
|
1064
|
+
export class IndexedDbSchemaUpgradeBlockedError extends Error {
|
|
1065
|
+
databaseName;
|
|
1066
|
+
oldVersion;
|
|
1067
|
+
requestedVersion;
|
|
1068
|
+
name = "IndexedDbSchemaUpgradeBlockedError";
|
|
1069
|
+
constructor(databaseName, oldVersion, requestedVersion) {
|
|
1070
|
+
super(`IndexedDB schema upgrade for ${databaseName} from version ${String(oldVersion)} to ` +
|
|
1071
|
+
`${String(requestedVersion)} is blocked by another open connection; close older tabs and retry`);
|
|
1072
|
+
this.databaseName = databaseName;
|
|
1073
|
+
this.oldVersion = oldVersion;
|
|
1074
|
+
this.requestedVersion = requestedVersion;
|
|
1075
|
+
}
|
|
1076
|
+
}
|
|
1077
|
+
/** Persisted metadata or payload failed an adapter's fail-closed integrity checks. */
|
|
1078
|
+
export class StorageCorruptionError extends Error {
|
|
1079
|
+
backend;
|
|
1080
|
+
location;
|
|
1081
|
+
name = "StorageCorruptionError";
|
|
1082
|
+
constructor(backend, location, message) {
|
|
1083
|
+
super(`${backend} storage corruption at ${location}: ${message}`);
|
|
1084
|
+
this.backend = backend;
|
|
1085
|
+
this.location = location;
|
|
1086
|
+
}
|
|
1087
|
+
}
|
|
1088
|
+
/**
|
|
1089
|
+
* A remote OPFS leader may have committed a mutation whose acknowledgement was lost.
|
|
1090
|
+
* Reconcile stable identities or revisions before retrying the named operation.
|
|
1091
|
+
*/
|
|
1092
|
+
export class OpfsUncertainOutcomeError extends Error {
|
|
1093
|
+
method;
|
|
1094
|
+
name = "OpfsUncertainOutcomeError";
|
|
1095
|
+
constructor(method) {
|
|
1096
|
+
super(`The OPFS leader changed before ${method} was acknowledged; the mutation may have committed`);
|
|
1097
|
+
this.method = method;
|
|
1098
|
+
}
|
|
1099
|
+
}
|
|
1100
|
+
/** Bounded calls used to seed an arbitrarily large UNIQUE namespace without heap materialization. */
|
|
1101
|
+
export const MAX_UNIQUE_KEY_BUILD_TOKENS_PER_CHUNK = 4_096;
|
|
1102
|
+
export const MAX_UNIQUE_KEY_BUILD_CHUNK_BYTES = 4 * 1024 * 1024;
|
|
1103
|
+
export const MAX_UNIQUE_KEY_BUILD_TTL_MS = 60 * 60 * 1_000;
|
|
1104
|
+
export const MAX_UNIQUE_KEY_BUILD_STAGED_BYTES = 512 * 1024 * 1024;
|
|
1105
|
+
export const MAX_UNIQUE_KEY_BUILD_STAGED_BYTES_TOTAL = 1024 * 1024 * 1024;
|
|
1106
|
+
/** Validates one staged seed call before an adapter opens a transaction or WAL frame. */
|
|
1107
|
+
export function uniqueKeyBuildChunkRetainedBytes(keyTokens) {
|
|
1108
|
+
if (keyTokens.length === 0 || keyTokens.length > MAX_UNIQUE_KEY_BUILD_TOKENS_PER_CHUNK) {
|
|
1109
|
+
throw new RangeError(`A UNIQUE build chunk must contain 1-${String(MAX_UNIQUE_KEY_BUILD_TOKENS_PER_CHUNK)} tokens`);
|
|
1110
|
+
}
|
|
1111
|
+
const seen = new Set();
|
|
1112
|
+
let bytes = 0;
|
|
1113
|
+
for (const token of keyTokens) {
|
|
1114
|
+
if (typeof token !== "string" ||
|
|
1115
|
+
token.length === 0 ||
|
|
1116
|
+
token.length > MAX_FTS_POSTING_TERM_CHARACTERS) {
|
|
1117
|
+
throw new TypeError("A UNIQUE build token has invalid length");
|
|
1118
|
+
}
|
|
1119
|
+
assertWellFormedString(token, "UNIQUE build token");
|
|
1120
|
+
if (seen.has(token))
|
|
1121
|
+
throw new UniqueKeyConflictError("UNIQUE build", token);
|
|
1122
|
+
seen.add(token);
|
|
1123
|
+
bytes = safeSum([bytes, 16 + token.length * 2], "UNIQUE build chunk bytes");
|
|
1124
|
+
if (bytes > MAX_UNIQUE_KEY_BUILD_CHUNK_BYTES) {
|
|
1125
|
+
throw new RangeError(`A UNIQUE build chunk cannot exceed ${String(MAX_UNIQUE_KEY_BUILD_CHUNK_BYTES)} retained bytes`);
|
|
1126
|
+
}
|
|
1127
|
+
}
|
|
1128
|
+
return bytes;
|
|
1129
|
+
}
|
|
1130
|
+
export class UniqueKeyBuildConflictError extends Error {
|
|
1131
|
+
buildId;
|
|
1132
|
+
reason;
|
|
1133
|
+
name = "UniqueKeyBuildConflictError";
|
|
1134
|
+
constructor(buildId, reason) {
|
|
1135
|
+
super(`UNIQUE build ${buildId} cannot continue: ${reason}`);
|
|
1136
|
+
this.buildId = buildId;
|
|
1137
|
+
this.reason = reason;
|
|
1138
|
+
}
|
|
1139
|
+
}
|
|
1140
|
+
export const MAX_POSTING_BUILD_TTL_MS = 60 * 60 * 1_000;
|
|
1141
|
+
export class PostingBuildConflictError extends Error {
|
|
1142
|
+
buildId;
|
|
1143
|
+
ownerId;
|
|
1144
|
+
reason;
|
|
1145
|
+
name = "PostingBuildConflictError";
|
|
1146
|
+
constructor(buildId, ownerId, reason) {
|
|
1147
|
+
super(`Posting build ${buildId} owned by ${ownerId}: ${reason}`);
|
|
1148
|
+
this.buildId = buildId;
|
|
1149
|
+
this.ownerId = ownerId;
|
|
1150
|
+
this.reason = reason;
|
|
1151
|
+
}
|
|
1152
|
+
}
|
|
1153
|
+
/** In-memory and atomic-publish bounds for one transaction's accelerator/key deltas. */
|
|
1154
|
+
export const MAX_TRANSACTION_COMMIT_DELTA_BYTES = 64 * 1024 * 1024;
|
|
1155
|
+
export const MAX_TRANSACTION_COMMIT_DELTA_ENTRIES = 1_048_576;
|
|
1156
|
+
export function transactionCommitDeltaRetainedBytes(uniqueKeyChanges, ftsChanges) {
|
|
1157
|
+
let bytes = 0;
|
|
1158
|
+
let entries = 0;
|
|
1159
|
+
const add = (amount) => {
|
|
1160
|
+
bytes += amount;
|
|
1161
|
+
if (!Number.isSafeInteger(bytes) || bytes > MAX_TRANSACTION_COMMIT_DELTA_BYTES) {
|
|
1162
|
+
throw new RangeError(`Transaction commit deltas exceed ${String(MAX_TRANSACTION_COMMIT_DELTA_BYTES)} retained bytes`);
|
|
1163
|
+
}
|
|
1164
|
+
};
|
|
1165
|
+
const addEntry = () => {
|
|
1166
|
+
entries += 1;
|
|
1167
|
+
if (entries > MAX_TRANSACTION_COMMIT_DELTA_ENTRIES) {
|
|
1168
|
+
throw new RangeError(`Transaction commit deltas exceed ${String(MAX_TRANSACTION_COMMIT_DELTA_ENTRIES)} entries`);
|
|
1169
|
+
}
|
|
1170
|
+
};
|
|
1171
|
+
for (const change of uniqueKeyChanges) {
|
|
1172
|
+
add(48 + change.tableId.length * 2);
|
|
1173
|
+
for (const token of change.keyTokens) {
|
|
1174
|
+
addEntry();
|
|
1175
|
+
add(16 + token.length * 2);
|
|
1176
|
+
}
|
|
1177
|
+
}
|
|
1178
|
+
for (const change of ftsChanges) {
|
|
1179
|
+
add(32 + change.tableId.length * 2);
|
|
1180
|
+
for (const column of change.columns) {
|
|
1181
|
+
add(48 + column.columnId.length * 2);
|
|
1182
|
+
for (const posting of column.postings) {
|
|
1183
|
+
addEntry();
|
|
1184
|
+
add(32 + posting.term.length * 2);
|
|
1185
|
+
if (posting.rowIds.length !== posting.tf.length) {
|
|
1186
|
+
throw new TypeError("Full-text posting row and frequency counts differ");
|
|
1187
|
+
}
|
|
1188
|
+
entries += posting.rowIds.length;
|
|
1189
|
+
if (entries > MAX_TRANSACTION_COMMIT_DELTA_ENTRIES) {
|
|
1190
|
+
throw new RangeError(`Transaction commit deltas exceed ${String(MAX_TRANSACTION_COMMIT_DELTA_ENTRIES)} entries`);
|
|
1191
|
+
}
|
|
1192
|
+
add(posting.rowIds.length * 16);
|
|
1193
|
+
}
|
|
1194
|
+
}
|
|
1195
|
+
}
|
|
1196
|
+
return { bytes, entries };
|
|
1197
|
+
}
|
|
1198
|
+
/** Aggregate row-id ceiling for one postings candidate read. Overflow falls back to a scan. */
|
|
1199
|
+
export const MAX_FTS_CANDIDATE_ROW_IDS = 65_536;
|
|
1200
|
+
/** Query-term cardinality shared by the parser and every postings adapter boundary. */
|
|
1201
|
+
export const MAX_FTS_QUERY_TERMS = 32;
|
|
1202
|
+
/** Retained metadata/value ceiling for one ordered postings read. Overflow falls back to a scan. */
|
|
1203
|
+
export const MAX_FTS_ORDERED_READ_BYTES = 64 * 1024 * 1024;
|
|
1204
|
+
/** Hard public mutation bounds for one streamed postings-build chunk. */
|
|
1205
|
+
export const MAX_FTS_POSTINGS_PER_CHUNK = 65_536;
|
|
1206
|
+
export const MAX_FTS_POSTING_ROW_IDS_PER_CHUNK = 1_048_576;
|
|
1207
|
+
export const MAX_FTS_POSTING_TERM_CHARACTERS = 65_536;
|
|
1208
|
+
/** Indexed string values are bounded; non-indexed block strings retain the physical block cap. */
|
|
1209
|
+
export const MAX_INDEXED_STRING_CHARACTERS = 16_384;
|
|
1210
|
+
/** Maximum normalized tokens retained from one indexed document. */
|
|
1211
|
+
export const MAX_FTS_TOKENS_PER_DOCUMENT = 4_096;
|
|
1212
|
+
/** Bounded catalog/TOC cardinality. Oversized accelerators invalidate and queries scan. */
|
|
1213
|
+
export const MAX_FTS_BASE_CHUNKS = 4_096;
|
|
1214
|
+
export const MAX_FTS_DELTA_CHUNKS = 128;
|
|
1215
|
+
/** Strict, bounded runtime validation shared by every adapter before any postings read. */
|
|
1216
|
+
export function validateFtsPostingQueries(value, label = "Full-text query terms") {
|
|
1217
|
+
if (!Array.isArray(value))
|
|
1218
|
+
throw new TypeError(`${label} must be an array`);
|
|
1219
|
+
if (value.length > MAX_FTS_QUERY_TERMS) {
|
|
1220
|
+
throw new RangeError(`${label} cannot exceed ${String(MAX_FTS_QUERY_TERMS)} items`);
|
|
1221
|
+
}
|
|
1222
|
+
for (const queryValue of value) {
|
|
1223
|
+
if (typeof queryValue !== "object" || queryValue === null || Array.isArray(queryValue)) {
|
|
1224
|
+
throw new TypeError("Full-text posting query must be an object");
|
|
1225
|
+
}
|
|
1226
|
+
const query = queryValue;
|
|
1227
|
+
const keys = Object.keys(query);
|
|
1228
|
+
if ("term" in query) {
|
|
1229
|
+
if (keys.length !== 2 ||
|
|
1230
|
+
keys.some((key) => key !== "term" && key !== "prefix") ||
|
|
1231
|
+
typeof query.term !== "string" ||
|
|
1232
|
+
query.term.length === 0 ||
|
|
1233
|
+
query.term.length > MAX_FTS_POSTING_TERM_CHARACTERS ||
|
|
1234
|
+
typeof query.prefix !== "boolean") {
|
|
1235
|
+
throw new TypeError("Full-text exact/prefix query is invalid");
|
|
1236
|
+
}
|
|
1237
|
+
assertWellFormedString(query.term, "Full-text query term");
|
|
1238
|
+
continue;
|
|
1239
|
+
}
|
|
1240
|
+
if (keys.some((key) => key !== "lower" &&
|
|
1241
|
+
key !== "lowerInclusive" &&
|
|
1242
|
+
key !== "upper" &&
|
|
1243
|
+
key !== "upperInclusive") ||
|
|
1244
|
+
(query.lower !== undefined &&
|
|
1245
|
+
(typeof query.lower !== "string" ||
|
|
1246
|
+
query.lower.length > MAX_FTS_POSTING_TERM_CHARACTERS)) ||
|
|
1247
|
+
(query.upper !== undefined &&
|
|
1248
|
+
(typeof query.upper !== "string" ||
|
|
1249
|
+
query.upper.length > MAX_FTS_POSTING_TERM_CHARACTERS)) ||
|
|
1250
|
+
(query.lowerInclusive !== undefined && typeof query.lowerInclusive !== "boolean") ||
|
|
1251
|
+
(query.upperInclusive !== undefined && typeof query.upperInclusive !== "boolean")) {
|
|
1252
|
+
throw new TypeError("Full-text range query is invalid");
|
|
1253
|
+
}
|
|
1254
|
+
if (typeof query.lower === "string") {
|
|
1255
|
+
assertWellFormedString(query.lower, "Full-text query lower bound");
|
|
1256
|
+
}
|
|
1257
|
+
if (typeof query.upper === "string") {
|
|
1258
|
+
assertWellFormedString(query.upper, "Full-text query upper bound");
|
|
1259
|
+
}
|
|
1260
|
+
}
|
|
1261
|
+
}
|
|
1262
|
+
/** Whether a stored term belongs to one exact, prefix, or range lookup. */
|
|
1263
|
+
export function ftsPostingQueryMatches(term, query) {
|
|
1264
|
+
if ("term" in query)
|
|
1265
|
+
return query.prefix ? term.startsWith(query.term) : term === query.term;
|
|
1266
|
+
if (query.lower !== undefined) {
|
|
1267
|
+
if (term < query.lower || (term === query.lower && query.lowerInclusive === false))
|
|
1268
|
+
return false;
|
|
1269
|
+
}
|
|
1270
|
+
if (query.upper !== undefined) {
|
|
1271
|
+
if (term > query.upper || (term === query.upper && query.upperInclusive === false))
|
|
1272
|
+
return false;
|
|
1273
|
+
}
|
|
1274
|
+
return true;
|
|
1275
|
+
}
|
|
166
1276
|
/**
|
|
167
1277
|
* Shared candidate-merge core for both stores: fetching chunks is store-specific, but the
|
|
168
1278
|
* term-match rule (exact, or prefix as a term range) and the sorted-unique row-id shape must
|
|
169
1279
|
* never drift between backends — pruning would silently differ per store.
|
|
170
1280
|
*/
|
|
171
|
-
export function collectFtsCandidates(chunkLists, terms) {
|
|
1281
|
+
export function collectFtsCandidates(chunkLists, terms, maxRowIds = MAX_FTS_CANDIDATE_ROW_IDS) {
|
|
1282
|
+
validateFtsPostingQueries(terms);
|
|
1283
|
+
if (!Number.isSafeInteger(maxRowIds) || maxRowIds < 1 || maxRowIds > MAX_FTS_CANDIDATE_ROW_IDS) {
|
|
1284
|
+
throw new RangeError(`Full-text candidate limit must be between 1 and ${String(MAX_FTS_CANDIDATE_ROW_IDS)}`);
|
|
1285
|
+
}
|
|
172
1286
|
const sets = terms.map(() => new Set());
|
|
1287
|
+
let retainedRowIds = 0;
|
|
173
1288
|
for (const postings of chunkLists) {
|
|
174
1289
|
for (const posting of postings) {
|
|
175
1290
|
for (let index = 0; index < terms.length; index += 1) {
|
|
176
1291
|
const term = terms[index];
|
|
177
1292
|
if (term === undefined)
|
|
178
1293
|
continue;
|
|
179
|
-
const matches = term
|
|
180
|
-
? posting.term.startsWith(term.term)
|
|
181
|
-
: posting.term === term.term;
|
|
1294
|
+
const matches = ftsPostingQueryMatches(posting.term, term);
|
|
182
1295
|
if (!matches)
|
|
183
1296
|
continue;
|
|
184
1297
|
const set = sets[index];
|
|
185
|
-
if (set !== undefined)
|
|
186
|
-
for (const rowId of posting.rowIds)
|
|
1298
|
+
if (set !== undefined) {
|
|
1299
|
+
for (const rowId of posting.rowIds) {
|
|
1300
|
+
if (set.has(rowId))
|
|
1301
|
+
continue;
|
|
1302
|
+
if (retainedRowIds === maxRowIds) {
|
|
1303
|
+
return { rowIdsByTerm: terms.map(() => []), overflow: true };
|
|
1304
|
+
}
|
|
187
1305
|
set.add(rowId);
|
|
1306
|
+
retainedRowIds += 1;
|
|
1307
|
+
}
|
|
1308
|
+
}
|
|
188
1309
|
}
|
|
189
1310
|
}
|
|
190
1311
|
}
|
|
191
1312
|
return {
|
|
192
1313
|
rowIdsByTerm: sets.map((set) => [...set].sort((left, right) => (left < right ? -1 : left > right ? 1 : 0))),
|
|
1314
|
+
overflow: false,
|
|
1315
|
+
};
|
|
1316
|
+
}
|
|
1317
|
+
/** Validates the fixed memory ceilings accepted by one ordered postings read. */
|
|
1318
|
+
export function validateFtsOrderedReadLimits(maxRowIds = MAX_FTS_CANDIDATE_ROW_IDS, maxRetainedBytes = MAX_FTS_ORDERED_READ_BYTES) {
|
|
1319
|
+
if (!Number.isSafeInteger(maxRowIds) || maxRowIds < 1 || maxRowIds > MAX_FTS_CANDIDATE_ROW_IDS) {
|
|
1320
|
+
throw new RangeError(`Ordered postings row limit must be between 1 and ${String(MAX_FTS_CANDIDATE_ROW_IDS)}`);
|
|
1321
|
+
}
|
|
1322
|
+
if (!Number.isSafeInteger(maxRetainedBytes) ||
|
|
1323
|
+
maxRetainedBytes < 1 ||
|
|
1324
|
+
maxRetainedBytes > MAX_FTS_ORDERED_READ_BYTES) {
|
|
1325
|
+
throw new RangeError(`Ordered postings byte limit must be between 1 and ${String(MAX_FTS_ORDERED_READ_BYTES)}`);
|
|
1326
|
+
}
|
|
1327
|
+
}
|
|
1328
|
+
/**
|
|
1329
|
+
* K-way merges sorted base/delta chunks into canonical term order. Only one term's row map and
|
|
1330
|
+
* one cursor per chunk stay live beyond the returned postings, avoiding a second index-sized map.
|
|
1331
|
+
*/
|
|
1332
|
+
export function collectFtsPostings(chunkLists) {
|
|
1333
|
+
return mergeFtsPostings(chunkLists, Number.POSITIVE_INFINITY, Number.POSITIVE_INFINITY).postings;
|
|
1334
|
+
}
|
|
1335
|
+
/** Bounded ordered merge; overflow deliberately returns no partial authoritative answer. */
|
|
1336
|
+
export function collectFtsPostingsBounded(chunkLists, maxRowIds = MAX_FTS_CANDIDATE_ROW_IDS, maxRetainedBytes = MAX_FTS_ORDERED_READ_BYTES) {
|
|
1337
|
+
validateFtsOrderedReadLimits(maxRowIds, maxRetainedBytes);
|
|
1338
|
+
return mergeFtsPostings(chunkLists, maxRowIds, maxRetainedBytes);
|
|
1339
|
+
}
|
|
1340
|
+
function mergeFtsPostings(chunkLists, maxRowIds, maxRetainedBytes) {
|
|
1341
|
+
const heap = [];
|
|
1342
|
+
const before = (left, right) => {
|
|
1343
|
+
const leftTerm = left.postings[left.position]?.term ?? "";
|
|
1344
|
+
const rightTerm = right.postings[right.position]?.term ?? "";
|
|
1345
|
+
return leftTerm < rightTerm || (leftTerm === rightTerm && left.ordinal < right.ordinal);
|
|
193
1346
|
};
|
|
1347
|
+
const cursorAt = (position) => {
|
|
1348
|
+
const cursor = heap[position];
|
|
1349
|
+
if (cursor === undefined)
|
|
1350
|
+
throw new Error("Posting merge heap is inconsistent");
|
|
1351
|
+
return cursor;
|
|
1352
|
+
};
|
|
1353
|
+
const push = (cursor) => {
|
|
1354
|
+
heap.push(cursor);
|
|
1355
|
+
let child = heap.length - 1;
|
|
1356
|
+
while (child > 0) {
|
|
1357
|
+
const parent = (child - 1) >>> 1;
|
|
1358
|
+
const childCursor = cursorAt(child);
|
|
1359
|
+
const parentCursor = cursorAt(parent);
|
|
1360
|
+
if (!before(childCursor, parentCursor))
|
|
1361
|
+
break;
|
|
1362
|
+
heap[parent] = childCursor;
|
|
1363
|
+
heap[child] = parentCursor;
|
|
1364
|
+
child = parent;
|
|
1365
|
+
}
|
|
1366
|
+
};
|
|
1367
|
+
const pop = () => {
|
|
1368
|
+
const first = heap[0];
|
|
1369
|
+
const last = heap.pop();
|
|
1370
|
+
if (first === undefined || last === undefined)
|
|
1371
|
+
return first;
|
|
1372
|
+
if (heap.length === 0)
|
|
1373
|
+
return first;
|
|
1374
|
+
heap[0] = last;
|
|
1375
|
+
let parent = 0;
|
|
1376
|
+
for (;;) {
|
|
1377
|
+
const left = parent * 2 + 1;
|
|
1378
|
+
if (left >= heap.length)
|
|
1379
|
+
break;
|
|
1380
|
+
const right = left + 1;
|
|
1381
|
+
const child = right < heap.length && before(cursorAt(right), cursorAt(left)) ? right : left;
|
|
1382
|
+
const childCursor = cursorAt(child);
|
|
1383
|
+
const parentCursor = cursorAt(parent);
|
|
1384
|
+
if (!before(childCursor, parentCursor))
|
|
1385
|
+
break;
|
|
1386
|
+
heap[parent] = childCursor;
|
|
1387
|
+
heap[child] = parentCursor;
|
|
1388
|
+
parent = child;
|
|
1389
|
+
}
|
|
1390
|
+
return first;
|
|
1391
|
+
};
|
|
1392
|
+
let ordinal = 0;
|
|
1393
|
+
for (const postings of chunkLists) {
|
|
1394
|
+
if (postings.length > 0)
|
|
1395
|
+
push({ postings, position: 0, ordinal });
|
|
1396
|
+
ordinal += 1;
|
|
1397
|
+
}
|
|
1398
|
+
const output = [];
|
|
1399
|
+
let retainedRowIds = 0;
|
|
1400
|
+
let retainedBytes = 0;
|
|
1401
|
+
while (heap.length > 0) {
|
|
1402
|
+
const term = heap[0]?.postings[heap[0].position]?.term;
|
|
1403
|
+
if (term === undefined)
|
|
1404
|
+
break;
|
|
1405
|
+
const rows = new Map();
|
|
1406
|
+
while (heap[0]?.postings[heap[0].position]?.term === term) {
|
|
1407
|
+
const cursor = pop();
|
|
1408
|
+
if (cursor === undefined)
|
|
1409
|
+
break;
|
|
1410
|
+
const posting = cursor.postings[cursor.position];
|
|
1411
|
+
if (posting === undefined)
|
|
1412
|
+
continue;
|
|
1413
|
+
posting.rowIds.forEach((rowId, index) => {
|
|
1414
|
+
const existing = rows.get(rowId);
|
|
1415
|
+
if (existing === undefined) {
|
|
1416
|
+
rows.set(rowId, posting.tf[index] ?? 1);
|
|
1417
|
+
}
|
|
1418
|
+
else {
|
|
1419
|
+
rows.set(rowId, Math.max(existing, posting.tf[index] ?? 1));
|
|
1420
|
+
}
|
|
1421
|
+
});
|
|
1422
|
+
cursor.position += 1;
|
|
1423
|
+
if (cursor.position < cursor.postings.length)
|
|
1424
|
+
push(cursor);
|
|
1425
|
+
}
|
|
1426
|
+
const nextRetainedRowIds = retainedRowIds + rows.size;
|
|
1427
|
+
const nextRetainedBytes = retainedBytes + 64 + term.length * 2 + rows.size * 32;
|
|
1428
|
+
if (nextRetainedRowIds > maxRowIds || nextRetainedBytes > maxRetainedBytes) {
|
|
1429
|
+
return { postings: [], overflow: true };
|
|
1430
|
+
}
|
|
1431
|
+
const ordered = [...rows].sort(([left], [right]) => (left < right ? -1 : left > right ? 1 : 0));
|
|
1432
|
+
output.push({
|
|
1433
|
+
term,
|
|
1434
|
+
rowIds: ordered.map(([rowId]) => rowId),
|
|
1435
|
+
tf: ordered.map(([, frequency]) => frequency),
|
|
1436
|
+
});
|
|
1437
|
+
retainedRowIds = nextRetainedRowIds;
|
|
1438
|
+
retainedBytes = nextRetainedBytes;
|
|
1439
|
+
}
|
|
1440
|
+
return { postings: output, overflow: false };
|
|
194
1441
|
}
|
|
195
1442
|
/**
|
|
196
1443
|
* Shared stale-writer policy for both stores' commit steps: a commit that adds segments to a
|
|
@@ -215,7 +1462,48 @@ export function invalidateUncoveredFtsColumns(record, coveredColumnIds) {
|
|
|
215
1462
|
}
|
|
216
1463
|
if (!invalidated)
|
|
217
1464
|
return undefined;
|
|
218
|
-
return {
|
|
1465
|
+
return {
|
|
1466
|
+
...record,
|
|
1467
|
+
ftsColumns: next,
|
|
1468
|
+
revision: safeSum([record.revision, 1], "Table revision"),
|
|
1469
|
+
};
|
|
1470
|
+
}
|
|
1471
|
+
/**
|
|
1472
|
+
* Scalar-index half of the stale-writer rule. A writer that staged table data from catalog
|
|
1473
|
+
* metadata older than a newly building/ready index cannot provide its postings, so the commit
|
|
1474
|
+
* invalidates that index atomically. Readers then scan until a rebuild closes the gap.
|
|
1475
|
+
*/
|
|
1476
|
+
export function invalidateUncoveredSecondaryIndexes(record, coveredStorageColumnIds) {
|
|
1477
|
+
const indexes = record.secondaryIndexes;
|
|
1478
|
+
if (indexes === undefined)
|
|
1479
|
+
return undefined;
|
|
1480
|
+
let invalidated = false;
|
|
1481
|
+
const next = {};
|
|
1482
|
+
for (const [indexId, index] of Object.entries(indexes)) {
|
|
1483
|
+
if (index.state !== "invalid" && !coveredStorageColumnIds.has(index.storageColumnId)) {
|
|
1484
|
+
const { buildId: _abandonedBuild, ...invalid } = index;
|
|
1485
|
+
void _abandonedBuild;
|
|
1486
|
+
next[indexId] = { ...invalid, state: "invalid" };
|
|
1487
|
+
invalidated = true;
|
|
1488
|
+
}
|
|
1489
|
+
else {
|
|
1490
|
+
next[indexId] = { ...index };
|
|
1491
|
+
}
|
|
1492
|
+
}
|
|
1493
|
+
if (!invalidated)
|
|
1494
|
+
return undefined;
|
|
1495
|
+
return {
|
|
1496
|
+
...record,
|
|
1497
|
+
secondaryIndexes: next,
|
|
1498
|
+
revision: safeSum([record.revision, 1], "Table revision"),
|
|
1499
|
+
};
|
|
1500
|
+
}
|
|
1501
|
+
/** Physical posting IDs a current catalog record authorizes a commit to write. */
|
|
1502
|
+
export function activePostingStorageColumnIds(record) {
|
|
1503
|
+
return new Set([
|
|
1504
|
+
...Object.entries(record.ftsColumns ?? {}).flatMap(([columnId, index]) => index.state === "invalid" ? [] : [columnId]),
|
|
1505
|
+
...Object.values(record.secondaryIndexes ?? {}).flatMap((index) => index.state === "invalid" ? [] : [index.storageColumnId]),
|
|
1506
|
+
]);
|
|
219
1507
|
}
|
|
220
1508
|
export class UniqueKeyConflictError extends Error {
|
|
221
1509
|
tableId;
|
|
@@ -227,6 +1515,17 @@ export class UniqueKeyConflictError extends Error {
|
|
|
227
1515
|
this.keyToken = keyToken;
|
|
228
1516
|
}
|
|
229
1517
|
}
|
|
1518
|
+
/** A writer prepared before an enforced UNIQUE index existed and therefore cannot enforce it. */
|
|
1519
|
+
export class UniqueIndexCoverageError extends Error {
|
|
1520
|
+
tableId;
|
|
1521
|
+
indexName;
|
|
1522
|
+
name = "UniqueIndexCoverageError";
|
|
1523
|
+
constructor(tableId, indexName) {
|
|
1524
|
+
super(`Write did not cover enforced UNIQUE index ${indexName} on table ${tableId}`);
|
|
1525
|
+
this.tableId = tableId;
|
|
1526
|
+
this.indexName = indexName;
|
|
1527
|
+
}
|
|
1528
|
+
}
|
|
230
1529
|
export class WriteConflictError extends Error {
|
|
231
1530
|
expectedVersion;
|
|
232
1531
|
actualVersion;
|
|
@@ -237,6 +1536,17 @@ export class WriteConflictError extends Error {
|
|
|
237
1536
|
this.actualVersion = actualVersion;
|
|
238
1537
|
}
|
|
239
1538
|
}
|
|
1539
|
+
/** Staged write artifacts were prepared against a structurally different catalog. */
|
|
1540
|
+
export class SchemaConflictError extends Error {
|
|
1541
|
+
expectedEpoch;
|
|
1542
|
+
actualEpoch;
|
|
1543
|
+
name = "SchemaConflictError";
|
|
1544
|
+
constructor(expectedEpoch, actualEpoch) {
|
|
1545
|
+
super(`Schema changed while the write was staged: expected epoch ${String(expectedEpoch)}, found ${String(actualEpoch)}`);
|
|
1546
|
+
this.expectedEpoch = expectedEpoch;
|
|
1547
|
+
this.actualEpoch = actualEpoch;
|
|
1548
|
+
}
|
|
1549
|
+
}
|
|
240
1550
|
export class TransactionRecordConflictError extends Error {
|
|
241
1551
|
transactionId;
|
|
242
1552
|
expectedRevision;
|
|
@@ -249,6 +1559,37 @@ export class TransactionRecordConflictError extends Error {
|
|
|
249
1559
|
this.actualRevision = actualRevision;
|
|
250
1560
|
}
|
|
251
1561
|
}
|
|
1562
|
+
/** Hard storage-boundary limits for one scratch-page write. */
|
|
1563
|
+
export const MAX_TEMP_RUN_PAGES_PER_BATCH = 64;
|
|
1564
|
+
export const MAX_TEMP_RUN_PAGE_BYTES = MAX_STORED_BLOCK_BYTE_LENGTH;
|
|
1565
|
+
export const MAX_TEMP_RUN_BATCH_BYTES = MAX_STORED_BLOCK_BYTE_LENGTH;
|
|
1566
|
+
/**
|
|
1567
|
+
* Refuses an oversized scratch write before an adapter starts a transaction or appends a WAL
|
|
1568
|
+
* record. Query execution splits pages and batches to these limits, so the limits bound atomic
|
|
1569
|
+
* storage work without limiting the total size of a spill.
|
|
1570
|
+
*/
|
|
1571
|
+
export function assertTempRunPageBatchLimits(pages) {
|
|
1572
|
+
if (pages.length > MAX_TEMP_RUN_PAGES_PER_BATCH) {
|
|
1573
|
+
throw new RangeError(`Temp run page batch exceeds ${String(MAX_TEMP_RUN_PAGES_PER_BATCH)} pages`);
|
|
1574
|
+
}
|
|
1575
|
+
let totalBytes = 0;
|
|
1576
|
+
for (const page of pages) {
|
|
1577
|
+
if (!(page.bytes instanceof Uint8Array)) {
|
|
1578
|
+
throw new TypeError("Temp run page bytes must be a Uint8Array");
|
|
1579
|
+
}
|
|
1580
|
+
if (page.bytes.byteLength > MAX_TEMP_RUN_PAGE_BYTES) {
|
|
1581
|
+
throw new RangeError(`Temp run page exceeds ${String(MAX_TEMP_RUN_PAGE_BYTES)} bytes`);
|
|
1582
|
+
}
|
|
1583
|
+
totalBytes += page.bytes.byteLength;
|
|
1584
|
+
if (!Number.isSafeInteger(totalBytes)) {
|
|
1585
|
+
throw new RangeError("Temp run page batch bytes exceed the safe integer range");
|
|
1586
|
+
}
|
|
1587
|
+
if (totalBytes > MAX_TEMP_RUN_BATCH_BYTES) {
|
|
1588
|
+
throw new RangeError(`Temp run page batch exceeds ${String(MAX_TEMP_RUN_BATCH_BYTES)} bytes`);
|
|
1589
|
+
}
|
|
1590
|
+
}
|
|
1591
|
+
}
|
|
1592
|
+
export const MAX_TEMP_OWNER_TTL_MS = 60 * 60 * 1_000;
|
|
252
1593
|
export class TempOwnerConflictError extends Error {
|
|
253
1594
|
ownerId;
|
|
254
1595
|
expectedRevision;
|
|
@@ -261,37 +1602,122 @@ export class TempOwnerConflictError extends Error {
|
|
|
261
1602
|
this.actualRevision = actualRevision;
|
|
262
1603
|
}
|
|
263
1604
|
}
|
|
1605
|
+
/**
|
|
1606
|
+
* Bulk payload storage: immutable, opaque byte blobs keyed by structured ids.
|
|
1607
|
+
*
|
|
1608
|
+
* Blocks are written only through transaction staging, which creates their durable provenance
|
|
1609
|
+
* in the same atomic step and rejects duplicate ids without a partial write. Ids contain `/` separators
|
|
1610
|
+
* (`table/<uuid>/segment/<uuid>/part/000001`) and sort lexically; treat them as opaque keys.
|
|
1611
|
+
* Reads MUST return bytes the caller may mutate freely (a fresh copy or freshly deserialized
|
|
1612
|
+
* buffer), and writes MUST NOT alias the caller's buffer — the engine may reuse it.
|
|
1613
|
+
*
|
|
1614
|
+
* Published blocks are retired by superseding them in a commit; only the store's atomic
|
|
1615
|
+
* lease-aware collection step can physically delete payloads once no reader can reach them.
|
|
1616
|
+
*/
|
|
1617
|
+
/** Maximum manifest-membership ids accepted by one storage call. */
|
|
1618
|
+
export const MAX_MANIFEST_BLOCK_PRESENCE_IDS = 1_024;
|
|
1619
|
+
/** Maximum UTF-16 code units in any opaque storage identity. */
|
|
1620
|
+
export const MAX_STORAGE_ID_CHARACTERS = 1_024;
|
|
1621
|
+
/** Maximum UTF-16 code units in a persisted catalog name. */
|
|
1622
|
+
export const MAX_CATALOG_NAME_CHARACTERS = 1_024;
|
|
1623
|
+
/** Public adapter database-name bound; names commonly become substrate keys or path segments. */
|
|
1624
|
+
export const MAX_STORAGE_DATABASE_NAME_CHARACTERS = 256;
|
|
1625
|
+
/** Maximum positional items accepted by one public bulk-read operation. */
|
|
1626
|
+
export const MAX_STORAGE_BULK_READ_ITEMS = 1_024;
|
|
1627
|
+
/** Maximum aggregate payload bytes returned by one getBlocks call. */
|
|
1628
|
+
export const MAX_BLOCK_READ_BATCH_BYTES = MAX_STORED_BLOCK_BYTE_LENGTH;
|
|
1629
|
+
export class BlockReadBatchTooLargeError extends RangeError {
|
|
1630
|
+
requestedBytes;
|
|
1631
|
+
limitBytes;
|
|
1632
|
+
name = "BlockReadBatchTooLargeError";
|
|
1633
|
+
constructor(requestedBytes, limitBytes = MAX_BLOCK_READ_BATCH_BYTES) {
|
|
1634
|
+
super(`Block read batch requires ${String(requestedBytes)} bytes; limit is ${String(limitBytes)}`);
|
|
1635
|
+
this.requestedBytes = requestedBytes;
|
|
1636
|
+
this.limitBytes = limitBytes;
|
|
1637
|
+
}
|
|
1638
|
+
}
|
|
1639
|
+
export function assertStorageBulkReadItems(items, label) {
|
|
1640
|
+
if (items.length > MAX_STORAGE_BULK_READ_ITEMS) {
|
|
1641
|
+
throw new RangeError(`${label} cannot exceed ${String(MAX_STORAGE_BULK_READ_ITEMS)} items`);
|
|
1642
|
+
}
|
|
1643
|
+
}
|
|
1644
|
+
export function validateStorageId(value, label = "Storage ID") {
|
|
1645
|
+
if (typeof value !== "string" || value.length === 0) {
|
|
1646
|
+
throw new TypeError(`${label} cannot be empty`);
|
|
1647
|
+
}
|
|
1648
|
+
if (value.length > MAX_STORAGE_ID_CHARACTERS) {
|
|
1649
|
+
throw new TypeError(`${label} exceeds ${String(MAX_STORAGE_ID_CHARACTERS)} characters`);
|
|
1650
|
+
}
|
|
1651
|
+
assertWellFormedString(value, label);
|
|
1652
|
+
return value;
|
|
1653
|
+
}
|
|
1654
|
+
export function validateCatalogName(value, label = "Catalog name") {
|
|
1655
|
+
if (typeof value !== "string" || value.trim().length === 0) {
|
|
1656
|
+
throw new TypeError(`${label} cannot be empty`);
|
|
1657
|
+
}
|
|
1658
|
+
if (value.length > MAX_CATALOG_NAME_CHARACTERS) {
|
|
1659
|
+
throw new TypeError(`${label} exceeds ${String(MAX_CATALOG_NAME_CHARACTERS)} characters`);
|
|
1660
|
+
}
|
|
1661
|
+
assertWellFormedString(value, label);
|
|
1662
|
+
return value;
|
|
1663
|
+
}
|
|
1664
|
+
export function validateStorageDatabaseName(value) {
|
|
1665
|
+
if (typeof value !== "string" || value.trim().length === 0) {
|
|
1666
|
+
throw new TypeError("Storage database name cannot be empty");
|
|
1667
|
+
}
|
|
1668
|
+
if (value.length > MAX_STORAGE_DATABASE_NAME_CHARACTERS) {
|
|
1669
|
+
throw new TypeError(`Storage database name exceeds ${String(MAX_STORAGE_DATABASE_NAME_CHARACTERS)} characters`);
|
|
1670
|
+
}
|
|
1671
|
+
assertWellFormedString(value, "Storage database name");
|
|
1672
|
+
return value;
|
|
1673
|
+
}
|
|
264
1674
|
export function createManifest(input) {
|
|
265
1675
|
return {
|
|
266
|
-
version: input.expectedVersion === null ? 0 : input.expectedVersion
|
|
1676
|
+
version: input.expectedVersion === null ? 0 : safeSum([input.expectedVersion, 1], "Manifest version"),
|
|
267
1677
|
previousVersion: input.expectedVersion,
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
1678
|
+
liveBlockCount: nonNegativeWholeNumber(input.liveBlockCount, "Manifest live block count"),
|
|
1679
|
+
liveBlockBytes: nonNegativeWholeNumber(input.liveBlockBytes, "Manifest live block bytes"),
|
|
1680
|
+
createdAt: input.createdAt ?? dateIsoString(new Date()),
|
|
1681
|
+
changedTableIds: canonicalManifestChangedTableIds(input.changedTableIds),
|
|
271
1682
|
};
|
|
272
1683
|
}
|
|
273
|
-
/**
|
|
274
|
-
* Normalizes the additive L2 partition metadata while preserving legacy segment records verbatim.
|
|
275
|
-
*/
|
|
1684
|
+
/** Strictly clones the one canonical v1 segment shape. */
|
|
276
1685
|
export function normalizeSegmentRecord(record) {
|
|
277
|
-
|
|
1686
|
+
switch (record.kind) {
|
|
1687
|
+
case "insert":
|
|
1688
|
+
case "upsert":
|
|
1689
|
+
case "update":
|
|
1690
|
+
case "delete":
|
|
1691
|
+
case "base":
|
|
1692
|
+
break;
|
|
1693
|
+
default:
|
|
1694
|
+
throw new TypeError("Segment kind is invalid");
|
|
1695
|
+
}
|
|
1696
|
+
const level = nonNegativeWholeNumber(record.level, "Segment level");
|
|
1697
|
+
if (level > 2)
|
|
1698
|
+
throw new RangeError("Segment level must be between zero and two");
|
|
1699
|
+
nonNegativeFiniteNumber(record.logicalOrder, "Segment logical order");
|
|
1700
|
+
nonNegativeWholeNumber(record.commitOrdinal, "Segment commit ordinal");
|
|
1701
|
+
if (!Array.isArray(record.rowIdSpans)) {
|
|
1702
|
+
throw new TypeError("Segment row ID spans must be an array");
|
|
1703
|
+
}
|
|
1704
|
+
if (record.partitionOrdinal === undefined) {
|
|
1705
|
+
if (level === 2)
|
|
1706
|
+
throw new TypeError("A level-two segment requires a partition ordinal");
|
|
278
1707
|
return structuredClone(record);
|
|
1708
|
+
}
|
|
279
1709
|
const partitionOrdinal = nonNegativeWholeNumber(record.partitionOrdinal, "Segment partition ordinal");
|
|
280
1710
|
if (record.level !== 2) {
|
|
281
1711
|
throw new TypeError("A partitioned segment must have explicit level two");
|
|
282
1712
|
}
|
|
283
|
-
const kind = record.kind
|
|
1713
|
+
const kind = record.kind;
|
|
284
1714
|
if (kind !== "insert" && kind !== "base") {
|
|
285
1715
|
throw new TypeError("A partitioned segment must be an insert or a merged base");
|
|
286
1716
|
}
|
|
287
|
-
if (record.logicalOrder === undefined) {
|
|
288
|
-
throw new TypeError("A partitioned segment requires an explicit logical order");
|
|
289
|
-
}
|
|
290
|
-
nonNegativeFiniteNumber(record.logicalOrder, "Segment logical order");
|
|
291
1717
|
const rowCount = positiveWholeNumber(record.rowCount, "Segment row count");
|
|
292
1718
|
if (kind === "insert") {
|
|
293
1719
|
// Append-row-range partition: one contiguous positive row-ID interval, no spans.
|
|
294
|
-
if (record.rowIdSpans !==
|
|
1720
|
+
if (record.rowIdSpans.length !== 0) {
|
|
295
1721
|
throw new TypeError("A partitioned segment cannot contain row ID spans");
|
|
296
1722
|
}
|
|
297
1723
|
if (typeof record.rowIdStart !== "bigint" || record.rowIdStart <= 0n) {
|
|
@@ -305,7 +1731,7 @@ export function normalizeSegmentRecord(record) {
|
|
|
305
1731
|
}
|
|
306
1732
|
// Keyed multi-range partition: a merged full-row base whose live rows keep their original
|
|
307
1733
|
// ids, described by positive, sorted, non-overlapping spans that sum to the row count.
|
|
308
|
-
if (record.rowIdSpans
|
|
1734
|
+
if (record.rowIdSpans.length === 0) {
|
|
309
1735
|
throw new TypeError("A merged partitioned segment requires row ID spans");
|
|
310
1736
|
}
|
|
311
1737
|
let spanRows = 0;
|
|
@@ -329,20 +1755,34 @@ export function normalizeSegmentRecord(record) {
|
|
|
329
1755
|
return structuredClone({ ...record, partitionOrdinal });
|
|
330
1756
|
}
|
|
331
1757
|
export function updateTransactionRecord(record, update) {
|
|
332
|
-
|
|
1758
|
+
const updated = {
|
|
333
1759
|
...record,
|
|
334
1760
|
...(update.snapshotVersion === undefined ? {} : { snapshotVersion: update.snapshotVersion }),
|
|
335
1761
|
...(update.pendingBlockIds === undefined
|
|
336
1762
|
? {}
|
|
337
|
-
: {
|
|
1763
|
+
: {
|
|
1764
|
+
pendingBlockIds: orderedUniqueIds(update.pendingBlockIds, "Transaction pending block ID"),
|
|
1765
|
+
}),
|
|
338
1766
|
...(update.pendingSegmentIds === undefined
|
|
339
1767
|
? {}
|
|
340
|
-
: {
|
|
1768
|
+
: {
|
|
1769
|
+
pendingSegmentIds: orderedUniqueIds(update.pendingSegmentIds, "Transaction pending segment ID"),
|
|
1770
|
+
}),
|
|
341
1771
|
...(update.status === undefined ? {} : { status: update.status }),
|
|
342
1772
|
...(update.committedVersion === undefined ? {} : { committedVersion: update.committedVersion }),
|
|
1773
|
+
...(update.pendingTableNextRowId === undefined
|
|
1774
|
+
? {}
|
|
1775
|
+
: { pendingTableNextRowId: update.pendingTableNextRowId }),
|
|
343
1776
|
updatedAt: update.updatedAt,
|
|
344
|
-
revision: record.revision
|
|
1777
|
+
revision: safeSum([record.revision, 1], "Transaction revision"),
|
|
345
1778
|
};
|
|
1779
|
+
if (updated.status !== "active") {
|
|
1780
|
+
delete updated.pendingTable;
|
|
1781
|
+
delete updated.pendingTableNextRowId;
|
|
1782
|
+
delete updated.catalogEpochGuard;
|
|
1783
|
+
delete updated.schemaEpochGuard;
|
|
1784
|
+
}
|
|
1785
|
+
return updated;
|
|
346
1786
|
}
|
|
347
1787
|
export function createGarbageCollectionJobRecord(input) {
|
|
348
1788
|
const candidateManifestVersions = uniqueWholeNumbers(input.candidateManifestVersions, "Garbage collection candidate manifest version");
|
|
@@ -350,12 +1790,16 @@ export function createGarbageCollectionJobRecord(input) {
|
|
|
350
1790
|
const candidateBlockIds = uniqueIds(input.candidateBlockIds, "Garbage collection candidate block ID", true);
|
|
351
1791
|
const candidateTransactionIds = uniqueIds(input.candidateTransactionIds ?? [], "Garbage collection candidate transaction ID", true);
|
|
352
1792
|
const createdAt = validTimestamp(input.createdAt, "Garbage collection creation timestamp");
|
|
353
|
-
const
|
|
1793
|
+
const discovery = input.discovery === undefined
|
|
1794
|
+
? undefined
|
|
1795
|
+
: normalizeGarbageCollectionDiscovery(input.discovery);
|
|
1796
|
+
const complete = (discovery === undefined || discovery.phase === "complete") &&
|
|
1797
|
+
candidateManifestVersions.length === 0 &&
|
|
354
1798
|
candidateSegmentIds.length === 0 &&
|
|
355
1799
|
candidateBlockIds.length === 0 &&
|
|
356
1800
|
candidateTransactionIds.length === 0;
|
|
357
1801
|
return {
|
|
358
|
-
id:
|
|
1802
|
+
id: validateStorageId(input.id, "Garbage collection job ID"),
|
|
359
1803
|
candidateManifestVersions,
|
|
360
1804
|
candidateSegmentIds,
|
|
361
1805
|
candidateBlockIds,
|
|
@@ -380,20 +1824,169 @@ export function createGarbageCollectionJobRecord(input) {
|
|
|
380
1824
|
leaseCutoff: validTimestamp(input.leaseCutoff, "Garbage collection lease cutoff"),
|
|
381
1825
|
createdAt,
|
|
382
1826
|
updatedAt: createdAt,
|
|
1827
|
+
...(discovery === undefined ? {} : { discovery }),
|
|
1828
|
+
};
|
|
1829
|
+
}
|
|
1830
|
+
export function normalizeGarbageCollectionDiscovery(discovery) {
|
|
1831
|
+
const runtime = discovery;
|
|
1832
|
+
if (typeof runtime !== "object" || runtime === null) {
|
|
1833
|
+
throw new TypeError("Garbage collection discovery must be an object");
|
|
1834
|
+
}
|
|
1835
|
+
const phase = runtime.phase;
|
|
1836
|
+
if (phase !== "manifests" &&
|
|
1837
|
+
phase !== "manifest-blocks" &&
|
|
1838
|
+
phase !== "segments" &&
|
|
1839
|
+
phase !== "transactions" &&
|
|
1840
|
+
phase !== "compactions" &&
|
|
1841
|
+
phase !== "complete") {
|
|
1842
|
+
throw new TypeError(`Invalid garbage collection discovery phase: ${String(phase)}`);
|
|
1843
|
+
}
|
|
1844
|
+
const stringCursor = (value, name) => value === null ? null : validateStorageId(value, name);
|
|
1845
|
+
const currentManifestVersion = discovery.currentManifestVersion === null
|
|
1846
|
+
? null
|
|
1847
|
+
: nonNegativeWholeNumber(discovery.currentManifestVersion, "Garbage collection discovery manifest version");
|
|
1848
|
+
const manifestCursor = discovery.manifestCursor === null
|
|
1849
|
+
? null
|
|
1850
|
+
: nonNegativeWholeNumber(discovery.manifestCursor, "Garbage collection manifest discovery cursor");
|
|
1851
|
+
if (!Number.isSafeInteger(discovery.retainAboveVersion)) {
|
|
1852
|
+
throw new RangeError("Garbage collection retained version floor must be a safe integer");
|
|
1853
|
+
}
|
|
1854
|
+
if (!Number.isSafeInteger(discovery.retainAfter)) {
|
|
1855
|
+
throw new RangeError("Garbage collection retained timestamp floor must be a safe integer");
|
|
1856
|
+
}
|
|
1857
|
+
const resumePhase = runtime.resumePhase;
|
|
1858
|
+
if (resumePhase !== undefined &&
|
|
1859
|
+
resumePhase !== null &&
|
|
1860
|
+
resumePhase !== "manifests" &&
|
|
1861
|
+
resumePhase !== "manifest-blocks" &&
|
|
1862
|
+
resumePhase !== "segments" &&
|
|
1863
|
+
resumePhase !== "transactions" &&
|
|
1864
|
+
resumePhase !== "compactions") {
|
|
1865
|
+
throw new TypeError("Invalid garbage collection resume phase");
|
|
1866
|
+
}
|
|
1867
|
+
const postManifestPhase = runtime.postManifestPhase;
|
|
1868
|
+
if (postManifestPhase !== undefined &&
|
|
1869
|
+
postManifestPhase !== null &&
|
|
1870
|
+
postManifestPhase !== "manifest-blocks" &&
|
|
1871
|
+
postManifestPhase !== "segments" &&
|
|
1872
|
+
postManifestPhase !== "transactions" &&
|
|
1873
|
+
postManifestPhase !== "compactions") {
|
|
1874
|
+
throw new TypeError("Invalid garbage collection post-manifest phase");
|
|
1875
|
+
}
|
|
1876
|
+
const artifactCursor = runtime.artifactCursor;
|
|
1877
|
+
let normalizedArtifactCursor;
|
|
1878
|
+
if (artifactCursor !== undefined && artifactCursor !== null) {
|
|
1879
|
+
if (typeof artifactCursor !== "object") {
|
|
1880
|
+
throw new TypeError("Garbage collection artifact cursor must be an object");
|
|
1881
|
+
}
|
|
1882
|
+
const rawArtifactCursor = artifactCursor;
|
|
1883
|
+
if (rawArtifactCursor.family !== "manifest" &&
|
|
1884
|
+
rawArtifactCursor.family !== "transaction" &&
|
|
1885
|
+
rawArtifactCursor.family !== "compaction") {
|
|
1886
|
+
throw new TypeError(`Invalid garbage collection artifact cursor family: ${String(rawArtifactCursor.family)}`);
|
|
1887
|
+
}
|
|
1888
|
+
normalizedArtifactCursor = {
|
|
1889
|
+
family: rawArtifactCursor.family,
|
|
1890
|
+
recordId: validateStorageId(rawArtifactCursor.recordId, "Garbage collection artifact cursor record ID"),
|
|
1891
|
+
blockId: rawArtifactCursor.blockId === null
|
|
1892
|
+
? null
|
|
1893
|
+
: validateStorageId(rawArtifactCursor.blockId, "Garbage collection artifact cursor block ID"),
|
|
1894
|
+
blockIndex: nonNegativeWholeNumber(rawArtifactCursor.blockIndex, "Garbage collection artifact block cursor"),
|
|
1895
|
+
segmentIndex: nonNegativeWholeNumber(rawArtifactCursor.segmentIndex, "Garbage collection artifact segment cursor"),
|
|
1896
|
+
};
|
|
1897
|
+
}
|
|
1898
|
+
else {
|
|
1899
|
+
normalizedArtifactCursor = artifactCursor;
|
|
1900
|
+
}
|
|
1901
|
+
return {
|
|
1902
|
+
phase,
|
|
1903
|
+
currentManifestVersion,
|
|
1904
|
+
retainAboveVersion: discovery.retainAboveVersion,
|
|
1905
|
+
retainAfter: discovery.retainAfter,
|
|
1906
|
+
maxPlanningItems: boundedMaintenanceBatchItems(discovery.maxPlanningItems, "Garbage collection planning item limit"),
|
|
1907
|
+
manifestCursor,
|
|
1908
|
+
segmentCursor: stringCursor(discovery.segmentCursor, "Garbage collection segment discovery cursor"),
|
|
1909
|
+
transactionCursor: stringCursor(discovery.transactionCursor, "Garbage collection transaction discovery cursor"),
|
|
1910
|
+
compactionCursor: stringCursor(discovery.compactionCursor, "Garbage collection compaction discovery cursor"),
|
|
1911
|
+
visitedRecords: nonNegativeWholeNumber(discovery.visitedRecords, "Garbage collection discovery visited records"),
|
|
1912
|
+
...(resumePhase === undefined ? {} : { resumePhase }),
|
|
1913
|
+
...(postManifestPhase === undefined ? {} : { postManifestPhase }),
|
|
1914
|
+
...(normalizedArtifactCursor === undefined ? {} : { artifactCursor: normalizedArtifactCursor }),
|
|
383
1915
|
};
|
|
384
1916
|
}
|
|
1917
|
+
export function updateGarbageCollectionPlanningRecord(record, input) {
|
|
1918
|
+
const current = normalizeGarbageCollectionJobRecord(record);
|
|
1919
|
+
if (current.id !== input.jobId || current.revision !== input.expectedRevision) {
|
|
1920
|
+
throw new GarbageCollectionJobConflictError(input.jobId, input.expectedRevision, current.id === input.jobId ? current.revision : null);
|
|
1921
|
+
}
|
|
1922
|
+
if (current.state !== "planned" || current.discovery?.phase === "complete") {
|
|
1923
|
+
throw new TypeError("Only a discovering planned garbage collection job can be updated");
|
|
1924
|
+
}
|
|
1925
|
+
const appendIds = (existing, additions, label) => {
|
|
1926
|
+
const normalized = uniqueIds(additions ?? [], label, true);
|
|
1927
|
+
const seen = new Set(existing);
|
|
1928
|
+
for (const id of normalized) {
|
|
1929
|
+
if (seen.has(id))
|
|
1930
|
+
throw new TypeError(`${label} is already planned: ${id}`);
|
|
1931
|
+
seen.add(id);
|
|
1932
|
+
}
|
|
1933
|
+
return [...existing, ...normalized];
|
|
1934
|
+
};
|
|
1935
|
+
const candidateManifestVersions = [...current.candidateManifestVersions];
|
|
1936
|
+
const manifestSet = new Set(candidateManifestVersions);
|
|
1937
|
+
for (const version of uniqueWholeNumbers(input.candidateManifestVersions ?? [], "Garbage collection candidate manifest version")) {
|
|
1938
|
+
if (manifestSet.has(version)) {
|
|
1939
|
+
throw new TypeError(`Garbage collection manifest is already planned: ${String(version)}`);
|
|
1940
|
+
}
|
|
1941
|
+
manifestSet.add(version);
|
|
1942
|
+
candidateManifestVersions.push(version);
|
|
1943
|
+
}
|
|
1944
|
+
candidateManifestVersions.sort((left, right) => left - right);
|
|
1945
|
+
const candidateSegmentIds = appendIds(current.candidateSegmentIds, input.candidateSegmentIds, "Garbage collection candidate segment ID");
|
|
1946
|
+
const candidateBlockIds = appendIds(current.candidateBlockIds, input.candidateBlockIds, "Garbage collection candidate block ID");
|
|
1947
|
+
const candidateTransactionIds = appendIds(current.candidateTransactionIds, input.candidateTransactionIds, "Garbage collection candidate transaction ID");
|
|
1948
|
+
const discovery = normalizeGarbageCollectionDiscovery(input.discovery);
|
|
1949
|
+
const fixedDiscovery = current.discovery;
|
|
1950
|
+
if (discovery.currentManifestVersion !== fixedDiscovery?.currentManifestVersion ||
|
|
1951
|
+
discovery.retainAboveVersion !== fixedDiscovery.retainAboveVersion ||
|
|
1952
|
+
discovery.retainAfter !== fixedDiscovery.retainAfter ||
|
|
1953
|
+
discovery.maxPlanningItems !== fixedDiscovery.maxPlanningItems) {
|
|
1954
|
+
throw new TypeError("Garbage collection discovery snapshot, retention boundary, and item limit are immutable");
|
|
1955
|
+
}
|
|
1956
|
+
if (candidateManifestVersions.length +
|
|
1957
|
+
candidateSegmentIds.length +
|
|
1958
|
+
candidateBlockIds.length +
|
|
1959
|
+
candidateTransactionIds.length >
|
|
1960
|
+
discovery.maxPlanningItems) {
|
|
1961
|
+
throw new RangeError("Garbage collection planning candidates exceed the persisted limit");
|
|
1962
|
+
}
|
|
1963
|
+
const updated = {
|
|
1964
|
+
...current,
|
|
1965
|
+
candidateManifestVersions,
|
|
1966
|
+
candidateSegmentIds,
|
|
1967
|
+
candidateBlockIds,
|
|
1968
|
+
candidateTransactionIds,
|
|
1969
|
+
discovery,
|
|
1970
|
+
state: "planned",
|
|
1971
|
+
revision: safeSum([current.revision, 1], "Garbage collection job revision"),
|
|
1972
|
+
updatedAt: validTimestamp(input.updatedAt, "Garbage collection update timestamp"),
|
|
1973
|
+
};
|
|
1974
|
+
if (garbageCollectionJobComplete(updated))
|
|
1975
|
+
updated.state = "completed";
|
|
1976
|
+
return normalizeGarbageCollectionJobRecord(updated);
|
|
1977
|
+
}
|
|
385
1978
|
export function normalizeGarbageCollectionJobRecord(record) {
|
|
386
|
-
const legacy = record;
|
|
387
1979
|
const candidateManifestVersions = uniqueWholeNumbers(record.candidateManifestVersions, "Garbage collection candidate manifest version");
|
|
388
1980
|
const candidateSegmentIds = uniqueIds(record.candidateSegmentIds, "Garbage collection candidate segment ID", true);
|
|
389
1981
|
const candidateBlockIds = uniqueIds(record.candidateBlockIds, "Garbage collection candidate block ID", true);
|
|
390
|
-
|
|
391
|
-
// phase, which lets an old planned/running job resume after an upgrade without migration.
|
|
392
|
-
const candidateTransactionIds = uniqueIds(legacy.candidateTransactionIds ?? [], "Garbage collection candidate transaction ID", true);
|
|
1982
|
+
const candidateTransactionIds = uniqueIds(record.candidateTransactionIds, "Garbage collection candidate transaction ID", true);
|
|
393
1983
|
const cursor = normalizeGarbageCollectionCursor(record.cursor);
|
|
1984
|
+
const discovery = record.discovery === undefined
|
|
1985
|
+
? undefined
|
|
1986
|
+
: normalizeGarbageCollectionDiscovery(record.discovery);
|
|
394
1987
|
const normalized = {
|
|
395
1988
|
...record,
|
|
396
|
-
id:
|
|
1989
|
+
id: validateStorageId(record.id, "Garbage collection job ID"),
|
|
397
1990
|
candidateManifestVersions,
|
|
398
1991
|
candidateSegmentIds,
|
|
399
1992
|
candidateBlockIds,
|
|
@@ -410,15 +2003,24 @@ export function normalizeGarbageCollectionJobRecord(record) {
|
|
|
410
2003
|
retainedBlockCount: nonNegativeWholeNumber(record.retainedBlockCount, "Garbage collection retained block count"),
|
|
411
2004
|
missingBlockCount: nonNegativeWholeNumber(record.missingBlockCount, "Garbage collection missing block count"),
|
|
412
2005
|
reclaimedBlockBytes: nonNegativeWholeNumber(record.reclaimedBlockBytes, "Garbage collection reclaimed block bytes"),
|
|
413
|
-
reclaimedTransactionCount: nonNegativeWholeNumber(
|
|
414
|
-
retainedTransactionCount: nonNegativeWholeNumber(
|
|
415
|
-
missingTransactionCount: nonNegativeWholeNumber(
|
|
2006
|
+
reclaimedTransactionCount: nonNegativeWholeNumber(record.reclaimedTransactionCount, "Garbage collection reclaimed transaction count"),
|
|
2007
|
+
retainedTransactionCount: nonNegativeWholeNumber(record.retainedTransactionCount, "Garbage collection retained transaction count"),
|
|
2008
|
+
missingTransactionCount: nonNegativeWholeNumber(record.missingTransactionCount, "Garbage collection missing transaction count"),
|
|
416
2009
|
state: garbageCollectionJobState(record.state),
|
|
417
2010
|
revision: nonNegativeWholeNumber(record.revision, "Garbage collection job revision"),
|
|
418
2011
|
leaseCutoff: validTimestamp(record.leaseCutoff, "Garbage collection lease cutoff"),
|
|
419
2012
|
createdAt: validTimestamp(record.createdAt, "Garbage collection creation timestamp"),
|
|
420
2013
|
updatedAt: validTimestamp(record.updatedAt, "Garbage collection update timestamp"),
|
|
2014
|
+
...(discovery === undefined ? {} : { discovery }),
|
|
421
2015
|
};
|
|
2016
|
+
if (discovery !== undefined &&
|
|
2017
|
+
candidateManifestVersions.length +
|
|
2018
|
+
candidateSegmentIds.length +
|
|
2019
|
+
candidateBlockIds.length +
|
|
2020
|
+
candidateTransactionIds.length >
|
|
2021
|
+
discovery.maxPlanningItems) {
|
|
2022
|
+
throw new RangeError("Garbage collection planning candidates exceed the persisted limit");
|
|
2023
|
+
}
|
|
422
2024
|
if (safeSum([
|
|
423
2025
|
normalized.prunedManifestCount,
|
|
424
2026
|
normalized.alreadyPrunedManifestCount,
|
|
@@ -461,6 +2063,9 @@ export function normalizeGarbageCollectionJobRecord(record) {
|
|
|
461
2063
|
}
|
|
462
2064
|
export function advanceGarbageCollectionJobRecord(record, accounting) {
|
|
463
2065
|
const current = normalizeGarbageCollectionJobRecord(record);
|
|
2066
|
+
if (current.discovery !== undefined && current.discovery.phase !== "complete") {
|
|
2067
|
+
throw new TypeError("Garbage collection discovery must complete before reclamation starts");
|
|
2068
|
+
}
|
|
464
2069
|
if (current.state === "completed")
|
|
465
2070
|
return current;
|
|
466
2071
|
const increments = {
|
|
@@ -534,7 +2139,7 @@ export function advanceGarbageCollectionJobRecord(record, accounting) {
|
|
|
534
2139
|
retainedTransactionCount: safeSum([current.retainedTransactionCount, increments.retainedTransactionCount], "Garbage collection retained transaction count"),
|
|
535
2140
|
missingTransactionCount: safeSum([current.missingTransactionCount, increments.missingTransactionCount], "Garbage collection missing transaction count"),
|
|
536
2141
|
state: "running",
|
|
537
|
-
revision: current.revision
|
|
2142
|
+
revision: safeSum([current.revision, 1], "Garbage collection revision"),
|
|
538
2143
|
updatedAt: validTimestamp(accounting.updatedAt, "Garbage collection update timestamp"),
|
|
539
2144
|
};
|
|
540
2145
|
if (garbageCollectionJobComplete(updated))
|
|
@@ -550,22 +2155,14 @@ export function normalizeCompactionJobRecord(record) {
|
|
|
550
2155
|
const logicalBytes = nonNegativeWholeNumber(record.logicalBytes, "Compaction logical bytes");
|
|
551
2156
|
const sourceStoredBytes = nonNegativeWholeNumber(record.sourceStoredBytes, "Compaction source stored bytes");
|
|
552
2157
|
const outputStoredBytes = nonNegativeWholeNumber(record.outputStoredBytes, "Compaction output stored bytes");
|
|
553
|
-
const
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
anchorSourceStoredBytes: nonNegativeWholeNumber(record.anchorSourceStoredBytes, "Compaction anchor source stored bytes"),
|
|
562
|
-
}
|
|
563
|
-
: undefined;
|
|
564
|
-
if (sourceLevelStoredBytes !== undefined &&
|
|
565
|
-
safeSum([
|
|
566
|
-
sourceLevelStoredBytes.level0SourceStoredBytes,
|
|
567
|
-
sourceLevelStoredBytes.anchorSourceStoredBytes,
|
|
568
|
-
], "Compaction source-level stored bytes") !== sourceStoredBytes) {
|
|
2158
|
+
const sourceLevelStoredBytes = {
|
|
2159
|
+
level0SourceStoredBytes: nonNegativeWholeNumber(record.level0SourceStoredBytes, "Compaction level-zero source stored bytes"),
|
|
2160
|
+
anchorSourceStoredBytes: nonNegativeWholeNumber(record.anchorSourceStoredBytes, "Compaction anchor source stored bytes"),
|
|
2161
|
+
};
|
|
2162
|
+
if (safeSum([
|
|
2163
|
+
sourceLevelStoredBytes.level0SourceStoredBytes,
|
|
2164
|
+
sourceLevelStoredBytes.anchorSourceStoredBytes,
|
|
2165
|
+
], "Compaction source-level stored bytes") !== sourceStoredBytes) {
|
|
569
2166
|
throw new TypeError("Compaction source-level stored bytes must equal source stored bytes");
|
|
570
2167
|
}
|
|
571
2168
|
const level2PolicyValues = [
|
|
@@ -592,13 +2189,10 @@ export function normalizeCompactionJobRecord(record) {
|
|
|
592
2189
|
// Append-row-range promotions consume pure level-zero prefixes; keyed merge promotions may
|
|
593
2190
|
// also fold a retained level-one anchor, whose bytes never count toward the L0 ceiling.
|
|
594
2191
|
if (rewritePlan.kind === "rechunk-v1" &&
|
|
595
|
-
(sourceLevelStoredBytes
|
|
2192
|
+
(sourceLevelStoredBytes.level0SourceStoredBytes !== sourceStoredBytes ||
|
|
596
2193
|
sourceLevelStoredBytes.anchorSourceStoredBytes !== 0)) {
|
|
597
2194
|
throw new TypeError("Append-row-range L2 compaction requires only level-zero source bytes");
|
|
598
2195
|
}
|
|
599
|
-
if (rewritePlan.kind === "merge-v1" && sourceLevelStoredBytes === undefined) {
|
|
600
|
-
throw new TypeError("Keyed L2 compaction requires source-level byte accounting");
|
|
601
|
-
}
|
|
602
2196
|
const outputPartitionOrdinal = nonNegativeWholeNumber(record.outputPartitionOrdinal, "Compaction output partition ordinal");
|
|
603
2197
|
const maxWriteAmplification = positiveFiniteNumber(record.maxWriteAmplification, "Compaction maximum write amplification");
|
|
604
2198
|
const maximumOutputStoredBytes = positiveWholeNumber(record.maximumOutputStoredBytes, "Compaction maximum output stored bytes");
|
|
@@ -628,8 +2222,8 @@ export function normalizeCompactionJobRecord(record) {
|
|
|
628
2222
|
}
|
|
629
2223
|
const normalized = {
|
|
630
2224
|
...record,
|
|
631
|
-
id:
|
|
632
|
-
tableId:
|
|
2225
|
+
id: validateStorageId(record.id, "Compaction job ID"),
|
|
2226
|
+
tableId: validateStorageId(record.tableId, "Compaction job table ID"),
|
|
633
2227
|
sourceManifestVersion: nonNegativeWholeNumber(record.sourceManifestVersion, "Compaction source manifest version"),
|
|
634
2228
|
sourceSegmentIds: rewritePlan.kind === "copy-v1"
|
|
635
2229
|
? uniqueIds(record.sourceSegmentIds, "Compaction source segment ID", false)
|
|
@@ -647,12 +2241,12 @@ export function normalizeCompactionJobRecord(record) {
|
|
|
647
2241
|
logicalBytes,
|
|
648
2242
|
rewritePlan,
|
|
649
2243
|
outputCursor: normalizeCompactionOutputCursor(record.outputCursor, rewritePlan),
|
|
650
|
-
memoryBudgetBytes: nonNegativeWholeNumber(record.memoryBudgetBytes
|
|
651
|
-
minimumMemoryBytes: nonNegativeWholeNumber(record.minimumMemoryBytes
|
|
652
|
-
...
|
|
2244
|
+
memoryBudgetBytes: nonNegativeWholeNumber(record.memoryBudgetBytes, "Compaction memory budget"),
|
|
2245
|
+
minimumMemoryBytes: nonNegativeWholeNumber(record.minimumMemoryBytes, "Compaction minimum memory"),
|
|
2246
|
+
...sourceLevelStoredBytes,
|
|
653
2247
|
...(level2Policy ?? {}),
|
|
654
|
-
peakWorkingBytes: nonNegativeWholeNumber(record.peakWorkingBytes
|
|
655
|
-
outputLogicalBytes: nonNegativeWholeNumber(record.outputLogicalBytes
|
|
2248
|
+
peakWorkingBytes: nonNegativeWholeNumber(record.peakWorkingBytes, "Compaction peak working bytes"),
|
|
2249
|
+
outputLogicalBytes: nonNegativeWholeNumber(record.outputLogicalBytes, "Compaction output logical bytes"),
|
|
656
2250
|
targetLevel: nonNegativeWholeNumber(record.targetLevel, "Compaction target level"),
|
|
657
2251
|
state: compactionJobState(record.state),
|
|
658
2252
|
transactionId: nullableId(record.transactionId, "Compaction transaction ID"),
|
|
@@ -703,7 +2297,7 @@ export function updateCompactionJobRecord(record, update) {
|
|
|
703
2297
|
}
|
|
704
2298
|
}
|
|
705
2299
|
}
|
|
706
|
-
const mirrorCopyLogicalBytes = current.rewritePlan
|
|
2300
|
+
const mirrorCopyLogicalBytes = current.rewritePlan.kind === "copy-v1" &&
|
|
707
2301
|
update.logicalBytes !== undefined &&
|
|
708
2302
|
update.outputLogicalBytes === undefined;
|
|
709
2303
|
const updated = {
|
|
@@ -730,7 +2324,7 @@ export function updateCompactionJobRecord(record, update) {
|
|
|
730
2324
|
...(update.outputSegmentId === undefined ? {} : { outputSegmentId: update.outputSegmentId }),
|
|
731
2325
|
...(update.publishedVersion === undefined ? {} : { publishedVersion: update.publishedVersion }),
|
|
732
2326
|
updatedAt: update.updatedAt,
|
|
733
|
-
revision: current.revision
|
|
2327
|
+
revision: safeSum([current.revision, 1], "Compaction revision"),
|
|
734
2328
|
};
|
|
735
2329
|
if (update.error === null)
|
|
736
2330
|
delete updated.error;
|
|
@@ -742,7 +2336,7 @@ export function updateCompactionJobRecord(record, update) {
|
|
|
742
2336
|
return normalized;
|
|
743
2337
|
}
|
|
744
2338
|
function validateCompactionJobState(record) {
|
|
745
|
-
const plan = record.rewritePlan
|
|
2339
|
+
const plan = record.rewritePlan;
|
|
746
2340
|
if (plan.kind === "merge-v1") {
|
|
747
2341
|
if (plan.totalRows === 0 && record.outputSegmentId !== null) {
|
|
748
2342
|
throw new TypeError("An empty merge compaction cannot have an output segment");
|
|
@@ -755,7 +2349,7 @@ function validateCompactionJobState(record) {
|
|
|
755
2349
|
throw new TypeError("A cancelled compaction cannot contain an error");
|
|
756
2350
|
}
|
|
757
2351
|
if (record.state === "planned") {
|
|
758
|
-
const isCopy = record.rewritePlan
|
|
2352
|
+
const isCopy = record.rewritePlan.kind === "copy-v1";
|
|
759
2353
|
const hasProgress = record.cursor.sourceSegmentIndex !== 0 ||
|
|
760
2354
|
record.cursor.sourceBlockIndex !== 0 ||
|
|
761
2355
|
record.outputBlockIds.length !== 0 ||
|
|
@@ -785,7 +2379,7 @@ function validateCompactionJobState(record) {
|
|
|
785
2379
|
}
|
|
786
2380
|
if (isOutputDrivenCompactionPlan(record.rewritePlan) &&
|
|
787
2381
|
expectedCompactionOutputCount(record) > 0 &&
|
|
788
|
-
|
|
2382
|
+
record.peakWorkingBytes < record.minimumMemoryBytes) {
|
|
789
2383
|
throw new TypeError(`${record.state} compaction requires complete memory accounting`);
|
|
790
2384
|
}
|
|
791
2385
|
}
|
|
@@ -799,11 +2393,11 @@ function validateCompactionJobState(record) {
|
|
|
799
2393
|
}
|
|
800
2394
|
}
|
|
801
2395
|
function validateCompactionRewrite(record) {
|
|
802
|
-
const plan = record.rewritePlan
|
|
803
|
-
const outputLogicalBytes = record.outputLogicalBytes
|
|
804
|
-
const memoryBudgetBytes = record.memoryBudgetBytes
|
|
805
|
-
const minimumMemoryBytes = record.minimumMemoryBytes
|
|
806
|
-
const peakWorkingBytes = record.peakWorkingBytes
|
|
2396
|
+
const plan = record.rewritePlan;
|
|
2397
|
+
const outputLogicalBytes = record.outputLogicalBytes;
|
|
2398
|
+
const memoryBudgetBytes = record.memoryBudgetBytes;
|
|
2399
|
+
const minimumMemoryBytes = record.minimumMemoryBytes;
|
|
2400
|
+
const peakWorkingBytes = record.peakWorkingBytes;
|
|
807
2401
|
if (plan.kind === "copy-v1") {
|
|
808
2402
|
if (record.outputCursor !== null) {
|
|
809
2403
|
throw new TypeError("A copy compaction cannot have an output cursor");
|
|
@@ -868,7 +2462,7 @@ function validateCompactionRewrite(record) {
|
|
|
868
2462
|
}
|
|
869
2463
|
}
|
|
870
2464
|
const cursor = record.outputCursor;
|
|
871
|
-
if (cursor === null
|
|
2465
|
+
if (cursor === null) {
|
|
872
2466
|
throw new TypeError("An output-driven compaction requires an output cursor");
|
|
873
2467
|
}
|
|
874
2468
|
const completedOutputs = safeSum([
|
|
@@ -891,8 +2485,8 @@ function validateCompactionJobProgress(previous, next) {
|
|
|
891
2485
|
["source stored bytes", previous.sourceStoredBytes, next.sourceStoredBytes],
|
|
892
2486
|
["output stored bytes", previous.outputStoredBytes, next.outputStoredBytes],
|
|
893
2487
|
["logical bytes", previous.logicalBytes, next.logicalBytes],
|
|
894
|
-
["output logical bytes", previous.outputLogicalBytes
|
|
895
|
-
["peak working bytes", previous.peakWorkingBytes
|
|
2488
|
+
["output logical bytes", previous.outputLogicalBytes, next.outputLogicalBytes],
|
|
2489
|
+
["peak working bytes", previous.peakWorkingBytes, next.peakWorkingBytes],
|
|
896
2490
|
]) {
|
|
897
2491
|
if (nextValue < previousValue) {
|
|
898
2492
|
throw new RangeError(`Compaction ${label} cannot decrease`);
|
|
@@ -928,7 +2522,7 @@ function validateCompactionJobProgress(previous, next) {
|
|
|
928
2522
|
}
|
|
929
2523
|
}
|
|
930
2524
|
function isInitialOutputCursor(record) {
|
|
931
|
-
const plan = record.rewritePlan
|
|
2525
|
+
const plan = record.rewritePlan;
|
|
932
2526
|
if (plan.kind === "copy-v1")
|
|
933
2527
|
return record.outputCursor === null;
|
|
934
2528
|
const cursor = record.outputCursor;
|
|
@@ -936,7 +2530,7 @@ function isInitialOutputCursor(record) {
|
|
|
936
2530
|
return (cursor?.outputIndex === 0 && cursor.columnIndex === 0 && cursor.rowStart === initialRowStart);
|
|
937
2531
|
}
|
|
938
2532
|
function hasCompletedCompactionCursor(record) {
|
|
939
|
-
const plan = record.rewritePlan
|
|
2533
|
+
const plan = record.rewritePlan;
|
|
940
2534
|
if (plan.kind === "copy-v1") {
|
|
941
2535
|
return record.cursor.sourceSegmentIndex === record.sourceSegmentIds.length;
|
|
942
2536
|
}
|
|
@@ -946,7 +2540,7 @@ function hasCompletedCompactionCursor(record) {
|
|
|
946
2540
|
cursor.rowStart === plan.totalRows);
|
|
947
2541
|
}
|
|
948
2542
|
function expectedCompactionOutputCount(record) {
|
|
949
|
-
const plan = record.rewritePlan
|
|
2543
|
+
const plan = record.rewritePlan;
|
|
950
2544
|
return plan.kind === "copy-v1"
|
|
951
2545
|
? record.sourceBlockIds.length
|
|
952
2546
|
: safeProduct(plan.outputs.length, plan.columns.length, "Rechunk output block count");
|
|
@@ -954,7 +2548,7 @@ function expectedCompactionOutputCount(record) {
|
|
|
954
2548
|
function compactionOutputOrdinal(record) {
|
|
955
2549
|
const plan = record.rewritePlan;
|
|
956
2550
|
const cursor = record.outputCursor;
|
|
957
|
-
if (!isOutputDrivenCompactionPlan(plan) || cursor === null
|
|
2551
|
+
if (!isOutputDrivenCompactionPlan(plan) || cursor === null)
|
|
958
2552
|
return 0;
|
|
959
2553
|
return safeSum([
|
|
960
2554
|
safeProduct(cursor.outputIndex, plan.columns.length, "Rechunk output cursor"),
|
|
@@ -962,7 +2556,7 @@ function compactionOutputOrdinal(record) {
|
|
|
962
2556
|
], "Rechunk output cursor");
|
|
963
2557
|
}
|
|
964
2558
|
function isOutputDrivenCompactionPlan(plan) {
|
|
965
|
-
return plan
|
|
2559
|
+
return plan.kind === "rechunk-v1" || plan.kind === "merge-v1";
|
|
966
2560
|
}
|
|
967
2561
|
function validateCompactionJobTransition(previous, next) {
|
|
968
2562
|
const allowed = {
|
|
@@ -987,8 +2581,6 @@ function normalizeCompactionJobCursor(value) {
|
|
|
987
2581
|
};
|
|
988
2582
|
}
|
|
989
2583
|
function normalizeCompactionRewritePlan(value) {
|
|
990
|
-
if (value === undefined)
|
|
991
|
-
return { kind: "copy-v1" };
|
|
992
2584
|
if (typeof value !== "object" || value === null) {
|
|
993
2585
|
throw new TypeError("Compaction rewrite plan must be an object");
|
|
994
2586
|
}
|
|
@@ -1025,7 +2617,7 @@ function normalizeCompactionRewritePlan(value) {
|
|
|
1025
2617
|
}
|
|
1026
2618
|
return {
|
|
1027
2619
|
rowStart: nonNegativeWholeNumber(Reflect.get(output, "rowStart"), `Rechunk output window ${String(index)} row start`),
|
|
1028
|
-
rowCount:
|
|
2620
|
+
rowCount: blockRowCount(Reflect.get(output, "rowCount"), `Rechunk output window ${String(index)} row count`),
|
|
1029
2621
|
};
|
|
1030
2622
|
});
|
|
1031
2623
|
validateContiguousRows(outputs, totalRows, "Rechunk output windows");
|
|
@@ -1059,9 +2651,9 @@ function normalizeRechunkSourceColumn(value, totalRows, columnIndex) {
|
|
|
1059
2651
|
throw new TypeError(`Rechunk source block ${String(columnIndex)}:${String(blockIndex)} must be an object`);
|
|
1060
2652
|
}
|
|
1061
2653
|
return {
|
|
1062
|
-
blockId:
|
|
2654
|
+
blockId: validateStorageId(Reflect.get(block, "blockId"), `Rechunk source block ${String(columnIndex)}:${String(blockIndex)} ID`),
|
|
1063
2655
|
rowStart: nonNegativeWholeNumber(Reflect.get(block, "rowStart"), `Rechunk source block ${String(columnIndex)}:${String(blockIndex)} row start`),
|
|
1064
|
-
rowCount:
|
|
2656
|
+
rowCount: blockRowCount(Reflect.get(block, "rowCount"), `Rechunk source block ${String(columnIndex)}:${String(blockIndex)} row count`),
|
|
1065
2657
|
storedBytes: positiveWholeNumber(Reflect.get(block, "storedBytes"), `Rechunk source block ${String(columnIndex)}:${String(blockIndex)} stored bytes`),
|
|
1066
2658
|
encodedBytes: nonNegativeWholeNumber(Reflect.get(block, "encodedBytes"), `Rechunk source block ${String(columnIndex)}:${String(blockIndex)} encoded bytes`),
|
|
1067
2659
|
checksum: uint32(Reflect.get(block, "checksum"), `Rechunk source block ${String(columnIndex)}:${String(blockIndex)} checksum`),
|
|
@@ -1069,7 +2661,7 @@ function normalizeRechunkSourceColumn(value, totalRows, columnIndex) {
|
|
|
1069
2661
|
});
|
|
1070
2662
|
validateContiguousRows(sourceBlocks, totalRows, `Rechunk source column ${String(columnIndex)} blocks`);
|
|
1071
2663
|
return {
|
|
1072
|
-
columnId:
|
|
2664
|
+
columnId: validateStorageId(Reflect.get(value, "columnId"), `Rechunk source column ${String(columnIndex)} ID`),
|
|
1073
2665
|
type: simpleDataType(Reflect.get(value, "type")),
|
|
1074
2666
|
sourceBlocks,
|
|
1075
2667
|
};
|
|
@@ -1079,7 +2671,7 @@ function normalizeMergeCompactionRewritePlan(value) {
|
|
|
1079
2671
|
const rowIdStart = nonNegativeBigInt(Reflect.get(value, "rowIdStart"), "Merge row ID start");
|
|
1080
2672
|
const rowIdEndExclusive = nonNegativeBigInt(Reflect.get(value, "rowIdEndExclusive"), "Merge row ID end");
|
|
1081
2673
|
const rowIdSpans = normalizeRowIdSpans(Reflect.get(value, "rowIdSpans"), totalRows, rowIdStart, rowIdEndExclusive, "Merge output row ID spans");
|
|
1082
|
-
const keyColumnId =
|
|
2674
|
+
const keyColumnId = validateStorageId(Reflect.get(value, "keyColumnId"), "Merge key column ID");
|
|
1083
2675
|
const sourceSegmentsValue = Reflect.get(value, "sourceSegments");
|
|
1084
2676
|
if (!Array.isArray(sourceSegmentsValue) || sourceSegmentsValue.length === 0) {
|
|
1085
2677
|
throw new TypeError("A merge plan requires at least one source segment");
|
|
@@ -1138,7 +2730,7 @@ function normalizeMergeCompactionRewritePlan(value) {
|
|
|
1138
2730
|
}
|
|
1139
2731
|
return {
|
|
1140
2732
|
rowStart: nonNegativeWholeNumber(Reflect.get(output, "rowStart"), `Merge output window ${String(index)} row start`),
|
|
1141
|
-
rowCount:
|
|
2733
|
+
rowCount: blockRowCount(Reflect.get(output, "rowCount"), `Merge output window ${String(index)} row count`),
|
|
1142
2734
|
};
|
|
1143
2735
|
});
|
|
1144
2736
|
validateContiguousRows(outputs, totalRows, "Merge output windows");
|
|
@@ -1245,8 +2837,8 @@ function normalizeMergeSourceSegment(value, segmentIndex) {
|
|
|
1245
2837
|
throw new TypeError(`${label} cannot contain duplicate source columns`);
|
|
1246
2838
|
}
|
|
1247
2839
|
return {
|
|
1248
|
-
segmentId:
|
|
1249
|
-
transactionId:
|
|
2840
|
+
segmentId: validateStorageId(Reflect.get(value, "segmentId"), `${label} ID`),
|
|
2841
|
+
transactionId: validateStorageId(Reflect.get(value, "transactionId"), `${label} transaction ID`),
|
|
1250
2842
|
committedVersion: nonNegativeWholeNumber(Reflect.get(value, "committedVersion"), `${label} committed version`),
|
|
1251
2843
|
kind,
|
|
1252
2844
|
keyColumnId: nullableId(Reflect.get(value, "keyColumnId"), `${label} key column ID`),
|
|
@@ -1273,9 +2865,9 @@ function normalizeMergeSourceColumn(value, segmentRowCount, segmentIndex, column
|
|
|
1273
2865
|
throw new TypeError(`${label} block ${String(blockIndex)} must be an object`);
|
|
1274
2866
|
}
|
|
1275
2867
|
return {
|
|
1276
|
-
blockId:
|
|
2868
|
+
blockId: validateStorageId(Reflect.get(block, "blockId"), `${label} block ID`),
|
|
1277
2869
|
rowStart: nonNegativeWholeNumber(Reflect.get(block, "rowStart"), `${label} block row start`),
|
|
1278
|
-
rowCount:
|
|
2870
|
+
rowCount: blockRowCount(Reflect.get(block, "rowCount"), `${label} block row count`),
|
|
1279
2871
|
storedBytes: positiveWholeNumber(Reflect.get(block, "storedBytes"), `${label} block stored bytes`),
|
|
1280
2872
|
encodedBytes: nonNegativeWholeNumber(Reflect.get(block, "encodedBytes"), `${label} block encoded bytes`),
|
|
1281
2873
|
checksum: uint32(Reflect.get(block, "checksum"), `${label} block checksum`),
|
|
@@ -1283,7 +2875,7 @@ function normalizeMergeSourceColumn(value, segmentRowCount, segmentIndex, column
|
|
|
1283
2875
|
});
|
|
1284
2876
|
validateContiguousRows(sourceBlocks, segmentRowCount, `${label} blocks`);
|
|
1285
2877
|
return {
|
|
1286
|
-
columnId:
|
|
2878
|
+
columnId: validateStorageId(Reflect.get(value, "columnId"), `${label} ID`),
|
|
1287
2879
|
type: simpleDataType(Reflect.get(value, "type")),
|
|
1288
2880
|
sourceBlocks,
|
|
1289
2881
|
};
|
|
@@ -1293,7 +2885,7 @@ function normalizeMergeOutputColumn(value, totalRows, sourceBlocks, columnIndex)
|
|
|
1293
2885
|
if (typeof value !== "object" || value === null) {
|
|
1294
2886
|
throw new TypeError(`${label} must be an object`);
|
|
1295
2887
|
}
|
|
1296
|
-
const columnId =
|
|
2888
|
+
const columnId = validateStorageId(Reflect.get(value, "columnId"), `${label} ID`);
|
|
1297
2889
|
const type = simpleDataType(Reflect.get(value, "type"));
|
|
1298
2890
|
const rangesValue = Reflect.get(value, "sourceRanges");
|
|
1299
2891
|
if (!Array.isArray(rangesValue))
|
|
@@ -1304,9 +2896,9 @@ function normalizeMergeOutputColumn(value, totalRows, sourceBlocks, columnIndex)
|
|
|
1304
2896
|
}
|
|
1305
2897
|
const normalized = {
|
|
1306
2898
|
outputRowStart: nonNegativeWholeNumber(Reflect.get(range, "outputRowStart"), `${label} source range output row start`),
|
|
1307
|
-
sourceBlockId:
|
|
2899
|
+
sourceBlockId: validateStorageId(Reflect.get(range, "sourceBlockId"), `${label} source range block ID`),
|
|
1308
2900
|
sourceRowStart: nonNegativeWholeNumber(Reflect.get(range, "sourceRowStart"), `${label} source range block row start`),
|
|
1309
|
-
rowCount:
|
|
2901
|
+
rowCount: blockRowCount(Reflect.get(range, "rowCount"), `${label} source range row count`),
|
|
1310
2902
|
};
|
|
1311
2903
|
const source = sourceBlocks.get(normalized.sourceBlockId);
|
|
1312
2904
|
if (source === undefined)
|
|
@@ -1432,16 +3024,13 @@ function compareMergeSourceSegments(left, right) {
|
|
|
1432
3024
|
}
|
|
1433
3025
|
function normalizeCompactionOutputCursor(value, plan) {
|
|
1434
3026
|
if (plan.kind === "copy-v1") {
|
|
1435
|
-
if (value !==
|
|
3027
|
+
if (value !== null) {
|
|
1436
3028
|
throw new TypeError("A copy compaction cannot have an output cursor");
|
|
1437
3029
|
}
|
|
1438
3030
|
return null;
|
|
1439
3031
|
}
|
|
1440
3032
|
if (value === undefined) {
|
|
1441
|
-
|
|
1442
|
-
throw new TypeError("A merge compaction requires an explicit output cursor");
|
|
1443
|
-
}
|
|
1444
|
-
return { outputIndex: 0, columnIndex: 0, rowStart: plan.outputs[0]?.rowStart ?? 0 };
|
|
3033
|
+
throw new TypeError("An output-driven compaction requires an explicit output cursor");
|
|
1445
3034
|
}
|
|
1446
3035
|
if (typeof value !== "object" || value === null) {
|
|
1447
3036
|
throw new TypeError("Rechunk output cursor must be an object");
|
|
@@ -1522,11 +3111,12 @@ function normalizeGarbageCollectionCursor(value) {
|
|
|
1522
3111
|
manifestIndex: nonNegativeWholeNumber(Reflect.get(value, "manifestIndex"), "Garbage collection manifest cursor"),
|
|
1523
3112
|
segmentIndex: nonNegativeWholeNumber(Reflect.get(value, "segmentIndex"), "Garbage collection segment cursor"),
|
|
1524
3113
|
blockIndex: nonNegativeWholeNumber(Reflect.get(value, "blockIndex"), "Garbage collection block cursor"),
|
|
1525
|
-
transactionIndex: nonNegativeWholeNumber(Reflect.get(value, "transactionIndex")
|
|
3114
|
+
transactionIndex: nonNegativeWholeNumber(Reflect.get(value, "transactionIndex"), "Garbage collection transaction cursor"),
|
|
1526
3115
|
};
|
|
1527
3116
|
}
|
|
1528
3117
|
function garbageCollectionJobComplete(record) {
|
|
1529
|
-
return (record.
|
|
3118
|
+
return ((record.discovery === undefined || record.discovery.phase === "complete") &&
|
|
3119
|
+
record.cursor.manifestIndex === record.candidateManifestVersions.length &&
|
|
1530
3120
|
record.cursor.segmentIndex === record.candidateSegmentIds.length &&
|
|
1531
3121
|
record.cursor.blockIndex === record.candidateBlockIds.length &&
|
|
1532
3122
|
record.cursor.transactionIndex === record.candidateTransactionIds.length);
|
|
@@ -1534,13 +3124,13 @@ function garbageCollectionJobComplete(record) {
|
|
|
1534
3124
|
function uniqueIds(ids, label, sort) {
|
|
1535
3125
|
if (!Array.isArray(ids))
|
|
1536
3126
|
throw new TypeError(`${label}s must be an array`);
|
|
1537
|
-
const unique = [...new Set(ids.map((id) =>
|
|
3127
|
+
const unique = [...new Set(ids.map((id) => validateStorageId(id, label)))];
|
|
1538
3128
|
return sort ? unique.sort() : unique;
|
|
1539
3129
|
}
|
|
1540
3130
|
function orderedUniqueIds(ids, label) {
|
|
1541
3131
|
if (!Array.isArray(ids))
|
|
1542
3132
|
throw new TypeError(`${label}s must be an array`);
|
|
1543
|
-
const normalized = ids.map((id) =>
|
|
3133
|
+
const normalized = ids.map((id) => validateStorageId(id, label));
|
|
1544
3134
|
if (new Set(normalized).size !== normalized.length) {
|
|
1545
3135
|
throw new TypeError(`${label}s cannot contain duplicates`);
|
|
1546
3136
|
}
|
|
@@ -1552,7 +3142,7 @@ function uniqueWholeNumbers(values, label) {
|
|
|
1552
3142
|
return [...new Set(values.map((value) => nonNegativeWholeNumber(value, label)))].sort((left, right) => left - right);
|
|
1553
3143
|
}
|
|
1554
3144
|
function nullableId(id, label) {
|
|
1555
|
-
return id === null ? null :
|
|
3145
|
+
return id === null ? null : validateStorageId(id, label);
|
|
1556
3146
|
}
|
|
1557
3147
|
function nonEmptyString(value, label) {
|
|
1558
3148
|
if (typeof value !== "string" || value.length === 0) {
|
|
@@ -1562,8 +3152,16 @@ function nonEmptyString(value, label) {
|
|
|
1562
3152
|
}
|
|
1563
3153
|
function validTimestamp(value, label) {
|
|
1564
3154
|
const timestamp = nonEmptyString(value, label);
|
|
1565
|
-
|
|
1566
|
-
|
|
3155
|
+
// Storage timestamps are internal coordination metadata, not SQL date values. Keeping the
|
|
3156
|
+
// canonical v1 form fixed at 24 UTF-8 bytes makes byte reservations exact (notably the
|
|
3157
|
+
// manifest-pruning tombstone) and avoids engine-dependent Date.parse spellings.
|
|
3158
|
+
if (!/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$/u.test(timestamp)) {
|
|
3159
|
+
throw new TypeError(`${label} must be canonical UTC ISO-8601`);
|
|
3160
|
+
}
|
|
3161
|
+
const milliseconds = Date.parse(timestamp);
|
|
3162
|
+
if (!Number.isFinite(milliseconds) || dateIsoString(new Date(milliseconds)) !== timestamp) {
|
|
3163
|
+
throw new TypeError(`${label} must be canonical UTC ISO-8601`);
|
|
3164
|
+
}
|
|
1567
3165
|
return timestamp;
|
|
1568
3166
|
}
|
|
1569
3167
|
function nonNegativeWholeNumber(value, label) {
|
|
@@ -1608,6 +3206,13 @@ function positiveUint32(value, label) {
|
|
|
1608
3206
|
throw new RangeError(`${label} must be positive`);
|
|
1609
3207
|
return normalized;
|
|
1610
3208
|
}
|
|
3209
|
+
function blockRowCount(value, label) {
|
|
3210
|
+
const normalized = positiveUint32(value, label);
|
|
3211
|
+
if (normalized > MAX_BLOCK_ROW_COUNT) {
|
|
3212
|
+
throw new RangeError(`${label} exceeds the block format row limit`);
|
|
3213
|
+
}
|
|
3214
|
+
return normalized;
|
|
3215
|
+
}
|
|
1611
3216
|
function safeSum(values, label) {
|
|
1612
3217
|
let total = 0;
|
|
1613
3218
|
for (const value of values) {
|
|
@@ -1647,4 +3252,3 @@ export function floorWholeNumberProduct(left, right, label) {
|
|
|
1647
3252
|
}
|
|
1648
3253
|
return Number(product);
|
|
1649
3254
|
}
|
|
1650
|
-
//# sourceMappingURL=types.js.map
|