@quereus/store 4.4.0 → 4.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (33) hide show
  1. package/README.md +512 -486
  2. package/dist/src/common/cached-kv-store.js +4 -0
  3. package/dist/src/common/cached-kv-store.js.map +1 -1
  4. package/dist/src/common/encoding.d.ts +32 -1
  5. package/dist/src/common/encoding.d.ts.map +1 -1
  6. package/dist/src/common/encoding.js +42 -10
  7. package/dist/src/common/encoding.js.map +1 -1
  8. package/dist/src/common/index.d.ts +3 -2
  9. package/dist/src/common/index.d.ts.map +1 -1
  10. package/dist/src/common/index.js +3 -1
  11. package/dist/src/common/index.js.map +1 -1
  12. package/dist/src/common/json-key.d.ts +96 -0
  13. package/dist/src/common/json-key.d.ts.map +1 -0
  14. package/dist/src/common/json-key.js +204 -0
  15. package/dist/src/common/json-key.js.map +1 -0
  16. package/dist/src/common/key-builder.d.ts +41 -5
  17. package/dist/src/common/key-builder.d.ts.map +1 -1
  18. package/dist/src/common/key-builder.js +60 -21
  19. package/dist/src/common/key-builder.js.map +1 -1
  20. package/dist/src/common/kv-store.d.ts +12 -2
  21. package/dist/src/common/kv-store.d.ts.map +1 -1
  22. package/dist/src/common/store-module.d.ts +64 -15
  23. package/dist/src/common/store-module.d.ts.map +1 -1
  24. package/dist/src/common/store-module.js +439 -133
  25. package/dist/src/common/store-module.js.map +1 -1
  26. package/dist/src/common/store-table.d.ts +226 -12
  27. package/dist/src/common/store-table.d.ts.map +1 -1
  28. package/dist/src/common/store-table.js +454 -37
  29. package/dist/src/common/store-table.js.map +1 -1
  30. package/dist/src/testing/kv-conformance.d.ts.map +1 -1
  31. package/dist/src/testing/kv-conformance.js +32 -0
  32. package/dist/src/testing/kv-conformance.js.map +1 -1
  33. package/package.json +6 -6
@@ -12,15 +12,15 @@
12
12
  * - {prefix}.__stats__ - Unified stats store (row counts for all tables)
13
13
  * - Catalog store: __catalog__ - DDL metadata keyed by {schema}.{table}
14
14
  */
15
- import { AccessPlanBuilder, QuereusError, StatusCode, buildColumnIndexMap, columnDefToSchema, compilePredicate, inferType, tryFoldLiteral, validateAndParse, buildAdvertisementsFromTags, resolveNamedConstraintClass, validateCollationForType, buildUniqueConstraintSchema, buildForeignKeyConstraintSchema, buildCheckConstraintSchema, validateForeignKeyOverExistingRows, extractColumnLevelCheckConstraints, extractColumnLevelForeignKeys, appendIndexToTableSchema, logicalTypeCanHoldText, serializeRowKey, isMaintainedTable, renameColumnInIndexPredicates, renameTableInIndexPredicates, renameColumnInCheckConstraints, renameTableInCheckConstraints } from '@quereus/quereus';
15
+ import { AccessPlanBuilder, QuereusError, StatusCode, buildColumnIndexMap, columnDefToSchema, compilePredicate, equalitySeekKeyCount, hasSemanticOrdering, inferType, isMultiValueEquality, tryFoldLiteral, validateAndParse, buildAdvertisementsFromTags, resolveNamedConstraintClass, validateCollationForType, buildUniqueConstraintSchema, buildForeignKeyConstraintSchema, buildCheckConstraintSchema, validateForeignKeyOverExistingRows, appendIndexToTableSchema, logicalTypeCanHoldText, serializeKey, isMaintainedTable, renameColumnInIndexPredicates, renameTableInIndexPredicates, renameColumnInCheckConstraints, renameTableInCheckConstraints } from '@quereus/quereus';
16
16
  import { TransactionCoordinator } from './transaction.js';
17
17
  import { StoreConnection } from './store-connection.js';
18
18
  import { StoreBackingHost } from './backing-host.js';
19
- import { StoreTable, resolvePkKeyCollations, columnCanHoldText, keyOrderMatchesCollation, pkOrderPreservingPrefixLength, withImplicitUniqueIndexes, implicitUniqueIndexName } from './store-table.js';
19
+ import { StoreTable, resolvePkKeyCollations, resolvePkKeyTransforms, resolveIndexKeyTransforms, storeSemanticKeyTransform, columnCanHoldText, keyOrderMatchesCollation, pkOrderPreservingPrefixLength, withImplicitUniqueIndexes, implicitUniqueIndexName } from './store-table.js';
20
20
  import { buildCatalogKey, buildCatalogScanBounds, buildDataKey, buildDataStoreName, buildIndexKey, buildIndexStoreName, buildFullScanBounds, buildStatsKey, buildViewCatalogKey, buildMaterializedViewCatalogKey, parseMaterializedViewCatalogKey, buildMetaCatalogKey, CLEAN_SHUTDOWN_META_NAME, STALE_MVS_META_NAME, classifyCatalogKey, } from './key-builder.js';
21
21
  import { assertNoUnpairedSurrogate } from './encoding.js';
22
22
  import { deserializeRow } from './serialization.js';
23
- import { generateTableDDL, generateIndexDDL, generateViewDDL, generateMaintainedTableDDL, generateIndexTagsDDL, isHiddenImplicitIndex, exposedImplicitIndexes } from '@quereus/quereus';
23
+ import { generateTableDDL, generateIndexDDL, generateDropTableDDL, generateDropIndexDDL, generateViewDDL, generateMaintainedTableDDL, generateIndexTagsDDL, isHiddenImplicitIndex, exposedImplicitIndexes } from '@quereus/quereus';
24
24
  /**
25
25
  * Default serialized-byte budget for a single index-build write batch. Bounds
26
26
  * heap directly: {@link StoreModule.buildIndexEntries} flushes and starts a fresh
@@ -76,9 +76,28 @@ function resolveMaxBatchBytes(raw) {
76
76
  * handled only when it falls in the group the engine's access-path rule will consume.
77
77
  */
78
78
  const EQ_OPS = ['='];
79
+ /**
80
+ * Secondary-index arm ONLY: an IN is N equalities served as one multi-seek (`plan=5`),
81
+ * whether its members are a literal list or a runtime-valued set. The primary-key arms
82
+ * deliberately keep {@link EQ_OPS} — a `_primary_` multi-seek's emission order would
83
+ * break the isolation layer's primary-key merge — so an IN on the PK declines here,
84
+ * runtime-valued or not. PK IN support is deferred; see
85
+ * tickets/backlog/feat-store-pk-in-list-multiseek.
86
+ */
87
+ const EQ_OR_IN_OPS = ['=', 'IN'];
79
88
  const LOWER_BOUND_OPS = ['>', '>='];
80
89
  const UPPER_BOUND_OPS = ['<', '<='];
81
90
  const RANGE_OPS = [...LOWER_BOUND_OPS, ...UPPER_BOUND_OPS];
91
+ /**
92
+ * Ceiling on the seek keys a single IN-list multi-seek may claim (the cross-product
93
+ * for a composite seek). Above it the plan declines to cost-only — residual retained,
94
+ * answer right, only the speed-up lost. The FK RESTRICT batch chunks at 500 keys, so
95
+ * 1000 leaves headroom while stopping an `a in (1..100) and b in (1..100)` 10k-seek
96
+ * explosion.
97
+ */
98
+ const MAX_MULTI_SEEK_KEYS = 1000;
99
+ /** Cost of positioning one index seek, in the same unit as the per-row fetch cost. */
100
+ const INDEX_SEEK_COST = 0.5;
82
101
  /**
83
102
  * Build `handledFilters` by claiming ONLY the constraints `rule-select-access-path`
84
103
  * actually consumes: per seek column the FIRST '=' (equality seek), or the FIRST lower
@@ -99,7 +118,11 @@ const RANGE_OPS = [...LOWER_BOUND_OPS, ...UPPER_BOUND_OPS];
99
118
  function claimFirstPerRole(filters, roles) {
100
119
  const claimed = new Set();
101
120
  for (const { colIdx, ops } of roles) {
102
- const i = filters.findIndex(f => f.columnIndex === colIdx && ops.includes(f.op));
121
+ // An 'IN' fills an equality role only when well-formed — a non-empty literal list
122
+ // or a runtime-valued set. Same shape gate the rule's pick applies, so a malformed
123
+ // IN is neither claimed here nor seeked there and survives as a residual.
124
+ const i = filters.findIndex(f => f.columnIndex === colIdx && ops.includes(f.op)
125
+ && (f.op !== 'IN' || equalitySeekKeyCount(f) !== null));
103
126
  if (i >= 0)
104
127
  claimed.add(i);
105
128
  }
@@ -109,9 +132,49 @@ function claimFirstPerRole(filters, roles) {
109
132
  function rangeRoles(colIdx) {
110
133
  return [{ colIdx, ops: LOWER_BOUND_OPS }, { colIdx, ops: UPPER_BOUND_OPS }];
111
134
  }
112
- /** The one equality role each seek column of an equality/prefix seek fills. */
113
- function equalityRoles(colIdxs) {
114
- return colIdxs.map(colIdx => ({ colIdx, ops: EQ_OPS }));
135
+ /**
136
+ * The one equality role each seek column of an equality/prefix seek fills. `ops` is
137
+ * explicit so the PK arm ({@link EQ_OPS}) and the secondary-index arm
138
+ * ({@link EQ_OR_IN_OPS}) cannot drift into each other — see the note on EQ_OR_IN_OPS.
139
+ */
140
+ function equalityRoles(colIdxs, ops) {
141
+ return colIdxs.map(colIdx => ({ colIdx, ops }));
142
+ }
143
+ /**
144
+ * Guard the FULL persisted schema text, not just the object's own name: a lone
145
+ * surrogate anywhere in it — a quoted column name, a `default '…'` string literal, a
146
+ * `check` constraint's string constant — folds to U+FFFD under `TextEncoder` and would
147
+ * read back as different schema text than what was created. Independent of the
148
+ * catalog-key builders' identifier guard; it catches what they cannot see.
149
+ */
150
+ function assertPersistableDdlText(ddl) {
151
+ assertNoUnpairedSurrogate(ddl, 'persisted schema text');
152
+ }
153
+ /**
154
+ * The catalog entry a plain view would be persisted as. Building it is the check:
155
+ * {@link buildViewCatalogKey} rejects an unencodable name, and the caller then runs
156
+ * {@link assertPersistableDdlText} over `ddl`. Shared by the write path
157
+ * ({@link StoreModule.saveViewDDL}) and the pre-flight veto
158
+ * ({@link StoreModule.assertCatalogObjectPersistable}) so the two cannot drift.
159
+ */
160
+ function viewCatalogEntry(view) {
161
+ return {
162
+ key: buildViewCatalogKey(view.schemaName, view.name),
163
+ ddl: generateViewDDL(view),
164
+ };
165
+ }
166
+ /**
167
+ * The catalog entry a materialized view would be persisted as, or `undefined` when
168
+ * `table` carries no derivation (not an MV — nothing lands under the MV prefix).
169
+ * Counterpart of {@link viewCatalogEntry}; same write/veto sharing.
170
+ */
171
+ function maintainedViewCatalogEntry(table) {
172
+ if (!isMaintainedTable(table))
173
+ return undefined;
174
+ return {
175
+ key: buildMaterializedViewCatalogKey(table.schemaName, table.name),
176
+ ddl: generateMaintainedTableDDL(table),
177
+ };
115
178
  }
116
179
  /**
117
180
  * Generic store module that works with any KVStoreProvider.
@@ -372,6 +435,14 @@ export class StoreModule {
372
435
  * stores in the set therefore causes no false rejects and keeps the
373
436
  * reject-before-any-side-effect guarantee uniform.
374
437
  */
438
+ // NOTE: the builders below carry the unpaired-surrogate identifier guard, so this
439
+ // enumeration can THROW on an EXISTING schema object, not just on the incoming one —
440
+ // an unrelated CREATE/RENAME in the same schema would then fail with a confusing
441
+ // "cannot store the identifier ..." error. Unreachable today: `create` throws before
442
+ // the table is registered, and a catalog entry written before the guard existed was
443
+ // folded to a real U+FFFD character, which passes. If a lone-surrogate-named table ever
444
+ // does reach the schema manager by another route, skip such an object here (it occupies
445
+ // no store) rather than letting the enumeration throw.
375
446
  collectOccupiedStoreNames(db, schemaName) {
376
447
  const names = new Map();
377
448
  const add = (s) => {
@@ -561,12 +632,15 @@ export class StoreModule {
561
632
  // torn down too — the engine-facing `.indexes` omits them and would leak them.
562
633
  const indexNames = this.materializedIndexNames(table, currentSchema);
563
634
  await this.tearDownTableStorage(schemaName, tableName, indexNames);
564
- // Emit schema change event for table drop
635
+ // Emit schema change event for table drop. The `ddl` is what a sync peer
636
+ // re-executes to replicate the drop — a drop with no DDL crosses the wire as
637
+ // an empty statement and silently changes nothing on the receiver.
565
638
  this.eventEmitter?.emitSchemaChange({
566
639
  type: 'drop',
567
640
  objectType: 'table',
568
641
  schemaName,
569
642
  objectName: tableName,
643
+ ddl: generateDropTableDDL(schemaName, tableName),
570
644
  });
571
645
  }
572
646
  /**
@@ -745,12 +819,17 @@ export class StoreModule {
745
819
  // skips (identical) — see persistCatalogIfChanged.
746
820
  await this.saveTableDDL(updatedSchema);
747
821
  table.markDdlSaved();
748
- // Emit schema change event
822
+ // Emit schema change event. `updatedSchema` (not the pre-create
823
+ // `tableSchema`) is the owner the canonical CREATE INDEX renders against —
824
+ // it is only read for the table's qualified name and column names, but
825
+ // passing the post-create schema keeps the rendering consistent with what a
826
+ // receiving peer regenerates when it compares definitions.
749
827
  this.eventEmitter?.emitSchemaChange({
750
828
  type: 'create',
751
829
  objectType: 'index',
752
830
  schemaName,
753
831
  objectName: indexSchema.name,
832
+ ddl: generateIndexDDL(indexSchema, updatedSchema),
754
833
  });
755
834
  }
756
835
  /**
@@ -808,6 +887,7 @@ export class StoreModule {
808
887
  objectType: 'index',
809
888
  schemaName,
810
889
  objectName: indexName,
890
+ ddl: generateDropIndexDDL(schemaName, indexName),
811
891
  });
812
892
  }
813
893
  /**
@@ -841,6 +921,11 @@ export class StoreModule {
841
921
  const encodeOptions = { collation: keyCollation, normalizers };
842
922
  const pkDirections = tableSchema.primaryKeyDefinition.map(pk => !!pk.desc);
843
923
  const pkCollations = resolvePkKeyCollations(tableSchema.primaryKeyDefinition, tableSchema.columns, keyCollation);
924
+ // Key-identity transforms for both halves, mirroring `StoreTable`'s maintenance
925
+ // writes (`updateSecondaryIndexes` / `encodeDataKey`) — a rebuild that skipped them
926
+ // would re-encode a TIMESPAN member under different bytes than DML writes.
927
+ const pkTransforms = resolvePkKeyTransforms(tableSchema.primaryKeyDefinition, tableSchema.columns);
928
+ const indexTransforms = resolveIndexKeyTransforms(indexSchema, tableSchema.columns);
844
929
  const indexDirections = indexSchema.columns.map(col => !!col.desc);
845
930
  const predicate = indexSchema.predicate
846
931
  ? compilePredicate(indexSchema.predicate, tableSchema.columns, tableSchema.name)
@@ -868,9 +953,9 @@ export class StoreModule {
868
953
  // Extract index column values
869
954
  const indexValues = indexSchema.columns.map(col => row[col.index]);
870
955
  if (seen) {
871
- // serializeRowKey returns null when any indexed column is NULL —
956
+ // The signature returns null when any indexed column is NULL —
872
957
  // SQL UNIQUE allows multiple NULLs, so those rows never collide.
873
- const keySig = serializeRowKey(row, indexColIndices, indexNormalizers);
958
+ const keySig = dedupeRowSignature(row, indexColIndices, indexNormalizers, indexTransforms);
874
959
  if (keySig !== null) {
875
960
  if (seen.has(keySig)) {
876
961
  const colNames = indexSchema.columns
@@ -882,13 +967,13 @@ export class StoreModule {
882
967
  }
883
968
  }
884
969
  // Build and store index key
885
- const indexKey = buildIndexKey(indexValues, pkValues, encodeOptions, indexDirections, pkDirections, pkCollations);
970
+ const indexKey = buildIndexKey(indexValues, pkValues, encodeOptions, indexDirections, pkDirections, pkCollations, indexTransforms, pkTransforms);
886
971
  // Index value = the row's encoded DATA key, so an index scan resolves each
887
972
  // entry back to its base row via a direct data-store read (see
888
973
  // `StoreTable.scanIndex`). Encoded under the same PK directions + per-column
889
- // PK collations as `buildDataKey` / `updateSecondaryIndexes`, so the value
890
- // byte-matches the data store's key for this row.
891
- const dataKey = buildDataKey(pkValues, encodeOptions, pkDirections, pkCollations);
974
+ // PK collations + key transforms as `buildDataKey` / `updateSecondaryIndexes`,
975
+ // so the value byte-matches the data store's key for this row.
976
+ const dataKey = buildDataKey(pkValues, encodeOptions, pkDirections, pkCollations, pkTransforms);
892
977
  batch.put(indexKey, dataKey);
893
978
  // Bound heap: once the accumulated serialized key bytes cross the budget, flush
894
979
  // the batch and start a fresh one. Both callers ITERATE the data store and WRITE
@@ -920,11 +1005,20 @@ export class StoreModule {
920
1005
  * connection's `db.getKeyNormalizerResolver()` — see {@link buildIndexEntries}.
921
1006
  *
922
1007
  * NOTE: reads the data store committed-only and writes the index stores outside the
923
- * coordinator. Sound only because both callers (the two re-key arms) call
924
- * {@link ddlCommitPendingOps} first, so "committed" is "everything live". A caller
925
- * that skips that flush would rebuild an index missing its transaction's pending rows.
926
- */
927
- async rebuildSecondaryIndexes(schemaName, tableName, table, schema, normalizers) {
1008
+ * coordinator. Sound only because every caller calls {@link ddlCommitPendingOps}
1009
+ * first, so this module's "committed" is "everything live". A caller that skips that
1010
+ * flush would rebuild an index missing its transaction's pending rows.
1011
+ *
1012
+ * `skipDuplicateCheck` suppresses {@link buildIndexEntries}' in-pass UNIQUE check.
1013
+ * The value-rewriting / key-transform ALTER COLUMN arm sets it: its pre-mutation
1014
+ * UNIQUE re-validation already judged the ISSUING connection's effective rows (a
1015
+ * wrapper's overlay included), and this rebuild sees only THIS module's committed
1016
+ * rows — a superset that may retain a row the wrapper's transaction has deleted, so
1017
+ * judging it here would spuriously reject a duplicate pair the probe correctly
1018
+ * accepted. Mirrors the memory module's deliberately non-enforcing base rebuild
1019
+ * (the probe is the only guard).
1020
+ */
1021
+ async rebuildSecondaryIndexes(schemaName, tableName, table, schema, normalizers, skipDuplicateCheck = false) {
928
1022
  const keyCollation = (table.getConfig().collation || 'NOCASE').toUpperCase();
929
1023
  const dataStore = await this.getStore(schemaName, tableName, table.getConfig());
930
1024
  const maxBatchBytes = table.getConfig().maxBatchBytes ?? DEFAULT_MAX_BATCH_BYTES;
@@ -942,7 +1036,7 @@ export class StoreModule {
942
1036
  clearBatch.delete(entry.key);
943
1037
  }
944
1038
  await clearBatch.write();
945
- await this.buildIndexEntries(dataStore.iterate(buildFullScanBounds()), indexStore, schema, indexSchema, keyCollation, normalizers, false, maxBatchBytes);
1039
+ await this.buildIndexEntries(dataStore.iterate(buildFullScanBounds()), indexStore, schema, indexSchema, keyCollation, normalizers, skipDuplicateCheck, maxBatchBytes);
946
1040
  }
947
1041
  }
948
1042
  /**
@@ -965,7 +1059,7 @@ export class StoreModule {
965
1059
  *
966
1060
  * No index store is written — store UNIQUE enforcement is a full-scan over
967
1061
  * `uniqueConstraints` at write time. The signature is built by
968
- * {@link serializeRowKey} with one normalizer per constrained column, resolved from
1062
+ * {@link dedupeRowSignature} with one normalizer per constrained column, resolved from
969
1063
  * `tableSchema.columns[idx].collation` through the connection's
970
1064
  * `db.getKeyNormalizerResolver()`, so a per-column collation registered with
971
1065
  * `db.registerCollation` is honored (matching write-time `compareSqlValues`
@@ -1183,8 +1277,16 @@ export class StoreModule {
1183
1277
  * ADD COLUMN arm of {@link alterTable}: append the new column, eagerly migrate
1184
1278
  * each row (literal or per-row backfill), and persist. Behavior-preserving
1185
1279
  * extraction of the former `switch` arm.
1280
+ *
1281
+ * The store always appends. A caller-chosen `insertAtIndex` (module-API only; SQL never
1282
+ * produces one) is rejected unless it names the append position, rather than silently
1283
+ * landing the column somewhere the caller did not ask for.
1186
1284
  */
1187
1285
  async alterAddColumn(db, schemaName, tableName, table, oldSchema, change, defaultNotNull) {
1286
+ if (change.insertAtIndex !== undefined && change.insertAtIndex !== oldSchema.columns.length) {
1287
+ throw new QuereusError(`Store-backed table '${schemaName}.${tableName}' can only ADD COLUMN at the end `
1288
+ + `(position ${oldSchema.columns.length}), not at position ${change.insertAtIndex}`, StatusCode.UNSUPPORTED);
1289
+ }
1188
1290
  // Honor the session `default_collation` for an ADD COLUMN that omits an
1189
1291
  // explicit COLLATE, matching the CREATE path so an ADD-COLUMN-ed text column
1190
1292
  // gets the same collation a CREATE-d one would. The persisted DDL re-emits an
@@ -1221,19 +1323,11 @@ export class StoreModule {
1221
1323
  columns: updatedColumns,
1222
1324
  columnIndexMap: buildColumnIndexMap(updatedColumns),
1223
1325
  };
1224
- // Extract any column-level CHECK / FK to persist (see the persist block below).
1225
- // Hoisted above the row migration so a malformed constraint (e.g. a multi-column
1226
- // FK on a single ADD COLUMN, which `extractColumnLevelForeignKeys` rejects) throws
1227
- // BEFORE any rows are migrated or the in-memory schema is swapped — validate-before-
1228
- // mutate, matching the engine's ordering in `runAddColumn`.
1229
- const newCheckConstraints = extractColumnLevelCheckConstraints(change.columnDef);
1230
- const newForeignKeys = extractColumnLevelForeignKeys(change.columnDef, schemaName);
1231
1326
  // Physical rewrite ahead: flush the module's buffered writes so `migrateRows`
1232
1327
  // (a committed-store scan + batch) sees this transaction's rows and re-encodes
1233
1328
  // them under the new column layout. See {@link ddlCommitPendingOps}. Placed
1234
1329
  // after every throw-only check above — the NOT NULL rejection reads effectively
1235
- // (`hasAnyRows`) and `extractColumnLevel*` reads no rows — so a rejected ALTER
1236
- // leaves the enclosing transaction intact.
1330
+ // (`hasAnyRows`) — so a rejected ALTER leaves the enclosing transaction intact.
1237
1331
  await this.ddlCommitPendingOps();
1238
1332
  // Migrate rows: append the new column's value — a single literal default, or a
1239
1333
  // per-row value derived from the existing row when a backfill evaluator is set.
@@ -1241,33 +1335,13 @@ export class StoreModule {
1241
1335
  await table.migrateRows(remap, defaultValue, backfillEvaluator
1242
1336
  ? { evaluator: backfillEvaluator, notNull: newColSchema.notNull, columnName: newColSchema.name }
1243
1337
  : undefined);
1244
- // Update table schema (column-only) and persist DDL.
1245
- //
1246
- // The engine's `runAddColumn` re-merges the column-level FK/CHECK extracted
1247
- // from `columnDef.constraints` into the LIVE in-memory schema AFTER this hook
1248
- // returns, so the schema handed back to it must stay column-only — returning a
1249
- // constrained schema would double the constraint in the live SchemaManager (and,
1250
- // on the next persist, in the DDL). But that engine-side merge is in-memory only:
1251
- // it never reaches the catalog, so persistence must carry the column-level
1252
- // CHECK/FK itself or they vanish on `rehydrateCatalog`. Build a separate
1253
- // `persistedSchema` for `saveTableDDL` when (and only when) the column declares
1254
- // such a constraint; the common path persists `updatedSchema` unchanged. This is
1255
- // unconditional on the default kind — a per-row (evaluator) DEFAULT extracts the
1256
- // same AST constraints as a literal one.
1338
+ // Update table schema (column-only) and persist DDL. Any constraint declared inline
1339
+ // on the added column (UNIQUE / CHECK / FOREIGN KEY) arrives as its own follow-up
1340
+ // `addConstraint` call from the engine's `runAddColumn`, which is what persists it —
1341
+ // so this arm neither installs nor persists them, and the schema it hands back stays
1342
+ // column-only.
1257
1343
  table.updateSchema(updatedSchema);
1258
- let persistedSchema = updatedSchema;
1259
- if (newCheckConstraints.length > 0 || newForeignKeys.length > 0) {
1260
- // The new column is appended last; resolve each FK's child column to its index
1261
- // (matching how the engine resolves `resolvedForeignKeys` via columnIndexMap).
1262
- const newColIdx = updatedColumns.length - 1;
1263
- const resolvedForeignKeys = newForeignKeys.map(fk => ({ ...fk, columns: Object.freeze([newColIdx]) }));
1264
- persistedSchema = {
1265
- ...updatedSchema,
1266
- checkConstraints: Object.freeze([...updatedSchema.checkConstraints, ...newCheckConstraints]),
1267
- foreignKeys: Object.freeze([...(updatedSchema.foreignKeys ?? []), ...resolvedForeignKeys]),
1268
- };
1269
- }
1270
- await this.saveTableDDL(persistedSchema);
1344
+ await this.saveTableDDL(updatedSchema);
1271
1345
  this.eventEmitter?.emitSchemaChange({
1272
1346
  type: 'alter',
1273
1347
  objectType: 'table',
@@ -1313,6 +1387,18 @@ export class StoreModule {
1313
1387
  const updatedUniqueConstraints = (oldSchema.uniqueConstraints ?? [])
1314
1388
  .filter(uc => !uc.columns.includes(colIndex))
1315
1389
  .map(uc => ({ ...uc, columns: Object.freeze(uc.columns.map(i => i > colIndex ? i - 1 : i)) }));
1390
+ // Shift or prune this table's own foreign keys, mirroring the UNIQUE pruning above
1391
+ // and the memory module's `dropColumn`. `foreignKeys[].columns` are indices into
1392
+ // THIS table, so they renumber over the removed slot; a key that loses ANY of its
1393
+ // child columns is removed outright (a key missing a child column is a different
1394
+ // constraint against the parent's key, not a narrowed one). Left unshifted, a
1395
+ // surviving key's index dangles past the column array or silently slides onto an
1396
+ // unrelated column — and `saveTableDDL` below persists whichever it ends up being.
1397
+ // `referencedColumnNames` is resolved against the parent by name at enforcement
1398
+ // time, so the parent side needs nothing here.
1399
+ const updatedForeignKeys = (oldSchema.foreignKeys ?? [])
1400
+ .filter(fk => !fk.columns.includes(colIndex))
1401
+ .map(fk => ({ ...fk, columns: Object.freeze(fk.columns.map(i => i > colIndex ? i - 1 : i)) }));
1316
1402
  const updatedSchema = {
1317
1403
  ...oldSchema,
1318
1404
  columns: Object.freeze(updatedColumns),
@@ -1322,6 +1408,9 @@ export class StoreModule {
1322
1408
  uniqueConstraints: updatedUniqueConstraints.length > 0
1323
1409
  ? Object.freeze(updatedUniqueConstraints)
1324
1410
  : undefined,
1411
+ foreignKeys: updatedForeignKeys.length > 0
1412
+ ? Object.freeze(updatedForeignKeys)
1413
+ : undefined,
1325
1414
  };
1326
1415
  // Physical rewrite ahead — flush buffered writes so `migrateRows` re-encodes
1327
1416
  // this transaction's rows too. See {@link ddlCommitPendingOps}.
@@ -1581,16 +1670,17 @@ export class StoreModule {
1581
1670
  const oldCol = oldSchema.columns[colIndex];
1582
1671
  // Pull exactly one attribute from the change. Each sub-helper returns the new
1583
1672
  // column schema plus whether the collation bytes changed (`collationChanged` gates
1584
- // the existing-row UNIQUE re-validation and PK re-key below), or null when the column
1585
- // is already in the desired state — preserving the pre-refactor early exits
1586
- // (SET NOT NULL no-op, SET COLLATE already-explicit) that leave the table and any
1587
- // open transaction untouched.
1673
+ // the existing-row UNIQUE re-validation and PK re-key below) plus a DEFERRED
1674
+ // `valueConvert` closure when the change rewrites stored values — the sub-helper
1675
+ // itself mutates nothing. Or null when the column is already in the desired state —
1676
+ // preserving the pre-refactor early exits (SET NOT NULL no-op, SET COLLATE
1677
+ // already-explicit) that leave the table and any open transaction untouched.
1588
1678
  let attr;
1589
1679
  if (change.setNotNull !== undefined) {
1590
1680
  attr = await this.alterColumnSetNotNull(table, oldSchema, oldCol, colIndex, change, rows);
1591
1681
  }
1592
1682
  else if (change.setDataType !== undefined) {
1593
- attr = await this.alterColumnSetDataType(table, oldCol, colIndex, change);
1683
+ attr = await this.alterColumnSetDataType(table, oldSchema, oldCol, colIndex, change);
1594
1684
  }
1595
1685
  else if (change.setDefault !== undefined) {
1596
1686
  attr = { newCol: { ...oldCol, defaultValue: change.setDefault }, collationChanged: false };
@@ -1604,7 +1694,12 @@ export class StoreModule {
1604
1694
  if (attr === null) {
1605
1695
  return oldSchema;
1606
1696
  }
1607
- const { newCol, collationChanged, valuesRewritten } = attr;
1697
+ const { newCol, collationChanged, valueConvert } = attr;
1698
+ // Named for the PENDING act, not a completed one: the deferred value rewrite (SET DATA
1699
+ // TYPE conversion / SET NOT NULL backfill) is applied below, after every throw-only
1700
+ // check. Both gates that read this run BEFORE it, while the store still holds the old
1701
+ // values.
1702
+ const rewritesValues = valueConvert !== undefined;
1608
1703
  const updatedColumns = oldSchema.columns.map((c, i) => i === colIndex ? newCol : c);
1609
1704
  // Mirror the memory module (MemoryTableManager.alterColumn): a per-column
1610
1705
  // collation change propagates into every index column ordering by this
@@ -1630,21 +1725,47 @@ export class StoreModule {
1630
1725
  columnIndexMap: buildColumnIndexMap(updatedColumns),
1631
1726
  indexes: updatedIndexes ? Object.freeze(updatedIndexes) : updatedIndexes,
1632
1727
  };
1633
- // SET COLLATE existing-row re-validation (Option A, non-PK UNIQUE): a new
1634
- // per-column collation can make rows that were distinct under the old
1635
- // collation collide. Re-scan every UNIQUE constraint covering the altered
1636
- // column under the NEW collation (`updatedSchema` carries it). The first
1637
- // collision throws CONSTRAINT BEFORE any mutation/persist, so the table is
1638
- // left unchanged and writable (matches the ADD CONSTRAINT rollback shape).
1728
+ // SET DATA TYPE onto/off a type with a key transform (TIMESPAN's total-seconds
1729
+ // groupKey, JSON's structural encoder — see `storeSemanticKeyTransform`)
1730
+ // changes the physical KEY BYTES a value encodes to, even when the stored
1731
+ // VALUE bytes don't change (text → timespan keeps TEXT physical). Treated
1732
+ // exactly like a collation change below: existing-row UNIQUE re-validation
1733
+ // (equal-elapsed spellings now collide), a PK re-key when the column is a PK
1734
+ // member, and a secondary-index rebuild (index keys encode the column's
1735
+ // values). A same-type no-op ALTER never gets here (`inferType` returns the
1736
+ // shared type object, so the identity check short-circuits).
1737
+ const keyTransformChanged = oldCol.logicalType !== newCol.logicalType
1738
+ && (storeSemanticKeyTransform(oldCol.logicalType) !== undefined
1739
+ || storeSemanticKeyTransform(newCol.logicalType) !== undefined);
1740
+ // Existing-row UNIQUE re-validation (Option A, non-PK UNIQUE). Two ways an ALTER
1741
+ // COLUMN can make rows that were distinct collide, both judged here BEFORE any
1742
+ // mutation/persist so the first collision throws CONSTRAINT and leaves the table
1743
+ // unchanged and writable (matches the ADD CONSTRAINT rollback shape):
1744
+ // - SET COLLATE / a key-transform change: the comparators change ('a'/'A' collide
1745
+ // under NOCASE). Re-scan under the NEW collation (`updatedSchema` carries it).
1746
+ // - a value-REWRITING change (`valueConvert`: SET DATA TYPE, or a SET NOT NULL
1747
+ // null → DEFAULT backfill): the comparators are unchanged but the VALUES collapse
1748
+ // ('1'/'01' → 1; two NULLs → the same DEFAULT). The probe judges the CONVERTED
1749
+ // rows — the same closure the deferred rewrite below applies — while the store
1750
+ // still holds the old values. A standalone `create unique index` synthesizes a
1751
+ // `derivedFromIndex` entry in `uniqueConstraints`, so both declaration shapes are
1752
+ // reachable through this walk.
1639
1753
  // The PK is intentionally excluded — it never appears in `uniqueConstraints`;
1640
- // its physical re-key/re-validation is the `isPkColumn` block below.
1641
- if (collationChanged) {
1754
+ // its physical re-key/re-validation is the `pkRekeyNeeded` block below.
1755
+ //
1756
+ // NOTE: one full row scan per covering constraint, and this gate now fires on the
1757
+ // COMMON retype/backfill path, not only on the rare collation/transform ones. Fine
1758
+ // while tables covered by several UNIQUE constraints over one column are rare; if a
1759
+ // wide-constraint table makes ALTER COLUMN slow, judge all covering constraints in a
1760
+ // single pass (one `seen` Set per constraint, one stream).
1761
+ if (collationChanged || keyTransformChanged || rewritesValues) {
1642
1762
  const coveringConstraints = (updatedSchema.uniqueConstraints ?? [])
1643
1763
  .filter(uc => uc.columns.includes(colIndex));
1644
1764
  for (const uc of coveringConstraints) {
1645
1765
  // Fresh generator per constraint — an async generator is single-shot. An
1646
1766
  // `EffectiveRowSource` is re-callable for exactly this reason.
1647
- await this.validateUniqueOverExistingRows(rows ? rows() : rowsFromEntries(table.iterateEffectiveEntries(buildFullScanBounds())), updatedSchema, uc, db.getKeyNormalizerResolver());
1767
+ const effectiveRows = rows ? rows() : rowsFromEntries(table.iterateEffectiveEntries(buildFullScanBounds()));
1768
+ await this.validateUniqueOverExistingRows(valueConvert ? convertRowsAtIndex(effectiveRows, colIndex, valueConvert) : effectiveRows, updatedSchema, uc, db.getKeyNormalizerResolver());
1648
1769
  }
1649
1770
  }
1650
1771
  // SET COLLATE on a PRIMARY KEY member (Option B physical re-key): re-encode
@@ -1656,7 +1777,9 @@ export class StoreModule {
1656
1777
  // ALTER PRIMARY KEY. Runs AFTER the non-PK UNIQUE re-validation above so both
1657
1778
  // throw-only checks precede the first store mutation. `updatedSchema.columns`
1658
1779
  // carries the new collation, so the new key bytes follow it.
1659
- if (collationChanged && oldSchema.primaryKeyDefinition.some(def => def.index === colIndex)) {
1780
+ const pkRekeyNeeded = (collationChanged || keyTransformChanged)
1781
+ && oldSchema.primaryKeyDefinition.some(def => def.index === colIndex);
1782
+ if (pkRekeyNeeded) {
1660
1783
  // Physical re-key ahead — flush buffered writes (see
1661
1784
  // {@link ddlCommitPendingOps}). Deliberately AFTER the non-PK UNIQUE
1662
1785
  // re-validation above: that check reads effectively and throws without
@@ -1674,18 +1797,46 @@ export class StoreModule {
1674
1797
  // key collation too (`updatedSchema` carries none on its own).
1675
1798
  await this.rebuildSecondaryIndexes(schemaName, tableName, table, withImplicitUniqueIndexes(updatedSchema), db.getKeyNormalizerResolver());
1676
1799
  }
1677
- // SET DATA TYPE physical conversion / SET NOT NULL DEFAULT backfill rewrote stored
1678
- // column values in place (same PK, new value) via mapRowsAtIndex. Secondary index KEY
1679
- // bytes encode the indexed column VALUES, so any index covering this column still points
1680
- // at the OLD value bytes until rebuilt — an index-backed lookup for the new value finds
1681
- // nothing (mirrors the memory module's `valuesRewritten` rebuild in MemoryTableManager.
1682
- // alterColumn). The sub-helper already ran ddlCommitPendingOps before its rewrite, so the
1683
- // committed data store rebuildSecondaryIndexes reads is "everything live". Mutually
1684
- // exclusive with the collation re-key above (distinct attributes), so no double rebuild.
1685
- // Materialize so an implicit `_uc_*` over the rewritten column is rebuilt against the
1686
- // new value bytes too.
1687
- if (valuesRewritten) {
1688
- await this.rebuildSecondaryIndexes(schemaName, tableName, table, withImplicitUniqueIndexes(updatedSchema), db.getKeyNormalizerResolver());
1800
+ // Deferred value rewrite (SET DATA TYPE physical conversion / SET NOT NULL DEFAULT
1801
+ // backfill): every throw-only check above has passed, so the store mutation is now
1802
+ // safe — a rejected ALTER never reaches this point, leaving values, DDL and the
1803
+ // transaction untouched. Flush buffered writes first so `mapRowsAtIndex` rewrites this
1804
+ // transaction's rows too; unflushed, they would replay under the OLD physical type
1805
+ // (see {@link ddlCommitPendingOps}; a re-flush after the PK re-key's own is a no-op).
1806
+ //
1807
+ // NOTE: `valueConvert` and `pkRekeyNeeded` cannot both be set today, and this ordering
1808
+ // depends on that. `valueConvert` comes only from SET DATA TYPE or SET NOT NULL;
1809
+ // SET NOT NULL changes neither collation nor logical type, so it never sets
1810
+ // `pkRekeyNeeded`, and SET DATA TYPE on a PK member is refused both upstream (every live
1811
+ // caller — the engine's ALTER COLUMN emitter, see `runAlterColumn` in
1812
+ // runtime/emit/alter-table.ts, and the materialized-view reshape, which declares a
1813
+ // key-column retype inexpressible) AND locally, by `alterColumnSetDataType` itself. If a
1814
+ // PK-member retype is ever admitted (that local guard removed), this rewrite must move IN
1815
+ // FRONT of the `pkRekeyNeeded` block: otherwise `rekeyRows` and its index rebuild encode
1816
+ // the PRE-rewrite values and the rebuild below is skipped, leaving keys and indexes
1817
+ // disagreeing with the stored values.
1818
+ if (valueConvert) {
1819
+ await this.ddlCommitPendingOps();
1820
+ await table.mapRowsAtIndex(colIndex, valueConvert);
1821
+ }
1822
+ // The value rewrite above changed stored column values in place (same PK, new value) —
1823
+ // and a key-identity transform change re-encodes the column's index-key bytes even with
1824
+ // values untouched. Secondary index KEY bytes encode the indexed column VALUES, so any
1825
+ // index covering this column still points at the OLD bytes until rebuilt — an
1826
+ // index-backed lookup for the new value finds nothing (mirrors the memory module's
1827
+ // `valueConvert` rebuild in MemoryTableManager.alterColumn). `rebuildSecondaryIndexes`
1828
+ // reads committed-only, so flush buffered writes first — the value rewrite above
1829
+ // already did, and the transform-only path (text → timespan: no value rewrite) must
1830
+ // too; a re-flush with nothing pending is a no-op. Skipped when the PK re-key above
1831
+ // already rebuilt every index, so no double rebuild. Materialize so an implicit `_uc_*`
1832
+ // over the rewritten column is rebuilt against the new value bytes too. The rebuild is
1833
+ // NON-enforcing (`skipDuplicateCheck`): the UNIQUE re-validation above already judged
1834
+ // the issuer's effective rows, and this module's committed rows may retain a row a
1835
+ // wrapper's transaction has deleted whose converted value duplicates a survivor —
1836
+ // enforcing here would spuriously reject what the probe correctly accepted.
1837
+ if ((rewritesValues || keyTransformChanged) && !pkRekeyNeeded) {
1838
+ await this.ddlCommitPendingOps();
1839
+ await this.rebuildSecondaryIndexes(schemaName, tableName, table, withImplicitUniqueIndexes(updatedSchema), db.getKeyNormalizerResolver(), true);
1689
1840
  }
1690
1841
  table.updateSchema(updatedSchema);
1691
1842
  await this.saveTableDDL(updatedSchema);
@@ -1700,8 +1851,10 @@ export class StoreModule {
1700
1851
  /**
1701
1852
  * SET NOT NULL / DROP NOT NULL sub-branch of {@link alterColumnChange}. Returns the
1702
1853
  * new column schema, or null when the column is already in the desired nullability
1703
- * (the pre-refactor `return oldSchema` no-op). The NULL-backfill probe and rewrite
1704
- * order (throw-only probe before {@link ddlCommitPendingOps}) is unchanged.
1854
+ * (the pre-refactor `return oldSchema` no-op). Mutates nothing: the NULL-backfill
1855
+ * probe is throw-only, and the backfill itself is returned as a deferred
1856
+ * `valueConvert` (null → DEFAULT) the caller applies only after every throw-only
1857
+ * check — including the UNIQUE re-validation over the backfilled values — has passed.
1705
1858
  *
1706
1859
  * `rows` is the wrapper-supplied effective row source (the isolation overlay). When present,
1707
1860
  * the reject-vs-backfill decision scans it instead of `table.rowsWithNullAtIndex`: behind the
@@ -1712,7 +1865,7 @@ export class StoreModule {
1712
1865
  */
1713
1866
  async alterColumnSetNotNull(table, oldSchema, oldCol, colIndex, change, rows) {
1714
1867
  let newCol;
1715
- let valuesRewritten = false;
1868
+ let valueConvert;
1716
1869
  if (change.setNotNull === true && !oldCol.notNull) {
1717
1870
  // Backfill NULLs from a literal DEFAULT, or throw.
1718
1871
  let defaultLiteral;
@@ -1743,14 +1896,12 @@ export class StoreModule {
1743
1896
  throw new QuereusError(`column ${change.columnName} contains NULL values`, StatusCode.CONSTRAINT);
1744
1897
  }
1745
1898
  const fill = defaultLiteral;
1746
- // Physical rewrite ahead — flush buffered writes so `mapRowsAtIndex` backfills this
1747
- // store's own NULL rows. Run directly, that also fills the transaction's rows (the
1748
- // probe saw them). Behind isolation the issuer's overlay-resident NULL rows are
1749
- // filled by the isolation layer's overlay migration, not here — this store never
1750
- // holds them. See {@link ddlCommitPendingOps}.
1751
- await this.ddlCommitPendingOps();
1752
- await table.mapRowsAtIndex(colIndex, (v) => v === null ? fill : v);
1753
- valuesRewritten = true;
1899
+ // Backfill DEFERRED: the caller flushes + `mapRowsAtIndex`es this closure only
1900
+ // after its throw-only checks pass. Run directly, that fills the transaction's
1901
+ // rows too (the probe saw them). Behind isolation the issuer's overlay-resident
1902
+ // NULL rows are filled by the isolation layer's overlay migration, not here —
1903
+ // this store never holds them.
1904
+ valueConvert = (v) => v === null ? fill : v;
1754
1905
  }
1755
1906
  newCol = { ...oldCol, notNull: true };
1756
1907
  }
@@ -1763,18 +1914,40 @@ export class StoreModule {
1763
1914
  else {
1764
1915
  return null; // already in desired state
1765
1916
  }
1766
- return { newCol, collationChanged: false, valuesRewritten };
1917
+ return { newCol, collationChanged: false, valueConvert };
1767
1918
  }
1768
1919
  /**
1769
1920
  * SET DATA TYPE sub-branch of {@link alterColumnChange}. Returns the retyped column
1770
- * schema. When the physical type changes, a throw-only convert pass over the live
1771
- * rows precedes {@link ddlCommitPendingOps} and the rewrite — order unchanged.
1772
- */
1773
- async alterColumnSetDataType(table, oldCol, colIndex, change) {
1921
+ * schema. Mutates nothing: for every retype between DIFFERENT logical types, a
1922
+ * throw-only convert pass over the live rows proves every value convertible, and the
1923
+ * conversion itself is returned as a deferred `valueConvert` the caller applies only
1924
+ * after every throw-only check — including the UNIQUE re-validation over the converted
1925
+ * values — has passed. Gated on logical-type IDENTITY, not the physical storage class:
1926
+ * `inferType` flattens aliases to the shared type object (`varchar(50)` IS `TEXT_TYPE`),
1927
+ * so an alias retype is schema-only, while a same-class retype (text → date) still
1928
+ * rejects values the new type refuses and rewrites the rest to the new type's
1929
+ * canonical spelling ('2024-06-05T00:00:00Z' → '2024-06-05') — exactly as an INSERT
1930
+ * would have stored them.
1931
+ *
1932
+ * A retype of a PRIMARY KEY member is refused here rather than converted: the rewrite
1933
+ * below is `mapRowsAtIndex`, a payload-only rewrite that reuses `entry.key` verbatim, so
1934
+ * a PK column's physical key bytes would stay encoded under the OLD type while the value
1935
+ * moves to the new one — unfindable by any lookup under the new encoding, and the same
1936
+ * `keyTransformChanged` path in the caller would re-key from the pre-rewrite values before
1937
+ * this rewrite ever ran (see the NOTE above the `valueConvert` block in
1938
+ * {@link alterColumnChange}). Every live SQL caller already refuses this earlier (the
1939
+ * engine's `runAlterColumn` and the materialized-view reshape's inexpressibility check),
1940
+ * so this only guards a direct module call — mirrors the memory backend's carve-out
1941
+ * (`MemoryTableManager.alterColumn`).
1942
+ */
1943
+ async alterColumnSetDataType(table, oldSchema, oldCol, colIndex, change) {
1774
1944
  const newLogicalType = inferType(change.setDataType);
1775
- let valuesRewritten = false;
1776
- if (newLogicalType.physicalType !== oldCol.logicalType.physicalType) {
1777
- // Physical conversion required — walk every row and attempt parse.
1945
+ let valueConvert;
1946
+ if (newLogicalType !== oldCol.logicalType) {
1947
+ if (oldSchema.primaryKeyDefinition.some(def => def.index === colIndex)) {
1948
+ throw new QuereusError(`Cannot change the data type of primary key column '${change.columnName}' of table '${oldSchema.name}'.`, StatusCode.CONSTRAINT);
1949
+ }
1950
+ // Conversion required — walk every row and attempt parse.
1778
1951
  const convert = (v) => {
1779
1952
  try {
1780
1953
  return validateAndParse(v, newLogicalType, change.columnName);
@@ -1783,21 +1956,21 @@ export class StoreModule {
1783
1956
  throw new QuereusError(`Cannot convert value in '${change.columnName}' to ${change.setDataType}`, StatusCode.MISMATCH);
1784
1957
  }
1785
1958
  };
1786
- // Throw-only pass over the LIVE rows first, so an unconvertible value
1787
- // this transaction inserted rejects the ALTER with the transaction still
1788
- // intact. The rewrite below reads only committed rows.
1959
+ // Throw-only pass over the LIVE rows, so an unconvertible value this
1960
+ // transaction inserted rejects the ALTER with the transaction still intact.
1961
+ // The rewrite itself is DEFERRED: the caller flushes + `mapRowsAtIndex`es
1962
+ // this closure only after its remaining throw-only checks pass.
1963
+ // NOTE: pre-existing gap — this scan reads the store's own effective rows and
1964
+ // ignores a wrapper-supplied `rows` stream (unlike the memory module's arm). No
1965
+ // hole under isolation: the wrapper's `validateOverlayMigration` converts every
1966
+ // staged overlay value itself before the underlying mutates.
1789
1967
  for await (const value of table.iterateEffectiveValuesAtIndex(colIndex)) {
1790
1968
  if (value !== null)
1791
1969
  convert(value);
1792
1970
  }
1793
- // Physical rewrite ahead — flush buffered writes so `mapRowsAtIndex`
1794
- // converts this transaction's rows too; unflushed, they would replay
1795
- // under the OLD physical type. See {@link ddlCommitPendingOps}.
1796
- await this.ddlCommitPendingOps();
1797
- await table.mapRowsAtIndex(colIndex, (v) => v === null ? v : convert(v));
1798
- valuesRewritten = true;
1971
+ valueConvert = (v) => v === null ? v : convert(v);
1799
1972
  }
1800
- return { newCol: { ...oldCol, logicalType: newLogicalType }, collationChanged: false, valuesRewritten };
1973
+ return { newCol: { ...oldCol, logicalType: newLogicalType }, collationChanged: false, valueConvert };
1801
1974
  }
1802
1975
  /**
1803
1976
  * SET COLLATE sub-branch of {@link alterColumnChange}. Returns the recollated column
@@ -1928,6 +2101,17 @@ export class StoreModule {
1928
2101
  }
1929
2102
  }
1930
2103
  // Move physical storage (data directory + index directories).
2104
+ //
2105
+ // NOTE: the relocation runs BEFORE the catalog rewrite below, and nothing undoes it.
2106
+ // Any failure after this point — an IO error, or the DDL-text guard in
2107
+ // `encodeCatalogDDL` firing on an unpaired surrogate in a column name or a `default`
2108
+ // literal — strands the rows under the NEW physical name while the catalog still
2109
+ // names the old table, so the table reads as empty with only the raised error as a
2110
+ // clue. Harmless for today's validation cases: the store-name guard in
2111
+ // `buildDataStoreName` (called above, before any side effect) refuses a bad target
2112
+ // name outright, and a table whose DDL text is unpersistable can never have held
2113
+ // rows in the first place. If a new post-relocation failure mode appears, the
2114
+ // relocation must be undone here or deferred until after the catalog write.
1931
2115
  if (this.provider.renameTableStores) {
1932
2116
  await this.provider.renameTableStores(schemaName, oldName, newName, indexNames);
1933
2117
  }
@@ -2071,6 +2255,13 @@ export class StoreModule {
2071
2255
  * needs the connection's collation registry, hence `db` is threaded down.
2072
2256
  */
2073
2257
  getBestAccessPlan(db, tableInfo, request) {
2258
+ // NOTE: this module does not call `validateAccessPlan` / `validateAccessPlanRequest`,
2259
+ // so a malformed request (e.g. a `runtimeSet` whose `maxCount` is not a positive
2260
+ // integer) is not rejected at this boundary the way it is for the memory module. It
2261
+ // degrades safely instead: `equalitySeekKeyCount` returns null for every malformed
2262
+ // shape, so the filter is simply never claimed and survives as a residual. If an
2263
+ // engine bug ever needs catching here rather than absorbing, call
2264
+ // `validateAccessPlan(request, plan)` on the way out.
2074
2265
  return { ...this.computeBestAccessPlan(db, tableInfo, request), honorsCollatedRangeBounds: true };
2075
2266
  }
2076
2267
  computeBestAccessPlan(db, tableInfo, request) {
@@ -2095,7 +2286,7 @@ export class StoreModule {
2095
2286
  // Full PK match - point lookup (single row; no monotonic advertisement)
2096
2287
  return AccessPlanBuilder
2097
2288
  .eqMatch(1, 0.1)
2098
- .setHandledFilters(claimFirstPerRole(request.filters, equalityRoles(pkColumns)))
2289
+ .setHandledFilters(claimFirstPerRole(request.filters, equalityRoles(pkColumns, EQ_OPS)))
2099
2290
  .setIsSet(true)
2100
2291
  .setIndexName('_primary_')
2101
2292
  .setExplanation('Store primary key lookup')
@@ -2213,12 +2404,33 @@ export class StoreModule {
2213
2404
  if (index.predicate)
2214
2405
  return null;
2215
2406
  const indexColIndexes = index.columns.map(c => c.index);
2216
- // Contiguous leading-prefix EQ → point/prefix seek.
2407
+ // Contiguous leading-prefix equality → point/prefix seek. An IN filter counts as
2408
+ // an equality here (it is N equalities, served as one multi-seek by
2409
+ // StoreTable.scanMultiSeek); `inCount` is the cross-product of the per-column
2410
+ // seek-key counts (1 for a plain '='), matching the rule's seek-key count. The
2411
+ // FIRST role-filling filter per column is what the rule seeks on, so its
2412
+ // cardinality is the one that counts.
2413
+ //
2414
+ // A runtime-valued IN set contributes its `maxCount` ceiling — the worst case the
2415
+ // engine may deliver — so every gate below (the MAX_MULTI_SEEK_KEYS cap, the
2416
+ // semantic-ordering decline) judges it against the largest multi-seek it could
2417
+ // ever be asked to perform.
2217
2418
  const eqCols = [];
2419
+ let inCount = 1;
2420
+ // `isMultiSeek` is NOT `inCount > 1`: a runtime set is delivered as a `plan=5`
2421
+ // multi-seek even at `maxCount === 1`, so the gates below must judge it as one
2422
+ // rather than as the plain EQ its ceiling arithmetic would otherwise suggest.
2423
+ // (`inCount > 1` implies this flag — every factor above 1 is a multi-value
2424
+ // equality — so the flag only ever *adds* the maxCount-1 runtime-set case.)
2425
+ let isMultiSeek = false;
2218
2426
  for (const colIdx of indexColIndexes) {
2219
- if (!request.filters.some(f => f.columnIndex === colIdx && f.op === '='))
2427
+ const eqFilter = request.filters.find(f => f.columnIndex === colIdx && equalitySeekKeyCount(f) !== null);
2428
+ if (!eqFilter)
2220
2429
  break;
2221
2430
  eqCols.push(colIdx);
2431
+ inCount *= equalitySeekKeyCount(eqFilter);
2432
+ if (isMultiValueEquality(eqFilter))
2433
+ isMultiSeek = true;
2222
2434
  }
2223
2435
  const leadingCol = indexColIndexes[0];
2224
2436
  const hasLeadingRange = request.filters.some(f => f.columnIndex === leadingCol && RANGE_OPS.includes(f.op));
@@ -2272,8 +2484,35 @@ export class StoreModule {
2272
2484
  if (!seekCols.every(isRange ? rangeSafeToHandle : eqSafeToHandle)) {
2273
2485
  return costOnly('cost-only; key collation may under-fetch');
2274
2486
  }
2487
+ // Multi-seek declines. Cost-only keeps the residual, so the answer stays right
2488
+ // and only the speed-up is lost.
2489
+ if (inCount > MAX_MULTI_SEEK_KEYS) {
2490
+ return costOnly(`cost-only; IN cross-product of ${inCount} exceeds the ${MAX_MULTI_SEEK_KEYS}-seek cap`);
2491
+ }
2492
+ if (isMultiSeek && seekCols.some(colIdx => hasSemanticOrdering(tableInfo.columns[colIdx]?.logicalType))) {
2493
+ // A plain EQ on a TIMESPAN/JSON column degrades safely (StoreTable.analyzeIndexAccess
2494
+ // breaks its prefix there and the full-scan residual re-filters under the type's
2495
+ // compare), but a multi-seek drops the residual and its byte-equality windows
2496
+ // under-fetch the type's equality ('PT1H' ≡ 'PT60M', byte-distinct raw values).
2497
+ return costOnly('cost-only; semantic-ordering seek column cannot multi-seek');
2498
+ }
2275
2499
  // Claim positionally — see {@link claimFirstPerRole}.
2276
- const handledFilters = claimFirstPerRole(request.filters, isRange ? rangeRoles(leadingCol) : equalityRoles(eqCols));
2500
+ const handledFilters = claimFirstPerRole(request.filters, isRange ? rangeRoles(leadingCol) : equalityRoles(eqCols, EQ_OR_IN_OPS));
2501
+ if (isMultiSeek) {
2502
+ // Multi-seek (plan=5): inCount point seeks, `rows` matched per seek key. The
2503
+ // per-seek positioning term keeps a 500-key IN over a 10-row table from pricing
2504
+ // below a full scan and issuing 500 seeks to read 10 rows. `isSet` false mirrors
2505
+ // MemoryTableModule.evaluateIndexAccess's setIsSet(!isMultiSeek). No ordering is
2506
+ // advertised — window emission order is encoded-key order, not any column order.
2507
+ const multiRows = Math.min(estimatedRows, inCount * rows);
2508
+ return AccessPlanBuilder.eqMatch(multiRows, inCount * INDEX_SEEK_COST)
2509
+ .setIsSet(false)
2510
+ .setHandledFilters(handledFilters)
2511
+ .setIndexName(index.name)
2512
+ .setSeekColumns(seekCols)
2513
+ .setExplanation(`Store index multi-seek(${inCount}) on ${index.name}`)
2514
+ .build();
2515
+ }
2277
2516
  return (isRange ? AccessPlanBuilder.rangeScan(rows, 0.2) : AccessPlanBuilder.eqMatch(rows, 0.3))
2278
2517
  .setHandledFilters(handledFilters)
2279
2518
  .setIndexName(index.name)
@@ -2463,15 +2702,11 @@ export class StoreModule {
2463
2702
  }
2464
2703
  /**
2465
2704
  * Encode a bundle of persisted schema text (DDL) for the catalog store.
2466
- *
2467
- * Guards the FULL text, not just the object's name: a lone surrogate anywhere in it — a
2468
- * quoted column name, a `default '…'` string literal, a `check` constraint's string
2469
- * constant — would otherwise fold to U+FFFD under `TextEncoder` and read back as
2470
- * different schema text than what was created. This does not rely on the catalog-key
2471
- * builders' identifier guard having already caught it; it catches it independently.
2705
+ * The guard is {@link assertPersistableDdlText} — see it for why the FULL text
2706
+ * is checked, not just the object's name.
2472
2707
  */
2473
2708
  encodeCatalogDDL(ddl) {
2474
- assertNoUnpairedSurrogate(ddl, 'persisted schema text');
2709
+ assertPersistableDdlText(ddl);
2475
2710
  return new TextEncoder().encode(ddl);
2476
2711
  }
2477
2712
  /**
@@ -2792,7 +3027,8 @@ export class StoreModule {
2792
3027
  * {@link persistObjectCatalogEntryIfChanged}.
2793
3028
  */
2794
3029
  async saveViewDDL(view) {
2795
- await this.persistObjectCatalogEntryIfChanged(buildViewCatalogKey(view.schemaName, view.name), generateViewDDL(view));
3030
+ const { key, ddl } = viewCatalogEntry(view);
3031
+ await this.persistObjectCatalogEntryIfChanged(key, ddl);
2796
3032
  }
2797
3033
  /** Remove a plain view's catalog entry (on DROP VIEW). */
2798
3034
  async removeViewDDL(schemaName, viewName) {
@@ -2806,13 +3042,50 @@ export class StoreModule {
2806
3042
  * {@link persistObjectCatalogEntryIfChanged}.
2807
3043
  */
2808
3044
  async saveMaterializedViewDDL(mv) {
2809
- await this.persistObjectCatalogEntryIfChanged(buildMaterializedViewCatalogKey(mv.schemaName, mv.name), generateMaintainedTableDDL(mv));
3045
+ const entry = maintainedViewCatalogEntry(mv);
3046
+ // Callers only reach here with a maintained table; the guard keeps the shared
3047
+ // helper's `undefined` arm honest rather than asserting non-null.
3048
+ if (!entry)
3049
+ return;
3050
+ await this.persistObjectCatalogEntryIfChanged(entry.key, entry.ddl);
2810
3051
  }
2811
3052
  /** Remove a materialized view's catalog entry (on DROP MATERIALIZED VIEW). */
2812
3053
  async removeMaterializedViewDDL(schemaName, mvName) {
2813
3054
  const catalogStore = await this.provider.getCatalogStore();
2814
3055
  await catalogStore.delete(buildMaterializedViewCatalogKey(schemaName, mvName));
2815
3056
  }
3057
+ /**
3058
+ * Pre-flight veto (see `VirtualTableModule.assertCatalogObjectPersistable`): refuse a
3059
+ * view / materialized view whose catalog entry this module could not durably write.
3060
+ *
3061
+ * It runs exactly the derivation the write path runs — {@link viewCatalogEntry} /
3062
+ * {@link maintainedViewCatalogEntry} build the key (rejecting an unencodable
3063
+ * identifier) and {@link assertPersistableDdlText} checks the generated DDL text —
3064
+ * so the veto and the write cannot disagree about what is persistable. It is the only
3065
+ * synchronous path a rejection can travel: the actual save is chained onto
3066
+ * `persistQueue` behind a `SchemaChangeNotifier` listener, and both layers swallow.
3067
+ *
3068
+ * Gated on `subscribedDb === db`: this module persists a view/MV only while subscribed
3069
+ * to that `Database`'s change notifier ({@link ensureSchemaSubscription}, driven by the
3070
+ * first `create`/`connect`/`alterTable`/`rehydrateCatalog`). A module registered but
3071
+ * never handed a `db` writes nothing, so vetoing there would reject a definition that
3072
+ * loses nothing. The gate becomes removable if persistence is ever made unconditional
3073
+ * (`bug-store-untouched-table-and-early-view-never-persisted`).
3074
+ */
3075
+ // NOTE: regenerates the object's full DDL text on every CREATE VIEW / CREATE MATERIALIZED
3076
+ // VIEW / view-or-MV SET TAGS, and the persist that follows regenerates it again. DDL is
3077
+ // rare and the text is small, so the duplicate render is not worth caching today; if a
3078
+ // schema-heavy workload (a large `apply schema` deploying many views) ever shows up hot
3079
+ // here, thread the already-built CatalogEntry from the veto through to the write.
3080
+ assertCatalogObjectPersistable(db, kind, object) {
3081
+ if (this.subscribedDb !== db)
3082
+ return;
3083
+ const entry = kind === 'view'
3084
+ ? viewCatalogEntry(object)
3085
+ : maintainedViewCatalogEntry(object);
3086
+ if (entry)
3087
+ assertPersistableDdlText(entry.ddl);
3088
+ }
2816
3089
  /**
2817
3090
  * Compare-write a view/MV catalog entry: write only when the entry is absent or its
2818
3091
  * DDL differs from `newDDL` (skip identical). Unlike the table path's
@@ -3203,6 +3476,21 @@ async function* rowsFromEntries(entries) {
3203
3476
  yield deserializeRow(entry.value);
3204
3477
  }
3205
3478
  }
3479
+ /**
3480
+ * Read-side wrap for a value-rewriting ALTER COLUMN's pre-mutation UNIQUE probe: each row's
3481
+ * value at `colIndex` run through `convert` (the sub-branch's deferred `valueConvert`, which
3482
+ * owns its own NULL handling), so the probe judges the POST-alter values while the store
3483
+ * still holds the old ones. Mutates nothing — the memory module's `convertRowAtIndex`
3484
+ * counterpart. A conversion failure propagates: the setDataType pre-pass already proved
3485
+ * every visible value convertible, so a throw here is unreachable, and surfacing it beats
3486
+ * probing a stale value that could mask a collision.
3487
+ */
3488
+ async function* convertRowsAtIndex(rows, colIndex, convert) {
3489
+ for await (const row of rows) {
3490
+ const newVal = convert(row[colIndex]);
3491
+ yield newVal === row[colIndex] ? row : row.map((v, i) => i === colIndex ? newVal : v);
3492
+ }
3493
+ }
3206
3494
  /**
3207
3495
  * Per-column key normalizers for a UNIQUE INDEX dedupe signature, drawing each column's
3208
3496
  * collation from the index column (if it carries one) else the underlying table column —
@@ -3210,7 +3498,7 @@ async function* rowsFromEntries(entries) {
3210
3498
  * write-time enforcement.
3211
3499
  *
3212
3500
  * A column whose declared type can never hold text takes the identity normalizer regardless
3213
- * of its collation (`serializeRowKey` normalizes only string values), so a comparator-only
3501
+ * of its collation (the signature serializer normalizes only string values), so a comparator-only
3214
3502
  * collation named on an integer column is not rejected here when the engine's own hash sites
3215
3503
  * would accept it.
3216
3504
  */
@@ -3222,20 +3510,38 @@ function indexDedupeNormalizers(tableSchema, indexSchema, keyNormalizers) {
3222
3510
  : undefined);
3223
3511
  });
3224
3512
  }
3513
+ /**
3514
+ * Collation- and identity-aware dedupe signature of `row` over `colIndices`: each value
3515
+ * runs through its column's key transform (`storeSemanticKeyTransform` — TIMESPAN's
3516
+ * total-seconds `groupKey`, so 'PT1H' and 'PT60M' sign identically; JSON's structural
3517
+ * bytes, so reorder-equal objects sign identically) before `serializeKey` applies the
3518
+ * per-column normalizer. Returns null when any covered value
3519
+ * is NULL (SQL UNIQUE allows multiple NULLs). The build/validate-time twin of the
3520
+ * write-time typed compare in `StoreTable.uniqueColumnComparators`.
3521
+ */
3522
+ function dedupeRowSignature(row, colIndices, normalizers, transforms) {
3523
+ const values = colIndices.map((c, i) => {
3524
+ const v = row[c];
3525
+ const transform = transforms[i];
3526
+ return transform && v !== null ? transform(v) : v;
3527
+ });
3528
+ return serializeKey(values, normalizers);
3529
+ }
3225
3530
  /**
3226
3531
  * Throws CONSTRAINT on the first pair of `rows` that share a signature over `columnIndices`.
3227
3532
  *
3228
- * SQL NULL semantics: `serializeRowKey` returns null when any constrained column is NULL, and
3229
- * such rows never collide. A partial `predicate` restricts the judged set to rows it accepts
3230
- * unambiguously. Shared by {@link StoreModule.validateUniqueOverExistingRows} (constraint
3231
- * columns) and {@link StoreModule.validateUniqueIndexOverRows} (index columns).
3533
+ * SQL NULL semantics: {@link dedupeRowSignature} returns null when any constrained column is
3534
+ * NULL, and such rows never collide. A partial `predicate` restricts the judged set to rows
3535
+ * it accepts unambiguously. Shared by {@link StoreModule.validateUniqueOverExistingRows}
3536
+ * (constraint columns) and {@link StoreModule.validateUniqueIndexOverRows} (index columns).
3232
3537
  */
3233
3538
  async function assertNoDuplicateRows(rows, tableSchema, columnIndices, normalizers, predicate) {
3539
+ const transforms = columnIndices.map(i => storeSemanticKeyTransform(tableSchema.columns[i]?.logicalType));
3234
3540
  const seen = new Set();
3235
3541
  for await (const row of rows) {
3236
3542
  if (predicate && predicate.evaluate(row) !== true)
3237
3543
  continue;
3238
- const keySig = serializeRowKey(row, columnIndices, normalizers);
3544
+ const keySig = dedupeRowSignature(row, columnIndices, normalizers, transforms);
3239
3545
  if (keySig === null)
3240
3546
  continue;
3241
3547
  if (seen.has(keySig)) {