@quereus/store 3.3.0 → 4.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. package/README.md +436 -317
  2. package/dist/src/common/backing-host.d.ts +198 -0
  3. package/dist/src/common/backing-host.d.ts.map +1 -0
  4. package/dist/src/common/backing-host.js +452 -0
  5. package/dist/src/common/backing-host.js.map +1 -0
  6. package/dist/src/common/bytes.d.ts +16 -0
  7. package/dist/src/common/bytes.d.ts.map +1 -0
  8. package/dist/src/common/bytes.js +33 -0
  9. package/dist/src/common/bytes.js.map +1 -0
  10. package/dist/src/common/cached-kv-store.d.ts +3 -3
  11. package/dist/src/common/cached-kv-store.d.ts.map +1 -1
  12. package/dist/src/common/cached-kv-store.js +6 -4
  13. package/dist/src/common/cached-kv-store.js.map +1 -1
  14. package/dist/src/common/encoding.d.ts +9 -1
  15. package/dist/src/common/encoding.d.ts.map +1 -1
  16. package/dist/src/common/encoding.js +12 -2
  17. package/dist/src/common/encoding.js.map +1 -1
  18. package/dist/src/common/index.d.ts +8 -6
  19. package/dist/src/common/index.d.ts.map +1 -1
  20. package/dist/src/common/index.js +7 -3
  21. package/dist/src/common/index.js.map +1 -1
  22. package/dist/src/common/key-builder.d.ts +125 -4
  23. package/dist/src/common/key-builder.d.ts.map +1 -1
  24. package/dist/src/common/key-builder.js +186 -9
  25. package/dist/src/common/key-builder.js.map +1 -1
  26. package/dist/src/common/kv-store.d.ts +75 -4
  27. package/dist/src/common/kv-store.d.ts.map +1 -1
  28. package/dist/src/common/memory-store.d.ts +3 -3
  29. package/dist/src/common/memory-store.d.ts.map +1 -1
  30. package/dist/src/common/memory-store.js +4 -2
  31. package/dist/src/common/memory-store.js.map +1 -1
  32. package/dist/src/common/store-connection.d.ts +11 -4
  33. package/dist/src/common/store-connection.d.ts.map +1 -1
  34. package/dist/src/common/store-connection.js +12 -6
  35. package/dist/src/common/store-connection.js.map +1 -1
  36. package/dist/src/common/store-module.d.ts +580 -20
  37. package/dist/src/common/store-module.d.ts.map +1 -1
  38. package/dist/src/common/store-module.js +1590 -135
  39. package/dist/src/common/store-module.js.map +1 -1
  40. package/dist/src/common/store-table.d.ts +347 -14
  41. package/dist/src/common/store-table.d.ts.map +1 -1
  42. package/dist/src/common/store-table.js +751 -98
  43. package/dist/src/common/store-table.js.map +1 -1
  44. package/dist/src/common/transaction.d.ts +135 -27
  45. package/dist/src/common/transaction.d.ts.map +1 -1
  46. package/dist/src/common/transaction.js +216 -55
  47. package/dist/src/common/transaction.js.map +1 -1
  48. package/package.json +8 -8
@@ -12,12 +12,13 @@
12
12
  * - {prefix}.__stats__ - Unified stats store (row counts for all tables)
13
13
  * - Catalog store: __catalog__ - DDL metadata keyed by {schema}.{table}
14
14
  */
15
- import { AccessPlanBuilder, QuereusError, StatusCode, buildColumnIndexMap, columnDefToSchema, compilePredicate, inferType, tryFoldLiteral, validateAndParse } from '@quereus/quereus';
15
+ import { AccessPlanBuilder, QuereusError, StatusCode, buildColumnIndexMap, columnDefToSchema, compilePredicate, inferType, tryFoldLiteral, validateAndParse, buildAdvertisementsFromTags, resolveNamedConstraintClass, validateCollationForType, buildUniqueConstraintSchema, buildForeignKeyConstraintSchema, buildCheckConstraintSchema, validateForeignKeyOverExistingRows, extractColumnLevelCheckConstraints, extractColumnLevelForeignKeys, appendIndexToTableSchema, resolveKeyNormalizer, serializeRowKey, isMaintainedTable } from '@quereus/quereus';
16
16
  import { TransactionCoordinator } from './transaction.js';
17
- import { StoreTable } from './store-table.js';
18
- import { buildCatalogKey, buildCatalogScanBounds, buildIndexKey, buildFullScanBounds, buildStatsKey, } from './key-builder.js';
17
+ import { StoreBackingHost } from './backing-host.js';
18
+ import { StoreTable, resolvePkKeyCollations } from './store-table.js';
19
+ import { buildCatalogKey, buildCatalogScanBounds, buildDataStoreName, buildIndexKey, buildIndexStoreName, buildFullScanBounds, buildStatsKey, buildViewCatalogKey, buildMaterializedViewCatalogKey, parseMaterializedViewCatalogKey, buildMetaCatalogKey, CLEAN_SHUTDOWN_META_NAME, STALE_MVS_META_NAME, classifyCatalogKey, } from './key-builder.js';
19
20
  import { deserializeRow } from './serialization.js';
20
- import { generateTableDDL } from '@quereus/quereus';
21
+ import { generateTableDDL, generateIndexDDL, generateViewDDL, generateMaintainedTableDDL, generateIndexTagsDDL, isHiddenImplicitIndex, exposedImplicitIndexes } from '@quereus/quereus';
21
22
  /**
22
23
  * Generic store module that works with any KVStoreProvider.
23
24
  *
@@ -34,19 +35,73 @@ import { generateTableDDL } from '@quereus/quereus';
34
35
  export class StoreModule {
35
36
  provider;
36
37
  stores = new Map();
37
- coordinators = new Map();
38
+ /**
39
+ * The single transaction coordinator shared by every {@link StoreTable} this
40
+ * module owns — the unit of cross-table atomicity (see {@link getCoordinator}).
41
+ * Lives for the module's lifetime: a single table's drop must NOT evict it
42
+ * (sibling tables still use it); only {@link closeAll} clears it.
43
+ */
44
+ moduleCoordinator;
38
45
  tables = new Map();
39
46
  eventEmitter;
47
+ /**
48
+ * Optional listener wired by the host (the worker) to forward a logical
49
+ * `apply schema` lens deployment to the sync layer's lifecycle bookkeeping.
50
+ * The store module is the only basis-backing host with both persistence and a
51
+ * `db` handle, so it is the forwarder; `@quereus/store` must not depend on
52
+ * `@quereus/sync`, so the listener is a plain callback the worker binds.
53
+ */
54
+ lensDeploymentListener;
55
+ /** Unsubscribe thunk for the engine `SchemaChangeNotifier` listener, set on first hook with a `db`. */
56
+ schemaListenerUnsub;
57
+ /** The `Database` whose notifier we subscribed to. One module instance serves one `Database`. */
58
+ subscribedDb;
59
+ /**
60
+ * Serialized chain of pending catalog writes triggered by engine schema-change
61
+ * events (catalog-only tag swaps). `notifyChange` invokes listeners synchronously
62
+ * and does not await them, so the actual read-compare-write runs here, in order,
63
+ * and is drained by `closeAll` (and `whenCatalogPersisted`) before the provider closes.
64
+ */
65
+ persistQueue = Promise.resolve();
66
+ /**
67
+ * The JSON-serialized stale-MV set last enqueued onto {@link persistQueue} (the
68
+ * `\x00meta\x00stale_mvs` value), or `undefined` when no baseline is established
69
+ * for the current subscription. The schema-change listener compares each
70
+ * recomputed set against this so an unrelated `table_modified` (e.g. a tag swap
71
+ * that changes no MV's staleness) costs no extra fsync. Reset to `undefined`
72
+ * whenever a fresh `Database` is subscribed ({@link ensureSchemaSubscription}) so
73
+ * a reopened module re-establishes the baseline from the first relevant event (or
74
+ * the rehydrate tail).
75
+ */
76
+ lastPersistedStaleMvs;
77
+ /**
78
+ * Whether the provider exposes an atomic cross-store commit domain
79
+ * ({@link KVStoreProvider.beginAtomicBatch}) — the capability the MV adopt fast
80
+ * path's gate-5 drop hinges on. Fixed for the provider's lifetime, so cached at
81
+ * construction (and used as the single gate for both writing AND reading the durable
82
+ * stale-MV set). Gating the **write** on it is load-bearing for soundness, not just an
83
+ * fsync saving: only an atomic-capable session — whose commits can never tear a source
84
+ * from its same-module backing — ever writes the durable set, so the set a later atomic
85
+ * session reads was necessarily written under that same no-tear guarantee. A persistent
86
+ * non-atomic provider therefore can never leave a torn-but-"not-stale" set for an atomic
87
+ * reopen to trust (it writes none); such a store falls back to the clean-shutdown marker.
88
+ */
89
+ atomicProvider;
40
90
  constructor(provider, eventEmitter) {
41
91
  this.provider = provider;
42
92
  this.eventEmitter = eventEmitter;
93
+ this.atomicProvider = typeof provider.beginAtomicBatch === 'function';
43
94
  }
44
95
  /**
45
96
  * Returns capability flags for this module.
46
97
  *
47
- * The base StoreModule does NOT provide transaction isolation.
48
- * Without isolation, queries see only committed data (no read-your-own-writes
49
- * within a transaction). To enable isolation, wrap with IsolationModule:
98
+ * The base StoreModule does NOT provide transaction isolation: there is no
99
+ * snapshot isolation and no cross-connection isolation (readers on other
100
+ * connections see only committed data). Within a transaction, reads through
101
+ * the table's shared coordinator DO see that transaction's own pending
102
+ * writes (read-your-own-writes — `StoreTable.query` merges the pending op
103
+ * view over the committed store). For full isolation, wrap with
104
+ * IsolationModule:
50
105
  *
51
106
  * ```typescript
52
107
  * import { IsolationModule, MemoryTableModule } from '@quereus/quereus';
@@ -63,24 +118,207 @@ export class StoreModule {
63
118
  getCapabilities() {
64
119
  return {
65
120
  isolation: false,
66
- savepoints: false,
121
+ // Coordinator-buffered ops support savepoint create/release/rollback-to
122
+ // within a transaction (advisory flag — not engine-consulted).
123
+ savepoints: true,
67
124
  persistent: true,
68
125
  secondaryIndexes: true,
69
126
  rangeScans: true,
70
127
  };
71
128
  }
129
+ /**
130
+ * Generic-module mapping advertisements: assembled from the `quereus.lens.decomp.*`
131
+ * reserved tags on this basis schema's tables. Returns `[]` when the schema has no
132
+ * such tags, leaving the lens default mapper on its name-match path.
133
+ * See `docs/lens.md` § The Default Mapper.
134
+ */
135
+ getMappingAdvertisements(_db, basisSchema) {
136
+ return buildAdvertisementsFromTags(basisSchema);
137
+ }
138
+ /**
139
+ * Backing-host capability (engine `vtab/backing-host.ts`): resolve the
140
+ * privileged surface for a store table this module owns, or undefined when the
141
+ * table is unknown to it. The host binds the CURRENT (StoreTable, coordinator)
142
+ * pair — `destroy` evicts both maps, so a drop+recreate yields fresh instances
143
+ * and the returned host is pinned to one backing-table incarnation (the engine
144
+ * resolves hosts fresh per call, never caching them). Resolution goes through
145
+ * {@link getOrReconnectTable} so a rehydrated-but-untouched (or rename-evicted)
146
+ * backing still resolves; the ownership pre-check keeps the reconnect fallback
147
+ * from adopting a registered table owned by a different module (`vtabModule`
148
+ * must be this StoreModule, or a wrapper — IsolationModule — exposing it as
149
+ * `underlying`). Attaching the coordinator eagerly makes the shared
150
+ * StoreTable's read paths merge the host's pending writes (reads-own-writes
151
+ * for a `select` from the MV mid-transaction).
152
+ */
153
+ getBackingHost(db, schemaName, tableName) {
154
+ const table = this.resolveOwnedTable(db, schemaName, tableName);
155
+ if (!table)
156
+ return undefined;
157
+ return new StoreBackingHost(table, table.attachCoordinator());
158
+ }
159
+ /**
160
+ * Resolve a {@link StoreTable} this module owns, reconnecting a
161
+ * rehydrated-but-untouched (or rename-evicted) table via
162
+ * {@link getOrReconnectTable}. The ownership pre-check keeps the reconnect
163
+ * fallback from adopting a registered table owned by a DIFFERENT module:
164
+ * `vtabModule` must be this StoreModule, or a wrapper (IsolationModule)
165
+ * exposing it as `underlying`. Returns undefined for an unknown/non-owned
166
+ * table. Shared by {@link getBackingHost} and
167
+ * {@link getTableForExternalWrite}; neither attaches a coordinator here —
168
+ * each layers its own pending-state policy on top.
169
+ */
170
+ resolveOwnedTable(db, schemaName, tableName) {
171
+ const tableKey = `${schemaName}.${tableName}`.toLowerCase();
172
+ if (!this.tables.has(tableKey)) {
173
+ const registered = db.schemaManager.getTable(schemaName, tableName);
174
+ const wrapper = registered?.vtabModule;
175
+ if (!registered || (registered.vtabModule !== this && wrapper?.underlying !== this)) {
176
+ return undefined;
177
+ }
178
+ }
179
+ return this.getOrReconnectTable(db, schemaName, tableName);
180
+ }
181
+ /**
182
+ * Resolve the live {@link StoreTable} for an externally-applied write to a
183
+ * SOURCE table (committed put/delete + secondary-index + stats maintenance via
184
+ * {@link StoreTable.applyExternalRowChanges}). Returns undefined when the table
185
+ * is not this module's.
186
+ *
187
+ * Resolution goes through {@link resolveOwnedTable} (the shared ownership
188
+ * pre-check + reconnect fallback). Unlike `getBackingHost` it attaches no
189
+ * coordinator: external writes target committed storage, and the before-image
190
+ * read (`readEffectiveRowByKey`) merges any already-attached coordinator's
191
+ * pending state on its own (none when the table is freshly reconnected).
192
+ */
193
+ getTableForExternalWrite(db, schemaName, tableName) {
194
+ return this.resolveOwnedTable(db, schemaName, tableName);
195
+ }
72
196
  /**
73
197
  * Get the event emitter for this module.
74
198
  */
75
199
  getEventEmitter() {
76
200
  return this.eventEmitter;
77
201
  }
202
+ /**
203
+ * Bind (or clear, with `undefined`) the lens-deployment forwarder. The host
204
+ * (worker) calls this to route a logical `apply schema`'s deployment to the
205
+ * sync layer; see {@link notifyLensDeployment}.
206
+ */
207
+ setLensDeploymentListener(listener) {
208
+ this.lensDeploymentListener = listener;
209
+ }
210
+ /**
211
+ * Engine `notifyLensDeployment` hook: a logical `apply schema X` fires this on
212
+ * every registered module once the lens catalog mutation + snapshot rotation
213
+ * complete (see `VirtualTableModule.notifyLensDeployment`). The store module —
214
+ * the basis-backing host — forwards the deployment to the bound listener so the
215
+ * sync layer can update its basis-table lifecycle bookkeeping.
216
+ *
217
+ * INVERSION OF THE ENGINE FIRING CONTRACT: the engine documents that a throwing
218
+ * notification aborts `apply schema X`. We deliberately wrap the listener in
219
+ * try/catch and SWALLOW (structured-log only) here — lifecycle bookkeeping is
220
+ * advisory and must never brick a schema apply. A bookkeeping bug is a logged
221
+ * warning, not a failed deploy.
222
+ */
223
+ async notifyLensDeployment(db, logicalSchemaName, snapshot) {
224
+ if (!this.lensDeploymentListener)
225
+ return;
226
+ try {
227
+ await this.lensDeploymentListener(db, logicalSchemaName, snapshot);
228
+ }
229
+ catch (e) {
230
+ // Advisory bookkeeping — swallow so a listener bug cannot abort the deploy.
231
+ console.warn(`[StoreModule] lens-deployment listener failed for logical schema '${logicalSchemaName}'; `
232
+ + `lifecycle bookkeeping skipped: ${e instanceof Error ? e.message : String(e)}`);
233
+ }
234
+ }
78
235
  /**
79
236
  * Get the KVStoreProvider used by this module.
80
237
  */
81
238
  getProvider() {
82
239
  return this.provider;
83
240
  }
241
+ /**
242
+ * Build the set of physical store names this module's data/index stores
243
+ * currently occupy in `schemaName`, mapping each name to a human description
244
+ * of the logical object that owns it (for sited collision messages).
245
+ *
246
+ * Physical store names are built by string concatenation (`{schema}.{table}` /
247
+ * `{schema}.{table}_idx_{index}`) and the `_idx_` delimiter is itself a legal
248
+ * substring of any identifier, so two distinct logical objects (e.g. index
249
+ * `archive` on `t` and a sibling table `t_idx_archive`) can collapse to the
250
+ * same physical name. This set is the authoritative occupancy used by
251
+ * {@link assertStoreNameFree} to reject such a collision at CREATE time.
252
+ *
253
+ * The occupancy is the union of two sources, robust to lazy connection and the
254
+ * isolation wrapper (see the ticket's "Enumeration source" section):
255
+ * 1. `this.tables` — every store table this module touched this session.
256
+ * 2. the target schema's catalog tables whose `vtabModule === this` and which
257
+ * are not views — store-backed tables not yet lazily connected.
258
+ * Names embed the schema prefix, so cross-schema entries never collide and no
259
+ * per-schema filter on `this.tables` is needed. Memory-backed siblings and
260
+ * views own no store in this provider and are excluded (the `=== this` /
261
+ * `!isView` filter) to avoid false-positive rejects.
262
+ *
263
+ * No self-exclusion: at each guarded call the candidate object is not yet
264
+ * registered (create/createIndex run before the engine adds it) so it cannot
265
+ * self-collide; and for renameTable the renamed table's OWN stores stay in the
266
+ * set deliberately. Any overlap between a name the rename introduces and an
267
+ * own current store is a footprint-swap rename (`t` with index `x` → `t_idx_x`,
268
+ * or table `u_idx_x` with index `x` → `u`) that providers cannot relocate
269
+ * safely — relocation order determines whether a source is clobbered before it
270
+ * is moved — while no benign rename produces such an overlap. Keeping own
271
+ * stores in the set therefore causes no false rejects and keeps the
272
+ * reject-before-any-side-effect guarantee uniform.
273
+ */
274
+ collectOccupiedStoreNames(db, schemaName) {
275
+ const names = new Map();
276
+ const add = (s) => {
277
+ const dataName = buildDataStoreName(s.schemaName, s.name);
278
+ if (!names.has(dataName)) {
279
+ names.set(dataName, `data store of table '${s.schemaName}.${s.name}'`);
280
+ }
281
+ for (const idx of s.indexes ?? []) {
282
+ const idxName = buildIndexStoreName(s.schemaName, s.name, idx.name);
283
+ if (!names.has(idxName)) {
284
+ names.set(idxName, `index store of index '${idx.name}' on table '${s.schemaName}.${s.name}'`);
285
+ }
286
+ }
287
+ };
288
+ for (const t of this.tables.values()) {
289
+ add(t.getSchema());
290
+ }
291
+ for (const t of db.schemaManager.getSchemaOrFail(schemaName).getAllTables()) {
292
+ if (t.vtabModule !== this || t.isView)
293
+ continue;
294
+ add(t);
295
+ }
296
+ return names;
297
+ }
298
+ /**
299
+ * Throws `StatusCode.ERROR` when `candidate` (a physical store name produced by
300
+ * the key-builder for an object about to be created/renamed) already names an
301
+ * existing data or index store in `schemaName`. `candidateDesc` describes the
302
+ * incoming logical object; the message names the candidate physical store and
303
+ * both conflicting logical objects and is actionable (rename one of them).
304
+ *
305
+ * Must run BEFORE any storage side-effect (`getStore` / `getIndexStore` / the
306
+ * physical relocation): the provider opens/creates the store eagerly, so a
307
+ * guard that ran after would already have aliased the colliding object's store.
308
+ *
309
+ * Callers checking several candidates against the same occupancy (renameTable
310
+ * checks the new data store plus every relocated index store) pass a
311
+ * precomputed `occupied` map so the occupancy is collected once, not per call.
312
+ */
313
+ assertStoreNameFree(db, schemaName, candidate, candidateDesc, occupied) {
314
+ const occupiedBy = (occupied ?? this.collectOccupiedStoreNames(db, schemaName)).get(candidate);
315
+ if (occupiedBy !== undefined) {
316
+ throw new QuereusError(`Physical store-name collision: the ${candidateDesc} would map to physical store `
317
+ + `'${candidate}', which already backs the ${occupiedBy}. These two objects would `
318
+ + `share one physical store and silently corrupt each other. Rename the table or the `
319
+ + `colliding index.`, StatusCode.ERROR);
320
+ }
321
+ }
84
322
  /**
85
323
  * Creates a new store-backed table.
86
324
  * Called by CREATE TABLE.
@@ -90,17 +328,32 @@ export class StoreModule {
90
328
  * event handlers (like sync module) try to access it.
91
329
  */
92
330
  async create(db, tableSchema) {
331
+ this.ensureSchemaSubscription(db);
93
332
  const tableKey = `${tableSchema.schemaName}.${tableSchema.name}`.toLowerCase();
94
333
  if (this.tables.has(tableKey)) {
95
334
  throw new QuereusError(`Store table '${tableSchema.name}' already exists in schema '${tableSchema.schemaName}'`, StatusCode.ERROR);
96
335
  }
97
336
  const config = this.parseConfig(tableSchema.vtabArgs);
337
+ // Apply the store's default key collation K to any IMPLICIT-default text PK column
338
+ // (an explicit per-column PK collation is honored natively by the per-column key
339
+ // encoding — see reconcilePkCollations / StoreTable.pkKeyCollations). The reconciled
340
+ // schema is what StoreTable holds and what `finalizeCreatedTableSchema` registers, so
341
+ // an undecorated text PK reports/keys under K rather than the engine BINARY default.
342
+ const keyCollation = (config.collation || 'NOCASE').toUpperCase();
343
+ const reconciledSchema = reconcilePkCollations(tableSchema, keyCollation);
344
+ // Reject when this new table's physical data store name already names an
345
+ // existing store (the only real positive here is data-vs-index: a sibling
346
+ // index store `{schema}.t_idx_<x>` already occupies `{schema}.{thisTable}` —
347
+ // data-vs-data is prevented by engine table-name uniqueness). Must precede
348
+ // `getStore`, which eagerly opens/creates the directory.
349
+ const dataStoreName = buildDataStoreName(tableSchema.schemaName, tableSchema.name);
350
+ this.assertStoreNameFree(db, tableSchema.schemaName, dataStoreName, `data store of new table '${tableSchema.schemaName}.${tableSchema.name}'`);
98
351
  // Eagerly initialize the store BEFORE creating the table or emitting events.
99
352
  // This ensures the underlying storage (e.g., IndexedDB object store) exists
100
353
  // before any schema change handlers try to access it.
101
354
  const store = await this.provider.getStore(tableSchema.schemaName, tableSchema.name);
102
355
  this.stores.set(tableKey, store);
103
- const table = new StoreTable(db, this, tableSchema, config, this.eventEmitter
356
+ const table = new StoreTable(db, this, reconciledSchema, config, this.eventEmitter
104
357
  // isConnected defaults to false for newly created tables
105
358
  );
106
359
  this.tables.set(tableKey, table);
@@ -110,7 +363,7 @@ export class StoreModule {
110
363
  objectType: 'table',
111
364
  schemaName: tableSchema.schemaName,
112
365
  objectName: tableSchema.name,
113
- ddl: generateTableDDL(tableSchema),
366
+ ddl: generateTableDDL(reconciledSchema),
114
367
  });
115
368
  return table;
116
369
  }
@@ -119,6 +372,7 @@ export class StoreModule {
119
372
  * Called when loading schema from persistent storage.
120
373
  */
121
374
  async connect(db, _pAux, _moduleName, schemaName, tableName, options, importedTableSchema) {
375
+ this.ensureSchemaSubscription(db);
122
376
  const tableKey = `${schemaName}.${tableName}`.toLowerCase();
123
377
  // Check if we already have this table connected
124
378
  const existing = this.tables.get(tableKey);
@@ -153,7 +407,6 @@ export class StoreModule {
153
407
  columnIndexMap: new Map(),
154
408
  primaryKeyDefinition: [],
155
409
  checkConstraints: Object.freeze([]),
156
- isTemporary: false,
157
410
  isView: false,
158
411
  vtabModuleName: 'store',
159
412
  vtabArgs,
@@ -163,6 +416,19 @@ export class StoreModule {
163
416
  }
164
417
  }
165
418
  const config = this.parseConfig(vtabArgs);
419
+ // The load path does NOT reconcile PK collations: a persisted / hand-authored
420
+ // DDL stays loadable as-declared. Physical key bytes are always K-encoded by
421
+ // `StoreTable.encodeOptions`, so a legacy divergent text-PK collation is a
422
+ // stale `table_info` declaration, not a correctness risk. Reconciling the
423
+ // transient `StoreTable` here would be pointless anyway — `importCatalog`'s
424
+ // post-import reconcile loop (`table.updateSchema(fresh)`) immediately
425
+ // overwrites it with the `SchemaManager`-registered schema. A genuine
426
+ // reopen-time migration (an engine import-path hook reconciling the
427
+ // *registered* schema to K) was considered and deliberately not built:
428
+ // only pre-`store-pk-collate-create-time-divergence` data can carry such a
429
+ // DDL, and backwards compatibility is out of scope (AGENTS.md). If legacy
430
+ // migration ever comes into scope, the fix is a module-consulted
431
+ // normalization hook on `importCatalog`/`rehydrateCatalog`.
166
432
  const table = new StoreTable(db, this, tableSchema, config, this.eventEmitter, true // isConnected - DDL already exists in storage
167
433
  );
168
434
  this.tables.set(tableKey, table);
@@ -171,21 +437,81 @@ export class StoreModule {
171
437
  /**
172
438
  * Destroys a store table and its storage.
173
439
  */
174
- async destroy(_db, _pAux, _moduleName, schemaName, tableName) {
440
+ async destroy(db, _pAux, _moduleName, schemaName, tableName) {
441
+ const tableKey = `${schemaName}.${tableName}`.toLowerCase();
442
+ // Capture the schema BEFORE we drop in-memory references so we can hand the
443
+ // provider the authoritative index list (exact store names) rather than let
444
+ // it prefix-scan `{table}_idx_`, which would also delete a sibling table
445
+ // literally named `{table}_idx_<x>`. Prefer the cached StoreTable's own
446
+ // schema; fall back to the schema manager (mirrors renameTable). If neither
447
+ // yields a schema (already deregistered), fall back to [] — index stores
448
+ // can't be swept by name then, but that is no worse than today for the
449
+ // no-sibling case and strictly safer for the sibling case.
450
+ const table = this.tables.get(tableKey);
451
+ const currentSchema = table?.getSchema() ?? db.schemaManager.getTable(schemaName, tableName);
452
+ const indexNames = (currentSchema?.indexes ?? []).map(i => i.name);
453
+ await this.tearDownTableStorage(schemaName, tableName, indexNames);
454
+ // Emit schema change event for table drop
455
+ this.eventEmitter?.emitSchemaChange({
456
+ type: 'drop',
457
+ objectType: 'table',
458
+ schemaName,
459
+ objectName: tableName,
460
+ });
461
+ }
462
+ /**
463
+ * Reclaim the local storage of a DETACHED basis table by name — the store-side
464
+ * target of the sync layer's basis-eviction sweep
465
+ * (`SyncManager.evictExpiredBasisTables`, `docs/migration.md` § 4 Contract).
466
+ *
467
+ * Unlike {@link destroy}, the table is no longer in the engine schema (it was
468
+ * removed from the basis on detach; only its physical storage lingered), so
469
+ * there is no `db`/schema to consult and NO schema-change event is emitted — the
470
+ * engine already saw the detach. The caller (the sync recorder) supplies the
471
+ * captured secondary-index name list it retained from before detach, because the
472
+ * table schema (and its index list) is gone: passing the exact names avoids the
473
+ * provider prefix-scanning `{table}_idx_`, which can clobber a sibling table
474
+ * literally named `{table}_idx_<x>`.
475
+ *
476
+ * Idempotent: any cached handles are evicted and disconnected, then the data /
477
+ * index / stats stores and the catalog DDL are removed. Storage already gone (a
478
+ * prior real `drop table` ran `destroy`) is treated as success — the provider's
479
+ * `deleteTableStores` no-ops on absent stores and `removeTableDDL` no-ops on an
480
+ * absent key.
481
+ */
482
+ async reclaimDetachedTable(schemaName, tableName, indexNames) {
483
+ await this.tearDownTableStorage(schemaName, tableName, indexNames);
484
+ }
485
+ /**
486
+ * Tear down a table's in-memory handles and physical storage: evict the cached
487
+ * StoreTable / store / coordinator (synchronously, before any await, so a
488
+ * concurrent reconnect cannot observe a stale instance mid-teardown), disconnect
489
+ * the handle, delete the provider's data / index / stats stores (by the exact
490
+ * index names — never a `{table}_idx_` prefix scan, which could clobber a sibling
491
+ * table literally named `{table}_idx_<x>`), and remove the catalog DDL.
492
+ *
493
+ * Shared by {@link destroy} (live `drop table`) and {@link reclaimDetachedTable}
494
+ * (post-detach eviction). The caller owns the index-name source (live schema vs.
495
+ * the captured pre-detach list) and any schema-change event. Idempotent: the
496
+ * provider no-ops on absent stores and `removeTableDDL` no-ops on an absent key.
497
+ */
498
+ async tearDownTableStorage(schemaName, tableName, indexNames) {
175
499
  const tableKey = `${schemaName}.${tableName}`.toLowerCase();
176
- // Clear internal maps synchronously before any await, so a concurrent
177
- // create() cannot observe the stale table/store/coordinator across a
178
- // microtask boundary mid-destroy.
179
500
  const table = this.tables.get(tableKey);
180
501
  this.tables.delete(tableKey);
181
502
  this.stores.delete(tableKey);
182
- this.coordinators.delete(tableKey);
503
+ // NOTE: the coordinator is module-wide and shared by sibling tables, so a
504
+ // single table's teardown must NOT evict the coordinator itself. But the
505
+ // evicted StoreTable's stats-callback pair MUST be deregistered, or its
506
+ // closures (capturing this instance) stay pinned on the shared coordinator
507
+ // for the module's lifetime — a leak bounded by drop/recreate count.
508
+ // table.dispose() both flushes pending stats and runs that disposer.
183
509
  if (table) {
184
- await table.disconnect();
510
+ await table.dispose();
185
511
  }
186
512
  // Delete all stores for this table (data, indexes, stats)
187
513
  if (this.provider.deleteTableStores) {
188
- await this.provider.deleteTableStores(schemaName, tableName);
514
+ await this.provider.deleteTableStores(schemaName, tableName, indexNames);
189
515
  }
190
516
  else {
191
517
  // Fallback: just close the data store
@@ -193,56 +519,75 @@ export class StoreModule {
193
519
  }
194
520
  // Remove DDL from catalog
195
521
  await this.removeTableDDL(schemaName, tableName);
196
- // Emit schema change event for table drop
197
- this.eventEmitter?.emitSchemaChange({
198
- type: 'drop',
199
- objectType: 'table',
200
- schemaName,
201
- objectName: tableName,
202
- });
522
+ }
523
+ /**
524
+ * Returns the connected StoreTable for `schemaName.tableName`, lazily
525
+ * reconnecting from the engine's schema registry when absent.
526
+ *
527
+ * `renameTable` evicts the old key from `this.tables` and expects the next
528
+ * `connect()` to repopulate under the new name, but `apply schema` can run
529
+ * follow-up DDL (ALTER TABLE, CREATE/DROP INDEX) against the new name
530
+ * without an intervening connect. Mirror connect()'s schemaManager lookup
531
+ * so that DDL finds the moved table. Safe for the index paths because
532
+ * SchemaManager calls the module BEFORE mutating the registered table
533
+ * schema, so the reconnected cache matches what a connected instance holds.
534
+ */
535
+ getOrReconnectTable(db, schemaName, tableName) {
536
+ const tableKey = `${schemaName}.${tableName}`.toLowerCase();
537
+ let table = this.tables.get(tableKey);
538
+ if (!table) {
539
+ const registeredSchema = db.schemaManager.getTable(schemaName, tableName);
540
+ if (registeredSchema) {
541
+ table = new StoreTable(db, this, registeredSchema, this.parseConfig(registeredSchema.vtabArgs ?? {}), this.eventEmitter, true);
542
+ this.tables.set(tableKey, table);
543
+ }
544
+ }
545
+ return table;
203
546
  }
204
547
  /**
205
548
  * Creates an index on a store-backed table.
206
549
  */
207
- async createIndex(_db, schemaName, tableName, indexSchema) {
550
+ async createIndex(db, schemaName, tableName, indexSchema) {
208
551
  const tableKey = `${schemaName}.${tableName}`.toLowerCase();
209
- const table = this.tables.get(tableKey);
552
+ const table = this.getOrReconnectTable(db, schemaName, tableName);
210
553
  if (!table) {
211
554
  throw new QuereusError(`Store table '${tableName}' not found in schema '${schemaName}'`, StatusCode.NOTFOUND);
212
555
  }
556
+ // Reject when this new index's physical store name already names an existing
557
+ // store: a sibling table's data store (`{schema}.t_idx_<x>` == sibling table
558
+ // `t_idx_<x>`) or another table's index store (index-vs-index, e.g.
559
+ // `a.b_idx_c` vs `a_idx_b.c` both → `{schema}.a_idx_b_idx_c`). The new index
560
+ // is not yet registered in the schema or the table's cached schema at this
561
+ // point, so the candidate cannot self-collide. Must precede `getIndexStore`,
562
+ // which opens/creates the directory that `buildIndexEntries` then writes into.
563
+ const indexStoreName = buildIndexStoreName(schemaName, tableName, indexSchema.name);
564
+ this.assertStoreNameFree(db, schemaName, indexStoreName, `index store of new index '${indexSchema.name}' on table '${schemaName}.${tableName}'`);
213
565
  // Create the index store
214
566
  const indexStore = await this.provider.getIndexStore(schemaName, tableName, indexSchema.name);
215
567
  // Build index entries for existing rows
216
568
  const dataStore = await this.getStore(tableKey, table.getConfig());
217
569
  const tableSchema = table.getSchema();
218
- await this.buildIndexEntries(dataStore, indexStore, tableSchema, indexSchema);
570
+ const keyCollation = (table.getConfig().collation || 'NOCASE').toUpperCase();
571
+ await this.buildIndexEntries(dataStore, indexStore, tableSchema, indexSchema, keyCollation);
219
572
  // Refresh the connected table's cached schema so subsequent DML
220
573
  // maintains the new index (the engine's schema registry is updated
221
574
  // separately by SchemaManager.createIndex, but the StoreTable instance
222
- // holds its own reference captured at connect time). Mirrors
223
- // SchemaManager.addIndexToTableSchema, including the UNIQUE → derived
224
- // uniqueConstraint entry so checkUniqueConstraints enforces it.
225
- const updatedIndexes = Object.freeze([
226
- ...(tableSchema.indexes ?? []),
227
- indexSchema,
228
- ]);
229
- const updatedSchema = { ...tableSchema, indexes: updatedIndexes };
230
- if (indexSchema.unique) {
231
- // `derivedFromIndex` tags this synthesized constraint so a future
232
- // `StoreModule.dropIndex` can filter it out symmetrically (mirrors
233
- // SchemaManager.dropIndex / MemoryTableManager.dropIndex). Without that
234
- // filter on the drop side, the UNIQUE check would survive the index.
235
- updatedSchema.uniqueConstraints = Object.freeze([
236
- ...(tableSchema.uniqueConstraints ?? []),
237
- {
238
- name: indexSchema.name,
239
- columns: Object.freeze(indexSchema.columns.map(c => c.index)),
240
- predicate: indexSchema.predicate,
241
- derivedFromIndex: indexSchema.name,
242
- },
243
- ]);
244
- }
575
+ // holds its own reference captured at connect time). The shared
576
+ // appendIndexToTableSchema also synthesizes the UNIQUE → derived
577
+ // uniqueConstraint entry so checkUniqueConstraints enforces it; the
578
+ // `derivedFromIndex` tag lets StoreModule.dropIndex filter it back out
579
+ // symmetrically (mirrors SchemaManager.dropIndex / MemoryTableManager.dropIndex).
580
+ const updatedSchema = appendIndexToTableSchema(tableSchema, indexSchema);
245
581
  table.updateSchema(updatedSchema);
582
+ // Authoritative catalog write: persist the table's bundle now (including the
583
+ // new index), so the index survives close → reopen even when the table has
584
+ // no rows yet and was never lazily persisted. `markDdlSaved` suppresses the
585
+ // later lazy table-only write on first store access (StoreTable.ddlSaved), so
586
+ // this is the only catalog write the createIndex produces. SchemaManager fires
587
+ // a follow-up `table_modified` whose listener regenerates the SAME bundle and
588
+ // skips (identical) — see persistCatalogIfChanged.
589
+ await this.saveTableDDL(updatedSchema);
590
+ table.markDdlSaved();
246
591
  // Emit schema change event
247
592
  this.eventEmitter?.emitSchemaChange({
248
593
  type: 'create',
@@ -259,9 +604,8 @@ export class StoreModule {
259
604
  * synthesized from it, tagged with `derivedFromIndex`), releases the
260
605
  * cached index-store handle, and tears down the underlying index store.
261
606
  */
262
- async dropIndex(_db, schemaName, tableName, indexName) {
263
- const tableKey = `${schemaName}.${tableName}`.toLowerCase();
264
- const table = this.tables.get(tableKey);
607
+ async dropIndex(db, schemaName, tableName, indexName) {
608
+ const table = this.getOrReconnectTable(db, schemaName, tableName);
265
609
  if (!table) {
266
610
  throw new QuereusError(`Store table '${tableName}' not found in schema '${schemaName}'`, StatusCode.NOTFOUND);
267
611
  }
@@ -284,6 +628,13 @@ export class StoreModule {
284
628
  // failure of the physical drop doesn't leave the schema enforcing an
285
629
  // index whose backing store has already been mutated.
286
630
  table.updateSchema(updatedSchema);
631
+ // Rewrite the catalog bundle without the dropped index, before the physical
632
+ // teardown — so on reopen the index does not resurrect even if the store
633
+ // delete below fails. `markDdlSaved` keeps the lazy first-access save from
634
+ // re-writing the same bundle. SchemaManager's follow-up `table_modified`
635
+ // regenerates an identical bundle and skips.
636
+ await this.saveTableDDL(updatedSchema);
637
+ table.markDdlSaved();
287
638
  // Drop the cached handle on the table side and tear down the
288
639
  // underlying KVStore. `deleteIndexStore` (if the provider implements
289
640
  // it) closes the handle before removing the directory; otherwise we
@@ -311,14 +662,26 @@ export class StoreModule {
311
662
  * populateNewIndex so `CREATE UNIQUE INDEX` over duplicated data fails
312
663
  * atomically.
313
664
  */
314
- async buildIndexEntries(dataStore, indexStore, tableSchema, indexSchema) {
315
- const encodeOptions = { collation: 'NOCASE' };
665
+ async buildIndexEntries(dataStore, indexStore, tableSchema, indexSchema, keyCollation) {
666
+ // Index COLUMN values use the table-level key collation K; the PK SUFFIX uses
667
+ // each PK column's own key collation, so the suffix bytes match the data-store
668
+ // keys (and `StoreTable.updateSecondaryIndexes`' maintenance writes) exactly.
669
+ const encodeOptions = { collation: keyCollation };
316
670
  const pkDirections = tableSchema.primaryKeyDefinition.map(pk => !!pk.desc);
671
+ const pkCollations = resolvePkKeyCollations(tableSchema.primaryKeyDefinition, tableSchema.columns, keyCollation);
317
672
  const indexDirections = indexSchema.columns.map(col => !!col.desc);
318
673
  const predicate = indexSchema.predicate
319
674
  ? compilePredicate(indexSchema.predicate, tableSchema.columns)
320
675
  : undefined;
321
676
  const seen = indexSchema.unique ? new Set() : undefined;
677
+ // Per-column normalizers for the in-pass UNIQUE dup check, drawing each
678
+ // column's collation from the index column (if it carries one) else the
679
+ // underlying table column — so the dedup signature honors a per-column
680
+ // NOCASE/RTRIM collation, matching write-time enforcement.
681
+ const indexColIndices = indexSchema.columns.map(col => col.index);
682
+ const indexNormalizers = seen
683
+ ? indexSchema.columns.map(col => resolveKeyNormalizer(col.collation ?? tableSchema.columns[col.index].collation))
684
+ : undefined;
322
685
  // Scan all data rows
323
686
  const bounds = buildFullScanBounds();
324
687
  const batch = indexStore.batch();
@@ -332,11 +695,10 @@ export class StoreModule {
332
695
  // Extract index column values
333
696
  const indexValues = indexSchema.columns.map(col => row[col.index]);
334
697
  if (seen) {
335
- // SQL UNIQUE allows multiple NULLs: skip dup detection when any
336
- // indexed column is NULL for this row.
337
- const hasNull = indexValues.some(v => v === null);
338
- if (!hasNull) {
339
- const keySig = JSON.stringify(indexValues);
698
+ // serializeRowKey returns null when any indexed column is NULL
699
+ // SQL UNIQUE allows multiple NULLs, so those rows never collide.
700
+ const keySig = serializeRowKey(row, indexColIndices, indexNormalizers);
701
+ if (keySig !== null) {
340
702
  if (seen.has(keySig)) {
341
703
  const colNames = indexSchema.columns
342
704
  .map(c => tableSchema.columns[c.index]?.name ?? String(c.index))
@@ -347,31 +709,85 @@ export class StoreModule {
347
709
  }
348
710
  }
349
711
  // Build and store index key
350
- const indexKey = buildIndexKey(indexValues, pkValues, encodeOptions, indexDirections, pkDirections);
712
+ const indexKey = buildIndexKey(indexValues, pkValues, encodeOptions, indexDirections, pkDirections, pkCollations);
351
713
  batch.put(indexKey, new Uint8Array(0)); // Index value is empty
352
714
  }
353
715
  await batch.write();
354
716
  }
717
+ /**
718
+ * Clear and rebuild every secondary index of `schema` against the (already
719
+ * re-encoded) data store. Secondary-index keys embed the PK suffix, so they must
720
+ * be rewritten whenever the data-store PK key bytes change — which happens both
721
+ * on `ALTER PRIMARY KEY` (the PK columns change) and on an `ALTER COLUMN … SET
722
+ * COLLATE` on a PK member (a PK column's key collation changes). Shared by both
723
+ * arms so the clear-then-rebuild stays identical. `schema` must already be the
724
+ * post-ALTER schema (its `primaryKeyDefinition` + column collations drive the new
725
+ * PK-suffix encoding via {@link buildIndexEntries}).
726
+ */
727
+ async rebuildSecondaryIndexes(schemaName, tableName, tableKey, table, schema) {
728
+ const keyCollation = (table.getConfig().collation || 'NOCASE').toUpperCase();
729
+ const dataStore = await this.getStore(tableKey, table.getConfig());
730
+ for (const indexSchema of schema.indexes ?? []) {
731
+ const indexStore = await this.getIndexStore(schemaName, tableName, indexSchema.name);
732
+ const clearBatch = indexStore.batch();
733
+ for await (const entry of indexStore.iterate(buildFullScanBounds())) {
734
+ clearBatch.delete(entry.key);
735
+ }
736
+ await clearBatch.write();
737
+ await this.buildIndexEntries(dataStore, indexStore, schema, indexSchema, keyCollation);
738
+ }
739
+ }
740
+ /**
741
+ * Validates the existing rows in `dataStore` against a UNIQUE constraint,
742
+ * throwing `CONSTRAINT` on the first duplicate before any schema mutation.
743
+ * Used by `ADD CONSTRAINT UNIQUE` (validate against the current collation) and
744
+ * by `SET COLLATE` (pass an `updatedSchema` whose altered column carries the
745
+ * NEW collation, so the dedup is performed under it). Mirrors the duplicate
746
+ * detection in {@link buildIndexEntries}: a `seen` Set keyed on a per-column
747
+ * collation-aware signature of the constrained values, with SQL NULL semantics
748
+ * (a row with any NULL constrained value never counts as a duplicate) and the
749
+ * partial `predicate` honored.
750
+ *
751
+ * No index store is written — store UNIQUE enforcement is a full-scan over
752
+ * `uniqueConstraints` at write time. The signature is built by
753
+ * {@link serializeRowKey} with one normalizer per constrained column drawn from
754
+ * `tableSchema.columns[idx].collation`, so a per-column NOCASE/RTRIM collation
755
+ * is honored (matching write-time `compareSqlValues` enforcement). Residual: a
756
+ * custom comparator-only collation has no string normalizer and falls back to
757
+ * BINARY for the dedup (see docs/schema.md store-collation note).
758
+ */
759
+ async validateUniqueOverExistingRows(dataStore, tableSchema, uc) {
760
+ const predicate = uc.predicate
761
+ ? compilePredicate(uc.predicate, tableSchema.columns)
762
+ : undefined;
763
+ const normalizers = uc.columns.map(idx => resolveKeyNormalizer(tableSchema.columns[idx].collation));
764
+ const seen = new Set();
765
+ for await (const entry of dataStore.iterate(buildFullScanBounds())) {
766
+ const row = deserializeRow(entry.value);
767
+ // Partial constraint: only rows the predicate unambiguously accepts count.
768
+ if (predicate && predicate.evaluate(row) !== true)
769
+ continue;
770
+ // serializeRowKey returns null when any constrained column is NULL —
771
+ // SQL UNIQUE allows multiple NULLs, so those rows never collide.
772
+ const keySig = serializeRowKey(row, uc.columns, normalizers);
773
+ if (keySig === null)
774
+ continue;
775
+ if (seen.has(keySig)) {
776
+ const colNames = uc.columns.map(i => tableSchema.columns[i]?.name ?? String(i)).join(', ');
777
+ throw new QuereusError(`UNIQUE constraint failed: ${tableSchema.name} (${colNames})`, StatusCode.CONSTRAINT);
778
+ }
779
+ seen.add(keySig);
780
+ }
781
+ }
355
782
  /**
356
783
  * Alters an existing store table's structure (ADD/DROP/RENAME COLUMN).
357
784
  * Performs eager row migration for ADD and DROP, schema-only update for RENAME.
358
785
  * Returns the updated TableSchema for the engine to register.
359
786
  */
360
787
  async alterTable(db, schemaName, tableName, change) {
788
+ this.ensureSchemaSubscription(db);
361
789
  const tableKey = `${schemaName}.${tableName}`.toLowerCase();
362
- // Lazy-connect: `renameTable` evicts the old key from `this.tables` and
363
- // expects the next `connect()` to repopulate under the new name, but
364
- // `apply schema` can call `alterTable` immediately after a rename without
365
- // an intervening connect. Mirror connect()'s schemaManager lookup so the
366
- // follow-up ALTER finds the moved table.
367
- let table = this.tables.get(tableKey);
368
- if (!table) {
369
- const registeredSchema = db.schemaManager.getTable(schemaName, tableName);
370
- if (registeredSchema) {
371
- table = new StoreTable(db, this, registeredSchema, this.parseConfig(registeredSchema.vtabArgs ?? {}), this.eventEmitter, true);
372
- this.tables.set(tableKey, table);
373
- }
374
- }
790
+ const table = this.getOrReconnectTable(db, schemaName, tableName);
375
791
  if (!table) {
376
792
  throw new QuereusError(`Store table '${tableName}' not found in schema '${schemaName}'. Cannot alter.`, StatusCode.ERROR);
377
793
  }
@@ -379,7 +795,11 @@ export class StoreModule {
379
795
  const defaultNotNull = db.options.getStringOption('default_column_nullability') === 'not_null';
380
796
  switch (change.type) {
381
797
  case 'addColumn': {
382
- const newColSchema = columnDefToSchema(change.columnDef, defaultNotNull);
798
+ // Honor the session `default_collation` for an ADD COLUMN that omits an
799
+ // explicit COLLATE, matching the CREATE path so an ADD-COLUMN-ed text column
800
+ // gets the same collation a CREATE-d one would. The persisted DDL re-emits an
801
+ // explicit COLLATE for any non-BINARY collation, so reopen stays stable.
802
+ const newColSchema = columnDefToSchema(change.columnDef, defaultNotNull, db.options.getStringOption('default_collation'));
383
803
  // Extract default value from column def constraints. Use the shared
384
804
  // `tryFoldLiteral` helper so signed numerics like `-123.0`
385
805
  // (a UnaryExpr in the AST) are recognized — matching the
@@ -392,8 +812,13 @@ export class StoreModule {
392
812
  defaultValue = folded;
393
813
  }
394
814
  }
395
- // Refuse NOT NULL without a literal DEFAULT on a non-empty table (SQLite-compatible).
396
- if (newColSchema.notNull && defaultValue === null) {
815
+ // A non-foldable DEFAULT (e.g. `new.<col>`) backfills each existing row from
816
+ // its own value via the engine-supplied evaluator (mirrors the memory path).
817
+ const backfillEvaluator = change.backfillEvaluator;
818
+ // Refuse NOT NULL without a usable DEFAULT on a non-empty table
819
+ // (SQLite-compatible). A per-row evaluator IS usable — its NOT NULL is enforced
820
+ // per row during migration — so it is exempt from this no-default rejection.
821
+ if (newColSchema.notNull && defaultValue === null && !backfillEvaluator) {
397
822
  if (await table.hasAnyRows()) {
398
823
  throw new QuereusError(`Cannot add NOT NULL column '${newColSchema.name}' to non-empty table `
399
824
  + `'${schemaName}.${tableName}' without a DEFAULT value`, StatusCode.CONSTRAINT);
@@ -406,12 +831,46 @@ export class StoreModule {
406
831
  columns: updatedColumns,
407
832
  columnIndexMap: buildColumnIndexMap(updatedColumns),
408
833
  };
409
- // Migrate rows: append default value to each row
834
+ // Extract any column-level CHECK / FK to persist (see the persist block below).
835
+ // Hoisted above the row migration so a malformed constraint (e.g. a multi-column
836
+ // FK on a single ADD COLUMN, which `extractColumnLevelForeignKeys` rejects) throws
837
+ // BEFORE any rows are migrated or the in-memory schema is swapped — validate-before-
838
+ // mutate, matching the engine's ordering in `runAddColumn`.
839
+ const newCheckConstraints = extractColumnLevelCheckConstraints(change.columnDef);
840
+ const newForeignKeys = extractColumnLevelForeignKeys(change.columnDef, schemaName);
841
+ // Migrate rows: append the new column's value — a single literal default, or a
842
+ // per-row value derived from the existing row when a backfill evaluator is set.
410
843
  const remap = buildColumnRemap(oldSchema.columns.map(c => c.name), updatedColumns.map(c => c.name));
411
- await table.migrateRows(remap, defaultValue);
412
- // Update table schema and persist DDL
844
+ await table.migrateRows(remap, defaultValue, backfillEvaluator
845
+ ? { evaluator: backfillEvaluator, notNull: newColSchema.notNull, columnName: newColSchema.name }
846
+ : undefined);
847
+ // Update table schema (column-only) and persist DDL.
848
+ //
849
+ // The engine's `runAddColumn` re-merges the column-level FK/CHECK extracted
850
+ // from `columnDef.constraints` into the LIVE in-memory schema AFTER this hook
851
+ // returns, so the schema handed back to it must stay column-only — returning a
852
+ // constrained schema would double the constraint in the live SchemaManager (and,
853
+ // on the next persist, in the DDL). But that engine-side merge is in-memory only:
854
+ // it never reaches the catalog, so persistence must carry the column-level
855
+ // CHECK/FK itself or they vanish on `rehydrateCatalog`. Build a separate
856
+ // `persistedSchema` for `saveTableDDL` when (and only when) the column declares
857
+ // such a constraint; the common path persists `updatedSchema` unchanged. This is
858
+ // unconditional on the default kind — a per-row (evaluator) DEFAULT extracts the
859
+ // same AST constraints as a literal one.
413
860
  table.updateSchema(updatedSchema);
414
- await this.saveTableDDL(updatedSchema);
861
+ let persistedSchema = updatedSchema;
862
+ if (newCheckConstraints.length > 0 || newForeignKeys.length > 0) {
863
+ // The new column is appended last; resolve each FK's child column to its index
864
+ // (matching how the engine resolves `resolvedForeignKeys` via columnIndexMap).
865
+ const newColIdx = updatedColumns.length - 1;
866
+ const resolvedForeignKeys = newForeignKeys.map(fk => ({ ...fk, columns: Object.freeze([newColIdx]) }));
867
+ persistedSchema = {
868
+ ...updatedSchema,
869
+ checkConstraints: Object.freeze([...updatedSchema.checkConstraints, ...newCheckConstraints]),
870
+ foreignKeys: Object.freeze([...(updatedSchema.foreignKeys ?? []), ...resolvedForeignKeys]),
871
+ };
872
+ }
873
+ await this.saveTableDDL(persistedSchema);
415
874
  this.eventEmitter?.emitSchemaChange({
416
875
  type: 'alter',
417
876
  objectType: 'table',
@@ -444,12 +903,26 @@ export class StoreModule {
444
903
  .map(ic => ({ ...ic, index: ic.index > colIndex ? ic.index - 1 : ic.index })),
445
904
  }))
446
905
  .filter(idx => idx.columns.length > 0);
906
+ // Prune any UNIQUE constraint over the dropped column, mirroring the index
907
+ // filtering above. Store-backed UNIQUE is enforced by a full scan over
908
+ // `uniqueConstraints`, so a stranded constraint whose column index dangles past
909
+ // the column array would break the next insert's validation (and the persisted
910
+ // DDL). A UNIQUE that includes the dropped column is removed outright; remaining
911
+ // constraints have their column indices shifted to track the removed slot. This
912
+ // also covers the engine's ADD COLUMN + inline-UNIQUE revert, which drops the
913
+ // just-added (uniquely-constrained) column.
914
+ const updatedUniqueConstraints = (oldSchema.uniqueConstraints ?? [])
915
+ .filter(uc => !uc.columns.includes(colIndex))
916
+ .map(uc => ({ ...uc, columns: Object.freeze(uc.columns.map(i => i > colIndex ? i - 1 : i)) }));
447
917
  const updatedSchema = {
448
918
  ...oldSchema,
449
919
  columns: Object.freeze(updatedColumns),
450
920
  columnIndexMap: buildColumnIndexMap(updatedColumns),
451
921
  primaryKeyDefinition: Object.freeze(updatedPkDef),
452
922
  indexes: Object.freeze(updatedIndexes),
923
+ uniqueConstraints: updatedUniqueConstraints.length > 0
924
+ ? Object.freeze(updatedUniqueConstraints)
925
+ : undefined,
453
926
  };
454
927
  // Migrate rows: remove the dropped column slot
455
928
  const remap = buildColumnRemap(oldSchema.columns.map(c => c.name), updatedColumns.map(c => c.name));
@@ -509,16 +982,7 @@ export class StoreModule {
509
982
  await table.rekeyRows(newPkColumns);
510
983
  // Secondary index keys embed the PK suffix — clear + rebuild every
511
984
  // index against the now-rekeyed data store.
512
- const dataStore = await this.getStore(tableKey, table.getConfig());
513
- for (const indexSchema of oldSchema.indexes ?? []) {
514
- const indexStore = await this.getIndexStore(schemaName, tableName, indexSchema.name);
515
- const clearBatch = indexStore.batch();
516
- for await (const entry of indexStore.iterate(buildFullScanBounds())) {
517
- clearBatch.delete(entry.key);
518
- }
519
- await clearBatch.write();
520
- await this.buildIndexEntries(dataStore, indexStore, updatedSchema, indexSchema);
521
- }
985
+ await this.rebuildSecondaryIndexes(schemaName, tableName, tableKey, table, updatedSchema);
522
986
  table.updateSchema(updatedSchema);
523
987
  await this.saveTableDDL(updatedSchema);
524
988
  this.eventEmitter?.emitSchemaChange({
@@ -530,7 +994,117 @@ export class StoreModule {
530
994
  return updatedSchema;
531
995
  }
532
996
  case 'addConstraint': {
533
- throw new QuereusError(`Store table does not support ADD CONSTRAINT ${change.constraint.type}`, StatusCode.UNSUPPORTED);
997
+ const constraint = change.constraint;
998
+ let updatedSchema;
999
+ if (constraint.type === 'unique') {
1000
+ // Store enforces inline UNIQUE by full-scan over `uniqueConstraints`
1001
+ // (no separate index store), so there is nothing physical to build —
1002
+ // but we must validate the existing rows before persisting.
1003
+ const uc = buildUniqueConstraintSchema(constraint, oldSchema.columnIndexMap);
1004
+ const dataStore = await this.getStore(tableKey, table.getConfig());
1005
+ await this.validateUniqueOverExistingRows(dataStore, oldSchema, uc);
1006
+ updatedSchema = {
1007
+ ...oldSchema,
1008
+ uniqueConstraints: Object.freeze([...(oldSchema.uniqueConstraints ?? []), uc]),
1009
+ };
1010
+ }
1011
+ else if (constraint.type === 'foreignKey') {
1012
+ const fk = buildForeignKeyConstraintSchema(constraint, oldSchema.columnIndexMap, oldSchema.name, oldSchema.schemaName);
1013
+ updatedSchema = {
1014
+ ...oldSchema,
1015
+ foreignKeys: Object.freeze([...(oldSchema.foreignKeys ?? []), fk]),
1016
+ };
1017
+ // Pragma-gated existing-row validation; throws before persistence on an orphan.
1018
+ await validateForeignKeyOverExistingRows(db, updatedSchema, fk);
1019
+ }
1020
+ else if (constraint.type === 'check') {
1021
+ // Schema-only: a CHECK has no physical structure and (matching the
1022
+ // engine's prior in-emitter behavior) no existing-row scan. Routing it
1023
+ // here — rather than catalog-only — keeps the persisted DDL and the
1024
+ // connected-table schema in lock-step so DROP/RENAME CONSTRAINT resolve it.
1025
+ const check = buildCheckConstraintSchema(constraint, oldSchema.checkConstraints.length);
1026
+ updatedSchema = {
1027
+ ...oldSchema,
1028
+ checkConstraints: Object.freeze([...oldSchema.checkConstraints, check]),
1029
+ };
1030
+ }
1031
+ else {
1032
+ throw new QuereusError(`Store table ADD CONSTRAINT does not support constraint type '${constraint.type}'`, StatusCode.UNSUPPORTED);
1033
+ }
1034
+ table.updateSchema(updatedSchema);
1035
+ await this.saveTableDDL(updatedSchema);
1036
+ this.eventEmitter?.emitSchemaChange({
1037
+ type: 'alter',
1038
+ objectType: 'table',
1039
+ schemaName,
1040
+ objectName: tableName,
1041
+ });
1042
+ return updatedSchema;
1043
+ }
1044
+ case 'dropConstraint': {
1045
+ // Schema-only catalog rewrite: store-backed UNIQUE enforcement is a
1046
+ // full-scan over `uniqueConstraints` (no separate index store for an
1047
+ // inline UNIQUE), so dropping the constraint stops enforcement with no
1048
+ // physical teardown. A UNIQUE derived from a CREATE UNIQUE INDEX is
1049
+ // rejected upstream (drop the index instead), so we never strand a store.
1050
+ const constraintClass = resolveNamedConstraintClass(oldSchema, change.constraintName);
1051
+ const lower = change.constraintName.toLowerCase();
1052
+ let updatedSchema;
1053
+ if (constraintClass === 'check') {
1054
+ updatedSchema = {
1055
+ ...oldSchema,
1056
+ checkConstraints: Object.freeze(oldSchema.checkConstraints.filter(c => c.name?.toLowerCase() !== lower)),
1057
+ };
1058
+ }
1059
+ else if (constraintClass === 'foreignKey') {
1060
+ const remaining = (oldSchema.foreignKeys ?? []).filter(c => c.name?.toLowerCase() !== lower);
1061
+ updatedSchema = { ...oldSchema, foreignKeys: remaining.length > 0 ? Object.freeze(remaining) : undefined };
1062
+ }
1063
+ else {
1064
+ const remaining = (oldSchema.uniqueConstraints ?? []).filter(c => c.name?.toLowerCase() !== lower);
1065
+ updatedSchema = { ...oldSchema, uniqueConstraints: remaining.length > 0 ? Object.freeze(remaining) : undefined };
1066
+ }
1067
+ table.updateSchema(updatedSchema);
1068
+ await this.saveTableDDL(updatedSchema);
1069
+ this.eventEmitter?.emitSchemaChange({
1070
+ type: 'alter',
1071
+ objectType: 'table',
1072
+ schemaName,
1073
+ objectName: tableName,
1074
+ });
1075
+ return updatedSchema;
1076
+ }
1077
+ case 'renameConstraint': {
1078
+ const constraintClass = resolveNamedConstraintClass(oldSchema, change.oldName);
1079
+ const oldLower = change.oldName.toLowerCase();
1080
+ let updatedSchema;
1081
+ if (constraintClass === 'check') {
1082
+ updatedSchema = {
1083
+ ...oldSchema,
1084
+ checkConstraints: Object.freeze(oldSchema.checkConstraints.map(c => (c.name?.toLowerCase() === oldLower ? { ...c, name: change.newName } : c))),
1085
+ };
1086
+ }
1087
+ else if (constraintClass === 'foreignKey') {
1088
+ updatedSchema = {
1089
+ ...oldSchema,
1090
+ foreignKeys: Object.freeze(oldSchema.foreignKeys.map(c => (c.name?.toLowerCase() === oldLower ? { ...c, name: change.newName } : c))),
1091
+ };
1092
+ }
1093
+ else {
1094
+ updatedSchema = {
1095
+ ...oldSchema,
1096
+ uniqueConstraints: Object.freeze(oldSchema.uniqueConstraints.map(c => (c.name?.toLowerCase() === oldLower ? { ...c, name: change.newName } : c))),
1097
+ };
1098
+ }
1099
+ table.updateSchema(updatedSchema);
1100
+ await this.saveTableDDL(updatedSchema);
1101
+ this.eventEmitter?.emitSchemaChange({
1102
+ type: 'alter',
1103
+ objectType: 'table',
1104
+ schemaName,
1105
+ objectName: tableName,
1106
+ });
1107
+ return updatedSchema;
534
1108
  }
535
1109
  case 'alterColumn': {
536
1110
  const colNameLower = change.columnName.toLowerCase();
@@ -540,6 +1114,9 @@ export class StoreModule {
540
1114
  }
541
1115
  const oldCol = oldSchema.columns[colIndex];
542
1116
  let newCol = oldCol;
1117
+ // A collation change needs existing-row UNIQUE re-validation below
1118
+ // (non-PK, Option A); the other attribute changes do not.
1119
+ let collationChanged = false;
543
1120
  // Pull exactly one of the three attributes from the change.
544
1121
  if (change.setNotNull !== undefined) {
545
1122
  if (change.setNotNull === true && !oldCol.notNull) {
@@ -589,15 +1166,94 @@ export class StoreModule {
589
1166
  else if (change.setDefault !== undefined) {
590
1167
  newCol = { ...oldCol, defaultValue: change.setDefault };
591
1168
  }
1169
+ else if (change.setCollation !== undefined) {
1170
+ // Per-column collation update. PRIMARY KEY uniqueness/ordering is enforced
1171
+ // PHYSICALLY in the key bytes under a PER-COLUMN key collation
1172
+ // (`StoreTable.pkKeyCollations`), so a PK-column SET COLLATE is honored
1173
+ // natively by physically re-keying the data store + rebuilding every
1174
+ // secondary index under the new collation (the `isPkColumn` block below),
1175
+ // mirroring the memory module's primary re-key. A re-key that would collide
1176
+ // under the new collation throws CONSTRAINT before any mutation. For non-PK
1177
+ // UNIQUE constraints we re-validate existing rows under the new collation
1178
+ // (Option A). Query-layer ORDER BY / `=` / `table_info().collation` pick the
1179
+ // new collation up from the column schema once this updated schema re-registers.
1180
+ const normalized = validateCollationForType(change.setCollation, oldCol.logicalType, change.columnName);
1181
+ const nameMatches = normalized === (oldCol.collation || 'BINARY');
1182
+ if (nameMatches && oldCol.collationExplicit) {
1183
+ return oldSchema; // already explicit in the desired collation — no scan, no re-key, no re-persist
1184
+ }
1185
+ // SET COLLATE is a user declaration with the same standing as a
1186
+ // CREATE-time COLLATE clause, so mark the collation explicit (rank 2
1187
+ // in the comparison lattice) regardless of the column's creation
1188
+ // history — including SET COLLATE binary. When only the name matches
1189
+ // but the column was not yet explicit (a defaulted collation, or one
1190
+ // inherited from session default_collation), flip the flag as a
1191
+ // METADATA-ONLY change: the collation bytes are unchanged, so keep
1192
+ // collationChanged false to skip rekeyRows / validateUniqueOverExistingRows
1193
+ // below while still re-registering the schema and re-persisting DDL.
1194
+ // A different name takes the full physical re-key path AND sets the flag.
1195
+ newCol = { ...oldCol, collation: normalized, collationExplicit: true };
1196
+ collationChanged = !nameMatches;
1197
+ }
592
1198
  else {
593
1199
  throw new QuereusError('ALTER COLUMN requires an attribute to change', StatusCode.INTERNAL);
594
1200
  }
595
1201
  const updatedColumns = oldSchema.columns.map((c, i) => i === colIndex ? newCol : c);
1202
+ // Mirror the memory module (MemoryTableManager.alterColumn): a per-column
1203
+ // collation change propagates into every index column ordering by this
1204
+ // column, so a `derivedFromIndex` UNIQUE re-keys its enforcement under the
1205
+ // new collation. StoreTable.uniqueEnforcementCollations reads the index's
1206
+ // per-column collation, so without this the index entry would stay stale
1207
+ // and the derived UNIQUE would keep enforcing the OLD collation after the
1208
+ // ALTER. Metadata-only: the store's index KEY bytes use the table-level key
1209
+ // collation K (see buildIndexEntries / updateSecondaryIndexes), so no index
1210
+ // entry re-encode is required for a non-PK column. An index column with an
1211
+ // explicit COLLATE is re-collated too — matching memory, which clobbers it
1212
+ // the same way (no surface preserves a differing index COLLATE across an
1213
+ // ALTER COLUMN SET COLLATE on its column).
1214
+ const updatedIndexes = (collationChanged && oldSchema.indexes)
1215
+ ? oldSchema.indexes.map(idx => ({
1216
+ ...idx,
1217
+ columns: idx.columns.map(ic => ic.index === colIndex ? { ...ic, collation: newCol.collation } : ic),
1218
+ }))
1219
+ : oldSchema.indexes;
596
1220
  const updatedSchema = {
597
1221
  ...oldSchema,
598
1222
  columns: Object.freeze(updatedColumns),
599
1223
  columnIndexMap: buildColumnIndexMap(updatedColumns),
1224
+ indexes: updatedIndexes ? Object.freeze(updatedIndexes) : updatedIndexes,
600
1225
  };
1226
+ // SET COLLATE existing-row re-validation (Option A, non-PK UNIQUE): a new
1227
+ // per-column collation can make rows that were distinct under the old
1228
+ // collation collide. Re-scan every UNIQUE constraint covering the altered
1229
+ // column under the NEW collation (`updatedSchema` carries it). The first
1230
+ // collision throws CONSTRAINT BEFORE any mutation/persist, so the table is
1231
+ // left unchanged and writable (matches the ADD CONSTRAINT rollback shape).
1232
+ // The PK is intentionally excluded — it never appears in `uniqueConstraints`;
1233
+ // its physical re-key/re-validation is the `isPkColumn` block below.
1234
+ if (collationChanged) {
1235
+ const coveringConstraints = (updatedSchema.uniqueConstraints ?? [])
1236
+ .filter(uc => uc.columns.includes(colIndex));
1237
+ if (coveringConstraints.length > 0) {
1238
+ const dataStore = await this.getStore(tableKey, table.getConfig());
1239
+ for (const uc of coveringConstraints) {
1240
+ await this.validateUniqueOverExistingRows(dataStore, updatedSchema, uc);
1241
+ }
1242
+ }
1243
+ }
1244
+ // SET COLLATE on a PRIMARY KEY member (Option B physical re-key): re-encode
1245
+ // every data-store key under the column's new key collation, then rebuild every
1246
+ // secondary index (its keys embed the PK suffix). `rekeyRows` validates in a
1247
+ // first pass and throws CONSTRAINT on a collision under the new collation WITHOUT
1248
+ // mutating the store — so a coarser collation that collapses two distinct PKs
1249
+ // (e.g. 'a'/'A' under BINARY→NOCASE) is rejected all-or-nothing, mirroring
1250
+ // ALTER PRIMARY KEY. Runs AFTER the non-PK UNIQUE re-validation above so both
1251
+ // throw-only checks precede the first store mutation. `updatedSchema.columns`
1252
+ // carries the new collation, so the new key bytes follow it.
1253
+ if (collationChanged && oldSchema.primaryKeyDefinition.some(def => def.index === colIndex)) {
1254
+ await table.rekeyRows(oldSchema.primaryKeyDefinition, updatedSchema.columns);
1255
+ await this.rebuildSecondaryIndexes(schemaName, tableName, tableKey, table, updatedSchema);
1256
+ }
601
1257
  table.updateSchema(updatedSchema);
602
1258
  await this.saveTableDDL(updatedSchema);
603
1259
  this.eventEmitter?.emitSchemaChange({
@@ -626,25 +1282,54 @@ export class StoreModule {
626
1282
  if (this.tables.has(newKey)) {
627
1283
  throw new QuereusError(`Store table '${newName}' already exists in schema '${schemaName}'`, StatusCode.ERROR);
628
1284
  }
629
- // Capture the current schema BEFORE we drop in-memory references, so the
630
- // new catalog DDL reflects the real column set.
1285
+ // Capture the current schema BEFORE the guard (and before we drop in-memory
1286
+ // references): the guard needs the index list to compute every relocated
1287
+ // store name, and the new catalog DDL must reflect the real column set.
631
1288
  const existing = this.tables.get(oldKey);
632
1289
  const currentSchema = existing?.getSchema() ?? db.schemaManager.getTable(schemaName, oldName);
1290
+ // Authoritative index list (exact store names): the provider relocates
1291
+ // exactly these index stores instead of prefix-scanning `{oldName}_idx_`,
1292
+ // which would also catch a sibling table named `{oldName}_idx_<x>`.
1293
+ const indexNames = (currentSchema?.indexes ?? []).map(i => i.name);
1294
+ // Reject when ANY physical name the rename introduces — the new data store
1295
+ // AND each relocated index store `{schema}.{newName}_idx_{x}` — already
1296
+ // names an existing store. E.g. rename some table to `q_idx_archive` while
1297
+ // table `q` has index `archive` (both → `{schema}.q_idx_archive`); or rename
1298
+ // `t`→`u` while `t` has index `x` and a sibling table is literally named
1299
+ // `u_idx_x`, which would relocate `t`'s index onto the sibling's data store.
1300
+ // The renamed table's own current stores stay in the occupied set (see
1301
+ // collectOccupiedStoreNames): an introduced name can only equal an own store
1302
+ // in a footprint-swap rename providers cannot relocate safely. All checks
1303
+ // run before the FIRST side effect (the coordinator commit, disconnect, and
1304
+ // cache evictions below, then the physical relocation) so a colliding
1305
+ // rename is a clean no-op.
1306
+ const occupied = this.collectOccupiedStoreNames(db, schemaName);
1307
+ this.assertStoreNameFree(db, schemaName, buildDataStoreName(schemaName, newName), `data store of table '${schemaName}.${newName}' (rename target)`, occupied);
1308
+ for (const indexName of indexNames) {
1309
+ this.assertStoreNameFree(db, schemaName, buildIndexStoreName(schemaName, newName, indexName), `index store of index '${indexName}' on table '${schemaName}.${newName}' (rename target)`, occupied);
1310
+ }
633
1311
  // ALTER TABLE is effectively DDL-committing on a store-backed table:
634
1312
  // once we move the on-disk directory, prior buffered writes can no
635
1313
  // longer be rolled back through the coordinator. Flush any pending
636
- // ops to the old store NOW, before its handle is closed. Subsequent
1314
+ // ops NOW, before the old store's handle is closed. Subsequent
637
1315
  // commit() calls on the same coordinator are no-ops (inTransaction
638
1316
  // is cleared), which keeps the enclosing transaction safe.
639
- const coordinator = this.coordinators.get(oldKey);
640
- if (coordinator?.isInTransaction()) {
641
- await coordinator.commit();
1317
+ //
1318
+ // The coordinator is module-wide, so this DDL-commits the WHOLE module
1319
+ // transaction — every table's pending ops, not just the renamed table's —
1320
+ // in one all-or-nothing batch. That is the correct, consistent posture
1321
+ // for a store DDL-commit: an ALTER cannot half-commit some sibling tables.
1322
+ if (this.moduleCoordinator?.isInTransaction()) {
1323
+ await this.moduleCoordinator.commit();
642
1324
  }
643
- // Flush any lazy stats the cached handle was buffering; disconnect failures
644
- // must not block the rename.
1325
+ // Hard-dispose the evicted handle: flush any lazy stats it was buffering AND
1326
+ // deregister its coordinator stats-callback pair (the renamed instance is
1327
+ // gone after this — the next connect()/getOrReconnectTable mints a fresh one
1328
+ // that re-registers against the shared coordinator). Dispose failures must
1329
+ // not block the physical rename.
645
1330
  if (existing) {
646
1331
  try {
647
- await existing.disconnect();
1332
+ await existing.dispose();
648
1333
  }
649
1334
  catch {
650
1335
  /* ignore — physical rename must proceed */
@@ -652,10 +1337,20 @@ export class StoreModule {
652
1337
  }
653
1338
  this.tables.delete(oldKey);
654
1339
  this.stores.delete(oldKey);
655
- this.coordinators.delete(oldKey);
1340
+ // The coordinator is module-wide (flushed above); it is not per-table, so
1341
+ // it is not evicted here.
1342
+ // Evict the disposed instance's registered engine connection. It is bound to
1343
+ // the OLD qualified name and its owning StoreTable is now disposed, so it is
1344
+ // definitively stale. Unlike drop — where the engine's schema manager calls
1345
+ // `removeConnectionsForTable` for us — the generic rename path
1346
+ // (`alter-table.ts` renameTableImpl) does NOT, so the store must evict it
1347
+ // here or the connection leaks one per rename. Safe because the module
1348
+ // DDL-commit above already flushed its pending ops (no uncommitted writes to
1349
+ // lose).
1350
+ db.removeConnectionsForTable(schemaName, oldName);
656
1351
  // Move physical storage (data directory + index directories).
657
1352
  if (this.provider.renameTableStores) {
658
- await this.provider.renameTableStores(schemaName, oldName, newName);
1353
+ await this.provider.renameTableStores(schemaName, oldName, newName, indexNames);
659
1354
  }
660
1355
  // Rewrite persistent catalog under the new name. Write the new DDL first
661
1356
  // so a crash mid-rename leaves the table discoverable under at least one
@@ -683,8 +1378,25 @@ export class StoreModule {
683
1378
  }
684
1379
  /**
685
1380
  * Modern access planning interface.
1381
+ *
1382
+ * Every plan is stamped with `honorsCollatedRangeBounds`: the store's post-fetch
1383
+ * row filter (`StoreTable.matchesFilters` → `compareValues`) compares each pushed
1384
+ * constraint — including LT/LE/GT/GE range bounds — under the column's declared
1385
+ * collation, so the access path's collation-cover analysis may keep a
1386
+ * collation-matched non-BINARY (NOCASE/RTRIM) PK range/BETWEEN seek instead of
1387
+ * declining to a SeqScan + residual (see `classifyConstraintCover` in
1388
+ * rule-select-access-path.ts). The seek really does narrow: `StoreTable.scanPKRange`
1389
+ * (via `StoreTable.buildPKRangeBounds`) encodes the LT/LE/GT/GE bounds under the
1390
+ * same per-column key collations the data keys use and iterates that
1391
+ * seek-start/early-termination window. The window is a SUPERSET, so the post-fetch
1392
+ * row filter still reproduces the exact collation semantics — and a comparator-only
1393
+ * collation with no byte encoder safely falls back to a full scan. Mirrors the
1394
+ * memory module's advertisement.
686
1395
  */
687
1396
  getBestAccessPlan(_db, tableInfo, request) {
1397
+ return { ...this.computeBestAccessPlan(tableInfo, request), honorsCollatedRangeBounds: true };
1398
+ }
1399
+ computeBestAccessPlan(tableInfo, request) {
688
1400
  const estimatedRows = request.estimatedRows ?? 1000;
689
1401
  // Check for primary key equality constraints
690
1402
  const pkColumns = tableInfo.primaryKeyDefinition.map(pk => pk.index);
@@ -716,9 +1428,9 @@ export class StoreModule {
716
1428
  if (rangeFilters.length > 0) {
717
1429
  // Range scan on first PK column. Iteration is by PK key order (see
718
1430
  // StoreTable.scanPKRange), so we can advertise monotonic emission on
719
- // the leading PK column. The scan still visits the entire data store
720
- // today (TODO in scanPKRange to refine bounds), but the order
721
- // guarantee already holds.
1431
+ // the leading PK column. The scan seeks to the window start and
1432
+ // early-terminates (StoreTable.buildPKRangeBounds derives the encoded
1433
+ // bounds), and the leading-PK order guarantee holds throughout.
722
1434
  const handledFilters = request.filters.map(f => rangeFilters.some(rf => rf.columnIndex === f.columnIndex && rf.op === f.op));
723
1435
  const rangeRows = Math.max(1, Math.floor(estimatedRows * 0.3));
724
1436
  const plan = AccessPlanBuilder
@@ -861,22 +1573,82 @@ export class StoreModule {
861
1573
  return this.provider.getStatsStore(schemaName, tableName);
862
1574
  }
863
1575
  /**
864
- * Get or create a transaction coordinator for a table.
1576
+ * Get (lazily constructing) the SINGLE transaction coordinator shared by every
1577
+ * table this module owns — the unit of cross-table atomicity. Every
1578
+ * {@link StoreTable} attaches to the same instance, so a transaction touching
1579
+ * several of the module's tables commits/rolls back as one all-or-nothing
1580
+ * batch.
1581
+ *
1582
+ * Synchronously constructible: ops are addressed by explicit store handle, so
1583
+ * the coordinator never needs to open a store to be created. This lets callers
1584
+ * that must stay synchronous (e.g. a `BackingHost.connect()`) obtain a working
1585
+ * coordinator before any table's store has been opened.
865
1586
  */
866
- async getCoordinator(tableKey, config) {
867
- let coordinator = this.coordinators.get(tableKey);
868
- if (!coordinator) {
869
- const store = await this.getStore(tableKey, config);
870
- coordinator = new TransactionCoordinator(store, this.eventEmitter);
871
- this.coordinators.set(tableKey, coordinator);
1587
+ getCoordinator() {
1588
+ if (!this.moduleCoordinator) {
1589
+ this.moduleCoordinator = new TransactionCoordinator(this.eventEmitter,
1590
+ // Re-evaluated per commit (see TransactionCoordinator.atomicBatchFactory)
1591
+ // so a provider that gains/loses the capability is always honored.
1592
+ () => this.provider.beginAtomicBatch?.());
872
1593
  }
873
- return coordinator;
1594
+ return this.moduleCoordinator;
874
1595
  }
875
1596
  /**
876
- * Save table DDL to the catalog store.
1597
+ * Build the catalog entry for a table: its CREATE TABLE DDL, one
1598
+ * `CREATE [UNIQUE] INDEX` statement per persistable secondary index, and one
1599
+ * `alter index … set tags (…)` statement per exposed implicit index carrying
1600
+ * user tags, newline-joined into a single multi-statement bundle keyed by
1601
+ * `{schema}.{table}`.
1602
+ *
1603
+ * Bundling the indexes into the table's own entry means every existing
1604
+ * re-persist path — `saveTableDDL` (each `alterTable` arm, `renameTable`) and
1605
+ * the `table_modified` listener (`persistCatalogIfChanged`) — carries the
1606
+ * indexes along for free, and `removeTableDDL` drops them with the table. The
1607
+ * bundle is consumed on reopen by `rehydrateCatalog` → `importCatalog`, whose
1608
+ * `parser.parseAll` splits it AST-by-AST (never on `\n`) and imports each
1609
+ * statement in document order, so the table registers before its indexes.
1610
+ *
1611
+ * Both the table DDL and the index DDL are emitted without a `db` arg, keeping
1612
+ * the persistence-safe fully-qualified form. Hidden implicit covering indexes
1613
+ * (the auto-built BTree backing a declared UNIQUE constraint) are excluded —
1614
+ * they are a backing detail that round-trips via the table's UNIQUE constraint,
1615
+ * not as a standalone CREATE INDEX. For store tables `buildTableSchemaFromAST`
1616
+ * synthesizes none of those, so in practice every index is included; the guard
1617
+ * is defensive. A `CREATE UNIQUE INDEX`'s derived UNIQUE constraint is already
1618
+ * excluded from the table DDL by the generator, so it round-trips solely via
1619
+ * its own `CREATE UNIQUE INDEX` line — no doubling.
1620
+ *
1621
+ * An *exposed* implicit index (a non-derived UNIQUE constraint tagged
1622
+ * `quereus.expose_implicit_index = true`, never materialized in store mode)
1623
+ * must NOT get a synthetic `CREATE INDEX` line — re-import would materialize a
1624
+ * real `IndexSchema` and change the store-mode shape. Its user tags
1625
+ * (`UniqueConstraintSchema.exposedIndexTags`) instead ride a trailing
1626
+ * `alter index … set tags` line, which `importDDL` re-applies silently after
1627
+ * the CREATE TABLE in the same entry has registered the constraint. Empty tag
1628
+ * records emit no line, and `exposedImplicitIndexes` returns `[]` for
1629
+ * memory-mode tables (name materialized) and orders descriptors by the
1630
+ * `uniqueConstraints` array, so the bundle stays byte-deterministic — which
1631
+ * the compare-write in `persistCatalogIfChanged` relies on.
1632
+ */
1633
+ buildCatalogEntry(tableSchema) {
1634
+ const parts = [generateTableDDL(tableSchema)];
1635
+ for (const idx of tableSchema.indexes ?? []) {
1636
+ if (isHiddenImplicitIndex(tableSchema, idx.name))
1637
+ continue;
1638
+ parts.push(generateIndexDDL(idx, tableSchema));
1639
+ }
1640
+ for (const desc of exposedImplicitIndexes(tableSchema)) {
1641
+ if (desc.tags && Object.keys(desc.tags).length > 0) {
1642
+ parts.push(generateIndexTagsDDL(tableSchema.schemaName, desc.name, desc.tags));
1643
+ }
1644
+ }
1645
+ return parts.join('\n');
1646
+ }
1647
+ /**
1648
+ * Save table DDL (bundled with its secondary index DDL) to the catalog store.
877
1649
  */
878
1650
  async saveTableDDL(tableSchema) {
879
- const ddl = generateTableDDL(tableSchema);
1651
+ const ddl = this.buildCatalogEntry(tableSchema);
880
1652
  const catalogKey = buildCatalogKey(tableSchema.schemaName, tableSchema.name);
881
1653
  const encoder = new TextEncoder();
882
1654
  const encodedDDL = encoder.encode(ddl);
@@ -884,8 +1656,12 @@ export class StoreModule {
884
1656
  await catalogStore.put(catalogKey, encodedDDL);
885
1657
  }
886
1658
  /**
887
- * Load all DDL statements from the catalog store.
888
- * Used to restore persisted tables on startup.
1659
+ * Load all DDL **values** from the catalog store (keys discarded).
1660
+ * Used to restore persisted tables on startup and by tests asserting persisted DDL.
1661
+ * Note: this returns table, view, AND materialized-view entries intermixed —
1662
+ * {@link rehydrateCatalog} uses {@link loadCatalogEntries} (keys retained) to
1663
+ * classify them. Meta entries (e.g. the clean-shutdown marker) are not DDL
1664
+ * and are filtered out.
889
1665
  */
890
1666
  async loadAllDDL() {
891
1667
  const catalogStore = await this.provider.getCatalogStore();
@@ -893,42 +1669,286 @@ export class StoreModule {
893
1669
  const decoder = new TextDecoder();
894
1670
  const ddlStatements = [];
895
1671
  for await (const entry of catalogStore.iterate(bounds)) {
1672
+ if (classifyCatalogKey(entry.key) === 'meta')
1673
+ continue;
896
1674
  const ddl = decoder.decode(entry.value);
897
1675
  ddlStatements.push(ddl);
898
1676
  }
899
1677
  return ddlStatements;
900
1678
  }
901
1679
  /**
902
- * Rehydrate persisted catalog into the in-memory schema manager.
1680
+ * Load every catalog entry as `{ key, ddl }`. Unlike {@link loadAllDDL} (values
1681
+ * only) this retains the key so {@link rehydrateCatalog} can classify each entry
1682
+ * (table / view / materialized view) by its reserved key prefix.
1683
+ */
1684
+ async loadCatalogEntries() {
1685
+ const catalogStore = await this.provider.getCatalogStore();
1686
+ const decoder = new TextDecoder();
1687
+ const entries = [];
1688
+ for await (const entry of catalogStore.iterate(buildCatalogScanBounds())) {
1689
+ entries.push({ key: entry.key, ddl: decoder.decode(entry.value) });
1690
+ }
1691
+ return entries;
1692
+ }
1693
+ /**
1694
+ * Rehydrate the persisted catalog into the in-memory schema manager, in
1695
+ * dependency order.
1696
+ *
1697
+ * Establishes the engine schema-change subscription up front (so a reopened DB
1698
+ * persists subsequent DDL even when its first post-reopen statement is a view/MV,
1699
+ * which never routes through a module hook — all the lazy subscription points are
1700
+ * table hooks). Then loads every catalog entry once, classifies each by its key
1701
+ * prefix into {tables, views, materialized views}, and imports in three phases:
903
1702
  *
904
- * Loads all DDL from the catalog store and imports each entry
905
- * individually. Parse failures are collected rather than fatal,
906
- * so a single corrupt entry does not prevent other tables from
907
- * loading.
1703
+ * 1. **Tables** `importCatalog` (connect to existing storage; refresh connected
1704
+ * `StoreTable` schemas).
1705
+ * 2. **Views** `importCatalog` (engine silent-register; body validation deferred
1706
+ * to query time, so order among views — and view-over-MV / view-over-view —
1707
+ * does not matter, and no schema-change event fires → phase 2 writes nothing).
1708
+ * 3. **Materialized views** — `importCatalog` per entry (engine re-materialize:
1709
+ * rebuilds the memory backing from current source data, re-registers row-time
1710
+ * maintenance, re-runs the eligibility gate — the same core the create emitter
1711
+ * uses, but silent: no `materialized_view_added` fires, so phase 3 writes
1712
+ * nothing back to the catalog). A store-hosted backing that phase 1 already
1713
+ * rehydrated is **adopted without the refill** when the engine's adopt gates
1714
+ * pass — see the clean-shutdown marker below.
908
1715
  *
909
- * Call after `db.registerModule()` (and `db.setDefaultVtabName()`
910
- * if DDL may lack a USING clause).
1716
+ * **Clean-shutdown marker.** Before anything loads, the reserved
1717
+ * `\x00meta\x00clean_shutdown` catalog entry (written by {@link closeAll} after
1718
+ * every batch flushed) is consumed: parsed into `{ trusted, staleAtClose }`, then
1719
+ * **deleted immediately** — single-use, so a crash later in this session (or a
1720
+ * second rehydrate without an intervening clean close) is detected at the next open
1721
+ * and every adopt falls back to the always-correct drop+refill, self-healing any
1722
+ * crash-window divergence (coordinated commit is not 2PC across stores). The marker
1723
+ * payload is the JSON set of MVs that were **stale-at-close** (row-time maintenance
1724
+ * detached, so the durable backing may be behind); phase 3 withholds trust per-entry
1725
+ * for those — `trustBackings: trusted && !staleAtClose.has(name)` — so a
1726
+ * stale-at-close MV refills (recomputing content and re-arming maintenance) while
1727
+ * every live-at-close MV keeps the fast path. The one shared `adoptedBackings` set
1728
+ * composes across fixpoint rounds (an upstream MV adopted in round 1 enables its
1729
+ * dependent in round 2, while a refilled — or stale-at-close — upstream is never
1730
+ * added to it, forcing dependents to refill).
1731
+ *
1732
+ * **MV-over-MV ordering** is handled by a fixpoint retry rather than a static topo
1733
+ * sort: an MV's resolved `sourceTables` are computed at import time, not serialized
1734
+ * in the DDL, so they are unavailable before import. Each round passes the names of
1735
+ * every OTHER still-pending MV entry as `pendingDerivations`; the engine defers any
1736
+ * entry whose body reads one (its source already pre-exists as a phase-1 plain
1737
+ * table, so the body would otherwise plan against content the upstream's own import
1738
+ * may be about to replace). The loop repeats while any MV makes progress — robust to
1739
+ * arbitrary nesting depth. A genuinely unbuildable MV — a missing (e.g. memory)
1740
+ * source, or an unresolvable cycle — makes no progress in a round and is recorded in
1741
+ * `errors`.
1742
+ *
1743
+ * Per-entry errors in any phase are collected (not fatal) so one bad object does not
1744
+ * abort the rest.
1745
+ *
1746
+ * Call after `db.registerModule()` (and `db.setDefaultVtabName()` if DDL may lack a
1747
+ * USING clause).
911
1748
  */
912
1749
  async rehydrateCatalog(db) {
913
- const ddlStatements = await this.loadAllDDL();
914
- const result = { tables: [], indexes: [], errors: [] };
915
- if (ddlStatements.length === 0) {
1750
+ // Subscribe up front: a reopened DB whose first post-reopen DDL is a view/MV
1751
+ // would otherwise miss the event (the lazy `ensureSchemaSubscription` points are
1752
+ // all table hooks). Done even for an empty catalog. (Documented gap: a brand-new
1753
+ // DB — never rehydrated — whose very first DDL is a view still relies on a prior
1754
+ // store-table create/connect to establish the subscription.)
1755
+ this.ensureSchemaSubscription(db);
1756
+ // Capability gate (method presence — matching the coordinator's own gate): an
1757
+ // atomic-commit provider commits a source write and its same-module backing in
1758
+ // one all-or-nothing batch, so a crash can no longer tear them apart (the
1759
+ // crash-divergence window gate 5 historically guarded). In that domain, gate 4
1760
+ // alone governs same-module backings and a non-stale backing adopts after a crash
1761
+ // too — the LOGICAL-staleness window is closed instead by the durable stale-MV
1762
+ // set, which (unlike the clean-shutdown marker) survives a crash. A non-atomic
1763
+ // provider never writes the set (see {@link atomicProvider}), so skip the read.
1764
+ const atomic = this.atomicProvider;
1765
+ const durableStale = atomic ? await this.readDurableStaleMvSet() : { present: false };
1766
+ // Consume the clean-shutdown marker FIRST (before the catalog scan): in the
1767
+ // non-atomic domain its presence is the adopt trust basis for this rehydration
1768
+ // only, and deleting it immediately makes it single-use. Its payload names the MVs
1769
+ // that were stale-at-close. Always consumed for single-use hygiene even in the
1770
+ // atomic branch (where its trust bit is ignored in favor of the durable set).
1771
+ const { trusted, staleAtClose } = await this.consumeCleanShutdownMarker();
1772
+ // Per-entry adopt trust basis, capability-aware:
1773
+ // - Atomic domain WITH a durable stale-set on disk → gate 4 alone governs; the
1774
+ // marker is NOT required, so a non-stale backing adopts even after a crash. A
1775
+ // present-but-unparseable stale-set (`stale === null`) refills everything.
1776
+ // - Otherwise (non-atomic provider, OR atomic but no stale-set yet — the upgrade
1777
+ // path) → today's marker path, byte-for-byte.
1778
+ const trustBacking = (name) => atomic && durableStale.present
1779
+ ? durableStale.stale !== null && !durableStale.stale.has(name)
1780
+ : trusted && !staleAtClose.has(name);
1781
+ const entries = await this.loadCatalogEntries();
1782
+ const result = { tables: [], indexes: [], views: [], materializedViews: [], errors: [] };
1783
+ if (entries.length === 0)
916
1784
  return result;
1785
+ const recordError = (ddl, e) => {
1786
+ const error = e instanceof Error ? e : new Error(String(e));
1787
+ console.warn(`[StoreModule] Failed to rehydrate DDL entry, skipping: ${error.message}\n DDL: ${ddl.substring(0, 120)}`);
1788
+ result.errors.push({ ddl, error });
1789
+ };
1790
+ // Classify every loaded entry by key prefix. The full-range catalog scan returns
1791
+ // table, view, and MV entries intermixed; each must reach the correct phase — a
1792
+ // view/MV entry fed to the table-phase importCatalog would fail-loud or mis-handle.
1793
+ const tableDDLs = [];
1794
+ const viewDDLs = [];
1795
+ // MV entries retain their qualified `schema.mv` name (derived from the catalog
1796
+ // key) so phase 3 can withhold the adopt fast path per-entry for any MV that
1797
+ // was stale-at-close.
1798
+ const mvEntries = [];
1799
+ for (const { key, ddl } of entries) {
1800
+ switch (classifyCatalogKey(key)) {
1801
+ case 'view': {
1802
+ viewDDLs.push(ddl);
1803
+ break;
1804
+ }
1805
+ case 'materializedView': {
1806
+ mvEntries.push({ name: parseMaterializedViewCatalogKey(key), ddl });
1807
+ break;
1808
+ }
1809
+ // Meta entries are store-internal, never DDL. (The marker itself was
1810
+ // already consumed above; this guards any future meta key.)
1811
+ case 'meta': {
1812
+ break;
1813
+ }
1814
+ default: {
1815
+ tableDDLs.push(ddl);
1816
+ break;
1817
+ }
1818
+ }
917
1819
  }
918
- for (const ddl of ddlStatements) {
1820
+ // Phase 1 tables. Per-entry import isolates a corrupt entry so the rest load.
1821
+ for (const ddl of tableDDLs) {
919
1822
  try {
920
1823
  const imported = await db.schemaManager.importCatalog([ddl]);
921
1824
  result.tables.push(...imported.tables);
922
1825
  result.indexes.push(...imported.indexes);
923
1826
  }
924
1827
  catch (e) {
925
- const error = e instanceof Error ? e : new Error(String(e));
926
- console.warn(`[StoreModule] Failed to rehydrate DDL entry, skipping: ${error.message}\n DDL: ${ddl.substring(0, 120)}`);
927
- result.errors.push({ ddl, error });
1828
+ recordError(ddl, e);
1829
+ }
1830
+ }
1831
+ // Phase 2 — views (silent register; deferred body validation → order-independent).
1832
+ for (const ddl of viewDDLs) {
1833
+ try {
1834
+ const imported = await db.schemaManager.importCatalog([ddl]);
1835
+ result.views.push(...imported.views);
1836
+ }
1837
+ catch (e) {
1838
+ recordError(ddl, e);
928
1839
  }
929
1840
  }
1841
+ // Phase 3 — materialized views, dependency-ordered via fixpoint retry (see
1842
+ // docstring). One shared adopt ledger across all rounds: adopted upstream
1843
+ // backings unlock their dependents' adoption in later rounds.
1844
+ const adoptedBackings = new Set();
1845
+ let pending = mvEntries;
1846
+ while (pending.length > 0) {
1847
+ const failed = [];
1848
+ let progressed = false;
1849
+ for (const entry of pending) {
1850
+ try {
1851
+ // Ordering gate (unified model): a dependent's source may already
1852
+ // pre-exist as the upstream's phase-1 *plain* table, so its body
1853
+ // PLANS before the upstream's own MV entry has imported. Pass the
1854
+ // names of every OTHER still-pending MV entry; the engine defers
1855
+ // (throws → retried next round) any entry whose body reads one.
1856
+ const pendingDerivations = new Set(pending.filter(p => p !== entry).map(p => p.name));
1857
+ // Trust this backing only when the per-entry capability-aware basis
1858
+ // (`trustBacking`) holds: in the atomic domain a non-stale backing is
1859
+ // trusted (even after a crash); otherwise a clean shutdown AND not
1860
+ // stale-at-close. A withheld MV refills — recomputing content and
1861
+ // re-arming maintenance (clearing `stale`) — and is never added to
1862
+ // `adoptedBackings`, so the ledger gate forces its MV-over-MV dependents
1863
+ // to refill too.
1864
+ const imported = await db.schemaManager.importCatalog([entry.ddl], {
1865
+ trustBackings: trustBacking(entry.name),
1866
+ adoptedBackings,
1867
+ pendingDerivations,
1868
+ });
1869
+ result.materializedViews.push(...imported.materializedViews);
1870
+ progressed = true;
1871
+ }
1872
+ catch (e) {
1873
+ failed.push({ entry, error: e });
1874
+ }
1875
+ }
1876
+ if (!progressed) {
1877
+ // No MV built this round → the remaining failures are genuine (missing
1878
+ // source, ineligible body, unresolvable cycle). Record them and stop.
1879
+ for (const f of failed)
1880
+ recordError(f.entry.ddl, f.error);
1881
+ break;
1882
+ }
1883
+ pending = failed.map(f => f.entry);
1884
+ }
1885
+ // Refresh each connected StoreTable from the now-current registry. During
1886
+ // import, `importTable` connects a StoreTable holding the table-only schema,
1887
+ // then `importIndex` appends the index (and its derived UNIQUE constraint) to
1888
+ // the SchemaManager's registered schema but NOT to that live StoreTable
1889
+ // instance — `importCatalog` deliberately skips module hooks to stay generic,
1890
+ // so the store module reconciles here. Without this, DML on a rehydrated table
1891
+ // would not maintain its indexes and the derived UNIQUE would not enforce.
1892
+ for (const table of this.tables.values()) {
1893
+ const current = table.getSchema();
1894
+ const fresh = db.schemaManager.getTable(current.schemaName, current.name);
1895
+ if (fresh)
1896
+ table.updateSchema(fresh);
1897
+ }
1898
+ // Recompute the durable stale-MV set from the now-current flags and compare-write
1899
+ // it: refilled MVs cleared `stale`, adopted MVs were never stale, so the on-disk
1900
+ // entry (which a crash may have left naming an MV this rehydrate just refilled)
1901
+ // now reflects post-rehydrate truth — preventing a wasteful re-refill at the next
1902
+ // open. Phase 3's `importCatalog` is silent (no schema-change events fire), so the
1903
+ // listener never ran this rehydration; this explicit tail write establishes the
1904
+ // session baseline (`lastPersistedStaleMvs`). It rides `persistQueue`, drained by
1905
+ // the next `closeAll`/`whenCatalogPersisted`. (An optimization, not soundness: a
1906
+ // crash before it lands leaves a stale-set that only over-names → a sound refill.)
1907
+ this.persistStaleMvSetIfChanged();
930
1908
  return result;
931
1909
  }
1910
+ /**
1911
+ * Consume the clean-shutdown marker, deleting it in the same breath (single-use).
1912
+ *
1913
+ * Returns `{ trusted, staleAtClose }`:
1914
+ * - `trusted` — the marker was present AND its payload parsed as a string array.
1915
+ * Absence (a fresh store, a crash, or a prior rehydrate without an intervening
1916
+ * {@link closeAll}) yields `false`, so every adopt this session falls back to
1917
+ * refill.
1918
+ * - `staleAtClose` — the qualified lowercased `schema.mv` names that were
1919
+ * stale-at-close (written by {@link closeAll}); those MVs refill even under trust.
1920
+ *
1921
+ * Conservative parse: any unparseable / wrong-shape payload (including a legacy
1922
+ * bare `'1'`, which parses to a number, not an array) degrades to
1923
+ * `{ trusted: false, staleAtClose: ∅ }` — refill everything — rather than
1924
+ * trust-everything. The marker is deleted regardless of parse outcome.
1925
+ */
1926
+ async consumeCleanShutdownMarker() {
1927
+ const catalogStore = await this.provider.getCatalogStore();
1928
+ const markerKey = buildMetaCatalogKey(CLEAN_SHUTDOWN_META_NAME);
1929
+ const raw = await catalogStore.get(markerKey);
1930
+ if (raw === undefined)
1931
+ return { trusted: false, staleAtClose: new Set() };
1932
+ // Single-use AND durable-before-session-writes: forcing this delete to stable
1933
+ // storage before any of the session's (independently flushed, different-store)
1934
+ // data writes can land closes the power-loss window where a persisted data write
1935
+ // outlives a lost marker-delete and resurrects a consumed marker. (Backends
1936
+ // without a durability knob no-op the hint — losing it is conservative there, and
1937
+ // memory has no crash.) See docs/materialized-views.md § Cross-module atomicity.
1938
+ await catalogStore.delete(markerKey, { sync: true }); // single-use, regardless of parse outcome
1939
+ try {
1940
+ const parsed = JSON.parse(new TextDecoder().decode(raw));
1941
+ if (!Array.isArray(parsed) || !parsed.every((s) => typeof s === 'string')) {
1942
+ console.warn('[StoreModule] clean-shutdown marker payload is not a string array; refilling all backings.');
1943
+ return { trusted: false, staleAtClose: new Set() };
1944
+ }
1945
+ return { trusted: true, staleAtClose: new Set(parsed) };
1946
+ }
1947
+ catch (e) {
1948
+ console.warn(`[StoreModule] clean-shutdown marker payload did not parse as JSON; refilling all backings: ${e instanceof Error ? e.message : String(e)}`);
1949
+ return { trusted: false, staleAtClose: new Set() };
1950
+ }
1951
+ }
932
1952
  /**
933
1953
  * Remove DDL from the catalog store when a table is dropped.
934
1954
  */
@@ -937,6 +1957,50 @@ export class StoreModule {
937
1957
  const catalogStore = await this.provider.getCatalogStore();
938
1958
  await catalogStore.delete(catalogKey);
939
1959
  }
1960
+ /**
1961
+ * Persist a plain view's catalog entry (DDL via `generateViewDDL`), keyed by its
1962
+ * reserved view prefix. Compare-write (skip identical) — see
1963
+ * {@link persistObjectCatalogEntryIfChanged}.
1964
+ */
1965
+ async saveViewDDL(view) {
1966
+ await this.persistObjectCatalogEntryIfChanged(buildViewCatalogKey(view.schemaName, view.name), generateViewDDL(view));
1967
+ }
1968
+ /** Remove a plain view's catalog entry (on DROP VIEW). */
1969
+ async removeViewDDL(schemaName, viewName) {
1970
+ const catalogStore = await this.provider.getCatalogStore();
1971
+ await catalogStore.delete(buildViewCatalogKey(schemaName, viewName));
1972
+ }
1973
+ /**
1974
+ * Persist a materialized view's catalog entry (DDL via `generateMaintainedTableDDL`
1975
+ * over the maintained table — the unified record), keyed by its reserved MV
1976
+ * prefix. Compare-write (skip identical) — see
1977
+ * {@link persistObjectCatalogEntryIfChanged}.
1978
+ */
1979
+ async saveMaterializedViewDDL(mv) {
1980
+ await this.persistObjectCatalogEntryIfChanged(buildMaterializedViewCatalogKey(mv.schemaName, mv.name), generateMaintainedTableDDL(mv));
1981
+ }
1982
+ /** Remove a materialized view's catalog entry (on DROP MATERIALIZED VIEW). */
1983
+ async removeMaterializedViewDDL(schemaName, mvName) {
1984
+ const catalogStore = await this.provider.getCatalogStore();
1985
+ await catalogStore.delete(buildMaterializedViewCatalogKey(schemaName, mvName));
1986
+ }
1987
+ /**
1988
+ * Compare-write a view/MV catalog entry: write only when the entry is absent or its
1989
+ * DDL differs from `newDDL` (skip identical). Unlike the table path's
1990
+ * {@link persistCatalogIfChanged} there is **no** absent→skip self-filter — a view/MV
1991
+ * belongs to this db's catalog unconditionally (one module instance serves one
1992
+ * Database). Skipping identical writes makes any event whose regenerated DDL is
1993
+ * unchanged (e.g. a `materialized_view_refreshed`) a no-op, so a second consecutive
1994
+ * reopen yields identical catalog bytes. (Rehydration itself fires no view/MV events
1995
+ * at all — `importCatalog` is silent.)
1996
+ */
1997
+ async persistObjectCatalogEntryIfChanged(key, newDDL) {
1998
+ const catalogStore = await this.provider.getCatalogStore();
1999
+ const existing = await catalogStore.get(key);
2000
+ if (existing !== undefined && new TextDecoder().decode(existing) === newDDL)
2001
+ return;
2002
+ await catalogStore.put(key, new TextEncoder().encode(newDDL));
2003
+ }
940
2004
  /**
941
2005
  * Parse module configuration from vtab args.
942
2006
  */
@@ -945,15 +2009,351 @@ export class StoreModule {
945
2009
  collation: args?.collation || 'NOCASE',
946
2010
  };
947
2011
  }
2012
+ // --- Engine schema-change subscription (catalog-only tag persistence) ---
2013
+ /**
2014
+ * Subscribe (once) to the engine's `SchemaChangeNotifier` so catalog-only
2015
+ * mutations that bypass `module.alterTable` — notably `ALTER … SET TAGS` and the
2016
+ * programmatic `setTableTags`/`setColumnTags`/`setConstraintTags` — still re-persist
2017
+ * the table's catalog DDL. Called lazily from the first `create`/`connect`/
2018
+ * `alterTable` hook that hands us a `db`.
2019
+ *
2020
+ * One `StoreModule` instance is assumed to serve one `Database`. A later hook
2021
+ * carrying a *different* `db` keeps the existing subscription (multi-database
2022
+ * sharing of a single module instance is out of scope) and logs.
2023
+ */
2024
+ ensureSchemaSubscription(db) {
2025
+ if (this.schemaListenerUnsub) {
2026
+ if (this.subscribedDb && this.subscribedDb !== db) {
2027
+ console.warn('[StoreModule] ensureSchemaSubscription called with a different Database; '
2028
+ + 'keeping the existing subscription (one module instance is assumed to serve one Database).');
2029
+ }
2030
+ return;
2031
+ }
2032
+ this.subscribedDb = db;
2033
+ // A fresh subscription has no on-disk stale-set baseline yet (this module never
2034
+ // wrote one this session); reset so the first relevant event — or the rehydrate
2035
+ // tail — re-establishes it rather than comparing against a prior subscription's.
2036
+ this.lastPersistedStaleMvs = undefined;
2037
+ this.schemaListenerUnsub = db.schemaManager.getChangeNotifier().addListener(this.onEngineSchemaChange);
2038
+ }
2039
+ /**
2040
+ * Engine schema-change listener. Persists the catalog incrementally for the events
2041
+ * that bypass `module.alterTable` / `module.destroy`:
2042
+ *
2043
+ * - `table_modified` — every catalog-only tag swap (and the redundant follow-up a
2044
+ * structural ALTER fires). Keeps a connected `StoreTable`'s cached schema consistent
2045
+ * (SET TAGS does not call `updateSchema`) then read-compare-writes the table bundle.
2046
+ * - `view_added` / `view_modified` / `view_removed` — plain `CREATE`/`ALTER … SET TAGS`/
2047
+ * `DROP VIEW` (the engine fires these from the runtime emitters).
2048
+ * - `materialized_view_added` / `_modified` / `_refreshed` / `_removed` — MV lifecycle.
2049
+ * Like `table_modified`, the `_added`/`_modified`/`_refreshed` arms also synchronously
2050
+ * refresh the connected `StoreTable`'s cached schema so a tag change (e.g.
2051
+ * `quereus.sync.replicate`) takes effect immediately without reopen.
2052
+ *
2053
+ * Unlike the table path there is **no** catalog-absent self-filter for view/MV
2054
+ * add/remove: one `StoreModule` instance serves one `Database`, so that database's
2055
+ * views/MVs belong in its catalog unconditionally. A MEMORY-hosted maintained table
2056
+ * fires `table_added`/`table_removed`/`table_modified` like any table; those stay
2057
+ * ignored (`table_added`/`table_removed` fall through; its `table_modified` is
2058
+ * catalog-absent → skipped), so only the MV entry persists for it. A STORE-hosted
2059
+ * maintained table additionally persists its own table bundle through the ordinary
2060
+ * store-table machinery (which phase-1 rehydrate connects for the adopt fast path).
2061
+ *
2062
+ * Synchronous by contract (`notifyChange` does not await listeners); every async write
2063
+ * rides `persistQueue`, drained by `closeAll`/`whenCatalogPersisted`.
2064
+ *
2065
+ * After dispatching the event's own catalog persistence, the listener recomputes the
2066
+ * durable stale-MV set and compare-writes it (see {@link persistStaleMvSetIfChanged}).
2067
+ * Because the engine's MV manager subscribes to the same notifier in the `Database`
2068
+ * constructor — before this lazy subscription — its listener runs first, so the
2069
+ * `derivation.stale` flags are already current when this recompute reads them.
2070
+ */
2071
+ onEngineSchemaChange = (event) => {
2072
+ this.dispatchSchemaChange(event);
2073
+ // Every staleness SET transition is bracketed by an event observed here (a source
2074
+ // `table_modified`/`table_removed`, or the synthetic backing-invalidation
2075
+ // `table_modified` for an MV-over-MV cascade), and every CLEAR but the no-event
2076
+ // rename-restore fires one too — so recompute on each event keeps the durable set
2077
+ // current. Enqueued on `persistQueue` AFTER the dispatch above, so the `sync` lands
2078
+ // after the event's own source-DDL write is queued (the durability ordering the
2079
+ // adopt soundness argument relies on — see `docs/materialized-views.md`).
2080
+ this.persistStaleMvSetIfChanged();
2081
+ };
2082
+ /** Dispatch a single engine schema-change event to its catalog-persistence arm. */
2083
+ dispatchSchemaChange(event) {
2084
+ switch (event.type) {
2085
+ case 'table_modified': {
2086
+ // SET TAGS does not call `table.updateSchema`, so a connected instance's cached
2087
+ // schema would otherwise go stale (and a later lazy `saveTableDDL` could re-write
2088
+ // tag-less DDL). Persistence below always reads `newObject`, never this cache.
2089
+ const tableKey = `${event.schemaName}.${event.objectName}`.toLowerCase();
2090
+ const connected = this.tables.get(tableKey);
2091
+ if (connected)
2092
+ connected.updateSchema(event.newObject);
2093
+ const key = buildCatalogKey(event.schemaName, event.objectName);
2094
+ const newObject = event.newObject;
2095
+ this.enqueuePersist(() => this.persistCatalogIfChanged(key, newObject));
2096
+ return;
2097
+ }
2098
+ case 'view_added':
2099
+ case 'view_modified': {
2100
+ const view = event.newObject;
2101
+ this.enqueuePersist(() => this.saveViewDDL(view));
2102
+ return;
2103
+ }
2104
+ case 'view_removed': {
2105
+ const { schemaName, objectName } = event;
2106
+ this.enqueuePersist(() => this.removeViewDDL(schemaName, objectName));
2107
+ return;
2108
+ }
2109
+ case 'materialized_view_added':
2110
+ case 'materialized_view_modified':
2111
+ // Unified model: the payload is the maintained table itself.
2112
+ this.refreshConnectedMaterializedView(event.schemaName, event.objectName, event.newObject);
2113
+ return;
2114
+ case 'materialized_view_refreshed':
2115
+ // DDL is usually unchanged by a REFRESH (body/tags identical) → compare-skip,
2116
+ // but re-read tags in case they were updated alongside the refresh.
2117
+ this.refreshConnectedMaterializedView(event.schemaName, event.objectName, event.object);
2118
+ return;
2119
+ case 'materialized_view_removed': {
2120
+ const { schemaName, objectName } = event;
2121
+ // DROP MAINTAINED detaches catalog-only: the engine has already swapped the
2122
+ // catalog entry to a plain (derivation-less) schema before firing this event,
2123
+ // but a connected `StoreTable` still caches the maintained schema. The store's
2124
+ // `alterTable` reads that cache (`getSchema`), so a following structural ALTER
2125
+ // would spread the stale `derivation` onto the rebuilt schema and re-register
2126
+ // the table as a materialized view (rejecting the next ALTER). Refresh the cache
2127
+ // to the now-plain catalog entry. When the entry is gone entirely (DROP TABLE /
2128
+ // DROP MATERIALIZED VIEW), there is nothing to refresh — `destroy` retires it.
2129
+ const plain = this.subscribedDb?.schemaManager.getTable(schemaName, objectName);
2130
+ if (plain && !isMaintainedTable(plain)) {
2131
+ const connected = this.tables.get(`${schemaName}.${objectName}`.toLowerCase());
2132
+ if (connected)
2133
+ connected.updateSchema(plain);
2134
+ }
2135
+ this.enqueuePersist(() => this.removeMaterializedViewDDL(schemaName, objectName));
2136
+ return;
2137
+ }
2138
+ default:
2139
+ return;
2140
+ }
2141
+ }
2142
+ /**
2143
+ * The qualified lowercased `schema.mv` names of every maintained table currently
2144
+ * marked `derivation.stale` — an MV whose row-time maintenance detached mid-session
2145
+ * (a body-relevant `table_modified`/`table_removed` on a source, or a cascade
2146
+ * backing-invalidation), so its durable backing may be behind. The single source of
2147
+ * truth for both the clean-shutdown marker payload and the durable stale-MV set.
2148
+ *
2149
+ * No subscribed db ⇒ the empty set: every path that can mark an MV stale requires a
2150
+ * session in which this module observed the db (a store source create/connect or
2151
+ * `rehydrateCatalog`, both of which subscribe), so a session without `subscribedDb`
2152
+ * never detached any persisted MV's maintenance. Memory-backed MVs that appear here
2153
+ * are harmless — their catalog entries always refill (no phase-1 pre-existing
2154
+ * backing), so withholding trust from them is a no-op.
2155
+ */
2156
+ computeStaleMvSet() {
2157
+ return this.subscribedDb
2158
+ ? this.subscribedDb.schemaManager.getAllMaintainedTables()
2159
+ .filter(mv => mv.derivation.stale)
2160
+ .map(mv => `${mv.schemaName}.${mv.name}`.toLowerCase())
2161
+ : [];
2162
+ }
2163
+ /**
2164
+ * Recompute the durable stale-MV set and, only when it differs from the last value
2165
+ * enqueued this subscription ({@link lastPersistedStaleMvs}), enqueue a `sync: true`
2166
+ * point-write of it onto {@link persistQueue}. The compare-skip keeps an unrelated
2167
+ * `table_modified` (a tag swap that changes no MV's staleness) from costing an fsync.
2168
+ *
2169
+ * The recompute reads `derivation.stale` synchronously (the flags are current at
2170
+ * listener-dispatch time); the field is updated synchronously at enqueue time, and
2171
+ * `persistQueue` serializes the writes in order, so the field always names the last
2172
+ * enqueued value. Riding `persistQueue` (rather than a bare `put`) also serializes
2173
+ * this `sync` behind the triggering event's own source-DDL compare-write, so the
2174
+ * stale-set becomes durable no-later-than the source DDL that caused the staleness —
2175
+ * the ordering the adopt soundness argument depends on.
2176
+ */
2177
+ persistStaleMvSetIfChanged() {
2178
+ // Only an atomic-capable session writes the durable set — it is the sole reader and
2179
+ // trust basis (see {@link atomicProvider}); a non-atomic session's trust basis is the
2180
+ // clean-shutdown marker, so writing the set there is both useless and a soundness
2181
+ // hazard (it could be trusted by a later atomic reopen). Skip before recomputing.
2182
+ if (!this.atomicProvider)
2183
+ return;
2184
+ const json = JSON.stringify(this.computeStaleMvSet());
2185
+ if (json === this.lastPersistedStaleMvs)
2186
+ return;
2187
+ this.lastPersistedStaleMvs = json;
2188
+ this.enqueuePersist(() => this.writeDurableStaleMvSet(json));
2189
+ }
2190
+ /**
2191
+ * Write the durable stale-MV set (`\x00meta\x00stale_mvs`) with `sync: true`. Unlike
2192
+ * the clean-shutdown marker this entry is persistent current-truth — overwritten as
2193
+ * staleness changes, never deleted — so a crash leaves the last synced value intact.
2194
+ * The `sync` mirrors the marker-consume delete's durability discipline (and carries
2195
+ * the same documented backend caveat: a backend without a durability knob no-ops it).
2196
+ */
2197
+ async writeDurableStaleMvSet(json) {
2198
+ const catalogStore = await this.provider.getCatalogStore();
2199
+ await catalogStore.put(buildMetaCatalogKey(STALE_MVS_META_NAME), new TextEncoder().encode(json), { sync: true });
2200
+ }
2201
+ /**
2202
+ * Read + conservative-parse the durable stale-MV set, mirroring
2203
+ * {@link consumeCleanShutdownMarker}'s parse discipline (but READ-only — never
2204
+ * deleted). Returns:
2205
+ * - `{ present: false }` — no entry on disk (a pre-atomic store, or an upgrade where
2206
+ * the capability was gained but no stale-set has been written yet) → the caller
2207
+ * falls back to the clean-shutdown marker path.
2208
+ * - `{ present: true, stale }` — the parsed set of qualified `schema.mv` names.
2209
+ * - `{ present: true, stale: null }` — a present but unparseable / wrong-shape payload
2210
+ * → the caller refills everything (the always-safe posture).
2211
+ */
2212
+ async readDurableStaleMvSet() {
2213
+ const catalogStore = await this.provider.getCatalogStore();
2214
+ const raw = await catalogStore.get(buildMetaCatalogKey(STALE_MVS_META_NAME));
2215
+ if (raw === undefined)
2216
+ return { present: false };
2217
+ try {
2218
+ const parsed = JSON.parse(new TextDecoder().decode(raw));
2219
+ if (!Array.isArray(parsed) || !parsed.every((s) => typeof s === 'string')) {
2220
+ console.warn('[StoreModule] durable stale-MV set payload is not a string array; refilling all backings.');
2221
+ return { present: true, stale: null };
2222
+ }
2223
+ return { present: true, stale: new Set(parsed) };
2224
+ }
2225
+ catch (e) {
2226
+ console.warn(`[StoreModule] durable stale-MV set payload did not parse as JSON; refilling all backings: ${e instanceof Error ? e.message : String(e)}`);
2227
+ return { present: true, stale: null };
2228
+ }
2229
+ }
2230
+ /**
2231
+ * Shared MV add/modify/refresh handling. Narrow defensively — a derivation-less
2232
+ * payload would be an engine bug, so skip. Otherwise, mirror `table_modified`:
2233
+ * synchronously refresh a connected `StoreTable`'s cached schema (so a tag change
2234
+ * such as `quereus.sync.replicate` takes effect immediately without reopen) before
2235
+ * enqueuing the catalog DDL persist.
2236
+ */
2237
+ refreshConnectedMaterializedView(schemaName, objectName, payload) {
2238
+ if (!isMaintainedTable(payload))
2239
+ return;
2240
+ const key = `${schemaName}.${objectName}`.toLowerCase();
2241
+ const connected = this.tables.get(key);
2242
+ if (connected)
2243
+ connected.updateSchema(payload);
2244
+ this.enqueuePersist(() => this.saveMaterializedViewDDL(payload));
2245
+ }
2246
+ /**
2247
+ * Append a catalog-persistence task to the serialized `persistQueue`, so successive
2248
+ * mutations (e.g. SET TAGS (a=1) then SET TAGS ()) apply in order and are drained by
2249
+ * `closeAll`/`whenCatalogPersisted` before the provider closes. Errors are
2250
+ * swallowed+logged to mirror `notifyChange`'s own try/catch contract — a listener
2251
+ * rejection must never escape.
2252
+ */
2253
+ enqueuePersist(work) {
2254
+ this.persistQueue = this.persistQueue
2255
+ .then(work)
2256
+ .catch((err) => {
2257
+ const message = err instanceof Error ? err.message : String(err);
2258
+ console.warn(`[StoreModule] Failed to persist catalog DDL after schema change: ${message}`);
2259
+ });
2260
+ }
2261
+ /**
2262
+ * Read-compare-write the catalog DDL for a table that just fired `table_modified`.
2263
+ *
2264
+ * - **Absent** catalog entry → the table is not store-backed in this catalog (a
2265
+ * memory table in the same `db`, or a store table never persisted) → skip. This
2266
+ * self-filters foreign-module tables without relying on `vtabModule` identity
2267
+ * (which points at the isolation wrapper when wrapped).
2268
+ * - **Present** but identical DDL → skip (no redundant write). This is what makes a
2269
+ * structural ALTER — whose own `alterTable` already wrote the final DDL, then fires
2270
+ * `table_modified` with that same final schema — a no-op here (no double-write).
2271
+ * - **Present** and different DDL → re-persist (the tag swap; or a beneficial
2272
+ * propagated rewrite of a dependent store table).
2273
+ */
2274
+ async persistCatalogIfChanged(key, newObject) {
2275
+ const catalogStore = await this.provider.getCatalogStore();
2276
+ const existing = await catalogStore.get(key);
2277
+ if (existing === undefined)
2278
+ return; // not store-backed in this catalog — skip
2279
+ // Regenerate the full bundle (table DDL + index DDL + exposed-implicit-index
2280
+ // tag DDL): a SET TAGS on an index fires `table_modified` on the OWNING table
2281
+ // with the updated index in `tableSchema.indexes` — or, for an exposed
2282
+ // implicit index, with the updated `exposedIndexTags` on the originating
2283
+ // UNIQUE constraint — so the changed DDL re-persists here with no
2284
+ // index-specific plumbing. An identical bundle is what makes a structural
2285
+ // ALTER (and the createIndex follow-up event) a no-op — no double-write.
2286
+ const newDDL = this.buildCatalogEntry(newObject);
2287
+ const existingDDL = new TextDecoder().decode(existing);
2288
+ if (existingDDL === newDDL)
2289
+ return; // identical — no redundant write
2290
+ await catalogStore.put(key, new TextEncoder().encode(newDDL));
2291
+ }
2292
+ /**
2293
+ * Resolve once all catalog writes queued by async schema-change listeners
2294
+ * (catalog-only tag swaps) have settled. A durability barrier: `closeAll` awaits
2295
+ * it internally; callers/tests that need the persisted catalog current without a
2296
+ * full close can await it directly. Never rejects — queued errors are logged in
2297
+ * the chain.
2298
+ */
2299
+ async whenCatalogPersisted() {
2300
+ await this.persistQueue;
2301
+ }
948
2302
  /**
949
2303
  * Close all stores.
950
2304
  */
951
2305
  async closeAll() {
2306
+ // Capture the stale-at-close MV set BEFORE the unsubscribe block clears
2307
+ // `subscribedDb`. Nothing between here and the marker write can change these
2308
+ // flags (closeAll only drains the persist queue and disconnects tables).
2309
+ // `stale` is in-memory-only runtime state — an MV whose row-time maintenance was
2310
+ // detached mid-session (any `table_modified` on a source: an ALTER, even a
2311
+ // `create index`) so subsequent source writes never reached its backing. Carrying
2312
+ // the names lets the next open exclude exactly those from the adopt fast path.
2313
+ //
2314
+ // No subscribed db (this module was opened but never rehydrated and never had a
2315
+ // store table created/connected) ⇒ the empty set: every path that can mark an MV
2316
+ // stale requires a session in which this module observed the db — a store source
2317
+ // create/connect (both call `ensureSchemaSubscription`) or `rehydrateCatalog`
2318
+ // (subscribes up front) — so a session without `subscribedDb` never detached any
2319
+ // persisted MV's maintenance. Memory-backed MVs that appear in the set are
2320
+ // harmless: their catalog entries always refill (no phase-1 pre-existing backing),
2321
+ // so withholding trust from them is a no-op.
2322
+ const staleAtClose = this.computeStaleMvSet();
2323
+ // Stop listening first so no new persist work is enqueued mid-close, then drain
2324
+ // the queued catalog writes (tag swaps) before the provider closes.
2325
+ if (this.schemaListenerUnsub) {
2326
+ this.schemaListenerUnsub();
2327
+ this.schemaListenerUnsub = undefined;
2328
+ this.subscribedDb = undefined;
2329
+ }
2330
+ await this.persistQueue;
952
2331
  for (const table of this.tables.values()) {
953
2332
  await table.disconnect();
954
2333
  }
955
2334
  this.tables.clear();
956
- this.coordinators.clear();
2335
+ this.moduleCoordinator = undefined;
2336
+ // Every batch has flushed (persist queue drained, tables disconnected):
2337
+ // attest the clean shutdown so the next open may take the materialized-view
2338
+ // adopt fast path. The marker VALUE is the JSON stale-at-close set (`[]` when
2339
+ // nothing is stale) — `rehydrateCatalog` excludes those MVs from the fast path
2340
+ // so they refill. Consumed (single-use) by `rehydrateCatalog`. Written LAST,
2341
+ // immediately before the provider closes — anything that dies before this line
2342
+ // leaves no marker and the next open refills everything.
2343
+ const catalogStore = await this.provider.getCatalogStore();
2344
+ await catalogStore.put(buildMetaCatalogKey(CLEAN_SHUTDOWN_META_NAME), new TextEncoder().encode(JSON.stringify(staleAtClose)));
2345
+ // Also write the DURABLE stale-MV set (persistent current-truth, NOT single-use) —
2346
+ // but ONLY for an atomic provider, the sole reader/truster of it (see
2347
+ // {@link atomicProvider}). Same array as the marker payload, under a separate
2348
+ // reserved meta key that the next open READS (never deletes). In the atomic-commit
2349
+ // domain this — not the crash-losable marker — is the adopt fast path's
2350
+ // logical-staleness exclusion basis, so a clean close (a) corrects any rename-restore
2351
+ // drift (the one staleness CLEAR transition that fires no event) and (b) guarantees
2352
+ // the entry exists even in a session with no staleness events. `provider.closeAll()`
2353
+ // below flushes it.
2354
+ if (this.atomicProvider) {
2355
+ await catalogStore.put(buildMetaCatalogKey(STALE_MVS_META_NAME), new TextEncoder().encode(JSON.stringify(staleAtClose)));
2356
+ }
957
2357
  await this.provider.closeAll();
958
2358
  this.stores.clear();
959
2359
  }
@@ -965,6 +2365,61 @@ export class StoreModule {
965
2365
  return this.tables.get(tableKey);
966
2366
  }
967
2367
  }
2368
+ /**
2369
+ * Reconcile each text PRIMARY KEY column's declared collation at CREATE time.
2370
+ *
2371
+ * The store now encodes PRIMARY KEY uniqueness/ordering PHYSICALLY with a
2372
+ * PER-COLUMN key collation (`StoreTable.pkKeyCollations`, drawn from each PK
2373
+ * column's declared `collation`), so ANY declared PK collation is honored
2374
+ * natively — an explicit `collate binary` text PK is keyed under BINARY, a
2375
+ * `collate nocase` under NOCASE, etc. The only thing this reconcile still does is
2376
+ * supply the store's table-level DEFAULT (`keyCollation` = K = `config.collation`,
2377
+ * default NOCASE) to a text PK column that declares NO explicit collation, so an
2378
+ * implicit-default text PK keeps the store's historical NOCASE-keyed behavior
2379
+ * (rather than the engine's BINARY column default). An EXPLICIT collation — even
2380
+ * one diverging from K — is left exactly as declared and re-keyed natively.
2381
+ *
2382
+ * Collation governs key bytes only for text, so only PK members whose logical type
2383
+ * is textual are touched (an `integer primary key` keeps its BINARY; temporal
2384
+ * text-physical types carry no collation). Non-PK columns are never touched.
2385
+ *
2386
+ * This is the CREATE path only — the load path (`connect` / rehydrate) does not
2387
+ * reconcile; the persisted DDL is the source of truth on reopen, and the column's
2388
+ * (BINARY-elided) `COLLATE` clause round-trips the per-column key collation.
2389
+ *
2390
+ * Returns the schema unchanged when no implicit text PK column needs the default
2391
+ * applied; otherwise a new schema with a rebuilt `columns` array + `columnIndexMap`.
2392
+ */
2393
+ function reconcilePkCollations(schema, keyCollation) {
2394
+ const pkIndices = new Set(schema.primaryKeyDefinition.map(def => def.index));
2395
+ if (pkIndices.size === 0)
2396
+ return schema;
2397
+ let changed = false;
2398
+ const newColumns = schema.columns.map((col, idx) => {
2399
+ if (!pkIndices.has(idx))
2400
+ return col;
2401
+ // Collation governs key bytes only for text; non-text PK columns (integer,
2402
+ // real, blob, …) are encoded type-natively and keep their declared collation.
2403
+ if (!col.logicalType.isTextual)
2404
+ return col;
2405
+ // An EXPLICIT collation is honored as-declared — the per-column key encoding
2406
+ // keys the column under it (Option B physical re-key parity with memory).
2407
+ if (col.collationExplicit)
2408
+ return col;
2409
+ const declared = (col.collation || 'BINARY').toUpperCase();
2410
+ if (declared === keyCollation)
2411
+ return col; // implicit default already == K
2412
+ // Implicit default diverges from K (e.g. the engine's BINARY column default
2413
+ // under the store's NOCASE K): apply the store's table-level default so an
2414
+ // undecorated text PK keeps the store's historical NOCASE-keyed semantics.
2415
+ changed = true;
2416
+ return { ...col, collation: keyCollation };
2417
+ });
2418
+ if (!changed)
2419
+ return schema;
2420
+ const columns = Object.freeze(newColumns);
2421
+ return { ...schema, columns, columnIndexMap: buildColumnIndexMap(columns) };
2422
+ }
968
2423
  /**
969
2424
  * Build a column remap array: newColumnIndex -> oldColumnIndex | -1.
970
2425
  * Maps each column in the new layout to its position in the old layout.