@quereus/store 4.5.0 → 4.7.0

package/dist/src/common/store-module-schema-sync.d.ts

@@ -0,0 +1,232 @@
+/**
+ * Keeping the engine's live schema and the persisted catalog in agreement, in both
+ * directions: reading the catalog back at open (rehydration), lazily reconnecting a
+ * table the engine already knows about, and reacting to the engine's schema-change
+ * notifications (catalog-only tag swaps, view/materialized-view DDL upkeep, and the
+ * stale-materialized-view set that survives a restart).
+ *
+ * Third layer of the store-module chain:
+ *   StoreModuleBase -> StoreModuleCatalog -> StoreModuleSchemaSync -> StoreModuleIndex
+ *   -> StoreModuleAlterColumn -> StoreModuleAlter -> StoreModuleRename -> StoreModule
+ */
+import type { Database } from '@quereus/quereus';
+import { StoreTable } from './store-table.js';
+import { StoreModuleCatalog } from './store-module-catalog.js';
+/**
+ * Result of catalog rehydration.
+ *
+ * `views` / `materializedViews` are additive (existing consumers — e.g.
+ * `quoomb-web` — read only `.errors`). Errors from any phase land in `errors`.
+ */
+export interface RehydrationResult {
+    tables: string[];
+    indexes: string[];
+    views: string[];
+    materializedViews: string[];
+    errors: RehydrationError[];
+}
+/**
+ * An error encountered while rehydrating a single DDL entry.
+ */
+export interface RehydrationError {
+    ddl: string;
+    error: Error;
+}
+export declare abstract class StoreModuleSchemaSync extends StoreModuleCatalog {
+    /**
+     * Returns the connected StoreTable for `schemaName.tableName`, lazily
+     * reconnecting from the engine's schema registry when absent.
+     *
+     * `renameTable` evicts the old key from `this.tables` and expects the next
+     * `connect()` to repopulate under the new name, but `apply schema` can run
+     * follow-up DDL (ALTER TABLE, CREATE/DROP INDEX) against the new name
+     * without an intervening connect. Mirror connect()'s schemaManager lookup
+     * so that DDL finds the moved table. Safe for the index paths because
+     * SchemaManager calls the module BEFORE mutating the registered table
+     * schema, so the reconnected cache matches what a connected instance holds.
+     */
+    protected getOrReconnectTable(db: Database, schemaName: string, tableName: string): StoreTable | undefined;
+    /**
+     * Rehydrate the persisted catalog into the in-memory schema manager, in
+     * dependency order.
+     *
+     * Re-asserts the engine schema-change subscription up front — normally already
+     * established by `StoreModule.onRegister` at `Database.registerModule`, but this
+     * entry point is also reachable with a `db` this module was never registered on, and
+     * phase 3 cannot observe MV staleness without it. Then loads every catalog entry
+     * once, classifies each by its key prefix into {tables, views, materialized views},
+     * and imports in three phases:
+     *
+     *   1. **Tables** — `importCatalog` (connect to existing storage; refresh connected
+     *      `StoreTable` schemas).
+     *   2. **Views** — `importCatalog` (engine silent-register; body validation deferred
+     *      to query time, so order among views — and view-over-MV / view-over-view —
+     *      does not matter, and no schema-change event fires → phase 2 writes nothing).
+     *   3. **Materialized views** — `importCatalog` per entry (engine re-materialize:
+     *      rebuilds the memory backing from current source data, re-registers row-time
+     *      maintenance, re-runs the eligibility gate — the same core the create emitter
+     *      uses, but silent: no `materialized_view_added` fires, so phase 3 writes
+     *      nothing back to the catalog). A store-hosted backing that phase 1 already
+     *      rehydrated is **adopted without the refill** when the engine's adopt gates
+     *      pass — see the clean-shutdown marker below.
+     *
+     * **Clean-shutdown marker.** Before anything loads, the reserved
+     * `\x00meta\x00clean_shutdown` catalog entry (written by `StoreModule.closeAll` after
+     * every batch flushed) is consumed: parsed into `{ trusted, staleAtClose }`, then
+     * **deleted immediately** — single-use, so a crash later in this session (or a
+     * second rehydrate without an intervening clean close) is detected at the next open
+     * and every adopt falls back to the always-correct drop+refill, self-healing any
+     * crash-window divergence (coordinated commit is not 2PC across stores). The marker
+     * payload is the JSON set of MVs that were **stale-at-close** (row-time maintenance
+     * detached, so the durable backing may be behind); phase 3 withholds trust per-entry
+     * for those — `trustBackings: trusted && !staleAtClose.has(name)` — so a
+     * stale-at-close MV refills (recomputing content and re-arming maintenance) while
+     * every live-at-close MV keeps the fast path. The one shared `adoptedBackings` set
+     * composes across fixpoint rounds (an upstream MV adopted in round 1 enables its
+     * dependent in round 2, while a refilled — or stale-at-close — upstream is never
+     * added to it, forcing dependents to refill).
+     *
+     * **MV-over-MV ordering** is handled by a fixpoint retry rather than a static topo
+     * sort: an MV's resolved `sourceTables` are computed at import time, not serialized
+     * in the DDL, so they are unavailable before import. Each round passes the names of
+     * every OTHER still-pending MV entry as `pendingDerivations`; the engine defers any
+     * entry whose body reads one (its source already pre-exists as a phase-1 plain
+     * table, so the body would otherwise plan against content the upstream's own import
+     * may be about to replace). The loop repeats while any MV makes progress — robust to
+     * arbitrary nesting depth. A genuinely unbuildable MV — a missing (e.g. memory)
+     * source, or an unresolvable cycle — makes no progress in a round and is recorded in
+     * `errors`.
+     *
+     * Per-entry errors in any phase are collected (not fatal) so one bad object does not
+     * abort the rest.
+     *
+     * Call after `db.registerModule()` (and `db.setDefaultVtabName()` if DDL may lack a
+     * USING clause).
+     */
+    rehydrateCatalog(db: Database): Promise<RehydrationResult>;
+    /**
+     * Subscribe (once) to the engine's `SchemaChangeNotifier`, the channel every
+     * event-driven catalog write rides: `table_added` (a new store table's entry),
+     * catalog-only mutations that bypass `module.alterTable` — notably `ALTER … SET TAGS`
+     * and the programmatic `setTableTags`/`setColumnTags`/`setConstraintTags` — and the
+     * whole view / materialized-view lifecycle.
+     *
+     * Normally called from `StoreModule.onRegister` at `Database.registerModule`, before
+     * any statement runs; also called from `rehydrateCatalog` and the `create`/`connect`/
+     * `alterTable` hooks, which are reachable with a `db` this module was not registered
+     * on. Idempotent.
+     *
+     * One `StoreModule` instance is assumed to serve one `Database`. A later hook
+     * carrying a *different* `db` keeps the existing subscription (multi-database
+     * sharing of a single module instance is out of scope) and logs.
+     */
+    protected ensureSchemaSubscription(db: Database): void;
+    /**
+     * Engine schema-change listener. Persists the catalog incrementally for the events
+     * that bypass `module.alterTable` / `module.destroy`:
+     *
+     * - `table_added` — a table the engine has just fully validated and registered. This is
+     *   what persists a store table nobody ever reads or writes (the lazy
+     *   first-storage-access save never runs for one).
+     * - `table_modified` — every catalog-only tag swap (and the redundant follow-up a
+     *   structural ALTER fires). Keeps a connected `StoreTable`'s cached schema consistent
+     *   (SET TAGS does not call `updateSchema`) then read-compare-writes the table bundle.
+     * - `view_added` / `view_modified` / `view_removed` — plain `CREATE`/`ALTER … SET TAGS`/
+     *   `DROP VIEW` (the engine fires these from the runtime emitters).
+     * - `materialized_view_added` / `_modified` / `_refreshed` / `_removed` — MV lifecycle.
+     *   Like `table_modified`, the `_added`/`_modified`/`_refreshed` arms also synchronously
+     *   refresh the connected `StoreTable`'s cached schema so a tag change (e.g.
+     *   `quereus.sync.replicate`) takes effect immediately without reopen.
+     *
+     * Unlike the table path there is **no** catalog-absent self-filter for view/MV
+     * add/remove: one `StoreModule` instance serves one `Database`, so that database's
+     * views/MVs belong in its catalog unconditionally. A MEMORY-hosted maintained table
+     * fires `table_added`/`table_removed`/`table_modified` like any table; those stay
+     * ignored (`table_added` fails the ownership filter, `table_removed` falls through,
+     * its `table_modified` is catalog-absent → skipped), so only the MV entry persists for
+     * it. A STORE-hosted maintained table additionally persists its own table bundle
+     * through the ordinary store-table machinery (which phase-1 rehydrate connects for the
+     * adopt fast path).
+     *
+     * Synchronous by contract (`notifyChange` does not await listeners); every async write
+     * rides `persistQueue`, drained by `closeAll`/`whenCatalogPersisted`.
+     *
+     * After dispatching the event's own catalog persistence, the listener recomputes the
+     * durable stale-MV set and compare-writes it (see {@link persistStaleMvSetIfChanged}).
+     * Because the engine's MV manager subscribes to the same notifier in the `Database`
+     * constructor — before this lazy subscription — its listener runs first, so the
+     * `derivation.stale` flags are already current when this recompute reads them.
+     */
+    private onEngineSchemaChange;
+    /**
+     * Dispatch a single engine schema-change event to its catalog-persistence arm.
+     *
+     * The `table_added` arm is where a store table's catalog entry is written. It fires
+     * from `SchemaManager.createTable` only AFTER the engine has fully validated the table
+     * (notably `validateForeignKeyCollations`, which runs after `module.create` returns and
+     * throws WITHOUT calling `destroy`) and registered it — so, unlike an eager write
+     * inside `StoreModule.create`, it can never leave an entry behind for a table the user
+     * never got, which would reopen as a phantom table.
+     *
+     * Two things distinguish this arm from the view/MV arms above it:
+     *
+     * - **It self-filters on ownership.** Every table the engine registers is announced,
+     *   including memory tables and other modules'. `ownsTableCatalogEntry` is the test for
+     *   "would this module write that table's entry" (and already handles the isolation
+     *   wrapper, whose `vtabModule` is the wrapper exposing us as `underlying`). The
+     *   view/MV arms need no such filter — views and MVs are not owned by a module.
+     * - **It compare-writes rather than blind-puts.** `table_added` is re-delivered during
+     *   rehydration for a store-hosted materialized-view backing
+     *   (`SchemaManager.createBackingTable`), where the entry phase 1 already wrote is
+     *   current; comparing keeps that a no-op so a second consecutive reopen still yields
+     *   identical catalog bytes. (Plain table import during rehydration is silent — no
+     *   `table_added` fires at all.)
+     *
+     * A failure on this path is logged and swallowed by `enqueuePersist`, not raised on the
+     * statement — identical to how `create view` / `create materialized view` already
+     * persist. That is why `StoreTableBase.initializeStore` keeps its first-storage-access
+     * catalog write as a compare-write BACKSTOP: it is the only site where a table whose
+     * DDL text cannot be encoded at all (a lone surrogate in a quoted column name, a
+     * DEFAULT string literal, a CHECK constant) still raises on a statement.
+     */
+    private dispatchSchemaChange;
+    /**
+     * Shared MV add/modify/refresh handling. Narrow defensively — a derivation-less
+     * payload would be an engine bug, so skip. Otherwise, mirror `table_modified`:
+     * synchronously refresh a connected `StoreTable`'s cached schema (so a tag change
+     * such as `quereus.sync.replicate` takes effect immediately without reopen) before
+     * enqueuing the catalog DDL persist.
+     */
+    private refreshConnectedMaterializedView;
+    /**
+     * The qualified lowercased `schema.mv` names of every maintained table currently
+     * marked `derivation.stale` — an MV whose row-time maintenance detached mid-session
+     * (a body-relevant `table_modified`/`table_removed` on a source, or a cascade
+     * backing-invalidation), so its durable backing may be behind. The single source of
+     * truth for both the clean-shutdown marker payload and the durable stale-MV set.
+     *
+     * No subscribed db ⇒ the empty set. Since `StoreModule.onRegister` subscribes at
+     * `Database.registerModule`, that now means a module never registered on any
+     * `Database` (constructed and closed directly) — which by construction observed no
+     * schema at all, so it detached no persisted MV's maintenance. Memory-backed MVs that appear here
+     * are harmless — their catalog entries always refill (no phase-1 pre-existing
+     * backing), so withholding trust from them is a no-op.
+     */
+    protected computeStaleMvSet(): string[];
+    /**
+     * Recompute the durable stale-MV set and, only when it differs from the last value
+     * enqueued this subscription (`StoreModuleBase.lastPersistedStaleMvs`), enqueue a `sync: true`
+     * point-write of it onto `StoreModuleBase.persistQueue`. The compare-skip keeps an unrelated
+     * `table_modified` (a tag swap that changes no MV's staleness) from costing an fsync.
+     *
+     * The recompute reads `derivation.stale` synchronously (the flags are current at
+     * listener-dispatch time); the field is updated synchronously at enqueue time, and
+     * `persistQueue` serializes the writes in order, so the field always names the last
+     * enqueued value. Riding `persistQueue` (rather than a bare `put`) also serializes
+     * this `sync` behind the triggering event's own source-DDL compare-write, so the
+     * stale-set becomes durable no-later-than the source DDL that caused the staleness —
+     * the ordering the adopt soundness argument depends on.
+     */
+    private persistStaleMvSetIfChanged;
+}
+//# sourceMappingURL=store-module-schema-sync.d.ts.map

package/dist/src/common/store-module-schema-sync.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"store-module-schema-sync.d.ts","sourceRoot":"","sources":["../../../src/common/store-module-schema-sync.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,KAAK,EAAE,QAAQ,EAA6D,MAAM,kBAAkB,CAAC;AAE5G,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAE9C,OAAO,EAAE,kBAAkB,EAAE,MAAM,2BAA2B,CAAC;AAE/D;;;;;GAKG;AACH,MAAM,WAAW,iBAAiB;IACjC,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,iBAAiB,EAAE,MAAM,EAAE,CAAC;IAC5B,MAAM,EAAE,gBAAgB,EAAE,CAAC;CAC3B;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAChC,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,KAAK,CAAC;CACb;AAED,8BAAsB,qBAAsB,SAAQ,kBAAkB;IACrE;;;;;;;;;;;OAWG;IACH,SAAS,CAAC,mBAAmB,CAAC,EAAE,EAAE,QAAQ,EAAE,UAAU,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,UAAU,GAAG,SAAS;IAoB1G;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAwDG;IACG,gBAAgB,CAAC,EAAE,EAAE,QAAQ,GAAG,OAAO,CAAC,iBAAiB,CAAC;IAuLhE;;;;;;;;;;;;;;;OAeG;IACH,SAAS,CAAC,wBAAwB,CAAC,EAAE,EAAE,QAAQ,GAAG,IAAI;IAkBtD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAmCG;IACH,OAAO,CAAC,oBAAoB,CAU1B;IAEF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA8BG;IACH,OAAO,CAAC,oBAAoB;IA0E5B;;;;;;OAMG;IACH,OAAO,CAAC,gCAAgC;IAQxC;;;;;;;;;;;;;OAaG;IACH,SAAS,CAAC,iBAAiB,IAAI,MAAM,EAAE;IAQvC;;;;;;;;;;;;;OAaG;IACH,OAAO,CAAC,0BAA0B;CAWlC"}