@proteinjs/db 1.34.3 → 1.35.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@proteinjs/db",
-  "version": "1.34.3",
+  "version": "1.35.0",
   "main": "./dist/generated/index.js",
   "types": "./dist/generated/index.d.ts",
   "exports": {
@@ -41,9 +41,9 @@
     "test": "jest --passWithNoTests"
   },
   "dependencies": {
-    "@proteinjs/db-query": "^1.7.1",
+    "@proteinjs/db-query": "^1.7.2",
     "@proteinjs/logger": "^1.0.21",
-    "@proteinjs/reflection": "^1.1.14",
+    "@proteinjs/reflection": "^1.2.0",
     "@proteinjs/serializer": "^1.1.10",
     "@proteinjs/server-api": "^3.0.11",
     "@proteinjs/service": "^1.5.1",
@@ -66,5 +66,5 @@
     "ts-jest": "29.1.1",
     "typescript": "5.2.2"
   },
-  "gitHead": "d3a98c4cf7140cc2d93e588d89011037b3b8987c"
+  "gitHead": "521d5d1ca2f86f0ed2bef0309b0dde30506f132b"
 }

package/src/Db.ts

@@ -145,8 +145,27 @@ export class Db<R extends Record = Record> implements DbService<R> {
     return defaultTransactionContextFactory;
   }
 
+  /**
+   * Boot-time reconciliation, in THE load-bearing order:
+   * 1. ensure the database exists;
+   * 2. pre-schema-sync migrations ({@link MigrationRunner.runPreSchemaSyncMigrations}) — data
+   *    repairs new schema invariants depend on (e.g. deduplicating rows before a unique-index
+   *    backfill, which fails loudly over violating data); an immediate no-op unless a migration
+   *    declares {@link Migration.preSchemaSync};
+   * 3. schema sync — create/alter every registered table (the DDL);
+   * 4. source-record sync.
+   */
   async init(): Promise<void> {
     await this.dbDriver.createDbIfNotExists();
+    // Resolved lazily, on purpose: Db sits at the BOTTOM of this package's module graph
+    // (Table -> Record -> Columns -> ReferenceArray -> Db), while MigrationRunner sits above it
+    // (it loads MigrationTable, whose class `extends Table`). A top-level import here closes
+    // that loop downward — an entry module that starts anywhere in the lower chain would then
+    // evaluate MigrationTable's `extends Table` while Table.ts is still mid-load (ES5
+    // downlevel: "Class extends value undefined is not a constructor"). A call-time import
+    // keeps the edge runtime-only, when the whole graph is complete.
+    const { MigrationRunner } = await import('./MigrationRunner');
+    await new MigrationRunner().runPreSchemaSyncMigrations(this.dbDriver.getTableManager());
     await this.dbDriver.getTableManager().loadTables();
     await new SourceRecordLoader().load();
   }

package/src/MigrationRunner.ts

@@ -2,9 +2,12 @@ import { Moment, moment } from './opt/moment';
 import { Db, getDb, getDbAsSystem } from './Db';
 import { Table } from './Table';
 import { SourceRecordRepo } from './source/SourceRecordRepo';
+import { SourceRecordLoader } from './source/SourceRecordLoader';
+import { getSourceRecordLoaders } from './source/SourceRecord';
 import { MigrationRunnerService, getMigrationRunnerService } from './services/MigrationRunnerService';
 import { Migration, MigrationTable } from './tables/MigrationTable';
 import { QueryBuilderFactory } from './QueryBuilderFactory';
+import { TableManager } from './schema/TableManager';
 import { Service } from '@proteinjs/service';
 import { Logger } from '@proteinjs/logger';
 
@@ -92,6 +95,65 @@ export class MigrationRunner implements MigrationRunnerService {
     return migration;
   }
 
+  /**
+   * Pre-schema-sync phase — called by {@link Db.init} between database creation and schema sync:
+   * runs every source-declared migration flagged {@link Migration.preSchemaSync}, so data repairs
+   * a new schema invariant depends on (e.g. deduplicating rows a new unique index would reject —
+   * TableManager's unique-index preflight fails loudly over violating data) land BEFORE the DDL
+   * that needs them. The ordinary series ({@link runPendingMigrations}) runs after init — too
+   * late for this class by construction.
+   *
+   * ZERO-COST WHEN UNUSED: no flagged migrations -> immediate return (no ledger IO, no DDL) —
+   * the common boot pays nothing.
+   *
+   * Bootstrap: the phase runs before schema sync, so it fronts the two pieces it needs itself —
+   * the migration TABLE's own schema (framework-owned, never data-hazardous) and the migration
+   * table's source-record sync (ledger rows + SourceRecordRepo registration, which
+   * {@link ensureMigrationRun}'s resolveMigration reads). The full loadTables/source sync that
+   * follows re-reconciles both idempotently.
+   *
+   * Runs through {@link ensureMigrationRun} — full ledger semantics: skip on 'success', retry on
+   * 'failure'/'running' (a crashed earlier boot). Multiple flagged migrations run in series
+   * ordered by id (deterministic; their ledger rows may not exist yet, so created-order cannot
+   * apply). A non-success outcome THROWS: Db.init must fail as loudly as the schema sync it
+   * protects would have — the boot crash / deploy-Job failure names the migration instead of an
+   * opaque index-backfill error, and the recorded failure row retries on the next boot.
+   */
+  async runPreSchemaSyncMigrations(tableManager: TableManager): Promise<void> {
+    const migrationTable: Table<Migration> = new MigrationTable();
+    const flagged = getSourceRecordLoaders<Migration>()
+      // db >=1.34.4: declarations are { source, loader } pairs — the loader carries table/record.
+      .filter(({ loader }) => loader.table.name === migrationTable.name && (loader.record as Migration).preSchemaSync)
+      .map(({ loader }) => loader.record as Migration)
+      .sort((a, b) => a.id.localeCompare(b.id));
+    if (flagged.length === 0) {
+      return;
+    }
+
+    const contradictions = flagged.filter((migration) => migration.manual);
+    if (contradictions.length > 0) {
+      throw new Error(
+        `Migration(s) declare both preSchemaSync and manual — a contradiction (the pre-schema-sync phase ` +
+          `exists to run unattended before DDL): ${contradictions.map((migration) => migration.id).join(', ')}`
+      );
+    }
+
+    await tableManager.loadTable(migrationTable);
+    await new SourceRecordLoader().load(migrationTable);
+    this.logger.info({
+      message: `Running ${flagged.length} pre-schema-sync migration${flagged.length === 1 ? '' : 's'} before schema sync`,
+      obj: { ids: flagged.map((migration) => migration.id) },
+    });
+    for (const migration of flagged) {
+      const outcome = await this.ensureMigrationRun(migration.id);
+      if (outcome.status !== 'success') {
+        throw new Error(
+          `Pre-schema-sync migration (${outcome.id}) failed; schema sync not attempted: ${outcome.failureMessage}`
+        );
+      }
+    }
+  }
+
   /**
    * Deploy-path API (plans/POST_RELEASE_QUEUE.md 27f): the deploy pipeline's migration Job calls
    * this AFTER `new Db().init()` (schema sync + source-record sync — every source-declared

package/src/source/SourceRecord.ts

@@ -1,10 +1,24 @@
 import { Loadable, SourceRepository } from '@proteinjs/reflection';
 import { Columns, Table, getTables } from '../Table';
 import { Record as DbRecord, withRecordColumns } from '../Record';
-import { BooleanColumn } from '../Columns';
+import { BooleanColumn, StringColumn } from '../Columns';
 
-export const getSourceRecordLoaders = <T extends SourceRecord = SourceRecord>() =>
-  SourceRepository.get().objects<SourceRecordLoader<T>>('@proteinjs/db/SourceRecordLoader');
+/**
+ * A source record declaration paired with its owning source: the package that compiled the
+ * declaration into this build (from the declaration's reflection qualified name). The source is
+ * the ownership grain of the sync — {@link SourceRecordLoader} stamps it on every row it writes
+ * and prunes only within it, so servers running different builds against one shared database
+ * never delete each other's rows.
+ */
+export type SourceRecordLoaderDeclaration<T extends SourceRecord = SourceRecord> = {
+  source: string;
+  loader: SourceRecordLoader<T>;
+};
+
+export const getSourceRecordLoaders = <T extends SourceRecord = SourceRecord>(): SourceRecordLoaderDeclaration<T>[] =>
+  SourceRepository.get()
+    .objectsWithNames<SourceRecordLoader<T>>('@proteinjs/db/SourceRecordLoader')
+    .map(({ packageName, object }) => ({ source: packageName, loader: object }));
 
 export function getSourceRecordTables() {
   const tables = getTables();
@@ -31,11 +45,31 @@ export function isSourceRecordTable(table: Table<any>) {
 
 export interface SourceRecord extends DbRecord {
   isLoadedFromSource?: boolean;
+  /**
+   * The package whose declaration owns this row (the declaring loader's package, from its
+   * reflection qualified name). Stamped by {@link SourceRecordLoader} on every row it writes;
+   * the removed-reconcile prunes only rows whose `sourcePackage` matches a package the running
+   * build actually declares from — so a build never deletes rows owned by a package it does not
+   * carry (e.g. another server's types on a shared database). Rows written before this column
+   * existed carry NULL until their owning package's next boot adopts and stamps them.
+   */
+  sourcePackage?: string;
+  /**
+   * The declaring package's version at the time this row was last stamped (from the package's
+   * own package.json, resolved at runtime). This is the ordering WITHIN a package that makes
+   * version skew safe on a shared database: a boot never prunes, flags, or rewrites a row stamped
+   * by a strictly NEWER version of the same package, so an older build cannot delete the types a
+   * newer build added or churn the ones it redefined. NULL (legacy rows, or builds whose package
+   * version could not be resolved) carries no ordering and keeps the last-writer-wins semantics.
+   */
+  sourcePackageVersion?: string;
 }
 
 const getSourceRecordColumns = (hideFromUi = true) => {
   return {
     isLoadedFromSource: new BooleanColumn('is_loaded_from_source', { ui: { hidden: hideFromUi } }),
+    sourcePackage: new StringColumn('source_package', { ui: { hidden: hideFromUi } }),
+    sourcePackageVersion: new StringColumn('source_package_version', { ui: { hidden: hideFromUi } }),
   };
 };
 
@@ -78,7 +112,11 @@ type OptionalProperties<T> = Pick<
  *
  * On Db.init, the record will be inserted if it doesn't exist, and updated if it does exist to mirror what is in source.
  *
- * If the SourceRecordLoader is deleted from source, the record will be deleted from the db on server startup. This will also be the behavior if id is changed - the record with the old id will be deleted.
+ * If the SourceRecordLoader is deleted from source, the record will be deleted from the db on server startup (per the
+ * table's `onSourceRemoved` policy) — by the next boot of a build that still carries the declaring package at its
+ * version or newer. Ownership is per package: a boot only reconciles rows owned by packages it carries, and never
+ * rows a newer version of the same package stamped; a package removed from every build leaves its rows behind.
+ * This will also be the behavior if id is changed - the record with the old id will be deleted.
  */
 export interface SourceRecordLoader<T extends SourceRecord> extends Loadable {
   table: Table<T>;