@proteinjs/db 1.33.0 → 1.34.1

package/index.ts

@@ -13,6 +13,10 @@ export * from './src/RecordIterator';
 export * from './src/CursorWindowPager';
 export * from './src/source/SourceRecord';
 export * from './src/source/SourceRecordRepo';
+// The sync runner behind Db.init's source-record leg, aliased: its class name is taken in this
+// index by the SourceRecordLoader declaration interface. Public so consumer test harnesses can
+// run one boot of the sync directly (the exports map blocks deep dist imports).
+export { SourceRecordLoader as SourceRecordSyncRunner } from './src/source/SourceRecordLoader';
 export * from './src/MigrationRunner';
 export * from './src/QueryBuilderFactory';
 export * from './src/TableWatcher';

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@proteinjs/db",
-  "version": "1.33.0",
+  "version": "1.34.1",
   "main": "./dist/generated/index.js",
   "types": "./dist/generated/index.d.ts",
   "exports": {
@@ -66,5 +66,5 @@
     "ts-jest": "29.1.1",
     "typescript": "5.2.2"
   },
-  "gitHead": "23b168ca8f82ba392500b545230adc6e8c761b4e"
+  "gitHead": "a5a56218a5757e47576cbe79858c9aacc6026816"
 }

package/src/Db.ts

@@ -8,8 +8,8 @@ import {
   tableByName,
   addDefaultFieldValues,
   addUpdateFieldValues,
-  getTables,
 } from './Table';
+import { ReverseCascadeEdgeIndex } from './ReverseCascadeEdgeIndex';
 import { Record, RecordSerializer, SerializedRecord } from './Record';
 import { Logger } from '@proteinjs/logger';
 import { SourceRecordLoader } from './source/SourceRecordLoader';
@@ -380,6 +380,8 @@ export class Db<R extends Record = Record> implements DbService<R> {
 
   /**
    * Reverse cascades driven by column-level flags on reference columns only.
+   * Edges come from the `ReverseCascadeEdgeIndex` (derived once per process from the static
+   * table registry) instead of a per-delete scan over every registered table's columns.
    * Supports:
    *  - ReferenceColumn
    *  - DynamicReferenceColumn
@@ -392,68 +394,41 @@ export class Db<R extends Record = Record> implements DbService<R> {
     }
 
     const deletedIdSet = new Set<string>(deletedIds);
-    const allTables = getTables();
+    for (const edge of ReverseCascadeEdgeIndex.get().getEdges(table.name)) {
+      const { referencingTable, columnPropertyName } = edge;
 
-    for (const referencingTable of allTables) {
-      for (const colPropName in referencingTable.columns) {
-        const col = referencingTable.columns[colPropName] as any;
+      if (edge.refKind === 'dynamicReference') {
+        const qb = new QueryBuilderFactory().getQueryBuilder(referencingTable);
+        await this.addColumnQueries(referencingTable, qb, 'read');
 
-        // Only act if the column explicitly opted in
-        if (!col || col.reverseCascadeDelete !== true) {
-          continue;
-        }
-
-        // DynamicReferenceColumn: has dynamicRefTableColName
-        if (typeof col.dynamicRefTableColName === 'string' && col.dynamicRefTableColName.length > 0) {
-          const dynTableProp = getColumnPropertyName(referencingTable, col.dynamicRefTableColName);
-          if (!dynTableProp) {
-            continue;
-          }
-
-          const qb = new QueryBuilderFactory().getQueryBuilder(referencingTable);
-          await this.addColumnQueries(referencingTable, qb, 'read');
+        qb.condition({ field: edge.dynamicRefTableColumnPropertyName as any, operator: '=', value: table.name as any });
+        qb.condition({ field: columnPropertyName as any, operator: 'IN', value: deletedIds as any });
 
-          qb.condition({ field: dynTableProp as any, operator: '=', value: table.name as any });
-          qb.condition({ field: colPropName as any, operator: 'IN', value: deletedIds as any });
-
-          this.logger.info({
-            message: `Executing reverse cascade (dynamic) for table: ${table.name}`,
-            obj: { referencingTable: referencingTable.name, columnPropertyName: colPropName, deletedIds },
-          });
+        this.logger.info({
+          message: `Executing reverse cascade (dynamic) for table: ${table.name}`,
+          obj: { referencingTable: referencingTable.name, columnPropertyName, deletedIds },
+        });
 
-          const deleteCount = await this.delete(referencingTable, qb);
-          this.logger.info({
-            message: `Reverse cascade (dynamic) deleted ${deleteCount} record${deleteCount == 1 ? '' : 's'}`,
-          });
-          continue;
-        }
+        const deleteCount = await this.delete(referencingTable, qb);
+        this.logger.info({
+          message: `Reverse cascade (dynamic) deleted ${deleteCount} record${deleteCount == 1 ? '' : 's'}`,
+        });
+      } else if (edge.refKind === 'reference') {
+        const qb = new QueryBuilderFactory().getQueryBuilder(referencingTable);
+        await this.addColumnQueries(referencingTable, qb, 'read');
+        qb.condition({ field: columnPropertyName as any, operator: 'IN', value: deletedIds as any });
 
-        // ReferenceColumn/ReferenceArrayColumn must match the target table
-        if (col.referenceTable !== table.name) {
-          continue;
-        }
+        this.logger.info({
+          message: `Executing reverse cascade (ReferenceColumn) for table: ${table.name}`,
+          obj: { referencingTable: referencingTable.name, columnPropertyName, deletedIds },
+        });
 
-        const ctorName = col.constructor?.name;
-
-        if (ctorName === 'ReferenceColumn') {
-          const qb = new QueryBuilderFactory().getQueryBuilder(referencingTable);
-          await this.addColumnQueries(referencingTable, qb, 'read');
-          qb.condition({ field: colPropName as any, operator: 'IN', value: deletedIds as any });
-
-          this.logger.info({
-            message: `Executing reverse cascade (ReferenceColumn) for table: ${table.name}`,
-            obj: { referencingTable: referencingTable.name, columnPropertyName: colPropName, deletedIds },
-          });
-
-          const deleteCount = await this.delete(referencingTable, qb);
-          this.logger.info({
-            message: `Reverse cascade (ReferenceColumn) deleted ${deleteCount} record${deleteCount == 1 ? '' : 's'}`,
-          });
-        } else if (ctorName === 'ReferenceArrayColumn') {
-          await this.reverseDeleteReferenceArrayHolders(referencingTable, colPropName, deletedIds, deletedIdSet);
-        } else {
-          continue;
-        }
+        const deleteCount = await this.delete(referencingTable, qb);
+        this.logger.info({
+          message: `Reverse cascade (ReferenceColumn) deleted ${deleteCount} record${deleteCount == 1 ? '' : 's'}`,
+        });
+      } else {
+        await this.reverseDeleteReferenceArrayHolders(referencingTable, columnPropertyName, deletedIds, deletedIdSet);
       }
     }
   }

package/src/MigrationRunner.ts

@@ -206,7 +206,7 @@ export class MigrationRunner implements MigrationRunnerService {
     const db = getRunDb();
     migration.status = 'running';
     migration.startTime = moment();
-    await db.update(migrationTable, migration);
+    await db.update(migrationTable, this.definedFields(migration));
     this.logger.info({ message: `Running migration (${migration.id}) ${migration.description}` });
     try {
       migration.output = await migration.run();
@@ -222,12 +222,31 @@ export class MigrationRunner implements MigrationRunnerService {
       migration.endTime = moment();
     }
     migration.duration = this.duration(migration.startTime, migration.endTime);
-    await db.update(migrationTable, migration);
+    await db.update(migrationTable, this.definedFields(migration));
     this.logger.info({
       message: `[${migration.status}] (${migration.duration}) Finished running migration (${migration.id}) ${migration.description}`,
     });
   }
 
+  /**
+   * The run-state payload with `undefined`-valued fields OMITTED. Several of the record's fields
+   * are legitimately absent depending on the run (`output` for a void `run()`, `failureMessage`/
+   * `failureStack` for a non-Error throw), but the migration object carries them as explicit
+   * `undefined` assignments — and `RecordSerializer` rejects any payload field holding `undefined`
+   * (never a partial write), which would strand the row at 'running' status with the run's real
+   * outcome lost. Absent means omitted, never undefined — for EVERY optional field of the payload,
+   * not per-field.
+   */
+  private definedFields(migration: Migration): Partial<Migration> {
+    const payload: Partial<Migration> = {};
+    for (const [field, value] of Object.entries(migration)) {
+      if (value !== undefined) {
+        (payload as any)[field] = value;
+      }
+    }
+    return payload;
+  }
+
   private resolveMigration(migrationTable: Table<Migration>, id: string): Migration {
     const migration = new SourceRecordRepo().getSourceRecord<Migration>(migrationTable.name, id);
     if (!migration) {

package/src/ReverseCascadeEdgeIndex.ts

@@ -0,0 +1,126 @@
+import { Table, getColumnPropertyName, getTables } from './Table';
+
+/**
+ * One reverse-cascade edge: a column flagged `reverseCascadeDelete: true` whose rows die when
+ * the record they point at is deleted. `dynamicReference` edges carry the resolved property
+ * name of the sibling column that stores the target table's name per row.
+ */
+export type ReverseCascadeEdge = {
+  /** The table holding the flagged column (its rows are the reverse-cascade victims). */
+  referencingTable: Table<any>;
+  /** Property name of the flagged column on `referencingTable.columns`. */
+  columnPropertyName: string;
+  /** Position in the registry walk (table order, then column order) — lookups preserve it. */
+  order: number;
+} & (
+  | { refKind: 'reference' | 'referenceArray' }
+  | { refKind: 'dynamicReference'; dynamicRefTableColumnPropertyName: string }
+);
+
+/**
+ * Derived in-memory reverse-cascade edge index over the static table registry.
+ *
+ * Maps the referenced-target dimension (`referencedTableName → [(referencingTable,
+ * columnProperty, refKind)]`) for every column flagged `reverseCascadeDelete: true`, so
+ * `Db.delete` consults an O(edges-for-target) lookup instead of re-scanning every registered
+ * table's columns on every delete. Table definitions are the source of truth and are static
+ * per process, so the index is built once — lazily on first use — and never invalidated.
+ *
+ * Edge derivation mirrors the three reference column shapes:
+ * - `ReferenceColumn` / `ReferenceArrayColumn` edges are keyed under their declared
+ *   `referenceTable` — they can only ever point at that one target.
+ * - `DynamicReferenceColumn` edges are target-agnostic (the target table name lives in a
+ *   sibling column per row), so they apply to every target table and are returned for all
+ *   lookups; the per-row table-name condition stays in the delete query, exactly as the
+ *   pre-index scan issued it.
+ */
+export class ReverseCascadeEdgeIndex {
+  private static instance: ReverseCascadeEdgeIndex | undefined;
+  private edgesByTargetTable = new Map<string, ReverseCascadeEdge[]>();
+  private dynamicEdges: ReverseCascadeEdge[] = [];
+
+  static get(): ReverseCascadeEdgeIndex {
+    if (!ReverseCascadeEdgeIndex.instance) {
+      ReverseCascadeEdgeIndex.instance = new ReverseCascadeEdgeIndex();
+    }
+
+    return ReverseCascadeEdgeIndex.instance;
+  }
+
+  /**
+   * All reverse-cascade edges that can point at `targetTableName`: the edges keyed under it
+   * plus every dynamic (target-agnostic) edge, in the same registry-walk order the pre-index
+   * all-tables scan visited them.
+   */
+  getEdges(targetTableName: string): ReverseCascadeEdge[] {
+    const exactEdges = this.edgesByTargetTable.get(targetTableName) ?? [];
+    if (exactEdges.length === 0) {
+      return this.dynamicEdges;
+    }
+
+    if (this.dynamicEdges.length === 0) {
+      return exactEdges;
+    }
+
+    return [...exactEdges, ...this.dynamicEdges].sort((a, b) => a.order - b.order);
+  }
+
+  private constructor() {
+    this.build();
+  }
+
+  private build(): void {
+    let order = 0;
+    for (const referencingTable of getTables()) {
+      for (const columnPropertyName in referencingTable.columns) {
+        const column = referencingTable.columns[columnPropertyName] as any;
+        if (!column || column.reverseCascadeDelete !== true) {
+          continue;
+        }
+
+        // DynamicReferenceColumn: identified by its sibling table-name column pointer
+        if (typeof column.dynamicRefTableColName === 'string' && column.dynamicRefTableColName.length > 0) {
+          const dynamicRefTableColumnPropertyName = getColumnPropertyName(
+            referencingTable,
+            column.dynamicRefTableColName
+          );
+          if (!dynamicRefTableColumnPropertyName) {
+            continue;
+          }
+
+          this.dynamicEdges.push({
+            referencingTable,
+            columnPropertyName,
+            refKind: 'dynamicReference',
+            dynamicRefTableColumnPropertyName,
+            order: order++,
+          });
+          continue;
+        }
+
+        if (typeof column.referenceTable !== 'string') {
+          continue;
+        }
+
+        const ctorName = column.constructor?.name;
+        const refKind =
+          ctorName === 'ReferenceColumn'
+            ? 'reference'
+            : ctorName === 'ReferenceArrayColumn'
+              ? 'referenceArray'
+              : undefined;
+        if (!refKind) {
+          continue;
+        }
+
+        let targetEdges = this.edgesByTargetTable.get(column.referenceTable);
+        if (!targetEdges) {
+          targetEdges = [];
+          this.edgesByTargetTable.set(column.referenceTable, targetEdges);
+        }
+
+        targetEdges.push({ referencingTable, columnPropertyName, refKind, order: order++ });
+      }
+    }
+  }
+}

package/src/Table.ts

@@ -87,13 +87,8 @@ export abstract class Table<T extends Record> implements Loadable, CustomSeriali
   public indexes: { columns: (keyof T)[]; name?: string; unique?: boolean }[] = [];
   /** When records are deleted, delete records having references pointing to deleted records */
   public cascadeDeleteReferences: () => { table: string; referenceColumn: string }[] = () => [];
-  /**
-   * Options for configuring SourceRecords
-   * @param doNotDeleteSourceRecordsFromDb if true, the SourceRecordLoader will not delete source records from the db if they no longer exist on the file system
-   */
-  public sourceRecordOptions: SourceRecordOptions = {
-    doNotDeleteSourceRecordsFromDb: false,
-  };
+  /** Options for configuring SourceRecords (see {@link SourceRecordOptions}) */
+  public sourceRecordOptions: SourceRecordOptions<T> = {};
   public auth?: {
     db?: TableOperationsAuth;
     service?: TableOperationsAuth;
@@ -180,8 +175,36 @@ export type ColumnOptions = {
   };
 };
 
-export type SourceRecordOptions = {
-  doNotDeleteSourceRecordsFromDb?: boolean;
+export type SourceRecordOptions<T = any> = {
+  /**
+   * What the source-record sync does with rows it previously loaded from source
+   * (`is_loaded_from_source = true`) whose declaration no longer exists:
+   * - `'delete'` (default): delete the rows.
+   * - `'keep'`: leave the rows untouched (e.g. the migration ledger — run history outlives the
+   *   migration class).
+   * - `{ update }`: apply the patch to the removed rows — e.g. a machine-account table flagging
+   *   removed accounts `{ update: { status: 'deactivated' } }` instead of deleting them. The
+   *   patch is applied only to rows whose fields actually differ (idempotent boots), through
+   *   `Db.update`, so table watchers observe the write. Re-declaring the record reverts the
+   *   patch via normal drift reversion — removal is reversible in source.
+   *
+   * Rows never loaded from source are structurally untouchable by every policy: the removed
+   * reconcile only ever matches `is_loaded_from_source = true`.
+   */
+  onSourceRemoved?: 'delete' | 'keep' | { update: Partial<T> };
+  /**
+   * When set, the sync keys records on this column instead of `id` — matching, adoption, and
+   * the removed reconcile all use it. An existing row matched by natural key is ADOPTED in
+   * place: it keeps its id (the declared id is used only for fresh inserts — existing ids may
+   * be referenced from other tables), gets stamped `is_loaded_from_source = true`, and has its
+   * declared fields reverted to source. Drift comparison excludes `id`, so adoption converges.
+   *
+   * Preconditions, validated at boot by the loader (loud failures):
+   * - the column is declared unique (`ColumnOptions.unique` or a single-column unique index in
+   *   {@link Table.indexes});
+   * - every declaration provides the natural key, and no two declarations share a value.
+   */
+  naturalKey?: keyof T & string;
   ui?: {
     hideColumns?: boolean;
   };

package/src/schema/TableManager.ts

@@ -1,9 +1,33 @@
 import { Logger } from '@proteinjs/logger';
-import { Column, Table, getTables } from '../Table';
+import { QueryBuilder } from '@proteinjs/db-query';
+import { Column, Table, getColumnPropertyName, getTables } from '../Table';
 import { SchemaOperations, TableChanges } from './SchemaOperations';
 import { SchemaMetadata } from './SchemaMetadata';
 import { DbDriver } from '../Db';
 import { DynamicReferenceColumn, DynamicReferenceTableNameColumn } from '../Columns';
+import { StatementConfigFactory } from '../StatementConfigFactory';
+
+/**
+ * Thrown by the pre-sync duplicate check when a unique index is about to be added to an existing
+ * table whose data already violates it. Without the preflight the backend's index backfill fails
+ * with an opaque driver error mid-boot; this names the table, the columns, and sample offending
+ * values so the operator can resolve the duplicates and boot again.
+ */
+export class DuplicateValuesForUniqueIndexError extends Error {
+  constructor(tableName: string, columnNames: string[], duplicateValues: unknown[]) {
+    super(
+      `(${tableName}) Cannot create a unique index on (${columnNames.join(', ')}): existing rows hold duplicate ` +
+        `values. Resolve the duplicates, then boot again. Duplicate values (up to ` +
+        `${DuplicateValuesForUniqueIndexError.MAX_REPORTED_VALUES} shown): ${JSON.stringify(duplicateValues)}`
+    );
+    this.name = 'DuplicateValuesForUniqueIndexError';
+    // ES5 down-leveled `extends Error` loses the subclass prototype — restore it so
+    // `instanceof DuplicateValuesForUniqueIndexError` holds for catchers.
+    Object.setPrototypeOf(this, DuplicateValuesForUniqueIndexError.prototype);
+  }
+
+  static readonly MAX_REPORTED_VALUES = 20;
+}
 
 const getEnvVar = (key: string): string | undefined =>
   typeof process !== 'undefined' && process.env ? process.env[key] : undefined;
@@ -22,6 +46,7 @@ export class TableManager {
   private static readonly RECONCILE_VERIFY_ATTEMPTS = 5;
   private static readonly RECONCILE_VERIFY_RETRY_DELAY_MS = 200;
   private logger = new Logger({ name: this.constructor.name, logLevel: getEnvVar('DB_LOG_LEVEL') as any });
+  private dbDriver: DbDriver;
   public columnTypeFactory: ColumnTypeFactory;
   public schemaOperations: SchemaOperations;
   public schemaMetadata: SchemaMetadata;
@@ -32,6 +57,7 @@ export class TableManager {
     schemaOperations: SchemaOperations,
     schemaMetadata?: SchemaMetadata
   ) {
+    this.dbDriver = dbDriver;
     this.columnTypeFactory = columnTypeFactory;
     this.schemaOperations = schemaOperations;
     this.schemaMetadata = schemaMetadata ? schemaMetadata : new SchemaMetadata(dbDriver);
@@ -95,6 +121,7 @@ export class TableManager {
   private async alterTableIfChanged(table: Table<any>): Promise<void> {
     const tableChanges = await this.getTableChanges(table);
     if (this.shouldAlterTable(tableChanges)) {
+      await this.preflightUniqueIndexAdditions(table, tableChanges);
       this.logger.info({ message: `Altering table: ${table.name}` });
       try {
         await this.schemaOperations.alterTable(table, tableChanges);
@@ -431,4 +458,74 @@ export class TableManager {
 
     return { indexesToCreate, indexesToDrop };
   }
+
+  /**
+   * Pre-sync duplicate check: before an alter adds a unique index (via `ColumnOptions.unique` or
+   * a unique entry in `Table.indexes`) to an EXISTING table, verify the data can satisfy it.
+   * Existing duplicates would otherwise surface as an opaque backend backfill failure mid-boot;
+   * this throws {@link DuplicateValuesForUniqueIndexError} naming the table, columns, and sample
+   * offending values instead. Runs only on the alter path (a freshly created table has no rows),
+   * and only when a unique index is actually being added — steady-state boots never pay for it.
+   */
+  private async preflightUniqueIndexAdditions(table: Table<any>, tableChanges: TableChanges): Promise<void> {
+    const uniqueAdditions: string[][] = tableChanges.columnsWithUniqueConstraintsToCreate.map((columnName) => [
+      columnName,
+    ]);
+    for (const index of tableChanges.indexesToCreate) {
+      if (index.unique) {
+        uniqueAdditions.push(Array.isArray(index.columns) ? index.columns : [index.columns]);
+      }
+    }
+
+    for (const columnNames of uniqueAdditions) {
+      await this.assertNoDuplicateValues(table, columnNames);
+    }
+  }
+
+  /**
+   * Read the key column(s) of every row (NULLs excluded — a unique index admits them) and group
+   * in memory. A full-column read is acceptable here: it runs once, at boot, only when a NEW
+   * unique index is being added over existing data.
+   */
+  private async assertNoDuplicateValues(table: Table<any>, columnNames: string[]): Promise<void> {
+    const propertyNames = columnNames.map((columnName) => {
+      const propertyName = getColumnPropertyName(table, columnName);
+      if (!propertyName) {
+        throw new Error(`(${table.name}) Unable to resolve a column property for column: ${columnName}`);
+      }
+
+      return propertyName;
+    });
+
+    const qb = new QueryBuilder(table.name).select({ fields: propertyNames });
+    for (const propertyName of propertyNames) {
+      qb.condition({ field: propertyName, operator: 'IS NOT NULL' });
+    }
+
+    const statementConfigFactory = new StatementConfigFactory(this.dbDriver.getDbName());
+    const rows = await this.dbDriver.runQuery((config) => qb.toSql(statementConfigFactory.getStatementConfig(config)));
+
+    const counts = new Map<string, { count: number; values: unknown[] }>();
+    for (const row of rows) {
+      const values = columnNames.map((columnName) => (row as any)[columnName]);
+      const key = JSON.stringify(values);
+      const entry = counts.get(key) ?? { count: 0, values };
+      entry.count += 1;
+      counts.set(key, entry);
+    }
+
+    const duplicates: unknown[] = [];
+    for (const entry of Array.from(counts.values())) {
+      if (entry.count > 1) {
+        duplicates.push(entry.values.length === 1 ? entry.values[0] : entry.values);
+        if (duplicates.length >= DuplicateValuesForUniqueIndexError.MAX_REPORTED_VALUES) {
+          break;
+        }
+      }
+    }
+
+    if (duplicates.length > 0) {
+      throw new DuplicateValuesForUniqueIndexError(table.name, columnNames, duplicates);
+    }
+  }
 }