@proteinjs/db 1.54.0 → 1.54.2

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@proteinjs/db",
-  "version": "1.54.0",
+  "version": "1.54.2",
   "main": "./dist/generated/index.js",
   "types": "./dist/generated/index.d.ts",
   "exports": {
@@ -66,5 +66,5 @@
     "ts-jest": "29.1.1",
     "typescript": "5.2.2"
   },
-  "gitHead": "856ac58e8bb61e5d46261ffb57434994fb808390"
+  "gitHead": "d90bd35b32b628c2b9cf9d8ddb16b4a2b9140f11"
 }

package/src/Columns.ts

@@ -74,7 +74,7 @@ export class BooleanColumn implements Column<boolean | null, boolean | null> {
     return null;
   }
 
-  async deserialize(serializedFieldValue: boolean | null): Promise<boolean | null> {
+  deserialize(serializedFieldValue: boolean | null): boolean | null {
     if (serializedFieldValue) {
       return true;
     }
@@ -95,6 +95,13 @@ export class DateColumn implements Column<Date, Date> {
 }
 
 export class DateTimeColumn implements Column<moment.Moment | null, Date | null> {
+  /**
+   * The moment is built on the field's first read, not at hydration (`Column.deserializeOnFirstRead`):
+   * a row carries several of these and most are never read, and a moment costs more to build than
+   * the stored Date it is built from.
+   */
+  readonly deserializeOnFirstRead = true;
+
   constructor(
     public name: string,
     public options?: ColumnOptions
@@ -112,7 +119,7 @@ export class DateTimeColumn implements Column<moment.Moment | null, Date | null>
     return fieldValue.toDate();
   }
 
-  async deserialize(serializedFieldValue: Date | null): Promise<moment.Moment | null> {
+  deserialize(serializedFieldValue: Date | null): moment.Moment | null {
     if (serializedFieldValue === undefined || serializedFieldValue === null) {
       return null;
     }
@@ -186,7 +193,7 @@ export class ObjectColumn<T> extends StringColumn<T> {
     return Serializer.serialize(fieldValue);
   }
 
-  async deserialize(serializedFieldValue: string): Promise<T | null> {
+  deserialize(serializedFieldValue: string): T | null {
     if (serializedFieldValue === undefined || serializedFieldValue == null) {
       return null;
     }
@@ -247,8 +254,8 @@ export class ReferenceArrayColumn<T extends Record> extends ObjectColumn<Referen
     return await super.serialize(returnIds as any);
   }
 
-  async deserialize(serializedFieldValue: string): Promise<ReferenceArray<T> | null> {
-    let ids = (await super.deserialize(serializedFieldValue)) as string[] | null;
+  deserialize(serializedFieldValue: string): ReferenceArray<T> | null {
+    let ids = super.deserialize(serializedFieldValue) as string[] | null;
     if (ids === null) {
       ids = [];
     }
@@ -315,7 +322,7 @@ export class ReferenceColumn<T extends Record> extends StringColumn<Reference<T>
     return fieldValue._id;
   }
 
-  async deserialize(serializedFieldValue: string): Promise<Reference<T> | null> {
+  deserialize(serializedFieldValue: string): Reference<T> | null {
     const reference = new Reference(this.referenceTable, serializedFieldValue);
     if (reference._id === null) {
       return null;
@@ -487,7 +494,7 @@ export class DynamicReferenceColumn<T extends Record> extends StringColumn<Refer
     return fieldValue._id;
   }
 
-  async deserialize(serializedFieldValue: string, serializedRecord: any): Promise<Reference<T> | null> {
+  deserialize(serializedFieldValue: string, serializedRecord: any): Reference<T> | null {
     const reference = new Reference('', serializedFieldValue);
     if (reference._id === null) {
       return null;

package/src/Db.ts

@@ -750,10 +750,9 @@ export class Db<R extends Record = Record> implements DbService<R> {
     const generateQuery = (config: DbDriverQueryStatementConfig) =>
       qb.toSql(this.statementConfigFactory.getStatementConfig(config));
     const serializedRecords = await this.dbDriver.runQuery(generateQuery, this.transactionForDriver());
-    const recordSerializer = new RecordSerializer(table);
-    const records = await Promise.all(
-      serializedRecords.map(async (serializedRecord) => recordSerializer.deserialize(serializedRecord))
-    );
+    // One pass over the result set — synchronous when no row needs an await (see
+    // `RecordSerializer.deserializeAll`): no promise per row, no yield of the loop between rows.
+    const records = await new RecordSerializer<T>(table).deserializeAll(serializedRecords);
     await this.preloadReferences(records, options);
     return records;
   }

package/src/Record.ts

@@ -2,6 +2,7 @@ import { Logger } from '@proteinjs/logger';
 import { DateTimeColumn, UuidColumn } from './Columns';
 import { Column, Table, Columns } from './Table';
 import { moment } from './opt/moment';
+import type { EncryptionRecordHooks } from './encryption/EncryptionRecordHooks';
 
 export interface Record {
   id: string;
@@ -51,10 +52,26 @@ export type SerializedRecord = { [columnName: string]: any };
  *  (see `EncryptionRecordHooks`). Resolved by `Db` for writes touching encrypted columns. */
 export type RecordEncryptionContext = { keyOwner: string };
 
+/** One column's seat in a hydration: the property the record carries it under, and the column. */
+interface HydrationSlot {
+  propertyName: string;
+  column: Column<any, any>;
+}
+
+/**
+ * What a hydration resolves ONCE per serializer, never per row: the decryption hook and every
+ * column's seat, keyed by the name a stored row carries it under.
+ */
+interface HydrationPlan {
+  hooks: EncryptionRecordHooks;
+  slotsByColumnName: { [columnName: string]: HydrationSlot | undefined };
+}
+
 export class RecordSerializer<T extends Record> {
   private logger = new Logger({ name: this.constructor.name });
   private table: Table<T>;
   private encryptionContext?: RecordEncryptionContext;
+  private plan?: HydrationPlan;
 
   constructor(table: Table<T>, encryptionContext?: RecordEncryptionContext) {
     this.table = table;
@@ -97,35 +114,152 @@ export class RecordSerializer<T extends Record> {
     return serialized;
   }
 
+  /**
+   * A result set's rows → records, in ONE pass. A row whose hook and columns all answer directly
+   * is hydrated synchronously, and when every row did the array returns synchronously: no promise
+   * per row, no yield of the event loop between rows. A row that needs an await — a ciphertext
+   * envelope to decrypt, a column whose `deserialize` answers with a promise — settles through one
+   * promise for the whole set. The hook and the column seats are resolved once per serializer.
+   */
+  deserializeAll(serializedRecords: SerializedRecord[]): T[] | Promise<T[]> {
+    const plan = this.hydrationPlan();
+    const records: (T | Promise<T>)[] = new Array(serializedRecords.length);
+    let settles = false;
+    for (let i = 0; i < serializedRecords.length; i++) {
+      const record = this.hydrate(plan, serializedRecords[i]);
+      if (this.isPromise(record)) {
+        settles = true;
+      }
+      records[i] = record;
+    }
+
+    return settles ? Promise.all(records) : (records as T[]);
+  }
+
+  /** One row → one record (`deserializeAll` is the door for a result set). */
   async deserialize(serializedRecord: SerializedRecord): Promise<T> {
+    return await this.hydrate(this.hydrationPlan(), serializedRecord);
+  }
+
+  private hydrate(plan: HydrationPlan, serializedRecord: SerializedRecord): T | Promise<T> {
     // The transparent decryption seam: ciphertext envelopes decrypt (self-describing — the
-    // envelope names its key), framework companion columns drop out of the result.
-    const { EncryptionRecordHooks } = await import('./encryption/EncryptionRecordHooks');
-    const prepared = await new EncryptionRecordHooks().onDeserialize(this.table, serializedRecord);
+    // envelope names its key), framework companion columns drop out of the result. The hook
+    // answers directly for a row holding no envelope.
+    const prepared = plan.hooks.onDeserialize(this.table, serializedRecord);
+    if (this.isPromise(prepared)) {
+      return prepared.then((row) => this.hydratePrepared(plan, row));
+    }
 
-    const deserialized: any = {};
-    const fieldSerializer = new FieldSerializer(this.table);
-    const omittedFields: string[] = [];
+    return this.hydratePrepared(plan, prepared);
+  }
+
+  private hydratePrepared(plan: HydrationPlan, prepared: SerializedRecord): T | Promise<T> {
+    const record: any = {};
+    let settling: Promise<void>[] | undefined;
+    let omittedFields: string[] | undefined;
     for (const columnName in prepared) {
-      const serializedFieldValue = prepared[columnName];
-      try {
-        const { fieldPropertyName, fieldValue } = await fieldSerializer.deserialize(
-          columnName,
-          serializedFieldValue,
-          prepared
+      const slot = plan.slotsByColumnName[columnName];
+      if (!slot) {
+        // a column exists in the db that is no longer defined in Table.columns
+        (omittedFields ??= []).push(columnName);
+        continue;
+      }
+
+      const { propertyName, column } = slot;
+      const storedValue = prepared[columnName];
+      if (!column.deserialize) {
+        record[propertyName] = storedValue;
+        continue;
+      }
+
+      if (column.deserializeOnFirstRead && storedValue !== null && storedValue !== undefined) {
+        this.defineFieldReadOnFirstRead(record, propertyName, column, storedValue);
+        continue;
+      }
+
+      const fieldValue = column.deserialize(storedValue, prepared);
+      if (this.isPromise(fieldValue)) {
+        record[propertyName] = undefined; // seats the property in column order; the value lands when it settles
+        (settling ??= []).push(
+          fieldValue.then((value) => {
+            record[propertyName] = value;
+          })
         );
-        deserialized[fieldPropertyName] = fieldValue;
-      } catch (MissingFieldError) {
-        omittedFields.push(columnName);
+        continue;
       }
+
+      record[propertyName] = fieldValue;
     }
 
-    if (omittedFields.length > 0) {
+    if (omittedFields) {
       // expected when passing a base table into the query api
       this.logger.debug({ message: `Fields were omitted during deserialization`, obj: { omittedFields } });
     }
 
-    return deserialized;
+    if (settling) {
+      return Promise.all(settling).then(() => record as T);
+    }
+
+    return record as T;
+  }
+
+  /**
+   * The field as an own, enumerable property whose value is deserialized on its first read, once
+   * (`Column.deserializeOnFirstRead`); a write replaces it like any other field.
+   */
+  private defineFieldReadOnFirstRead(
+    record: any,
+    propertyName: string,
+    column: Column<any, any>,
+    storedValue: unknown
+  ): void {
+    let value: unknown;
+    let read = false;
+    Object.defineProperty(record, propertyName, {
+      enumerable: true,
+      configurable: true,
+      get() {
+        if (!read) {
+          value = column.deserialize!(storedValue, undefined);
+          read = true;
+        }
+
+        return value;
+      },
+      set(next: unknown) {
+        value = next;
+        read = true;
+      },
+    });
+  }
+
+  private hydrationPlan(): HydrationPlan {
+    if (this.plan) {
+      return this.plan;
+    }
+
+    // Required at call time — this module sits below the encryption machinery in the package's
+    // module graph (the seam `Table.ts` uses for the same reason).
+    // eslint-disable-next-line @typescript-eslint/no-var-requires
+    const { EncryptionRecordHooks } =
+      require('./encryption/EncryptionRecordHooks') as typeof import('./encryption/EncryptionRecordHooks');
+    const slotsByColumnName: HydrationPlan['slotsByColumnName'] = {};
+    const columns: { [propertyName: string]: Column<any, any> } = this.table.columns;
+    // Seated by column name, then by property name — a stored key that is a property name
+    // resolves to that property's column first, as it always has.
+    for (const propertyName in columns) {
+      slotsByColumnName[columns[propertyName].name] = { propertyName, column: columns[propertyName] };
+    }
+    for (const propertyName in columns) {
+      slotsByColumnName[propertyName] = { propertyName, column: columns[propertyName] };
+    }
+
+    this.plan = { hooks: new EncryptionRecordHooks(), slotsByColumnName };
+    return this.plan;
+  }
+
+  private isPromise<V>(value: V | Promise<V>): value is Promise<V> {
+    return typeof (value as { then?: unknown } | null | undefined)?.then === 'function';
   }
 }
 

package/src/Table.ts

@@ -263,7 +263,27 @@ export type Column<T, Serialized> = {
   oldName?: string;
   options?: ColumnOptions;
   serialize?: (fieldValue: T | null | undefined) => Promise<Serialized | null | undefined>;
-  deserialize?: (serializedFieldValue: Serialized | null, serializedRecord: any) => Promise<T | null | void>;
+  /**
+   * The stored value → the field's value. Answers directly when building the value needs no
+   * await (every column this package ships does), or with a promise when it does: record
+   * hydration (`RecordSerializer.deserializeAll`) awaits only what comes back as a promise, so a
+   * result set whose rows all answer directly hydrates synchronously — no promise per row, no
+   * yield of the event loop between rows.
+   */
+  deserialize?: (
+    serializedFieldValue: Serialized | null,
+    serializedRecord: any
+  ) => T | null | void | Promise<T | null | void>;
+  /**
+   * When true, a hydrated record builds this column's field on the FIRST READ of the field, not
+   * at hydration: `deserialize` runs then, once, with the stored value alone (the row is not
+   * kept), so a deserializer under this flag reads nothing but its value and answers directly,
+   * never with a promise. For a pure, synchronous deserializer whose value costs more to build
+   * than to store and is often never read on a row — a DateTime column's moment. The record's
+   * shape is unchanged: the field is an own, enumerable property that reads as the deserialized
+   * value and takes a write like any other; a stored null reads null, undeferred.
+   */
+  deserializeOnFirstRead?: boolean;
   /**
    * The column's QUERY-side contract, paralleling `serialize`/`deserialize` on the storage
    * side: how uses of this column in a query (conditions, ORDER BY, aggregation, GROUP BY)

package/src/encryption/EncryptionRecordHooks.ts

@@ -9,6 +9,9 @@ import { SearchTokenizer } from './SearchTokenizer';
 /** Who a row being written belongs to — the data key its values encrypt under. */
 export type EncryptionWriteContext = { keyOwner: string };
 
+/** A stored cell holding a ciphertext envelope, and the key the envelope names. */
+type EnvelopeCell = { columnName: string; owner: string; version: number };
+
 /**
  * The transparent encrypt/decrypt seam, invoked by `RecordSerializer` on every write's
  * serialize and every read's deserialize — the same per-column hook layer every caller
@@ -31,6 +34,7 @@ export class EncryptionRecordHooks {
   private encryptedColumns = new EncryptedColumns();
   private envelope = new EncryptionEnvelope();
   private tokenizer = new SearchTokenizer();
+  private internalNames?: { table: Table<any>; names: Set<string> };
 
   /** Encrypt encrypted-column values in `serialized` (column-name-keyed) and add companions. */
   async onSerialize(table: Table<any>, serialized: SerializedRecord, context?: EncryptionWriteContext): Promise<void> {
@@ -88,34 +92,46 @@ export class EncryptionRecordHooks {
   }
 
   /**
-   * A copy of `serializedRecord` with framework companion columns dropped and every
-   * envelope value decrypted (any column — self-describing ciphertext also covers the
-   * decrypt-out transition, where the config no longer marks the column).
+   * `serializedRecord` with framework companion columns dropped and every envelope value
+   * decrypted (any column — self-describing ciphertext also covers the decrypt-out transition,
+   * where the config no longer marks the column). Answers directly — the row itself when
+   * nothing drops or decrypts, else a copy — unless an envelope must be decrypted: its key is
+   * read asynchronously, so the copy settles through a promise.
    */
-  async onDeserialize(table: Table<any>, serializedRecord: SerializedRecord): Promise<SerializedRecord> {
+  onDeserialize(table: Table<any>, serializedRecord: SerializedRecord): SerializedRecord | Promise<SerializedRecord> {
     this.encryptedColumns.ensureSchema(table);
-    const prepared: SerializedRecord = {};
     const internalColumnNames = this.internalColumnNames(table);
-    let decrypted = false;
-    for (const columnName of Object.keys(serializedRecord)) {
+    let dropsInternal = false;
+    let envelopes: EnvelopeCell[] | undefined;
+    for (const columnName in serializedRecord) {
       if (internalColumnNames.has(columnName)) {
+        dropsInternal = true;
         continue;
       }
 
-      const value = serializedRecord[columnName];
-      const parsed = this.envelope.parse(value);
-      if (!parsed) {
-        prepared[columnName] = value;
-        continue;
+      const parsed = this.envelope.parse(serializedRecord[columnName]);
+      if (parsed) {
+        (envelopes ??= []).push({ columnName, owner: parsed.owner, version: parsed.version });
       }
+    }
 
-      this.assertServerSide();
-      const key = await new DataKeyStore().getKeyByVersion(parsed.owner, parsed.version);
-      prepared[columnName] = this.envelope.decrypt(value as string, key);
-      decrypted = true;
+    if (!dropsInternal && !envelopes) {
+      return serializedRecord;
     }
 
-    return decrypted || internalColumnNames.size > 0 ? prepared : serializedRecord;
+    const prepared: SerializedRecord = {};
+    for (const columnName in serializedRecord) {
+      if (!internalColumnNames.has(columnName)) {
+        prepared[columnName] = serializedRecord[columnName];
+      }
+    }
+
+    if (!envelopes) {
+      return prepared;
+    }
+
+    this.assertServerSide();
+    return this.decryptInto(prepared, envelopes);
   }
 
   /**
@@ -144,7 +160,21 @@ export class EncryptionRecordHooks {
     );
   }
 
+  private async decryptInto(prepared: SerializedRecord, envelopes: EnvelopeCell[]): Promise<SerializedRecord> {
+    for (const cell of envelopes) {
+      const key = await new DataKeyStore().getKeyByVersion(cell.owner, cell.version);
+      prepared[cell.columnName] = this.envelope.decrypt(prepared[cell.columnName] as string, key);
+    }
+
+    return prepared;
+  }
+
+  /** The table's framework-derived companion column names — resolved once per table per instance, not per row. */
   private internalColumnNames(table: Table<any>): Set<string> {
+    if (this.internalNames?.table === table) {
+      return this.internalNames.names;
+    }
+
     const names = new Set<string>();
     for (const prop of Object.keys(table.columns)) {
       const column = (table.columns as any)[prop];
@@ -153,6 +183,7 @@ export class EncryptionRecordHooks {
       }
     }
 
+    this.internalNames = { table, names };
     return names;
   }
 

package/src/services/TransactionRunnerService.ts

@@ -17,7 +17,9 @@ export type TransactionRunOptions = {
    * Ids of rows the operations depend on that may not exist yet: the row an update or a
    * membership update targets while its insert is still in flight, the permission source a
    * child row attaches to while the root's own insert (and the owner grant that commits with
-   * it) is still landing. Every id must be a row the operations themselves reference. The
+   * it) is still landing, any row an insert names through one of its table's reference columns
+   * while that row is still being born. Every id must be a row the operations themselves
+   * reference (an insert references every row its reference columns name). The
    * runner waits, bounded, for each to exist AND be visible to the caller before running the
    * operations; a row that never appears fails the request with a plain clause — never a
    * silent zero-row write, never a hang.

package/src/transaction/TransactionRunner.ts

@@ -1,6 +1,11 @@
 import { Logger } from '@proteinjs/logger';
+import { isInstanceOf } from '@proteinjs/util';
 import { Db, getDb } from '../Db';
-import { Table, tableByName } from '../Table';
+import { Column, Table, getColumnPropertyName, tableByName } from '../Table';
+// After Table: Columns ↔ Table ↔ Record load in a cycle that resolves only when Record is entered
+// before Columns (Record constructs its columns at module load); a module that enters Columns
+// first — as this one would when it is the first module loaded — leaves those constructors undefined.
+import { DynamicReferenceColumn, ReferenceColumn } from '../Columns';
 import { getDefaultTransactionContextFactory } from './TransactionContextFactory';
 import {
   TransactionRunnerService,
@@ -135,10 +140,11 @@ export class TransactionRunner implements TransactionRunnerService {
   }
 
   /**
-   * The rows the operations reference, by id → the table(s) they are read from: the permission
-   * source a row attaches to (an insert whose `permissionSource` is another row), the row an
-   * update / preserving update / membership update / delete targets. The rows the operations
-   * insert themselves are not dependencies.
+   * The rows the operations reference, by id → the table(s) they are read from. An insert
+   * references every row its table's reference columns name (`rowsTheInsertReferences`) — the
+   * permission source a row attaches to is one of these, a row named through any other reference
+   * column another. An update / preserving update / membership update / delete references the row
+   * it targets. The rows the operations insert themselves are not dependencies.
    */
   private static rowsTheOperationsReference(ops: Operation<any>[]): Map<string, Table<any>[]> {
     const inserted: string[] = [];
@@ -161,16 +167,12 @@ export class TransactionRunner implements TransactionRunnerService {
       const args = op.args as unknown[];
       const table = args[0] as Table<any> | undefined;
       if (op.name === 'insert') {
-        const record = args[1] as
-          | { id?: string; permissionSource?: { _table?: string; _id?: string }; permissionSourceTable?: string }
-          | undefined;
+        const record = args[1] as { id?: string } | undefined;
         if (record?.id) {
           inserted.push(record.id);
         }
-        const source = record?.permissionSource;
-        if (source?._id && source._id !== record?.id) {
-          const sourceTableName = record?.permissionSourceTable ?? source._table;
-          reference(sourceTableName ? TransactionRunner.tableNamed(sourceTableName, table) : table, source._id);
+        for (const row of TransactionRunner.rowsTheInsertReferences(table, record)) {
+          reference(TransactionRunner.tableNamed(row.tableName, table), row.id);
         }
       } else if (op.name === 'update' || op.name === 'updatePreserving') {
         const record = args[1] as { id?: string } | undefined;
@@ -188,7 +190,45 @@ export class TransactionRunner implements TransactionRunnerService {
     return referenced;
   }
 
-  /** The table a permission source lives in: the insert's own table when it names the same one. */
+  /**
+   * The rows an inserted record names through its table's reference columns: for each column
+   * whose value carries an id, that id and the table it lives in — a `ReferenceColumn`'s declared
+   * table; for a `DynamicReferenceColumn`, the table its paired table-name column carries on the
+   * record (a client's queued insert ran the column's default), else the reference's own.
+   */
+  private static rowsTheInsertReferences(
+    table: Table<any> | undefined,
+    record: { [field: string]: unknown } | undefined
+  ): { id: string; tableName: string }[] {
+    if (!table || !record) {
+      return [];
+    }
+
+    const rows: { id: string; tableName: string }[] = [];
+    for (const [field, column] of Object.entries(table.columns as { [field: string]: Column<any, any> })) {
+      const value = record[field] as { _id?: unknown; _table?: unknown } | null | undefined;
+      if (!value || typeof value._id !== 'string') {
+        continue;
+      }
+      let tableName: unknown;
+      if (isInstanceOf(column, ReferenceColumn)) {
+        tableName = (column as ReferenceColumn<any>).referenceTable;
+      } else if (isInstanceOf(column, DynamicReferenceColumn)) {
+        const tableNameField = getColumnPropertyName(
+          table,
+          (column as DynamicReferenceColumn<any>).dynamicRefTableColName
+        );
+        tableName = (tableNameField && record[tableNameField]) || value._table;
+      }
+      if (typeof tableName === 'string' && tableName) {
+        rows.push({ id: value._id, tableName });
+      }
+    }
+
+    return rows;
+  }
+
+  /** The table a referenced row lives in: the insert's own table when it names the same one. */
   private static tableNamed(tableName: string, inserting: Table<any> | undefined): Table<any> {
     return inserting && inserting.name === tableName ? inserting : tableByName(tableName);
   }