@proteinjs/db 1.54.1 → 1.54.3

package/src/Record.ts

@@ -2,6 +2,7 @@ import { Logger } from '@proteinjs/logger';
 import { DateTimeColumn, UuidColumn } from './Columns';
 import { Column, Table, Columns } from './Table';
 import { moment } from './opt/moment';
+import type { EncryptionRecordHooks } from './encryption/EncryptionRecordHooks';
 
 export interface Record {
   id: string;
@@ -51,10 +52,33 @@ export type SerializedRecord = { [columnName: string]: any };
  *  (see `EncryptionRecordHooks`). Resolved by `Db` for writes touching encrypted columns. */
 export type RecordEncryptionContext = { keyOwner: string };
 
+/** One column's seat in a hydration: the property the record carries it under, and the column. */
+interface HydrationSlot {
+  propertyName: string;
+  column: Column<any, any>;
+}
+
+/**
+ * What a hydration resolves ONCE per serializer, never per row: the decryption hook and every
+ * column's seat, keyed by the name a stored row carries it under.
+ */
+interface HydrationPlan {
+  hooks: EncryptionRecordHooks;
+  slotsByColumnName: { [columnName: string]: HydrationSlot | undefined };
+}
+
 export class RecordSerializer<T extends Record> {
+  /**
+   * The table-and-column pairs whose stored cells have failed to deserialize in this process
+   * (`omitCellNotDeserialized`): each is warned once. A legacy cell is read again by every query
+   * that touches its row — a background sweep reads some rows every few seconds — and the warning
+   * is a schema fact, not a per-read event.
+   */
+  private static legacyCellsWarned = new Set<string>();
   private logger = new Logger({ name: this.constructor.name });
   private table: Table<T>;
   private encryptionContext?: RecordEncryptionContext;
+  private plan?: HydrationPlan;
 
   constructor(table: Table<T>, encryptionContext?: RecordEncryptionContext) {
     this.table = table;
@@ -97,35 +121,197 @@ export class RecordSerializer<T extends Record> {
     return serialized;
   }
 
+  /**
+   * A result set's rows → records, in ONE pass. A row whose hook and columns all answer directly
+   * is hydrated synchronously, and when every row did the array returns synchronously: no promise
+   * per row, no yield of the event loop between rows. A row that needs an await — a ciphertext
+   * envelope to decrypt, a column whose `deserialize` answers with a promise — settles through one
+   * promise for the whole set. The hook and the column seats are resolved once per serializer.
+   */
+  deserializeAll(serializedRecords: SerializedRecord[]): T[] | Promise<T[]> {
+    const plan = this.hydrationPlan();
+    const records: (T | Promise<T>)[] = new Array(serializedRecords.length);
+    let settles = false;
+    for (let i = 0; i < serializedRecords.length; i++) {
+      const record = this.hydrate(plan, serializedRecords[i]);
+      if (this.isPromise(record)) {
+        settles = true;
+      }
+      records[i] = record;
+    }
+
+    return settles ? Promise.all(records) : (records as T[]);
+  }
+
+  /** One row → one record (`deserializeAll` is the door for a result set). */
   async deserialize(serializedRecord: SerializedRecord): Promise<T> {
+    return await this.hydrate(this.hydrationPlan(), serializedRecord);
+  }
+
+  private hydrate(plan: HydrationPlan, serializedRecord: SerializedRecord): T | Promise<T> {
     // The transparent decryption seam: ciphertext envelopes decrypt (self-describing — the
-    // envelope names its key), framework companion columns drop out of the result.
-    const { EncryptionRecordHooks } = await import('./encryption/EncryptionRecordHooks');
-    const prepared = await new EncryptionRecordHooks().onDeserialize(this.table, serializedRecord);
+    // envelope names its key), framework companion columns drop out of the result. The hook
+    // answers directly for a row holding no envelope.
+    const prepared = plan.hooks.onDeserialize(this.table, serializedRecord);
+    if (this.isPromise(prepared)) {
+      return prepared.then((row) => this.hydratePrepared(plan, row));
+    }
 
-    const deserialized: any = {};
-    const fieldSerializer = new FieldSerializer(this.table);
-    const omittedFields: string[] = [];
+    return this.hydratePrepared(plan, prepared);
+  }
+
+  private hydratePrepared(plan: HydrationPlan, prepared: SerializedRecord): T | Promise<T> {
+    const record: any = {};
+    let settling: Promise<void>[] | undefined;
+    let omittedFields: string[] | undefined;
     for (const columnName in prepared) {
-      const serializedFieldValue = prepared[columnName];
+      const slot = plan.slotsByColumnName[columnName];
+      if (!slot) {
+        // a column exists in the db that is no longer defined in Table.columns
+        (omittedFields ??= []).push(columnName);
+        continue;
+      }
+
+      const { propertyName, column } = slot;
+      const storedValue = prepared[columnName];
+      if (!column.deserialize) {
+        record[propertyName] = storedValue;
+        continue;
+      }
+
+      if (column.deserializeOnFirstRead && storedValue !== null && storedValue !== undefined) {
+        this.defineFieldReadOnFirstRead(record, propertyName, column, storedValue);
+        continue;
+      }
+
+      let fieldValue: unknown;
       try {
-        const { fieldPropertyName, fieldValue } = await fieldSerializer.deserialize(
-          columnName,
-          serializedFieldValue,
-          prepared
+        fieldValue = column.deserialize(storedValue, prepared);
+      } catch (error) {
+        this.omitCellNotDeserialized(columnName, prepared, error);
+        continue;
+      }
+
+      if (this.isPromise(fieldValue)) {
+        record[propertyName] = undefined; // seats the property in column order; the value lands when it settles
+        (settling ??= []).push(
+          fieldValue.then(
+            (value) => {
+              record[propertyName] = value;
+            },
+            (error) => {
+              delete record[propertyName];
+              this.omitCellNotDeserialized(columnName, prepared, error);
+            }
+          )
         );
-        deserialized[fieldPropertyName] = fieldValue;
-      } catch (MissingFieldError) {
-        omittedFields.push(columnName);
+        continue;
       }
+
+      record[propertyName] = fieldValue;
     }
 
-    if (omittedFields.length > 0) {
+    if (omittedFields) {
       // expected when passing a base table into the query api
       this.logger.debug({ message: `Fields were omitted during deserialization`, obj: { omittedFields } });
     }
 
-    return deserialized;
+    if (settling) {
+      return Promise.all(settling).then(() => record as T);
+    }
+
+    return record as T;
+  }
+
+  /**
+   * The field as an own, enumerable property whose value is deserialized on its first read, once
+   * (`Column.deserializeOnFirstRead`); a write replaces it like any other field.
+   */
+  private defineFieldReadOnFirstRead(
+    record: any,
+    propertyName: string,
+    column: Column<any, any>,
+    storedValue: unknown
+  ): void {
+    let value: unknown;
+    let read = false;
+    const omit = (error: unknown) => this.omitCellNotDeserialized(column.name, record, error);
+    Object.defineProperty(record, propertyName, {
+      enumerable: true,
+      configurable: true,
+      get() {
+        if (!read) {
+          read = true;
+          try {
+            value = column.deserialize!(storedValue, undefined);
+          } catch (error) {
+            delete record[propertyName];
+            omit(error);
+          }
+        }
+
+        return value;
+      },
+      set(next: unknown) {
+        value = next;
+        read = true;
+      },
+    });
+  }
+
+  private hydrationPlan(): HydrationPlan {
+    if (this.plan) {
+      return this.plan;
+    }
+
+    // Required at call time — this module sits below the encryption machinery in the package's
+    // module graph (the seam `Table.ts` uses for the same reason).
+    // eslint-disable-next-line @typescript-eslint/no-var-requires
+    const { EncryptionRecordHooks } =
+      require('./encryption/EncryptionRecordHooks') as typeof import('./encryption/EncryptionRecordHooks');
+    const slotsByColumnName: HydrationPlan['slotsByColumnName'] = {};
+    const columns: { [propertyName: string]: Column<any, any> } = this.table.columns;
+    // Seated by column name, then by property name — a stored key that is a property name
+    // resolves to that property's column first, as it always has.
+    for (const propertyName in columns) {
+      slotsByColumnName[columns[propertyName].name] = { propertyName, column: columns[propertyName] };
+    }
+    for (const propertyName in columns) {
+      slotsByColumnName[propertyName] = { propertyName, column: columns[propertyName] };
+    }
+
+    this.plan = { hooks: new EncryptionRecordHooks(), slotsByColumnName };
+    return this.plan;
+  }
+
+  /**
+   * A stored cell the column cannot deserialize — a value written past the serializer, the way a
+   * hand-run statement or an older writer leaves it — never takes the row, the result set or the
+   * process with it. The field is omitted from the record (the law the per-row hydration held:
+   * `try { deserialize } catch { omit }`; what a reader of an optional field already takes as
+   * absent), and one warning per table and column per process names the table, the column and the
+   * first row seen — never the stored value, which the parser's own message quotes.
+   */
+  private omitCellNotDeserialized(columnName: string, row: SerializedRecord, error: unknown): void {
+    const cell = `${this.table.name}.${columnName}`;
+    if (RecordSerializer.legacyCellsWarned.has(cell)) {
+      return;
+    }
+
+    RecordSerializer.legacyCellsWarned.add(cell);
+    this.logger.warn({
+      message: `A stored cell could not be deserialized; the field is omitted from the record`,
+      obj: {
+        table: this.table.name,
+        column: columnName,
+        id: row['id'],
+        error: error instanceof Error ? error.name : typeof error,
+      },
+    });
+  }
+
+  private isPromise<V>(value: V | Promise<V>): value is Promise<V> {
+    return typeof (value as { then?: unknown } | null | undefined)?.then === 'function';
   }
 }
 

package/src/Table.ts

@@ -263,7 +263,27 @@ export type Column<T, Serialized> = {
   oldName?: string;
   options?: ColumnOptions;
   serialize?: (fieldValue: T | null | undefined) => Promise<Serialized | null | undefined>;
-  deserialize?: (serializedFieldValue: Serialized | null, serializedRecord: any) => Promise<T | null | void>;
+  /**
+   * The stored value → the field's value. Answers directly when building the value needs no
+   * await (every column this package ships does), or with a promise when it does: record
+   * hydration (`RecordSerializer.deserializeAll`) awaits only what comes back as a promise, so a
+   * result set whose rows all answer directly hydrates synchronously — no promise per row, no
+   * yield of the event loop between rows.
+   */
+  deserialize?: (
+    serializedFieldValue: Serialized | null,
+    serializedRecord: any
+  ) => T | null | void | Promise<T | null | void>;
+  /**
+   * When true, a hydrated record builds this column's field on the FIRST READ of the field, not
+   * at hydration: `deserialize` runs then, once, with the stored value alone (the row is not
+   * kept), so a deserializer under this flag reads nothing but its value and answers directly,
+   * never with a promise. For a pure, synchronous deserializer whose value costs more to build
+   * than to store and is often never read on a row — a DateTime column's moment. The record's
+   * shape is unchanged: the field is an own, enumerable property that reads as the deserialized
+   * value and takes a write like any other; a stored null reads null, undeferred.
+   */
+  deserializeOnFirstRead?: boolean;
   /**
    * The column's QUERY-side contract, paralleling `serialize`/`deserialize` on the storage
    * side: how uses of this column in a query (conditions, ORDER BY, aggregation, GROUP BY)

package/src/encryption/EncryptionRecordHooks.ts

@@ -9,6 +9,9 @@ import { SearchTokenizer } from './SearchTokenizer';
 /** Who a row being written belongs to — the data key its values encrypt under. */
 export type EncryptionWriteContext = { keyOwner: string };
 
+/** A stored cell holding a ciphertext envelope, and the key the envelope names. */
+type EnvelopeCell = { columnName: string; owner: string; version: number };
+
 /**
  * The transparent encrypt/decrypt seam, invoked by `RecordSerializer` on every write's
  * serialize and every read's deserialize — the same per-column hook layer every caller
@@ -31,6 +34,7 @@ export class EncryptionRecordHooks {
   private encryptedColumns = new EncryptedColumns();
   private envelope = new EncryptionEnvelope();
   private tokenizer = new SearchTokenizer();
+  private internalNames?: { table: Table<any>; names: Set<string> };
 
   /** Encrypt encrypted-column values in `serialized` (column-name-keyed) and add companions. */
   async onSerialize(table: Table<any>, serialized: SerializedRecord, context?: EncryptionWriteContext): Promise<void> {
@@ -88,34 +92,46 @@ export class EncryptionRecordHooks {
   }
 
   /**
-   * A copy of `serializedRecord` with framework companion columns dropped and every
-   * envelope value decrypted (any column — self-describing ciphertext also covers the
-   * decrypt-out transition, where the config no longer marks the column).
+   * `serializedRecord` with framework companion columns dropped and every envelope value
+   * decrypted (any column — self-describing ciphertext also covers the decrypt-out transition,
+   * where the config no longer marks the column). Answers directly — the row itself when
+   * nothing drops or decrypts, else a copy — unless an envelope must be decrypted: its key is
+   * read asynchronously, so the copy settles through a promise.
    */
-  async onDeserialize(table: Table<any>, serializedRecord: SerializedRecord): Promise<SerializedRecord> {
+  onDeserialize(table: Table<any>, serializedRecord: SerializedRecord): SerializedRecord | Promise<SerializedRecord> {
     this.encryptedColumns.ensureSchema(table);
-    const prepared: SerializedRecord = {};
     const internalColumnNames = this.internalColumnNames(table);
-    let decrypted = false;
-    for (const columnName of Object.keys(serializedRecord)) {
+    let dropsInternal = false;
+    let envelopes: EnvelopeCell[] | undefined;
+    for (const columnName in serializedRecord) {
       if (internalColumnNames.has(columnName)) {
+        dropsInternal = true;
         continue;
       }
 
-      const value = serializedRecord[columnName];
-      const parsed = this.envelope.parse(value);
-      if (!parsed) {
-        prepared[columnName] = value;
-        continue;
+      const parsed = this.envelope.parse(serializedRecord[columnName]);
+      if (parsed) {
+        (envelopes ??= []).push({ columnName, owner: parsed.owner, version: parsed.version });
       }
+    }
 
-      this.assertServerSide();
-      const key = await new DataKeyStore().getKeyByVersion(parsed.owner, parsed.version);
-      prepared[columnName] = this.envelope.decrypt(value as string, key);
-      decrypted = true;
+    if (!dropsInternal && !envelopes) {
+      return serializedRecord;
     }
 
-    return decrypted || internalColumnNames.size > 0 ? prepared : serializedRecord;
+    const prepared: SerializedRecord = {};
+    for (const columnName in serializedRecord) {
+      if (!internalColumnNames.has(columnName)) {
+        prepared[columnName] = serializedRecord[columnName];
+      }
+    }
+
+    if (!envelopes) {
+      return prepared;
+    }
+
+    this.assertServerSide();
+    return this.decryptInto(prepared, envelopes);
   }
 
   /**
@@ -144,7 +160,21 @@ export class EncryptionRecordHooks {
     );
   }
 
+  private async decryptInto(prepared: SerializedRecord, envelopes: EnvelopeCell[]): Promise<SerializedRecord> {
+    for (const cell of envelopes) {
+      const key = await new DataKeyStore().getKeyByVersion(cell.owner, cell.version);
+      prepared[cell.columnName] = this.envelope.decrypt(prepared[cell.columnName] as string, key);
+    }
+
+    return prepared;
+  }
+
+  /** The table's framework-derived companion column names — resolved once per table per instance, not per row. */
   private internalColumnNames(table: Table<any>): Set<string> {
+    if (this.internalNames?.table === table) {
+      return this.internalNames.names;
+    }
+
     const names = new Set<string>();
     for (const prop of Object.keys(table.columns)) {
       const column = (table.columns as any)[prop];
@@ -153,6 +183,7 @@ export class EncryptionRecordHooks {
       }
     }
 
+    this.internalNames = { table, names };
     return names;
   }
 

package/test/DateTimeFieldReadOnFirstRead.test.ts

@@ -0,0 +1,162 @@
+/**
+ * A DateTime column's moment is built on the field's FIRST READ, not at hydration
+ * (`Column.deserializeOnFirstRead`): a row carries many such columns and most are never read, and
+ * every moment built at hydration was event-loop time under a large read. What a reader sees is
+ * unchanged — an own, enumerable field holding a moment at the stored instant, the same instance on
+ * every read, replaceable by a write, present to Object.keys, spread and JSON alike.
+ */
+jest.mock('moment', () => {
+  const actual = jest.requireActual('moment');
+  const factory = jest.fn((...args: unknown[]) => actual(...args));
+  Object.assign(factory, actual);
+  return factory;
+});
+
+import '../src/Table';
+import moment from 'moment';
+import { Table } from '../src/Table';
+import { Record, RecordSerializer, SerializedRecord, withRecordColumns } from '../src/Record';
+import { DateTimeColumn, StringColumn } from '../src/Columns';
+
+const momentFactory = moment as unknown as jest.Mock;
+
+interface Stamped extends Record {
+  title: string;
+  t0: moment.Moment | null;
+  t1: moment.Moment | null;
+  t2: moment.Moment | null;
+  t3: moment.Moment | null;
+  t4: moment.Moment | null;
+  t5: moment.Moment | null;
+  t6: moment.Moment | null;
+  t7: moment.Moment | null;
+  t8: moment.Moment | null;
+  t9: moment.Moment | null;
+}
+
+class StampedTable extends Table<Stamped> {
+  public name = 'stamped';
+  public columns = withRecordColumns<Stamped>({
+    title: new StringColumn('title'),
+    t0: new DateTimeColumn('t0'),
+    t1: new DateTimeColumn('t1'),
+    t2: new DateTimeColumn('t2'),
+    t3: new DateTimeColumn('t3'),
+    t4: new DateTimeColumn('t4'),
+    t5: new DateTimeColumn('t5'),
+    t6: new DateTimeColumn('t6'),
+    t7: new DateTimeColumn('t7'),
+    t8: new DateTimeColumn('t8'),
+    t9: new DateTimeColumn('t9'),
+  });
+}
+
+const stampedTable = new StampedTable() as Table<Stamped>;
+
+/** What a driver hands the column: a Date (Spanner's is a Date subclass) — the instant is what the column takes. */
+const stampsOf = (row: number) => Array.from({ length: 10 }, (_, i) => new Date(1_700_000_000_000 + row * 100 + i));
+
+const storedRows = (count: number): SerializedRecord[] =>
+  Array.from({ length: count }, (_, row) => {
+    const stamps = stampsOf(row);
+    const stored: SerializedRecord = {
+      id: `row-${row}`,
+      created: new Date(1_600_000_000_000 + row),
+      updated: new Date(1_600_000_000_000 + row),
+      title: `title ${row}`,
+    };
+    stamps.forEach((stamp, i) => {
+      stored[`t${i}`] = i === 9 ? null : stamp;
+    });
+    return stored;
+  });
+
+/** Through the per-row door; the result-set door shares the hydrator (SyncRowHydration proves its one pass). */
+const hydrate = (rows: SerializedRecord[]): Promise<Stamped[]> => {
+  const serializer = new RecordSerializer<Stamped>(stampedTable);
+  return Promise.all(rows.map((row) => serializer.deserialize(row)));
+};
+
+describe('a DateTime field builds its moment on the first read', () => {
+  beforeEach(() => {
+    momentFactory.mockClear();
+  });
+
+  it('hydrating rows with ten DateTime columns builds no moment', async () => {
+    const records = await hydrate(storedRows(50));
+
+    expect(records).toHaveLength(50);
+    expect(momentFactory).not.toHaveBeenCalled();
+  });
+
+  it('the first read builds the moment at the stored instant, once; every read after returns that instance', async () => {
+    const [record] = await hydrate(storedRows(1));
+    const stamps = stampsOf(0);
+
+    const first = record.t3!;
+
+    expect(moment.isMoment(first)).toBe(true);
+    expect(first.valueOf()).toBe(stamps[3].getTime());
+    expect(momentFactory).toHaveBeenCalledTimes(1);
+    expect(record.t3).toBe(first);
+    expect(momentFactory).toHaveBeenCalledTimes(1);
+    expect(record.created.valueOf()).toBe(1_600_000_000_000);
+    expect(momentFactory).toHaveBeenCalledTimes(2);
+  });
+
+  it('a stored null reads null; a write replaces the field', async () => {
+    const [record] = await hydrate(storedRows(1));
+
+    expect(record.t9).toBeNull();
+    expect(momentFactory).not.toHaveBeenCalled();
+
+    const assigned = moment(0);
+    record.t4 = assigned;
+    expect(record.t4).toBe(assigned);
+    expect(momentFactory).toHaveBeenCalledTimes(1);
+  });
+
+  it('the record keeps its shape: own enumerable fields in column order, under spread and JSON alike', async () => {
+    const [record] = await hydrate(storedRows(1));
+    const stamps = stampsOf(0);
+
+    expect(Object.keys(record)).toEqual([
+      'id',
+      'created',
+      'updated',
+      'title',
+      't0',
+      't1',
+      't2',
+      't3',
+      't4',
+      't5',
+      't6',
+      't7',
+      't8',
+      't9',
+    ]);
+    expect(Object.prototype.hasOwnProperty.call(record, 't5')).toBe(true);
+    expect(momentFactory).not.toHaveBeenCalled();
+
+    const copy = { ...record };
+    expect(moment.isMoment(copy.t5)).toBe(true);
+    expect(copy.t5).toBe(record.t5);
+
+    const json = JSON.parse(JSON.stringify(record));
+    expect(json.t7).toBe(stamps[7].toISOString());
+    expect(json.t9).toBeNull();
+    expect(json.title).toBe('title 0');
+  });
+
+  it('a column read directly still answers the moment itself', () => {
+    const column = stampedTable.columns.t0 as DateTimeColumn;
+    const stamp = new Date(1_700_000_000_123);
+
+    const value = column.deserialize(stamp);
+
+    expect(moment.isMoment(value)).toBe(true);
+    expect(value!.valueOf()).toBe(1_700_000_000_123);
+    expect(column.deserialize(null)).toBeNull();
+  });
+});