@proteinjs/db 1.54.1 → 1.54.2

package/src/Table.ts

@@ -263,7 +263,27 @@ export type Column<T, Serialized> = {
   oldName?: string;
   options?: ColumnOptions;
   serialize?: (fieldValue: T | null | undefined) => Promise<Serialized | null | undefined>;
-  deserialize?: (serializedFieldValue: Serialized | null, serializedRecord: any) => Promise<T | null | void>;
+  /**
+   * The stored value → the field's value. Answers directly when building the value needs no
+   * await (every column this package ships does), or with a promise when it does: record
+   * hydration (`RecordSerializer.deserializeAll`) awaits only what comes back as a promise, so a
+   * result set whose rows all answer directly hydrates synchronously — no promise per row, no
+   * yield of the event loop between rows.
+   */
+  deserialize?: (
+    serializedFieldValue: Serialized | null,
+    serializedRecord: any
+  ) => T | null | void | Promise<T | null | void>;
+  /**
+   * When true, a hydrated record builds this column's field on the FIRST READ of the field, not
+   * at hydration: `deserialize` runs then, once, with the stored value alone (the row is not
+   * kept), so a deserializer under this flag reads nothing but its value and answers directly,
+   * never with a promise. For a pure, synchronous deserializer whose value costs more to build
+   * than to store and is often never read on a row — a DateTime column's moment. The record's
+   * shape is unchanged: the field is an own, enumerable property that reads as the deserialized
+   * value and takes a write like any other; a stored null reads null, undeferred.
+   */
+  deserializeOnFirstRead?: boolean;
   /**
    * The column's QUERY-side contract, paralleling `serialize`/`deserialize` on the storage
    * side: how uses of this column in a query (conditions, ORDER BY, aggregation, GROUP BY)

package/src/encryption/EncryptionRecordHooks.ts

@@ -9,6 +9,9 @@ import { SearchTokenizer } from './SearchTokenizer';
 /** Who a row being written belongs to — the data key its values encrypt under. */
 export type EncryptionWriteContext = { keyOwner: string };
 
+/** A stored cell holding a ciphertext envelope, and the key the envelope names. */
+type EnvelopeCell = { columnName: string; owner: string; version: number };
+
 /**
  * The transparent encrypt/decrypt seam, invoked by `RecordSerializer` on every write's
  * serialize and every read's deserialize — the same per-column hook layer every caller
@@ -31,6 +34,7 @@ export class EncryptionRecordHooks {
   private encryptedColumns = new EncryptedColumns();
   private envelope = new EncryptionEnvelope();
   private tokenizer = new SearchTokenizer();
+  private internalNames?: { table: Table<any>; names: Set<string> };
 
   /** Encrypt encrypted-column values in `serialized` (column-name-keyed) and add companions. */
   async onSerialize(table: Table<any>, serialized: SerializedRecord, context?: EncryptionWriteContext): Promise<void> {
@@ -88,34 +92,46 @@ export class EncryptionRecordHooks {
   }
 
   /**
-   * A copy of `serializedRecord` with framework companion columns dropped and every
-   * envelope value decrypted (any column — self-describing ciphertext also covers the
-   * decrypt-out transition, where the config no longer marks the column).
+   * `serializedRecord` with framework companion columns dropped and every envelope value
+   * decrypted (any column — self-describing ciphertext also covers the decrypt-out transition,
+   * where the config no longer marks the column). Answers directly — the row itself when
+   * nothing drops or decrypts, else a copy — unless an envelope must be decrypted: its key is
+   * read asynchronously, so the copy settles through a promise.
    */
-  async onDeserialize(table: Table<any>, serializedRecord: SerializedRecord): Promise<SerializedRecord> {
+  onDeserialize(table: Table<any>, serializedRecord: SerializedRecord): SerializedRecord | Promise<SerializedRecord> {
     this.encryptedColumns.ensureSchema(table);
-    const prepared: SerializedRecord = {};
     const internalColumnNames = this.internalColumnNames(table);
-    let decrypted = false;
-    for (const columnName of Object.keys(serializedRecord)) {
+    let dropsInternal = false;
+    let envelopes: EnvelopeCell[] | undefined;
+    for (const columnName in serializedRecord) {
       if (internalColumnNames.has(columnName)) {
+        dropsInternal = true;
         continue;
       }
 
-      const value = serializedRecord[columnName];
-      const parsed = this.envelope.parse(value);
-      if (!parsed) {
-        prepared[columnName] = value;
-        continue;
+      const parsed = this.envelope.parse(serializedRecord[columnName]);
+      if (parsed) {
+        (envelopes ??= []).push({ columnName, owner: parsed.owner, version: parsed.version });
       }
+    }
 
-      this.assertServerSide();
-      const key = await new DataKeyStore().getKeyByVersion(parsed.owner, parsed.version);
-      prepared[columnName] = this.envelope.decrypt(value as string, key);
-      decrypted = true;
+    if (!dropsInternal && !envelopes) {
+      return serializedRecord;
     }
 
-    return decrypted || internalColumnNames.size > 0 ? prepared : serializedRecord;
+    const prepared: SerializedRecord = {};
+    for (const columnName in serializedRecord) {
+      if (!internalColumnNames.has(columnName)) {
+        prepared[columnName] = serializedRecord[columnName];
+      }
+    }
+
+    if (!envelopes) {
+      return prepared;
+    }
+
+    this.assertServerSide();
+    return this.decryptInto(prepared, envelopes);
   }
 
   /**
@@ -144,7 +160,21 @@ export class EncryptionRecordHooks {
     );
   }
 
+  private async decryptInto(prepared: SerializedRecord, envelopes: EnvelopeCell[]): Promise<SerializedRecord> {
+    for (const cell of envelopes) {
+      const key = await new DataKeyStore().getKeyByVersion(cell.owner, cell.version);
+      prepared[cell.columnName] = this.envelope.decrypt(prepared[cell.columnName] as string, key);
+    }
+
+    return prepared;
+  }
+
+  /** The table's framework-derived companion column names — resolved once per table per instance, not per row. */
   private internalColumnNames(table: Table<any>): Set<string> {
+    if (this.internalNames?.table === table) {
+      return this.internalNames.names;
+    }
+
     const names = new Set<string>();
     for (const prop of Object.keys(table.columns)) {
       const column = (table.columns as any)[prop];
@@ -153,6 +183,7 @@ export class EncryptionRecordHooks {
       }
     }
 
+    this.internalNames = { table, names };
     return names;
   }
 

package/test/DateTimeFieldReadOnFirstRead.test.ts

@@ -0,0 +1,162 @@
+/**
+ * A DateTime column's moment is built on the field's FIRST READ, not at hydration
+ * (`Column.deserializeOnFirstRead`): a row carries many such columns and most are never read, and
+ * every moment built at hydration was event-loop time under a large read. What a reader sees is
+ * unchanged — an own, enumerable field holding a moment at the stored instant, the same instance on
+ * every read, replaceable by a write, present to Object.keys, spread and JSON alike.
+ */
+jest.mock('moment', () => {
+  const actual = jest.requireActual('moment');
+  const factory = jest.fn((...args: unknown[]) => actual(...args));
+  Object.assign(factory, actual);
+  return factory;
+});
+
+import '../src/Table';
+import moment from 'moment';
+import { Table } from '../src/Table';
+import { Record, RecordSerializer, SerializedRecord, withRecordColumns } from '../src/Record';
+import { DateTimeColumn, StringColumn } from '../src/Columns';
+
+const momentFactory = moment as unknown as jest.Mock;
+
+interface Stamped extends Record {
+  title: string;
+  t0: moment.Moment | null;
+  t1: moment.Moment | null;
+  t2: moment.Moment | null;
+  t3: moment.Moment | null;
+  t4: moment.Moment | null;
+  t5: moment.Moment | null;
+  t6: moment.Moment | null;
+  t7: moment.Moment | null;
+  t8: moment.Moment | null;
+  t9: moment.Moment | null;
+}
+
+class StampedTable extends Table<Stamped> {
+  public name = 'stamped';
+  public columns = withRecordColumns<Stamped>({
+    title: new StringColumn('title'),
+    t0: new DateTimeColumn('t0'),
+    t1: new DateTimeColumn('t1'),
+    t2: new DateTimeColumn('t2'),
+    t3: new DateTimeColumn('t3'),
+    t4: new DateTimeColumn('t4'),
+    t5: new DateTimeColumn('t5'),
+    t6: new DateTimeColumn('t6'),
+    t7: new DateTimeColumn('t7'),
+    t8: new DateTimeColumn('t8'),
+    t9: new DateTimeColumn('t9'),
+  });
+}
+
+const stampedTable = new StampedTable() as Table<Stamped>;
+
+/** What a driver hands the column: a Date (Spanner's is a Date subclass) — the instant is what the column takes. */
+const stampsOf = (row: number) => Array.from({ length: 10 }, (_, i) => new Date(1_700_000_000_000 + row * 100 + i));
+
+const storedRows = (count: number): SerializedRecord[] =>
+  Array.from({ length: count }, (_, row) => {
+    const stamps = stampsOf(row);
+    const stored: SerializedRecord = {
+      id: `row-${row}`,
+      created: new Date(1_600_000_000_000 + row),
+      updated: new Date(1_600_000_000_000 + row),
+      title: `title ${row}`,
+    };
+    stamps.forEach((stamp, i) => {
+      stored[`t${i}`] = i === 9 ? null : stamp;
+    });
+    return stored;
+  });
+
+/** Through the per-row door; the result-set door shares the hydrator (SyncRowHydration proves its one pass). */
+const hydrate = (rows: SerializedRecord[]): Promise<Stamped[]> => {
+  const serializer = new RecordSerializer<Stamped>(stampedTable);
+  return Promise.all(rows.map((row) => serializer.deserialize(row)));
+};
+
+describe('a DateTime field builds its moment on the first read', () => {
+  beforeEach(() => {
+    momentFactory.mockClear();
+  });
+
+  it('hydrating rows with ten DateTime columns builds no moment', async () => {
+    const records = await hydrate(storedRows(50));
+
+    expect(records).toHaveLength(50);
+    expect(momentFactory).not.toHaveBeenCalled();
+  });
+
+  it('the first read builds the moment at the stored instant, once; every read after returns that instance', async () => {
+    const [record] = await hydrate(storedRows(1));
+    const stamps = stampsOf(0);
+
+    const first = record.t3!;
+
+    expect(moment.isMoment(first)).toBe(true);
+    expect(first.valueOf()).toBe(stamps[3].getTime());
+    expect(momentFactory).toHaveBeenCalledTimes(1);
+    expect(record.t3).toBe(first);
+    expect(momentFactory).toHaveBeenCalledTimes(1);
+    expect(record.created.valueOf()).toBe(1_600_000_000_000);
+    expect(momentFactory).toHaveBeenCalledTimes(2);
+  });
+
+  it('a stored null reads null; a write replaces the field', async () => {
+    const [record] = await hydrate(storedRows(1));
+
+    expect(record.t9).toBeNull();
+    expect(momentFactory).not.toHaveBeenCalled();
+
+    const assigned = moment(0);
+    record.t4 = assigned;
+    expect(record.t4).toBe(assigned);
+    expect(momentFactory).toHaveBeenCalledTimes(1);
+  });
+
+  it('the record keeps its shape: own enumerable fields in column order, under spread and JSON alike', async () => {
+    const [record] = await hydrate(storedRows(1));
+    const stamps = stampsOf(0);
+
+    expect(Object.keys(record)).toEqual([
+      'id',
+      'created',
+      'updated',
+      'title',
+      't0',
+      't1',
+      't2',
+      't3',
+      't4',
+      't5',
+      't6',
+      't7',
+      't8',
+      't9',
+    ]);
+    expect(Object.prototype.hasOwnProperty.call(record, 't5')).toBe(true);
+    expect(momentFactory).not.toHaveBeenCalled();
+
+    const copy = { ...record };
+    expect(moment.isMoment(copy.t5)).toBe(true);
+    expect(copy.t5).toBe(record.t5);
+
+    const json = JSON.parse(JSON.stringify(record));
+    expect(json.t7).toBe(stamps[7].toISOString());
+    expect(json.t9).toBeNull();
+    expect(json.title).toBe('title 0');
+  });
+
+  it('a column read directly still answers the moment itself', () => {
+    const column = stampedTable.columns.t0 as DateTimeColumn;
+    const stamp = new Date(1_700_000_000_123);
+
+    const value = column.deserialize(stamp);
+
+    expect(moment.isMoment(value)).toBe(true);
+    expect(value!.valueOf()).toBe(1_700_000_000_123);
+    expect(column.deserialize(null)).toBeNull();
+  });
+});

package/test/HydrationBenchmark.test.ts

@@ -0,0 +1,209 @@
+import { createHook } from 'async_hooks';
+import { SourceRepository } from '@proteinjs/reflection';
+import { Db, DbDriver } from '../src/Db';
+import { TableWatcherRunner } from '../src/TableWatcherRunner';
+import { Table } from '../src/Table';
+import { Record, SerializedRecord, withRecordColumns } from '../src/Record';
+import { DateTimeColumn, IntegerColumn, ObjectColumn, StringColumn } from '../src/Columns';
+import { DefaultTransactionContextFactory, TransactionContextData } from '../src/transaction/TransactionContextFactory';
+import type { Moment } from 'moment';
+
+/**
+ * A MEASURE, never a gate: hydrating 1,000 rows with ten DateTime columns through `Db.query`, read
+ * as wall time, promises created (async_hooks) and microtask rounds — the numbers are printed for
+ * the person reading the run and nothing here asserts a threshold (CI's machines vary). The one
+ * assertion is that the rows came back.
+ */
+
+interface Wide extends Record {
+  title: string;
+  body: string;
+  count: number;
+  payload: { tags: string[]; n: number } | null;
+  t0: Moment | null;
+  t1: Moment | null;
+  t2: Moment | null;
+  t3: Moment | null;
+  t4: Moment | null;
+  t5: Moment | null;
+  t6: Moment | null;
+  t7: Moment | null;
+  t8: Moment | null;
+  t9: Moment | null;
+}
+
+class WideTable extends Table<Wide> {
+  public name = 'hydration_wide';
+  public columns = withRecordColumns<Wide>({
+    title: new StringColumn('title'),
+    body: new StringColumn('body', undefined, 'MAX'),
+    count: new IntegerColumn('count'),
+    payload: new ObjectColumn('payload'),
+    t0: new DateTimeColumn('t0'),
+    t1: new DateTimeColumn('t1'),
+    t2: new DateTimeColumn('t2'),
+    t3: new DateTimeColumn('t3'),
+    t4: new DateTimeColumn('t4'),
+    t5: new DateTimeColumn('t5'),
+    t6: new DateTimeColumn('t6'),
+    t7: new DateTimeColumn('t7'),
+    t8: new DateTimeColumn('t8'),
+    t9: new DateTimeColumn('t9'),
+  });
+}
+
+const wideTable = new WideTable() as Table<Wide>;
+
+class TestTransactionContextFactory implements DefaultTransactionContextFactory {
+  private context: TransactionContextData = {};
+
+  getTransactionContext(): TransactionContextData {
+    return this.context;
+  }
+
+  async runInContext<T>(context: TransactionContextData, fn: () => Promise<T>): Promise<T> {
+    const previous = this.context;
+    this.context = context;
+    try {
+      return await fn();
+    } finally {
+      this.context = previous;
+    }
+  }
+}
+
+class RowsDriver implements DbDriver {
+  constructor(private readonly rows: SerializedRecord[]) {}
+
+  getDbName(): string {
+    return 'test';
+  }
+
+  async createDbIfNotExists(): Promise<void> {}
+
+  async createDb(): Promise<void> {}
+
+  async dropDb(): Promise<void> {}
+
+  getTableManager(): never {
+    throw new Error('not used by this measure');
+  }
+
+  async runQuery(): Promise<SerializedRecord[]> {
+    return this.rows;
+  }
+
+  async runDml(): Promise<number> {
+    return 0;
+  }
+
+  async runTransaction<T>(fn: (transaction: unknown) => Promise<T>): Promise<T> {
+    return await fn({ id: 'transaction' });
+  }
+
+  getOperationDeadlineMs(): number {
+    return 1000;
+  }
+}
+
+type SourceRepositoryInternals = { objectCache: { [qualifiedName: string]: unknown[] } };
+type TableWatcherRunnerStatics = { tableWatcherMap?: unknown };
+const objectCache = () => (SourceRepository.get() as unknown as SourceRepositoryInternals).objectCache;
+let originalWatchers: unknown[] | undefined;
+
+// No reflection graph in a unit test: the watcher registry is empty by declaration.
+beforeAll(() => {
+  originalWatchers = objectCache()['@proteinjs/db/TableWatcher'];
+  objectCache()['@proteinjs/db/TableWatcher'] = [];
+  (TableWatcherRunner as unknown as TableWatcherRunnerStatics).tableWatcherMap = undefined;
+});
+
+afterAll(() => {
+  if (originalWatchers) {
+    objectCache()['@proteinjs/db/TableWatcher'] = originalWatchers;
+  } else {
+    delete objectCache()['@proteinjs/db/TableWatcher'];
+  }
+  (TableWatcherRunner as unknown as TableWatcherRunnerStatics).tableWatcherMap = undefined;
+});
+
+const storedRows = (count: number): SerializedRecord[] =>
+  Array.from({ length: count }, (_, row) => {
+    const stored: SerializedRecord = {
+      id: `row-${row}`,
+      created: new Date(1_600_000_000_000 + row),
+      updated: new Date(1_600_000_000_000 + row),
+      title: `title ${row}`,
+      body: 'x'.repeat(400),
+      count: row,
+      payload: JSON.stringify({ tags: ['a', 'b', 'c'], n: row }),
+    };
+    for (let i = 0; i < 10; i++) {
+      stored[`t${i}`] = new Date(1_700_000_000_000 + row * 100 + i);
+    }
+    return stored;
+  });
+
+type Measure = { ms: number; promises: number; microtaskRounds: number };
+
+/** Wall time, promises created and microtask rounds while `act` runs. */
+const measure = async (act: () => Promise<unknown>): Promise<Measure> => {
+  let promises = 0;
+  const hook = createHook({
+    init(_asyncId, type) {
+      if (type === 'PROMISE') {
+        promises++;
+      }
+    },
+  });
+  let microtaskRounds = 0;
+  let running = true;
+  const round = () => {
+    if (!running) {
+      return;
+    }
+    microtaskRounds++;
+    queueMicrotask(round);
+  };
+  hook.enable();
+  const started = process.hrtime.bigint();
+  queueMicrotask(round);
+  await act();
+  running = false;
+  const ms = Number(process.hrtime.bigint() - started) / 1_000_000;
+  hook.disable();
+  return { ms: Math.round(ms * 10) / 10, promises, microtaskRounds };
+};
+
+describe('hydration measure', () => {
+  it('reports 1,000 rows with ten DateTime columns through Db.query (wall ms, promises, microtask rounds)', async () => {
+    const rows = storedRows(1000);
+    const db = new Db(new RowsDriver(rows), () => wideTable, new TestTransactionContextFactory(), true);
+    // one warm-up pass, then the measured passes
+    await db.query(wideTable, {});
+
+    const passes: Measure[] = [];
+    let records: Wide[] = [];
+    for (let i = 0; i < 5; i++) {
+      passes.push(await measure(async () => (records = await db.query(wideTable, {}))));
+    }
+    const readAll = await measure(async () => {
+      for (const record of records) {
+        record.t0!.valueOf();
+        record.created.valueOf();
+      }
+    });
+
+    process.stdout.write(
+      `hydration measure — 1,000 rows × (10 DateTime + created/updated + 4 plain columns), Db.query, 5 passes:\n` +
+        passes
+          .map(
+            (pass, i) =>
+              `  pass ${i + 1}: ${pass.ms} ms · ${pass.promises} promises · ${pass.microtaskRounds} microtask rounds`
+          )
+          .join('\n') +
+        `\n  then reading t0 + created on every record: ${readAll.ms} ms · ${readAll.promises} promises\n`
+    );
+    expect(records).toHaveLength(1000);
+  });
+});