@proteinjs/db 1.54.2 → 1.55.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@proteinjs/db",
-  "version": "1.54.2",
+  "version": "1.55.0",
   "main": "./dist/generated/index.js",
   "types": "./dist/generated/index.d.ts",
   "exports": {
@@ -41,7 +41,7 @@
     "test": "jest --passWithNoTests"
   },
   "dependencies": {
-    "@proteinjs/db-query": "^1.11.0",
+    "@proteinjs/db-query": "^1.11.1",
     "@proteinjs/logger": "^1.2.0",
     "@proteinjs/reflection": "^1.2.0",
     "@proteinjs/serializer": "^1.1.10",
@@ -66,5 +66,5 @@
     "ts-jest": "29.1.1",
     "typescript": "5.2.2"
   },
-  "gitHead": "d90bd35b32b628c2b9cf9d8ddb16b4a2b9140f11"
+  "gitHead": "e2cb123d846e60e8cddccbca06b5076a009817ed"
 }

package/src/Record.ts

@@ -25,10 +25,26 @@ const recordColumns: Columns<Record> = {
   }),
 };
 
-class MissingFieldError extends Error {
+/** A record field no declared column owns — the one serialize failure a write omits instead of failing on. */
+export class MissingFieldError extends Error {
   constructor(tableName: string, fieldName: string) {
     super(`Table ${tableName} is missing field: ${fieldName}`);
     this.name = 'MissingFieldError';
+    // ES5 down-leveled `extends Error` loses the subclass prototype — restore it so
+    // `instanceof MissingFieldError` holds for the write-side catch.
+    Object.setPrototypeOf(this, MissingFieldError.prototype);
+  }
+}
+
+/** Two declared columns of one table share a stored name: the second would silently take the first's seat. */
+export class DuplicateColumnNameError extends Error {
+  constructor(tableName: string, columnName: string, propertyNames: [string, string]) {
+    super(
+      `(${tableName}) Columns ${propertyNames[0]} and ${propertyNames[1]} both declare the column name ` +
+        `'${columnName}'; a stored cell can have only one seat`
+    );
+    this.name = 'DuplicateColumnNameError';
+    Object.setPrototypeOf(this, DuplicateColumnNameError.prototype);
   }
 }
 
@@ -68,6 +84,13 @@ interface HydrationPlan {
 }
 
 export class RecordSerializer<T extends Record> {
+  /**
+   * The table-and-column pairs whose stored cells have failed to deserialize in this process
+   * (`omitCellNotDeserialized`): each is warned once. A legacy cell is read again by every query
+   * that touches its row — a background sweep reads some rows every few seconds — and the warning
+   * is a schema fact, not a per-read event.
+   */
+  private static legacyCellsWarned = new Set<string>();
   private logger = new Logger({ name: this.constructor.name });
   private table: Table<T>;
   private encryptionContext?: RecordEncryptionContext;
@@ -95,7 +118,13 @@ export class RecordSerializer<T extends Record> {
       try {
         const { columnName, serializedFieldValue } = await fieldSerializer.serialize(fieldPropertyName, fieldValue);
         serialized[columnName] = serializedFieldValue;
-      } catch (MissingFieldError) {
+      } catch (error) {
+        // Only a field no column owns is omitted; a column that cannot serialize its value fails
+        // the write — nothing is stored for a cell the column could not make, and the caller hears it.
+        if (!(error instanceof MissingFieldError)) {
+          throw error;
+        }
+
         omittedFields.push(fieldPropertyName);
       }
     }
@@ -125,17 +154,45 @@ export class RecordSerializer<T extends Record> {
     const plan = this.hydrationPlan();
     const records: (T | Promise<T>)[] = new Array(serializedRecords.length);
     let settles = false;
-    for (let i = 0; i < serializedRecords.length; i++) {
-      const record = this.hydrate(plan, serializedRecords[i]);
-      if (this.isPromise(record)) {
-        settles = true;
+    try {
+      for (let i = 0; i < serializedRecords.length; i++) {
+        const record = this.hydrate(plan, serializedRecords[i]);
+        if (this.isPromise(record)) {
+          settles = true;
+        }
+        records[i] = record;
       }
-      records[i] = record;
+    } catch (error) {
+      return this.failAfterSettling(records, error);
     }
 
     return settles ? Promise.all(records) : (records as T[]);
   }
 
+  /**
+   * Settle, then fail: a row (or cell) that threw synchronously fails the set — but never before
+   * every promise an earlier row or cell already created has settled. A promise left behind by a
+   * synchronous throw is observed by nobody: its rejection is an `unhandledRejection` no caller
+   * can catch. So the throw becomes the set's rejection, after the created promises settle.
+   */
+  private failAfterSettling(created: unknown[], error: unknown): Promise<never> {
+    const settling: Promise<void>[] = [];
+    for (const value of created) {
+      if (this.isPromise(value)) {
+        settling.push(
+          value.then(
+            () => undefined,
+            () => undefined
+          )
+        );
+      }
+    }
+
+    return Promise.all(settling).then(() => {
+      throw error;
+    });
+  }
+
   /** One row → one record (`deserializeAll` is the door for a result set). */
   async deserialize(serializedRecord: SerializedRecord): Promise<T> {
     return await this.hydrate(this.hydrationPlan(), serializedRecord);
@@ -157,38 +214,56 @@ export class RecordSerializer<T extends Record> {
     const record: any = {};
     let settling: Promise<void>[] | undefined;
     let omittedFields: string[] | undefined;
-    for (const columnName in prepared) {
-      const slot = plan.slotsByColumnName[columnName];
-      if (!slot) {
-        // a column exists in the db that is no longer defined in Table.columns
-        (omittedFields ??= []).push(columnName);
-        continue;
-      }
+    try {
+      for (const columnName in prepared) {
+        const slot = plan.slotsByColumnName[columnName];
+        if (!slot) {
+          // a column exists in the db that no declared column owns (a column the table no longer
+          // declares, or one a rename left behind under the property's old name)
+          (omittedFields ??= []).push(columnName);
+          continue;
+        }
 
-      const { propertyName, column } = slot;
-      const storedValue = prepared[columnName];
-      if (!column.deserialize) {
-        record[propertyName] = storedValue;
-        continue;
-      }
+        const { propertyName, column } = slot;
+        const storedValue = prepared[columnName];
+        if (!column.deserialize) {
+          record[propertyName] = storedValue;
+          continue;
+        }
 
-      if (column.deserializeOnFirstRead && storedValue !== null && storedValue !== undefined) {
-        this.defineFieldReadOnFirstRead(record, propertyName, column, storedValue);
-        continue;
-      }
+        if (column.deserializeOnFirstRead && storedValue !== null && storedValue !== undefined) {
+          this.defineFieldReadOnFirstRead(record, propertyName, column, storedValue);
+          continue;
+        }
 
-      const fieldValue = column.deserialize(storedValue, prepared);
-      if (this.isPromise(fieldValue)) {
-        record[propertyName] = undefined; // seats the property in column order; the value lands when it settles
-        (settling ??= []).push(
-          fieldValue.then((value) => {
-            record[propertyName] = value;
-          })
-        );
-        continue;
-      }
+        let fieldValue: unknown;
+        try {
+          fieldValue = column.deserialize(storedValue, prepared);
+        } catch (error) {
+          this.omitCellNotDeserialized(columnName, prepared, error);
+          continue;
+        }
+
+        if (this.isPromise(fieldValue)) {
+          record[propertyName] = undefined; // seats the property in column order; the value lands when it settles
+          (settling ??= []).push(
+            fieldValue.then(
+              (value) => {
+                record[propertyName] = value;
+              },
+              (error) => {
+                delete record[propertyName];
+                this.omitCellNotDeserialized(columnName, prepared, error);
+              }
+            )
+          );
+          continue;
+        }
 
-      record[propertyName] = fieldValue;
+        record[propertyName] = fieldValue;
+      }
+    } catch (error) {
+      return this.failAfterSettling(settling ?? [], error);
     }
 
     if (omittedFields) {
@@ -215,13 +290,19 @@ export class RecordSerializer<T extends Record> {
   ): void {
     let value: unknown;
     let read = false;
+    const omit = (error: unknown) => this.omitCellNotDeserialized(column.name, record, error);
     Object.defineProperty(record, propertyName, {
       enumerable: true,
       configurable: true,
       get() {
         if (!read) {
-          value = column.deserialize!(storedValue, undefined);
           read = true;
+          try {
+            value = column.deserialize!(storedValue, undefined);
+          } catch (error) {
+            delete record[propertyName];
+            omit(error);
+          }
         }
 
         return value;
@@ -245,19 +326,50 @@ export class RecordSerializer<T extends Record> {
       require('./encryption/EncryptionRecordHooks') as typeof import('./encryption/EncryptionRecordHooks');
     const slotsByColumnName: HydrationPlan['slotsByColumnName'] = {};
     const columns: { [propertyName: string]: Column<any, any> } = this.table.columns;
-    // Seated by column name, then by property name — a stored key that is a property name
-    // resolves to that property's column first, as it always has.
-    for (const propertyName in columns) {
-      slotsByColumnName[columns[propertyName].name] = { propertyName, column: columns[propertyName] };
-    }
+    // Seated by the declared COLUMN name alone — the name a stored row carries a cell under (a
+    // driver's rows are keyed by column name, always). A stored key that is only a property's name
+    // — a column a rename retired and left physically present — names no seat: it is omitted like
+    // any other undeclared column and never reaches a declared column's parser.
     for (const propertyName in columns) {
-      slotsByColumnName[propertyName] = { propertyName, column: columns[propertyName] };
+      const columnName = columns[propertyName].name;
+      const seated = slotsByColumnName[columnName];
+      if (seated) {
+        throw new DuplicateColumnNameError(this.table.name, columnName, [seated.propertyName, propertyName]);
+      }
+
+      slotsByColumnName[columnName] = { propertyName, column: columns[propertyName] };
     }
 
     this.plan = { hooks: new EncryptionRecordHooks(), slotsByColumnName };
     return this.plan;
   }
 
+  /**
+   * A stored cell the column cannot deserialize — a value written past the serializer, the way a
+   * hand-run statement or an older writer leaves it — never takes the row, the result set or the
+   * process with it. The field is omitted from the record (the law the per-row hydration held:
+   * `try { deserialize } catch { omit }`; what a reader of an optional field already takes as
+   * absent), and one warning per table and column per process names the table, the column and the
+   * first row seen — never the stored value, which the parser's own message quotes.
+   */
+  private omitCellNotDeserialized(columnName: string, row: SerializedRecord, error: unknown): void {
+    const cell = `${this.table.name}.${columnName}`;
+    if (RecordSerializer.legacyCellsWarned.has(cell)) {
+      return;
+    }
+
+    RecordSerializer.legacyCellsWarned.add(cell);
+    this.logger.warn({
+      message: `A stored cell could not be deserialized; the field is omitted from the record`,
+      obj: {
+        table: this.table.name,
+        column: columnName,
+        id: row['id'],
+        error: error instanceof Error ? error.name : typeof error,
+      },
+    });
+  }
+
   private isPromise<V>(value: V | Promise<V>): value is Promise<V> {
     return typeof (value as { then?: unknown } | null | undefined)?.then === 'function';
   }
@@ -283,22 +395,21 @@ export class FieldSerializer<T extends Record> {
 
   async deserialize(columnName: string, serializedFieldValue: any, serializedRecord: SerializedRecord) {
     const columns: { [prop: string]: Column<any, any> } = this.table.columns;
-    let fieldPropertyName = columnName;
-    let column = columns[columnName]; // the scenario that the column name is the same as the property name
-    if (!column) {
-      for (const columnPropertyName in columns) {
-        const checkColumn = (this.table.columns as any)[columnPropertyName];
-        if (checkColumn && columnName == checkColumn.name) {
-          fieldPropertyName = columnPropertyName;
-          column = checkColumn;
-          break;
-        }
+    // One seat per stored cell: the declared column whose NAME the cell is stored under (the law
+    // `RecordSerializer` hydrates by) — never a property that merely shares the stored key's name.
+    let fieldPropertyName: string | undefined;
+    let column: Column<any, any> | undefined;
+    for (const columnPropertyName in columns) {
+      if (columnName === columns[columnPropertyName].name) {
+        fieldPropertyName = columnPropertyName;
+        column = columns[columnPropertyName];
+        break;
       }
     }
 
-    if (!column) {
-      // this is the case where a column exists in the db that is no longer defined in Table.columns
-      throw new MissingFieldError(this.table.name, fieldPropertyName);
+    if (!column || !fieldPropertyName) {
+      // this is the case where a column exists in the db that no declared column owns
+      throw new MissingFieldError(this.table.name, columnName);
     }
 
     let fieldValue = serializedFieldValue;

package/src/auth/TableServiceAuth.ts

@@ -1,7 +1,7 @@
 import { Logger } from '@proteinjs/logger';
 import { ServiceError, ServiceRefusal } from '@proteinjs/service';
 import { Table, isTable } from '../Table';
-import { TableAuth, TableAuthError } from './TableAuth';
+import { TableAuth } from './TableAuth';
 
 export class TableServiceAuth {
   private logger = new Logger({ name: this.constructor.name });

package/src/schema/TableManager.ts

@@ -419,6 +419,30 @@ export class TableManager {
     return true;
   }
 
+  /**
+   * The census, at every schema sync of an existing table: the physical columns no declared column
+   * of the table owns — a column a rename retired (`tags` beside the `tag_list` that replaced it),
+   * a column the table dropped from its declaration. Such a column is never read or altered by the
+   * schema sync and never seated by the row reader; it rides every row of the table until a
+   * migration retires it. One WARN per table naming the columns — names only, never a value.
+   */
+  private warnUndeclaredColumns(table: Table<any>, physicalColumnNames: string[]): void {
+    const declared = new Set<string>();
+    for (const columnPropertyName in table.columns) {
+      declared.add(table.columns[columnPropertyName].name);
+    }
+
+    const undeclared = physicalColumnNames.filter((columnName) => !declared.has(columnName));
+    if (undeclared.length === 0) {
+      return;
+    }
+
+    this.logger.warn({
+      message: `Physical columns no declared column owns; a migration retires them`,
+      obj: { table: table.name, columns: undeclared },
+    });
+  }
+
   private async getTableChanges(table: Table<any>) {
     const { indexesToCreate, indexesToDrop } = await this.getIndexOperations(table);
     const tableChanges: TableChanges = {
@@ -440,6 +464,7 @@ export class TableManager {
     const columnMetadata = await this.schemaMetadata.getColumnMetadata(table);
     const uniqueColumns = await this.schemaMetadata.getUniqueColumns(table);
     const foreignKeys = await this.schemaMetadata.getForeignKeys(table);
+    this.warnUndeclaredColumns(table, Object.keys(columnMetadata));
     for (const columnPropertyName in table.columns) {
       const column = table.columns[columnPropertyName];
       if (columnMetadata[column.name]) {

package/src/source/SourceRecordSyncRunner.ts

@@ -662,7 +662,7 @@ export class SourceRecordSyncRunner {
             break;
           }
         }
-      } catch (error) {
+      } catch {
         // Unresolvable from cwd — reported below.
       }
     }

package/test/LegacyCellHydration.test.ts

@@ -0,0 +1,234 @@
+import { SourceRepository } from '@proteinjs/reflection';
+import { Db, DbDriver } from '../src/Db';
+import { TableWatcherRunner } from '../src/TableWatcherRunner';
+import { Column, Table } from '../src/Table';
+import { Record, RecordSerializer, SerializedRecord, withRecordColumns } from '../src/Record';
+import { ArrayColumn, ObjectColumn, StringColumn } from '../src/Columns';
+import { DefaultTransactionContextFactory, TransactionContextData } from '../src/transaction/TransactionContextFactory';
+
+/**
+ * A stored cell a column cannot deserialize — a legacy value written outside the serializer, say a
+ * bare word in a column that stores JSON — never takes the row, the result set or the process with
+ * it. The law the per-row hydration held (`try { deserialize } catch { omit }`) and the one-pass
+ * hydration dropped: the field is OMITTED from the record, every other field lands, the query's
+ * promise resolves, and ONE warning per table and column per process names the table, the column
+ * and the first row seen — never the stored value (the parser's own message quotes it).
+ */
+
+interface Doc extends Record {
+  title: string;
+  tags: string[];
+  settings: { theme: string };
+  after: string;
+}
+
+class DocTable extends Table<Doc> {
+  public name = 'legacy_cell_doc';
+  public columns = withRecordColumns<Doc>({
+    title: new StringColumn('title'),
+    tags: new ArrayColumn<string>('tag_list'),
+    settings: new ObjectColumn<{ theme: string }>('settings'),
+    after: new StringColumn('after'),
+  });
+}
+
+const docTable = new DocTable() as Table<Doc>;
+
+class TestTransactionContextFactory implements DefaultTransactionContextFactory {
+  private context: TransactionContextData = {};
+
+  getTransactionContext(): TransactionContextData {
+    return this.context;
+  }
+
+  async runInContext<T>(context: TransactionContextData, fn: () => Promise<T>): Promise<T> {
+    const previous = this.context;
+    this.context = context;
+    try {
+      return await fn();
+    } finally {
+      this.context = previous;
+    }
+  }
+}
+
+/** A driver that answers every query with the rows it was handed — the stored cells as the database holds them. */
+class RowsDriver implements DbDriver {
+  constructor(private readonly rows: SerializedRecord[]) {}
+
+  getDbName(): string {
+    return 'test';
+  }
+
+  async createDbIfNotExists(): Promise<void> {}
+
+  async createDb(): Promise<void> {}
+
+  async dropDb(): Promise<void> {}
+
+  getTableManager(): never {
+    throw new Error('not used by these tests');
+  }
+
+  async runQuery(): Promise<SerializedRecord[]> {
+    return this.rows;
+  }
+
+  async runDml(): Promise<number> {
+    return 0;
+  }
+
+  async runTransaction<T>(fn: (transaction: unknown) => Promise<T>): Promise<T> {
+    return await fn({ id: 'transaction' });
+  }
+
+  getOperationDeadlineMs(): number {
+    return 1000;
+  }
+}
+
+const dbOver = (table: Table<any>, rows: SerializedRecord[]) =>
+  new Db(new RowsDriver(rows), () => table, new TestTransactionContextFactory(), true);
+
+const storedRow = (id: string, cells: Partial<SerializedRecord>): SerializedRecord => ({
+  id,
+  created: new Date(1_700_000_000_000),
+  updated: new Date(1_700_000_000_000),
+  title: `title ${id}`,
+  tag_list: '["a","b"]',
+  settings: '{"theme":"dark"}',
+  after: `after ${id}`,
+  ...cells,
+});
+
+/** The stored value no warning may ever carry — distinctive, so a leak is unmistakable. */
+const LEGACY_WORD = 'legacyword7';
+
+type SourceRepositoryInternals = { objectCache: { [qualifiedName: string]: unknown[] } };
+type TableWatcherRunnerStatics = { tableWatcherMap?: unknown };
+type SerializerInternals = { logger: { warn: (...args: unknown[]) => void; debug: (...args: unknown[]) => void } };
+/** The process-wide once-per-table-and-column registry behind the warning (reset per test). */
+type SerializerStatics = { legacyCellsWarned: Set<string> };
+const objectCache = () => (SourceRepository.get() as unknown as SourceRepositoryInternals).objectCache;
+let originalWatchers: unknown[] | undefined;
+
+beforeEach(() => {
+  (RecordSerializer as unknown as SerializerStatics).legacyCellsWarned?.clear();
+});
+
+// No reflection graph in a unit test: the watcher registry is empty by declaration.
+beforeAll(() => {
+  originalWatchers = objectCache()['@proteinjs/db/TableWatcher'];
+  objectCache()['@proteinjs/db/TableWatcher'] = [];
+  (TableWatcherRunner as unknown as TableWatcherRunnerStatics).tableWatcherMap = undefined;
+});
+
+afterAll(() => {
+  if (originalWatchers) {
+    objectCache()['@proteinjs/db/TableWatcher'] = originalWatchers;
+  } else {
+    delete objectCache()['@proteinjs/db/TableWatcher'];
+  }
+  (TableWatcherRunner as unknown as TableWatcherRunnerStatics).tableWatcherMap = undefined;
+});
+
+/** Every warning a serializer logs, as the logger received it. */
+const warningsOf = (serializer: RecordSerializer<any>): unknown[][] => {
+  const calls: unknown[][] = [];
+  const logger = (serializer as unknown as SerializerInternals).logger;
+  logger.warn = (...args: unknown[]) => {
+    calls.push(args);
+  };
+  return calls;
+};
+
+describe('a stored cell a column cannot deserialize', () => {
+  it('a query over a row holding a legacy value resolves: the field is omitted, every other field lands', async () => {
+    const rows = [storedRow('r1', {}), storedRow('r2', { tag_list: LEGACY_WORD }), storedRow('r3', {})];
+
+    const records = await dbOver(docTable, rows).query(docTable, {});
+
+    expect(records.map((record) => record.id)).toEqual(['r1', 'r2', 'r3']);
+    expect(records[0].tags).toEqual(['a', 'b']);
+    expect(records[2].tags).toEqual(['a', 'b']);
+    expect(records[1]).not.toHaveProperty('tags');
+    expect(records[1]).toMatchObject({
+      id: 'r2',
+      title: 'title r2',
+      settings: { theme: 'dark' },
+      after: 'after r2',
+    });
+  });
+
+  it('a get of the row answers the record, not a rejection', async () => {
+    const rows = [storedRow('r2', { settings: LEGACY_WORD })];
+
+    const record = await dbOver(docTable, rows).get(docTable, { id: 'r2' });
+
+    expect(record).toMatchObject({ id: 'r2', title: 'title r2', tags: ['a', 'b'], after: 'after r2' });
+    expect(record).not.toHaveProperty('settings');
+  });
+
+  it('warns once per table and column per process, naming the table, the column and the first row — never the value', () => {
+    const serializer = new RecordSerializer<Doc>(docTable);
+    const warnings = warningsOf(serializer);
+    const rows = [
+      storedRow('r1', { tag_list: LEGACY_WORD }),
+      storedRow('r2', { tag_list: LEGACY_WORD }),
+      storedRow('r3', { settings: LEGACY_WORD }),
+    ];
+
+    const records = serializer.deserializeAll(rows) as Doc[];
+
+    expect(Array.isArray(records)).toBe(true);
+    expect(records.map((record) => record.id)).toEqual(['r1', 'r2', 'r3']);
+    expect(warnings).toHaveLength(2);
+    expect(warnings[0][0]).toMatchObject({
+      obj: { table: 'legacy_cell_doc', column: 'tag_list', id: 'r1', error: 'SyntaxError' },
+    });
+    expect(warnings[1][0]).toMatchObject({
+      obj: { table: 'legacy_cell_doc', column: 'settings', id: 'r3', error: 'SyntaxError' },
+    });
+    expect(JSON.stringify(warnings)).not.toContain(LEGACY_WORD);
+  });
+
+  it('a column whose deserialize answers with a rejecting promise is omitted the same way', async () => {
+    class LaterColumn implements Column<number, number> {
+      constructor(public name: string) {}
+
+      deserialize(storedValue: number | null): Promise<number | null> {
+        return storedValue === -1 ? Promise.reject(new RangeError('unreadable')) : Promise.resolve(storedValue);
+      }
+    }
+    interface Later extends Record {
+      title: string;
+      later: number;
+      after: string;
+    }
+    class LaterTable extends Table<Later> {
+      public name = 'legacy_cell_later';
+      public columns = withRecordColumns<Later>({
+        title: new StringColumn('title'),
+        later: new LaterColumn('later'),
+        after: new StringColumn('after'),
+      });
+    }
+    const laterTable = new LaterTable() as Table<Later>;
+    const serializer = new RecordSerializer<Later>(laterTable);
+    const warnings = warningsOf(serializer);
+    const rows: SerializedRecord[] = [
+      { id: 'l1', title: 'one', later: 1, after: 'x' },
+      { id: 'l2', title: 'two', later: -1, after: 'y' },
+    ];
+
+    const records = await serializer.deserializeAll(rows);
+
+    expect(records[0]).toMatchObject({ id: 'l1', title: 'one', later: 1, after: 'x' });
+    expect(records[1]).toMatchObject({ id: 'l2', title: 'two', after: 'y' });
+    expect(records[1]).not.toHaveProperty('later');
+    expect(warnings).toHaveLength(1);
+    expect(warnings[0][0]).toMatchObject({
+      obj: { table: 'legacy_cell_later', column: 'later', id: 'l2', error: 'RangeError' },
+    });
+  });
+});