@proteinjs/db 1.51.0 → 1.52.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@proteinjs/db",
-  "version": "1.51.0",
+  "version": "1.52.0",
   "main": "./dist/generated/index.js",
   "types": "./dist/generated/index.d.ts",
   "exports": {
@@ -41,7 +41,7 @@
     "test": "jest --passWithNoTests"
   },
   "dependencies": {
-    "@proteinjs/db-query": "^1.10.2",
+    "@proteinjs/db-query": "^1.11.0",
     "@proteinjs/logger": "^1.2.0",
     "@proteinjs/reflection": "^1.2.0",
     "@proteinjs/serializer": "^1.1.10",
@@ -66,5 +66,5 @@
     "ts-jest": "29.1.1",
     "typescript": "5.2.2"
   },
-  "gitHead": "1128b1e6374c7f52cd5de172620b619b8f92e119"
+  "gitHead": "57c2ed1dca68a508bbd7b194ea7695ac4cf93344"
 }

package/src/Db.ts

@@ -104,6 +104,14 @@ export interface DbDriver {
    * configured its database to tolerate rather than a number of its own.
    */
   getOperationDeadlineMs(): number;
+  /**
+   * The most bound parameters one statement may carry on this driver's backend, when the backend
+   * bounds it — `Db.insertMany` sizes its multi-row statements by it (`columns × rows` parameters
+   * each), so a batch wider than the bound goes as the fewest statements that fit instead of one
+   * the backend refuses. A driver whose backend declares no such bound leaves this undeclared,
+   * and a batch on it is always one statement.
+   */
+  getStatementParameterLimit?(): number;
 }
 
 export class Db<R extends Record = Record> implements DbService<R> {
@@ -228,6 +236,77 @@ export class Db<R extends Record = Record> implements DbService<R> {
     return recordCopy as T;
   }
 
+  /**
+   * Insert `records` as ONE statement (a multi-row INSERT) instead of one statement per row,
+   * each row keeping every contract {@link insert} gives a single one: the table's default field
+   * values, the before-insert table watchers, the columns' before-insert hooks, the encryption
+   * context, then — after the statement — the search-token upkeep, the columns' after-insert
+   * hooks and the after-insert table watchers. Each phase runs per row, in the order given, and
+   * sees the row exactly as `insert` would; what differs from N single inserts is only that every
+   * row's before-phases precede the statement and every row's after-phases follow it.
+   *
+   * Atomic: inside a transaction the statement rides it; outside one this opens a transaction
+   * of its own around the batch, so a row the database refuses fails the whole batch and no row
+   * of it lands. A batch wider than the driver's statement parameter bound
+   * ({@link DbDriver.getStatementParameterLimit}) goes as the fewest statements that fit, in
+   * order, inside that same transaction.
+   *
+   * Returns the rows as inserted (defaults applied, watchers' changes carried), in order. An
+   * empty list inserts nothing and returns [].
+   */
+  async insertMany<T extends R>(table: Table<T>, records: Omit<T, keyof R>[]): Promise<T[]> {
+    if (!this.runAsSystem) {
+      this.auth.canInsert(table);
+    }
+    if (records.length === 0) {
+      return [];
+    }
+    if (!this.transactionContextFactory.getTransactionContext().currentTransaction) {
+      const db = this.newSelfWrapDb();
+      return await db.runTransaction(async () => await db.insertMany(table, records));
+    }
+
+    const prepared: {
+      recordCopy: any;
+      encryptionContext?: { keyOwner: string };
+      serializedRecord: SerializedRecord;
+    }[] = [];
+    for (const record of records) {
+      let recordCopy = Object.assign({}, record);
+      await addDefaultFieldValues(table, recordCopy, this.runAsSystem);
+      recordCopy = await this.tableWatcherRunner.runBeforeInsertTableWatchers(table, recordCopy);
+      await this.addColumnInsertHooks(table, recordCopy);
+      const encryptionContext = await this.encryptionWriteContext(table, recordCopy);
+      const serializedRecord = await new RecordSerializer(table, encryptionContext).serialize(recordCopy);
+      prepared.push({ recordCopy, encryptionContext, serializedRecord });
+    }
+
+    for (const statementRows of this.chunkByParameterLimit(prepared.map((row) => row.serializedRecord))) {
+      const generateInsert = (config: DbDriverDmlStatementConfig) =>
+        new StatementFactory<T>().insertMany(
+          table.name,
+          statementRows as Partial<T>[],
+          this.statementConfigFactory.getStatementConfig(config)
+        );
+      await this.dbDriver.runDml(generateInsert, this.transactionForDriver());
+    }
+
+    for (const { recordCopy, encryptionContext } of prepared) {
+      if (encryptionContext) {
+        const { EncryptionTokenMaintenance } = await import('./encryption/EncryptionTokenMaintenance');
+        await new EncryptionTokenMaintenance().afterInsert(
+          table,
+          recordCopy,
+          encryptionContext.keyOwner,
+          this.newSystemDb()
+        );
+      }
+      await this.runColumnAfterInsertHooks(table, recordCopy);
+      await this.tableWatcherRunner.runAfterInsertTableWatchers(table, recordCopy as T);
+    }
+    return prepared.map((row) => row.recordCopy as T);
+  }
+
   async update<T extends R>(table: Table<T>, record: Partial<T>, query?: Query<T>): Promise<number> {
     if (!this.runAsSystem) {
       this.auth.canUpdate(table);
@@ -1045,6 +1124,29 @@ export class Db<R extends Record = Record> implements DbService<R> {
     return context.currentTransaction;
   }
 
+  /**
+   * The rows of one multi-row insert split into the fewest statements the driver's statement
+   * parameter bound admits (`insertMany`): each statement binds `columns × rows` parameters,
+   * the columns being the union over the whole batch (what `StatementFactory.insertMany` lists),
+   * so the split follows the batch's widest shape rather than the first statement's. One
+   * statement when the driver declares no bound.
+   */
+  private chunkByParameterLimit(rows: SerializedRecord[]): SerializedRecord[][] {
+    const limit = this.dbDriver.getStatementParameterLimit?.();
+    if (limit === undefined) {
+      return [rows];
+    }
+
+    const columns = new Set<string>();
+    for (const row of rows) {
+      for (const column of Object.keys(row)) {
+        columns.add(column);
+      }
+    }
+    const rowsPerStatement = Math.max(1, Math.floor(limit / Math.max(1, columns.size)));
+    return this.chunk(rows, rowsPerStatement);
+  }
+
   // Utility: simple chunker
   private chunk<T>(arr: T[], size: number): T[][] {
     if (size <= 0) {

package/test/InsertMany.test.ts

@@ -0,0 +1,298 @@
+import { SourceRepository } from '@proteinjs/reflection';
+import { Statement } from '@proteinjs/db-query';
+import { Db, DbDriver, DbDriverDmlStatementConfig } from '../src/Db';
+import { Table } from '../src/Table';
+import { Record, withRecordColumns } from '../src/Record';
+import { BooleanColumn, StringColumn } from '../src/Columns';
+import { TableWatcher } from '../src/TableWatcher';
+import { TableWatcherRunner } from '../src/TableWatcherRunner';
+import { DefaultTransactionContextFactory, TransactionContextData } from '../src/transaction/TransactionContextFactory';
+
+/**
+ * `Db.insertMany`'s contract against a recording driver (no database): N rows issue ONE statement
+ * carrying every row with its defaults; each per-row hook — the before-insert table watcher, the
+ * columns' before/after-insert hooks, the after-insert table watcher — fires once per row, in
+ * the order given, and sees the row exactly as the single `insert` road shows it; a batch wider
+ * than the driver's statement parameter bound splits into the fewest statements that fit; the
+ * batch is one transaction (the ambient one, else its own); a refused statement runs no
+ * after-phase. What the database does with the statement — every row landing, the rollback of a
+ * refused batch — is the drivers' emulator suites' (the reusable CRUD and transaction tests).
+ */
+
+interface Doc extends Record {
+  title: string;
+  note?: string | null;
+  flagged?: boolean;
+}
+
+type HookEvent = [phase: string, title: string, idPresent: boolean];
+const log: HookEvent[] = [];
+
+class DocTable extends Table<Doc> {
+  public name = 'insert_many_doc';
+  public columns = withRecordColumns<Doc>({
+    title: new StringColumn('title', {
+      onBeforeInsert: async (_table, record) => {
+        log.push(['column.before', record.title, typeof record.id === 'string']);
+        record.title = `${record.title}!`;
+      },
+      onAfterInsert: async (_table, record) => {
+        log.push(['column.after', record.title, typeof record.id === 'string']);
+      },
+    }),
+    note: new StringColumn('note'),
+    flagged: new BooleanColumn('flagged', { defaultValue: async () => false }),
+  });
+}
+
+const docTable = new DocTable() as Table<Doc>;
+
+class RecordingWatcher implements TableWatcher<Doc> {
+  name(): string {
+    return 'RecordingWatcher';
+  }
+
+  table(): Table<Doc> {
+    return docTable;
+  }
+
+  async beforeInsert<T extends Doc>(record: Omit<T, keyof Doc>): Promise<Omit<T, keyof Doc>> {
+    const doc = record as unknown as Doc;
+    log.push(['watcher.before', doc.title, typeof doc.id === 'string']);
+    return { ...record, note: `seen ${doc.title}` };
+  }
+
+  async afterInsert<T extends Doc>(record: T): Promise<void> {
+    log.push(['watcher.after', record.title, typeof record.id === 'string']);
+  }
+}
+
+class TestTransactionContextFactory implements DefaultTransactionContextFactory {
+  private context: TransactionContextData = {};
+
+  getTransactionContext(): TransactionContextData {
+    return this.context;
+  }
+
+  async runInContext<T>(context: TransactionContextData, fn: () => Promise<T>): Promise<T> {
+    const previous = this.context;
+    this.context = context;
+    try {
+      return await fn();
+    } finally {
+      this.context = previous;
+    }
+  }
+}
+
+/** Records every DML statement the Db hands it, as the Spanner driver would generate it. */
+class RecordingDriver implements DbDriver {
+  dml: { sql: string; params: { [name: string]: unknown }; transaction: unknown }[] = [];
+  transactions = 0;
+  refuse?: (sql: string) => boolean;
+
+  getDbName(): string {
+    return 'test';
+  }
+
+  async createDbIfNotExists(): Promise<void> {}
+
+  async createDb(): Promise<void> {}
+
+  async dropDb(): Promise<void> {}
+
+  getTableManager(): never {
+    throw new Error('not used by these tests');
+  }
+
+  async runQuery(): Promise<never[]> {
+    return [];
+  }
+
+  async runDml(generateStatement: (config: DbDriverDmlStatementConfig) => Statement, transaction?: unknown) {
+    const statement = generateStatement({
+      useParams: true,
+      useNamedParams: true,
+      prefixTablesWithDb: false,
+      getDriverColumnType: () => 'STRING(MAX)',
+    });
+    this.dml.push({ sql: statement.sql, params: statement.namedParams?.params ?? {}, transaction });
+    if (this.refuse?.(statement.sql)) {
+      throw new Error(`refused: ${statement.sql}`);
+    }
+    return tuplesOf(statement.sql);
+  }
+
+  async runTransaction<T>(fn: (transaction: unknown) => Promise<T>): Promise<T> {
+    this.transactions += 1;
+    return await fn({ id: `transaction-${this.transactions}` });
+  }
+
+  getOperationDeadlineMs(): number {
+    return 1000;
+  }
+}
+
+class BoundedDriver extends RecordingDriver {
+  constructor(private readonly limit: number) {
+    super();
+  }
+
+  getStatementParameterLimit(): number {
+    return this.limit;
+  }
+}
+
+const tuplesOf = (sql: string): number => (sql.match(/\(@param\d+/g) ?? []).length;
+
+type SourceRepositoryInternals = { objectCache: { [qualifiedName: string]: unknown[] } };
+type TableWatcherRunnerStatics = { tableWatcherMap?: unknown };
+const objectCache = () => (SourceRepository.get() as unknown as SourceRepositoryInternals).objectCache;
+
+/** A system Db over `driver`, its watcher map rebuilt from the recording watcher alone. */
+const dbOver = (driver: DbDriver): Db<Doc> => {
+  (TableWatcherRunner as unknown as TableWatcherRunnerStatics).tableWatcherMap = undefined;
+  return new Db<Doc>(driver, () => docTable, new TestTransactionContextFactory(), true);
+};
+
+const rowsOf = (titles: string[]): Omit<Doc, keyof Record>[] => titles.map((title) => ({ title }));
+const eventsFor = (title: string) => log.filter(([, t]) => t === title || t === `${title}!`);
+const shapeOf = (doc: Doc) => ({ title: doc.title, note: doc.note, flagged: doc.flagged });
+
+describe('Db.insertMany', () => {
+  let originalWatchers: unknown[] | undefined;
+
+  beforeAll(() => {
+    originalWatchers = objectCache()['@proteinjs/db/TableWatcher'];
+    objectCache()['@proteinjs/db/TableWatcher'] = [new RecordingWatcher()];
+  });
+
+  afterAll(() => {
+    if (originalWatchers) {
+      objectCache()['@proteinjs/db/TableWatcher'] = originalWatchers;
+    } else {
+      delete objectCache()['@proteinjs/db/TableWatcher'];
+    }
+    (TableWatcherRunner as unknown as TableWatcherRunnerStatics).tableWatcherMap = undefined;
+  });
+
+  beforeEach(() => {
+    log.length = 0;
+  });
+
+  test('N rows issue exactly ONE statement, every row in it with its defaults, returned in order', async () => {
+    const driver = new RecordingDriver();
+    const db = dbOver(driver);
+
+    const inserted = await db.insertMany(docTable, rowsOf(['a', 'b', 'c']));
+
+    expect(driver.dml).toHaveLength(1);
+    expect(tuplesOf(driver.dml[0].sql)).toBe(3);
+    expect(driver.dml[0].sql).toMatch(/^INSERT INTO `insert_many_doc` \(.+\) VALUES \(.+\), \(.+\), \(.+\);$/);
+    expect(inserted.map(shapeOf)).toEqual([
+      { title: 'a!', note: 'seen a', flagged: false },
+      { title: 'b!', note: 'seen b', flagged: false },
+      { title: 'c!', note: 'seen c', flagged: false },
+    ]);
+    for (const doc of inserted) {
+      expect(typeof doc.id).toBe('string');
+      expect(doc.created).toBeDefined();
+      expect(doc.updated).toBeDefined();
+    }
+    // The statement carries what the rows carry: the ids the defaults set and the watcher's note.
+    const bound = Object.values(driver.dml[0].params);
+    for (const doc of inserted) {
+      expect(bound).toContain(doc.id);
+      expect(bound).toContain(doc.note);
+    }
+  });
+
+  test('every per-row hook fires once per row, in row order, and sees the row as the single insert shows it', async () => {
+    const serial = dbOver(new RecordingDriver());
+    for (const row of rowsOf(['a', 'b'])) {
+      await serial.insert(docTable, row);
+    }
+    const serialLog = [...log];
+    log.length = 0;
+
+    const batched = dbOver(new RecordingDriver());
+    await batched.insertMany(docTable, rowsOf(['a', 'b']));
+
+    // Each row's own sequence — what fired, in what order, with the id already set — is the
+    // serial road's exactly.
+    for (const title of ['a', 'b']) {
+      const serialEvents = serialLog.filter(([, t]) => t === title || t === `${title}!`);
+      expect(eventsFor(title)).toEqual(serialEvents);
+      expect(eventsFor(title).map(([phase]) => phase)).toEqual([
+        'watcher.before',
+        'column.before',
+        'column.after',
+        'watcher.after',
+      ]);
+    }
+    // Across rows, each phase runs in row order; the before-phases of every row precede the
+    // statement, the after-phases follow it.
+    expect(log.map(([phase, title]) => `${phase}:${title}`)).toEqual([
+      'watcher.before:a',
+      'column.before:a',
+      'watcher.before:b',
+      'column.before:b',
+      'column.after:a!',
+      'watcher.after:a!',
+      'column.after:b!',
+      'watcher.after:b!',
+    ]);
+  });
+
+  test("a batch wider than the driver's parameter bound goes as the fewest statements that fit, in order", async () => {
+    // Six columns per row (id, created, updated, title, note, flagged); a bound of 13 admits two rows a statement.
+    const driver = new BoundedDriver(13);
+    const db = dbOver(driver);
+
+    const inserted = await db.insertMany(docTable, rowsOf(['a', 'b', 'c', 'd', 'e']));
+
+    expect(driver.dml.map((statement) => tuplesOf(statement.sql))).toEqual([2, 2, 1]);
+    for (const statement of driver.dml) {
+      expect(Object.keys(statement.params).length).toBeLessThanOrEqual(13);
+    }
+    expect(inserted.map((doc) => doc.title)).toEqual(['a!', 'b!', 'c!', 'd!', 'e!']);
+    // One transaction around the whole batch: every statement rides the same one.
+    expect(driver.transactions).toBe(1);
+    expect(new Set(driver.dml.map((statement) => statement.transaction)).size).toBe(1);
+  });
+
+  test('the batch rides the ambient transaction when there is one, and opens its own when there is none', async () => {
+    const ambient = new RecordingDriver();
+    const db = dbOver(ambient);
+    await db.runTransaction(async () => {
+      await db.insertMany(docTable, rowsOf(['a', 'b']));
+    });
+    expect(ambient.transactions).toBe(1);
+    expect(ambient.dml[0].transaction).toEqual({ id: 'transaction-1' });
+
+    const own = new RecordingDriver();
+    await dbOver(own).insertMany(docTable, rowsOf(['a', 'b']));
+    expect(own.transactions).toBe(1);
+    expect(own.dml[0].transaction).toEqual({ id: 'transaction-1' });
+  });
+
+  test('no rows: nothing issued, nothing opened, an empty result', async () => {
+    const driver = new RecordingDriver();
+    const inserted = await dbOver(driver).insertMany(docTable, []);
+
+    expect(inserted).toEqual([]);
+    expect(driver.dml).toHaveLength(0);
+    expect(driver.transactions).toBe(0);
+    expect(log).toHaveLength(0);
+  });
+
+  test('a refused statement fails the batch: the error surfaces and no after-phase runs for any row', async () => {
+    const driver = new RecordingDriver();
+    driver.refuse = () => true;
+
+    await expect(dbOver(driver).insertMany(docTable, rowsOf(['a', 'b']))).rejects.toThrow('refused');
+
+    expect(driver.dml).toHaveLength(1);
+    expect(log.map(([phase]) => phase).filter((phase) => phase.endsWith('.after'))).toEqual([]);
+  });
+});

package/test/TableServiceAuth.test.ts

@@ -1,4 +1,5 @@
 import { UserAuth } from '@proteinjs/user-auth';
+import { Logger } from '@proteinjs/logger';
 import { Table } from '../src/Table';
 import { withRecordColumns, Record } from '../src/Record';
 import { StringColumn } from '../src/Columns';
@@ -267,6 +268,17 @@ describe('TableServiceAuth — default deny (no auth block)', () => {
     expect(auth().canAccess('update', [table, { id: 'x', title: 't' }])).toBe(true);
     expect(auth().canAccess('delete', [table, { id: 'x' }])).toBe(true);
   });
+
+  it('the multi-row insert is a server-side door only: the service refuses it, admin included', () => {
+    setUser(['admin']);
+    const table = new NoAuthTable();
+    const errorSpy = jest.spyOn(Logger.prototype, 'error').mockImplementation(() => undefined);
+    try {
+      expect(auth().canAccess('insertMany', [table, [{ title: 't' }]])).toBe(false);
+    } finally {
+      errorSpy.mockRestore();
+    }
+  });
 });
 
 describe('TableServiceAuth — updateArrayMembership / updatePreserving (update parity)', () => {

package/test/reusable/CrudTests.ts

@@ -25,6 +25,36 @@ export const crudTests = (
       await db.delete(emplyeeTable, { id: fetchedEmployee.id });
     });
 
+    test('Insert many: every row lands with its defaults, as its own single insert would', async () => {
+      const emplyeeTable: Table<Employee> = crudTestTables.Employee;
+      const rows: Omit<Employee, keyof Record>[] = [
+        { name: 'Veronica', department: 'Cake Factory' },
+        { name: 'Ada', department: 'Cake Factory', jobTitle: null, isRemote: true },
+        { name: 'Sean' },
+      ];
+      const inserted = await db.insertMany(emplyeeTable, rows);
+      expect(inserted.map((employee) => employee.name)).toEqual(['Veronica', 'Ada', 'Sean']);
+      const qb = new QueryBuilder<Employee>(emplyeeTable.name).condition({
+        field: 'id',
+        operator: 'IN',
+        value: inserted.map((employee) => employee.id),
+      });
+      const fetched = await db.query(emplyeeTable, qb);
+      expect(fetched.length).toBe(3);
+      const byName = new Map(fetched.map((employee) => [employee.name, employee]));
+      expect(byName.get('Veronica')?.department).toBe('Cake Factory');
+      // A column one row carries and another leaves out stores NULL for the row without it —
+      // the stored value of a single-row insert that left it out.
+      expect(byName.get('Veronica')?.jobTitle).toBeNull();
+      expect(byName.get('Ada')?.isRemote).toBe(true);
+      expect(byName.get('Sean')?.department).toBeNull();
+      for (const employee of fetched) {
+        expect(employee.created).toBeTruthy();
+        expect(employee.updated).toBeTruthy();
+      }
+      await db.delete(emplyeeTable, qb);
+    });
+
     test('Update', async () => {
       const testEmployee: Omit<Employee, keyof Record> = { name: 'Veronica' };
       const emplyeeTable: Table<Employee> = crudTestTables.Employee;

package/test/reusable/TransactionTests.ts

@@ -88,6 +88,52 @@ export const transactionTests = (
       }
     });
 
+    test('Insert many inside a transaction: a refused row rolls the whole batch back, and the writes before it', async () => {
+      const emplyeeTable: Table<TransactionEmployee> = transactionTestTables.TransactionEmployee;
+      let singleId: string | undefined;
+      let batchIds: string[] = [];
+
+      await expect(
+        db.runTransaction(async () => {
+          const single = await db.insert(emplyeeTable, { name: 'Veronica', department: 'Engineering' });
+          singleId = single.id;
+          // The third row reuses the first row's id: the statement is refused as a whole.
+          const rows: Omit<TransactionEmployee, keyof Record>[] = [
+            { name: 'Ada', department: 'Engineering' },
+            { name: 'Sean', department: 'Engineering' },
+            { name: 'Cassidy', department: 'Engineering' },
+          ];
+          const first = await db.insertMany(emplyeeTable, [rows[0]]);
+          batchIds = [first[0].id];
+          await db.insertMany(emplyeeTable, [
+            rows[1],
+            { ...rows[2], id: first[0].id } as Omit<TransactionEmployee, keyof Record>,
+          ]);
+        })
+      ).rejects.toThrow();
+
+      expect(singleId).toBeDefined();
+      expect(await db.get(emplyeeTable, { id: singleId })).toBeFalsy();
+      expect(await db.get(emplyeeTable, { id: batchIds[0] })).toBeFalsy();
+      expect(await db.query(emplyeeTable, { department: 'Engineering' })).toHaveLength(0);
+    });
+
+    test('Insert many outside a transaction is one unit: a refused row lands none of the batch', async () => {
+      const emplyeeTable: Table<TransactionEmployee> = transactionTestTables.TransactionEmployee;
+      const anchor = await db.insert(emplyeeTable, { name: 'Veronica', department: 'Sales' });
+
+      await expect(
+        db.insertMany(emplyeeTable, [
+          { name: 'Ada', department: 'Sales' },
+          { name: 'Sean', department: 'Sales', id: anchor.id } as Omit<TransactionEmployee, keyof Record>,
+        ])
+      ).rejects.toThrow();
+
+      const sales = await db.query(emplyeeTable, { department: 'Sales' });
+      expect(sales.map((employee) => employee.name)).toEqual(['Veronica']);
+      await db.delete(emplyeeTable, { id: anchor.id });
+    });
+
     test('Nested transactions are not allowed', async () => {
       const testEmployee: Omit<TransactionEmployee, keyof Record> = {
         name: 'Veronica',