@stonyx/orm 0.3.2-beta.326 → 0.3.2-beta.328

package/src/mysql/mysql-db.ts

@@ -4,7 +4,7 @@ import { ensureMigrationsTable, getAppliedMigrations, getMigrationFiles, applyMi
 import { introspectModels, introspectViews, getTopologicalOrder, schemasToSnapshot } from './schema-introspector.js';
 import type { ModelSchema, ViewSchema, SnapshotEntry } from './schema-introspector.js';
 import { loadLatestSnapshot, detectSchemaDrift } from './migration-generator.js';
-import { buildInsert, buildUpdate, buildDelete, buildSelect } from './query-builder.js';
+import { buildInsert, buildInsertMany, buildUpdate, buildDelete, buildSelect } from './query-builder.js';
 import { store } from '@stonyx/orm';
 import { createRecord } from '../manage-record.js';
 import { confirm } from '@stonyx/utils/prompt';
@@ -32,6 +32,43 @@ interface ExecuteResult {
   insertId?: number;
 }
 
+interface QueuedWrite {
+  operation: string;
+  modelName: string;
+  context: PersistContext;
+  response: PersistResponse;
+  resolve: () => void;
+  reject: (error: unknown) => void;
+}
+
+interface PreparedCreate {
+  write: QueuedWrite;
+  modelName: string;
+  record: OrmRecord;
+  row: Record<string, unknown>;
+  isPendingId: boolean;
+}
+
+type ExitEmitter = Pick<NodeJS.Process, 'once' | 'removeListener'>;
+
+const MYSQL_MAX_PLACEHOLDERS = 65535;
+const DEFAULT_WRITE_BATCH_SIZE = 500;
+const DEFAULT_WRITE_QUEUE_LIMIT = 10000;
+
+export class WriteQueueFullError extends Error {
+  constructor(limit: number) {
+    super(`MysqlDB write queue is full (${limit} queued writes, writeQueueLimit); the write was rejected`);
+    this.name = 'WriteQueueFullError';
+  }
+}
+
+export class WriteQueueClosedError extends Error {
+  constructor() {
+    super('MysqlDB is shutting down; the write was rejected');
+    this.name = 'WriteQueueClosedError';
+  }
+}
+
 interface OrmStore {
   get(key: string): Map<number | string, unknown> | undefined;
   get(key: string, id: number | string): unknown;
@@ -54,6 +91,7 @@ interface MysqlDBDeps {
   loadLatestSnapshot: typeof loadLatestSnapshot;
   detectSchemaDrift: typeof detectSchemaDrift;
   buildInsert: typeof buildInsert;
+  buildInsertMany: typeof buildInsertMany;
   buildUpdate: typeof buildUpdate;
   buildDelete: typeof buildDelete;
   buildSelect: typeof buildSelect;
@@ -65,6 +103,7 @@ interface MysqlDBDeps {
   config: typeof config;
   log: Record<string, ((...args: unknown[]) => void) | undefined>;
   path: typeof path;
+  process: ExitEmitter;
 }
 
 const defaultDeps: MysqlDBDeps = {
@@ -72,9 +111,9 @@ const defaultDeps: MysqlDBDeps = {
   getMigrationFiles, applyMigration, parseMigrationFile,
   introspectModels, introspectViews, getTopologicalOrder, schemasToSnapshot,
   loadLatestSnapshot, detectSchemaDrift,
-  buildInsert, buildUpdate, buildDelete, buildSelect,
+  buildInsert, buildInsertMany, buildUpdate, buildDelete, buildSelect,
   createRecord, store: store as OrmStore, confirm, readFile, getPluralName,
-  config, log, path
+  config, log, path, process
 };
 
 export default class MysqlDB {
@@ -84,14 +123,11 @@ export default class MysqlDB {
   pool!: Pool | null;
   mysqlConfig!: MysqlConfig;
 
-  /**
-   * Promise-chain mutex for write serialization (#156).
-   * All persist() calls chain through this single queue so concurrent
-   * fire-and-forget writes never produce parallel InnoDB transactions
-   * on FK-linked rows (which cause deadlocks).
-   * Reads are NOT affected — only persist() serializes.
-   */
-  private _writeQueue: Promise<void> = Promise.resolve();
+  private _queuedWrites: QueuedWrite[] = [];
+  private _inFlightWrites = new Set<QueuedWrite>();
+  private _drainingWrites: Promise<void> | null = null;
+  private _acceptingWrites = true;
+  private _exitReport: (() => void) | null = null;
 
   constructor(deps: Partial<MysqlDBDeps> = {}) {
     if (MysqlDB.instance) return MysqlDB.instance;
@@ -113,6 +149,20 @@ export default class MysqlDB {
     this.pool = await this.deps.getPool(this.mysqlConfig);
     await this.deps.ensureMigrationsTable(this.pool, this.mysqlConfig.migrationsTable);
     await this.loadMemoryRecords();
+    this._acceptingWrites = true;
+    this._registerExitReport();
+  }
+
+  get writeBatchSize(): number {
+    return Number(this.mysqlConfig?.writeBatchSize) || DEFAULT_WRITE_BATCH_SIZE;
+  }
+
+  get writeQueueLimit(): number {
+    return Number(this.mysqlConfig?.writeQueueLimit) || DEFAULT_WRITE_QUEUE_LIMIT;
+  }
+
+  get pendingWriteCount(): number {
+    return this._inFlightWrites.size + this._queuedWrites.length;
   }
 
   async startup(): Promise<void> {
@@ -200,10 +250,36 @@ export default class MysqlDB {
   }
 
   async shutdown(): Promise<void> {
+    this._acceptingWrites = false;
+    while (this._drainingWrites) await this._drainingWrites;
+    this._unregisterExitReport();
     await this.deps.closePool();
     this.pool = null;
   }
 
+  private _registerExitReport(): void {
+    if (this._exitReport) return;
+    this._exitReport = () => this._reportUnpersistedWrites();
+    this.deps.process.once('exit', this._exitReport);
+  }
+
+  private _unregisterExitReport(): void {
+    if (!this._exitReport) return;
+    this.deps.process.removeListener('exit', this._exitReport);
+    this._exitReport = null;
+  }
+
+  private _reportUnpersistedWrites(): void {
+    const lostWrites = [...this._inFlightWrites, ...this._queuedWrites];
+    if (lostWrites.length === 0) return;
+
+    const lostByModel = new Map<string, number>();
+    for (const { modelName } of lostWrites) lostByModel.set(modelName, (lostByModel.get(modelName) ?? 0) + 1);
+    const breakdown = [...lostByModel].map(([modelName, count]) => `${modelName}: ${count}`).join(', ');
+
+    this.deps.log.error?.(`[MysqlDB] Process exiting with ${lostWrites.length} queued write(s) not persisted; they are lost (${breakdown})`);
+  }
+
   async save(): Promise<void> {
     // No-op: MySQL persists data immediately via persist()
   }
@@ -419,76 +495,185 @@ export default class MysqlDB {
   }
 
   async persist(operation: string, modelName: string, context: PersistContext, response: PersistResponse): Promise<void> {
-    // Views are read-only — no-op for all write operations
     const Orm = (await import('@stonyx/orm')).default;
     if ((Orm as unknown as { instance?: { isView?: (name: string) => boolean } }).instance?.isView?.(modelName)) return;
 
-    const work = async () => {
-      switch (operation) {
-        case 'create':
-          return this._persistCreate(modelName, context, response);
-        case 'update':
-          return this._persistUpdate(modelName, context, response);
-        case 'delete':
-          return this._persistDelete(modelName, context);
+    if (!this._acceptingWrites) throw new WriteQueueClosedError();
+    if (this._queuedWrites.length >= this.writeQueueLimit) throw new WriteQueueFullError(this.writeQueueLimit);
+
+    return new Promise<void>((resolve, reject) => {
+      const write: QueuedWrite = {
+        operation, modelName, context, response,
+        resolve: () => {
+          this._inFlightWrites.delete(write);
+          resolve();
+        },
+        reject: (error: unknown) => {
+          this._inFlightWrites.delete(write);
+          reject(error);
+        },
+      };
+      this._queuedWrites.push(write);
+      this._drainWritesInBackground();
+    });
+  }
+
+  private _drainWritesInBackground(): void {
+    if (this._drainingWrites) return;
+    this._drainingWrites = this._drainQueuedWrites().finally(() => {
+      this._drainingWrites = null;
+    });
+  }
+
+  private async _drainQueuedWrites(): Promise<void> {
+    while (this._queuedWrites.length > 0) {
+      const writes = this._takeNextWriteGroup();
+      writes.forEach(write => this._inFlightWrites.add(write));
+      try {
+        await this._executeWriteGroup(writes);
+      } catch (error) {
+        writes.filter(write => this._inFlightWrites.has(write)).forEach(write => write.reject(error));
       }
-    };
+    }
+  }
+
+  private _takeNextWriteGroup(): QueuedWrite[] {
+    const [first] = this._queuedWrites;
+    let groupSize = 1;
+
+    if (first.operation === 'create') {
+      while (
+        groupSize < this.writeBatchSize
+        && groupSize < this._queuedWrites.length
+        && this._queuedWrites[groupSize].operation === 'create'
+        && this._queuedWrites[groupSize].modelName === first.modelName
+      ) groupSize++;
+    }
 
-    // Chain through the write queue — .then(work, work) ensures the queue
-    // advances even when a previous persist rejects (#156).
-    this._writeQueue = this._writeQueue.then(work, work);
-    return this._writeQueue;
+    return this._queuedWrites.splice(0, groupSize);
   }
 
-  private async _persistCreate(modelName: string, context: PersistContext, response: PersistResponse): Promise<void> {
-    const schemas = this.deps.introspectModels();
-    const schema = schemas[modelName];
+  private async _executeWriteGroup(writes: QueuedWrite[]): Promise<void> {
+    const [first] = writes;
+    if (first.operation === 'create') return this._persistCreates(first.modelName, writes);
 
-    if (!schema) return;
+    try {
+      if (first.operation === 'update') await this._persistUpdate(first.modelName, first.context, first.response);
+      if (first.operation === 'delete') await this._persistDelete(first.modelName, first.context);
+      first.resolve();
+    } catch (error) {
+      first.reject(error);
+    }
+  }
+
+  private async _persistCreates(modelName: string, writes: QueuedWrite[]): Promise<void> {
+    const schema = this.deps.introspectModels()[modelName];
+    const prepared: PreparedCreate[] = [];
+
+    for (const write of writes) {
+      try {
+        const create = schema ? this._prepareCreate(modelName, schema, write) : null;
+        if (create) prepared.push(create);
+        else write.resolve();
+      } catch (error) {
+        write.reject(error);
+      }
+    }
+
+    for (const chunk of this._chunkForSingleInsert(prepared)) {
+      await this._insertCreates(schema, chunk);
+    }
+  }
 
+  private _prepareCreate(modelName: string, schema: ModelSchema, write: QueuedWrite): PreparedCreate | null {
+    const { context, response } = write;
     const recordId = response?.data?.id;
     const record = recordId != null ? this.deps.store.get(modelName, (isNaN(recordId as number) ? recordId : parseInt(recordId as string)) as number | string) as OrmRecord | null : null;
 
-    if (!record) return;
-
-    const insertData = this._recordToRow(record, schema);
+    if (!record) return null;
 
-    // For auto-increment models, remove the pending ID.
-    // Check context.rawData (not record.__data) because __pendingSqlId is not a model
-    // attribute and gets lost during serialization.
+    const row = this._recordToRow(record, schema);
     const isPendingId = context.rawData?.__pendingSqlId === true;
 
-    if (isPendingId) {
-      delete insertData.id;
-    } else if (insertData.id !== undefined) {
-      // Keep user-provided ID (string IDs or explicit numeric IDs)
+    if (isPendingId) delete row.id;
+
+    return { write, modelName, record, row, isPendingId };
+  }
+
+  private _chunkForSingleInsert(prepared: PreparedCreate[]): PreparedCreate[][] {
+    const chunks: PreparedCreate[][] = [];
+    let chunk: PreparedCreate[] = [];
+    let chunkColumns = new Set<string>();
+
+    for (const create of prepared) {
+      const columnsWithCreate = new Set([...chunkColumns, ...Object.keys(create.row)]);
+      const mixesIdSources = chunk.length > 0 && chunk[0].isPendingId !== create.isPendingId;
+      const exceedsPlaceholders = (chunk.length + 1) * columnsWithCreate.size > MYSQL_MAX_PLACEHOLDERS;
+
+      if (chunk.length > 0 && (mixesIdSources || exceedsPlaceholders)) {
+        chunks.push(chunk);
+        chunk = [];
+        chunkColumns = new Set(Object.keys(create.row));
+      } else {
+        chunkColumns = columnsWithCreate;
+      }
+
+      chunk.push(create);
     }
 
-    const { sql, values } = this.deps.buildInsert(schema.table, insertData);
+    if (chunk.length > 0) chunks.push(chunk);
+    return chunks;
+  }
 
-    const [result] = await this.requirePool().execute(sql, values) as [ExecuteResult, unknown];
+  private async _insertCreates(schema: ModelSchema, creates: PreparedCreate[]): Promise<void> {
+    if (creates.length === 1) return this._insertCreatesOneByOne(schema, creates);
 
-    // Re-key the record in the store if MySQL generated the ID
-    if (isPendingId && result.insertId) {
-      const pendingId = record.id;
-      const realId = result.insertId;
-      const modelStore = this.deps.store.get(modelName);
-      if (!modelStore) throw new Error(`Model "${modelName}" not found in store during ID re-key`);
+    let firstInsertId: number | undefined;
 
-      modelStore.delete(pendingId as number | string);
-      record.__data.id = realId;
-      record.id = realId;
-      modelStore.set(realId as number | string, record);
+    try {
+      const { sql, values } = this.deps.buildInsertMany(schema.table, creates.map(({ row }) => row));
+      const [result] = await this.requirePool().execute(sql, values) as [ExecuteResult, unknown];
+      firstInsertId = result.insertId;
+    } catch {
+      return this._insertCreatesOneByOne(schema, creates);
+    }
 
-      // Update the response data with the real ID
-      if (response?.data) {
-        response.data.id = realId;
-      }
+    creates.forEach((create, index) => {
+      if (firstInsertId) this._adoptGeneratedId(create, firstInsertId + index);
+      create.write.resolve();
+    });
+  }
 
-      delete record.__data.__pendingSqlId;
+  private async _insertCreatesOneByOne(schema: ModelSchema, creates: PreparedCreate[]): Promise<void> {
+    for (const create of creates) {
+      try {
+        const { sql, values } = this.deps.buildInsert(schema.table, create.row);
+        const [result] = await this.requirePool().execute(sql, values) as [ExecuteResult, unknown];
+        if (result.insertId) this._adoptGeneratedId(create, result.insertId);
+        create.write.resolve();
+      } catch (error) {
+        create.write.reject(error);
+      }
     }
   }
 
+  private _adoptGeneratedId({ modelName, record, isPendingId, write }: PreparedCreate, realId: number): void {
+    if (!isPendingId) return;
+
+    const pendingId = record.id;
+    const modelStore = this.deps.store.get(modelName);
+    if (!modelStore) throw new Error(`Model "${modelName}" not found in store during ID re-key`);
+
+    modelStore.delete(pendingId as number | string);
+    record.__data.id = realId;
+    record.id = realId;
+    modelStore.set(realId, record);
+
+    if (write.response?.data) write.response.data.id = realId;
+
+    delete record.__data.__pendingSqlId;
+  }
+
   private async _persistUpdate(modelName: string, context: PersistContext, response: PersistResponse): Promise<void> {
     const schemas = this.deps.introspectModels();
     const schema = schemas[modelName];

package/src/mysql/query-builder.ts

@@ -27,6 +27,27 @@ export function buildInsert(table: string, data: Record<string, unknown>): Query
   return { sql, values };
 }
 
+export function buildInsertMany(table: string, rows: Record<string, unknown>[]): QueryResult {
+  validateIdentifier(table, 'table name');
+
+  const columns = [...new Set(rows.flatMap(row => Object.keys(row)))];
+  columns.forEach(column => validateIdentifier(column, 'column name'));
+
+  const values: unknown[] = [];
+  const tuples = rows.map(row => {
+    const cells = columns.map(column => {
+      if (!Object.hasOwn(row, column)) return 'DEFAULT';
+      values.push(row[column]);
+      return '?';
+    });
+    return `(${cells.join(', ')})`;
+  });
+
+  const sql = `INSERT INTO \`${table}\` (${columns.map(column => `\`${column}\``).join(', ')}) VALUES ${tuples.join(', ')}`;
+
+  return { sql, values };
+}
+
 export function buildUpdate(table: string, id: unknown, data: Record<string, unknown>): QueryResult {
   validateIdentifier(table, 'table name');
 

package/src/postgres/connection.ts

@@ -21,7 +21,7 @@ export async function getPool(pgConfig: PgConfig, extensions: string[] = ['vecto
 
   const { default: pg } = await import('pg');
 
-  const { host, port, user, password, database, connectionLimit, migrationsDir, migrationsTable, autoMigrate, ...poolOpts } = pgConfig;
+  const { host, port, user, password, database, connectionLimit, migrationsDir, migrationsTable, autoMigrate, writeBatchSize, writeQueueLimit, ...poolOpts } = pgConfig;
 
   pool = new pg.Pool({
     host,