@stonyx/orm 0.3.2-beta.327 → 0.3.2-beta.328

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -19,6 +19,8 @@ const {
19
19
  MYSQL_DATABASE,
20
20
  MYSQL_CONNECTION_LIMIT,
21
21
  MYSQL_MIGRATIONS_DIR,
22
+ MYSQL_WRITE_BATCH_SIZE,
23
+ MYSQL_WRITE_QUEUE_LIMIT,
22
24
  PG_HOST,
23
25
  PG_PORT,
24
26
  PG_USER,
@@ -70,6 +72,8 @@ export default {
70
72
  connectionLimit: parseInt(MYSQL_CONNECTION_LIMIT ?? '10'),
71
73
  migrationsDir: MYSQL_MIGRATIONS_DIR ?? 'migrations',
72
74
  migrationsTable: '__migrations',
75
+ writeBatchSize: parseInt(MYSQL_WRITE_BATCH_SIZE ?? '500'),
76
+ writeQueueLimit: parseInt(MYSQL_WRITE_QUEUE_LIMIT ?? '10000'),
73
77
  } : undefined,
74
78
  postgres: PG_HOST ? {
75
79
  host: PG_HOST ?? 'localhost',
@@ -9,6 +9,8 @@ interface MysqlConfig {
9
9
  migrationsTable?: string;
10
10
  migrationsDir?: string;
11
11
  autoMigrate?: boolean;
12
+ writeBatchSize?: number;
13
+ writeQueueLimit?: number;
12
14
  }
13
15
  export declare function getPool(mysqlConfig: MysqlConfig): Promise<Pool>;
14
16
  export declare function closePool(): Promise<void>;
@@ -3,7 +3,7 @@ import type { MysqlConfig } from './connection.js';
3
3
  import { ensureMigrationsTable, getAppliedMigrations, getMigrationFiles, applyMigration, parseMigrationFile } from './migration-runner.js';
4
4
  import { introspectModels, introspectViews, getTopologicalOrder, schemasToSnapshot } from './schema-introspector.js';
5
5
  import { loadLatestSnapshot, detectSchemaDrift } from './migration-generator.js';
6
- import { buildInsert, buildUpdate, buildDelete, buildSelect } from './query-builder.js';
6
+ import { buildInsert, buildInsertMany, buildUpdate, buildDelete, buildSelect } from './query-builder.js';
7
7
  import { createRecord } from '../manage-record.js';
8
8
  import { confirm } from '@stonyx/utils/prompt';
9
9
  import { readFile } from '@stonyx/utils/file';
@@ -23,6 +23,13 @@ interface PersistResponse {
23
23
  id?: unknown;
24
24
  };
25
25
  }
26
+ type ExitEmitter = Pick<NodeJS.Process, 'once' | 'removeListener'>;
27
+ export declare class WriteQueueFullError extends Error {
28
+ constructor(limit: number);
29
+ }
30
+ export declare class WriteQueueClosedError extends Error {
31
+ constructor();
32
+ }
26
33
  interface OrmStore {
27
34
  get(key: string): Map<number | string, unknown> | undefined;
28
35
  get(key: string, id: number | string): unknown;
@@ -44,6 +51,7 @@ interface MysqlDBDeps {
44
51
  loadLatestSnapshot: typeof loadLatestSnapshot;
45
52
  detectSchemaDrift: typeof detectSchemaDrift;
46
53
  buildInsert: typeof buildInsert;
54
+ buildInsertMany: typeof buildInsertMany;
47
55
  buildUpdate: typeof buildUpdate;
48
56
  buildDelete: typeof buildDelete;
49
57
  buildSelect: typeof buildSelect;
@@ -55,25 +63,29 @@ interface MysqlDBDeps {
55
63
  config: typeof config;
56
64
  log: Record<string, ((...args: unknown[]) => void) | undefined>;
57
65
  path: typeof path;
66
+ process: ExitEmitter;
58
67
  }
59
68
  export default class MysqlDB {
60
69
  static instance: MysqlDB | undefined;
61
70
  deps: MysqlDBDeps;
62
71
  pool: Pool | null;
63
72
  mysqlConfig: MysqlConfig;
64
- /**
65
- * Promise-chain mutex for write serialization (#156).
66
- * All persist() calls chain through this single queue so concurrent
67
- * fire-and-forget writes never produce parallel InnoDB transactions
68
- * on FK-linked rows (which cause deadlocks).
69
- * Reads are NOT affected — only persist() serializes.
70
- */
71
- private _writeQueue;
73
+ private _queuedWrites;
74
+ private _inFlightWrites;
75
+ private _drainingWrites;
76
+ private _acceptingWrites;
77
+ private _exitReport;
72
78
  constructor(deps?: Partial<MysqlDBDeps>);
73
79
  private requirePool;
74
80
  init(): Promise<void>;
81
+ get writeBatchSize(): number;
82
+ get writeQueueLimit(): number;
83
+ get pendingWriteCount(): number;
75
84
  startup(): Promise<void>;
76
85
  shutdown(): Promise<void>;
86
+ private _registerExitReport;
87
+ private _unregisterExitReport;
88
+ private _reportUnpersistedWrites;
77
89
  save(): Promise<void>;
78
90
  /**
79
91
  * Loads only models with memory: true into the in-memory store on startup.
@@ -101,7 +113,16 @@ export default class MysqlDB {
101
113
  private _evictIfNotMemory;
102
114
  private _rowToRawData;
103
115
  persist(operation: string, modelName: string, context: PersistContext, response: PersistResponse): Promise<void>;
104
- private _persistCreate;
116
+ private _drainWritesInBackground;
117
+ private _drainQueuedWrites;
118
+ private _takeNextWriteGroup;
119
+ private _executeWriteGroup;
120
+ private _persistCreates;
121
+ private _prepareCreate;
122
+ private _chunkForSingleInsert;
123
+ private _insertCreates;
124
+ private _insertCreatesOneByOne;
125
+ private _adoptGeneratedId;
105
126
  private _persistUpdate;
106
127
  private _persistDelete;
107
128
  private _recordToRow;
@@ -2,7 +2,7 @@ import { getPool, closePool } from './connection.js';
2
2
  import { ensureMigrationsTable, getAppliedMigrations, getMigrationFiles, applyMigration, parseMigrationFile } from './migration-runner.js';
3
3
  import { introspectModels, introspectViews, getTopologicalOrder, schemasToSnapshot } from './schema-introspector.js';
4
4
  import { loadLatestSnapshot, detectSchemaDrift } from './migration-generator.js';
5
- import { buildInsert, buildUpdate, buildDelete, buildSelect } from './query-builder.js';
5
+ import { buildInsert, buildInsertMany, buildUpdate, buildDelete, buildSelect } from './query-builder.js';
6
6
  import { store } from '@stonyx/orm';
7
7
  import { createRecord } from '../manage-record.js';
8
8
  import { confirm } from '@stonyx/utils/prompt';
@@ -12,28 +12,40 @@ import { isDbError } from '../utils.js';
12
12
  import config from 'stonyx/config';
13
13
  import log from 'stonyx/log';
14
14
  import path from 'path';
15
+ const MYSQL_MAX_PLACEHOLDERS = 65535;
16
+ const DEFAULT_WRITE_BATCH_SIZE = 500;
17
+ const DEFAULT_WRITE_QUEUE_LIMIT = 10000;
18
+ export class WriteQueueFullError extends Error {
19
+ constructor(limit) {
20
+ super(`MysqlDB write queue is full (${limit} queued writes, writeQueueLimit); the write was rejected`);
21
+ this.name = 'WriteQueueFullError';
22
+ }
23
+ }
24
+ export class WriteQueueClosedError extends Error {
25
+ constructor() {
26
+ super('MysqlDB is shutting down; the write was rejected');
27
+ this.name = 'WriteQueueClosedError';
28
+ }
29
+ }
15
30
  const defaultDeps = {
16
31
  getPool, closePool, ensureMigrationsTable, getAppliedMigrations,
17
32
  getMigrationFiles, applyMigration, parseMigrationFile,
18
33
  introspectModels, introspectViews, getTopologicalOrder, schemasToSnapshot,
19
34
  loadLatestSnapshot, detectSchemaDrift,
20
- buildInsert, buildUpdate, buildDelete, buildSelect,
35
+ buildInsert, buildInsertMany, buildUpdate, buildDelete, buildSelect,
21
36
  createRecord, store: store, confirm, readFile, getPluralName,
22
- config, log, path
37
+ config, log, path, process
23
38
  };
24
39
  export default class MysqlDB {
25
40
  static instance;
26
41
  deps;
27
42
  pool;
28
43
  mysqlConfig;
29
- /**
30
- * Promise-chain mutex for write serialization (#156).
31
- * All persist() calls chain through this single queue so concurrent
32
- * fire-and-forget writes never produce parallel InnoDB transactions
33
- * on FK-linked rows (which cause deadlocks).
34
- * Reads are NOT affected — only persist() serializes.
35
- */
36
- _writeQueue = Promise.resolve();
44
+ _queuedWrites = [];
45
+ _inFlightWrites = new Set();
46
+ _drainingWrites = null;
47
+ _acceptingWrites = true;
48
+ _exitReport = null;
37
49
  constructor(deps = {}) {
38
50
  if (MysqlDB.instance)
39
51
  return MysqlDB.instance;
@@ -54,6 +66,17 @@ export default class MysqlDB {
54
66
  this.pool = await this.deps.getPool(this.mysqlConfig);
55
67
  await this.deps.ensureMigrationsTable(this.pool, this.mysqlConfig.migrationsTable);
56
68
  await this.loadMemoryRecords();
69
+ this._acceptingWrites = true;
70
+ this._registerExitReport();
71
+ }
72
+ get writeBatchSize() {
73
+ return Number(this.mysqlConfig?.writeBatchSize) || DEFAULT_WRITE_BATCH_SIZE;
74
+ }
75
+ get writeQueueLimit() {
76
+ return Number(this.mysqlConfig?.writeQueueLimit) || DEFAULT_WRITE_QUEUE_LIMIT;
77
+ }
78
+ get pendingWriteCount() {
79
+ return this._inFlightWrites.size + this._queuedWrites.length;
57
80
  }
58
81
  async startup() {
59
82
  if (!this.mysqlConfig.migrationsDir)
@@ -134,9 +157,35 @@ export default class MysqlDB {
134
157
  }
135
158
  }
136
159
  async shutdown() {
160
+ this._acceptingWrites = false;
161
+ while (this._drainingWrites)
162
+ await this._drainingWrites;
163
+ this._unregisterExitReport();
137
164
  await this.deps.closePool();
138
165
  this.pool = null;
139
166
  }
167
+ _registerExitReport() {
168
+ if (this._exitReport)
169
+ return;
170
+ this._exitReport = () => this._reportUnpersistedWrites();
171
+ this.deps.process.once('exit', this._exitReport);
172
+ }
173
+ _unregisterExitReport() {
174
+ if (!this._exitReport)
175
+ return;
176
+ this.deps.process.removeListener('exit', this._exitReport);
177
+ this._exitReport = null;
178
+ }
179
+ _reportUnpersistedWrites() {
180
+ const lostWrites = [...this._inFlightWrites, ...this._queuedWrites];
181
+ if (lostWrites.length === 0)
182
+ return;
183
+ const lostByModel = new Map();
184
+ for (const { modelName } of lostWrites)
185
+ lostByModel.set(modelName, (lostByModel.get(modelName) ?? 0) + 1);
186
+ const breakdown = [...lostByModel].map(([modelName, count]) => `${modelName}: ${count}`).join(', ');
187
+ this.deps.log.error?.(`[MysqlDB] Process exiting with ${lostWrites.length} queued write(s) not persisted; they are lost (${breakdown})`);
188
+ }
140
189
  async save() {
141
190
  // No-op: MySQL persists data immediately via persist()
142
191
  }
@@ -326,64 +375,174 @@ export default class MysqlDB {
326
375
  return rawData;
327
376
  }
328
377
  async persist(operation, modelName, context, response) {
329
- // Views are read-only — no-op for all write operations
330
378
  const Orm = (await import('@stonyx/orm')).default;
331
379
  if (Orm.instance?.isView?.(modelName))
332
380
  return;
333
- const work = async () => {
334
- switch (operation) {
335
- case 'create':
336
- return this._persistCreate(modelName, context, response);
337
- case 'update':
338
- return this._persistUpdate(modelName, context, response);
339
- case 'delete':
340
- return this._persistDelete(modelName, context);
341
- }
342
- };
343
- // Chain through the write queue — .then(work, work) ensures the queue
344
- // advances even when a previous persist rejects (#156).
345
- this._writeQueue = this._writeQueue.then(work, work);
346
- return this._writeQueue;
347
- }
348
- async _persistCreate(modelName, context, response) {
349
- const schemas = this.deps.introspectModels();
350
- const schema = schemas[modelName];
351
- if (!schema)
381
+ if (!this._acceptingWrites)
382
+ throw new WriteQueueClosedError();
383
+ if (this._queuedWrites.length >= this.writeQueueLimit)
384
+ throw new WriteQueueFullError(this.writeQueueLimit);
385
+ return new Promise((resolve, reject) => {
386
+ const write = {
387
+ operation, modelName, context, response,
388
+ resolve: () => {
389
+ this._inFlightWrites.delete(write);
390
+ resolve();
391
+ },
392
+ reject: (error) => {
393
+ this._inFlightWrites.delete(write);
394
+ reject(error);
395
+ },
396
+ };
397
+ this._queuedWrites.push(write);
398
+ this._drainWritesInBackground();
399
+ });
400
+ }
401
+ _drainWritesInBackground() {
402
+ if (this._drainingWrites)
352
403
  return;
404
+ this._drainingWrites = this._drainQueuedWrites().finally(() => {
405
+ this._drainingWrites = null;
406
+ });
407
+ }
408
+ async _drainQueuedWrites() {
409
+ while (this._queuedWrites.length > 0) {
410
+ const writes = this._takeNextWriteGroup();
411
+ writes.forEach(write => this._inFlightWrites.add(write));
412
+ try {
413
+ await this._executeWriteGroup(writes);
414
+ }
415
+ catch (error) {
416
+ writes.filter(write => this._inFlightWrites.has(write)).forEach(write => write.reject(error));
417
+ }
418
+ }
419
+ }
420
+ _takeNextWriteGroup() {
421
+ const [first] = this._queuedWrites;
422
+ let groupSize = 1;
423
+ if (first.operation === 'create') {
424
+ while (groupSize < this.writeBatchSize
425
+ && groupSize < this._queuedWrites.length
426
+ && this._queuedWrites[groupSize].operation === 'create'
427
+ && this._queuedWrites[groupSize].modelName === first.modelName)
428
+ groupSize++;
429
+ }
430
+ return this._queuedWrites.splice(0, groupSize);
431
+ }
432
+ async _executeWriteGroup(writes) {
433
+ const [first] = writes;
434
+ if (first.operation === 'create')
435
+ return this._persistCreates(first.modelName, writes);
436
+ try {
437
+ if (first.operation === 'update')
438
+ await this._persistUpdate(first.modelName, first.context, first.response);
439
+ if (first.operation === 'delete')
440
+ await this._persistDelete(first.modelName, first.context);
441
+ first.resolve();
442
+ }
443
+ catch (error) {
444
+ first.reject(error);
445
+ }
446
+ }
447
+ async _persistCreates(modelName, writes) {
448
+ const schema = this.deps.introspectModels()[modelName];
449
+ const prepared = [];
450
+ for (const write of writes) {
451
+ try {
452
+ const create = schema ? this._prepareCreate(modelName, schema, write) : null;
453
+ if (create)
454
+ prepared.push(create);
455
+ else
456
+ write.resolve();
457
+ }
458
+ catch (error) {
459
+ write.reject(error);
460
+ }
461
+ }
462
+ for (const chunk of this._chunkForSingleInsert(prepared)) {
463
+ await this._insertCreates(schema, chunk);
464
+ }
465
+ }
466
+ _prepareCreate(modelName, schema, write) {
467
+ const { context, response } = write;
353
468
  const recordId = response?.data?.id;
354
469
  const record = recordId != null ? this.deps.store.get(modelName, (isNaN(recordId) ? recordId : parseInt(recordId))) : null;
355
470
  if (!record)
356
- return;
357
- const insertData = this._recordToRow(record, schema);
358
- // For auto-increment models, remove the pending ID.
359
- // Check context.rawData (not record.__data) because __pendingSqlId is not a model
360
- // attribute and gets lost during serialization.
471
+ return null;
472
+ const row = this._recordToRow(record, schema);
361
473
  const isPendingId = context.rawData?.__pendingSqlId === true;
362
- if (isPendingId) {
363
- delete insertData.id;
474
+ if (isPendingId)
475
+ delete row.id;
476
+ return { write, modelName, record, row, isPendingId };
477
+ }
478
+ _chunkForSingleInsert(prepared) {
479
+ const chunks = [];
480
+ let chunk = [];
481
+ let chunkColumns = new Set();
482
+ for (const create of prepared) {
483
+ const columnsWithCreate = new Set([...chunkColumns, ...Object.keys(create.row)]);
484
+ const mixesIdSources = chunk.length > 0 && chunk[0].isPendingId !== create.isPendingId;
485
+ const exceedsPlaceholders = (chunk.length + 1) * columnsWithCreate.size > MYSQL_MAX_PLACEHOLDERS;
486
+ if (chunk.length > 0 && (mixesIdSources || exceedsPlaceholders)) {
487
+ chunks.push(chunk);
488
+ chunk = [];
489
+ chunkColumns = new Set(Object.keys(create.row));
490
+ }
491
+ else {
492
+ chunkColumns = columnsWithCreate;
493
+ }
494
+ chunk.push(create);
364
495
  }
365
- else if (insertData.id !== undefined) {
366
- // Keep user-provided ID (string IDs or explicit numeric IDs)
496
+ if (chunk.length > 0)
497
+ chunks.push(chunk);
498
+ return chunks;
499
+ }
500
+ async _insertCreates(schema, creates) {
501
+ if (creates.length === 1)
502
+ return this._insertCreatesOneByOne(schema, creates);
503
+ let firstInsertId;
504
+ try {
505
+ const { sql, values } = this.deps.buildInsertMany(schema.table, creates.map(({ row }) => row));
506
+ const [result] = await this.requirePool().execute(sql, values);
507
+ firstInsertId = result.insertId;
367
508
  }
368
- const { sql, values } = this.deps.buildInsert(schema.table, insertData);
369
- const [result] = await this.requirePool().execute(sql, values);
370
- // Re-key the record in the store if MySQL generated the ID
371
- if (isPendingId && result.insertId) {
372
- const pendingId = record.id;
373
- const realId = result.insertId;
374
- const modelStore = this.deps.store.get(modelName);
375
- if (!modelStore)
376
- throw new Error(`Model "${modelName}" not found in store during ID re-key`);
377
- modelStore.delete(pendingId);
378
- record.__data.id = realId;
379
- record.id = realId;
380
- modelStore.set(realId, record);
381
- // Update the response data with the real ID
382
- if (response?.data) {
383
- response.data.id = realId;
384
- }
385
- delete record.__data.__pendingSqlId;
509
+ catch {
510
+ return this._insertCreatesOneByOne(schema, creates);
386
511
  }
512
+ creates.forEach((create, index) => {
513
+ if (firstInsertId)
514
+ this._adoptGeneratedId(create, firstInsertId + index);
515
+ create.write.resolve();
516
+ });
517
+ }
518
+ async _insertCreatesOneByOne(schema, creates) {
519
+ for (const create of creates) {
520
+ try {
521
+ const { sql, values } = this.deps.buildInsert(schema.table, create.row);
522
+ const [result] = await this.requirePool().execute(sql, values);
523
+ if (result.insertId)
524
+ this._adoptGeneratedId(create, result.insertId);
525
+ create.write.resolve();
526
+ }
527
+ catch (error) {
528
+ create.write.reject(error);
529
+ }
530
+ }
531
+ }
532
+ _adoptGeneratedId({ modelName, record, isPendingId, write }, realId) {
533
+ if (!isPendingId)
534
+ return;
535
+ const pendingId = record.id;
536
+ const modelStore = this.deps.store.get(modelName);
537
+ if (!modelStore)
538
+ throw new Error(`Model "${modelName}" not found in store during ID re-key`);
539
+ modelStore.delete(pendingId);
540
+ record.__data.id = realId;
541
+ record.id = realId;
542
+ modelStore.set(realId, record);
543
+ if (write.response?.data)
544
+ write.response.data.id = realId;
545
+ delete record.__data.__pendingSqlId;
387
546
  }
388
547
  async _persistUpdate(modelName, context, response) {
389
548
  const schemas = this.deps.introspectModels();
@@ -4,6 +4,7 @@ interface QueryResult {
4
4
  }
5
5
  export declare function validateIdentifier(name: string, context?: string): string;
6
6
  export declare function buildInsert(table: string, data: Record<string, unknown>): QueryResult;
7
+ export declare function buildInsertMany(table: string, rows: Record<string, unknown>[]): QueryResult;
7
8
  export declare function buildUpdate(table: string, id: unknown, data: Record<string, unknown>): QueryResult;
8
9
  export declare function buildDelete(table: string, id: unknown): QueryResult;
9
10
  export declare function buildSelect(table: string, conditions?: Record<string, unknown>): QueryResult;
@@ -14,6 +14,23 @@ export function buildInsert(table, data) {
14
14
  const sql = `INSERT INTO \`${table}\` (${keys.map(k => `\`${k}\``).join(', ')}) VALUES (${placeholders.join(', ')})`;
15
15
  return { sql, values };
16
16
  }
17
+ export function buildInsertMany(table, rows) {
18
+ validateIdentifier(table, 'table name');
19
+ const columns = [...new Set(rows.flatMap(row => Object.keys(row)))];
20
+ columns.forEach(column => validateIdentifier(column, 'column name'));
21
+ const values = [];
22
+ const tuples = rows.map(row => {
23
+ const cells = columns.map(column => {
24
+ if (!Object.hasOwn(row, column))
25
+ return 'DEFAULT';
26
+ values.push(row[column]);
27
+ return '?';
28
+ });
29
+ return `(${cells.join(', ')})`;
30
+ });
31
+ const sql = `INSERT INTO \`${table}\` (${columns.map(column => `\`${column}\``).join(', ')}) VALUES ${tuples.join(', ')}`;
32
+ return { sql, values };
33
+ }
17
34
  export function buildUpdate(table, id, data) {
18
35
  validateIdentifier(table, 'table name');
19
36
  const keys = Object.keys(data);
package/package.json CHANGED
@@ -4,7 +4,7 @@
4
4
  "stonyx-async",
5
5
  "stonyx-module"
6
6
  ],
7
- "version": "0.3.2-beta.327",
7
+ "version": "0.3.2-beta.328",
8
8
  "description": "",
9
9
  "main": "dist/index.js",
10
10
  "type": "module",
@@ -10,6 +10,8 @@ interface MysqlConfig {
10
10
  migrationsTable?: string;
11
11
  migrationsDir?: string;
12
12
  autoMigrate?: boolean;
13
+ writeBatchSize?: number;
14
+ writeQueueLimit?: number;
13
15
  }
14
16
 
15
17
  let pool: Pool | null = null;
@@ -4,7 +4,7 @@ import { ensureMigrationsTable, getAppliedMigrations, getMigrationFiles, applyMi
4
4
  import { introspectModels, introspectViews, getTopologicalOrder, schemasToSnapshot } from './schema-introspector.js';
5
5
  import type { ModelSchema, ViewSchema, SnapshotEntry } from './schema-introspector.js';
6
6
  import { loadLatestSnapshot, detectSchemaDrift } from './migration-generator.js';
7
- import { buildInsert, buildUpdate, buildDelete, buildSelect } from './query-builder.js';
7
+ import { buildInsert, buildInsertMany, buildUpdate, buildDelete, buildSelect } from './query-builder.js';
8
8
  import { store } from '@stonyx/orm';
9
9
  import { createRecord } from '../manage-record.js';
10
10
  import { confirm } from '@stonyx/utils/prompt';
@@ -32,6 +32,43 @@ interface ExecuteResult {
32
32
  insertId?: number;
33
33
  }
34
34
 
35
+ interface QueuedWrite {
36
+ operation: string;
37
+ modelName: string;
38
+ context: PersistContext;
39
+ response: PersistResponse;
40
+ resolve: () => void;
41
+ reject: (error: unknown) => void;
42
+ }
43
+
44
+ interface PreparedCreate {
45
+ write: QueuedWrite;
46
+ modelName: string;
47
+ record: OrmRecord;
48
+ row: Record<string, unknown>;
49
+ isPendingId: boolean;
50
+ }
51
+
52
+ type ExitEmitter = Pick<NodeJS.Process, 'once' | 'removeListener'>;
53
+
54
+ const MYSQL_MAX_PLACEHOLDERS = 65535;
55
+ const DEFAULT_WRITE_BATCH_SIZE = 500;
56
+ const DEFAULT_WRITE_QUEUE_LIMIT = 10000;
57
+
58
+ export class WriteQueueFullError extends Error {
59
+ constructor(limit: number) {
60
+ super(`MysqlDB write queue is full (${limit} queued writes, writeQueueLimit); the write was rejected`);
61
+ this.name = 'WriteQueueFullError';
62
+ }
63
+ }
64
+
65
+ export class WriteQueueClosedError extends Error {
66
+ constructor() {
67
+ super('MysqlDB is shutting down; the write was rejected');
68
+ this.name = 'WriteQueueClosedError';
69
+ }
70
+ }
71
+
35
72
  interface OrmStore {
36
73
  get(key: string): Map<number | string, unknown> | undefined;
37
74
  get(key: string, id: number | string): unknown;
@@ -54,6 +91,7 @@ interface MysqlDBDeps {
54
91
  loadLatestSnapshot: typeof loadLatestSnapshot;
55
92
  detectSchemaDrift: typeof detectSchemaDrift;
56
93
  buildInsert: typeof buildInsert;
94
+ buildInsertMany: typeof buildInsertMany;
57
95
  buildUpdate: typeof buildUpdate;
58
96
  buildDelete: typeof buildDelete;
59
97
  buildSelect: typeof buildSelect;
@@ -65,6 +103,7 @@ interface MysqlDBDeps {
65
103
  config: typeof config;
66
104
  log: Record<string, ((...args: unknown[]) => void) | undefined>;
67
105
  path: typeof path;
106
+ process: ExitEmitter;
68
107
  }
69
108
 
70
109
  const defaultDeps: MysqlDBDeps = {
@@ -72,9 +111,9 @@ const defaultDeps: MysqlDBDeps = {
72
111
  getMigrationFiles, applyMigration, parseMigrationFile,
73
112
  introspectModels, introspectViews, getTopologicalOrder, schemasToSnapshot,
74
113
  loadLatestSnapshot, detectSchemaDrift,
75
- buildInsert, buildUpdate, buildDelete, buildSelect,
114
+ buildInsert, buildInsertMany, buildUpdate, buildDelete, buildSelect,
76
115
  createRecord, store: store as OrmStore, confirm, readFile, getPluralName,
77
- config, log, path
116
+ config, log, path, process
78
117
  };
79
118
 
80
119
  export default class MysqlDB {
@@ -84,14 +123,11 @@ export default class MysqlDB {
84
123
  pool!: Pool | null;
85
124
  mysqlConfig!: MysqlConfig;
86
125
 
87
- /**
88
- * Promise-chain mutex for write serialization (#156).
89
- * All persist() calls chain through this single queue so concurrent
90
- * fire-and-forget writes never produce parallel InnoDB transactions
91
- * on FK-linked rows (which cause deadlocks).
92
- * Reads are NOT affected — only persist() serializes.
93
- */
94
- private _writeQueue: Promise<void> = Promise.resolve();
126
+ private _queuedWrites: QueuedWrite[] = [];
127
+ private _inFlightWrites = new Set<QueuedWrite>();
128
+ private _drainingWrites: Promise<void> | null = null;
129
+ private _acceptingWrites = true;
130
+ private _exitReport: (() => void) | null = null;
95
131
 
96
132
  constructor(deps: Partial<MysqlDBDeps> = {}) {
97
133
  if (MysqlDB.instance) return MysqlDB.instance;
@@ -113,6 +149,20 @@ export default class MysqlDB {
113
149
  this.pool = await this.deps.getPool(this.mysqlConfig);
114
150
  await this.deps.ensureMigrationsTable(this.pool, this.mysqlConfig.migrationsTable);
115
151
  await this.loadMemoryRecords();
152
+ this._acceptingWrites = true;
153
+ this._registerExitReport();
154
+ }
155
+
156
+ get writeBatchSize(): number {
157
+ return Number(this.mysqlConfig?.writeBatchSize) || DEFAULT_WRITE_BATCH_SIZE;
158
+ }
159
+
160
+ get writeQueueLimit(): number {
161
+ return Number(this.mysqlConfig?.writeQueueLimit) || DEFAULT_WRITE_QUEUE_LIMIT;
162
+ }
163
+
164
+ get pendingWriteCount(): number {
165
+ return this._inFlightWrites.size + this._queuedWrites.length;
116
166
  }
117
167
 
118
168
  async startup(): Promise<void> {
@@ -200,10 +250,36 @@ export default class MysqlDB {
200
250
  }
201
251
 
202
252
  async shutdown(): Promise<void> {
253
+ this._acceptingWrites = false;
254
+ while (this._drainingWrites) await this._drainingWrites;
255
+ this._unregisterExitReport();
203
256
  await this.deps.closePool();
204
257
  this.pool = null;
205
258
  }
206
259
 
260
+ private _registerExitReport(): void {
261
+ if (this._exitReport) return;
262
+ this._exitReport = () => this._reportUnpersistedWrites();
263
+ this.deps.process.once('exit', this._exitReport);
264
+ }
265
+
266
+ private _unregisterExitReport(): void {
267
+ if (!this._exitReport) return;
268
+ this.deps.process.removeListener('exit', this._exitReport);
269
+ this._exitReport = null;
270
+ }
271
+
272
+ private _reportUnpersistedWrites(): void {
273
+ const lostWrites = [...this._inFlightWrites, ...this._queuedWrites];
274
+ if (lostWrites.length === 0) return;
275
+
276
+ const lostByModel = new Map<string, number>();
277
+ for (const { modelName } of lostWrites) lostByModel.set(modelName, (lostByModel.get(modelName) ?? 0) + 1);
278
+ const breakdown = [...lostByModel].map(([modelName, count]) => `${modelName}: ${count}`).join(', ');
279
+
280
+ this.deps.log.error?.(`[MysqlDB] Process exiting with ${lostWrites.length} queued write(s) not persisted; they are lost (${breakdown})`);
281
+ }
282
+
207
283
  async save(): Promise<void> {
208
284
  // No-op: MySQL persists data immediately via persist()
209
285
  }
@@ -419,76 +495,185 @@ export default class MysqlDB {
419
495
  }
420
496
 
421
497
  async persist(operation: string, modelName: string, context: PersistContext, response: PersistResponse): Promise<void> {
422
- // Views are read-only — no-op for all write operations
423
498
  const Orm = (await import('@stonyx/orm')).default;
424
499
  if ((Orm as unknown as { instance?: { isView?: (name: string) => boolean } }).instance?.isView?.(modelName)) return;
425
500
 
426
- const work = async () => {
427
- switch (operation) {
428
- case 'create':
429
- return this._persistCreate(modelName, context, response);
430
- case 'update':
431
- return this._persistUpdate(modelName, context, response);
432
- case 'delete':
433
- return this._persistDelete(modelName, context);
501
+ if (!this._acceptingWrites) throw new WriteQueueClosedError();
502
+ if (this._queuedWrites.length >= this.writeQueueLimit) throw new WriteQueueFullError(this.writeQueueLimit);
503
+
504
+ return new Promise<void>((resolve, reject) => {
505
+ const write: QueuedWrite = {
506
+ operation, modelName, context, response,
507
+ resolve: () => {
508
+ this._inFlightWrites.delete(write);
509
+ resolve();
510
+ },
511
+ reject: (error: unknown) => {
512
+ this._inFlightWrites.delete(write);
513
+ reject(error);
514
+ },
515
+ };
516
+ this._queuedWrites.push(write);
517
+ this._drainWritesInBackground();
518
+ });
519
+ }
520
+
521
+ private _drainWritesInBackground(): void {
522
+ if (this._drainingWrites) return;
523
+ this._drainingWrites = this._drainQueuedWrites().finally(() => {
524
+ this._drainingWrites = null;
525
+ });
526
+ }
527
+
528
+ private async _drainQueuedWrites(): Promise<void> {
529
+ while (this._queuedWrites.length > 0) {
530
+ const writes = this._takeNextWriteGroup();
531
+ writes.forEach(write => this._inFlightWrites.add(write));
532
+ try {
533
+ await this._executeWriteGroup(writes);
534
+ } catch (error) {
535
+ writes.filter(write => this._inFlightWrites.has(write)).forEach(write => write.reject(error));
434
536
  }
435
- };
537
+ }
538
+ }
539
+
540
+ private _takeNextWriteGroup(): QueuedWrite[] {
541
+ const [first] = this._queuedWrites;
542
+ let groupSize = 1;
543
+
544
+ if (first.operation === 'create') {
545
+ while (
546
+ groupSize < this.writeBatchSize
547
+ && groupSize < this._queuedWrites.length
548
+ && this._queuedWrites[groupSize].operation === 'create'
549
+ && this._queuedWrites[groupSize].modelName === first.modelName
550
+ ) groupSize++;
551
+ }
436
552
 
437
- // Chain through the write queue — .then(work, work) ensures the queue
438
- // advances even when a previous persist rejects (#156).
439
- this._writeQueue = this._writeQueue.then(work, work);
440
- return this._writeQueue;
553
+ return this._queuedWrites.splice(0, groupSize);
441
554
  }
442
555
 
443
- private async _persistCreate(modelName: string, context: PersistContext, response: PersistResponse): Promise<void> {
444
- const schemas = this.deps.introspectModels();
445
- const schema = schemas[modelName];
556
+ private async _executeWriteGroup(writes: QueuedWrite[]): Promise<void> {
557
+ const [first] = writes;
558
+ if (first.operation === 'create') return this._persistCreates(first.modelName, writes);
446
559
 
447
- if (!schema) return;
560
+ try {
561
+ if (first.operation === 'update') await this._persistUpdate(first.modelName, first.context, first.response);
562
+ if (first.operation === 'delete') await this._persistDelete(first.modelName, first.context);
563
+ first.resolve();
564
+ } catch (error) {
565
+ first.reject(error);
566
+ }
567
+ }
568
+
569
+ private async _persistCreates(modelName: string, writes: QueuedWrite[]): Promise<void> {
570
+ const schema = this.deps.introspectModels()[modelName];
571
+ const prepared: PreparedCreate[] = [];
572
+
573
+ for (const write of writes) {
574
+ try {
575
+ const create = schema ? this._prepareCreate(modelName, schema, write) : null;
576
+ if (create) prepared.push(create);
577
+ else write.resolve();
578
+ } catch (error) {
579
+ write.reject(error);
580
+ }
581
+ }
582
+
583
+ for (const chunk of this._chunkForSingleInsert(prepared)) {
584
+ await this._insertCreates(schema, chunk);
585
+ }
586
+ }
448
587
 
588
+ private _prepareCreate(modelName: string, schema: ModelSchema, write: QueuedWrite): PreparedCreate | null {
589
+ const { context, response } = write;
449
590
  const recordId = response?.data?.id;
450
591
  const record = recordId != null ? this.deps.store.get(modelName, (isNaN(recordId as number) ? recordId : parseInt(recordId as string)) as number | string) as OrmRecord | null : null;
451
592
 
452
- if (!record) return;
453
-
454
- const insertData = this._recordToRow(record, schema);
593
+ if (!record) return null;
455
594
 
456
- // For auto-increment models, remove the pending ID.
457
- // Check context.rawData (not record.__data) because __pendingSqlId is not a model
458
- // attribute and gets lost during serialization.
595
+ const row = this._recordToRow(record, schema);
459
596
  const isPendingId = context.rawData?.__pendingSqlId === true;
460
597
 
461
- if (isPendingId) {
462
- delete insertData.id;
463
- } else if (insertData.id !== undefined) {
464
- // Keep user-provided ID (string IDs or explicit numeric IDs)
598
+ if (isPendingId) delete row.id;
599
+
600
+ return { write, modelName, record, row, isPendingId };
601
+ }
602
+
603
+ private _chunkForSingleInsert(prepared: PreparedCreate[]): PreparedCreate[][] {
604
+ const chunks: PreparedCreate[][] = [];
605
+ let chunk: PreparedCreate[] = [];
606
+ let chunkColumns = new Set<string>();
607
+
608
+ for (const create of prepared) {
609
+ const columnsWithCreate = new Set([...chunkColumns, ...Object.keys(create.row)]);
610
+ const mixesIdSources = chunk.length > 0 && chunk[0].isPendingId !== create.isPendingId;
611
+ const exceedsPlaceholders = (chunk.length + 1) * columnsWithCreate.size > MYSQL_MAX_PLACEHOLDERS;
612
+
613
+ if (chunk.length > 0 && (mixesIdSources || exceedsPlaceholders)) {
614
+ chunks.push(chunk);
615
+ chunk = [];
616
+ chunkColumns = new Set(Object.keys(create.row));
617
+ } else {
618
+ chunkColumns = columnsWithCreate;
619
+ }
620
+
621
+ chunk.push(create);
465
622
  }
466
623
 
467
- const { sql, values } = this.deps.buildInsert(schema.table, insertData);
624
+ if (chunk.length > 0) chunks.push(chunk);
625
+ return chunks;
626
+ }
468
627
 
469
- const [result] = await this.requirePool().execute(sql, values) as [ExecuteResult, unknown];
628
+ private async _insertCreates(schema: ModelSchema, creates: PreparedCreate[]): Promise<void> {
629
+ if (creates.length === 1) return this._insertCreatesOneByOne(schema, creates);
470
630
 
471
- // Re-key the record in the store if MySQL generated the ID
472
- if (isPendingId && result.insertId) {
473
- const pendingId = record.id;
474
- const realId = result.insertId;
475
- const modelStore = this.deps.store.get(modelName);
476
- if (!modelStore) throw new Error(`Model "${modelName}" not found in store during ID re-key`);
631
+ let firstInsertId: number | undefined;
477
632
 
478
- modelStore.delete(pendingId as number | string);
479
- record.__data.id = realId;
480
- record.id = realId;
481
- modelStore.set(realId as number | string, record);
633
+ try {
634
+ const { sql, values } = this.deps.buildInsertMany(schema.table, creates.map(({ row }) => row));
635
+ const [result] = await this.requirePool().execute(sql, values) as [ExecuteResult, unknown];
636
+ firstInsertId = result.insertId;
637
+ } catch {
638
+ return this._insertCreatesOneByOne(schema, creates);
639
+ }
482
640
 
483
- // Update the response data with the real ID
484
- if (response?.data) {
485
- response.data.id = realId;
486
- }
641
+ creates.forEach((create, index) => {
642
+ if (firstInsertId) this._adoptGeneratedId(create, firstInsertId + index);
643
+ create.write.resolve();
644
+ });
645
+ }
487
646
 
488
- delete record.__data.__pendingSqlId;
647
+ private async _insertCreatesOneByOne(schema: ModelSchema, creates: PreparedCreate[]): Promise<void> {
648
+ for (const create of creates) {
649
+ try {
650
+ const { sql, values } = this.deps.buildInsert(schema.table, create.row);
651
+ const [result] = await this.requirePool().execute(sql, values) as [ExecuteResult, unknown];
652
+ if (result.insertId) this._adoptGeneratedId(create, result.insertId);
653
+ create.write.resolve();
654
+ } catch (error) {
655
+ create.write.reject(error);
656
+ }
489
657
  }
490
658
  }
491
659
 
660
+ private _adoptGeneratedId({ modelName, record, isPendingId, write }: PreparedCreate, realId: number): void {
661
+ if (!isPendingId) return;
662
+
663
+ const pendingId = record.id;
664
+ const modelStore = this.deps.store.get(modelName);
665
+ if (!modelStore) throw new Error(`Model "${modelName}" not found in store during ID re-key`);
666
+
667
+ modelStore.delete(pendingId as number | string);
668
+ record.__data.id = realId;
669
+ record.id = realId;
670
+ modelStore.set(realId, record);
671
+
672
+ if (write.response?.data) write.response.data.id = realId;
673
+
674
+ delete record.__data.__pendingSqlId;
675
+ }
676
+
492
677
  private async _persistUpdate(modelName: string, context: PersistContext, response: PersistResponse): Promise<void> {
493
678
  const schemas = this.deps.introspectModels();
494
679
  const schema = schemas[modelName];
@@ -27,6 +27,27 @@ export function buildInsert(table: string, data: Record<string, unknown>): Query
27
27
  return { sql, values };
28
28
  }
29
29
 
30
+ export function buildInsertMany(table: string, rows: Record<string, unknown>[]): QueryResult {
31
+ validateIdentifier(table, 'table name');
32
+
33
+ const columns = [...new Set(rows.flatMap(row => Object.keys(row)))];
34
+ columns.forEach(column => validateIdentifier(column, 'column name'));
35
+
36
+ const values: unknown[] = [];
37
+ const tuples = rows.map(row => {
38
+ const cells = columns.map(column => {
39
+ if (!Object.hasOwn(row, column)) return 'DEFAULT';
40
+ values.push(row[column]);
41
+ return '?';
42
+ });
43
+ return `(${cells.join(', ')})`;
44
+ });
45
+
46
+ const sql = `INSERT INTO \`${table}\` (${columns.map(column => `\`${column}\``).join(', ')}) VALUES ${tuples.join(', ')}`;
47
+
48
+ return { sql, values };
49
+ }
50
+
30
51
  export function buildUpdate(table: string, id: unknown, data: Record<string, unknown>): QueryResult {
31
52
  validateIdentifier(table, 'table name');
32
53