@stonyx/orm 0.3.2-alpha.122 → 0.3.2-alpha.123

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -26,6 +26,8 @@ const {
26
26
  PG_DATABASE,
27
27
  PG_CONNECTION_LIMIT,
28
28
  PG_MIGRATIONS_DIR,
29
+ PG_WRITE_BATCH_SIZE,
30
+ PG_WRITE_QUEUE_LIMIT,
29
31
  TIMESCALE_HOST,
30
32
  TIMESCALE_PORT,
31
33
  TIMESCALE_USER,
@@ -33,6 +35,8 @@ const {
33
35
  TIMESCALE_DATABASE,
34
36
  TIMESCALE_CONNECTION_LIMIT,
35
37
  TIMESCALE_MIGRATIONS_DIR,
38
+ TIMESCALE_WRITE_BATCH_SIZE,
39
+ TIMESCALE_WRITE_QUEUE_LIMIT,
36
40
  DYNAMODB_REGION,
37
41
  DYNAMODB_ENDPOINT,
38
42
  DYNAMODB_TABLE_PREFIX,
@@ -76,6 +80,8 @@ export default {
76
80
  connectionLimit: parseInt(PG_CONNECTION_LIMIT ?? '10'),
77
81
  migrationsDir: PG_MIGRATIONS_DIR ?? 'migrations',
78
82
  migrationsTable: '__migrations',
83
+ writeBatchSize: parseInt(PG_WRITE_BATCH_SIZE ?? '500'),
84
+ writeQueueLimit: parseInt(PG_WRITE_QUEUE_LIMIT ?? '10000'),
79
85
  } : undefined,
80
86
  timescale: TIMESCALE_HOST ? {
81
87
  host: TIMESCALE_HOST ?? 'localhost',
@@ -86,6 +92,8 @@ export default {
86
92
  connectionLimit: parseInt(TIMESCALE_CONNECTION_LIMIT ?? '10'),
87
93
  migrationsDir: TIMESCALE_MIGRATIONS_DIR ?? 'migrations',
88
94
  migrationsTable: '__migrations',
95
+ writeBatchSize: parseInt(TIMESCALE_WRITE_BATCH_SIZE ?? '500'),
96
+ writeQueueLimit: parseInt(TIMESCALE_WRITE_QUEUE_LIMIT ?? '10000'),
89
97
  } : undefined,
90
98
  dynamodb: DYNAMODB_REGION ? {
91
99
  region: DYNAMODB_REGION,
@@ -7,7 +7,7 @@ export async function getPool(pgConfig, extensions = ['vector']) {
7
7
  if (pool)
8
8
  return pool;
9
9
  const { default: pg } = await import('pg');
10
- const { host, port, user, password, database, connectionLimit, migrationsDir, migrationsTable, autoMigrate, ...poolOpts } = pgConfig;
10
+ const { host, port, user, password, database, connectionLimit, migrationsDir, migrationsTable, autoMigrate, writeBatchSize, writeQueueLimit, ...poolOpts } = pgConfig;
11
11
  pool = new pg.Pool({
12
12
  host,
13
13
  port,
@@ -2,7 +2,7 @@ import { getPool, closePool } from './connection.js';
2
2
  import { ensureMigrationsTable, getAppliedMigrations, getMigrationFiles, applyMigration, parseMigrationFile } from './migration-runner.js';
3
3
  import { introspectModels, introspectViews, getTopologicalOrder, schemasToSnapshot } from './schema-introspector.js';
4
4
  import { loadLatestSnapshot, detectSchemaDrift } from './migration-generator.js';
5
- import { buildInsert, buildUpdate, buildDelete, buildSelect, buildVectorSearch, buildHybridSearch } from './query-builder.js';
5
+ import { buildInsert, buildInsertMany, buildUpdate, buildDelete, buildSelect, buildVectorSearch, buildHybridSearch } from './query-builder.js';
6
6
  import { store } from '@stonyx/orm';
7
7
  import { createRecord } from '../manage-record.js';
8
8
  import { confirm } from '@stonyx/utils/prompt';
@@ -26,6 +26,13 @@ interface PersistResponse {
26
26
  id: unknown;
27
27
  };
28
28
  }
29
+ type ExitEmitter = Pick<NodeJS.Process, 'once' | 'removeListener'>;
30
+ export declare class WriteQueueFullError extends Error {
31
+ constructor(limit: number);
32
+ }
33
+ export declare class WriteQueueClosedError extends Error {
34
+ constructor();
35
+ }
29
36
  interface SearchResult {
30
37
  record: OrmRecord;
31
38
  distance: number;
@@ -49,6 +56,7 @@ interface PostgresDeps {
49
56
  loadLatestSnapshot: typeof loadLatestSnapshot;
50
57
  detectSchemaDrift: typeof detectSchemaDrift;
51
58
  buildInsert: typeof buildInsert;
59
+ buildInsertMany: typeof buildInsertMany;
52
60
  buildUpdate: typeof buildUpdate;
53
61
  buildDelete: typeof buildDelete;
54
62
  buildSelect: typeof buildSelect;
@@ -64,6 +72,7 @@ interface PostgresDeps {
64
72
  config: typeof config;
65
73
  log: typeof log;
66
74
  path: typeof path;
75
+ process: ExitEmitter;
67
76
  [key: string]: unknown;
68
77
  }
69
78
  export default class PostgresDB {
@@ -76,19 +85,22 @@ export default class PostgresDB {
76
85
  deps: PostgresDeps;
77
86
  pool: Pool | null;
78
87
  pgConfig: Record<string, unknown>;
79
- /**
80
- * Promise-chain mutex for write serialization (#156).
81
- * All persist() calls chain through this single queue so concurrent
82
- * fire-and-forget writes never produce parallel transactions
83
- * on FK-linked rows (which cause deadlocks).
84
- * Reads are NOT affected — only persist() serializes.
85
- */
86
- private _writeQueue;
88
+ private _queuedWrites;
89
+ private _inFlightWrites;
90
+ private _drainingWrites;
91
+ private _acceptingWrites;
92
+ private _exitReport;
87
93
  constructor(deps?: Partial<PostgresDeps>);
88
94
  protected requirePool(): Pool;
89
95
  init(): Promise<void>;
96
+ get writeBatchSize(): number;
97
+ get writeQueueLimit(): number;
98
+ get pendingWriteCount(): number;
90
99
  startup(): Promise<void>;
91
100
  shutdown(): Promise<void>;
101
+ private _registerExitReport;
102
+ private _unregisterExitReport;
103
+ private _reportUnpersistedWrites;
92
104
  save(): Promise<void>;
93
105
  /**
94
106
  * Loads only models with memory: true into the in-memory store on startup.
@@ -124,7 +136,16 @@ export default class PostgresDB {
124
136
  private _evictIfNotMemory;
125
137
  private _rowToRawData;
126
138
  persist(operation: string, modelName: string, context: PersistContext, response: PersistResponse): Promise<void>;
127
- private _persistCreate;
139
+ private _drainWritesInBackground;
140
+ private _drainQueuedWrites;
141
+ private _takeNextWriteGroup;
142
+ private _executeWriteGroup;
143
+ private _persistCreates;
144
+ private _prepareCreate;
145
+ private _chunkWithinBindParameterLimit;
146
+ private _insertCreates;
147
+ private _insertCreatesOneByOne;
148
+ private _adoptGeneratedId;
128
149
  private _persistUpdate;
129
150
  private _persistDelete;
130
151
  private _recordToRow;
@@ -2,7 +2,7 @@ import { getPool, closePool } from './connection.js';
2
2
  import { ensureMigrationsTable, getAppliedMigrations, getMigrationFiles, applyMigration, parseMigrationFile } from './migration-runner.js';
3
3
  import { introspectModels, introspectViews, getTopologicalOrder, schemasToSnapshot } from './schema-introspector.js';
4
4
  import { loadLatestSnapshot, detectSchemaDrift } from './migration-generator.js';
5
- import { buildInsert, buildUpdate, buildDelete, buildSelect, buildVectorSearch, buildHybridSearch } from './query-builder.js';
5
+ import { buildInsert, buildInsertMany, buildUpdate, buildDelete, buildSelect, buildVectorSearch, buildHybridSearch } from './query-builder.js';
6
6
  import { store } from '@stonyx/orm';
7
7
  import { createRecord } from '../manage-record.js';
8
8
  import { confirm } from '@stonyx/utils/prompt';
@@ -14,13 +14,28 @@ import { isDbError } from '../utils.js';
14
14
  import config from 'stonyx/config';
15
15
  import log from 'stonyx/log';
16
16
  import path from 'path';
17
+ const POSTGRES_MAX_BIND_PARAMETERS = 65535;
18
+ const DEFAULT_WRITE_BATCH_SIZE = 500;
19
+ const DEFAULT_WRITE_QUEUE_LIMIT = 10000;
20
+ export class WriteQueueFullError extends Error {
21
+ constructor(limit) {
22
+ super(`PostgresDB write queue is full (${limit} queued writes, writeQueueLimit); the write was rejected`);
23
+ this.name = 'WriteQueueFullError';
24
+ }
25
+ }
26
+ export class WriteQueueClosedError extends Error {
27
+ constructor() {
28
+ super('PostgresDB is shutting down; the write was rejected');
29
+ this.name = 'WriteQueueClosedError';
30
+ }
31
+ }
17
32
  const defaultDeps = {
18
33
  getPool, closePool, ensureMigrationsTable, getAppliedMigrations,
19
34
  getMigrationFiles, applyMigration, parseMigrationFile,
20
35
  introspectModels, introspectViews, getTopologicalOrder, schemasToSnapshot,
21
36
  loadLatestSnapshot, detectSchemaDrift,
22
- buildInsert, buildUpdate, buildDelete, buildSelect, buildVectorSearch, buildHybridSearch,
23
- createRecord, store, confirm, readFile, getPluralName, config, log, path,
37
+ buildInsert, buildInsertMany, buildUpdate, buildDelete, buildSelect, buildVectorSearch, buildHybridSearch,
38
+ createRecord, store, confirm, readFile, getPluralName, config, log, path, process,
24
39
  convertRow, postgresJsonRule
25
40
  };
26
41
  export default class PostgresDB {
@@ -33,14 +48,11 @@ export default class PostgresDB {
33
48
  deps;
34
49
  pool;
35
50
  pgConfig;
36
- /**
37
- * Promise-chain mutex for write serialization (#156).
38
- * All persist() calls chain through this single queue so concurrent
39
- * fire-and-forget writes never produce parallel transactions
40
- * on FK-linked rows (which cause deadlocks).
41
- * Reads are NOT affected — only persist() serializes.
42
- */
43
- _writeQueue = Promise.resolve();
51
+ _queuedWrites = [];
52
+ _inFlightWrites = new Set();
53
+ _drainingWrites = null;
54
+ _acceptingWrites = true;
55
+ _exitReport = null;
44
56
  constructor(deps = {}) {
45
57
  const Ctor = this.constructor;
46
58
  if (Ctor.instance)
@@ -59,6 +71,17 @@ export default class PostgresDB {
59
71
  this.pool = await this.deps.getPool(this.pgConfig, this.constructor.extensions);
60
72
  await this.deps.ensureMigrationsTable(this.pool, this.pgConfig.migrationsTable);
61
73
  await this.loadMemoryRecords();
74
+ this._acceptingWrites = true;
75
+ this._registerExitReport();
76
+ }
77
+ get writeBatchSize() {
78
+ return Number(this.pgConfig?.writeBatchSize) || DEFAULT_WRITE_BATCH_SIZE;
79
+ }
80
+ get writeQueueLimit() {
81
+ return Number(this.pgConfig?.writeQueueLimit) || DEFAULT_WRITE_QUEUE_LIMIT;
82
+ }
83
+ get pendingWriteCount() {
84
+ return this._inFlightWrites.size + this._queuedWrites.length;
62
85
  }
63
86
  async startup() {
64
87
  const migrationsPath = this.deps.path.resolve(this.deps.config.rootPath, this.pgConfig.migrationsDir);
@@ -135,9 +158,35 @@ export default class PostgresDB {
135
158
  }
136
159
  }
137
160
  async shutdown() {
161
+ this._acceptingWrites = false;
162
+ while (this._drainingWrites)
163
+ await this._drainingWrites;
164
+ this._unregisterExitReport();
138
165
  await this.deps.closePool();
139
166
  this.pool = null;
140
167
  }
168
+ _registerExitReport() {
169
+ if (this._exitReport)
170
+ return;
171
+ this._exitReport = () => this._reportUnpersistedWrites();
172
+ this.deps.process.once('exit', this._exitReport);
173
+ }
174
+ _unregisterExitReport() {
175
+ if (!this._exitReport)
176
+ return;
177
+ this.deps.process.removeListener('exit', this._exitReport);
178
+ this._exitReport = null;
179
+ }
180
+ _reportUnpersistedWrites() {
181
+ const lostWrites = [...this._inFlightWrites, ...this._queuedWrites];
182
+ if (lostWrites.length === 0)
183
+ return;
184
+ const lostByModel = new Map();
185
+ for (const { modelName } of lostWrites)
186
+ lostByModel.set(modelName, (lostByModel.get(modelName) ?? 0) + 1);
187
+ const breakdown = [...lostByModel].map(([modelName, count]) => `${modelName}: ${count}`).join(', ');
188
+ this.deps.log.error?.(`[PostgresDB] Process exiting with ${lostWrites.length} queued write(s) not persisted; they are lost (${breakdown})`);
189
+ }
141
190
  async save() {
142
191
  // No-op: PostgreSQL persists data immediately via persist()
143
192
  }
@@ -383,62 +432,165 @@ export default class PostgresDB {
383
432
  return rawData;
384
433
  }
385
434
  async persist(operation, modelName, context, response) {
386
- // Views are read-only -- no-op for all write operations
387
435
  const Orm = (await import('@stonyx/orm')).default;
388
436
  if (Orm.instance?.isView?.(modelName))
389
437
  return;
390
- const work = async () => {
391
- switch (operation) {
392
- case 'create':
393
- return this._persistCreate(modelName, context, response);
394
- case 'update':
395
- return this._persistUpdate(modelName, context, response);
396
- case 'delete':
397
- return this._persistDelete(modelName, context);
398
- }
399
- };
400
- // Chain through the write queue — .then(work, work) ensures the queue
401
- // advances even when a previous persist rejects (#156).
402
- this._writeQueue = this._writeQueue.then(work, work);
403
- return this._writeQueue;
438
+ if (!this._acceptingWrites)
439
+ throw new WriteQueueClosedError();
440
+ if (this._queuedWrites.length >= this.writeQueueLimit)
441
+ throw new WriteQueueFullError(this.writeQueueLimit);
442
+ return new Promise((resolve, reject) => {
443
+ const write = {
444
+ operation, modelName, context, response,
445
+ resolve: () => {
446
+ this._inFlightWrites.delete(write);
447
+ resolve();
448
+ },
449
+ reject: (error) => {
450
+ this._inFlightWrites.delete(write);
451
+ reject(error);
452
+ },
453
+ };
454
+ this._queuedWrites.push(write);
455
+ this._drainWritesInBackground();
456
+ });
404
457
  }
405
- async _persistCreate(modelName, context, response) {
406
- const schemas = this.deps.introspectModels();
407
- const schema = schemas[modelName];
408
- if (!schema)
458
+ _drainWritesInBackground() {
459
+ if (this._drainingWrites)
409
460
  return;
461
+ this._drainingWrites = this._drainQueuedWrites().finally(() => {
462
+ this._drainingWrites = null;
463
+ });
464
+ }
465
+ async _drainQueuedWrites() {
466
+ while (this._queuedWrites.length > 0) {
467
+ const writes = this._takeNextWriteGroup();
468
+ writes.forEach(write => this._inFlightWrites.add(write));
469
+ await this._executeWriteGroup(writes);
470
+ }
471
+ }
472
+ _takeNextWriteGroup() {
473
+ const [first] = this._queuedWrites;
474
+ let groupSize = 1;
475
+ if (first.operation === 'create') {
476
+ while (groupSize < this.writeBatchSize
477
+ && groupSize < this._queuedWrites.length
478
+ && this._queuedWrites[groupSize].operation === 'create'
479
+ && this._queuedWrites[groupSize].modelName === first.modelName)
480
+ groupSize++;
481
+ }
482
+ return this._queuedWrites.splice(0, groupSize);
483
+ }
484
+ async _executeWriteGroup(writes) {
485
+ const [first] = writes;
486
+ if (first.operation === 'create')
487
+ return this._persistCreates(first.modelName, writes);
488
+ try {
489
+ if (first.operation === 'update')
490
+ await this._persistUpdate(first.modelName, first.context, first.response);
491
+ if (first.operation === 'delete')
492
+ await this._persistDelete(first.modelName, first.context);
493
+ first.resolve();
494
+ }
495
+ catch (error) {
496
+ first.reject(error);
497
+ }
498
+ }
499
+ async _persistCreates(modelName, writes) {
500
+ const schema = this.deps.introspectModels()[modelName];
501
+ const prepared = [];
502
+ for (const write of writes) {
503
+ try {
504
+ const create = schema ? this._prepareCreate(modelName, schema, write) : null;
505
+ if (create)
506
+ prepared.push(create);
507
+ else
508
+ write.resolve();
509
+ }
510
+ catch (error) {
511
+ write.reject(error);
512
+ }
513
+ }
514
+ for (const chunk of this._chunkWithinBindParameterLimit(prepared)) {
515
+ await this._insertCreates(schema, chunk);
516
+ }
517
+ }
518
+ _prepareCreate(modelName, schema, write) {
519
+ const { context, response } = write;
410
520
  const recordId = response?.data?.id;
411
521
  const storeRef = this.deps.store;
412
522
  const record = recordId != null ? storeRef.get(modelName, isNaN(recordId) ? recordId : parseInt(recordId)) : null;
413
523
  if (!record)
414
- return;
524
+ return null;
415
525
  const insertData = this._recordToRow(record, schema, context.rawData);
416
- // For auto-increment models, remove the pending ID.
417
- // Check context.rawData (not record.__data) because __pendingSqlId is not a model
418
- // attribute and gets lost during serialization.
419
526
  const isPendingId = context.rawData?.__pendingSqlId === true;
420
- if (isPendingId) {
527
+ if (isPendingId)
421
528
  delete insertData.id;
422
- }
423
- // Convert the *complete* payload (attributes + id + FKs) in one place (#292).
424
529
  const row = this.deps.convertRow(insertData, schema.columns, this.deps.postgresJsonRule);
425
- const { sql, values } = this.deps.buildInsert(schema.table, row);
426
- const result = await this.requirePool().query(sql, values);
427
- // Re-key the record in the store if PostgreSQL generated the ID (via RETURNING)
428
- if (isPendingId && result.rows.length > 0) {
429
- const pendingId = record.id;
430
- const realId = result.rows[0].id;
431
- const modelStore = this.deps.store.get(modelName);
432
- modelStore.delete(pendingId);
433
- record.__data.id = realId;
434
- record.id = realId;
435
- modelStore.set(realId, record);
436
- // Update the response data with the real ID
437
- if (response?.data) {
438
- response.data.id = realId;
530
+ return { write, modelName, record, row, isPendingId };
531
+ }
532
+ _chunkWithinBindParameterLimit(prepared) {
533
+ const chunks = [];
534
+ let chunk = [];
535
+ let chunkColumns = new Set();
536
+ for (const create of prepared) {
537
+ const columnsWithCreate = new Set([...chunkColumns, ...Object.keys(create.row)]);
538
+ if (chunk.length > 0 && (chunk.length + 1) * columnsWithCreate.size > POSTGRES_MAX_BIND_PARAMETERS) {
539
+ chunks.push(chunk);
540
+ chunk = [];
541
+ chunkColumns = new Set(Object.keys(create.row));
542
+ }
543
+ else {
544
+ chunkColumns = columnsWithCreate;
439
545
  }
440
- delete record.__data.__pendingSqlId;
546
+ chunk.push(create);
441
547
  }
548
+ if (chunk.length > 0)
549
+ chunks.push(chunk);
550
+ return chunks;
551
+ }
552
+ async _insertCreates(schema, creates) {
553
+ if (creates.length === 1)
554
+ return this._insertCreatesOneByOne(schema, creates);
555
+ const { sql, values } = this.deps.buildInsertMany(schema.table, creates.map(({ row }) => row));
556
+ let result;
557
+ try {
558
+ result = await this.requirePool().query(sql, values);
559
+ }
560
+ catch {
561
+ return this._insertCreatesOneByOne(schema, creates);
562
+ }
563
+ creates.forEach((create, index) => {
564
+ this._adoptGeneratedId(create, result.rows[index]);
565
+ create.write.resolve();
566
+ });
567
+ }
568
+ async _insertCreatesOneByOne(schema, creates) {
569
+ for (const create of creates) {
570
+ try {
571
+ const { sql, values } = this.deps.buildInsert(schema.table, create.row);
572
+ const result = await this.requirePool().query(sql, values);
573
+ this._adoptGeneratedId(create, result.rows[0]);
574
+ create.write.resolve();
575
+ }
576
+ catch (error) {
577
+ create.write.reject(error);
578
+ }
579
+ }
580
+ }
581
+ _adoptGeneratedId({ modelName, record, isPendingId, write }, returnedRow) {
582
+ if (!isPendingId || !returnedRow)
583
+ return;
584
+ const pendingId = record.id;
585
+ const realId = returnedRow.id;
586
+ const modelStore = this.deps.store.get(modelName);
587
+ modelStore.delete(pendingId);
588
+ record.__data.id = realId;
589
+ record.id = realId;
590
+ modelStore.set(realId, record);
591
+ if (write.response?.data)
592
+ write.response.data.id = realId;
593
+ delete record.__data.__pendingSqlId;
442
594
  }
443
595
  async _persistUpdate(modelName, context, _response) {
444
596
  const schemas = this.deps.introspectModels();
@@ -12,6 +12,7 @@ interface HybridSearchOptions {
12
12
  }
13
13
  export declare function validateIdentifier(name: string, context?: string): string;
14
14
  export declare function buildInsert(table: string, data: Record<string, unknown>): QueryResult;
15
+ export declare function buildInsertMany(table: string, rows: Record<string, unknown>[]): QueryResult;
15
16
  export declare function buildUpdate(table: string, id: unknown, data: Record<string, unknown>): QueryResult;
16
17
  export declare function buildDelete(table: string, id: unknown): QueryResult;
17
18
  export declare function buildSelect(table: string, conditions?: Record<string, unknown>): QueryResult;
@@ -14,6 +14,23 @@ export function buildInsert(table, data) {
14
14
  const sql = `INSERT INTO "${table}" (${keys.map(k => `"${k}"`).join(', ')}) VALUES (${placeholders.join(', ')}) RETURNING "id"`;
15
15
  return { sql, values };
16
16
  }
17
+ export function buildInsertMany(table, rows) {
18
+ validateIdentifier(table, 'table name');
19
+ const columns = [...new Set(rows.flatMap(row => Object.keys(row)))];
20
+ columns.forEach(column => validateIdentifier(column, 'column name'));
21
+ const values = [];
22
+ const tuples = rows.map(row => {
23
+ const cells = columns.map(column => {
24
+ if (!Object.hasOwn(row, column))
25
+ return 'DEFAULT';
26
+ values.push(row[column]);
27
+ return `$${values.length}`;
28
+ });
29
+ return `(${cells.join(', ')})`;
30
+ });
31
+ const sql = `INSERT INTO "${table}" (${columns.map(column => `"${column}"`).join(', ')}) VALUES ${tuples.join(', ')} RETURNING "id"`;
32
+ return { sql, values };
33
+ }
17
34
  export function buildUpdate(table, id, data) {
18
35
  validateIdentifier(table, 'table name');
19
36
  const keys = Object.keys(data);
@@ -1,4 +1,4 @@
1
- export { validateIdentifier, buildInsert, buildUpdate, buildDelete, buildSelect } from '../postgres/query-builder.js';
1
+ export { validateIdentifier, buildInsert, buildInsertMany, buildUpdate, buildDelete, buildSelect } from '../postgres/query-builder.js';
2
2
  export declare function validateInterval(interval: string, context?: string): string;
3
3
  export declare function validateAggregate(expr: string, context?: string): string;
4
4
  interface QueryResult {
@@ -1,5 +1,4 @@
1
- // Re-export all base PostgreSQL query builders
2
- export { validateIdentifier, buildInsert, buildUpdate, buildDelete, buildSelect } from '../postgres/query-builder.js';
1
+ export { validateIdentifier, buildInsert, buildInsertMany, buildUpdate, buildDelete, buildSelect } from '../postgres/query-builder.js';
3
2
  import { validateIdentifier } from '../postgres/query-builder.js';
4
3
  const SAFE_INTERVAL = /^\d+\s+(microsecond|millisecond|second|minute|hour|day|week|month|year)s?$/i;
5
4
  export function validateInterval(interval, context = 'interval') {
package/package.json CHANGED
@@ -4,7 +4,7 @@
4
4
  "stonyx-async",
5
5
  "stonyx-module"
6
6
  ],
7
- "version": "0.3.2-alpha.122",
7
+ "version": "0.3.2-alpha.123",
8
8
  "description": "",
9
9
  "main": "dist/index.js",
10
10
  "type": "module",
@@ -61,10 +61,10 @@
61
61
  },
62
62
  "homepage": "https://github.com/abofs/stonyx-orm#readme",
63
63
  "dependencies": {
64
- "@stonyx/cron": "0.2.1-beta.172",
65
- "@stonyx/events": "0.1.1-beta.72",
64
+ "@stonyx/cron": "0.2.1-beta.174",
65
+ "@stonyx/events": "0.1.1-beta.73",
66
66
  "@stonyx/utils": "0.2.3-beta.29",
67
- "stonyx": "0.2.3-beta.107"
67
+ "stonyx": "0.2.3-beta.108"
68
68
  },
69
69
  "peerDependencies": {
70
70
  "@aws-sdk/client-dynamodb": "^3.0.0",
@@ -91,7 +91,7 @@
91
91
  }
92
92
  },
93
93
  "devDependencies": {
94
- "@stonyx/rest-server": "0.2.1-beta.175",
94
+ "@stonyx/rest-server": "0.2.1-beta.177",
95
95
  "@types/node": "^25.6.0",
96
96
  "mysql2": "^3.20.0",
97
97
  "pg": "^8.20.0",
@@ -21,7 +21,7 @@ export async function getPool(pgConfig: PgConfig, extensions: string[] = ['vecto
21
21
 
22
22
  const { default: pg } = await import('pg');
23
23
 
24
- const { host, port, user, password, database, connectionLimit, migrationsDir, migrationsTable, autoMigrate, ...poolOpts } = pgConfig;
24
+ const { host, port, user, password, database, connectionLimit, migrationsDir, migrationsTable, autoMigrate, writeBatchSize, writeQueueLimit, ...poolOpts } = pgConfig;
25
25
 
26
26
  pool = new pg.Pool({
27
27
  host,
@@ -2,7 +2,7 @@ import { getPool, closePool } from './connection.js';
2
2
  import { ensureMigrationsTable, getAppliedMigrations, getMigrationFiles, applyMigration, parseMigrationFile } from './migration-runner.js';
3
3
  import { introspectModels, introspectViews, getTopologicalOrder, schemasToSnapshot } from './schema-introspector.js';
4
4
  import { loadLatestSnapshot, detectSchemaDrift } from './migration-generator.js';
5
- import { buildInsert, buildUpdate, buildDelete, buildSelect, buildVectorSearch, buildHybridSearch } from './query-builder.js';
5
+ import { buildInsert, buildInsertMany, buildUpdate, buildDelete, buildSelect, buildVectorSearch, buildHybridSearch } from './query-builder.js';
6
6
  import { store } from '@stonyx/orm';
7
7
  import { createRecord } from '../manage-record.js';
8
8
  import { confirm } from '@stonyx/utils/prompt';
@@ -28,6 +28,43 @@ interface PersistResponse {
28
28
  data?: { id: unknown };
29
29
  }
30
30
 
31
+ interface QueuedWrite {
32
+ operation: string;
33
+ modelName: string;
34
+ context: PersistContext;
35
+ response: PersistResponse;
36
+ resolve: () => void;
37
+ reject: (error: unknown) => void;
38
+ }
39
+
40
+ interface PreparedCreate {
41
+ write: QueuedWrite;
42
+ modelName: string;
43
+ record: OrmRecord;
44
+ row: Record<string, unknown>;
45
+ isPendingId: boolean;
46
+ }
47
+
48
+ type ExitEmitter = Pick<NodeJS.Process, 'once' | 'removeListener'>;
49
+
50
+ const POSTGRES_MAX_BIND_PARAMETERS = 65535;
51
+ const DEFAULT_WRITE_BATCH_SIZE = 500;
52
+ const DEFAULT_WRITE_QUEUE_LIMIT = 10000;
53
+
54
+ export class WriteQueueFullError extends Error {
55
+ constructor(limit: number) {
56
+ super(`PostgresDB write queue is full (${limit} queued writes, writeQueueLimit); the write was rejected`);
57
+ this.name = 'WriteQueueFullError';
58
+ }
59
+ }
60
+
61
+ export class WriteQueueClosedError extends Error {
62
+ constructor() {
63
+ super('PostgresDB is shutting down; the write was rejected');
64
+ this.name = 'WriteQueueClosedError';
65
+ }
66
+ }
67
+
31
68
  interface SearchResult {
32
69
  record: OrmRecord;
33
70
  distance: number;
@@ -53,6 +90,7 @@ interface PostgresDeps {
53
90
  loadLatestSnapshot: typeof loadLatestSnapshot;
54
91
  detectSchemaDrift: typeof detectSchemaDrift;
55
92
  buildInsert: typeof buildInsert;
93
+ buildInsertMany: typeof buildInsertMany;
56
94
  buildUpdate: typeof buildUpdate;
57
95
  buildDelete: typeof buildDelete;
58
96
  buildSelect: typeof buildSelect;
@@ -68,6 +106,7 @@ interface PostgresDeps {
68
106
  config: typeof config;
69
107
  log: typeof log;
70
108
  path: typeof path;
109
+ process: ExitEmitter;
71
110
  [key: string]: unknown;
72
111
  }
73
112
 
@@ -76,8 +115,8 @@ const defaultDeps: PostgresDeps = {
76
115
  getMigrationFiles, applyMigration, parseMigrationFile,
77
116
  introspectModels, introspectViews, getTopologicalOrder, schemasToSnapshot,
78
117
  loadLatestSnapshot, detectSchemaDrift,
79
- buildInsert, buildUpdate, buildDelete, buildSelect, buildVectorSearch, buildHybridSearch,
80
- createRecord, store, confirm, readFile, getPluralName, config, log, path,
118
+ buildInsert, buildInsertMany, buildUpdate, buildDelete, buildSelect, buildVectorSearch, buildHybridSearch,
119
+ createRecord, store, confirm, readFile, getPluralName, config, log, path, process,
81
120
  convertRow, postgresJsonRule
82
121
  };
83
122
 
@@ -95,14 +134,11 @@ export default class PostgresDB {
95
134
  pool!: Pool | null;
96
135
  pgConfig!: Record<string, unknown>;
97
136
 
98
- /**
99
- * Promise-chain mutex for write serialization (#156).
100
- * All persist() calls chain through this single queue so concurrent
101
- * fire-and-forget writes never produce parallel transactions
102
- * on FK-linked rows (which cause deadlocks).
103
- * Reads are NOT affected — only persist() serializes.
104
- */
105
- private _writeQueue: Promise<void> = Promise.resolve();
137
+ private _queuedWrites: QueuedWrite[] = [];
138
+ private _inFlightWrites = new Set<QueuedWrite>();
139
+ private _drainingWrites: Promise<void> | null = null;
140
+ private _acceptingWrites = true;
141
+ private _exitReport: (() => void) | null = null;
106
142
 
107
143
  constructor(deps: Partial<PostgresDeps> = {}) {
108
144
  const Ctor = this.constructor as typeof PostgresDB;
@@ -126,6 +162,20 @@ export default class PostgresDB {
126
162
  );
127
163
  await this.deps.ensureMigrationsTable(this.pool, this.pgConfig.migrationsTable as string | undefined);
128
164
  await this.loadMemoryRecords();
165
+ this._acceptingWrites = true;
166
+ this._registerExitReport();
167
+ }
168
+
169
+ get writeBatchSize(): number {
170
+ return Number(this.pgConfig?.writeBatchSize) || DEFAULT_WRITE_BATCH_SIZE;
171
+ }
172
+
173
+ get writeQueueLimit(): number {
174
+ return Number(this.pgConfig?.writeQueueLimit) || DEFAULT_WRITE_QUEUE_LIMIT;
175
+ }
176
+
177
+ get pendingWriteCount(): number {
178
+ return this._inFlightWrites.size + this._queuedWrites.length;
129
179
  }
130
180
 
131
181
  async startup(): Promise<void> {
@@ -211,10 +261,36 @@ export default class PostgresDB {
211
261
  }
212
262
 
213
263
  async shutdown(): Promise<void> {
264
+ this._acceptingWrites = false;
265
+ while (this._drainingWrites) await this._drainingWrites;
266
+ this._unregisterExitReport();
214
267
  await this.deps.closePool();
215
268
  this.pool = null;
216
269
  }
217
270
 
271
+ private _registerExitReport(): void {
272
+ if (this._exitReport) return;
273
+ this._exitReport = () => this._reportUnpersistedWrites();
274
+ this.deps.process.once('exit', this._exitReport);
275
+ }
276
+
277
+ private _unregisterExitReport(): void {
278
+ if (!this._exitReport) return;
279
+ this.deps.process.removeListener('exit', this._exitReport);
280
+ this._exitReport = null;
281
+ }
282
+
283
+ private _reportUnpersistedWrites(): void {
284
+ const lostWrites = [...this._inFlightWrites, ...this._queuedWrites];
285
+ if (lostWrites.length === 0) return;
286
+
287
+ const lostByModel = new Map<string, number>();
288
+ for (const { modelName } of lostWrites) lostByModel.set(modelName, (lostByModel.get(modelName) ?? 0) + 1);
289
+ const breakdown = [...lostByModel].map(([modelName, count]) => `${modelName}: ${count}`).join(', ');
290
+
291
+ this.deps.log.error?.(`[PostgresDB] Process exiting with ${lostWrites.length} queued write(s) not persisted; they are lost (${breakdown})`);
292
+ }
293
+
218
294
  async save(): Promise<void> {
219
295
  // No-op: PostgreSQL persists data immediately via persist()
220
296
  }
@@ -494,77 +570,181 @@ export default class PostgresDB {
494
570
  }
495
571
 
496
572
  async persist(operation: string, modelName: string, context: PersistContext, response: PersistResponse): Promise<void> {
497
- // Views are read-only -- no-op for all write operations
498
573
  const Orm = (await import('@stonyx/orm')).default;
499
574
  if ((Orm.instance as { isView?: (name: string) => boolean })?.isView?.(modelName)) return;
500
575
 
501
- const work = async () => {
502
- switch (operation) {
503
- case 'create':
504
- return this._persistCreate(modelName, context, response);
505
- case 'update':
506
- return this._persistUpdate(modelName, context, response);
507
- case 'delete':
508
- return this._persistDelete(modelName, context);
509
- }
510
- };
576
+ if (!this._acceptingWrites) throw new WriteQueueClosedError();
577
+ if (this._queuedWrites.length >= this.writeQueueLimit) throw new WriteQueueFullError(this.writeQueueLimit);
578
+
579
+ return new Promise<void>((resolve, reject) => {
580
+ const write: QueuedWrite = {
581
+ operation, modelName, context, response,
582
+ resolve: () => {
583
+ this._inFlightWrites.delete(write);
584
+ resolve();
585
+ },
586
+ reject: (error: unknown) => {
587
+ this._inFlightWrites.delete(write);
588
+ reject(error);
589
+ },
590
+ };
591
+ this._queuedWrites.push(write);
592
+ this._drainWritesInBackground();
593
+ });
594
+ }
511
595
 
512
- // Chain through the write queue — .then(work, work) ensures the queue
513
- // advances even when a previous persist rejects (#156).
514
- this._writeQueue = this._writeQueue.then(work, work);
515
- return this._writeQueue;
596
+ private _drainWritesInBackground(): void {
597
+ if (this._drainingWrites) return;
598
+ this._drainingWrites = this._drainQueuedWrites().finally(() => {
599
+ this._drainingWrites = null;
600
+ });
516
601
  }
517
602
 
518
- private async _persistCreate(modelName: string, context: PersistContext, response: PersistResponse): Promise<void> {
519
- const schemas = this.deps.introspectModels();
520
- const schema = schemas[modelName];
603
+ private async _drainQueuedWrites(): Promise<void> {
604
+ while (this._queuedWrites.length > 0) {
605
+ const writes = this._takeNextWriteGroup();
606
+ writes.forEach(write => this._inFlightWrites.add(write));
607
+ await this._executeWriteGroup(writes);
608
+ }
609
+ }
521
610
 
522
- if (!schema) return;
611
+ private _takeNextWriteGroup(): QueuedWrite[] {
612
+ const [first] = this._queuedWrites;
613
+ let groupSize = 1;
614
+
615
+ if (first.operation === 'create') {
616
+ while (
617
+ groupSize < this.writeBatchSize
618
+ && groupSize < this._queuedWrites.length
619
+ && this._queuedWrites[groupSize].operation === 'create'
620
+ && this._queuedWrites[groupSize].modelName === first.modelName
621
+ ) groupSize++;
622
+ }
623
+
624
+ return this._queuedWrites.splice(0, groupSize);
625
+ }
626
+
627
+ private async _executeWriteGroup(writes: QueuedWrite[]): Promise<void> {
628
+ const [first] = writes;
629
+ if (first.operation === 'create') return this._persistCreates(first.modelName, writes);
523
630
 
631
+ try {
632
+ if (first.operation === 'update') await this._persistUpdate(first.modelName, first.context, first.response);
633
+ if (first.operation === 'delete') await this._persistDelete(first.modelName, first.context);
634
+ first.resolve();
635
+ } catch (error) {
636
+ first.reject(error);
637
+ }
638
+ }
639
+
640
+ private async _persistCreates(modelName: string, writes: QueuedWrite[]): Promise<void> {
641
+ const schema = this.deps.introspectModels()[modelName];
642
+ const prepared: PreparedCreate[] = [];
643
+
644
+ for (const write of writes) {
645
+ try {
646
+ const create = schema ? this._prepareCreate(modelName, schema, write) : null;
647
+ if (create) prepared.push(create);
648
+ else write.resolve();
649
+ } catch (error) {
650
+ write.reject(error);
651
+ }
652
+ }
653
+
654
+ for (const chunk of this._chunkWithinBindParameterLimit(prepared)) {
655
+ await this._insertCreates(schema, chunk);
656
+ }
657
+ }
658
+
659
+ private _prepareCreate(modelName: string, schema: ModelSchema, write: QueuedWrite): PreparedCreate | null {
660
+ const { context, response } = write;
524
661
  const recordId = response?.data?.id;
525
662
  const storeRef = this.deps.store as unknown as {
526
663
  get(name: string, id: unknown): OrmRecord | null;
527
664
  };
528
665
  const record = recordId != null ? storeRef.get(modelName, isNaN(recordId as number) ? recordId : parseInt(recordId as string)) : null;
529
666
 
530
- if (!record) return;
667
+ if (!record) return null;
531
668
 
532
669
  const insertData = this._recordToRow(record, schema, context.rawData);
533
-
534
- // For auto-increment models, remove the pending ID.
535
- // Check context.rawData (not record.__data) because __pendingSqlId is not a model
536
- // attribute and gets lost during serialization.
537
670
  const isPendingId = context.rawData?.__pendingSqlId === true;
538
671
 
539
- if (isPendingId) {
540
- delete insertData.id;
541
- }
672
+ if (isPendingId) delete insertData.id;
542
673
 
543
- // Convert the *complete* payload (attributes + id + FKs) in one place (#292).
544
674
  const row = this.deps.convertRow(insertData, schema.columns, this.deps.postgresJsonRule);
545
675
 
546
- const { sql, values } = this.deps.buildInsert(schema.table, row);
547
-
548
- const result = await this.requirePool().query(sql, values);
676
+ return { write, modelName, record, row, isPendingId };
677
+ }
549
678
 
550
- // Re-key the record in the store if PostgreSQL generated the ID (via RETURNING)
551
- if (isPendingId && result.rows.length > 0) {
552
- const pendingId = record.id;
553
- const realId = result.rows[0].id as string | number;
554
- const modelStore = (this.deps.store as unknown as { get(name: string): Map<unknown, unknown> }).get(modelName);
679
+ private _chunkWithinBindParameterLimit(prepared: PreparedCreate[]): PreparedCreate[][] {
680
+ const chunks: PreparedCreate[][] = [];
681
+ let chunk: PreparedCreate[] = [];
682
+ let chunkColumns = new Set<string>();
555
683
 
556
- modelStore.delete(pendingId);
557
- record.__data.id = realId;
558
- record.id = realId;
559
- modelStore.set(realId, record);
684
+ for (const create of prepared) {
685
+ const columnsWithCreate = new Set([...chunkColumns, ...Object.keys(create.row)]);
560
686
 
561
- // Update the response data with the real ID
562
- if (response?.data) {
563
- response.data.id = realId;
687
+ if (chunk.length > 0 && (chunk.length + 1) * columnsWithCreate.size > POSTGRES_MAX_BIND_PARAMETERS) {
688
+ chunks.push(chunk);
689
+ chunk = [];
690
+ chunkColumns = new Set(Object.keys(create.row));
691
+ } else {
692
+ chunkColumns = columnsWithCreate;
564
693
  }
565
694
 
566
- delete record.__data.__pendingSqlId;
695
+ chunk.push(create);
567
696
  }
697
+
698
+ if (chunk.length > 0) chunks.push(chunk);
699
+ return chunks;
700
+ }
701
+
702
+ private async _insertCreates(schema: ModelSchema, creates: PreparedCreate[]): Promise<void> {
703
+ if (creates.length === 1) return this._insertCreatesOneByOne(schema, creates);
704
+
705
+ const { sql, values } = this.deps.buildInsertMany(schema.table, creates.map(({ row }) => row));
706
+ let result: { rows: Record<string, unknown>[] };
707
+
708
+ try {
709
+ result = await this.requirePool().query(sql, values);
710
+ } catch {
711
+ return this._insertCreatesOneByOne(schema, creates);
712
+ }
713
+
714
+ creates.forEach((create, index) => {
715
+ this._adoptGeneratedId(create, result.rows[index]);
716
+ create.write.resolve();
717
+ });
718
+ }
719
+
720
+ private async _insertCreatesOneByOne(schema: ModelSchema, creates: PreparedCreate[]): Promise<void> {
721
+ for (const create of creates) {
722
+ try {
723
+ const { sql, values } = this.deps.buildInsert(schema.table, create.row);
724
+ const result = await this.requirePool().query(sql, values);
725
+ this._adoptGeneratedId(create, result.rows[0]);
726
+ create.write.resolve();
727
+ } catch (error) {
728
+ create.write.reject(error);
729
+ }
730
+ }
731
+ }
732
+
733
+ private _adoptGeneratedId({ modelName, record, isPendingId, write }: PreparedCreate, returnedRow: Record<string, unknown> | undefined): void {
734
+ if (!isPendingId || !returnedRow) return;
735
+
736
+ const pendingId = record.id;
737
+ const realId = returnedRow.id as string | number;
738
+ const modelStore = (this.deps.store as unknown as { get(name: string): Map<unknown, unknown> }).get(modelName);
739
+
740
+ modelStore.delete(pendingId);
741
+ record.__data.id = realId;
742
+ record.id = realId;
743
+ modelStore.set(realId, record);
744
+
745
+ if (write.response?.data) write.response.data.id = realId;
746
+
747
+ delete record.__data.__pendingSqlId;
568
748
  }
569
749
 
570
750
  private async _persistUpdate(modelName: string, context: PersistContext, _response: PersistResponse): Promise<void> {
@@ -37,6 +37,27 @@ export function buildInsert(table: string, data: Record<string, unknown>): Query
37
37
  return { sql, values };
38
38
  }
39
39
 
40
+ export function buildInsertMany(table: string, rows: Record<string, unknown>[]): QueryResult {
41
+ validateIdentifier(table, 'table name');
42
+
43
+ const columns = [...new Set(rows.flatMap(row => Object.keys(row)))];
44
+ columns.forEach(column => validateIdentifier(column, 'column name'));
45
+
46
+ const values: unknown[] = [];
47
+ const tuples = rows.map(row => {
48
+ const cells = columns.map(column => {
49
+ if (!Object.hasOwn(row, column)) return 'DEFAULT';
50
+ values.push(row[column]);
51
+ return `$${values.length}`;
52
+ });
53
+ return `(${cells.join(', ')})`;
54
+ });
55
+
56
+ const sql = `INSERT INTO "${table}" (${columns.map(column => `"${column}"`).join(', ')}) VALUES ${tuples.join(', ')} RETURNING "id"`;
57
+
58
+ return { sql, values };
59
+ }
60
+
40
61
  export function buildUpdate(table: string, id: unknown, data: Record<string, unknown>): QueryResult {
41
62
  validateIdentifier(table, 'table name');
42
63
 
@@ -1,5 +1,4 @@
1
- // Re-export all base PostgreSQL query builders
2
- export { validateIdentifier, buildInsert, buildUpdate, buildDelete, buildSelect } from '../postgres/query-builder.js';
1
+ export { validateIdentifier, buildInsert, buildInsertMany, buildUpdate, buildDelete, buildSelect } from '../postgres/query-builder.js';
3
2
 
4
3
  import { validateIdentifier } from '../postgres/query-builder.js';
5
4