@stonyx/orm 0.3.2-beta.326 → 0.3.2-beta.327

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -26,6 +26,8 @@ const {
26
26
  PG_DATABASE,
27
27
  PG_CONNECTION_LIMIT,
28
28
  PG_MIGRATIONS_DIR,
29
+ PG_WRITE_BATCH_SIZE,
30
+ PG_WRITE_QUEUE_LIMIT,
29
31
  TIMESCALE_HOST,
30
32
  TIMESCALE_PORT,
31
33
  TIMESCALE_USER,
@@ -33,6 +35,8 @@ const {
33
35
  TIMESCALE_DATABASE,
34
36
  TIMESCALE_CONNECTION_LIMIT,
35
37
  TIMESCALE_MIGRATIONS_DIR,
38
+ TIMESCALE_WRITE_BATCH_SIZE,
39
+ TIMESCALE_WRITE_QUEUE_LIMIT,
36
40
  DYNAMODB_REGION,
37
41
  DYNAMODB_ENDPOINT,
38
42
  DYNAMODB_TABLE_PREFIX,
@@ -76,6 +80,8 @@ export default {
76
80
  connectionLimit: parseInt(PG_CONNECTION_LIMIT ?? '10'),
77
81
  migrationsDir: PG_MIGRATIONS_DIR ?? 'migrations',
78
82
  migrationsTable: '__migrations',
83
+ writeBatchSize: parseInt(PG_WRITE_BATCH_SIZE ?? '500'),
84
+ writeQueueLimit: parseInt(PG_WRITE_QUEUE_LIMIT ?? '10000'),
79
85
  } : undefined,
80
86
  timescale: TIMESCALE_HOST ? {
81
87
  host: TIMESCALE_HOST ?? 'localhost',
@@ -86,6 +92,8 @@ export default {
86
92
  connectionLimit: parseInt(TIMESCALE_CONNECTION_LIMIT ?? '10'),
87
93
  migrationsDir: TIMESCALE_MIGRATIONS_DIR ?? 'migrations',
88
94
  migrationsTable: '__migrations',
95
+ writeBatchSize: parseInt(TIMESCALE_WRITE_BATCH_SIZE ?? '500'),
96
+ writeQueueLimit: parseInt(TIMESCALE_WRITE_QUEUE_LIMIT ?? '10000'),
89
97
  } : undefined,
90
98
  dynamodb: DYNAMODB_REGION ? {
91
99
  region: DYNAMODB_REGION,
@@ -7,7 +7,7 @@ export async function getPool(pgConfig, extensions = ['vector']) {
7
7
  if (pool)
8
8
  return pool;
9
9
  const { default: pg } = await import('pg');
10
- const { host, port, user, password, database, connectionLimit, migrationsDir, migrationsTable, autoMigrate, ...poolOpts } = pgConfig;
10
+ const { host, port, user, password, database, connectionLimit, migrationsDir, migrationsTable, autoMigrate, writeBatchSize, writeQueueLimit, ...poolOpts } = pgConfig;
11
11
  pool = new pg.Pool({
12
12
  host,
13
13
  port,
@@ -2,7 +2,7 @@ import { getPool, closePool } from './connection.js';
2
2
  import { ensureMigrationsTable, getAppliedMigrations, getMigrationFiles, applyMigration, parseMigrationFile } from './migration-runner.js';
3
3
  import { introspectModels, introspectViews, getTopologicalOrder, schemasToSnapshot } from './schema-introspector.js';
4
4
  import { loadLatestSnapshot, detectSchemaDrift } from './migration-generator.js';
5
- import { buildInsert, buildUpdate, buildDelete, buildSelect, buildVectorSearch, buildHybridSearch } from './query-builder.js';
5
+ import { buildInsert, buildInsertMany, buildUpdate, buildDelete, buildSelect, buildVectorSearch, buildHybridSearch } from './query-builder.js';
6
6
  import { store } from '@stonyx/orm';
7
7
  import { createRecord } from '../manage-record.js';
8
8
  import { confirm } from '@stonyx/utils/prompt';
@@ -26,6 +26,13 @@ interface PersistResponse {
26
26
  id: unknown;
27
27
  };
28
28
  }
29
+ type ExitEmitter = Pick<NodeJS.Process, 'once' | 'removeListener'>;
30
+ export declare class WriteQueueFullError extends Error {
31
+ constructor(limit: number);
32
+ }
33
+ export declare class WriteQueueClosedError extends Error {
34
+ constructor();
35
+ }
29
36
  interface SearchResult {
30
37
  record: OrmRecord;
31
38
  distance: number;
@@ -49,6 +56,7 @@ interface PostgresDeps {
49
56
  loadLatestSnapshot: typeof loadLatestSnapshot;
50
57
  detectSchemaDrift: typeof detectSchemaDrift;
51
58
  buildInsert: typeof buildInsert;
59
+ buildInsertMany: typeof buildInsertMany;
52
60
  buildUpdate: typeof buildUpdate;
53
61
  buildDelete: typeof buildDelete;
54
62
  buildSelect: typeof buildSelect;
@@ -64,6 +72,7 @@ interface PostgresDeps {
64
72
  config: typeof config;
65
73
  log: typeof log;
66
74
  path: typeof path;
75
+ process: ExitEmitter;
67
76
  [key: string]: unknown;
68
77
  }
69
78
  export default class PostgresDB {
@@ -76,19 +85,22 @@ export default class PostgresDB {
76
85
  deps: PostgresDeps;
77
86
  pool: Pool | null;
78
87
  pgConfig: Record<string, unknown>;
79
- /**
80
- * Promise-chain mutex for write serialization (#156).
81
- * All persist() calls chain through this single queue so concurrent
82
- * fire-and-forget writes never produce parallel transactions
83
- * on FK-linked rows (which cause deadlocks).
84
- * Reads are NOT affected — only persist() serializes.
85
- */
86
- private _writeQueue;
88
+ private _queuedWrites;
89
+ private _inFlightWrites;
90
+ private _drainingWrites;
91
+ private _acceptingWrites;
92
+ private _exitReport;
87
93
  constructor(deps?: Partial<PostgresDeps>);
88
94
  protected requirePool(): Pool;
89
95
  init(): Promise<void>;
96
+ get writeBatchSize(): number;
97
+ get writeQueueLimit(): number;
98
+ get pendingWriteCount(): number;
90
99
  startup(): Promise<void>;
91
100
  shutdown(): Promise<void>;
101
+ private _registerExitReport;
102
+ private _unregisterExitReport;
103
+ private _reportUnpersistedWrites;
92
104
  save(): Promise<void>;
93
105
  /**
94
106
  * Loads only models with memory: true into the in-memory store on startup.
@@ -124,7 +136,16 @@ export default class PostgresDB {
124
136
  private _evictIfNotMemory;
125
137
  private _rowToRawData;
126
138
  persist(operation: string, modelName: string, context: PersistContext, response: PersistResponse): Promise<void>;
127
- private _persistCreate;
139
+ private _drainWritesInBackground;
140
+ private _drainQueuedWrites;
141
+ private _takeNextWriteGroup;
142
+ private _executeWriteGroup;
143
+ private _persistCreates;
144
+ private _prepareCreate;
145
+ private _chunkWithinBindParameterLimit;
146
+ private _insertCreates;
147
+ private _insertCreatesOneByOne;
148
+ private _adoptGeneratedId;
128
149
  private _persistUpdate;
129
150
  private _persistDelete;
130
151
  private _recordToRow;
@@ -2,7 +2,7 @@ import { getPool, closePool } from './connection.js';
2
2
  import { ensureMigrationsTable, getAppliedMigrations, getMigrationFiles, applyMigration, parseMigrationFile } from './migration-runner.js';
3
3
  import { introspectModels, introspectViews, getTopologicalOrder, schemasToSnapshot } from './schema-introspector.js';
4
4
  import { loadLatestSnapshot, detectSchemaDrift } from './migration-generator.js';
5
- import { buildInsert, buildUpdate, buildDelete, buildSelect, buildVectorSearch, buildHybridSearch } from './query-builder.js';
5
+ import { buildInsert, buildInsertMany, buildUpdate, buildDelete, buildSelect, buildVectorSearch, buildHybridSearch } from './query-builder.js';
6
6
  import { store } from '@stonyx/orm';
7
7
  import { createRecord } from '../manage-record.js';
8
8
  import { confirm } from '@stonyx/utils/prompt';
@@ -14,13 +14,28 @@ import { isDbError } from '../utils.js';
14
14
  import config from 'stonyx/config';
15
15
  import log from 'stonyx/log';
16
16
  import path from 'path';
17
+ const POSTGRES_MAX_BIND_PARAMETERS = 65535;
18
+ const DEFAULT_WRITE_BATCH_SIZE = 500;
19
+ const DEFAULT_WRITE_QUEUE_LIMIT = 10000;
20
+ export class WriteQueueFullError extends Error {
21
+ constructor(limit) {
22
+ super(`PostgresDB write queue is full (${limit} queued writes, writeQueueLimit); the write was rejected`);
23
+ this.name = 'WriteQueueFullError';
24
+ }
25
+ }
26
+ export class WriteQueueClosedError extends Error {
27
+ constructor() {
28
+ super('PostgresDB is shutting down; the write was rejected');
29
+ this.name = 'WriteQueueClosedError';
30
+ }
31
+ }
17
32
  const defaultDeps = {
18
33
  getPool, closePool, ensureMigrationsTable, getAppliedMigrations,
19
34
  getMigrationFiles, applyMigration, parseMigrationFile,
20
35
  introspectModels, introspectViews, getTopologicalOrder, schemasToSnapshot,
21
36
  loadLatestSnapshot, detectSchemaDrift,
22
- buildInsert, buildUpdate, buildDelete, buildSelect, buildVectorSearch, buildHybridSearch,
23
- createRecord, store, confirm, readFile, getPluralName, config, log, path,
37
+ buildInsert, buildInsertMany, buildUpdate, buildDelete, buildSelect, buildVectorSearch, buildHybridSearch,
38
+ createRecord, store, confirm, readFile, getPluralName, config, log, path, process,
24
39
  convertRow, postgresJsonRule
25
40
  };
26
41
  export default class PostgresDB {
@@ -33,14 +48,11 @@ export default class PostgresDB {
33
48
  deps;
34
49
  pool;
35
50
  pgConfig;
36
- /**
37
- * Promise-chain mutex for write serialization (#156).
38
- * All persist() calls chain through this single queue so concurrent
39
- * fire-and-forget writes never produce parallel transactions
40
- * on FK-linked rows (which cause deadlocks).
41
- * Reads are NOT affected — only persist() serializes.
42
- */
43
- _writeQueue = Promise.resolve();
51
+ _queuedWrites = [];
52
+ _inFlightWrites = new Set();
53
+ _drainingWrites = null;
54
+ _acceptingWrites = true;
55
+ _exitReport = null;
44
56
  constructor(deps = {}) {
45
57
  const Ctor = this.constructor;
46
58
  if (Ctor.instance)
@@ -59,6 +71,17 @@ export default class PostgresDB {
59
71
  this.pool = await this.deps.getPool(this.pgConfig, this.constructor.extensions);
60
72
  await this.deps.ensureMigrationsTable(this.pool, this.pgConfig.migrationsTable);
61
73
  await this.loadMemoryRecords();
74
+ this._acceptingWrites = true;
75
+ this._registerExitReport();
76
+ }
77
+ get writeBatchSize() {
78
+ return Number(this.pgConfig?.writeBatchSize) || DEFAULT_WRITE_BATCH_SIZE;
79
+ }
80
+ get writeQueueLimit() {
81
+ return Number(this.pgConfig?.writeQueueLimit) || DEFAULT_WRITE_QUEUE_LIMIT;
82
+ }
83
+ get pendingWriteCount() {
84
+ return this._inFlightWrites.size + this._queuedWrites.length;
62
85
  }
63
86
  async startup() {
64
87
  const migrationsPath = this.deps.path.resolve(this.deps.config.rootPath, this.pgConfig.migrationsDir);
@@ -135,9 +158,35 @@ export default class PostgresDB {
135
158
  }
136
159
  }
137
160
  async shutdown() {
161
+ this._acceptingWrites = false;
162
+ while (this._drainingWrites)
163
+ await this._drainingWrites;
164
+ this._unregisterExitReport();
138
165
  await this.deps.closePool();
139
166
  this.pool = null;
140
167
  }
168
+ _registerExitReport() {
169
+ if (this._exitReport)
170
+ return;
171
+ this._exitReport = () => this._reportUnpersistedWrites();
172
+ this.deps.process.once('exit', this._exitReport);
173
+ }
174
+ _unregisterExitReport() {
175
+ if (!this._exitReport)
176
+ return;
177
+ this.deps.process.removeListener('exit', this._exitReport);
178
+ this._exitReport = null;
179
+ }
180
+ _reportUnpersistedWrites() {
181
+ const lostWrites = [...this._inFlightWrites, ...this._queuedWrites];
182
+ if (lostWrites.length === 0)
183
+ return;
184
+ const lostByModel = new Map();
185
+ for (const { modelName } of lostWrites)
186
+ lostByModel.set(modelName, (lostByModel.get(modelName) ?? 0) + 1);
187
+ const breakdown = [...lostByModel].map(([modelName, count]) => `${modelName}: ${count}`).join(', ');
188
+ this.deps.log.error?.(`[PostgresDB] Process exiting with ${lostWrites.length} queued write(s) not persisted; they are lost (${breakdown})`);
189
+ }
141
190
  async save() {
142
191
  // No-op: PostgreSQL persists data immediately via persist()
143
192
  }
@@ -383,62 +432,170 @@ export default class PostgresDB {
383
432
  return rawData;
384
433
  }
385
434
  async persist(operation, modelName, context, response) {
386
- // Views are read-only -- no-op for all write operations
387
435
  const Orm = (await import('@stonyx/orm')).default;
388
436
  if (Orm.instance?.isView?.(modelName))
389
437
  return;
390
- const work = async () => {
391
- switch (operation) {
392
- case 'create':
393
- return this._persistCreate(modelName, context, response);
394
- case 'update':
395
- return this._persistUpdate(modelName, context, response);
396
- case 'delete':
397
- return this._persistDelete(modelName, context);
398
- }
399
- };
400
- // Chain through the write queue — .then(work, work) ensures the queue
401
- // advances even when a previous persist rejects (#156).
402
- this._writeQueue = this._writeQueue.then(work, work);
403
- return this._writeQueue;
404
- }
405
- async _persistCreate(modelName, context, response) {
406
- const schemas = this.deps.introspectModels();
407
- const schema = schemas[modelName];
408
- if (!schema)
438
+ if (!this._acceptingWrites)
439
+ throw new WriteQueueClosedError();
440
+ if (this._queuedWrites.length >= this.writeQueueLimit)
441
+ throw new WriteQueueFullError(this.writeQueueLimit);
442
+ return new Promise((resolve, reject) => {
443
+ const write = {
444
+ operation, modelName, context, response,
445
+ resolve: () => {
446
+ this._inFlightWrites.delete(write);
447
+ resolve();
448
+ },
449
+ reject: (error) => {
450
+ this._inFlightWrites.delete(write);
451
+ reject(error);
452
+ },
453
+ };
454
+ this._queuedWrites.push(write);
455
+ this._drainWritesInBackground();
456
+ });
457
+ }
458
+ _drainWritesInBackground() {
459
+ if (this._drainingWrites)
409
460
  return;
461
+ this._drainingWrites = this._drainQueuedWrites().finally(() => {
462
+ this._drainingWrites = null;
463
+ });
464
+ }
465
+ async _drainQueuedWrites() {
466
+ while (this._queuedWrites.length > 0) {
467
+ const writes = this._takeNextWriteGroup();
468
+ writes.forEach(write => this._inFlightWrites.add(write));
469
+ try {
470
+ await this._executeWriteGroup(writes);
471
+ }
472
+ catch (error) {
473
+ writes.filter(write => this._inFlightWrites.has(write)).forEach(write => write.reject(error));
474
+ }
475
+ }
476
+ }
477
+ _takeNextWriteGroup() {
478
+ const [first] = this._queuedWrites;
479
+ let groupSize = 1;
480
+ if (first.operation === 'create') {
481
+ while (groupSize < this.writeBatchSize
482
+ && groupSize < this._queuedWrites.length
483
+ && this._queuedWrites[groupSize].operation === 'create'
484
+ && this._queuedWrites[groupSize].modelName === first.modelName)
485
+ groupSize++;
486
+ }
487
+ return this._queuedWrites.splice(0, groupSize);
488
+ }
489
+ async _executeWriteGroup(writes) {
490
+ const [first] = writes;
491
+ if (first.operation === 'create')
492
+ return this._persistCreates(first.modelName, writes);
493
+ try {
494
+ if (first.operation === 'update')
495
+ await this._persistUpdate(first.modelName, first.context, first.response);
496
+ if (first.operation === 'delete')
497
+ await this._persistDelete(first.modelName, first.context);
498
+ first.resolve();
499
+ }
500
+ catch (error) {
501
+ first.reject(error);
502
+ }
503
+ }
504
+ async _persistCreates(modelName, writes) {
505
+ const schema = this.deps.introspectModels()[modelName];
506
+ const prepared = [];
507
+ for (const write of writes) {
508
+ try {
509
+ const create = schema ? this._prepareCreate(modelName, schema, write) : null;
510
+ if (create)
511
+ prepared.push(create);
512
+ else
513
+ write.resolve();
514
+ }
515
+ catch (error) {
516
+ write.reject(error);
517
+ }
518
+ }
519
+ for (const chunk of this._chunkWithinBindParameterLimit(prepared)) {
520
+ await this._insertCreates(schema, chunk);
521
+ }
522
+ }
523
+ _prepareCreate(modelName, schema, write) {
524
+ const { context, response } = write;
410
525
  const recordId = response?.data?.id;
411
526
  const storeRef = this.deps.store;
412
527
  const record = recordId != null ? storeRef.get(modelName, isNaN(recordId) ? recordId : parseInt(recordId)) : null;
413
528
  if (!record)
414
- return;
529
+ return null;
415
530
  const insertData = this._recordToRow(record, schema, context.rawData);
416
- // For auto-increment models, remove the pending ID.
417
- // Check context.rawData (not record.__data) because __pendingSqlId is not a model
418
- // attribute and gets lost during serialization.
419
531
  const isPendingId = context.rawData?.__pendingSqlId === true;
420
- if (isPendingId) {
532
+ if (isPendingId)
421
533
  delete insertData.id;
422
- }
423
- // Convert the *complete* payload (attributes + id + FKs) in one place (#292).
424
534
  const row = this.deps.convertRow(insertData, schema.columns, this.deps.postgresJsonRule);
425
- const { sql, values } = this.deps.buildInsert(schema.table, row);
426
- const result = await this.requirePool().query(sql, values);
427
- // Re-key the record in the store if PostgreSQL generated the ID (via RETURNING)
428
- if (isPendingId && result.rows.length > 0) {
429
- const pendingId = record.id;
430
- const realId = result.rows[0].id;
431
- const modelStore = this.deps.store.get(modelName);
432
- modelStore.delete(pendingId);
433
- record.__data.id = realId;
434
- record.id = realId;
435
- modelStore.set(realId, record);
436
- // Update the response data with the real ID
437
- if (response?.data) {
438
- response.data.id = realId;
439
- }
440
- delete record.__data.__pendingSqlId;
535
+ return { write, modelName, record, row, isPendingId };
536
+ }
537
+ _chunkWithinBindParameterLimit(prepared) {
538
+ const chunks = [];
539
+ let chunk = [];
540
+ let chunkColumns = new Set();
541
+ for (const create of prepared) {
542
+ const columnsWithCreate = new Set([...chunkColumns, ...Object.keys(create.row)]);
543
+ if (chunk.length > 0 && (chunk.length + 1) * columnsWithCreate.size > POSTGRES_MAX_BIND_PARAMETERS) {
544
+ chunks.push(chunk);
545
+ chunk = [];
546
+ chunkColumns = new Set(Object.keys(create.row));
547
+ }
548
+ else {
549
+ chunkColumns = columnsWithCreate;
550
+ }
551
+ chunk.push(create);
552
+ }
553
+ if (chunk.length > 0)
554
+ chunks.push(chunk);
555
+ return chunks;
556
+ }
557
+ async _insertCreates(schema, creates) {
558
+ if (creates.length === 1)
559
+ return this._insertCreatesOneByOne(schema, creates);
560
+ let result;
561
+ try {
562
+ const { sql, values } = this.deps.buildInsertMany(schema.table, creates.map(({ row }) => row));
563
+ result = await this.requirePool().query(sql, values);
564
+ }
565
+ catch {
566
+ return this._insertCreatesOneByOne(schema, creates);
441
567
  }
568
+ creates.forEach((create, index) => {
569
+ this._adoptGeneratedId(create, result.rows[index]);
570
+ create.write.resolve();
571
+ });
572
+ }
573
+ async _insertCreatesOneByOne(schema, creates) {
574
+ for (const create of creates) {
575
+ try {
576
+ const { sql, values } = this.deps.buildInsert(schema.table, create.row);
577
+ const result = await this.requirePool().query(sql, values);
578
+ this._adoptGeneratedId(create, result.rows[0]);
579
+ create.write.resolve();
580
+ }
581
+ catch (error) {
582
+ create.write.reject(error);
583
+ }
584
+ }
585
+ }
586
+ _adoptGeneratedId({ modelName, record, isPendingId, write }, returnedRow) {
587
+ if (!isPendingId || !returnedRow)
588
+ return;
589
+ const pendingId = record.id;
590
+ const realId = returnedRow.id;
591
+ const modelStore = this.deps.store.get(modelName);
592
+ modelStore.delete(pendingId);
593
+ record.__data.id = realId;
594
+ record.id = realId;
595
+ modelStore.set(realId, record);
596
+ if (write.response?.data)
597
+ write.response.data.id = realId;
598
+ delete record.__data.__pendingSqlId;
442
599
  }
443
600
  async _persistUpdate(modelName, context, _response) {
444
601
  const schemas = this.deps.introspectModels();
@@ -12,6 +12,7 @@ interface HybridSearchOptions {
12
12
  }
13
13
  export declare function validateIdentifier(name: string, context?: string): string;
14
14
  export declare function buildInsert(table: string, data: Record<string, unknown>): QueryResult;
15
+ export declare function buildInsertMany(table: string, rows: Record<string, unknown>[]): QueryResult;
15
16
  export declare function buildUpdate(table: string, id: unknown, data: Record<string, unknown>): QueryResult;
16
17
  export declare function buildDelete(table: string, id: unknown): QueryResult;
17
18
  export declare function buildSelect(table: string, conditions?: Record<string, unknown>): QueryResult;
@@ -14,6 +14,23 @@ export function buildInsert(table, data) {
14
14
  const sql = `INSERT INTO "${table}" (${keys.map(k => `"${k}"`).join(', ')}) VALUES (${placeholders.join(', ')}) RETURNING "id"`;
15
15
  return { sql, values };
16
16
  }
17
+ export function buildInsertMany(table, rows) {
18
+ validateIdentifier(table, 'table name');
19
+ const columns = [...new Set(rows.flatMap(row => Object.keys(row)))];
20
+ columns.forEach(column => validateIdentifier(column, 'column name'));
21
+ const values = [];
22
+ const tuples = rows.map(row => {
23
+ const cells = columns.map(column => {
24
+ if (!Object.hasOwn(row, column))
25
+ return 'DEFAULT';
26
+ values.push(row[column]);
27
+ return `$${values.length}`;
28
+ });
29
+ return `(${cells.join(', ')})`;
30
+ });
31
+ const sql = `INSERT INTO "${table}" (${columns.map(column => `"${column}"`).join(', ')}) VALUES ${tuples.join(', ')} RETURNING "id"`;
32
+ return { sql, values };
33
+ }
17
34
  export function buildUpdate(table, id, data) {
18
35
  validateIdentifier(table, 'table name');
19
36
  const keys = Object.keys(data);
@@ -1,4 +1,4 @@
1
- export { validateIdentifier, buildInsert, buildUpdate, buildDelete, buildSelect } from '../postgres/query-builder.js';
1
+ export { validateIdentifier, buildInsert, buildInsertMany, buildUpdate, buildDelete, buildSelect } from '../postgres/query-builder.js';
2
2
  export declare function validateInterval(interval: string, context?: string): string;
3
3
  export declare function validateAggregate(expr: string, context?: string): string;
4
4
  interface QueryResult {
@@ -1,5 +1,4 @@
1
- // Re-export all base PostgreSQL query builders
2
- export { validateIdentifier, buildInsert, buildUpdate, buildDelete, buildSelect } from '../postgres/query-builder.js';
1
+ export { validateIdentifier, buildInsert, buildInsertMany, buildUpdate, buildDelete, buildSelect } from '../postgres/query-builder.js';
3
2
  import { validateIdentifier } from '../postgres/query-builder.js';
4
3
  const SAFE_INTERVAL = /^\d+\s+(microsecond|millisecond|second|minute|hour|day|week|month|year)s?$/i;
5
4
  export function validateInterval(interval, context = 'interval') {
package/package.json CHANGED
@@ -4,7 +4,7 @@
4
4
  "stonyx-async",
5
5
  "stonyx-module"
6
6
  ],
7
- "version": "0.3.2-beta.326",
7
+ "version": "0.3.2-beta.327",
8
8
  "description": "",
9
9
  "main": "dist/index.js",
10
10
  "type": "module",
@@ -21,7 +21,7 @@ export async function getPool(pgConfig: PgConfig, extensions: string[] = ['vecto
21
21
 
22
22
  const { default: pg } = await import('pg');
23
23
 
24
- const { host, port, user, password, database, connectionLimit, migrationsDir, migrationsTable, autoMigrate, ...poolOpts } = pgConfig;
24
+ const { host, port, user, password, database, connectionLimit, migrationsDir, migrationsTable, autoMigrate, writeBatchSize, writeQueueLimit, ...poolOpts } = pgConfig;
25
25
 
26
26
  pool = new pg.Pool({
27
27
  host,
@@ -2,7 +2,7 @@ import { getPool, closePool } from './connection.js';
2
2
  import { ensureMigrationsTable, getAppliedMigrations, getMigrationFiles, applyMigration, parseMigrationFile } from './migration-runner.js';
3
3
  import { introspectModels, introspectViews, getTopologicalOrder, schemasToSnapshot } from './schema-introspector.js';
4
4
  import { loadLatestSnapshot, detectSchemaDrift } from './migration-generator.js';
5
- import { buildInsert, buildUpdate, buildDelete, buildSelect, buildVectorSearch, buildHybridSearch } from './query-builder.js';
5
+ import { buildInsert, buildInsertMany, buildUpdate, buildDelete, buildSelect, buildVectorSearch, buildHybridSearch } from './query-builder.js';
6
6
  import { store } from '@stonyx/orm';
7
7
  import { createRecord } from '../manage-record.js';
8
8
  import { confirm } from '@stonyx/utils/prompt';
@@ -28,6 +28,43 @@ interface PersistResponse {
28
28
  data?: { id: unknown };
29
29
  }
30
30
 
31
+ interface QueuedWrite {
32
+ operation: string;
33
+ modelName: string;
34
+ context: PersistContext;
35
+ response: PersistResponse;
36
+ resolve: () => void;
37
+ reject: (error: unknown) => void;
38
+ }
39
+
40
+ interface PreparedCreate {
41
+ write: QueuedWrite;
42
+ modelName: string;
43
+ record: OrmRecord;
44
+ row: Record<string, unknown>;
45
+ isPendingId: boolean;
46
+ }
47
+
48
+ type ExitEmitter = Pick<NodeJS.Process, 'once' | 'removeListener'>;
49
+
50
+ const POSTGRES_MAX_BIND_PARAMETERS = 65535;
51
+ const DEFAULT_WRITE_BATCH_SIZE = 500;
52
+ const DEFAULT_WRITE_QUEUE_LIMIT = 10000;
53
+
54
+ export class WriteQueueFullError extends Error {
55
+ constructor(limit: number) {
56
+ super(`PostgresDB write queue is full (${limit} queued writes, writeQueueLimit); the write was rejected`);
57
+ this.name = 'WriteQueueFullError';
58
+ }
59
+ }
60
+
61
+ export class WriteQueueClosedError extends Error {
62
+ constructor() {
63
+ super('PostgresDB is shutting down; the write was rejected');
64
+ this.name = 'WriteQueueClosedError';
65
+ }
66
+ }
67
+
31
68
  interface SearchResult {
32
69
  record: OrmRecord;
33
70
  distance: number;
@@ -53,6 +90,7 @@ interface PostgresDeps {
53
90
  loadLatestSnapshot: typeof loadLatestSnapshot;
54
91
  detectSchemaDrift: typeof detectSchemaDrift;
55
92
  buildInsert: typeof buildInsert;
93
+ buildInsertMany: typeof buildInsertMany;
56
94
  buildUpdate: typeof buildUpdate;
57
95
  buildDelete: typeof buildDelete;
58
96
  buildSelect: typeof buildSelect;
@@ -68,6 +106,7 @@ interface PostgresDeps {
68
106
  config: typeof config;
69
107
  log: typeof log;
70
108
  path: typeof path;
109
+ process: ExitEmitter;
71
110
  [key: string]: unknown;
72
111
  }
73
112
 
@@ -76,8 +115,8 @@ const defaultDeps: PostgresDeps = {
76
115
  getMigrationFiles, applyMigration, parseMigrationFile,
77
116
  introspectModels, introspectViews, getTopologicalOrder, schemasToSnapshot,
78
117
  loadLatestSnapshot, detectSchemaDrift,
79
- buildInsert, buildUpdate, buildDelete, buildSelect, buildVectorSearch, buildHybridSearch,
80
- createRecord, store, confirm, readFile, getPluralName, config, log, path,
118
+ buildInsert, buildInsertMany, buildUpdate, buildDelete, buildSelect, buildVectorSearch, buildHybridSearch,
119
+ createRecord, store, confirm, readFile, getPluralName, config, log, path, process,
81
120
  convertRow, postgresJsonRule
82
121
  };
83
122
 
@@ -95,14 +134,11 @@ export default class PostgresDB {
95
134
  pool!: Pool | null;
96
135
  pgConfig!: Record<string, unknown>;
97
136
 
98
- /**
99
- * Promise-chain mutex for write serialization (#156).
100
- * All persist() calls chain through this single queue so concurrent
101
- * fire-and-forget writes never produce parallel transactions
102
- * on FK-linked rows (which cause deadlocks).
103
- * Reads are NOT affected — only persist() serializes.
104
- */
105
- private _writeQueue: Promise<void> = Promise.resolve();
137
+ private _queuedWrites: QueuedWrite[] = [];
138
+ private _inFlightWrites = new Set<QueuedWrite>();
139
+ private _drainingWrites: Promise<void> | null = null;
140
+ private _acceptingWrites = true;
141
+ private _exitReport: (() => void) | null = null;
106
142
 
107
143
  constructor(deps: Partial<PostgresDeps> = {}) {
108
144
  const Ctor = this.constructor as typeof PostgresDB;
@@ -126,6 +162,20 @@ export default class PostgresDB {
126
162
  );
127
163
  await this.deps.ensureMigrationsTable(this.pool, this.pgConfig.migrationsTable as string | undefined);
128
164
  await this.loadMemoryRecords();
165
+ this._acceptingWrites = true;
166
+ this._registerExitReport();
167
+ }
168
+
169
+ get writeBatchSize(): number {
170
+ return Number(this.pgConfig?.writeBatchSize) || DEFAULT_WRITE_BATCH_SIZE;
171
+ }
172
+
173
+ get writeQueueLimit(): number {
174
+ return Number(this.pgConfig?.writeQueueLimit) || DEFAULT_WRITE_QUEUE_LIMIT;
175
+ }
176
+
177
+ get pendingWriteCount(): number {
178
+ return this._inFlightWrites.size + this._queuedWrites.length;
129
179
  }
130
180
 
131
181
  async startup(): Promise<void> {
@@ -211,10 +261,36 @@ export default class PostgresDB {
211
261
  }
212
262
 
213
263
  async shutdown(): Promise<void> {
264
+ this._acceptingWrites = false;
265
+ while (this._drainingWrites) await this._drainingWrites;
266
+ this._unregisterExitReport();
214
267
  await this.deps.closePool();
215
268
  this.pool = null;
216
269
  }
217
270
 
271
+ private _registerExitReport(): void {
272
+ if (this._exitReport) return;
273
+ this._exitReport = () => this._reportUnpersistedWrites();
274
+ this.deps.process.once('exit', this._exitReport);
275
+ }
276
+
277
+ private _unregisterExitReport(): void {
278
+ if (!this._exitReport) return;
279
+ this.deps.process.removeListener('exit', this._exitReport);
280
+ this._exitReport = null;
281
+ }
282
+
283
+ private _reportUnpersistedWrites(): void {
284
+ const lostWrites = [...this._inFlightWrites, ...this._queuedWrites];
285
+ if (lostWrites.length === 0) return;
286
+
287
+ const lostByModel = new Map<string, number>();
288
+ for (const { modelName } of lostWrites) lostByModel.set(modelName, (lostByModel.get(modelName) ?? 0) + 1);
289
+ const breakdown = [...lostByModel].map(([modelName, count]) => `${modelName}: ${count}`).join(', ');
290
+
291
+ this.deps.log.error?.(`[PostgresDB] Process exiting with ${lostWrites.length} queued write(s) not persisted; they are lost (${breakdown})`);
292
+ }
293
+
218
294
  async save(): Promise<void> {
219
295
  // No-op: PostgreSQL persists data immediately via persist()
220
296
  }
@@ -494,77 +570,185 @@ export default class PostgresDB {
494
570
  }
495
571
 
496
572
  async persist(operation: string, modelName: string, context: PersistContext, response: PersistResponse): Promise<void> {
497
- // Views are read-only -- no-op for all write operations
498
573
  const Orm = (await import('@stonyx/orm')).default;
499
574
  if ((Orm.instance as { isView?: (name: string) => boolean })?.isView?.(modelName)) return;
500
575
 
501
- const work = async () => {
502
- switch (operation) {
503
- case 'create':
504
- return this._persistCreate(modelName, context, response);
505
- case 'update':
506
- return this._persistUpdate(modelName, context, response);
507
- case 'delete':
508
- return this._persistDelete(modelName, context);
576
+ if (!this._acceptingWrites) throw new WriteQueueClosedError();
577
+ if (this._queuedWrites.length >= this.writeQueueLimit) throw new WriteQueueFullError(this.writeQueueLimit);
578
+
579
+ return new Promise<void>((resolve, reject) => {
580
+ const write: QueuedWrite = {
581
+ operation, modelName, context, response,
582
+ resolve: () => {
583
+ this._inFlightWrites.delete(write);
584
+ resolve();
585
+ },
586
+ reject: (error: unknown) => {
587
+ this._inFlightWrites.delete(write);
588
+ reject(error);
589
+ },
590
+ };
591
+ this._queuedWrites.push(write);
592
+ this._drainWritesInBackground();
593
+ });
594
+ }
595
+
596
+ private _drainWritesInBackground(): void {
597
+ if (this._drainingWrites) return;
598
+ this._drainingWrites = this._drainQueuedWrites().finally(() => {
599
+ this._drainingWrites = null;
600
+ });
601
+ }
602
+
603
+ private async _drainQueuedWrites(): Promise<void> {
604
+ while (this._queuedWrites.length > 0) {
605
+ const writes = this._takeNextWriteGroup();
606
+ writes.forEach(write => this._inFlightWrites.add(write));
607
+ try {
608
+ await this._executeWriteGroup(writes);
609
+ } catch (error) {
610
+ writes.filter(write => this._inFlightWrites.has(write)).forEach(write => write.reject(error));
509
611
  }
510
- };
612
+ }
613
+ }
511
614
 
512
- // Chain through the write queue — .then(work, work) ensures the queue
513
- // advances even when a previous persist rejects (#156).
514
- this._writeQueue = this._writeQueue.then(work, work);
515
- return this._writeQueue;
615
+ private _takeNextWriteGroup(): QueuedWrite[] {
616
+ const [first] = this._queuedWrites;
617
+ let groupSize = 1;
618
+
619
+ if (first.operation === 'create') {
620
+ while (
621
+ groupSize < this.writeBatchSize
622
+ && groupSize < this._queuedWrites.length
623
+ && this._queuedWrites[groupSize].operation === 'create'
624
+ && this._queuedWrites[groupSize].modelName === first.modelName
625
+ ) groupSize++;
626
+ }
627
+
628
+ return this._queuedWrites.splice(0, groupSize);
516
629
  }
517
630
 
518
- private async _persistCreate(modelName: string, context: PersistContext, response: PersistResponse): Promise<void> {
519
- const schemas = this.deps.introspectModels();
520
- const schema = schemas[modelName];
631
+ private async _executeWriteGroup(writes: QueuedWrite[]): Promise<void> {
632
+ const [first] = writes;
633
+ if (first.operation === 'create') return this._persistCreates(first.modelName, writes);
521
634
 
522
- if (!schema) return;
635
+ try {
636
+ if (first.operation === 'update') await this._persistUpdate(first.modelName, first.context, first.response);
637
+ if (first.operation === 'delete') await this._persistDelete(first.modelName, first.context);
638
+ first.resolve();
639
+ } catch (error) {
640
+ first.reject(error);
641
+ }
642
+ }
523
643
 
644
+ private async _persistCreates(modelName: string, writes: QueuedWrite[]): Promise<void> {
645
+ const schema = this.deps.introspectModels()[modelName];
646
+ const prepared: PreparedCreate[] = [];
647
+
648
+ for (const write of writes) {
649
+ try {
650
+ const create = schema ? this._prepareCreate(modelName, schema, write) : null;
651
+ if (create) prepared.push(create);
652
+ else write.resolve();
653
+ } catch (error) {
654
+ write.reject(error);
655
+ }
656
+ }
657
+
658
+ for (const chunk of this._chunkWithinBindParameterLimit(prepared)) {
659
+ await this._insertCreates(schema, chunk);
660
+ }
661
+ }
662
+
663
+ private _prepareCreate(modelName: string, schema: ModelSchema, write: QueuedWrite): PreparedCreate | null {
664
+ const { context, response } = write;
524
665
  const recordId = response?.data?.id;
525
666
  const storeRef = this.deps.store as unknown as {
526
667
  get(name: string, id: unknown): OrmRecord | null;
527
668
  };
528
669
  const record = recordId != null ? storeRef.get(modelName, isNaN(recordId as number) ? recordId : parseInt(recordId as string)) : null;
529
670
 
530
- if (!record) return;
671
+ if (!record) return null;
531
672
 
532
673
  const insertData = this._recordToRow(record, schema, context.rawData);
533
-
534
- // For auto-increment models, remove the pending ID.
535
- // Check context.rawData (not record.__data) because __pendingSqlId is not a model
536
- // attribute and gets lost during serialization.
537
674
  const isPendingId = context.rawData?.__pendingSqlId === true;
538
675
 
539
- if (isPendingId) {
540
- delete insertData.id;
541
- }
676
+ if (isPendingId) delete insertData.id;
542
677
 
543
- // Convert the *complete* payload (attributes + id + FKs) in one place (#292).
544
678
  const row = this.deps.convertRow(insertData, schema.columns, this.deps.postgresJsonRule);
545
679
 
546
- const { sql, values } = this.deps.buildInsert(schema.table, row);
547
-
548
- const result = await this.requirePool().query(sql, values);
680
+ return { write, modelName, record, row, isPendingId };
681
+ }
549
682
 
550
- // Re-key the record in the store if PostgreSQL generated the ID (via RETURNING)
551
- if (isPendingId && result.rows.length > 0) {
552
- const pendingId = record.id;
553
- const realId = result.rows[0].id as string | number;
554
- const modelStore = (this.deps.store as unknown as { get(name: string): Map<unknown, unknown> }).get(modelName);
683
+ private _chunkWithinBindParameterLimit(prepared: PreparedCreate[]): PreparedCreate[][] {
684
+ const chunks: PreparedCreate[][] = [];
685
+ let chunk: PreparedCreate[] = [];
686
+ let chunkColumns = new Set<string>();
555
687
 
556
- modelStore.delete(pendingId);
557
- record.__data.id = realId;
558
- record.id = realId;
559
- modelStore.set(realId, record);
688
+ for (const create of prepared) {
689
+ const columnsWithCreate = new Set([...chunkColumns, ...Object.keys(create.row)]);
560
690
 
561
- // Update the response data with the real ID
562
- if (response?.data) {
563
- response.data.id = realId;
691
+ if (chunk.length > 0 && (chunk.length + 1) * columnsWithCreate.size > POSTGRES_MAX_BIND_PARAMETERS) {
692
+ chunks.push(chunk);
693
+ chunk = [];
694
+ chunkColumns = new Set(Object.keys(create.row));
695
+ } else {
696
+ chunkColumns = columnsWithCreate;
564
697
  }
565
698
 
566
- delete record.__data.__pendingSqlId;
699
+ chunk.push(create);
567
700
  }
701
+
702
+ if (chunk.length > 0) chunks.push(chunk);
703
+ return chunks;
704
+ }
705
+
706
+ private async _insertCreates(schema: ModelSchema, creates: PreparedCreate[]): Promise<void> {
707
+ if (creates.length === 1) return this._insertCreatesOneByOne(schema, creates);
708
+
709
+ let result: { rows: Record<string, unknown>[] };
710
+
711
+ try {
712
+ const { sql, values } = this.deps.buildInsertMany(schema.table, creates.map(({ row }) => row));
713
+ result = await this.requirePool().query(sql, values);
714
+ } catch {
715
+ return this._insertCreatesOneByOne(schema, creates);
716
+ }
717
+
718
+ creates.forEach((create, index) => {
719
+ this._adoptGeneratedId(create, result.rows[index]);
720
+ create.write.resolve();
721
+ });
722
+ }
723
+
724
+ private async _insertCreatesOneByOne(schema: ModelSchema, creates: PreparedCreate[]): Promise<void> {
725
+ for (const create of creates) {
726
+ try {
727
+ const { sql, values } = this.deps.buildInsert(schema.table, create.row);
728
+ const result = await this.requirePool().query(sql, values);
729
+ this._adoptGeneratedId(create, result.rows[0]);
730
+ create.write.resolve();
731
+ } catch (error) {
732
+ create.write.reject(error);
733
+ }
734
+ }
735
+ }
736
+
737
+ private _adoptGeneratedId({ modelName, record, isPendingId, write }: PreparedCreate, returnedRow: Record<string, unknown> | undefined): void {
738
+ if (!isPendingId || !returnedRow) return;
739
+
740
+ const pendingId = record.id;
741
+ const realId = returnedRow.id as string | number;
742
+ const modelStore = (this.deps.store as unknown as { get(name: string): Map<unknown, unknown> }).get(modelName);
743
+
744
+ modelStore.delete(pendingId);
745
+ record.__data.id = realId;
746
+ record.id = realId;
747
+ modelStore.set(realId, record);
748
+
749
+ if (write.response?.data) write.response.data.id = realId;
750
+
751
+ delete record.__data.__pendingSqlId;
568
752
  }
569
753
 
570
754
  private async _persistUpdate(modelName: string, context: PersistContext, _response: PersistResponse): Promise<void> {
@@ -37,6 +37,27 @@ export function buildInsert(table: string, data: Record<string, unknown>): Query
37
37
  return { sql, values };
38
38
  }
39
39
 
40
+ export function buildInsertMany(table: string, rows: Record<string, unknown>[]): QueryResult {
41
+ validateIdentifier(table, 'table name');
42
+
43
+ const columns = [...new Set(rows.flatMap(row => Object.keys(row)))];
44
+ columns.forEach(column => validateIdentifier(column, 'column name'));
45
+
46
+ const values: unknown[] = [];
47
+ const tuples = rows.map(row => {
48
+ const cells = columns.map(column => {
49
+ if (!Object.hasOwn(row, column)) return 'DEFAULT';
50
+ values.push(row[column]);
51
+ return `$${values.length}`;
52
+ });
53
+ return `(${cells.join(', ')})`;
54
+ });
55
+
56
+ const sql = `INSERT INTO "${table}" (${columns.map(column => `"${column}"`).join(', ')}) VALUES ${tuples.join(', ')} RETURNING "id"`;
57
+
58
+ return { sql, values };
59
+ }
60
+
40
61
  export function buildUpdate(table: string, id: unknown, data: Record<string, unknown>): QueryResult {
41
62
  validateIdentifier(table, 'table name');
42
63
 
@@ -1,5 +1,4 @@
1
- // Re-export all base PostgreSQL query builders
2
- export { validateIdentifier, buildInsert, buildUpdate, buildDelete, buildSelect } from '../postgres/query-builder.js';
1
+ export { validateIdentifier, buildInsert, buildInsertMany, buildUpdate, buildDelete, buildSelect } from '../postgres/query-builder.js';
3
2
 
4
3
  import { validateIdentifier } from '../postgres/query-builder.js';
5
4