@stonyx/orm 0.3.2-beta.326 → 0.3.2-beta.327
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/config/environment.js +8 -0
- package/dist/postgres/connection.js +1 -1
- package/dist/postgres/postgres-db.d.ts +31 -10
- package/dist/postgres/postgres-db.js +211 -54
- package/dist/postgres/query-builder.d.ts +1 -0
- package/dist/postgres/query-builder.js +17 -0
- package/dist/timescale/query-builder.d.ts +1 -1
- package/dist/timescale/query-builder.js +1 -2
- package/package.json +1 -1
- package/src/postgres/connection.ts +1 -1
- package/src/postgres/postgres-db.ts +238 -54
- package/src/postgres/query-builder.ts +21 -0
- package/src/timescale/query-builder.ts +1 -2
package/config/environment.js
CHANGED
|
@@ -26,6 +26,8 @@ const {
|
|
|
26
26
|
PG_DATABASE,
|
|
27
27
|
PG_CONNECTION_LIMIT,
|
|
28
28
|
PG_MIGRATIONS_DIR,
|
|
29
|
+
PG_WRITE_BATCH_SIZE,
|
|
30
|
+
PG_WRITE_QUEUE_LIMIT,
|
|
29
31
|
TIMESCALE_HOST,
|
|
30
32
|
TIMESCALE_PORT,
|
|
31
33
|
TIMESCALE_USER,
|
|
@@ -33,6 +35,8 @@ const {
|
|
|
33
35
|
TIMESCALE_DATABASE,
|
|
34
36
|
TIMESCALE_CONNECTION_LIMIT,
|
|
35
37
|
TIMESCALE_MIGRATIONS_DIR,
|
|
38
|
+
TIMESCALE_WRITE_BATCH_SIZE,
|
|
39
|
+
TIMESCALE_WRITE_QUEUE_LIMIT,
|
|
36
40
|
DYNAMODB_REGION,
|
|
37
41
|
DYNAMODB_ENDPOINT,
|
|
38
42
|
DYNAMODB_TABLE_PREFIX,
|
|
@@ -76,6 +80,8 @@ export default {
|
|
|
76
80
|
connectionLimit: parseInt(PG_CONNECTION_LIMIT ?? '10'),
|
|
77
81
|
migrationsDir: PG_MIGRATIONS_DIR ?? 'migrations',
|
|
78
82
|
migrationsTable: '__migrations',
|
|
83
|
+
writeBatchSize: parseInt(PG_WRITE_BATCH_SIZE ?? '500'),
|
|
84
|
+
writeQueueLimit: parseInt(PG_WRITE_QUEUE_LIMIT ?? '10000'),
|
|
79
85
|
} : undefined,
|
|
80
86
|
timescale: TIMESCALE_HOST ? {
|
|
81
87
|
host: TIMESCALE_HOST ?? 'localhost',
|
|
@@ -86,6 +92,8 @@ export default {
|
|
|
86
92
|
connectionLimit: parseInt(TIMESCALE_CONNECTION_LIMIT ?? '10'),
|
|
87
93
|
migrationsDir: TIMESCALE_MIGRATIONS_DIR ?? 'migrations',
|
|
88
94
|
migrationsTable: '__migrations',
|
|
95
|
+
writeBatchSize: parseInt(TIMESCALE_WRITE_BATCH_SIZE ?? '500'),
|
|
96
|
+
writeQueueLimit: parseInt(TIMESCALE_WRITE_QUEUE_LIMIT ?? '10000'),
|
|
89
97
|
} : undefined,
|
|
90
98
|
dynamodb: DYNAMODB_REGION ? {
|
|
91
99
|
region: DYNAMODB_REGION,
|
|
@@ -7,7 +7,7 @@ export async function getPool(pgConfig, extensions = ['vector']) {
|
|
|
7
7
|
if (pool)
|
|
8
8
|
return pool;
|
|
9
9
|
const { default: pg } = await import('pg');
|
|
10
|
-
const { host, port, user, password, database, connectionLimit, migrationsDir, migrationsTable, autoMigrate, ...poolOpts } = pgConfig;
|
|
10
|
+
const { host, port, user, password, database, connectionLimit, migrationsDir, migrationsTable, autoMigrate, writeBatchSize, writeQueueLimit, ...poolOpts } = pgConfig;
|
|
11
11
|
pool = new pg.Pool({
|
|
12
12
|
host,
|
|
13
13
|
port,
|
|
@@ -2,7 +2,7 @@ import { getPool, closePool } from './connection.js';
|
|
|
2
2
|
import { ensureMigrationsTable, getAppliedMigrations, getMigrationFiles, applyMigration, parseMigrationFile } from './migration-runner.js';
|
|
3
3
|
import { introspectModels, introspectViews, getTopologicalOrder, schemasToSnapshot } from './schema-introspector.js';
|
|
4
4
|
import { loadLatestSnapshot, detectSchemaDrift } from './migration-generator.js';
|
|
5
|
-
import { buildInsert, buildUpdate, buildDelete, buildSelect, buildVectorSearch, buildHybridSearch } from './query-builder.js';
|
|
5
|
+
import { buildInsert, buildInsertMany, buildUpdate, buildDelete, buildSelect, buildVectorSearch, buildHybridSearch } from './query-builder.js';
|
|
6
6
|
import { store } from '@stonyx/orm';
|
|
7
7
|
import { createRecord } from '../manage-record.js';
|
|
8
8
|
import { confirm } from '@stonyx/utils/prompt';
|
|
@@ -26,6 +26,13 @@ interface PersistResponse {
|
|
|
26
26
|
id: unknown;
|
|
27
27
|
};
|
|
28
28
|
}
|
|
29
|
+
type ExitEmitter = Pick<NodeJS.Process, 'once' | 'removeListener'>;
|
|
30
|
+
export declare class WriteQueueFullError extends Error {
|
|
31
|
+
constructor(limit: number);
|
|
32
|
+
}
|
|
33
|
+
export declare class WriteQueueClosedError extends Error {
|
|
34
|
+
constructor();
|
|
35
|
+
}
|
|
29
36
|
interface SearchResult {
|
|
30
37
|
record: OrmRecord;
|
|
31
38
|
distance: number;
|
|
@@ -49,6 +56,7 @@ interface PostgresDeps {
|
|
|
49
56
|
loadLatestSnapshot: typeof loadLatestSnapshot;
|
|
50
57
|
detectSchemaDrift: typeof detectSchemaDrift;
|
|
51
58
|
buildInsert: typeof buildInsert;
|
|
59
|
+
buildInsertMany: typeof buildInsertMany;
|
|
52
60
|
buildUpdate: typeof buildUpdate;
|
|
53
61
|
buildDelete: typeof buildDelete;
|
|
54
62
|
buildSelect: typeof buildSelect;
|
|
@@ -64,6 +72,7 @@ interface PostgresDeps {
|
|
|
64
72
|
config: typeof config;
|
|
65
73
|
log: typeof log;
|
|
66
74
|
path: typeof path;
|
|
75
|
+
process: ExitEmitter;
|
|
67
76
|
[key: string]: unknown;
|
|
68
77
|
}
|
|
69
78
|
export default class PostgresDB {
|
|
@@ -76,19 +85,22 @@ export default class PostgresDB {
|
|
|
76
85
|
deps: PostgresDeps;
|
|
77
86
|
pool: Pool | null;
|
|
78
87
|
pgConfig: Record<string, unknown>;
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
* Reads are NOT affected — only persist() serializes.
|
|
85
|
-
*/
|
|
86
|
-
private _writeQueue;
|
|
88
|
+
private _queuedWrites;
|
|
89
|
+
private _inFlightWrites;
|
|
90
|
+
private _drainingWrites;
|
|
91
|
+
private _acceptingWrites;
|
|
92
|
+
private _exitReport;
|
|
87
93
|
constructor(deps?: Partial<PostgresDeps>);
|
|
88
94
|
protected requirePool(): Pool;
|
|
89
95
|
init(): Promise<void>;
|
|
96
|
+
get writeBatchSize(): number;
|
|
97
|
+
get writeQueueLimit(): number;
|
|
98
|
+
get pendingWriteCount(): number;
|
|
90
99
|
startup(): Promise<void>;
|
|
91
100
|
shutdown(): Promise<void>;
|
|
101
|
+
private _registerExitReport;
|
|
102
|
+
private _unregisterExitReport;
|
|
103
|
+
private _reportUnpersistedWrites;
|
|
92
104
|
save(): Promise<void>;
|
|
93
105
|
/**
|
|
94
106
|
* Loads only models with memory: true into the in-memory store on startup.
|
|
@@ -124,7 +136,16 @@ export default class PostgresDB {
|
|
|
124
136
|
private _evictIfNotMemory;
|
|
125
137
|
private _rowToRawData;
|
|
126
138
|
persist(operation: string, modelName: string, context: PersistContext, response: PersistResponse): Promise<void>;
|
|
127
|
-
private
|
|
139
|
+
private _drainWritesInBackground;
|
|
140
|
+
private _drainQueuedWrites;
|
|
141
|
+
private _takeNextWriteGroup;
|
|
142
|
+
private _executeWriteGroup;
|
|
143
|
+
private _persistCreates;
|
|
144
|
+
private _prepareCreate;
|
|
145
|
+
private _chunkWithinBindParameterLimit;
|
|
146
|
+
private _insertCreates;
|
|
147
|
+
private _insertCreatesOneByOne;
|
|
148
|
+
private _adoptGeneratedId;
|
|
128
149
|
private _persistUpdate;
|
|
129
150
|
private _persistDelete;
|
|
130
151
|
private _recordToRow;
|
|
@@ -2,7 +2,7 @@ import { getPool, closePool } from './connection.js';
|
|
|
2
2
|
import { ensureMigrationsTable, getAppliedMigrations, getMigrationFiles, applyMigration, parseMigrationFile } from './migration-runner.js';
|
|
3
3
|
import { introspectModels, introspectViews, getTopologicalOrder, schemasToSnapshot } from './schema-introspector.js';
|
|
4
4
|
import { loadLatestSnapshot, detectSchemaDrift } from './migration-generator.js';
|
|
5
|
-
import { buildInsert, buildUpdate, buildDelete, buildSelect, buildVectorSearch, buildHybridSearch } from './query-builder.js';
|
|
5
|
+
import { buildInsert, buildInsertMany, buildUpdate, buildDelete, buildSelect, buildVectorSearch, buildHybridSearch } from './query-builder.js';
|
|
6
6
|
import { store } from '@stonyx/orm';
|
|
7
7
|
import { createRecord } from '../manage-record.js';
|
|
8
8
|
import { confirm } from '@stonyx/utils/prompt';
|
|
@@ -14,13 +14,28 @@ import { isDbError } from '../utils.js';
|
|
|
14
14
|
import config from 'stonyx/config';
|
|
15
15
|
import log from 'stonyx/log';
|
|
16
16
|
import path from 'path';
|
|
17
|
+
const POSTGRES_MAX_BIND_PARAMETERS = 65535;
|
|
18
|
+
const DEFAULT_WRITE_BATCH_SIZE = 500;
|
|
19
|
+
const DEFAULT_WRITE_QUEUE_LIMIT = 10000;
|
|
20
|
+
export class WriteQueueFullError extends Error {
|
|
21
|
+
constructor(limit) {
|
|
22
|
+
super(`PostgresDB write queue is full (${limit} queued writes, writeQueueLimit); the write was rejected`);
|
|
23
|
+
this.name = 'WriteQueueFullError';
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
export class WriteQueueClosedError extends Error {
|
|
27
|
+
constructor() {
|
|
28
|
+
super('PostgresDB is shutting down; the write was rejected');
|
|
29
|
+
this.name = 'WriteQueueClosedError';
|
|
30
|
+
}
|
|
31
|
+
}
|
|
17
32
|
const defaultDeps = {
|
|
18
33
|
getPool, closePool, ensureMigrationsTable, getAppliedMigrations,
|
|
19
34
|
getMigrationFiles, applyMigration, parseMigrationFile,
|
|
20
35
|
introspectModels, introspectViews, getTopologicalOrder, schemasToSnapshot,
|
|
21
36
|
loadLatestSnapshot, detectSchemaDrift,
|
|
22
|
-
buildInsert, buildUpdate, buildDelete, buildSelect, buildVectorSearch, buildHybridSearch,
|
|
23
|
-
createRecord, store, confirm, readFile, getPluralName, config, log, path,
|
|
37
|
+
buildInsert, buildInsertMany, buildUpdate, buildDelete, buildSelect, buildVectorSearch, buildHybridSearch,
|
|
38
|
+
createRecord, store, confirm, readFile, getPluralName, config, log, path, process,
|
|
24
39
|
convertRow, postgresJsonRule
|
|
25
40
|
};
|
|
26
41
|
export default class PostgresDB {
|
|
@@ -33,14 +48,11 @@ export default class PostgresDB {
|
|
|
33
48
|
deps;
|
|
34
49
|
pool;
|
|
35
50
|
pgConfig;
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
* Reads are NOT affected — only persist() serializes.
|
|
42
|
-
*/
|
|
43
|
-
_writeQueue = Promise.resolve();
|
|
51
|
+
_queuedWrites = [];
|
|
52
|
+
_inFlightWrites = new Set();
|
|
53
|
+
_drainingWrites = null;
|
|
54
|
+
_acceptingWrites = true;
|
|
55
|
+
_exitReport = null;
|
|
44
56
|
constructor(deps = {}) {
|
|
45
57
|
const Ctor = this.constructor;
|
|
46
58
|
if (Ctor.instance)
|
|
@@ -59,6 +71,17 @@ export default class PostgresDB {
|
|
|
59
71
|
this.pool = await this.deps.getPool(this.pgConfig, this.constructor.extensions);
|
|
60
72
|
await this.deps.ensureMigrationsTable(this.pool, this.pgConfig.migrationsTable);
|
|
61
73
|
await this.loadMemoryRecords();
|
|
74
|
+
this._acceptingWrites = true;
|
|
75
|
+
this._registerExitReport();
|
|
76
|
+
}
|
|
77
|
+
get writeBatchSize() {
|
|
78
|
+
return Number(this.pgConfig?.writeBatchSize) || DEFAULT_WRITE_BATCH_SIZE;
|
|
79
|
+
}
|
|
80
|
+
get writeQueueLimit() {
|
|
81
|
+
return Number(this.pgConfig?.writeQueueLimit) || DEFAULT_WRITE_QUEUE_LIMIT;
|
|
82
|
+
}
|
|
83
|
+
get pendingWriteCount() {
|
|
84
|
+
return this._inFlightWrites.size + this._queuedWrites.length;
|
|
62
85
|
}
|
|
63
86
|
async startup() {
|
|
64
87
|
const migrationsPath = this.deps.path.resolve(this.deps.config.rootPath, this.pgConfig.migrationsDir);
|
|
@@ -135,9 +158,35 @@ export default class PostgresDB {
|
|
|
135
158
|
}
|
|
136
159
|
}
|
|
137
160
|
async shutdown() {
|
|
161
|
+
this._acceptingWrites = false;
|
|
162
|
+
while (this._drainingWrites)
|
|
163
|
+
await this._drainingWrites;
|
|
164
|
+
this._unregisterExitReport();
|
|
138
165
|
await this.deps.closePool();
|
|
139
166
|
this.pool = null;
|
|
140
167
|
}
|
|
168
|
+
_registerExitReport() {
|
|
169
|
+
if (this._exitReport)
|
|
170
|
+
return;
|
|
171
|
+
this._exitReport = () => this._reportUnpersistedWrites();
|
|
172
|
+
this.deps.process.once('exit', this._exitReport);
|
|
173
|
+
}
|
|
174
|
+
_unregisterExitReport() {
|
|
175
|
+
if (!this._exitReport)
|
|
176
|
+
return;
|
|
177
|
+
this.deps.process.removeListener('exit', this._exitReport);
|
|
178
|
+
this._exitReport = null;
|
|
179
|
+
}
|
|
180
|
+
_reportUnpersistedWrites() {
|
|
181
|
+
const lostWrites = [...this._inFlightWrites, ...this._queuedWrites];
|
|
182
|
+
if (lostWrites.length === 0)
|
|
183
|
+
return;
|
|
184
|
+
const lostByModel = new Map();
|
|
185
|
+
for (const { modelName } of lostWrites)
|
|
186
|
+
lostByModel.set(modelName, (lostByModel.get(modelName) ?? 0) + 1);
|
|
187
|
+
const breakdown = [...lostByModel].map(([modelName, count]) => `${modelName}: ${count}`).join(', ');
|
|
188
|
+
this.deps.log.error?.(`[PostgresDB] Process exiting with ${lostWrites.length} queued write(s) not persisted; they are lost (${breakdown})`);
|
|
189
|
+
}
|
|
141
190
|
async save() {
|
|
142
191
|
// No-op: PostgreSQL persists data immediately via persist()
|
|
143
192
|
}
|
|
@@ -383,62 +432,170 @@ export default class PostgresDB {
|
|
|
383
432
|
return rawData;
|
|
384
433
|
}
|
|
385
434
|
async persist(operation, modelName, context, response) {
|
|
386
|
-
// Views are read-only -- no-op for all write operations
|
|
387
435
|
const Orm = (await import('@stonyx/orm')).default;
|
|
388
436
|
if (Orm.instance?.isView?.(modelName))
|
|
389
437
|
return;
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
438
|
+
if (!this._acceptingWrites)
|
|
439
|
+
throw new WriteQueueClosedError();
|
|
440
|
+
if (this._queuedWrites.length >= this.writeQueueLimit)
|
|
441
|
+
throw new WriteQueueFullError(this.writeQueueLimit);
|
|
442
|
+
return new Promise((resolve, reject) => {
|
|
443
|
+
const write = {
|
|
444
|
+
operation, modelName, context, response,
|
|
445
|
+
resolve: () => {
|
|
446
|
+
this._inFlightWrites.delete(write);
|
|
447
|
+
resolve();
|
|
448
|
+
},
|
|
449
|
+
reject: (error) => {
|
|
450
|
+
this._inFlightWrites.delete(write);
|
|
451
|
+
reject(error);
|
|
452
|
+
},
|
|
453
|
+
};
|
|
454
|
+
this._queuedWrites.push(write);
|
|
455
|
+
this._drainWritesInBackground();
|
|
456
|
+
});
|
|
457
|
+
}
|
|
458
|
+
_drainWritesInBackground() {
|
|
459
|
+
if (this._drainingWrites)
|
|
409
460
|
return;
|
|
461
|
+
this._drainingWrites = this._drainQueuedWrites().finally(() => {
|
|
462
|
+
this._drainingWrites = null;
|
|
463
|
+
});
|
|
464
|
+
}
|
|
465
|
+
async _drainQueuedWrites() {
|
|
466
|
+
while (this._queuedWrites.length > 0) {
|
|
467
|
+
const writes = this._takeNextWriteGroup();
|
|
468
|
+
writes.forEach(write => this._inFlightWrites.add(write));
|
|
469
|
+
try {
|
|
470
|
+
await this._executeWriteGroup(writes);
|
|
471
|
+
}
|
|
472
|
+
catch (error) {
|
|
473
|
+
writes.filter(write => this._inFlightWrites.has(write)).forEach(write => write.reject(error));
|
|
474
|
+
}
|
|
475
|
+
}
|
|
476
|
+
}
|
|
477
|
+
_takeNextWriteGroup() {
|
|
478
|
+
const [first] = this._queuedWrites;
|
|
479
|
+
let groupSize = 1;
|
|
480
|
+
if (first.operation === 'create') {
|
|
481
|
+
while (groupSize < this.writeBatchSize
|
|
482
|
+
&& groupSize < this._queuedWrites.length
|
|
483
|
+
&& this._queuedWrites[groupSize].operation === 'create'
|
|
484
|
+
&& this._queuedWrites[groupSize].modelName === first.modelName)
|
|
485
|
+
groupSize++;
|
|
486
|
+
}
|
|
487
|
+
return this._queuedWrites.splice(0, groupSize);
|
|
488
|
+
}
|
|
489
|
+
async _executeWriteGroup(writes) {
|
|
490
|
+
const [first] = writes;
|
|
491
|
+
if (first.operation === 'create')
|
|
492
|
+
return this._persistCreates(first.modelName, writes);
|
|
493
|
+
try {
|
|
494
|
+
if (first.operation === 'update')
|
|
495
|
+
await this._persistUpdate(first.modelName, first.context, first.response);
|
|
496
|
+
if (first.operation === 'delete')
|
|
497
|
+
await this._persistDelete(first.modelName, first.context);
|
|
498
|
+
first.resolve();
|
|
499
|
+
}
|
|
500
|
+
catch (error) {
|
|
501
|
+
first.reject(error);
|
|
502
|
+
}
|
|
503
|
+
}
|
|
504
|
+
async _persistCreates(modelName, writes) {
|
|
505
|
+
const schema = this.deps.introspectModels()[modelName];
|
|
506
|
+
const prepared = [];
|
|
507
|
+
for (const write of writes) {
|
|
508
|
+
try {
|
|
509
|
+
const create = schema ? this._prepareCreate(modelName, schema, write) : null;
|
|
510
|
+
if (create)
|
|
511
|
+
prepared.push(create);
|
|
512
|
+
else
|
|
513
|
+
write.resolve();
|
|
514
|
+
}
|
|
515
|
+
catch (error) {
|
|
516
|
+
write.reject(error);
|
|
517
|
+
}
|
|
518
|
+
}
|
|
519
|
+
for (const chunk of this._chunkWithinBindParameterLimit(prepared)) {
|
|
520
|
+
await this._insertCreates(schema, chunk);
|
|
521
|
+
}
|
|
522
|
+
}
|
|
523
|
+
_prepareCreate(modelName, schema, write) {
|
|
524
|
+
const { context, response } = write;
|
|
410
525
|
const recordId = response?.data?.id;
|
|
411
526
|
const storeRef = this.deps.store;
|
|
412
527
|
const record = recordId != null ? storeRef.get(modelName, isNaN(recordId) ? recordId : parseInt(recordId)) : null;
|
|
413
528
|
if (!record)
|
|
414
|
-
return;
|
|
529
|
+
return null;
|
|
415
530
|
const insertData = this._recordToRow(record, schema, context.rawData);
|
|
416
|
-
// For auto-increment models, remove the pending ID.
|
|
417
|
-
// Check context.rawData (not record.__data) because __pendingSqlId is not a model
|
|
418
|
-
// attribute and gets lost during serialization.
|
|
419
531
|
const isPendingId = context.rawData?.__pendingSqlId === true;
|
|
420
|
-
if (isPendingId)
|
|
532
|
+
if (isPendingId)
|
|
421
533
|
delete insertData.id;
|
|
422
|
-
}
|
|
423
|
-
// Convert the *complete* payload (attributes + id + FKs) in one place (#292).
|
|
424
534
|
const row = this.deps.convertRow(insertData, schema.columns, this.deps.postgresJsonRule);
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
535
|
+
return { write, modelName, record, row, isPendingId };
|
|
536
|
+
}
|
|
537
|
+
_chunkWithinBindParameterLimit(prepared) {
|
|
538
|
+
const chunks = [];
|
|
539
|
+
let chunk = [];
|
|
540
|
+
let chunkColumns = new Set();
|
|
541
|
+
for (const create of prepared) {
|
|
542
|
+
const columnsWithCreate = new Set([...chunkColumns, ...Object.keys(create.row)]);
|
|
543
|
+
if (chunk.length > 0 && (chunk.length + 1) * columnsWithCreate.size > POSTGRES_MAX_BIND_PARAMETERS) {
|
|
544
|
+
chunks.push(chunk);
|
|
545
|
+
chunk = [];
|
|
546
|
+
chunkColumns = new Set(Object.keys(create.row));
|
|
547
|
+
}
|
|
548
|
+
else {
|
|
549
|
+
chunkColumns = columnsWithCreate;
|
|
550
|
+
}
|
|
551
|
+
chunk.push(create);
|
|
552
|
+
}
|
|
553
|
+
if (chunk.length > 0)
|
|
554
|
+
chunks.push(chunk);
|
|
555
|
+
return chunks;
|
|
556
|
+
}
|
|
557
|
+
async _insertCreates(schema, creates) {
|
|
558
|
+
if (creates.length === 1)
|
|
559
|
+
return this._insertCreatesOneByOne(schema, creates);
|
|
560
|
+
let result;
|
|
561
|
+
try {
|
|
562
|
+
const { sql, values } = this.deps.buildInsertMany(schema.table, creates.map(({ row }) => row));
|
|
563
|
+
result = await this.requirePool().query(sql, values);
|
|
564
|
+
}
|
|
565
|
+
catch {
|
|
566
|
+
return this._insertCreatesOneByOne(schema, creates);
|
|
441
567
|
}
|
|
568
|
+
creates.forEach((create, index) => {
|
|
569
|
+
this._adoptGeneratedId(create, result.rows[index]);
|
|
570
|
+
create.write.resolve();
|
|
571
|
+
});
|
|
572
|
+
}
|
|
573
|
+
async _insertCreatesOneByOne(schema, creates) {
|
|
574
|
+
for (const create of creates) {
|
|
575
|
+
try {
|
|
576
|
+
const { sql, values } = this.deps.buildInsert(schema.table, create.row);
|
|
577
|
+
const result = await this.requirePool().query(sql, values);
|
|
578
|
+
this._adoptGeneratedId(create, result.rows[0]);
|
|
579
|
+
create.write.resolve();
|
|
580
|
+
}
|
|
581
|
+
catch (error) {
|
|
582
|
+
create.write.reject(error);
|
|
583
|
+
}
|
|
584
|
+
}
|
|
585
|
+
}
|
|
586
|
+
_adoptGeneratedId({ modelName, record, isPendingId, write }, returnedRow) {
|
|
587
|
+
if (!isPendingId || !returnedRow)
|
|
588
|
+
return;
|
|
589
|
+
const pendingId = record.id;
|
|
590
|
+
const realId = returnedRow.id;
|
|
591
|
+
const modelStore = this.deps.store.get(modelName);
|
|
592
|
+
modelStore.delete(pendingId);
|
|
593
|
+
record.__data.id = realId;
|
|
594
|
+
record.id = realId;
|
|
595
|
+
modelStore.set(realId, record);
|
|
596
|
+
if (write.response?.data)
|
|
597
|
+
write.response.data.id = realId;
|
|
598
|
+
delete record.__data.__pendingSqlId;
|
|
442
599
|
}
|
|
443
600
|
async _persistUpdate(modelName, context, _response) {
|
|
444
601
|
const schemas = this.deps.introspectModels();
|
|
@@ -12,6 +12,7 @@ interface HybridSearchOptions {
|
|
|
12
12
|
}
|
|
13
13
|
export declare function validateIdentifier(name: string, context?: string): string;
|
|
14
14
|
export declare function buildInsert(table: string, data: Record<string, unknown>): QueryResult;
|
|
15
|
+
export declare function buildInsertMany(table: string, rows: Record<string, unknown>[]): QueryResult;
|
|
15
16
|
export declare function buildUpdate(table: string, id: unknown, data: Record<string, unknown>): QueryResult;
|
|
16
17
|
export declare function buildDelete(table: string, id: unknown): QueryResult;
|
|
17
18
|
export declare function buildSelect(table: string, conditions?: Record<string, unknown>): QueryResult;
|
|
@@ -14,6 +14,23 @@ export function buildInsert(table, data) {
|
|
|
14
14
|
const sql = `INSERT INTO "${table}" (${keys.map(k => `"${k}"`).join(', ')}) VALUES (${placeholders.join(', ')}) RETURNING "id"`;
|
|
15
15
|
return { sql, values };
|
|
16
16
|
}
|
|
17
|
+
export function buildInsertMany(table, rows) {
|
|
18
|
+
validateIdentifier(table, 'table name');
|
|
19
|
+
const columns = [...new Set(rows.flatMap(row => Object.keys(row)))];
|
|
20
|
+
columns.forEach(column => validateIdentifier(column, 'column name'));
|
|
21
|
+
const values = [];
|
|
22
|
+
const tuples = rows.map(row => {
|
|
23
|
+
const cells = columns.map(column => {
|
|
24
|
+
if (!Object.hasOwn(row, column))
|
|
25
|
+
return 'DEFAULT';
|
|
26
|
+
values.push(row[column]);
|
|
27
|
+
return `$${values.length}`;
|
|
28
|
+
});
|
|
29
|
+
return `(${cells.join(', ')})`;
|
|
30
|
+
});
|
|
31
|
+
const sql = `INSERT INTO "${table}" (${columns.map(column => `"${column}"`).join(', ')}) VALUES ${tuples.join(', ')} RETURNING "id"`;
|
|
32
|
+
return { sql, values };
|
|
33
|
+
}
|
|
17
34
|
export function buildUpdate(table, id, data) {
|
|
18
35
|
validateIdentifier(table, 'table name');
|
|
19
36
|
const keys = Object.keys(data);
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
export { validateIdentifier, buildInsert, buildUpdate, buildDelete, buildSelect } from '../postgres/query-builder.js';
|
|
1
|
+
export { validateIdentifier, buildInsert, buildInsertMany, buildUpdate, buildDelete, buildSelect } from '../postgres/query-builder.js';
|
|
2
2
|
export declare function validateInterval(interval: string, context?: string): string;
|
|
3
3
|
export declare function validateAggregate(expr: string, context?: string): string;
|
|
4
4
|
interface QueryResult {
|
|
@@ -1,5 +1,4 @@
|
|
|
1
|
-
|
|
2
|
-
export { validateIdentifier, buildInsert, buildUpdate, buildDelete, buildSelect } from '../postgres/query-builder.js';
|
|
1
|
+
export { validateIdentifier, buildInsert, buildInsertMany, buildUpdate, buildDelete, buildSelect } from '../postgres/query-builder.js';
|
|
3
2
|
import { validateIdentifier } from '../postgres/query-builder.js';
|
|
4
3
|
const SAFE_INTERVAL = /^\d+\s+(microsecond|millisecond|second|minute|hour|day|week|month|year)s?$/i;
|
|
5
4
|
export function validateInterval(interval, context = 'interval') {
|
package/package.json
CHANGED
|
@@ -21,7 +21,7 @@ export async function getPool(pgConfig: PgConfig, extensions: string[] = ['vecto
|
|
|
21
21
|
|
|
22
22
|
const { default: pg } = await import('pg');
|
|
23
23
|
|
|
24
|
-
const { host, port, user, password, database, connectionLimit, migrationsDir, migrationsTable, autoMigrate, ...poolOpts } = pgConfig;
|
|
24
|
+
const { host, port, user, password, database, connectionLimit, migrationsDir, migrationsTable, autoMigrate, writeBatchSize, writeQueueLimit, ...poolOpts } = pgConfig;
|
|
25
25
|
|
|
26
26
|
pool = new pg.Pool({
|
|
27
27
|
host,
|
|
@@ -2,7 +2,7 @@ import { getPool, closePool } from './connection.js';
|
|
|
2
2
|
import { ensureMigrationsTable, getAppliedMigrations, getMigrationFiles, applyMigration, parseMigrationFile } from './migration-runner.js';
|
|
3
3
|
import { introspectModels, introspectViews, getTopologicalOrder, schemasToSnapshot } from './schema-introspector.js';
|
|
4
4
|
import { loadLatestSnapshot, detectSchemaDrift } from './migration-generator.js';
|
|
5
|
-
import { buildInsert, buildUpdate, buildDelete, buildSelect, buildVectorSearch, buildHybridSearch } from './query-builder.js';
|
|
5
|
+
import { buildInsert, buildInsertMany, buildUpdate, buildDelete, buildSelect, buildVectorSearch, buildHybridSearch } from './query-builder.js';
|
|
6
6
|
import { store } from '@stonyx/orm';
|
|
7
7
|
import { createRecord } from '../manage-record.js';
|
|
8
8
|
import { confirm } from '@stonyx/utils/prompt';
|
|
@@ -28,6 +28,43 @@ interface PersistResponse {
|
|
|
28
28
|
data?: { id: unknown };
|
|
29
29
|
}
|
|
30
30
|
|
|
31
|
+
interface QueuedWrite {
|
|
32
|
+
operation: string;
|
|
33
|
+
modelName: string;
|
|
34
|
+
context: PersistContext;
|
|
35
|
+
response: PersistResponse;
|
|
36
|
+
resolve: () => void;
|
|
37
|
+
reject: (error: unknown) => void;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
interface PreparedCreate {
|
|
41
|
+
write: QueuedWrite;
|
|
42
|
+
modelName: string;
|
|
43
|
+
record: OrmRecord;
|
|
44
|
+
row: Record<string, unknown>;
|
|
45
|
+
isPendingId: boolean;
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
type ExitEmitter = Pick<NodeJS.Process, 'once' | 'removeListener'>;
|
|
49
|
+
|
|
50
|
+
const POSTGRES_MAX_BIND_PARAMETERS = 65535;
|
|
51
|
+
const DEFAULT_WRITE_BATCH_SIZE = 500;
|
|
52
|
+
const DEFAULT_WRITE_QUEUE_LIMIT = 10000;
|
|
53
|
+
|
|
54
|
+
export class WriteQueueFullError extends Error {
|
|
55
|
+
constructor(limit: number) {
|
|
56
|
+
super(`PostgresDB write queue is full (${limit} queued writes, writeQueueLimit); the write was rejected`);
|
|
57
|
+
this.name = 'WriteQueueFullError';
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
export class WriteQueueClosedError extends Error {
|
|
62
|
+
constructor() {
|
|
63
|
+
super('PostgresDB is shutting down; the write was rejected');
|
|
64
|
+
this.name = 'WriteQueueClosedError';
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
|
|
31
68
|
interface SearchResult {
|
|
32
69
|
record: OrmRecord;
|
|
33
70
|
distance: number;
|
|
@@ -53,6 +90,7 @@ interface PostgresDeps {
|
|
|
53
90
|
loadLatestSnapshot: typeof loadLatestSnapshot;
|
|
54
91
|
detectSchemaDrift: typeof detectSchemaDrift;
|
|
55
92
|
buildInsert: typeof buildInsert;
|
|
93
|
+
buildInsertMany: typeof buildInsertMany;
|
|
56
94
|
buildUpdate: typeof buildUpdate;
|
|
57
95
|
buildDelete: typeof buildDelete;
|
|
58
96
|
buildSelect: typeof buildSelect;
|
|
@@ -68,6 +106,7 @@ interface PostgresDeps {
|
|
|
68
106
|
config: typeof config;
|
|
69
107
|
log: typeof log;
|
|
70
108
|
path: typeof path;
|
|
109
|
+
process: ExitEmitter;
|
|
71
110
|
[key: string]: unknown;
|
|
72
111
|
}
|
|
73
112
|
|
|
@@ -76,8 +115,8 @@ const defaultDeps: PostgresDeps = {
|
|
|
76
115
|
getMigrationFiles, applyMigration, parseMigrationFile,
|
|
77
116
|
introspectModels, introspectViews, getTopologicalOrder, schemasToSnapshot,
|
|
78
117
|
loadLatestSnapshot, detectSchemaDrift,
|
|
79
|
-
buildInsert, buildUpdate, buildDelete, buildSelect, buildVectorSearch, buildHybridSearch,
|
|
80
|
-
createRecord, store, confirm, readFile, getPluralName, config, log, path,
|
|
118
|
+
buildInsert, buildInsertMany, buildUpdate, buildDelete, buildSelect, buildVectorSearch, buildHybridSearch,
|
|
119
|
+
createRecord, store, confirm, readFile, getPluralName, config, log, path, process,
|
|
81
120
|
convertRow, postgresJsonRule
|
|
82
121
|
};
|
|
83
122
|
|
|
@@ -95,14 +134,11 @@ export default class PostgresDB {
|
|
|
95
134
|
pool!: Pool | null;
|
|
96
135
|
pgConfig!: Record<string, unknown>;
|
|
97
136
|
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
* Reads are NOT affected — only persist() serializes.
|
|
104
|
-
*/
|
|
105
|
-
private _writeQueue: Promise<void> = Promise.resolve();
|
|
137
|
+
private _queuedWrites: QueuedWrite[] = [];
|
|
138
|
+
private _inFlightWrites = new Set<QueuedWrite>();
|
|
139
|
+
private _drainingWrites: Promise<void> | null = null;
|
|
140
|
+
private _acceptingWrites = true;
|
|
141
|
+
private _exitReport: (() => void) | null = null;
|
|
106
142
|
|
|
107
143
|
constructor(deps: Partial<PostgresDeps> = {}) {
|
|
108
144
|
const Ctor = this.constructor as typeof PostgresDB;
|
|
@@ -126,6 +162,20 @@ export default class PostgresDB {
|
|
|
126
162
|
);
|
|
127
163
|
await this.deps.ensureMigrationsTable(this.pool, this.pgConfig.migrationsTable as string | undefined);
|
|
128
164
|
await this.loadMemoryRecords();
|
|
165
|
+
this._acceptingWrites = true;
|
|
166
|
+
this._registerExitReport();
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
get writeBatchSize(): number {
|
|
170
|
+
return Number(this.pgConfig?.writeBatchSize) || DEFAULT_WRITE_BATCH_SIZE;
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
get writeQueueLimit(): number {
|
|
174
|
+
return Number(this.pgConfig?.writeQueueLimit) || DEFAULT_WRITE_QUEUE_LIMIT;
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
get pendingWriteCount(): number {
|
|
178
|
+
return this._inFlightWrites.size + this._queuedWrites.length;
|
|
129
179
|
}
|
|
130
180
|
|
|
131
181
|
async startup(): Promise<void> {
|
|
@@ -211,10 +261,36 @@ export default class PostgresDB {
|
|
|
211
261
|
}
|
|
212
262
|
|
|
213
263
|
async shutdown(): Promise<void> {
|
|
264
|
+
this._acceptingWrites = false;
|
|
265
|
+
while (this._drainingWrites) await this._drainingWrites;
|
|
266
|
+
this._unregisterExitReport();
|
|
214
267
|
await this.deps.closePool();
|
|
215
268
|
this.pool = null;
|
|
216
269
|
}
|
|
217
270
|
|
|
271
|
+
private _registerExitReport(): void {
|
|
272
|
+
if (this._exitReport) return;
|
|
273
|
+
this._exitReport = () => this._reportUnpersistedWrites();
|
|
274
|
+
this.deps.process.once('exit', this._exitReport);
|
|
275
|
+
}
|
|
276
|
+
|
|
277
|
+
private _unregisterExitReport(): void {
|
|
278
|
+
if (!this._exitReport) return;
|
|
279
|
+
this.deps.process.removeListener('exit', this._exitReport);
|
|
280
|
+
this._exitReport = null;
|
|
281
|
+
}
|
|
282
|
+
|
|
283
|
+
private _reportUnpersistedWrites(): void {
|
|
284
|
+
const lostWrites = [...this._inFlightWrites, ...this._queuedWrites];
|
|
285
|
+
if (lostWrites.length === 0) return;
|
|
286
|
+
|
|
287
|
+
const lostByModel = new Map<string, number>();
|
|
288
|
+
for (const { modelName } of lostWrites) lostByModel.set(modelName, (lostByModel.get(modelName) ?? 0) + 1);
|
|
289
|
+
const breakdown = [...lostByModel].map(([modelName, count]) => `${modelName}: ${count}`).join(', ');
|
|
290
|
+
|
|
291
|
+
this.deps.log.error?.(`[PostgresDB] Process exiting with ${lostWrites.length} queued write(s) not persisted; they are lost (${breakdown})`);
|
|
292
|
+
}
|
|
293
|
+
|
|
218
294
|
async save(): Promise<void> {
|
|
219
295
|
// No-op: PostgreSQL persists data immediately via persist()
|
|
220
296
|
}
|
|
@@ -494,77 +570,185 @@ export default class PostgresDB {
|
|
|
494
570
|
}
|
|
495
571
|
|
|
496
572
|
async persist(operation: string, modelName: string, context: PersistContext, response: PersistResponse): Promise<void> {
|
|
497
|
-
// Views are read-only -- no-op for all write operations
|
|
498
573
|
const Orm = (await import('@stonyx/orm')).default;
|
|
499
574
|
if ((Orm.instance as { isView?: (name: string) => boolean })?.isView?.(modelName)) return;
|
|
500
575
|
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
576
|
+
if (!this._acceptingWrites) throw new WriteQueueClosedError();
|
|
577
|
+
if (this._queuedWrites.length >= this.writeQueueLimit) throw new WriteQueueFullError(this.writeQueueLimit);
|
|
578
|
+
|
|
579
|
+
return new Promise<void>((resolve, reject) => {
|
|
580
|
+
const write: QueuedWrite = {
|
|
581
|
+
operation, modelName, context, response,
|
|
582
|
+
resolve: () => {
|
|
583
|
+
this._inFlightWrites.delete(write);
|
|
584
|
+
resolve();
|
|
585
|
+
},
|
|
586
|
+
reject: (error: unknown) => {
|
|
587
|
+
this._inFlightWrites.delete(write);
|
|
588
|
+
reject(error);
|
|
589
|
+
},
|
|
590
|
+
};
|
|
591
|
+
this._queuedWrites.push(write);
|
|
592
|
+
this._drainWritesInBackground();
|
|
593
|
+
});
|
|
594
|
+
}
|
|
595
|
+
|
|
596
|
+
private _drainWritesInBackground(): void {
|
|
597
|
+
if (this._drainingWrites) return;
|
|
598
|
+
this._drainingWrites = this._drainQueuedWrites().finally(() => {
|
|
599
|
+
this._drainingWrites = null;
|
|
600
|
+
});
|
|
601
|
+
}
|
|
602
|
+
|
|
603
|
+
private async _drainQueuedWrites(): Promise<void> {
|
|
604
|
+
while (this._queuedWrites.length > 0) {
|
|
605
|
+
const writes = this._takeNextWriteGroup();
|
|
606
|
+
writes.forEach(write => this._inFlightWrites.add(write));
|
|
607
|
+
try {
|
|
608
|
+
await this._executeWriteGroup(writes);
|
|
609
|
+
} catch (error) {
|
|
610
|
+
writes.filter(write => this._inFlightWrites.has(write)).forEach(write => write.reject(error));
|
|
509
611
|
}
|
|
510
|
-
}
|
|
612
|
+
}
|
|
613
|
+
}
|
|
511
614
|
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
615
|
+
private _takeNextWriteGroup(): QueuedWrite[] {
|
|
616
|
+
const [first] = this._queuedWrites;
|
|
617
|
+
let groupSize = 1;
|
|
618
|
+
|
|
619
|
+
if (first.operation === 'create') {
|
|
620
|
+
while (
|
|
621
|
+
groupSize < this.writeBatchSize
|
|
622
|
+
&& groupSize < this._queuedWrites.length
|
|
623
|
+
&& this._queuedWrites[groupSize].operation === 'create'
|
|
624
|
+
&& this._queuedWrites[groupSize].modelName === first.modelName
|
|
625
|
+
) groupSize++;
|
|
626
|
+
}
|
|
627
|
+
|
|
628
|
+
return this._queuedWrites.splice(0, groupSize);
|
|
516
629
|
}
|
|
517
630
|
|
|
518
|
-
private async
|
|
519
|
-
const
|
|
520
|
-
|
|
631
|
+
private async _executeWriteGroup(writes: QueuedWrite[]): Promise<void> {
|
|
632
|
+
const [first] = writes;
|
|
633
|
+
if (first.operation === 'create') return this._persistCreates(first.modelName, writes);
|
|
521
634
|
|
|
522
|
-
|
|
635
|
+
try {
|
|
636
|
+
if (first.operation === 'update') await this._persistUpdate(first.modelName, first.context, first.response);
|
|
637
|
+
if (first.operation === 'delete') await this._persistDelete(first.modelName, first.context);
|
|
638
|
+
first.resolve();
|
|
639
|
+
} catch (error) {
|
|
640
|
+
first.reject(error);
|
|
641
|
+
}
|
|
642
|
+
}
|
|
523
643
|
|
|
644
|
+
private async _persistCreates(modelName: string, writes: QueuedWrite[]): Promise<void> {
|
|
645
|
+
const schema = this.deps.introspectModels()[modelName];
|
|
646
|
+
const prepared: PreparedCreate[] = [];
|
|
647
|
+
|
|
648
|
+
for (const write of writes) {
|
|
649
|
+
try {
|
|
650
|
+
const create = schema ? this._prepareCreate(modelName, schema, write) : null;
|
|
651
|
+
if (create) prepared.push(create);
|
|
652
|
+
else write.resolve();
|
|
653
|
+
} catch (error) {
|
|
654
|
+
write.reject(error);
|
|
655
|
+
}
|
|
656
|
+
}
|
|
657
|
+
|
|
658
|
+
for (const chunk of this._chunkWithinBindParameterLimit(prepared)) {
|
|
659
|
+
await this._insertCreates(schema, chunk);
|
|
660
|
+
}
|
|
661
|
+
}
|
|
662
|
+
|
|
663
|
+
private _prepareCreate(modelName: string, schema: ModelSchema, write: QueuedWrite): PreparedCreate | null {
|
|
664
|
+
const { context, response } = write;
|
|
524
665
|
const recordId = response?.data?.id;
|
|
525
666
|
const storeRef = this.deps.store as unknown as {
|
|
526
667
|
get(name: string, id: unknown): OrmRecord | null;
|
|
527
668
|
};
|
|
528
669
|
const record = recordId != null ? storeRef.get(modelName, isNaN(recordId as number) ? recordId : parseInt(recordId as string)) : null;
|
|
529
670
|
|
|
530
|
-
if (!record) return;
|
|
671
|
+
if (!record) return null;
|
|
531
672
|
|
|
532
673
|
const insertData = this._recordToRow(record, schema, context.rawData);
|
|
533
|
-
|
|
534
|
-
// For auto-increment models, remove the pending ID.
|
|
535
|
-
// Check context.rawData (not record.__data) because __pendingSqlId is not a model
|
|
536
|
-
// attribute and gets lost during serialization.
|
|
537
674
|
const isPendingId = context.rawData?.__pendingSqlId === true;
|
|
538
675
|
|
|
539
|
-
if (isPendingId)
|
|
540
|
-
delete insertData.id;
|
|
541
|
-
}
|
|
676
|
+
if (isPendingId) delete insertData.id;
|
|
542
677
|
|
|
543
|
-
// Convert the *complete* payload (attributes + id + FKs) in one place (#292).
|
|
544
678
|
const row = this.deps.convertRow(insertData, schema.columns, this.deps.postgresJsonRule);
|
|
545
679
|
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
const result = await this.requirePool().query(sql, values);
|
|
680
|
+
return { write, modelName, record, row, isPendingId };
|
|
681
|
+
}
|
|
549
682
|
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
const modelStore = (this.deps.store as unknown as { get(name: string): Map<unknown, unknown> }).get(modelName);
|
|
683
|
+
private _chunkWithinBindParameterLimit(prepared: PreparedCreate[]): PreparedCreate[][] {
|
|
684
|
+
const chunks: PreparedCreate[][] = [];
|
|
685
|
+
let chunk: PreparedCreate[] = [];
|
|
686
|
+
let chunkColumns = new Set<string>();
|
|
555
687
|
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
record.id = realId;
|
|
559
|
-
modelStore.set(realId, record);
|
|
688
|
+
for (const create of prepared) {
|
|
689
|
+
const columnsWithCreate = new Set([...chunkColumns, ...Object.keys(create.row)]);
|
|
560
690
|
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
691
|
+
if (chunk.length > 0 && (chunk.length + 1) * columnsWithCreate.size > POSTGRES_MAX_BIND_PARAMETERS) {
|
|
692
|
+
chunks.push(chunk);
|
|
693
|
+
chunk = [];
|
|
694
|
+
chunkColumns = new Set(Object.keys(create.row));
|
|
695
|
+
} else {
|
|
696
|
+
chunkColumns = columnsWithCreate;
|
|
564
697
|
}
|
|
565
698
|
|
|
566
|
-
|
|
699
|
+
chunk.push(create);
|
|
567
700
|
}
|
|
701
|
+
|
|
702
|
+
if (chunk.length > 0) chunks.push(chunk);
|
|
703
|
+
return chunks;
|
|
704
|
+
}
|
|
705
|
+
|
|
706
|
+
private async _insertCreates(schema: ModelSchema, creates: PreparedCreate[]): Promise<void> {
|
|
707
|
+
if (creates.length === 1) return this._insertCreatesOneByOne(schema, creates);
|
|
708
|
+
|
|
709
|
+
let result: { rows: Record<string, unknown>[] };
|
|
710
|
+
|
|
711
|
+
try {
|
|
712
|
+
const { sql, values } = this.deps.buildInsertMany(schema.table, creates.map(({ row }) => row));
|
|
713
|
+
result = await this.requirePool().query(sql, values);
|
|
714
|
+
} catch {
|
|
715
|
+
return this._insertCreatesOneByOne(schema, creates);
|
|
716
|
+
}
|
|
717
|
+
|
|
718
|
+
creates.forEach((create, index) => {
|
|
719
|
+
this._adoptGeneratedId(create, result.rows[index]);
|
|
720
|
+
create.write.resolve();
|
|
721
|
+
});
|
|
722
|
+
}
|
|
723
|
+
|
|
724
|
+
private async _insertCreatesOneByOne(schema: ModelSchema, creates: PreparedCreate[]): Promise<void> {
|
|
725
|
+
for (const create of creates) {
|
|
726
|
+
try {
|
|
727
|
+
const { sql, values } = this.deps.buildInsert(schema.table, create.row);
|
|
728
|
+
const result = await this.requirePool().query(sql, values);
|
|
729
|
+
this._adoptGeneratedId(create, result.rows[0]);
|
|
730
|
+
create.write.resolve();
|
|
731
|
+
} catch (error) {
|
|
732
|
+
create.write.reject(error);
|
|
733
|
+
}
|
|
734
|
+
}
|
|
735
|
+
}
|
|
736
|
+
|
|
737
|
+
private _adoptGeneratedId({ modelName, record, isPendingId, write }: PreparedCreate, returnedRow: Record<string, unknown> | undefined): void {
|
|
738
|
+
if (!isPendingId || !returnedRow) return;
|
|
739
|
+
|
|
740
|
+
const pendingId = record.id;
|
|
741
|
+
const realId = returnedRow.id as string | number;
|
|
742
|
+
const modelStore = (this.deps.store as unknown as { get(name: string): Map<unknown, unknown> }).get(modelName);
|
|
743
|
+
|
|
744
|
+
modelStore.delete(pendingId);
|
|
745
|
+
record.__data.id = realId;
|
|
746
|
+
record.id = realId;
|
|
747
|
+
modelStore.set(realId, record);
|
|
748
|
+
|
|
749
|
+
if (write.response?.data) write.response.data.id = realId;
|
|
750
|
+
|
|
751
|
+
delete record.__data.__pendingSqlId;
|
|
568
752
|
}
|
|
569
753
|
|
|
570
754
|
private async _persistUpdate(modelName: string, context: PersistContext, _response: PersistResponse): Promise<void> {
|
|
@@ -37,6 +37,27 @@ export function buildInsert(table: string, data: Record<string, unknown>): Query
|
|
|
37
37
|
return { sql, values };
|
|
38
38
|
}
|
|
39
39
|
|
|
40
|
+
export function buildInsertMany(table: string, rows: Record<string, unknown>[]): QueryResult {
|
|
41
|
+
validateIdentifier(table, 'table name');
|
|
42
|
+
|
|
43
|
+
const columns = [...new Set(rows.flatMap(row => Object.keys(row)))];
|
|
44
|
+
columns.forEach(column => validateIdentifier(column, 'column name'));
|
|
45
|
+
|
|
46
|
+
const values: unknown[] = [];
|
|
47
|
+
const tuples = rows.map(row => {
|
|
48
|
+
const cells = columns.map(column => {
|
|
49
|
+
if (!Object.hasOwn(row, column)) return 'DEFAULT';
|
|
50
|
+
values.push(row[column]);
|
|
51
|
+
return `$${values.length}`;
|
|
52
|
+
});
|
|
53
|
+
return `(${cells.join(', ')})`;
|
|
54
|
+
});
|
|
55
|
+
|
|
56
|
+
const sql = `INSERT INTO "${table}" (${columns.map(column => `"${column}"`).join(', ')}) VALUES ${tuples.join(', ')} RETURNING "id"`;
|
|
57
|
+
|
|
58
|
+
return { sql, values };
|
|
59
|
+
}
|
|
60
|
+
|
|
40
61
|
export function buildUpdate(table: string, id: unknown, data: Record<string, unknown>): QueryResult {
|
|
41
62
|
validateIdentifier(table, 'table name');
|
|
42
63
|
|
|
@@ -1,5 +1,4 @@
|
|
|
1
|
-
|
|
2
|
-
export { validateIdentifier, buildInsert, buildUpdate, buildDelete, buildSelect } from '../postgres/query-builder.js';
|
|
1
|
+
export { validateIdentifier, buildInsert, buildInsertMany, buildUpdate, buildDelete, buildSelect } from '../postgres/query-builder.js';
|
|
3
2
|
|
|
4
3
|
import { validateIdentifier } from '../postgres/query-builder.js';
|
|
5
4
|
|