@stonyx/orm 0.3.2-beta.326 → 0.3.2-beta.328

package/dist/postgres/postgres-db.js

@@ -2,7 +2,7 @@ import { getPool, closePool } from './connection.js';
 import { ensureMigrationsTable, getAppliedMigrations, getMigrationFiles, applyMigration, parseMigrationFile } from './migration-runner.js';
 import { introspectModels, introspectViews, getTopologicalOrder, schemasToSnapshot } from './schema-introspector.js';
 import { loadLatestSnapshot, detectSchemaDrift } from './migration-generator.js';
-import { buildInsert, buildUpdate, buildDelete, buildSelect, buildVectorSearch, buildHybridSearch } from './query-builder.js';
+import { buildInsert, buildInsertMany, buildUpdate, buildDelete, buildSelect, buildVectorSearch, buildHybridSearch } from './query-builder.js';
 import { store } from '@stonyx/orm';
 import { createRecord } from '../manage-record.js';
 import { confirm } from '@stonyx/utils/prompt';
@@ -14,13 +14,28 @@ import { isDbError } from '../utils.js';
 import config from 'stonyx/config';
 import log from 'stonyx/log';
 import path from 'path';
+const POSTGRES_MAX_BIND_PARAMETERS = 65535;
+const DEFAULT_WRITE_BATCH_SIZE = 500;
+const DEFAULT_WRITE_QUEUE_LIMIT = 10000;
+export class WriteQueueFullError extends Error {
+    constructor(limit) {
+        super(`PostgresDB write queue is full (${limit} queued writes, writeQueueLimit); the write was rejected`);
+        this.name = 'WriteQueueFullError';
+    }
+}
+export class WriteQueueClosedError extends Error {
+    constructor() {
+        super('PostgresDB is shutting down; the write was rejected');
+        this.name = 'WriteQueueClosedError';
+    }
+}
 const defaultDeps = {
     getPool, closePool, ensureMigrationsTable, getAppliedMigrations,
     getMigrationFiles, applyMigration, parseMigrationFile,
     introspectModels, introspectViews, getTopologicalOrder, schemasToSnapshot,
     loadLatestSnapshot, detectSchemaDrift,
-    buildInsert, buildUpdate, buildDelete, buildSelect, buildVectorSearch, buildHybridSearch,
-    createRecord, store, confirm, readFile, getPluralName, config, log, path,
+    buildInsert, buildInsertMany, buildUpdate, buildDelete, buildSelect, buildVectorSearch, buildHybridSearch,
+    createRecord, store, confirm, readFile, getPluralName, config, log, path, process,
     convertRow, postgresJsonRule
 };
 export default class PostgresDB {
@@ -33,14 +48,11 @@ export default class PostgresDB {
     deps;
     pool;
     pgConfig;
-    /**
-     * Promise-chain mutex for write serialization (#156).
-     * All persist() calls chain through this single queue so concurrent
-     * fire-and-forget writes never produce parallel transactions
-     * on FK-linked rows (which cause deadlocks).
-     * Reads are NOT affected — only persist() serializes.
-     */
-    _writeQueue = Promise.resolve();
+    _queuedWrites = [];
+    _inFlightWrites = new Set();
+    _drainingWrites = null;
+    _acceptingWrites = true;
+    _exitReport = null;
     constructor(deps = {}) {
         const Ctor = this.constructor;
         if (Ctor.instance)
@@ -59,6 +71,17 @@ export default class PostgresDB {
         this.pool = await this.deps.getPool(this.pgConfig, this.constructor.extensions);
         await this.deps.ensureMigrationsTable(this.pool, this.pgConfig.migrationsTable);
         await this.loadMemoryRecords();
+        this._acceptingWrites = true;
+        this._registerExitReport();
+    }
+    get writeBatchSize() {
+        return Number(this.pgConfig?.writeBatchSize) || DEFAULT_WRITE_BATCH_SIZE;
+    }
+    get writeQueueLimit() {
+        return Number(this.pgConfig?.writeQueueLimit) || DEFAULT_WRITE_QUEUE_LIMIT;
+    }
+    get pendingWriteCount() {
+        return this._inFlightWrites.size + this._queuedWrites.length;
     }
     async startup() {
         const migrationsPath = this.deps.path.resolve(this.deps.config.rootPath, this.pgConfig.migrationsDir);
@@ -135,9 +158,35 @@ export default class PostgresDB {
         }
     }
     async shutdown() {
+        this._acceptingWrites = false;
+        while (this._drainingWrites)
+            await this._drainingWrites;
+        this._unregisterExitReport();
         await this.deps.closePool();
         this.pool = null;
     }
+    _registerExitReport() {
+        if (this._exitReport)
+            return;
+        this._exitReport = () => this._reportUnpersistedWrites();
+        this.deps.process.once('exit', this._exitReport);
+    }
+    _unregisterExitReport() {
+        if (!this._exitReport)
+            return;
+        this.deps.process.removeListener('exit', this._exitReport);
+        this._exitReport = null;
+    }
+    _reportUnpersistedWrites() {
+        const lostWrites = [...this._inFlightWrites, ...this._queuedWrites];
+        if (lostWrites.length === 0)
+            return;
+        const lostByModel = new Map();
+        for (const { modelName } of lostWrites)
+            lostByModel.set(modelName, (lostByModel.get(modelName) ?? 0) + 1);
+        const breakdown = [...lostByModel].map(([modelName, count]) => `${modelName}: ${count}`).join(', ');
+        this.deps.log.error?.(`[PostgresDB] Process exiting with ${lostWrites.length} queued write(s) not persisted; they are lost (${breakdown})`);
+    }
     async save() {
         // No-op: PostgreSQL persists data immediately via persist()
     }
@@ -383,62 +432,170 @@ export default class PostgresDB {
         return rawData;
     }
     async persist(operation, modelName, context, response) {
-        // Views are read-only -- no-op for all write operations
         const Orm = (await import('@stonyx/orm')).default;
         if (Orm.instance?.isView?.(modelName))
             return;
-        const work = async () => {
-            switch (operation) {
-                case 'create':
-                    return this._persistCreate(modelName, context, response);
-                case 'update':
-                    return this._persistUpdate(modelName, context, response);
-                case 'delete':
-                    return this._persistDelete(modelName, context);
-            }
-        };
-        // Chain through the write queue — .then(work, work) ensures the queue
-        // advances even when a previous persist rejects (#156).
-        this._writeQueue = this._writeQueue.then(work, work);
-        return this._writeQueue;
-    }
-    async _persistCreate(modelName, context, response) {
-        const schemas = this.deps.introspectModels();
-        const schema = schemas[modelName];
-        if (!schema)
+        if (!this._acceptingWrites)
+            throw new WriteQueueClosedError();
+        if (this._queuedWrites.length >= this.writeQueueLimit)
+            throw new WriteQueueFullError(this.writeQueueLimit);
+        return new Promise((resolve, reject) => {
+            const write = {
+                operation, modelName, context, response,
+                resolve: () => {
+                    this._inFlightWrites.delete(write);
+                    resolve();
+                },
+                reject: (error) => {
+                    this._inFlightWrites.delete(write);
+                    reject(error);
+                },
+            };
+            this._queuedWrites.push(write);
+            this._drainWritesInBackground();
+        });
+    }
+    _drainWritesInBackground() {
+        if (this._drainingWrites)
             return;
+        this._drainingWrites = this._drainQueuedWrites().finally(() => {
+            this._drainingWrites = null;
+        });
+    }
+    async _drainQueuedWrites() {
+        while (this._queuedWrites.length > 0) {
+            const writes = this._takeNextWriteGroup();
+            writes.forEach(write => this._inFlightWrites.add(write));
+            try {
+                await this._executeWriteGroup(writes);
+            }
+            catch (error) {
+                writes.filter(write => this._inFlightWrites.has(write)).forEach(write => write.reject(error));
+            }
+        }
+    }
+    _takeNextWriteGroup() {
+        const [first] = this._queuedWrites;
+        let groupSize = 1;
+        if (first.operation === 'create') {
+            while (groupSize < this.writeBatchSize
+                && groupSize < this._queuedWrites.length
+                && this._queuedWrites[groupSize].operation === 'create'
+                && this._queuedWrites[groupSize].modelName === first.modelName)
+                groupSize++;
+        }
+        return this._queuedWrites.splice(0, groupSize);
+    }
+    async _executeWriteGroup(writes) {
+        const [first] = writes;
+        if (first.operation === 'create')
+            return this._persistCreates(first.modelName, writes);
+        try {
+            if (first.operation === 'update')
+                await this._persistUpdate(first.modelName, first.context, first.response);
+            if (first.operation === 'delete')
+                await this._persistDelete(first.modelName, first.context);
+            first.resolve();
+        }
+        catch (error) {
+            first.reject(error);
+        }
+    }
+    async _persistCreates(modelName, writes) {
+        const schema = this.deps.introspectModels()[modelName];
+        const prepared = [];
+        for (const write of writes) {
+            try {
+                const create = schema ? this._prepareCreate(modelName, schema, write) : null;
+                if (create)
+                    prepared.push(create);
+                else
+                    write.resolve();
+            }
+            catch (error) {
+                write.reject(error);
+            }
+        }
+        for (const chunk of this._chunkWithinBindParameterLimit(prepared)) {
+            await this._insertCreates(schema, chunk);
+        }
+    }
+    _prepareCreate(modelName, schema, write) {
+        const { context, response } = write;
         const recordId = response?.data?.id;
         const storeRef = this.deps.store;
         const record = recordId != null ? storeRef.get(modelName, isNaN(recordId) ? recordId : parseInt(recordId)) : null;
         if (!record)
-            return;
+            return null;
         const insertData = this._recordToRow(record, schema, context.rawData);
-        // For auto-increment models, remove the pending ID.
-        // Check context.rawData (not record.__data) because __pendingSqlId is not a model
-        // attribute and gets lost during serialization.
         const isPendingId = context.rawData?.__pendingSqlId === true;
-        if (isPendingId) {
+        if (isPendingId)
             delete insertData.id;
-        }
-        // Convert the *complete* payload (attributes + id + FKs) in one place (#292).
         const row = this.deps.convertRow(insertData, schema.columns, this.deps.postgresJsonRule);
-        const { sql, values } = this.deps.buildInsert(schema.table, row);
-        const result = await this.requirePool().query(sql, values);
-        // Re-key the record in the store if PostgreSQL generated the ID (via RETURNING)
-        if (isPendingId && result.rows.length > 0) {
-            const pendingId = record.id;
-            const realId = result.rows[0].id;
-            const modelStore = this.deps.store.get(modelName);
-            modelStore.delete(pendingId);
-            record.__data.id = realId;
-            record.id = realId;
-            modelStore.set(realId, record);
-            // Update the response data with the real ID
-            if (response?.data) {
-                response.data.id = realId;
-            }
-            delete record.__data.__pendingSqlId;
+        return { write, modelName, record, row, isPendingId };
+    }
+    _chunkWithinBindParameterLimit(prepared) {
+        const chunks = [];
+        let chunk = [];
+        let chunkColumns = new Set();
+        for (const create of prepared) {
+            const columnsWithCreate = new Set([...chunkColumns, ...Object.keys(create.row)]);
+            if (chunk.length > 0 && (chunk.length + 1) * columnsWithCreate.size > POSTGRES_MAX_BIND_PARAMETERS) {
+                chunks.push(chunk);
+                chunk = [];
+                chunkColumns = new Set(Object.keys(create.row));
+            }
+            else {
+                chunkColumns = columnsWithCreate;
+            }
+            chunk.push(create);
+        }
+        if (chunk.length > 0)
+            chunks.push(chunk);
+        return chunks;
+    }
+    async _insertCreates(schema, creates) {
+        if (creates.length === 1)
+            return this._insertCreatesOneByOne(schema, creates);
+        let result;
+        try {
+            const { sql, values } = this.deps.buildInsertMany(schema.table, creates.map(({ row }) => row));
+            result = await this.requirePool().query(sql, values);
+        }
+        catch {
+            return this._insertCreatesOneByOne(schema, creates);
         }
+        creates.forEach((create, index) => {
+            this._adoptGeneratedId(create, result.rows[index]);
+            create.write.resolve();
+        });
+    }
+    async _insertCreatesOneByOne(schema, creates) {
+        for (const create of creates) {
+            try {
+                const { sql, values } = this.deps.buildInsert(schema.table, create.row);
+                const result = await this.requirePool().query(sql, values);
+                this._adoptGeneratedId(create, result.rows[0]);
+                create.write.resolve();
+            }
+            catch (error) {
+                create.write.reject(error);
+            }
+        }
+    }
+    _adoptGeneratedId({ modelName, record, isPendingId, write }, returnedRow) {
+        if (!isPendingId || !returnedRow)
+            return;
+        const pendingId = record.id;
+        const realId = returnedRow.id;
+        const modelStore = this.deps.store.get(modelName);
+        modelStore.delete(pendingId);
+        record.__data.id = realId;
+        record.id = realId;
+        modelStore.set(realId, record);
+        if (write.response?.data)
+            write.response.data.id = realId;
+        delete record.__data.__pendingSqlId;
     }
     async _persistUpdate(modelName, context, _response) {
         const schemas = this.deps.introspectModels();

package/dist/postgres/query-builder.d.ts

@@ -12,6 +12,7 @@ interface HybridSearchOptions {
 }
 export declare function validateIdentifier(name: string, context?: string): string;
 export declare function buildInsert(table: string, data: Record<string, unknown>): QueryResult;
+export declare function buildInsertMany(table: string, rows: Record<string, unknown>[]): QueryResult;
 export declare function buildUpdate(table: string, id: unknown, data: Record<string, unknown>): QueryResult;
 export declare function buildDelete(table: string, id: unknown): QueryResult;
 export declare function buildSelect(table: string, conditions?: Record<string, unknown>): QueryResult;

package/dist/postgres/query-builder.js

@@ -14,6 +14,23 @@ export function buildInsert(table, data) {
     const sql = `INSERT INTO "${table}" (${keys.map(k => `"${k}"`).join(', ')}) VALUES (${placeholders.join(', ')}) RETURNING "id"`;
     return { sql, values };
 }
+export function buildInsertMany(table, rows) {
+    validateIdentifier(table, 'table name');
+    const columns = [...new Set(rows.flatMap(row => Object.keys(row)))];
+    columns.forEach(column => validateIdentifier(column, 'column name'));
+    const values = [];
+    const tuples = rows.map(row => {
+        const cells = columns.map(column => {
+            if (!Object.hasOwn(row, column))
+                return 'DEFAULT';
+            values.push(row[column]);
+            return `$${values.length}`;
+        });
+        return `(${cells.join(', ')})`;
+    });
+    const sql = `INSERT INTO "${table}" (${columns.map(column => `"${column}"`).join(', ')}) VALUES ${tuples.join(', ')} RETURNING "id"`;
+    return { sql, values };
+}
 export function buildUpdate(table, id, data) {
     validateIdentifier(table, 'table name');
     const keys = Object.keys(data);

package/dist/timescale/query-builder.d.ts

@@ -1,4 +1,4 @@
-export { validateIdentifier, buildInsert, buildUpdate, buildDelete, buildSelect } from '../postgres/query-builder.js';
+export { validateIdentifier, buildInsert, buildInsertMany, buildUpdate, buildDelete, buildSelect } from '../postgres/query-builder.js';
 export declare function validateInterval(interval: string, context?: string): string;
 export declare function validateAggregate(expr: string, context?: string): string;
 interface QueryResult {

package/dist/timescale/query-builder.js

@@ -1,5 +1,4 @@
-// Re-export all base PostgreSQL query builders
-export { validateIdentifier, buildInsert, buildUpdate, buildDelete, buildSelect } from '../postgres/query-builder.js';
+export { validateIdentifier, buildInsert, buildInsertMany, buildUpdate, buildDelete, buildSelect } from '../postgres/query-builder.js';
 import { validateIdentifier } from '../postgres/query-builder.js';
 const SAFE_INTERVAL = /^\d+\s+(microsecond|millisecond|second|minute|hour|day|week|month|year)s?$/i;
 export function validateInterval(interval, context = 'interval') {

package/package.json

@@ -4,7 +4,7 @@
     "stonyx-async",
     "stonyx-module"
   ],
-  "version": "0.3.2-beta.326",
+  "version": "0.3.2-beta.328",
   "description": "",
   "main": "dist/index.js",
   "type": "module",

package/src/mysql/connection.ts

@@ -10,6 +10,8 @@ interface MysqlConfig {
   migrationsTable?: string;
   migrationsDir?: string;
   autoMigrate?: boolean;
+  writeBatchSize?: number;
+  writeQueueLimit?: number;
 }
 
 let pool: Pool | null = null;