@stonyx/orm 0.3.2-beta.33 → 0.3.2-beta.330

package/src/postgres/postgres-db.ts

@@ -2,12 +2,14 @@ import { getPool, closePool } from './connection.js';
 import { ensureMigrationsTable, getAppliedMigrations, getMigrationFiles, applyMigration, parseMigrationFile } from './migration-runner.js';
 import { introspectModels, introspectViews, getTopologicalOrder, schemasToSnapshot } from './schema-introspector.js';
 import { loadLatestSnapshot, detectSchemaDrift } from './migration-generator.js';
-import { buildInsert, buildUpdate, buildDelete, buildSelect, buildVectorSearch, buildHybridSearch } from './query-builder.js';
+import { buildInsert, buildInsertMany, buildUpdate, buildDelete, buildSelect, buildVectorSearch, buildHybridSearch } from './query-builder.js';
 import { store } from '@stonyx/orm';
 import { createRecord } from '../manage-record.js';
 import { confirm } from '@stonyx/utils/prompt';
 import { readFile } from '@stonyx/utils/file';
 import { getPluralName } from '../plural-registry.js';
+import { convertRow } from '../column-conversion.js';
+import { postgresJsonRule } from './column-rules.js';
 import { isDbError } from '../utils.js';
 import config from 'stonyx/config';
 import log from 'stonyx/log';
@@ -26,6 +28,43 @@ interface PersistResponse {
   data?: { id: unknown };
 }
 
+interface QueuedWrite {
+  operation: string;
+  modelName: string;
+  context: PersistContext;
+  response: PersistResponse;
+  resolve: () => void;
+  reject: (error: unknown) => void;
+}
+
+interface PreparedCreate {
+  write: QueuedWrite;
+  modelName: string;
+  record: OrmRecord;
+  row: Record<string, unknown>;
+  isPendingId: boolean;
+}
+
+type ExitEmitter = Pick<NodeJS.Process, 'once' | 'removeListener'>;
+
+const POSTGRES_MAX_BIND_PARAMETERS = 65535;
+const DEFAULT_WRITE_BATCH_SIZE = 500;
+const DEFAULT_WRITE_QUEUE_LIMIT = 10000;
+
+export class WriteQueueFullError extends Error {
+  constructor(limit: number) {
+    super(`PostgresDB write queue is full (${limit} queued writes, writeQueueLimit); the write was rejected`);
+    this.name = 'WriteQueueFullError';
+  }
+}
+
+export class WriteQueueClosedError extends Error {
+  constructor() {
+    super('PostgresDB is shutting down; the write was rejected');
+    this.name = 'WriteQueueClosedError';
+  }
+}
+
 interface SearchResult {
   record: OrmRecord;
   distance: number;
@@ -51,6 +90,7 @@ interface PostgresDeps {
   loadLatestSnapshot: typeof loadLatestSnapshot;
   detectSchemaDrift: typeof detectSchemaDrift;
   buildInsert: typeof buildInsert;
+  buildInsertMany: typeof buildInsertMany;
   buildUpdate: typeof buildUpdate;
   buildDelete: typeof buildDelete;
   buildSelect: typeof buildSelect;
@@ -61,9 +101,12 @@ interface PostgresDeps {
   confirm: typeof confirm;
   readFile: typeof readFile;
   getPluralName: typeof getPluralName;
+  convertRow: typeof convertRow;
+  postgresJsonRule: typeof postgresJsonRule;
   config: typeof config;
   log: typeof log;
   path: typeof path;
+  process: ExitEmitter;
   [key: string]: unknown;
 }
 
@@ -72,8 +115,9 @@ const defaultDeps: PostgresDeps = {
   getMigrationFiles, applyMigration, parseMigrationFile,
   introspectModels, introspectViews, getTopologicalOrder, schemasToSnapshot,
   loadLatestSnapshot, detectSchemaDrift,
-  buildInsert, buildUpdate, buildDelete, buildSelect, buildVectorSearch, buildHybridSearch,
-  createRecord, store, confirm, readFile, getPluralName, config, log, path
+  buildInsert, buildInsertMany, buildUpdate, buildDelete, buildSelect, buildVectorSearch, buildHybridSearch,
+  createRecord, store, confirm, readFile, getPluralName, config, log, path, process,
+  convertRow, postgresJsonRule
 };
 
 export default class PostgresDB {
@@ -90,6 +134,13 @@ export default class PostgresDB {
   pool!: Pool | null;
   pgConfig!: Record<string, unknown>;
 
+  readonly persistResolvesAfterWrite = true;
+  private _queuedWrites: QueuedWrite[] = [];
+  private _inFlightWrites = new Set<QueuedWrite>();
+  private _drainingWrites: Promise<void> | null = null;
+  private _acceptingWrites = true;
+  private _exitReport: (() => void) | null = null;
+
   constructor(deps: Partial<PostgresDeps> = {}) {
     const Ctor = this.constructor as typeof PostgresDB;
     if (Ctor.instance) return Ctor.instance;
@@ -112,6 +163,20 @@ export default class PostgresDB {
     );
     await this.deps.ensureMigrationsTable(this.pool, this.pgConfig.migrationsTable as string | undefined);
     await this.loadMemoryRecords();
+    this._acceptingWrites = true;
+    this._registerExitReport();
+  }
+
+  get writeBatchSize(): number {
+    return Number(this.pgConfig?.writeBatchSize) || DEFAULT_WRITE_BATCH_SIZE;
+  }
+
+  get writeQueueLimit(): number {
+    return Number(this.pgConfig?.writeQueueLimit) || DEFAULT_WRITE_QUEUE_LIMIT;
+  }
+
+  get pendingWriteCount(): number {
+    return this._inFlightWrites.size + this._queuedWrites.length;
   }
 
   async startup(): Promise<void> {
@@ -125,7 +190,15 @@ export default class PostgresDB {
     if (pending.length > 0) {
       this.deps.log.db?.(`${pending.length} pending migration(s) found.`);
 
-      const shouldApply = await this.deps.confirm(`${pending.length} pending migration(s) found. Apply now?`);
+      let shouldApply: boolean;
+      if (this.pgConfig.autoMigrate === true) {
+        shouldApply = true;
+      } else if (this.pgConfig.autoMigrate === false) {
+        shouldApply = false;
+        this.deps.log.warn?.(`autoMigrate is false — skipping ${pending.length} pending migration(s).`);
+      } else {
+        shouldApply = await this.deps.confirm(`${pending.length} pending migration(s) found. Apply now?`);
+      }
 
       if (shouldApply) {
         for (const filename of pending) {
@@ -146,9 +219,17 @@ export default class PostgresDB {
       const modelCount = Object.keys(schemas).length;
 
       if (modelCount > 0) {
-        const shouldGenerate = await this.deps.confirm(
-          `No migrations found but ${modelCount} model(s) detected. Generate and apply initial migration?`
-        );
+        let shouldGenerate: boolean;
+        if (this.pgConfig.autoMigrate === true) {
+          shouldGenerate = true;
+        } else if (this.pgConfig.autoMigrate === false) {
+          shouldGenerate = false;
+          this.deps.log.warn?.(`autoMigrate is false — skipping initial migration generation for ${modelCount} model(s).`);
+        } else {
+          shouldGenerate = await this.deps.confirm(
+            `No migrations found but ${modelCount} model(s) detected. Generate and apply initial migration?`
+          );
+        }
 
         if (shouldGenerate) {
           const { generateMigration } = await import('./migration-generator.js');
@@ -181,10 +262,36 @@ export default class PostgresDB {
   }
 
   async shutdown(): Promise<void> {
+    this._acceptingWrites = false;
+    while (this._drainingWrites) await this._drainingWrites;
+    this._unregisterExitReport();
     await this.deps.closePool();
     this.pool = null;
   }
 
+  private _registerExitReport(): void {
+    if (this._exitReport) return;
+    this._exitReport = () => this._reportUnpersistedWrites();
+    this.deps.process.once('exit', this._exitReport);
+  }
+
+  private _unregisterExitReport(): void {
+    if (!this._exitReport) return;
+    this.deps.process.removeListener('exit', this._exitReport);
+    this._exitReport = null;
+  }
+
+  private _reportUnpersistedWrites(): void {
+    const lostWrites = [...this._inFlightWrites, ...this._queuedWrites];
+    if (lostWrites.length === 0) return;
+
+    const lostByModel = new Map<string, number>();
+    for (const { modelName } of lostWrites) lostByModel.set(modelName, (lostByModel.get(modelName) ?? 0) + 1);
+    const breakdown = [...lostByModel].map(([modelName, count]) => `${modelName}: ${count}`).join(', ');
+
+    this.deps.log.error?.(`[PostgresDB] Process exiting with ${lostWrites.length} queued write(s) not persisted; they are lost (${breakdown})`);
+  }
+
   async save(): Promise<void> {
     // No-op: PostgreSQL persists data immediately via persist()
   }
@@ -464,67 +571,185 @@ export default class PostgresDB {
   }
 
   async persist(operation: string, modelName: string, context: PersistContext, response: PersistResponse): Promise<void> {
-    // Views are read-only -- no-op for all write operations
     const Orm = (await import('@stonyx/orm')).default;
     if ((Orm.instance as { isView?: (name: string) => boolean })?.isView?.(modelName)) return;
 
-    switch (operation) {
-      case 'create':
-        return this._persistCreate(modelName, context, response);
-      case 'update':
-        return this._persistUpdate(modelName, context, response);
-      case 'delete':
-        return this._persistDelete(modelName, context);
+    if (!this._acceptingWrites) throw new WriteQueueClosedError();
+    if (this._queuedWrites.length >= this.writeQueueLimit) throw new WriteQueueFullError(this.writeQueueLimit);
+
+    return new Promise<void>((resolve, reject) => {
+      const write: QueuedWrite = {
+        operation, modelName, context, response,
+        resolve: () => {
+          this._inFlightWrites.delete(write);
+          resolve();
+        },
+        reject: (error: unknown) => {
+          this._inFlightWrites.delete(write);
+          reject(error);
+        },
+      };
+      this._queuedWrites.push(write);
+      this._drainWritesInBackground();
+    });
+  }
+
+  private _drainWritesInBackground(): void {
+    if (this._drainingWrites) return;
+    this._drainingWrites = this._drainQueuedWrites().finally(() => {
+      this._drainingWrites = null;
+    });
+  }
+
+  private async _drainQueuedWrites(): Promise<void> {
+    while (this._queuedWrites.length > 0) {
+      const writes = this._takeNextWriteGroup();
+      writes.forEach(write => this._inFlightWrites.add(write));
+      try {
+        await this._executeWriteGroup(writes);
+      } catch (error) {
+        writes.filter(write => this._inFlightWrites.has(write)).forEach(write => write.reject(error));
+      }
     }
   }
 
-  private async _persistCreate(modelName: string, context: PersistContext, response: PersistResponse): Promise<void> {
-    const schemas = this.deps.introspectModels();
-    const schema = schemas[modelName];
+  private _takeNextWriteGroup(): QueuedWrite[] {
+    const [first] = this._queuedWrites;
+    let groupSize = 1;
+
+    if (first.operation === 'create') {
+      while (
+        groupSize < this.writeBatchSize
+        && groupSize < this._queuedWrites.length
+        && this._queuedWrites[groupSize].operation === 'create'
+        && this._queuedWrites[groupSize].modelName === first.modelName
+      ) groupSize++;
+    }
 
-    if (!schema) return;
+    return this._queuedWrites.splice(0, groupSize);
+  }
+
+  private async _executeWriteGroup(writes: QueuedWrite[]): Promise<void> {
+    const [first] = writes;
+    if (first.operation === 'create') return this._persistCreates(first.modelName, writes);
+
+    try {
+      if (first.operation === 'update') await this._persistUpdate(first.modelName, first.context, first.response);
+      if (first.operation === 'delete') await this._persistDelete(first.modelName, first.context);
+      first.resolve();
+    } catch (error) {
+      first.reject(error);
+    }
+  }
+
+  private async _persistCreates(modelName: string, writes: QueuedWrite[]): Promise<void> {
+    const schema = this.deps.introspectModels()[modelName];
+    const prepared: PreparedCreate[] = [];
+
+    for (const write of writes) {
+      try {
+        const create = schema ? this._prepareCreate(modelName, schema, write) : null;
+        if (create) prepared.push(create);
+        else write.resolve();
+      } catch (error) {
+        write.reject(error);
+      }
+    }
+
+    for (const chunk of this._chunkWithinBindParameterLimit(prepared)) {
+      await this._insertCreates(schema, chunk);
+    }
+  }
 
+  private _prepareCreate(modelName: string, schema: ModelSchema, write: QueuedWrite): PreparedCreate | null {
+    const { context, response } = write;
     const recordId = response?.data?.id;
     const storeRef = this.deps.store as unknown as {
       get(name: string, id: unknown): OrmRecord | null;
     };
     const record = recordId != null ? storeRef.get(modelName, isNaN(recordId as number) ? recordId : parseInt(recordId as string)) : null;
 
-    if (!record) return;
+    if (!record) return null;
 
     const insertData = this._recordToRow(record, schema, context.rawData);
-
-    // For auto-increment models, remove the pending ID.
-    // Check context.rawData (not record.__data) because __pendingSqlId is not a model
-    // attribute and gets lost during serialization.
     const isPendingId = context.rawData?.__pendingSqlId === true;
 
-    if (isPendingId) {
-      delete insertData.id;
-    }
+    if (isPendingId) delete insertData.id;
 
-    const { sql, values } = this.deps.buildInsert(schema.table, insertData);
+    const row = this.deps.convertRow(insertData, schema.columns, this.deps.postgresJsonRule);
 
-    const result = await this.requirePool().query(sql, values);
+    return { write, modelName, record, row, isPendingId };
+  }
 
-    // Re-key the record in the store if PostgreSQL generated the ID (via RETURNING)
-    if (isPendingId && result.rows.length > 0) {
-      const pendingId = record.id;
-      const realId = result.rows[0].id as string | number;
-      const modelStore = (this.deps.store as unknown as { get(name: string): Map<unknown, unknown> }).get(modelName);
+  private _chunkWithinBindParameterLimit(prepared: PreparedCreate[]): PreparedCreate[][] {
+    const chunks: PreparedCreate[][] = [];
+    let chunk: PreparedCreate[] = [];
+    let chunkColumns = new Set<string>();
 
-      modelStore.delete(pendingId);
-      record.__data.id = realId;
-      record.id = realId;
-      modelStore.set(realId, record);
+    for (const create of prepared) {
+      const columnsWithCreate = new Set([...chunkColumns, ...Object.keys(create.row)]);
 
-      // Update the response data with the real ID
-      if (response?.data) {
-        response.data.id = realId;
+      if (chunk.length > 0 && (chunk.length + 1) * columnsWithCreate.size > POSTGRES_MAX_BIND_PARAMETERS) {
+        chunks.push(chunk);
+        chunk = [];
+        chunkColumns = new Set(Object.keys(create.row));
+      } else {
+        chunkColumns = columnsWithCreate;
       }
 
-      delete record.__data.__pendingSqlId;
+      chunk.push(create);
     }
+
+    if (chunk.length > 0) chunks.push(chunk);
+    return chunks;
+  }
+
+  private async _insertCreates(schema: ModelSchema, creates: PreparedCreate[]): Promise<void> {
+    if (creates.length === 1) return this._insertCreatesOneByOne(schema, creates);
+
+    let result: { rows: Record<string, unknown>[] };
+
+    try {
+      const { sql, values } = this.deps.buildInsertMany(schema.table, creates.map(({ row }) => row));
+      result = await this.requirePool().query(sql, values);
+    } catch {
+      return this._insertCreatesOneByOne(schema, creates);
+    }
+
+    creates.forEach((create, index) => {
+      this._adoptGeneratedId(create, result.rows[index]);
+      create.write.resolve();
+    });
+  }
+
+  private async _insertCreatesOneByOne(schema: ModelSchema, creates: PreparedCreate[]): Promise<void> {
+    for (const create of creates) {
+      try {
+        const { sql, values } = this.deps.buildInsert(schema.table, create.row);
+        const result = await this.requirePool().query(sql, values);
+        this._adoptGeneratedId(create, result.rows[0]);
+        create.write.resolve();
+      } catch (error) {
+        create.write.reject(error);
+      }
+    }
+  }
+
+  private _adoptGeneratedId({ modelName, record, isPendingId, write }: PreparedCreate, returnedRow: Record<string, unknown> | undefined): void {
+    if (!isPendingId || !returnedRow) return;
+
+    const pendingId = record.id;
+    const realId = returnedRow.id as string | number;
+    const modelStore = (this.deps.store as unknown as { get(name: string): Map<unknown, unknown> }).get(modelName);
+
+    modelStore.delete(pendingId);
+    record.__data.id = realId;
+    record.id = realId;
+    modelStore.set(realId, record);
+
+    if (write.response?.data) write.response.data.id = realId;
+
+    delete record.__data.__pendingSqlId;
   }
 
   private async _persistUpdate(modelName: string, context: PersistContext, _response: PersistResponse): Promise<void> {
@@ -568,7 +793,12 @@ export default class PostgresDB {
     // PostgreSQL doesn't have ON UPDATE CURRENT_TIMESTAMP -- set updated_at manually
     changedData.updated_at = new Date();
 
-    const { sql, values } = this.deps.buildUpdate(schema.table, id, changedData);
+    // Same conversion, same choke point as the create path (#292). Columns
+    // appended after the attribute loop (FKs, updated_at) are absent from
+    // schema.columns and pass through untouched.
+    const row = this.deps.convertRow(changedData, schema.columns, this.deps.postgresJsonRule);
+
+    const { sql, values } = this.deps.buildUpdate(schema.table, id, row);
     await this.requirePool().query(sql, values);
   }
 
@@ -594,13 +824,11 @@ export default class PostgresDB {
       row.id = data.id;
     }
 
-    // Attribute columns
-    for (const [col, pgType] of Object.entries(schema.columns)) {
+    // Attribute columns. Type conversion is NOT applied here -- it is applied
+    // once to the finished payload by the caller, via convertRow (#292).
+    for (const col of Object.keys(schema.columns)) {
       if (data[col] !== undefined) {
-        // JSONB columns: stringify non-string values for PostgreSQL JSONB storage
-        row[col] = pgType === 'JSONB' && typeof data[col] !== 'string'
-          ? JSON.stringify(data[col])
-          : data[col];
+        row[col] = data[col];
       }
     }
 

package/src/postgres/query-builder.ts

@@ -37,6 +37,27 @@ export function buildInsert(table: string, data: Record<string, unknown>): Query
   return { sql, values };
 }
 
+export function buildInsertMany(table: string, rows: Record<string, unknown>[]): QueryResult {
+  validateIdentifier(table, 'table name');
+
+  const columns = [...new Set(rows.flatMap(row => Object.keys(row)))];
+  columns.forEach(column => validateIdentifier(column, 'column name'));
+
+  const values: unknown[] = [];
+  const tuples = rows.map(row => {
+    const cells = columns.map(column => {
+      if (!Object.hasOwn(row, column)) return 'DEFAULT';
+      values.push(row[column]);
+      return `$${values.length}`;
+    });
+    return `(${cells.join(', ')})`;
+  });
+
+  const sql = `INSERT INTO "${table}" (${columns.map(column => `"${column}"`).join(', ')}) VALUES ${tuples.join(', ')} RETURNING "id"`;
+
+  return { sql, values };
+}
+
 export function buildUpdate(table: string, id: unknown, data: Record<string, unknown>): QueryResult {
   validateIdentifier(table, 'table name');
 

package/src/record.ts

@@ -87,12 +87,15 @@ export default class Record {
 
     for (const [key, childRecord] of Object.entries(this.__relationships)) {
       if (Array.isArray(childRecord)) {
+        // Filter out cleaned records (those with no __model)
+        const live = childRecord.filter((r: Record) => r?.__model);
+
         // Deduplicate by record ID — keep last occurrence (latest data wins)
         const seen = new Set<unknown>();
         const unique: Record[] = [];
 
-        for (let i = childRecord.length - 1; i >= 0; i--) {
-          const r = childRecord[i] as Record;
+        for (let i = live.length - 1; i >= 0; i--) {
+          const r = live[i] as Record;
           if (!seen.has(r.id)) {
             seen.add(r.id);
             unique.push(r);
@@ -102,7 +105,7 @@ export default class Record {
         unique.reverse();
         records[key] = unique.map((r: Record) => r.serialize());
       } else {
-        records[key] = (childRecord as Record)?.serialize() ?? null;
+        records[key] = (childRecord as Record)?.__model ? (childRecord as Record).serialize() : null;
       }
     }
 
@@ -136,8 +139,8 @@ export default class Record {
       if (fields && !fields.has(key)) continue;
 
       const relationshipData = Array.isArray(childRecord)
-        ? childRecord.map((r: Record) => ({ type: r.__model.__name, id: r.id }))
-        : childRecord ? { type: (childRecord as Record).__model.__name, id: (childRecord as Record).id } : null;
+        ? childRecord.filter((r: Record) => r?.__model).map((r: Record) => ({ type: r.__model.__name, id: r.id }))
+        : (childRecord && (childRecord as Record).__model) ? { type: (childRecord as Record).__model.__name, id: (childRecord as Record).id } : null;
 
       // Dasherize the key for URL paths (e.g., accessLinks -> access-links)
       const dasherizedKey = camelCaseToKebabCase(key);

package/src/relationships.ts

@@ -51,4 +51,4 @@ export function getPendingBelongsToRegistry(): PendingBelongsToMap {
   return relationships.get('pendingBelongsTo') as PendingBelongsToMap;
 }
 
-export const TYPES: string[] = ['global', 'hasMany', 'belongsTo', 'pending'];
+export const TYPES: string[] = ['global', 'hasMany', 'belongsTo', 'pending', 'pendingBelongsTo'];

package/src/serializer.ts

@@ -94,8 +94,45 @@ export default class Serializer {
         const handlerOptions = { ...options, _relationshipKey: key };
         const childRecord = handler(record, data, handlerOptions);
 
-        rec[key] = childRecord;
-        relatedRecords[key] = childRecord;
+        // hasMany relationships use a getter so format()/toJSON() always read
+        // the live registry array instead of a stale snapshot captured at
+        // serialization time.  This is critical when child records are created
+        // in a later async frame — the belongsTo inverse wiring pushes into
+        // the shared registry array, and the getter ensures the parent sees it.
+        const isHasMany = (handler as { __relationshipType?: string }).__relationshipType === 'hasMany';
+
+        if (isHasMany) {
+          // `childRecord` IS the shared registry array — define a getter that
+          // always dereferences through the same array reference.
+          const registryArray = childRecord as unknown[];
+          Object.defineProperty(rec, key, {
+            enumerable: true,
+            configurable: true,
+            get: () => registryArray,
+            set(v: unknown) { relatedRecords[key] = v; }
+          });
+          Object.defineProperty(relatedRecords, key, {
+            enumerable: true,
+            configurable: true,
+            get: () => registryArray,
+            set(v: unknown) { Object.defineProperty(relatedRecords, key, { value: v, writable: true, enumerable: true, configurable: true }); }
+          });
+        } else {
+          rec[key] = childRecord;
+          relatedRecords[key] = childRecord;
+
+          // Preserve the raw FK value in __data when the belongsTo handler
+          // couldn't resolve the target (e.g., memory:false model not loaded).
+          // This allows adapters to read the FK from __data as a fallback
+          // when __relationships[key] is null.  Only store when `data` is a
+          // truthy non-object — i.e., a raw FK string/number that the handler
+          // attempted but failed to resolve.  When `data` is null/undefined
+          // (optional empty relationship) we intentionally skip to preserve
+          // the existing behavior of not populating __data for empty FKs.
+          if (childRecord === null && data && typeof data !== 'object') {
+            parsedData[key] = data;
+          }
+        }
 
         continue;
       }