@stonyx/orm 0.3.2-beta.33 → 0.3.2-beta.330

package/src/mysql/mysql-db.ts

@@ -4,7 +4,7 @@ import { ensureMigrationsTable, getAppliedMigrations, getMigrationFiles, applyMi
 import { introspectModels, introspectViews, getTopologicalOrder, schemasToSnapshot } from './schema-introspector.js';
 import type { ModelSchema, ViewSchema, SnapshotEntry } from './schema-introspector.js';
 import { loadLatestSnapshot, detectSchemaDrift } from './migration-generator.js';
-import { buildInsert, buildUpdate, buildDelete, buildSelect } from './query-builder.js';
+import { buildInsert, buildInsertMany, buildUpdate, buildDelete, buildSelect } from './query-builder.js';
 import { store } from '@stonyx/orm';
 import { createRecord } from '../manage-record.js';
 import { confirm } from '@stonyx/utils/prompt';
@@ -32,6 +32,43 @@ interface ExecuteResult {
   insertId?: number;
 }
 
+interface QueuedWrite {
+  operation: string;
+  modelName: string;
+  context: PersistContext;
+  response: PersistResponse;
+  resolve: () => void;
+  reject: (error: unknown) => void;
+}
+
+interface PreparedCreate {
+  write: QueuedWrite;
+  modelName: string;
+  record: OrmRecord;
+  row: Record<string, unknown>;
+  isPendingId: boolean;
+}
+
+type ExitEmitter = Pick<NodeJS.Process, 'once' | 'removeListener'>;
+
+const MYSQL_MAX_PLACEHOLDERS = 65535;
+const DEFAULT_WRITE_BATCH_SIZE = 500;
+const DEFAULT_WRITE_QUEUE_LIMIT = 10000;
+
+export class WriteQueueFullError extends Error {
+  constructor(limit: number) {
+    super(`MysqlDB write queue is full (${limit} queued writes, writeQueueLimit); the write was rejected`);
+    this.name = 'WriteQueueFullError';
+  }
+}
+
+export class WriteQueueClosedError extends Error {
+  constructor() {
+    super('MysqlDB is shutting down; the write was rejected');
+    this.name = 'WriteQueueClosedError';
+  }
+}
+
 interface OrmStore {
   get(key: string): Map<number | string, unknown> | undefined;
   get(key: string, id: number | string): unknown;
@@ -54,6 +91,7 @@ interface MysqlDBDeps {
   loadLatestSnapshot: typeof loadLatestSnapshot;
   detectSchemaDrift: typeof detectSchemaDrift;
   buildInsert: typeof buildInsert;
+  buildInsertMany: typeof buildInsertMany;
   buildUpdate: typeof buildUpdate;
   buildDelete: typeof buildDelete;
   buildSelect: typeof buildSelect;
@@ -65,6 +103,7 @@ interface MysqlDBDeps {
   config: typeof config;
   log: Record<string, ((...args: unknown[]) => void) | undefined>;
   path: typeof path;
+  process: ExitEmitter;
 }
 
 const defaultDeps: MysqlDBDeps = {
@@ -72,9 +111,9 @@ const defaultDeps: MysqlDBDeps = {
   getMigrationFiles, applyMigration, parseMigrationFile,
   introspectModels, introspectViews, getTopologicalOrder, schemasToSnapshot,
   loadLatestSnapshot, detectSchemaDrift,
-  buildInsert, buildUpdate, buildDelete, buildSelect,
+  buildInsert, buildInsertMany, buildUpdate, buildDelete, buildSelect,
   createRecord, store: store as OrmStore, confirm, readFile, getPluralName,
-  config, log, path
+  config, log, path, process
 };
 
 export default class MysqlDB {
@@ -84,6 +123,13 @@ export default class MysqlDB {
   pool!: Pool | null;
   mysqlConfig!: MysqlConfig;
 
+  readonly persistResolvesAfterWrite = true;
+  private _queuedWrites: QueuedWrite[] = [];
+  private _inFlightWrites = new Set<QueuedWrite>();
+  private _drainingWrites: Promise<void> | null = null;
+  private _acceptingWrites = true;
+  private _exitReport: (() => void) | null = null;
+
   constructor(deps: Partial<MysqlDBDeps> = {}) {
     if (MysqlDB.instance) return MysqlDB.instance;
     MysqlDB.instance = this;
@@ -104,6 +150,20 @@ export default class MysqlDB {
     this.pool = await this.deps.getPool(this.mysqlConfig);
     await this.deps.ensureMigrationsTable(this.pool, this.mysqlConfig.migrationsTable);
     await this.loadMemoryRecords();
+    this._acceptingWrites = true;
+    this._registerExitReport();
+  }
+
+  get writeBatchSize(): number {
+    return Number(this.mysqlConfig?.writeBatchSize) || DEFAULT_WRITE_BATCH_SIZE;
+  }
+
+  get writeQueueLimit(): number {
+    return Number(this.mysqlConfig?.writeQueueLimit) || DEFAULT_WRITE_QUEUE_LIMIT;
+  }
+
+  get pendingWriteCount(): number {
+    return this._inFlightWrites.size + this._queuedWrites.length;
   }
 
   async startup(): Promise<void> {
@@ -118,7 +178,15 @@ export default class MysqlDB {
     if (pending.length > 0) {
       this.deps.log.db?.(`${pending.length} pending migration(s) found.`);
 
-      const shouldApply = await this.deps.confirm(`${pending.length} pending migration(s) found. Apply now?`);
+      let shouldApply: boolean;
+      if (this.mysqlConfig.autoMigrate === true) {
+        shouldApply = true;
+      } else if (this.mysqlConfig.autoMigrate === false) {
+        shouldApply = false;
+        this.deps.log.warn?.(`autoMigrate is false — skipping ${pending.length} pending migration(s).`);
+      } else {
+        shouldApply = await this.deps.confirm(`${pending.length} pending migration(s) found. Apply now?`);
+      }
 
       if (shouldApply) {
         for (const filename of pending) {
@@ -139,9 +207,17 @@ export default class MysqlDB {
       const modelCount = Object.keys(schemas).length;
 
       if (modelCount > 0) {
-        const shouldGenerate = await this.deps.confirm(
-          `No migrations found but ${modelCount} model(s) detected. Generate and apply initial migration?`
-        );
+        let shouldGenerate: boolean;
+        if (this.mysqlConfig.autoMigrate === true) {
+          shouldGenerate = true;
+        } else if (this.mysqlConfig.autoMigrate === false) {
+          shouldGenerate = false;
+          this.deps.log.warn?.(`autoMigrate is false — skipping initial migration generation for ${modelCount} model(s).`);
+        } else {
+          shouldGenerate = await this.deps.confirm(
+            `No migrations found but ${modelCount} model(s) detected. Generate and apply initial migration?`
+          );
+        }
 
         if (shouldGenerate) {
           const { generateMigration } = await import('./migration-generator.js');
@@ -175,10 +251,36 @@ export default class MysqlDB {
   }
 
   async shutdown(): Promise<void> {
+    this._acceptingWrites = false;
+    while (this._drainingWrites) await this._drainingWrites;
+    this._unregisterExitReport();
     await this.deps.closePool();
     this.pool = null;
   }
 
+  private _registerExitReport(): void {
+    if (this._exitReport) return;
+    this._exitReport = () => this._reportUnpersistedWrites();
+    this.deps.process.once('exit', this._exitReport);
+  }
+
+  private _unregisterExitReport(): void {
+    if (!this._exitReport) return;
+    this.deps.process.removeListener('exit', this._exitReport);
+    this._exitReport = null;
+  }
+
+  private _reportUnpersistedWrites(): void {
+    const lostWrites = [...this._inFlightWrites, ...this._queuedWrites];
+    if (lostWrites.length === 0) return;
+
+    const lostByModel = new Map<string, number>();
+    for (const { modelName } of lostWrites) lostByModel.set(modelName, (lostByModel.get(modelName) ?? 0) + 1);
+    const breakdown = [...lostByModel].map(([modelName, count]) => `${modelName}: ${count}`).join(', ');
+
+    this.deps.log.error?.(`[MysqlDB] Process exiting with ${lostWrites.length} queued write(s) not persisted; they are lost (${breakdown})`);
+  }
+
   async save(): Promise<void> {
     // No-op: MySQL persists data immediately via persist()
   }
@@ -394,69 +496,185 @@ export default class MysqlDB {
   }
 
   async persist(operation: string, modelName: string, context: PersistContext, response: PersistResponse): Promise<void> {
-    // Views are read-only — no-op for all write operations
     const Orm = (await import('@stonyx/orm')).default;
     if ((Orm as unknown as { instance?: { isView?: (name: string) => boolean } }).instance?.isView?.(modelName)) return;
 
-    switch (operation) {
-      case 'create':
-        return this._persistCreate(modelName, context, response);
-      case 'update':
-        return this._persistUpdate(modelName, context, response);
-      case 'delete':
-        return this._persistDelete(modelName, context);
+    if (!this._acceptingWrites) throw new WriteQueueClosedError();
+    if (this._queuedWrites.length >= this.writeQueueLimit) throw new WriteQueueFullError(this.writeQueueLimit);
+
+    return new Promise<void>((resolve, reject) => {
+      const write: QueuedWrite = {
+        operation, modelName, context, response,
+        resolve: () => {
+          this._inFlightWrites.delete(write);
+          resolve();
+        },
+        reject: (error: unknown) => {
+          this._inFlightWrites.delete(write);
+          reject(error);
+        },
+      };
+      this._queuedWrites.push(write);
+      this._drainWritesInBackground();
+    });
+  }
+
+  private _drainWritesInBackground(): void {
+    if (this._drainingWrites) return;
+    this._drainingWrites = this._drainQueuedWrites().finally(() => {
+      this._drainingWrites = null;
+    });
+  }
+
+  private async _drainQueuedWrites(): Promise<void> {
+    while (this._queuedWrites.length > 0) {
+      const writes = this._takeNextWriteGroup();
+      writes.forEach(write => this._inFlightWrites.add(write));
+      try {
+        await this._executeWriteGroup(writes);
+      } catch (error) {
+        writes.filter(write => this._inFlightWrites.has(write)).forEach(write => write.reject(error));
+      }
     }
   }
 
-  private async _persistCreate(modelName: string, context: PersistContext, response: PersistResponse): Promise<void> {
-    const schemas = this.deps.introspectModels();
-    const schema = schemas[modelName];
+  private _takeNextWriteGroup(): QueuedWrite[] {
+    const [first] = this._queuedWrites;
+    let groupSize = 1;
+
+    if (first.operation === 'create') {
+      while (
+        groupSize < this.writeBatchSize
+        && groupSize < this._queuedWrites.length
+        && this._queuedWrites[groupSize].operation === 'create'
+        && this._queuedWrites[groupSize].modelName === first.modelName
+      ) groupSize++;
+    }
 
-    if (!schema) return;
+    return this._queuedWrites.splice(0, groupSize);
+  }
+
+  private async _executeWriteGroup(writes: QueuedWrite[]): Promise<void> {
+    const [first] = writes;
+    if (first.operation === 'create') return this._persistCreates(first.modelName, writes);
 
+    try {
+      if (first.operation === 'update') await this._persistUpdate(first.modelName, first.context, first.response);
+      if (first.operation === 'delete') await this._persistDelete(first.modelName, first.context);
+      first.resolve();
+    } catch (error) {
+      first.reject(error);
+    }
+  }
+
+  private async _persistCreates(modelName: string, writes: QueuedWrite[]): Promise<void> {
+    const schema = this.deps.introspectModels()[modelName];
+    const prepared: PreparedCreate[] = [];
+
+    for (const write of writes) {
+      try {
+        const create = schema ? this._prepareCreate(modelName, schema, write) : null;
+        if (create) prepared.push(create);
+        else write.resolve();
+      } catch (error) {
+        write.reject(error);
+      }
+    }
+
+    for (const chunk of this._chunkForSingleInsert(prepared)) {
+      await this._insertCreates(schema, chunk);
+    }
+  }
+
+  private _prepareCreate(modelName: string, schema: ModelSchema, write: QueuedWrite): PreparedCreate | null {
+    const { context, response } = write;
     const recordId = response?.data?.id;
     const record = recordId != null ? this.deps.store.get(modelName, (isNaN(recordId as number) ? recordId : parseInt(recordId as string)) as number | string) as OrmRecord | null : null;
 
-    if (!record) return;
-
-    const insertData = this._recordToRow(record, schema);
+    if (!record) return null;
 
-    // For auto-increment models, remove the pending ID.
-    // Check context.rawData (not record.__data) because __pendingSqlId is not a model
-    // attribute and gets lost during serialization.
+    const row = this._recordToRow(record, schema);
     const isPendingId = context.rawData?.__pendingSqlId === true;
 
-    if (isPendingId) {
-      delete insertData.id;
-    } else if (insertData.id !== undefined) {
-      // Keep user-provided ID (string IDs or explicit numeric IDs)
+    if (isPendingId) delete row.id;
+
+    return { write, modelName, record, row, isPendingId };
+  }
+
+  private _chunkForSingleInsert(prepared: PreparedCreate[]): PreparedCreate[][] {
+    const chunks: PreparedCreate[][] = [];
+    let chunk: PreparedCreate[] = [];
+    let chunkColumns = new Set<string>();
+
+    for (const create of prepared) {
+      const columnsWithCreate = new Set([...chunkColumns, ...Object.keys(create.row)]);
+      const mixesIdSources = chunk.length > 0 && chunk[0].isPendingId !== create.isPendingId;
+      const exceedsPlaceholders = (chunk.length + 1) * columnsWithCreate.size > MYSQL_MAX_PLACEHOLDERS;
+
+      if (chunk.length > 0 && (mixesIdSources || exceedsPlaceholders)) {
+        chunks.push(chunk);
+        chunk = [];
+        chunkColumns = new Set(Object.keys(create.row));
+      } else {
+        chunkColumns = columnsWithCreate;
+      }
+
+      chunk.push(create);
     }
 
-    const { sql, values } = this.deps.buildInsert(schema.table, insertData);
+    if (chunk.length > 0) chunks.push(chunk);
+    return chunks;
+  }
 
-    const [result] = await this.requirePool().execute(sql, values) as [ExecuteResult, unknown];
+  private async _insertCreates(schema: ModelSchema, creates: PreparedCreate[]): Promise<void> {
+    if (creates.length === 1) return this._insertCreatesOneByOne(schema, creates);
 
-    // Re-key the record in the store if MySQL generated the ID
-    if (isPendingId && result.insertId) {
-      const pendingId = record.id;
-      const realId = result.insertId;
-      const modelStore = this.deps.store.get(modelName);
-      if (!modelStore) throw new Error(`Model "${modelName}" not found in store during ID re-key`);
+    let firstInsertId: number | undefined;
 
-      modelStore.delete(pendingId as number | string);
-      record.__data.id = realId;
-      record.id = realId;
-      modelStore.set(realId as number | string, record);
+    try {
+      const { sql, values } = this.deps.buildInsertMany(schema.table, creates.map(({ row }) => row));
+      const [result] = await this.requirePool().execute(sql, values) as [ExecuteResult, unknown];
+      firstInsertId = result.insertId;
+    } catch {
+      return this._insertCreatesOneByOne(schema, creates);
+    }
 
-      // Update the response data with the real ID
-      if (response?.data) {
-        response.data.id = realId;
-      }
+    creates.forEach((create, index) => {
+      if (firstInsertId) this._adoptGeneratedId(create, firstInsertId + index);
+      create.write.resolve();
+    });
+  }
 
-      delete record.__data.__pendingSqlId;
+  private async _insertCreatesOneByOne(schema: ModelSchema, creates: PreparedCreate[]): Promise<void> {
+    for (const create of creates) {
+      try {
+        const { sql, values } = this.deps.buildInsert(schema.table, create.row);
+        const [result] = await this.requirePool().execute(sql, values) as [ExecuteResult, unknown];
+        if (result.insertId) this._adoptGeneratedId(create, result.insertId);
+        create.write.resolve();
+      } catch (error) {
+        create.write.reject(error);
+      }
     }
   }
 
+  private _adoptGeneratedId({ modelName, record, isPendingId, write }: PreparedCreate, realId: number): void {
+    if (!isPendingId) return;
+
+    const pendingId = record.id;
+    const modelStore = this.deps.store.get(modelName);
+    if (!modelStore) throw new Error(`Model "${modelName}" not found in store during ID re-key`);
+
+    modelStore.delete(pendingId as number | string);
+    record.__data.id = realId;
+    record.id = realId;
+    modelStore.set(realId, record);
+
+    if (write.response?.data) write.response.data.id = realId;
+
+    delete record.__data.__pendingSqlId;
+  }
+
   private async _persistUpdate(modelName: string, context: PersistContext, response: PersistResponse): Promise<void> {
     const schemas = this.deps.introspectModels();
     const schema = schemas[modelName];

package/src/mysql/query-builder.ts

@@ -27,6 +27,27 @@ export function buildInsert(table: string, data: Record<string, unknown>): Query
   return { sql, values };
 }
 
+export function buildInsertMany(table: string, rows: Record<string, unknown>[]): QueryResult {
+  validateIdentifier(table, 'table name');
+
+  const columns = [...new Set(rows.flatMap(row => Object.keys(row)))];
+  columns.forEach(column => validateIdentifier(column, 'column name'));
+
+  const values: unknown[] = [];
+  const tuples = rows.map(row => {
+    const cells = columns.map(column => {
+      if (!Object.hasOwn(row, column)) return 'DEFAULT';
+      values.push(row[column]);
+      return '?';
+    });
+    return `(${cells.join(', ')})`;
+  });
+
+  const sql = `INSERT INTO \`${table}\` (${columns.map(column => `\`${column}\``).join(', ')}) VALUES ${tuples.join(', ')}`;
+
+  return { sql, values };
+}
+
 export function buildUpdate(table: string, id: unknown, data: Record<string, unknown>): QueryResult {
   validateIdentifier(table, 'table name');
 

package/src/normalize-record-id.ts

@@ -0,0 +1,88 @@
+/*
+ * Copyright 2025 Stone Costa
+ *
+ * Licensed under the Apache License, Version 2.0 (the 'License');
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an 'AS IS' BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * The ONE place a URL id is turned into the value a record is resolved by —
+ * abofs/stonyx-orm#270.
+ *
+ * Before this existed this coercion was written out seven times: twice in
+ * README.md, once in docs/usage-patterns.md, and four times inside the
+ * framework. Seven copies, not seven identical copies — the three persistence
+ * ones (abofs/stonyx-orm#282) omit the `if (!id) return ''` guard below, which
+ * is the point: nothing held them together, so they had already drifted.
+ * The framework's copy — `getId()`, module-private in
+ * orm-request.ts, unreachable through the package `exports` map — was the one
+ * that decided which record a request addressed, and a consumer's `access()`
+ * predicate had no way to obtain it. So the framework resolved the record by
+ * one value and asked the consumer to authorize on a different one.
+ *
+ * Measured consequence at the time of filing: with the documented predicate
+ * applied verbatim to a numeric-id model, `GET /animals/007`, `/7.9`, `/7e0`,
+ * `/0x7`, `/%207`, `/%2B7` and `/7%0A` each served the protected record, and
+ * `DELETE /animals/007` destroyed it — because `parseInt` folds every one of
+ * those onto `7` while `'007' === '7'` is false.
+ *
+ * Two things follow from this being a single exported function, and both are
+ * the point:
+ *
+ *  1. A permissive change here is HARMLESS, because both sides move together.
+ *     Lowercasing string ids used to disclose and destroy `owner:angela` with
+ *     the whole suite green; with one implementation the predicate simply sees
+ *     the same lowercased value and still refuses.
+ *  2. A divergence — a second, private normaliser at the resolution site — is
+ *     what is now dangerous, and that is what the tests pin, by observing the
+ *     key the resolution path actually used.
+ *
+ * SEMANTICS ARE UNCHANGED FROM `getId()`, deliberately and byte-for-byte.
+ * Whether `7.9` / `0x7` / `%0A` *should* resolve to record 7 at all is a real
+ * question, and it is a behaviour change on the record-resolution path for
+ * every consumer rather than an authorization fix — it belongs in its own
+ * issue with its own compatibility argument (issue body scope item 4). #270
+ * preserves today's semantics exactly and makes both sides agree on them.
+ *
+ * Two details are load-bearing and are pinned by
+ * test/unit/normalize-record-id-test.ts:
+ *
+ *  - `parseInt` is called with NO RADIX. `parseInt('0x7')` is 7;
+ *    `parseInt('0x7', 10)` is 0. Adding the radix looks like a cleanup and
+ *    silently changes which record every hex-spelled URL addresses.
+ *  - The coercion applies only when the id LOOKS numeric, so a model with
+ *    string ids is passed through untouched, case included.
+ *
+ * Must stay synchronous: `auth()` is invoked without `await` by
+ * @stonyx/rest-server, so a promise here would be handed to `access()` as the
+ * record id.
+ *
+ * FALSY AND `NaN` RETURNS ARE LOAD-BEARING ELSEWHERE — abofs/stonyx-orm#287.
+ * `''` is returned for any falsy id because `store.get(key, undefined)` returns
+ * the whole model Map rather than a record (abofs/stonyx-orm#167). But `''` is
+ * not the only falsy return: `'0'`, `'00'`, `'-0'` and `'0x0'` all normalise to
+ * `0`, and `' '`, `'\t'`, `'\n'`, `'\u00a0'` all normalise to `NaN` (their
+ * `Number()` is `0`, so the `isNaN` guard does not fire and `parseInt` runs).
+ * `store.remove(key, id)` branches on truthiness, so those spellings reach a
+ * fall-through this function does not own. Tracked as #287; every row is pinned
+ * in test/unit/normalize-record-id-test.ts so a cleanup here cannot move the
+ * boundary #287 is measured against.
+ *
+ * @param id the raw, already-URL-decoded id text from `request.params.id`
+ * @returns the value the ORM resolves the record by
+ */
+export default function normalizeRecordId(id?: string | null): string | number {
+  if (!id) return '';
+  if (isNaN(id as unknown as number)) return id;
+
+  return parseInt(id);
+}