@lenne.tech/nest-server 12.1.0 → 12.2.0

package/src/core/common/persistence/sql/mysql/mysql-data-store.ts

@@ -2,8 +2,10 @@ import { BadRequestException, Logger } from '@nestjs/common';
 
 import type {
   DataStoreUpdate,
+  DeleteOptions,
   FindOptions,
   PushOptions,
+  ReadOptions,
   StoreCapabilities,
   StoredRecord,
   UpdateOptions,
@@ -21,7 +23,9 @@ import {
   assertSelectFields,
 } from '../../store-guards.js';
 import { castFilterValue } from '../sql-cast.js';
-import { POSTGRES_STORE_CAPABILITIES, SqlDataStore, type SqlDataStoreOptions } from '../sql-data-store.js';
+import { POSTGRES_STORE_CAPABILITIES, SqlDataStore, type SqlDataStoreOptions, type SqlRun } from '../sql-data-store.js';
+import { tableName } from '../sql-schema.shared.js';
+import { MYSQL_SYNC_COUNTERS } from './mysql-schema.compiler.js';
 import { DuplicateKeyError, UnsupportedQueryError } from '../sql-errors.js';
 import type { SqlQueryResult, SqlStatement } from '../sql-executor.interface.js';
 import { columnName, quoteMysqlIdentifier, SqlParams } from '../sql-naming.js';
@@ -71,7 +75,6 @@ type Run = <R = Record<string, any>>(statement: SqlStatement) => Promise<SqlQuer
  * gives MongoDB's findOneAndUpdate guarantee without a retry.
  */
 export class MysqlDataStore<T = Record<string, any>> extends SqlDataStore<T> {
-  override readonly capabilities = MYSQL_STORE_CAPABILITIES;
   override readonly driver: string = 'mysql';
   protected override readonly logger = new Logger(MysqlDataStore.name);
   private readonly variant: MysqlVariant;
@@ -82,13 +85,44 @@ export class MysqlDataStore<T = Record<string, any>> extends SqlDataStore<T> {
     this.driver = executor.variant;
   }
 
+  protected override baseCapabilities(): Readonly<StoreCapabilities> {
+    return MYSQL_STORE_CAPABILITIES;
+  }
+
+  /**
+   * Every store write of a synchronized model takes its change number last: `FOR UPDATE` locks index gaps here, and an
+   * insert waits for them after its BEFORE INSERT trigger ran — a trigger holding the counter meanwhile would deadlock
+   * with a multi-row write that locked those gaps and now asks for the counter (measured, DEV-3900 review)
+   */
+  protected override numbersLast(): boolean {
+    return true;
+  }
+
+  /** A single-column write (pullFromArray, a one-stage push) — in a transaction, so a synchronized row is numbered */
+  protected override async updateColumn(id: string, column: string, value: string, params: SqlParams): Promise<void> {
+    if (!this.model.sync) {
+      return super.updateColumn(id, column, value, params);
+    }
+    const { $setOnInsert: _insertOnly, ...stamped } = await this.beforeUpdate({});
+    const assignments = [`${column} = ${value}`, ...this.compileUpdate(stamped, params)];
+    await this.transaction(async (run) => {
+      const { rowCount } = await run({
+        params: params.values,
+        text: `UPDATE ${this.table} SET ${assignments.join(', ')} WHERE ${this.where({ id } as Where<T>, params)}`,
+      });
+      if (rowCount) {
+        await this.numberPending(run, [id]);
+      }
+    });
+  }
+
   // -- reads ----------------------------------------------------------------------------------------------
 
-  override async count(where?: Where<T>): Promise<number> {
+  override async count(where?: Where<T>, options: ReadOptions = {}): Promise<number> {
     const params = new SqlParams();
     const { rows } = await this.run<{ n: number | string }>({
       params: params.values,
-      text: `SELECT COUNT(*) AS n FROM ${this.table} WHERE ${this.where(where, params)}`,
+      text: `SELECT COUNT(*) AS n FROM ${this.table} WHERE ${this.where(where, params, options)}`,
     });
     return Number(rows[0].n);
   }
@@ -109,10 +143,10 @@ export class MysqlDataStore<T = Record<string, any>> extends SqlDataStore<T> {
     return { count: Number(rows[0].n), sum: Number(rows[0].total) };
   }
 
-  override async distinct(field: string, where?: Where<T>): Promise<unknown[]> {
+  override async distinct(field: string, where?: Where<T>, options: ReadOptions = {}): Promise<unknown[]> {
     const params = new SqlParams();
     const path = compileMysqlPath(this.variant, this.model, field, params);
-    const condition = this.where(where, params);
+    const condition = this.where(where, params, options);
     if (path.kind === 'array') {
       const { rows } = await this.run<{ v: unknown }>({
         params: params.values,
@@ -133,7 +167,7 @@ export class MysqlDataStore<T = Record<string, any>> extends SqlDataStore<T> {
   override async find(where?: Where<T>, options: FindOptions = {}): Promise<StoredRecord<T>[]> {
     const sample = options.sample === undefined ? undefined : assertSampleSize(options.sample);
     const params = new SqlParams();
-    let text = `SELECT ${this.selectList(options.select, options.exclude)} FROM ${this.table} WHERE ${this.where(where, params)}`;
+    let text = `SELECT ${this.selectList(options.select, options.exclude)} FROM ${this.table} WHERE ${this.where(where, params, options)}`;
     text += this.orderBy(options, params);
     if (options.limit !== undefined) {
       text += ` LIMIT ${params.add(assertNonNegativeInteger(options.limit, 'limit'))}`;
@@ -153,14 +187,17 @@ export class MysqlDataStore<T = Record<string, any>> extends SqlDataStore<T> {
 
   override async insert(record: Partial<T>): Promise<StoredRecord<T>> {
     const doc = await this.beforeInsert({ ...(record as Record<string, any>) });
-    const params = new SqlParams();
-    const { columns, id, values } = this.insertValues(doc, params);
-    return this.transaction(async (run) => {
-      await run({
-        params: params.values,
-        text: `INSERT INTO ${this.table} (${columns.join(', ')}) VALUES (${values.join(', ')})`,
+    return this.withServerId(doc, (prepared) => {
+      const params = new SqlParams();
+      const { columns, id, values } = this.insertValues(prepared, params);
+      return this.transaction(async (run) => {
+        await run({
+          params: params.values,
+          text: `INSERT INTO ${this.table} (${columns.join(', ')}) VALUES (${values.join(', ')})`,
+        });
+        await this.numberPending(run, [id]);
+        return (await this.readById(run, id))!;
       });
-      return (await this.readById(run, id))!;
     });
   }
 
@@ -173,28 +210,36 @@ export class MysqlDataStore<T = Record<string, any>> extends SqlDataStore<T> {
     const params = new SqlParams();
     const assignments = this.compileUpdate(patch, params);
     if (!assignments.length) {
-      return this.findById(id);
+      return this.findById(id, options);
     }
-    const condition = this.where({ id } as Where<T>, params);
+    const condition = this.where({ id } as Where<T>, params, options);
     return this.transaction(async (run) => {
       const { rowCount } = await run({
         params: params.values,
         text: `UPDATE ${this.table} SET ${assignments.join(', ')} WHERE ${condition}`,
       });
       // The driver counts MATCHED rows (CLIENT_FOUND_ROWS), so an update that changes nothing still finds its record
-      return rowCount ? this.readById(run, id) : null;
+      if (!rowCount) {
+        return null;
+      }
+      await this.numberPending(run, [id]);
+      return this.readById(run, id);
     });
   }
 
-  override async updateOne(where: Where<T>, update: DataStoreUpdate<T>): Promise<null | StoredRecord<T>> {
-    const patch = await this.preparePatch(update);
+  override async updateOne(
+    where: Where<T>,
+    update: DataStoreUpdate<T>,
+    options: UpdateOptions = {},
+  ): Promise<null | StoredRecord<T>> {
+    const patch = await this.preparePatch(update, options);
     const params = new SqlParams();
     const assignments = this.compileUpdate(patch, params);
     if (!assignments.length) {
-      return this.findOne(where);
+      return this.findOne(where, options);
     }
     return this.transaction(async (run) => {
-      const id = await this.lockFirst(run, where);
+      const id = await this.lockFirst(run, where, options);
       if (id === undefined) {
         return null;
       }
@@ -202,16 +247,19 @@ export class MysqlDataStore<T = Record<string, any>> extends SqlDataStore<T> {
         params: params.values,
         text: `UPDATE ${this.table} SET ${assignments.join(', ')} WHERE \`id\` = ${params.add(id)}`,
       });
+      await this.numberPending(run, [id]);
       return this.readById(run, id);
     });
   }
 
-  override async deleteById(id: string): Promise<null | StoredRecord<T>> {
-    return this.deleteFirst({ id } as Where<T>);
+  override async deleteById(id: string, options: DeleteOptions = {}): Promise<null | StoredRecord<T>> {
+    return this.model.sync && !options.hardDelete
+      ? super.deleteById(id, options)
+      : this.deleteFirst({ id } as Where<T>, options);
   }
 
-  override async deleteOne(where: Where<T>): Promise<null | StoredRecord<T>> {
-    return this.deleteFirst(where);
+  override async deleteOne(where: Where<T>, options: DeleteOptions = {}): Promise<null | StoredRecord<T>> {
+    return this.model.sync && !options.hardDelete ? super.deleteOne(where, options) : this.deleteFirst(where, options);
   }
 
   override async pushToArray(
@@ -307,6 +355,7 @@ export class MysqlDataStore<T = Record<string, any>> extends SqlDataStore<T> {
           return;
         }
       }
+      await this.numberPending(run, [id]);
     });
   }
 
@@ -418,6 +467,30 @@ export class MysqlDataStore<T = Record<string, any>> extends SqlDataStore<T> {
     return super.fromColumn(field, value);
   }
 
+  /**
+   * The counter row of this table (plan §3.4, C1d — MySQL and MariaDB update it in place, so a long snapshot costs no
+   * versions): raised by `count` at once, its row lock held until the transaction ends
+   */
+  protected override async takeChangeNumbers(run: SqlRun, count: number): Promise<string[]> {
+    const params = new SqlParams();
+    const key = tableName(this.model);
+    await run({
+      params: params.values,
+      text: `UPDATE ${this.quote(MYSQL_SYNC_COUNTERS)} SET \`seq\` = \`seq\` + ${params.add(count)} WHERE \`model\` = ${params.add(key)}`,
+    });
+    const read = new SqlParams();
+    const { rows } = await run<{ seq: number | string }>({
+      params: read.values,
+      text: `SELECT \`seq\` FROM ${this.quote(MYSQL_SYNC_COUNTERS)} WHERE \`model\` = ${read.add(key)}`,
+    });
+    const last = BigInt(rows[0].seq);
+    return Array.from({ length: count }, (_, index) => (last - BigInt(count - 1 - index)).toString());
+  }
+
+  protected override bigintParam(params: SqlParams, value: string): string {
+    return params.add(value);
+  }
+
   /**
    * MySQL / MariaDB errors as the answers the MongoDB path gives for the same situation. Matched by error NUMBER and
    * server: the driver names numbers from MySQL's list, and MariaDB's 4025 (a failed CHECK) is another error there
@@ -425,7 +498,8 @@ export class MysqlDataStore<T = Record<string, any>> extends SqlDataStore<T> {
   protected override mapError(error: unknown): unknown {
     const errno = Number((error as { errno?: unknown })?.errno);
     if (errno === MYSQL_ERRORS.duplicateEntry) {
-      return new DuplicateKeyError(this.modelName);
+      const message = String((error as { message?: unknown })?.message ?? '');
+      return new DuplicateKeyError(this.modelName, /for key '(?:[^']*\.)?PRIMARY'/.test(message));
     }
     if (errno === MYSQL_ERRORS.dataTooLong) {
       return new BadRequestException(`${this.modelName}: a value is longer than its column allows`);
@@ -460,21 +534,25 @@ export class MysqlDataStore<T = Record<string, any>> extends SqlDataStore<T> {
   }
 
   /** The id of the first match, its row locked until the transaction ends */
-  private async lockFirst(run: Run, where: Where<T>): Promise<string | undefined> {
+  private async lockFirst(
+    run: Run,
+    where: Where<T>,
+    options: { includeTombstones?: boolean } = {},
+  ): Promise<string | undefined> {
     const params = new SqlParams();
     const { rows } = await run<{ id: string }>({
       params: params.values,
-      text: `SELECT \`id\` FROM ${this.table} WHERE ${this.where(where, params)} LIMIT 1 FOR UPDATE`,
+      text: `SELECT \`id\` FROM ${this.table} WHERE ${this.where(where, params, options)} LIMIT 1 FOR UPDATE`,
     });
     return rows[0]?.id;
   }
 
-  private async deleteFirst(where: Where<T>): Promise<null | StoredRecord<T>> {
+  private async deleteFirst(where: Where<T>, options: DeleteOptions = {}): Promise<null | StoredRecord<T>> {
     return this.transaction(async (run) => {
       const params = new SqlParams();
       const { rows } = await run({
         params: params.values,
-        text: `SELECT ${this.selectList()} FROM ${this.table} WHERE ${this.where(where, params)} LIMIT 1 FOR UPDATE`,
+        text: `SELECT ${this.selectList()} FROM ${this.table} WHERE ${this.where(where, params, options)} LIMIT 1 FOR UPDATE`,
       });
       if (!rows.length) {
         return null;

package/src/core/common/persistence/sql/mysql/mysql-schema.compiler.ts

@@ -2,6 +2,7 @@ import type { FieldDescriptor, IndexDescriptor, ModelDescriptor } from '../../mo
 import { UnsupportedQueryError } from '../sql-errors.js';
 import { columnName, quoteMysqlIdentifier } from '../sql-naming.js';
 import { schemaHash, shortName, tableName } from '../sql-schema.shared.js';
+import { SYNC_DELETED_FIELD, SYNC_SEQUENCE_FIELD } from '../../sync-model.js';
 import type { MysqlVariant } from './mysql-session.js';
 
 /**
@@ -48,6 +49,12 @@ export interface MysqlDesiredCheck {
   name: string;
 }
 
+/** The triggers that number a synchronized model's changes (offline-sync plan §3.4) */
+export interface MysqlDesiredSync {
+  insertTrigger: string;
+  updateTrigger: string;
+}
+
 export interface MysqlDesiredTable {
   checks: MysqlDesiredCheck[];
   columns: MysqlDesiredColumn[];
@@ -56,14 +63,29 @@ export interface MysqlDesiredTable {
   name: string;
   /** What the model declares but the table cannot carry, reported once */
   notices: string[];
+  /** Present for a synchronized model: its change-order triggers */
+  sync?: MysqlDesiredSync;
 }
 
+/** The counter rows of every synchronized table (offline-sync plan §3.4, C1d): one row per table, updated in place */
+export const MYSQL_SYNC_COUNTERS = 'lt_sync_counters';
+/**
+ * Bumped whenever the change-order triggers change: their names carry the version, so the migrator finds an outdated
+ * trigger by name and replaces it
+ */
+export const MYSQL_SYNC_TRIGGER_VERSION = 'v1';
+
 /** Text types whose collation is `utf8mb4_bin` */
 const TEXT_TYPES = new Set(['char(36)', 'longtext', 'varchar(255)']);
 /** Kinds stored as strings — the null sentinel of a unique index is a string */
 const STRING_KINDS = new Set(['enum', 'id', 'ref', 'string', 'uuid']);
 /** Prefix of the virtual columns that carry a unique index's null sentinel */
 export const NULL_SENTINEL_PREFIX = 'lt_nn_';
+/**
+ * Prefix of the virtual columns a synchronized model's unique index reads: the value while the record is live, NULL
+ * once it is a tombstone — MySQL and MariaDB have no partial index, and NULL may repeat in a unique one (plan §3.6)
+ */
+export const LIVE_KEY_PREFIX = 'lt_lk_';
 /** InnoDB's limit for the columns of one index, in bytes (utf8mb4: four per character) */
 const MAX_INDEX_BYTES = 3072;
 const INDEX_BYTES: Record<string, number> = {
@@ -141,7 +163,14 @@ export function desiredMysqlTable(model: ModelDescriptor, variant: MysqlVariant)
       tableIndexes.push(...result);
     }
   }
-  return { checks, columns, indexes: tableIndexes, model, name, notices };
+  const sync =
+    model.sync && model.fields[SYNC_SEQUENCE_FIELD]
+      ? {
+          insertTrigger: shortName(`lt_sync_${MYSQL_SYNC_TRIGGER_VERSION}`, name, ['bi']),
+          updateTrigger: shortName(`lt_sync_${MYSQL_SYNC_TRIGGER_VERSION}`, name, ['bu']),
+        }
+      : undefined;
+  return { checks, columns, indexes: tableIndexes, model, name, notices, ...(sync ? { sync } : {}) };
 }
 
 export function variantLabel(variant: MysqlVariant): string {
@@ -248,10 +277,14 @@ function desiredIndexes(
   }
   const unique = index.options.unique === true;
   const partial = index.options.partialFilterExpression as Record<string, unknown> | undefined;
+  // A synchronized model's unique index applies to live records only: it reads virtual live-key columns
+  const live = unique && !!partial && livePartial(partial, entries);
+  const liveSparse = live && Object.keys(partial!).length > 1;
   // A partial unique index whose filter only says "every indexed field exists" is a sparse one — MySQL's own NULL
   // handling. Any other filter has no counterpart, and a uniqueness checked nowhere must not pass for one that holds
-  const sparse = index.options.sparse === true || (unique && !!partial && requiresIndexedFields(partial, entries));
-  if (unique && partial && !sparse) {
+  const sparse =
+    !live && (index.options.sparse === true || (unique && !!partial && requiresIndexedFields(partial, entries)));
+  if (unique && partial && !sparse && !live) {
     throw new UnsupportedQueryError(
       `${model.name}: the partial unique index ${JSON.stringify(index.fields)} has no MySQL / MariaDB counterpart — ` +
         'uniqueness would be checked nowhere. Use a sparse unique index, or enforce it in the service',
@@ -273,7 +306,9 @@ function desiredIndexes(
       return 'an index over an array, JSON or binary field';
     }
     let target = column.name;
-    if (unique && !sparse && !column.primaryKey && STRING_KINDS.has(field.kind) && !field.isArray) {
+    if (live && !column.primaryKey) {
+      target = liveKeyColumn(column, columns, !liveSparse && STRING_KINDS.has(field.kind) && !field.isArray);
+    } else if (unique && !sparse && !column.primaryKey && STRING_KINDS.has(field.kind) && !field.isArray) {
       // A missing value is a value of its own, as in MongoDB: the index reads it from a virtual column
       target = nullSentinelColumn(column, columns);
     }
@@ -331,6 +366,36 @@ function desiredIndexes(
   return result;
 }
 
+/**
+ * Whether a partial filter is a synchronized model's live-record condition (`{ deletedAt: null }`), alone or with
+ * "every indexed field exists" — the rewrite of a sparse unique index (`sync-model.ts` → `liveUniqueIndex()`)
+ */
+function livePartial(partial: Record<string, unknown>, entries: [string, unknown][]): boolean {
+  if (!(SYNC_DELETED_FIELD in partial) || partial[SYNC_DELETED_FIELD] !== null) {
+    return false;
+  }
+  const { [SYNC_DELETED_FIELD]: _live, ...rest } = partial;
+  return !Object.keys(rest).length || requiresIndexedFields(rest, entries);
+}
+
+/**
+ * The virtual column carrying `column`'s value while the record is live — and NULL for a tombstone, which a unique
+ * index lets repeat. `sentinel`: a live NULL becomes the null sentinel, so it collides like a second live value
+ */
+function liveKeyColumn(column: MysqlDesiredColumn, columns: MysqlDesiredColumn[], sentinel: boolean): string {
+  const name = `${LIVE_KEY_PREFIX}${column.name}`.slice(0, 63);
+  if (!columns.some((candidate) => candidate.name === name)) {
+    const value = quoteMysqlIdentifier(column.name);
+    columns.push({
+      generated: `IF(${quoteMysqlIdentifier(columnName(SYNC_DELETED_FIELD))} IS NULL, ${sentinel ? `IFNULL(${value}, CHAR(0 USING utf8mb4))` : value}, NULL)`,
+      name,
+      primaryKey: false,
+      type: column.type,
+    });
+  }
+  return name;
+}
+
 /** Whether a partial filter says exactly "every indexed field exists" (`{ field: { $exists: true } }` each) */
 function requiresIndexedFields(partial: Record<string, unknown>, entries: [string, unknown][]): boolean {
   const keys = Object.keys(partial);
@@ -384,6 +449,44 @@ export function mysqlColumnDefinition(column: MysqlDesiredColumn): string {
   return `${quoteMysqlIdentifier(column.name)} ${type}${collation}${key}${fallback}`;
 }
 
+/**
+ * The counter table, created once for every synchronized table of the database. `triggers` records the version of the
+ * table's triggers once they are in place: the application account may not read the triggers themselves (that needs
+ * the TRIGGER privilege it must never hold), so a boot without the owner checks this column instead
+ */
+export function mysqlSyncCountersStatement(): string {
+  return `CREATE TABLE IF NOT EXISTS ${quoteMysqlIdentifier(MYSQL_SYNC_COUNTERS)} (\`model\` varchar(64) COLLATE utf8mb4_bin NOT NULL PRIMARY KEY, \`seq\` bigint NOT NULL, \`triggers\` varchar(16) COLLATE utf8mb4_bin NULL) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin`;
+}
+
+/**
+ * A synchronized table's counter row, its two triggers and the numbering of rows written before it was synchronized.
+ *
+ * A BEFORE ROW trigger runs after the row is locked, so a write takes its row first and the counter last; the counter
+ * row's lock is held until COMMIT, so sequence order is commit order (C1d). A value the statement sets itself is kept
+ * — the store's multi-row writes take their numbers and set them in their final statement. The numbering of existing
+ * rows runs in an explicit transaction holding the counter row: a DDL statement commits on its own, and a write firing
+ * the new trigger meanwhile must wait rather than draw a number twice.
+ */
+export function mysqlSyncTableStatements(table: string, sync: MysqlDesiredSync): string[] {
+  const quoted = quoteMysqlIdentifier(table);
+  const counters = quoteMysqlIdentifier(MYSQL_SYNC_COUNTERS);
+  const key = mysqlLiteral(table);
+  const seq = quoteMysqlIdentifier(columnName(SYNC_SEQUENCE_FIELD));
+  const take = `UPDATE ${counters} SET \`seq\` = \`seq\` + 1 WHERE \`model\` = ${key}; SET NEW.${seq} = (SELECT \`seq\` FROM ${counters} WHERE \`model\` = ${key});`;
+  return [
+    `INSERT IGNORE INTO ${counters} (\`model\`, \`seq\`) VALUES (${key}, 0)`,
+    `DROP TRIGGER IF EXISTS ${quoteMysqlIdentifier(sync.insertTrigger)}`,
+    `DROP TRIGGER IF EXISTS ${quoteMysqlIdentifier(sync.updateTrigger)}`,
+    `CREATE TRIGGER ${quoteMysqlIdentifier(sync.insertTrigger)} BEFORE INSERT ON ${quoted} FOR EACH ROW BEGIN IF NEW.${seq} IS NULL THEN ${take} END IF; END`,
+    `CREATE TRIGGER ${quoteMysqlIdentifier(sync.updateTrigger)} BEFORE UPDATE ON ${quoted} FOR EACH ROW BEGIN IF NEW.${seq} <=> OLD.${seq} THEN ${take} END IF; END`,
+    'START TRANSACTION',
+    `SELECT \`seq\` INTO @lt_sync_base FROM ${counters} WHERE \`model\` = ${key} FOR UPDATE`,
+    `UPDATE ${quoted} AS t JOIN (SELECT \`id\`, ROW_NUMBER() OVER (ORDER BY \`id\`) AS n FROM ${quoted} WHERE ${seq} IS NULL) AS o ON o.\`id\` = t.\`id\` SET t.${seq} = @lt_sync_base + o.n`,
+    `UPDATE ${counters} SET \`seq\` = GREATEST(\`seq\`, (SELECT COALESCE(MAX(${seq}), 0) FROM ${quoted})), \`triggers\` = ${mysqlLiteral(MYSQL_SYNC_TRIGGER_VERSION)} WHERE \`model\` = ${key}`,
+    'COMMIT',
+  ];
+}
+
 export function addMysqlCheckStatement(table: string, check: MysqlDesiredCheck): string {
   return `ALTER TABLE ${quoteMysqlIdentifier(table)} ADD CONSTRAINT ${quoteMysqlIdentifier(check.name)} CHECK (${check.expression})`;
 }