@lenne.tech/nest-server 12.1.1 → 12.3.0

package/src/core/common/persistence/mongoose-data-store.ts

@@ -2,6 +2,22 @@ import { Logger } from '@nestjs/common';
 import type { Model } from 'mongoose';
 
 import { getStringIds } from '../helpers/id.helper.js';
+import {
+  includeTombstones,
+  mongoChangeWindowMs,
+  mongoServerNow,
+  syncOfSchema,
+  tombstoneWrite,
+} from './mongo/sync-schema.plugin.js';
+import type { SyncDescriptor } from './model-descriptor.interface.js';
+import {
+  assertClientId,
+  insertWithServerId,
+  suppliedId,
+  syncInsertRecord,
+  tombstoneClearedFields,
+} from './policies/sync.policy.js';
+import { SYNC_CLOCK_FIELD, SYNC_DELETED_FIELD } from './sync-model.js';
 import {
   assertAccessReason,
   assertArrayField,
@@ -10,10 +26,13 @@ import {
   assertSelectFields,
 } from './store-guards.js';
 import type {
+  ChangeClock,
   DataStore,
   DataStoreUpdate,
+  DeleteOptions,
   FindOptions,
   PushOptions,
+  ReadOptions,
   StoreCapabilities,
   StoredRecord,
   UpdateOptions,
@@ -42,18 +61,34 @@ export const MONGOOSE_STORE_CAPABILITIES: Readonly<StoreCapabilities> = Object.f
  * are the Mongo-operator form — except that the neutral `id` is addressed as `_id`.
  */
 export class MongooseDataStore<T = Record<string, any>> implements DataStore<T> {
-  readonly capabilities = MONGOOSE_STORE_CAPABILITIES;
+  readonly capabilities: Readonly<StoreCapabilities>;
   readonly driver = 'mongodb';
   protected readonly logger = new Logger(MongooseDataStore.name);
+  /** The model's sync declaration (`@PersistedModel({ sync })`), read from its schema */
+  protected readonly sync?: SyncDescriptor;
 
-  constructor(protected readonly model: Model<any>) {}
+  constructor(protected readonly model: Model<any>) {
+    this.sync = syncOfSchema(model?.schema);
+    this.capabilities = this.sync
+      ? Object.freeze({ ...MONGOOSE_STORE_CAPABILITIES, changeOrder: 'clock', idType: this.sync.idFormat })
+      : MONGOOSE_STORE_CAPABILITIES;
+  }
 
   get modelName(): string {
     return this.model.modelName;
   }
 
-  async count(where?: Where<T>): Promise<number> {
-    return this.model.countDocuments(toMongoFilter(where)).exec();
+  /** The server clock and the window W of a synchronized model (its change order is the clock, plan §3.4) */
+  async changeClock(): Promise<ChangeClock> {
+    if (!this.sync) {
+      throw new Error(`changeClock(): ${this.modelName} is not a synchronized model (@PersistedModel({ sync }))`);
+    }
+    const [now, windowMs] = await Promise.all([mongoServerNow(this.model.db), mongoChangeWindowMs(this.model.db)]);
+    return { now, windowMs };
+  }
+
+  async count(where?: Where<T>, options: ReadOptions = {}): Promise<number> {
+    return this.tombstones(this.model.countDocuments(toMongoFilter(where)), options).exec();
   }
 
   async sum(field: string, where?: Where<T>): Promise<{ count: number; sum: number }> {
@@ -69,28 +104,43 @@ export class MongooseDataStore<T = Record<string, any>> implements DataStore<T>
     return { count: row?.count ?? 0, sum: row?.sum ?? 0 };
   }
 
-  async deleteById(id: string): Promise<null | StoredRecord<T>> {
-    return toRecord<T>(await this.model.findByIdAndDelete(id).lean().exec());
+  async deleteById(id: string, options: DeleteOptions = {}): Promise<null | StoredRecord<T>> {
+    if (this.sync && !options.hardDelete) {
+      return this.tombstone({ _id: id }, options);
+    }
+    return toRecord<T>(await this.tombstones(this.model.findByIdAndDelete(id), options).lean().exec());
   }
 
-  async deleteMany(where: Where<T>): Promise<number> {
-    const result = await this.model.deleteMany(toMongoFilter(where)).exec();
+  async deleteMany(where: Where<T>, options: DeleteOptions = {}): Promise<number> {
+    if (this.sync && !options.hardDelete) {
+      const query = tombstoneWrite(this.model.updateMany(toMongoFilter(where), this.tombstoneUpdate() as any));
+      const result = await this.tombstones(query, options).exec();
+      return result.matchedCount;
+    }
+    const result = await this.tombstones(this.model.deleteMany(toMongoFilter(where)), options).exec();
     return result.deletedCount;
   }
 
-  async deleteOne(where: Where<T>): Promise<null | StoredRecord<T>> {
-    return toRecord<T>(await this.model.findOneAndDelete(toMongoFilter(where)).lean().exec());
+  async deleteOne(where: Where<T>, options: DeleteOptions = {}): Promise<null | StoredRecord<T>> {
+    if (this.sync && !options.hardDelete) {
+      return this.tombstone(toMongoFilter(where), options);
+    }
+    return toRecord<T>(
+      await this.tombstones(this.model.findOneAndDelete(toMongoFilter(where)), options)
+        .lean()
+        .exec(),
+    );
   }
 
-  async distinct(field: string, where?: Where<T>): Promise<unknown[]> {
-    return this.model.distinct(field, toMongoFilter(where)).exec();
+  async distinct(field: string, where?: Where<T>, options: ReadOptions = {}): Promise<unknown[]> {
+    return this.tombstones(this.model.distinct(field, toMongoFilter(where)), options).exec();
   }
 
   async find(where?: Where<T>, options: FindOptions = {}): Promise<StoredRecord<T>[]> {
     const sample = options.sample === undefined ? undefined : assertSampleSize(options.sample);
-    let query = this.model.find(toMongoFilter(where));
+    let query = this.tombstones(this.model.find(toMongoFilter(where)), options);
     if (options.select || options.exclude) query = query.select(toProjection(options));
-    if (options.sort) query = query.sort(options.sort);
+    if (options.sort) query = query.sort(toMongoSort(options.sort));
     if (options.skip !== undefined) query = query.skip(options.skip);
     if (options.limit !== undefined) query = query.limit(options.limit);
     if (options.collation) query = query.collation(options.collation as any);
@@ -101,21 +151,27 @@ export class MongooseDataStore<T = Record<string, any>> implements DataStore<T>
     return sample === undefined ? records : randomSample(records, sample);
   }
 
-  async findById(id: string, options: Pick<FindOptions, 'exclude' | 'select'> = {}): Promise<null | StoredRecord<T>> {
-    let query = this.model.findById(id);
+  async findById(
+    id: string,
+    options: Pick<FindOptions, 'exclude' | 'includeTombstones' | 'select'> = {},
+  ): Promise<null | StoredRecord<T>> {
+    let query = this.tombstones(this.model.findById(id), options);
     if (options.select || options.exclude) query = query.select(toProjection(options));
     return toRecord<T>(await query.lean().exec());
   }
 
   async findOne(where: Where<T>, options: Omit<FindOptions, 'limit' | 'skip'> = {}): Promise<null | StoredRecord<T>> {
-    let query = this.model.findOne(toMongoFilter(where));
+    let query = this.tombstones(this.model.findOne(toMongoFilter(where)), options);
     if (options.select || options.exclude) query = query.select(toProjection(options));
-    if (options.sort) query = query.sort(options.sort);
+    if (options.sort) query = query.sort(toMongoSort(options.sort));
     if (options.collation) query = query.collation(options.collation as any);
     return toRecord<T>(await query.lean().exec());
   }
 
   async insert(record: Partial<T>): Promise<StoredRecord<T>> {
+    if (this.sync) {
+      return this.insertSynchronized(record as Record<string, any>);
+    }
     // `create()` runs the save hooks, so tenant stamping, audit fields and password hashing apply
     const created = await this.model.create(record);
     return toRecord<T>(created.toObject());
@@ -161,30 +217,95 @@ export class MongooseDataStore<T = Record<string, any>> implements DataStore<T>
     update: DataStoreUpdate<T>,
     options: UpdateOptions = {},
   ): Promise<null | StoredRecord<T>> {
-    const updated = await this.model
-      .findByIdAndUpdate(id, update as any, {
-        returnDocument: 'after',
-        ...(options.timestamps === false ? { timestamps: false } : {}),
-      })
-      .lean()
-      .exec();
-    return toRecord<T>(updated);
+    const query = this.model.findByIdAndUpdate(id, update as any, {
+      returnDocument: 'after',
+      ...(options.timestamps === false ? { timestamps: false } : {}),
+    });
+    return toRecord<T>(await this.tombstones(query, options).lean().exec());
   }
 
-  async updateMany(where: Where<T>, update: DataStoreUpdate<T>): Promise<number> {
-    const result = await this.model.updateMany(toMongoFilter(where), update as any).exec();
+  async updateMany(where: Where<T>, update: DataStoreUpdate<T>, options: UpdateOptions = {}): Promise<number> {
+    const query = this.model.updateMany(
+      toMongoFilter(where),
+      update as any,
+      options.timestamps === false ? { timestamps: false } : {},
+    );
+    const result = await this.tombstones(query, options).exec();
     return result.matchedCount;
   }
 
-  async updateOne(where: Where<T>, update: DataStoreUpdate<T>): Promise<null | StoredRecord<T>> {
-    const updated = await this.model
-      .findOneAndUpdate(toMongoFilter(where), update as any, { returnDocument: 'after' })
-      .lean()
-      .exec();
-    return toRecord<T>(updated);
+  async updateOne(
+    where: Where<T>,
+    update: DataStoreUpdate<T>,
+    options: UpdateOptions = {},
+  ): Promise<null | StoredRecord<T>> {
+    const query = this.model.findOneAndUpdate(toMongoFilter(where), update as any, {
+      returnDocument: 'after',
+      ...(options.timestamps === false ? { timestamps: false } : {}),
+    });
+    return toRecord<T>(await this.tombstones(query, options).lean().exec());
+  }
+
+  // -- synchronized models ------------------------------------------------------------------------------------
+
+  /** A query that also sees tombstones when the caller asked for them (synchronized models only) */
+  protected tombstones<Q extends object>(query: Q, options: { includeTombstones?: boolean }): Q {
+    return this.sync && options.includeTombstones ? includeTombstones(query) : query;
+  }
+
+  /**
+   * The insert of a synchronized model: system fields set by the store, a client id in the model's format kept, a
+   * server id generated anew when it is taken (plan §3.3, E6). `create()` runs every save hook, as for any model.
+   */
+  protected async insertSynchronized(record: Record<string, any>): Promise<StoredRecord<T>> {
+    const { _id: _ignored, id: _alsoIgnored, ...doc } = syncInsertRecord(record);
+    const supplied = suppliedId(record);
+    if (supplied !== undefined && supplied !== null) {
+      const created = await this.model.create({
+        ...doc,
+        _id: assertClientId(supplied, this.sync.idFormat, this.modelName),
+      });
+      return toRecord<T>(created.toObject());
+    }
+    const created = await insertWithServerId(
+      this.sync.idFormat,
+      (id) => this.model.create({ ...doc, _id: id }),
+      (error) => isDuplicateIdError(error),
+    );
+    return toRecord<T>(created.toObject());
+  }
+
+  /** The update that turns a record into a tombstone: content cleared, `deletedAt` from the server clock */
+  protected tombstoneUpdate(): Record<string, unknown> {
+    const fields = Object.keys(this.model.schema.paths).map((path) => path.split('.', 1)[0]);
+    const cleared = tombstoneClearedFields(new Set(fields), this.sync);
+    return {
+      $currentDate: { [SYNC_CLOCK_FIELD]: true, [SYNC_DELETED_FIELD]: true },
+      ...(cleared.length ? { $unset: Object.fromEntries(cleared.map((field) => [field, 1])) } : {}),
+    };
+  }
+
+  /** Turn the first record matching `filter` into a tombstone; returns it as it was before */
+  protected async tombstone(filter: Record<string, any>, options: DeleteOptions): Promise<null | StoredRecord<T>> {
+    const query = tombstoneWrite(
+      this.model.findOneAndUpdate(filter, this.tombstoneUpdate() as any, { returnDocument: 'before' }),
+    );
+    return toRecord<T>(await this.tombstones(query, options).lean().exec());
   }
 }
 
+/** A duplicate-key error raised by the `_id` index — the case a server-generated id is replaced for */
+export function isDuplicateIdError(error: unknown): boolean {
+  const candidate = error as { code?: unknown; keyPattern?: Record<string, unknown>; message?: unknown };
+  if (candidate?.code !== 11000) {
+    return false;
+  }
+  if (candidate.keyPattern) {
+    return Object.keys(candidate.keyPattern).length === 1 && '_id' in candidate.keyPattern;
+  }
+  return typeof candidate.message === 'string' && /index: _id_ /.test(candidate.message);
+}
+
 /** The neutral `id` is `_id` in MongoDB — at the top level and inside `$and` / `$or` / `$nor`. */
 export function toMongoFilter(where: undefined | Where<any>): Record<string, any> {
   if (!where) {
@@ -203,13 +324,28 @@ export function toMongoFilter(where: undefined | Where<any>): Record<string, any
   return filter;
 }
 
+/** The neutral `id` is `_id` in a sort too */
+function toMongoSort(sort: Record<string, -1 | 1>): Record<string, -1 | 1> {
+  return Object.fromEntries(
+    Object.entries(sort).map(([field, direction]) => [field === 'id' ? '_id' : field, direction]),
+  );
+}
+
 /** A lean document as a neutral record: the id as a string under `id`, no `_id`, no version key. */
 function toRecord<T>(doc: unknown): StoredRecord<T> {
   if (!doc) {
     return null;
   }
   const { __v: _version, _id, ...fields } = doc as Record<string, any>;
-  return { ...fields, id: _id === undefined ? fields.id : getStringIds(_id) } as StoredRecord<T>;
+  return { ...fields, id: _id === undefined ? fields.id : storedIdString(_id) } as StoredRecord<T>;
+}
+
+/** An id as the neutral string: a BSON UUID (a uuidv7 model's id) in its hyphenated form, anything else as before */
+function storedIdString(id: unknown): string {
+  if (id && typeof id === 'object' && (id as { sub_type?: unknown }).sub_type === 4) {
+    return String(id);
+  }
+  return getStringIds(id);
 }
 
 function toProjection(options: Pick<FindOptions, 'exclude' | 'select'>): string {

package/src/core/common/persistence/persisted-model.decorator.ts

@@ -1,3 +1,5 @@
+import type { SyncModelOptions } from './sync-model.js';
+
 /**
  * Persistence options a model class declares in code.
  */
@@ -8,6 +10,13 @@ export interface PersistedModelOptions {
    * @default the default source
    */
   dataSource?: string;
+  /**
+   * The model is synchronized to offline clients (`docs/offline-sync-protocol.md`): its records carry `version`,
+   * `deletedAt` and a change position, a delete leaves a tombstone, and unique indexes apply to live records only.
+   * `true` declares it with the defaults. Takes effect for a model registered with `CorePersistenceModule.forFeature()`.
+   * @default not synchronized
+   */
+  sync?: boolean | SyncModelOptions;
 }
 
 const PERSISTED_MODEL_OPTIONS = Symbol('PERSISTED_MODEL_OPTIONS');
@@ -59,3 +68,20 @@ export function getPersistedModelOptionsByName(name: string): PersistedModelOpti
   }
   return classes.length ? getPersistedModelOptions(classes[0]) : undefined;
 }
+
+/**
+ * The `sync` option declared for the model registered under `name` without its class. Whether a model is synchronized
+ * changes what its table holds and what a delete does, so two decorated classes of that name that disagree are never
+ * guessed between either.
+ */
+export function getPersistedSyncByName(name: string): PersistedModelOptions['sync'] {
+  const classes = declaredByName.get(name) ?? [];
+  const syncs = new Set(classes.map((target) => JSON.stringify(getPersistedModelOptions(target)?.sync ?? false)));
+  if (syncs.size > 1) {
+    throw new Error(
+      `@PersistedModel: ${classes.length} classes named "${name}" declare different sync options; pass \`model\` ` +
+        'in CorePersistenceModule.forFeature() to say which one this model is',
+    );
+  }
+  return classes.length ? getPersistedModelOptions(classes[0])?.sync : undefined;
+}

package/src/core/common/persistence/policies/sync.policy.ts

@@ -0,0 +1,121 @@
+import { BadRequestException, ConflictException } from '@nestjs/common';
+
+import { ErrorCode } from '../../../modules/error-code/error-codes.js';
+import { generateId, generateUuidV7, isUuidV7, isWellFormedId } from '../id-format.js';
+import type { SyncDescriptor, SyncIdFormat } from '../model-descriptor.interface.js';
+import { SYNC_DELETED_FIELD, SYNC_SYSTEM_FIELDS, SYNC_VERSION_FIELD } from '../sync-model.js';
+
+/**
+ * Write rules of synchronized models, shared by every data store (offline-sync plan §3.3) — like the tenant policy,
+ * a fix here reaches every database.
+ */
+
+/** How often an insert with a SERVER-generated id is tried again when that id is taken (plan §3.3, E6) */
+export const SERVER_ID_ATTEMPTS = 3;
+
+/** Fields a tombstone keeps besides the declared scope fields (protocol §2) */
+const TOMBSTONE_KEPT = new Set([
+  '_id',
+  '__v',
+  'createdAt',
+  'createdBy',
+  'id',
+  'tenantId',
+  'updatedAt',
+  'updatedBy',
+  ...SYNC_SYSTEM_FIELDS,
+]);
+
+/**
+ * The record a caller inserts, as a synchronized model stores it: the system fields are the store's (`version: 1`,
+ * `deletedAt: null`), whatever the caller sent
+ */
+export function syncInsertRecord(record: Record<string, any>): Record<string, any> {
+  const doc = { ...record };
+  for (const field of SYNC_SYSTEM_FIELDS) {
+    delete doc[field];
+  }
+  doc[SYNC_VERSION_FIELD] = 1;
+  doc[SYNC_DELETED_FIELD] = null;
+  return doc;
+}
+
+/** The content fields a delete clears: everything but the id, the system, audit and tenant fields and the scope */
+export function tombstoneClearedFields(fields: Iterable<string>, sync: SyncDescriptor): string[] {
+  const scope = new Set(sync.scope);
+  return [...fields].filter((field) => !TOMBSTONE_KEPT.has(field) && !scope.has(field));
+}
+
+/** A new id in the model's format */
+export function generateSyncId(format: SyncIdFormat): string {
+  return format === 'uuidv7' ? generateUuidV7() : generateId();
+}
+
+/** Whether a value is an id in the model's format */
+export function isSyncId(value: unknown, format: SyncIdFormat): boolean {
+  return format === 'uuidv7' ? isUuidV7(value) : typeof value === 'string' && isWellFormedId(value);
+}
+
+/**
+ * A client-supplied id, checked against the model's format (protocol §3: `INVALID_ID`). Unlike an ordinary model —
+ * which replaces a malformed id with a new one — a synchronized model refuses it: the client keeps its own id and
+ * would lose track of the record.
+ */
+export function assertClientId(value: unknown, format: SyncIdFormat, model: string): string {
+  const id = typeof value === 'string' ? value : value?.toString?.();
+  if (format === 'objectId' && typeof id === 'string' && isWellFormedId(id)) {
+    return id.toLowerCase();
+  }
+  if (format === 'uuidv7' && isUuidV7(id)) {
+    return id;
+  }
+  throw new BadRequestException(
+    `${ErrorCode.INVALID_FIELD_FORMAT} [id]: ${model} takes ${format === 'uuidv7' ? 'a lowercase uuidv7' : 'a 24-character hex ObjectId'} as id`,
+  );
+}
+
+/** The id an insert names, if any (`id` or `_id`) */
+export function suppliedId(record: Record<string, any>): unknown {
+  return record.id ?? record._id;
+}
+
+/**
+ * Insert with a server-generated id, generating a fresh one when it is taken (plan §3.3, E6): server ids are
+ * predictable from one observed id, so a client could occupy the next ones; a server id was never shown to anyone, so
+ * replacing it is invisible. A client-supplied id is never replaced — its duplicate is the caller's answer.
+ */
+export async function insertWithServerId<R>(
+  format: SyncIdFormat,
+  attempt: (id: string) => Promise<R>,
+  isIdTaken: (error: unknown) => boolean,
+): Promise<R> {
+  for (let tries = 1; ; tries++) {
+    try {
+      return await attempt(generateSyncId(format));
+    } catch (error) {
+      if (tries >= SERVER_ID_ATTEMPTS || !isIdTaken(error)) {
+        throw error;
+      }
+    }
+  }
+}
+
+/** A conditional change (`expectedVersion`) whose record has another version: the caller resolves and retries */
+export class VersionConflictError extends ConflictException {
+  override readonly name = 'VersionConflictError';
+
+  constructor(
+    readonly model: string,
+    readonly id: string,
+  ) {
+    super(`${ErrorCode.VERSION_CONFLICT}: ${model} ${id} was changed in the meantime`);
+  }
+}
+
+/** Whether `expectedVersion` is a usable version number */
+export function assertExpectedVersion(value: unknown): number {
+  if (typeof value !== 'number' || !Number.isInteger(value) || value < 1) {
+    throw new BadRequestException(`${ErrorCode.INVALID_FIELD_FORMAT} [expectedVersion]: a positive integer`);
+  }
+  return value;
+}