@lenne.tech/nest-server 12.1.1 → 12.2.0

package/src/core/common/interfaces/server-options.interface.ts

@@ -3509,6 +3509,16 @@ export interface IServerOptions {
     path?: string;
   };
 
+  /**
+   * Offline sync (`docs/offline-sync-protocol.md`): `GET/POST /sync/{model}` for the models whose service carries
+   * `@SyncService()` and whose model is declared with `@PersistedModel({ sync })`. Presence implies enabled — `true`
+   * or `{}` enables it with the defaults, `{ enabled: false }` pre-configures without enabling. Absent: no route, no
+   * store, no code path.
+   *
+   * @since 12.2.0
+   */
+  sync?: boolean | ISync;
+
   /**
    * System setup configuration for initial admin creation.
    *
@@ -3831,6 +3841,125 @@ export interface IS3Config {
   stagingBucket?: string;
 }
 
+/**
+ * Offline sync (`sync`, presence implies enabled). See `src/core/modules/sync/README.md`.
+ *
+ * @since 12.2.0
+ */
+export interface ISync {
+  /**
+   * Secret the pull cursors are signed with (HMAC; the key is derived from it). Falls back to the
+   * `SECRETS_ENCRYPTION_KEY` environment variable. REQUIRED in `production` / `staging` — the boot fails without it;
+   * elsewhere an insecure development default is used with a warning. Changing it answers every client's next pull
+   * with a full resync (`410 invalid`).
+   */
+  cursorSecret?: string;
+
+  /**
+   * Pre-configure without enabling.
+   * @default true (when the object is present)
+   */
+  enabled?: boolean;
+
+  /**
+   * Hints that a synchronized model changed (protocol §9): SSE `GET /sync/hints` and the GraphQL subscription
+   * `syncHint`, carrying `{ model, at }` only, to callers with the model's pull roles and — on a tenant-scoped model —
+   * for changes in their own tenants. A hint only means "pull now": a client works without them. `false` switches both
+   * channels off; with `redis` a hint reaches the subscribers of every replica.
+   * @default on (both channels)
+   */
+  hints?:
+    | boolean
+    | {
+        /**
+         * At most one hint per model and tenant within this many milliseconds — a burst of writes is one hint.
+         * @default 250
+         */
+        debounceMs?: number;
+        /** @default true (when the object is present) */
+        enabled?: boolean;
+        /**
+         * The GraphQL subscription `syncHint` (only where GraphQL is enabled).
+         * @default true
+         */
+        graphql?: boolean;
+        /**
+         * The SSE stream `GET /sync/hints`.
+         * @default true
+         */
+        sse?: boolean;
+      };
+
+  /**
+   * Route prefix of the sync endpoints.
+   * @default 'sync'
+   */
+  path?: string;
+
+  pull?: {
+    /**
+     * Records per page when the client sends no `limit`.
+     * @default 200
+     */
+    defaultLimit?: number;
+    /**
+     * Upper bound of the serialized size of one page; a page ends before it would exceed it (it always holds at least
+     * one item when one is pending).
+     * @default 1048576 (1 MiB)
+     */
+    maxBytes?: number;
+    /**
+     * Highest `limit` a client may ask for (at most 1000).
+     * @default 1000
+     */
+    maxLimit?: number;
+    /**
+     * MongoDB only: the window W a pull run re-reads below its previous start (plan §3.4). Changes stamped within W
+     * before a run started may be repeated; a write that takes longer than W from its stamp until it is visible would
+     * be missed, so W must cover the longest such write.
+     * @default replica set / sharded: the server's `transactionLifetimeLimitSeconds` (60 when unreadable) + 30;
+     *   standalone: 30
+     */
+    windowSeconds?: number;
+  };
+
+  push?: {
+    /**
+     * Items per push request; more answer 400.
+     * @default 100
+     */
+    maxItems?: number;
+  };
+
+  /**
+   * Per-user rate limits of pull and push (fixed window; exact across replicas when `redis` is configured). `false`
+   * switches both off, `{ pull: false }` one of them.
+   * @default pull 120 / 60 s, push 60 / 60 s
+   */
+  rateLimit?:
+    | false
+    | {
+        pull?: false | { max?: number; windowSeconds?: number };
+        push?: false | { max?: number; windowSeconds?: number };
+      };
+
+  retention?: {
+    /**
+     * How long the outcome of a pushed mutation is kept, so a repeat is answered with it (protocol §6.5).
+     * @default 30
+     */
+    mutationsDays?: number;
+    /**
+     * How long a tombstone is kept. Every hour (on one replica, under a lease) the purge first moves the model's
+     * horizon to this age, then deletes the older tombstones for good; a client whose cursor is older than the horizon
+     * answers its next pull with a full resync (`410 horizon`). `false` keeps tombstones for ever; an invalid value
+     * (zero, negative, not a number) falls back to the default — never to "purge everything".
+     * @default 90
+     */
+    tombstonesDays?: false | number;
+  };
+}
+
 export interface ISystemSetup {
   /**
    * Whether system setup is enabled.
@@ -4831,6 +4960,26 @@ export interface ICoreModuleOverrides {
     service?: Type<any>;
   };
 
+  /**
+   * Override the offline sync endpoints, the hint subscription and their service (`sync` config).
+   *
+   * - `controller` must extend `CoreSyncController` (e.g. to release a sync scope to API tokens)
+   * - `hintsController` must extend `CoreSyncHintsController` (`GET /sync/hints`, e.g. for API tokens there too)
+   * - `resolver` must extend `CoreSyncResolver` (the `syncHint` subscription)
+   * - `service` must extend `CoreSyncService`
+   *
+   * @example
+   * ```typescript
+   * { sync: { controller: SyncController } }
+   * ```
+   */
+  sync?: {
+    controller?: Type<any>;
+    hintsController?: Type<any>;
+    resolver?: Type<any>;
+    service?: Type<any>;
+  };
+
   /**
    * Override AI module collaborators with project-specific subclasses.
    *

package/src/core/common/interfaces/service-options.interface.ts

@@ -32,6 +32,11 @@ export interface ServiceOptions {
   // If truly: create mode is activated
   create?: boolean;
 
+  // The id `create()` gives the new record: on a synchronized model the id a client chose offline, in the model's
+  // format (refused otherwise), on any other model a 24-character hex id. Passed beside the input because an input
+  // class keeps only the fields it declares: an `id` inside the input is lost the moment `inputType` maps it
+  createId?: string;
+
   // Current user to set ownership, check rights and other things
   currentUser?: {
     [key: string]: any;
@@ -42,9 +47,19 @@ export interface ServiceOptions {
   // Field selection for results (will be overwritten by populate if populate was set)
   fieldSelection?: FieldSelection;
 
+  // Conditional change of a synchronized model (`@PersistedModel({ sync })`): `update()` and `delete()` apply only
+  // when the stored record still has this version, in one atomic statement, and throw a 409 (`VersionConflictError`)
+  // otherwise. Refused on a model that is not synchronized
+  expectedVersion?: number;
+
   // Determines whether all restrictions are ignored
   force?: boolean;
 
+  // Read a synchronized model's tombstones too — its deleted records, which every other read leaves out (get, find,
+  // findOne, findAndCount, distinct). Reads only: update() and delete() never touch a tombstone and answer 404 for one.
+  // Applies to the main model only: a populated reference to a tombstone stays empty
+  includeTombstones?: boolean;
+
   // Overwrites type of input (array items)
   inputType?: new (...params: any[]) => any;
 

package/src/core/common/persistence/core-persistence.module.ts

@@ -4,8 +4,15 @@ import { getConnectionToken, getModelToken } from './mongo/mongoose-compat.js';
 import type { Model } from 'mongoose';
 
 import { registerModelSchema } from './model-metadata.registry.js';
+import { isDeclaredSchema } from './mongo/model-declarations.js';
+import { applySyncSchema, assertSyncCollectionReady } from './mongo/sync-schema.plugin.js';
 import { MongooseDataStore } from './mongoose-data-store.js';
-import { getPersistedModelOptions, getPersistedModelOptionsByName } from './persisted-model.decorator.js';
+import {
+  getPersistedModelOptions,
+  getPersistedModelOptionsByName,
+  getPersistedSyncByName,
+} from './persisted-model.decorator.js';
+import { resolveSyncOptions } from './sync-model.js';
 import {
   bindStoreToModel,
   getPersistenceRuntime,
@@ -46,7 +53,15 @@ export interface PersistenceModelDefinition extends ModelDefinition {
 export class CorePersistenceModule {
   static forFeature(models: PersistenceModelDefinition[]): DynamicModule {
     for (const model of models) {
-      registerModelSchema(model.name, model.schema);
+      const sync = resolveSyncOptions(
+        model.model ? getPersistedModelOptions(model.model)?.sync : getPersistedSyncByName(model.name),
+        model.name,
+      );
+      // Described as declared first, then prepared: the descriptor must not see the Mongoose sync paths
+      registerModelSchema(model.name, model.schema, sync);
+      if (sync && !isDeclaredSchema(model.schema)) {
+        applySyncSchema(model.schema, sync);
+      }
       registerPersistedModel({
         declaredSource:
           model.dataSource ??
@@ -88,6 +103,10 @@ export class CorePersistenceModule {
             ? // No SQL source and no MongoDB connection: the project runs without a database (`database: false`)
               unavailableStore(model.name, 'the project runs without a database (`database: false`)')
             : new MongooseDataStore(mongooseModel);
+        // A synchronized model on MongoDB: the collection must hold nothing from before it was synchronized
+        if (store.capabilities.mongoosePopulate && store.capabilities.changeOrder) {
+          await assertSyncCollectionReady(mongooseModel);
+        }
         registerDataStore(model.name, store);
         bindStoreToModel(mongooseModel, store);
         // A stand-in already refuses, naming the cause; a real Mongoose model of a model routed elsewhere is made to

package/src/core/common/persistence/data-source.runtime.ts

@@ -13,6 +13,7 @@ import {
 } from './data-source.config.js';
 import type { ModelDescriptor } from './model-descriptor.interface.js';
 import { getModelDescriptor } from './model-metadata.registry.js';
+import { withSequenceField } from './sync-model.js';
 import { MongooseDataStore } from './mongoose-data-store.js';
 import {
   getPersistedModels,
@@ -190,10 +191,12 @@ export class DataSourceRuntime implements PersistenceRuntime {
   }
 
   private async openStore(source: ResolvedDataSource, name: string): Promise<DataStore<any>> {
-    const model = getModelDescriptor(name);
-    if (!model) {
+    const declared = getModelDescriptor(name);
+    if (!declared) {
       throw new Error(`No descriptor for model "${name}" — register it with CorePersistenceModule.forFeature()`);
     }
+    // A synchronized model's table also carries its commit-ordered change position (offline-sync plan §3.4)
+    const model = withSequenceField(declared);
     const app = await this.executor(source);
     const dialect = this.dialect(source);
     let changes: SchemaChange[];

package/src/core/common/persistence/data-store.interface.ts

@@ -54,6 +54,11 @@ export interface FindOptions {
    * Not combinable with `select`, as MongoDB does not combine them either.
    */
   exclude?: readonly string[];
+  /**
+   * Read tombstones too (a synchronized model's deleted records, `deletedAt` set). Every read of such a model leaves
+   * them out unless asked — only the sync module asks. No effect on other models.
+   */
+  includeTombstones?: boolean;
   /** Field names to return; the id is always returned */
   select?: readonly string[];
   skip?: number;
@@ -61,7 +66,15 @@ export interface FindOptions {
   sort?: Record<string, -1 | 1>;
 }
 
+/** Options of a read that has no paging (`count`, `distinct`, `findById`) */
+export interface ReadOptions {
+  /** See `FindOptions.includeTombstones` */
+  includeTombstones?: boolean;
+}
+
 export interface UpdateOptions {
+  /** Match tombstones too (a synchronized model's deleted records) — see `FindOptions.includeTombstones` */
+  includeTombstones?: boolean;
   /**
    * `false` leaves the update time (`updatedAt`) as it is — for bookkeeping writes such as a last-used stamp, which
    * must not read as a change of the record (Mongoose's `timestamps: false`). @default true
@@ -69,6 +82,17 @@ export interface UpdateOptions {
   timestamps?: boolean;
 }
 
+export interface DeleteOptions {
+  /**
+   * Remove the record for good. A synchronized model's delete otherwise leaves a tombstone (`deletedAt` set, the
+   * content cleared, the version incremented), which is how clients learn of it; a hard delete is the tombstone purge's
+   * (with `includeTombstones`). No effect on other models, whose delete always removes.
+   */
+  hardDelete?: boolean;
+  /** Match tombstones too — see `FindOptions.includeTombstones` */
+  includeTombstones?: boolean;
+}
+
 export interface PushOptions {
   /** Insert position (MongoDB `$position`) */
   position?: number;
@@ -99,28 +123,57 @@ export interface StoreCapabilities {
   textSearch: boolean;
   /** Multi-statement transactions */
   transactions: boolean;
-  /** Format of generated ids */
-  idType: 'objectId';
+  /** Format of the model's ids — `uuidv7` only for a synchronized model that declares it */
+  idType: 'objectId' | 'uuidv7';
+  /**
+   * How a synchronized model orders its changes (offline-sync plan §3.4): `seq` — the commit-ordered `syncSeq` (SQL);
+   * `clock` — the server clock `syncAt`, read with a window (MongoDB). Absent for a model that is not synchronized.
+   */
+  changeOrder?: 'clock' | 'seq';
+}
+
+/** The database's clock as a pull over `changeOrder: 'clock'` reads it (offline-sync plan §3.4) */
+export interface ChangeClock {
+  /** The database server's current time */
+  now: Date;
+  /**
+   * The window W: how long a write may take from its stamp until it is visible. A pull run starts this far below the
+   * previous run's start, so a change stamped before that start but committed after it is not passed.
+   */
+  windowMs: number;
 }
 
 export interface DataStore<T = Record<string, any>> {
   readonly capabilities: StoreCapabilities;
+  /**
+   * The database's clock and the window W of a synchronized model whose change order is the clock
+   * (`capabilities.changeOrder === 'clock'`); absent on every other store. What the sync module reads instead of
+   * asking for the driver.
+   */
+  changeClock?(): Promise<ChangeClock>;
   /** The database behind the store (`mongodb`, `postgres`, …) */
   readonly driver: string;
   readonly modelName: string;
 
-  count(where?: Where<T>): Promise<number>;
-  deleteById(id: string): Promise<null | StoredRecord<T>>;
-  /** Remove every record matching `where`; returns how many were removed. */
-  deleteMany(where: Where<T>): Promise<number>;
+  count(where?: Where<T>, options?: ReadOptions): Promise<number>;
+  /**
+   * Remove the record and return it as it was; `null` when there is none. On a synchronized model: turn it into a
+   * tombstone instead, unless `hardDelete` (see `DeleteOptions`).
+   */
+  deleteById(id: string, options?: DeleteOptions): Promise<null | StoredRecord<T>>;
+  /** Remove every record matching `where`; returns how many were removed (tombstoned on a synchronized model). */
+  deleteMany(where: Where<T>, options?: DeleteOptions): Promise<number>;
   /**
    * Remove the first record matching `where`, atomically (MongoDB's `findOneAndDelete`), and return it; `null` when
-   * nothing matches.
+   * nothing matches (tombstoned on a synchronized model).
    */
-  deleteOne(where: Where<T>): Promise<null | StoredRecord<T>>;
-  distinct(field: string, where?: Where<T>): Promise<unknown[]>;
+  deleteOne(where: Where<T>, options?: DeleteOptions): Promise<null | StoredRecord<T>>;
+  distinct(field: string, where?: Where<T>, options?: ReadOptions): Promise<unknown[]>;
   find(where?: Where<T>, options?: FindOptions): Promise<StoredRecord<T>[]>;
-  findById(id: string, options?: Pick<FindOptions, 'exclude' | 'select'>): Promise<null | StoredRecord<T>>;
+  findById(
+    id: string,
+    options?: Pick<FindOptions, 'exclude' | 'includeTombstones' | 'select'>,
+  ): Promise<null | StoredRecord<T>>;
   findOne(where: Where<T>, options?: Omit<FindOptions, 'limit' | 'skip'>): Promise<null | StoredRecord<T>>;
   insert(record: Partial<T>): Promise<StoredRecord<T>>;
   /**
@@ -145,11 +198,11 @@ export interface DataStore<T = Record<string, any>> {
   /** Apply `update` to the record and return it as stored afterwards; `null` when there is none. */
   updateById(id: string, update: DataStoreUpdate<T>, options?: UpdateOptions): Promise<null | StoredRecord<T>>;
   /** Apply `update` to every record matching `where`; returns how many records matched. */
-  updateMany(where: Where<T>, update: DataStoreUpdate<T>): Promise<number>;
+  updateMany(where: Where<T>, update: DataStoreUpdate<T>, options?: UpdateOptions): Promise<number>;
   /**
    * Apply `update` to the first record matching `where`, atomically: the record still matches when it is changed, and
    * concurrent callers never change the same record on the strength of one match (MongoDB's `findOneAndUpdate`).
    * Returns the record as stored afterwards; `null` when nothing matches.
    */
-  updateOne(where: Where<T>, update: DataStoreUpdate<T>): Promise<null | StoredRecord<T>>;
+  updateOne(where: Where<T>, update: DataStoreUpdate<T>, options?: UpdateOptions): Promise<null | StoredRecord<T>>;
 }

package/src/core/common/persistence/id-format.ts

@@ -41,3 +41,29 @@ export function generateId(): string {
   bytes.writeUIntBE(counter, 9, 3);
   return bytes.toString('hex');
 }
+
+/**
+ * A uuidv7 as the framework stores it (plan §3.5): lowercase, hyphenated, version nibble `7`, variant `8`–`b`. The id
+ * format a synchronized model may choose instead of the ObjectId (`@PersistedModel({ sync: { idFormat: 'uuidv7' } })`).
+ */
+const UUID_V7 = /^[0-9a-f]{8}-[0-9a-f]{4}-7[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/;
+
+/** Whether a value is a uuidv7 in the stored form — lowercase only, so one id has exactly one spelling */
+export function isUuidV7(value: unknown): value is string {
+  return typeof value === 'string' && UUID_V7.test(value);
+}
+
+/**
+ * A new uuidv7 (RFC 9562): 48 bits of milliseconds, then 74 random bits — sortable by creation time like an ObjectId,
+ * but with nothing after the timestamp that one observed id would let anybody predict.
+ */
+export function generateUuidV7(): string {
+  const bytes = randomBytes(16);
+  const millis = Date.now();
+  bytes.writeUIntBE(Math.floor(millis / 0x10000), 0, 4);
+  bytes.writeUInt16BE(millis % 0x10000, 4);
+  bytes[6] = (bytes[6] & 0x0f) | 0x70;
+  bytes[8] = (bytes[8] & 0x3f) | 0x80;
+  const hex = bytes.toString('hex');
+  return `${hex.slice(0, 8)}-${hex.slice(8, 12)}-${hex.slice(12, 16)}-${hex.slice(16, 20)}-${hex.slice(20)}`;
+}

package/src/core/common/persistence/model-descriptor.interface.ts

@@ -77,12 +77,24 @@ export interface IndexDescriptor {
   options: Record<string, unknown>;
 }
 
+/** The id format of a synchronized model: one per model, never mixed (offline-sync plan §3.5) */
+export type SyncIdFormat = 'objectId' | 'uuidv7';
+
+/** How a model is synchronized to offline clients — see `sync-model.ts` */
+export interface SyncDescriptor {
+  idFormat: SyncIdFormat;
+  /** User-id fields a tombstone keeps besides `tenantId` */
+  scope: readonly string[];
+}
+
 export interface ModelDescriptor {
   /** Explicit collection / table name, when the model declares one */
   collection?: string;
   fields: Record<string, FieldDescriptor>;
   indexes: IndexDescriptor[];
   name: string;
+  /** Present when the model is synchronized (`@PersistedModel({ sync })`); absent for every other model */
+  sync?: SyncDescriptor;
   /** Whether the model carries a `tenantId` — the same criterion the tenant plugin uses */
   tenantScoped: boolean;
   /** `createdAt` / `updatedAt` maintenance: `true`, `false`, or the configured field names */

package/src/core/common/persistence/model-metadata.registry.ts

@@ -2,6 +2,7 @@ import type { FieldDescriptor, FieldKind, IndexDescriptor, ModelDescriptor } fro
 import { declaredSchemaView } from './mongo/declared-schema.view.js';
 import { type DeclaredSchema, isDeclaredSchema } from './mongo/model-declarations.js';
 import { SchemaFactory } from './mongo/mongoose-compat.js';
+import { type SyncDescriptor, withSyncDescriptor } from './sync-model.js';
 
 /**
  * Registry of database-neutral model descriptors.
@@ -70,12 +71,40 @@ export function describeModelClass(modelClass: abstract new (...args: any[]) =>
   return descriptor;
 }
 
+/**
+ * The descriptor of a schema as it was declared, per schema object and model name. A synchronized model's Mongoose
+ * schema gains its system paths and partial indexes once it is registered (`mongo/sync-schema.plugin.ts`); a second
+ * registration of the same schema must describe what was declared, not what the plugin added.
+ */
+const declaredDescriptors = new WeakMap<object, Map<string, ModelDescriptor>>();
+
 /**
  * Register the schema a model is persisted with, under its model name. A later registration under
  * the same name replaces the earlier one — the last `forFeature` wins, as it does in Mongoose.
+ *
+ * `sync` makes it the descriptor of a synchronized model (`sync-model.ts`).
  */
-export function registerModelSchema(name: string, schema: DeclaredSchema | SchemaLike): ModelDescriptor {
-  const descriptor = describeSchema(name, schema);
+export function registerModelSchema(
+  name: string,
+  schema: DeclaredSchema | SchemaLike,
+  sync?: SyncDescriptor,
+): ModelDescriptor {
+  let descriptor: ModelDescriptor;
+  if (sync) {
+    let byName = declaredDescriptors.get(schema);
+    if (!byName) {
+      byName = new Map();
+      declaredDescriptors.set(schema, byName);
+    }
+    let declared = byName.get(name);
+    if (!declared) {
+      declared = describeSchema(name, schema);
+      byName.set(name, declared);
+    }
+    descriptor = withSyncDescriptor(declared, sync);
+  } else {
+    descriptor = describeSchema(name, schema);
+  }
   descriptorsByName.set(name, descriptor);
   return descriptor;
 }