@lenne.tech/nest-server 12.1.1 → 12.3.0

package/src/core/modules/sync/core-sync-state.store.ts

@@ -0,0 +1,89 @@
+import type { DataStore } from '../../common/persistence/data-store.interface.js';
+import { isDuplicateKey } from './core-sync-mutation.store.js';
+
+interface StateRecord {
+  epoch?: null | number;
+  horizon?: Date | null;
+  key: string;
+}
+
+/**
+ * Access epochs and tombstone horizons (offline-sync plan §3.7, §3.8), keyed by string:
+ *
+ * - `epoch:t:<model>:<tenant>` — bumped when what a tenant's members may see changes (`invalidateTenant()`, a record
+ *   moved to another tenant). A cursor carries the epochs it was issued under; another value answers 410 `access`.
+ * - `epoch:u:<model>:<tenant>:<user>` — bumped when a scope field (`sync.scope`) changes away from that user, so only
+ *   that user's cursors resync.
+ * - `horizon:<model>` — the oldest point a cursor may continue from; a cursor whose state is older answers 410 `horizon`.
+ *
+ * A bump is one atomic increment; a key that does not exist yet is created with a duplicate-key retry, so two
+ * concurrent bumps both count.
+ */
+export class CoreSyncStateStore {
+  constructor(protected readonly store: DataStore<StateRecord>) {}
+
+  static tenantEpochKey(model: string, tenantId: string): string {
+    return `epoch:t:${model}:${tenantId}`;
+  }
+
+  static userEpochKey(model: string, tenantId: string, userId: string): string {
+    return `epoch:u:${model}:${tenantId}:${userId}`;
+  }
+
+  static horizonKey(model: string): string {
+    return `horizon:${model}`;
+  }
+
+  /** The current epochs of `keys`; a key never bumped is `0` */
+  async epochs(keys: readonly string[]): Promise<Map<string, number>> {
+    const result = new Map<string, number>(keys.map((key) => [key, 0]));
+    if (!keys.length) {
+      return result;
+    }
+    for (const row of await this.store.find({ key: { $in: [...keys] } }, { select: ['epoch', 'key'] })) {
+      result.set(row.key, typeof row.epoch === 'number' ? row.epoch : 0);
+    }
+    return result;
+  }
+
+  async bump(key: string): Promise<void> {
+    if (await this.store.updateOne({ key }, { $inc: { epoch: 1 } })) {
+      return;
+    }
+    try {
+      await this.store.insert({ epoch: 1, key });
+    } catch (error) {
+      if (!isDuplicateKey(error)) {
+        throw error;
+      }
+      // Created concurrently: this bump still counts
+      await this.store.updateOne({ key }, { $inc: { epoch: 1 } });
+    }
+  }
+
+  /** The tombstone horizon of a model, `undefined` while no purge ran */
+  async horizon(model: string): Promise<Date | undefined> {
+    const row = await this.store.findOne({ key: CoreSyncStateStore.horizonKey(model) });
+    return row?.horizon ? new Date(row.horizon) : undefined;
+  }
+
+  /** Move the horizon forward to `at` — never back, so a slower purge cannot reopen a gap a faster one closed */
+  async advanceHorizon(model: string, at: Date): Promise<void> {
+    const key = CoreSyncStateStore.horizonKey(model);
+    const behind = { $or: [{ horizon: null }, { horizon: { $lt: at } }], key };
+    if (await this.store.updateOne(behind, { $set: { horizon: at } })) {
+      return;
+    }
+    if (await this.store.findOne({ key })) {
+      return;
+    }
+    try {
+      await this.store.insert({ horizon: at, key });
+    } catch (error) {
+      if (!isDuplicateKey(error)) {
+        throw error;
+      }
+      await this.store.updateOne(behind, { $set: { horizon: at } });
+    }
+  }
+}

package/src/core/modules/sync/core-sync.constants.ts

@@ -0,0 +1,51 @@
+/**
+ * Constants of the sync module (offline sync, `docs/offline-sync-protocol.md`). Import-free leaf: tokens and numbers
+ * only, so every file of the module may import it without an import cycle.
+ */
+
+/** The framework model holding one entry per pushed mutation (claim, then outcome; protocol §6.5) */
+export const SYNC_MUTATION_MODEL = 'SyncMutation';
+
+/** The framework model holding the access epochs and the tombstone horizons (plan §3.7, §3.8) */
+export const SYNC_STATE_MODEL = 'SyncState';
+
+/** Metadata of `@SyncService()` on a project's CrudService class */
+export const SYNC_SERVICE_METADATA = 'lt:sync:service';
+
+/** Metadata of `@SyncField()`: field name → strategy, on the model class */
+export const SYNC_FIELD_METADATA = 'lt:sync:fields';
+
+/**
+ * A mutation claimed this recently is still being written (protocol §6.5): a repeat answers `retry`. An older claim
+ * belongs to a request that died between the claim and its outcome, and is taken over.
+ */
+export const SYNC_CLAIM_TIMEOUT_MS = 30_000;
+
+/** How often a write applied on top of the current state (`lww`, `merge`, `custom`) is tried before `retry` */
+export const SYNC_REAPPLY_ATTEMPTS = 3;
+
+/** Defaults of the `sync` configuration (plan §3.10) */
+export const SYNC_DEFAULTS = Object.freeze({
+  hintDebounceMs: 250,
+  maxBytes: 1_048_576,
+  maxItems: 100,
+  maxLimit: 1000,
+  mutationsDays: 30,
+  path: 'sync',
+  pullLimit: 200,
+  rateLimitPull: { max: 120, windowSeconds: 60 },
+  rateLimitPush: { max: 60, windowSeconds: 60 },
+  tombstonesDays: 90,
+});
+
+/** How often the tombstone purge runs (on one replica per run, under a lease; plan §3.7, E8) */
+/** Tombstones per purge statement: small enough that a statement never holds many row locks at once */
+export const SYNC_PURGE_BATCH = 1000;
+
+export const SYNC_PURGE_INTERVAL_MS = 60 * 60_000;
+
+/** How often an open hint stream sends a comment, so proxies and load balancers keep it open */
+export const SYNC_HINT_KEEPALIVE_MS = 25_000;
+
+/** The hard upper bound of `sync.pull.maxLimit` (protocol §5.1) */
+export const SYNC_PULL_LIMIT_CEILING = 1000;

package/src/core/modules/sync/core-sync.controller.ts

@@ -0,0 +1,68 @@
+import { Body, Controller, Get, HttpCode, HttpStatus, Param, Post, Query, Req, Res } from '@nestjs/common';
+import { ApiBearerAuth, ApiOperation, ApiTags } from '@nestjs/swagger';
+
+import { CurrentUser } from '../../common/decorators/current-user.decorator.js';
+import { Roles } from '../../common/decorators/roles.decorator.js';
+import { RoleEnum } from '../../common/enums/role.enum.js';
+import { CoreSyncBaseController } from './core-sync-base.controller.js';
+import { SYNC_DEFAULTS } from './core-sync.constants.js';
+import { SyncResyncRequiredException } from './core-sync.service.js';
+import type { ISyncPullResult, ISyncPushResult } from './interfaces/core-sync.interfaces.js';
+
+/**
+ * The sync routes (protocol `docs/offline-sync-protocol.md` §4–§6): `GET /sync/{model}` pulls a page of changes,
+ * `POST /sync/{model}` pushes changes. Both need a signed-in caller — a session cookie or a bearer token, exactly as on
+ * every other route; the model's own operation roles decide the rest (`@PersistedModel({ sync: { roles } })`). The
+ * change hints (`GET /sync/hints`) are `CoreSyncHintsController`'s, registered before this one.
+ *
+ * API tokens are refused here like on every route without `@ApiTokenScopes()`. A project that wants them extends the
+ * controller, re-declares the handlers with `@ApiTokenScopes('sync')` and passes it as `sync.controller` override.
+ */
+@ApiTags('sync')
+@Controller(SYNC_DEFAULTS.path)
+@Roles(RoleEnum.S_USER)
+export class CoreSyncController extends CoreSyncBaseController {
+  @ApiBearerAuth()
+  @ApiOperation({ description: 'One page of changes after the cursor (protocol §5)', summary: 'Pull changes' })
+  @Get(':model')
+  @Roles(RoleEnum.S_USER)
+  async pull(
+    @Param('model') model: string,
+    @Query('cursor') cursor: string,
+    @Query('limit') limit: string,
+    @CurrentUser() currentUser: any,
+    @Req() req: any,
+    @Res({ passthrough: true }) res: any,
+  ): Promise<ISyncPullResult | object> {
+    await this.limit('pull', currentUser, res);
+    await this.scopeTenants(req, currentUser);
+    try {
+      return await this.syncService.pull(model, { cursor, limit }, currentUser);
+    } catch (error) {
+      if (error instanceof SyncResyncRequiredException) {
+        // The protocol's body (protocol §8), not the shape of whatever exception filter the project registered
+        res?.status?.(error.getStatus());
+        return error.getResponse() as object;
+      }
+      throw error;
+    }
+  }
+
+  @ApiBearerAuth()
+  @ApiOperation({ description: 'Apply pushed changes item by item (protocol §6)', summary: 'Push changes' })
+  @HttpCode(HttpStatus.OK)
+  @Post(':model')
+  @Roles(RoleEnum.S_USER)
+  async push(
+    @Param('model') model: string,
+    @Body() body: any,
+    @CurrentUser() currentUser: any,
+    @Req() req: any,
+    @Res({ passthrough: true }) res: any,
+  ): Promise<{ results: ISyncPushResult[] }> {
+    this.syncService.assertPushRequest(req);
+    await this.limit('push', currentUser, res);
+    await this.scopeTenants(req, currentUser);
+    return this.syncService.push(model, body, currentUser);
+  }
+}

package/src/core/modules/sync/core-sync.decorators.ts

@@ -0,0 +1,84 @@
+import type { Type } from '@nestjs/common';
+
+import { SYNC_FIELD_METADATA, SYNC_SERVICE_METADATA } from './core-sync.constants.js';
+
+/** What `@SyncService()` declares: the model it serves under `/sync` and the inputs a push is validated against */
+export interface SyncServiceOptions {
+  /** Input of a push that creates a record — validated and whitelisted like a request body */
+  createInput: Type<any>;
+  /** The model class, declared with `@PersistedModel({ sync })` */
+  model: Type<any>;
+  /**
+   * The name the model is registered under in `CorePersistenceModule.forFeature()`, which is also its name in the sync
+   * path (`/sync/{model}`)
+   * @default the model class's name
+   */
+  modelName?: string;
+  /** Input of a push that changes a record */
+  updateInput: Type<any>;
+}
+
+/**
+ * Offer a model to offline clients through the project's CrudService: `GET/POST /sync/{model}` read and write through
+ * this service — its `process()` pipeline, `securityCheck()`, the tenant policy — with the operation roles the model
+ * declares in `@PersistedModel({ sync: { roles } })`. Needs the `sync` configuration; without it nothing is registered.
+ *
+ * @example
+ * ```typescript
+ * @SyncService({ createInput: NoteCreateInput, model: Note, updateInput: NoteInput })
+ * @Injectable()
+ * export class NoteService extends CrudService<Note, NoteCreateInput, NoteInput> { … }
+ * ```
+ */
+export function SyncService(options: SyncServiceOptions): ClassDecorator {
+  return (target) => {
+    Reflect.defineMetadata(SYNC_SERVICE_METADATA, { ...options }, target);
+  };
+}
+
+/** The `@SyncService()` declaration of a service class (inherited from a parent class), if any */
+export function getSyncServiceOptions(target: Function): SyncServiceOptions | undefined {
+  return Reflect.getMetadata(SYNC_SERVICE_METADATA, target);
+}
+
+/**
+ * How a field takes part in a `merge` (protocol §6.4): `lww` — the client's value wins when it changed the field;
+ * `reject` — a client change of this field still conflicts when the record changed in the meantime.
+ */
+export type SyncFieldStrategy = 'lww' | 'reject';
+
+/**
+ * Declare how a field is merged when the model's strategy is `merge`. Fields without the decorator merge as `lww`.
+ *
+ * @example
+ * ```typescript
+ * @SyncField('reject')
+ * @UnifiedField({ description: 'Price, never merged silently' })
+ * price?: number;
+ * ```
+ */
+export function SyncField(strategy: SyncFieldStrategy): PropertyDecorator {
+  if (strategy !== 'lww' && strategy !== 'reject') {
+    throw new Error(`@SyncField: strategy must be 'lww' or 'reject', got ${JSON.stringify(strategy)}`);
+  }
+  return (target, property) => {
+    const own: Record<string, SyncFieldStrategy> = Reflect.hasOwnMetadata(SYNC_FIELD_METADATA, target.constructor)
+      ? Reflect.getOwnMetadata(SYNC_FIELD_METADATA, target.constructor)
+      : {};
+    Reflect.defineMetadata(SYNC_FIELD_METADATA, { ...own, [String(property)]: strategy }, target.constructor);
+  };
+}
+
+/** The `@SyncField()` strategies of a model class, its parents' included (a child's declaration wins) */
+export function getSyncFieldStrategies(modelClass: Function): Record<string, SyncFieldStrategy> {
+  const chain: Function[] = [];
+  for (let current = modelClass; current && current !== Function.prototype; current = Object.getPrototypeOf(current)) {
+    chain.unshift(current);
+  }
+  return Object.assign(
+    {},
+    ...chain.map((current) =>
+      Reflect.hasOwnMetadata(SYNC_FIELD_METADATA, current) ? Reflect.getOwnMetadata(SYNC_FIELD_METADATA, current) : {},
+    ),
+  );
+}

package/src/core/modules/sync/core-sync.module.ts

@@ -0,0 +1,75 @@
+import { DynamicModule, Module, Type } from '@nestjs/common';
+import { PATH_METADATA } from '@nestjs/common/constants';
+import { DiscoveryModule } from '@nestjs/core';
+
+import { CorePersistenceModule } from '../../common/persistence/core-persistence.module.js';
+import { SYNC_DEFAULTS, SYNC_MUTATION_MODEL, SYNC_STATE_MODEL } from './core-sync.constants.js';
+import { CoreSyncController } from './core-sync.controller.js';
+import { CoreSyncHintsController } from './core-sync-hints.controller.js';
+import { CoreSyncHintService } from './core-sync-hints.service.js';
+import { SyncMutationSchema, SyncStateSchema } from './core-sync-infrastructure.models.js';
+import { CoreSyncResolver } from './core-sync.resolver.js';
+import { CoreSyncService } from './core-sync.service.js';
+
+/** Options for `CoreSyncModule.forRoot()` */
+export interface CoreSyncModuleOptions {
+  /** Custom controller (must extend CoreSyncController) */
+  controller?: Type<CoreSyncController>;
+  /**
+   * Register the GraphQL hint subscription `syncHint` — `false` where GraphQL is disabled or `sync.hints.graphql` /
+   * `sync.hints` is `false`
+   * @default true
+   */
+  graphQlHints?: boolean;
+  /** Custom controller of `GET /sync/hints` (must extend CoreSyncHintsController) */
+  hintsController?: Type<CoreSyncHintsController>;
+  /** Route prefix (`sync.path`) @default 'sync' */
+  path?: string;
+  /** Custom resolver of the hint subscription (must extend CoreSyncResolver) */
+  resolver?: Type<CoreSyncResolver>;
+  /** Custom service (must extend CoreSyncService) */
+  service?: Type<CoreSyncService>;
+}
+
+/**
+ * Offline sync (`sync` config, protocol `docs/offline-sync-protocol.md`).
+ *
+ * Registered automatically by `CoreModule.forRoot()` when `sync` is configured. Provides the sync routes, the change
+ * hints (SSE and, with GraphQL, the `syncHint` subscription), the service, the tombstone purge and the module's two
+ * infrastructure models (mutations; epochs and horizons), which follow `database` like every
+ * other model. Which models are offered is decided by the project: a CrudService with `@SyncService()` whose model is
+ * declared with `@PersistedModel({ sync })`.
+ */
+@Module({})
+export class CoreSyncModule {
+  static forRoot(options: CoreSyncModuleOptions = {}): DynamicModule {
+    const controller = options.controller ?? CoreSyncController;
+    const hintsController = options.hintsController ?? CoreSyncHintsController;
+    // The configured prefix wins over the class's declaration, as for the TUS routes
+    for (const routes of [hintsController, controller]) {
+      Reflect.defineMetadata(PATH_METADATA, options.path || SYNC_DEFAULTS.path, routes);
+    }
+    return {
+      // The hints first: Nest registers routes in this order, and `GET /sync/hints` must win over `GET /sync/:model`
+      // whatever a project's controller re-declares
+      controllers: [hintsController, controller],
+      exports: [CoreSyncHintService, CoreSyncService],
+      // Global, as the tenant and API-token modules: a project's own service injects CoreSyncService to report writes
+      // past the CrudService (`notifyChange()`, `bumpAccessEpoch()`), and CoreModule does not re-export its imports
+      global: true,
+      imports: [
+        DiscoveryModule,
+        CorePersistenceModule.forFeature([
+          { name: SYNC_MUTATION_MODEL, schema: SyncMutationSchema },
+          { name: SYNC_STATE_MODEL, schema: SyncStateSchema },
+        ]),
+      ],
+      module: CoreSyncModule,
+      providers: [
+        CoreSyncHintService,
+        options.service ? { provide: CoreSyncService, useClass: options.service } : CoreSyncService,
+        ...(options.graphQlHints === false ? [] : [options.resolver ?? CoreSyncResolver]),
+      ],
+    };
+  }
+}

package/src/core/modules/sync/core-sync.resolver.ts

@@ -0,0 +1,35 @@
+import { Args, Resolver, Subscription } from '@nestjs/graphql';
+
+import { GraphQLUser } from '../../common/decorators/graphql-user.decorator.js';
+import { Roles } from '../../common/decorators/roles.decorator.js';
+import { RoleEnum } from '../../common/enums/role.enum.js';
+import { CoreSyncService } from './core-sync.service.js';
+import { CoreSyncHintModel } from './models/core-sync-hint.model.js';
+
+/**
+ * The GraphQL hint channel of offline sync (protocol §9): `subscription { syncHint(models: ["Note"]) { model at } }`
+ * (`models` optional), registered when
+ * GraphQL is enabled and `sync.hints.graphql` is not `false`. Hints reach only subscribers with the model's pull roles,
+ * for changes in their own tenants; the WebSocket it runs on is re-checked by `CoreModule`, so changed rights close it.
+ *
+ * Extend it and pass it as `CoreModule.forRoot(env, { sync: { resolver } })` to change the channel — re-declare
+ * `@Subscription()` and `@Roles()` on an override, as on every resolver.
+ */
+@Resolver()
+@Roles(RoleEnum.S_USER)
+export class CoreSyncResolver {
+  constructor(protected readonly syncService: CoreSyncService) {}
+
+  @Roles(RoleEnum.S_USER)
+  @Subscription(() => CoreSyncHintModel, {
+    description: 'A hint whenever a synchronized model the subscriber may pull changes',
+    name: 'syncHint',
+  })
+  syncHint(
+    @GraphQLUser() currentUser: any,
+    @Args('models', { description: 'Only these models (all when absent)', nullable: true, type: () => [String] })
+    models?: string[],
+  ): Promise<AsyncIterableIterator<{ syncHint: CoreSyncHintModel }>> {
+    return this.syncService.hintIterator(currentUser, models);
+  }
+}