@lenne.tech/nest-server 12.1.1 → 12.2.0

package/src/core/common/persistence/sql/postgres/postgres-schema.compiler.ts

@@ -2,6 +2,7 @@ import type { FieldDescriptor, IndexDescriptor, ModelDescriptor } from '../../mo
 import { UnsupportedQueryError } from '../sql-errors.js';
 import { columnName, quoteIdentifier, SqlLiterals, sqlLiteral } from '../sql-naming.js';
 import { schemaHash, shortName, tableName } from '../sql-schema.shared.js';
+import { isLiveFilter, SYNC_SEQUENCE_FIELD } from '../../sync-model.js';
 import { compilePostgresWhere } from './postgres-where.compiler.js';
 
 export { LOCK_MODEL, shortName, tableName } from '../sql-schema.shared.js';
@@ -23,6 +24,15 @@ export { LOCK_MODEL, shortName, tableName } from '../sql-schema.shared.js';
 /** Any string item that starts with `s_` after leading whitespace, case-insensitive (`looksLikeSystemRole`) */
 const SYSTEM_ROLE_PATH = '$[*] ? (@.type() == "string" && @ like_regex "^\\\\s*s_" flag "i")';
 
+/**
+ * Bumped whenever the change-order trigger (`lt_sync_stamp`) changes, so the migrator replaces the function and
+ * every table's trigger — the trigger's name carries the version, because a trigger has no comment the catalog query
+ * could read alongside the table
+ */
+export const SYNC_TRIGGER_VERSION = 'lt:sync:v1';
+export const SYNC_STAMP_FUNCTION = 'lt_sync_stamp';
+export const SYNC_TRIGGER = 'lt_sync_stamp_v1';
+
 /** Bumped whenever the generated guard function or policies change, so the migrator replaces them */
 export const RLS_VERSION = 'lt:tenant:v1';
 export const TENANT_FILTER_FUNCTION = 'lt_tenant_filter';
@@ -41,6 +51,12 @@ export interface DesiredColumn {
   renamedFrom?: string;
 }
 
+/** The objects that number a synchronized model's changes (offline-sync plan §3.4) */
+export interface DesiredSync {
+  /** The table's sequence: `nextval()` under the model's advisory lock gives commit order */
+  sequence: string;
+}
+
 export interface DesiredIndex {
   /** The CREATE statement */
   create: string;
@@ -61,6 +77,8 @@ export interface DesiredTable {
   name: string;
   /** What the model declares but the table cannot carry (text indexes, …), reported once */
   notices: string[];
+  /** Present for a synchronized model: its change-order sequence and trigger */
+  sync?: DesiredSync;
   /** Row-level security on `tenant_id` */
   tenantIsolated: boolean;
 }
@@ -119,7 +137,13 @@ export function desiredTable(model: ModelDescriptor, options: PostgresSchemaOpti
     indexes.push(desiredIndex(model, name, { fields: { tenantId: 1 }, options: {} }) as DesiredIndex);
   }
 
-  return { checks, columns, indexes, model, name, notices, tenantIsolated };
+  const sync = model.sync && model.fields[SYNC_SEQUENCE_FIELD] ? { sequence: syncSequenceName(name) } : undefined;
+  return { checks, columns, indexes, model, name, notices, ...(sync ? { sync } : {}), tenantIsolated };
+}
+
+/** The change-order sequence of a synchronized model's table */
+export function syncSequenceName(table: string): string {
+  return shortName('lt_sync', table, ['seq']);
 }
 
 function desiredColumn(field: FieldDescriptor): DesiredColumn {
@@ -223,7 +247,9 @@ function desiredIndex(model: ModelDescriptor, table: string, index: IndexDescrip
   const using = gin.length
     ? ` USING gin (${gin[0].expression} jsonb_path_ops)`
     : ` (${parts.map((part) => part.sql).join(', ')})`;
-  const nulls = unique && !conditions.length ? ' NULLS NOT DISTINCT' : '';
+  // A partial index admits NULL repeatedly unless told otherwise. That is harmless where the condition excludes NULL
+  // (sparse), but a synchronized model's live-record condition keeps MongoDB's rule: a second live NULL collides
+  const nulls = unique && (!conditions.length || (!sparse && isLiveFilter(partial))) ? ' NULLS NOT DISTINCT' : '';
   const where = conditions.length ? ` WHERE ${conditions.join(' AND ')}` : '';
   return {
     create: `CREATE ${unique ? 'UNIQUE ' : ''}INDEX ${quoteIdentifier(name)} ON ${quoteIdentifier(table)}${using}${nulls}${where}`,
@@ -280,6 +306,52 @@ export function grantStatement(table: string, appRole: string): string {
   return `GRANT SELECT, INSERT, UPDATE, DELETE ON ${quoteIdentifier(table)} TO ${quoteIdentifier(appRole)}`;
 }
 
+/**
+ * The trigger function that numbers a synchronized model's writes (offline-sync plan §3.4, C1d; DEV-3900 measured
+ * the advisory lock against a counter row). A BEFORE ROW trigger runs after the row is locked, so a write takes its
+ * row first and the order last; the transaction-scoped advisory lock is released only after the commit is visible,
+ * so sequence order is commit order, and `nextval()` leaves no row version a long snapshot could pile up.
+ *
+ * A value the statement sets itself is kept: the store's multi-row writes lock their rows, take their numbers and set
+ * them explicitly in their final statement.
+ */
+export function syncFunctionStatements(): string[] {
+  const fn = quoteIdentifier(SYNC_STAMP_FUNCTION);
+  return [
+    `CREATE OR REPLACE FUNCTION ${fn}() RETURNS trigger
+LANGUAGE plpgsql SET search_path = pg_catalog AS $lt$
+BEGIN
+  IF TG_OP = 'INSERT' THEN
+    IF NEW.sync_seq IS NOT NULL THEN
+      RETURN NEW;
+    END IF;
+  ELSIF NEW.sync_seq IS DISTINCT FROM OLD.sync_seq THEN
+    RETURN NEW;
+  END IF;
+  PERFORM pg_advisory_xact_lock(hashtext('lenne.tech:sync:' || TG_TABLE_SCHEMA || '.' || TG_TABLE_NAME));
+  NEW.sync_seq := nextval(format('%I.%I', TG_TABLE_SCHEMA, TG_ARGV[0])::regclass);
+  RETURN NEW;
+END
+$lt$`,
+    `COMMENT ON FUNCTION ${fn}() IS ${sqlLiteral(SYNC_TRIGGER_VERSION)}`,
+  ];
+}
+
+/**
+ * A synchronized table's sequence, the application role's use of it, the trigger, and the numbering of rows written
+ * before the table was synchronized (in the order of their ids, i.e. of their creation)
+ */
+export function syncTableStatements(table: string, sync: DesiredSync, appRole: string): string[] {
+  const quoted = quoteIdentifier(table);
+  const sequence = quoteIdentifier(sync.sequence);
+  return [
+    `CREATE SEQUENCE IF NOT EXISTS ${sequence}`,
+    `GRANT USAGE ON SEQUENCE ${sequence} TO ${quoteIdentifier(appRole)}`,
+    `CREATE TRIGGER ${quoteIdentifier(SYNC_TRIGGER)} BEFORE INSERT OR UPDATE ON ${quoted} FOR EACH ROW EXECUTE FUNCTION ${quoteIdentifier(SYNC_STAMP_FUNCTION)}(${sqlLiteral(sync.sequence)})`,
+    `UPDATE ${quoted} AS t SET "sync_seq" = n.seq FROM (SELECT "id", nextval(${sqlLiteral(sync.sequence)}::regclass) AS seq FROM (SELECT "id" FROM ${quoted} WHERE "sync_seq" IS NULL ORDER BY "id") AS o) AS n WHERE t."id" = n."id"`,
+  ];
+}
+
 /**
  * The guard every tenant policy calls. It reads the scope the store wrote into the transaction (`sql-session.ts`)
  * and RAISES `42501` without one, so a query on a tenant table outside the framework's context — a forgotten scope,

package/src/core/common/persistence/sql/postgres/postgres-schema.migrator.ts

@@ -18,6 +18,11 @@ import {
   grantStatement,
   type PostgresSchemaOptions,
   RLS_VERSION,
+  SYNC_STAMP_FUNCTION,
+  SYNC_TRIGGER,
+  SYNC_TRIGGER_VERSION,
+  syncFunctionStatements,
+  syncTableStatements,
   TENANT_FILTER_FUNCTION,
   TENANT_POLICY,
   tenantFilterFunctionStatements,
@@ -63,10 +68,15 @@ interface CatalogTable {
   granted: boolean;
   policyVersion: null | string;
   rls: boolean;
+  /** Change-order triggers (`lt_sync_*`) on the table */
+  syncTriggers: Set<string>;
 }
 
 interface Catalog {
   functionVersion: null | string;
+  /** Sequences of synchronized tables → whether the application role may use them */
+  sequences: Map<string, boolean>;
+  syncFunctionVersion: null | string;
   schema: string;
   /** Whether the application role may use the schema */
   schemaUsage: boolean;
@@ -124,9 +134,18 @@ export async function planPostgresSchema(
     run,
     tables.map((table) => table.name),
     options.appRole,
+    tables.flatMap((table) => (table.sync ? [table.sync.sequence] : [])),
   );
   const changes: SchemaChange[] = [];
 
+  if (tables.some((table) => table.sync) && catalog.syncFunctionVersion !== SYNC_TRIGGER_VERSION) {
+    changes.push({
+      description: `${catalog.syncFunctionVersion ? 'update' : 'create'} the change-order function ${SYNC_STAMP_FUNCTION}()`,
+      kind: 'automatic',
+      statements: syncFunctionStatements(),
+    });
+  }
+
   if (tables.some((table) => table.tenantIsolated) && catalog.functionVersion !== RLS_VERSION) {
     changes.push({
       description: `${catalog.functionVersion ? 'update' : 'create'} the tenant guard function ${TENANT_FILTER_FUNCTION}()`,
@@ -146,7 +165,9 @@ export async function planPostgresSchema(
       changes.push({ description: notice, kind: 'notice', statements: [], table: table.name });
     }
     const existing = catalog.tables.get(table.name);
-    changes.push(...(existing ? planExistingTable(table, existing, options) : planNewTable(table, options)));
+    changes.push(
+      ...(existing ? planExistingTable(table, existing, catalog.sequences, options) : planNewTable(table, options)),
+    );
   }
   return changes;
 }
@@ -161,6 +182,7 @@ function planNewTable(table: DesiredTable, options: PostgresSchemaOptions): Sche
         ...table.indexes.map((index) => index.create),
         ...(table.tenantIsolated ? tenantPolicyStatements(table.name) : []),
         grantStatement(table.name, options.appRole),
+        ...(table.sync ? syncTableStatements(table.name, table.sync, options.appRole) : []),
       ],
       table: table.name,
     },
@@ -170,6 +192,7 @@ function planNewTable(table: DesiredTable, options: PostgresSchemaOptions): Sche
 function planExistingTable(
   table: DesiredTable,
   existing: CatalogTable,
+  sequences: Map<string, boolean>,
   options: PostgresSchemaOptions,
 ): SchemaChange[] {
   const changes: SchemaChange[] = [];
@@ -249,7 +272,7 @@ function planExistingTable(
       description: `create ${index.unique ? 'unique ' : ''}index ${index.name} on ${name}`,
       // A unique index can fail on the rows already stored, and it changes what may be written: a reviewed decision
       kind: index.unique ? 'migration' : 'automatic',
-      statements: [index.create],
+      statements: index.unique ? liveUniqueMigration(table, existing, index, wanted) : [index.create],
       table: name,
     });
   }
@@ -294,9 +317,97 @@ function planExistingTable(
       table: name,
     });
   }
+  changes.push(...planSync(table, existing, sequences, options));
   return changes;
 }
 
+/**
+ * A synchronized table's change order: the trigger of the current version, its sequence and the application role's
+ * use of it. A trigger of an earlier version is replaced; a table that is no longer synchronized keeps its trigger
+ * (and its numbering) until a migration removes it
+ */
+function planSync(
+  table: DesiredTable,
+  existing: CatalogTable,
+  sequences: Map<string, boolean>,
+  options: PostgresSchemaOptions,
+): SchemaChange[] {
+  const name = table.name;
+  const quoted = quoteIdentifier(name);
+  if (!table.sync) {
+    return existing.syncTriggers.size
+      ? [
+          {
+            description: `${name} is no longer synchronized; its change-order trigger is kept until a migration drops it`,
+            kind: 'notice',
+            statements: [],
+            table: name,
+          },
+        ]
+      : [];
+  }
+  if (!existing.syncTriggers.has(SYNC_TRIGGER)) {
+    const outdated = [...existing.syncTriggers].map(
+      (trigger) => `DROP TRIGGER IF EXISTS ${quoteIdentifier(trigger)} ON ${quoted}`,
+    );
+    return [
+      {
+        description: `${outdated.length ? 'replace' : 'add'} the change-order trigger of ${name}`,
+        kind: 'automatic',
+        statements: [...outdated, ...syncTableStatements(name, table.sync, options.appRole)],
+        table: name,
+      },
+    ];
+  }
+  if (!sequences.get(table.sync.sequence)) {
+    return [
+      {
+        description: `let ${options.appRole} use the change-order sequence of ${name}`,
+        kind: 'automatic',
+        statements: syncTableStatements(name, table.sync, options.appRole).slice(0, 2),
+        table: name,
+      },
+    ];
+  }
+  return [];
+}
+
+/**
+ * The statements of a new unique index's migration. A synchronized model's index reads `deleted_at`: when the table
+ * does not have it yet (an existing model that became synchronized), the column comes first — the boot that refuses the
+ * index never adds it. A framework unique index over the same fields that the model no longer declares is the one the
+ * live-only index replaces: it goes last, because a tombstone (which keeps no values) would collide on it
+ */
+function liveUniqueMigration(
+  table: DesiredTable,
+  existing: CatalogTable,
+  index: DesiredTable['indexes'][number],
+  wanted: Set<string>,
+): string[] {
+  const statements: string[] = [];
+  const deletedAt = table.columns.find((column) => column.name === 'deleted_at');
+  if (table.sync && deletedAt && !existing.columns.has('deleted_at') && index.create.includes('"deleted_at"')) {
+    statements.push(
+      `ALTER TABLE ${quoteIdentifier(table.name)} ADD COLUMN ${columnDefinition({ ...deletedAt, primaryKey: false })}`,
+    );
+  }
+  statements.push(index.create);
+  if (table.sync) {
+    const stem = indexStem(index.name);
+    for (const [old, unique] of existing.indexes) {
+      if (unique && !wanted.has(old) && indexStem(old) === stem) {
+        statements.push(`DROP INDEX IF EXISTS ${quoteIdentifier(old)}`);
+      }
+    }
+  }
+  return statements;
+}
+
+/** A framework index name without its definition hash (`ux_<table>__<fields>_<hash>`) */
+function indexStem(name: string): string {
+  return name.replace(/_[0-9a-f]{8}$/, '');
+}
+
 function sameType(
   column: DesiredTable['columns'][number],
   current: { collation: null | string; type: string },
@@ -321,7 +432,7 @@ function typeChange(
   };
 }
 
-async function readCatalog(run: Run, tables: string[], appRole: string): Promise<Catalog> {
+async function readCatalog(run: Run, tables: string[], appRole: string, sequences: string[]): Promise<Catalog> {
   // One query after the other: inside the migration transaction they share a single connection
   const schemaResult = await run({
     params: [appRole],
@@ -359,14 +470,31 @@ async function readCatalog(run: Run, tables: string[], appRole: string): Promise
       WHERE n.nspname = current_schema() AND c.relname = ANY($1) AND k.contype = 'c'`,
   });
   const functionResult = await run({
-    params: [TENANT_FILTER_FUNCTION],
-    text: `SELECT obj_description(p.oid, 'pg_proc') AS version
+    params: [[TENANT_FILTER_FUNCTION, SYNC_STAMP_FUNCTION]],
+    text: `SELECT p.proname AS name, obj_description(p.oid, 'pg_proc') AS version
       FROM pg_proc p JOIN pg_namespace n ON n.oid = p.pronamespace
-      WHERE n.nspname = current_schema() AND p.proname = $1 AND p.pronargs = 0`,
+      WHERE n.nspname = current_schema() AND p.proname = ANY($1) AND p.pronargs = 0`,
   });
+  const triggerResult = await run({
+    params: [tables],
+    text: `SELECT c.relname AS table, t.tgname AS name
+      FROM pg_trigger t JOIN pg_class c ON c.oid = t.tgrelid JOIN pg_namespace n ON n.oid = c.relnamespace
+      WHERE n.nspname = current_schema() AND c.relname = ANY($1) AND NOT t.tgisinternal AND t.tgname LIKE 'lt\\_sync\\_%'`,
+  });
+  const sequenceResult = sequences.length
+    ? await run({
+        params: [sequences, appRole],
+        text: `SELECT c.relname AS name, has_sequence_privilege($2, c.oid, 'USAGE') AS usage
+          FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace
+          WHERE n.nspname = current_schema() AND c.relkind = 'S' AND c.relname = ANY($1)`,
+      })
+    : { rowCount: 0, rows: [] };
+  const versionOf = (fn: string) => functionResult.rows.find((row) => row.name === fn)?.version ?? null;
 
   const catalog: Catalog = {
-    functionVersion: functionResult.rows[0]?.version ?? null,
+    functionVersion: versionOf(TENANT_FILTER_FUNCTION),
+    sequences: new Map(sequenceResult.rows.map((row: { name: string; usage: boolean }) => [row.name, row.usage])),
+    syncFunctionVersion: versionOf(SYNC_STAMP_FUNCTION),
     schema: schemaResult.rows[0].schema,
     schemaUsage: schemaResult.rows[0].usage,
     tables: new Map(),
@@ -380,8 +508,12 @@ async function readCatalog(run: Run, tables: string[], appRole: string): Promise
       indexes: new Map(),
       policyVersion: row.policy_version ?? null,
       rls: row.rls,
+      syncTriggers: new Set(),
     });
   }
+  for (const row of triggerResult.rows) {
+    catalog.tables.get(row.table)?.syncTriggers.add(row.name);
+  }
   for (const row of columnResult.rows) {
     catalog.tables.get(row.table)?.columns.set(row.name, { collation: row.collation ?? null, type: row.type });
   }