@syncular/server 0.8.0 → 0.10.0

package/src/push.ts

@@ -25,6 +25,7 @@ import {
   type PushOperationResult,
   type PushResultFrame,
   parseBlobRef,
+  type RejectionDetails,
   type RowValue,
 } from '@syncular/core';
 import type { BlobStore } from './blob-store';
@@ -41,8 +42,18 @@ import type {
   StoredCommit,
   StoredPushResult,
 } from './storage';
-import type { ValidateOpKind, ValidatorRegistry } from './validate';
-import { toValidateRow, ValidationRejection } from './validate';
+import type {
+  CommitValidationReader,
+  CommitValidator,
+  ValidateCommitOperation,
+  ValidateOpKind,
+  ValidatorRegistry,
+} from './validate';
+import {
+  CommitValidationRejection,
+  toValidateRow,
+  ValidationRejection,
+} from './validate';
 
 /**
  * Extract the blobIds a decoded row references through its `blob_ref`
@@ -62,7 +73,11 @@ function blobIdsInRow(
 }
 
 type OperationOutcome =
-  | { readonly kind: 'applied'; readonly change: NewChange | undefined }
+  | {
+      readonly kind: 'applied';
+      readonly change: NewChange | undefined;
+      readonly operation: ValidateCommitOperation;
+    }
   | { readonly kind: 'terminate'; readonly record: PushOperationResult };
 
 function errorRecord(
@@ -70,10 +85,18 @@ function errorRecord(
   code: string,
   message: string,
   retryable = false,
+  details?: RejectionDetails,
 ): OperationOutcome {
   return {
     kind: 'terminate',
-    record: { opIndex, status: 'error', code, message, retryable },
+    record: {
+      opIndex,
+      status: 'error',
+      code,
+      message,
+      retryable,
+      ...(details !== undefined ? { details } : {}),
+    },
   };
 }
 
@@ -141,7 +164,13 @@ async function runValidator(
     );
   } catch (error) {
     if (error instanceof ValidationRejection) {
-      return errorRecord(opIndex, error.code, error.message);
+      return errorRecord(
+        opIndex,
+        error.code,
+        error.message,
+        false,
+        error.details,
+      );
     }
     // §6.7: a non-ValidationRejection throw is still a rejection, mapped to
     // the generic server-side constraint code (§10.2) — the validator's
@@ -237,7 +266,18 @@ async function applyOperation(
   if (op.op === 'delete') {
     if (stored === undefined) {
       // Deleting an absent row is applied (idempotent, §6.2); no change.
-      return { kind: 'applied', change: undefined };
+      return {
+        kind: 'applied',
+        change: undefined,
+        operation: {
+          opIndex,
+          op: 'delete',
+          table: table.name,
+          rowId: op.rowId,
+          row: undefined,
+          stored: undefined,
+        },
+      };
     }
     if (!authorizeWrite(table, stored.scopes, resolved)) {
       return errorRecord(
@@ -257,13 +297,14 @@ async function applyOperation(
     // §6.7: validate the delete against the stored row (row = undefined,
     // stored = the row about to be removed). Only reached for an existing
     // row — an absent-row delete is an idempotent no-op above.
+    const storedValues = decodeRow(table.columns, stored.payload);
     const deleteReject = await runValidator(
       validators,
       table,
       'delete',
       op.rowId,
       undefined,
-      decodeRow(table.columns, stored.payload),
+      storedValues,
       opIndex,
       partition,
       actorId,
@@ -278,6 +319,15 @@ async function applyOperation(
         op: 'delete',
         scopes: stored.scopes,
       },
+      operation: {
+        opIndex,
+        op: 'delete',
+        table: table.name,
+        rowId: op.rowId,
+        row: undefined,
+        stored: toValidateRow(table.columns, storedValues),
+        storedServerVersion: stored.serverVersion,
+      },
     };
   }
 
@@ -396,6 +446,16 @@ async function applyOperation(
         scopes: stored.scopes,
         payload: newPayload,
       },
+      operation: {
+        opIndex,
+        op: 'upsert',
+        table: table.name,
+        rowId: op.rowId,
+        row: toValidateRow(table.columns, values),
+        stored: toValidateRow(table.columns, storedValues),
+        storedServerVersion: stored.serverVersion,
+        nextServerVersion: newVersion,
+      },
     };
   }
 
@@ -491,6 +551,15 @@ async function applyOperation(
       scopes: extracted.scopes,
       payload: insertPayload,
     },
+    operation: {
+      opIndex,
+      op: 'upsert',
+      table: table.name,
+      rowId: op.rowId,
+      row: toValidateRow(table.columns, values),
+      stored: undefined,
+      nextServerVersion: 1,
+    },
   };
 }
 
@@ -547,6 +616,121 @@ function missingScopeVariable(
   return undefined;
 }
 
+function commitValidationReader(
+  tx: StorageTransaction,
+  schema: CompiledSchema,
+): CommitValidationReader {
+  const tableFor = (name: string): CompiledTable => {
+    const table = schema.tables.get(name);
+    if (table === undefined) {
+      throw new Error(
+        `commit validator requested unknown table ${JSON.stringify(name)}`,
+      );
+    }
+    return table;
+  };
+  return {
+    getRow: async (tableName, rowId) => {
+      const table = tableFor(tableName);
+      const stored = await tx.getRow(tableName, rowId);
+      if (stored === undefined) return undefined;
+      return {
+        row: toValidateRow(
+          table.columns,
+          decodeRow(table.columns, stored.payload),
+        ),
+        serverVersion: stored.serverVersion,
+      };
+    },
+    scanRows: async ({
+      table: tableName,
+      scopeFilter,
+      afterRowId = null,
+      limit = 100,
+    }) => {
+      const table = tableFor(tableName);
+      if (!Number.isSafeInteger(limit) || limit < 1 || limit > 1_000) {
+        throw new Error(
+          'commit validator scan limit must be an integer from 1 to 1,000',
+        );
+      }
+      if (tx.scanRows === undefined) {
+        throw new Error(
+          'storage transaction does not support commit-validator scans',
+        );
+      }
+      const rows = await tx.scanRows({
+        table: tableName,
+        scopeFilter,
+        afterRowId,
+        limit,
+      });
+      return rows.map((stored) => ({
+        row: toValidateRow(
+          table.columns,
+          decodeRow(table.columns, stored.payload),
+        ),
+        serverVersion: stored.serverVersion,
+      }));
+    },
+  };
+}
+
+async function runCommitValidator(
+  validator: CommitValidator | undefined,
+  tx: StorageTransaction,
+  schema: CompiledSchema,
+  clientId: string,
+  clientCommitId: string,
+  actorId: string,
+  partition: string,
+  operations: readonly ValidateCommitOperation[],
+): Promise<OperationOutcome | undefined> {
+  if (validator === undefined) return undefined;
+  try {
+    await validator({
+      clientId,
+      clientCommitId,
+      actorId,
+      partition,
+      operations,
+      read: commitValidationReader(tx, schema),
+    });
+  } catch (error) {
+    if (error instanceof CommitValidationRejection) {
+      if (error.opIndex >= operations.length) {
+        return errorRecord(
+          0,
+          'sync.constraint_violation',
+          `commit validator rejection names unavailable opIndex ${error.opIndex}`,
+        );
+      }
+      return errorRecord(
+        error.opIndex,
+        error.code,
+        error.message,
+        false,
+        error.details,
+      );
+    }
+    if (error instanceof ValidationRejection) {
+      return errorRecord(
+        operations[0]?.opIndex ?? 0,
+        error.code,
+        error.message,
+        false,
+        error.details,
+      );
+    }
+    return errorRecord(
+      operations[0]?.opIndex ?? 0,
+      'sync.constraint_violation',
+      `whole-commit validator threw: ${error instanceof Error ? error.message : String(error)}`,
+    );
+  }
+  return undefined;
+}
+
 function resultFrame(
   clientCommitId: string,
   stored: StoredPushResult,
@@ -565,6 +749,26 @@ function resultFrame(
   };
 }
 
+function idempotencyCacheMissFrame(
+  clientCommitId: string,
+  error: SyncError,
+): PushResultFrame {
+  return {
+    type: 'PUSH_RESULT',
+    clientCommitId,
+    status: 'rejected',
+    results: [
+      {
+        opIndex: 0,
+        status: 'error',
+        code: 'sync.idempotency_cache_miss',
+        message: error.message,
+        retryable: true,
+      },
+    ],
+  };
+}
+
 export interface AppliedCommitEvent {
   readonly commit: StoredCommit;
 }
@@ -595,20 +799,7 @@ export async function processPushCommit(
     ) {
       // §6.3: answer the retryable cache-miss for this commit rather than
       // re-applying. Not persisted — a retry may find a readable record.
-      return {
-        type: 'PUSH_RESULT',
-        clientCommitId: frame.clientCommitId,
-        status: 'rejected',
-        results: [
-          {
-            opIndex: 0,
-            status: 'error',
-            code: 'sync.idempotency_cache_miss',
-            message: error.message,
-            retryable: true,
-          },
-        ],
-      };
+      return idempotencyCacheMissFrame(frame.clientCommitId, error);
     }
     throw error;
   }
@@ -620,10 +811,47 @@ export async function processPushCommit(
   const blobCtx: BlobApplyContext = { store: ctx.blobs, partition };
   const crdtMergers = ctx.crdtMergers;
   const validators = ctx.validators;
+  const commitValidator = ctx.commitValidator;
   const tx = await storage.begin(partition);
+  const commitRejectedPushResult = tx.commitRejectedPushResult?.bind(tx);
   try {
+    if (commitValidator !== undefined) {
+      if (
+        tx.lockPartitionForCommitValidation === undefined ||
+        commitRejectedPushResult === undefined
+      ) {
+        throw new Error(
+          'storage transaction does not support atomic whole-commit validation finalization',
+        );
+      }
+      await tx.lockPartitionForCommitValidation();
+      // The optimistic lookup above may have raced another request for the
+      // same idempotency key. Re-check after acquiring partition serialization
+      // so a concurrent duplicate never reruns the aggregate validator.
+      try {
+        const serializedPersisted = await storage.getPushResult(
+          partition,
+          clientId,
+          frame.clientCommitId,
+        );
+        if (serializedPersisted !== undefined) {
+          await tx.rollback();
+          return resultFrame(frame.clientCommitId, serializedPersisted, true);
+        }
+      } catch (error) {
+        if (
+          error instanceof SyncError &&
+          error.code === 'sync.idempotency_cache_miss'
+        ) {
+          await tx.rollback();
+          return idempotencyCacheMissFrame(frame.clientCommitId, error);
+        }
+        throw error;
+      }
+    }
     const results: PushOperationResult[] = [];
     const changes: NewChange[] = [];
+    const validatedOperations: ValidateCommitOperation[] = [];
     let terminated: PushOperationResult | undefined;
     for (let opIndex = 0; opIndex < frame.operations.length; opIndex++) {
       const op = frame.operations[opIndex];
@@ -645,24 +873,57 @@ export async function processPushCommit(
         break;
       }
       results.push({ opIndex, status: 'applied' });
+      validatedOperations.push(outcome.operation);
       if (outcome.change !== undefined) changes.push(outcome.change);
     }
 
+    if (terminated === undefined) {
+      const commitReject = await runCommitValidator(
+        commitValidator,
+        tx,
+        schema,
+        clientId,
+        frame.clientCommitId,
+        ctx.actorId,
+        partition,
+        validatedOperations,
+      );
+      if (commitReject?.kind === 'terminate') {
+        terminated = commitReject.record;
+      }
+    }
+
     if (terminated !== undefined) {
       // §6.3 rejected: only the terminating operation's record; §6.4:
       // every write of the commit rolls back.
-      await tx.rollback();
       const stored: StoredPushResult = {
         status: 'rejected',
         results: [terminated],
       };
-      const rejectionTx = await storage.begin(partition);
-      try {
-        await rejectionTx.putPushResult(clientId, frame.clientCommitId, stored);
-        await rejectionTx.commit();
-      } catch (error) {
-        await rejectionTx.rollback();
-        throw error;
+      if (commitValidator !== undefined) {
+        // Discard candidate rows and persist the rejection while retaining the
+        // same partition lock. This closes the duplicate-request race between
+        // rollback and the durable idempotency outcome.
+        if (commitRejectedPushResult === undefined) {
+          throw new Error(
+            'storage transaction lost whole-commit rejection finalization support',
+          );
+        }
+        await commitRejectedPushResult(clientId, frame.clientCommitId, stored);
+      } else {
+        await tx.rollback();
+        const rejectionTx = await storage.begin(partition);
+        try {
+          await rejectionTx.putPushResult(
+            clientId,
+            frame.clientCommitId,
+            stored,
+          );
+          await rejectionTx.commit();
+        } catch (error) {
+          await rejectionTx.rollback();
+          throw error;
+        }
       }
       return resultFrame(frame.clientCommitId, stored, false);
     }

package/src/realtime.ts

@@ -50,11 +50,16 @@ import {
 import type { SegmentStore } from './segment-store';
 import type { SegmentUrlConfig } from './signed-url';
 import type { ServerStorage, StoredCommit } from './storage';
+import type { CommitValidator, ValidatorRegistry } from './validate';
 
 export interface RealtimeHubConfig {
   readonly schema: ServerSchema;
   readonly storage: ServerStorage;
   readonly resolveScopes: ResolveScopes;
+  /** §6.7 validators used by sync rounds carried over this socket. */
+  readonly validators?: ValidatorRegistry;
+  /** §6.8 whole-commit validator shared with HTTP sync rounds. */
+  readonly commitValidator?: CommitValidator;
   readonly clock?: () => number;
   /** Deltas larger than this become `delta-too-large` wake-ups (§8.2). */
   readonly maxDeltaBytes?: number;
@@ -926,6 +931,12 @@ export class RealtimeHub {
       storage: this.#config.storage,
       segments,
       resolveScopes: this.#config.resolveScopes,
+      ...(this.#config.validators !== undefined
+        ? { validators: this.#config.validators }
+        : {}),
+      ...(this.#config.commitValidator !== undefined
+        ? { commitValidator: this.#config.commitValidator }
+        : {}),
       ...(this.#config.clock !== undefined
         ? { clock: this.#config.clock }
         : {}),

package/src/sqlite-dialect.ts

@@ -126,6 +126,7 @@ interface SerializedResult {
   serverVersion?: number;
   serverRow?: string;
   retryable?: boolean;
+  details?: import('@syncular/core').RejectionDetails;
 }
 
 /** Serialize a push result to the JSON `TEXT` stored in `sync_push_results`. */
@@ -151,6 +152,7 @@ export function serializePushResult(result: StoredPushResult): string {
           code: record.code,
           message: record.message,
           retryable: record.retryable,
+          ...(record.details !== undefined ? { details: record.details } : {}),
         };
       }
       return { opIndex: record.opIndex, status: record.status };
@@ -182,6 +184,7 @@ export function deserializePushResult(text: string): StoredPushResult {
         code: record.code ?? '',
         message: record.message ?? '',
         retryable: record.retryable ?? false,
+        ...(record.details !== undefined ? { details: record.details } : {}),
       };
     }
     return { opIndex: record.opIndex, status: 'applied' };

package/src/sqlite-storage.ts

@@ -60,6 +60,7 @@ class SqliteTransaction implements StorageTransaction {
   #storage: SqliteServerStorage;
   #partition: string;
   #open = true;
+  #commitValidationSavepoint = false;
 
   constructor(storage: SqliteServerStorage, partition: string) {
     this.#storage = storage;
@@ -76,6 +77,40 @@ class SqliteTransaction implements StorageTransaction {
     return this.#storage.getRow(this.#partition, table, rowId);
   }
 
+  scanRows(query: RowScanQuery): Promise<StoredRow[]> {
+    this.#assertOpen();
+    return this.#storage.scanRows(this.#partition, query);
+  }
+
+  async lockPartitionForCommitValidation(): Promise<void> {
+    this.#assertOpen();
+    // BEGIN IMMEDIATE in the constructor already owns SQLite's writer lock.
+    this.#storage.db.exec('SAVEPOINT syncular_commit_validation_candidate');
+    this.#commitValidationSavepoint = true;
+  }
+
+  async commitRejectedPushResult(
+    clientId: string,
+    clientCommitId: string,
+    result: StoredPushResult,
+  ): Promise<void> {
+    this.#assertOpen();
+    if (!this.#commitValidationSavepoint) {
+      throw new Error(
+        'whole-commit rejection requires its validation savepoint',
+      );
+    }
+    this.#storage.db.exec(
+      'ROLLBACK TO SAVEPOINT syncular_commit_validation_candidate',
+    );
+    this.#storage.db.exec(
+      'RELEASE SAVEPOINT syncular_commit_validation_candidate',
+    );
+    this.#commitValidationSavepoint = false;
+    await this.putPushResult(clientId, clientCommitId, result);
+    await this.commit();
+  }
+
   async upsertRow(table: string, row: StoredRow): Promise<void> {
     this.#assertOpen();
     this.#storage.writeRow(this.#partition, table, row);

package/src/storage.ts

@@ -166,6 +166,28 @@ export interface ScopeActivityQuery {
  */
 export interface StorageTransaction {
   getRow(table: string, rowId: string): Promise<StoredRow | undefined>;
+  /**
+   * Optional candidate-state scan used only by whole-commit validation.
+   * In-tree SQLite/Postgres/D1 backends implement it with read-your-own-writes
+   * semantics. A custom backend may omit it until `commitValidator` is used.
+   */
+  scanRows?(query: RowScanQuery): Promise<StoredRow[]>;
+  /**
+   * Serialize candidate-state validation for this partition before any row
+   * read/write. Required at runtime when `commitValidator` is configured.
+   */
+  lockPartitionForCommitValidation?(): Promise<void>;
+  /**
+   * §6.8 rejection finalization while the validation serialization lock is
+   * still held: discard every candidate write, persist the rejected
+   * idempotency result, and finish the transaction atomically. Required when
+   * `commitValidator` is configured so a concurrent duplicate cannot rerun it.
+   */
+  commitRejectedPushResult?(
+    clientId: string,
+    clientCommitId: string,
+    result: StoredPushResult,
+  ): Promise<void>;
   upsertRow(table: string, row: StoredRow): Promise<void>;
   deleteRow(table: string, rowId: string): Promise<void>;
   /** Allocates the next per-partition commitSeq and appends the commit. */

package/src/validate.ts

@@ -13,7 +13,13 @@
  * path pays only an `undefined` check per operation and builds no context
  * object — zero cost, the events-seam discipline.
  */
-import type { RowColumn, RowValue } from '@syncular/core';
+import {
+  normalizeRejectionDetails,
+  type RejectionDetails,
+  type RowColumn,
+  type RowValue,
+  type ScopeMap,
+} from '@syncular/core';
 
 /**
  * §6.7 reserved code prefixes. A host validator code MUST NOT start with
@@ -59,6 +65,63 @@ export interface ValidateOperation {
   readonly stored: ValidateRow | undefined;
 }
 
+/**
+ * One authorized, decoded operation presented to the whole-commit validator.
+ * `row` is the final candidate row after scope stripping and CRDT merge;
+ * `stored` is the state observed immediately before this operation. Multiple
+ * operations targeting one row therefore retain their sequential evidence.
+ */
+export interface ValidateCommitOperation extends ValidateOperation {
+  readonly opIndex: number;
+  readonly storedServerVersion?: number;
+  readonly nextServerVersion?: number;
+}
+
+/** One candidate-state row read from inside the still-open commit transaction. */
+export interface CommitValidationRow {
+  readonly row: ValidateRow;
+  readonly serverVersion: number;
+}
+
+export interface CommitValidationScanInput {
+  readonly table: string;
+  /** Exact scope filter, using the same AND-across-keys semantics as sync. */
+  readonly scopeFilter: ScopeMap;
+  readonly afterRowId?: string | null;
+  /** Bounded per call by the server to 1..1,000; defaults to 100. */
+  readonly limit?: number;
+}
+
+/**
+ * Candidate-state reads bound to the same storage transaction as the commit.
+ * Reads observe every staged sibling operation and no uncommitted competing
+ * transaction when the storage's commit-validation lock contract is honored.
+ */
+export interface CommitValidationReader {
+  getRow(
+    table: string,
+    rowId: string,
+  ): Promise<CommitValidationRow | undefined>;
+  scanRows(input: CommitValidationScanInput): Promise<CommitValidationRow[]>;
+}
+
+export interface ValidateCommitInput {
+  readonly clientId: string;
+  readonly clientCommitId: string;
+  readonly actorId: string;
+  readonly partition: string;
+  readonly operations: readonly ValidateCommitOperation[];
+  readonly read: CommitValidationReader;
+}
+
+/**
+ * Runs once after every operation passed protocol/scope/row validation and was
+ * staged, but before commit-log/idempotency append and transaction commit.
+ */
+export type CommitValidator = (
+  input: ValidateCommitInput,
+) => void | Promise<void>;
+
 /** Ambient context a validator may consult (§6.7). */
 export interface ValidateContext {
   /** Host-authenticated actor (§1.1) performing the write. */
@@ -93,8 +156,13 @@ export type ValidatorRegistry = Readonly<Record<string, Validator>>;
 export class ValidationRejection extends Error {
   override readonly name = 'ValidationRejection';
   readonly code: string;
+  /**
+   * Bounded code-like metadata explicitly safe to replicate to authorized
+   * clients. Never place diagnostic prose, secrets, or clinical values here.
+   */
+  readonly details: RejectionDetails | undefined;
 
-  constructor(code: string, message?: string) {
+  constructor(code: string, message?: string, details?: RejectionDetails) {
     super(message ?? code);
     if (code.length === 0) {
       throw new Error('ValidationRejection code must be non-empty (§6.7)');
@@ -107,6 +175,29 @@ export class ValidationRejection extends Error {
       }
     }
     this.code = code;
+    this.details =
+      details === undefined ? undefined : normalizeRejectionDetails(details);
+  }
+}
+
+/**
+ * A whole-commit rejection attributed to one operation for the existing
+ * per-operation PUSH_RESULT envelope. The validator may still describe
+ * multiple affected fields in `details.fieldPaths`.
+ */
+export class CommitValidationRejection extends ValidationRejection {
+  constructor(
+    readonly opIndex: number,
+    code: string,
+    message?: string,
+    details?: RejectionDetails,
+  ) {
+    super(code, message, details);
+    if (!Number.isSafeInteger(opIndex) || opIndex < 0) {
+      throw new Error(
+        'CommitValidationRejection opIndex must be a non-negative safe integer',
+      );
+    }
   }
 }