@syncular/server 0.9.0 → 0.11.0

package/dist/validate.js

@@ -57,6 +57,21 @@ export class ValidationRejection extends Error {
             details === undefined ? undefined : normalizeRejectionDetails(details);
     }
 }
+/**
+ * A whole-commit rejection attributed to one operation for the existing
+ * per-operation PUSH_RESULT envelope. The validator may still describe
+ * multiple affected fields in `details.fieldPaths`.
+ */
+export class CommitValidationRejection extends ValidationRejection {
+    opIndex;
+    constructor(opIndex, code, message, details) {
+        super(code, message, details);
+        this.opIndex = opIndex;
+        if (!Number.isSafeInteger(opIndex) || opIndex < 0) {
+            throw new Error('CommitValidationRejection opIndex must be a non-negative safe integer');
+        }
+    }
+}
 /** Build the column-keyed row object a validator inspects (§6.7). */
 export function toValidateRow(columns, values) {
     const row = {};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@syncular/server",
-  "version": "0.9.0",
+  "version": "0.11.0",
   "description": "Syncular server: handleSyncRequest + storage/auth interfaces for the sync protocol",
   "license": "Apache-2.0",
   "author": "Benjamin Kniffler",
@@ -53,7 +53,7 @@
     "!dist/**/*.test.d.ts"
   ],
   "dependencies": {
-    "@syncular/core": "0.9.0"
+    "@syncular/core": "0.11.0"
   },
   "devDependencies": {
     "@electric-sql/pglite": "^0.5.4"

package/src/context.ts

@@ -18,7 +18,7 @@ import type {
 } from './signed-url';
 import type { SqliteImageBuilder } from './sqlite-image';
 import type { ServerStorage, StoredCommit } from './storage';
-import type { ValidatorRegistry } from './validate';
+import type { CommitValidator, ValidatorRegistry } from './validate';
 
 /** SSP2 body content type (§1.1). */
 export const SSP2_CONTENT_TYPE = 'application/vnd.syncular.sync.v2';
@@ -112,6 +112,13 @@ export interface SyncServerConfig {
    * process, never on the wire.
    */
   readonly validators?: ValidatorRegistry;
+  /**
+   * §6.8 whole-commit validator. When present, the storage serializes this
+   * partition before operation reads/writes, then invokes the callback once
+   * over every staged decoded operation and candidate-state reader before
+   * commit-log/idempotency append. A throw rolls back the complete commit.
+   */
+  readonly commitValidator?: CommitValidator;
   readonly resolveScopes: ResolveScopes;
   /**
    * §7.3 auth leases. Absent ⇒ the feature is off: no `LEASE` frame is

package/src/postgres-storage.ts

@@ -396,6 +396,7 @@ class PostgresTransaction implements StorageTransaction {
   #partition: string;
   #resolveTable: (name: string) => CompiledTable;
   #open = true;
+  #commitValidationSavepoint = false;
   /** Resolves/rejects the `transaction(fn)` wrapper (see `begin`). */
   #resolve: () => void;
   #reject: (error: unknown) => void;
@@ -428,6 +429,81 @@ class PostgresTransaction implements StorageTransaction {
     );
   }
 
+  async scanRows(query: RowScanQuery): Promise<StoredRow[]> {
+    this.#assertOpen();
+    const variables = Object.keys(query.scopeFilter).sort();
+    const firstVariable = variables[0];
+    if (firstVariable === undefined) return [];
+    const firstValues = query.scopeFilter[firstVariable] ?? [];
+    if (firstValues.length === 0) return [];
+    const sql = scanRowPageSql(
+      this.#resolveTable(query.table),
+      firstValues.length,
+      'postgres',
+    );
+    const rows: StoredRow[] = [];
+    let afterRowId = query.afterRowId ?? '';
+    const batchSize = Math.max(64, query.limit);
+    while (rows.length < query.limit) {
+      const { rows: records } = await this.#client.query<RowRecord>(sql, [
+        this.#partition,
+        query.table,
+        firstVariable,
+        ...firstValues,
+        afterRowId,
+        batchSize,
+      ]);
+      if (records.length === 0) break;
+      for (const record of records) {
+        afterRowId = record.row_id;
+        if (record.payload === null || record.payload === undefined) continue;
+        const stored = toStoredRow(record);
+        if (!matchesEffective(stored.scopes, query.scopeFilter)) continue;
+        rows.push(stored);
+        if (rows.length >= query.limit) break;
+      }
+      if (records.length < batchSize) break;
+    }
+    return rows;
+  }
+
+  async lockPartitionForCommitValidation(): Promise<void> {
+    this.#assertOpen();
+    await this.#client.query(
+      `INSERT INTO sync_partitions(partition, max_commit_seq) VALUES ($1, 0)
+       ON CONFLICT (partition) DO NOTHING`,
+      [this.#partition],
+    );
+    await this.#client.query(
+      'SELECT max_commit_seq FROM sync_partitions WHERE partition=$1 FOR UPDATE',
+      [this.#partition],
+    );
+    await this.#client.query('SAVEPOINT syncular_commit_validation_candidate');
+    this.#commitValidationSavepoint = true;
+  }
+
+  async commitRejectedPushResult(
+    clientId: string,
+    clientCommitId: string,
+    result: StoredPushResult,
+  ): Promise<void> {
+    this.#assertOpen();
+    if (!this.#commitValidationSavepoint) {
+      throw new Error(
+        'whole-commit rejection requires its validation savepoint',
+      );
+    }
+    await this.#client.query(
+      'ROLLBACK TO SAVEPOINT syncular_commit_validation_candidate',
+    );
+    await this.#client.query(
+      'RELEASE SAVEPOINT syncular_commit_validation_candidate',
+    );
+    this.#commitValidationSavepoint = false;
+    await this.putPushResult(clientId, clientCommitId, result);
+    await this.commit();
+  }
+
   async upsertRow(table: string, row: StoredRow): Promise<void> {
     this.#assertOpen();
     await writeRowOn(

package/src/push.ts

@@ -42,8 +42,18 @@ import type {
   StoredCommit,
   StoredPushResult,
 } from './storage';
-import type { ValidateOpKind, ValidatorRegistry } from './validate';
-import { toValidateRow, ValidationRejection } from './validate';
+import type {
+  CommitValidationReader,
+  CommitValidator,
+  ValidateCommitOperation,
+  ValidateOpKind,
+  ValidatorRegistry,
+} from './validate';
+import {
+  CommitValidationRejection,
+  toValidateRow,
+  ValidationRejection,
+} from './validate';
 
 /**
  * Extract the blobIds a decoded row references through its `blob_ref`
@@ -63,7 +73,11 @@ function blobIdsInRow(
 }
 
 type OperationOutcome =
-  | { readonly kind: 'applied'; readonly change: NewChange | undefined }
+  | {
+      readonly kind: 'applied';
+      readonly change: NewChange | undefined;
+      readonly operation: ValidateCommitOperation;
+    }
   | { readonly kind: 'terminate'; readonly record: PushOperationResult };
 
 function errorRecord(
@@ -252,7 +266,18 @@ async function applyOperation(
   if (op.op === 'delete') {
     if (stored === undefined) {
       // Deleting an absent row is applied (idempotent, §6.2); no change.
-      return { kind: 'applied', change: undefined };
+      return {
+        kind: 'applied',
+        change: undefined,
+        operation: {
+          opIndex,
+          op: 'delete',
+          table: table.name,
+          rowId: op.rowId,
+          row: undefined,
+          stored: undefined,
+        },
+      };
     }
     if (!authorizeWrite(table, stored.scopes, resolved)) {
       return errorRecord(
@@ -272,13 +297,14 @@ async function applyOperation(
     // §6.7: validate the delete against the stored row (row = undefined,
     // stored = the row about to be removed). Only reached for an existing
     // row — an absent-row delete is an idempotent no-op above.
+    const storedValues = decodeRow(table.columns, stored.payload);
     const deleteReject = await runValidator(
       validators,
       table,
       'delete',
       op.rowId,
       undefined,
-      decodeRow(table.columns, stored.payload),
+      storedValues,
       opIndex,
       partition,
       actorId,
@@ -293,6 +319,15 @@ async function applyOperation(
         op: 'delete',
         scopes: stored.scopes,
       },
+      operation: {
+        opIndex,
+        op: 'delete',
+        table: table.name,
+        rowId: op.rowId,
+        row: undefined,
+        stored: toValidateRow(table.columns, storedValues),
+        storedServerVersion: stored.serverVersion,
+      },
     };
   }
 
@@ -411,6 +446,16 @@ async function applyOperation(
         scopes: stored.scopes,
         payload: newPayload,
       },
+      operation: {
+        opIndex,
+        op: 'upsert',
+        table: table.name,
+        rowId: op.rowId,
+        row: toValidateRow(table.columns, values),
+        stored: toValidateRow(table.columns, storedValues),
+        storedServerVersion: stored.serverVersion,
+        nextServerVersion: newVersion,
+      },
     };
   }
 
@@ -506,6 +551,15 @@ async function applyOperation(
       scopes: extracted.scopes,
       payload: insertPayload,
     },
+    operation: {
+      opIndex,
+      op: 'upsert',
+      table: table.name,
+      rowId: op.rowId,
+      row: toValidateRow(table.columns, values),
+      stored: undefined,
+      nextServerVersion: 1,
+    },
   };
 }
 
@@ -562,6 +616,121 @@ function missingScopeVariable(
   return undefined;
 }
 
+function commitValidationReader(
+  tx: StorageTransaction,
+  schema: CompiledSchema,
+): CommitValidationReader {
+  const tableFor = (name: string): CompiledTable => {
+    const table = schema.tables.get(name);
+    if (table === undefined) {
+      throw new Error(
+        `commit validator requested unknown table ${JSON.stringify(name)}`,
+      );
+    }
+    return table;
+  };
+  return {
+    getRow: async (tableName, rowId) => {
+      const table = tableFor(tableName);
+      const stored = await tx.getRow(tableName, rowId);
+      if (stored === undefined) return undefined;
+      return {
+        row: toValidateRow(
+          table.columns,
+          decodeRow(table.columns, stored.payload),
+        ),
+        serverVersion: stored.serverVersion,
+      };
+    },
+    scanRows: async ({
+      table: tableName,
+      scopeFilter,
+      afterRowId = null,
+      limit = 100,
+    }) => {
+      const table = tableFor(tableName);
+      if (!Number.isSafeInteger(limit) || limit < 1 || limit > 1_000) {
+        throw new Error(
+          'commit validator scan limit must be an integer from 1 to 1,000',
+        );
+      }
+      if (tx.scanRows === undefined) {
+        throw new Error(
+          'storage transaction does not support commit-validator scans',
+        );
+      }
+      const rows = await tx.scanRows({
+        table: tableName,
+        scopeFilter,
+        afterRowId,
+        limit,
+      });
+      return rows.map((stored) => ({
+        row: toValidateRow(
+          table.columns,
+          decodeRow(table.columns, stored.payload),
+        ),
+        serverVersion: stored.serverVersion,
+      }));
+    },
+  };
+}
+
+async function runCommitValidator(
+  validator: CommitValidator | undefined,
+  tx: StorageTransaction,
+  schema: CompiledSchema,
+  clientId: string,
+  clientCommitId: string,
+  actorId: string,
+  partition: string,
+  operations: readonly ValidateCommitOperation[],
+): Promise<OperationOutcome | undefined> {
+  if (validator === undefined) return undefined;
+  try {
+    await validator({
+      clientId,
+      clientCommitId,
+      actorId,
+      partition,
+      operations,
+      read: commitValidationReader(tx, schema),
+    });
+  } catch (error) {
+    if (error instanceof CommitValidationRejection) {
+      if (error.opIndex >= operations.length) {
+        return errorRecord(
+          0,
+          'sync.constraint_violation',
+          `commit validator rejection names unavailable opIndex ${error.opIndex}`,
+        );
+      }
+      return errorRecord(
+        error.opIndex,
+        error.code,
+        error.message,
+        false,
+        error.details,
+      );
+    }
+    if (error instanceof ValidationRejection) {
+      return errorRecord(
+        operations[0]?.opIndex ?? 0,
+        error.code,
+        error.message,
+        false,
+        error.details,
+      );
+    }
+    return errorRecord(
+      operations[0]?.opIndex ?? 0,
+      'sync.constraint_violation',
+      `whole-commit validator threw: ${error instanceof Error ? error.message : String(error)}`,
+    );
+  }
+  return undefined;
+}
+
 function resultFrame(
   clientCommitId: string,
   stored: StoredPushResult,
@@ -580,6 +749,26 @@ function resultFrame(
   };
 }
 
+function idempotencyCacheMissFrame(
+  clientCommitId: string,
+  error: SyncError,
+): PushResultFrame {
+  return {
+    type: 'PUSH_RESULT',
+    clientCommitId,
+    status: 'rejected',
+    results: [
+      {
+        opIndex: 0,
+        status: 'error',
+        code: 'sync.idempotency_cache_miss',
+        message: error.message,
+        retryable: true,
+      },
+    ],
+  };
+}
+
 export interface AppliedCommitEvent {
   readonly commit: StoredCommit;
 }
@@ -610,20 +799,7 @@ export async function processPushCommit(
     ) {
       // §6.3: answer the retryable cache-miss for this commit rather than
       // re-applying. Not persisted — a retry may find a readable record.
-      return {
-        type: 'PUSH_RESULT',
-        clientCommitId: frame.clientCommitId,
-        status: 'rejected',
-        results: [
-          {
-            opIndex: 0,
-            status: 'error',
-            code: 'sync.idempotency_cache_miss',
-            message: error.message,
-            retryable: true,
-          },
-        ],
-      };
+      return idempotencyCacheMissFrame(frame.clientCommitId, error);
     }
     throw error;
   }
@@ -635,10 +811,47 @@ export async function processPushCommit(
   const blobCtx: BlobApplyContext = { store: ctx.blobs, partition };
   const crdtMergers = ctx.crdtMergers;
   const validators = ctx.validators;
+  const commitValidator = ctx.commitValidator;
   const tx = await storage.begin(partition);
+  const commitRejectedPushResult = tx.commitRejectedPushResult?.bind(tx);
   try {
+    if (commitValidator !== undefined) {
+      if (
+        tx.lockPartitionForCommitValidation === undefined ||
+        commitRejectedPushResult === undefined
+      ) {
+        throw new Error(
+          'storage transaction does not support atomic whole-commit validation finalization',
+        );
+      }
+      await tx.lockPartitionForCommitValidation();
+      // The optimistic lookup above may have raced another request for the
+      // same idempotency key. Re-check after acquiring partition serialization
+      // so a concurrent duplicate never reruns the aggregate validator.
+      try {
+        const serializedPersisted = await storage.getPushResult(
+          partition,
+          clientId,
+          frame.clientCommitId,
+        );
+        if (serializedPersisted !== undefined) {
+          await tx.rollback();
+          return resultFrame(frame.clientCommitId, serializedPersisted, true);
+        }
+      } catch (error) {
+        if (
+          error instanceof SyncError &&
+          error.code === 'sync.idempotency_cache_miss'
+        ) {
+          await tx.rollback();
+          return idempotencyCacheMissFrame(frame.clientCommitId, error);
+        }
+        throw error;
+      }
+    }
     const results: PushOperationResult[] = [];
     const changes: NewChange[] = [];
+    const validatedOperations: ValidateCommitOperation[] = [];
     let terminated: PushOperationResult | undefined;
     for (let opIndex = 0; opIndex < frame.operations.length; opIndex++) {
       const op = frame.operations[opIndex];
@@ -660,24 +873,57 @@ export async function processPushCommit(
         break;
       }
       results.push({ opIndex, status: 'applied' });
+      validatedOperations.push(outcome.operation);
       if (outcome.change !== undefined) changes.push(outcome.change);
     }
 
+    if (terminated === undefined) {
+      const commitReject = await runCommitValidator(
+        commitValidator,
+        tx,
+        schema,
+        clientId,
+        frame.clientCommitId,
+        ctx.actorId,
+        partition,
+        validatedOperations,
+      );
+      if (commitReject?.kind === 'terminate') {
+        terminated = commitReject.record;
+      }
+    }
+
     if (terminated !== undefined) {
       // §6.3 rejected: only the terminating operation's record; §6.4:
       // every write of the commit rolls back.
-      await tx.rollback();
       const stored: StoredPushResult = {
         status: 'rejected',
         results: [terminated],
       };
-      const rejectionTx = await storage.begin(partition);
-      try {
-        await rejectionTx.putPushResult(clientId, frame.clientCommitId, stored);
-        await rejectionTx.commit();
-      } catch (error) {
-        await rejectionTx.rollback();
-        throw error;
+      if (commitValidator !== undefined) {
+        // Discard candidate rows and persist the rejection while retaining the
+        // same partition lock. This closes the duplicate-request race between
+        // rollback and the durable idempotency outcome.
+        if (commitRejectedPushResult === undefined) {
+          throw new Error(
+            'storage transaction lost whole-commit rejection finalization support',
+          );
+        }
+        await commitRejectedPushResult(clientId, frame.clientCommitId, stored);
+      } else {
+        await tx.rollback();
+        const rejectionTx = await storage.begin(partition);
+        try {
+          await rejectionTx.putPushResult(
+            clientId,
+            frame.clientCommitId,
+            stored,
+          );
+          await rejectionTx.commit();
+        } catch (error) {
+          await rejectionTx.rollback();
+          throw error;
+        }
       }
       return resultFrame(frame.clientCommitId, stored, false);
     }

package/src/realtime.ts

@@ -50,7 +50,7 @@ import {
 import type { SegmentStore } from './segment-store';
 import type { SegmentUrlConfig } from './signed-url';
 import type { ServerStorage, StoredCommit } from './storage';
-import type { ValidatorRegistry } from './validate';
+import type { CommitValidator, ValidatorRegistry } from './validate';
 
 export interface RealtimeHubConfig {
   readonly schema: ServerSchema;
@@ -58,6 +58,8 @@ export interface RealtimeHubConfig {
   readonly resolveScopes: ResolveScopes;
   /** §6.7 validators used by sync rounds carried over this socket. */
   readonly validators?: ValidatorRegistry;
+  /** §6.8 whole-commit validator shared with HTTP sync rounds. */
+  readonly commitValidator?: CommitValidator;
   readonly clock?: () => number;
   /** Deltas larger than this become `delta-too-large` wake-ups (§8.2). */
   readonly maxDeltaBytes?: number;
@@ -932,6 +934,9 @@ export class RealtimeHub {
       ...(this.#config.validators !== undefined
         ? { validators: this.#config.validators }
         : {}),
+      ...(this.#config.commitValidator !== undefined
+        ? { commitValidator: this.#config.commitValidator }
+        : {}),
       ...(this.#config.clock !== undefined
         ? { clock: this.#config.clock }
         : {}),

package/src/sqlite-storage.ts

@@ -60,6 +60,7 @@ class SqliteTransaction implements StorageTransaction {
   #storage: SqliteServerStorage;
   #partition: string;
   #open = true;
+  #commitValidationSavepoint = false;
 
   constructor(storage: SqliteServerStorage, partition: string) {
     this.#storage = storage;
@@ -76,6 +77,40 @@ class SqliteTransaction implements StorageTransaction {
     return this.#storage.getRow(this.#partition, table, rowId);
   }
 
+  scanRows(query: RowScanQuery): Promise<StoredRow[]> {
+    this.#assertOpen();
+    return this.#storage.scanRows(this.#partition, query);
+  }
+
+  async lockPartitionForCommitValidation(): Promise<void> {
+    this.#assertOpen();
+    // BEGIN IMMEDIATE in the constructor already owns SQLite's writer lock.
+    this.#storage.db.exec('SAVEPOINT syncular_commit_validation_candidate');
+    this.#commitValidationSavepoint = true;
+  }
+
+  async commitRejectedPushResult(
+    clientId: string,
+    clientCommitId: string,
+    result: StoredPushResult,
+  ): Promise<void> {
+    this.#assertOpen();
+    if (!this.#commitValidationSavepoint) {
+      throw new Error(
+        'whole-commit rejection requires its validation savepoint',
+      );
+    }
+    this.#storage.db.exec(
+      'ROLLBACK TO SAVEPOINT syncular_commit_validation_candidate',
+    );
+    this.#storage.db.exec(
+      'RELEASE SAVEPOINT syncular_commit_validation_candidate',
+    );
+    this.#commitValidationSavepoint = false;
+    await this.putPushResult(clientId, clientCommitId, result);
+    await this.commit();
+  }
+
   async upsertRow(table: string, row: StoredRow): Promise<void> {
     this.#assertOpen();
     this.#storage.writeRow(this.#partition, table, row);

package/src/storage.ts

@@ -166,6 +166,28 @@ export interface ScopeActivityQuery {
  */
 export interface StorageTransaction {
   getRow(table: string, rowId: string): Promise<StoredRow | undefined>;
+  /**
+   * Optional candidate-state scan used only by whole-commit validation.
+   * In-tree SQLite/Postgres/D1 backends implement it with read-your-own-writes
+   * semantics. A custom backend may omit it until `commitValidator` is used.
+   */
+  scanRows?(query: RowScanQuery): Promise<StoredRow[]>;
+  /**
+   * Serialize candidate-state validation for this partition before any row
+   * read/write. Required at runtime when `commitValidator` is configured.
+   */
+  lockPartitionForCommitValidation?(): Promise<void>;
+  /**
+   * §6.8 rejection finalization while the validation serialization lock is
+   * still held: discard every candidate write, persist the rejected
+   * idempotency result, and finish the transaction atomically. Required when
+   * `commitValidator` is configured so a concurrent duplicate cannot rerun it.
+   */
+  commitRejectedPushResult?(
+    clientId: string,
+    clientCommitId: string,
+    result: StoredPushResult,
+  ): Promise<void>;
   upsertRow(table: string, row: StoredRow): Promise<void>;
   deleteRow(table: string, rowId: string): Promise<void>;
   /** Allocates the next per-partition commitSeq and appends the commit. */