@syncular/server 0.42.2 → 0.44.0

package/src/push.ts

@@ -48,6 +48,7 @@ import { authorizeWrite, renderScopeValue, storedScopesForRow } from './scopes';
 import { decodeColumnVersions, encodeColumnVersions } from './relational-rows';
 import type {
   NewChange,
+  PreparedPushApply,
   StorageTransaction,
   StoredCommit,
   StoredPushResult,
@@ -108,12 +109,28 @@ type TerminatingOutcome = {
 type OperationOutcome =
   | {
       readonly kind: 'applied';
-      readonly change: NewChange | undefined;
-      readonly operation: ValidateCommitOperation;
+      readonly changes: readonly NewChange[];
+      readonly operations: readonly ValidateCommitOperation[];
       readonly staged: StagedWrite;
     }
   | TerminatingOutcome;
 
+/**
+ * A server-authoritative scope move (§2.2). Commands emit it through
+ * `CommandMutation`; it is not an SSP2 wire operation, so a client cannot
+ * request one. Apply rewrites the stored row in place under the new scopes
+ * and emits the delete/upsert change pair in one commit.
+ */
+export interface RescopeOperation {
+  readonly table: string;
+  readonly rowId: string;
+  readonly op: 'rescope';
+  /** Full-row payload (the row's values under the new scopes). */
+  readonly payload: Uint8Array;
+}
+
+export type ApplyOperation = PushOperation | RescopeOperation;
+
 function errorRecord(
   opIndex: number,
   code: string,
@@ -296,7 +313,7 @@ async function applyOperation(
   schema: CompiledSchema,
   wireSchema: CompiledSchema,
   resolved: ResolvedScopes,
-  op: PushOperation,
+  op: ApplyOperation,
   opIndex: number,
   blobCtx: BlobApplyContext,
   crdtMergers: CrdtMergerRegistry | undefined,
@@ -322,20 +339,224 @@ async function applyOperation(
   }
   const stored = await tx.getRow(op.table, op.rowId);
 
-  if (op.op === 'delete') {
+  if (op.op === 'rescope') {
+    // §2.2 explicit scope move. One stored row changes its scope values;
+    // apply rewrites it in place, keeps `server_version`, and emits the
+    // delete under the old scopes plus the upsert under the new ones.
     if (stored === undefined) {
-      // Deleting an absent row is applied (idempotent, §6.2); no change.
-      return {
-        kind: 'applied',
-        change: undefined,
-        operation: {
+      return errorRecord(
+        opIndex,
+        'sync.row_missing',
+        'rescope targets an absent row (§2.2)',
+      );
+    }
+    if (!authorizeWrite(table, stored.scopes, resolved)) {
+      return errorRecord(
+        opIndex,
+        'sync.forbidden',
+        'rescope denied by scope authorization (§3.4)',
+      );
+    }
+    const missing = missingScopeVariable(table, stored.scopes);
+    if (missing !== undefined) {
+      return errorRecord(
+        opIndex,
+        'sync.missing_scopes',
+        `stored row lacks scope variable ${JSON.stringify(missing)} (§3.1)`,
+      );
+    }
+    let sparse: SparseRowValue[];
+    try {
+      sparse = decodeSparseRow(
+        wireTable.columns,
+        wireTable.primaryKeyIndex,
+        op.payload,
+      );
+    } catch (error) {
+      return errorRecord(
+        opIndex,
+        'sync.invalid_request',
+        `rescope payload failed row-codec decode (§1.7): ${error instanceof Error ? error.message : String(error)}`,
+      );
+    }
+    if (
+      sparse.length < table.columns.length ||
+      sparse.some((value) => value === undefined)
+    ) {
+      return errorRecord(
+        opIndex,
+        'sync.invalid_request',
+        'rescope requires a full-row payload (§2.2)',
+      );
+    }
+    const values: RowValue[] = sparse.map((value) => value ?? null);
+    const pkValue = renderScopeValue(values[table.primaryKeyIndex] ?? null);
+    if (pkValue !== op.rowId) {
+      return errorRecord(
+        opIndex,
+        'sync.invalid_request',
+        'payload primary key does not match rowId',
+      );
+    }
+    const storedValues = decodeRow(table.columns, stored.payload);
+    const scopeIndexes = new Set(
+      table.scopePatterns.map((pattern) => pattern.columnIndex),
+    );
+    const sameValue = (left: RowValue, right: RowValue): boolean => {
+      if (left instanceof Uint8Array || right instanceof Uint8Array) {
+        return (
+          left instanceof Uint8Array &&
+          right instanceof Uint8Array &&
+          left.length === right.length &&
+          left.every((byte, index) => byte === right[index])
+        );
+      }
+      return left === right;
+    };
+    for (let index = 0; index < table.columns.length; index++) {
+      if (index === table.primaryKeyIndex || scopeIndexes.has(index)) continue;
+      if (!sameValue(values[index] ?? null, storedValues[index] ?? null)) {
+        return errorRecord(
+          opIndex,
+          'sync.invalid_request',
+          'rescope may change only scope columns (§2.2)',
+        );
+      }
+    }
+    const extracted = storedScopesForRow(table, values);
+    if ('missing' in extracted) {
+      return errorRecord(
+        opIndex,
+        'sync.forbidden',
+        `rescope missing scope column value for ${JSON.stringify(extracted.missing)} (§3.4)`,
+      );
+    }
+    if (!authorizeWrite(table, extracted.scopes, resolved)) {
+      return errorRecord(
+        opIndex,
+        'sync.forbidden',
+        'rescope denied by scope authorization of the new scopes (§3.4)',
+      );
+    }
+    const deleteReject = await runValidator(
+      validators,
+      table,
+      'delete',
+      op.rowId,
+      undefined,
+      storedValues,
+      opIndex,
+      tx,
+      partition,
+      actorId,
+    );
+    if (deleteReject !== undefined) return deleteReject;
+    const upsertReject = await runValidator(
+      validators,
+      table,
+      'upsert',
+      op.rowId,
+      values,
+      storedValues,
+      opIndex,
+      tx,
+      partition,
+      actorId,
+    );
+    if (upsertReject !== undefined) return upsertReject;
+    const blobCheck = await checkAndRecordBlobs(
+      tx,
+      table,
+      op.rowId,
+      values,
+      opIndex,
+      blobCtx,
+    );
+    if (blobCheck !== undefined) return blobCheck;
+    const payload = encodeRow(table.columns, values);
+    // The move clears any tombstone from an earlier incarnation: the row is
+    // live at the end of this commit, so the next move must not hit §5.2.
+    await tx.clearTombstone(op.table, op.rowId);
+    await tx.upsertRow(
+      op.table,
+      {
+        rowId: op.rowId,
+        serverVersion: stored.serverVersion,
+        scopes: extracted.scopes,
+        payload,
+        ...(stored.columnVersions !== undefined
+          ? { columnVersions: stored.columnVersions }
+          : {}),
+      },
+      { opIndex },
+    );
+    const storedRow = toValidateRow(table.columns, storedValues);
+    return {
+      kind: 'applied',
+      changes: [
+        {
+          table: op.table,
+          rowId: op.rowId,
+          op: 'delete',
+          scopes: stored.scopes,
+        },
+        {
+          table: op.table,
+          rowId: op.rowId,
+          op: 'upsert',
+          rowVersion: stored.serverVersion,
+          scopes: extracted.scopes,
+          payload,
+        },
+      ],
+      operations: [
+        {
           opIndex,
           op: 'delete',
           table: table.name,
           rowId: op.rowId,
           row: undefined,
-          stored: undefined,
+          stored: storedRow,
+          storedServerVersion: stored.serverVersion,
         },
+        {
+          opIndex,
+          op: 'upsert',
+          table: table.name,
+          rowId: op.rowId,
+          row: toValidateRow(table.columns, values),
+          stored: storedRow,
+          storedServerVersion: stored.serverVersion,
+          nextServerVersion: stored.serverVersion,
+        },
+      ],
+      staged: {
+        table,
+        op: 'upsert',
+        rowId: op.rowId,
+        opIndex,
+        values,
+        stored,
+      },
+    };
+  }
+
+  if (op.op === 'delete') {
+    if (stored === undefined) {
+      // Deleting an absent row is applied (idempotent, §6.2); no change.
+      return {
+        kind: 'applied',
+        changes: [],
+        operations: [
+          {
+            opIndex,
+            op: 'delete',
+            table: table.name,
+            rowId: op.rowId,
+            row: undefined,
+            stored: undefined,
+          },
+        ],
         staged: {
           table,
           op: 'delete',
@@ -380,21 +601,25 @@ async function applyOperation(
     await tx.deleteRow(op.table, op.rowId);
     return {
       kind: 'applied',
-      change: {
-        table: op.table,
-        rowId: op.rowId,
-        op: 'delete',
-        scopes: stored.scopes,
-      },
-      operation: {
-        opIndex,
-        op: 'delete',
-        table: table.name,
-        rowId: op.rowId,
-        row: undefined,
-        stored: toValidateRow(table.columns, storedValues),
-        storedServerVersion: stored.serverVersion,
-      },
+      changes: [
+        {
+          table: op.table,
+          rowId: op.rowId,
+          op: 'delete',
+          scopes: stored.scopes,
+        },
+      ],
+      operations: [
+        {
+          opIndex,
+          op: 'delete',
+          table: table.name,
+          rowId: op.rowId,
+          row: undefined,
+          stored: toValidateRow(table.columns, storedValues),
+          storedServerVersion: stored.serverVersion,
+        },
+      ],
       staged: { table, op: 'delete', rowId: op.rowId, opIndex, stored },
     };
   }
@@ -586,24 +811,28 @@ async function applyOperation(
     await tx.upsertRow(op.table, newRow, { opIndex });
     return {
       kind: 'applied',
-      change: {
-        table: op.table,
-        rowId: op.rowId,
-        op: 'upsert',
-        rowVersion: newVersion,
-        scopes: stored.scopes,
-        payload: newPayload,
-      },
-      operation: {
-        opIndex,
-        op: 'upsert',
-        table: table.name,
-        rowId: op.rowId,
-        row: toValidateRow(table.columns, values),
-        stored: toValidateRow(table.columns, storedValues),
-        storedServerVersion: stored.serverVersion,
-        nextServerVersion: newVersion,
-      },
+      changes: [
+        {
+          table: op.table,
+          rowId: op.rowId,
+          op: 'upsert',
+          rowVersion: newVersion,
+          scopes: stored.scopes,
+          payload: newPayload,
+        },
+      ],
+      operations: [
+        {
+          opIndex,
+          op: 'upsert',
+          table: table.name,
+          rowId: op.rowId,
+          row: toValidateRow(table.columns, values),
+          stored: toValidateRow(table.columns, storedValues),
+          storedServerVersion: stored.serverVersion,
+          nextServerVersion: newVersion,
+        },
+      ],
       staged: { table, op: 'upsert', rowId: op.rowId, opIndex, values, stored },
     };
   }
@@ -721,23 +950,27 @@ async function applyOperation(
   await tx.upsertRow(op.table, newRow, { opIndex });
   return {
     kind: 'applied',
-    change: {
-      table: op.table,
-      rowId: op.rowId,
-      op: 'upsert',
-      rowVersion: 1,
-      scopes: extracted.scopes,
-      payload: insertPayload,
-    },
-    operation: {
-      opIndex,
-      op: 'upsert',
-      table: table.name,
-      rowId: op.rowId,
-      row: toValidateRow(table.columns, values),
-      stored: undefined,
-      nextServerVersion: 1,
-    },
+    changes: [
+      {
+        table: op.table,
+        rowId: op.rowId,
+        op: 'upsert',
+        rowVersion: 1,
+        scopes: extracted.scopes,
+        payload: insertPayload,
+      },
+    ],
+    operations: [
+      {
+        opIndex,
+        op: 'upsert',
+        table: table.name,
+        rowId: op.rowId,
+        row: toValidateRow(table.columns, values),
+        stored: undefined,
+        nextServerVersion: 1,
+      },
+    ],
     staged: {
       table,
       op: 'upsert',
@@ -1342,6 +1575,16 @@ export async function processPushCommitWithTrace(
   frame: PushCommitFrame,
   wireSchema: CompiledSchema = schema,
 ): Promise<ProcessedPushCommit> {
+  // The frame names its rows before the transaction opens, so the admission
+  // read carries the commit's read-ahead and the apply issues no separate
+  // prefetch statement.
+  const targets = new Map<string, string[]>();
+  for (const op of frame.operations) {
+    if (!schema.tables.has(op.table)) continue;
+    const ids = targets.get(op.table) ?? [];
+    ids.push(op.rowId);
+    targets.set(op.table, ids);
+  }
   return processPushOperationsWithTrace(
     ctx,
     schema,
@@ -1350,6 +1593,9 @@ export async function processPushCommitWithTrace(
     frame.clientCommitId,
     async () => frame.operations,
     wireSchema,
+    targets.size === 0
+      ? undefined
+      : [...targets].map(([table, rowIds]) => ({ table, rowIds })),
   );
 }
 
@@ -1366,88 +1612,48 @@ export async function processPushOperationsWithTrace(
   clientCommitId: string,
   buildOperations: (
     tx: StorageTransaction,
-  ) => Promise<readonly PushOperation[]>,
+  ) => Promise<readonly ApplyOperation[]>,
   wireSchema: CompiledSchema = schema,
+  prefetchTargets?: readonly {
+    readonly table: string;
+    readonly rowIds: readonly string[];
+  }[],
 ): Promise<ProcessedPushCommit> {
-  const { storage, partition } = ctx;
-  let persisted: StoredPushResult | undefined;
-  try {
-    persisted = await storage.getPushResult(
-      partition,
-      clientId,
-      clientCommitId,
-    );
-  } catch (error) {
-    if (
-      error instanceof SyncError &&
-      error.code === 'sync.idempotency_cache_miss'
-    ) {
-      // §6.3: answer the retryable cache-miss for this commit rather than
-      // re-applying. Not persisted — a retry may find a readable record.
-      return {
-        frame: idempotencyCacheMissFrame(clientCommitId, error),
-        replayed: false,
-      };
-    }
-    throw error;
-  }
-  if (persisted !== undefined) {
-    return processedPushCommit(clientCommitId, persisted, true);
-  }
-
+  const { partition } = ctx;
   const createdAtMs = clockOf(ctx)();
   const blobCtx: BlobApplyContext = { store: ctx.blobs, partition };
   const crdtMergers = ctx.crdtMergers;
   const validators = ctx.validators;
   const commitValidator = ctx.commitValidator;
   const reactionPlanner = ctx.reactionPlanner;
-  const tx = await storage.begin(partition);
-  const lockPartitionForPush =
-    tx.lockPartitionForPush?.bind(tx) ??
-    tx.lockPartitionForCommitValidation?.bind(tx);
-  const commitRejectedPushResult = tx.commitRejectedPushResult?.bind(tx);
+  const tx = await ctx.storage.begin(partition);
   try {
-    if (
-      lockPartitionForPush === undefined ||
-      commitRejectedPushResult === undefined ||
-      (reactionPlanner !== undefined && tx.enqueueReactions === undefined)
-    ) {
+    if (reactionPlanner !== undefined && tx.enqueueReactions === undefined) {
       throw new Error(
-        'storage transaction does not support serialized push apply, atomic rejection finalization, and configured durable reactions',
+        'storage transaction does not support configured durable reactions',
       );
     }
-    await lockPartitionForPush();
-    // RFC 0007: evaluate the gate on this transaction's own connection while
-    // the partition lock is held, so the migration check and the write share
-    // one transaction and no migration can interleave in between.
-    const gate = await tx.readServeGate(schema.version);
-    const refusal = serveGateRefusal(gate, schema.version, ctx.checkpoints);
-    if (refusal !== undefined) {
-      await tx.rollback();
-      throw serveNotReadyError(refusal);
-    }
-    // The optimistic lookup above may have raced another delivery. Re-check
-    // only after acquiring partition serialization and before any operation
-    // read, validation, merge, or staged write. The re-check runs on the
-    // transaction's own connection when the backend provides one: a pooled
-    // Postgres client holding the partition lock would otherwise wait for a
-    // second pool slot and deadlock against pushes waiting on the lock. A
-    // racing duplicate's result commits before the lock releases, so the
-    // transaction-scoped read observes it.
+    // The storage takes the partition lock and then reads the admission token
+    // on the transaction's own connection: the RFC 0007 serve gate and the
+    // §2.3 idempotency record, plus any read-ahead the caller already knows.
+    // A racing duplicate's result commits before the lock releases, so the
+    // later read is exact and a duplicate never reruns an operation,
+    // validator, or merge.
+    let prepared: PreparedPushApply;
     try {
-      const serializedPersisted =
-        tx.getPushResult !== undefined
-          ? await tx.getPushResult(clientId, clientCommitId)
-          : await storage.getPushResult(partition, clientId, clientCommitId);
-      if (serializedPersisted !== undefined) {
-        await tx.rollback();
-        return processedPushCommit(clientCommitId, serializedPersisted, true);
-      }
+      prepared = await tx.preparePushApply(
+        clientId,
+        clientCommitId,
+        schema.version,
+        prefetchTargets,
+      );
     } catch (error) {
       if (
         error instanceof SyncError &&
         error.code === 'sync.idempotency_cache_miss'
       ) {
+        // §6.3: answer the retryable cache-miss for this commit rather than
+        // re-applying. Not persisted — a retry may find a readable record.
         await tx.rollback();
         return {
           frame: idempotencyCacheMissFrame(clientCommitId, error),
@@ -1456,6 +1662,22 @@ export async function processPushOperationsWithTrace(
       }
       throw error;
     }
+    // RFC 0007: the gate is evaluated while the partition lock is held, so
+    // the migration check and the write share one transaction and no
+    // migration can interleave in between.
+    const refusal = serveGateRefusal(
+      prepared.gate,
+      schema.version,
+      ctx.checkpoints,
+    );
+    if (refusal !== undefined) {
+      await tx.rollback();
+      throw serveNotReadyError(refusal);
+    }
+    if (prepared.persisted !== undefined) {
+      await tx.rollback();
+      return processedPushCommit(clientCommitId, prepared.persisted, true);
+    }
     const results: PushOperationResult[] = [];
     const changes: NewChange[] = [];
     const validatedOperations: ValidateCommitOperation[] = [];
@@ -1470,8 +1692,10 @@ export async function processPushOperationsWithTrace(
         ids.push(op.rowId);
         targets.set(op.table, ids);
       }
-      for (const [table, rowIds] of targets) {
-        await tx.prefetchRows(table, rowIds);
+      if (targets.size > 0) {
+        await tx.prefetchRows(
+          [...targets].map(([table, rowIds]) => ({ table, rowIds })),
+        );
       }
     }
     for (let opIndex = 0; opIndex < operations.length; opIndex++) {
@@ -1495,9 +1719,9 @@ export async function processPushOperationsWithTrace(
         break;
       }
       results.push({ opIndex, status: 'applied' });
-      validatedOperations.push(outcome.operation);
+      validatedOperations.push(...outcome.operations);
       stagedWrites.push(outcome.staged);
-      if (outcome.change !== undefined) changes.push(outcome.change);
+      changes.push(...outcome.changes);
     }
 
     // §6.11: reference enforcement runs after every client operation is
@@ -1570,18 +1794,13 @@ export async function processPushOperationsWithTrace(
         results: [terminated],
       });
       // Discard candidates and persist the rejection while retaining the same
-      // partition lock. There is no unlock gap in which a duplicate can rerun.
-      await commitRejectedPushResult(clientId, clientCommitId, stored);
-      const canonical = await storage.getPushResult(
-        partition,
+      // partition lock. There is no unlock gap in which a duplicate can rerun,
+      // and the finalization returns the canonical outcome it persisted.
+      const canonical = await tx.commitRejectedPushResult(
         clientId,
         clientCommitId,
+        stored,
       );
-      if (canonical === undefined) {
-        throw new Error(
-          'push rejection finalization did not persist an outcome',
-        );
-      }
       return processedPushCommit(
         clientCommitId,
         canonical,
@@ -1663,24 +1882,12 @@ export async function processPushOperationsWithTrace(
           },
         ],
       });
-      if (commitRejectedPushResult === undefined) {
-        await tx.rollback();
-        throw new Error(
-          'storage transaction lost atomic push rejection finalization support',
-        );
-      }
       try {
-        await commitRejectedPushResult(clientId, clientCommitId, stored);
-        const canonical = await storage.getPushResult(
-          partition,
+        const canonical = await tx.commitRejectedPushResult(
           clientId,
           clientCommitId,
+          stored,
         );
-        if (canonical === undefined) {
-          throw new Error(
-            'push rejection finalization did not persist an outcome',
-          );
-        }
         return processedPushCommit(
           clientCommitId,
           canonical,