@syncular/server 0.43.0 → 0.45.0

package/src/push.ts

@@ -48,6 +48,7 @@ import { authorizeWrite, renderScopeValue, storedScopesForRow } from './scopes';
 import { decodeColumnVersions, encodeColumnVersions } from './relational-rows';
 import type {
   NewChange,
+  PreparedPushApply,
   StorageTransaction,
   StoredCommit,
   StoredPushResult,
@@ -1574,6 +1575,16 @@ export async function processPushCommitWithTrace(
   frame: PushCommitFrame,
   wireSchema: CompiledSchema = schema,
 ): Promise<ProcessedPushCommit> {
+  // The frame names its rows before the transaction opens, so the admission
+  // read carries the commit's read-ahead and the apply issues no separate
+  // prefetch statement.
+  const targets = new Map<string, string[]>();
+  for (const op of frame.operations) {
+    if (!schema.tables.has(op.table)) continue;
+    const ids = targets.get(op.table) ?? [];
+    ids.push(op.rowId);
+    targets.set(op.table, ids);
+  }
   return processPushOperationsWithTrace(
     ctx,
     schema,
@@ -1582,6 +1593,9 @@ export async function processPushCommitWithTrace(
     frame.clientCommitId,
     async () => frame.operations,
     wireSchema,
+    targets.size === 0
+      ? undefined
+      : [...targets].map(([table, rowIds]) => ({ table, rowIds })),
   );
 }
 
@@ -1600,86 +1614,46 @@ export async function processPushOperationsWithTrace(
     tx: StorageTransaction,
   ) => Promise<readonly ApplyOperation[]>,
   wireSchema: CompiledSchema = schema,
+  prefetchTargets?: readonly {
+    readonly table: string;
+    readonly rowIds: readonly string[];
+  }[],
 ): Promise<ProcessedPushCommit> {
-  const { storage, partition } = ctx;
-  let persisted: StoredPushResult | undefined;
-  try {
-    persisted = await storage.getPushResult(
-      partition,
-      clientId,
-      clientCommitId,
-    );
-  } catch (error) {
-    if (
-      error instanceof SyncError &&
-      error.code === 'sync.idempotency_cache_miss'
-    ) {
-      // §6.3: answer the retryable cache-miss for this commit rather than
-      // re-applying. Not persisted — a retry may find a readable record.
-      return {
-        frame: idempotencyCacheMissFrame(clientCommitId, error),
-        replayed: false,
-      };
-    }
-    throw error;
-  }
-  if (persisted !== undefined) {
-    return processedPushCommit(clientCommitId, persisted, true);
-  }
-
+  const { partition } = ctx;
   const createdAtMs = clockOf(ctx)();
   const blobCtx: BlobApplyContext = { store: ctx.blobs, partition };
   const crdtMergers = ctx.crdtMergers;
   const validators = ctx.validators;
   const commitValidator = ctx.commitValidator;
   const reactionPlanner = ctx.reactionPlanner;
-  const tx = await storage.begin(partition);
-  const lockPartitionForPush =
-    tx.lockPartitionForPush?.bind(tx) ??
-    tx.lockPartitionForCommitValidation?.bind(tx);
-  const commitRejectedPushResult = tx.commitRejectedPushResult?.bind(tx);
+  const tx = await ctx.storage.begin(partition);
   try {
-    if (
-      lockPartitionForPush === undefined ||
-      commitRejectedPushResult === undefined ||
-      (reactionPlanner !== undefined && tx.enqueueReactions === undefined)
-    ) {
+    if (reactionPlanner !== undefined && tx.enqueueReactions === undefined) {
       throw new Error(
-        'storage transaction does not support serialized push apply, atomic rejection finalization, and configured durable reactions',
+        'storage transaction does not support configured durable reactions',
       );
     }
-    await lockPartitionForPush();
-    // RFC 0007: evaluate the gate on this transaction's own connection while
-    // the partition lock is held, so the migration check and the write share
-    // one transaction and no migration can interleave in between.
-    const gate = await tx.readServeGate(schema.version);
-    const refusal = serveGateRefusal(gate, schema.version, ctx.checkpoints);
-    if (refusal !== undefined) {
-      await tx.rollback();
-      throw serveNotReadyError(refusal);
-    }
-    // The optimistic lookup above may have raced another delivery. Re-check
-    // only after acquiring partition serialization and before any operation
-    // read, validation, merge, or staged write. The re-check runs on the
-    // transaction's own connection when the backend provides one: a pooled
-    // Postgres client holding the partition lock would otherwise wait for a
-    // second pool slot and deadlock against pushes waiting on the lock. A
-    // racing duplicate's result commits before the lock releases, so the
-    // transaction-scoped read observes it.
+    // The storage takes the partition lock and then reads the admission token
+    // on the transaction's own connection: the RFC 0007 serve gate and the
+    // §2.3 idempotency record, plus any read-ahead the caller already knows.
+    // A racing duplicate's result commits before the lock releases, so the
+    // later read is exact and a duplicate never reruns an operation,
+    // validator, or merge.
+    let prepared: PreparedPushApply;
     try {
-      const serializedPersisted =
-        tx.getPushResult !== undefined
-          ? await tx.getPushResult(clientId, clientCommitId)
-          : await storage.getPushResult(partition, clientId, clientCommitId);
-      if (serializedPersisted !== undefined) {
-        await tx.rollback();
-        return processedPushCommit(clientCommitId, serializedPersisted, true);
-      }
+      prepared = await tx.preparePushApply(
+        clientId,
+        clientCommitId,
+        schema.version,
+        prefetchTargets,
+      );
     } catch (error) {
       if (
         error instanceof SyncError &&
         error.code === 'sync.idempotency_cache_miss'
       ) {
+        // §6.3: answer the retryable cache-miss for this commit rather than
+        // re-applying. Not persisted — a retry may find a readable record.
         await tx.rollback();
         return {
           frame: idempotencyCacheMissFrame(clientCommitId, error),
@@ -1688,6 +1662,22 @@ export async function processPushOperationsWithTrace(
       }
       throw error;
     }
+    // RFC 0007: the gate is evaluated while the partition lock is held, so
+    // the migration check and the write share one transaction and no
+    // migration can interleave in between.
+    const refusal = serveGateRefusal(
+      prepared.gate,
+      schema.version,
+      ctx.checkpoints,
+    );
+    if (refusal !== undefined) {
+      await tx.rollback();
+      throw serveNotReadyError(refusal);
+    }
+    if (prepared.persisted !== undefined) {
+      await tx.rollback();
+      return processedPushCommit(clientCommitId, prepared.persisted, true);
+    }
     const results: PushOperationResult[] = [];
     const changes: NewChange[] = [];
     const validatedOperations: ValidateCommitOperation[] = [];
@@ -1702,8 +1692,10 @@ export async function processPushOperationsWithTrace(
         ids.push(op.rowId);
         targets.set(op.table, ids);
       }
-      for (const [table, rowIds] of targets) {
-        await tx.prefetchRows(table, rowIds);
+      if (targets.size > 0) {
+        await tx.prefetchRows(
+          [...targets].map(([table, rowIds]) => ({ table, rowIds })),
+        );
       }
     }
     for (let opIndex = 0; opIndex < operations.length; opIndex++) {
@@ -1802,18 +1794,13 @@ export async function processPushOperationsWithTrace(
         results: [terminated],
       });
       // Discard candidates and persist the rejection while retaining the same
-      // partition lock. There is no unlock gap in which a duplicate can rerun.
-      await commitRejectedPushResult(clientId, clientCommitId, stored);
-      const canonical = await storage.getPushResult(
-        partition,
+      // partition lock. There is no unlock gap in which a duplicate can rerun,
+      // and the finalization returns the canonical outcome it persisted.
+      const canonical = await tx.commitRejectedPushResult(
         clientId,
         clientCommitId,
+        stored,
       );
-      if (canonical === undefined) {
-        throw new Error(
-          'push rejection finalization did not persist an outcome',
-        );
-      }
       return processedPushCommit(
         clientCommitId,
         canonical,
@@ -1895,24 +1882,12 @@ export async function processPushOperationsWithTrace(
           },
         ],
       });
-      if (commitRejectedPushResult === undefined) {
-        await tx.rollback();
-        throw new Error(
-          'storage transaction lost atomic push rejection finalization support',
-        );
-      }
       try {
-        await commitRejectedPushResult(clientId, clientCommitId, stored);
-        const canonical = await storage.getPushResult(
-          partition,
+        const canonical = await tx.commitRejectedPushResult(
           clientId,
           clientCommitId,
+          stored,
         );
-        if (canonical === undefined) {
-          throw new Error(
-            'push rejection finalization did not persist an outcome',
-          );
-        }
         return processedPushCommit(
           clientCommitId,
           canonical,

package/src/relational-rows.ts

@@ -807,17 +807,38 @@ export const COMMIT_WINDOW_BATCH_SQL = `SELECT s.i AS sub, c.commit_seq AS commi
      ORDER BY s.i, c.commit_seq, ch.idx`;
 
 /**
- * PostgreSQL push read-ahead: every requested row id with its stored row
- * (NULL columns when absent, `selectRowSql` shape) and its §5 delete
- * tombstone. Params: [partition, rowIds as JSON array text, tbl].
+ * PostgreSQL push read-ahead across every table a commit targets, in one
+ * statement: each requested row id with its stored row (NULL columns when
+ * absent, `selectRowSql` shape) and its §5 delete tombstone. Params:
+ * [partition, specs as JSON text], `specs` = `{tbl, ids: string[]}` records.
  */
-export function prefetchRowsSql(table: CompiledTable): string {
-  return `SELECT ids.id AS lookup_id, r.${quoteIdent(SYNC_ROW_ID_COLUMN)} AS row_id, r.${quoteIdent(SYNC_VERSION_COLUMN)} AS server_version, r.${quoteIdent(SYNC_SCOPES_COLUMN)} AS scopes, r.${quoteIdent(SYNC_PAYLOAD_COLUMN)} AS payload, r.${quoteIdent(SYNC_COLUMN_VERSIONS_COLUMN)} AS column_versions, t.commit_seq AS tombstone_seq
-     FROM jsonb_array_elements_text($2::text::jsonb) AS ids(id)
-     LEFT JOIN ${quoteIdent(table.name)} r
-       ON r.${quoteIdent(SYNC_PARTITION_COLUMN)}=$1 AND r.${quoteIdent(SYNC_ROW_ID_COLUMN)}=ids.id
-     LEFT JOIN sync_tombstones t
-       ON t.partition=$1 AND t.tbl=$3 AND t.row_id=ids.id`;
+export function prefetchRowsBatchSql(
+  tables: readonly CompiledTable[],
+  params: { readonly partition: string; readonly specs: string } = {
+    partition: '$1',
+    specs: '$2',
+  },
+): string {
+  // One branch per table: a lateral join cannot switch its relation per
+  // record, and the union keeps the whole read-ahead one round trip.
+  const projection = tables
+    .map((table) => {
+      const quoted = quoteIdent(table.name);
+      return `SELECT ids.tbl AS tbl, ids.id AS lookup_id, r.${quoteIdent(SYNC_ROW_ID_COLUMN)} AS row_id, r.${quoteIdent(SYNC_VERSION_COLUMN)} AS server_version, r.${quoteIdent(SYNC_SCOPES_COLUMN)} AS scopes, r.${quoteIdent(SYNC_PAYLOAD_COLUMN)} AS payload, r.${quoteIdent(SYNC_COLUMN_VERSIONS_COLUMN)} AS column_versions, t.commit_seq AS tombstone_seq
+       FROM ids
+       LEFT JOIN ${quoted} r
+         ON r.${quoteIdent(SYNC_PARTITION_COLUMN)}=${params.partition} AND r.${quoteIdent(SYNC_ROW_ID_COLUMN)}=ids.id
+       LEFT JOIN sync_tombstones t
+         ON t.partition=${params.partition} AND t.tbl=ids.tbl AND t.row_id=ids.id
+      WHERE ids.tbl='${table.name.replaceAll("'", "''")}'`;
+    })
+    .join('\n     UNION ALL ');
+  return `WITH ids AS (
+       SELECT spec.tbl AS tbl, entry.id AS id
+         FROM jsonb_to_recordset(${params.specs}::text::jsonb) AS spec(tbl text, ids jsonb)
+         CROSS JOIN LATERAL jsonb_array_elements_text(spec.ids) AS entry(id)
+     )
+     ${projection}`;
 }
 
 /**

package/src/sqlite-storage.ts

@@ -101,6 +101,7 @@ import type {
   IndexRowScanQuery,
   NewCommit,
   NewReaction,
+  PreparedPushApply,
   PartitionRegistryEntry,
   PrunedReactionCounts,
   ReactionClaimQuery,
@@ -255,20 +256,6 @@ class SqliteTransaction implements StorageTransaction {
       .run(this.#partition, table, rowId);
   }
 
-  getPushResult(
-    clientId: string,
-    clientCommitId: string,
-  ): Promise<StoredPushResult | undefined> {
-    this.#assertOpen();
-    // One shared SQLite connection: this read runs inside this
-    // transaction's BEGIN IMMEDIATE.
-    return this.#storage.getPushResult(
-      this.#partition,
-      clientId,
-      clientCommitId,
-    );
-  }
-
   scanRows(query: RowScanQuery): Promise<StoredRow[]> {
     this.#assertOpen();
     return this.#storage.scanRows(this.#partition, query);
@@ -288,11 +275,30 @@ class SqliteTransaction implements StorageTransaction {
     return this.#storage.runAuthoritativeQuery(this.#partition, query);
   }
 
-  async lockPartitionForPush(): Promise<void> {
+  async preparePushApply(
+    clientId: string,
+    clientCommitId: string,
+    runningSchemaVersion: number,
+    _targets: readonly {
+      readonly table: string;
+      readonly rowIds: readonly string[];
+    }[] = [],
+  ): Promise<PreparedPushApply> {
     this.#assertOpen();
     // BEGIN IMMEDIATE in the constructor already owns SQLite's writer lock.
     this.#storage.db.exec('SAVEPOINT syncular_push_candidate');
     this.#pushApplySavepoint = true;
+    return {
+      gate: await this.#storage.readServeGate(
+        this.#partition,
+        runningSchemaVersion,
+      ),
+      persisted: await this.#storage.getPushResult(
+        this.#partition,
+        clientId,
+        clientCommitId,
+      ),
+    };
   }
 
   async advanceCheckpoint(
@@ -324,7 +330,7 @@ class SqliteTransaction implements StorageTransaction {
     clientId: string,
     clientCommitId: string,
     result: StoredPushResult,
-  ): Promise<void> {
+  ): Promise<StoredPushResult> {
     this.#assertOpen();
     if (!this.#pushApplySavepoint) {
       throw new Error('push rejection requires its apply savepoint');
@@ -334,6 +340,9 @@ class SqliteTransaction implements StorageTransaction {
     this.#pushApplySavepoint = false;
     await this.putPushResult(clientId, clientCommitId, result);
     await this.commit();
+    // The idempotency re-check found no record under the writer lock, so the
+    // insert above is the canonical outcome.
+    return result;
   }
 
   async upsertRow(

package/src/storage.ts

@@ -74,6 +74,18 @@ export interface ServeGate {
   }[];
 }
 
+/**
+ * The push apply's admission token: the partition lock, the serve gate, and
+ * the idempotency record read in one round trip. A storage with per-statement
+ * network cost answers all three with one statement (see
+ * `PostgresServerStorage`); the lock is held from this call through commit.
+ */
+export interface PreparedPushApply {
+  readonly gate: ServeGate;
+  /** The already-persisted outcome for this key, or `undefined` on a miss. */
+  readonly persisted: StoredPushResult | undefined;
+}
+
 /** Why `serveGate` refuses, or `undefined` when this request may serve. */
 export type ServeGateRefusal =
   | {
@@ -547,12 +559,17 @@ export interface StorageTransaction {
   getRow(table: string, rowId: string): Promise<StoredRow | undefined>;
   /**
    * Optional read-ahead: the push layer names every row a commit's
-   * operations target, per table, before it applies them. A storage with
-   * per-statement network cost reads them in one statement and answers the
-   * following `getRow` calls from that snapshot until the row is written in
-   * this transaction.
+   * operations target, across every table, before it applies them. A storage
+   * with per-statement network cost reads them in one statement (one branch
+   * per table) and answers the following `getRow` calls from that snapshot
+   * until the row is written in this transaction.
    */
-  prefetchRows?(table: string, rowIds: readonly string[]): Promise<void>;
+  prefetchRows?(
+    targets: readonly {
+      readonly table: string;
+      readonly rowIds: readonly string[];
+    }[],
+  ): Promise<void>;
   /**
    * §5 delete-precedence tombstone read: the `commit_seq` of the newest
    * applied delete of this row still within the pruning horizon, or
@@ -567,20 +584,6 @@ export interface StorageTransaction {
    * over an absent row; a no-op when no tombstone exists.
    */
   clearTombstone(table: string, rowId: string): Promise<void>;
-  /**
-   * Optional transaction-scoped idempotency lookup (§2.3) with the same
-   * semantics as `ServerStorage.getPushResult`, including the
-   * `sync.idempotency_cache_miss` throw. The push layer prefers it for the
-   * post-serialization duplicate re-check so the read runs on the
-   * transaction's own connection — a pooled Postgres client that holds the
-   * partition lock must never wait for a second pool slot mid-push. In-tree
-   * SQLite/Postgres/D1 adapters implement it; a custom adapter that omits it
-   * keeps the pool-level read.
-   */
-  getPushResult?(
-    clientId: string,
-    clientCommitId: string,
-  ): Promise<StoredPushResult | undefined>;
   /**
    * Optional candidate-state scan used only by whole-commit validation.
    * In-tree SQLite/Postgres/D1 backends implement it with read-your-own-writes
@@ -610,28 +613,37 @@ export interface StorageTransaction {
   ): Promise<AuthoritativeQueryResult>;
   /**
    * Serialize every push apply for this partition before any operation read,
-   * validation, merge, or write. The push layer re-checks idempotency only
-   * after this resolves and retains the lock through terminal-result commit.
+   * validation, merge, or write, and read the admission token in the same
+   * round trip: the RFC 0007 serve gate and the §2.3 idempotency record for
+   * `clientCommitId`. The lock is retained through terminal-result commit,
+   * so the idempotency read is exact and a racing duplicate cannot rerun.
+   * `targets` are the rows the commit is known to touch before its operations
+   * are built; a storage with per-statement network cost folds that read-ahead
+   * into the admission read and answers the following `getRow` calls from it.
    * Missing support fails closed before an app-row mutation.
    */
-  lockPartitionForPush?(): Promise<void>;
-  /**
-   * @deprecated Implement `lockPartitionForPush`. Kept as a compatibility
-   * bridge for custom adapters whose existing implementation already locks
-   * the complete partition from before candidate reads through commit.
-   */
-  lockPartitionForCommitValidation?(): Promise<void>;
+  preparePushApply(
+    clientId: string,
+    clientCommitId: string,
+    runningSchemaVersion: number,
+    targets?: readonly {
+      readonly table: string;
+      readonly rowIds: readonly string[];
+    }[],
+  ): Promise<PreparedPushApply>;
   /**
    * Rejection finalization while the push-apply serialization lock is still
    * held: discard every candidate write, persist the rejected idempotency
-   * result, and finish the transaction atomically. Required for every push so
-   * a concurrent duplicate cannot rerun operations, validators, or merges.
+   * result, and finish the transaction atomically. Returns the persisted
+   * canonical outcome, so the caller never re-reads it. Required for every
+   * push so a concurrent duplicate cannot rerun operations, validators, or
+   * merges.
    */
-  commitRejectedPushResult?(
+  commitRejectedPushResult(
     clientId: string,
     clientCommitId: string,
     result: StoredPushResult,
-  ): Promise<void>;
+  ): Promise<StoredPushResult>;
   upsertRow(
     table: string,
     row: StoredRow,
@@ -744,10 +756,10 @@ export interface ServerStorage {
    * same command races a push on Postgres and a SQLite-only test conceals it.
    *
    * A trusted command must therefore run through `registerRemoteCommand`,
-   * which builds after `lockPartitionForPush` and the idempotency recheck and
-   * shares the committed transaction, or call `lockPartitionForPush` before
-   * its first authority, version, or replay read. SSP2 request handling does
-   * this for you; only host-authored commands choose.
+   * which builds after `preparePushApply` and shares the committed
+   * transaction, or call `preparePushApply` before its first authority,
+   * version, or replay read. SSP2 request handling does this for you; only
+   * host-authored commands choose.
    */
   begin(partition: string): Promise<StorageTransaction>;