@powersync/service-core 1.23.0 → 1.23.2

package/src/sync/sync.ts

@@ -9,11 +9,19 @@ import * as util from '../util/util-index.js';
 
 import { Logger, logger as defaultLogger } from '@powersync/lib-services-framework';
 import { mergeAsyncIterables } from '../streams/streams-index.js';
-import { BucketChecksumState, CheckpointLine } from './BucketChecksumState.js';
+import { PerformanceTracer, type Span } from '../tracing/PerformanceTracer.js';
+import { BucketChecksumState, CheckpointLine, type SyncCheckpointTraceCategory } from './BucketChecksumState.js';
 import { OperationsSentStats, RequestTracker, statsForBatch } from './RequestTracker.js';
 import { SyncContext } from './SyncContext.js';
 import { TokenStreamOptions, acquireSemaphoreAbortable, settledPromise, tokenStream } from './util.js';
 
+type CheckpointTiming = Record<string, number>;
+
+interface ActiveCheckpointTrace {
+  tracer: PerformanceTracer<SyncCheckpointTraceCategory>;
+  span: Span;
+}
+
 export interface SyncStreamParameters {
   syncContext: SyncContext;
   bucketStorage: storage.SyncRulesBucketStorage;
@@ -119,8 +127,9 @@ async function* streamResponseInner(
   const newCheckpoints = stream[Symbol.asyncIterator]();
 
   type CheckpointAndLine = {
-    checkpoint: bigint;
+    checkpoint: storage.ReplicationCheckpoint;
     line: CheckpointLine | null;
+    trace: ActiveCheckpointTrace | null;
   };
 
   async function waitForNewCheckpointLine(): Promise<IteratorResult<CheckpointAndLine>> {
@@ -128,9 +137,22 @@ async function* streamResponseInner(
     if (next.done) {
       return { done: true, value: undefined };
     }
+    const cp = next.value.base;
 
-    const line = await checksumState.buildNextCheckpointLine(next.value);
-    return { done: false, value: { checkpoint: next.value.base.checkpoint, line } };
+    const tracer = new PerformanceTracer<SyncCheckpointTraceCategory>(`sync checkpoint ${cp.checkpoint}`);
+    const trace = {
+      tracer,
+      span: tracer.span('checkpoint')
+    };
+
+    try {
+      const line = await checksumState.buildNextCheckpointLine(next.value, trace.tracer);
+      return { done: false, value: { checkpoint: cp, line, trace: line == null ? null : trace } };
+    } catch (e) {
+      // Only end the span if we error. If we return normally, we pass ownership on to the caller.
+      trace.span.end();
+      throw e;
+    }
   }
 
   try {
@@ -155,12 +177,18 @@ async function* streamResponseInner(
         // No update to sync
         continue;
       }
+      const trace = next.value.value.trace!;
+      const checkpoint = next.value.value.checkpoint;
+      const tracer = trace.tracer;
 
-      const { checkpointLine, bucketsToFetch } = line;
+      const { checkpointLine, bucketsToFetch, bucketDataRequestHint } = line;
 
       // Since yielding can block, we update the state just before yielding the line.
       line.advance();
-      yield checkpointLine;
+      {
+        using _ = tracer.span('sending', 'checkpoint');
+        yield checkpointLine;
+      }
 
       // Start syncing data for buckets up to the checkpoint. As soon as we have completed at least one priority and
       // at least 1000 operations, we also start listening for new checkpoints concurrently. When a new checkpoint comes
@@ -169,6 +197,8 @@ async function* streamResponseInner(
       const abortCheckpointController = new AbortController();
       let syncedOperations = 0;
 
+      let checkpointResult: CheckpointResult | null = null;
+
       const abortCheckpointSignal = AbortSignal.any([abortCheckpointController.signal, signal]);
 
       const bucketsByPriority = [...Map.groupBy(bucketsToFetch, (bucket) => bucket.priority).entries()];
@@ -190,6 +220,11 @@ async function* streamResponseInner(
             while (true) {
               const next = await settledPromise(waitForNewCheckpointLine());
               if (next.status == 'rejected') {
+                if (next.reason instanceof AbortError) {
+                  checkpointResult = { result: 'invalidated', invalidationReason: 'checkpoint_cancelled' };
+                } else {
+                  checkpointResult = { result: 'invalidated', invalidationReason: 'checkpoint_error' };
+                }
                 abortCheckpointController.abort();
               } else if (!next.value.done) {
                 if (next.value.value.line == null) {
@@ -200,6 +235,7 @@ async function* streamResponseInner(
 
                 // A new sync line can be emitted. Stop running the bucketDataInBatches() iterations, making the
                 // main flow reach the new checkpoint.
+                checkpointResult = { result: 'invalidated', invalidationReason: 'checkpoint_superseded' };
                 abortCheckpointController.abort();
               }
 
@@ -215,6 +251,14 @@ async function* streamResponseInner(
         maybeRaceForNewCheckpoint();
       }
 
+      function checkpointLogDetails(cp: storage.ReplicationCheckpoint) {
+        return {
+          checkpoint: cp.checkpoint,
+          user_id: tokenPayload.userIdJson,
+          ...tracker.getIncrementalCheckpointStats()
+        };
+      }
+
       // This incrementally updates dataBuckets with each individual bucket position.
       // At the end of this, we can be sure that all buckets have data up to the checkpoint.
       for (const [priority, buckets] of priorityBatches) {
@@ -223,11 +267,12 @@ async function* streamResponseInner(
           break;
         }
 
-        yield* bucketDataInBatches({
+        const batchResult = yield* bucketDataInBatches({
           syncContext: syncContext,
           bucketStorage: bucketStorage,
-          checkpoint: next.value.value.checkpoint,
+          checkpoint,
           bucketsToFetch: buckets,
+          requestHint: bucketDataRequestHint,
           checkpointLine: line,
           legacyDataLines: !isEncodingAsBson && params.raw_data != true,
           onRowsSent: markOperationsSent,
@@ -237,7 +282,35 @@ async function* streamResponseInner(
           // Passing null here will emit a full sync complete message at the end. If we pass a priority, we'll emit a partial
           // sync complete message instead.
           forPriority: !isLast ? priority : null,
-          logger
+          logger,
+          tracer
+        });
+        if (batchResult?.result == 'partial_checkpoint_complete') {
+          // We don't specifically include timing details here, since that would end the span.
+          // Timings are included in the checkpoint_complete following this.
+          logger.info(`partial_checkpoint_complete: ${checkpoint.checkpoint}`, {
+            priority,
+            ...checkpointLogDetails(checkpoint)
+          });
+        } else if (batchResult?.result != null) {
+          checkpointResult = batchResult;
+          break;
+        }
+      }
+
+      if (checkpointResult?.result == 'checkpoint_complete') {
+        logger.info(`checkpoint_complete: ${checkpoint.checkpoint}`, {
+          // Incremental stats since the last checkpoint_complete, partial_checkpoint_complete or checkpoint_invalidated
+          ...checkpointLogDetails(checkpoint),
+          // Timings since the last checkpoint or checkpoint_diff
+          ms: getCheckpointTraceTimings(trace.span)
+        });
+      } else if (checkpointResult?.result == 'invalidated') {
+        // A new checkpoint and checkpoint_complete should follow soon, but we log the stats already
+        logger.info(`checkpoint_invalidated: ${checkpoint.checkpoint}`, {
+          reason: checkpointResult.invalidationReason,
+          ...checkpointLogDetails(checkpoint),
+          ms: getCheckpointTraceTimings(trace.span)
         });
       }
 
@@ -253,7 +326,8 @@ async function* streamResponseInner(
 interface BucketDataRequest {
   syncContext: SyncContext;
   bucketStorage: storage.SyncRulesBucketStorage;
-  checkpoint: util.InternalOpId;
+  checkpoint: storage.ReplicationCheckpoint;
+  requestHint: storage.BucketRequestHint;
   /** Contains current bucket state. Modified by the request as data is sent. */
   checkpointLine: CheckpointLine;
   /** Subset of checkpointLine.bucketsToFetch, filtered by priority. */
@@ -272,9 +346,20 @@ interface BucketDataRequest {
   forPriority: BucketPriority | null;
   onRowsSent: (stats: OperationsSentStats) => void;
   logger: Logger;
+  tracer: PerformanceTracer<SyncCheckpointTraceCategory>;
 }
 
-async function* bucketDataInBatches(request: BucketDataRequest) {
+type CheckpointResult =
+  | { result: 'partial_checkpoint_complete' }
+  | { result: 'checkpoint_complete' }
+  | {
+      result: 'invalidated';
+      invalidationReason: 'compacted' | 'checkpoint_error' | 'checkpoint_superseded' | 'checkpoint_cancelled';
+    };
+
+async function* bucketDataInBatches(
+  request: BucketDataRequest
+): AsyncGenerator<util.StreamingSyncLine | string | null, CheckpointResult | null> {
   let isDone = false;
   while (!request.abort_batch.aborted && !isDone) {
     // The code below is functionally the same as this for-await loop below.
@@ -291,9 +376,13 @@ async function* bucketDataInBatches(request: BucketDataRequest) {
       while (true) {
         const { value, done: iterDone } = await iter.next();
         if (iterDone) {
+          if (value != null) {
+            return value;
+          }
           break;
         } else {
           const { done, data } = value;
+          using _ = request.tracer.span('sending', 'data');
           yield data;
           if (done) {
             isDone = true;
@@ -301,9 +390,10 @@ async function* bucketDataInBatches(request: BucketDataRequest) {
         }
       }
     } finally {
-      await iter.return();
+      await iter.return(null);
     }
   }
+  return null;
 }
 
 interface BucketDataBatchResult {
@@ -314,11 +404,14 @@ interface BucketDataBatchResult {
 /**
  * Extracted as a separate internal function just to avoid memory leaks.
  */
-async function* bucketDataBatch(request: BucketDataRequest): AsyncGenerator<BucketDataBatchResult, void> {
+async function* bucketDataBatch(
+  request: BucketDataRequest
+): AsyncGenerator<BucketDataBatchResult, CheckpointResult | null> {
   const {
     syncContext,
     bucketStorage: storage,
     checkpoint,
+    requestHint,
     bucketsToFetch,
     checkpointLine,
     legacyDataLines,
@@ -328,42 +421,34 @@ async function* bucketDataBatch(request: BucketDataRequest): AsyncGenerator<Buck
     logger
   } = request;
 
+  const tracer = request.tracer;
+
   let checkpointInvalidated = false;
 
-  if (syncContext.syncSemaphore.isLocked()) {
-    logger.info('Sync concurrency limit reached, waiting for lock', { user_id: request.userIdForLogs });
-  }
-  const acquired = await acquireSemaphoreAbortable(syncContext.syncSemaphore, AbortSignal.any([abort_batch]));
+  const acquired = await tracer.span('acquiring_lock').with(async () => {
+    return acquireSemaphoreAbortable(syncContext.syncSemaphore, AbortSignal.any([abort_batch]));
+  });
   if (acquired === 'aborted') {
-    return;
+    return null;
   }
 
-  const [value, release] = acquired;
+  const [, release] = acquired;
   try {
-    if (value <= 3) {
-      // This can be noisy, so we only log when we get close to the
-      // concurrency limit.
-      logger.info(`Got sync lock. Slots available: ${value - 1}`, {
-        user_id: request.userIdForLogs,
-        sync_data_slots: value - 1
-      });
-    }
+    using _ = tracer.span('bucket_data');
+    let has_more = false;
     // Optimization: Only fetch buckets for which the checksums have changed since the last checkpoint
     // For the first batch, this will be all buckets.
     const filteredBuckets = checkpointLine.getFilteredBucketPositions(bucketsToFetch);
-    const dataBatches = storage.getBucketDataBatch(checkpoint, filteredBuckets);
-
-    let has_more = false;
-
+    const dataBatches = storage.getBucketDataBatch(checkpoint, filteredBuckets, { requestHint });
     for await (let { chunkData: r, targetOp } of dataBatches) {
       // Abort in current batch if the connection is closed
       if (abort_connection.aborted) {
-        return;
+        return null;
       }
       if (r.has_more) {
         has_more = true;
       }
-      if (targetOp != null && targetOp > checkpoint) {
+      if (targetOp != null && targetOp > checkpoint.checkpoint) {
         checkpointInvalidated = true;
       }
       if (r.data.length == 0) {
@@ -371,7 +456,11 @@ async function* bucketDataBatch(request: BucketDataRequest): AsyncGenerator<Buck
         // storage can emit an empty chunk for a bucket that has no operations in the
         // requested range, and its position must advance so the bucket is not
         // re-requested indefinitely.
-        checkpointLine.updateBucketPosition({ bucket: r.bucket, nextAfter: BigInt(r.next_after), hasMore: r.has_more });
+        checkpointLine.updateBucketPosition({
+          bucket: r.bucket,
+          nextAfter: BigInt(r.next_after),
+          hasMore: r.has_more
+        });
         continue;
       }
       logger.debug(`Sending data for ${r.bucket}`);
@@ -398,7 +487,7 @@ async function* bucketDataBatch(request: BucketDataRequest): AsyncGenerator<Buck
       // Check if syncing bucket data is supposed to stop before fetching more data
       // from storage.
       if (abort_batch.aborted) {
-        return;
+        return null;
       }
     }
 
@@ -408,35 +497,50 @@ async function* bucketDataBatch(request: BucketDataRequest): AsyncGenerator<Buck
         // Don't send the checkpoint_complete line in this case.
         // More data should be available immediately for a new checkpoint.
         yield { data: null, done: true };
+        return { result: 'invalidated', invalidationReason: 'compacted' };
       } else {
         if (request.forPriority != null) {
           const line: util.StreamingSyncCheckpointPartiallyComplete = {
             partial_checkpoint_complete: {
-              last_op_id: util.internalToExternalOpId(checkpoint),
+              last_op_id: util.internalToExternalOpId(checkpoint.checkpoint),
               priority: request.forPriority
             }
           };
           yield { data: line, done: true };
+          return { result: 'partial_checkpoint_complete' };
         } else {
           const line: util.StreamingSyncCheckpointComplete = {
             checkpoint_complete: {
-              last_op_id: util.internalToExternalOpId(checkpoint)
+              last_op_id: util.internalToExternalOpId(checkpoint.checkpoint)
             }
           };
           yield { data: line, done: true };
+          return { result: 'checkpoint_complete' };
         }
       }
     }
   } finally {
-    if (value <= 3) {
-      // This can be noisy, so we only log when we get close to the
-      // concurrency limit.
-      logger.info(`Releasing sync lock`, {
-        user_id: request.userIdForLogs
-      });
-    }
     release();
   }
+  return null;
+}
+
+function getCheckpointTraceTimings(span: Span): CheckpointTiming {
+  const timings = span.end();
+  const result: CheckpointTiming = {};
+  for (let key in timings) {
+    addTiming(result, key, timings[key]);
+  }
+  addTiming(result, 'other', span.selfDuration);
+  addTiming(result, 'total', span.endAt - span.startAt);
+  return result;
+}
+
+function addTiming(timings: CheckpointTiming, key: string, durationMicros: number) {
+  const ms = Math.round(durationMicros / 1000);
+  if (ms > 0) {
+    timings[key] = (timings[key] ?? 0) + ms;
+  }
 }
 
 function transformLegacyResponse(bucketData: util.SyncBucketData): util.SyncBucketData<util.ProtocolOplogData> {

package/src/system/ServiceContext.ts

@@ -19,6 +19,7 @@ export interface ServiceContext {
   storageEngine: storage.StorageEngine;
   migrations: PowerSyncMigrationManager;
   syncContext: SyncContext;
+  writeCheckpointBatcher: utils.WriteCheckpointBatcher;
   serviceMode: ServiceContextMode;
   eventsEngine: EventsEngine;
 }
@@ -50,6 +51,7 @@ export class ServiceContextContainer implements ServiceContext {
   eventsEngine: EventsEngine;
   syncContext: SyncContext;
   routerEngine: routes.RouterEngine;
+  writeCheckpointBatcher: utils.WriteCheckpointBatcher;
   serviceMode: ServiceContextMode;
 
   constructor(options: ServiceContextOptions) {
@@ -84,6 +86,11 @@ export class ServiceContextContainer implements ServiceContext {
       stop: (routerEngine) => routerEngine.shutDown()
     });
 
+    this.writeCheckpointBatcher = new utils.WriteCheckpointBatcher(
+      () => this.routerEngine.getAPI(),
+      () => this.storageEngine.activeBucketStorage
+    );
+
     this.syncContext = new SyncContext({
       maxDataFetchConcurrency: configuration.api_parameters.max_data_fetch_concurrency,
       maxBuckets: configuration.api_parameters.max_buckets_per_connection,

package/src/tracing/PerformanceTracer.ts

@@ -8,10 +8,13 @@ export interface Span extends Disposable {
   startAt: number;
   /**
    * End time in microseconds since an arbitrary epoch.
+   *
+   * 0 if the span hasn't ended yet.
    */
   endAt: number;
+
   /**
-   * Time spent not in nested spans.
+   * Time spent not in nested spans, in microseconds.
    */
   selfDuration: number;
 
@@ -36,6 +39,11 @@ export interface Span extends Disposable {
    * Returns an aggregate record of category -> "selfDuration", in microseconds.
    */
   end(): Record<string, number>;
+
+  /**
+   * Call a nested function, then end the span, returning the function's result.
+   */
+  with<T>(cb: () => Promise<T>): Promise<T>;
 }
 
 function now() {
@@ -138,8 +146,17 @@ export class PerformanceTracer<K extends string> {
       get durationMillis() {
         return Math.ceil((this.endAt - this.startAt) / 1000);
       },
+
       [Symbol.dispose]() {
         this.end();
+      },
+
+      async with<T>(cb: () => Promise<T>): Promise<T> {
+        try {
+          return await cb();
+        } finally {
+          this.end();
+        }
       }
     };
     this.stack.push(s);

package/src/util/checkpointing.ts

@@ -1,33 +1,14 @@
-import { ErrorCode, ServiceError } from '@powersync/lib-services-framework';
-import { RouteAPI } from '../api/RouteAPI.js';
-import { BucketStorageFactory } from '../storage/storage-index.js';
+import { WriteCheckpointBatcher } from './write-checkpoint-batcher.js';
 
 export interface CreateWriteCheckpointOptions {
   userId: string | undefined;
   clientId: string | undefined;
-  api: RouteAPI;
-  storage: BucketStorageFactory;
+  batcher: WriteCheckpointBatcher;
 }
 export async function createWriteCheckpoint(options: CreateWriteCheckpointOptions) {
   const full_user_id = checkpointUserId(options.userId, options.clientId);
 
-  const syncBucketStorage = (await options.storage.getActiveSyncConfig())?.storage;
-  if (!syncBucketStorage) {
-    throw new ServiceError(ErrorCode.PSYNC_S2302, `Cannot create Write Checkpoint since no sync config is active.`);
-  }
-
-  const { writeCheckpoint, currentCheckpoint } = await options.api.createReplicationHead(async (currentCheckpoint) => {
-    const writeCheckpoint = await syncBucketStorage.createManagedWriteCheckpoint({
-      user_id: full_user_id,
-      heads: { '1': currentCheckpoint }
-    });
-    return { writeCheckpoint, currentCheckpoint };
-  });
-
-  return {
-    writeCheckpoint: String(writeCheckpoint),
-    replicationHead: currentCheckpoint
-  };
+  return options.batcher.enqueue(full_user_id);
 }
 
 export function checkpointUserId(user_id: string | undefined, client_id: string | undefined) {

package/src/util/config/compound-config-collector.ts

@@ -6,6 +6,7 @@ import { Base64ConfigCollector } from './collectors/impl/base64-config-collector
 import { FallbackConfigCollector } from './collectors/impl/fallback-config-collector.js';
 import { FileSystemConfigCollector } from './collectors/impl/filesystem-config-collector.js';
 import {
+  DEFAULT_CHECKSUM_CACHE_TTL_MINUTES,
   DEFAULT_MAX_BUCKETS_PER_CONNECTION,
   DEFAULT_MAX_CONCURRENT_CONNECTIONS,
   DEFAULT_MAX_DATA_FETCH_CONCURRENCY,
@@ -193,7 +194,10 @@ export class CompoundConfigCollector {
         max_concurrent_connections:
           baseConfig.api?.parameters?.max_concurrent_connections ?? DEFAULT_MAX_CONCURRENT_CONNECTIONS,
         max_data_fetch_concurrency:
-          baseConfig.api?.parameters?.max_data_fetch_concurrency ?? DEFAULT_MAX_DATA_FETCH_CONCURRENCY
+          baseConfig.api?.parameters?.max_data_fetch_concurrency ?? DEFAULT_MAX_DATA_FETCH_CONCURRENCY,
+        bucket_count_cache_ttl_minutes: normalizeChecksumCacheTtlMinutes(
+          baseConfig.api?.parameters?.bucket_count_cache_ttl_minutes
+        )
       },
       // TODO maybe move this out of the connection or something
       slot_name_prefix: baseConfig.replication?.connections?.[0]?.slot_name_prefix ?? 'powersync_',
@@ -249,3 +253,11 @@ export class CompoundConfigCollector {
     };
   }
 }
+
+function normalizeChecksumCacheTtlMinutes(ttlMinutes: number | undefined): number {
+  const normalized = ttlMinutes ?? DEFAULT_CHECKSUM_CACHE_TTL_MINUTES;
+  if (!Number.isFinite(normalized) || !Number.isInteger(normalized) || normalized < 1) {
+    throw new Error('api.parameters.bucket_count_cache_ttl_minutes must be a positive integer');
+  }
+  return normalized;
+}

package/src/util/config/defaults.ts

@@ -3,3 +3,4 @@ export const DEFAULT_MAX_CONCURRENT_CONNECTIONS = 200;
 export const DEFAULT_MAX_DATA_FETCH_CONCURRENCY = 10;
 export const DEFAULT_MAX_BUCKETS_PER_CONNECTION = 1000;
 export const DEFAULT_MAX_PARAMETER_QUERY_RESULTS = 1000;
+export const DEFAULT_CHECKSUM_CACHE_TTL_MINUTES = 1 * 60;

package/src/util/config/types.ts

@@ -53,6 +53,7 @@ export type ResolvedPowerSyncConfig = {
     max_data_fetch_concurrency: number;
     max_buckets_per_connection: number;
     max_parameter_query_results: number;
+    bucket_count_cache_ttl_minutes: number;
   };
 
   /** Prefix for postgres replication slot names, as well as replication stream names. */

package/src/util/util-index.ts

@@ -8,6 +8,7 @@ export * from './protocol-types.js';
 export * from './secs.js';
 export * from './utils.js';
 export * from './version.js';
+export * from './write-checkpoint-batcher.js';
 
 export * from './config.js';
 export * from './config/compound-config-collector.js';

package/src/util/utils.ts

@@ -223,7 +223,7 @@ export function isCompleteRow<V>(
  * All other operations are replaced with a single CLEAR operation,
  * summing their checksums, and using a 0 as an op_id.
  *
- * This is the function $r(B)$, as described in /docs/bucket-properties.md.
+ * This is the function $r(B)$, as described in /docs/storage/bucket-properties.md.
  *
  * Used for tests.
  */

package/src/util/write-checkpoint-batcher.ts

@@ -0,0 +1,122 @@
+import { ErrorCode, ServiceAssertionError, ServiceError } from '@powersync/lib-services-framework';
+import { RouteAPI } from '../api/RouteAPI.js';
+import { BucketStorageFactory, SyncRulesBucketStorage } from '../storage/storage-index.js';
+
+// Keep up to three source-head/storage batches executing concurrently under load.
+// There is intentionally no explicit batch-size cap here: the HTTP request queue
+// already bounds how many requests can be waiting in this process.
+// Smaller values here have better efficiency, while larger numbers here may give slight improvements in latency.
+// We want to limit the number of database connections in use for write checkpoints, so we keep the numbers low here.
+const MAX_IN_FLIGHT_WRITE_CHECKPOINT_BATCHES = 3;
+
+export interface CreateWriteCheckpointResult {
+  writeCheckpoint: string;
+  replicationHead: string;
+}
+
+interface QueuedWriteCheckpoint {
+  userId: string;
+  resolvers: PromiseWithResolvers<CreateWriteCheckpointResult>;
+}
+
+export class WriteCheckpointBatcher {
+  private pending: QueuedWriteCheckpoint[] = [];
+  private inFlight = 0;
+  private scheduledPump: NodeJS.Timeout | undefined;
+
+  constructor(
+    private readonly getAPI: () => RouteAPI,
+    private readonly getStorage: () => BucketStorageFactory
+  ) {}
+
+  enqueue(userId: string): Promise<CreateWriteCheckpointResult> {
+    const resolvers = Promise.withResolvers<CreateWriteCheckpointResult>();
+    this.pending.push({ userId, resolvers });
+    this.schedulePump();
+    return resolvers.promise;
+  }
+
+  private schedulePump() {
+    if (this.scheduledPump != null) {
+      return;
+    }
+
+    // Delay dispatch by one timer turn so requests arriving together can share
+    // one source head and one storage write batch.
+    // While normally we wouldn't have http requests arriving at the exact same time, this is relevant when
+    // requests are queued by fastify: All current requests are typically batched together, and finish at the
+    // same time. Then we get many requests from the queue arriving at the same time. Without this delayed dispatch,
+    // the first request would form its own batch, and the rest will then wait for a new batch to become available.
+    // The delayed dispatch allow all those requests to be batched together.
+    this.scheduledPump = setTimeout(() => {
+      this.scheduledPump = undefined;
+      this.pump();
+    }, 0);
+  }
+
+  private pump() {
+    while (this.inFlight < MAX_IN_FLIGHT_WRITE_CHECKPOINT_BATCHES && this.pending.length > 0) {
+      const batch = this.pending;
+      this.pending = [];
+      this.inFlight++;
+
+      void this.executeBatch(batch).finally(() => {
+        this.inFlight--;
+        this.schedulePump();
+      });
+    }
+  }
+
+  private async executeBatch(batch: QueuedWriteCheckpoint[]) {
+    try {
+      const syncBucketStorage = (await this.getStorage().getActiveSyncConfig())?.storage;
+      if (!syncBucketStorage) {
+        throw new ServiceError(ErrorCode.PSYNC_S2302, `Cannot create Write Checkpoint since no sync config is active.`);
+      }
+
+      const { writeCheckpoints, currentCheckpoint } = await this.getAPI().createReplicationHead(
+        async (currentCheckpoint) => {
+          const writeCheckpoints = await this.createBatchWriteCheckpoints(syncBucketStorage, batch, currentCheckpoint);
+          return { writeCheckpoints, currentCheckpoint };
+        }
+      );
+
+      const results = batch.map((request) => {
+        const writeCheckpoint = writeCheckpoints.get(request.userId);
+        if (writeCheckpoint == null) {
+          throw new ServiceAssertionError(
+            `Write checkpoint storage did not return a checkpoint for user ${request.userId}`
+          );
+        }
+
+        return { request, writeCheckpoint };
+      });
+
+      for (const { request, writeCheckpoint } of results) {
+        request.resolvers.resolve({
+          writeCheckpoint: String(writeCheckpoint),
+          replicationHead: currentCheckpoint
+        });
+      }
+    } catch (error) {
+      // Do not retry here. The route should pass failures through so clients can
+      // apply their normal write-checkpoint retry policy.
+      for (const request of batch) {
+        request.resolvers.reject(error);
+      }
+    }
+  }
+
+  private async createBatchWriteCheckpoints(
+    syncBucketStorage: SyncRulesBucketStorage,
+    batch: QueuedWriteCheckpoint[],
+    currentCheckpoint: string
+  ) {
+    return syncBucketStorage.createManagedWriteCheckpoints(
+      batch.map((request) => ({
+        user_id: request.userId,
+        heads: { '1': currentCheckpoint }
+      }))
+    );
+  }
+}