@powersync/service-core 1.23.1 → 1.23.3

package/dist/util/utils.d.ts

@@ -62,7 +62,7 @@ export declare function isCompleteRow<V>(storeData: boolean, row: sync_rules.Toa
  * All other operations are replaced with a single CLEAR operation,
  * summing their checksums, and using a 0 as an op_id.
  *
- * This is the function $r(B)$, as described in /docs/bucket-properties.md.
+ * This is the function $r(B)$, as described in /docs/storage/bucket-properties.md.
  *
  * Used for tests.
  */

package/dist/util/utils.js

@@ -165,7 +165,7 @@ export function isCompleteRow(storeData, row) {
  * All other operations are replaced with a single CLEAR operation,
  * summing their checksums, and using a 0 as an op_id.
  *
- * This is the function $r(B)$, as described in /docs/bucket-properties.md.
+ * This is the function $r(B)$, as described in /docs/storage/bucket-properties.md.
  *
  * Used for tests.
  */

package/dist/util/write-checkpoint-batcher.d.ts

@@ -0,0 +1,19 @@
+import { RouteAPI } from '../api/RouteAPI.js';
+import { BucketStorageFactory } from '../storage/storage-index.js';
+export interface CreateWriteCheckpointResult {
+    writeCheckpoint: string;
+    replicationHead: string;
+}
+export declare class WriteCheckpointBatcher {
+    private readonly getAPI;
+    private readonly getStorage;
+    private pending;
+    private inFlight;
+    private scheduledPump;
+    constructor(getAPI: () => RouteAPI, getStorage: () => BucketStorageFactory);
+    enqueue(userId: string): Promise<CreateWriteCheckpointResult>;
+    private schedulePump;
+    private pump;
+    private executeBatch;
+    private createBatchWriteCheckpoints;
+}

package/dist/util/write-checkpoint-batcher.js

@@ -0,0 +1,90 @@
+import { ErrorCode, ServiceAssertionError, ServiceError } from '@powersync/lib-services-framework';
+// Keep up to three source-head/storage batches executing concurrently under load.
+// There is intentionally no explicit batch-size cap here: the HTTP request queue
+// already bounds how many requests can be waiting in this process.
+// Smaller values here have better efficiency, while larger numbers here may give slight improvements in latency.
+// We want to limit the number of database connections in use for write checkpoints, so we keep the numbers low here.
+const MAX_IN_FLIGHT_WRITE_CHECKPOINT_BATCHES = 3;
+export class WriteCheckpointBatcher {
+    getAPI;
+    getStorage;
+    pending = [];
+    inFlight = 0;
+    scheduledPump;
+    constructor(getAPI, getStorage) {
+        this.getAPI = getAPI;
+        this.getStorage = getStorage;
+    }
+    enqueue(userId) {
+        const resolvers = Promise.withResolvers();
+        this.pending.push({ userId, resolvers });
+        this.schedulePump();
+        return resolvers.promise;
+    }
+    schedulePump() {
+        if (this.scheduledPump != null) {
+            return;
+        }
+        // Delay dispatch by one timer turn so requests arriving together can share
+        // one source head and one storage write batch.
+        // While normally we wouldn't have http requests arriving at the exact same time, this is relevant when
+        // requests are queued by fastify: All current requests are typically batched together, and finish at the
+        // same time. Then we get many requests from the queue arriving at the same time. Without this delayed dispatch,
+        // the first request would form its own batch, and the rest will then wait for a new batch to become available.
+        // The delayed dispatch allow all those requests to be batched together.
+        this.scheduledPump = setTimeout(() => {
+            this.scheduledPump = undefined;
+            this.pump();
+        }, 0);
+    }
+    pump() {
+        while (this.inFlight < MAX_IN_FLIGHT_WRITE_CHECKPOINT_BATCHES && this.pending.length > 0) {
+            const batch = this.pending;
+            this.pending = [];
+            this.inFlight++;
+            void this.executeBatch(batch).finally(() => {
+                this.inFlight--;
+                this.schedulePump();
+            });
+        }
+    }
+    async executeBatch(batch) {
+        try {
+            const syncBucketStorage = (await this.getStorage().getActiveSyncConfig())?.storage;
+            if (!syncBucketStorage) {
+                throw new ServiceError(ErrorCode.PSYNC_S2302, `Cannot create Write Checkpoint since no sync config is active.`);
+            }
+            const { writeCheckpoints, currentCheckpoint } = await this.getAPI().createReplicationHead(async (currentCheckpoint) => {
+                const writeCheckpoints = await this.createBatchWriteCheckpoints(syncBucketStorage, batch, currentCheckpoint);
+                return { writeCheckpoints, currentCheckpoint };
+            });
+            const results = batch.map((request) => {
+                const writeCheckpoint = writeCheckpoints.get(request.userId);
+                if (writeCheckpoint == null) {
+                    throw new ServiceAssertionError(`Write checkpoint storage did not return a checkpoint for user ${request.userId}`);
+                }
+                return { request, writeCheckpoint };
+            });
+            for (const { request, writeCheckpoint } of results) {
+                request.resolvers.resolve({
+                    writeCheckpoint: String(writeCheckpoint),
+                    replicationHead: currentCheckpoint
+                });
+            }
+        }
+        catch (error) {
+            // Do not retry here. The route should pass failures through so clients can
+            // apply their normal write-checkpoint retry policy.
+            for (const request of batch) {
+                request.resolvers.reject(error);
+            }
+        }
+    }
+    async createBatchWriteCheckpoints(syncBucketStorage, batch, currentCheckpoint) {
+        return syncBucketStorage.createManagedWriteCheckpoints(batch.map((request) => ({
+            user_id: request.userId,
+            heads: { '1': currentCheckpoint }
+        })));
+    }
+}
+//# sourceMappingURL=write-checkpoint-batcher.js.map

package/dist/util/write-checkpoint-batcher.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"write-checkpoint-batcher.js","sourceRoot":"","sources":["../../src/util/write-checkpoint-batcher.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,qBAAqB,EAAE,YAAY,EAAE,MAAM,mCAAmC,CAAC;AAInG,kFAAkF;AAClF,iFAAiF;AACjF,mEAAmE;AACnE,iHAAiH;AACjH,qHAAqH;AACrH,MAAM,sCAAsC,GAAG,CAAC,CAAC;AAYjD,MAAM,OAAO,sBAAsB;IAMd;IACA;IANX,OAAO,GAA4B,EAAE,CAAC;IACtC,QAAQ,GAAG,CAAC,CAAC;IACb,aAAa,CAA6B;IAElD,YACmB,MAAsB,EACtB,UAAsC;QADtC,WAAM,GAAN,MAAM,CAAgB;QACtB,eAAU,GAAV,UAAU,CAA4B;IACtD,CAAC;IAEJ,OAAO,CAAC,MAAc;QACpB,MAAM,SAAS,GAAG,OAAO,CAAC,aAAa,EAA+B,CAAC;QACvE,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC,CAAC;QACzC,IAAI,CAAC,YAAY,EAAE,CAAC;QACpB,OAAO,SAAS,CAAC,OAAO,CAAC;IAC3B,CAAC;IAEO,YAAY;QAClB,IAAI,IAAI,CAAC,aAAa,IAAI,IAAI,EAAE,CAAC;YAC/B,OAAO;QACT,CAAC;QAED,2EAA2E;QAC3E,+CAA+C;QAC/C,uGAAuG;QACvG,yGAAyG;QACzG,gHAAgH;QAChH,+GAA+G;QAC/G,wEAAwE;QACxE,IAAI,CAAC,aAAa,GAAG,UAAU,CAAC,GAAG,EAAE;YACnC,IAAI,CAAC,aAAa,GAAG,SAAS,CAAC;YAC/B,IAAI,CAAC,IAAI,EAAE,CAAC;QACd,CAAC,EAAE,CAAC,CAAC,CAAC;IACR,CAAC;IAEO,IAAI;QACV,OAAO,IAAI,CAAC,QAAQ,GAAG,sCAAsC,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACzF,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC;YAC3B,IAAI,CAAC,OAAO,GAAG,EAAE,CAAC;YAClB,IAAI,CAAC,QAAQ,EAAE,CAAC;YAEhB,KAAK,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,GAAG,EAAE;gBACzC,IAAI,CAAC,QAAQ,EAAE,CAAC;gBAChB,IAAI,CAAC,YAAY,EAAE,CAAC;YACtB,CAAC,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,YAAY,CAAC,KAA8B;QACvD,IAAI,CAAC;YACH,MAAM,iBAAiB,GAAG,CAAC,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC,mBAAmB,EAAE,CAAC,EAAE,OAAO,CAAC;YACnF,IAAI,CAAC,iBAAiB,EAAE,CAAC;gBACvB,MAAM,IAAI,YAAY,CAAC,SAAS,CAAC,WAAW,EAAE,gEAAgE,CAAC,CAAC;YAClH,CAAC;YAED,MAAM,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,GAAG,MAAM,IAAI,CAAC,MAAM,EAAE,CAAC,qBAAqB,CACvF,KAAK,EAAE,iBAAiB,EAAE,EAAE;gBAC1B,MAAM,gBAAgB,GAAG,MAAM,IAAI,CAAC,2BAA2B,CAAC,iBAAiB,EAAE,KAAK,EAAE,iBAAiB,CAAC,CAAC;gBAC7G,OAAO,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,CAAC;YACjD,CAAC,CACF,CAAC;YAEF,MAAM,OAAO,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE;gBACpC,MAAM,eAAe,GAAG,gBAAgB,CAAC,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;gBAC7D,IAAI,eAAe,IAAI,IAAI,EAAE,CAAC;oBAC5B,MAAM,IAAI,qBAAqB,CAC7B,iEAAiE,OAAO,CAAC,MAAM,EAAE,CAClF,CAAC;gBACJ,CAAC;gBAED,OAAO,EAAE,OAAO,EAAE,eAAe,EAAE,CAAC;YACtC,CAAC,CAAC,CAAC;YAEH,KAAK,MAAM,EAAE,OAAO,EAAE,eAAe,EAAE,IAAI,OAAO,EAAE,CAAC;gBACnD,OAAO,CAAC,SAAS,CAAC,OAAO,CAAC;oBACxB,eAAe,EAAE,MAAM,CAAC,eAAe,CAAC;oBACxC,eAAe,EAAE,iBAAiB;iBACnC,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,2EAA2E;YAC3E,oDAAoD;YACpD,KAAK,MAAM,OAAO,IAAI,KAAK,EAAE,CAAC;gBAC5B,OAAO,CAAC,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YAClC,CAAC;QACH,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,2BAA2B,CACvC,iBAAyC,EACzC,KAA8B,EAC9B,iBAAyB;QAEzB,OAAO,iBAAiB,CAAC,6BAA6B,CACpD,KAAK,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;YACtB,OAAO,EAAE,OAAO,CAAC,MAAM;YACvB,KAAK,EAAE,EAAE,GAAG,EAAE,iBAAiB,EAAE;SAClC,CAAC,CAAC,CACJ,CAAC;IACJ,CAAC;CACF"}

package/package.json

@@ -4,7 +4,7 @@
   "publishConfig": {
     "access": "public"
   },
-  "version": "1.23.1",
+  "version": "1.23.3",
   "license": "FSL-1.1-ALv2",
   "type": "module",
   "exports": "./dist/index.js",
@@ -32,11 +32,11 @@
     "uuid": "^14.0.0",
     "winston": "^3.13.0",
     "yaml": "^2.8.3",
-    "@powersync/lib-services-framework": "0.9.6",
+    "@powersync/lib-services-framework": "0.9.8",
     "@powersync/service-jsonbig": "0.17.13",
-    "@powersync/service-rsocket-router": "0.2.22",
-    "@powersync/service-sync-rules": "0.38.0",
-    "@powersync/service-types": "0.16.0"
+    "@powersync/service-rsocket-router": "0.2.24",
+    "@powersync/service-sync-rules": "0.39.0",
+    "@powersync/service-types": "0.16.1"
   },
   "devDependencies": {
     "@types/async": "^3.2.24",

package/src/api/RouteAPI.ts

@@ -25,7 +25,8 @@ export interface SlotWalBudgetOptions {
 }
 
 /**
- *  Describes all the methods currently required to service the sync API endpoints.
+ * Source-specific capabilities used by admin routes, sync config validation,
+ * checkpointing, diagnostics, and the sync API.
  */
 export interface RouteAPI {
   /**
@@ -40,16 +41,17 @@ export interface RouteAPI {
   getConnectionStatus(): Promise<types.ConnectionStatusV2>;
 
   /**
-   * Generates replication table information from a given pattern of tables.
+   * Expand sync config table patterns into table metadata from the source connection.
+   *
+   * This is used by validation and diagnostics paths to explain what will be
+   * replicated for the current sync config.
    *
    * @param tablePatterns A set of table patterns which typically come from
    *          the tables listed in sync config definitions.
    *
    * @param sqlSyncRules
-   * @returns A result of all the tables and columns which should be replicated
-   *           based off the input patterns. Certain tests are executed on the
-   *           tables to ensure syncing should function according to the input
-   *           pattern. Debug errors and warnings are reported per table.
+   * @returns Matching tables or collections, their columns or fields where available,
+   *          and per-table warnings or errors for the requested table patterns.
    */
   getDebugTablesInfo(tablePatterns: TablePattern[], sqlSyncRules: SyncConfig): Promise<PatternResult[]>;
 
@@ -69,14 +71,21 @@ export interface RouteAPI {
   /**
    * Get the current LSN or equivalent replication HEAD position identifier.
    *
-   * The position is provided to the callback. After the callback returns,
-   * the replication head or a greater one will be streamed on the replication stream.
+   * The position is provided to the callback so the caller can persist its
+   * write-checkpoint mapping before the source adapter forces any required
+   * source-side marker or keepalive. After the callback returns, the adapter
+   * must ensure that the replication stream will observe this position or a
+   * greater one, even when the source is otherwise idle.
    */
   createReplicationHead<T>(callback: ReplicationHeadCallback<T>): Promise<T>;
 
   /**
-   * @returns The schema for tables inside the connected database. This is typically
-   *          used to validate sync config.
+   * @returns The connected source schema in service-friendly table and column
+   *          formats. Strict-schema sources should include actual tables,
+   *          collections, columns, and types. Schemaless sources should return
+   *          the best metadata available and document validation limitations in
+   *          their adapter behavior. This is typically used to validate sync
+   *          config and generate client schemas.
    */
   getConnectionSchema(): Promise<types.DatabaseSchema[]>;
 
@@ -92,7 +101,12 @@ export interface RouteAPI {
   shutdown(): Promise<void>;
 
   /**
-   * Get the default schema (or database) when only a table name is specified in sync config.
+   * Get source-specific sync config parsing defaults, such as the default
+   * schema or database to use when sync config references an unqualified table
+   * name.
+   *
+   * This is used wherever sync config is parsed for this source, including
+   * validation, diagnostics, and sync routing.
    */
   getParseSyncRulesOptions(): ParseSyncConfigOptions;
 }

package/src/replication/AbstractReplicator.ts

@@ -72,6 +72,11 @@ export abstract class AbstractReplicator<T extends AbstractReplicationJob = Abst
     this.logger = logger.child({ name: `Replicator:${options.id}` });
   }
 
+  /**
+   * Create the source-specific replication job for a locked storage stream.
+   *
+   * The job owns the per-stream snapshot and streaming work for this attempt.
+   */
   abstract createJob(options: CreateJobOptions): T;
 
   /**
@@ -374,6 +379,9 @@ export abstract class AbstractReplicator<T extends AbstractReplicationJob = Abst
     syncRuleStorage.logger.info(`Successfully terminated replication stream`);
   }
 
+  /**
+   * Check source connectivity using the replicator's decoded connection config.
+   */
   abstract testConnection(): Promise<ConnectionTestResult>;
 
   /**

package/src/routes/RouterEngine.ts

@@ -11,6 +11,7 @@ export type RouterSetupResponse = {
 
 export type RouterEngineRoutes = {
   api_routes: RouteDefinition[];
+  checkpointing_routes: RouteDefinition[];
   stream_routes: RouteDefinition[];
   socket_routes: SocketRouteGenerator[];
 };
@@ -42,6 +43,7 @@ export class RouterEngine {
 
     this.routes = {
       api_routes: [],
+      checkpointing_routes: [],
       stream_routes: [],
       socket_routes: []
     };
@@ -49,13 +51,17 @@ export class RouterEngine {
 
   public registerRoutes(routes: Partial<RouterEngineRoutes>) {
     this.routes.api_routes.push(...(routes.api_routes ?? []));
+    this.routes.checkpointing_routes.push(...(routes.checkpointing_routes ?? []));
     this.routes.stream_routes.push(...(routes.stream_routes ?? []));
     this.routes.socket_routes.push(...(routes.socket_routes ?? []));
   }
 
   public get hasRoutes() {
     return (
-      this.routes.api_routes.length > 0 || this.routes.stream_routes.length > 0 || this.routes.socket_routes.length > 0
+      this.routes.api_routes.length > 0 ||
+      this.routes.checkpointing_routes.length > 0 ||
+      this.routes.stream_routes.length > 0 ||
+      this.routes.socket_routes.length > 0
     );
   }
 

package/src/routes/configure-fastify.ts

@@ -31,6 +31,7 @@ export type RouteRegistrationOptions = {
  */
 export type RouteDefinitions = {
   api?: Partial<RouteRegistrationOptions>;
+  checkpointing?: Partial<RouteRegistrationOptions>;
   sync_stream?: Partial<RouteRegistrationOptions>;
 };
 
@@ -41,12 +42,20 @@ export type FastifyServerConfig = {
 
 export const DEFAULT_ROUTE_OPTIONS = {
   api: {
-    routes: [...ADMIN_ROUTES, ...CHECKPOINT_ROUTES, ...SYNC_RULES_ROUTES, ...PROBES_ROUTES],
+    routes: [...ADMIN_ROUTES, ...SYNC_RULES_ROUTES, ...PROBES_ROUTES],
     queue_options: {
       concurrency: 10,
       max_queue_depth: 20
     }
   },
+  checkpointing: {
+    routes: [...CHECKPOINT_ROUTES],
+    queue_options: {
+      // Note that the values here has an effect on WriteCheckpointBatcher
+      concurrency: 100,
+      max_queue_depth: 100
+    }
+  },
   sync_stream: {
     routes: [...SYNC_STREAM_ROUTES],
     queue_options: {
@@ -72,6 +81,7 @@ export function configureFastifyServer(server: fastify.FastifyInstance, options:
 
   // Set on the outer server so both child scopes inherit.
   registerFastifyErrorHandler(server);
+  registerFastifyNotFoundHandler(server);
 
   /**
    * Fastify creates an encapsulated context for each `.register` call.
@@ -81,7 +91,6 @@ export function configureFastifyServer(server: fastify.FastifyInstance, options:
    */
   server.register(async function (childContext) {
     registerFastifyRoutes(childContext, generateContext, routes.api?.routes ?? DEFAULT_ROUTE_OPTIONS.api.routes);
-    registerFastifyNotFoundHandler(childContext);
 
     // Limit the active concurrent requests
     childContext.addHook(
@@ -90,6 +99,20 @@ export function configureFastifyServer(server: fastify.FastifyInstance, options:
     );
   });
 
+  server.register(async function (childContext) {
+    registerFastifyRoutes(
+      childContext,
+      generateContext,
+      routes.checkpointing?.routes ?? DEFAULT_ROUTE_OPTIONS.checkpointing.routes
+    );
+
+    // Limit the active concurrent requests
+    childContext.addHook(
+      'onRequest',
+      createRequestQueueHook(routes.checkpointing?.queue_options ?? DEFAULT_ROUTE_OPTIONS.checkpointing.queue_options)
+    );
+  });
+
   // Create a separate context for concurrency queueing
   server.register(async function (childContext) {
     registerFastifyRoutes(

package/src/routes/endpoints/checkpointing.ts

@@ -54,13 +54,10 @@ export const writeCheckpoint2 = routeDefinition({
   handler: async (payload) => {
     const { token_payload, service_context } = payload.context;
 
-    const apiHandler = service_context.routerEngine.getAPI();
-
     const { replicationHead, writeCheckpoint } = await util.createWriteCheckpoint({
       userId: token_payload!.userIdString,
       clientId: payload.params.client_id,
-      api: apiHandler,
-      storage: service_context.storageEngine.activeBucketStorage
+      batcher: service_context.writeCheckpointBatcher
     });
 
     logger.info(

package/src/routes/endpoints/socket-route.ts

@@ -163,8 +163,9 @@ export const syncStreamReactive: SocketRouteGenerator = (router) =>
       } catch (ex) {
         // Convert to our standard form before responding.
         // This ensures the error can be serialized.
+        // However, use the original error for the logs, so that we have the stack trace.
         const error = new errors.InternalServerError(ex);
-        logger.error('Sync stream error', error);
+        logger.error('Sync stream error', ex);
         closeReason ??= 'stream error';
         responder.onError(error);
       } finally {

package/src/storage/BucketStorageBatch.ts

@@ -12,6 +12,15 @@ export const DEFAULT_BUCKET_BATCH_COMMIT_OPTIONS: ResolvedBucketBatchCommitOptio
   oldestUncommittedChange: null
 };
 
+/**
+ * Storage writer for a source-specific unit of replication work.
+ *
+ * A batch evaluates source row changes against the hydrated sync config,
+ * persists bucket operations and parameter lookup rows, tracks source table and
+ * snapshot state, records the committed source position for visible
+ * checkpoints, and stores restart cursors for sources with PowerSync-managed
+ * resume state.
+ */
 export interface BucketStorageBatch extends ObserverClient<BucketBatchStorageListener>, AsyncDisposable {
   /**
    * Alias for [Symbol.asyncDispose]
@@ -29,21 +38,28 @@ export interface BucketStorageBatch extends ObserverClient<BucketBatchStorageLis
   readonly skipExistingRows: boolean;
 
   /**
-   * Save an op, and potentially flush.
+   * Queue one normalized source row operation and potentially flush.
    *
-   * This can be an insert, update or delete op.
+   * When flushed, the operation is evaluated against data and parameter
+   * queries. That may produce bucket operations, parameter lookup rows, and
+   * current-data updates. `save()` may trigger that flush automatically when
+   * the batch reaches its size limits, but a flush does not create a checkpoint
+   * by itself.
    */
   save(record: SaveOptions): Promise<FlushedResult | null>;
 
   /**
-   * Replicate a truncate op - deletes all data in the specified tables.
+   * Replicate a truncate operation by removing all currently replicated rows
+   * for the specified source tables.
    */
   truncate(sourceTables: SourceTable[]): Promise<FlushedResult | null>;
 
   /**
-   * Drop one or more tables.
+   * Drop one or more source table mappings.
    *
-   * This is the same as truncate, but additionally removes the SourceTable record.
+   * This performs the same logical row removal as `truncate()`, then removes
+   * the stored `SourceTable` record. It is used for table drops, renames,
+   * replica identity changes, and superseded mappings.
    */
   drop(sourceTables: SourceTable[]): Promise<FlushedResult | null>;
 
@@ -58,21 +74,26 @@ export interface BucketStorageBatch extends ObserverClient<BucketBatchStorageLis
   flush(options?: BatchBucketFlushOptions): Promise<FlushedResult | null>;
 
   /**
-   * Flush and commit any saved ops. This creates a new checkpoint by default.
+   * Flush saved ops and advance the committed source checkpoint position.
    *
-   * Only call this after a transaction.
+   * Only call this at a source transaction, page, snapshot, or other boundary
+   * where it is valid for clients to observe all flushed changes together. A
+   * single commit may include multiple complete source transactions, but must
+   * not split one source transaction across checkpoint boundaries.
    */
   commit(lsn: string, options?: BucketBatchCommitOptions): Promise<CheckpointResult>;
 
   /**
-   * Advance the checkpoint LSN position, without any associated op.
+   * Advance the committed source checkpoint position without associated row
+   * operations.
    *
-   * This must only be called when not inside a transaction.
+   * Use this for source heartbeats, marker-only events, and idle-source write
+   * checkpoint progress. This must only be called outside a source transaction.
    */
   keepalive(lsn: string): Promise<CheckpointResult>;
 
   /**
-   * Set the LSN that replication should resume from.
+   * Set the source position that replication should resume from after restart.
    *
    * This can be used for:
    * 1. Setting the LSN for a snapshot, before starting replication.
@@ -80,6 +101,10 @@ export interface BucketStorageBatch extends ObserverClient<BucketBatchStorageLis
    *
    * Not required if the source database keeps track of this, for example with
    * PostgreSQL logical replication slots.
+   *
+   * This is for restart safety only. If the same source position should become
+   * visible to clients, also call `commit()` or `keepalive()` at a valid source
+   * boundary.
    */
   setResumeLsn(lsn: string): Promise<void>;
 
@@ -90,7 +115,21 @@ export interface BucketStorageBatch extends ObserverClient<BucketBatchStorageLis
    */
   resumeFromLsn: string | null;
 
+  /**
+   * Mark table snapshots complete and record the source position that
+   * checkpoints must not precede for those tables.
+   */
   markTableSnapshotDone(tables: SourceTable[], no_checkpoint_before_lsn?: string): Promise<SourceTable[]>;
+
+  /**
+   * Mark a source table as requiring snapshot work again.
+   *
+   * Use this when a table that was previously considered snapshot-complete
+   * must be copied again before it can safely contribute to checkpoints. Common
+   * cases include detected schema or replica identity changes, interrupted
+   * re-snapshot work, or other source metadata changes that make the existing
+   * replicated rows no longer trustworthy.
+   */
   markTableSnapshotRequired(table: SourceTable): Promise<void>;
 
   /**
@@ -113,6 +152,10 @@ export interface BucketStorageBatch extends ObserverClient<BucketBatchStorageLis
    */
   markSnapshotDone(no_checkpoint_before_lsn: string, options?: { throwOnConflict?: boolean }): Promise<void>;
 
+  /**
+   * Persist resumable table snapshot progress such as row counts or source
+   * cursors.
+   */
   updateTableProgress(table: SourceTable, progress: Partial<TableSnapshotStatus>): Promise<SourceTable>;
 
   /**
@@ -120,10 +163,17 @@ export interface BucketStorageBatch extends ObserverClient<BucketBatchStorageLis
    */
   getSourceTableStatus(table: SourceTable): Promise<SourceTable | null>;
 
+  /**
+   * Resolve discovered source metadata against the active sync config and
+   * persisted source table state.
+   *
+   * @returns Current mappings that should receive replicated rows, plus
+   *          outdated mappings that the source connector should drop.
+   */
   resolveTables(options: ResolveTablesOptions): Promise<ResolveTablesResult>;
 
   /**
-   * Queues the creation of a custom Write Checkpoint. This will be persisted after operations are flushed.
+   * Queue a custom write checkpoint to be persisted after operations are flushed.
    */
   addCustomWriteCheckpoint(checkpoint: BatchedCustomWriteCheckpointOptions): void;
 }

package/src/storage/ChecksumCache.ts

@@ -5,7 +5,7 @@ import { BucketChecksum } from '../util/protocol-types.js';
 import { addBucketChecksums, ChecksumMap, InternalOpId, PartialChecksum } from '../util/utils.js';
 import { BucketChecksumRequest } from './SyncRulesBucketStorage.js';
 
-interface ChecksumFetchContext {
+interface ChecksumFetchContext<TContext> {
   fetch(bucket: string): Promise<BucketChecksum>;
   checkpoint: InternalOpId;
 }
@@ -20,26 +20,35 @@ export interface FetchPartialBucketChecksum {
 export type PartialOrFullChecksum = PartialChecksum | BucketChecksum;
 export type PartialChecksumMap = Map<string, PartialOrFullChecksum>;
 
-export type FetchChecksums = (batch: FetchPartialBucketChecksum[]) => Promise<PartialChecksumMap>;
+export type FetchChecksums<TContext = undefined> = (
+  batch: FetchPartialBucketChecksum[],
+  context: TContext
+) => Promise<PartialChecksumMap>;
 
-export interface ChecksumCacheOptions {
+export interface ChecksumCacheOptions<TContext = undefined> {
   /**
    * Upstream checksum implementation.
    *
    * This fetches a batch of either entire bucket checksums, or a partial range.
    */
-  fetchChecksums: FetchChecksums;
+  fetchChecksums: FetchChecksums<TContext>;
 
   /**
    * Maximum number of cached checksums.
    */
   maxSize?: number;
+
+  /**
+   * How long to keep cached checksums before refreshing them.
+   */
+  ttlMs?: number;
 }
 
 // Approximately 5MB of memory, if we assume 50 bytes per entry
 const DEFAULT_MAX_SIZE = 100_000;
 
-const TTL_MS = 3_600_000;
+// Default to 1h
+export const DEFAULT_CHECKSUM_CACHE_TTL_MS = 1 * 3_600_000;
 
 /**
  * Implement a LRU cache for checksum requests. Each (bucket, checkpoint) request is cached separately,
@@ -50,22 +59,22 @@ const TTL_MS = 3_600_000;
  *
  * We use the LRUCache fetchMethod to deduplicate in-progress requests.
  */
-export class ChecksumCache {
+export class ChecksumCache<TContext = undefined> {
   /**
    * The primary checksum cache, with key of `${checkpoint}/${bucket}`.
    */
-  private cache: LRUCache<string, BucketChecksum, ChecksumFetchContext>;
+  private cache: LRUCache<string, BucketChecksum, ChecksumFetchContext<TContext>>;
   /**
    * For each bucket, an ordered set of cached checkpoints.
    */
   private bucketCheckpoints = new Map<string, OrderedSet<bigint>>();
 
-  private fetchChecksums: FetchChecksums;
+  private fetchChecksums: FetchChecksums<TContext>;
 
-  constructor(options: ChecksumCacheOptions) {
+  constructor(options: ChecksumCacheOptions<TContext>) {
     this.fetchChecksums = options.fetchChecksums;
 
-    this.cache = new LRUCache<string, BucketChecksum, ChecksumFetchContext>({
+    this.cache = new LRUCache<string, BucketChecksum, ChecksumFetchContext<TContext>>({
       max: options.maxSize ?? DEFAULT_MAX_SIZE,
       fetchMethod: async (cacheKey, _staleValue, options) => {
         // Called when this checksum hasn't been cached yet.
@@ -105,7 +114,7 @@ export class ChecksumCache {
       // We use a TTL so that counts can eventually be refreshed
       // after a compact. This only has effect if the bucket has
       // not been checked in the meantime.
-      ttl: TTL_MS,
+      ttl: options.ttlMs ?? DEFAULT_CHECKSUM_CACHE_TTL_MS,
       ttlResolution: 1_000,
       allowStale: false
     });
@@ -129,7 +138,11 @@ export class ChecksumCache {
    *
    * @returns a Map with exactly one entry for each bucket requested
    */
-  async getChecksumMap(checkpoint: InternalOpId, buckets: BucketChecksumRequest[]): Promise<ChecksumMap> {
+  async getChecksumMap(
+    checkpoint: InternalOpId,
+    buckets: BucketChecksumRequest[],
+    fetchContext?: TContext
+  ): Promise<ChecksumMap> {
     // Buckets that don't have a cached checksum for this checkpoint yet
     let toFetch = new Set<string>();
 
@@ -147,7 +160,7 @@ export class ChecksumCache {
     // Accumulated results - both from cached checksums, and fetched checksums
     let finalResults = new Map<string, BucketChecksum>();
 
-    const context: ChecksumFetchContext = {
+    const context: ChecksumFetchContext<TContext> = {
       async fetch(bucket) {
         await fetchPromise;
         if (!toFetch.has(bucket)) {
@@ -260,7 +273,7 @@ export class ChecksumCache {
         }
 
         // Fetch partial checksums from upstream
-        const results = await this.fetchChecksums(bucketRequests);
+        const results = await this.fetchChecksums(bucketRequests, fetchContext as TContext);
 
         for (let bucket of toFetch) {
           const result = results.get(bucket);