@powersync/service-core 1.23.2 → 1.24.0

package/src/routes/endpoints/checkpointing.ts

@@ -1,10 +1,49 @@
-import { logger, router, schema } from '@powersync/lib-services-framework';
+import { codecs, logger, router, schema } from '@powersync/lib-services-framework';
 import * as t from 'ts-codec';
 
 import * as util from '../../util/util-index.js';
 import { authUser } from '../auth.js';
 import { routeDefinition } from '../router.js';
 
+const CHECKPOINT_REQUEST_ID_MAX = 9_223_372_036_854_775_807n;
+
+function isValidCheckpointRequestId(value: bigint) {
+  return value > 0n && value <= CHECKPOINT_REQUEST_ID_MAX;
+}
+
+export const CheckpointRequestPayload = t.object({
+  client_id: t.string,
+  // Positive int64, matching the managed write checkpoint id. Clients should
+  // send values larger than Number.MAX_SAFE_INTEGER as strings, since JSON
+  // numbers are only validated up to the safe-integer range.
+  checkpoint_request_id: codecs.bigint
+});
+
+const CheckpointRequestPayloadShapeValidator = schema.createTsCodecValidator(CheckpointRequestPayload, {
+  allowAdditional: true
+});
+
+const CheckpointRequestPayloadValidator = {
+  validate(params: t.Encoded<typeof CheckpointRequestPayload>) {
+    const shapeValidation = CheckpointRequestPayloadShapeValidator.validate(params);
+    if (!shapeValidation.valid) {
+      return shapeValidation;
+    }
+
+    const decodedParams = CheckpointRequestPayload.decode(params);
+    if (!isValidCheckpointRequestId(decodedParams.checkpoint_request_id)) {
+      return {
+        valid: false as const,
+        errors: [`Expected checkpoint_request_id between 1 and ${CHECKPOINT_REQUEST_ID_MAX}`]
+      };
+    }
+
+    return {
+      valid: true as const
+    };
+  }
+};
+
 const WriteCheckpointRequest = t.object({
   client_id: t.string.optional()
 });
@@ -24,7 +63,7 @@ export const writeCheckpoint = routeDefinition({
     // Since we don't use LSNs anymore, the only way to get that is to wait.
     const start = Date.now();
 
-    const head = await apiHandler.createReplicationHead(async (head) => head);
+    const head = await apiHandler.createReplicationHead(async (head) => ({ response: head, shouldAdvance: true }));
 
     const timeout = 50_000;
 
@@ -70,4 +109,39 @@ export const writeCheckpoint2 = routeDefinition({
   }
 });
 
-export const CHECKPOINT_ROUTES = [writeCheckpoint, writeCheckpoint2];
+export const checkpointRequest = routeDefinition({
+  path: '/sync/checkpoint-request',
+  method: router.HTTPMethod.POST,
+  authorize: authUser,
+  validator: CheckpointRequestPayloadValidator,
+  handler: async (request) => {
+    const { token_payload, service_context } = request.context;
+    const { params } = request;
+
+    const decodedParams = CheckpointRequestPayload.decode(params);
+
+    // Storage only applies supplied request ids that advance the stored managed checkpoint.
+    // Stale or duplicate ids return the stored checkpoint; storage decides whether
+    // the matched checkpoint still needs a source marker.
+    const { replicationHead, writeCheckpoint } = await util.createWriteCheckpoint({
+      userId: token_payload!.userIdString,
+      clientId: decodedParams.client_id,
+      batcher: service_context.writeCheckpointBatcher,
+      checkpointRequestId: decodedParams.checkpoint_request_id
+    });
+
+    logger.info(
+      `Requested checkpoint for ${token_payload!.userIdString}/${decodedParams.client_id}: ${writeCheckpoint} | ${replicationHead}`
+    );
+
+    // Return the checkpoint request id storage is actually at after this request.
+    // When an earlier request has already advanced the stored value beyond the
+    // supplied checkpoint_request_id, this returns that larger previous value so
+    // the client can treat the response as a stale-request acknowledgement.
+    return {
+      checkpoint_request_id: String(writeCheckpoint)
+    };
+  }
+});
+
+export const CHECKPOINT_ROUTES = [checkpointRequest, writeCheckpoint, writeCheckpoint2];

package/src/routes/endpoints/socket-route.ts

@@ -163,8 +163,9 @@ export const syncStreamReactive: SocketRouteGenerator = (router) =>
       } catch (ex) {
         // Convert to our standard form before responding.
         // This ensures the error can be serialized.
+        // However, use the original error for the logs, so that we have the stack trace.
         const error = new errors.InternalServerError(ex);
-        logger.error('Sync stream error', error);
+        logger.error('Sync stream error', ex);
         closeReason ??= 'stream error';
         responder.onError(error);
       } finally {

package/src/storage/BucketStorageBatch.ts

@@ -173,7 +173,10 @@ export interface BucketStorageBatch extends ObserverClient<BucketBatchStorageLis
   resolveTables(options: ResolveTablesOptions): Promise<ResolveTablesResult>;
 
   /**
-   * Queue a custom write checkpoint to be persisted after operations are flushed.
+   * Queue a custom checkpoint request to be persisted after operations are
+   * flushed. Set `checkpoint_requested_at` when the custom checkpoint came from
+   * a client checkpoint request and should be cleaned up by request-retention
+   * compaction; omit it for persistent source-owned checkpoints.
    */
   addCustomWriteCheckpoint(checkpoint: BatchedCustomWriteCheckpointOptions): void;
 }

package/src/storage/BucketStorageFactory.ts

@@ -1,8 +1,8 @@
 import { BaseObserver, logger } from '@powersync/lib-services-framework';
 import {
   PrecompiledSyncConfig,
+  SerializedSyncPlan as RawSerializedSyncPlan,
   SerializedCompatibilityContext,
-  SerializedSyncPlanV1,
   serializeSyncPlan,
   SqlSyncRules,
   SyncConfigWithErrors
@@ -171,7 +171,7 @@ export interface SerializedSyncPlan {
   /**
    * The serialized plan, from {@link serializeSyncPlan}.
    */
-  plan: SerializedSyncPlanV1;
+  plan: RawSerializedSyncPlan;
   compatibility: SerializedCompatibilityContext;
   /**
    * Event descriptors are not currently represented in the sync plan because they don't use the sync streams compiler

package/src/storage/CheckpointChecksumInvalidatedError.ts

@@ -0,0 +1,17 @@
+import { InternalOpId } from '../util/util-index.js';
+
+/**
+ * A checkpoint cannot be served because compaction rewrote a bucket-data
+ * document across its end boundary.
+ *
+ * The sync loop must skip this checkpoint before it sends its checkpoint line.
+ */
+export class CheckpointChecksumInvalidatedError extends Error {
+  constructor(
+    public readonly checkpoint: InternalOpId,
+    public readonly bucket: string
+  ) {
+    super(`Checkpoint ${checkpoint} was invalidated by compaction in bucket ${bucket}`);
+    this.name = 'CheckpointChecksumInvalidatedError';
+  }
+}

package/src/storage/SyncRulesBucketStorage.ts

@@ -133,6 +133,13 @@ export interface SyncRulesBucketStorage
    * 1. Separate buckets.
    * 2. Limit the size of each individual chunk according to options.batchSizeLimitBytes.
    *
+   * The batch may not contain all data for the checkpoint, if the checkpoint is large. The caller must
+   * continue querying if either:
+   * 1. The last chunk for any bucket has has_more = true.
+   * 2. A SyncBucketDataBatchEnd is returned with hasMore = true.
+   *
+   * The first check can be skipped if a SyncBucketDataBatchEnd is returned with hasMore = false.
+   *
    * @param checkpoint the checkpoint
    * @param dataBuckets current bucket states
    * @param options batch size options
@@ -141,7 +148,7 @@ export interface SyncRulesBucketStorage
     checkpoint: ReplicationCheckpoint,
     dataBuckets: BucketDataRequest[],
     options?: BucketDataBatchOptions
-  ): AsyncIterable<SyncBucketDataChunk>;
+  ): AsyncIterable<SyncBucketDataChunk | SyncBucketDataBatchEnd>;
 
   /**
    * Compute checksums for a given list of buckets.
@@ -308,6 +315,13 @@ export interface CompactOptions {
 
   compactParameterData?: boolean;
 
+  /**
+   * Delete client-requested write checkpoints created before this time.
+   *
+   * Generated write checkpoints are not affected.
+   */
+  deleteCheckpointRequestsBefore?: Date;
+
   /** Minimum of 2 */
   clearBatchLimit?: number;
 
@@ -396,6 +410,9 @@ export interface TerminateOptions extends ClearStorageOptions {
 export interface BucketDataBatchOptions {
   requestHint?: BucketRequestHint;
 
+  /** Abort any in-progress work for this batch, including object-storage downloads. */
+  signal?: AbortSignal;
+
   /** Limit number of documents returned. Defaults to 1000. */
   limit?: number;
 
@@ -416,6 +433,20 @@ export interface SyncBucketDataChunk {
   targetOp: util.InternalOpId | null;
 }
 
+export interface SyncBucketDataBatchEnd {
+  /**
+   * True if there may be more data for this checkpoint, and the caller should continue querying.
+   *
+   * This is different from `SyncBucketDataChunk.has_more`, which is per-bucket. This is a global signal for the
+   * entire request, and may be true even if there is no returned chunk with has_more: true.
+   */
+  hasMore: boolean;
+}
+
+export function isBatchEnd(chunk: SyncBucketDataChunk | SyncBucketDataBatchEnd): chunk is SyncBucketDataBatchEnd {
+  return (chunk as SyncBucketDataBatchEnd).hasMore !== undefined;
+}
+
 export interface ReplicationCheckpoint {
   readonly checkpoint: util.InternalOpId;
   readonly lsn: string | null;

package/src/storage/WriteCheckpointAPI.ts

@@ -19,6 +19,18 @@ export interface BaseWriteCheckpointIdentifier {
   user_id: string;
 }
 
+export interface ClientRequestedCheckpointOptions {
+  /**
+   * Supplied for client-generated checkpoint requests.
+   *
+   * If omitted, storage creates or stores a regular write checkpoint.
+   * If supplied for managed write checkpoints, storage only uses it when it is
+   * greater than the stored id for the user_id. Same or lower supplied ids are
+   * no-ops, and storage returns the current stored id.
+   */
+  checkpoint_request_id?: bigint;
+}
+
 export interface CustomWriteCheckpointFilters extends BaseWriteCheckpointIdentifier {
   /**
    * Replication stream which was active when this checkpoint was created.
@@ -28,9 +40,17 @@ export interface CustomWriteCheckpointFilters extends BaseWriteCheckpointIdentif
 
 export interface BatchedCustomWriteCheckpointOptions extends BaseWriteCheckpointIdentifier {
   /**
-   * A supplied incrementing Write Checkpoint number
+   * A supplied incrementing checkpoint request id. This is still named
+   * "write checkpoint" in storage APIs for backwards compatibility.
    */
   checkpoint: bigint;
+  /**
+   * Required when this custom checkpoint was created from a client checkpoint
+   * request and should be eligible for checkpoint request retention cleanup.
+   * Omit or set to null for persistent custom checkpoints owned by the source
+   * or integration.
+   */
+  checkpoint_requested_at?: Date | null;
 }
 
 export interface CustomWriteCheckpointOptions extends BatchedCustomWriteCheckpointOptions {
@@ -50,7 +70,90 @@ export interface ManagedWriteCheckpointFilters extends BaseWriteCheckpointIdenti
   heads: Record<string, string>;
 }
 
-export type ManagedWriteCheckpointOptions = ManagedWriteCheckpointFilters;
+export type ManagedWriteCheckpointOptions = ManagedWriteCheckpointFilters & ClientRequestedCheckpointOptions;
+
+export interface CreateManagedWriteCheckpointsResult {
+  /**
+   * Current managed checkpoint id for each full user_id after applying the batch.
+   */
+  writeCheckpoints: Map<string, bigint>;
+  /**
+   * True when the source marker must be forced so replication observes the
+   * stored head. Callers use this to advance the source marker once for the
+   * whole batch.
+   *
+   * This must be true whenever a checkpoint in the batch has not yet been
+   * processed by replication - both freshly created/advanced checkpoints and
+   * existing checkpoints matched by a stale or duplicate request that are still
+   * pending. Already-processed checkpoints do not need a new marker.
+   *
+   * The case this guards against is a lost source marker on a stale retry. The
+   * source marker is forced after the checkpoint is persisted, so the two are
+   * not atomic (storage and source are usually different databases). Without
+   * this flag, the following sequence strands the checkpoint:
+   *
+   *   1. A client-supplied checkpoint id is persisted (write checkpoint stored,
+   *      processed_at_lsn null).
+   *   2. Forcing the source marker fails, so the request errors and the client
+   *      retries with the same id.
+   *   3. The retry is a no-op for the stored id (monotonic, not greater), so if
+   *      we only advanced on "a row changed" we would skip the marker.
+   *   4. On an idle source nothing else moves replication past the stored head,
+   *      so the checkpoint never gets acknowledged and the client waits forever.
+   *
+   * Reporting true while the checkpoint is still pending makes the retry re-force
+   * the marker, which resolves the wait.
+   *
+   * Backends that do not track a per-row processed indicator may conservatively
+   * report true whenever any checkpoint was matched.
+   */
+  shouldAdvance: boolean;
+}
+
+export function uniqueManagedWriteCheckpoints(
+  checkpoints: ManagedWriteCheckpointOptions[]
+): ManagedWriteCheckpointOptions[] {
+  const byUser = new Map<string, ManagedWriteCheckpointOptions>();
+
+  for (const checkpoint of checkpoints) {
+    const existing = byUser.get(checkpoint.user_id);
+    // A batch can contain entries for many users, and different users may use
+    // different request types (generated vs client-supplied). For a single full
+    // user_id, though, all entries should be the same type - a batch can still
+    // hold multiple entries for that user (e.g. coalesced requests), so we
+    // collapse them to one. For client-supplied ids we keep the greatest
+    // requested value, so lower stale ids cannot hide the request that should
+    // advance storage. For generated ids the entries are equivalent, so any one
+    // is kept.
+    //
+    // Mixing generated and supplied entries for the same user_id is not expected;
+    // shouldReplaceManagedWriteCheckpoint still resolves it deterministically
+    // (supplied wins) if that invariant is ever violated.
+    if (existing == null || shouldReplaceManagedWriteCheckpoint(existing, checkpoint)) {
+      byUser.set(checkpoint.user_id, checkpoint);
+    }
+  }
+
+  return [...byUser.values()];
+}
+
+function shouldReplaceManagedWriteCheckpoint(
+  existing: ManagedWriteCheckpointOptions,
+  candidate: ManagedWriteCheckpointOptions
+) {
+  const existingRequestId = existing.checkpoint_request_id;
+  const candidateRequestId = candidate.checkpoint_request_id;
+
+  if (candidateRequestId == null) {
+    return existingRequestId == null;
+  }
+
+  if (existingRequestId == null) {
+    return true;
+  }
+
+  return candidateRequestId > existingRequestId;
+}
 
 export type SyncStorageLastWriteCheckpointFilters = BaseWriteCheckpointIdentifier | ManagedWriteCheckpointFilters;
 export type LastWriteCheckpointFilters = CustomWriteCheckpointFilters | ManagedWriteCheckpointFilters;
@@ -58,7 +161,9 @@ export type LastWriteCheckpointFilters = CustomWriteCheckpointFilters | ManagedW
 export interface BaseWriteCheckpointAPI {
   readonly writeCheckpointMode: WriteCheckpointMode;
   setWriteCheckpointMode(mode: WriteCheckpointMode): void;
-  createManagedWriteCheckpoints(checkpoints: ManagedWriteCheckpointOptions[]): Promise<Map<string, bigint>>;
+  createManagedWriteCheckpoints(
+    checkpoints: ManagedWriteCheckpointOptions[]
+  ): Promise<CreateManagedWriteCheckpointsResult>;
 }
 
 /**

package/src/storage/implementation/BucketDefinitionMapping.ts

@@ -10,13 +10,13 @@ import {
   SerializedParameterIndexLookupCreator,
   serializedStreamBucketDataSourceEquality,
   serializedStreamParameterIndexLookupCreatorEquality,
-  SerializedSyncPlanV1,
+  SerializedSyncPlan,
   SourceTableRef,
   SyncConfigWithErrors
 } from '@powersync/service-sync-rules';
 
 export interface SerializedSyncConfigWithMapping {
-  plan: SerializedSyncPlanV1;
+  plan: SerializedSyncPlan;
   mapping: SingleSyncConfigBucketDefinitionMapping;
 }
 
@@ -131,7 +131,7 @@ export class SingleSyncConfigBucketDefinitionMapping implements BucketDefinition
    */
   static constructIncrementalMappingFromSerializedPlans(
     compatibleConfigs: SerializedSyncConfigWithMapping[],
-    newPlan: SerializedSyncPlanV1,
+    newPlan: SerializedSyncPlan,
     reservedMappings: SingleSyncConfigBucketDefinitionMapping[]
   ): SingleSyncConfigBucketDefinitionMapping {
     return this.constructIncrementalMappingWithChanges(compatibleConfigs, newPlan, reservedMappings).mapping;
@@ -139,7 +139,7 @@ export class SingleSyncConfigBucketDefinitionMapping implements BucketDefinition
 
   static constructIncrementalMappingWithChanges(
     compatibleConfigs: SerializedSyncConfigWithMapping[],
-    newPlan: SerializedSyncPlanV1,
+    newPlan: SerializedSyncPlan,
     reservedMappings: SingleSyncConfigBucketDefinitionMapping[]
   ): IncrementalMappingResult {
     let nextBucketDefinitionId =

package/src/storage/storage-index.ts

@@ -2,6 +2,7 @@ export * from './bson.js';
 export * from './BucketStorage.js';
 export * from './BucketStorageBatch.js';
 export * from './BucketStorageFactory.js';
+export * from './CheckpointChecksumInvalidatedError.js';
 export * from './ChecksumCache.js';
 export * from './ParsedSyncConfigSet.js';
 export * from './PersistedReplicationStream.js';

package/src/sync/BucketChecksumState.ts

@@ -2,6 +2,7 @@ import {
   BucketParameterQuerier,
   BucketPriority,
   BucketSource,
+  BucketSourceType,
   HydratedSyncConfig,
   mergeBuckets,
   QuerierError,
@@ -75,6 +76,12 @@ export class BucketChecksumState {
    */
   private lastChecksums: util.ChecksumMap | null = null;
   private lastWriteCheckpoint: bigint | null = null;
+  /**
+   * The next storage checkpoint diff may be relative to a checksum-invalidated
+   * checkpoint that was never sent to the client. Re-check every bucket once
+   * instead of applying that diff to the last client-visible checksums.
+   */
+  private forceFullChecksumForNextCheckpoint = false;
   /**
    * Once we've sent the first full checkpoint line including all {@link util.Checkpoint.streams} that the user is
    * subscribed to, we keep an index of the stream names to their index in that array.
@@ -116,6 +123,10 @@ export class BucketChecksumState {
     }
   }
 
+  invalidateChecksumBaseline() {
+    this.forceFullChecksumForNextCheckpoint = true;
+  }
+
   /**
    * Build a new checkpoint line for an underlying storage checkpoint update if any buckets have changed.
    *
@@ -140,6 +151,7 @@ export class BucketChecksumState {
 
     const update = await this.parameterState.getCheckpointUpdate(next, tracer);
     const { buckets: allBuckets, updatedBuckets, usedParameterResults } = update;
+    const forceFullChecksum = this.forceFullChecksumForNextCheckpoint;
 
     /** Set of all buckets in this checkpoint. */
     const bucketDescriptionMap = new Map(allBuckets.map((b) => [b.bucket, b]));
@@ -164,8 +176,9 @@ export class BucketChecksumState {
         const count = bucketsByDefinition.get(definition) ?? 0;
         bucketsByDefinition.set(definition, count + 1);
       }
-
-      const breakdown = formatBucketDefinitionBreakdown(bucketsByDefinition);
+      // Only 1 type is allowed per sync config
+      const bucketSourceType = this.parameterState.syncRules.bucketSourceDefinitions[0].type;
+      const breakdown = formatBucketDefinitionBreakdown(bucketsByDefinition, bucketSourceType);
       errorMessage += breakdown.message;
       logData.buckets_by_definition = breakdown.countsByDefinition;
 
@@ -174,7 +187,7 @@ export class BucketChecksumState {
     }
 
     let checksumMap: util.ChecksumMap;
-    if (updatedBuckets != INVALIDATE_ALL_BUCKETS) {
+    if (!forceFullChecksum && updatedBuckets != INVALIDATE_ALL_BUCKETS) {
       if (this.lastChecksums == null) {
         throw new ServiceAssertionError(`Bucket diff received without existing checksums`);
       }
@@ -248,6 +261,9 @@ export class BucketChecksumState {
         diff.updatedBuckets.length == 0
       ) {
         // No changes - don't send anything to the client
+        if (forceFullChecksum) {
+          this.forceFullChecksumForNextCheckpoint = false;
+        }
         return null;
       }
 
@@ -402,6 +418,9 @@ export class BucketChecksumState {
         this.lastChecksums = checksumMap;
         this.lastWriteCheckpoint = writeCheckpoint;
         this.pendingBucketDownloads = pendingBucketDownloads;
+        if (forceFullChecksum) {
+          this.forceFullChecksumForNextCheckpoint = false;
+        }
         deferredLog();
       },
 
@@ -795,30 +814,36 @@ function logCheckpoint(
 }
 
 /**
- * Format a breakdown of dynamic bucket by sync stream definition.
+ * Format a breakdown of dynamic buckets by sync stream or legacy bucket definition.
  *
- * Sorts definitions by count (descending), includes the top 10, and returns both the
+ * Sorts definitions by count (descending), includes the top 100, and returns both the
  * formatted message string and the counts record suitable for structured log data.
  */
-function formatBucketDefinitionBreakdown(bucketsByDefinition: Map<string, number>): {
+function formatBucketDefinitionBreakdown(
+  bucketsByDefinition: Map<string, number>,
+  bucketSourceType: BucketSourceType
+): {
   message: string;
   countsByDefinition: Record<string, number>;
 } {
-  // Sort definitions by count (descending) and take top 10
+  const maxLoggedDefinitions = 100;
+
+  // Sort definitions by count (descending) and take the largest entries.
   const allSorted = Array.from(bucketsByDefinition.entries()).sort((a, b) => b[1] - a[1]);
-  const sortedDefinitions = allSorted.slice(0, 10);
+  const sortedDefinitions = allSorted.slice(0, maxLoggedDefinitions);
 
-  let message = '\Buckets by definition:';
+  const sourceLabel = bucketSourceType == BucketSourceType.SYNC_STREAM ? 'sync stream' : 'bucket definition';
+  let message = `\nBuckets by ${sourceLabel}:`;
   const countsByDefinition: Record<string, number> = {};
   for (const [definition, count] of sortedDefinitions) {
     message += `\n  ${definition}: ${count}`;
     countsByDefinition[definition] = count;
   }
 
-  if (allSorted.length > 10) {
-    const remainingResults = allSorted.slice(10).reduce((sum, [, count]) => sum + count, 0);
-    const remainingDefinitions = allSorted.length - 10;
-    message += `\n  ... and ${remainingResults} more results from ${remainingDefinitions} definitions`;
+  if (allSorted.length > maxLoggedDefinitions) {
+    const remainingResults = allSorted.slice(maxLoggedDefinitions).reduce((sum, [, count]) => sum + count, 0);
+    const remainingDefinitions = allSorted.length - maxLoggedDefinitions;
+    message += `\n  ... and ${remainingResults} more buckets from ${remainingDefinitions} ${sourceLabel}s`;
   }
 
   return { message, countsByDefinition };

package/src/sync/sync.ts

@@ -1,19 +1,18 @@
 import { JSONBig, JsonContainer } from '@powersync/service-jsonbig';
 import { BucketPriority, HydratedSyncConfig, ResolvedBucket, SqliteJsonValue } from '@powersync/service-sync-rules';
 
-import { AbortError } from 'ix/aborterror.js';
-
 import * as auth from '../auth/auth-index.js';
 import * as storage from '../storage/storage-index.js';
 import * as util from '../util/util-index.js';
 
 import { Logger, logger as defaultLogger } from '@powersync/lib-services-framework';
+import { isBatchEnd } from '../storage/storage-index.js';
 import { mergeAsyncIterables } from '../streams/streams-index.js';
 import { PerformanceTracer, type Span } from '../tracing/PerformanceTracer.js';
 import { BucketChecksumState, CheckpointLine, type SyncCheckpointTraceCategory } from './BucketChecksumState.js';
 import { OperationsSentStats, RequestTracker, statsForBatch } from './RequestTracker.js';
 import { SyncContext } from './SyncContext.js';
-import { TokenStreamOptions, acquireSemaphoreAbortable, settledPromise, tokenStream } from './util.js';
+import { TokenStreamOptions, acquireSemaphoreAbortable, isAbortError, settledPromise, tokenStream } from './util.js';
 
 type CheckpointTiming = Record<string, number>;
 
@@ -87,7 +86,7 @@ export async function* streamResponse(
   try {
     yield* merged;
   } catch (e) {
-    if (e instanceof AbortError) {
+    if (isAbortError(e)) {
       return;
     } else {
       throw e;
@@ -149,6 +148,22 @@ async function* streamResponseInner(
       const line = await checksumState.buildNextCheckpointLine(next.value, trace.tracer);
       return { done: false, value: { checkpoint: cp, line, trace: line == null ? null : trace } };
     } catch (e) {
+      if (e instanceof storage.CheckpointChecksumInvalidatedError) {
+        // The checksum was not usable, so buildNextCheckpointLine has not advanced
+        // the connection state. Drop this candidate and wait for a checkpoint that
+        // is not split by a compaction-produced bucket-data document.
+        // This is different from other checkpoint_invalidated cases in that we hit
+        // this during checksum calculation, instead of on data read.
+        trace.span.end();
+        checksumState.invalidateChecksumBaseline();
+        logger.info(`checkpoint_invalidated: ${cp.checkpoint}`, {
+          reason: 'compacted_before_checkpoint_line',
+          bucket: e.bucket,
+          checkpoint: cp.checkpoint,
+          user_id: tokenPayload.userIdJson
+        });
+        return { done: false, value: { checkpoint: cp, line: null, trace: null } };
+      }
       // Only end the span if we error. If we return normally, we pass ownership on to the caller.
       trace.span.end();
       throw e;
@@ -220,7 +235,7 @@ async function* streamResponseInner(
             while (true) {
               const next = await settledPromise(waitForNewCheckpointLine());
               if (next.status == 'rejected') {
-                if (next.reason instanceof AbortError) {
+                if (isAbortError(next.reason)) {
                   checkpointResult = { result: 'invalidated', invalidationReason: 'checkpoint_cancelled' };
                 } else {
                   checkpointResult = { result: 'invalidated', invalidationReason: 'checkpoint_error' };
@@ -439,12 +454,23 @@ async function* bucketDataBatch(
     // Optimization: Only fetch buckets for which the checksums have changed since the last checkpoint
     // For the first batch, this will be all buckets.
     const filteredBuckets = checkpointLine.getFilteredBucketPositions(bucketsToFetch);
-    const dataBatches = storage.getBucketDataBatch(checkpoint, filteredBuckets, { requestHint });
-    for await (let { chunkData: r, targetOp } of dataBatches) {
+    const dataBatches = storage.getBucketDataBatch(checkpoint, filteredBuckets, {
+      requestHint,
+      // Checkpoint supersession is a cooperative batch handoff. Only abort
+      // in-flight storage work when the connection itself is closed.
+      signal: abort_connection
+    });
+    for await (let chunk of dataBatches) {
       // Abort in current batch if the connection is closed
       if (abort_connection.aborted) {
         return null;
       }
+      if (isBatchEnd(chunk)) {
+        // This replaces any other has_more value, since the batch end is the last chunk.
+        has_more = chunk.hasMore;
+        break;
+      }
+      const { chunkData: r, targetOp } = chunk;
       if (r.has_more) {
         has_more = true;
       }

package/src/sync/util.ts

@@ -2,6 +2,7 @@ import * as timers from 'timers/promises';
 
 import { SemaphoreInterface } from 'async-mutex';
 import { serialize } from 'bson';
+import { AbortError } from 'ix/aborterror.js';
 import * as util from '../util/util-index.js';
 import { RequestTracker } from './RequestTracker.js';
 
@@ -23,6 +24,17 @@ const DEFAULT_TOKEN_STREAM_OPTIONS: TokenStreamOptions = {
   expire_warning_period: 20_000
 };
 
+/**
+ * Recognize both ix's AbortError and native abort errors, which Node represents
+ * as DOMExceptions with the name "AbortError".
+ */
+export function isAbortError(error: unknown): boolean {
+  return (
+    error instanceof AbortError ||
+    (typeof error == 'object' && error != null && 'name' in error && error.name === 'AbortError')
+  );
+}
+
 /**
  * An iterator that periodically yields token and optionally keepalive events, and returns once the
  * provided token expiry is reached.

package/src/util/checkpointing.ts

@@ -3,12 +3,13 @@ import { WriteCheckpointBatcher } from './write-checkpoint-batcher.js';
 export interface CreateWriteCheckpointOptions {
   userId: string | undefined;
   clientId: string | undefined;
+  checkpointRequestId?: bigint;
   batcher: WriteCheckpointBatcher;
 }
 export async function createWriteCheckpoint(options: CreateWriteCheckpointOptions) {
   const full_user_id = checkpointUserId(options.userId, options.clientId);
 
-  return options.batcher.enqueue(full_user_id);
+  return options.batcher.enqueue(full_user_id, options.checkpointRequestId);
 }
 
 export function checkpointUserId(user_id: string | undefined, client_id: string | undefined) {