@powersync/service-core 1.23.3 → 1.25.0

package/src/storage/SourceTableReconciler.ts

@@ -0,0 +1,188 @@
+import { ServiceAssertionError } from '@powersync/lib-services-framework';
+import { isDeepStrictEqual } from 'node:util';
+import { JsonValue, SourceEntityDescriptor } from './SourceEntity.js';
+import { SourceTable, SourceTableCandidate, SourceTableId, sourceTableIdEquals } from './SourceTable.js';
+
+/**
+ * A source connector's classification of overlapping persisted tables.
+ */
+export interface SourceTableCandidateResolution {
+  /**
+   * Records storage can reuse. Copies may include updated source metadata.
+   */
+  compatibleTables: ReadonlyArray<SourceTableCandidate>;
+
+  /**
+   * Records that cannot be reused. Every candidate must appear in exactly one result list.
+   */
+  incompatibleTables: ReadonlyArray<SourceTableCandidate>;
+
+  /**
+   * Values for records storage creates during this resolution.
+   */
+  newTableValues: SourceTableCreateValues;
+}
+
+export interface SourceTableCreateValues {
+  /**
+   * Source metadata for new records. Null means no metadata.
+   */
+  sourceMetadata: JsonValue;
+}
+
+/**
+ * Input to a source-owned reconciliation callback. The callback may run inside a storage
+ * transaction, so it must not mutate storage or perform slow external work.
+ */
+export interface SourceTableCandidateReconcilerInput {
+  /**
+   * Source entity being resolved.
+   */
+  source: SourceEntityDescriptor;
+
+  /**
+   * Persisted tables overlapping by name or object id.
+   */
+  candidates: ReadonlyArray<SourceTableCandidate>;
+}
+
+export type SourceTableCandidateReconciler = (
+  input: SourceTableCandidateReconcilerInput
+) => SourceTableCandidateResolution | Promise<SourceTableCandidateResolution>;
+
+/**
+ * Compare replica-id columns in order.
+ */
+export function sameReplicaIdColumns(
+  left: SourceTableCandidate['replicaIdColumns'],
+  right: SourceEntityDescriptor
+): boolean {
+  const target = right.replicaIdColumns;
+  return (
+    left.length == target.length &&
+    left.every(
+      (column, index) =>
+        column.name == target[index].name && column.type == target[index].type && column.typeId == target[index].typeId
+    )
+  );
+}
+
+/**
+ * Compare the shared source-table identity fields.
+ */
+export function sourceIdentityCompatible(source: SourceEntityDescriptor, candidate: SourceTableCandidate): boolean {
+  return (
+    candidate.schema == source.schema &&
+    candidate.name == source.name &&
+    (source.objectId == null || candidate.objectId == source.objectId) &&
+    sameReplicaIdColumns(candidate.replicaIdColumns, source)
+  );
+}
+
+/**
+ * Default identity-based reconciliation for connectors without source-specific metadata.
+ */
+export const defaultSourceTableReconciler: SourceTableCandidateReconciler = ({ source, candidates }) => {
+  const compatibleTables: SourceTableCandidate[] = [];
+  const incompatibleTables: SourceTableCandidate[] = [];
+  for (const candidate of candidates) {
+    if (sourceIdentityCompatible(source, candidate)) {
+      compatibleTables.push(candidate);
+    } else {
+      incompatibleTables.push(candidate);
+    }
+  }
+  return { compatibleTables, incompatibleTables, newTableValues: { sourceMetadata: null } };
+};
+
+/**
+ * Check that every candidate was classified exactly once.
+ */
+export function validateSourceTableCandidateResolution(
+  candidates: ReadonlyArray<SourceTableCandidate>,
+  resolution: SourceTableCandidateResolution
+): void {
+  const classifiedTables = [...resolution.compatibleTables, ...resolution.incompatibleTables];
+
+  for (const candidate of candidates) {
+    const classifications = classifiedTables.filter((table) => sourceTableIdEquals(table.id, candidate.id));
+    if (classifications.length !== 1) {
+      throw new ServiceAssertionError(
+        `Source table candidate ${candidate.id.toString()} must be classified exactly once, got ${classifications.length}`
+      );
+    }
+  }
+
+  for (const table of classifiedTables) {
+    if (!candidates.some((candidate) => sourceTableIdEquals(candidate.id, table.id))) {
+      throw new ServiceAssertionError(`Source table reconciliation returned unknown candidate ${table.id.toString()}`);
+    }
+  }
+}
+
+/**
+ * A source-metadata update to persist.
+ */
+export interface SourceTableMetadataUpdate {
+  id: SourceTableId;
+  sourceMetadata: JsonValue;
+}
+
+/**
+ * Rebuild a resolution from storage-owned tables, applying only reconciler-owned metadata to
+ * compatible tables. All other mutable table state comes from storage.
+ *
+ * Reconciler candidates are typed as read-only, but TypeScript types provide no runtime protection:
+ * callback code can cast a cloned candidate and mutate it. Rematerializing by id ensures those
+ * mutations are not trusted even when the compile-time boundary is bypassed.
+ */
+export function materializeSourceTableResolution(
+  tables: ReadonlyArray<SourceTable>,
+  resolution: SourceTableCandidateResolution
+): MaterializedSourceTableResolution {
+  const findTable = (candidate: SourceTableCandidate): SourceTable => {
+    const table = tables.find((table) => sourceTableIdEquals(table.id, candidate.id));
+    if (table == null) {
+      throw new ServiceAssertionError(`Source table candidate ${candidate.id.toString()} was not persisted`);
+    }
+    return table;
+  };
+  return {
+    compatibleTables: resolution.compatibleTables.map((candidate) =>
+      findTable(candidate).withSourceMetadata(candidate.sourceMetadata)
+    ),
+    incompatibleTables: resolution.incompatibleTables.map(findTable),
+    newTableValues: resolution.newTableValues
+  };
+}
+
+export interface MaterializedSourceTableResolution {
+  compatibleTables: SourceTable[];
+  incompatibleTables: SourceTable[];
+  newTableValues: SourceTableCreateValues;
+}
+
+/**
+ * Return source-metadata changes from compatible candidates, comparing metadata by value against
+ * the original storage-owned tables. The reconciler may mutate its isolated candidate clones, so
+ * those clones cannot be used as the persisted baseline.
+ */
+export function diffSourceTableUpdates(
+  persistedTables: ReadonlyArray<SourceTable>,
+  resolution: SourceTableCandidateResolution
+): SourceTableMetadataUpdate[] {
+  const updates: SourceTableMetadataUpdate[] = [];
+  for (const resolvedTable of resolution.compatibleTables) {
+    const persistedTable = persistedTables.find((table) => sourceTableIdEquals(table.id, resolvedTable.id));
+    if (persistedTable == null) {
+      throw new ServiceAssertionError(
+        `Source table reconciliation returned unknown candidate ${resolvedTable.id.toString()}`
+      );
+    }
+    if (isDeepStrictEqual(persistedTable.sourceMetadata, resolvedTable.sourceMetadata)) {
+      continue;
+    }
+    updates.push({ id: resolvedTable.id, sourceMetadata: resolvedTable.sourceMetadata });
+  }
+  return updates;
+}

package/src/storage/SyncRulesBucketStorage.ts

@@ -14,6 +14,7 @@ import { ParsedSyncConfigSet } from './ParsedSyncConfigSet.js';
 import { ParseSyncConfigOptions } from './PersistedSyncConfigContent.js';
 import { SourceEntityDescriptor } from './SourceEntity.js';
 import { SourceTable } from './SourceTable.js';
+import { SourceTableCandidateReconciler } from './SourceTableReconciler.js';
 import { StorageVersionConfig } from './StorageVersionConfig.js';
 import { SyncStorageWriteCheckpointAPI } from './WriteCheckpointAPI.js';
 
@@ -133,6 +134,13 @@ export interface SyncRulesBucketStorage
    * 1. Separate buckets.
    * 2. Limit the size of each individual chunk according to options.batchSizeLimitBytes.
    *
+   * The batch may not contain all data for the checkpoint, if the checkpoint is large. The caller must
+   * continue querying if either:
+   * 1. The last chunk for any bucket has has_more = true.
+   * 2. A SyncBucketDataBatchEnd is returned with hasMore = true.
+   *
+   * The first check can be skipped if a SyncBucketDataBatchEnd is returned with hasMore = false.
+   *
    * @param checkpoint the checkpoint
    * @param dataBuckets current bucket states
    * @param options batch size options
@@ -141,7 +149,7 @@ export interface SyncRulesBucketStorage
     checkpoint: ReplicationCheckpoint,
     dataBuckets: BucketDataRequest[],
     options?: BucketDataBatchOptions
-  ): AsyncIterable<SyncBucketDataChunk>;
+  ): AsyncIterable<SyncBucketDataChunk | SyncBucketDataBatchEnd>;
 
   /**
    * Compute checksums for a given list of buckets.
@@ -205,6 +213,11 @@ export interface ResolveTablesOptions {
    * Source table or collection metadata discovered during snapshot or streaming.
    */
   source: SourceEntityDescriptor;
+  /**
+   * Classifies overlapping persisted tables. Defaults to identity-based reconciliation.
+   * This may run inside a storage transaction and must not mutate storage.
+   */
+  reconcileSourceTables?: SourceTableCandidateReconciler;
   /**
    * For tests only - custom id generator for stable ids.
    */
@@ -308,6 +321,13 @@ export interface CompactOptions {
 
   compactParameterData?: boolean;
 
+  /**
+   * Delete client-requested write checkpoints created before this time.
+   *
+   * Generated write checkpoints are not affected.
+   */
+  deleteCheckpointRequestsBefore?: Date;
+
   /** Minimum of 2 */
   clearBatchLimit?: number;
 
@@ -396,6 +416,9 @@ export interface TerminateOptions extends ClearStorageOptions {
 export interface BucketDataBatchOptions {
   requestHint?: BucketRequestHint;
 
+  /** Abort any in-progress work for this batch, including object-storage downloads. */
+  signal?: AbortSignal;
+
   /** Limit number of documents returned. Defaults to 1000. */
   limit?: number;
 
@@ -416,6 +439,20 @@ export interface SyncBucketDataChunk {
   targetOp: util.InternalOpId | null;
 }
 
+export interface SyncBucketDataBatchEnd {
+  /**
+   * True if there may be more data for this checkpoint, and the caller should continue querying.
+   *
+   * This is different from `SyncBucketDataChunk.has_more`, which is per-bucket. This is a global signal for the
+   * entire request, and may be true even if there is no returned chunk with has_more: true.
+   */
+  hasMore: boolean;
+}
+
+export function isBatchEnd(chunk: SyncBucketDataChunk | SyncBucketDataBatchEnd): chunk is SyncBucketDataBatchEnd {
+  return (chunk as SyncBucketDataBatchEnd).hasMore !== undefined;
+}
+
 export interface ReplicationCheckpoint {
   readonly checkpoint: util.InternalOpId;
   readonly lsn: string | null;

package/src/storage/WriteCheckpointAPI.ts

@@ -19,6 +19,18 @@ export interface BaseWriteCheckpointIdentifier {
   user_id: string;
 }
 
+export interface ClientRequestedCheckpointOptions {
+  /**
+   * Supplied for client-generated checkpoint requests.
+   *
+   * If omitted, storage creates or stores a regular write checkpoint.
+   * If supplied for managed write checkpoints, storage only uses it when it is
+   * greater than the stored id for the user_id. Same or lower supplied ids are
+   * no-ops, and storage returns the current stored id.
+   */
+  checkpoint_request_id?: bigint;
+}
+
 export interface CustomWriteCheckpointFilters extends BaseWriteCheckpointIdentifier {
   /**
    * Replication stream which was active when this checkpoint was created.
@@ -28,9 +40,17 @@ export interface CustomWriteCheckpointFilters extends BaseWriteCheckpointIdentif
 
 export interface BatchedCustomWriteCheckpointOptions extends BaseWriteCheckpointIdentifier {
   /**
-   * A supplied incrementing Write Checkpoint number
+   * A supplied incrementing checkpoint request id. This is still named
+   * "write checkpoint" in storage APIs for backwards compatibility.
    */
   checkpoint: bigint;
+  /**
+   * Required when this custom checkpoint was created from a client checkpoint
+   * request and should be eligible for checkpoint request retention cleanup.
+   * Omit or set to null for persistent custom checkpoints owned by the source
+   * or integration.
+   */
+  checkpoint_requested_at?: Date | null;
 }
 
 export interface CustomWriteCheckpointOptions extends BatchedCustomWriteCheckpointOptions {
@@ -50,7 +70,90 @@ export interface ManagedWriteCheckpointFilters extends BaseWriteCheckpointIdenti
   heads: Record<string, string>;
 }
 
-export type ManagedWriteCheckpointOptions = ManagedWriteCheckpointFilters;
+export type ManagedWriteCheckpointOptions = ManagedWriteCheckpointFilters & ClientRequestedCheckpointOptions;
+
+export interface CreateManagedWriteCheckpointsResult {
+  /**
+   * Current managed checkpoint id for each full user_id after applying the batch.
+   */
+  writeCheckpoints: Map<string, bigint>;
+  /**
+   * True when the source marker must be forced so replication observes the
+   * stored head. Callers use this to advance the source marker once for the
+   * whole batch.
+   *
+   * This must be true whenever a checkpoint in the batch has not yet been
+   * processed by replication - both freshly created/advanced checkpoints and
+   * existing checkpoints matched by a stale or duplicate request that are still
+   * pending. Already-processed checkpoints do not need a new marker.
+   *
+   * The case this guards against is a lost source marker on a stale retry. The
+   * source marker is forced after the checkpoint is persisted, so the two are
+   * not atomic (storage and source are usually different databases). Without
+   * this flag, the following sequence strands the checkpoint:
+   *
+   *   1. A client-supplied checkpoint id is persisted (write checkpoint stored,
+   *      processed_at_lsn null).
+   *   2. Forcing the source marker fails, so the request errors and the client
+   *      retries with the same id.
+   *   3. The retry is a no-op for the stored id (monotonic, not greater), so if
+   *      we only advanced on "a row changed" we would skip the marker.
+   *   4. On an idle source nothing else moves replication past the stored head,
+   *      so the checkpoint never gets acknowledged and the client waits forever.
+   *
+   * Reporting true while the checkpoint is still pending makes the retry re-force
+   * the marker, which resolves the wait.
+   *
+   * Backends that do not track a per-row processed indicator may conservatively
+   * report true whenever any checkpoint was matched.
+   */
+  shouldAdvance: boolean;
+}
+
+export function uniqueManagedWriteCheckpoints(
+  checkpoints: ManagedWriteCheckpointOptions[]
+): ManagedWriteCheckpointOptions[] {
+  const byUser = new Map<string, ManagedWriteCheckpointOptions>();
+
+  for (const checkpoint of checkpoints) {
+    const existing = byUser.get(checkpoint.user_id);
+    // A batch can contain entries for many users, and different users may use
+    // different request types (generated vs client-supplied). For a single full
+    // user_id, though, all entries should be the same type - a batch can still
+    // hold multiple entries for that user (e.g. coalesced requests), so we
+    // collapse them to one. For client-supplied ids we keep the greatest
+    // requested value, so lower stale ids cannot hide the request that should
+    // advance storage. For generated ids the entries are equivalent, so any one
+    // is kept.
+    //
+    // Mixing generated and supplied entries for the same user_id is not expected;
+    // shouldReplaceManagedWriteCheckpoint still resolves it deterministically
+    // (supplied wins) if that invariant is ever violated.
+    if (existing == null || shouldReplaceManagedWriteCheckpoint(existing, checkpoint)) {
+      byUser.set(checkpoint.user_id, checkpoint);
+    }
+  }
+
+  return [...byUser.values()];
+}
+
+function shouldReplaceManagedWriteCheckpoint(
+  existing: ManagedWriteCheckpointOptions,
+  candidate: ManagedWriteCheckpointOptions
+) {
+  const existingRequestId = existing.checkpoint_request_id;
+  const candidateRequestId = candidate.checkpoint_request_id;
+
+  if (candidateRequestId == null) {
+    return existingRequestId == null;
+  }
+
+  if (existingRequestId == null) {
+    return true;
+  }
+
+  return candidateRequestId > existingRequestId;
+}
 
 export type SyncStorageLastWriteCheckpointFilters = BaseWriteCheckpointIdentifier | ManagedWriteCheckpointFilters;
 export type LastWriteCheckpointFilters = CustomWriteCheckpointFilters | ManagedWriteCheckpointFilters;
@@ -58,7 +161,9 @@ export type LastWriteCheckpointFilters = CustomWriteCheckpointFilters | ManagedW
 export interface BaseWriteCheckpointAPI {
   readonly writeCheckpointMode: WriteCheckpointMode;
   setWriteCheckpointMode(mode: WriteCheckpointMode): void;
-  createManagedWriteCheckpoints(checkpoints: ManagedWriteCheckpointOptions[]): Promise<Map<string, bigint>>;
+  createManagedWriteCheckpoints(
+    checkpoints: ManagedWriteCheckpointOptions[]
+  ): Promise<CreateManagedWriteCheckpointsResult>;
 }
 
 /**

package/src/storage/implementation/BucketDefinitionMapping.ts

@@ -10,13 +10,13 @@ import {
   SerializedParameterIndexLookupCreator,
   serializedStreamBucketDataSourceEquality,
   serializedStreamParameterIndexLookupCreatorEquality,
-  SerializedSyncPlanV1,
+  SerializedSyncPlan,
   SourceTableRef,
   SyncConfigWithErrors
 } from '@powersync/service-sync-rules';
 
 export interface SerializedSyncConfigWithMapping {
-  plan: SerializedSyncPlanV1;
+  plan: SerializedSyncPlan;
   mapping: SingleSyncConfigBucketDefinitionMapping;
 }
 
@@ -131,7 +131,7 @@ export class SingleSyncConfigBucketDefinitionMapping implements BucketDefinition
    */
   static constructIncrementalMappingFromSerializedPlans(
     compatibleConfigs: SerializedSyncConfigWithMapping[],
-    newPlan: SerializedSyncPlanV1,
+    newPlan: SerializedSyncPlan,
     reservedMappings: SingleSyncConfigBucketDefinitionMapping[]
   ): SingleSyncConfigBucketDefinitionMapping {
     return this.constructIncrementalMappingWithChanges(compatibleConfigs, newPlan, reservedMappings).mapping;
@@ -139,7 +139,7 @@ export class SingleSyncConfigBucketDefinitionMapping implements BucketDefinition
 
   static constructIncrementalMappingWithChanges(
     compatibleConfigs: SerializedSyncConfigWithMapping[],
-    newPlan: SerializedSyncPlanV1,
+    newPlan: SerializedSyncPlan,
     reservedMappings: SingleSyncConfigBucketDefinitionMapping[]
   ): IncrementalMappingResult {
     let nextBucketDefinitionId =

package/src/storage/storage-index.ts

@@ -2,6 +2,7 @@ export * from './bson.js';
 export * from './BucketStorage.js';
 export * from './BucketStorageBatch.js';
 export * from './BucketStorageFactory.js';
+export * from './CheckpointChecksumInvalidatedError.js';
 export * from './ChecksumCache.js';
 export * from './ParsedSyncConfigSet.js';
 export * from './PersistedReplicationStream.js';
@@ -12,6 +13,7 @@ export * from './ReplicationLock.js';
 export * from './ReportStorage.js';
 export * from './SourceEntity.js';
 export * from './SourceTable.js';
+export * from './SourceTableReconciler.js';
 export * from './storage-metrics.js';
 export * from './StorageEngine.js';
 export * from './StorageProvider.js';

package/src/sync/BucketChecksumState.ts

@@ -2,6 +2,7 @@ import {
   BucketParameterQuerier,
   BucketPriority,
   BucketSource,
+  BucketSourceType,
   HydratedSyncConfig,
   mergeBuckets,
   QuerierError,
@@ -75,6 +76,12 @@ export class BucketChecksumState {
    */
   private lastChecksums: util.ChecksumMap | null = null;
   private lastWriteCheckpoint: bigint | null = null;
+  /**
+   * The next storage checkpoint diff may be relative to a checksum-invalidated
+   * checkpoint that was never sent to the client. Re-check every bucket once
+   * instead of applying that diff to the last client-visible checksums.
+   */
+  private forceFullChecksumForNextCheckpoint = false;
   /**
    * Once we've sent the first full checkpoint line including all {@link util.Checkpoint.streams} that the user is
    * subscribed to, we keep an index of the stream names to their index in that array.
@@ -116,6 +123,10 @@ export class BucketChecksumState {
     }
   }
 
+  invalidateChecksumBaseline() {
+    this.forceFullChecksumForNextCheckpoint = true;
+  }
+
   /**
    * Build a new checkpoint line for an underlying storage checkpoint update if any buckets have changed.
    *
@@ -140,6 +151,7 @@ export class BucketChecksumState {
 
     const update = await this.parameterState.getCheckpointUpdate(next, tracer);
     const { buckets: allBuckets, updatedBuckets, usedParameterResults } = update;
+    const forceFullChecksum = this.forceFullChecksumForNextCheckpoint;
 
     /** Set of all buckets in this checkpoint. */
     const bucketDescriptionMap = new Map(allBuckets.map((b) => [b.bucket, b]));
@@ -164,8 +176,9 @@ export class BucketChecksumState {
         const count = bucketsByDefinition.get(definition) ?? 0;
         bucketsByDefinition.set(definition, count + 1);
       }
-
-      const breakdown = formatBucketDefinitionBreakdown(bucketsByDefinition);
+      // Only 1 type is allowed per sync config
+      const bucketSourceType = this.parameterState.syncRules.bucketSourceDefinitions[0].type;
+      const breakdown = formatBucketDefinitionBreakdown(bucketsByDefinition, bucketSourceType);
       errorMessage += breakdown.message;
       logData.buckets_by_definition = breakdown.countsByDefinition;
 
@@ -174,7 +187,7 @@ export class BucketChecksumState {
     }
 
     let checksumMap: util.ChecksumMap;
-    if (updatedBuckets != INVALIDATE_ALL_BUCKETS) {
+    if (!forceFullChecksum && updatedBuckets != INVALIDATE_ALL_BUCKETS) {
       if (this.lastChecksums == null) {
         throw new ServiceAssertionError(`Bucket diff received without existing checksums`);
       }
@@ -248,6 +261,9 @@ export class BucketChecksumState {
         diff.updatedBuckets.length == 0
       ) {
         // No changes - don't send anything to the client
+        if (forceFullChecksum) {
+          this.forceFullChecksumForNextCheckpoint = false;
+        }
         return null;
       }
 
@@ -402,6 +418,9 @@ export class BucketChecksumState {
         this.lastChecksums = checksumMap;
         this.lastWriteCheckpoint = writeCheckpoint;
         this.pendingBucketDownloads = pendingBucketDownloads;
+        if (forceFullChecksum) {
+          this.forceFullChecksumForNextCheckpoint = false;
+        }
         deferredLog();
       },
 
@@ -795,30 +814,36 @@ function logCheckpoint(
 }
 
 /**
- * Format a breakdown of dynamic bucket by sync stream definition.
+ * Format a breakdown of dynamic buckets by sync stream or legacy bucket definition.
  *
- * Sorts definitions by count (descending), includes the top 10, and returns both the
+ * Sorts definitions by count (descending), includes the top 100, and returns both the
  * formatted message string and the counts record suitable for structured log data.
  */
-function formatBucketDefinitionBreakdown(bucketsByDefinition: Map<string, number>): {
+function formatBucketDefinitionBreakdown(
+  bucketsByDefinition: Map<string, number>,
+  bucketSourceType: BucketSourceType
+): {
   message: string;
   countsByDefinition: Record<string, number>;
 } {
-  // Sort definitions by count (descending) and take top 10
+  const maxLoggedDefinitions = 100;
+
+  // Sort definitions by count (descending) and take the largest entries.
   const allSorted = Array.from(bucketsByDefinition.entries()).sort((a, b) => b[1] - a[1]);
-  const sortedDefinitions = allSorted.slice(0, 10);
+  const sortedDefinitions = allSorted.slice(0, maxLoggedDefinitions);
 
-  let message = '\Buckets by definition:';
+  const sourceLabel = bucketSourceType == BucketSourceType.SYNC_STREAM ? 'sync stream' : 'bucket definition';
+  let message = `\nBuckets by ${sourceLabel}:`;
   const countsByDefinition: Record<string, number> = {};
   for (const [definition, count] of sortedDefinitions) {
     message += `\n  ${definition}: ${count}`;
     countsByDefinition[definition] = count;
   }
 
-  if (allSorted.length > 10) {
-    const remainingResults = allSorted.slice(10).reduce((sum, [, count]) => sum + count, 0);
-    const remainingDefinitions = allSorted.length - 10;
-    message += `\n  ... and ${remainingResults} more results from ${remainingDefinitions} definitions`;
+  if (allSorted.length > maxLoggedDefinitions) {
+    const remainingResults = allSorted.slice(maxLoggedDefinitions).reduce((sum, [, count]) => sum + count, 0);
+    const remainingDefinitions = allSorted.length - maxLoggedDefinitions;
+    message += `\n  ... and ${remainingResults} more buckets from ${remainingDefinitions} ${sourceLabel}s`;
   }
 
   return { message, countsByDefinition };

package/src/sync/sync.ts

@@ -1,19 +1,18 @@
 import { JSONBig, JsonContainer } from '@powersync/service-jsonbig';
 import { BucketPriority, HydratedSyncConfig, ResolvedBucket, SqliteJsonValue } from '@powersync/service-sync-rules';
 
-import { AbortError } from 'ix/aborterror.js';
-
 import * as auth from '../auth/auth-index.js';
 import * as storage from '../storage/storage-index.js';
 import * as util from '../util/util-index.js';
 
 import { Logger, logger as defaultLogger } from '@powersync/lib-services-framework';
+import { isBatchEnd } from '../storage/storage-index.js';
 import { mergeAsyncIterables } from '../streams/streams-index.js';
 import { PerformanceTracer, type Span } from '../tracing/PerformanceTracer.js';
 import { BucketChecksumState, CheckpointLine, type SyncCheckpointTraceCategory } from './BucketChecksumState.js';
 import { OperationsSentStats, RequestTracker, statsForBatch } from './RequestTracker.js';
 import { SyncContext } from './SyncContext.js';
-import { TokenStreamOptions, acquireSemaphoreAbortable, settledPromise, tokenStream } from './util.js';
+import { TokenStreamOptions, acquireSemaphoreAbortable, isAbortError, settledPromise, tokenStream } from './util.js';
 
 type CheckpointTiming = Record<string, number>;
 
@@ -87,7 +86,7 @@ export async function* streamResponse(
   try {
     yield* merged;
   } catch (e) {
-    if (e instanceof AbortError) {
+    if (isAbortError(e)) {
       return;
     } else {
       throw e;
@@ -149,6 +148,22 @@ async function* streamResponseInner(
       const line = await checksumState.buildNextCheckpointLine(next.value, trace.tracer);
       return { done: false, value: { checkpoint: cp, line, trace: line == null ? null : trace } };
     } catch (e) {
+      if (e instanceof storage.CheckpointChecksumInvalidatedError) {
+        // The checksum was not usable, so buildNextCheckpointLine has not advanced
+        // the connection state. Drop this candidate and wait for a checkpoint that
+        // is not split by a compaction-produced bucket-data document.
+        // This is different from other checkpoint_invalidated cases in that we hit
+        // this during checksum calculation, instead of on data read.
+        trace.span.end();
+        checksumState.invalidateChecksumBaseline();
+        logger.info(`checkpoint_invalidated: ${cp.checkpoint}`, {
+          reason: 'compacted_before_checkpoint_line',
+          bucket: e.bucket,
+          checkpoint: cp.checkpoint,
+          user_id: tokenPayload.userIdJson
+        });
+        return { done: false, value: { checkpoint: cp, line: null, trace: null } };
+      }
       // Only end the span if we error. If we return normally, we pass ownership on to the caller.
       trace.span.end();
       throw e;
@@ -220,7 +235,7 @@ async function* streamResponseInner(
             while (true) {
               const next = await settledPromise(waitForNewCheckpointLine());
               if (next.status == 'rejected') {
-                if (next.reason instanceof AbortError) {
+                if (isAbortError(next.reason)) {
                   checkpointResult = { result: 'invalidated', invalidationReason: 'checkpoint_cancelled' };
                 } else {
                   checkpointResult = { result: 'invalidated', invalidationReason: 'checkpoint_error' };
@@ -439,12 +454,23 @@ async function* bucketDataBatch(
     // Optimization: Only fetch buckets for which the checksums have changed since the last checkpoint
     // For the first batch, this will be all buckets.
     const filteredBuckets = checkpointLine.getFilteredBucketPositions(bucketsToFetch);
-    const dataBatches = storage.getBucketDataBatch(checkpoint, filteredBuckets, { requestHint });
-    for await (let { chunkData: r, targetOp } of dataBatches) {
+    const dataBatches = storage.getBucketDataBatch(checkpoint, filteredBuckets, {
+      requestHint,
+      // Checkpoint supersession is a cooperative batch handoff. Only abort
+      // in-flight storage work when the connection itself is closed.
+      signal: abort_connection
+    });
+    for await (let chunk of dataBatches) {
       // Abort in current batch if the connection is closed
       if (abort_connection.aborted) {
         return null;
       }
+      if (isBatchEnd(chunk)) {
+        // This replaces any other has_more value, since the batch end is the last chunk.
+        has_more = chunk.hasMore;
+        break;
+      }
+      const { chunkData: r, targetOp } = chunk;
       if (r.has_more) {
         has_more = true;
       }