@powersync/service-core 1.24.0 → 1.26.0

package/src/storage/SourceTable.ts

@@ -2,19 +2,30 @@ import {
   BucketDataSource,
   BucketDefinitionId,
   DEFAULT_TAG,
+  EventDefinitionId,
   ParameterIndexId,
   ParameterIndexLookupCreator,
   SourceTableRef
 } from '@powersync/service-sync-rules';
 import { bson } from '../index.js';
 import * as util from '../util/util-index.js';
-import { ColumnDescriptor } from './SourceEntity.js';
+import { ColumnDescriptor, JsonValue } from './SourceEntity.js';
 
 /**
  * Format of the id depends on the bucket storage module. It should be consistent within the module.
  */
 export type SourceTableId = string | bson.ObjectId;
 
+/**
+ * Compare source-table ids without coercing between storage-specific types.
+ */
+export function sourceTableIdEquals(left: SourceTableId, right: SourceTableId): boolean {
+  if (typeof left === 'string' || typeof right === 'string') {
+    return typeof left === 'string' && typeof right === 'string' && left === right;
+  }
+  return left.equals(right);
+}
+
 export interface SourceTableOptions {
   id: SourceTableId;
   ref: SourceTableRef;
@@ -25,6 +36,17 @@ export interface SourceTableOptions {
   parameterLookupSources: ParameterIndexLookupCreator[];
   bucketDataSourceIds?: Set<BucketDefinitionId>;
   parameterLookupSourceIds?: Set<ParameterIndexId>;
+  /**
+   * Compiled event definitions assigned to this persisted source-table record.
+   *
+   * Undefined is the legacy/non-incremental representation where event selection is
+   * based on the table ref. V3 incremental storage always supplies this set.
+   */
+  eventDefinitionIds?: Set<EventDefinitionId>;
+  /**
+   * Source-specific metadata. Null when no metadata has been recorded.
+   */
+  sourceMetadata?: JsonValue;
 }
 
 export interface TableSnapshotStatus {
@@ -61,11 +83,11 @@ export class SourceTable {
   public syncParameters = true;
 
   /**
-   * True if this table should fire events for row changes.
+   * True if this table should evaluate event definitions for row changes.
    *
-   * This value is resolved externally, and cached here. When multiple SourceTables exist
-   * for the same SourceTableRef (v3 storage), resolveTables designates exactly one of them
-   * as the event carrier, so that a row change saved once per table fires each event once.
+   * This value is resolved externally, and cached here. V3 storage assigns disjoint
+   * event-definition ids to SourceTables for the same physical table. Multiple records
+   * may evaluate different events, but each event id is evaluated through at most one record.
    *
    * Defaults to true for tests.
    */
@@ -136,6 +158,14 @@ export class SourceTable {
     return this.options.parameterLookupSourceIds;
   }
 
+  get eventDefinitionIds() {
+    return this.options.eventDefinitionIds;
+  }
+
+  get sourceMetadata() {
+    return this.options.sourceMetadata ?? null;
+  }
+
   /**
    * Sanitized name of the entity in the format of "{schema}.{entity name}".
    * Suitable for safe use in Postgres queries.
@@ -152,23 +182,42 @@ export class SourceTable {
    * In-memory clone of the table status.
    */
   clone() {
+    return this.copyWithSourceMetadata(this.sourceMetadata);
+  }
+
+  /**
+   * Copy this table with different source metadata, preserving its resolved state.
+   */
+  withSourceMetadata(sourceMetadata: JsonValue) {
+    return this.copyWithSourceMetadata(sourceMetadata);
+  }
+
+  private copyWithSourceMetadata(sourceMetadata: JsonValue) {
     const copy = new SourceTable({
       id: this.id,
-      ref: this.options.ref,
+      ref: { ...this.options.ref },
       objectId: this.objectId,
-      replicaIdColumns: this.replicaIdColumns,
+      replicaIdColumns: this.replicaIdColumns.map((column) => ({ ...column })),
       snapshotComplete: this.snapshotComplete,
-      bucketDataSources: this.bucketDataSources,
-      parameterLookupSources: this.parameterLookupSources,
+      bucketDataSources: [...this.bucketDataSources],
+      parameterLookupSources: [...this.parameterLookupSources],
       bucketDataSourceIds: this.bucketDataSourceIds == null ? undefined : new Set(this.bucketDataSourceIds),
       parameterLookupSourceIds:
-        this.parameterLookupSourceIds == null ? undefined : new Set(this.parameterLookupSourceIds)
+        this.parameterLookupSourceIds == null ? undefined : new Set(this.parameterLookupSourceIds),
+      eventDefinitionIds: this.eventDefinitionIds == null ? undefined : new Set(this.eventDefinitionIds),
+      sourceMetadata: structuredClone(sourceMetadata)
     });
     copy.syncData = this.syncData;
     copy.syncParameters = this.syncParameters;
     copy.syncEvent = this.syncEvent;
     copy.storeCurrentData = this.storeCurrentData;
-    copy.snapshotStatus = this.snapshotStatus;
+    copy.snapshotStatus =
+      this.snapshotStatus == null
+        ? undefined
+        : {
+            ...this.snapshotStatus,
+            lastKey: this.snapshotStatus.lastKey?.slice() ?? null
+          };
     return copy;
   }
 
@@ -183,3 +232,11 @@ export class SourceTable {
     }
   }
 }
+
+/**
+ * A cloned SourceTable exposed to reconciliation with public fields typed as read-only.
+ * `options` is omitted so callers cannot mutate the underlying option bag without an explicit cast.
+ */
+export type SourceTableCandidate = Omit<Readonly<SourceTable>, 'options' | 'withSourceMetadata'> & {
+  withSourceMetadata(sourceMetadata: JsonValue): SourceTableCandidate;
+};

package/src/storage/SourceTableReconciler.ts

@@ -0,0 +1,188 @@
+import { ServiceAssertionError } from '@powersync/lib-services-framework';
+import { isDeepStrictEqual } from 'node:util';
+import { JsonValue, SourceEntityDescriptor } from './SourceEntity.js';
+import { SourceTable, SourceTableCandidate, SourceTableId, sourceTableIdEquals } from './SourceTable.js';
+
+/**
+ * A source connector's classification of overlapping persisted tables.
+ */
+export interface SourceTableCandidateResolution {
+  /**
+   * Records storage can reuse. Copies may include updated source metadata.
+   */
+  compatibleTables: ReadonlyArray<SourceTableCandidate>;
+
+  /**
+   * Records that cannot be reused. Every candidate must appear in exactly one result list.
+   */
+  incompatibleTables: ReadonlyArray<SourceTableCandidate>;
+
+  /**
+   * Values for records storage creates during this resolution.
+   */
+  newTableValues: SourceTableCreateValues;
+}
+
+export interface SourceTableCreateValues {
+  /**
+   * Source metadata for new records. Null means no metadata.
+   */
+  sourceMetadata: JsonValue;
+}
+
+/**
+ * Input to a source-owned reconciliation callback. The callback may run inside a storage
+ * transaction, so it must not mutate storage or perform slow external work.
+ */
+export interface SourceTableCandidateReconcilerInput {
+  /**
+   * Source entity being resolved.
+   */
+  source: SourceEntityDescriptor;
+
+  /**
+   * Persisted tables overlapping by name or object id.
+   */
+  candidates: ReadonlyArray<SourceTableCandidate>;
+}
+
+export type SourceTableCandidateReconciler = (
+  input: SourceTableCandidateReconcilerInput
+) => SourceTableCandidateResolution | Promise<SourceTableCandidateResolution>;
+
+/**
+ * Compare replica-id columns in order.
+ */
+export function sameReplicaIdColumns(
+  left: SourceTableCandidate['replicaIdColumns'],
+  right: SourceEntityDescriptor
+): boolean {
+  const target = right.replicaIdColumns;
+  return (
+    left.length == target.length &&
+    left.every(
+      (column, index) =>
+        column.name == target[index].name && column.type == target[index].type && column.typeId == target[index].typeId
+    )
+  );
+}
+
+/**
+ * Compare the shared source-table identity fields.
+ */
+export function sourceIdentityCompatible(source: SourceEntityDescriptor, candidate: SourceTableCandidate): boolean {
+  return (
+    candidate.schema == source.schema &&
+    candidate.name == source.name &&
+    (source.objectId == null || candidate.objectId == source.objectId) &&
+    sameReplicaIdColumns(candidate.replicaIdColumns, source)
+  );
+}
+
+/**
+ * Default identity-based reconciliation for connectors without source-specific metadata.
+ */
+export const defaultSourceTableReconciler: SourceTableCandidateReconciler = ({ source, candidates }) => {
+  const compatibleTables: SourceTableCandidate[] = [];
+  const incompatibleTables: SourceTableCandidate[] = [];
+  for (const candidate of candidates) {
+    if (sourceIdentityCompatible(source, candidate)) {
+      compatibleTables.push(candidate);
+    } else {
+      incompatibleTables.push(candidate);
+    }
+  }
+  return { compatibleTables, incompatibleTables, newTableValues: { sourceMetadata: null } };
+};
+
+/**
+ * Check that every candidate was classified exactly once.
+ */
+export function validateSourceTableCandidateResolution(
+  candidates: ReadonlyArray<SourceTableCandidate>,
+  resolution: SourceTableCandidateResolution
+): void {
+  const classifiedTables = [...resolution.compatibleTables, ...resolution.incompatibleTables];
+
+  for (const candidate of candidates) {
+    const classifications = classifiedTables.filter((table) => sourceTableIdEquals(table.id, candidate.id));
+    if (classifications.length !== 1) {
+      throw new ServiceAssertionError(
+        `Source table candidate ${candidate.id.toString()} must be classified exactly once, got ${classifications.length}`
+      );
+    }
+  }
+
+  for (const table of classifiedTables) {
+    if (!candidates.some((candidate) => sourceTableIdEquals(candidate.id, table.id))) {
+      throw new ServiceAssertionError(`Source table reconciliation returned unknown candidate ${table.id.toString()}`);
+    }
+  }
+}
+
+/**
+ * A source-metadata update to persist.
+ */
+export interface SourceTableMetadataUpdate {
+  id: SourceTableId;
+  sourceMetadata: JsonValue;
+}
+
+/**
+ * Rebuild a resolution from storage-owned tables, applying only reconciler-owned metadata to
+ * compatible tables. All other mutable table state comes from storage.
+ *
+ * Reconciler candidates are typed as read-only, but TypeScript types provide no runtime protection:
+ * callback code can cast a cloned candidate and mutate it. Rematerializing by id ensures those
+ * mutations are not trusted even when the compile-time boundary is bypassed.
+ */
+export function materializeSourceTableResolution(
+  tables: ReadonlyArray<SourceTable>,
+  resolution: SourceTableCandidateResolution
+): MaterializedSourceTableResolution {
+  const findTable = (candidate: SourceTableCandidate): SourceTable => {
+    const table = tables.find((table) => sourceTableIdEquals(table.id, candidate.id));
+    if (table == null) {
+      throw new ServiceAssertionError(`Source table candidate ${candidate.id.toString()} was not persisted`);
+    }
+    return table;
+  };
+  return {
+    compatibleTables: resolution.compatibleTables.map((candidate) =>
+      findTable(candidate).withSourceMetadata(candidate.sourceMetadata)
+    ),
+    incompatibleTables: resolution.incompatibleTables.map(findTable),
+    newTableValues: resolution.newTableValues
+  };
+}
+
+export interface MaterializedSourceTableResolution {
+  compatibleTables: SourceTable[];
+  incompatibleTables: SourceTable[];
+  newTableValues: SourceTableCreateValues;
+}
+
+/**
+ * Return source-metadata changes from compatible candidates, comparing metadata by value against
+ * the original storage-owned tables. The reconciler may mutate its isolated candidate clones, so
+ * those clones cannot be used as the persisted baseline.
+ */
+export function diffSourceTableUpdates(
+  persistedTables: ReadonlyArray<SourceTable>,
+  resolution: SourceTableCandidateResolution
+): SourceTableMetadataUpdate[] {
+  const updates: SourceTableMetadataUpdate[] = [];
+  for (const resolvedTable of resolution.compatibleTables) {
+    const persistedTable = persistedTables.find((table) => sourceTableIdEquals(table.id, resolvedTable.id));
+    if (persistedTable == null) {
+      throw new ServiceAssertionError(
+        `Source table reconciliation returned unknown candidate ${resolvedTable.id.toString()}`
+      );
+    }
+    if (isDeepStrictEqual(persistedTable.sourceMetadata, resolvedTable.sourceMetadata)) {
+      continue;
+    }
+    updates.push({ id: resolvedTable.id, sourceMetadata: resolvedTable.sourceMetadata });
+  }
+  return updates;
+}

package/src/storage/StorageVersionConfig.ts

@@ -39,6 +39,11 @@ export const STORAGE_VERSION_2 = 2;
  */
 export const STORAGE_VERSION_3 = 3;
 
+/**
+ * Same as version 3, but marked as stable.
+ */
+export const STORAGE_VERSION_4 = 4;
+
 /**
  * Oldest supported storage version.
  */
@@ -54,7 +59,7 @@ export const CURRENT_STORAGE_VERSION = STORAGE_VERSION_2;
  *
  * This includes unstable versions.
  */
-export const SUPPORTED_STORAGE_VERSIONS = [STORAGE_VERSION_1, STORAGE_VERSION_2, STORAGE_VERSION_3];
+export const SUPPORTED_STORAGE_VERSIONS = [STORAGE_VERSION_1, STORAGE_VERSION_2, STORAGE_VERSION_3, STORAGE_VERSION_4];
 
 /**
  * Shared storage-version behavior across storage implementations.
@@ -74,5 +79,10 @@ export const STORAGE_VERSION_CONFIG: Record<number, StorageVersionConfig | undef
     version: STORAGE_VERSION_3,
     versionedBuckets: true,
     softDeleteCurrentData: true
+  },
+  [STORAGE_VERSION_4]: {
+    version: STORAGE_VERSION_4,
+    versionedBuckets: true,
+    softDeleteCurrentData: true
   }
 };

package/src/storage/SyncRulesBucketStorage.ts

@@ -14,6 +14,7 @@ import { ParsedSyncConfigSet } from './ParsedSyncConfigSet.js';
 import { ParseSyncConfigOptions } from './PersistedSyncConfigContent.js';
 import { SourceEntityDescriptor } from './SourceEntity.js';
 import { SourceTable } from './SourceTable.js';
+import { SourceTableCandidateReconciler } from './SourceTableReconciler.js';
 import { StorageVersionConfig } from './StorageVersionConfig.js';
 import { SyncStorageWriteCheckpointAPI } from './WriteCheckpointAPI.js';
 
@@ -96,12 +97,20 @@ export interface SyncRulesBucketStorage
    */
   reportError(e: any): Promise<void>;
 
+  /**
+   * Whether parameter compaction can run when {@link CompactOptions.incrementalOnly} is set.
+   *
+   * Storage implementations that do not support incremental parameter compaction must return
+   * false; the compact command will skip parameter compaction in that mode.
+   */
+  supportsIncrementalParameterCompaction(): boolean;
+
   compact(options?: CompactOptions): Promise<void>;
 
   /**
-   * Lightweight "compact" process to populate the checksum cache, if any.
+   * Compact storage after initial replication, before the first checkpoint exists.
    */
-  populatePersistentChecksumCache(options: PopulateChecksumCacheOptions): Promise<PopulateChecksumCacheResults>;
+  compactInitialReplication(options: CompactInitialReplicationOptions): Promise<CompactInitialReplicationResults>;
 
   // ## Read operations
 
@@ -212,6 +221,11 @@ export interface ResolveTablesOptions {
    * Source table or collection metadata discovered during snapshot or streaming.
    */
   source: SourceEntityDescriptor;
+  /**
+   * Classifies overlapping persisted tables. Defaults to identity-based reconciliation.
+   * This may run inside a storage transaction and must not mutate storage.
+   */
+  reconcileSourceTables?: SourceTableCandidateReconciler;
   /**
    * For tests only - custom id generator for stable ids.
    */
@@ -271,6 +285,14 @@ export interface CreateWriterOptions extends ParseSyncConfigOptions {
   tracer?: PerformanceTracer<'storage' | 'evaluate'>;
 
   logger?: Logger;
+
+  /**
+   * Aborts long-running storage work started by this writer, such as uploads to object storage.
+   *
+   * This does not replace flushing or committing at the appropriate source boundary: it only
+   * cancels work that is still in flight when replication stops.
+   */
+  signal?: AbortSignal;
 }
 
 export interface StorageHooks {
@@ -315,6 +337,15 @@ export interface CompactOptions {
 
   compactParameterData?: boolean;
 
+  /**
+   * Only perform compaction that can be done incrementally.
+   *
+   * This includes full bucket compaction on MongoDB V3 storage.
+   *
+   * On MongoDB v1 and Postgres storage, this makes compacting a no-op.
+   */
+  incrementalOnly?: boolean;
+
   /**
    * Delete client-requested write checkpoints created before this time.
    *
@@ -348,6 +379,18 @@ export interface CompactOptions {
    */
   minChangeRatio?: number;
 
+  /** Minimum delay before a V3 bucket is checked for chunk compaction. Default: five minutes. */
+  minCompactChunkIntervalMs?: number;
+
+  /** Minimum elapsed write pressure before the v3 sliding-scale full compact. Default: two hours. */
+  minCompactFullIntervalMs?: number;
+
+  /** Maximum age of writes not covered by a v3 full compact. */
+  maxCompactFullIntervalMs?: number;
+
+  /** How long a v3 worker owns a claimed bucket before another worker may recover it. */
+  compactLeaseDurationMs?: number;
+
   /**
    * Internal/testing use: Cache size for compacting parameters.
    */
@@ -358,12 +401,12 @@ export interface CompactOptions {
   logger?: Logger;
 }
 
-export interface PopulateChecksumCacheOptions {
+export interface CompactInitialReplicationOptions {
   /**
-   * Compute checksums up to this op id.
+   * Compact data up to this op id.
    *
    * Defaults to the highest persisted op id for the replication stream, which covers
-   * the common case of populating the cache right after initial replication, before
+   * the common case of compacting right after initial replication, before
    * the first checkpoint exists.
    */
   maxOpId?: util.InternalOpId;
@@ -371,9 +414,9 @@ export interface PopulateChecksumCacheOptions {
   signal?: AbortSignal;
 }
 
-export interface PopulateChecksumCacheResults {
+export interface CompactInitialReplicationResults {
   /**
-   * Number of buckets we have calculated checksums for.
+   * Number of buckets processed.
    */
   buckets: number;
 }
@@ -413,6 +456,9 @@ export interface BucketDataBatchOptions {
   /** Abort any in-progress work for this batch, including object-storage downloads. */
   signal?: AbortSignal;
 
+  /** Traces data reads performed by the storage implementation. */
+  tracer?: PerformanceTracer<string>;
+
   /** Limit number of documents returned. Defaults to 1000. */
   limit?: number;
 

package/src/storage/WriteCheckpointAPI.ts

@@ -1,3 +1,5 @@
+import { EventDefinitionId } from '@powersync/service-sync-rules';
+
 export enum WriteCheckpointMode {
   /**
    * Raw mappings of `user_id` to `write_checkpoint`s should
@@ -31,7 +33,9 @@ export interface ClientRequestedCheckpointOptions {
   checkpoint_request_id?: bigint;
 }
 
-export interface CustomWriteCheckpointFilters extends BaseWriteCheckpointIdentifier {
+export type SyncStorageCustomWriteCheckpointFilters = BaseWriteCheckpointIdentifier;
+
+export interface CustomWriteCheckpointFilters extends SyncStorageCustomWriteCheckpointFilters {
   /**
    * Replication stream which was active when this checkpoint was created.
    */
@@ -44,6 +48,14 @@ export interface BatchedCustomWriteCheckpointOptions extends BaseWriteCheckpoint
    * "write checkpoint" in storage APIs for backwards compatibility.
    */
   checkpoint: bigint;
+  /**
+   * Compiled event definition which created the checkpoint.
+   *
+   * Incremental replication uses this to keep checkpoint records produced by
+   * active and processing sync configs separate. This is required by v3
+   * storage and unused by legacy storage versions.
+   */
+  event_id?: EventDefinitionId;
   /**
    * Required when this custom checkpoint was created from a client checkpoint
    * request and should be eligible for checkpoint request retention cleanup.
@@ -155,12 +167,29 @@ function shouldReplaceManagedWriteCheckpoint(
   return candidateRequestId > existingRequestId;
 }
 
-export type SyncStorageLastWriteCheckpointFilters = BaseWriteCheckpointIdentifier | ManagedWriteCheckpointFilters;
+export type SyncStorageLastWriteCheckpointFilters =
+  | SyncStorageCustomWriteCheckpointFilters
+  | ManagedWriteCheckpointFilters;
 export type LastWriteCheckpointFilters = CustomWriteCheckpointFilters | ManagedWriteCheckpointFilters;
 
+/** Configures checkpoint ownership and generation for a storage instance. */
+export type WriteCheckpointModeConfig =
+  | {
+      mode: WriteCheckpointMode.CUSTOM;
+      /**
+       * Name of the event whose custom checkpoints are served. Storage resolves this name through the active sync
+       * config's persisted mapping; integrations must not construct or persist event ids themselves. Required by
+       * storage implementations that scope custom checkpoints by event definition.
+       */
+      eventName?: string;
+    }
+  | {
+      mode: WriteCheckpointMode.MANAGED;
+    };
+
 export interface BaseWriteCheckpointAPI {
   readonly writeCheckpointMode: WriteCheckpointMode;
-  setWriteCheckpointMode(mode: WriteCheckpointMode): void;
+  setWriteCheckpointMode(config: WriteCheckpointModeConfig): void;
   createManagedWriteCheckpoints(
     checkpoints: ManagedWriteCheckpointOptions[]
   ): Promise<CreateManagedWriteCheckpointsResult>;