@powersync/service-core 1.25.0 → 1.26.0

package/src/storage/BucketStorageFactory.ts

@@ -37,10 +37,17 @@ export abstract class BucketStorageFactory
     const deploying = await this.getDeployingSyncConfig();
     const active = await this.getActiveSyncConfig();
 
-    if (deploying?.content.sync_rules_content == options.config.yaml) {
+    if (
+      deploying?.content.sync_rules_content == options.config.yaml &&
+      deploying.content.version_label == options.version_label
+    ) {
       logger.info('Sync config unchanged');
       return { updated: false };
-    } else if (deploying == null && active?.content.sync_rules_content == options.config.yaml) {
+    } else if (
+      deploying == null &&
+      active?.content.sync_rules_content == options.config.yaml &&
+      active.content.version_label == options.version_label
+    ) {
       logger.info('Sync config unchanged');
       return { updated: false };
     } else {
@@ -141,9 +148,32 @@ export interface StorageMetrics {
    * Size of current_data.
    */
   replication_size_bytes: number;
+
+  /**
+   * Size of active object-storage references, such as S3 bucket-data objects.
+   */
+  object_storage_size_bytes?: number;
+
+  /**
+   * Per-sync-config storage sizes, when the storage backend can provide them.
+   */
+  sync_config_metrics?: StorageSyncConfigMetrics[];
+}
+
+export interface StorageSyncConfigMetrics {
+  sync_config_id: string;
+  sync_config_state: string;
+  /** Optional operator-supplied label identifying this sync config version. */
+  version_label?: string;
+  attributed_bucket_data_bytes: number;
+  attributed_parameter_indexes_bytes: number;
+  attributed_source_records_bytes: number;
+  attributed_object_storage_bytes: number;
 }
 
 export interface UpdateSyncRulesOptions {
+  /** Optional operator-supplied label identifying this deployed sync config version. */
+  version_label?: string;
   config: {
     yaml: string;
     /**
@@ -161,6 +191,12 @@ export interface UpdateSyncRulesOptions {
   lock?: boolean;
   storageVersion?: number;
 
+  /**
+   * Start a replacement replication stream instead of incrementally reprocessing a
+   * compatible active stream. Used when the current stream needs a full reprocess.
+   */
+  forceNewReplicationStream?: boolean;
+
   /**
    * Only relevant if the result is used. This does not affect the persisted config.
    */
@@ -174,11 +210,10 @@ export interface SerializedSyncPlan {
   plan: RawSerializedSyncPlan;
   compatibility: SerializedCompatibilityContext;
   /**
-   * Event descriptors are not currently represented in the sync plan because they don't use the sync streams compiler
-   * yet.
+   * Raw event SQL persisted as a compatibility mirror for compiled {@link plan} events.
    *
-   * We might revisit that in the future, but for now we store SQL text of their definitions here to be able to restore
-   * them.
+   * Compiled events are an additive plan field. Older services ignore that field and restore these descriptors through
+   * the legacy evaluator. Keep dual-writing this field until a future plan version explicitly removes that support.
    */
   eventDescriptors: Record<string, string[]>;
   errors?: ReplicationError[];
@@ -206,13 +241,14 @@ export function updateSyncRulesFromConfig(
   const { config, errors } = parsed;
   if (config instanceof PrecompiledSyncConfig) {
     const eventDescriptors: Record<string, string[]> = {};
-    for (const event of config.eventDescriptors) {
-      eventDescriptors[event.name] = event.sourceQueries.map((q) => q.sql);
+    for (const event of config.plan.events) {
+      eventDescriptors[event.name] = event.sourceQueries.map((query) => query.sql);
     }
 
     plan = {
       compatibility: config.compatibility.serialize(),
       plan: serializeSyncPlan(config.plan),
+      // Dual-write raw SQL so older services can ignore additive compiled plan events without losing event behavior.
       eventDescriptors,
       errors: errors.map((e) => syncConfigYamlErrorToReplicationError(e))
     };

package/src/storage/PersistedSyncConfigContent.ts

@@ -2,6 +2,7 @@ import { logger as defaultLogger, ErrorCode, ServiceError } from '@powersync/lib
 import {
   CompatibilityContext,
   CompatibilityOption,
+  compileEventDefinitions,
   DEFAULT_HYDRATION_STATE,
   deserializeSyncPlan,
   ErrorLocation,
@@ -9,7 +10,6 @@ import {
   HydrationState,
   nodeSqlite,
   PrecompiledSyncConfig,
-  SqlEventDescriptor,
   SqlSyncRules,
   SyncConfigWithErrors,
   versionedHydrationState,
@@ -40,17 +40,28 @@ export function parsePersistedSyncConfigContent(options: ParsePersistedSyncConfi
 
   const plan = deserializeSyncPlan(compiledPlan.plan);
   const compatibility = CompatibilityContext.deserialize(compiledPlan.compatibility);
-  const eventDefinitions: SqlEventDescriptor[] = [];
-  for (const [name, queries] of Object.entries(compiledPlan.eventDescriptors)) {
-    const descriptor = new SqlEventDescriptor(name, compatibility);
-    for (const query of queries) {
-      descriptor.addSourceQuery(query, parseOptions);
+  const errors: YamlError[] = [];
+  // Compiled events are additive to plan versions 1 and 2. New readers prefer them when present; when an older plan
+  // does not contain them, normalize the dual-written raw SQL at this loading boundary. This keeps legacy event
+  // evaluators out of PrecompiledSyncConfig while older binaries can continue reading the same persisted config.
+  if (compiledPlan.plan.events == null) {
+    const normalized = compileEventDefinitions(compiledPlan.eventDescriptors, {
+      ...parseOptions,
+      // The legacy evaluator ignored event payload filters. Preserve that behavior for plans deployed before compiled
+      // events existed; a redeploy compiles and validates those filters before the replacement config is activated.
+      compileEventPayloadFilters: false
+    });
+    const fatalErrors = normalized.errors.filter((error) => error.type == 'fatal');
+    if (fatalErrors.length != 0) {
+      throw new Error(
+        `Failed to compile persisted replication events: ${fatalErrors.map((error) => error.message).join(', ')}`
+      );
     }
-
-    eventDefinitions.push(descriptor);
+    plan.events = normalized.events;
+    errors.push(...normalized.errors.map((error) => new YamlError(error)));
   }
 
-  const precompiled = new PrecompiledSyncConfig(plan, compatibility, eventDefinitions, {
+  const precompiled = new PrecompiledSyncConfig(plan, compatibility, {
     defaultSchema: parseOptions.defaultSchema,
     sourceText: content
   });
@@ -59,7 +70,6 @@ export function parsePersistedSyncConfigContent(options: ParsePersistedSyncConfi
   // This means asUpdateOptions will not change the storage version, even if the default changes.
   precompiled.storageVersion = storageVersion;
 
-  const errors: YamlError[] = [];
   if (compiledPlan.errors) {
     for (const error of compiledPlan.errors) {
       const location: ErrorLocation | undefined = error.location && {
@@ -93,6 +103,7 @@ export abstract class PersistedSyncConfigContent implements PersistedSyncConfigC
   readonly logger: Logger;
   readonly syncConfigId: PersistedSyncConfigId | null;
   readonly syncConfigState: SyncRuleState;
+  readonly version_label: string | undefined;
 
   constructor(data: PersistedSyncConfigContentData) {
     this.replicationStreamId = data.replicationStreamId;
@@ -102,7 +113,9 @@ export abstract class PersistedSyncConfigContent implements PersistedSyncConfigC
     this.storageVersion = data.storageVersion;
     this.syncConfigId = data.syncConfigId ?? null;
     this.syncConfigState = data.syncConfigState;
-    this.logger = defaultLogger.child({ prefix: `[${this.replicationStreamName}] ` });
+    this.version_label = data.version_label;
+    const versionPrefix = this.version_label == null ? '' : `[${this.version_label}]`;
+    this.logger = defaultLogger.child({ prefix: `[${this.replicationStreamName}]${versionPrefix} ` });
   }
 
   /**
@@ -167,6 +180,7 @@ export abstract class PersistedSyncConfigContent implements PersistedSyncConfigC
     const parsed = this.parseSingleConfig({ defaultSchema: 'not_applicable' });
     return {
       config: { yaml: this.sync_rules_content, plan: this.compiled_plan, parsed },
+      version_label: this.version_label,
       ...options
     };
   }
@@ -185,6 +199,7 @@ export interface PersistedSyncConfigContentData {
 
   readonly syncConfigId?: PersistedSyncConfigId | null;
   readonly syncConfigState: SyncRuleState;
+  readonly version_label?: string;
 }
 export type PersistedSyncConfigId = string;
 export interface ParseSyncConfigOptions {

package/src/storage/ReplicationEventPayload.ts

@@ -11,6 +11,11 @@ export type EventData = {
 export type ReplicationEventPayload = {
   batch: BucketStorageBatch;
   data: EventData;
-  event: sync_rules.SqlEventDescriptor;
+  event: sync_rules.HydratedEventDescriptor;
+  /**
+   * Storage-assigned id for this event definition. Present when storage uses persisted event mappings, allowing event
+   * handlers to route custom checkpoints without deriving an id from compiled event content.
+   */
+  event_id?: sync_rules.EventDefinitionId;
   table: SourceTable;
 };

package/src/storage/SourceTable.ts

@@ -2,6 +2,7 @@ import {
   BucketDataSource,
   BucketDefinitionId,
   DEFAULT_TAG,
+  EventDefinitionId,
   ParameterIndexId,
   ParameterIndexLookupCreator,
   SourceTableRef
@@ -35,6 +36,13 @@ export interface SourceTableOptions {
   parameterLookupSources: ParameterIndexLookupCreator[];
   bucketDataSourceIds?: Set<BucketDefinitionId>;
   parameterLookupSourceIds?: Set<ParameterIndexId>;
+  /**
+   * Compiled event definitions assigned to this persisted source-table record.
+   *
+   * Undefined is the legacy/non-incremental representation where event selection is
+   * based on the table ref. V3 incremental storage always supplies this set.
+   */
+  eventDefinitionIds?: Set<EventDefinitionId>;
   /**
    * Source-specific metadata. Null when no metadata has been recorded.
    */
@@ -75,11 +83,11 @@ export class SourceTable {
   public syncParameters = true;
 
   /**
-   * True if this table should fire events for row changes.
+   * True if this table should evaluate event definitions for row changes.
    *
-   * This value is resolved externally, and cached here. When multiple SourceTables exist
-   * for the same SourceTableRef (v3 storage), resolveTables designates exactly one of them
-   * as the event carrier, so that a row change saved once per table fires each event once.
+   * This value is resolved externally, and cached here. V3 storage assigns disjoint
+   * event-definition ids to SourceTables for the same physical table. Multiple records
+   * may evaluate different events, but each event id is evaluated through at most one record.
    *
    * Defaults to true for tests.
    */
@@ -150,6 +158,10 @@ export class SourceTable {
     return this.options.parameterLookupSourceIds;
   }
 
+  get eventDefinitionIds() {
+    return this.options.eventDefinitionIds;
+  }
+
   get sourceMetadata() {
     return this.options.sourceMetadata ?? null;
   }
@@ -192,6 +204,7 @@ export class SourceTable {
       bucketDataSourceIds: this.bucketDataSourceIds == null ? undefined : new Set(this.bucketDataSourceIds),
       parameterLookupSourceIds:
         this.parameterLookupSourceIds == null ? undefined : new Set(this.parameterLookupSourceIds),
+      eventDefinitionIds: this.eventDefinitionIds == null ? undefined : new Set(this.eventDefinitionIds),
       sourceMetadata: structuredClone(sourceMetadata)
     });
     copy.syncData = this.syncData;

package/src/storage/StorageVersionConfig.ts

@@ -39,6 +39,11 @@ export const STORAGE_VERSION_2 = 2;
  */
 export const STORAGE_VERSION_3 = 3;
 
+/**
+ * Same as version 3, but marked as stable.
+ */
+export const STORAGE_VERSION_4 = 4;
+
 /**
  * Oldest supported storage version.
  */
@@ -54,7 +59,7 @@ export const CURRENT_STORAGE_VERSION = STORAGE_VERSION_2;
  *
  * This includes unstable versions.
  */
-export const SUPPORTED_STORAGE_VERSIONS = [STORAGE_VERSION_1, STORAGE_VERSION_2, STORAGE_VERSION_3];
+export const SUPPORTED_STORAGE_VERSIONS = [STORAGE_VERSION_1, STORAGE_VERSION_2, STORAGE_VERSION_3, STORAGE_VERSION_4];
 
 /**
  * Shared storage-version behavior across storage implementations.
@@ -74,5 +79,10 @@ export const STORAGE_VERSION_CONFIG: Record<number, StorageVersionConfig | undef
     version: STORAGE_VERSION_3,
     versionedBuckets: true,
     softDeleteCurrentData: true
+  },
+  [STORAGE_VERSION_4]: {
+    version: STORAGE_VERSION_4,
+    versionedBuckets: true,
+    softDeleteCurrentData: true
   }
 };

package/src/storage/SyncRulesBucketStorage.ts

@@ -97,12 +97,20 @@ export interface SyncRulesBucketStorage
    */
   reportError(e: any): Promise<void>;
 
+  /**
+   * Whether parameter compaction can run when {@link CompactOptions.incrementalOnly} is set.
+   *
+   * Storage implementations that do not support incremental parameter compaction must return
+   * false; the compact command will skip parameter compaction in that mode.
+   */
+  supportsIncrementalParameterCompaction(): boolean;
+
   compact(options?: CompactOptions): Promise<void>;
 
   /**
-   * Lightweight "compact" process to populate the checksum cache, if any.
+   * Compact storage after initial replication, before the first checkpoint exists.
    */
-  populatePersistentChecksumCache(options: PopulateChecksumCacheOptions): Promise<PopulateChecksumCacheResults>;
+  compactInitialReplication(options: CompactInitialReplicationOptions): Promise<CompactInitialReplicationResults>;
 
   // ## Read operations
 
@@ -277,6 +285,14 @@ export interface CreateWriterOptions extends ParseSyncConfigOptions {
   tracer?: PerformanceTracer<'storage' | 'evaluate'>;
 
   logger?: Logger;
+
+  /**
+   * Aborts long-running storage work started by this writer, such as uploads to object storage.
+   *
+   * This does not replace flushing or committing at the appropriate source boundary: it only
+   * cancels work that is still in flight when replication stops.
+   */
+  signal?: AbortSignal;
 }
 
 export interface StorageHooks {
@@ -321,6 +337,15 @@ export interface CompactOptions {
 
   compactParameterData?: boolean;
 
+  /**
+   * Only perform compaction that can be done incrementally.
+   *
+   * This includes full bucket compaction on MongoDB V3 storage.
+   *
+   * On MongoDB v1 and Postgres storage, this makes compacting a no-op.
+   */
+  incrementalOnly?: boolean;
+
   /**
    * Delete client-requested write checkpoints created before this time.
    *
@@ -354,6 +379,18 @@ export interface CompactOptions {
    */
   minChangeRatio?: number;
 
+  /** Minimum delay before a V3 bucket is checked for chunk compaction. Default: five minutes. */
+  minCompactChunkIntervalMs?: number;
+
+  /** Minimum elapsed write pressure before the v3 sliding-scale full compact. Default: two hours. */
+  minCompactFullIntervalMs?: number;
+
+  /** Maximum age of writes not covered by a v3 full compact. */
+  maxCompactFullIntervalMs?: number;
+
+  /** How long a v3 worker owns a claimed bucket before another worker may recover it. */
+  compactLeaseDurationMs?: number;
+
   /**
    * Internal/testing use: Cache size for compacting parameters.
    */
@@ -364,12 +401,12 @@ export interface CompactOptions {
   logger?: Logger;
 }
 
-export interface PopulateChecksumCacheOptions {
+export interface CompactInitialReplicationOptions {
   /**
-   * Compute checksums up to this op id.
+   * Compact data up to this op id.
    *
    * Defaults to the highest persisted op id for the replication stream, which covers
-   * the common case of populating the cache right after initial replication, before
+   * the common case of compacting right after initial replication, before
    * the first checkpoint exists.
    */
   maxOpId?: util.InternalOpId;
@@ -377,9 +414,9 @@ export interface PopulateChecksumCacheOptions {
   signal?: AbortSignal;
 }
 
-export interface PopulateChecksumCacheResults {
+export interface CompactInitialReplicationResults {
   /**
-   * Number of buckets we have calculated checksums for.
+   * Number of buckets processed.
    */
   buckets: number;
 }
@@ -419,6 +456,9 @@ export interface BucketDataBatchOptions {
   /** Abort any in-progress work for this batch, including object-storage downloads. */
   signal?: AbortSignal;
 
+  /** Traces data reads performed by the storage implementation. */
+  tracer?: PerformanceTracer<string>;
+
   /** Limit number of documents returned. Defaults to 1000. */
   limit?: number;
 

package/src/storage/WriteCheckpointAPI.ts

@@ -1,3 +1,5 @@
+import { EventDefinitionId } from '@powersync/service-sync-rules';
+
 export enum WriteCheckpointMode {
   /**
    * Raw mappings of `user_id` to `write_checkpoint`s should
@@ -31,7 +33,9 @@ export interface ClientRequestedCheckpointOptions {
   checkpoint_request_id?: bigint;
 }
 
-export interface CustomWriteCheckpointFilters extends BaseWriteCheckpointIdentifier {
+export type SyncStorageCustomWriteCheckpointFilters = BaseWriteCheckpointIdentifier;
+
+export interface CustomWriteCheckpointFilters extends SyncStorageCustomWriteCheckpointFilters {
   /**
    * Replication stream which was active when this checkpoint was created.
    */
@@ -44,6 +48,14 @@ export interface BatchedCustomWriteCheckpointOptions extends BaseWriteCheckpoint
    * "write checkpoint" in storage APIs for backwards compatibility.
    */
   checkpoint: bigint;
+  /**
+   * Compiled event definition which created the checkpoint.
+   *
+   * Incremental replication uses this to keep checkpoint records produced by
+   * active and processing sync configs separate. This is required by v3
+   * storage and unused by legacy storage versions.
+   */
+  event_id?: EventDefinitionId;
   /**
    * Required when this custom checkpoint was created from a client checkpoint
    * request and should be eligible for checkpoint request retention cleanup.
@@ -155,12 +167,29 @@ function shouldReplaceManagedWriteCheckpoint(
   return candidateRequestId > existingRequestId;
 }
 
-export type SyncStorageLastWriteCheckpointFilters = BaseWriteCheckpointIdentifier | ManagedWriteCheckpointFilters;
+export type SyncStorageLastWriteCheckpointFilters =
+  | SyncStorageCustomWriteCheckpointFilters
+  | ManagedWriteCheckpointFilters;
 export type LastWriteCheckpointFilters = CustomWriteCheckpointFilters | ManagedWriteCheckpointFilters;
 
+/** Configures checkpoint ownership and generation for a storage instance. */
+export type WriteCheckpointModeConfig =
+  | {
+      mode: WriteCheckpointMode.CUSTOM;
+      /**
+       * Name of the event whose custom checkpoints are served. Storage resolves this name through the active sync
+       * config's persisted mapping; integrations must not construct or persist event ids themselves. Required by
+       * storage implementations that scope custom checkpoints by event definition.
+       */
+      eventName?: string;
+    }
+  | {
+      mode: WriteCheckpointMode.MANAGED;
+    };
+
 export interface BaseWriteCheckpointAPI {
   readonly writeCheckpointMode: WriteCheckpointMode;
-  setWriteCheckpointMode(mode: WriteCheckpointMode): void;
+  setWriteCheckpointMode(config: WriteCheckpointModeConfig): void;
   createManagedWriteCheckpoints(
     checkpoints: ManagedWriteCheckpointOptions[]
   ): Promise<CreateManagedWriteCheckpointsResult>;