@powersync/service-core 1.19.2 → 1.20.1

package/src/storage/BucketStorageFactory.ts

@@ -1,9 +1,16 @@
-import { ObserverClient } from '@powersync/lib-services-framework';
+import { BaseObserver, logger } from '@powersync/lib-services-framework';
 import { ParseSyncRulesOptions, PersistedSyncRules, PersistedSyncRulesContent } from './PersistedSyncRulesContent.js';
 import { ReplicationEventPayload } from './ReplicationEventPayload.js';
 import { ReplicationLock } from './ReplicationLock.js';
 import { SyncRulesBucketStorage } from './SyncRulesBucketStorage.js';
 import { ReportStorage } from './ReportStorage.js';
+import {
+  PrecompiledSyncConfig,
+  SerializedCompatibilityContext,
+  serializeSyncPlan,
+  SqlSyncRules,
+  SyncConfig
+} from '@powersync/service-sync-rules';
 
 /**
  * Represents a configured storage provider.
@@ -13,23 +20,41 @@ import { ReportStorage } from './ReportStorage.js';
  *
  * Storage APIs for a specific copy of sync rules are provided by the `SyncRulesBucketStorage` instances.
  */
-export interface BucketStorageFactory extends ObserverClient<BucketStorageFactoryListener>, AsyncDisposable {
+export abstract class BucketStorageFactory
+  extends BaseObserver<BucketStorageFactoryListener>
+  implements AsyncDisposable
+{
   /**
    * Update sync rules from configuration, if changed.
    */
-  configureSyncRules(
+  async configureSyncRules(
     options: UpdateSyncRulesOptions
-  ): Promise<{ updated: boolean; persisted_sync_rules?: PersistedSyncRulesContent; lock?: ReplicationLock }>;
+  ): Promise<{ updated: boolean; persisted_sync_rules?: PersistedSyncRulesContent; lock?: ReplicationLock }> {
+    const next = await this.getNextSyncRulesContent();
+    const active = await this.getActiveSyncRulesContent();
+
+    if (next?.sync_rules_content == options.config.yaml) {
+      logger.info('Sync rules from configuration unchanged');
+      return { updated: false };
+    } else if (next == null && active?.sync_rules_content == options.config.yaml) {
+      logger.info('Sync rules from configuration unchanged');
+      return { updated: false };
+    } else {
+      logger.info('Sync rules updated from configuration');
+      const persisted_sync_rules = await this.updateSyncRules(options);
+      return { updated: true, persisted_sync_rules, lock: persisted_sync_rules.current_lock ?? undefined };
+    }
+  }
 
   /**
    * Get a storage instance to query sync data for specific sync rules.
    */
-  getInstance(syncRules: PersistedSyncRulesContent, options?: GetIntanceOptions): SyncRulesBucketStorage;
+  abstract getInstance(syncRules: PersistedSyncRulesContent, options?: GetIntanceOptions): SyncRulesBucketStorage;
 
   /**
    * Deploy new sync rules.
    */
-  updateSyncRules(options: UpdateSyncRulesOptions): Promise<PersistedSyncRulesContent>;
+  abstract updateSyncRules(options: UpdateSyncRulesOptions): Promise<PersistedSyncRulesContent>;
 
   /**
    * Indicate that a slot was removed, and we should re-sync by creating
@@ -41,57 +66,65 @@ export interface BucketStorageFactory extends ObserverClient<BucketStorageFactor
    *
    * Replication should be restarted after this.
    */
-  restartReplication(sync_rules_group_id: number): Promise<void>;
+  abstract restartReplication(sync_rules_group_id: number): Promise<void>;
 
   /**
    * Get the sync rules used for querying.
    */
-  getActiveSyncRules(options: ParseSyncRulesOptions): Promise<PersistedSyncRules | null>;
+  async getActiveSyncRules(options: ParseSyncRulesOptions): Promise<PersistedSyncRules | null> {
+    const content = await this.getActiveSyncRulesContent();
+    return content?.parsed(options) ?? null;
+  }
 
   /**
    * Get the sync rules used for querying.
    */
-  getActiveSyncRulesContent(): Promise<PersistedSyncRulesContent | null>;
+  abstract getActiveSyncRulesContent(): Promise<PersistedSyncRulesContent | null>;
 
   /**
    * Get the sync rules that will be active next once done with initial replicatino.
    */
-  getNextSyncRules(options: ParseSyncRulesOptions): Promise<PersistedSyncRules | null>;
+  async getNextSyncRules(options: ParseSyncRulesOptions): Promise<PersistedSyncRules | null> {
+    const content = await this.getNextSyncRulesContent();
+    return content?.parsed(options) ?? null;
+  }
 
   /**
    * Get the sync rules that will be active next once done with initial replicatino.
    */
-  getNextSyncRulesContent(): Promise<PersistedSyncRulesContent | null>;
+  abstract getNextSyncRulesContent(): Promise<PersistedSyncRulesContent | null>;
 
   /**
    * Get all sync rules currently replicating. Typically this is the "active" and "next" sync rules.
    */
-  getReplicatingSyncRules(): Promise<PersistedSyncRulesContent[]>;
+  abstract getReplicatingSyncRules(): Promise<PersistedSyncRulesContent[]>;
 
   /**
    * Get all sync rules stopped but not terminated yet.
    */
-  getStoppedSyncRules(): Promise<PersistedSyncRulesContent[]>;
+  abstract getStoppedSyncRules(): Promise<PersistedSyncRulesContent[]>;
 
   /**
    * Get the active storage instance.
    */
-  getActiveStorage(): Promise<SyncRulesBucketStorage | null>;
+  abstract getActiveStorage(): Promise<SyncRulesBucketStorage | null>;
 
   /**
    * Get storage size of active sync rules.
    */
-  getStorageMetrics(): Promise<StorageMetrics>;
+  abstract getStorageMetrics(): Promise<StorageMetrics>;
 
   /**
    * Get the unique identifier for this instance of Powersync
    */
-  getPowerSyncInstanceId(): Promise<string>;
+  abstract getPowerSyncInstanceId(): Promise<string>;
 
   /**
    * Get a unique identifier for the system used for storage.
    */
-  getSystemIdentifier(): Promise<BucketStorageSystemIdentifier>;
+  abstract getSystemIdentifier(): Promise<BucketStorageSystemIdentifier>;
+
+  abstract [Symbol.asyncDispose](): PromiseLike<void>;
 }
 
 export interface BucketStorageFactoryListener {
@@ -119,9 +152,67 @@ export interface StorageMetrics {
 }
 
 export interface UpdateSyncRulesOptions {
-  content: string;
+  config: {
+    yaml: string;
+    /**
+     * The serialized sync plan for the sync configuration, or `null` for configurations not using the sync stream
+     * compiler.
+     */
+    plan: SerializedSyncPlan | null;
+  };
   lock?: boolean;
-  validate?: boolean;
+  storageVersion?: number;
+}
+
+export interface SerializedSyncPlan {
+  /**
+   * The serialized plan, from {@link serializeSyncPlan}.
+   */
+  plan: unknown;
+  compatibility: SerializedCompatibilityContext;
+  /**
+   * Event descriptors are not currently represented in the sync plan because they don't use the sync streams compiler
+   * yet.
+   *
+   * We might revisit that in the future, but for now we store SQL text of their definitions here to be able to restore
+   * them.
+   */
+  eventDescriptors: Record<string, string[]>;
+}
+
+export function updateSyncRulesFromYaml(
+  content: string,
+  options?: Omit<UpdateSyncRulesOptions, 'config'> & { validate?: boolean }
+): UpdateSyncRulesOptions {
+  const { config } = SqlSyncRules.fromYaml(content, {
+    // No schema-based validation at this point
+    schema: undefined,
+    defaultSchema: 'not_applicable', // Not needed for validation
+    throwOnError: options?.validate ?? false
+  });
+
+  return updateSyncRulesFromConfig(config, options);
+}
+
+export function updateSyncRulesFromConfig(
+  parsed: SyncConfig,
+  options?: Omit<UpdateSyncRulesOptions, 'config'>
+): UpdateSyncRulesOptions {
+  let plan: SerializedSyncPlan | null = null;
+  if (parsed instanceof PrecompiledSyncConfig) {
+    const eventDescriptors: Record<string, string[]> = {};
+    for (const event of parsed.eventDescriptors) {
+      eventDescriptors[event.name] = event.sourceQueries.map((q) => q.sql);
+    }
+
+    plan = {
+      compatibility: parsed.compatibility.serialize(),
+      plan: serializeSyncPlan(parsed.plan),
+      eventDescriptors
+    };
+  }
+
+  return { config: { yaml: parsed.content, plan }, ...options };
 }
 
 export interface GetIntanceOptions {
@@ -166,3 +257,9 @@ export interface TestStorageOptions {
 }
 export type TestStorageFactory = (options?: TestStorageOptions) => Promise<BucketStorageFactory>;
 export type TestReportStorageFactory = (options?: TestStorageOptions) => Promise<ReportStorage>;
+
+export interface TestStorageConfig {
+  factory: TestStorageFactory;
+  tableIdStrings: boolean;
+  storageVersion?: number;
+}

package/src/storage/ChecksumCache.ts

@@ -1,7 +1,9 @@
 import { OrderedSet } from '@js-sdsl/ordered-set';
 import { LRUCache } from 'lru-cache/min';
+import { BucketDataSource } from '@powersync/service-sync-rules';
 import { BucketChecksum } from '../util/protocol-types.js';
 import { addBucketChecksums, ChecksumMap, InternalOpId, PartialChecksum } from '../util/utils.js';
+import { BucketChecksumRequest } from './SyncRulesBucketStorage.js';
 
 interface ChecksumFetchContext {
   fetch(bucket: string): Promise<BucketChecksum>;
@@ -10,6 +12,7 @@ interface ChecksumFetchContext {
 
 export interface FetchPartialBucketChecksum {
   bucket: string;
+  source: BucketDataSource;
   start?: InternalOpId;
   end: InternalOpId;
 }
@@ -113,10 +116,10 @@ export class ChecksumCache {
     this.bucketCheckpoints.clear();
   }
 
-  async getChecksums(checkpoint: InternalOpId, buckets: string[]): Promise<BucketChecksum[]> {
+  async getChecksums(checkpoint: InternalOpId, buckets: BucketChecksumRequest[]): Promise<BucketChecksum[]> {
     const checksums = await this.getChecksumMap(checkpoint, buckets);
     // Return results in the same order as the request
-    return buckets.map((bucket) => checksums.get(bucket)!);
+    return buckets.map((bucket) => checksums.get(bucket.bucket)!);
   }
 
   /**
@@ -126,7 +129,7 @@ export class ChecksumCache {
    *
    * @returns a Map with exactly one entry for each bucket requested
    */
-  async getChecksumMap(checkpoint: InternalOpId, buckets: string[]): Promise<ChecksumMap> {
+  async getChecksumMap(checkpoint: InternalOpId, buckets: BucketChecksumRequest[]): Promise<ChecksumMap> {
     // Buckets that don't have a cached checksum for this checkpoint yet
     let toFetch = new Set<string>();
 
@@ -164,19 +167,21 @@ export class ChecksumCache {
     // One promise to await to ensure all fetch requests completed.
     let settledPromise: Promise<PromiseSettledResult<void>[]> | null = null;
 
+    const sourceMap = new Map<string, BucketDataSource>();
+
     try {
       // Individual cache fetch promises
       let cacheFetchPromises: Promise<void>[] = [];
 
       for (let bucket of buckets) {
-        const cacheKey = makeCacheKey(checkpoint, bucket);
+        const cacheKey = makeCacheKey(checkpoint, bucket.bucket);
         let status: LRUCache.Status<BucketChecksum> = {};
         const p = this.cache.fetch(cacheKey, { context: context, status: status }).then((checksums) => {
           if (checksums == null) {
             // Should never happen
             throw new Error(`Failed to get checksums for ${cacheKey}`);
           }
-          finalResults.set(bucket, checksums);
+          finalResults.set(bucket.bucket, checksums);
         });
         cacheFetchPromises.push(p);
         if (status.fetch == 'hit' || status.fetch == 'inflight') {
@@ -185,7 +190,8 @@ export class ChecksumCache {
           // In either case, we don't need to fetch a new checksum.
         } else {
           // We need a new request for this checksum.
-          toFetch.add(bucket);
+          toFetch.add(bucket.bucket);
+          sourceMap.set(bucket.bucket, bucket.source);
         }
       }
       // We do this directly after creating the promises, otherwise
@@ -220,6 +226,7 @@ export class ChecksumCache {
                 // Partial checksum found - make a partial checksum request
                 bucketRequest = {
                   bucket,
+                  source: sourceMap.get(bucket)!,
                   start: cp,
                   end: checkpoint
                 };
@@ -240,6 +247,7 @@ export class ChecksumCache {
             // No partial checksum found - make a new full checksum request
             bucketRequest = {
               bucket,
+              source: sourceMap.get(bucket)!,
               end: checkpoint
             };
             add.set(bucket, {

package/src/storage/PersistedSyncRulesContent.ts

@@ -1,18 +1,36 @@
-import { HydratedSyncRules, SyncConfig, SyncConfigWithErrors } from '@powersync/service-sync-rules';
+import { ErrorCode, ServiceError } from '@powersync/lib-services-framework';
+import {
+  CompatibilityContext,
+  CompatibilityOption,
+  DEFAULT_HYDRATION_STATE,
+  deserializeSyncPlan,
+  HydratedSyncRules,
+  HydrationState,
+  javaScriptExpressionEngine,
+  PrecompiledSyncConfig,
+  SqlEventDescriptor,
+  SqlSyncRules,
+  SyncConfigWithErrors,
+  versionedHydrationState
+} from '@powersync/service-sync-rules';
+import { SerializedSyncPlan, UpdateSyncRulesOptions } from './BucketStorageFactory.js';
 import { ReplicationLock } from './ReplicationLock.js';
+import { STORAGE_VERSION_CONFIG, StorageVersionConfig } from './StorageVersionConfig.js';
 
 export interface ParseSyncRulesOptions {
   defaultSchema: string;
 }
 
-export interface PersistedSyncRulesContent {
+export interface PersistedSyncRulesContentData {
   readonly id: number;
   readonly sync_rules_content: string;
+  readonly compiled_plan: SerializedSyncPlan | null;
   readonly slot_name: string;
   /**
    * True if this is the "active" copy of the sync rules.
    */
   readonly active: boolean;
+  readonly storageVersion: number;
 
   readonly last_checkpoint_lsn: string | null;
 
@@ -20,16 +38,113 @@ export interface PersistedSyncRulesContent {
   readonly last_fatal_error_ts?: Date | null;
   readonly last_keepalive_ts?: Date | null;
   readonly last_checkpoint_ts?: Date | null;
+}
+
+export abstract class PersistedSyncRulesContent implements PersistedSyncRulesContentData {
+  readonly id!: number;
+  readonly sync_rules_content!: string;
+  readonly compiled_plan!: SerializedSyncPlan | null;
+  readonly slot_name!: string;
+  readonly active!: boolean;
+  readonly storageVersion!: number;
+
+  readonly last_checkpoint_lsn!: string | null;
+
+  readonly last_fatal_error?: string | null;
+  readonly last_fatal_error_ts?: Date | null;
+  readonly last_keepalive_ts?: Date | null;
+  readonly last_checkpoint_ts?: Date | null;
+
+  abstract readonly current_lock: ReplicationLock | null;
+
+  constructor(data: PersistedSyncRulesContentData) {
+    Object.assign(this, data);
+  }
+
+  /**
+   * Load the storage config.
+   *
+   * This may throw if the persisted storage version is not supported.
+   */
+  getStorageConfig(): StorageVersionConfig {
+    const storageConfig = STORAGE_VERSION_CONFIG[this.storageVersion];
+    if (storageConfig == null) {
+      throw new ServiceError(
+        ErrorCode.PSYNC_S1005,
+        `Unsupported storage version ${this.storageVersion} for sync rules ${this.id}`
+      );
+    }
+    return storageConfig;
+  }
+
+  parsed(options: ParseSyncRulesOptions): PersistedSyncRules {
+    let hydrationState: HydrationState;
+
+    // Do we have a compiled sync plan? If so, restore from there instead of parsing everything again.
+    let config: SyncConfigWithErrors;
+    if (this.compiled_plan != null) {
+      const plan = deserializeSyncPlan(this.compiled_plan.plan);
+      const compatibility = CompatibilityContext.deserialize(this.compiled_plan.compatibility);
+      const eventDefinitions: SqlEventDescriptor[] = [];
+      for (const [name, queries] of Object.entries(this.compiled_plan.eventDescriptors)) {
+        const descriptor = new SqlEventDescriptor(name, compatibility);
+        for (const query of queries) {
+          descriptor.addSourceQuery(query, options);
+        }
+
+        eventDefinitions.push(descriptor);
+      }
+
+      const precompiled = new PrecompiledSyncConfig(plan, compatibility, eventDefinitions, {
+        defaultSchema: options.defaultSchema,
+        engine: javaScriptExpressionEngine(compatibility),
+        sourceText: this.sync_rules_content
+      });
 
-  parsed(options: ParseSyncRulesOptions): PersistedSyncRules;
+      config = { config: precompiled, errors: [] };
+    } else {
+      config = SqlSyncRules.fromYaml(this.sync_rules_content, options);
+    }
 
-  lock(): Promise<ReplicationLock>;
+    const storageConfig = this.getStorageConfig();
+    if (
+      storageConfig.versionedBuckets ||
+      config.config.compatibility.isEnabled(CompatibilityOption.versionedBucketIds)
+    ) {
+      hydrationState = versionedHydrationState(this.id);
+    } else {
+      hydrationState = DEFAULT_HYDRATION_STATE;
+    }
+
+    return {
+      id: this.id,
+      slot_name: this.slot_name,
+      sync_rules: config,
+      hydrationState,
+      hydratedSyncRules: () => {
+        return config.config.hydrate({ hydrationState });
+      }
+    };
+  }
+
+  asUpdateOptions(options?: Omit<UpdateSyncRulesOptions, 'config'>): UpdateSyncRulesOptions {
+    return {
+      config: { yaml: this.sync_rules_content, plan: this.compiled_plan },
+      ...options
+    };
+  }
+
+  abstract lock(): Promise<ReplicationLock>;
 }
 
 export interface PersistedSyncRules {
   readonly id: number;
   readonly sync_rules: SyncConfigWithErrors;
   readonly slot_name: string;
+  /**
+   * For testing only.
+   */
+  readonly hydrationState: HydrationState;
 
   hydratedSyncRules(): HydratedSyncRules;
 }

package/src/storage/SourceTable.ts

@@ -1,9 +1,15 @@
 import { DEFAULT_TAG } from '@powersync/service-sync-rules';
 import * as util from '../util/util-index.js';
 import { ColumnDescriptor, SourceEntityDescriptor } from './SourceEntity.js';
+import { bson } from '../index.js';
+
+/**
+ * Format of the id depends on the bucket storage module. It should be consistent within the module.
+ */
+export type SourceTableId = string | bson.ObjectId;
 
 export interface SourceTableOptions {
-  id: any;
+  id: SourceTableId;
   connectionTag: string;
   objectId: number | string | undefined;
   schema: string;

package/src/storage/StorageVersionConfig.ts

@@ -0,0 +1,78 @@
+export interface StorageVersionConfig {
+  version: number;
+
+  /**
+   * Whether versioned bucket names are automatically enabled.
+   *
+   * If this is false, bucket names may still be versioned depending on the sync config.
+   *
+   * Introduced in STORAGE_VERSION_2.
+   */
+  versionedBuckets: boolean;
+
+  /**
+   * Whether to use soft deletes for current_data, improving replication concurrency support.
+   *
+   * Introduced in STORAGE_VERSION_3.
+   */
+  softDeleteCurrentData: boolean;
+}
+
+/**
+ * Corresponds to the storage version initially used, before we started explicitly versioning storage.
+ */
+export const STORAGE_VERSION_1 = 1;
+
+/**
+ * First new storage version.
+ *
+ * Uses versioned bucket names.
+ *
+ * On MongoDB storage, this always uses Long for checksums.
+ */
+export const STORAGE_VERSION_2 = 2;
+
+/**
+ * This version is currently unstable, and not enabled by default yet.
+ *
+ * This is used to build towards incremental reprocessing.
+ */
+export const STORAGE_VERSION_3 = 3;
+
+/**
+ * Oldest supported storage version.
+ */
+export const LEGACY_STORAGE_VERSION = STORAGE_VERSION_1;
+
+/**
+ * Default storage version for newly persisted sync rules.
+ */
+export const CURRENT_STORAGE_VERSION = STORAGE_VERSION_2;
+
+/**
+ * All versions that can be loaded.
+ *
+ * This includes unstable versions.
+ */
+export const SUPPORTED_STORAGE_VERSIONS = [STORAGE_VERSION_1, STORAGE_VERSION_2, STORAGE_VERSION_3];
+
+/**
+ * Shared storage-version behavior across storage implementations.
+ */
+export const STORAGE_VERSION_CONFIG: Record<number, StorageVersionConfig | undefined> = {
+  [STORAGE_VERSION_1]: {
+    version: STORAGE_VERSION_1,
+    versionedBuckets: false,
+    softDeleteCurrentData: false
+  },
+  [STORAGE_VERSION_2]: {
+    version: STORAGE_VERSION_2,
+    versionedBuckets: true,
+    softDeleteCurrentData: false
+  },
+  [STORAGE_VERSION_3]: {
+    version: STORAGE_VERSION_3,
+    versionedBuckets: true,
+    softDeleteCurrentData: true
+  }
+};

package/src/storage/SyncRulesBucketStorage.ts

@@ -1,5 +1,10 @@
 import { Logger, ObserverClient } from '@powersync/lib-services-framework';
-import { HydratedSyncRules, ScopedParameterLookup, SqliteJsonRow } from '@powersync/service-sync-rules';
+import {
+  BucketDataSource,
+  HydratedSyncRules,
+  ScopedParameterLookup,
+  SqliteJsonRow
+} from '@powersync/service-sync-rules';
 import * as util from '../util/util-index.js';
 import { BucketStorageBatch, FlushedResult, SaveUpdate } from './BucketStorageBatch.js';
 import { BucketStorageFactory } from './BucketStorageFactory.js';
@@ -103,7 +108,7 @@ export interface SyncRulesBucketStorage
    */
   getBucketDataBatch(
     checkpoint: util.InternalOpId,
-    dataBuckets: Map<string, util.InternalOpId>,
+    dataBuckets: BucketDataRequest[],
     options?: BucketDataBatchOptions
   ): AsyncIterable<SyncBucketDataChunk>;
 
@@ -115,7 +120,7 @@ export interface SyncRulesBucketStorage
    * This may be slow, depending on the size of the buckets.
    * The checksums are cached internally to compensate for this, but does not cover all cases.
    */
-  getChecksums(checkpoint: util.InternalOpId, buckets: string[]): Promise<util.ChecksumMap>;
+  getChecksums(checkpoint: util.InternalOpId, buckets: BucketChecksumRequest[]): Promise<util.ChecksumMap>;
 
   /**
    * Clear checksum cache. Primarily intended for tests.
@@ -127,6 +132,16 @@ export interface SyncRulesBucketStorageListener {
   batchStarted: (batch: BucketStorageBatch) => void;
 }
 
+export interface BucketDataRequest {
+  bucket: string;
+  start: util.InternalOpId;
+  source: BucketDataSource;
+}
+export interface BucketChecksumRequest {
+  bucket: string;
+  source: BucketDataSource;
+}
+
 export interface SyncRuleStatus {
   checkpoint_lsn: string | null;
   active: boolean;
@@ -202,7 +217,8 @@ export interface CompactOptions {
    *
    * If not specified, compacts all buckets.
    *
-   * These can be individual bucket names, or bucket definition names.
+   * These must be full bucket names (e.g., "global[]", "mybucket[\"user1\"]").
+   * Bucket definition names (e.g., "global") are not supported.
    */
   compactBuckets?: string[];
 

package/src/storage/storage-index.ts

@@ -14,3 +14,4 @@ export * from './SyncRulesBucketStorage.js';
 export * from './PersistedSyncRulesContent.js';
 export * from './ReplicationLock.js';
 export * from './ReportStorage.js';
+export * from './StorageVersionConfig.js';