@powersync/service-core 0.0.0-dev-20260225160713 → 0.0.0-dev-20260511080634

package/src/storage/BucketStorageFactory.ts

@@ -1,31 +1,33 @@
 import { BaseObserver, logger } from '@powersync/lib-services-framework';
-import { ParseSyncRulesOptions, PersistedSyncRules, PersistedSyncRulesContent } from './PersistedSyncRulesContent.js';
-import { ReplicationEventPayload } from './ReplicationEventPayload.js';
-import { ReplicationLock } from './ReplicationLock.js';
-import { SyncRulesBucketStorage } from './SyncRulesBucketStorage.js';
-import { ReportStorage } from './ReportStorage.js';
 import {
   PrecompiledSyncConfig,
   SerializedCompatibilityContext,
   serializeSyncPlan,
   SqlSyncRules,
-  SyncConfig
+  SyncConfigWithErrors
 } from '@powersync/service-sync-rules';
+import { ReplicationError } from '@powersync/service-types';
+import { syncConfigYamlErrorToReplicationError } from '../util/errors.js';
+import { ParseSyncRulesOptions, PersistedSyncRules, PersistedSyncRulesContent } from './PersistedSyncRulesContent.js';
+import { ReplicationEventPayload } from './ReplicationEventPayload.js';
+import { ReplicationLock } from './ReplicationLock.js';
+import { ReportStorage } from './ReportStorage.js';
+import { SyncRulesBucketStorage } from './SyncRulesBucketStorage.js';
 
 /**
  * Represents a configured storage provider.
  *
- * The provider can handle multiple copies of sync rules concurrently, each with their own storage.
- * This is to handle replication of a new version of sync rules, while the old version is still active.
+ * The provider can handle multiple replication streams concurrently, each with their own storage.
+ * This is to handle replication of a new version of sync config, while the old replication stream is still active.
  *
- * Storage APIs for a specific copy of sync rules are provided by the `SyncRulesBucketStorage` instances.
+ * Storage APIs for a specific replication stream are provided by the `SyncRulesBucketStorage` instances.
  */
 export abstract class BucketStorageFactory
   extends BaseObserver<BucketStorageFactoryListener>
   implements AsyncDisposable
 {
   /**
-   * Update sync rules from configuration, if changed.
+   * Update sync config from configuration, if changed.
    */
   async configureSyncRules(
     options: UpdateSyncRulesOptions
@@ -34,42 +36,42 @@ export abstract class BucketStorageFactory
     const active = await this.getActiveSyncRulesContent();
 
     if (next?.sync_rules_content == options.config.yaml) {
-      logger.info('Sync rules from configuration unchanged');
+      logger.info('Sync config unchanged');
       return { updated: false };
     } else if (next == null && active?.sync_rules_content == options.config.yaml) {
-      logger.info('Sync rules from configuration unchanged');
+      logger.info('Sync config unchanged');
       return { updated: false };
     } else {
-      logger.info('Sync rules updated from configuration');
+      logger.info('Sync config updated');
       const persisted_sync_rules = await this.updateSyncRules(options);
       return { updated: true, persisted_sync_rules, lock: persisted_sync_rules.current_lock ?? undefined };
     }
   }
 
   /**
-   * Get a storage instance to query sync data for specific sync rules.
+   * Get a storage instance to query sync data for specific sync config.
    */
   abstract getInstance(syncRules: PersistedSyncRulesContent, options?: GetIntanceOptions): SyncRulesBucketStorage;
 
   /**
-   * Deploy new sync rules.
+   * Deploy new sync config.
    */
   abstract updateSyncRules(options: UpdateSyncRulesOptions): Promise<PersistedSyncRulesContent>;
 
   /**
    * Indicate that a slot was removed, and we should re-sync by creating
-   * a new sync rules instance.
+   * a new replication stream.
    *
    * This is roughly the same as deploying a new version of the current sync
-   * rules, but also accounts for cases where the current sync rules are not
-   * the latest ones.
+   * config, but also accounts for cases where the current sync config is not
+   * the latest one.
    *
    * Replication should be restarted after this.
    */
   abstract restartReplication(sync_rules_group_id: number): Promise<void>;
 
   /**
-   * Get the sync rules used for querying.
+   * Get the sync config used for querying.
    */
   async getActiveSyncRules(options: ParseSyncRulesOptions): Promise<PersistedSyncRules | null> {
     const content = await this.getActiveSyncRulesContent();
@@ -77,12 +79,12 @@ export abstract class BucketStorageFactory
   }
 
   /**
-   * Get the sync rules used for querying.
+   * Get the sync config used for querying.
    */
   abstract getActiveSyncRulesContent(): Promise<PersistedSyncRulesContent | null>;
 
   /**
-   * Get the sync rules that will be active next once done with initial replicatino.
+   * Get the sync config that will be active next once done with initial replicatino.
    */
   async getNextSyncRules(options: ParseSyncRulesOptions): Promise<PersistedSyncRules | null> {
     const content = await this.getNextSyncRulesContent();
@@ -90,17 +92,17 @@ export abstract class BucketStorageFactory
   }
 
   /**
-   * Get the sync rules that will be active next once done with initial replicatino.
+   * Get the sync config that will be active next once done with initial replicatino.
    */
   abstract getNextSyncRulesContent(): Promise<PersistedSyncRulesContent | null>;
 
   /**
-   * Get all sync rules currently replicating. Typically this is the "active" and "next" sync rules.
+   * Get all sync config currently replicating. Typically this is the "active" and "next" sync config.
    */
   abstract getReplicatingSyncRules(): Promise<PersistedSyncRulesContent[]>;
 
   /**
-   * Get all sync rules stopped but not terminated yet.
+   * Get all sync config stopped but not terminated yet.
    */
   abstract getStoppedSyncRules(): Promise<PersistedSyncRulesContent[]>;
 
@@ -110,12 +112,12 @@ export abstract class BucketStorageFactory
   abstract getActiveStorage(): Promise<SyncRulesBucketStorage | null>;
 
   /**
-   * Get storage size of active sync rules.
+   * Get storage size of active replication stream.
    */
   abstract getStorageMetrics(): Promise<StorageMetrics>;
 
   /**
-   * Get the unique identifier for this instance of Powersync
+   * Get the unique identifier for this instance of Powersync.
    */
   abstract getPowerSyncInstanceId(): Promise<string>;
 
@@ -159,6 +161,12 @@ export interface UpdateSyncRulesOptions {
      * compiler.
      */
     plan: SerializedSyncPlan | null;
+
+    /**
+     * Parsed sync config, primarily to generate a definition mapping.
+     * Not persisted, and the defaultSchema used for parsing is not relevant.
+     */
+    parsed: SyncConfigWithErrors;
   };
   lock?: boolean;
   storageVersion?: number;
@@ -178,13 +186,14 @@ export interface SerializedSyncPlan {
    * them.
    */
   eventDescriptors: Record<string, string[]>;
+  errors?: ReplicationError[];
 }
 
 export function updateSyncRulesFromYaml(
   content: string,
   options?: Omit<UpdateSyncRulesOptions, 'config'> & { validate?: boolean }
 ): UpdateSyncRulesOptions {
-  const { config } = SqlSyncRules.fromYaml(content, {
+  const config = SqlSyncRules.fromYaml(content, {
     // No schema-based validation at this point
     schema: undefined,
     defaultSchema: 'not_applicable', // Not needed for validation
@@ -195,24 +204,26 @@ export function updateSyncRulesFromYaml(
 }
 
 export function updateSyncRulesFromConfig(
-  parsed: SyncConfig,
+  parsed: SyncConfigWithErrors,
   options?: Omit<UpdateSyncRulesOptions, 'config'>
 ): UpdateSyncRulesOptions {
   let plan: SerializedSyncPlan | null = null;
-  if (parsed instanceof PrecompiledSyncConfig) {
+  const { config, errors } = parsed;
+  if (config instanceof PrecompiledSyncConfig) {
     const eventDescriptors: Record<string, string[]> = {};
-    for (const event of parsed.eventDescriptors) {
+    for (const event of config.eventDescriptors) {
       eventDescriptors[event.name] = event.sourceQueries.map((q) => q.sql);
     }
 
     plan = {
-      compatibility: parsed.compatibility.serialize(),
-      plan: serializeSyncPlan(parsed.plan),
-      eventDescriptors
+      compatibility: config.compatibility.serialize(),
+      plan: serializeSyncPlan(config.plan),
+      eventDescriptors,
+      errors: errors.map((e) => syncConfigYamlErrorToReplicationError(e))
     };
   }
 
-  return { config: { yaml: parsed.content, plan }, ...options };
+  return { config: { yaml: config.content, plan, parsed }, ...options };
 }
 
 export interface GetIntanceOptions {
@@ -257,3 +268,9 @@ export interface TestStorageOptions {
 }
 export type TestStorageFactory = (options?: TestStorageOptions) => Promise<BucketStorageFactory>;
 export type TestReportStorageFactory = (options?: TestStorageOptions) => Promise<ReportStorage>;
+
+export interface TestStorageConfig {
+  factory: TestStorageFactory;
+  tableIdStrings: boolean;
+  storageVersion?: number;
+}

package/src/storage/ChecksumCache.ts

@@ -1,7 +1,9 @@
 import { OrderedSet } from '@js-sdsl/ordered-set';
+import { BucketDataSource } from '@powersync/service-sync-rules';
 import { LRUCache } from 'lru-cache/min';
 import { BucketChecksum } from '../util/protocol-types.js';
 import { addBucketChecksums, ChecksumMap, InternalOpId, PartialChecksum } from '../util/utils.js';
+import { BucketChecksumRequest } from './SyncRulesBucketStorage.js';
 
 interface ChecksumFetchContext {
   fetch(bucket: string): Promise<BucketChecksum>;
@@ -10,6 +12,7 @@ interface ChecksumFetchContext {
 
 export interface FetchPartialBucketChecksum {
   bucket: string;
+  source: BucketDataSource;
   start?: InternalOpId;
   end: InternalOpId;
 }
@@ -113,10 +116,10 @@ export class ChecksumCache {
     this.bucketCheckpoints.clear();
   }
 
-  async getChecksums(checkpoint: InternalOpId, buckets: string[]): Promise<BucketChecksum[]> {
+  async getChecksums(checkpoint: InternalOpId, buckets: BucketChecksumRequest[]): Promise<BucketChecksum[]> {
     const checksums = await this.getChecksumMap(checkpoint, buckets);
     // Return results in the same order as the request
-    return buckets.map((bucket) => checksums.get(bucket)!);
+    return buckets.map((bucket) => checksums.get(bucket.bucket)!);
   }
 
   /**
@@ -126,7 +129,7 @@ export class ChecksumCache {
    *
    * @returns a Map with exactly one entry for each bucket requested
    */
-  async getChecksumMap(checkpoint: InternalOpId, buckets: string[]): Promise<ChecksumMap> {
+  async getChecksumMap(checkpoint: InternalOpId, buckets: BucketChecksumRequest[]): Promise<ChecksumMap> {
     // Buckets that don't have a cached checksum for this checkpoint yet
     let toFetch = new Set<string>();
 
@@ -164,19 +167,21 @@ export class ChecksumCache {
     // One promise to await to ensure all fetch requests completed.
     let settledPromise: Promise<PromiseSettledResult<void>[]> | null = null;
 
+    const sourceMap = new Map<string, BucketDataSource>();
+
     try {
       // Individual cache fetch promises
       let cacheFetchPromises: Promise<void>[] = [];
 
       for (let bucket of buckets) {
-        const cacheKey = makeCacheKey(checkpoint, bucket);
+        const cacheKey = makeCacheKey(checkpoint, bucket.bucket);
         let status: LRUCache.Status<BucketChecksum> = {};
         const p = this.cache.fetch(cacheKey, { context: context, status: status }).then((checksums) => {
           if (checksums == null) {
             // Should never happen
             throw new Error(`Failed to get checksums for ${cacheKey}`);
           }
-          finalResults.set(bucket, checksums);
+          finalResults.set(bucket.bucket, checksums);
         });
         cacheFetchPromises.push(p);
         if (status.fetch == 'hit' || status.fetch == 'inflight') {
@@ -185,7 +190,8 @@ export class ChecksumCache {
           // In either case, we don't need to fetch a new checksum.
         } else {
           // We need a new request for this checksum.
-          toFetch.add(bucket);
+          toFetch.add(bucket.bucket);
+          sourceMap.set(bucket.bucket, bucket.source);
         }
       }
       // We do this directly after creating the promises, otherwise
@@ -220,6 +226,7 @@ export class ChecksumCache {
                 // Partial checksum found - make a partial checksum request
                 bucketRequest = {
                   bucket,
+                  source: sourceMap.get(bucket)!,
                   start: cp,
                   end: checkpoint
                 };
@@ -240,6 +247,7 @@ export class ChecksumCache {
             // No partial checksum found - make a new full checksum request
             bucketRequest = {
               bucket,
+              source: sourceMap.get(bucket)!,
               end: checkpoint
             };
             add.set(bucket, {

package/src/storage/PersistedSyncRulesContent.ts

@@ -1,8 +1,10 @@
+import { logger as defaultLogger, ErrorCode, Logger, ServiceError } from '@powersync/lib-services-framework';
 import {
   CompatibilityContext,
   CompatibilityOption,
   DEFAULT_HYDRATION_STATE,
   deserializeSyncPlan,
+  ErrorLocation,
   HydratedSyncRules,
   HydrationState,
   javaScriptExpressionEngine,
@@ -10,12 +12,12 @@ import {
   SqlEventDescriptor,
   SqlSyncRules,
   SyncConfigWithErrors,
-  versionedHydrationState
+  versionedHydrationState,
+  YamlError
 } from '@powersync/service-sync-rules';
+import { SerializedSyncPlan, UpdateSyncRulesOptions } from './BucketStorageFactory.js';
 import { ReplicationLock } from './ReplicationLock.js';
 import { STORAGE_VERSION_CONFIG, StorageVersionConfig } from './StorageVersionConfig.js';
-import { ErrorCode, ServiceError } from '@powersync/lib-services-framework';
-import { SerializedSyncPlan, UpdateSyncRulesOptions } from './BucketStorageFactory.js';
 
 export interface ParseSyncRulesOptions {
   defaultSchema: string;
@@ -27,7 +29,7 @@ export interface PersistedSyncRulesContentData {
   readonly compiled_plan: SerializedSyncPlan | null;
   readonly slot_name: string;
   /**
-   * True if this is the "active" copy of the sync rules.
+   * True if this is the "active" copy of the sync config.
    */
   readonly active: boolean;
   readonly storageVersion: number;
@@ -47,6 +49,7 @@ export abstract class PersistedSyncRulesContent implements PersistedSyncRulesCon
   readonly slot_name!: string;
   readonly active!: boolean;
   readonly storageVersion!: number;
+  readonly logger: Logger;
 
   readonly last_checkpoint_lsn!: string | null;
 
@@ -59,6 +62,7 @@ export abstract class PersistedSyncRulesContent implements PersistedSyncRulesCon
 
   constructor(data: PersistedSyncRulesContentData) {
     Object.assign(this, data);
+    this.logger = defaultLogger.child({ prefix: `[${this.slot_name}] ` });
   }
 
   /**
@@ -71,7 +75,7 @@ export abstract class PersistedSyncRulesContent implements PersistedSyncRulesCon
     if (storageConfig == null) {
       throw new ServiceError(
         ErrorCode.PSYNC_S1005,
-        `Unsupported storage version ${this.storageVersion} for sync rules ${this.id}`
+        `Unsupported storage version ${this.storageVersion} for replication stream ${this.id}`
       );
     }
     return storageConfig;
@@ -101,7 +105,25 @@ export abstract class PersistedSyncRulesContent implements PersistedSyncRulesCon
         sourceText: this.sync_rules_content
       });
 
-      config = { config: precompiled, errors: [] };
+      // Note: If the original content did not define a storage version, this will still set the storage version.
+      // This means asUpdateOptions will not change the storage version, even if the default changes.
+      precompiled.storageVersion = this.storageVersion;
+
+      const errors: YamlError[] = [];
+      if (this.compiled_plan.errors) {
+        for (const error of this.compiled_plan.errors) {
+          const location: ErrorLocation | undefined = error.location && {
+            start: error.location.start_offset,
+            end: error.location.end_offset
+          };
+          const asYamlError = new YamlError(new Error(error.message), location);
+          asYamlError.type = error.level;
+
+          errors.push(asYamlError);
+        }
+      }
+
+      config = { config: precompiled, errors };
     } else {
       config = SqlSyncRules.fromYaml(this.sync_rules_content, options);
     }
@@ -120,6 +142,7 @@ export abstract class PersistedSyncRulesContent implements PersistedSyncRulesCon
       id: this.id,
       slot_name: this.slot_name,
       sync_rules: config,
+      hydrationState,
       hydratedSyncRules: () => {
         return config.config.hydrate({ hydrationState });
       }
@@ -127,8 +150,10 @@ export abstract class PersistedSyncRulesContent implements PersistedSyncRulesCon
   }
 
   asUpdateOptions(options?: Omit<UpdateSyncRulesOptions, 'config'>): UpdateSyncRulesOptions {
+    // defaultSchema is not relevant for the parsed version here
+    const parsed = this.parsed({ defaultSchema: 'not_applicable' });
     return {
-      config: { yaml: this.sync_rules_content, plan: this.compiled_plan },
+      config: { yaml: this.sync_rules_content, plan: this.compiled_plan, parsed: parsed.sync_rules },
       ...options
     };
   }
@@ -140,6 +165,10 @@ export interface PersistedSyncRules {
   readonly id: number;
   readonly sync_rules: SyncConfigWithErrors;
   readonly slot_name: string;
+  /**
+   * For testing only.
+   */
+  readonly hydrationState: HydrationState;
 
   hydratedSyncRules(): HydratedSyncRules;
 }

package/src/storage/ReplicationEventPayload.ts

@@ -1,6 +1,6 @@
 import * as sync_rules from '@powersync/service-sync-rules';
-import { SourceTable } from './SourceTable.js';
 import { BucketStorageBatch, SaveOp } from './BucketStorageBatch.js';
+import { SourceTable } from './SourceTable.js';
 
 export type EventData = {
   op: SaveOp;

package/src/storage/SourceTable.ts

@@ -1,9 +1,15 @@
 import { DEFAULT_TAG } from '@powersync/service-sync-rules';
+import { bson } from '../index.js';
 import * as util from '../util/util-index.js';
 import { ColumnDescriptor, SourceEntityDescriptor } from './SourceEntity.js';
 
+/**
+ * Format of the id depends on the bucket storage module. It should be consistent within the module.
+ */
+export type SourceTableId = string | bson.ObjectId;
+
 export interface SourceTableOptions {
-  id: any;
+  id: SourceTableId;
   connectionTag: string;
   objectId: number | string | undefined;
   schema: string;
@@ -22,7 +28,7 @@ export class SourceTable implements SourceEntityDescriptor {
   static readonly DEFAULT_TAG = DEFAULT_TAG;
 
   /**
-   * True if the table is used in sync rules for data queries.
+   * True if the table is used in sync config for data queries.
    *
    * This value is resolved externally, and cached here.
    *
@@ -31,7 +37,7 @@ export class SourceTable implements SourceEntityDescriptor {
   public syncData = true;
 
   /**
-   * True if the table is used in sync rules for data queries.
+   * True if the table is used in sync config for data queries.
    *
    * This value is resolved externally, and cached here.
    *
@@ -40,7 +46,7 @@ export class SourceTable implements SourceEntityDescriptor {
   public syncParameters = true;
 
   /**
-   * True if the table is used in sync rules for events.
+   * True if the table is used in sync config for events.
    *
    * This value is resolved externally, and cached here.
    *

package/src/storage/StorageVersionConfig.ts

@@ -1,30 +1,78 @@
 export interface StorageVersionConfig {
+  version: number;
+
   /**
    * Whether versioned bucket names are automatically enabled.
    *
    * If this is false, bucket names may still be versioned depending on the sync config.
+   *
+   * Introduced in STORAGE_VERSION_2.
    */
   versionedBuckets: boolean;
+
+  /**
+   * Whether to use soft deletes for current_data, improving replication concurrency support.
+   *
+   * Introduced in STORAGE_VERSION_3.
+   */
+  softDeleteCurrentData: boolean;
 }
 
+/**
+ * Corresponds to the storage version initially used, before we started explicitly versioning storage.
+ */
+export const STORAGE_VERSION_1 = 1;
+
+/**
+ * First new storage version.
+ *
+ * Uses versioned bucket names.
+ *
+ * On MongoDB storage, this always uses Long for checksums.
+ */
+export const STORAGE_VERSION_2 = 2;
+
+/**
+ * This version is currently unstable, and not enabled by default yet.
+ *
+ * This is used to build towards incremental reprocessing.
+ */
+export const STORAGE_VERSION_3 = 3;
+
 /**
  * Oldest supported storage version.
  */
-export const LEGACY_STORAGE_VERSION = 1;
+export const LEGACY_STORAGE_VERSION = STORAGE_VERSION_1;
 
 /**
- * Default storage version for newly persisted sync rules.
+ * Default storage version for newly persisted replication streams.
  */
-export const CURRENT_STORAGE_VERSION = 2;
+export const CURRENT_STORAGE_VERSION = STORAGE_VERSION_2;
+
+/**
+ * All versions that can be loaded.
+ *
+ * This includes unstable versions.
+ */
+export const SUPPORTED_STORAGE_VERSIONS = [STORAGE_VERSION_1, STORAGE_VERSION_2, STORAGE_VERSION_3];
 
 /**
  * Shared storage-version behavior across storage implementations.
  */
 export const STORAGE_VERSION_CONFIG: Record<number, StorageVersionConfig | undefined> = {
-  [LEGACY_STORAGE_VERSION]: {
-    versionedBuckets: false
+  [STORAGE_VERSION_1]: {
+    version: STORAGE_VERSION_1,
+    versionedBuckets: false,
+    softDeleteCurrentData: false
+  },
+  [STORAGE_VERSION_2]: {
+    version: STORAGE_VERSION_2,
+    versionedBuckets: true,
+    softDeleteCurrentData: false
   },
-  [CURRENT_STORAGE_VERSION]: {
-    versionedBuckets: true
+  [STORAGE_VERSION_3]: {
+    version: STORAGE_VERSION_3,
+    versionedBuckets: true,
+    softDeleteCurrentData: true
   }
 };