@powersync/service-core-tests 0.16.0 → 0.17.1

package/src/test-utils/AbstractStreamTestContext.ts

@@ -2,6 +2,7 @@ import { ReplicationAbortedError } from '@powersync/lib-services-framework';
 import {
   BucketStorageFactory,
   InternalOpId,
+  ReplicationCheckpoint,
   settledPromise,
   storage,
   SyncRulesBucketStorage,
@@ -14,7 +15,8 @@ import { fromAsync } from './stream_utils.js';
 
 export abstract class AbstractStreamTestContext implements AsyncDisposable {
   protected abortController = new AbortController();
-  protected syncRulesContent?: storage.PersistedSyncRulesContent;
+  protected syncRulesContent?: storage.PersistedSyncConfigContent;
+  protected replicationStream?: storage.PersistedReplicationStream;
   public storage?: SyncRulesBucketStorage;
   protected settledReplicationPromise?: Promise<PromiseSettledResult<void>>;
 
@@ -44,39 +46,42 @@ export abstract class AbstractStreamTestContext implements AsyncDisposable {
   }
 
   async updateSyncRules(content: string) {
-    const syncRules = await this.factory.updateSyncRules(
+    const stream = await this.factory.updateSyncRules(
       updateSyncRulesFromYaml(content, { validate: true, storageVersion: this.storageVersion })
     );
-    this.syncRulesContent = syncRules;
-    this.storage = this.factory.getInstance(syncRules);
+    this.replicationStream = stream;
+    this.syncRulesContent = stream.syncConfigContent[0];
+    this.storage = this.factory.getInstance(stream);
     return this.storage!;
   }
 
   async loadNextSyncRules() {
-    const syncRules = await this.factory.getNextSyncRulesContent();
-    if (syncRules == null) {
-      throw new Error(`Next sync rules not available`);
+    const syncConfig = await this.factory.getDeployingSyncConfig();
+    if (syncConfig == null) {
+      throw new Error(`Next sync config not available`);
     }
 
-    this.syncRulesContent = syncRules;
-    this.storage = this.factory.getInstance(syncRules);
+    this.syncRulesContent = syncConfig.content;
+    this.replicationStream = syncConfig.replicationStream;
+    this.storage = syncConfig.storage;
     return this.storage!;
   }
 
   async loadActiveSyncRules() {
-    const syncRules = await this.factory.getActiveSyncRulesContent();
-    if (syncRules == null) {
-      throw new Error(`Active sync rules not available`);
+    const syncConfig = await this.factory.getActiveSyncConfig();
+    if (syncConfig == null) {
+      throw new Error(`Active sync config not available`);
     }
 
-    this.syncRulesContent = syncRules;
-    this.storage = this.factory.getInstance(syncRules);
+    this.syncRulesContent = syncConfig.content;
+    this.replicationStream = syncConfig.replicationStream;
+    this.storage = syncConfig.storage;
     return this.storage!;
   }
 
-  private getSyncRulesContent(): storage.PersistedSyncRulesContent {
+  private getSyncConfigContent(): storage.PersistedSyncConfigContent {
     if (this.syncRulesContent == null) {
-      throw new Error('Sync rules not configured - call updateSyncRules() first');
+      throw new Error('Sync config not configured - call updateSyncRules() first');
     }
     return this.syncRulesContent;
   }
@@ -112,7 +117,7 @@ export abstract class AbstractStreamTestContext implements AsyncDisposable {
     }
   }
 
-  abstract getClientCheckpoint(options?: { timeout?: number }): Promise<bigint>;
+  abstract getClientCheckpoint(options?: { timeout?: number }): Promise<ReplicationCheckpoint>;
 
   async getCheckpoint(options?: { timeout?: number }) {
     let checkpoint = await Promise.race([
@@ -128,7 +133,7 @@ export abstract class AbstractStreamTestContext implements AsyncDisposable {
   }
 
   async getBucketsDataBatch(buckets: Record<string, InternalOpId>, options?: { timeout?: number }) {
-    const helpers = new StorageDataHelpers(this.storage!, this.getSyncRulesContent());
+    const helpers = new StorageDataHelpers(this.storage!, this.getSyncConfigContent());
     const checkpoint = await this.getCheckpoint(options);
     return helpers.getBucketsDataBatch(buckets, checkpoint);
   }
@@ -137,15 +142,15 @@ export abstract class AbstractStreamTestContext implements AsyncDisposable {
    * This waits for a client checkpoint.
    */
   async getBucketData(bucket: string, start?: InternalOpId | string | undefined, options?: { timeout?: number }) {
-    const helpers = new StorageDataHelpers(this.storage!, this.getSyncRulesContent());
+    const helpers = new StorageDataHelpers(this.storage!, this.getSyncConfigContent());
     const checkpoint = await this.getCheckpoint(options);
     return helpers.getBucketData(bucket, checkpoint, start);
   }
 
   async getChecksums(buckets: string[], options?: { timeout?: number }) {
     const checkpoint = await this.getCheckpoint(options);
-    const syncRules = this.getSyncRulesContent();
-    const versionedBuckets = buckets.map((bucket) => bucketRequest(syncRules, bucket, 0n));
+    const syncConfigContent = this.getSyncConfigContent();
+    const versionedBuckets = buckets.map((bucket) => bucketRequest(syncConfigContent, bucket, 0n));
     const checksums = await this.storage!.getChecksums(checkpoint, versionedBuckets);
 
     const unversioned = new Map();
@@ -169,9 +174,9 @@ export abstract class AbstractStreamTestContext implements AsyncDisposable {
     if (typeof start == 'string') {
       start = BigInt(start);
     }
-    const syncRules = this.getSyncRulesContent();
-    const { checkpoint } = await this.storage!.getCheckpoint();
-    const map = [bucketRequest(syncRules, bucket, start)];
+    const syncConfigContent = this.getSyncConfigContent();
+    const checkpoint = await this.storage!.getCheckpoint();
+    const map = [bucketRequest(syncConfigContent, bucket, start)];
     const batch = this.storage!.getBucketDataBatch(checkpoint, map);
     const batches = await fromAsync(batch);
     return batches[0]?.chunkData.data ?? [];

package/src/test-utils/StorageDataHelpers.ts

@@ -1,8 +1,9 @@
 import {
   InternalOpId,
   OplogEntry,
-  PersistedSyncRules,
-  PersistedSyncRulesContent,
+  ParsedSyncConfigSet,
+  PersistedSyncConfigContent,
+  ReplicationCheckpoint,
   SyncRulesBucketStorage
 } from '@powersync/service-core';
 import { bucketRequest } from './general-utils.js';
@@ -10,14 +11,14 @@ import { fromAsync } from './stream_utils.js';
 
 export class StorageDataHelpers {
   storage: SyncRulesBucketStorage;
-  syncRules: PersistedSyncRulesContent | PersistedSyncRules;
+  syncRules: PersistedSyncConfigContent | ParsedSyncConfigSet;
 
-  constructor(storage: SyncRulesBucketStorage, syncRules: PersistedSyncRulesContent | PersistedSyncRules) {
+  constructor(storage: SyncRulesBucketStorage, syncRules: PersistedSyncConfigContent | ParsedSyncConfigSet) {
     this.storage = storage;
     this.syncRules = syncRules;
   }
 
-  async getBucketData(bucket: string, checkpoint: InternalOpId, start?: InternalOpId | string | undefined) {
+  async getBucketData(bucket: string, checkpoint: ReplicationCheckpoint, start?: InternalOpId | string | undefined) {
     start ??= 0n;
     if (typeof start == 'string') {
       start = BigInt(start);
@@ -37,7 +38,7 @@ export class StorageDataHelpers {
     return data;
   }
 
-  async getBucketsDataBatch(buckets: Record<string, InternalOpId>, checkpoint: InternalOpId) {
+  async getBucketsDataBatch(buckets: Record<string, InternalOpId>, checkpoint: ReplicationCheckpoint) {
     const map = Object.entries(buckets).map(([bucket, start]) => bucketRequest(this.syncRules, bucket, start));
     return fromAsync(this.storage!.getBucketDataBatch(checkpoint, map));
   }

package/src/test-utils/bucket-validation.ts

@@ -9,7 +9,7 @@ import { expect } from 'vitest';
  * All other operations are replaced with a single CLEAR operation,
  * summing their checksums, and using a 0 as an op_id.
  *
- * This is the function $r(B)$, as described in /docs/bucket-properties.md.
+ * This is the function $r(B)$, as described in /docs/storage/bucket-properties.md.
  */
 export function reduceBucket(operations: utils.OplogEntry[]) {
   let rowState = new Map<string, utils.OplogEntry>();
@@ -63,7 +63,7 @@ import { reduceBucket } from '@/util/utils.js';
 import { expect } from 'vitest';
 
 /**
- * Validate this property, as described in /docs/bucket-properties.md:
+ * Validate this property, as described in /docs/storage/bucket-properties.md:
  *
  * $r(B_{[..id_n]}) = r(r(B_{[..id_i]}) \cup B_{[id_{i+1}..id_n]}) \;\forall\; i \in [1..n]$
  *
@@ -87,7 +87,7 @@ export function validateBucket(bucket: utils.OplogEntry[]) {
 
 /**
  * Validate these properties for a bucket $B$ and its compacted version $B'$,:
- * as described in /docs/bucket-properties.md:
+ * as described in /docs/storage/bucket-properties.md:
  *
  * 1. $r(B) = r(B')$
  * 2. $r(B_{[..c]}) = r(r(B_{[..c_i]}) \cup B'_{[c_i+1..c]}) \;\forall\; c_i \in B$

package/src/test-utils/general-utils.ts

@@ -4,7 +4,7 @@ import * as bson from 'bson';
 
 export const ZERO_LSN = '0/0';
 
-export const PARSE_OPTIONS: storage.ParseSyncRulesOptions = {
+export const PARSE_OPTIONS: storage.ParseSyncConfigOptions = {
   defaultSchema: 'public'
 };
 
@@ -14,6 +14,31 @@ export const BATCH_OPTIONS: storage.CreateWriterOptions = {
   storeCurrentData: true
 };
 
+export function testCheckpoint(checkpoint: InternalOpId, lsn: string | null = null): storage.ReplicationCheckpoint {
+  return {
+    checkpoint,
+    lsn,
+    async getParameterSets() {
+      return [];
+    }
+  };
+}
+
+/**
+ * Deploy a sync config and return both the replication stream (for {@link storage.BucketStorageFactory.getInstance})
+ * and the deployed config content (for parsing / bucket requests).
+ *
+ * Replication streams and sync config content are separate concerns since one stream can hold multiple configs.
+ */
+export async function deploySyncRules(
+  factory: storage.BucketStorageFactory,
+  options: storage.UpdateSyncRulesOptions
+): Promise<{ stream: storage.PersistedReplicationStream; content: storage.PersistedSyncConfigContent }> {
+  const stream = await factory.updateSyncRules(options);
+  const content = stream.syncConfigContent[0];
+  return { stream, content };
+}
+
 /**
  * With newer storage versions, we need actual test tables, resolved via the writer.
  */
@@ -78,9 +103,9 @@ export function getBatchData(
 }
 
 function isParsedSyncRules(
-  syncRules: storage.PersistedSyncRulesContent | storage.PersistedSyncRules
-): syncRules is storage.PersistedSyncRules {
-  return (syncRules as storage.PersistedSyncRules).syncConfigWithErrors !== undefined;
+  syncRules: storage.PersistedSyncConfigContent | storage.ParsedSyncConfigSet
+): syncRules is storage.ParsedSyncConfigSet {
+  return (syncRules as storage.ParsedSyncConfigSet).syncConfigs !== undefined;
 }
 
 /**
@@ -88,7 +113,7 @@ function isParsedSyncRules(
  * This converts a bucket name like "global[]" into the actual bucket name, for use in tests.
  */
 export function bucketRequest(
-  syncRules: storage.PersistedSyncRulesContent | storage.PersistedSyncRules,
+  syncRules: storage.PersistedSyncConfigContent | storage.ParsedSyncConfigSet,
   bucket: string,
   start?: InternalOpId | string | number
 ): BucketDataRequest {
@@ -97,10 +122,13 @@ export function bucketRequest(
   const parameterStart = bucket.indexOf('[');
   const definitionName = bucket.substring(0, parameterStart);
   const parameters = bucket.substring(parameterStart);
-  const source = parsed.syncConfigWithErrors.config.bucketDataSources.find((b) => b.uniqueName === definitionName);
+  const availableSources = parsed.syncConfigs.flatMap((config) => config.config.bucketDataSources);
+  const source = availableSources.find((b) => b.uniqueName === definitionName);
 
   if (source == null) {
-    throw new Error(`Failed to find global bucket ${bucket}`);
+    throw new Error(
+      `Failed to find global bucket ${bucket}. Available: ${availableSources.map((s) => s.uniqueName).join(',')}`
+    );
   }
   const bucketName = hydrationState.getBucketSourceScope(source).bucketPrefix + parameters;
   return {

package/src/tests/register-compacting-tests.ts

@@ -17,6 +17,7 @@ bucket_definitions:
     `)
     );
     const bucketStorage = factory.getInstance(syncRules);
+    const syncRulesContent = syncRules.syncConfigContent[0];
 
     await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
     const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
@@ -53,11 +54,13 @@ bucket_definitions:
 
     const checkpoint = writer.last_flushed_op!;
 
-    const request = bucketRequest(syncRules, 'global[]');
+    const request = bucketRequest(syncRulesContent, 'global[]');
 
-    const batchBefore = await test_utils.oneFromAsync(bucketStorage.getBucketDataBatch(checkpoint, [request]));
+    const batchBefore = await test_utils.oneFromAsync(
+      bucketStorage.getBucketDataBatch(test_utils.testCheckpoint(checkpoint), [request])
+    );
     const dataBefore = batchBefore.chunkData.data;
-    const checksumBefore = await bucketStorage.getChecksums(checkpoint, [request]);
+    const checksumBefore = await bucketStorage.getChecksums(test_utils.testCheckpoint(checkpoint), [request]);
 
     expect(dataBefore).toMatchObject([
       {
@@ -86,11 +89,13 @@ bucket_definitions:
       minChangeRatio: 0
     });
 
-    const batchAfter = await test_utils.oneFromAsync(bucketStorage.getBucketDataBatch(checkpoint, [request]));
+    const batchAfter = await test_utils.oneFromAsync(
+      bucketStorage.getBucketDataBatch(test_utils.testCheckpoint(checkpoint), [request])
+    );
     const dataAfter = batchAfter.chunkData.data;
-    const checksumAfter = await bucketStorage.getChecksums(checkpoint, [request]);
+    const checksumAfter = await bucketStorage.getChecksums(test_utils.testCheckpoint(checkpoint), [request]);
     bucketStorage.clearChecksumCache();
-    const checksumAfter2 = await bucketStorage.getChecksums(checkpoint, [request]);
+    const checksumAfter2 = await bucketStorage.getChecksums(test_utils.testCheckpoint(checkpoint), [request]);
 
     expect(batchAfter.targetOp).toEqual(3n);
     expect(dataAfter).toMatchObject([
@@ -124,6 +129,7 @@ bucket_definitions:
     `)
     );
     const bucketStorage = factory.getInstance(syncRules);
+    const syncRulesContent = syncRules.syncConfigContent[0];
 
     await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
     const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
@@ -168,11 +174,13 @@ bucket_definitions:
     await writer.flush();
 
     const checkpoint = writer.last_flushed_op!;
-    const request = bucketRequest(syncRules, 'global[]');
+    const request = bucketRequest(syncRulesContent, 'global[]');
 
-    const batchBefore = await test_utils.oneFromAsync(bucketStorage.getBucketDataBatch(checkpoint, [request]));
+    const batchBefore = await test_utils.oneFromAsync(
+      bucketStorage.getBucketDataBatch(test_utils.testCheckpoint(checkpoint), [request])
+    );
     const dataBefore = batchBefore.chunkData.data;
-    const checksumBefore = await bucketStorage.getChecksums(checkpoint, [request]);
+    const checksumBefore = await bucketStorage.getChecksums(test_utils.testCheckpoint(checkpoint), [request]);
 
     // op_id sequence depends on the storage implementation
     expect(dataBefore).toMatchObject([
@@ -202,10 +210,12 @@ bucket_definitions:
       minChangeRatio: 0
     });
 
-    const batchAfter = await test_utils.oneFromAsync(bucketStorage.getBucketDataBatch(checkpoint, [request]));
+    const batchAfter = await test_utils.oneFromAsync(
+      bucketStorage.getBucketDataBatch(test_utils.testCheckpoint(checkpoint), [request])
+    );
     const dataAfter = batchAfter.chunkData.data;
     bucketStorage.clearChecksumCache();
-    const checksumAfter = await bucketStorage.getChecksums(checkpoint, [request]);
+    const checksumAfter = await bucketStorage.getChecksums(test_utils.testCheckpoint(checkpoint), [request]);
 
     expect(batchAfter.targetOp).toBeLessThanOrEqual(checkpoint);
     expect(dataAfter).toMatchObject([
@@ -240,6 +250,7 @@ bucket_definitions:
     `)
     );
     const bucketStorage = factory.getInstance(syncRules);
+    const syncRulesContent = syncRules.syncConfigContent[0];
 
     await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
     const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
@@ -275,7 +286,7 @@ bucket_definitions:
     await writer.flush();
 
     const checkpoint1 = writer.last_flushed_op!;
-    const request = bucketRequest(syncRules, 'global[]');
+    const request = bucketRequest(syncRulesContent, 'global[]');
     await using writer2 = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
     const testTable2 = await test_utils.resolveTestTable(writer2, 'test', ['id'], config);
     await writer2.save({
@@ -298,10 +309,12 @@ bucket_definitions:
       minChangeRatio: 0
     });
 
-    const batchAfter = await test_utils.oneFromAsync(bucketStorage.getBucketDataBatch(checkpoint2, [request]));
+    const batchAfter = await test_utils.oneFromAsync(
+      bucketStorage.getBucketDataBatch(test_utils.testCheckpoint(checkpoint2), [request])
+    );
     const dataAfter = batchAfter.chunkData.data;
     await bucketStorage.clearChecksumCache();
-    const checksumAfter = await bucketStorage.getChecksums(checkpoint2, [request]);
+    const checksumAfter = await bucketStorage.getChecksums(test_utils.testCheckpoint(checkpoint2), [request]);
 
     expect(dataAfter).toMatchObject([
       {
@@ -328,6 +341,7 @@ bucket_definitions:
               - select * from test where b = bucket.b`)
     );
     const bucketStorage = factory.getInstance(syncRules);
+    const syncRulesContent = syncRules.syncConfigContent[0];
 
     await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
     const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
@@ -412,8 +426,8 @@ bucket_definitions:
 
     const batchAfter = await test_utils.fromAsync(
       bucketStorage.getBucketDataBatch(
-        checkpoint,
-        bucketRequestMap(syncRules, [
+        test_utils.testCheckpoint(checkpoint),
+        bucketRequestMap(syncRulesContent, [
           ['grouped["b1"]', 0n],
           ['grouped["b2"]', 0n]
         ])
@@ -458,6 +472,7 @@ bucket_definitions:
     `)
     );
     const bucketStorage = factory.getInstance(syncRules);
+    const syncRulesContent = syncRules.syncConfigContent[0];
 
     await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
     const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
@@ -513,9 +528,9 @@ bucket_definitions:
     await writer2.commit('2/1');
     await writer2.flush();
     const checkpoint2 = writer2.last_flushed_op!;
-    const request = bucketRequest(syncRules, 'global[]');
+    const request = bucketRequest(syncRulesContent, 'global[]');
     await bucketStorage.clearChecksumCache();
-    const checksumAfter = await bucketStorage.getChecksums(checkpoint2, [request]);
+    const checksumAfter = await bucketStorage.getChecksums(test_utils.testCheckpoint(checkpoint2), [request]);
     const globalChecksum = checksumAfter.get(request.bucket);
     expect(globalChecksum).toMatchObject({
       bucket: request.bucket,
@@ -536,6 +551,7 @@ bucket_definitions:
     `)
     );
     const bucketStorage = factory.getInstance(syncRules);
+    const syncRulesContent = syncRules.syncConfigContent[0];
 
     await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
     const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
@@ -562,7 +578,10 @@ bucket_definitions:
     await writer.flush();
 
     // Get checksums here just to populate the cache
-    await bucketStorage.getChecksums(writer.last_flushed_op!, bucketRequests(syncRules, ['global[]']));
+    await bucketStorage.getChecksums(
+      test_utils.testCheckpoint(writer.last_flushed_op!),
+      bucketRequests(syncRulesContent, ['global[]'])
+    );
     await using writer2 = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
     const testTable2 = await test_utils.resolveTestTable(writer2, 'test', ['id'], config);
     await writer2.save({
@@ -585,9 +604,9 @@ bucket_definitions:
     });
 
     const checkpoint2 = writer2.last_flushed_op!;
-    const request = bucketRequest(syncRules, 'global[]');
+    const request = bucketRequest(syncRulesContent, 'global[]');
     // Check that the checksum was correctly updated with the clear operation after having a cached checksum
-    const checksumAfter = await bucketStorage.getChecksums(checkpoint2, [request]);
+    const checksumAfter = await bucketStorage.getChecksums(test_utils.testCheckpoint(checkpoint2), [request]);
     const globalChecksum = checksumAfter.get(request.bucket);
     expect(globalChecksum).toMatchObject({
       bucket: request.bucket,
@@ -607,6 +626,7 @@ bucket_definitions:
       `)
     );
     const bucketStorage = factory.getInstance(syncRules);
+    const syncRulesContent = syncRules.syncConfigContent[0];
 
     await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
     const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
@@ -646,7 +666,10 @@ bucket_definitions:
     });
 
     const batchAfterDefaultCompact = await test_utils.oneFromAsync(
-      bucketStorage.getBucketDataBatch(checkpoint2, bucketRequestMap(syncRules, [['global[]', 0n]]))
+      bucketStorage.getBucketDataBatch(
+        test_utils.testCheckpoint(checkpoint2),
+        bucketRequestMap(syncRulesContent, [['global[]', 0n]])
+      )
     );
 
     // Operation 1 should remain a PUT because op_id=2 is above the default maxOpId checkpoint.

package/src/tests/register-data-storage-checkpoint-tests.ts

@@ -42,7 +42,7 @@ bucket_definitions:
     await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
     await writer.markAllSnapshotDone('1/1');
 
-    const writeCheckpoint = await bucketStorage.createManagedWriteCheckpoint({
+    const writeCheckpoint = await createManagedWriteCheckpoint(bucketStorage, {
       heads: { '1': '5/0' },
       user_id: 'user1'
     });
@@ -102,7 +102,7 @@ bucket_definitions:
       }
     });
 
-    const writeCheckpoint = await bucketStorage.createManagedWriteCheckpoint({
+    const writeCheckpoint = await createManagedWriteCheckpoint(bucketStorage, {
       heads: { '1': '6/0' },
       user_id: 'user1'
     });
@@ -302,3 +302,13 @@ bucket_definitions:
     });
   });
 }
+
+async function createManagedWriteCheckpoint(
+  bucketStorage: storage.SyncRulesBucketStorage,
+  checkpoint: storage.ManagedWriteCheckpointOptions
+) {
+  const checkpoints = await bucketStorage.createManagedWriteCheckpoints([checkpoint]);
+  const writeCheckpoint = checkpoints.get(checkpoint.user_id);
+  expect(writeCheckpoint).not.toBeUndefined();
+  return writeCheckpoint!;
+}