@powersync/service-core-tests 0.15.5 → 0.17.0

package/src/test-utils/StorageDataHelpers.ts

@@ -1,8 +1,8 @@
 import {
   InternalOpId,
   OplogEntry,
-  PersistedSyncRules,
-  PersistedSyncRulesContent,
+  ParsedSyncConfigSet,
+  PersistedSyncConfigContent,
   SyncRulesBucketStorage
 } from '@powersync/service-core';
 import { bucketRequest } from './general-utils.js';
@@ -10,9 +10,9 @@ import { fromAsync } from './stream_utils.js';
 
 export class StorageDataHelpers {
   storage: SyncRulesBucketStorage;
-  syncRules: PersistedSyncRulesContent | PersistedSyncRules;
+  syncRules: PersistedSyncConfigContent | ParsedSyncConfigSet;
 
-  constructor(storage: SyncRulesBucketStorage, syncRules: PersistedSyncRulesContent | PersistedSyncRules) {
+  constructor(storage: SyncRulesBucketStorage, syncRules: PersistedSyncConfigContent | ParsedSyncConfigSet) {
     this.storage = storage;
     this.syncRules = syncRules;
   }

package/src/test-utils/general-utils.ts

@@ -4,7 +4,7 @@ import * as bson from 'bson';
 
 export const ZERO_LSN = '0/0';
 
-export const PARSE_OPTIONS: storage.ParseSyncRulesOptions = {
+export const PARSE_OPTIONS: storage.ParseSyncConfigOptions = {
   defaultSchema: 'public'
 };
 
@@ -14,50 +14,65 @@ export const BATCH_OPTIONS: storage.CreateWriterOptions = {
   storeCurrentData: true
 };
 
-export function makeTestTable(
-  name: string,
-  replicaIdColumns?: string[] | undefined,
-  options?: { tableIdStrings: boolean }
-) {
-  const relId = utils.hashData('table', name, (replicaIdColumns ?? ['id']).join(','));
-  const id =
-    options?.tableIdStrings == false ? new bson.ObjectId('6544e3899293153fa7b38331') : '6544e3899293153fa7b38331';
-  return new storage.SourceTable({
-    id: id,
-    connectionTag: storage.SourceTable.DEFAULT_TAG,
-    objectId: relId,
-    schema: 'public',
-    name: name,
-    replicaIdColumns: (replicaIdColumns ?? ['id']).map((column) => ({ name: column, type: 'VARCHAR', typeId: 25 })),
-    snapshotComplete: true
-  });
-}
 /**
- * With incremental reprocessing, we need actual test tables, resolved via the writer.
+ * Deploy a sync config and return both the replication stream (for {@link storage.BucketStorageFactory.getInstance})
+ * and the deployed config content (for parsing / bucket requests).
  *
- * This prepares for it.
+ * Replication streams and sync config content are separate concerns since one stream can hold multiple configs.
+ */
+export async function deploySyncRules(
+  factory: storage.BucketStorageFactory,
+  options: storage.UpdateSyncRulesOptions
+): Promise<{ stream: storage.PersistedReplicationStream; content: storage.PersistedSyncConfigContent }> {
+  const stream = await factory.updateSyncRules(options);
+  const content = stream.syncConfigContent[0];
+  return { stream, content };
+}
+
+/**
+ * With newer storage versions, we need actual test tables, resolved via the writer.
  */
 export async function resolveTestTable(
-  _writer: storage.BucketStorageBatch,
+  writer: storage.BucketStorageBatch,
   name: string,
   replicaIdColumns: string[] | undefined,
   options: { tableIdStrings: boolean },
   idIndex: number = 1
 ) {
+  void idIndex;
+  void options;
+
   const relId = utils.hashData('table', name, (replicaIdColumns ?? ['id']).join(','));
-  // Generate unique ids per test table (if idIndex is specified), without completely
-  // breaking all the existing tests.
+  // Semi-hardcoded id for tests, to get consistent output.
+  // If the same test uses multiple tables, pass idIndex to get different ids.
   const idString = '6544e3899293153fa7b383' + (30 + idIndex).toString().padStart(2, '0');
   const id = options.tableIdStrings == false ? new bson.ObjectId(idString) : idString;
-  return new storage.SourceTable({
-    id: id,
+  let didGenerateId = false;
+
+  const source: storage.SourceEntityDescriptor = {
     connectionTag: storage.SourceTable.DEFAULT_TAG,
     objectId: relId,
     schema: 'public',
     name: name,
-    replicaIdColumns: (replicaIdColumns ?? ['id']).map((column) => ({ name: column, type: 'VARCHAR', typeId: 25 })),
-    snapshotComplete: true
+    replicaIdColumns: (replicaIdColumns ?? ['id']).map((column) => ({ name: column, type: 'VARCHAR', typeId: 25 }))
+  };
+  const resolved = await writer.resolveTables({
+    connection_id: 1,
+    source,
+    idGenerator: () => {
+      if (didGenerateId) {
+        throw new Error('idGenerator called multiple times - not supported in tests');
+      }
+      didGenerateId = true;
+      return id;
+    }
   });
+
+  const table = resolved.tables[0];
+  if (table == null) {
+    throw new Error(`Failed to resolve test table ${source.schema}.${source.name}`);
+  }
+  return table;
 }
 
 export function getBatchData(
@@ -78,9 +93,9 @@ export function getBatchData(
 }
 
 function isParsedSyncRules(
-  syncRules: storage.PersistedSyncRulesContent | storage.PersistedSyncRules
-): syncRules is storage.PersistedSyncRules {
-  return (syncRules as storage.PersistedSyncRules).sync_rules !== undefined;
+  syncRules: storage.PersistedSyncConfigContent | storage.ParsedSyncConfigSet
+): syncRules is storage.ParsedSyncConfigSet {
+  return (syncRules as storage.ParsedSyncConfigSet).syncConfigs !== undefined;
 }
 
 /**
@@ -88,7 +103,7 @@ function isParsedSyncRules(
  * This converts a bucket name like "global[]" into the actual bucket name, for use in tests.
  */
 export function bucketRequest(
-  syncRules: storage.PersistedSyncRulesContent | storage.PersistedSyncRules,
+  syncRules: storage.PersistedSyncConfigContent | storage.ParsedSyncConfigSet,
   bucket: string,
   start?: InternalOpId | string | number
 ): BucketDataRequest {
@@ -97,10 +112,13 @@ export function bucketRequest(
   const parameterStart = bucket.indexOf('[');
   const definitionName = bucket.substring(0, parameterStart);
   const parameters = bucket.substring(parameterStart);
-  const source = parsed.sync_rules.config.bucketDataSources.find((b) => b.uniqueName === definitionName);
+  const availableSources = parsed.syncConfigs.flatMap((config) => config.config.bucketDataSources);
+  const source = availableSources.find((b) => b.uniqueName === definitionName);
 
   if (source == null) {
-    throw new Error(`Failed to find global bucket ${bucket}`);
+    throw new Error(
+      `Failed to find global bucket ${bucket}. Available: ${availableSources.map((s) => s.uniqueName).join(',')}`
+    );
   }
   const bucketName = hydrationState.getBucketSourceScope(source).bucketPrefix + parameters;
   return {

package/src/test-utils/test-utils-index.ts

@@ -1,3 +1,4 @@
+export * from './AbstractStreamTestContext.js';
 export * from './bucket-validation.js';
 export * from './general-utils.js';
 export * from './MetricsHelper.js';

package/src/tests/register-compacting-tests.ts

@@ -17,6 +17,7 @@ bucket_definitions:
     `)
     );
     const bucketStorage = factory.getInstance(syncRules);
+    const syncRulesContent = syncRules.syncConfigContent[0];
 
     await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
     const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
@@ -53,7 +54,7 @@ bucket_definitions:
 
     const checkpoint = writer.last_flushed_op!;
 
-    const request = bucketRequest(syncRules, 'global[]');
+    const request = bucketRequest(syncRulesContent, 'global[]');
 
     const batchBefore = await test_utils.oneFromAsync(bucketStorage.getBucketDataBatch(checkpoint, [request]));
     const dataBefore = batchBefore.chunkData.data;
@@ -124,6 +125,7 @@ bucket_definitions:
     `)
     );
     const bucketStorage = factory.getInstance(syncRules);
+    const syncRulesContent = syncRules.syncConfigContent[0];
 
     await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
     const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
@@ -168,7 +170,7 @@ bucket_definitions:
     await writer.flush();
 
     const checkpoint = writer.last_flushed_op!;
-    const request = bucketRequest(syncRules, 'global[]');
+    const request = bucketRequest(syncRulesContent, 'global[]');
 
     const batchBefore = await test_utils.oneFromAsync(bucketStorage.getBucketDataBatch(checkpoint, [request]));
     const dataBefore = batchBefore.chunkData.data;
@@ -240,6 +242,7 @@ bucket_definitions:
     `)
     );
     const bucketStorage = factory.getInstance(syncRules);
+    const syncRulesContent = syncRules.syncConfigContent[0];
 
     await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
     const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
@@ -275,7 +278,7 @@ bucket_definitions:
     await writer.flush();
 
     const checkpoint1 = writer.last_flushed_op!;
-    const request = bucketRequest(syncRules, 'global[]');
+    const request = bucketRequest(syncRulesContent, 'global[]');
     await using writer2 = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
     const testTable2 = await test_utils.resolveTestTable(writer2, 'test', ['id'], config);
     await writer2.save({
@@ -328,6 +331,7 @@ bucket_definitions:
               - select * from test where b = bucket.b`)
     );
     const bucketStorage = factory.getInstance(syncRules);
+    const syncRulesContent = syncRules.syncConfigContent[0];
 
     await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
     const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
@@ -413,7 +417,7 @@ bucket_definitions:
     const batchAfter = await test_utils.fromAsync(
       bucketStorage.getBucketDataBatch(
         checkpoint,
-        bucketRequestMap(syncRules, [
+        bucketRequestMap(syncRulesContent, [
           ['grouped["b1"]', 0n],
           ['grouped["b2"]', 0n]
         ])
@@ -458,6 +462,7 @@ bucket_definitions:
     `)
     );
     const bucketStorage = factory.getInstance(syncRules);
+    const syncRulesContent = syncRules.syncConfigContent[0];
 
     await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
     const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
@@ -513,7 +518,7 @@ bucket_definitions:
     await writer2.commit('2/1');
     await writer2.flush();
     const checkpoint2 = writer2.last_flushed_op!;
-    const request = bucketRequest(syncRules, 'global[]');
+    const request = bucketRequest(syncRulesContent, 'global[]');
     await bucketStorage.clearChecksumCache();
     const checksumAfter = await bucketStorage.getChecksums(checkpoint2, [request]);
     const globalChecksum = checksumAfter.get(request.bucket);
@@ -536,6 +541,7 @@ bucket_definitions:
     `)
     );
     const bucketStorage = factory.getInstance(syncRules);
+    const syncRulesContent = syncRules.syncConfigContent[0];
 
     await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
     const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
@@ -562,7 +568,7 @@ bucket_definitions:
     await writer.flush();
 
     // Get checksums here just to populate the cache
-    await bucketStorage.getChecksums(writer.last_flushed_op!, bucketRequests(syncRules, ['global[]']));
+    await bucketStorage.getChecksums(writer.last_flushed_op!, bucketRequests(syncRulesContent, ['global[]']));
     await using writer2 = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
     const testTable2 = await test_utils.resolveTestTable(writer2, 'test', ['id'], config);
     await writer2.save({
@@ -585,7 +591,7 @@ bucket_definitions:
     });
 
     const checkpoint2 = writer2.last_flushed_op!;
-    const request = bucketRequest(syncRules, 'global[]');
+    const request = bucketRequest(syncRulesContent, 'global[]');
     // Check that the checksum was correctly updated with the clear operation after having a cached checksum
     const checksumAfter = await bucketStorage.getChecksums(checkpoint2, [request]);
     const globalChecksum = checksumAfter.get(request.bucket);
@@ -607,6 +613,7 @@ bucket_definitions:
       `)
     );
     const bucketStorage = factory.getInstance(syncRules);
+    const syncRulesContent = syncRules.syncConfigContent[0];
 
     await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
     const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
@@ -646,7 +653,7 @@ bucket_definitions:
     });
 
     const batchAfterDefaultCompact = await test_utils.oneFromAsync(
-      bucketStorage.getBucketDataBatch(checkpoint2, bucketRequestMap(syncRules, [['global[]', 0n]]))
+      bucketStorage.getBucketDataBatch(checkpoint2, bucketRequestMap(syncRulesContent, [['global[]', 0n]]))
     );
 
     // Operation 1 should remain a PUT because op_id=2 is above the default maxOpId checkpoint.