@powersync/service-core-tests 0.15.5 → 0.16.0

package/src/test-utils/AbstractStreamTestContext.ts

@@ -0,0 +1,179 @@
+import { ReplicationAbortedError } from '@powersync/lib-services-framework';
+import {
+  BucketStorageFactory,
+  InternalOpId,
+  settledPromise,
+  storage,
+  SyncRulesBucketStorage,
+  unsettledPromise,
+  updateSyncRulesFromYaml
+} from '@powersync/service-core';
+import { StorageDataHelpers } from './StorageDataHelpers.js';
+import { bucketRequest } from './general-utils.js';
+import { fromAsync } from './stream_utils.js';
+
+export abstract class AbstractStreamTestContext implements AsyncDisposable {
+  protected abortController = new AbortController();
+  protected syncRulesContent?: storage.PersistedSyncRulesContent;
+  public storage?: SyncRulesBucketStorage;
+  protected settledReplicationPromise?: Promise<PromiseSettledResult<void>>;
+
+  abstract get factory(): BucketStorageFactory;
+  protected abstract get storageVersion(): number;
+
+  async [Symbol.asyncDispose]() {
+    await this.dispose();
+  }
+
+  protected abstract _dispose(): Promise<void>;
+
+  async dispose() {
+    this.abortController.abort();
+    try {
+      await this.settledReplicationPromise;
+      await this._dispose();
+      await this.factory?.[Symbol.asyncDispose]();
+    } catch (e) {
+      // Throwing here may result in SuppressedError. The underlying errors often don't show up
+      // in the test output, so we log it here.
+      // If we could get vitest to log SuppressedError.error and SuppressedError.suppressed, we
+      // could remove this.
+      console.error('Error during ConvexStreamTestContext dispose', e);
+      throw e;
+    }
+  }
+
+  async updateSyncRules(content: string) {
+    const syncRules = await this.factory.updateSyncRules(
+      updateSyncRulesFromYaml(content, { validate: true, storageVersion: this.storageVersion })
+    );
+    this.syncRulesContent = syncRules;
+    this.storage = this.factory.getInstance(syncRules);
+    return this.storage!;
+  }
+
+  async loadNextSyncRules() {
+    const syncRules = await this.factory.getNextSyncRulesContent();
+    if (syncRules == null) {
+      throw new Error(`Next sync rules not available`);
+    }
+
+    this.syncRulesContent = syncRules;
+    this.storage = this.factory.getInstance(syncRules);
+    return this.storage!;
+  }
+
+  async loadActiveSyncRules() {
+    const syncRules = await this.factory.getActiveSyncRulesContent();
+    if (syncRules == null) {
+      throw new Error(`Active sync rules not available`);
+    }
+
+    this.syncRulesContent = syncRules;
+    this.storage = this.factory.getInstance(syncRules);
+    return this.storage!;
+  }
+
+  private getSyncRulesContent(): storage.PersistedSyncRulesContent {
+    if (this.syncRulesContent == null) {
+      throw new Error('Sync rules not configured - call updateSyncRules() first');
+    }
+    return this.syncRulesContent;
+  }
+
+  /**
+   * Replicate a snapshot, start streaming, and wait for a consistent checkpoint.
+   */
+  async initializeReplication() {
+    await this.replicateSnapshot();
+    // Make sure we're up to date
+    await this.getCheckpoint();
+  }
+
+  protected abstract triggerReplication(): Promise<void>;
+  protected abstract waitForInitialSnapshot(): Promise<void>;
+
+  /**
+   * Replicate the initial snapshot, and start streaming.
+   */
+  async replicateSnapshot() {
+    // Use a settledPromise to avoid unhandled rejections
+    this.settledReplicationPromise = settledPromise(this.triggerReplication());
+    try {
+      await Promise.race([unsettledPromise(this.settledReplicationPromise), this.waitForInitialSnapshot()]);
+    } catch (e) {
+      if (e instanceof ReplicationAbortedError && e.cause != null) {
+        // Edge case for tests: replicate() can throw an error, but we'd receive the ReplicationAbortedError from
+        // waitForInitialSnapshot() first. In that case, prioritize the cause, e.g. MissingReplicationSlotError.
+        // This is not a concern for production use, since we only use waitForInitialSnapshot() in tests.
+        throw e.cause;
+      }
+      throw e;
+    }
+  }
+
+  abstract getClientCheckpoint(options?: { timeout?: number }): Promise<bigint>;
+
+  async getCheckpoint(options?: { timeout?: number }) {
+    let checkpoint = await Promise.race([
+      this.getClientCheckpoint(options),
+      unsettledPromise(this.settledReplicationPromise!)
+    ]);
+    if (checkpoint == null) {
+      // This indicates an issue with the test setup - replicationPromise completed instead
+      // of getClientCheckpoint()
+      throw new Error('Test failure - replicationPromise completed');
+    }
+    return checkpoint;
+  }
+
+  async getBucketsDataBatch(buckets: Record<string, InternalOpId>, options?: { timeout?: number }) {
+    const helpers = new StorageDataHelpers(this.storage!, this.getSyncRulesContent());
+    const checkpoint = await this.getCheckpoint(options);
+    return helpers.getBucketsDataBatch(buckets, checkpoint);
+  }
+
+  /**
+   * This waits for a client checkpoint.
+   */
+  async getBucketData(bucket: string, start?: InternalOpId | string | undefined, options?: { timeout?: number }) {
+    const helpers = new StorageDataHelpers(this.storage!, this.getSyncRulesContent());
+    const checkpoint = await this.getCheckpoint(options);
+    return helpers.getBucketData(bucket, checkpoint, start);
+  }
+
+  async getChecksums(buckets: string[], options?: { timeout?: number }) {
+    const checkpoint = await this.getCheckpoint(options);
+    const syncRules = this.getSyncRulesContent();
+    const versionedBuckets = buckets.map((bucket) => bucketRequest(syncRules, bucket, 0n));
+    const checksums = await this.storage!.getChecksums(checkpoint, versionedBuckets);
+
+    const unversioned = new Map();
+    for (let i = 0; i < buckets.length; i++) {
+      unversioned.set(buckets[i], checksums.get(versionedBuckets[i].bucket)!);
+    }
+
+    return unversioned;
+  }
+
+  async getChecksum(bucket: string, options?: { timeout?: number }) {
+    const checksums = await this.getChecksums([bucket], options);
+    return checksums.get(bucket);
+  }
+
+  /**
+   * This does not wait for a client checkpoint.
+   */
+  async getCurrentBucketData(bucket: string, start?: InternalOpId | string | undefined) {
+    start ??= 0n;
+    if (typeof start == 'string') {
+      start = BigInt(start);
+    }
+    const syncRules = this.getSyncRulesContent();
+    const { checkpoint } = await this.storage!.getCheckpoint();
+    const map = [bucketRequest(syncRules, bucket, start)];
+    const batch = this.storage!.getBucketDataBatch(checkpoint, map);
+    const batches = await fromAsync(batch);
+    return batches[0]?.chunkData.data ?? [];
+  }
+}

package/src/test-utils/general-utils.ts

@@ -14,50 +14,50 @@ export const BATCH_OPTIONS: storage.CreateWriterOptions = {
   storeCurrentData: true
 };
 
-export function makeTestTable(
-  name: string,
-  replicaIdColumns?: string[] | undefined,
-  options?: { tableIdStrings: boolean }
-) {
-  const relId = utils.hashData('table', name, (replicaIdColumns ?? ['id']).join(','));
-  const id =
-    options?.tableIdStrings == false ? new bson.ObjectId('6544e3899293153fa7b38331') : '6544e3899293153fa7b38331';
-  return new storage.SourceTable({
-    id: id,
-    connectionTag: storage.SourceTable.DEFAULT_TAG,
-    objectId: relId,
-    schema: 'public',
-    name: name,
-    replicaIdColumns: (replicaIdColumns ?? ['id']).map((column) => ({ name: column, type: 'VARCHAR', typeId: 25 })),
-    snapshotComplete: true
-  });
-}
 /**
- * With incremental reprocessing, we need actual test tables, resolved via the writer.
- *
- * This prepares for it.
+ * With newer storage versions, we need actual test tables, resolved via the writer.
  */
 export async function resolveTestTable(
-  _writer: storage.BucketStorageBatch,
+  writer: storage.BucketStorageBatch,
   name: string,
   replicaIdColumns: string[] | undefined,
   options: { tableIdStrings: boolean },
   idIndex: number = 1
 ) {
+  void idIndex;
+  void options;
+
   const relId = utils.hashData('table', name, (replicaIdColumns ?? ['id']).join(','));
-  // Generate unique ids per test table (if idIndex is specified), without completely
-  // breaking all the existing tests.
+  // Semi-hardcoded id for tests, to get consistent output.
+  // If the same test uses multiple tables, pass idIndex to get different ids.
   const idString = '6544e3899293153fa7b383' + (30 + idIndex).toString().padStart(2, '0');
   const id = options.tableIdStrings == false ? new bson.ObjectId(idString) : idString;
-  return new storage.SourceTable({
-    id: id,
+  let didGenerateId = false;
+
+  const source: storage.SourceEntityDescriptor = {
     connectionTag: storage.SourceTable.DEFAULT_TAG,
     objectId: relId,
     schema: 'public',
     name: name,
-    replicaIdColumns: (replicaIdColumns ?? ['id']).map((column) => ({ name: column, type: 'VARCHAR', typeId: 25 })),
-    snapshotComplete: true
+    replicaIdColumns: (replicaIdColumns ?? ['id']).map((column) => ({ name: column, type: 'VARCHAR', typeId: 25 }))
+  };
+  const resolved = await writer.resolveTables({
+    connection_id: 1,
+    source,
+    idGenerator: () => {
+      if (didGenerateId) {
+        throw new Error('idGenerator called multiple times - not supported in tests');
+      }
+      didGenerateId = true;
+      return id;
+    }
   });
+
+  const table = resolved.tables[0];
+  if (table == null) {
+    throw new Error(`Failed to resolve test table ${source.schema}.${source.name}`);
+  }
+  return table;
 }
 
 export function getBatchData(
@@ -80,7 +80,7 @@ export function getBatchData(
 function isParsedSyncRules(
   syncRules: storage.PersistedSyncRulesContent | storage.PersistedSyncRules
 ): syncRules is storage.PersistedSyncRules {
-  return (syncRules as storage.PersistedSyncRules).sync_rules !== undefined;
+  return (syncRules as storage.PersistedSyncRules).syncConfigWithErrors !== undefined;
 }
 
 /**
@@ -97,7 +97,7 @@ export function bucketRequest(
   const parameterStart = bucket.indexOf('[');
   const definitionName = bucket.substring(0, parameterStart);
   const parameters = bucket.substring(parameterStart);
-  const source = parsed.sync_rules.config.bucketDataSources.find((b) => b.uniqueName === definitionName);
+  const source = parsed.syncConfigWithErrors.config.bucketDataSources.find((b) => b.uniqueName === definitionName);
 
   if (source == null) {
     throw new Error(`Failed to find global bucket ${bucket}`);

package/src/test-utils/test-utils-index.ts

@@ -1,3 +1,4 @@
+export * from './AbstractStreamTestContext.js';
 export * from './bucket-validation.js';
 export * from './general-utils.js';
 export * from './MetricsHelper.js';

package/src/tests/register-data-storage-data-tests.ts

@@ -826,7 +826,7 @@ bucket_definitions:
     const bucketStorage = factory.getInstance(syncRules);
 
     await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
-    const sourceTable = test_utils.makeTestTable('test', ['id', 'description'], config);
+    const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id', 'description'], config);
 
     // Pre-setup
     await writer.markAllSnapshotDone('1/1');
@@ -939,7 +939,7 @@ bucket_definitions:
     const bucketStorage = factory.getInstance(syncRules);
 
     await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
-    const sourceTable = test_utils.makeTestTable('test', ['id', 'description'], config);
+    const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id', 'description'], config);
 
     // Pre-setup
     await writer.markAllSnapshotDone('1/1');
@@ -1756,7 +1756,7 @@ streams:
 `)
     );
 
-    const { errors } = deployed.parsed({ defaultSchema: 'ignored' }).sync_rules;
+    const { errors } = deployed.parsed({ defaultSchema: 'ignored' }).syncConfigWithErrors;
     expect(errors).toHaveLength(1);
     expect(errors[0].message).toStrictEqual('Expected a SELECT statement');
     expect(errors[0].location).toStrictEqual({

package/src/tests/register-data-storage-parameter-tests.ts

@@ -41,7 +41,7 @@ bucket_definitions:
       )
     );
     const bucketStorage = factory.getInstance(syncRules);
-    const sync_rules = syncRules.parsed(test_utils.PARSE_OPTIONS).hydratedSyncRules();
+    const sync_rules = syncRules.parsed(test_utils.PARSE_OPTIONS).hydratedSyncConfig();
 
     await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
     const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
@@ -107,7 +107,7 @@ bucket_definitions:
       )
     );
     const bucketStorage = factory.getInstance(syncRules);
-    const sync_rules = syncRules.parsed(test_utils.PARSE_OPTIONS).hydratedSyncRules();
+    const sync_rules = syncRules.parsed(test_utils.PARSE_OPTIONS).hydratedSyncConfig();
 
     await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
     const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
@@ -178,7 +178,7 @@ bucket_definitions:
       )
     );
     const bucketStorage = factory.getInstance(syncRules);
-    const sync_rules = syncRules.parsed(test_utils.PARSE_OPTIONS).hydratedSyncRules();
+    const sync_rules = syncRules.parsed(test_utils.PARSE_OPTIONS).hydratedSyncConfig();
 
     await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
     const table = await test_utils.resolveTestTable(writer, 'todos', ['id', 'list_id'], config);
@@ -265,7 +265,7 @@ bucket_definitions:
       )
     );
     const bucketStorage = factory.getInstance(syncRules);
-    const sync_rules = syncRules.parsed(test_utils.PARSE_OPTIONS).hydratedSyncRules();
+    const sync_rules = syncRules.parsed(test_utils.PARSE_OPTIONS).hydratedSyncConfig();
 
     await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
     const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
@@ -333,7 +333,7 @@ bucket_definitions:
       )
     );
     const bucketStorage = factory.getInstance(syncRules);
-    const sync_rules = syncRules.parsed(test_utils.PARSE_OPTIONS).hydratedSyncRules();
+    const sync_rules = syncRules.parsed(test_utils.PARSE_OPTIONS).hydratedSyncConfig();
 
     await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
     const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
@@ -400,7 +400,7 @@ bucket_definitions:
         }
       )
     );
-    const sync_rules = syncRules.parsed(test_utils.PARSE_OPTIONS).hydratedSyncRules();
+    const sync_rules = syncRules.parsed(test_utils.PARSE_OPTIONS).hydratedSyncConfig();
     const bucketStorage = factory.getInstance(syncRules);
 
     await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
@@ -458,7 +458,7 @@ bucket_definitions:
         }
       )
     );
-    const sync_rules = syncRules.parsed(test_utils.PARSE_OPTIONS).hydratedSyncRules();
+    const sync_rules = syncRules.parsed(test_utils.PARSE_OPTIONS).hydratedSyncConfig();
     const bucketStorage = factory.getInstance(syncRules);
 
     await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
@@ -552,7 +552,7 @@ bucket_definitions:
         }
       )
     );
-    const sync_rules = syncRules.parsed(test_utils.PARSE_OPTIONS).hydratedSyncRules();
+    const sync_rules = syncRules.parsed(test_utils.PARSE_OPTIONS).hydratedSyncConfig();
     const bucketStorage = factory.getInstance(syncRules);
 
     await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
@@ -654,7 +654,7 @@ bucket_definitions:
       )
     );
     const bucketStorage = factory.getInstance(syncRules);
-    const sync_rules = syncRules.parsed(test_utils.PARSE_OPTIONS).hydratedSyncRules();
+    const sync_rules = syncRules.parsed(test_utils.PARSE_OPTIONS).hydratedSyncConfig();
 
     await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
     const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
@@ -745,7 +745,7 @@ streams:
     `)
     );
     const bucketStorage = factory.getInstance(syncRules);
-    const sync_rules = syncRules.parsed(test_utils.PARSE_OPTIONS).hydratedSyncRules();
+    const sync_rules = syncRules.parsed(test_utils.PARSE_OPTIONS).hydratedSyncConfig();
 
     await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
     const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
@@ -786,7 +786,6 @@ streams:
         return parameter_sets;
       }
     });
-    console.log('whatabuckets', buckets);
     expect(buckets).toHaveLength(1);
     expect(buckets).toMatchObject([
       {
@@ -817,7 +816,7 @@ streams:
       )
     );
     const bucketStorage = factory.getInstance(syncRules);
-    const sync_rules = syncRules.parsed(test_utils.PARSE_OPTIONS).hydratedSyncRules();
+    const sync_rules = syncRules.parsed(test_utils.PARSE_OPTIONS).hydratedSyncConfig();
 
     await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
     const testTable = await test_utils.resolveTestTable(writer, 'b', ['id'], config);
@@ -866,7 +865,7 @@ streams:
     `)
     );
     const bucketStorage = factory.getInstance(syncRules);
-    const sync_rules = syncRules.parsed(test_utils.PARSE_OPTIONS).hydratedSyncRules();
+    const sync_rules = syncRules.parsed(test_utils.PARSE_OPTIONS).hydratedSyncConfig();
 
     await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
     const tableB = await test_utils.resolveTestTable(writer, 'b', ['id'], config);
@@ -996,11 +995,11 @@ streams:
     const bucketStorage = factory.getInstance(syncRules);
     const parsedSyncRules = syncRules.parsed(test_utils.PARSE_OPTIONS);
     const hydrationState = parsedSyncRules.hydrationState;
-    const syncConfig = parsedSyncRules.sync_rules.config;
+    const syncConfig = parsedSyncRules.syncConfigWithErrors.config;
 
     await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
-    const paramATable = await test_utils.resolveTestTable(writer, 'param_a', ['id'], config);
-    const paramBTable = await test_utils.resolveTestTable(writer, 'param_b', ['id'], config);
+    const paramATable = await test_utils.resolveTestTable(writer, 'param_a', ['id'], config, 1);
+    const paramBTable = await test_utils.resolveTestTable(writer, 'param_b', ['id'], config, 2);
     const replicaId = test_utils.rid('id');
 
     // Insert the same row into param_a and param_b
@@ -1092,11 +1091,11 @@ streams:
       )
     );
     const bucketStorage = factory.getInstance(syncRules);
-    const sync_rules = syncRules.parsed(test_utils.PARSE_OPTIONS).hydratedSyncRules();
+    const sync_rules = syncRules.parsed(test_utils.PARSE_OPTIONS).hydratedSyncConfig();
 
     await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
-    const tableB = await test_utils.resolveTestTable(writer, 'b', ['id'], config);
-    const tableC = await test_utils.resolveTestTable(writer, 'c', ['id'], config);
+    const tableB = await test_utils.resolveTestTable(writer, 'b', ['id'], config, 1);
+    const tableC = await test_utils.resolveTestTable(writer, 'c', ['id'], config, 2);
 
     await writer.markAllSnapshotDone('1/1');
     await writer.save({

package/src/tests/util.ts

@@ -1,11 +1,11 @@
 import { storage } from '@powersync/service-core';
 import {
   ParameterIndexLookupCreator,
-  SourceTableInterface,
-  SqliteRow,
+  ParameterLookupDefinitionId,
+  ParameterLookupScope,
+  SourceTableRef,
   TablePattern
 } from '@powersync/service-sync-rules';
-import { ParameterLookupScope } from '@powersync/service-sync-rules/src/HydrationState.js';
 import { bucketRequest } from '../test-utils/general-utils.js';
 
 export function bucketRequestMap(
@@ -26,20 +26,23 @@ export function bucketRequests(
 }
 
 const EMPTY_LOOKUP_SOURCE: ParameterIndexLookupCreator = {
-  get defaultLookupScope(): ParameterLookupScope {
+  get sourceId(): ParameterLookupDefinitionId {
     return {
       lookupName: 'lookup',
-      queryId: '0',
-      source: EMPTY_LOOKUP_SOURCE
+      queryId: '0'
     };
   },
   getSourceTables(): Set<TablePattern> {
     return new Set();
   },
-  evaluateParameterRow(_sourceTable: SourceTableInterface, _row: SqliteRow) {
-    return [];
+  createEvaluator(input) {
+    return {
+      evaluateParameterRow(sourceTable, row) {
+        return [];
+      }
+    };
   },
-  tableSyncsParameters(_table: SourceTableInterface): boolean {
+  tableSyncsParameters(_table: SourceTableRef): boolean {
     return false;
   }
 };

package/tsconfig.json

@@ -17,6 +17,9 @@
     },
     {
       "path": "../service-core"
+    },
+    {
+      "path": "../../libs/lib-services"
     }
   ]
 }