@powersync/service-core-tests 0.13.1 → 0.14.0

package/src/tests/register-data-storage-parameter-tests.ts

@@ -1,9 +1,9 @@
-import { JwtPayload, storage } from '@powersync/service-core';
+import { JwtPayload, storage, updateSyncRulesFromYaml } from '@powersync/service-core';
 import { RequestParameters, ScopedParameterLookup, SqliteJsonRow } from '@powersync/service-sync-rules';
 import { expect, test } from 'vitest';
 import * as test_utils from '../test-utils/test-utils-index.js';
-import { TEST_TABLE } from './util.js';
-import { ParameterLookupScope } from '@powersync/service-sync-rules/src/HydrationState.js';
+import { bucketRequest, TEST_TABLE } from './util.js';
+import { ParameterLookupScope } from '@powersync/service-sync-rules';
 
 /**
  * @example
@@ -20,15 +20,15 @@ export function registerDataStorageParameterTests(generateStorageFactory: storag
 
   test('save and load parameters', async () => {
     await using factory = await generateStorageFactory();
-    const syncRules = await factory.updateSyncRules({
-      content: `
+    const syncRules = await factory.updateSyncRules(
+      updateSyncRulesFromYaml(`
 bucket_definitions:
   mybucket:
     parameters:
       - SELECT group_id FROM test WHERE id1 = token_parameters.user_id OR id2 = token_parameters.user_id
     data: []
-    `
-    });
+    `)
+    );
     const bucketStorage = factory.getInstance(syncRules);
 
     await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
@@ -70,15 +70,15 @@ bucket_definitions:
 
   test('it should use the latest version', async () => {
     await using factory = await generateStorageFactory();
-    const syncRules = await factory.updateSyncRules({
-      content: `
+    const syncRules = await factory.updateSyncRules(
+      updateSyncRulesFromYaml(`
 bucket_definitions:
   mybucket:
     parameters:
       - SELECT group_id FROM test WHERE id = token_parameters.user_id
     data: []
-    `
-    });
+    `)
+    );
     const bucketStorage = factory.getInstance(syncRules);
 
     await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
@@ -126,8 +126,8 @@ bucket_definitions:
 
   test('it should use the latest version after updates', async () => {
     await using factory = await generateStorageFactory();
-    const syncRules = await factory.updateSyncRules({
-      content: `
+    const syncRules = await factory.updateSyncRules(
+      updateSyncRulesFromYaml(`
 bucket_definitions:
   mybucket:
     parameters:
@@ -135,8 +135,8 @@ bucket_definitions:
         FROM todos
         WHERE list_id IN token_parameters.list_id
     data: []
-    `
-    });
+    `)
+    );
     const bucketStorage = factory.getInstance(syncRules);
 
     const table = test_utils.makeTestTable('todos', ['id', 'list_id']);
@@ -201,15 +201,15 @@ bucket_definitions:
 
   test('save and load parameters with different number types', async () => {
     await using factory = await generateStorageFactory();
-    const syncRules = await factory.updateSyncRules({
-      content: `
+    const syncRules = await factory.updateSyncRules(
+      updateSyncRulesFromYaml(`
 bucket_definitions:
   mybucket:
     parameters:
       - SELECT group_id FROM test WHERE n1 = token_parameters.n1 and f2 = token_parameters.f2 and f3 = token_parameters.f3
     data: []
-    `
-    });
+    `)
+    );
     const bucketStorage = factory.getInstance(syncRules);
 
     await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
@@ -251,15 +251,15 @@ bucket_definitions:
     // test this to ensure correct deserialization.
 
     await using factory = await generateStorageFactory();
-    const syncRules = await factory.updateSyncRules({
-      content: `
+    const syncRules = await factory.updateSyncRules(
+      updateSyncRulesFromYaml(`
 bucket_definitions:
   mybucket:
     parameters:
       - SELECT group_id FROM test WHERE n1 = token_parameters.n1
     data: []
-    `
-    });
+    `)
+    );
     const bucketStorage = factory.getInstance(syncRules);
 
     await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
@@ -304,16 +304,16 @@ bucket_definitions:
     const WORKSPACE_TABLE = test_utils.makeTestTable('workspace', ['id']);
 
     await using factory = await generateStorageFactory();
-    const syncRules = await factory.updateSyncRules({
-      content: `
+    const syncRules = await factory.updateSyncRules(
+      updateSyncRulesFromYaml(`
 bucket_definitions:
     by_workspace:
       parameters:
         - SELECT id as workspace_id FROM workspace WHERE
           workspace."userId" = token_parameters.user_id
       data: []
-    `
-    });
+    `)
+    );
     const sync_rules = syncRules.parsed(test_utils.PARSE_OPTIONS).hydratedSyncRules();
     const bucketStorage = factory.getInstance(syncRules);
 
@@ -345,7 +345,12 @@ bucket_definitions:
       }
     });
     expect(buckets).toEqual([
-      { bucket: 'by_workspace["workspace1"]', priority: 3, definition: 'by_workspace', inclusion_reasons: ['default'] }
+      {
+        bucket: bucketRequest(syncRules, 'by_workspace["workspace1"]'),
+        priority: 3,
+        definition: 'by_workspace',
+        inclusion_reasons: ['default']
+      }
     ]);
   });
 
@@ -353,16 +358,16 @@ bucket_definitions:
     const WORKSPACE_TABLE = test_utils.makeTestTable('workspace');
 
     await using factory = await generateStorageFactory();
-    const syncRules = await factory.updateSyncRules({
-      content: `
+    const syncRules = await factory.updateSyncRules(
+      updateSyncRulesFromYaml(`
 bucket_definitions:
     by_public_workspace:
       parameters:
         - SELECT id as workspace_id FROM workspace WHERE
           workspace.visibility = 'public'
       data: []
-    `
-    });
+    `)
+    );
     const sync_rules = syncRules.parsed(test_utils.PARSE_OPTIONS).hydratedSyncRules();
     const bucketStorage = factory.getInstance(syncRules);
 
@@ -421,13 +426,13 @@ bucket_definitions:
     buckets.sort((a, b) => a.bucket.localeCompare(b.bucket));
     expect(buckets).toEqual([
       {
-        bucket: 'by_public_workspace["workspace1"]',
+        bucket: bucketRequest(syncRules, 'by_public_workspace["workspace1"]'),
         priority: 3,
         definition: 'by_public_workspace',
         inclusion_reasons: ['default']
       },
       {
-        bucket: 'by_public_workspace["workspace3"]',
+        bucket: bucketRequest(syncRules, 'by_public_workspace["workspace3"]'),
         priority: 3,
         definition: 'by_public_workspace',
         inclusion_reasons: ['default']
@@ -439,8 +444,8 @@ bucket_definitions:
     const WORKSPACE_TABLE = test_utils.makeTestTable('workspace');
 
     await using factory = await generateStorageFactory();
-    const syncRules = await factory.updateSyncRules({
-      content: `
+    const syncRules = await factory.updateSyncRules(
+      updateSyncRulesFromYaml(`
 bucket_definitions:
     by_workspace:
       parameters:
@@ -449,8 +454,8 @@ bucket_definitions:
         - SELECT id as workspace_id FROM workspace WHERE
             workspace.user_id = token_parameters.user_id
       data: []
-    `
-    });
+    `)
+    );
     const sync_rules = syncRules.parsed(test_utils.PARSE_OPTIONS).hydratedSyncRules();
     const bucketStorage = factory.getInstance(syncRules);
 
@@ -528,20 +533,23 @@ bucket_definitions:
     expect(parameter_sets).toEqual([{ workspace_id: 'workspace1' }, { workspace_id: 'workspace3' }]);
 
     buckets.sort();
-    expect(buckets).toEqual(['by_workspace["workspace1"]', 'by_workspace["workspace3"]']);
+    expect(buckets).toEqual([
+      bucketRequest(syncRules, 'by_workspace["workspace1"]'),
+      bucketRequest(syncRules, 'by_workspace["workspace3"]')
+    ]);
   });
 
   test('truncate parameters', async () => {
     await using factory = await generateStorageFactory();
-    const syncRules = await factory.updateSyncRules({
-      content: `
+    const syncRules = await factory.updateSyncRules(
+      updateSyncRulesFromYaml(`
 bucket_definitions:
   mybucket:
     parameters:
       - SELECT group_id FROM test WHERE id1 = token_parameters.user_id OR id2 = token_parameters.user_id
     data: []
-    `
-    });
+    `)
+    );
     const bucketStorage = factory.getInstance(syncRules);
 
     await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
@@ -568,16 +576,16 @@ bucket_definitions:
 
   test('invalidate cached parsed sync rules', async () => {
     await using bucketStorageFactory = await generateStorageFactory();
-    const syncRules = await bucketStorageFactory.updateSyncRules({
-      content: `
+    const syncRules = await bucketStorageFactory.updateSyncRules(
+      updateSyncRulesFromYaml(`
 bucket_definitions:
     by_workspace:
       parameters:
         - SELECT id as workspace_id FROM workspace WHERE
           workspace."userId" = token_parameters.user_id
       data: []
-    `
-    });
+    `)
+    );
     const syncBucketStorage = bucketStorageFactory.getInstance(syncRules);
 
     const parsedSchema1 = syncBucketStorage.getParsedSyncRules({
@@ -600,4 +608,51 @@ bucket_definitions:
     expect(parsedSchema3).not.equals(parsedSchema2);
     expect(parsedSchema3.getSourceTables()[0].schema).equals('databasename');
   });
+
+  test('sync streams smoke test', async () => {
+    await using factory = await generateStorageFactory();
+    const syncRules = await factory.updateSyncRules(
+      updateSyncRulesFromYaml(`
+config:
+  edition: 3
+
+streams:
+  stream:
+    query: |
+      SELECT data.* FROM test AS data, test AS param
+      WHERE data.foo = param.bar AND param.baz = auth.user_id()
+    `)
+    );
+    const bucketStorage = factory.getInstance(syncRules);
+
+    await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+      await batch.save({
+        sourceTable: TEST_TABLE,
+        tag: storage.SaveOperationTag.INSERT,
+        after: {
+          baz: 'baz',
+          bar: 'bar'
+        },
+        afterReplicaId: test_utils.rid('t1')
+      });
+
+      await batch.commit('1/1');
+    });
+
+    const checkpoint = await bucketStorage.getCheckpoint();
+    const parameters = await checkpoint.getParameterSets([
+      ScopedParameterLookup.direct(
+        {
+          lookupName: 'lookup',
+          queryId: '0'
+        },
+        ['baz']
+      )
+    ]);
+    expect(parameters).toEqual([
+      {
+        '0': 'bar'
+      }
+    ]);
+  });
 }

package/src/tests/register-parameter-compacting-tests.ts

@@ -1,4 +1,4 @@
-import { storage } from '@powersync/service-core';
+import { storage, updateSyncRulesFromYaml } from '@powersync/service-core';
 import { ScopedParameterLookup } from '@powersync/service-sync-rules';
 import { expect, test } from 'vitest';
 import * as test_utils from '../test-utils/test-utils-index.js';
@@ -8,14 +8,14 @@ const TEST_TABLE = test_utils.makeTestTable('test', ['id']);
 export function registerParameterCompactTests(generateStorageFactory: storage.TestStorageFactory) {
   test('compacting parameters', async () => {
     await using factory = await generateStorageFactory();
-    const syncRules = await factory.updateSyncRules({
-      content: `
+    const syncRules = await factory.updateSyncRules(
+      updateSyncRulesFromYaml(`
 bucket_definitions:
   test:
     parameters: select id from test where id = request.user_id()
     data: []
-    `
-    });
+    `)
+    );
     const bucketStorage = factory.getInstance(syncRules);
 
     await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
@@ -91,14 +91,14 @@ bucket_definitions:
   for (let cacheLimit of [1, 10]) {
     test(`compacting deleted parameters with cache size ${cacheLimit}`, async () => {
       await using factory = await generateStorageFactory();
-      const syncRules = await factory.updateSyncRules({
-        content: `
+      const syncRules = await factory.updateSyncRules(
+        updateSyncRulesFromYaml(`
 bucket_definitions:
   test:
     parameters: select id from test where uid = request.user_id()
     data: []
-    `
-      });
+    `)
+      );
       const bucketStorage = factory.getInstance(syncRules);
 
       await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {

package/src/tests/register-sync-tests.ts

@@ -5,16 +5,17 @@ import {
   StreamingSyncCheckpoint,
   StreamingSyncCheckpointDiff,
   sync,
+  updateSyncRulesFromYaml,
   utils
 } from '@powersync/service-core';
 import { JSONBig } from '@powersync/service-jsonbig';
-import { BucketSourceType, RequestParameters } from '@powersync/service-sync-rules';
 import path from 'path';
 import * as timers from 'timers/promises';
 import { fileURLToPath } from 'url';
 import { expect, test } from 'vitest';
 import * as test_utils from '../test-utils/test-utils-index.js';
 import { METRICS_HELPER } from '../test-utils/test-utils-index.js';
+import { bucketRequest } from './util.js';
 
 const __filename = fileURLToPath(import.meta.url);
 const __dirname = path.dirname(__filename);
@@ -38,7 +39,7 @@ export const SYNC_SNAPSHOT_PATH = path.resolve(__dirname, '../__snapshots/sync.t
  * });
  * ```
  */
-export function registerSyncTests(factory: storage.TestStorageFactory) {
+export function registerSyncTests(factory: storage.TestStorageFactory, options: { storageVersion?: number } = {}) {
   createCoreAPIMetrics(METRICS_HELPER.metricsEngine);
   const tracker = new sync.RequestTracker(METRICS_HELPER.metricsEngine);
   const syncContext = new sync.SyncContext({
@@ -47,10 +48,19 @@ export function registerSyncTests(factory: storage.TestStorageFactory) {
     maxDataFetchConcurrency: 2
   });
 
+  const updateSyncRules = (bucketStorageFactory: storage.BucketStorageFactory, updateOptions: { content: string }) => {
+    return bucketStorageFactory.updateSyncRules(
+      updateSyncRulesFromYaml(updateOptions.content, {
+        validate: true,
+        storageVersion: options.storageVersion
+      })
+    );
+  };
+
   test('sync global data', async () => {
     await using f = await factory();
 
-    const syncRules = await f.updateSyncRules({
+    const syncRules = await updateSyncRules(f, {
       content: BASIC_SYNC_RULES
     });
 
@@ -101,7 +111,7 @@ export function registerSyncTests(factory: storage.TestStorageFactory) {
   test('sync buckets in order', async () => {
     await using f = await factory();
 
-    const syncRules = await f.updateSyncRules({
+    const syncRules = await updateSyncRules(f, {
       content: `
 bucket_definitions:
   b0:
@@ -162,7 +172,7 @@ bucket_definitions:
   test('sync interrupts low-priority buckets on new checkpoints', async () => {
     await using f = await factory();
 
-    const syncRules = await f.updateSyncRules({
+    const syncRules = await updateSyncRules(f, {
       content: `
 bucket_definitions:
   b0:
@@ -271,7 +281,7 @@ bucket_definitions:
   test('sync interruptions with unrelated data', async () => {
     await using f = await factory();
 
-    const syncRules = await f.updateSyncRules({
+    const syncRules = await updateSyncRules(f, {
       content: `
 bucket_definitions:
   b0:
@@ -409,7 +419,7 @@ bucket_definitions:
     // then interrupt checkpoint with new data for all buckets
     // -> data for all buckets should be sent in the new checkpoint
 
-    const syncRules = await f.updateSyncRules({
+    const syncRules = await updateSyncRules(f, {
       content: `
 bucket_definitions:
   b0a:
@@ -553,7 +563,7 @@ bucket_definitions:
   test('sends checkpoint complete line for empty checkpoint', async () => {
     await using f = await factory();
 
-    const syncRules = await f.updateSyncRules({
+    const syncRules = await updateSyncRules(f, {
       content: BASIC_SYNC_RULES
     });
     const bucketStorage = f.getInstance(syncRules);
@@ -616,7 +626,7 @@ bucket_definitions:
   test('sync legacy non-raw data', async () => {
     const f = await factory();
 
-    const syncRules = await f.updateSyncRules({
+    const syncRules = await updateSyncRules(f, {
       content: BASIC_SYNC_RULES
     });
 
@@ -660,7 +670,7 @@ bucket_definitions:
   test('expired token', async () => {
     await using f = await factory();
 
-    const syncRules = await f.updateSyncRules({
+    const syncRules = await updateSyncRules(f, {
       content: BASIC_SYNC_RULES
     });
 
@@ -687,7 +697,7 @@ bucket_definitions:
   test('sync updates to global data', async (context) => {
     await using f = await factory();
 
-    const syncRules = await f.updateSyncRules({
+    const syncRules = await updateSyncRules(f, {
       content: BASIC_SYNC_RULES
     });
 
@@ -753,7 +763,7 @@ bucket_definitions:
   test('sync updates to parameter query only', async (context) => {
     await using f = await factory();
 
-    const syncRules = await f.updateSyncRules({
+    const syncRules = await updateSyncRules(f, {
       content: `bucket_definitions:
   by_user:
     parameters: select users.id as user_id from users where users.id = request.user_id()
@@ -812,14 +822,14 @@ bucket_definitions:
     const checkpoint2 = await getCheckpointLines(iter);
     expect(
       (checkpoint2[0] as StreamingSyncCheckpointDiff).checkpoint_diff?.updated_buckets?.map((b) => b.bucket)
-    ).toEqual(['by_user["user1"]']);
+    ).toEqual([bucketRequest(syncRules, 'by_user["user1"]')]);
     expect(checkpoint2).toMatchSnapshot();
   });
 
   test('sync updates to data query only', async (context) => {
     await using f = await factory();
 
-    const syncRules = await f.updateSyncRules({
+    const syncRules = await updateSyncRules(f, {
       content: `bucket_definitions:
   by_user:
     parameters: select users.id as user_id from users where users.id = request.user_id()
@@ -867,7 +877,7 @@ bucket_definitions:
 
     const checkpoint1 = await getCheckpointLines(iter);
     expect((checkpoint1[0] as StreamingSyncCheckpoint).checkpoint?.buckets?.map((b) => b.bucket)).toEqual([
-      'by_user["user1"]'
+      bucketRequest(syncRules, 'by_user["user1"]')
     ]);
     expect(checkpoint1).toMatchSnapshot();
 
@@ -889,14 +899,14 @@ bucket_definitions:
     const checkpoint2 = await getCheckpointLines(iter);
     expect(
       (checkpoint2[0] as StreamingSyncCheckpointDiff).checkpoint_diff?.updated_buckets?.map((b) => b.bucket)
-    ).toEqual(['by_user["user1"]']);
+    ).toEqual([bucketRequest(syncRules, 'by_user["user1"]')]);
     expect(checkpoint2).toMatchSnapshot();
   });
 
   test('sync updates to parameter query + data', async (context) => {
     await using f = await factory();
 
-    const syncRules = await f.updateSyncRules({
+    const syncRules = await updateSyncRules(f, {
       content: `bucket_definitions:
   by_user:
     parameters: select users.id as user_id from users where users.id = request.user_id()
@@ -963,14 +973,14 @@ bucket_definitions:
     const checkpoint2 = await getCheckpointLines(iter);
     expect(
       (checkpoint2[0] as StreamingSyncCheckpointDiff).checkpoint_diff?.updated_buckets?.map((b) => b.bucket)
-    ).toEqual(['by_user["user1"]']);
+    ).toEqual([bucketRequest(syncRules, 'by_user["user1"]')]);
     expect(checkpoint2).toMatchSnapshot();
   });
 
   test('expiring token', async (context) => {
     await using f = await factory();
 
-    const syncRules = await f.updateSyncRules({
+    const syncRules = await updateSyncRules(f, {
       content: BASIC_SYNC_RULES
     });
 
@@ -1015,7 +1025,7 @@ bucket_definitions:
 
     await using f = await factory();
 
-    const syncRules = await f.updateSyncRules({
+    const syncRules = await updateSyncRules(f, {
       content: BASIC_SYNC_RULES
     });
 
@@ -1158,7 +1168,7 @@ bucket_definitions:
   test('write checkpoint', async () => {
     await using f = await factory();
 
-    const syncRules = await f.updateSyncRules({
+    const syncRules = await updateSyncRules(f, {
       content: BASIC_SYNC_RULES
     });
 
@@ -1229,7 +1239,7 @@ config:
 `;
 
     for (let i = 0; i < 2; i++) {
-      const syncRules = await f.updateSyncRules({
+      const syncRules = await updateSyncRules(f, {
         content: rules
       });
       const bucketStorage = f.getInstance(syncRules);

package/src/tests/util.ts

@@ -1,3 +1,24 @@
+import { storage } from '@powersync/service-core';
 import { test_utils } from '../index.js';
 
 export const TEST_TABLE = test_utils.makeTestTable('test', ['id']);
+
+export function bucketRequest(syncRules: storage.PersistedSyncRulesContent, bucketName: string): string {
+  if (/^\d+#/.test(bucketName)) {
+    return bucketName;
+  }
+
+  const versionedBuckets = storage.STORAGE_VERSION_CONFIG[syncRules.storageVersion]?.versionedBuckets ?? false;
+  return versionedBuckets ? `${syncRules.id}#${bucketName}` : bucketName;
+}
+
+export function bucketRequests(syncRules: storage.PersistedSyncRulesContent, bucketNames: string[]): string[] {
+  return bucketNames.map((bucketName) => bucketRequest(syncRules, bucketName));
+}
+
+export function bucketRequestMap(
+  syncRules: storage.PersistedSyncRulesContent,
+  buckets: Iterable<readonly [string, bigint]>
+): Map<string, bigint> {
+  return new Map(Array.from(buckets, ([bucketName, opId]) => [bucketRequest(syncRules, bucketName), opId]));
+}