@powersync/service-core 0.0.0-dev-20260225160713 → 0.0.0-dev-20260511080634

package/test/src/sync/BucketChecksumState.test.ts

@@ -7,6 +7,7 @@ import {
   ChecksumMap,
   InternalOpId,
   JwtPayload,
+  ParameterSetLimitExceededError,
   ReplicationCheckpoint,
   StreamingSyncRequest,
   SyncContext,
@@ -14,15 +15,42 @@ import {
 } from '@/index.js';
 import { JSONBig } from '@powersync/service-jsonbig';
 import {
-  RequestJwtPayload,
+  ParameterIndexLookupCreator,
   ScopedParameterLookup,
+  SourceTableInterface,
   SqliteJsonRow,
+  SqliteRow,
   SqlSyncRules,
+  TablePattern,
   versionedHydrationState
 } from '@powersync/service-sync-rules';
+import { ParameterLookupScope } from '@powersync/service-sync-rules/src/HydrationState.js';
 import { beforeEach, describe, expect, test } from 'vitest';
 
 describe('BucketChecksumState', () => {
+  const LOOKUP_SOURCE: ParameterIndexLookupCreator = {
+    get defaultLookupScope(): ParameterLookupScope {
+      return {
+        lookupName: 'lookup',
+        queryId: '0',
+        source: LOOKUP_SOURCE
+      };
+    },
+    getSourceTables(): Set<TablePattern> {
+      return new Set();
+    },
+    evaluateParameterRow(_sourceTable: SourceTableInterface, _row: SqliteRow) {
+      return [];
+    },
+    tableSyncsParameters(_table: SourceTableInterface): boolean {
+      return false;
+    }
+  };
+
+  function lookupScope(lookupName: string, queryId: string): ParameterLookupScope {
+    return { lookupName, queryId, source: LOOKUP_SOURCE };
+  }
+
   // Single global[] bucket.
   // We don't care about data in these tests
   const SYNC_RULES_GLOBAL = SqlSyncRules.fromYaml(
@@ -67,6 +95,10 @@ bucket_definitions:
   const syncRequest: StreamingSyncRequest = {};
   const tokenPayload = new JwtPayload({ sub: '' });
 
+  function bucketStarts(requests: { bucket: string; start: InternalOpId }[]) {
+    return new Map(requests.map((request) => [request.bucket, request.start]));
+  }
+
   test('global bucket with update', async () => {
     const storage = new MockBucketChecksumStateStorage();
     // Set intial state
@@ -94,14 +126,14 @@ bucket_definitions:
         streams: [{ name: 'global', is_default: true, errors: [] }]
       }
     });
-    expect(line.bucketsToFetch).toEqual([
+    expect(line.bucketsToFetch).toMatchObject([
       {
         bucket: '1#global[]',
         priority: 3
       }
     ]);
     // This is the bucket data to be fetched
-    expect(line.getFilteredBucketPositions()).toEqual(new Map([['1#global[]', 0n]]));
+    expect(bucketStarts(line.getFilteredBucketPositions())).toEqual(new Map([['1#global[]', 0n]]));
 
     // This similuates the bucket data being sent
     line.advance();
@@ -132,7 +164,7 @@ bucket_definitions:
         write_checkpoint: undefined
       }
     });
-    expect(line2.getFilteredBucketPositions()).toEqual(new Map([['1#global[]', 1n]]));
+    expect(bucketStarts(line2.getFilteredBucketPositions())).toEqual(new Map([['1#global[]', 1n]]));
   });
 
   test('global bucket with initial state', async () => {
@@ -166,14 +198,14 @@ bucket_definitions:
         streams: [{ name: 'global', is_default: true, errors: [] }]
       }
     });
-    expect(line.bucketsToFetch).toEqual([
+    expect(line.bucketsToFetch).toMatchObject([
       {
         bucket: '1#global[]',
         priority: 3
       }
     ]);
     // This is the main difference between this and the previous test
-    expect(line.getFilteredBucketPositions()).toEqual(new Map([['1#global[]', 1n]]));
+    expect(bucketStarts(line.getFilteredBucketPositions())).toEqual(new Map([['1#global[]', 1n]]));
   });
 
   test('multiple static buckets', async () => {
@@ -206,7 +238,7 @@ bucket_definitions:
         streams: [{ name: 'global', is_default: true, errors: [] }]
       }
     });
-    expect(line.bucketsToFetch).toEqual([
+    expect(line.bucketsToFetch).toMatchObject([
       {
         bucket: '2#global[1]',
         priority: 3
@@ -274,13 +306,13 @@ bucket_definitions:
         streams: [{ name: 'global', is_default: true, errors: [] }]
       }
     });
-    expect(line.bucketsToFetch).toEqual([
+    expect(line.bucketsToFetch).toMatchObject([
       {
         bucket: '1#global[]',
         priority: 3
       }
     ]);
-    expect(line.getFilteredBucketPositions()).toEqual(new Map([['1#global[]', 0n]]));
+    expect(bucketStarts(line.getFilteredBucketPositions())).toEqual(new Map([['1#global[]', 0n]]));
   });
 
   test('invalidating individual bucket', async () => {
@@ -337,7 +369,7 @@ bucket_definitions:
         write_checkpoint: undefined
       }
     });
-    expect(line2.bucketsToFetch).toEqual([{ bucket: '2#global[1]', priority: 3 }]);
+    expect(line2.bucketsToFetch).toMatchObject([{ bucket: '2#global[1]', priority: 3 }]);
   });
 
   test('invalidating all buckets', async () => {
@@ -387,7 +419,7 @@ bucket_definitions:
         write_checkpoint: undefined
       }
     });
-    expect(line2.bucketsToFetch).toEqual([
+    expect(line2.bucketsToFetch).toMatchObject([
       { bucket: '2#global[1]', priority: 3 },
       { bucket: '2#global[2]', priority: 3 }
     ]);
@@ -424,7 +456,7 @@ bucket_definitions:
         streams: [{ name: 'global', is_default: true, errors: [] }]
       }
     });
-    expect(line.bucketsToFetch).toEqual([
+    expect(line.bucketsToFetch).toMatchObject([
       {
         bucket: '2#global[1]',
         priority: 3
@@ -436,7 +468,7 @@ bucket_definitions:
     ]);
 
     // This is the bucket data to be fetched
-    expect(line.getFilteredBucketPositions()).toEqual(
+    expect(bucketStarts(line.getFilteredBucketPositions())).toEqual(
       new Map([
         ['2#global[1]', 0n],
         ['2#global[2]', 0n]
@@ -477,7 +509,7 @@ bucket_definitions:
       }
     });
     // This should contain both buckets, even though only one changed.
-    expect(line2.bucketsToFetch).toEqual([
+    expect(line2.bucketsToFetch).toMatchObject([
       {
         bucket: '2#global[1]',
         priority: 3
@@ -488,7 +520,7 @@ bucket_definitions:
       }
     ]);
 
-    expect(line2.getFilteredBucketPositions()).toEqual(
+    expect(bucketStarts(line2.getFilteredBucketPositions())).toEqual(
       new Map([
         ['2#global[1]', 3n],
         ['2#global[2]', 1n]
@@ -513,7 +545,7 @@ bucket_definitions:
 
     const line = (await state.buildNextCheckpointLine({
       base: storage.makeCheckpoint(1n, (lookups) => {
-        expect(lookups).toEqual([ScopedParameterLookup.direct({ lookupName: 'by_project', queryId: '1' }, ['u1'])]);
+        expect(lookups).toEqual([ScopedParameterLookup.direct(lookupScope('by_project', '1'), ['u1'])]);
         return [{ id: 1 }, { id: 2 }];
       }),
       writeCheckpoint: null,
@@ -548,7 +580,7 @@ bucket_definitions:
         write_checkpoint: undefined
       }
     });
-    expect(line.bucketsToFetch).toEqual([
+    expect(line.bucketsToFetch).toMatchObject([
       {
         bucket: '3#by_project[1]',
         priority: 3
@@ -560,7 +592,7 @@ bucket_definitions:
     ]);
     line.advance();
     // This is the bucket data to be fetched
-    expect(line.getFilteredBucketPositions()).toEqual(
+    expect(bucketStarts(line.getFilteredBucketPositions())).toEqual(
       new Map([
         ['3#by_project[1]', 0n],
         ['3#by_project[2]', 0n]
@@ -574,7 +606,7 @@ bucket_definitions:
     // Now we get a new line
     const line2 = (await state.buildNextCheckpointLine({
       base: storage.makeCheckpoint(2n, (lookups) => {
-        expect(lookups).toEqual([ScopedParameterLookup.direct({ lookupName: 'by_project', queryId: '1' }, ['u1'])]);
+        expect(lookups).toEqual([ScopedParameterLookup.direct(lookupScope('by_project', '1'), ['u1'])]);
         return [{ id: 1 }, { id: 2 }, { id: 3 }];
       }),
       writeCheckpoint: null,
@@ -602,7 +634,7 @@ bucket_definitions:
         write_checkpoint: undefined
       }
     });
-    expect(line2.getFilteredBucketPositions()).toEqual(new Map([['3#by_project[3]', 0n]]));
+    expect(bucketStarts(line2.getFilteredBucketPositions())).toEqual(new Map([['3#by_project[3]', 0n]]));
   });
 
   describe('streams', () => {
@@ -869,19 +901,127 @@ config:
       expect(logData[0].parameter_query_results).toBe(3);
     });
 
-    test('throws error with breakdown when parameter query limit is exceeded', async () => {
+    test('throws error with breakdown when too many parameters were fetched', async () => {
+      const syncRules = SqlSyncRules.fromYaml(
+        `
+config:
+  edition: 3
+
+streams:
+  a:
+    auto_subscribe: true
+    query: SELECT * FROM a WHERE id IN (SELECT id FROM magic_sequence WHERE count0 = auth.parameter('a'))
+  b:
+    auto_subscribe: true
+    query: SELECT * FROM a WHERE id IN (SELECT id FROM magic_sequence WHERE count1 = auth.parameter('b'))
+  c:
+    auto_subscribe: true
+    query: SELECT * FROM a WHERE id IN (SELECT id FROM magic_sequence WHERE count1 = auth.parameter('c'))
+`,
+        { defaultSchema: 'public' }
+      ).config.hydrate({
+        hydrationState: versionedHydrationState(1)
+      });
+
+      const storage = new MockBucketChecksumStateStorage();
+
+      const errorMessages: string[] = [];
+      const errorData: any[] = [];
+      const mockLogger = {
+        info: () => {},
+        error: (message: string, data: any) => {
+          errorMessages.push(message);
+          errorData.push(data);
+        },
+        warn: () => {},
+        debug: () => {}
+      };
+
+      const smallContext = new SyncContext({
+        maxBuckets: 100,
+        maxParameterQueryResults: 55,
+        maxDataFetchConcurrency: 10
+      });
+
+      const state = new BucketChecksumState({
+        syncContext: smallContext,
+        tokenPayload: new JwtPayload({
+          sub: 'u1',
+          // Create 60 total results: 30 a + 20 b + 10 c
+          a: BigInt(30),
+          b: BigInt(20),
+          c: BigInt(10)
+        }),
+        syncRequest,
+        syncRules,
+        bucketStorage: storage,
+        logger: mockLogger as any
+      });
+
+      await expect(
+        state.buildNextCheckpointLine({
+          base: storage.makeCheckpoint(1n, (lookups, limit) => {
+            expect(lookups).toHaveLength(1);
+            const [{ values }] = lookups;
+            expect(values[0]).toStrictEqual('lookup');
+
+            const count = Number(values[2]); // The count parameter from streams
+            if (count > limit) {
+              throw new ParameterSetLimitExceededError(limit);
+            }
+            return Array.from({ length: count }, (_, i) => ({ '0': BigInt(i) }));
+          }),
+          writeCheckpoint: null,
+          update: CHECKPOINT_INVALIDATE_ALL
+        })
+      ).rejects.toThrow('Too many parameter query results (limit of 55)');
+
+      // Verify error log includes breakdown
+      expect(errorMessages[0]).toContain('Invoked parameter queries by definition:');
+      expect(errorMessages[0]).toContain('Stream a evaluating parameter on magic_sequence: 30 results.');
+      expect(errorMessages[0]).toContain('Stream b evaluating parameter on magic_sequence: 20 results.');
+      expect(errorMessages[0]).toContain(
+        'Stream c evaluating parameter on magic_sequence exceeded the remaining limit of 5 available results.'
+      );
+
+      expect(errorData[0].checkpoint).toEqual(1n);
+      expect(errorData[0].parameterResults).toEqual([
+        {
+          definition: 'Stream a evaluating parameter on magic_sequence',
+          didExceedLimit: false,
+          resultsOrLimit: 30
+        },
+        {
+          definition: 'Stream b evaluating parameter on magic_sequence',
+          didExceedLimit: false,
+          resultsOrLimit: 20
+        },
+        {
+          definition: 'Stream c evaluating parameter on magic_sequence',
+          didExceedLimit: true,
+          resultsOrLimit: 5
+        }
+      ]);
+    });
+
+    test('throws error with breakdown when bucket limit is exceeded', async () => {
+      // These streams are designed to return buckets without consuming too many parameters (as exceeding that limit is
+      // a different error).
       const SYNC_RULES_MULTI = SqlSyncRules.fromYaml(
         `
-bucket_definitions:
+config:
+  edition: 3
+
+streams:
   projects:
-    parameters: select id from projects where user_id = request.user_id()
-    data: []
+    auto_subscribe: true
+    query: SELECT id FROM projects WHERE p IN auth.parameter('a')
   tasks:
-    parameters: select id from tasks where user_id = request.user_id()
-    data: []
+    auto_subscribe: true
+    query: SELECT id FROM tasks WHERE p IN auth.parameter('b')
   comments:
-    parameters: select id from comments where user_id = request.user_id()
-    data: []
+    auto_subscribe: true
+    query: SELECT id FROM comments WHERE p IN auth.parameter('c')
     `,
         { defaultSchema: 'public' }
       ).config.hydrate({ hydrationState: versionedHydrationState(4) });
@@ -901,74 +1041,66 @@ bucket_definitions:
       };
 
       const smallContext = new SyncContext({
-        maxBuckets: 100,
-        maxParameterQueryResults: 50,
+        maxBuckets: 50,
+        maxParameterQueryResults: 10,
         maxDataFetchConcurrency: 10
       });
 
       const state = new BucketChecksumState({
         syncContext: smallContext,
-        tokenPayload: new JwtPayload({ sub: 'u1' }),
+        tokenPayload: new JwtPayload({
+          sub: 'u1',
+          // Create 60 total results: 30 projects + 20 tasks + 10 comments
+          a: Array.from({ length: 30 }, (_, i) => BigInt(i)),
+          b: Array.from({ length: 20 }, (_, i) => BigInt(i)),
+          c: Array.from({ length: 10 }, (_, i) => BigInt(i))
+        }),
         syncRequest,
         syncRules: SYNC_RULES_MULTI,
         bucketStorage: storage,
         logger: mockLogger as any
       });
 
-      // Create 60 total results: 30 projects + 20 tasks + 10 comments
-      const projectIds = Array.from({ length: 30 }, (_, i) => ({ id: i + 1 }));
-      const taskIds = Array.from({ length: 20 }, (_, i) => ({ id: i + 1 }));
-      const commentIds = Array.from({ length: 10 }, (_, i) => ({ id: i + 1 }));
-
-      for (let i = 1; i <= 30; i++) {
-        storage.updateTestChecksum({ bucket: `projects[${i}]`, checksum: 1, count: 1 });
-      }
-      for (let i = 1; i <= 20; i++) {
-        storage.updateTestChecksum({ bucket: `tasks[${i}]`, checksum: 1, count: 1 });
-      }
-      for (let i = 1; i <= 10; i++) {
-        storage.updateTestChecksum({ bucket: `comments[${i}]`, checksum: 1, count: 1 });
-      }
-
+      let invokedParameterQueries = 0;
       await expect(
         state.buildNextCheckpointLine({
-          base: storage.makeCheckpoint(1n, (lookups) => {
-            const lookup = lookups[0];
-            const lookupName = lookup.values[0];
-            if (lookupName === 'projects') {
-              return projectIds;
-            } else if (lookupName === 'tasks') {
-              return taskIds;
-            } else {
-              return commentIds;
-            }
+          base: storage.makeCheckpoint(1n, () => {
+            invokedParameterQueries++;
+            return [{ '0': 'user' }];
           }),
           writeCheckpoint: null,
           update: CHECKPOINT_INVALIDATE_ALL
         })
-      ).rejects.toThrow('Too many parameter query results: 60 (limit of 50)');
+      ).rejects.toThrow('Too many buckets: 60 (limit of 50');
+      expect(invokedParameterQueries).toStrictEqual(0);
 
       // Verify error log includes breakdown
-      expect(errorMessages[0]).toContain('Parameter query results by definition:');
+      expect(errorMessages[0]).toContain('Buckets by definition:');
       expect(errorMessages[0]).toContain('projects: 30');
       expect(errorMessages[0]).toContain('tasks: 20');
       expect(errorMessages[0]).toContain('comments: 10');
 
-      expect(errorData[0].parameter_query_results).toBe(60);
-      expect(errorData[0].parameter_query_results_by_definition).toEqual({
+      expect(errorData[0].checkpoint).toEqual(1n);
+      expect(errorData[0].buckets).toBe(60);
+      expect(errorData[0].buckets_by_definition).toEqual({
         projects: 30,
         tasks: 20,
         comments: 10
       });
     });
 
-    test('limits breakdown to top 10 definitions', async () => {
-      // Create sync rules with 15 different definitions with dynamic parameters
-      let yamlDefinitions = 'bucket_definitions:\n';
+    test('limits bucket breakdown to top 10 definitions', async () => {
+      // Create sync streams with 15 different definitions with dynamic parameters
+      let yamlDefinitions = `
+config:
+  edition: 3
+
+streams:
+`;
       for (let i = 1; i <= 15; i++) {
         yamlDefinitions += `  def${i}:\n`;
-        yamlDefinitions += `    parameters: select id from def${i}_table where user_id = request.user_id()\n`;
-        yamlDefinitions += `    data: []\n`;
+        yamlDefinitions += `    auto_subscribe: true\n`;
+        yamlDefinitions += `    query: SELECT * FROM tbl WHERE b = auth.parameter('${i}')\n`;
       }
 
       const SYNC_RULES_MANY = SqlSyncRules.fromYaml(yamlDefinitions, { defaultSchema: 'public' }).config.hydrate({
@@ -988,35 +1120,32 @@ bucket_definitions:
       };
 
       const smallContext = new SyncContext({
-        maxBuckets: 100,
+        maxBuckets: 10,
         maxParameterQueryResults: 10,
         maxDataFetchConcurrency: 10
       });
 
       const state = new BucketChecksumState({
         syncContext: smallContext,
-        tokenPayload: new JwtPayload({ sub: 'u1' }),
+        tokenPayload: new JwtPayload({
+          sub: 'u1',
+          ...Object.fromEntries(Array.from({ length: 30 }, (_, i) => [i.toString(), BigInt(i)]))
+        }),
         syncRequest,
         syncRules: SYNC_RULES_MANY,
         bucketStorage: storage,
         logger: mockLogger as any
       });
 
-      // Each definition creates one bucket, total 15 buckets
-      for (let i = 1; i <= 15; i++) {
-        storage.updateTestChecksum({ bucket: `def${i}[${i}]`, checksum: 1, count: 1 });
-      }
-
       await expect(
         state.buildNextCheckpointLine({
           base: storage.makeCheckpoint(1n, (lookups) => {
-            // Return one result for each definition
-            return [{ id: 1 }];
+            return [{ '0': 'user' }];
           }),
           writeCheckpoint: null,
           update: CHECKPOINT_INVALIDATE_ALL
         })
-      ).rejects.toThrow('Too many parameter query results: 15 (limit of 10)');
+      ).rejects.toThrow('Too many buckets: 15 (limit of 10)');
 
       // Verify only top 10 are shown
       const errorMessage = errorMessages[0];
@@ -1044,9 +1173,9 @@ class MockBucketChecksumStateStorage implements BucketChecksumStateStorage {
     this.filter?.({ invalidate: true });
   }
 
-  async getChecksums(checkpoint: InternalOpId, buckets: string[]): Promise<ChecksumMap> {
+  async getChecksums(_checkpoint: InternalOpId, buckets: { bucket: string }[]): Promise<ChecksumMap> {
     return new Map<string, BucketChecksum>(
-      buckets.map((bucket) => {
+      buckets.map(({ bucket }) => {
         const checksum = this.state.get(bucket);
         return [
           bucket,
@@ -1062,16 +1191,16 @@ class MockBucketChecksumStateStorage implements BucketChecksumStateStorage {
 
   makeCheckpoint(
     opId: InternalOpId,
-    parameters?: (lookups: ScopedParameterLookup[]) => SqliteJsonRow[]
+    parameters?: (lookups: ScopedParameterLookup[], limit: number) => SqliteJsonRow[]
   ): ReplicationCheckpoint {
     return {
       checkpoint: opId,
       lsn: String(opId),
-      getParameterSets: async (lookups: ScopedParameterLookup[]) => {
+      getParameterSets: async (lookups: ScopedParameterLookup[], limit) => {
         if (parameters == null) {
           throw new Error(`getParametersSets not defined for checkpoint ${opId}`);
         }
-        return parameters(lookups);
+        return parameters(lookups, limit);
       }
     };
   }

package/test/src/util/protocol_types.test.ts

@@ -1,6 +1,6 @@
 import { StreamingSyncRequest } from '@/index.js';
 import { schema } from '@powersync/lib-services-framework';
-import { describe, test, expect, it } from 'vitest';
+import { describe, expect, test } from 'vitest';
 
 describe('protocol types', () => {
   describe('StreamingSyncRequest', () => {

package/test/src/utils.ts

@@ -0,0 +1,9 @@
+/**
+ * Removes the source property from an object.
+ *
+ * This is for tests where we don't care about this value, and it adds a lot of noise in the output.
+ */
+export function removeSource<T extends { source?: any }>(obj: T): Omit<T, 'source'> {
+  const { source, ...rest } = obj;
+  return rest;
+}

package/test/tsconfig.json

@@ -1,7 +1,6 @@
 {
   "extends": "../../../tsconfig.tests.json",
   "compilerOptions": {
-    "baseUrl": "./",
     "outDir": "dist",
     "paths": {
       "@/*": ["../src/*"]