@powersync/service-core 0.0.0-dev-20250813080357 → 0.0.0-dev-20250819134004

package/test/src/sync/BucketChecksumState.test.ts

@@ -1,17 +1,27 @@
 import {
   BucketChecksum,
   BucketChecksumState,
+  BucketChecksumStateOptions,
   BucketChecksumStateStorage,
   CHECKPOINT_INVALIDATE_ALL,
   ChecksumMap,
   InternalOpId,
   ReplicationCheckpoint,
+  StreamingSyncRequest,
   SyncContext,
   WatchFilterEvent
 } from '@/index.js';
 import { JSONBig } from '@powersync/service-jsonbig';
-import { ParameterLookup, RequestParameters, SqliteJsonRow, SqlSyncRules } from '@powersync/service-sync-rules';
-import { describe, expect, test } from 'vitest';
+import {
+  SqliteJsonRow,
+  ParameterLookup,
+  SqlSyncRules,
+  RequestJwtPayload,
+  BucketSource,
+  BucketSourceType,
+  BucketParameterQuerier
+} from '@powersync/service-sync-rules';
+import { describe, expect, test, beforeEach } from 'vitest';
 
 describe('BucketChecksumState', () => {
   // Single global[] bucket.
@@ -55,6 +65,9 @@ bucket_definitions:
     maxDataFetchConcurrency: 10
   });
 
+  const syncRequest: StreamingSyncRequest = {};
+  const tokenPayload: RequestJwtPayload = { sub: '' };
+
   test('global bucket with update', async () => {
     const storage = new MockBucketChecksumStateStorage();
     // Set intial state
@@ -62,7 +75,8 @@ bucket_definitions:
 
     const state = new BucketChecksumState({
       syncContext,
-      syncParams: new RequestParameters({ sub: '' }, {}),
+      syncRequest,
+      tokenPayload,
       syncRules: SYNC_RULES_GLOBAL,
       bucketStorage: storage
     });
@@ -75,9 +89,10 @@ bucket_definitions:
     line.advance();
     expect(line.checkpointLine).toEqual({
       checkpoint: {
-        buckets: [{ bucket: 'global[]', checksum: 1, count: 1, priority: 3 }],
+        buckets: [{ bucket: 'global[]', checksum: 1, count: 1, priority: 3, subscriptions: [{ default: 0 }] }],
         last_op_id: '1',
-        write_checkpoint: undefined
+        write_checkpoint: undefined,
+        streams: [{ name: 'global', is_default: true, errors: [] }]
       }
     });
     expect(line.bucketsToFetch).toEqual([
@@ -111,7 +126,7 @@ bucket_definitions:
     expect(line2.checkpointLine).toEqual({
       checkpoint_diff: {
         removed_buckets: [],
-        updated_buckets: [{ bucket: 'global[]', checksum: 2, count: 2, priority: 3 }],
+        updated_buckets: [{ bucket: 'global[]', checksum: 2, count: 2, priority: 3, subscriptions: [{ default: 0 }] }],
         last_op_id: '2',
         write_checkpoint: undefined
       }
@@ -129,9 +144,9 @@ bucket_definitions:
 
     const state = new BucketChecksumState({
       syncContext,
+      tokenPayload,
       // Client sets the initial state here
-      initialBucketPositions: [{ name: 'global[]', after: 1n }],
-      syncParams: new RequestParameters({ sub: '' }, {}),
+      syncRequest: { buckets: [{ name: 'global[]', after: '1' }] },
       syncRules: SYNC_RULES_GLOBAL,
       bucketStorage: storage
     });
@@ -144,9 +159,10 @@ bucket_definitions:
     line.advance();
     expect(line.checkpointLine).toEqual({
       checkpoint: {
-        buckets: [{ bucket: 'global[]', checksum: 1, count: 1, priority: 3 }],
+        buckets: [{ bucket: 'global[]', checksum: 1, count: 1, priority: 3, subscriptions: [{ default: 0 }] }],
         last_op_id: '1',
-        write_checkpoint: undefined
+        write_checkpoint: undefined,
+        streams: [{ name: 'global', is_default: true, errors: [] }]
       }
     });
     expect(line.bucketsToFetch).toEqual([
@@ -167,7 +183,8 @@ bucket_definitions:
 
     const state = new BucketChecksumState({
       syncContext,
-      syncParams: new RequestParameters({ sub: '' }, {}),
+      tokenPayload,
+      syncRequest,
       syncRules: SYNC_RULES_GLOBAL_TWO,
       bucketStorage: storage
     });
@@ -180,11 +197,12 @@ bucket_definitions:
     expect(line.checkpointLine).toEqual({
       checkpoint: {
         buckets: [
-          { bucket: 'global[1]', checksum: 1, count: 1, priority: 3 },
-          { bucket: 'global[2]', checksum: 1, count: 1, priority: 3 }
+          { bucket: 'global[1]', checksum: 1, count: 1, priority: 3, subscriptions: [{ default: 0 }] },
+          { bucket: 'global[2]', checksum: 1, count: 1, priority: 3, subscriptions: [{ default: 0 }] }
         ],
         last_op_id: '1',
-        write_checkpoint: undefined
+        write_checkpoint: undefined,
+        streams: [{ name: 'global', is_default: true, errors: [] }]
       }
     });
     expect(line.bucketsToFetch).toEqual([
@@ -215,8 +233,8 @@ bucket_definitions:
       checkpoint_diff: {
         removed_buckets: [],
         updated_buckets: [
-          { bucket: 'global[1]', checksum: 2, count: 2, priority: 3 },
-          { bucket: 'global[2]', checksum: 2, count: 2, priority: 3 }
+          { bucket: 'global[1]', checksum: 2, count: 2, priority: 3, subscriptions: [{ default: 0 }] },
+          { bucket: 'global[2]', checksum: 2, count: 2, priority: 3, subscriptions: [{ default: 0 }] }
         ],
         last_op_id: '2',
         write_checkpoint: undefined
@@ -232,9 +250,9 @@ bucket_definitions:
 
     const state = new BucketChecksumState({
       syncContext,
+      tokenPayload,
       // Client sets the initial state here
-      initialBucketPositions: [{ name: 'something_here[]', after: 1n }],
-      syncParams: new RequestParameters({ sub: '' }, {}),
+      syncRequest: { buckets: [{ name: 'something_here[]', after: '1' }] },
       syncRules: SYNC_RULES_GLOBAL,
       bucketStorage: storage
     });
@@ -249,9 +267,10 @@ bucket_definitions:
     line.advance();
     expect(line.checkpointLine).toEqual({
       checkpoint: {
-        buckets: [{ bucket: 'global[]', checksum: 1, count: 1, priority: 3 }],
+        buckets: [{ bucket: 'global[]', checksum: 1, count: 1, priority: 3, subscriptions: [{ default: 0 }] }],
         last_op_id: '1',
-        write_checkpoint: undefined
+        write_checkpoint: undefined,
+        streams: [{ name: 'global', is_default: true, errors: [] }]
       }
     });
     expect(line.bucketsToFetch).toEqual([
@@ -273,7 +292,8 @@ bucket_definitions:
 
     const state = new BucketChecksumState({
       syncContext,
-      syncParams: new RequestParameters({ sub: '' }, {}),
+      tokenPayload,
+      syncRequest,
       syncRules: SYNC_RULES_GLOBAL_TWO,
       bucketStorage: storage
     });
@@ -310,7 +330,7 @@ bucket_definitions:
         removed_buckets: [],
         updated_buckets: [
           // This does not include global[2], since it was not invalidated.
-          { bucket: 'global[1]', checksum: 2, count: 2, priority: 3 }
+          { bucket: 'global[1]', checksum: 2, count: 2, priority: 3, subscriptions: [{ default: 0 }] }
         ],
         last_op_id: '2',
         write_checkpoint: undefined
@@ -325,7 +345,8 @@ bucket_definitions:
 
     const state = new BucketChecksumState({
       syncContext,
-      syncParams: new RequestParameters({ sub: '' }, {}),
+      tokenPayload,
+      syncRequest,
       syncRules: SYNC_RULES_GLOBAL_TWO,
       bucketStorage: storage
     });
@@ -358,8 +379,8 @@ bucket_definitions:
       checkpoint_diff: {
         removed_buckets: [],
         updated_buckets: [
-          { bucket: 'global[1]', checksum: 2, count: 2, priority: 3 },
-          { bucket: 'global[2]', checksum: 2, count: 2, priority: 3 }
+          { bucket: 'global[1]', checksum: 2, count: 2, priority: 3, subscriptions: [{ default: 0 }] },
+          { bucket: 'global[2]', checksum: 2, count: 2, priority: 3, subscriptions: [{ default: 0 }] }
         ],
         last_op_id: '2',
         write_checkpoint: undefined
@@ -379,7 +400,8 @@ bucket_definitions:
 
     const state = new BucketChecksumState({
       syncContext,
-      syncParams: new RequestParameters({ sub: '' }, {}),
+      tokenPayload,
+      syncRequest,
       syncRules: SYNC_RULES_GLOBAL_TWO,
       bucketStorage: storage
     });
@@ -393,11 +415,12 @@ bucket_definitions:
     expect(line.checkpointLine).toEqual({
       checkpoint: {
         buckets: [
-          { bucket: 'global[1]', checksum: 3, count: 3, priority: 3 },
-          { bucket: 'global[2]', checksum: 3, count: 3, priority: 3 }
+          { bucket: 'global[1]', checksum: 3, count: 3, priority: 3, subscriptions: [{ default: 0 }] },
+          { bucket: 'global[2]', checksum: 3, count: 3, priority: 3, subscriptions: [{ default: 0 }] }
         ],
         last_op_id: '3',
-        write_checkpoint: undefined
+        write_checkpoint: undefined,
+        streams: [{ name: 'global', is_default: true, errors: [] }]
       }
     });
     expect(line.bucketsToFetch).toEqual([
@@ -444,7 +467,8 @@ bucket_definitions:
             bucket: 'global[1]',
             checksum: 4,
             count: 4,
-            priority: 3
+            priority: 3,
+            subscriptions: [{ default: 0 }]
           }
         ],
         last_op_id: '4',
@@ -480,7 +504,8 @@ bucket_definitions:
 
     const state = new BucketChecksumState({
       syncContext,
-      syncParams: new RequestParameters({ sub: 'u1' }, {}),
+      tokenPayload: { sub: 'u1' },
+      syncRequest,
       syncRules: SYNC_RULES_DYNAMIC,
       bucketStorage: storage
     });
@@ -496,10 +521,29 @@ bucket_definitions:
     expect(line.checkpointLine).toEqual({
       checkpoint: {
         buckets: [
-          { bucket: 'by_project[1]', checksum: 1, count: 1, priority: 3 },
-          { bucket: 'by_project[2]', checksum: 1, count: 1, priority: 3 }
+          {
+            bucket: 'by_project[1]',
+            checksum: 1,
+            count: 1,
+            priority: 3,
+            subscriptions: [{ default: 0 }]
+          },
+          {
+            bucket: 'by_project[2]',
+            checksum: 1,
+            count: 1,
+            priority: 3,
+            subscriptions: [{ default: 0 }]
+          }
         ],
         last_op_id: '1',
+        streams: [
+          {
+            is_default: true,
+            name: 'by_project',
+            errors: []
+          }
+        ],
         write_checkpoint: undefined
       }
     });
@@ -544,13 +588,301 @@ bucket_definitions:
     expect(line2.checkpointLine).toEqual({
       checkpoint_diff: {
         removed_buckets: [],
-        updated_buckets: [{ bucket: 'by_project[3]', checksum: 1, count: 1, priority: 3 }],
+        updated_buckets: [
+          {
+            bucket: 'by_project[3]',
+            checksum: 1,
+            count: 1,
+            priority: 3,
+            subscriptions: [{ default: 0 }]
+          }
+        ],
         last_op_id: '2',
         write_checkpoint: undefined
       }
     });
     expect(line2.getFilteredBucketPositions()).toEqual(new Map([['by_project[3]', 0n]]));
   });
+
+  describe('streams', () => {
+    let source: { -readonly [P in keyof BucketSource]: BucketSource[P] };
+    let storage: MockBucketChecksumStateStorage;
+    let staticBucketIds = ['stream|0[]'];
+
+    function checksumState(options?: Partial<BucketChecksumStateOptions>) {
+      const rules = new SqlSyncRules('');
+      rules.bucketSources.push(source);
+
+      return new BucketChecksumState({
+        syncContext,
+        syncRequest,
+        tokenPayload,
+        syncRules: rules,
+        bucketStorage: storage,
+        ...options
+      });
+    }
+
+    function createQuerier(ids: string[], subscription: number | null): BucketParameterQuerier {
+      return {
+        staticBuckets: ids.map((bucket) => ({
+          definition: 'stream',
+          inclusion_reasons: subscription == null ? ['default'] : [{ subscription }],
+          bucket,
+          priority: 3
+        })),
+        hasDynamicBuckets: false,
+        parameterQueryLookups: [],
+        queryDynamicBucketDescriptions: function (): never {
+          throw new Error('no dynamic buckets.');
+        }
+      };
+    }
+
+    beforeEach(() => {
+      // Currently using mocked streams before streams are actually implemented as parsable rules.
+      source = {
+        name: 'stream',
+        type: BucketSourceType.SYNC_STREAM,
+        subscribedToByDefault: false,
+        pushBucketParameterQueriers(result, options) {
+          // Create a fake querier that resolves the global stream["default"] bucket by default and allows extracting
+          // additional buckets from parameters.
+          const subscriptions = options.streams['stream'] ?? [];
+          if (!this.subscribedToByDefault && !subscriptions.length) {
+            return;
+          }
+
+          let hasExplicitDefaultSubscription = false;
+          for (const subscription of subscriptions) {
+            try {
+              let subscriptionParameters = [];
+
+              if (subscription.parameters != null) {
+                subscriptionParameters = JSON.parse(subscription.parameters['ids'] as string).map(
+                  (e: string) => `stream["${e}"]`
+                );
+              } else {
+                hasExplicitDefaultSubscription = true;
+              }
+
+              result.queriers.push(createQuerier([...subscriptionParameters], subscription.opaque_id));
+            } catch (e) {
+              result.errors.push({
+                descriptor: 'stream',
+                subscription,
+                message: `Error evaluating bucket ids: ${e.message}`
+              });
+            }
+          }
+
+          // If the stream is subscribed to by default and there is no explicit subscription that would match the default
+          // subscription, also include the default querier.
+          if (this.subscribedToByDefault && !hasExplicitDefaultSubscription) {
+            result.queriers.push(createQuerier(['stream["default"]'], null));
+          }
+        }
+      } satisfies Partial<BucketSource> as any;
+
+      storage = new MockBucketChecksumStateStorage();
+      storage.updateTestChecksum({ bucket: 'stream["default"]', checksum: 1, count: 1 });
+      storage.updateTestChecksum({ bucket: 'stream["a"]', checksum: 1, count: 1 });
+      storage.updateTestChecksum({ bucket: 'stream["b"]', checksum: 1, count: 1 });
+    });
+
+    test('includes defaults', async () => {
+      source.subscribedToByDefault = true;
+      const state = checksumState();
+
+      const line = await state.buildNextCheckpointLine({
+        base: storage.makeCheckpoint(1n),
+        writeCheckpoint: null,
+        update: CHECKPOINT_INVALIDATE_ALL
+      })!;
+      line?.advance();
+      expect(line?.checkpointLine).toEqual({
+        checkpoint: {
+          buckets: [
+            { bucket: 'stream["default"]', checksum: 1, count: 1, priority: 3, subscriptions: [{ default: 0 }] }
+          ],
+          last_op_id: '1',
+          write_checkpoint: undefined,
+          streams: [{ name: 'stream', is_default: true, errors: [] }]
+        }
+      });
+    });
+
+    test('can exclude defaults', async () => {
+      source.subscribedToByDefault = true;
+      const state = checksumState({ syncRequest: { streams: { include_defaults: false, subscriptions: [] } } });
+
+      const line = await state.buildNextCheckpointLine({
+        base: storage.makeCheckpoint(1n),
+        writeCheckpoint: null,
+        update: CHECKPOINT_INVALIDATE_ALL
+      })!;
+      line?.advance();
+      expect(line?.checkpointLine).toEqual({
+        checkpoint: {
+          buckets: [],
+          last_op_id: '1',
+          write_checkpoint: undefined,
+          streams: []
+        }
+      });
+    });
+
+    test('custom subscriptions', async () => {
+      source.subscribedToByDefault = true;
+
+      const state = checksumState({
+        syncRequest: {
+          streams: {
+            subscriptions: [
+              { stream: 'stream', parameters: { ids: '["a"]' }, override_priority: null },
+              { stream: 'stream', parameters: { ids: '["b"]' }, override_priority: 1 }
+            ]
+          }
+        }
+      });
+
+      const line = await state.buildNextCheckpointLine({
+        base: storage.makeCheckpoint(1n),
+        writeCheckpoint: null,
+        update: CHECKPOINT_INVALIDATE_ALL
+      })!;
+      line?.advance();
+      expect(line?.checkpointLine).toEqual({
+        checkpoint: {
+          buckets: [
+            { bucket: 'stream["a"]', checksum: 1, count: 1, priority: 3, subscriptions: [{ sub: 0 }] },
+            { bucket: 'stream["b"]', checksum: 1, count: 1, priority: 1, subscriptions: [{ sub: 1 }] },
+            { bucket: 'stream["default"]', checksum: 1, count: 1, priority: 3, subscriptions: [{ default: 0 }] }
+          ],
+          last_op_id: '1',
+          write_checkpoint: undefined,
+          streams: [{ name: 'stream', is_default: true, errors: [] }]
+        }
+      });
+    });
+
+    test('overlap between custom subscriptions', async () => {
+      const state = checksumState({
+        syncRequest: {
+          streams: {
+            subscriptions: [
+              { stream: 'stream', parameters: { ids: '["a", "b"]' }, override_priority: null },
+              { stream: 'stream', parameters: { ids: '["b"]' }, override_priority: 1 }
+            ]
+          }
+        }
+      });
+
+      const line = await state.buildNextCheckpointLine({
+        base: storage.makeCheckpoint(1n),
+        writeCheckpoint: null,
+        update: CHECKPOINT_INVALIDATE_ALL
+      })!;
+      line?.advance();
+      expect(line?.checkpointLine).toEqual({
+        checkpoint: {
+          buckets: [
+            { bucket: 'stream["a"]', checksum: 1, count: 1, priority: 3, subscriptions: [{ sub: 0 }] },
+            { bucket: 'stream["b"]', checksum: 1, count: 1, priority: 1, subscriptions: [{ sub: 0 }, { sub: 1 }] }
+          ],
+          last_op_id: '1',
+          write_checkpoint: undefined,
+          streams: [{ name: 'stream', is_default: false, errors: [] }]
+        }
+      });
+    });
+
+    test('overlap between default and custom subscription', async () => {
+      source.subscribedToByDefault = true;
+      const state = checksumState({
+        syncRequest: {
+          streams: {
+            subscriptions: [{ stream: 'stream', parameters: { ids: '["a", "default"]' }, override_priority: 1 }]
+          }
+        }
+      });
+
+      const line = await state.buildNextCheckpointLine({
+        base: storage.makeCheckpoint(1n),
+        writeCheckpoint: null,
+        update: CHECKPOINT_INVALIDATE_ALL
+      })!;
+      line?.advance();
+      expect(line?.checkpointLine).toEqual({
+        checkpoint: {
+          buckets: [
+            { bucket: 'stream["a"]', checksum: 1, count: 1, priority: 1, subscriptions: [{ sub: 0 }] },
+            {
+              bucket: 'stream["default"]',
+              checksum: 1,
+              count: 1,
+              priority: 1,
+              subscriptions: [{ sub: 0 }, { default: 0 }]
+            }
+          ],
+          last_op_id: '1',
+          write_checkpoint: undefined,
+          streams: [{ name: 'stream', is_default: true, errors: [] }]
+        }
+      });
+    });
+
+    test('reports errors', async () => {
+      source.subscribedToByDefault = true;
+
+      const state = checksumState({
+        syncRequest: {
+          streams: {
+            subscriptions: [
+              { stream: 'stream', parameters: { ids: '["a", "b"]' }, override_priority: 1 },
+              { stream: 'stream', parameters: { ids: 'invalid json' }, override_priority: null }
+            ]
+          }
+        }
+      });
+
+      const line = await state.buildNextCheckpointLine({
+        base: storage.makeCheckpoint(1n),
+        writeCheckpoint: null,
+        update: CHECKPOINT_INVALIDATE_ALL
+      })!;
+      line?.advance();
+      expect(line?.checkpointLine).toEqual({
+        checkpoint: {
+          buckets: [
+            { bucket: 'stream["a"]', checksum: 1, count: 1, priority: 1, subscriptions: [{ sub: 0 }] },
+            { bucket: 'stream["b"]', checksum: 1, count: 1, priority: 1, subscriptions: [{ sub: 0 }] },
+            {
+              bucket: 'stream["default"]',
+              checksum: 1,
+              count: 1,
+              priority: 3,
+              subscriptions: [{ default: 0 }]
+            }
+          ],
+          last_op_id: '1',
+          write_checkpoint: undefined,
+          streams: [
+            {
+              name: 'stream',
+              is_default: true,
+              errors: [
+                {
+                  message: 'Error evaluating bucket ids: Unexpected token \'i\', "invalid json" is not valid JSON',
+                  subscription: 1
+                }
+              ]
+            }
+          ]
+        }
+      });
+    });
+  });
 });
 
 class MockBucketChecksumStateStorage implements BucketChecksumStateStorage {