@powersync/service-core 1.26.1 → 1.27.1

package/test/src/source_table_config.test.ts

@@ -0,0 +1,104 @@
+import { isCompatible, parsePersistedSyncConfigContent, updateSyncRulesFromConfig } from '@/storage/storage-index.js';
+import { logger } from '@powersync/lib-services-framework';
+import { PrecompiledSyncConfig, SqlSyncRules } from '@powersync/service-sync-rules';
+import { describe, expect, test } from 'vitest';
+
+const CONFIGURED = /* yaml */ `
+  # sync config fixture.
+  config:
+    edition: 3
+    source_table_options:
+      orders:
+        mongodb_filter_expression: { $eq: ['$$doc.active', true] }
+  streams:
+    orders:
+      query: SELECT * FROM orders
+`;
+
+describe('source-table options', () => {
+  test('preserves filters through compiled storage', () => {
+    const parsed = SqlSyncRules.fromYaml(CONFIGURED, { defaultSchema: 'app' });
+    const update = updateSyncRulesFromConfig(parsed);
+    const restored = parsePersistedSyncConfigContent({
+      content: CONFIGURED,
+      compiledPlan: update.config.plan!,
+      storageVersion: 2,
+      parseOptions: { defaultSchema: 'app' }
+    });
+    expect((restored.config as PrecompiledSyncConfig).plan.sourceTableConfig).toEqual({
+      orders: { mongodb_filter_expression: { $eq: ['$$doc.active', true] } }
+    });
+  });
+  test('reports invalid expressions before storage', () => {
+    expect(() =>
+      SqlSyncRules.fromYaml(CONFIGURED.replace('$$doc.active', '$doc.active'), {
+        defaultSchema: 'app'
+      })
+    ).toThrow('Expected a source field');
+  });
+  test.each([
+    ['a..orders: {}', 'Source table patterns must use'],
+    ['Users: {}, users: {}', 'resolve to the same pattern']
+  ])('persists fatal diagnostics for invalid source-table options: %s', (options, message) => {
+    const yaml = /* yaml */ `
+      # sync config fixture.
+      config:
+        edition: 3
+        source_table_options: { ${options} }
+      streams:
+        orders:
+          query: SELECT * FROM orders
+    `;
+    const parsed = SqlSyncRules.fromYaml(yaml, { defaultSchema: 'app', throwOnError: false });
+    expect(parsed.errors).toContainEqual(
+      expect.objectContaining({ type: 'fatal', message: expect.stringContaining(message) })
+    );
+    const diagnostic = parsed.errors.find((error) => error.type === 'fatal' && error.message.includes(message))!;
+    const offendingKey = options.startsWith('a..') ? 'a..orders' : 'users';
+    expect(yaml.slice(diagnostic.location.start, diagnostic.location.end)).toContain(offendingKey);
+    const update = updateSyncRulesFromConfig(parsed);
+    expect(update.config.yaml).toBe(yaml);
+    const restored = parsePersistedSyncConfigContent({
+      content: update.config.yaml,
+      compiledPlan: update.config.plan!,
+      storageVersion: 2,
+      parseOptions: { defaultSchema: 'app' }
+    });
+    expect(restored.errors.map(({ message, type, location }) => ({ message, type, location }))).toEqual(
+      parsed.errors.map(({ message, type, location }) => ({ message, type, location }))
+    );
+    expect((restored.config as PrecompiledSyncConfig).plan.sourceTableConfig ?? {}).toEqual({});
+  });
+
+  test('keeps valid filters on a partial plan with an unrelated SQL error', () => {
+    const yaml = CONFIGURED.replace('SELECT * FROM orders', 'SELECT FROM');
+    const parsed = SqlSyncRules.fromYaml(yaml, { defaultSchema: 'app', throwOnError: false });
+    expect(parsed.errors.some((error) => error.type === 'fatal')).toBe(true);
+    const update = updateSyncRulesFromConfig(parsed);
+    expect(update.config.plan!.plan.sourceTableConfig).toEqual({
+      orders: { mongodb_filter_expression: { $eq: ['$$doc.active', true] } }
+    });
+  });
+  test('requires replacement processing when filters change or are removed', () => {
+    const deploy = (yaml: string) => updateSyncRulesFromConfig(SqlSyncRules.fromYaml(yaml, { defaultSchema: 'app' }));
+    const first = deploy(CONFIGURED);
+    const same = deploy(CONFIGURED.replace('SELECT *', 'SELECT id'));
+    const changed = deploy(CONFIGURED.replace('true', 'false'));
+    const removed = deploy(
+      CONFIGURED.replace(
+        "mongodb_filter_expression: { $eq: ['$$doc.active', true] }",
+        'mongodb_filter_expression: disabled'
+      )
+    );
+    expect(isCompatible([first.config.plan], same.config, logger)).toBe(true);
+    expect(isCompatible([first.config.plan], changed.config, logger)).toBe(false);
+    expect(isCompatible([first.config.plan], removed.config, logger)).toBe(false);
+  });
+  test('requires replacement processing when saved source-table options no longer validate', () => {
+    const update = updateSyncRulesFromConfig(SqlSyncRules.fromYaml(CONFIGURED, { defaultSchema: 'app' }));
+    const existing = structuredClone(update.config.plan!);
+    // A saved expression can become unsupported after a module or version change.
+    Object.assign(existing.plan.sourceTableConfig!.orders!, { mongodb_filter_expression: { $unsupported: [] } });
+    expect(isCompatible([existing], update.config, logger)).toBe(false);
+  });
+});

package/test/src/storage/BucketStorageFactory.test.ts

@@ -0,0 +1,46 @@
+import { describe, expect, test, vi } from 'vitest';
+import * as storage from '../../../src/storage/storage-index.js';
+
+describe('logSyncConfigErrors', () => {
+  function makeLogger() {
+    return { warn: vi.fn(), error: vi.fn() };
+  }
+
+  test('logs fatal errors and warnings when the sync config is not validated', () => {
+    // Without validation, as at startup with exit_on_error: false, parsing doesn't throw on fatal errors.
+    const updateOptions = storage.updateSyncRulesFromYaml(
+      `
+bucket_definitions:
+  mybucket:
+    data:
+      - SELECT FROM
+`,
+      { validate: false }
+    );
+    const log = makeLogger();
+
+    storage.logSyncConfigErrors(updateOptions.config.parsed, log as any);
+
+    expect(log.error).toHaveBeenCalledWith(expect.stringContaining('Sync config error:'));
+    expect(log.warn).toHaveBeenCalledWith(expect.stringContaining('Sync Rules (`bucket_definitions`) are deprecated'));
+  });
+
+  test('logs nothing for a valid Sync Streams config', () => {
+    const updateOptions = storage.updateSyncRulesFromYaml(
+      `
+config:
+  edition: 3
+streams:
+  global:
+    query: SELECT id FROM test
+`,
+      { validate: true }
+    );
+    const log = makeLogger();
+
+    storage.logSyncConfigErrors(updateOptions.config.parsed, log as any);
+
+    expect(log.error).not.toHaveBeenCalled();
+    expect(log.warn).not.toHaveBeenCalled();
+  });
+});

package/test/src/sync/BucketChecksumState.test.ts

@@ -719,6 +719,195 @@ config:
       storage.updateTestChecksum({ bucket: '1#stream|0["b"]', checksum: 1, count: 1 });
     });
 
+    // Auto-subscription includes profiles in both cases. The explicit subscription adds another
+    // inclusion reason and overrides priority 3 with priority 1; it does not enable the stream.
+    test.each([false, true])(
+      'fetches overlapping static and dynamic buckets only once (custom subscription: %s)',
+      async (subscribe) => {
+        // Alice's profile is selected directly through auth.user_id() and also
+        // through her membership in her own group. Both branches resolve to the same bucket.
+        const state = checksumState(
+          /* yaml */ `
+            # Select the same profile through static and dynamic parameters.
+            config:
+              edition: 3
+            streams:
+              profiles:
+                auto_subscribe: true
+                query: |
+                  SELECT id, name FROM profiles
+                  WHERE id = auth.user_id()
+                     OR id IN (SELECT member_id FROM memberships WHERE owner_id = auth.user_id())
+          `,
+          {
+            tokenPayload: new JwtPayload({ sub: 'alice' }),
+            syncRequest: {
+              streams: {
+                subscriptions: subscribe ? [{ stream: 'profiles', parameters: null, override_priority: 1 }] : []
+              }
+            }
+          }
+        );
+        const bucket = '1#profiles|0["alice"]';
+        storage.updateTestChecksum({ bucket, checksum: 1, count: 1 });
+
+        const line = (await state.buildNextCheckpointLine({
+          // Supply the membership lookup result without a database. '0' is the generated
+          // output column for member_id in the compiled parameter query.
+          base: storage.makeCheckpoint(1n, (lookups) => [{ lookup: lookups[0], rows: [{ '0': 'alice' }] }]),
+          writeCheckpoint: null,
+          update: CHECKPOINT_INVALIDATE_ALL
+        }))!;
+        line.advance();
+
+        // Check the request arrays directly: converting them to a Map would hide duplicates.
+        // Postgres fetches operations once per request, so duplicates here duplicate the data.
+        expect(line.bucketsToFetch.map((description) => description.bucket)).toEqual([bucket]);
+        expect(line.getFilteredBucketPositions().map((request) => request.bucket)).toEqual([bucket]);
+        // Merging must retain the default and any explicit subscription, using the lowest
+        // numeric priority (the highest sync priority) among them.
+        expect(line.checkpointLine).toMatchObject({
+          checkpoint: {
+            buckets: [
+              {
+                bucket,
+                checksum: 1,
+                count: 1,
+                priority: subscribe ? 1 : 3,
+                subscriptions: expect.arrayContaining(subscribe ? [{ default: 0 }, { sub: 0 }] : [{ default: 0 }])
+              }
+            ]
+          }
+        });
+
+        // Only bucket data changes at this checkpoint, so reuse the cached membership result.
+        // The first download is still pending; its next fetch must also contain only one request.
+        // Incremental requests are already deduplicated by a Set, so also check the description:
+        // it must retain the inclusion reasons from both the static and cached dynamic matches.
+        storage.updateTestChecksum({ bucket, checksum: 2, count: 2 });
+        const cachedLine = (await state.buildNextCheckpointLine({
+          base: storage.makeCheckpoint(2n),
+          writeCheckpoint: null,
+          update: {
+            invalidateDataBuckets: false,
+            invalidateParameterBuckets: false,
+            updatedDataBuckets: new Set([bucket]),
+            updatedParameterLookups: new Set()
+          }
+        }))!;
+        cachedLine.advance();
+        expect(cachedLine.getFilteredBucketPositions().map((request) => request.bucket)).toEqual([bucket]);
+        expect(cachedLine.bucketsToFetch).toMatchObject([
+          {
+            bucket,
+            priority: subscribe ? 1 : 3,
+            inclusion_reasons: expect.arrayContaining(subscribe ? ['default', { subscription: 0 }] : ['default'])
+          }
+        ]);
+        expect(cachedLine.checkpointLine).toMatchObject({
+          checkpoint_diff: {
+            updated_buckets: [
+              {
+                bucket,
+                checksum: 2,
+                count: 2,
+                priority: subscribe ? 1 : 3,
+                subscriptions: expect.arrayContaining(subscribe ? [{ default: 0 }, { sub: 0 }] : [{ default: 0 }])
+              }
+            ]
+          }
+        });
+
+        // Simulate Alice leaving the group by returning no membership rows. The direct auth
+        // match must keep her bucket. Change its checksum so a new checkpoint line is emitted.
+        storage.updateTestChecksum({ bucket, checksum: 3, count: 3 });
+        const staticLine = (await state.buildNextCheckpointLine({
+          base: storage.makeCheckpoint(3n, (lookups) => lookups.map((lookup) => ({ lookup, rows: [] }))),
+          writeCheckpoint: null,
+          update: CHECKPOINT_INVALIDATE_ALL
+        }))!;
+        staticLine.advance();
+        expect(staticLine.getFilteredBucketPositions().map((request) => request.bucket)).toEqual([bucket]);
+      }
+    );
+
+    test('preserves static subscription metadata when a dynamic match overlaps', async () => {
+      const state = checksumState(
+        /* yaml */ `
+          # The explicit subscription contributes priority through its static match.
+          config:
+            edition: 3
+          streams:
+            profiles:
+              auto_subscribe: true
+              query: |
+                SELECT id, name FROM profiles
+                WHERE id = auth.user_id()
+                   OR id IN (
+                     SELECT member_id FROM memberships
+                     WHERE owner_id = ifnull(subscription.parameter('owner_id'), auth.user_id())
+                   )
+        `,
+        {
+          tokenPayload: new JwtPayload({ sub: 'alice' }),
+          syncRequest: {
+            streams: {
+              subscriptions: [{ stream: 'profiles', parameters: { owner_id: 'bob' }, override_priority: 1 }]
+            }
+          }
+        }
+      );
+      const bucket = '1#profiles|0["alice"]';
+
+      // Both subscriptions select Alice statically, but only the default subscription finds
+      // her through memberships. Keeping only the dynamic description would lose sub: 0 and
+      // its priority override. Check both the initial and cached incremental checkpoints.
+      for (const checkpoint of [1n, 2n]) {
+        storage.updateTestChecksum({ bucket, checksum: Number(checkpoint), count: Number(checkpoint) });
+        const line = (await state.buildNextCheckpointLine({
+          base: storage.makeCheckpoint(
+            checkpoint,
+            checkpoint === 1n
+              ? (lookups) =>
+                  lookups.map((lookup) => ({
+                    lookup,
+                    rows: lookup.values.includes('alice') ? [{ '0': 'alice' }] : []
+                  }))
+              : undefined
+          ),
+          writeCheckpoint: null,
+          update:
+            checkpoint === 1n
+              ? CHECKPOINT_INVALIDATE_ALL
+              : {
+                  invalidateDataBuckets: false,
+                  invalidateParameterBuckets: false,
+                  updatedDataBuckets: new Set([bucket]),
+                  updatedParameterLookups: new Set()
+                }
+        }))!;
+        line.advance();
+        expect(line.bucketsToFetch).toMatchObject([
+          {
+            bucket,
+            priority: 1,
+            inclusion_reasons: expect.arrayContaining(['default', { subscription: 0 }])
+          }
+        ]);
+        const descriptions =
+          'checkpoint' in line.checkpointLine
+            ? line.checkpointLine.checkpoint.buckets
+            : line.checkpointLine.checkpoint_diff.updated_buckets;
+        expect(descriptions).toMatchObject([
+          {
+            bucket,
+            priority: 1,
+            subscriptions: expect.arrayContaining([{ default: 0 }, { sub: 0 }])
+          }
+        ]);
+      }
+    });
+
     test('includes defaults', async () => {
       const state = checksumState(true);
       const line = await state.buildNextCheckpointLine({
@@ -1183,7 +1372,7 @@ streams:
       ).rejects.toThrow('Too many buckets: 60 (limit of 50');
 
       // Verify error log includes breakdown
-      expect(errorMessages[0]).toContain('(limit of 50)\nBuckets by sync stream:');
+      expect(errorMessages[0]).toContain('(limit of 50)\nBuckets by Sync Stream:');
       expect(errorMessages[0]).toContain('projects: 30');
       expect(errorMessages[0]).toContain('tasks: 20');
       expect(errorMessages[0]).toContain('comments: 10');
@@ -1242,8 +1431,8 @@ bucket_definitions:
       expect(errorMessages[0]).toContain('projects: 60');
     });
 
-    test('limits bucket breakdown to top 100 sync streams', async () => {
-      // Create 105 sync streams with dynamic parameters.
+    test('limits bucket breakdown to top 100 Sync Streams', async () => {
+      // Create 105 Sync Streams with dynamic parameters.
       let yamlDefinitions = `
 config:
   edition: 3
@@ -1301,10 +1490,10 @@ streams:
 
       // Verify only the first 100 are shown.
       const errorMessage = errorMessages[0];
-      expect(errorMessage).toContain('Buckets by sync stream:');
-      expect(errorMessage).toContain('... and 5 more buckets from 5 sync streams');
+      expect(errorMessage).toContain('Buckets by Sync Stream:');
+      expect(errorMessage).toContain('... and 5 more buckets from 5 Sync Streams');
 
-      // Count how many sync streams are listed (should be exactly 100).
+      // Count how many Sync Streams are listed (should be exactly 100).
       const defMatches = errorMessage.match(/def\d+:/g);
       expect(defMatches?.length).toBe(100);
     });

package/test/src/util/param-logging.test.ts

@@ -0,0 +1,59 @@
+import { limitParamsForLogging } from '@/util/param-logging.js';
+import { StreamingSyncRequest } from '@/util/protocol-types.js';
+import { schema } from '@powersync/lib-services-framework';
+import { deserialize } from 'bson';
+import { describe, expect, test } from 'vitest';
+
+describe('parameter logging', () => {
+  test('handles BSON Undefined accepted by the sync request validator', () => {
+    // Raw BSON is required: the serializer normally omits undefined values.
+    // This encodes { parameters: { x: undefined } } using BSON type 0x06.
+    const request = deserialize(Buffer.from('1900000003706172616d657465727300080000000678000000', 'hex'));
+    const validator = schema.createTsCodecValidator(StreamingSyncRequest, { allowAdditional: true });
+
+    expect(validator.validate(request).valid).toBe(true);
+    expect(Object.hasOwn(request.parameters, 'x')).toBe(true);
+    expect(request.parameters.x).toBeUndefined();
+    expect(limitParamsForLogging(request.parameters)).toEqual({ x: '[undefined]' });
+  });
+
+  test('handles values without a JSON representation and serialization errors', () => {
+    const circular: Record<string, unknown> = {};
+    circular.self = circular;
+
+    expect(limitParamsForLogging({ missing: undefined, fn: () => {}, symbol: Symbol(), bigint: 1n, circular })).toEqual(
+      {
+        missing: '[undefined]',
+        fn: '[undefined]',
+        symbol: '[undefined]',
+        bigint: '[unserializable]',
+        circular: '[unserializable]'
+      }
+    );
+  });
+
+  test('bounds keys and values, including JSON-encoded values', () => {
+    const params = { ['k'.repeat(100_000)]: 'v'.repeat(1000), nested: { text: 'v'.repeat(1000) }, null: null };
+    const result = limitParamsForLogging(params);
+
+    expect(result['k'.repeat(97) + '...']).toBe('v'.repeat(97) + '...');
+    expect(result.nested).toHaveLength(100);
+    expect(result.null).toBe('null');
+    expect(Object.keys(params)[0]).toHaveLength(100_000);
+  });
+
+  test('omits excess entries without serializing their values', () => {
+    const params = Object.fromEntries(Array.from({ length: 20 }, (_, i) => ['key' + i, 'value']));
+    const result = limitParamsForLogging({
+      ...params,
+      omitted: {
+        toJSON: () => {
+          throw new Error('must not serialize');
+        }
+      },
+      alsoOmitted: 'value'
+    });
+
+    expect(result).toEqual({ ...params, '⚠️': 'Additional parameters omitted' });
+  });
+});