@powersync/service-core 1.26.0 → 1.27.0

package/src/util/param-logging.ts

@@ -18,8 +18,9 @@ export const DEFAULT_PARAM_LOGGING_FORMAT_OPTIONS: ParamLoggingFormatOptions = {
  * Formats potentially arbitrary parameters for logging.
  * This limits the number of keys and strings to a maximum length.
  * A warning key-value is added if the number of keys exceeds the maximum.
- * String values exceeding the maximum length are truncated.
+ * Keys and string values exceeding the maximum length are truncated.
  * Non-String values are stringified, the maximum length is then applied.
+ * Values that cannot be serialized are replaced with a placeholder.
  * @param params - The parameters to format.
  * @param options - The options to use.
  * @default DEFAULT_PARAM_LOGGING_FORMAT_OPTIONS
@@ -41,20 +42,26 @@ export function limitParamsForLogging(
     return value;
   }
 
-  return Object.fromEntries(
-    Object.entries(params).map(([key, value], index) => {
-      if (index == maxKeyCount) {
-        return ['⚠️', 'Additional parameters omitted'];
-      }
+  function formatValue(value: unknown): string {
+    if (typeof value == 'string') {
+      return trimString(value);
+    }
+    try {
+      return trimString(JSON.stringify(value) ?? '[undefined]');
+    } catch {
+      return trimString('[unserializable]');
+    }
+  }
 
-      if (index > maxKeyCount) {
-        return [];
-      }
+  return Object.fromEntries(
+    Object.entries(params)
+      .slice(0, maxKeyCount + 1)
+      .map(([key, value], index) => {
+        if (index == maxKeyCount) {
+          return ['⚠️', 'Additional parameters omitted'];
+        }
 
-      if (typeof value == 'string') {
-        return [key, trimString(value)];
-      }
-      return [key, trimString(JSON.stringify(value))];
-    })
+        return [trimString(key), formatValue(value)];
+      })
   );
 }

package/src/util/protocol-types.ts

@@ -19,7 +19,7 @@ export type BucketRequest = t.Decoded<typeof BucketRequest>;
  */
 export const RequestedStreamSubscription = t.object({
   /**
-   * The defined name of the stream as it appears in sync stream definitions.
+   * The defined name of the stream as it appears in Sync Stream definitions.
    */
   stream: t.string,
   /**
@@ -29,7 +29,7 @@ export const RequestedStreamSubscription = t.object({
   /**
    * Set when the client wishes to re-assign a different priority to this stream.
    *
-   * Streams and sync rules can also assign a default priority, but clients are allowed to override those. This can be
+   * Sync Streams and legacy bucket definitions can also assign a default priority, but clients are allowed to override those. This can be
    * useful when the priority for partial syncs depends on e.g. the current page opened in a client.
    */
   override_priority: orNull(t.number)
@@ -49,7 +49,7 @@ export const StreamSubscriptionRequest = t.object({
   include_defaults: t.boolean.optional(),
 
   /**
-   * An array of sync streams the client has opened explicitly.
+   * An array of Sync Streams the client has opened explicitly.
    */
   subscriptions: t.array(RequestedStreamSubscription)
 });
@@ -253,8 +253,8 @@ export interface BucketChecksum {
 /**
  * The reason a particular bucket is included in a checkpoint.
  *
- * This information allows clients to associate individual buckets with sync streams they're subscribed to. Having that
- * association is useful because it enables clients to track progress for individual sync streams.
+ * This information allows clients to associate individual buckets with Sync Streams they're subscribed to. Having that
+ * association is useful because it enables clients to track progress for individual Sync Streams.
  */
 export type BucketSubscriptionReason = BucketDerivedFromDefaultStream | BucketDerivedFromExplicitSubscription;
 

package/test/src/broadcast_iterable.test.ts

@@ -4,10 +4,16 @@ import { delayEach } from 'ix/asynciterable/operators/delayeach.js';
 import { take } from 'ix/asynciterable/operators/take.js';
 import { wrapWithAbort } from 'ix/asynciterable/operators/withabort.js';
 import { toArray } from 'ix/asynciterable/toarray.js';
-import * as timers from 'timers/promises';
-import { describe, expect, it } from 'vitest';
+import { afterEach, describe, expect, it, vi } from 'vitest';
+
+// The global setTimeout, not the one from timers/promises: vi.useFakeTimers() only replaces the global timers
+const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
 
 describe('BroadcastIterable', () => {
+  afterEach(() => {
+    vi.useRealTimers();
+  });
+
   it('should iterate', async () => {
     const range = AsyncIterableX.from([1, 2, 3]);
     const broadcast = new BroadcastIterable(() => range);
@@ -47,6 +53,7 @@ describe('BroadcastIterable', () => {
   });
 
   it('should handle indefinite sources', async () => {
+    vi.useFakeTimers();
     const source: IterableSource<number> = (signal) => {
       return wrapWithAbort(interval(1), signal);
     };
@@ -55,7 +62,9 @@ describe('BroadcastIterable', () => {
 
     const delayed = delayEach(10)(broadcast);
     expect(broadcast.active).toBe(false);
-    const results = await toArray(take(5)(delayed));
+    const resultsPromise = toArray(take(5)(delayed));
+    await vi.advanceTimersByTimeAsync(100);
+    const results = await resultsPromise;
 
     expect(results.length).toEqual(5);
     expect(results[0]).toEqual(0);
@@ -66,6 +75,7 @@ describe('BroadcastIterable', () => {
   });
 
   it('should handle multiple subscribers', async () => {
+    vi.useFakeTimers();
     let sourceIndex = 0;
     const source = async function* (signal: AbortSignal) {
       // Test value out by 1000 means it may have used the wrong iteration of the source
@@ -75,7 +85,7 @@ describe('BroadcastIterable', () => {
       });
       for (let i = 0; !signal.aborted; i++) {
         yield base + i;
-        await Promise.race([abortedPromise, timers.setTimeout(1)]);
+        await Promise.race([abortedPromise, sleep(1)]);
       }
       // Test value out by 100 means this wasn't reached
       sourceIndex += 100;
@@ -88,6 +98,7 @@ describe('BroadcastIterable', () => {
     expect(broadcast.active).toBe(false);
     const results1Promise = toArray(take(5)(delayed1));
     const results2Promise = toArray(take(5)(delayed2));
+    await vi.advanceTimersByTimeAsync(100);
     const [results1, results2] = [await results1Promise, await results2Promise];
 
     expect(broadcast.active).toBe(false);
@@ -104,7 +115,9 @@ describe('BroadcastIterable', () => {
 
     // This starts a new source
     const delayed3 = delayEach(10)(broadcast);
-    const results3 = await toArray(take(5)(delayed3));
+    const results3Promise = toArray(take(5)(delayed3));
+    await vi.advanceTimersByTimeAsync(100);
+    const results3 = await results3Promise;
     expect(results3.length).toEqual(5);
     expect(results3[0]).toEqual(2100);
     expect(results3[4]).toBeGreaterThan(2110);
@@ -112,6 +125,7 @@ describe('BroadcastIterable', () => {
   });
 
   it('should handle errors on multiple subscribers', async () => {
+    vi.useFakeTimers();
     let sourceIndex = 0;
     const source = async function* (signal: AbortSignal) {
       // Test value out by 1000 means it may have used the wrong iteration of the source
@@ -124,7 +138,7 @@ describe('BroadcastIterable', () => {
           throw new Error('simulated failure');
         }
         yield base + i;
-        await Promise.race([abortedPromise, timers.setTimeout(1)]);
+        await Promise.race([abortedPromise, sleep(1)]);
       }
       // Test value out by 100 means this wasn't reached
       sourceIndex += 100;
@@ -138,7 +152,9 @@ describe('BroadcastIterable', () => {
     const results1Promise = toArray(take(5)(delayed1)) as Promise<number[]>;
     const results2Promise = toArray(take(5)(delayed2)) as Promise<number[]>;
 
-    const [r1, r2] = await Promise.allSettled([results1Promise, results2Promise]);
+    const settled = Promise.allSettled([results1Promise, results2Promise]);
+    await vi.advanceTimersByTimeAsync(100);
+    const [r1, r2] = await settled;
 
     expect(r1).toEqual({ status: 'rejected', reason: new Error('simulated failure') });
     expect(r2).toEqual({ status: 'rejected', reason: new Error('simulated failure') });
@@ -147,7 +163,9 @@ describe('BroadcastIterable', () => {
 
     // This starts a new source
     const delayed3 = delayEach(10)(broadcast);
-    const results3 = await toArray(take(5)(delayed3));
+    const results3Promise = toArray(take(5)(delayed3));
+    await vi.advanceTimersByTimeAsync(100);
+    const results3 = await results3Promise;
     expect(results3.length).toEqual(5);
     expect(results3[0]).toEqual(2000);
     expect(results3[4]).toBeGreaterThan(2010);

package/test/src/sync/BucketChecksumState.test.ts

@@ -719,6 +719,195 @@ config:
       storage.updateTestChecksum({ bucket: '1#stream|0["b"]', checksum: 1, count: 1 });
     });
 
+    // Auto-subscription includes profiles in both cases. The explicit subscription adds another
+    // inclusion reason and overrides priority 3 with priority 1; it does not enable the stream.
+    test.each([false, true])(
+      'fetches overlapping static and dynamic buckets only once (custom subscription: %s)',
+      async (subscribe) => {
+        // Alice's profile is selected directly through auth.user_id() and also
+        // through her membership in her own group. Both branches resolve to the same bucket.
+        const state = checksumState(
+          /* yaml */ `
+            # Select the same profile through static and dynamic parameters.
+            config:
+              edition: 3
+            streams:
+              profiles:
+                auto_subscribe: true
+                query: |
+                  SELECT id, name FROM profiles
+                  WHERE id = auth.user_id()
+                     OR id IN (SELECT member_id FROM memberships WHERE owner_id = auth.user_id())
+          `,
+          {
+            tokenPayload: new JwtPayload({ sub: 'alice' }),
+            syncRequest: {
+              streams: {
+                subscriptions: subscribe ? [{ stream: 'profiles', parameters: null, override_priority: 1 }] : []
+              }
+            }
+          }
+        );
+        const bucket = '1#profiles|0["alice"]';
+        storage.updateTestChecksum({ bucket, checksum: 1, count: 1 });
+
+        const line = (await state.buildNextCheckpointLine({
+          // Supply the membership lookup result without a database. '0' is the generated
+          // output column for member_id in the compiled parameter query.
+          base: storage.makeCheckpoint(1n, (lookups) => [{ lookup: lookups[0], rows: [{ '0': 'alice' }] }]),
+          writeCheckpoint: null,
+          update: CHECKPOINT_INVALIDATE_ALL
+        }))!;
+        line.advance();
+
+        // Check the request arrays directly: converting them to a Map would hide duplicates.
+        // Postgres fetches operations once per request, so duplicates here duplicate the data.
+        expect(line.bucketsToFetch.map((description) => description.bucket)).toEqual([bucket]);
+        expect(line.getFilteredBucketPositions().map((request) => request.bucket)).toEqual([bucket]);
+        // Merging must retain the default and any explicit subscription, using the lowest
+        // numeric priority (the highest sync priority) among them.
+        expect(line.checkpointLine).toMatchObject({
+          checkpoint: {
+            buckets: [
+              {
+                bucket,
+                checksum: 1,
+                count: 1,
+                priority: subscribe ? 1 : 3,
+                subscriptions: expect.arrayContaining(subscribe ? [{ default: 0 }, { sub: 0 }] : [{ default: 0 }])
+              }
+            ]
+          }
+        });
+
+        // Only bucket data changes at this checkpoint, so reuse the cached membership result.
+        // The first download is still pending; its next fetch must also contain only one request.
+        // Incremental requests are already deduplicated by a Set, so also check the description:
+        // it must retain the inclusion reasons from both the static and cached dynamic matches.
+        storage.updateTestChecksum({ bucket, checksum: 2, count: 2 });
+        const cachedLine = (await state.buildNextCheckpointLine({
+          base: storage.makeCheckpoint(2n),
+          writeCheckpoint: null,
+          update: {
+            invalidateDataBuckets: false,
+            invalidateParameterBuckets: false,
+            updatedDataBuckets: new Set([bucket]),
+            updatedParameterLookups: new Set()
+          }
+        }))!;
+        cachedLine.advance();
+        expect(cachedLine.getFilteredBucketPositions().map((request) => request.bucket)).toEqual([bucket]);
+        expect(cachedLine.bucketsToFetch).toMatchObject([
+          {
+            bucket,
+            priority: subscribe ? 1 : 3,
+            inclusion_reasons: expect.arrayContaining(subscribe ? ['default', { subscription: 0 }] : ['default'])
+          }
+        ]);
+        expect(cachedLine.checkpointLine).toMatchObject({
+          checkpoint_diff: {
+            updated_buckets: [
+              {
+                bucket,
+                checksum: 2,
+                count: 2,
+                priority: subscribe ? 1 : 3,
+                subscriptions: expect.arrayContaining(subscribe ? [{ default: 0 }, { sub: 0 }] : [{ default: 0 }])
+              }
+            ]
+          }
+        });
+
+        // Simulate Alice leaving the group by returning no membership rows. The direct auth
+        // match must keep her bucket. Change its checksum so a new checkpoint line is emitted.
+        storage.updateTestChecksum({ bucket, checksum: 3, count: 3 });
+        const staticLine = (await state.buildNextCheckpointLine({
+          base: storage.makeCheckpoint(3n, (lookups) => lookups.map((lookup) => ({ lookup, rows: [] }))),
+          writeCheckpoint: null,
+          update: CHECKPOINT_INVALIDATE_ALL
+        }))!;
+        staticLine.advance();
+        expect(staticLine.getFilteredBucketPositions().map((request) => request.bucket)).toEqual([bucket]);
+      }
+    );
+
+    test('preserves static subscription metadata when a dynamic match overlaps', async () => {
+      const state = checksumState(
+        /* yaml */ `
+          # The explicit subscription contributes priority through its static match.
+          config:
+            edition: 3
+          streams:
+            profiles:
+              auto_subscribe: true
+              query: |
+                SELECT id, name FROM profiles
+                WHERE id = auth.user_id()
+                   OR id IN (
+                     SELECT member_id FROM memberships
+                     WHERE owner_id = ifnull(subscription.parameter('owner_id'), auth.user_id())
+                   )
+        `,
+        {
+          tokenPayload: new JwtPayload({ sub: 'alice' }),
+          syncRequest: {
+            streams: {
+              subscriptions: [{ stream: 'profiles', parameters: { owner_id: 'bob' }, override_priority: 1 }]
+            }
+          }
+        }
+      );
+      const bucket = '1#profiles|0["alice"]';
+
+      // Both subscriptions select Alice statically, but only the default subscription finds
+      // her through memberships. Keeping only the dynamic description would lose sub: 0 and
+      // its priority override. Check both the initial and cached incremental checkpoints.
+      for (const checkpoint of [1n, 2n]) {
+        storage.updateTestChecksum({ bucket, checksum: Number(checkpoint), count: Number(checkpoint) });
+        const line = (await state.buildNextCheckpointLine({
+          base: storage.makeCheckpoint(
+            checkpoint,
+            checkpoint === 1n
+              ? (lookups) =>
+                  lookups.map((lookup) => ({
+                    lookup,
+                    rows: lookup.values.includes('alice') ? [{ '0': 'alice' }] : []
+                  }))
+              : undefined
+          ),
+          writeCheckpoint: null,
+          update:
+            checkpoint === 1n
+              ? CHECKPOINT_INVALIDATE_ALL
+              : {
+                  invalidateDataBuckets: false,
+                  invalidateParameterBuckets: false,
+                  updatedDataBuckets: new Set([bucket]),
+                  updatedParameterLookups: new Set()
+                }
+        }))!;
+        line.advance();
+        expect(line.bucketsToFetch).toMatchObject([
+          {
+            bucket,
+            priority: 1,
+            inclusion_reasons: expect.arrayContaining(['default', { subscription: 0 }])
+          }
+        ]);
+        const descriptions =
+          'checkpoint' in line.checkpointLine
+            ? line.checkpointLine.checkpoint.buckets
+            : line.checkpointLine.checkpoint_diff.updated_buckets;
+        expect(descriptions).toMatchObject([
+          {
+            bucket,
+            priority: 1,
+            subscriptions: expect.arrayContaining([{ default: 0 }, { sub: 0 }])
+          }
+        ]);
+      }
+    });
+
     test('includes defaults', async () => {
       const state = checksumState(true);
       const line = await state.buildNextCheckpointLine({
@@ -1183,7 +1372,7 @@ streams:
       ).rejects.toThrow('Too many buckets: 60 (limit of 50');
 
       // Verify error log includes breakdown
-      expect(errorMessages[0]).toContain('(limit of 50)\nBuckets by sync stream:');
+      expect(errorMessages[0]).toContain('(limit of 50)\nBuckets by Sync Stream:');
       expect(errorMessages[0]).toContain('projects: 30');
       expect(errorMessages[0]).toContain('tasks: 20');
       expect(errorMessages[0]).toContain('comments: 10');
@@ -1242,8 +1431,8 @@ bucket_definitions:
       expect(errorMessages[0]).toContain('projects: 60');
     });
 
-    test('limits bucket breakdown to top 100 sync streams', async () => {
-      // Create 105 sync streams with dynamic parameters.
+    test('limits bucket breakdown to top 100 Sync Streams', async () => {
+      // Create 105 Sync Streams with dynamic parameters.
       let yamlDefinitions = `
 config:
   edition: 3
@@ -1301,10 +1490,10 @@ streams:
 
       // Verify only the first 100 are shown.
       const errorMessage = errorMessages[0];
-      expect(errorMessage).toContain('Buckets by sync stream:');
-      expect(errorMessage).toContain('... and 5 more buckets from 5 sync streams');
+      expect(errorMessage).toContain('Buckets by Sync Stream:');
+      expect(errorMessage).toContain('... and 5 more buckets from 5 Sync Streams');
 
-      // Count how many sync streams are listed (should be exactly 100).
+      // Count how many Sync Streams are listed (should be exactly 100).
       const defMatches = errorMessage.match(/def\d+:/g);
       expect(defMatches?.length).toBe(100);
     });

package/test/src/sync/util.test.ts

@@ -15,6 +15,25 @@ describe('isAbortError', () => {
 });
 
 describe('acquireSemaphoreAbortable', () => {
+  test('waits for an available slot without a signal', async () => {
+    const semaphore = new Semaphore(1);
+    const [, release] = await semaphore.acquire();
+    const resolved = vi.fn();
+    const waiting = acquireSemaphoreAbortable(semaphore).then((result) => {
+      resolved();
+      return result;
+    });
+    await Promise.resolve();
+    expect(resolved).not.toHaveBeenCalled();
+
+    release();
+    const acquired = await waiting;
+    expect(acquired).not.toBe('aborted');
+    expect(semaphore.getValue()).toBe(0);
+    (acquired as [number, SemaphoreInterface.Releaser])[1]();
+    expect(semaphore.getValue()).toBe(1);
+  });
+
   test('can acquire', async () => {
     const semaphore = new Semaphore(1);
     const controller = new AbortController();

package/test/src/util/param-logging.test.ts

@@ -0,0 +1,59 @@
+import { limitParamsForLogging } from '@/util/param-logging.js';
+import { StreamingSyncRequest } from '@/util/protocol-types.js';
+import { schema } from '@powersync/lib-services-framework';
+import { deserialize } from 'bson';
+import { describe, expect, test } from 'vitest';
+
+describe('parameter logging', () => {
+  test('handles BSON Undefined accepted by the sync request validator', () => {
+    // Raw BSON is required: the serializer normally omits undefined values.
+    // This encodes { parameters: { x: undefined } } using BSON type 0x06.
+    const request = deserialize(Buffer.from('1900000003706172616d657465727300080000000678000000', 'hex'));
+    const validator = schema.createTsCodecValidator(StreamingSyncRequest, { allowAdditional: true });
+
+    expect(validator.validate(request).valid).toBe(true);
+    expect(Object.hasOwn(request.parameters, 'x')).toBe(true);
+    expect(request.parameters.x).toBeUndefined();
+    expect(limitParamsForLogging(request.parameters)).toEqual({ x: '[undefined]' });
+  });
+
+  test('handles values without a JSON representation and serialization errors', () => {
+    const circular: Record<string, unknown> = {};
+    circular.self = circular;
+
+    expect(limitParamsForLogging({ missing: undefined, fn: () => {}, symbol: Symbol(), bigint: 1n, circular })).toEqual(
+      {
+        missing: '[undefined]',
+        fn: '[undefined]',
+        symbol: '[undefined]',
+        bigint: '[unserializable]',
+        circular: '[unserializable]'
+      }
+    );
+  });
+
+  test('bounds keys and values, including JSON-encoded values', () => {
+    const params = { ['k'.repeat(100_000)]: 'v'.repeat(1000), nested: { text: 'v'.repeat(1000) }, null: null };
+    const result = limitParamsForLogging(params);
+
+    expect(result['k'.repeat(97) + '...']).toBe('v'.repeat(97) + '...');
+    expect(result.nested).toHaveLength(100);
+    expect(result.null).toBe('null');
+    expect(Object.keys(params)[0]).toHaveLength(100_000);
+  });
+
+  test('omits excess entries without serializing their values', () => {
+    const params = Object.fromEntries(Array.from({ length: 20 }, (_, i) => ['key' + i, 'value']));
+    const result = limitParamsForLogging({
+      ...params,
+      omitted: {
+        toJSON: () => {
+          throw new Error('must not serialize');
+        }
+      },
+      alsoOmitted: 'value'
+    });
+
+    expect(result).toEqual({ ...params, '⚠️': 'Additional parameters omitted' });
+  });
+});