@powersync/service-core 1.23.3 → 1.25.0

package/test/src/source-table-reconciler.test.ts

@@ -0,0 +1,244 @@
+import { SourceEntityDescriptor } from '@/storage/SourceEntity.js';
+import { SourceTable, sourceTableIdEquals } from '@/storage/SourceTable.js';
+import {
+  defaultSourceTableReconciler,
+  diffSourceTableUpdates,
+  materializeSourceTableResolution,
+  sourceIdentityCompatible,
+  validateSourceTableCandidateResolution
+} from '@/storage/SourceTableReconciler.js';
+import * as bson from 'bson';
+import { describe, expect, it } from 'vitest';
+
+function descriptor(overrides: Partial<SourceEntityDescriptor> = {}): SourceEntityDescriptor {
+  return {
+    connectionTag: 'default',
+    schema: 'public',
+    name: 'users',
+    objectId: 100,
+    replicaIdColumns: [{ name: 'id', type: 'int', typeId: 23 }],
+    ...overrides
+  };
+}
+
+function candidate(overrides: Partial<ConstructorParameters<typeof SourceTable>[0]> = {}): SourceTable {
+  return new SourceTable({
+    id: overrides.id ?? 'table-1',
+    ref: overrides.ref ?? { connectionTag: 'default', schema: 'public', name: 'users' },
+    objectId: 'objectId' in overrides ? overrides.objectId! : 100,
+    replicaIdColumns: overrides.replicaIdColumns ?? [{ name: 'id', type: 'int', typeId: 23 }],
+    snapshotComplete: overrides.snapshotComplete ?? true,
+    bucketDataSources: overrides.bucketDataSources ?? [],
+    parameterLookupSources: overrides.parameterLookupSources ?? [],
+    sourceMetadata: overrides.sourceMetadata
+  });
+}
+
+describe('sourceIdentityCompatible', () => {
+  it('matches identical identity', () => {
+    expect(sourceIdentityCompatible(descriptor(), candidate())).toBe(true);
+  });
+
+  it('rejects a different object id', () => {
+    expect(sourceIdentityCompatible(descriptor({ objectId: 200 }), candidate({ objectId: 100 }))).toBe(false);
+  });
+
+  it('rejects a different schema/name', () => {
+    expect(sourceIdentityCompatible(descriptor({ name: 'accounts' }), candidate())).toBe(false);
+  });
+
+  it('rejects changed replica-id columns', () => {
+    expect(
+      sourceIdentityCompatible(
+        descriptor(),
+        candidate({ replicaIdColumns: [{ name: 'id', type: 'bigint', typeId: 20 }] })
+      )
+    ).toBe(false);
+  });
+
+  it('treats an undefined descriptor object id as a wildcard on object id', () => {
+    expect(sourceIdentityCompatible(descriptor({ objectId: undefined }), candidate({ objectId: 999 }))).toBe(true);
+  });
+});
+
+describe('defaultSourceTableReconciler', () => {
+  it('returns all identity-compatible candidates and no metadata', async () => {
+    const a = candidate({ id: 'a' });
+    const b = candidate({ id: 'b', ref: { connectionTag: 'default', schema: 'public', name: 'accounts' } });
+    const resolution = await defaultSourceTableReconciler({ source: descriptor(), candidates: [a, b] });
+    expect(resolution.compatibleTables).toEqual([a]);
+    expect(resolution.incompatibleTables).toEqual([b]);
+    expect(resolution.newTableValues).toEqual({ sourceMetadata: null });
+  });
+
+  it('returns an empty set when nothing matches', async () => {
+    const resolution = await defaultSourceTableReconciler({
+      source: descriptor({ objectId: 200 }),
+      candidates: [candidate({ objectId: 100 })]
+    });
+    expect(resolution.compatibleTables).toHaveLength(0);
+    expect(resolution.incompatibleTables.map((table) => table.id)).toEqual(['table-1']);
+  });
+});
+
+describe('validateSourceTableCandidateResolution', () => {
+  it('accepts an explicit compatible/incompatible partition', () => {
+    const a = candidate({ id: 'a' });
+    const b = candidate({ id: 'b' });
+    expect(() =>
+      validateSourceTableCandidateResolution([a, b], {
+        compatibleTables: [a],
+        incompatibleTables: [b],
+        newTableValues: { sourceMetadata: null }
+      })
+    ).not.toThrow();
+  });
+
+  it('rejects omitted, duplicate, and unknown candidates', () => {
+    const a = candidate({ id: 'a' });
+    expect(() =>
+      validateSourceTableCandidateResolution([a], {
+        compatibleTables: [],
+        incompatibleTables: [],
+        newTableValues: { sourceMetadata: null }
+      })
+    ).toThrow(/exactly once/);
+    expect(() =>
+      validateSourceTableCandidateResolution([a], {
+        compatibleTables: [a],
+        incompatibleTables: [a],
+        newTableValues: { sourceMetadata: null }
+      })
+    ).toThrow(/exactly once/);
+    expect(() =>
+      validateSourceTableCandidateResolution([a], {
+        compatibleTables: [a],
+        incompatibleTables: [candidate({ id: 'unknown' })],
+        newTableValues: { sourceMetadata: null }
+      })
+    ).toThrow(/unknown candidate/);
+  });
+});
+
+describe('diffSourceTableUpdates', () => {
+  function resolution(compatibleTables: SourceTable[]) {
+    return { compatibleTables, incompatibleTables: [], newTableValues: { sourceMetadata: null } };
+  }
+
+  it('returns nothing when the reconciler returned the candidates untouched', () => {
+    const a = candidate({ id: 'a', sourceMetadata: { captureTableObjectId: 7 } });
+    expect(diffSourceTableUpdates([a], resolution([a]))).toEqual([]);
+  });
+
+  it('compares by value, not by reference', () => {
+    // A new object with equal metadata should not cause a write.
+    const a = candidate({ id: 'a', sourceMetadata: { captureTableObjectId: 7 } });
+    const rebuilt = a.withSourceMetadata({ captureTableObjectId: 7 });
+
+    expect(rebuilt.sourceMetadata).not.toBe(a.sourceMetadata);
+    expect(diffSourceTableUpdates([a], resolution([rebuilt]))).toEqual([]);
+  });
+
+  it('returns changed metadata', () => {
+    const a = candidate({ id: 'a', sourceMetadata: { captureTableObjectId: 7 } });
+    expect(diffSourceTableUpdates([a], resolution([a.withSourceMetadata({ captureTableObjectId: 8 })]))).toEqual([
+      { id: 'a', sourceMetadata: { captureTableObjectId: 8 } }
+    ]);
+  });
+
+  it('returns metadata added to a legacy record', () => {
+    const legacy = candidate({ id: 'a' });
+    expect(legacy.sourceMetadata).toBeNull();
+    expect(
+      diffSourceTableUpdates([legacy], resolution([legacy.withSourceMetadata({ captureTableObjectId: 7 })]))
+    ).toEqual([{ id: 'a', sourceMetadata: { captureTableObjectId: 7 } }]);
+  });
+
+  it('returns cleared metadata as null', () => {
+    const a = candidate({ id: 'a', sourceMetadata: { captureTableObjectId: 7 } });
+    expect(diffSourceTableUpdates([a], resolution([a.withSourceMetadata(null)]))).toEqual([
+      { id: 'a', sourceMetadata: null }
+    ]);
+  });
+
+  it('materializes only source metadata from reconciler results', () => {
+    const persisted = candidate({ id: 'a', snapshotComplete: false });
+    const modified = persisted.withSourceMetadata({ captureTableObjectId: 8 });
+    modified.snapshotComplete = true;
+    modified.syncData = false;
+
+    const [materialized] = materializeSourceTableResolution([persisted], resolution([modified])).compatibleTables;
+
+    expect(materialized.sourceMetadata).toEqual({ captureTableObjectId: 8 });
+    expect(materialized.snapshotComplete).toBe(false);
+    expect(materialized.syncData).toBe(persisted.syncData);
+  });
+
+  it('does not trust incompatible candidate mutations made through a cast', () => {
+    const persisted = candidate({ id: 'a', snapshotComplete: false });
+    const exposed = persisted.clone();
+    // Simulate callback code bypassing the read-only type boundary.
+    (exposed as any).snapshotComplete = true;
+
+    const materialized = materializeSourceTableResolution([persisted], {
+      compatibleTables: [],
+      incompatibleTables: [exposed],
+      newTableValues: { sourceMetadata: null }
+    }).incompatibleTables[0];
+
+    expect(exposed.snapshotComplete).toBe(true);
+    expect(materialized).toBe(persisted);
+    expect(materialized.snapshotComplete).toBe(false);
+    expect(materialized.syncData).toBe(persisted.syncData);
+  });
+
+  it('only reports the candidates that changed', () => {
+    const a = candidate({ id: 'a', sourceMetadata: { captureTableObjectId: 7 } });
+    const b = candidate({ id: 'b', sourceMetadata: { captureTableObjectId: 7 } });
+    const updates = diffSourceTableUpdates([a, b], resolution([a, b.withSourceMetadata({ captureTableObjectId: 9 })]));
+    expect(updates).toEqual([{ id: 'b', sourceMetadata: { captureTableObjectId: 9 } }]);
+  });
+
+  it('rejects a compatible table that was never a candidate', () => {
+    expect(() => diffSourceTableUpdates([candidate({ id: 'a' })], resolution([candidate({ id: 'ghost' })]))).toThrow(
+      /unknown candidate/
+    );
+  });
+});
+
+describe('SourceTable.clone', () => {
+  it('isolates mutable identity and metadata values', () => {
+    const persisted = candidate({ id: 'a', sourceMetadata: { captureTableObjectId: 7 } });
+    const exposed = persisted.clone();
+
+    (exposed.ref as any).name = 'changed';
+    exposed.replicaIdColumns[0].name = 'changed';
+    (exposed.sourceMetadata as any).captureTableObjectId = 8;
+
+    expect(persisted.name).toBe('users');
+    expect(persisted.replicaIdColumns[0].name).toBe('id');
+    expect(persisted.sourceMetadata).toEqual({ captureTableObjectId: 7 });
+  });
+});
+
+describe('sourceTableIdEquals', () => {
+  it('compares string ids by value', () => {
+    expect(sourceTableIdEquals('table-1', 'table-1')).toBe(true);
+    expect(sourceTableIdEquals('table-1', 'table-2')).toBe(false);
+  });
+
+  it('compares BSON ObjectIds by value', () => {
+    const id = new bson.ObjectId();
+    const copy = new bson.ObjectId(id.toHexString());
+
+    expect(id).not.toBe(copy);
+    expect(sourceTableIdEquals(id, copy)).toBe(true);
+  });
+
+  it('does not mix string and BSON ObjectId representations', () => {
+    const id = new bson.ObjectId();
+
+    expect(sourceTableIdEquals(id.toHexString(), id)).toBe(false);
+    expect(sourceTableIdEquals(id, id.toHexString())).toBe(false);
+  });
+});

package/test/src/sync/BucketChecksumState.test.ts

@@ -1183,7 +1183,7 @@ streams:
       ).rejects.toThrow('Too many buckets: 60 (limit of 50');
 
       // Verify error log includes breakdown
-      expect(errorMessages[0]).toContain('Buckets by definition:');
+      expect(errorMessages[0]).toContain('(limit of 50)\nBuckets by sync stream:');
       expect(errorMessages[0]).toContain('projects: 30');
       expect(errorMessages[0]).toContain('tasks: 20');
       expect(errorMessages[0]).toContain('comments: 10');
@@ -1197,15 +1197,60 @@ streams:
       });
     });
 
-    test('limits bucket breakdown to top 10 definitions', async () => {
-      // Create sync streams with 15 different definitions with dynamic parameters
+    test('labels the bucket breakdown as bucket definitions for legacy sync rules', async () => {
+      const legacySyncRules = SqlSyncRules.fromYaml(
+        `
+bucket_definitions:
+  projects:
+    parameters: SELECT id FROM projects WHERE user_id = request.user_id()
+    data: []
+    `,
+        { defaultSchema: 'public' }
+      ).config.hydrate({ hydrationState: versionedHydrationState(5), sqlite: nodeSqlite(sqlite) });
+
+      const storage = new MockBucketChecksumStateStorage();
+      const errorMessages: string[] = [];
+      const state = new BucketChecksumState({
+        syncContext: new SyncContext({
+          maxBuckets: 50,
+          maxParameterQueryResults: 100,
+          maxDataFetchConcurrency: 10
+        }),
+        tokenPayload: new JwtPayload({ sub: 'u1' }),
+        syncRequest,
+        syncRules: legacySyncRules,
+        bucketStorage: storage,
+        logger: {
+          info: () => {},
+          error: (message: string) => errorMessages.push(message),
+          warn: () => {},
+          debug: () => {}
+        } as any
+      });
+
+      await expect(
+        state.buildNextCheckpointLine({
+          base: storage.makeCheckpoint(1n, (lookups) => [
+            { lookup: lookups[0], rows: Array.from({ length: 60 }, (_, id) => ({ id })) }
+          ]),
+          writeCheckpoint: null,
+          update: CHECKPOINT_INVALIDATE_ALL
+        })
+      ).rejects.toThrow('Too many buckets: 60 (limit of 50)');
+
+      expect(errorMessages[0]).toContain('(limit of 50)\nBuckets by bucket definition:');
+      expect(errorMessages[0]).toContain('projects: 60');
+    });
+
+    test('limits bucket breakdown to top 100 sync streams', async () => {
+      // Create 105 sync streams with dynamic parameters.
       let yamlDefinitions = `
 config:
   edition: 3
 
 streams:
 `;
-      for (let i = 1; i <= 15; i++) {
+      for (let i = 1; i <= 105; i++) {
         yamlDefinitions += `  def${i}:\n`;
         yamlDefinitions += `    auto_subscribe: true\n`;
         yamlDefinitions += `    query: SELECT * FROM tbl WHERE b = auth.parameter('${i}')\n`;
@@ -1229,7 +1274,7 @@ streams:
       };
 
       const smallContext = new SyncContext({
-        maxBuckets: 10,
+        maxBuckets: 100,
         maxParameterQueryResults: 10,
         maxDataFetchConcurrency: 10
       });
@@ -1238,7 +1283,7 @@ streams:
         syncContext: smallContext,
         tokenPayload: new JwtPayload({
           sub: 'u1',
-          ...Object.fromEntries(Array.from({ length: 30 }, (_, i) => [i.toString(), BigInt(i)]))
+          ...Object.fromEntries(Array.from({ length: 106 }, (_, i) => [i.toString(), BigInt(i)]))
         }),
         syncRequest,
         syncRules: SYNC_RULES_MANY,
@@ -1252,15 +1297,16 @@ streams:
           writeCheckpoint: null,
           update: CHECKPOINT_INVALIDATE_ALL
         })
-      ).rejects.toThrow('Too many buckets: 15 (limit of 10)');
+      ).rejects.toThrow('Too many buckets: 105 (limit of 100)');
 
-      // Verify only top 10 are shown
+      // Verify only the first 100 are shown.
       const errorMessage = errorMessages[0];
-      expect(errorMessage).toContain('... and 5 more results from 5 definitions');
+      expect(errorMessage).toContain('Buckets by sync stream:');
+      expect(errorMessage).toContain('... and 5 more buckets from 5 sync streams');
 
-      // Count how many definitions are listed (should be exactly 10)
+      // Count how many sync streams are listed (should be exactly 100).
       const defMatches = errorMessage.match(/def\d+:/g);
-      expect(defMatches?.length).toBe(10);
+      expect(defMatches?.length).toBe(100);
     });
   });
 });

package/test/src/sync/util.test.ts

@@ -1,7 +1,19 @@
-import { acquireSemaphoreAbortable } from '@/index.js';
+import { acquireSemaphoreAbortable, isAbortError } from '@/index.js';
 import { Semaphore, SemaphoreInterface } from 'async-mutex';
+import { AbortError } from 'ix/aborterror.js';
 import { describe, expect, test, vi } from 'vitest';
 
+describe('isAbortError', () => {
+  test('recognizes ix and native abort errors', () => {
+    const controller = new AbortController();
+    controller.abort();
+
+    expect(isAbortError(new AbortError())).toBe(true);
+    expect(isAbortError(controller.signal.reason)).toBe(true);
+    expect(isAbortError(new Error('not aborted'))).toBe(false);
+  });
+});
+
 describe('acquireSemaphoreAbortable', () => {
   test('can acquire', async () => {
     const semaphore = new Semaphore(1);

package/test/src/util/checkpointing.test.ts

@@ -1,4 +1,4 @@
-import { createWriteCheckpoint, WriteCheckpointBatcher } from '@/index.js';
+import { createWriteCheckpoint, storage, WriteCheckpointBatcher } from '@/index.js';
 import { describe, expect, test, vi } from 'vitest';
 
 function deferred<T = void>() {
@@ -10,11 +10,14 @@ async function waitForAsyncWork() {
   await new Promise((resolve) => setImmediate(resolve));
 }
 
-function createStorage() {
+function createStorage(options?: { shouldAdvance?: (checkpoint: { user_id: string }) => boolean }) {
   let nextCheckpoint = 1n;
   const bucketStorage = {
     createManagedWriteCheckpoints: vi.fn(async (checkpoints: { user_id: string }[]) => {
-      return new Map(checkpoints.map((checkpoint) => [checkpoint.user_id, nextCheckpoint++]));
+      return {
+        writeCheckpoints: new Map(checkpoints.map((checkpoint) => [checkpoint.user_id, nextCheckpoint++])),
+        shouldAdvance: checkpoints.some((checkpoint) => options?.shouldAdvance?.(checkpoint) ?? true)
+      };
     })
   };
 
@@ -25,6 +28,31 @@ function createStorage() {
   return { bucketStorage, storage };
 }
 
+/**
+ * Mock RouteAPI.createReplicationHead. The callback persists the write-checkpoint
+ * mapping and returns { response, shouldAdvance }; the source marker (tracked by
+ * `advanceMarker`) is only forced when shouldAdvance is true. An optional gate per
+ * batch keeps the call in flight so concurrency can be exercised.
+ */
+function createApi(options?: { gates?: PromiseWithResolvers<void>[] }) {
+  const advanceMarker = vi.fn(async (_head: string) => undefined);
+  const createReplicationHead = vi.fn(
+    async (callback: (head: string) => Promise<{ response: unknown; shouldAdvance: boolean }>) => {
+      const batch = createReplicationHead.mock.calls.length;
+      const head = `head-${batch}`;
+      const { response, shouldAdvance } = await callback(head);
+      if (shouldAdvance) {
+        await advanceMarker(head);
+      }
+      if (options?.gates) {
+        await options.gates[batch - 1].promise;
+      }
+      return response;
+    }
+  );
+  return { createReplicationHead, advanceMarker };
+}
+
 function createBatcher(api: any, storage: any) {
   return new WriteCheckpointBatcher(
     () => api,
@@ -33,17 +61,25 @@ function createBatcher(api: any, storage: any) {
 }
 
 describe('write checkpoint batching', () => {
+  test('deduplicates managed checkpoints by greatest supplied request id per user', () => {
+    expect(
+      storage.uniqueManagedWriteCheckpoints([
+        { user_id: 'user-a', heads: { '1': 'generated' } },
+        { user_id: 'user-a', heads: { '1': 'stale' }, checkpoint_request_id: 41n },
+        { user_id: 'user-a', heads: { '1': 'winner' }, checkpoint_request_id: 43n },
+        { user_id: 'user-a', heads: { '1': 'lower' }, checkpoint_request_id: 42n },
+        { user_id: 'user-b', heads: { '1': 'generated-b' } }
+      ])
+    ).toEqual([
+      { user_id: 'user-a', heads: { '1': 'winner' }, checkpoint_request_id: 43n },
+      { user_id: 'user-b', heads: { '1': 'generated-b' } }
+    ]);
+  });
+
   test('coalesces same-turn requests and dispatches queued requests as capacity becomes available', async () => {
     const gates = [deferred(), deferred()];
     const { bucketStorage, storage } = createStorage();
-    const api = {
-      createReplicationHead: vi.fn(async (callback: (head: string) => Promise<unknown>) => {
-        const batch = api.createReplicationHead.mock.calls.length;
-        const result = await callback(`head-${batch}`);
-        await gates[batch - 1].promise;
-        return result;
-      })
-    };
+    const api = createApi({ gates });
     const batcher = createBatcher(api, storage);
 
     const firstBatch = [
@@ -92,17 +128,58 @@ describe('write checkpoint batching', () => {
     ]);
   });
 
+  test('passes supplied checkpoint request ids into the storage batch', async () => {
+    const { bucketStorage, storage } = createStorage();
+    const api = createApi();
+    const batcher = createBatcher(api, storage);
+
+    await expect(
+      createWriteCheckpoint({
+        userId: 'user-a',
+        clientId: 'client-1',
+        checkpointRequestId: 42n,
+        batcher
+      })
+    ).resolves.toEqual({ writeCheckpoint: '1', replicationHead: 'head-1' });
+
+    expect(bucketStorage.createManagedWriteCheckpoints).toHaveBeenCalledWith([
+      {
+        user_id: 'user-a/client-1',
+        heads: { '1': 'head-1' },
+        checkpoint_request_id: 42n
+      }
+    ]);
+  });
+
+  test('does not advance the source marker when storage reports no checkpoint updates', async () => {
+    const { storage } = createStorage({ shouldAdvance: () => false });
+    const api = createApi();
+    const batcher = createBatcher(api, storage);
+
+    await expect(createWriteCheckpoint({ userId: 'user-a', clientId: undefined, batcher })).resolves.toEqual({
+      writeCheckpoint: '1',
+      replicationHead: 'head-1'
+    });
+
+    expect(api.createReplicationHead).toHaveBeenCalledTimes(1);
+    expect(api.advanceMarker).not.toHaveBeenCalled();
+  });
+
+  test('advances the source marker when storage reports a checkpoint update', async () => {
+    const { storage } = createStorage();
+    const api = createApi();
+    const batcher = createBatcher(api, storage);
+
+    await createWriteCheckpoint({ userId: 'user-a', clientId: undefined, batcher });
+
+    expect(api.advanceMarker).toHaveBeenCalledTimes(1);
+    expect(api.advanceMarker).toHaveBeenCalledWith('head-1');
+  });
+
   test('allows three executing batches and queues later requests until one completes', async () => {
     const gates = [deferred(), deferred(), deferred(), deferred()];
     const { storage } = createStorage();
-    const api = {
-      createReplicationHead: vi.fn(async (callback: (head: string) => Promise<unknown>) => {
-        const batch = api.createReplicationHead.mock.calls.length;
-        const result = await callback(`head-${batch}`);
-        await gates[batch - 1].promise;
-        return result;
-      })
-    };
+    const api = createApi({ gates });
     const batcher = createBatcher(api, storage);
 
     const first = createWriteCheckpoint({
@@ -160,11 +237,13 @@ describe('write checkpoint batching', () => {
     const api = {
       createReplicationHead: vi.fn(async () => {
         throw error;
-      })
+      }),
+      advanceMarker: vi.fn(async () => undefined)
     };
     const batcher = createBatcher(api, storage);
 
     await expect(createWriteCheckpoint({ userId: 'user-a', clientId: undefined, batcher })).rejects.toBe(error);
     expect(api.createReplicationHead).toHaveBeenCalledTimes(1);
+    expect(api.advanceMarker).not.toHaveBeenCalled();
   });
 });