@powersync/service-core 0.8.8 → 0.9.0

package/test/src/env.ts

@@ -2,7 +2,5 @@ import { utils } from '@powersync/lib-services-framework';
 
 export const env = utils.collectEnvironmentVariables({
   MONGO_TEST_URL: utils.type.string.default('mongodb://localhost:27017/powersync_test'),
-  PG_TEST_URL: utils.type.string.default('postgres://postgres:postgres@localhost:5432/powersync_test'),
-  CI: utils.type.boolean.default('false'),
-  SLOW_TESTS: utils.type.boolean.default('false')
+  CI: utils.type.boolean.default('false')
 });

package/test/src/merge_iterable.test.ts

@@ -1,11 +1,6 @@
+import { mergeAsyncIterablesNew, mergeAsyncIterablesOld } from '@/sync/merge.js';
 import * as timers from 'timers/promises';
 import { describe, expect, test } from 'vitest';
-import {
-  FixedMergeAsyncIterable,
-  mergeAsyncIterables,
-  mergeAsyncIterablesNew,
-  mergeAsyncIterablesOld
-} from '../../src/sync/merge.js';
 
 type MergeIteratorFunction = <T>(source: AsyncIterable<T>[]) => AsyncIterable<T>;
 

package/test/src/routes/probes.integration.test.ts

@@ -6,8 +6,8 @@ import * as system from '../../../src/system/system-index.js';
 import { configureFastifyServer } from '../../../src/index.js';
 import { ProbeRoutes } from '../../../src/routes/endpoints/probes.js';
 
-vi.mock("@powersync/lib-services-framework", async () => {
-  const actual = await vi.importActual("@powersync/lib-services-framework") as any;
+vi.mock('@powersync/lib-services-framework', async () => {
+  const actual = (await vi.importActual('@powersync/lib-services-framework')) as any;
   return {
     ...actual,
     container: {
@@ -15,18 +15,18 @@ vi.mock("@powersync/lib-services-framework", async () => {
       probes: {
         state: vi.fn()
       }
-    },
-  }
-})
+    }
+  };
+});
 
 describe('Probe Routes Integration', () => {
   let app: FastifyInstance;
-  let mockSystem: system.CorePowerSyncSystem;
+  let mockSystem: system.ServiceContext;
 
   beforeEach(async () => {
     app = Fastify();
-    mockSystem = {} as system.CorePowerSyncSystem;
-    await configureFastifyServer(app, { system: mockSystem });
+    mockSystem = { routerEngine: {} } as system.ServiceContext;
+    await configureFastifyServer(app, { service_context: mockSystem });
     await app.ready();
   });
 
@@ -46,7 +46,7 @@ describe('Probe Routes Integration', () => {
 
       const response = await app.inject({
         method: 'GET',
-        url: ProbeRoutes.STARTUP,
+        url: ProbeRoutes.STARTUP
       });
 
       expect(response.statusCode).toBe(200);
@@ -67,7 +67,7 @@ describe('Probe Routes Integration', () => {
 
       const response = await app.inject({
         method: 'GET',
-        url: ProbeRoutes.STARTUP,
+        url: ProbeRoutes.STARTUP
       });
 
       expect(response.statusCode).toBe(400);
@@ -90,7 +90,7 @@ describe('Probe Routes Integration', () => {
 
       const response = await app.inject({
         method: 'GET',
-        url: ProbeRoutes.LIVENESS,
+        url: ProbeRoutes.LIVENESS
       });
 
       expect(response.statusCode).toBe(200);
@@ -111,7 +111,7 @@ describe('Probe Routes Integration', () => {
 
       const response = await app.inject({
         method: 'GET',
-        url: ProbeRoutes.LIVENESS,
+        url: ProbeRoutes.LIVENESS
       });
 
       expect(response.statusCode).toBe(400);
@@ -134,7 +134,7 @@ describe('Probe Routes Integration', () => {
 
       const response = await app.inject({
         method: 'GET',
-        url: ProbeRoutes.READINESS,
+        url: ProbeRoutes.READINESS
       });
 
       expect(response.statusCode).toBe(200);
@@ -155,7 +155,7 @@ describe('Probe Routes Integration', () => {
 
       const response = await app.inject({
         method: 'GET',
-        url: ProbeRoutes.READINESS,
+        url: ProbeRoutes.READINESS
       });
 
       expect(response.statusCode).toBe(400);
@@ -172,17 +172,19 @@ describe('Probe Routes Integration', () => {
       vi.mocked(container.probes.state).mockReturnValue(mockState);
 
       // Create array of 15 concurrent requests (default concurrency is 10)
-      const requests = Array(15).fill(null).map(() =>
-        app.inject({
-          method: 'GET',
-          url: ProbeRoutes.STARTUP,
-        })
-      );
+      const requests = Array(15)
+        .fill(null)
+        .map(() =>
+          app.inject({
+            method: 'GET',
+            url: ProbeRoutes.STARTUP
+          })
+        );
 
       const responses = await Promise.all(requests);
 
       // All requests should complete successfully
-      responses.forEach(response => {
+      responses.forEach((response) => {
         expect(response.statusCode).toBe(200);
         expect(JSON.parse(response.payload)).toEqual({
           ...mockState,
@@ -196,18 +198,20 @@ describe('Probe Routes Integration', () => {
       vi.mocked(container.probes.state).mockReturnValue(mockState);
 
       // Create array of 35 concurrent requests (default max_queue_depth is 20)
-      const requests = Array(35).fill(null).map(() =>
-        app.inject({
-          method: 'GET',
-          url: ProbeRoutes.STARTUP,
-        })
-      );
+      const requests = Array(35)
+        .fill(null)
+        .map(() =>
+          app.inject({
+            method: 'GET',
+            url: ProbeRoutes.STARTUP
+          })
+        );
 
       const responses = await Promise.all(requests);
 
       // Some requests should succeed and some should fail with 429
-      const successCount = responses.filter(r => r.statusCode === 200).length;
-      const queueFullCount = responses.filter(r => r.statusCode === 429).length;
+      const successCount = responses.filter((r) => r.statusCode === 200).length;
+      const queueFullCount = responses.filter((r) => r.statusCode === 429).length;
 
       expect(successCount).toBeGreaterThan(0);
       expect(queueFullCount).toBeGreaterThan(0);
@@ -222,7 +226,7 @@ describe('Probe Routes Integration', () => {
 
       const response = await app.inject({
         method: 'GET',
-        url: ProbeRoutes.STARTUP,
+        url: ProbeRoutes.STARTUP
       });
 
       expect(response.headers['content-type']).toMatch(/application\/json/);

package/test/src/setup.ts

@@ -2,6 +2,6 @@ import { container } from '@powersync/lib-services-framework';
 import { beforeAll } from 'vitest';
 
 beforeAll(() => {
-  // Your setup code here
+  // Executes for every test file
   container.registerDefaults();
 });

package/test/src/stream_utils.ts

@@ -0,0 +1,42 @@
+import { OplogEntry } from '@/util/protocol-types.js';
+import { JSONBig } from '@powersync/service-jsonbig';
+
+export function putOp(table: string, data: Record<string, any>): Partial<OplogEntry> {
+  return {
+    op: 'PUT',
+    object_type: table,
+    object_id: data.id,
+    data: JSONBig.stringify(data)
+  };
+}
+
+export function removeOp(table: string, id: string): Partial<OplogEntry> {
+  return {
+    op: 'REMOVE',
+    object_type: table,
+    object_id: id
+  };
+}
+
+export function compareIds(a: OplogEntry, b: OplogEntry) {
+  return a.object_id!.localeCompare(b.object_id!);
+}
+
+export async function oneFromAsync<T>(source: Iterable<T> | AsyncIterable<T>): Promise<T> {
+  const items: T[] = [];
+  for await (const item of source) {
+    items.push(item);
+  }
+  if (items.length != 1) {
+    throw new Error(`One item expected, got: ${items.length}`);
+  }
+  return items[0];
+}
+
+export async function fromAsync<T>(source: Iterable<T> | AsyncIterable<T>): Promise<T[]> {
+  const items: T[] = [];
+  for await (const item of source) {
+    items.push(item);
+  }
+  return items;
+}

package/test/src/sync.test.ts

@@ -1,12 +1,12 @@
+import { SaveOperationTag } from '@/storage/storage-index.js';
 import { RequestTracker } from '@/sync/RequestTracker.js';
+import { streamResponse, SyncStreamParameters } from '@/sync/sync.js';
 import { StreamingSyncLine } from '@/util/protocol-types.js';
-import { lsnMakeComparable } from '@powersync/service-jpgwire';
 import { JSONBig } from '@powersync/service-jsonbig';
 import { RequestParameters } from '@powersync/service-sync-rules';
 import * as timers from 'timers/promises';
 import { describe, expect, test } from 'vitest';
-import { streamResponse } from '../../src/sync/sync.js';
-import { makeTestTable, MONGO_STORAGE_FACTORY, StorageFactory } from './util.js';
+import { BATCH_OPTIONS, makeTestTable, MONGO_STORAGE_FACTORY, PARSE_OPTIONS, StorageFactory } from './util.js';
 
 describe('sync - mongodb', function () {
   defineTests(MONGO_STORAGE_FACTORY);
@@ -31,29 +31,31 @@ function defineTests(factory: StorageFactory) {
       content: BASIC_SYNC_RULES
     });
 
-    const storage = await f.getInstance(syncRules.parsed());
+    const storage = f.getInstance(syncRules);
     await storage.autoActivate();
 
-    const result = await storage.startBatch({}, async (batch) => {
+    const result = await storage.startBatch(BATCH_OPTIONS, async (batch) => {
       await batch.save({
         sourceTable: TEST_TABLE,
-        tag: 'insert',
+        tag: SaveOperationTag.INSERT,
         after: {
           id: 't1',
           description: 'Test 1'
-        }
+        },
+        afterReplicaId: 't1'
       });
 
       await batch.save({
         sourceTable: TEST_TABLE,
-        tag: 'insert',
+        tag: SaveOperationTag.INSERT,
         after: {
           id: 't2',
           description: 'Test 2'
-        }
+        },
+        afterReplicaId: 't2'
       });
 
-      await batch.commit(lsnMakeComparable('0/1'));
+      await batch.commit('0/1');
     });
 
     const stream = streamResponse({
@@ -63,6 +65,7 @@ function defineTests(factory: StorageFactory) {
         include_checksum: true,
         raw_data: true
       },
+      parseOptions: PARSE_OPTIONS,
       tracker,
       syncParams: new RequestParameters({ sub: '' }, {}),
       token: { exp: Date.now() / 1000 + 10 } as any
@@ -79,21 +82,22 @@ function defineTests(factory: StorageFactory) {
       content: BASIC_SYNC_RULES
     });
 
-    const storage = await f.getInstance(syncRules.parsed());
+    const storage = await f.getInstance(syncRules);
     await storage.autoActivate();
 
-    const result = await storage.startBatch({}, async (batch) => {
+    const result = await storage.startBatch(BATCH_OPTIONS, async (batch) => {
       await batch.save({
         sourceTable: TEST_TABLE,
-        tag: 'insert',
+        tag: SaveOperationTag.INSERT,
         after: {
           id: 't1',
           description: 'Test\n"string"',
           large_num: 12345678901234567890n
-        }
+        },
+        afterReplicaId: 't1'
       });
 
-      await batch.commit(lsnMakeComparable('0/1'));
+      await batch.commit('0/1');
     });
 
     const stream = streamResponse({
@@ -103,6 +107,7 @@ function defineTests(factory: StorageFactory) {
         include_checksum: true,
         raw_data: false
       },
+      parseOptions: PARSE_OPTIONS,
       tracker,
       syncParams: new RequestParameters({ sub: '' }, {}),
       token: { exp: Date.now() / 1000 + 10 } as any
@@ -121,7 +126,7 @@ function defineTests(factory: StorageFactory) {
       content: BASIC_SYNC_RULES
     });
 
-    const storage = await f.getInstance(syncRules.parsed());
+    const storage = await f.getInstance(syncRules);
     await storage.autoActivate();
 
     const stream = streamResponse({
@@ -131,6 +136,7 @@ function defineTests(factory: StorageFactory) {
         include_checksum: true,
         raw_data: true
       },
+      parseOptions: PARSE_OPTIONS,
       tracker,
       syncParams: new RequestParameters({ sub: '' }, {}),
       token: { exp: 0 } as any
@@ -147,7 +153,7 @@ function defineTests(factory: StorageFactory) {
       content: BASIC_SYNC_RULES
     });
 
-    const storage = await f.getInstance(syncRules.parsed());
+    const storage = await f.getInstance(syncRules);
     await storage.autoActivate();
 
     const stream = streamResponse({
@@ -157,6 +163,7 @@ function defineTests(factory: StorageFactory) {
         include_checksum: true,
         raw_data: true
       },
+      parseOptions: PARSE_OPTIONS,
       tracker,
       syncParams: new RequestParameters({ sub: '' }, {}),
       token: { exp: Date.now() / 1000 + 10 } as any
@@ -165,32 +172,34 @@ function defineTests(factory: StorageFactory) {
 
     expect(await getCheckpointLines(iter)).toMatchSnapshot();
 
-    await storage.startBatch({}, async (batch) => {
+    await storage.startBatch(BATCH_OPTIONS, async (batch) => {
       await batch.save({
         sourceTable: TEST_TABLE,
-        tag: 'insert',
+        tag: SaveOperationTag.INSERT,
         after: {
           id: 't1',
           description: 'Test 1'
-        }
+        },
+        afterReplicaId: 't1'
       });
 
-      await batch.commit(lsnMakeComparable('0/1'));
+      await batch.commit('0/1');
     });
 
     expect(await getCheckpointLines(iter)).toMatchSnapshot();
 
-    await storage.startBatch({}, async (batch) => {
+    await storage.startBatch(BATCH_OPTIONS, async (batch) => {
       await batch.save({
         sourceTable: TEST_TABLE,
-        tag: 'insert',
+        tag: SaveOperationTag.INSERT,
         after: {
           id: 't2',
           description: 'Test 2'
-        }
+        },
+        afterReplicaId: 't2'
       });
 
-      await batch.commit(lsnMakeComparable('0/2'));
+      await batch.commit('0/2');
     });
 
     expect(await getCheckpointLines(iter)).toMatchSnapshot();
@@ -205,7 +214,7 @@ function defineTests(factory: StorageFactory) {
       content: BASIC_SYNC_RULES
     });
 
-    const storage = await f.getInstance(syncRules.parsed());
+    const storage = await f.getInstance(syncRules);
     await storage.autoActivate();
 
     const exp = Date.now() / 1000 + 0.1;
@@ -217,6 +226,7 @@ function defineTests(factory: StorageFactory) {
         include_checksum: true,
         raw_data: true
       },
+      parseOptions: PARSE_OPTIONS,
       tracker,
       syncParams: new RequestParameters({ sub: '' }, {}),
       token: { exp: exp } as any
@@ -242,29 +252,31 @@ function defineTests(factory: StorageFactory) {
       content: BASIC_SYNC_RULES
     });
 
-    const storage = await f.getInstance(syncRules.parsed());
+    const storage = await f.getInstance(syncRules);
     await storage.autoActivate();
 
-    await storage.startBatch({}, async (batch) => {
+    await storage.startBatch(BATCH_OPTIONS, async (batch) => {
       await batch.save({
         sourceTable: TEST_TABLE,
-        tag: 'insert',
+        tag: SaveOperationTag.INSERT,
         after: {
           id: 't1',
           description: 'Test 1'
-        }
+        },
+        afterReplicaId: 't1'
       });
 
       await batch.save({
         sourceTable: TEST_TABLE,
-        tag: 'insert',
+        tag: SaveOperationTag.INSERT,
         after: {
           id: 't2',
           description: 'Test 2'
-        }
+        },
+        afterReplicaId: 't2'
       });
 
-      await batch.commit(lsnMakeComparable('0/1'));
+      await batch.commit('0/1');
     });
 
     const stream = streamResponse({
@@ -274,6 +286,7 @@ function defineTests(factory: StorageFactory) {
         include_checksum: true,
         raw_data: true
       },
+      parseOptions: PARSE_OPTIONS,
       tracker,
       syncParams: new RequestParameters({ sub: '' }, {}),
       token: { exp: Date.now() / 1000 + 10 } as any
@@ -293,26 +306,28 @@ function defineTests(factory: StorageFactory) {
     // Now we save additional data AND compact before continuing.
     // This invalidates the checkpoint we've received above.
 
-    await storage.startBatch({}, async (batch) => {
+    await storage.startBatch(BATCH_OPTIONS, async (batch) => {
       await batch.save({
         sourceTable: TEST_TABLE,
-        tag: 'update',
+        tag: SaveOperationTag.UPDATE,
         after: {
           id: 't1',
           description: 'Test 1b'
-        }
+        },
+        afterReplicaId: 't1'
       });
 
       await batch.save({
         sourceTable: TEST_TABLE,
-        tag: 'update',
+        tag: SaveOperationTag.UPDATE,
         after: {
           id: 't2',
           description: 'Test 2b'
-        }
+        },
+        afterReplicaId: 't2'
       });
 
-      await batch.commit(lsnMakeComparable('0/2'));
+      await batch.commit('0/2');
     });
 
     await storage.compact();
@@ -366,6 +381,67 @@ function defineTests(factory: StorageFactory) {
       })
     });
   });
+
+  test('write checkpoint', async () => {
+    const f = await factory();
+
+    const syncRules = await f.updateSyncRules({
+      content: BASIC_SYNC_RULES
+    });
+
+    const storage = f.getInstance(syncRules);
+    await storage.autoActivate();
+
+    await storage.startBatch(BATCH_OPTIONS, async (batch) => {
+      // <= the managed write checkpoint LSN below
+      await batch.commit('0/1');
+    });
+
+    const checkpoint = await storage.createManagedWriteCheckpoint({
+      user_id: 'test',
+      heads: { '1': '1/0' }
+    });
+
+    const params: SyncStreamParameters = {
+      storage: f,
+      params: {
+        buckets: [],
+        include_checksum: true,
+        raw_data: true
+      },
+      parseOptions: PARSE_OPTIONS,
+      tracker,
+      syncParams: new RequestParameters({ sub: 'test' }, {}),
+      token: { sub: 'test', exp: Date.now() / 1000 + 10 } as any
+    };
+    const stream1 = streamResponse(params);
+    const lines1 = await consumeCheckpointLines(stream1);
+
+    // If write checkpoints are not correctly filtered, this may already
+    // contain the write checkpoint.
+    expect(lines1[0]).toMatchObject({
+      checkpoint: expect.objectContaining({
+        last_op_id: '0',
+        write_checkpoint: undefined
+      })
+    });
+
+    await storage.startBatch(BATCH_OPTIONS, async (batch) => {
+      // must be >= the managed write checkpoint LSN
+      await batch.commit('1/0');
+    });
+
+    // At this point the LSN has advanced, so the write checkpoint should be
+    // included in the next checkpoint message.
+    const stream2 = streamResponse(params);
+    const lines2 = await consumeCheckpointLines(stream2);
+    expect(lines2[0]).toMatchObject({
+      checkpoint: expect.objectContaining({
+        last_op_id: '0',
+        write_checkpoint: `${checkpoint}`
+      })
+    });
+  });
 }
 
 /**