@powersync/service-core 0.8.7 → 0.9.0

package/test/src/sync.test.ts

@@ -1,12 +1,12 @@
+import { SaveOperationTag } from '@/storage/storage-index.js';
 import { RequestTracker } from '@/sync/RequestTracker.js';
+import { streamResponse, SyncStreamParameters } from '@/sync/sync.js';
 import { StreamingSyncLine } from '@/util/protocol-types.js';
-import { lsnMakeComparable } from '@powersync/service-jpgwire';
 import { JSONBig } from '@powersync/service-jsonbig';
 import { RequestParameters } from '@powersync/service-sync-rules';
 import * as timers from 'timers/promises';
 import { describe, expect, test } from 'vitest';
-import { streamResponse } from '../../src/sync/sync.js';
-import { makeTestTable, MONGO_STORAGE_FACTORY, StorageFactory } from './util.js';
+import { BATCH_OPTIONS, makeTestTable, MONGO_STORAGE_FACTORY, PARSE_OPTIONS, StorageFactory } from './util.js';
 
 describe('sync - mongodb', function () {
   defineTests(MONGO_STORAGE_FACTORY);
@@ -31,29 +31,31 @@ function defineTests(factory: StorageFactory) {
       content: BASIC_SYNC_RULES
     });
 
-    const storage = await f.getInstance(syncRules.parsed());
+    const storage = f.getInstance(syncRules);
     await storage.autoActivate();
 
-    const result = await storage.startBatch({}, async (batch) => {
+    const result = await storage.startBatch(BATCH_OPTIONS, async (batch) => {
       await batch.save({
         sourceTable: TEST_TABLE,
-        tag: 'insert',
+        tag: SaveOperationTag.INSERT,
         after: {
           id: 't1',
           description: 'Test 1'
-        }
+        },
+        afterReplicaId: 't1'
       });
 
       await batch.save({
         sourceTable: TEST_TABLE,
-        tag: 'insert',
+        tag: SaveOperationTag.INSERT,
         after: {
           id: 't2',
           description: 'Test 2'
-        }
+        },
+        afterReplicaId: 't2'
       });
 
-      await batch.commit(lsnMakeComparable('0/1'));
+      await batch.commit('0/1');
     });
 
     const stream = streamResponse({
@@ -63,6 +65,7 @@ function defineTests(factory: StorageFactory) {
         include_checksum: true,
         raw_data: true
       },
+      parseOptions: PARSE_OPTIONS,
       tracker,
       syncParams: new RequestParameters({ sub: '' }, {}),
       token: { exp: Date.now() / 1000 + 10 } as any
@@ -79,21 +82,22 @@ function defineTests(factory: StorageFactory) {
       content: BASIC_SYNC_RULES
     });
 
-    const storage = await f.getInstance(syncRules.parsed());
+    const storage = await f.getInstance(syncRules);
     await storage.autoActivate();
 
-    const result = await storage.startBatch({}, async (batch) => {
+    const result = await storage.startBatch(BATCH_OPTIONS, async (batch) => {
       await batch.save({
         sourceTable: TEST_TABLE,
-        tag: 'insert',
+        tag: SaveOperationTag.INSERT,
         after: {
           id: 't1',
           description: 'Test\n"string"',
           large_num: 12345678901234567890n
-        }
+        },
+        afterReplicaId: 't1'
       });
 
-      await batch.commit(lsnMakeComparable('0/1'));
+      await batch.commit('0/1');
     });
 
     const stream = streamResponse({
@@ -103,6 +107,7 @@ function defineTests(factory: StorageFactory) {
         include_checksum: true,
         raw_data: false
       },
+      parseOptions: PARSE_OPTIONS,
       tracker,
       syncParams: new RequestParameters({ sub: '' }, {}),
       token: { exp: Date.now() / 1000 + 10 } as any
@@ -121,7 +126,7 @@ function defineTests(factory: StorageFactory) {
       content: BASIC_SYNC_RULES
     });
 
-    const storage = await f.getInstance(syncRules.parsed());
+    const storage = await f.getInstance(syncRules);
     await storage.autoActivate();
 
     const stream = streamResponse({
@@ -131,6 +136,7 @@ function defineTests(factory: StorageFactory) {
         include_checksum: true,
         raw_data: true
       },
+      parseOptions: PARSE_OPTIONS,
       tracker,
       syncParams: new RequestParameters({ sub: '' }, {}),
       token: { exp: 0 } as any
@@ -147,7 +153,7 @@ function defineTests(factory: StorageFactory) {
       content: BASIC_SYNC_RULES
     });
 
-    const storage = await f.getInstance(syncRules.parsed());
+    const storage = await f.getInstance(syncRules);
     await storage.autoActivate();
 
     const stream = streamResponse({
@@ -157,6 +163,7 @@ function defineTests(factory: StorageFactory) {
         include_checksum: true,
         raw_data: true
       },
+      parseOptions: PARSE_OPTIONS,
       tracker,
       syncParams: new RequestParameters({ sub: '' }, {}),
       token: { exp: Date.now() / 1000 + 10 } as any
@@ -165,32 +172,34 @@ function defineTests(factory: StorageFactory) {
 
     expect(await getCheckpointLines(iter)).toMatchSnapshot();
 
-    await storage.startBatch({}, async (batch) => {
+    await storage.startBatch(BATCH_OPTIONS, async (batch) => {
       await batch.save({
         sourceTable: TEST_TABLE,
-        tag: 'insert',
+        tag: SaveOperationTag.INSERT,
         after: {
           id: 't1',
           description: 'Test 1'
-        }
+        },
+        afterReplicaId: 't1'
       });
 
-      await batch.commit(lsnMakeComparable('0/1'));
+      await batch.commit('0/1');
     });
 
     expect(await getCheckpointLines(iter)).toMatchSnapshot();
 
-    await storage.startBatch({}, async (batch) => {
+    await storage.startBatch(BATCH_OPTIONS, async (batch) => {
       await batch.save({
         sourceTable: TEST_TABLE,
-        tag: 'insert',
+        tag: SaveOperationTag.INSERT,
         after: {
           id: 't2',
           description: 'Test 2'
-        }
+        },
+        afterReplicaId: 't2'
       });
 
-      await batch.commit(lsnMakeComparable('0/2'));
+      await batch.commit('0/2');
     });
 
     expect(await getCheckpointLines(iter)).toMatchSnapshot();
@@ -205,7 +214,7 @@ function defineTests(factory: StorageFactory) {
       content: BASIC_SYNC_RULES
     });
 
-    const storage = await f.getInstance(syncRules.parsed());
+    const storage = await f.getInstance(syncRules);
     await storage.autoActivate();
 
     const exp = Date.now() / 1000 + 0.1;
@@ -217,6 +226,7 @@ function defineTests(factory: StorageFactory) {
         include_checksum: true,
         raw_data: true
       },
+      parseOptions: PARSE_OPTIONS,
       tracker,
       syncParams: new RequestParameters({ sub: '' }, {}),
       token: { exp: exp } as any
@@ -242,29 +252,31 @@ function defineTests(factory: StorageFactory) {
       content: BASIC_SYNC_RULES
     });
 
-    const storage = await f.getInstance(syncRules.parsed());
+    const storage = await f.getInstance(syncRules);
     await storage.autoActivate();
 
-    await storage.startBatch({}, async (batch) => {
+    await storage.startBatch(BATCH_OPTIONS, async (batch) => {
       await batch.save({
         sourceTable: TEST_TABLE,
-        tag: 'insert',
+        tag: SaveOperationTag.INSERT,
         after: {
           id: 't1',
           description: 'Test 1'
-        }
+        },
+        afterReplicaId: 't1'
       });
 
       await batch.save({
         sourceTable: TEST_TABLE,
-        tag: 'insert',
+        tag: SaveOperationTag.INSERT,
         after: {
           id: 't2',
           description: 'Test 2'
-        }
+        },
+        afterReplicaId: 't2'
       });
 
-      await batch.commit(lsnMakeComparable('0/1'));
+      await batch.commit('0/1');
     });
 
     const stream = streamResponse({
@@ -274,6 +286,7 @@ function defineTests(factory: StorageFactory) {
         include_checksum: true,
         raw_data: true
       },
+      parseOptions: PARSE_OPTIONS,
       tracker,
       syncParams: new RequestParameters({ sub: '' }, {}),
       token: { exp: Date.now() / 1000 + 10 } as any
@@ -293,26 +306,28 @@ function defineTests(factory: StorageFactory) {
     // Now we save additional data AND compact before continuing.
     // This invalidates the checkpoint we've received above.
 
-    await storage.startBatch({}, async (batch) => {
+    await storage.startBatch(BATCH_OPTIONS, async (batch) => {
       await batch.save({
         sourceTable: TEST_TABLE,
-        tag: 'update',
+        tag: SaveOperationTag.UPDATE,
         after: {
           id: 't1',
           description: 'Test 1b'
-        }
+        },
+        afterReplicaId: 't1'
       });
 
       await batch.save({
         sourceTable: TEST_TABLE,
-        tag: 'update',
+        tag: SaveOperationTag.UPDATE,
         after: {
           id: 't2',
           description: 'Test 2b'
-        }
+        },
+        afterReplicaId: 't2'
       });
 
-      await batch.commit(lsnMakeComparable('0/2'));
+      await batch.commit('0/2');
     });
 
     await storage.compact();
@@ -366,6 +381,67 @@ function defineTests(factory: StorageFactory) {
       })
     });
   });
+
+  test('write checkpoint', async () => {
+    const f = await factory();
+
+    const syncRules = await f.updateSyncRules({
+      content: BASIC_SYNC_RULES
+    });
+
+    const storage = f.getInstance(syncRules);
+    await storage.autoActivate();
+
+    await storage.startBatch(BATCH_OPTIONS, async (batch) => {
+      // <= the managed write checkpoint LSN below
+      await batch.commit('0/1');
+    });
+
+    const checkpoint = await storage.createManagedWriteCheckpoint({
+      user_id: 'test',
+      heads: { '1': '1/0' }
+    });
+
+    const params: SyncStreamParameters = {
+      storage: f,
+      params: {
+        buckets: [],
+        include_checksum: true,
+        raw_data: true
+      },
+      parseOptions: PARSE_OPTIONS,
+      tracker,
+      syncParams: new RequestParameters({ sub: 'test' }, {}),
+      token: { sub: 'test', exp: Date.now() / 1000 + 10 } as any
+    };
+    const stream1 = streamResponse(params);
+    const lines1 = await consumeCheckpointLines(stream1);
+
+    // If write checkpoints are not correctly filtered, this may already
+    // contain the write checkpoint.
+    expect(lines1[0]).toMatchObject({
+      checkpoint: expect.objectContaining({
+        last_op_id: '0',
+        write_checkpoint: undefined
+      })
+    });
+
+    await storage.startBatch(BATCH_OPTIONS, async (batch) => {
+      // must be >= the managed write checkpoint LSN
+      await batch.commit('1/0');
+    });
+
+    // At this point the LSN has advanced, so the write checkpoint should be
+    // included in the next checkpoint message.
+    const stream2 = streamResponse(params);
+    const lines2 = await consumeCheckpointLines(stream2);
+    expect(lines2[0]).toMatchObject({
+      checkpoint: expect.objectContaining({
+        last_op_id: '0',
+        write_checkpoint: `${checkpoint}`
+      })
+    });
+  });
 }
 
 /**

package/test/src/util.ts

@@ -1,16 +1,20 @@
-import * as pgwire from '@powersync/service-jpgwire';
-import { normalizeConnection } from '@powersync/service-types';
-import * as mongo from 'mongodb';
-import { BucketStorageFactory, SyncBucketDataBatch } from '../../src/storage/BucketStorage.js';
-import { MongoBucketStorage } from '../../src/storage/MongoBucketStorage.js';
-import { PowerSyncMongo } from '../../src/storage/mongo/db.js';
-import { escapeIdentifier } from '../../src/util/pgwire_utils.js';
-import { env } from './env.js';
 import { Metrics } from '@/metrics/Metrics.js';
-import { hashData } from '@/util/utils.js';
+import {
+  BucketStorageFactory,
+  ParseSyncRulesOptions,
+  PersistedSyncRulesContent,
+  StartBatchOptions,
+  SyncBucketDataBatch
+} from '@/storage/BucketStorage.js';
+import { MongoBucketStorage } from '@/storage/MongoBucketStorage.js';
 import { SourceTable } from '@/storage/SourceTable.js';
-import * as bson from 'bson';
+import { PowerSyncMongo } from '@/storage/mongo/db.js';
 import { SyncBucketData } from '@/util/protocol-types.js';
+import { getUuidReplicaIdentityBson, hashData } from '@/util/utils.js';
+import { SqlSyncRules } from '@powersync/service-sync-rules';
+import * as bson from 'bson';
+import * as mongo from 'mongodb';
+import { env } from './env.js';
 
 // The metrics need to be initialised before they can be used
 await Metrics.initialise({
@@ -20,8 +24,6 @@ await Metrics.initialise({
 });
 Metrics.getInstance().resetCounters();
 
-export const TEST_URI = env.PG_TEST_URL;
-
 export interface StorageOptions {
   /**
    * By default, collections are only cleared/
@@ -41,45 +43,34 @@ export const MONGO_STORAGE_FACTORY: StorageFactory = async (options?: StorageOpt
   return new MongoBucketStorage(db, { slot_name_prefix: 'test_' });
 };
 
-export async function clearTestDb(db: pgwire.PgClient) {
-  await db.query(
-    "select pg_drop_replication_slot(slot_name) from pg_replication_slots where active = false and slot_name like 'test_%'"
-  );
+export const ZERO_LSN = '0/0';
 
-  await db.query(`CREATE EXTENSION IF NOT EXISTS "uuid-ossp"`);
-  try {
-    await db.query(`DROP PUBLICATION powersync`);
-  } catch (e) {
-    // Ignore
-  }
+export const PARSE_OPTIONS: ParseSyncRulesOptions = {
+  defaultSchema: 'public'
+};
 
-  await db.query(`CREATE PUBLICATION powersync FOR ALL TABLES`);
+export const BATCH_OPTIONS: StartBatchOptions = {
+  ...PARSE_OPTIONS,
+  zeroLSN: ZERO_LSN,
+  storeCurrentData: true
+};
 
-  const tableRows = pgwire.pgwireRows(
-    await db.query(`SELECT table_name FROM information_schema.tables where table_schema = 'public'`)
-  );
-  for (let row of tableRows) {
-    const name = row.table_name;
-    if (name.startsWith('test_')) {
-      await db.query(`DROP TABLE public.${escapeIdentifier(name)}`);
+export function testRules(content: string): PersistedSyncRulesContent {
+  return {
+    id: 1,
+    sync_rules_content: content,
+    slot_name: 'test',
+    parsed(options) {
+      return {
+        id: 1,
+        sync_rules: SqlSyncRules.fromYaml(content, options),
+        slot_name: 'test'
+      };
+    },
+    lock() {
+      throw new Error('Not implemented');
     }
-  }
-}
-
-export const TEST_CONNECTION_OPTIONS = normalizeConnection({
-  type: 'postgresql',
-  uri: TEST_URI,
-  sslmode: 'disable'
-});
-
-export async function connectPgWire(type?: 'replication' | 'standard') {
-  const db = await pgwire.connectPgWire(TEST_CONNECTION_OPTIONS, { type });
-  return db;
-}
-
-export function connectPgPool() {
-  const db = pgwire.connectPgWirePool(TEST_CONNECTION_OPTIONS);
-  return db;
+  };
 }
 
 export async function connectMongo() {
@@ -90,8 +81,7 @@ export async function connectMongo() {
     socketTimeoutMS: env.CI ? 15_000 : 5_000,
     serverSelectionTimeoutMS: env.CI ? 15_000 : 2_500
   });
-  const db = new PowerSyncMongo(client);
-  return db;
+  return new PowerSyncMongo(client);
 }
 
 export function makeTestTable(name: string, columns?: string[] | undefined) {
@@ -101,9 +91,9 @@ export function makeTestTable(name: string, columns?: string[] | undefined) {
     id,
     SourceTable.DEFAULT_TAG,
     relId,
-    SourceTable.DEFAULT_SCHEMA,
+    'public',
     name,
-    (columns ?? ['id']).map((column) => ({ name: column, typeOid: 25 })),
+    (columns ?? ['id']).map((column) => ({ name: column, type: 'VARCHAR', typeId: 25 })),
     true
   );
 }
@@ -149,3 +139,10 @@ function getFirst(batch: SyncBucketData[] | SyncBucketDataBatch[] | SyncBucketDa
     return first as SyncBucketData;
   }
 }
+
+/**
+ * Replica id in the old Postgres format, for backwards-compatible tests.
+ */
+export function rid(id: string): bson.UUID {
+  return getUuidReplicaIdentityBson({ id: id }, [{ name: 'id', type: 'VARCHAR', typeId: 25 }]);
+}

package/test/tsconfig.json

@@ -2,8 +2,8 @@
   "extends": "../../../tsconfig.base.json",
   "compilerOptions": {
     "rootDir": "src",
-    "noEmit": true,
     "baseUrl": "./",
+    "outDir": "dist",
     "esModuleInterop": true,
     "skipLibCheck": true,
     "sourceMap": true,