@powersync/service-core 0.0.0-dev-20240725112650 → 0.0.0-dev-20240918082156

package/test/src/env.ts

@@ -2,7 +2,5 @@ import { utils } from '@powersync/lib-services-framework';
 
 export const env = utils.collectEnvironmentVariables({
   MONGO_TEST_URL: utils.type.string.default('mongodb://localhost:27017/powersync_test'),
-  PG_TEST_URL: utils.type.string.default('postgres://postgres:postgres@localhost:5432/powersync_test'),
-  CI: utils.type.boolean.default('false'),
-  SLOW_TESTS: utils.type.boolean.default('false')
+  CI: utils.type.boolean.default('false')
 });

package/test/src/merge_iterable.test.ts

@@ -1,11 +1,6 @@
+import { mergeAsyncIterablesNew, mergeAsyncIterablesOld } from '@/sync/merge.js';
 import * as timers from 'timers/promises';
 import { describe, expect, test } from 'vitest';
-import {
-  FixedMergeAsyncIterable,
-  mergeAsyncIterables,
-  mergeAsyncIterablesNew,
-  mergeAsyncIterablesOld
-} from '../../src/sync/merge.js';
 
 type MergeIteratorFunction = <T>(source: AsyncIterable<T>[]) => AsyncIterable<T>;
 

package/test/src/setup.ts

@@ -2,6 +2,6 @@ import { container } from '@powersync/lib-services-framework';
 import { beforeAll } from 'vitest';
 
 beforeAll(() => {
-  // Your setup code here
+  // Executes for every test file
   container.registerDefaults();
 });

package/test/src/stream_utils.ts

@@ -0,0 +1,42 @@
+import { OplogEntry } from '@/util/protocol-types.js';
+import { JSONBig } from '@powersync/service-jsonbig';
+
+export function putOp(table: string, data: Record<string, any>): Partial<OplogEntry> {
+  return {
+    op: 'PUT',
+    object_type: table,
+    object_id: data.id,
+    data: JSONBig.stringify(data)
+  };
+}
+
+export function removeOp(table: string, id: string): Partial<OplogEntry> {
+  return {
+    op: 'REMOVE',
+    object_type: table,
+    object_id: id
+  };
+}
+
+export function compareIds(a: OplogEntry, b: OplogEntry) {
+  return a.object_id!.localeCompare(b.object_id!);
+}
+
+export async function oneFromAsync<T>(source: Iterable<T> | AsyncIterable<T>): Promise<T> {
+  const items: T[] = [];
+  for await (const item of source) {
+    items.push(item);
+  }
+  if (items.length != 1) {
+    throw new Error(`One item expected, got: ${items.length}`);
+  }
+  return items[0];
+}
+
+export async function fromAsync<T>(source: Iterable<T> | AsyncIterable<T>): Promise<T[]> {
+  const items: T[] = [];
+  for await (const item of source) {
+    items.push(item);
+  }
+  return items;
+}

package/test/src/sync.test.ts

@@ -1,13 +1,19 @@
 import { RequestTracker } from '@/sync/RequestTracker.js';
+import { streamResponse } from '@/sync/sync.js';
 import { StreamingSyncLine } from '@/util/protocol-types.js';
-import { lsnMakeComparable } from '@powersync/service-jpgwire';
 import { JSONBig } from '@powersync/service-jsonbig';
 import { RequestParameters } from '@powersync/service-sync-rules';
 import * as timers from 'timers/promises';
 import { describe, expect, test } from 'vitest';
-import { ZERO_LSN } from '../../src/replication/WalStream.js';
-import { streamResponse } from '../../src/sync/sync.js';
-import { makeTestTable, MONGO_STORAGE_FACTORY, StorageFactory } from './util.js';
+import {
+  BATCH_OPTIONS,
+  makeTestTable,
+  MONGO_STORAGE_FACTORY,
+  PARSE_OPTIONS,
+  StorageFactory,
+  ZERO_LSN
+} from './util.js';
+import { ParseSyncRulesOptions, StartBatchOptions } from '@/storage/BucketStorage.js';
 
 describe('sync - mongodb', function () {
   defineTests(MONGO_STORAGE_FACTORY);
@@ -32,18 +38,19 @@ function defineTests(factory: StorageFactory) {
       content: BASIC_SYNC_RULES
     });
 
-    const storage = await f.getInstance(syncRules.parsed());
+    const storage = await f.getInstance(syncRules);
     await storage.setSnapshotDone(ZERO_LSN);
     await storage.autoActivate();
 
-    const result = await storage.startBatch({}, async (batch) => {
+    const result = await storage.startBatch(BATCH_OPTIONS, async (batch) => {
       await batch.save({
         sourceTable: TEST_TABLE,
         tag: 'insert',
         after: {
           id: 't1',
           description: 'Test 1'
-        }
+        },
+        afterReplicaId: 't1'
       });
 
       await batch.save({
@@ -52,10 +59,11 @@ function defineTests(factory: StorageFactory) {
         after: {
           id: 't2',
           description: 'Test 2'
-        }
+        },
+        afterReplicaId: 't2'
       });
 
-      await batch.commit(lsnMakeComparable('0/1'));
+      await batch.commit('0/1');
     });
 
     const stream = streamResponse({
@@ -65,6 +73,7 @@ function defineTests(factory: StorageFactory) {
         include_checksum: true,
         raw_data: true
       },
+      parseOptions: PARSE_OPTIONS,
       tracker,
       syncParams: new RequestParameters({ sub: '' }, {}),
       token: { exp: Date.now() / 1000 + 10 } as any
@@ -81,11 +90,11 @@ function defineTests(factory: StorageFactory) {
       content: BASIC_SYNC_RULES
     });
 
-    const storage = await f.getInstance(syncRules.parsed());
+    const storage = await f.getInstance(syncRules);
     await storage.setSnapshotDone(ZERO_LSN);
     await storage.autoActivate();
 
-    const result = await storage.startBatch({}, async (batch) => {
+    const result = await storage.startBatch(BATCH_OPTIONS, async (batch) => {
       await batch.save({
         sourceTable: TEST_TABLE,
         tag: 'insert',
@@ -93,10 +102,11 @@ function defineTests(factory: StorageFactory) {
           id: 't1',
           description: 'Test\n"string"',
           large_num: 12345678901234567890n
-        }
+        },
+        afterReplicaId: 't1'
       });
 
-      await batch.commit(lsnMakeComparable('0/1'));
+      await batch.commit('0/1');
     });
 
     const stream = streamResponse({
@@ -106,6 +116,7 @@ function defineTests(factory: StorageFactory) {
         include_checksum: true,
         raw_data: false
       },
+      parseOptions: PARSE_OPTIONS,
       tracker,
       syncParams: new RequestParameters({ sub: '' }, {}),
       token: { exp: Date.now() / 1000 + 10 } as any
@@ -124,7 +135,7 @@ function defineTests(factory: StorageFactory) {
       content: BASIC_SYNC_RULES
     });
 
-    const storage = await f.getInstance(syncRules.parsed());
+    const storage = await f.getInstance(syncRules);
     await storage.setSnapshotDone(ZERO_LSN);
     await storage.autoActivate();
 
@@ -135,6 +146,7 @@ function defineTests(factory: StorageFactory) {
         include_checksum: true,
         raw_data: true
       },
+      parseOptions: PARSE_OPTIONS,
       tracker,
       syncParams: new RequestParameters({ sub: '' }, {}),
       token: { exp: 0 } as any
@@ -151,7 +163,7 @@ function defineTests(factory: StorageFactory) {
       content: BASIC_SYNC_RULES
     });
 
-    const storage = await f.getInstance(syncRules.parsed());
+    const storage = await f.getInstance(syncRules);
     await storage.setSnapshotDone(ZERO_LSN);
     await storage.autoActivate();
 
@@ -162,6 +174,7 @@ function defineTests(factory: StorageFactory) {
         include_checksum: true,
         raw_data: true
       },
+      parseOptions: PARSE_OPTIONS,
       tracker,
       syncParams: new RequestParameters({ sub: '' }, {}),
       token: { exp: Date.now() / 1000 + 10 } as any
@@ -170,32 +183,34 @@ function defineTests(factory: StorageFactory) {
 
     expect(await getCheckpointLines(iter)).toMatchSnapshot();
 
-    await storage.startBatch({}, async (batch) => {
+    await storage.startBatch(BATCH_OPTIONS, async (batch) => {
       await batch.save({
         sourceTable: TEST_TABLE,
         tag: 'insert',
         after: {
           id: 't1',
           description: 'Test 1'
-        }
+        },
+        afterReplicaId: 't1'
       });
 
-      await batch.commit(lsnMakeComparable('0/1'));
+      await batch.commit('0/1');
     });
 
     expect(await getCheckpointLines(iter)).toMatchSnapshot();
 
-    await storage.startBatch({}, async (batch) => {
+    await storage.startBatch(BATCH_OPTIONS, async (batch) => {
       await batch.save({
         sourceTable: TEST_TABLE,
         tag: 'insert',
         after: {
           id: 't2',
           description: 'Test 2'
-        }
+        },
+        afterReplicaId: 't2'
       });
 
-      await batch.commit(lsnMakeComparable('0/2'));
+      await batch.commit('0/2');
     });
 
     expect(await getCheckpointLines(iter)).toMatchSnapshot();
@@ -210,7 +225,7 @@ function defineTests(factory: StorageFactory) {
       content: BASIC_SYNC_RULES
     });
 
-    const storage = await f.getInstance(syncRules.parsed());
+    const storage = await f.getInstance(syncRules);
     await storage.setSnapshotDone(ZERO_LSN);
     await storage.autoActivate();
 
@@ -223,6 +238,7 @@ function defineTests(factory: StorageFactory) {
         include_checksum: true,
         raw_data: true
       },
+      parseOptions: PARSE_OPTIONS,
       tracker,
       syncParams: new RequestParameters({ sub: '' }, {}),
       token: { exp: exp } as any
@@ -248,18 +264,19 @@ function defineTests(factory: StorageFactory) {
       content: BASIC_SYNC_RULES
     });
 
-    const storage = await f.getInstance(syncRules.parsed());
+    const storage = await f.getInstance(syncRules);
     await storage.setSnapshotDone(ZERO_LSN);
     await storage.autoActivate();
 
-    await storage.startBatch({}, async (batch) => {
+    await storage.startBatch(BATCH_OPTIONS, async (batch) => {
       await batch.save({
         sourceTable: TEST_TABLE,
         tag: 'insert',
         after: {
           id: 't1',
           description: 'Test 1'
-        }
+        },
+        afterReplicaId: 't1'
       });
 
       await batch.save({
@@ -268,10 +285,11 @@ function defineTests(factory: StorageFactory) {
         after: {
           id: 't2',
           description: 'Test 2'
-        }
+        },
+        afterReplicaId: 't2'
       });
 
-      await batch.commit(lsnMakeComparable('0/1'));
+      await batch.commit('0/1');
     });
 
     const stream = streamResponse({
@@ -281,6 +299,7 @@ function defineTests(factory: StorageFactory) {
         include_checksum: true,
         raw_data: true
       },
+      parseOptions: PARSE_OPTIONS,
       tracker,
       syncParams: new RequestParameters({ sub: '' }, {}),
       token: { exp: Date.now() / 1000 + 10 } as any
@@ -300,14 +319,15 @@ function defineTests(factory: StorageFactory) {
     // Now we save additional data AND compact before continuing.
     // This invalidates the checkpoint we've received above.
 
-    await storage.startBatch({}, async (batch) => {
+    await storage.startBatch(BATCH_OPTIONS, async (batch) => {
       await batch.save({
         sourceTable: TEST_TABLE,
         tag: 'update',
         after: {
           id: 't1',
           description: 'Test 1b'
-        }
+        },
+        afterReplicaId: 't1'
       });
 
       await batch.save({
@@ -316,10 +336,11 @@ function defineTests(factory: StorageFactory) {
         after: {
           id: 't2',
           description: 'Test 2b'
-        }
+        },
+        afterReplicaId: 't2'
       });
 
-      await batch.commit(lsnMakeComparable('0/2'));
+      await batch.commit('0/2');
     });
 
     await storage.compact();

package/test/src/util.ts

@@ -1,16 +1,21 @@
-import * as pgwire from '@powersync/service-jpgwire';
-import { normalizeConnection } from '@powersync/service-types';
-import * as mongo from 'mongodb';
-import { BucketStorageFactory, SyncBucketDataBatch } from '../../src/storage/BucketStorage.js';
-import { MongoBucketStorage } from '../../src/storage/MongoBucketStorage.js';
-import { PowerSyncMongo } from '../../src/storage/mongo/db.js';
-import { escapeIdentifier } from '../../src/util/pgwire_utils.js';
-import { env } from './env.js';
 import { Metrics } from '@/metrics/Metrics.js';
-import { hashData } from '@/util/utils.js';
+import {
+  BucketStorageFactory,
+  ParseSyncRulesOptions,
+  PersistedSyncRulesContent,
+  StartBatchOptions,
+  SyncBucketDataBatch
+} from '@/storage/BucketStorage.js';
+import { MongoBucketStorage } from '@/storage/MongoBucketStorage.js';
 import { SourceTable } from '@/storage/SourceTable.js';
-import * as bson from 'bson';
+import { PowerSyncMongo } from '@/storage/mongo/db.js';
 import { SyncBucketData } from '@/util/protocol-types.js';
+import { getUuidReplicaIdentityBson, hashData } from '@/util/utils.js';
+import * as bson from 'bson';
+import * as mongo from 'mongodb';
+import { env } from './env.js';
+import { SqlSyncRules } from '@powersync/service-sync-rules';
+import { ReplicaId } from '@/storage/storage-index.js';
 
 // The metrics need to be initialised before they can be used
 await Metrics.initialise({
@@ -20,8 +25,6 @@ await Metrics.initialise({
 });
 Metrics.getInstance().resetCounters();
 
-export const TEST_URI = env.PG_TEST_URL;
-
 export type StorageFactory = () => Promise<BucketStorageFactory>;
 
 export const MONGO_STORAGE_FACTORY: StorageFactory = async () => {
@@ -30,45 +33,33 @@ export const MONGO_STORAGE_FACTORY: StorageFactory = async () => {
   return new MongoBucketStorage(db, { slot_name_prefix: 'test_' });
 };
 
-export async function clearTestDb(db: pgwire.PgClient) {
-  await db.query(
-    "select pg_drop_replication_slot(slot_name) from pg_replication_slots where active = false and slot_name like 'test_%'"
-  );
+export const ZERO_LSN = '0/0';
 
-  await db.query(`CREATE EXTENSION IF NOT EXISTS "uuid-ossp"`);
-  try {
-    await db.query(`DROP PUBLICATION powersync`);
-  } catch (e) {
-    // Ignore
-  }
+export const PARSE_OPTIONS: ParseSyncRulesOptions = {
+  defaultSchema: 'public'
+};
 
-  await db.query(`CREATE PUBLICATION powersync FOR ALL TABLES`);
+export const BATCH_OPTIONS: StartBatchOptions = {
+  ...PARSE_OPTIONS,
+  zeroLSN: ZERO_LSN
+};
 
-  const tableRows = pgwire.pgwireRows(
-    await db.query(`SELECT table_name FROM information_schema.tables where table_schema = 'public'`)
-  );
-  for (let row of tableRows) {
-    const name = row.table_name;
-    if (name.startsWith('test_')) {
-      await db.query(`DROP TABLE public.${escapeIdentifier(name)}`);
+export function testRules(content: string): PersistedSyncRulesContent {
+  return {
+    id: 1,
+    sync_rules_content: content,
+    slot_name: 'test',
+    parsed(options) {
+      return {
+        id: 1,
+        sync_rules: SqlSyncRules.fromYaml(content, PARSE_OPTIONS),
+        slot_name: 'test'
+      };
+    },
+    lock() {
+      throw new Error('Not implemented');
     }
-  }
-}
-
-export const TEST_CONNECTION_OPTIONS = normalizeConnection({
-  type: 'postgresql',
-  uri: TEST_URI,
-  sslmode: 'disable'
-});
-
-export async function connectPgWire(type?: 'replication' | 'standard') {
-  const db = await pgwire.connectPgWire(TEST_CONNECTION_OPTIONS, { type });
-  return db;
-}
-
-export function connectPgPool() {
-  const db = pgwire.connectPgWirePool(TEST_CONNECTION_OPTIONS);
-  return db;
+  };
 }
 
 export async function connectMongo() {
@@ -79,8 +70,7 @@ export async function connectMongo() {
     socketTimeoutMS: env.CI ? 15_000 : 5_000,
     serverSelectionTimeoutMS: env.CI ? 15_000 : 2_500
   });
-  const db = new PowerSyncMongo(client);
-  return db;
+  return new PowerSyncMongo(client);
 }
 
 export function makeTestTable(name: string, columns?: string[] | undefined) {
@@ -90,9 +80,9 @@ export function makeTestTable(name: string, columns?: string[] | undefined) {
     id,
     SourceTable.DEFAULT_TAG,
     relId,
-    SourceTable.DEFAULT_SCHEMA,
+    'public',
     name,
-    (columns ?? ['id']).map((column) => ({ name: column, typeOid: 25 })),
+    (columns ?? ['id']).map((column) => ({ name: column, type: 'VARCHAR', typeId: 25 })),
     true
   );
 }
@@ -138,3 +128,10 @@ function getFirst(batch: SyncBucketData[] | SyncBucketDataBatch[] | SyncBucketDa
     return first as SyncBucketData;
   }
 }
+
+/**
+ * Replica id in the old Postgres format, for backwards-compatible tests.
+ */
+export function rid(id: string): bson.UUID {
+  return getUuidReplicaIdentityBson({ id: id }, [{ name: 'id', type: 'VARCHAR', typeId: 25 }]);
+}

package/test/tsconfig.json

@@ -2,8 +2,8 @@
   "extends": "../../../tsconfig.base.json",
   "compilerOptions": {
     "rootDir": "src",
-    "noEmit": true,
     "baseUrl": "./",
+    "outDir": "dist",
     "esModuleInterop": true,
     "skipLibCheck": true,
     "sourceMap": true,