@powersync/service-core 0.0.0-dev-20240718134716 → 0.0.0-dev-20240918082156

package/test/src/env.ts

@@ -2,7 +2,5 @@ import { utils } from '@powersync/lib-services-framework';
 
 export const env = utils.collectEnvironmentVariables({
   MONGO_TEST_URL: utils.type.string.default('mongodb://localhost:27017/powersync_test'),
-  PG_TEST_URL: utils.type.string.default('postgres://postgres:postgres@localhost:5432/powersync_test'),
-  CI: utils.type.boolean.default('false'),
-  SLOW_TESTS: utils.type.boolean.default('false')
+  CI: utils.type.boolean.default('false')
 });

package/test/src/merge_iterable.test.ts

@@ -1,11 +1,6 @@
+import { mergeAsyncIterablesNew, mergeAsyncIterablesOld } from '@/sync/merge.js';
 import * as timers from 'timers/promises';
 import { describe, expect, test } from 'vitest';
-import {
-  FixedMergeAsyncIterable,
-  mergeAsyncIterables,
-  mergeAsyncIterablesNew,
-  mergeAsyncIterablesOld
-} from '../../src/sync/merge.js';
 
 type MergeIteratorFunction = <T>(source: AsyncIterable<T>[]) => AsyncIterable<T>;
 

package/test/src/setup.ts

@@ -2,6 +2,6 @@ import { container } from '@powersync/lib-services-framework';
 import { beforeAll } from 'vitest';
 
 beforeAll(() => {
-  // Your setup code here
+  // Executes for every test file
   container.registerDefaults();
 });

package/test/src/stream_utils.ts

@@ -0,0 +1,42 @@
+import { OplogEntry } from '@/util/protocol-types.js';
+import { JSONBig } from '@powersync/service-jsonbig';
+
+export function putOp(table: string, data: Record<string, any>): Partial<OplogEntry> {
+  return {
+    op: 'PUT',
+    object_type: table,
+    object_id: data.id,
+    data: JSONBig.stringify(data)
+  };
+}
+
+export function removeOp(table: string, id: string): Partial<OplogEntry> {
+  return {
+    op: 'REMOVE',
+    object_type: table,
+    object_id: id
+  };
+}
+
+export function compareIds(a: OplogEntry, b: OplogEntry) {
+  return a.object_id!.localeCompare(b.object_id!);
+}
+
+export async function oneFromAsync<T>(source: Iterable<T> | AsyncIterable<T>): Promise<T> {
+  const items: T[] = [];
+  for await (const item of source) {
+    items.push(item);
+  }
+  if (items.length != 1) {
+    throw new Error(`One item expected, got: ${items.length}`);
+  }
+  return items[0];
+}
+
+export async function fromAsync<T>(source: Iterable<T> | AsyncIterable<T>): Promise<T[]> {
+  const items: T[] = [];
+  for await (const item of source) {
+    items.push(item);
+  }
+  return items;
+}

package/test/src/sync.test.ts

@@ -1,34 +1,24 @@
-import * as bson from 'bson';
-import { describe, expect, test } from 'vitest';
-import { ZERO_LSN } from '../../src/replication/WalStream.js';
-import { SourceTable } from '../../src/storage/SourceTable.js';
-import { hashData } from '../../src/util/utils.js';
-import { MONGO_STORAGE_FACTORY, StorageFactory } from './util.js';
+import { RequestTracker } from '@/sync/RequestTracker.js';
+import { streamResponse } from '@/sync/sync.js';
+import { StreamingSyncLine } from '@/util/protocol-types.js';
 import { JSONBig } from '@powersync/service-jsonbig';
-import { streamResponse } from '../../src/sync/sync.js';
-import * as timers from 'timers/promises';
-import { lsnMakeComparable } from '@powersync/service-jpgwire';
 import { RequestParameters } from '@powersync/service-sync-rules';
-import { RequestTracker } from '@/sync/RequestTracker.js';
+import * as timers from 'timers/promises';
+import { describe, expect, test } from 'vitest';
+import {
+  BATCH_OPTIONS,
+  makeTestTable,
+  MONGO_STORAGE_FACTORY,
+  PARSE_OPTIONS,
+  StorageFactory,
+  ZERO_LSN
+} from './util.js';
+import { ParseSyncRulesOptions, StartBatchOptions } from '@/storage/BucketStorage.js';
 
 describe('sync - mongodb', function () {
   defineTests(MONGO_STORAGE_FACTORY);
 });
 
-function makeTestTable(name: string, columns?: string[] | undefined) {
-  const relId = hashData('table', name, (columns ?? ['id']).join(','));
-  const id = new bson.ObjectId('6544e3899293153fa7b38331');
-  return new SourceTable(
-    id,
-    SourceTable.DEFAULT_TAG,
-    relId,
-    SourceTable.DEFAULT_SCHEMA,
-    name,
-    (columns ?? ['id']).map((column) => ({ name: column, typeOid: 25 })),
-    true
-  );
-}
-
 const TEST_TABLE = makeTestTable('test', ['id']);
 
 const BASIC_SYNC_RULES = `
@@ -48,18 +38,19 @@ function defineTests(factory: StorageFactory) {
       content: BASIC_SYNC_RULES
     });
 
-    const storage = await f.getInstance(syncRules.parsed());
+    const storage = await f.getInstance(syncRules);
     await storage.setSnapshotDone(ZERO_LSN);
     await storage.autoActivate();
 
-    const result = await storage.startBatch({}, async (batch) => {
+    const result = await storage.startBatch(BATCH_OPTIONS, async (batch) => {
       await batch.save({
         sourceTable: TEST_TABLE,
         tag: 'insert',
         after: {
           id: 't1',
           description: 'Test 1'
-        }
+        },
+        afterReplicaId: 't1'
       });
 
       await batch.save({
@@ -68,10 +59,11 @@ function defineTests(factory: StorageFactory) {
         after: {
           id: 't2',
           description: 'Test 2'
-        }
+        },
+        afterReplicaId: 't2'
       });
 
-      await batch.commit(lsnMakeComparable('0/1'));
+      await batch.commit('0/1');
     });
 
     const stream = streamResponse({
@@ -81,6 +73,7 @@ function defineTests(factory: StorageFactory) {
         include_checksum: true,
         raw_data: true
       },
+      parseOptions: PARSE_OPTIONS,
       tracker,
       syncParams: new RequestParameters({ sub: '' }, {}),
       token: { exp: Date.now() / 1000 + 10 } as any
@@ -97,11 +90,11 @@ function defineTests(factory: StorageFactory) {
       content: BASIC_SYNC_RULES
     });
 
-    const storage = await f.getInstance(syncRules.parsed());
+    const storage = await f.getInstance(syncRules);
     await storage.setSnapshotDone(ZERO_LSN);
     await storage.autoActivate();
 
-    const result = await storage.startBatch({}, async (batch) => {
+    const result = await storage.startBatch(BATCH_OPTIONS, async (batch) => {
       await batch.save({
         sourceTable: TEST_TABLE,
         tag: 'insert',
@@ -109,10 +102,11 @@ function defineTests(factory: StorageFactory) {
           id: 't1',
           description: 'Test\n"string"',
           large_num: 12345678901234567890n
-        }
+        },
+        afterReplicaId: 't1'
       });
 
-      await batch.commit(lsnMakeComparable('0/1'));
+      await batch.commit('0/1');
     });
 
     const stream = streamResponse({
@@ -122,6 +116,7 @@ function defineTests(factory: StorageFactory) {
         include_checksum: true,
         raw_data: false
       },
+      parseOptions: PARSE_OPTIONS,
       tracker,
       syncParams: new RequestParameters({ sub: '' }, {}),
       token: { exp: Date.now() / 1000 + 10 } as any
@@ -140,7 +135,7 @@ function defineTests(factory: StorageFactory) {
       content: BASIC_SYNC_RULES
     });
 
-    const storage = await f.getInstance(syncRules.parsed());
+    const storage = await f.getInstance(syncRules);
     await storage.setSnapshotDone(ZERO_LSN);
     await storage.autoActivate();
 
@@ -151,6 +146,7 @@ function defineTests(factory: StorageFactory) {
         include_checksum: true,
         raw_data: true
       },
+      parseOptions: PARSE_OPTIONS,
       tracker,
       syncParams: new RequestParameters({ sub: '' }, {}),
       token: { exp: 0 } as any
@@ -167,7 +163,7 @@ function defineTests(factory: StorageFactory) {
       content: BASIC_SYNC_RULES
     });
 
-    const storage = await f.getInstance(syncRules.parsed());
+    const storage = await f.getInstance(syncRules);
     await storage.setSnapshotDone(ZERO_LSN);
     await storage.autoActivate();
 
@@ -178,6 +174,7 @@ function defineTests(factory: StorageFactory) {
         include_checksum: true,
         raw_data: true
       },
+      parseOptions: PARSE_OPTIONS,
       tracker,
       syncParams: new RequestParameters({ sub: '' }, {}),
       token: { exp: Date.now() / 1000 + 10 } as any
@@ -186,32 +183,34 @@ function defineTests(factory: StorageFactory) {
 
     expect(await getCheckpointLines(iter)).toMatchSnapshot();
 
-    await storage.startBatch({}, async (batch) => {
+    await storage.startBatch(BATCH_OPTIONS, async (batch) => {
       await batch.save({
         sourceTable: TEST_TABLE,
         tag: 'insert',
         after: {
           id: 't1',
           description: 'Test 1'
-        }
+        },
+        afterReplicaId: 't1'
       });
 
-      await batch.commit(lsnMakeComparable('0/1'));
+      await batch.commit('0/1');
     });
 
     expect(await getCheckpointLines(iter)).toMatchSnapshot();
 
-    await storage.startBatch({}, async (batch) => {
+    await storage.startBatch(BATCH_OPTIONS, async (batch) => {
       await batch.save({
         sourceTable: TEST_TABLE,
         tag: 'insert',
         after: {
           id: 't2',
           description: 'Test 2'
-        }
+        },
+        afterReplicaId: 't2'
       });
 
-      await batch.commit(lsnMakeComparable('0/2'));
+      await batch.commit('0/2');
     });
 
     expect(await getCheckpointLines(iter)).toMatchSnapshot();
@@ -226,7 +225,7 @@ function defineTests(factory: StorageFactory) {
       content: BASIC_SYNC_RULES
     });
 
-    const storage = await f.getInstance(syncRules.parsed());
+    const storage = await f.getInstance(syncRules);
     await storage.setSnapshotDone(ZERO_LSN);
     await storage.autoActivate();
 
@@ -239,6 +238,7 @@ function defineTests(factory: StorageFactory) {
         include_checksum: true,
         raw_data: true
       },
+      parseOptions: PARSE_OPTIONS,
       tracker,
       syncParams: new RequestParameters({ sub: '' }, {}),
       token: { exp: exp } as any
@@ -251,15 +251,161 @@ function defineTests(factory: StorageFactory) {
     const expLines = await getCheckpointLines(iter);
     expect(expLines).toMatchSnapshot();
   });
+
+  test('compacting data - invalidate checkpoint', async () => {
+    // This tests a case of a compact operation invalidating a checkpoint in the
+    // middle of syncing data.
+    // This is expected to be rare in practice, but it is important to handle
+    // this case correctly to maintain consistency on the client.
+
+    const f = await factory();
+
+    const syncRules = await f.updateSyncRules({
+      content: BASIC_SYNC_RULES
+    });
+
+    const storage = await f.getInstance(syncRules);
+    await storage.setSnapshotDone(ZERO_LSN);
+    await storage.autoActivate();
+
+    await storage.startBatch(BATCH_OPTIONS, async (batch) => {
+      await batch.save({
+        sourceTable: TEST_TABLE,
+        tag: 'insert',
+        after: {
+          id: 't1',
+          description: 'Test 1'
+        },
+        afterReplicaId: 't1'
+      });
+
+      await batch.save({
+        sourceTable: TEST_TABLE,
+        tag: 'insert',
+        after: {
+          id: 't2',
+          description: 'Test 2'
+        },
+        afterReplicaId: 't2'
+      });
+
+      await batch.commit('0/1');
+    });
+
+    const stream = streamResponse({
+      storage: f,
+      params: {
+        buckets: [],
+        include_checksum: true,
+        raw_data: true
+      },
+      parseOptions: PARSE_OPTIONS,
+      tracker,
+      syncParams: new RequestParameters({ sub: '' }, {}),
+      token: { exp: Date.now() / 1000 + 10 } as any
+    });
+
+    const iter = stream[Symbol.asyncIterator]();
+
+    // Only consume the first "checkpoint" message, and pause before receiving data.
+    const lines = await consumeIterator(iter, { consume: false, isDone: (line) => (line as any)?.checkpoint != null });
+    expect(lines).toMatchSnapshot();
+    expect(lines[0]).toEqual({
+      checkpoint: expect.objectContaining({
+        last_op_id: '2'
+      })
+    });
+
+    // Now we save additional data AND compact before continuing.
+    // This invalidates the checkpoint we've received above.
+
+    await storage.startBatch(BATCH_OPTIONS, async (batch) => {
+      await batch.save({
+        sourceTable: TEST_TABLE,
+        tag: 'update',
+        after: {
+          id: 't1',
+          description: 'Test 1b'
+        },
+        afterReplicaId: 't1'
+      });
+
+      await batch.save({
+        sourceTable: TEST_TABLE,
+        tag: 'update',
+        after: {
+          id: 't2',
+          description: 'Test 2b'
+        },
+        afterReplicaId: 't2'
+      });
+
+      await batch.commit('0/2');
+    });
+
+    await storage.compact();
+
+    const lines2 = await getCheckpointLines(iter, { consume: true });
+
+    // Snapshot test checks for changes in general.
+    // The tests after that documents the specific things we're looking for
+    // in this test.
+    expect(lines2).toMatchSnapshot();
+
+    expect(lines2[0]).toEqual({
+      data: expect.objectContaining({
+        has_more: false,
+        data: [
+          // The first two ops have been replaced by a single CLEAR op
+          expect.objectContaining({
+            op: 'CLEAR'
+          })
+        ]
+      })
+    });
+
+    // Note: No checkpoint_complete here, since the checkpoint has been
+    // invalidated by the CLEAR op.
+
+    expect(lines2[1]).toEqual({
+      checkpoint_diff: expect.objectContaining({
+        last_op_id: '4'
+      })
+    });
+
+    expect(lines2[2]).toEqual({
+      data: expect.objectContaining({
+        has_more: false,
+        data: [
+          expect.objectContaining({
+            op: 'PUT'
+          }),
+          expect.objectContaining({
+            op: 'PUT'
+          })
+        ]
+      })
+    });
+
+    // Now we get a checkpoint_complete
+    expect(lines2[3]).toEqual({
+      checkpoint_complete: expect.objectContaining({
+        last_op_id: '4'
+      })
+    });
+  });
 }
 
 /**
- * Get lines on an iterator until the next checkpoint_complete.
+ * Get lines on an iterator until isDone(line) == true.
  *
- * Does not stop the iterator.
+ * Does not stop the iterator unless options.consume is true.
  */
-async function getCheckpointLines(iter: AsyncIterator<any>, options?: { consume?: boolean }): Promise<any[]> {
-  let lines: any[] = [];
+async function consumeIterator<T>(
+  iter: AsyncIterator<T>,
+  options: { isDone: (line: T) => boolean; consume?: boolean }
+) {
+  let lines: T[] = [];
   try {
     const controller = new AbortController();
     const timeout = timers.setTimeout(1500, { value: null, done: 'timeout' }, { signal: controller.signal });
@@ -274,7 +420,7 @@ async function getCheckpointLines(iter: AsyncIterator<any>, options?: { consume?
       if (value) {
         lines.push(value);
       }
-      if (done || value.checkpoint_complete) {
+      if (done || options.isDone(value)) {
         break;
       }
     }
@@ -292,11 +438,26 @@ async function getCheckpointLines(iter: AsyncIterator<any>, options?: { consume?
   }
 }
 
+/**
+ * Get lines on an iterator until the next checkpoint_complete.
+ *
+ * Does not stop the iterator unless options.consume is true.
+ */
+async function getCheckpointLines(
+  iter: AsyncIterator<StreamingSyncLine | string | null>,
+  options?: { consume?: boolean }
+) {
+  return consumeIterator(iter, {
+    consume: options?.consume,
+    isDone: (line) => (line as any)?.checkpoint_complete
+  });
+}
+
 /**
  * Get lines on an iterator until the next checkpoint_complete.
  *
  * Stops the iterator afterwards.
  */
-async function consumeCheckpointLines(iterable: AsyncIterable<any>): Promise<any[]> {
+async function consumeCheckpointLines(iterable: AsyncIterable<StreamingSyncLine | string | null>): Promise<any[]> {
   return getCheckpointLines(iterable[Symbol.asyncIterator](), { consume: true });
 }

package/test/src/util.ts

@@ -1,28 +1,29 @@
-import * as pgwire from '@powersync/service-jpgwire';
-import { normalizeConnection } from '@powersync/service-types';
+import { Metrics } from '@/metrics/Metrics.js';
+import {
+  BucketStorageFactory,
+  ParseSyncRulesOptions,
+  PersistedSyncRulesContent,
+  StartBatchOptions,
+  SyncBucketDataBatch
+} from '@/storage/BucketStorage.js';
+import { MongoBucketStorage } from '@/storage/MongoBucketStorage.js';
+import { SourceTable } from '@/storage/SourceTable.js';
+import { PowerSyncMongo } from '@/storage/mongo/db.js';
+import { SyncBucketData } from '@/util/protocol-types.js';
+import { getUuidReplicaIdentityBson, hashData } from '@/util/utils.js';
+import * as bson from 'bson';
 import * as mongo from 'mongodb';
-import { BucketStorageFactory } from '../../src/storage/BucketStorage.js';
-import { MongoBucketStorage } from '../../src/storage/MongoBucketStorage.js';
-import { PowerSyncMongo } from '../../src/storage/mongo/db.js';
-import { escapeIdentifier } from '../../src/util/pgwire_utils.js';
 import { env } from './env.js';
-import { Metrics } from '@/metrics/Metrics.js';
-import { container } from '@powersync/lib-services-framework';
-import { MeterProvider } from '@opentelemetry/sdk-metrics';
-import { PrometheusExporter } from '@opentelemetry/exporter-prometheus';
+import { SqlSyncRules } from '@powersync/service-sync-rules';
+import { ReplicaId } from '@/storage/storage-index.js';
 
 // The metrics need to be initialised before they can be used
-const prometheus = new PrometheusExporter();
-const metrics = new Metrics(
-  new MeterProvider({
-    readers: [prometheus]
-  }),
-  prometheus
-);
-container.register(Metrics, metrics);
-metrics.resetCounters();
-
-export const TEST_URI = env.PG_TEST_URL;
+await Metrics.initialise({
+  disable_telemetry_sharing: true,
+  powersync_instance_id: 'test',
+  internal_metrics_endpoint: 'unused.for.tests.com'
+});
+Metrics.getInstance().resetCounters();
 
 export type StorageFactory = () => Promise<BucketStorageFactory>;
 
@@ -32,41 +33,33 @@ export const MONGO_STORAGE_FACTORY: StorageFactory = async () => {
   return new MongoBucketStorage(db, { slot_name_prefix: 'test_' });
 };
 
-export async function clearTestDb(db: pgwire.PgClient) {
-  await db.query(`CREATE EXTENSION IF NOT EXISTS "uuid-ossp"`);
-  try {
-    await db.query(`DROP PUBLICATION powersync`);
-  } catch (e) {
-    // Ignore
-  }
+export const ZERO_LSN = '0/0';
 
-  await db.query(`CREATE PUBLICATION powersync FOR ALL TABLES`);
-
-  const tableRows = pgwire.pgwireRows(
-    await db.query(`SELECT table_name FROM information_schema.tables where table_schema = 'public'`)
-  );
-  for (let row of tableRows) {
-    const name = row.table_name;
-    if (name.startsWith('test_')) {
-      await db.query(`DROP TABLE public.${escapeIdentifier(name)}`);
-    }
-  }
-}
-
-export const TEST_CONNECTION_OPTIONS = normalizeConnection({
-  type: 'postgresql',
-  uri: TEST_URI,
-  sslmode: 'disable'
-});
+export const PARSE_OPTIONS: ParseSyncRulesOptions = {
+  defaultSchema: 'public'
+};
 
-export async function connectPgWire(type?: 'replication' | 'standard') {
-  const db = await pgwire.connectPgWire(TEST_CONNECTION_OPTIONS, { type });
-  return db;
-}
+export const BATCH_OPTIONS: StartBatchOptions = {
+  ...PARSE_OPTIONS,
+  zeroLSN: ZERO_LSN
+};
 
-export function connectPgPool() {
-  const db = pgwire.connectPgWirePool(TEST_CONNECTION_OPTIONS);
-  return db;
+export function testRules(content: string): PersistedSyncRulesContent {
+  return {
+    id: 1,
+    sync_rules_content: content,
+    slot_name: 'test',
+    parsed(options) {
+      return {
+        id: 1,
+        sync_rules: SqlSyncRules.fromYaml(content, PARSE_OPTIONS),
+        slot_name: 'test'
+      };
+    },
+    lock() {
+      throw new Error('Not implemented');
+    }
+  };
 }
 
 export async function connectMongo() {
@@ -77,6 +70,68 @@ export async function connectMongo() {
     socketTimeoutMS: env.CI ? 15_000 : 5_000,
     serverSelectionTimeoutMS: env.CI ? 15_000 : 2_500
   });
-  const db = new PowerSyncMongo(client);
-  return db;
+  return new PowerSyncMongo(client);
+}
+
+export function makeTestTable(name: string, columns?: string[] | undefined) {
+  const relId = hashData('table', name, (columns ?? ['id']).join(','));
+  const id = new bson.ObjectId('6544e3899293153fa7b38331');
+  return new SourceTable(
+    id,
+    SourceTable.DEFAULT_TAG,
+    relId,
+    'public',
+    name,
+    (columns ?? ['id']).map((column) => ({ name: column, type: 'VARCHAR', typeId: 25 })),
+    true
+  );
+}
+
+export function getBatchData(batch: SyncBucketData[] | SyncBucketDataBatch[] | SyncBucketDataBatch) {
+  const first = getFirst(batch);
+  if (first == null) {
+    return [];
+  }
+  return first.data.map((d) => {
+    return {
+      op_id: d.op_id,
+      op: d.op,
+      object_id: d.object_id,
+      checksum: d.checksum
+    };
+  });
+}
+
+export function getBatchMeta(batch: SyncBucketData[] | SyncBucketDataBatch[] | SyncBucketDataBatch) {
+  const first = getFirst(batch);
+  if (first == null) {
+    return null;
+  }
+  return {
+    has_more: first.has_more,
+    after: first.after,
+    next_after: first.next_after
+  };
+}
+
+function getFirst(batch: SyncBucketData[] | SyncBucketDataBatch[] | SyncBucketDataBatch): SyncBucketData | null {
+  if (!Array.isArray(batch)) {
+    return batch.batch;
+  }
+  if (batch.length == 0) {
+    return null;
+  }
+  let first = batch[0];
+  if ((first as SyncBucketDataBatch).batch != null) {
+    return (first as SyncBucketDataBatch).batch;
+  } else {
+    return first as SyncBucketData;
+  }
+}
+
+/**
+ * Replica id in the old Postgres format, for backwards-compatible tests.
+ */
+export function rid(id: string): bson.UUID {
+  return getUuidReplicaIdentityBson({ id: id }, [{ name: 'id', type: 'VARCHAR', typeId: 25 }]);
 }

package/test/tsconfig.json

@@ -2,8 +2,8 @@
   "extends": "../../../tsconfig.base.json",
   "compilerOptions": {
     "rootDir": "src",
-    "noEmit": true,
     "baseUrl": "./",
+    "outDir": "dist",
     "esModuleInterop": true,
     "skipLibCheck": true,
     "sourceMap": true,