@powersync/service-core 0.0.0-dev-20240718134716 → 0.0.0-dev-20240725112650

package/test/src/sync.test.ts

@@ -1,34 +1,18 @@
-import * as bson from 'bson';
+import { RequestTracker } from '@/sync/RequestTracker.js';
+import { StreamingSyncLine } from '@/util/protocol-types.js';
+import { lsnMakeComparable } from '@powersync/service-jpgwire';
+import { JSONBig } from '@powersync/service-jsonbig';
+import { RequestParameters } from '@powersync/service-sync-rules';
+import * as timers from 'timers/promises';
 import { describe, expect, test } from 'vitest';
 import { ZERO_LSN } from '../../src/replication/WalStream.js';
-import { SourceTable } from '../../src/storage/SourceTable.js';
-import { hashData } from '../../src/util/utils.js';
-import { MONGO_STORAGE_FACTORY, StorageFactory } from './util.js';
-import { JSONBig } from '@powersync/service-jsonbig';
 import { streamResponse } from '../../src/sync/sync.js';
-import * as timers from 'timers/promises';
-import { lsnMakeComparable } from '@powersync/service-jpgwire';
-import { RequestParameters } from '@powersync/service-sync-rules';
-import { RequestTracker } from '@/sync/RequestTracker.js';
+import { makeTestTable, MONGO_STORAGE_FACTORY, StorageFactory } from './util.js';
 
 describe('sync - mongodb', function () {
   defineTests(MONGO_STORAGE_FACTORY);
 });
 
-function makeTestTable(name: string, columns?: string[] | undefined) {
-  const relId = hashData('table', name, (columns ?? ['id']).join(','));
-  const id = new bson.ObjectId('6544e3899293153fa7b38331');
-  return new SourceTable(
-    id,
-    SourceTable.DEFAULT_TAG,
-    relId,
-    SourceTable.DEFAULT_SCHEMA,
-    name,
-    (columns ?? ['id']).map((column) => ({ name: column, typeOid: 25 })),
-    true
-  );
-}
-
 const TEST_TABLE = makeTestTable('test', ['id']);
 
 const BASIC_SYNC_RULES = `
@@ -251,15 +235,156 @@ function defineTests(factory: StorageFactory) {
     const expLines = await getCheckpointLines(iter);
     expect(expLines).toMatchSnapshot();
   });
+
+  test('compacting data - invalidate checkpoint', async () => {
+    // This tests a case of a compact operation invalidating a checkpoint in the
+    // middle of syncing data.
+    // This is expected to be rare in practice, but it is important to handle
+    // this case correctly to maintain consistency on the client.
+
+    const f = await factory();
+
+    const syncRules = await f.updateSyncRules({
+      content: BASIC_SYNC_RULES
+    });
+
+    const storage = await f.getInstance(syncRules.parsed());
+    await storage.setSnapshotDone(ZERO_LSN);
+    await storage.autoActivate();
+
+    await storage.startBatch({}, async (batch) => {
+      await batch.save({
+        sourceTable: TEST_TABLE,
+        tag: 'insert',
+        after: {
+          id: 't1',
+          description: 'Test 1'
+        }
+      });
+
+      await batch.save({
+        sourceTable: TEST_TABLE,
+        tag: 'insert',
+        after: {
+          id: 't2',
+          description: 'Test 2'
+        }
+      });
+
+      await batch.commit(lsnMakeComparable('0/1'));
+    });
+
+    const stream = streamResponse({
+      storage: f,
+      params: {
+        buckets: [],
+        include_checksum: true,
+        raw_data: true
+      },
+      tracker,
+      syncParams: new RequestParameters({ sub: '' }, {}),
+      token: { exp: Date.now() / 1000 + 10 } as any
+    });
+
+    const iter = stream[Symbol.asyncIterator]();
+
+    // Only consume the first "checkpoint" message, and pause before receiving data.
+    const lines = await consumeIterator(iter, { consume: false, isDone: (line) => (line as any)?.checkpoint != null });
+    expect(lines).toMatchSnapshot();
+    expect(lines[0]).toEqual({
+      checkpoint: expect.objectContaining({
+        last_op_id: '2'
+      })
+    });
+
+    // Now we save additional data AND compact before continuing.
+    // This invalidates the checkpoint we've received above.
+
+    await storage.startBatch({}, async (batch) => {
+      await batch.save({
+        sourceTable: TEST_TABLE,
+        tag: 'update',
+        after: {
+          id: 't1',
+          description: 'Test 1b'
+        }
+      });
+
+      await batch.save({
+        sourceTable: TEST_TABLE,
+        tag: 'update',
+        after: {
+          id: 't2',
+          description: 'Test 2b'
+        }
+      });
+
+      await batch.commit(lsnMakeComparable('0/2'));
+    });
+
+    await storage.compact();
+
+    const lines2 = await getCheckpointLines(iter, { consume: true });
+
+    // Snapshot test checks for changes in general.
+    // The tests after that documents the specific things we're looking for
+    // in this test.
+    expect(lines2).toMatchSnapshot();
+
+    expect(lines2[0]).toEqual({
+      data: expect.objectContaining({
+        has_more: false,
+        data: [
+          // The first two ops have been replaced by a single CLEAR op
+          expect.objectContaining({
+            op: 'CLEAR'
+          })
+        ]
+      })
+    });
+
+    // Note: No checkpoint_complete here, since the checkpoint has been
+    // invalidated by the CLEAR op.
+
+    expect(lines2[1]).toEqual({
+      checkpoint_diff: expect.objectContaining({
+        last_op_id: '4'
+      })
+    });
+
+    expect(lines2[2]).toEqual({
+      data: expect.objectContaining({
+        has_more: false,
+        data: [
+          expect.objectContaining({
+            op: 'PUT'
+          }),
+          expect.objectContaining({
+            op: 'PUT'
+          })
+        ]
+      })
+    });
+
+    // Now we get a checkpoint_complete
+    expect(lines2[3]).toEqual({
+      checkpoint_complete: expect.objectContaining({
+        last_op_id: '4'
+      })
+    });
+  });
 }
 
 /**
- * Get lines on an iterator until the next checkpoint_complete.
+ * Get lines on an iterator until isDone(line) == true.
  *
- * Does not stop the iterator.
+ * Does not stop the iterator unless options.consume is true.
  */
-async function getCheckpointLines(iter: AsyncIterator<any>, options?: { consume?: boolean }): Promise<any[]> {
-  let lines: any[] = [];
+async function consumeIterator<T>(
+  iter: AsyncIterator<T>,
+  options: { isDone: (line: T) => boolean; consume?: boolean }
+) {
+  let lines: T[] = [];
   try {
     const controller = new AbortController();
     const timeout = timers.setTimeout(1500, { value: null, done: 'timeout' }, { signal: controller.signal });
@@ -274,7 +399,7 @@ async function getCheckpointLines(iter: AsyncIterator<any>, options?: { consume?
       if (value) {
         lines.push(value);
       }
-      if (done || value.checkpoint_complete) {
+      if (done || options.isDone(value)) {
         break;
       }
     }
@@ -292,11 +417,26 @@ async function getCheckpointLines(iter: AsyncIterator<any>, options?: { consume?
   }
 }
 
+/**
+ * Get lines on an iterator until the next checkpoint_complete.
+ *
+ * Does not stop the iterator unless options.consume is true.
+ */
+async function getCheckpointLines(
+  iter: AsyncIterator<StreamingSyncLine | string | null>,
+  options?: { consume?: boolean }
+) {
+  return consumeIterator(iter, {
+    consume: options?.consume,
+    isDone: (line) => (line as any)?.checkpoint_complete
+  });
+}
+
 /**
  * Get lines on an iterator until the next checkpoint_complete.
  *
  * Stops the iterator afterwards.
  */
-async function consumeCheckpointLines(iterable: AsyncIterable<any>): Promise<any[]> {
+async function consumeCheckpointLines(iterable: AsyncIterable<StreamingSyncLine | string | null>): Promise<any[]> {
   return getCheckpointLines(iterable[Symbol.asyncIterator](), { consume: true });
 }

package/test/src/util.ts

@@ -1,26 +1,24 @@
 import * as pgwire from '@powersync/service-jpgwire';
 import { normalizeConnection } from '@powersync/service-types';
 import * as mongo from 'mongodb';
-import { BucketStorageFactory } from '../../src/storage/BucketStorage.js';
+import { BucketStorageFactory, SyncBucketDataBatch } from '../../src/storage/BucketStorage.js';
 import { MongoBucketStorage } from '../../src/storage/MongoBucketStorage.js';
 import { PowerSyncMongo } from '../../src/storage/mongo/db.js';
 import { escapeIdentifier } from '../../src/util/pgwire_utils.js';
 import { env } from './env.js';
 import { Metrics } from '@/metrics/Metrics.js';
-import { container } from '@powersync/lib-services-framework';
-import { MeterProvider } from '@opentelemetry/sdk-metrics';
-import { PrometheusExporter } from '@opentelemetry/exporter-prometheus';
+import { hashData } from '@/util/utils.js';
+import { SourceTable } from '@/storage/SourceTable.js';
+import * as bson from 'bson';
+import { SyncBucketData } from '@/util/protocol-types.js';
 
 // The metrics need to be initialised before they can be used
-const prometheus = new PrometheusExporter();
-const metrics = new Metrics(
-  new MeterProvider({
-    readers: [prometheus]
-  }),
-  prometheus
-);
-container.register(Metrics, metrics);
-metrics.resetCounters();
+await Metrics.initialise({
+  disable_telemetry_sharing: true,
+  powersync_instance_id: 'test',
+  internal_metrics_endpoint: 'unused.for.tests.com'
+});
+Metrics.getInstance().resetCounters();
 
 export const TEST_URI = env.PG_TEST_URL;
 
@@ -33,6 +31,10 @@ export const MONGO_STORAGE_FACTORY: StorageFactory = async () => {
 };
 
 export async function clearTestDb(db: pgwire.PgClient) {
+  await db.query(
+    "select pg_drop_replication_slot(slot_name) from pg_replication_slots where active = false and slot_name like 'test_%'"
+  );
+
   await db.query(`CREATE EXTENSION IF NOT EXISTS "uuid-ossp"`);
   try {
     await db.query(`DROP PUBLICATION powersync`);
@@ -80,3 +82,59 @@ export async function connectMongo() {
   const db = new PowerSyncMongo(client);
   return db;
 }
+
+export function makeTestTable(name: string, columns?: string[] | undefined) {
+  const relId = hashData('table', name, (columns ?? ['id']).join(','));
+  const id = new bson.ObjectId('6544e3899293153fa7b38331');
+  return new SourceTable(
+    id,
+    SourceTable.DEFAULT_TAG,
+    relId,
+    SourceTable.DEFAULT_SCHEMA,
+    name,
+    (columns ?? ['id']).map((column) => ({ name: column, typeOid: 25 })),
+    true
+  );
+}
+
+export function getBatchData(batch: SyncBucketData[] | SyncBucketDataBatch[] | SyncBucketDataBatch) {
+  const first = getFirst(batch);
+  if (first == null) {
+    return [];
+  }
+  return first.data.map((d) => {
+    return {
+      op_id: d.op_id,
+      op: d.op,
+      object_id: d.object_id,
+      checksum: d.checksum
+    };
+  });
+}
+
+export function getBatchMeta(batch: SyncBucketData[] | SyncBucketDataBatch[] | SyncBucketDataBatch) {
+  const first = getFirst(batch);
+  if (first == null) {
+    return null;
+  }
+  return {
+    has_more: first.has_more,
+    after: first.after,
+    next_after: first.next_after
+  };
+}
+
+function getFirst(batch: SyncBucketData[] | SyncBucketDataBatch[] | SyncBucketDataBatch): SyncBucketData | null {
+  if (!Array.isArray(batch)) {
+    return batch.batch;
+  }
+  if (batch.length == 0) {
+    return null;
+  }
+  let first = batch[0];
+  if ((first as SyncBucketDataBatch).batch != null) {
+    return (first as SyncBucketDataBatch).batch;
+  } else {
+    return first as SyncBucketData;
+  }
+}

package/test/src/wal_stream.test.ts

@@ -5,7 +5,6 @@ import { MONGO_STORAGE_FACTORY } from './util.js';
 import { putOp, removeOp, walStreamTest } from './wal_stream_utils.js';
 import { pgwireRows } from '@powersync/service-jpgwire';
 import { Metrics } from '@/metrics/Metrics.js';
-import { container } from '@powersync/lib-services-framework';
 
 type StorageFactory = () => Promise<BucketStorageFactory>;
 
@@ -42,9 +41,10 @@ bucket_definitions:
 
       await context.replicateSnapshot();
 
-      const metrics = container.getImplementation(Metrics);
-      const startRowCount = (await metrics.getMetricValueForTests('powersync_rows_replicated_total')) ?? 0;
-      const startTxCount = (await metrics.getMetricValueForTests('powersync_transactions_replicated_total')) ?? 0;
+      const startRowCount =
+        (await Metrics.getInstance().getMetricValueForTests('powersync_rows_replicated_total')) ?? 0;
+      const startTxCount =
+        (await Metrics.getInstance().getMetricValueForTests('powersync_transactions_replicated_total')) ?? 0;
 
       context.startStreaming();
 
@@ -59,8 +59,9 @@ bucket_definitions:
       expect(data).toMatchObject([
         putOp('test_data', { id: test_id, description: 'test1', num: 1152921504606846976n })
       ]);
-      const endRowCount = (await metrics.getMetricValueForTests('powersync_rows_replicated_total')) ?? 0;
-      const endTxCount = (await metrics.getMetricValueForTests('powersync_transactions_replicated_total')) ?? 0;
+      const endRowCount = (await Metrics.getInstance().getMetricValueForTests('powersync_rows_replicated_total')) ?? 0;
+      const endTxCount =
+        (await Metrics.getInstance().getMetricValueForTests('powersync_transactions_replicated_total')) ?? 0;
       expect(endRowCount - startRowCount).toEqual(1);
       expect(endTxCount - startTxCount).toEqual(1);
     })
@@ -82,9 +83,10 @@ bucket_definitions:
 
       await context.replicateSnapshot();
 
-      const metrics = container.getImplementation(Metrics);
-      const startRowCount = (await metrics.getMetricValueForTests('powersync_rows_replicated_total')) ?? 0;
-      const startTxCount = (await metrics.getMetricValueForTests('powersync_transactions_replicated_total')) ?? 0;
+      const startRowCount =
+        (await Metrics.getInstance().getMetricValueForTests('powersync_rows_replicated_total')) ?? 0;
+      const startTxCount =
+        (await Metrics.getInstance().getMetricValueForTests('powersync_transactions_replicated_total')) ?? 0;
 
       context.startStreaming();
 
@@ -95,8 +97,9 @@ bucket_definitions:
       const data = await context.getBucketData('global[]');
 
       expect(data).toMatchObject([putOp('test_DATA', { id: test_id, description: 'test1' })]);
-      const endRowCount = (await metrics.getMetricValueForTests('powersync_rows_replicated_total')) ?? 0;
-      const endTxCount = (await metrics.getMetricValueForTests('powersync_transactions_replicated_total')) ?? 0;
+      const endRowCount = (await Metrics.getInstance().getMetricValueForTests('powersync_rows_replicated_total')) ?? 0;
+      const endTxCount =
+        (await Metrics.getInstance().getMetricValueForTests('powersync_transactions_replicated_total')) ?? 0;
       expect(endRowCount - startRowCount).toEqual(1);
       expect(endTxCount - startTxCount).toEqual(1);
     })
@@ -290,9 +293,10 @@ bucket_definitions:
 
       await context.replicateSnapshot();
 
-      const metrics = container.getImplementation(Metrics);
-      const startRowCount = (await metrics.getMetricValueForTests('powersync_rows_replicated_total')) ?? 0;
-      const startTxCount = (await metrics.getMetricValueForTests('powersync_transactions_replicated_total')) ?? 0;
+      const startRowCount =
+        (await Metrics.getInstance().getMetricValueForTests('powersync_rows_replicated_total')) ?? 0;
+      const startTxCount =
+        (await Metrics.getInstance().getMetricValueForTests('powersync_transactions_replicated_total')) ?? 0;
 
       context.startStreaming();
 
@@ -303,8 +307,9 @@ bucket_definitions:
       const data = await context.getBucketData('global[]');
 
       expect(data).toMatchObject([]);
-      const endRowCount = (await metrics.getMetricValueForTests('powersync_rows_replicated_total')) ?? 0;
-      const endTxCount = (await metrics.getMetricValueForTests('powersync_transactions_replicated_total')) ?? 0;
+      const endRowCount = (await Metrics.getInstance().getMetricValueForTests('powersync_rows_replicated_total')) ?? 0;
+      const endTxCount =
+        (await Metrics.getInstance().getMetricValueForTests('powersync_transactions_replicated_total')) ?? 0;
 
       // There was a transaction, but we should not replicate any actual data
       expect(endRowCount - startRowCount).toEqual(0);

package/test/src/wal_stream_utils.ts

@@ -20,9 +20,7 @@ export function walStreamTest(
   return async () => {
     const f = await factory();
     const connections = new PgManager(TEST_CONNECTION_OPTIONS, {});
-    await connections.pool.query(
-      'select pg_drop_replication_slot(slot_name) from pg_replication_slots where active = false'
-    );
+
     await clearTestDb(connections.pool);
     const context = new WalStreamTestContext(f, connections);
     try {
@@ -113,7 +111,7 @@ export class WalStreamTestContext {
     const map = new Map<string, string>([[bucket, start]]);
     const batch = await this.storage!.getBucketDataBatch(checkpoint, map);
     const batches = await fromAsync(batch);
-    return batches[0]?.data ?? [];
+    return batches[0]?.batch.data ?? [];
   }
 }
 
@@ -145,3 +143,14 @@ export async function fromAsync<T>(source: Iterable<T> | AsyncIterable<T>): Prom
   }
   return items;
 }
+
+export async function oneFromAsync<T>(source: Iterable<T> | AsyncIterable<T>): Promise<T> {
+  const items: T[] = [];
+  for await (const item of source) {
+    items.push(item);
+  }
+  if (items.length != 1) {
+    throw new Error(`One item expected, got: ${items.length}`);
+  }
+  return items[0];
+}