@powersync/service-core 0.4.2 → 0.5.1

package/test/src/slow_tests.test.ts

@@ -4,12 +4,15 @@ import { afterEach, describe, expect, test } from 'vitest';
 import { WalStream, WalStreamOptions } from '../../src/replication/WalStream.js';
 import { getClientCheckpoint } from '../../src/util/utils.js';
 import { env } from './env.js';
-import { MONGO_STORAGE_FACTORY, StorageFactory, TEST_CONNECTION_OPTIONS, connectPgPool } from './util.js';
+import { MONGO_STORAGE_FACTORY, StorageFactory, TEST_CONNECTION_OPTIONS, clearTestDb, connectPgPool } from './util.js';
 
 import * as pgwire from '@powersync/service-jpgwire';
 import { SqliteRow } from '@powersync/service-sync-rules';
 import { MongoBucketStorage } from '../../src/storage/MongoBucketStorage.js';
 import { PgManager } from '../../src/util/PgManager.js';
+import { mapOpEntry } from '@/storage/storage-index.js';
+import { reduceBucket, validateCompactedBucket, validateBucket } from './bucket_validation.js';
+import * as timers from 'node:timers/promises';
 
 describe('slow tests - mongodb', function () {
   // These are slow, inconsistent tests.
@@ -51,130 +54,169 @@ function defineSlowTests(factory: StorageFactory) {
   // * Skipping LSNs after a keepalive message
   // * Skipping LSNs when source transactions overlap
   test(
-    'repeated replication',
+    'repeated replication - basic',
     async () => {
-      const connections = new PgManager(TEST_CONNECTION_OPTIONS, {});
-      const replicationConnection = await connections.replicationConnection();
-      const pool = connections.pool;
-      const f = (await factory()) as MongoBucketStorage;
+      await testRepeatedReplication({ compact: false, maxBatchSize: 50, numBatches: 5 });
+    },
+    { timeout: TEST_DURATION_MS + TIMEOUT_MARGIN_MS }
+  );
 
-      const syncRuleContent = `
+  test(
+    'repeated replication - compacted',
+    async () => {
+      await testRepeatedReplication({ compact: true, maxBatchSize: 100, numBatches: 2 });
+    },
+    { timeout: TEST_DURATION_MS + TIMEOUT_MARGIN_MS }
+  );
+
+  async function testRepeatedReplication(testOptions: { compact: boolean; maxBatchSize: number; numBatches: number }) {
+    const connections = new PgManager(TEST_CONNECTION_OPTIONS, {});
+    const replicationConnection = await connections.replicationConnection();
+    const pool = connections.pool;
+    await clearTestDb(pool);
+    const f = (await factory()) as MongoBucketStorage;
+
+    const syncRuleContent = `
 bucket_definitions:
   global:
     data:
       - SELECT * FROM "test_data"
 `;
-      const syncRules = await f.updateSyncRules({ content: syncRuleContent });
-      const storage = f.getInstance(syncRules.parsed());
-      abortController = new AbortController();
-      const options: WalStreamOptions = {
-        abort_signal: abortController.signal,
-        connections,
-        storage: storage,
-        factory: f
-      };
-      walStream = new WalStream(options);
-
-      await pool.query(`DROP TABLE IF EXISTS test_data`);
-      await pool.query(
-        `CREATE TABLE test_data(id uuid primary key default uuid_generate_v4(), description text, num decimal)`
-      );
-      await pool.query(`ALTER TABLE test_data REPLICA IDENTITY FULL`);
-
-      await walStream.initReplication(replicationConnection);
-      await storage.autoActivate();
-      let abort = false;
-      streamPromise = walStream.streamChanges(replicationConnection).finally(() => {
-        abort = true;
-      });
-      const start = Date.now();
+    const syncRules = await f.updateSyncRules({ content: syncRuleContent });
+    const storage = f.getInstance(syncRules.parsed());
+    abortController = new AbortController();
+    const options: WalStreamOptions = {
+      abort_signal: abortController.signal,
+      connections,
+      storage: storage,
+      factory: f
+    };
+    walStream = new WalStream(options);
+
+    await pool.query(
+      `CREATE TABLE test_data(id uuid primary key default uuid_generate_v4(), description text, num decimal)`
+    );
+    await pool.query(`ALTER TABLE test_data REPLICA IDENTITY FULL`);
+
+    await walStream.initReplication(replicationConnection);
+    await storage.autoActivate();
+    let abort = false;
+    streamPromise = walStream.streamChanges(replicationConnection).finally(() => {
+      abort = true;
+    });
+    const start = Date.now();
+
+    while (!abort && Date.now() - start < TEST_DURATION_MS) {
+      const bg = async () => {
+        for (let j = 0; j < testOptions.numBatches && !abort; j++) {
+          const n = Math.max(1, Math.floor(Math.random() * testOptions.maxBatchSize));
+          let statements: pgwire.Statement[] = [];
+          for (let i = 0; i < n; i++) {
+            const description = `test${i}`;
+            statements.push({
+              statement: `INSERT INTO test_data(description, num) VALUES($1, $2) returning id as test_id`,
+              params: [
+                { type: 'varchar', value: description },
+                { type: 'float8', value: Math.random() }
+              ]
+            });
+          }
+          const results = await pool.query(...statements);
+          const ids = results.results.map((sub) => {
+            return sub.rows[0][0] as string;
+          });
+          await new Promise((resolve) => setTimeout(resolve, Math.random() * 30));
 
-      while (!abort && Date.now() - start < TEST_DURATION_MS) {
-        const bg = async () => {
-          for (let j = 0; j < 1 && !abort; j++) {
-            const n = 1;
-            let statements: pgwire.Statement[] = [];
-            for (let i = 0; i < n; i++) {
-              const description = `test${i}`;
-              statements.push({
-                statement: `INSERT INTO test_data(description, num) VALUES($1, $2) returning id as test_id`,
+          if (Math.random() > 0.5) {
+            const updateStatements: pgwire.Statement[] = ids.map((id) => {
+              return {
+                statement: `UPDATE test_data SET num = $2 WHERE id = $1`,
                 params: [
-                  { type: 'varchar', value: description },
+                  { type: 'uuid', value: id },
                   { type: 'float8', value: Math.random() }
                 ]
-              });
-            }
-            const results = await pool.query(...statements);
-            const ids = results.results.map((sub) => {
-              return sub.rows[0][0] as string;
+              };
             });
-            await new Promise((resolve) => setTimeout(resolve, Math.random() * 30));
 
+            await pool.query(...updateStatements);
             if (Math.random() > 0.5) {
-              const updateStatements: pgwire.Statement[] = ids.map((id) => {
-                return {
-                  statement: `UPDATE test_data SET num = $2 WHERE id = $1`,
-                  params: [
-                    { type: 'uuid', value: id },
-                    { type: 'float8', value: Math.random() }
-                  ]
-                };
-              });
-
+              // Special case - an update that doesn't change data
               await pool.query(...updateStatements);
-              if (Math.random() > 0.5) {
-                // Special case - an update that doesn't change data
-                await pool.query(...updateStatements);
-              }
             }
-
-            const deleteStatements: pgwire.Statement[] = ids.map((id) => {
-              return {
-                statement: `DELETE FROM test_data WHERE id = $1`,
-                params: [{ type: 'uuid', value: id }]
-              };
-            });
-            await pool.query(...deleteStatements);
-
-            await new Promise((resolve) => setTimeout(resolve, Math.random() * 10));
           }
-        };
 
-        // Call the above loop multiple times concurrently
-        let promises = [1, 2, 3].map((i) => bg());
-        await Promise.all(promises);
+          const deleteStatements: pgwire.Statement[] = ids.map((id) => {
+            return {
+              statement: `DELETE FROM test_data WHERE id = $1`,
+              params: [{ type: 'uuid', value: id }]
+            };
+          });
+          await pool.query(...deleteStatements);
 
-        // Wait for replication to finish
-        let checkpoint = await getClientCheckpoint(pool, storage.factory, { timeout: TIMEOUT_MARGIN_MS });
+          await new Promise((resolve) => setTimeout(resolve, Math.random() * 10));
+        }
+      };
 
-        // Check that all inserts have been deleted again
-        const docs = await f.db.current_data.find().toArray();
-        const transformed = docs.map((doc) => {
-          return bson.deserialize((doc.data as mongo.Binary).buffer) as SqliteRow;
-        });
-        expect(transformed).toEqual([]);
-
-        // Check that each PUT has a REMOVE
-        const ops = await f.db.bucket_data.find().sort({ _id: 1 }).toArray();
-        let active = new Set<string>();
-        for (let op of ops) {
-          const key = op.source_key.toHexString();
-          if (op.op == 'PUT') {
-            active.add(key);
-          } else if (op.op == 'REMOVE') {
-            active.delete(key);
+      let compactController = new AbortController();
+
+      const bgCompact = async () => {
+        // Repeatedly compact, and check that the compact conditions hold
+        while (!compactController.signal.aborted) {
+          const delay = Math.random() * 50;
+          try {
+            await timers.setTimeout(delay, undefined, { signal: compactController.signal });
+          } catch (e) {
+            break;
           }
+
+          const checkpoint = BigInt((await storage.getCheckpoint()).checkpoint);
+          const opsBefore = (await f.db.bucket_data.find().sort({ _id: 1 }).toArray())
+            .filter((row) => row._id.o <= checkpoint)
+            .map(mapOpEntry);
+          await storage.compact({ maxOpId: checkpoint });
+          const opsAfter = (await f.db.bucket_data.find().sort({ _id: 1 }).toArray())
+            .filter((row) => row._id.o <= checkpoint)
+            .map(mapOpEntry);
+
+          validateCompactedBucket(opsBefore, opsAfter);
         }
-        if (active.size > 0) {
-          throw new Error(`${active.size} rows not removed`);
+      };
+
+      // Call the above loop multiple times concurrently
+      const promises = [1, 2, 3].map((i) => bg());
+      const compactPromise = testOptions.compact ? bgCompact() : null;
+      await Promise.all(promises);
+      compactController.abort();
+      await compactPromise;
+
+      // Wait for replication to finish
+      let checkpoint = await getClientCheckpoint(pool, storage.factory, { timeout: TIMEOUT_MARGIN_MS });
+
+      // Check that all inserts have been deleted again
+      const docs = await f.db.current_data.find().toArray();
+      const transformed = docs.map((doc) => {
+        return bson.deserialize((doc.data as mongo.Binary).buffer) as SqliteRow;
+      });
+      expect(transformed).toEqual([]);
+
+      // Check that each PUT has a REMOVE
+      const ops = await f.db.bucket_data.find().sort({ _id: 1 }).toArray();
+
+      // All a single bucket in this test
+      const bucket = ops.map((op) => mapOpEntry(op));
+      const reduced = reduceBucket(bucket);
+      expect(reduced).toMatchObject([
+        {
+          op_id: '0',
+          op: 'CLEAR'
         }
-      }
+        // Should contain no additional data
+      ]);
+    }
 
-      abortController.abort();
-      await streamPromise;
-    },
-    { timeout: TEST_DURATION_MS + TIMEOUT_MARGIN_MS }
-  );
+    abortController.abort();
+    await streamPromise;
+  }
 
   // Test repeatedly performing initial replication.
   //
@@ -184,6 +226,7 @@ bucket_definitions:
     'repeated initial replication',
     async () => {
       const pool = await connectPgPool();
+      await clearTestDb(pool);
       const f = await factory();
 
       const syncRuleContent = `
@@ -196,7 +239,6 @@ bucket_definitions:
       const storage = f.getInstance(syncRules.parsed());
 
       // 1. Setup some base data that will be replicated in initial replication
-      await pool.query(`DROP TABLE IF EXISTS test_data`);
       await pool.query(`CREATE TABLE test_data(id uuid primary key default uuid_generate_v4(), description text)`);
 
       let statements: pgwire.Statement[] = [];

package/test/src/sync.test.ts

@@ -1,34 +1,18 @@
-import * as bson from 'bson';
+import { RequestTracker } from '@/sync/RequestTracker.js';
+import { StreamingSyncLine } from '@/util/protocol-types.js';
+import { lsnMakeComparable } from '@powersync/service-jpgwire';
+import { JSONBig } from '@powersync/service-jsonbig';
+import { RequestParameters } from '@powersync/service-sync-rules';
+import * as timers from 'timers/promises';
 import { describe, expect, test } from 'vitest';
 import { ZERO_LSN } from '../../src/replication/WalStream.js';
-import { SourceTable } from '../../src/storage/SourceTable.js';
-import { hashData } from '../../src/util/utils.js';
-import { MONGO_STORAGE_FACTORY, StorageFactory } from './util.js';
-import { JSONBig } from '@powersync/service-jsonbig';
 import { streamResponse } from '../../src/sync/sync.js';
-import * as timers from 'timers/promises';
-import { lsnMakeComparable } from '@powersync/service-jpgwire';
-import { RequestParameters } from '@powersync/service-sync-rules';
-import { RequestTracker } from '@/sync/RequestTracker.js';
+import { makeTestTable, MONGO_STORAGE_FACTORY, StorageFactory } from './util.js';
 
 describe('sync - mongodb', function () {
   defineTests(MONGO_STORAGE_FACTORY);
 });
 
-function makeTestTable(name: string, columns?: string[] | undefined) {
-  const relId = hashData('table', name, (columns ?? ['id']).join(','));
-  const id = new bson.ObjectId('6544e3899293153fa7b38331');
-  return new SourceTable(
-    id,
-    SourceTable.DEFAULT_TAG,
-    relId,
-    SourceTable.DEFAULT_SCHEMA,
-    name,
-    (columns ?? ['id']).map((column) => ({ name: column, typeOid: 25 })),
-    true
-  );
-}
-
 const TEST_TABLE = makeTestTable('test', ['id']);
 
 const BASIC_SYNC_RULES = `
@@ -251,15 +235,156 @@ function defineTests(factory: StorageFactory) {
     const expLines = await getCheckpointLines(iter);
     expect(expLines).toMatchSnapshot();
   });
+
+  test('compacting data - invalidate checkpoint', async () => {
+    // This tests a case of a compact operation invalidating a checkpoint in the
+    // middle of syncing data.
+    // This is expected to be rare in practice, but it is important to handle
+    // this case correctly to maintain consistency on the client.
+
+    const f = await factory();
+
+    const syncRules = await f.updateSyncRules({
+      content: BASIC_SYNC_RULES
+    });
+
+    const storage = await f.getInstance(syncRules.parsed());
+    await storage.setSnapshotDone(ZERO_LSN);
+    await storage.autoActivate();
+
+    await storage.startBatch({}, async (batch) => {
+      await batch.save({
+        sourceTable: TEST_TABLE,
+        tag: 'insert',
+        after: {
+          id: 't1',
+          description: 'Test 1'
+        }
+      });
+
+      await batch.save({
+        sourceTable: TEST_TABLE,
+        tag: 'insert',
+        after: {
+          id: 't2',
+          description: 'Test 2'
+        }
+      });
+
+      await batch.commit(lsnMakeComparable('0/1'));
+    });
+
+    const stream = streamResponse({
+      storage: f,
+      params: {
+        buckets: [],
+        include_checksum: true,
+        raw_data: true
+      },
+      tracker,
+      syncParams: new RequestParameters({ sub: '' }, {}),
+      token: { exp: Date.now() / 1000 + 10 } as any
+    });
+
+    const iter = stream[Symbol.asyncIterator]();
+
+    // Only consume the first "checkpoint" message, and pause before receiving data.
+    const lines = await consumeIterator(iter, { consume: false, isDone: (line) => (line as any)?.checkpoint != null });
+    expect(lines).toMatchSnapshot();
+    expect(lines[0]).toEqual({
+      checkpoint: expect.objectContaining({
+        last_op_id: '2'
+      })
+    });
+
+    // Now we save additional data AND compact before continuing.
+    // This invalidates the checkpoint we've received above.
+
+    await storage.startBatch({}, async (batch) => {
+      await batch.save({
+        sourceTable: TEST_TABLE,
+        tag: 'update',
+        after: {
+          id: 't1',
+          description: 'Test 1b'
+        }
+      });
+
+      await batch.save({
+        sourceTable: TEST_TABLE,
+        tag: 'update',
+        after: {
+          id: 't2',
+          description: 'Test 2b'
+        }
+      });
+
+      await batch.commit(lsnMakeComparable('0/2'));
+    });
+
+    await storage.compact();
+
+    const lines2 = await getCheckpointLines(iter, { consume: true });
+
+    // Snapshot test checks for changes in general.
+    // The tests after that documents the specific things we're looking for
+    // in this test.
+    expect(lines2).toMatchSnapshot();
+
+    expect(lines2[0]).toEqual({
+      data: expect.objectContaining({
+        has_more: false,
+        data: [
+          // The first two ops have been replaced by a single CLEAR op
+          expect.objectContaining({
+            op: 'CLEAR'
+          })
+        ]
+      })
+    });
+
+    // Note: No checkpoint_complete here, since the checkpoint has been
+    // invalidated by the CLEAR op.
+
+    expect(lines2[1]).toEqual({
+      checkpoint_diff: expect.objectContaining({
+        last_op_id: '4'
+      })
+    });
+
+    expect(lines2[2]).toEqual({
+      data: expect.objectContaining({
+        has_more: false,
+        data: [
+          expect.objectContaining({
+            op: 'PUT'
+          }),
+          expect.objectContaining({
+            op: 'PUT'
+          })
+        ]
+      })
+    });
+
+    // Now we get a checkpoint_complete
+    expect(lines2[3]).toEqual({
+      checkpoint_complete: expect.objectContaining({
+        last_op_id: '4'
+      })
+    });
+  });
 }
 
 /**
- * Get lines on an iterator until the next checkpoint_complete.
+ * Get lines on an iterator until isDone(line) == true.
  *
- * Does not stop the iterator.
+ * Does not stop the iterator unless options.consume is true.
  */
-async function getCheckpointLines(iter: AsyncIterator<any>, options?: { consume?: boolean }): Promise<any[]> {
-  let lines: any[] = [];
+async function consumeIterator<T>(
+  iter: AsyncIterator<T>,
+  options: { isDone: (line: T) => boolean; consume?: boolean }
+) {
+  let lines: T[] = [];
   try {
     const controller = new AbortController();
     const timeout = timers.setTimeout(1500, { value: null, done: 'timeout' }, { signal: controller.signal });
@@ -274,7 +399,7 @@ async function getCheckpointLines(iter: AsyncIterator<any>, options?: { consume?
       if (value) {
         lines.push(value);
       }
-      if (done || value.checkpoint_complete) {
+      if (done || options.isDone(value)) {
         break;
       }
     }
@@ -292,11 +417,26 @@ async function getCheckpointLines(iter: AsyncIterator<any>, options?: { consume?
   }
 }
 
+/**
+ * Get lines on an iterator until the next checkpoint_complete.
+ *
+ * Does not stop the iterator unless options.consume is true.
+ */
+async function getCheckpointLines(
+  iter: AsyncIterator<StreamingSyncLine | string | null>,
+  options?: { consume?: boolean }
+) {
+  return consumeIterator(iter, {
+    consume: options?.consume,
+    isDone: (line) => (line as any)?.checkpoint_complete
+  });
+}
+
 /**
  * Get lines on an iterator until the next checkpoint_complete.
  *
  * Stops the iterator afterwards.
  */
-async function consumeCheckpointLines(iterable: AsyncIterable<any>): Promise<any[]> {
+async function consumeCheckpointLines(iterable: AsyncIterable<StreamingSyncLine | string | null>): Promise<any[]> {
   return getCheckpointLines(iterable[Symbol.asyncIterator](), { consume: true });
 }

package/test/src/util.ts

@@ -1,12 +1,16 @@
 import * as pgwire from '@powersync/service-jpgwire';
 import { normalizeConnection } from '@powersync/service-types';
 import * as mongo from 'mongodb';
-import { BucketStorageFactory } from '../../src/storage/BucketStorage.js';
+import { BucketStorageFactory, SyncBucketDataBatch } from '../../src/storage/BucketStorage.js';
 import { MongoBucketStorage } from '../../src/storage/MongoBucketStorage.js';
 import { PowerSyncMongo } from '../../src/storage/mongo/db.js';
 import { escapeIdentifier } from '../../src/util/pgwire_utils.js';
 import { env } from './env.js';
 import { Metrics } from '@/metrics/Metrics.js';
+import { hashData } from '@/util/utils.js';
+import { SourceTable } from '@/storage/SourceTable.js';
+import * as bson from 'bson';
+import { SyncBucketData } from '@/util/protocol-types.js';
 
 // The metrics need to be initialised before they can be used
 await Metrics.initialise({
@@ -27,6 +31,10 @@ export const MONGO_STORAGE_FACTORY: StorageFactory = async () => {
 };
 
 export async function clearTestDb(db: pgwire.PgClient) {
+  await db.query(
+    "select pg_drop_replication_slot(slot_name) from pg_replication_slots where active = false and slot_name like 'test_%'"
+  );
+
   await db.query(`CREATE EXTENSION IF NOT EXISTS "uuid-ossp"`);
   try {
     await db.query(`DROP PUBLICATION powersync`);
@@ -74,3 +82,59 @@ export async function connectMongo() {
   const db = new PowerSyncMongo(client);
   return db;
 }
+
+export function makeTestTable(name: string, columns?: string[] | undefined) {
+  const relId = hashData('table', name, (columns ?? ['id']).join(','));
+  const id = new bson.ObjectId('6544e3899293153fa7b38331');
+  return new SourceTable(
+    id,
+    SourceTable.DEFAULT_TAG,
+    relId,
+    SourceTable.DEFAULT_SCHEMA,
+    name,
+    (columns ?? ['id']).map((column) => ({ name: column, typeOid: 25 })),
+    true
+  );
+}
+
+export function getBatchData(batch: SyncBucketData[] | SyncBucketDataBatch[] | SyncBucketDataBatch) {
+  const first = getFirst(batch);
+  if (first == null) {
+    return [];
+  }
+  return first.data.map((d) => {
+    return {
+      op_id: d.op_id,
+      op: d.op,
+      object_id: d.object_id,
+      checksum: d.checksum
+    };
+  });
+}
+
+export function getBatchMeta(batch: SyncBucketData[] | SyncBucketDataBatch[] | SyncBucketDataBatch) {
+  const first = getFirst(batch);
+  if (first == null) {
+    return null;
+  }
+  return {
+    has_more: first.has_more,
+    after: first.after,
+    next_after: first.next_after
+  };
+}
+
+function getFirst(batch: SyncBucketData[] | SyncBucketDataBatch[] | SyncBucketDataBatch): SyncBucketData | null {
+  if (!Array.isArray(batch)) {
+    return batch.batch;
+  }
+  if (batch.length == 0) {
+    return null;
+  }
+  let first = batch[0];
+  if ((first as SyncBucketDataBatch).batch != null) {
+    return (first as SyncBucketDataBatch).batch;
+  } else {
+    return first as SyncBucketData;
+  }
+}

package/test/src/wal_stream_utils.ts

@@ -20,9 +20,7 @@ export function walStreamTest(
   return async () => {
     const f = await factory();
     const connections = new PgManager(TEST_CONNECTION_OPTIONS, {});
-    await connections.pool.query(
-      'select pg_drop_replication_slot(slot_name) from pg_replication_slots where active = false'
-    );
+
     await clearTestDb(connections.pool);
     const context = new WalStreamTestContext(f, connections);
     try {
@@ -113,7 +111,7 @@ export class WalStreamTestContext {
     const map = new Map<string, string>([[bucket, start]]);
     const batch = await this.storage!.getBucketDataBatch(checkpoint, map);
     const batches = await fromAsync(batch);
-    return batches[0]?.data ?? [];
+    return batches[0]?.batch.data ?? [];
   }
 }
 
@@ -145,3 +143,14 @@ export async function fromAsync<T>(source: Iterable<T> | AsyncIterable<T>): Prom
   }
   return items;
 }
+
+export async function oneFromAsync<T>(source: Iterable<T> | AsyncIterable<T>): Promise<T> {
+  const items: T[] = [];
+  for await (const item of source) {
+    items.push(item);
+  }
+  if (items.length != 1) {
+    throw new Error(`One item expected, got: ${items.length}`);
+  }
+  return items[0];
+}