@powersync/service-core 0.8.8 → 0.9.0

package/src/util/utils.ts

@@ -1,15 +1,21 @@
+import * as sync_rules from '@powersync/service-sync-rules';
+import * as bson from 'bson';
 import crypto from 'crypto';
+import * as uuid from 'uuid';
+import { BucketChecksum, OpId } from './protocol-types.js';
 
-import { logger } from '@powersync/lib-services-framework';
-import * as pgwire from '@powersync/service-jpgwire';
-import { pgwireRows } from '@powersync/service-jpgwire';
-import { PartialChecksum } from '../storage/ChecksumCache.js';
 import * as storage from '../storage/storage-index.js';
-import { retriedQuery } from './pgwire_utils.js';
-import { BucketChecksum, OpId } from './protocol-types.js';
+
+import { PartialChecksum } from '../storage/ChecksumCache.js';
 
 export type ChecksumMap = Map<string, BucketChecksum>;
 
+export const ID_NAMESPACE = 'a396dd91-09fc-4017-a28d-3df722f651e9';
+
+export function escapeIdentifier(identifier: string) {
+  return `"${identifier.replace(/"/g, '""').replace(/\./g, '"."')}"`;
+}
+
 export function hashData(type: string, id: string, data: string): number {
   const hash = crypto.createHash('sha256');
   hash.update(`put.${type}.${id}.${data}`);
@@ -83,49 +89,50 @@ export function addBucketChecksums(a: BucketChecksum, b: PartialChecksum | null)
   }
 }
 
-export async function getClientCheckpoint(
-  db: pgwire.PgClient,
-  bucketStorage: storage.BucketStorageFactory,
-  options?: { timeout?: number }
-): Promise<OpId> {
-  const start = Date.now();
+function getRawReplicaIdentity(
+  tuple: sync_rules.ToastableSqliteRow,
+  columns: storage.ColumnDescriptor[]
+): Record<string, any> {
+  let result: Record<string, any> = {};
+  for (let column of columns) {
+    const name = column.name;
+    result[name] = tuple[name];
+  }
+  return result;
+}
 
-  const [{ lsn }] = pgwireRows(await db.query(`SELECT pg_logical_emit_message(false, 'powersync', 'ping') as lsn`));
+export function getUuidReplicaIdentityBson(
+  tuple: sync_rules.ToastableSqliteRow,
+  columns: storage.ColumnDescriptor[]
+): bson.UUID {
+  if (columns.length == 0) {
+    // REPLICA IDENTITY NOTHING - generate random id
+    return new bson.UUID(uuid.v4());
+  }
+  const rawIdentity = getRawReplicaIdentity(tuple, columns);
 
-  // This old API needs a persisted checkpoint id.
-  // Since we don't use LSNs anymore, the only way to get that is to wait.
+  return uuidForRowBson(rawIdentity);
+}
 
-  const timeout = options?.timeout ?? 50_000;
+export function uuidForRowBson(row: sync_rules.SqliteRow): bson.UUID {
+  // Important: This must not change, since it will affect how ids are generated.
+  // Use BSON so that it's a well-defined format without encoding ambiguities.
+  const repr = bson.serialize(row);
+  const buffer = Buffer.alloc(16);
+  return new bson.UUID(uuid.v5(repr, ID_NAMESPACE, buffer));
+}
 
-  logger.info(`Waiting for LSN checkpoint: ${lsn}`);
-  while (Date.now() - start < timeout) {
-    const cp = await bucketStorage.getActiveCheckpoint();
-    if (!cp.hasSyncRules()) {
-      throw new Error('No sync rules available');
-    }
-    if (cp.lsn >= lsn) {
-      logger.info(`Got write checkpoint: ${lsn} : ${cp.checkpoint}`);
-      return cp.checkpoint;
+export function hasToastedValues(row: sync_rules.ToastableSqliteRow) {
+  for (let key in row) {
+    if (typeof row[key] == 'undefined') {
+      return true;
     }
-
-    await new Promise((resolve) => setTimeout(resolve, 30));
   }
-
-  throw new Error('Timeout while waiting for checkpoint');
+  return false;
 }
 
-export async function createWriteCheckpoint(
-  db: pgwire.PgClient,
-  bucketStorage: storage.BucketStorageFactory,
-  user_id: string
-): Promise<bigint> {
-  const [{ lsn }] = pgwireRows(
-    await retriedQuery(db, `SELECT pg_logical_emit_message(false, 'powersync', 'ping') as lsn`)
-  );
-
-  const id = await bucketStorage.createWriteCheckpoint(user_id, { '1': lsn });
-  logger.info(`Write checkpoint 2: ${JSON.stringify({ lsn, id: String(id) })}`);
-  return id;
+export function isCompleteRow(row: sync_rules.ToastableSqliteRow): row is sync_rules.SqliteRow {
+  return !hasToastedValues(row);
 }
 
 export function checkpointUserId(user_id: string | undefined, client_id: string | undefined) {

package/test/src/__snapshots__/sync.test.ts.snap

@@ -56,21 +56,21 @@ exports[`sync - mongodb > compacting data - invalidate checkpoint 2`] = `
       "data": [
         {
           "checksum": 1859363232n,
-          "data": "{\\"id\\":\\"t1\\",\\"description\\":\\"Test 1b\\"}",
+          "data": "{"id":"t1","description":"Test 1b"}",
           "object_id": "t1",
           "object_type": "test",
           "op": "PUT",
           "op_id": "3",
-          "subkey": "6544e3899293153fa7b38331/117ab485-4b42-58a2-ab32-0053a22c3423",
+          "subkey": "e5aa2ddc-1328-58fa-a000-0b5ed31eaf1a",
         },
         {
           "checksum": 3028503153n,
-          "data": "{\\"id\\":\\"t2\\",\\"description\\":\\"Test 2b\\"}",
+          "data": "{"id":"t2","description":"Test 2b"}",
           "object_id": "t2",
           "object_type": "test",
           "op": "PUT",
           "op_id": "4",
-          "subkey": "6544e3899293153fa7b38331/ec27c691-b47a-5d92-927a-9944feb89eee",
+          "subkey": "13423353-9f27-59b4-baf0-64a5e09f1769",
         },
       ],
       "has_more": false,
@@ -146,21 +146,21 @@ exports[`sync - mongodb > sync global data 1`] = `
       "data": [
         {
           "checksum": 920318466n,
-          "data": "{\\"id\\":\\"t1\\",\\"description\\":\\"Test 1\\"}",
+          "data": "{"id":"t1","description":"Test 1"}",
           "object_id": "t1",
           "object_type": "test",
           "op": "PUT",
           "op_id": "1",
-          "subkey": "6544e3899293153fa7b38331/117ab485-4b42-58a2-ab32-0053a22c3423",
+          "subkey": "e5aa2ddc-1328-58fa-a000-0b5ed31eaf1a",
         },
         {
           "checksum": 3280762209n,
-          "data": "{\\"id\\":\\"t2\\",\\"description\\":\\"Test 2\\"}",
+          "data": "{"id":"t2","description":"Test 2"}",
           "object_id": "t2",
           "object_type": "test",
           "op": "PUT",
           "op_id": "2",
-          "subkey": "6544e3899293153fa7b38331/ec27c691-b47a-5d92-927a-9944feb89eee",
+          "subkey": "13423353-9f27-59b4-baf0-64a5e09f1769",
         },
       ],
       "has_more": false,
@@ -199,7 +199,7 @@ exports[`sync - mongodb > sync legacy non-raw data 1`] = `
           "checksum": 3442149460n,
           "data": {
             "description": "Test
-\\"string\\"",
+"string"",
             "id": "t1",
             "large_num": 12345678901234567890n,
           },
@@ -207,7 +207,7 @@ exports[`sync - mongodb > sync legacy non-raw data 1`] = `
           "object_type": "test",
           "op": "PUT",
           "op_id": "1",
-          "subkey": "6544e3899293153fa7b38331/117ab485-4b42-58a2-ab32-0053a22c3423",
+          "subkey": "e5aa2ddc-1328-58fa-a000-0b5ed31eaf1a",
         },
       ],
       "has_more": false,
@@ -268,12 +268,12 @@ exports[`sync - mongodb > sync updates to global data 2`] = `
       "data": [
         {
           "checksum": 920318466n,
-          "data": "{\\"id\\":\\"t1\\",\\"description\\":\\"Test 1\\"}",
+          "data": "{"id":"t1","description":"Test 1"}",
           "object_id": "t1",
           "object_type": "test",
           "op": "PUT",
           "op_id": "1",
-          "subkey": "6544e3899293153fa7b38331/117ab485-4b42-58a2-ab32-0053a22c3423",
+          "subkey": "e5aa2ddc-1328-58fa-a000-0b5ed31eaf1a",
         },
       ],
       "has_more": false,
@@ -311,12 +311,12 @@ exports[`sync - mongodb > sync updates to global data 3`] = `
       "data": [
         {
           "checksum": 3280762209n,
-          "data": "{\\"id\\":\\"t2\\",\\"description\\":\\"Test 2\\"}",
+          "data": "{"id":"t2","description":"Test 2"}",
           "object_id": "t2",
           "object_type": "test",
           "op": "PUT",
           "op_id": "2",
-          "subkey": "6544e3899293153fa7b38331/ec27c691-b47a-5d92-927a-9944feb89eee",
+          "subkey": "13423353-9f27-59b4-baf0-64a5e09f1769",
         },
       ],
       "has_more": false,

package/test/src/auth.test.ts

@@ -1,11 +1,11 @@
-import { describe, expect, test } from 'vitest';
-import { StaticKeyCollector } from '../../src/auth/StaticKeyCollector.js';
+import { CachedKeyCollector } from '@/auth/CachedKeyCollector.js';
+import { KeyResult } from '@/auth/KeyCollector.js';
+import { KeySpec } from '@/auth/KeySpec.js';
+import { KeyStore } from '@/auth/KeyStore.js';
+import { RemoteJWKSCollector } from '@/auth/RemoteJWKSCollector.js';
+import { StaticKeyCollector } from '@/auth/StaticKeyCollector.js';
 import * as jose from 'jose';
-import { KeyStore } from '../../src/auth/KeyStore.js';
-import { KeySpec } from '../../src/auth/KeySpec.js';
-import { RemoteJWKSCollector } from '../../src/auth/RemoteJWKSCollector.js';
-import { KeyResult } from '../../src/auth/KeyCollector.js';
-import { CachedKeyCollector } from '../../src/auth/CachedKeyCollector.js';
+import { describe, expect, test } from 'vitest';
 
 const publicKey: jose.JWK = {
   use: 'sig',

package/test/src/broadcast_iterable.test.ts

@@ -1,3 +1,4 @@
+import { BroadcastIterable, IterableSource } from '@/sync/BroadcastIterable.js';
 import { AsyncIterableX, interval } from 'ix/asynciterable/index.js';
 import { delayEach } from 'ix/asynciterable/operators/delayeach.js';
 import { take } from 'ix/asynciterable/operators/take.js';
@@ -5,7 +6,6 @@ import { wrapWithAbort } from 'ix/asynciterable/operators/withabort.js';
 import { toArray } from 'ix/asynciterable/toarray.js';
 import * as timers from 'timers/promises';
 import { describe, expect, test } from 'vitest';
-import { BroadcastIterable, IterableSource } from '../../src/sync/BroadcastIterable.js';
 
 describe('BroadcastIterable', () => {
   test('should iterate', async () => {

package/test/src/compacting.test.ts

@@ -1,9 +1,9 @@
-import { SqlSyncRules } from '@powersync/service-sync-rules';
-import { describe, expect, test } from 'vitest';
-import { makeTestTable, MONGO_STORAGE_FACTORY } from './util.js';
-import { oneFromAsync } from './wal_stream_utils.js';
+import { SaveOperationTag } from '@/storage/BucketStorage.js';
 import { MongoCompactOptions } from '@/storage/mongo/MongoCompactor.js';
-import { reduceBucket, validateCompactedBucket, validateBucket } from './bucket_validation.js';
+import { describe, expect, test } from 'vitest';
+import { validateCompactedBucket } from './bucket_validation.js';
+import { oneFromAsync } from './stream_utils.js';
+import { BATCH_OPTIONS, makeTestTable, MONGO_STORAGE_FACTORY, rid, testRules } from './util.js';
 
 const TEST_TABLE = makeTestTable('test', ['id']);
 
@@ -18,37 +18,40 @@ function compactTests(compactOptions: MongoCompactOptions) {
   const factory = MONGO_STORAGE_FACTORY;
 
   test('compacting (1)', async () => {
-    const sync_rules = SqlSyncRules.fromYaml(`
+    const sync_rules = testRules(`
 bucket_definitions:
   global:
     data: [select * from test]
     `);
 
-    const storage = (await factory()).getInstance({ id: 1, sync_rules, slot_name: 'test' });
+    const storage = (await factory()).getInstance(sync_rules);
 
-    const result = await storage.startBatch({}, async (batch) => {
+    const result = await storage.startBatch(BATCH_OPTIONS, async (batch) => {
       await batch.save({
         sourceTable: TEST_TABLE,
-        tag: 'insert',
+        tag: SaveOperationTag.INSERT,
         after: {
           id: 't1'
-        }
+        },
+        afterReplicaId: rid('t1')
       });
 
       await batch.save({
         sourceTable: TEST_TABLE,
-        tag: 'insert',
+        tag: SaveOperationTag.INSERT,
         after: {
           id: 't2'
-        }
+        },
+        afterReplicaId: rid('t2')
       });
 
       await batch.save({
         sourceTable: TEST_TABLE,
-        tag: 'update',
+        tag: SaveOperationTag.UPDATE,
         after: {
           id: 't2'
-        }
+        },
+        afterReplicaId: rid('t2')
       });
     });
 
@@ -112,45 +115,49 @@ bucket_definitions:
   });
 
   test('compacting (2)', async () => {
-    const sync_rules = SqlSyncRules.fromYaml(`
+    const sync_rules = testRules(`
 bucket_definitions:
   global:
     data: [select * from test]
     `);
 
-    const storage = (await factory()).getInstance({ id: 1, sync_rules, slot_name: 'test' });
+    const storage = (await factory()).getInstance(sync_rules);
 
-    const result = await storage.startBatch({}, async (batch) => {
+    const result = await storage.startBatch(BATCH_OPTIONS, async (batch) => {
       await batch.save({
         sourceTable: TEST_TABLE,
-        tag: 'insert',
+        tag: SaveOperationTag.INSERT,
         after: {
           id: 't1'
-        }
+        },
+        afterReplicaId: rid('t1')
       });
 
       await batch.save({
         sourceTable: TEST_TABLE,
-        tag: 'insert',
+        tag: SaveOperationTag.INSERT,
         after: {
           id: 't2'
-        }
+        },
+        afterReplicaId: rid('t2')
       });
 
       await batch.save({
         sourceTable: TEST_TABLE,
-        tag: 'delete',
+        tag: SaveOperationTag.DELETE,
         before: {
           id: 't1'
-        }
+        },
+        beforeReplicaId: rid('t1')
       });
 
       await batch.save({
         sourceTable: TEST_TABLE,
-        tag: 'update',
+        tag: SaveOperationTag.UPDATE,
         after: {
           id: 't2'
-        }
+        },
+        afterReplicaId: rid('t2')
       });
     });
 
@@ -213,51 +220,54 @@ bucket_definitions:
   });
 
   test('compacting (3)', async () => {
-    const sync_rules = SqlSyncRules.fromYaml(`
+    const sync_rules = testRules(`
 bucket_definitions:
   global:
     data: [select * from test]
     `);
 
-    const storage = (await factory()).getInstance({ id: 1, sync_rules, slot_name: 'test' });
+    const storage = (await factory()).getInstance(sync_rules);
 
-    const result = await storage.startBatch({}, async (batch) => {
+    const result = await storage.startBatch(BATCH_OPTIONS, async (batch) => {
       await batch.save({
         sourceTable: TEST_TABLE,
-        tag: 'insert',
+        tag: SaveOperationTag.INSERT,
         after: {
           id: 't1'
-        }
+        },
+        afterReplicaId: 't1'
       });
 
       await batch.save({
         sourceTable: TEST_TABLE,
-        tag: 'insert',
+        tag: SaveOperationTag.INSERT,
         after: {
           id: 't2'
-        }
+        },
+        afterReplicaId: 't2'
       });
 
       await batch.save({
         sourceTable: TEST_TABLE,
-        tag: 'delete',
+        tag: SaveOperationTag.DELETE,
         before: {
           id: 't1'
-        }
+        },
+        beforeReplicaId: 't1'
       });
     });
 
     const checkpoint1 = result!.flushed_op;
     const checksumBefore = await storage.getChecksums(checkpoint1, ['global[]']);
-    console.log('before', checksumBefore);
 
-    const result2 = await storage.startBatch({}, async (batch) => {
+    const result2 = await storage.startBatch(BATCH_OPTIONS, async (batch) => {
       await batch.save({
         sourceTable: TEST_TABLE,
-        tag: 'delete',
+        tag: SaveOperationTag.DELETE,
         before: {
           id: 't2'
-        }
+        },
+        beforeReplicaId: 't2'
       });
     });
     const checkpoint2 = result2!.flushed_op;
@@ -271,7 +281,7 @@ bucket_definitions:
     expect(batchAfter.targetOp).toEqual(4n);
     expect(dataAfter).toMatchObject([
       {
-        checksum: 857217610,
+        checksum: 1874612650,
         op: 'CLEAR',
         op_id: '4'
       }
@@ -279,7 +289,7 @@ bucket_definitions:
     expect(checksumAfter.get('global[]')).toEqual({
       bucket: 'global[]',
       count: 1,
-      checksum: 857217610
+      checksum: 1874612650
     });
   });
 }