@powersync/service-core 0.0.0-dev-20240725112650 → 0.0.0-dev-20240918092408

package/src/util/utils.ts

@@ -1,14 +1,19 @@
+import * as sync_rules from '@powersync/service-sync-rules';
+import * as bson from 'bson';
 import crypto from 'crypto';
-import * as pgwire from '@powersync/service-jpgwire';
-import { pgwireRows } from '@powersync/service-jpgwire';
+import * as uuid from 'uuid';
+import { BucketChecksum, OpId } from './protocol-types.js';
 
 import * as storage from '../storage/storage-index.js';
-import { BucketChecksum, OpId } from './protocol-types.js';
-import { retriedQuery } from './pgwire_utils.js';
-import { logger } from '@powersync/lib-services-framework';
 
 export type ChecksumMap = Map<string, BucketChecksum>;
 
+export const ID_NAMESPACE = 'a396dd91-09fc-4017-a28d-3df722f651e9';
+
+export function escapeIdentifier(identifier: string) {
+  return `"${identifier.replace(/"/g, '""').replace(/\./g, '"."')}"`;
+}
+
 export function hashData(type: string, id: string, data: string): number {
   const hash = crypto.createHash('sha256');
   hash.update(`put.${type}.${id}.${data}`);
@@ -76,47 +81,58 @@ export function addBucketChecksums(a: BucketChecksum, b: BucketChecksum | null):
   }
 }
 
-export async function getClientCheckpoint(
-  db: pgwire.PgClient,
-  bucketStorage: storage.BucketStorageFactory,
-  options?: { timeout?: number }
-): Promise<OpId> {
-  const start = Date.now();
+function getRawReplicaIdentity(
+  tuple: sync_rules.ToastableSqliteRow,
+  columns: storage.ColumnDescriptor[]
+): Record<string, any> {
+  let result: Record<string, any> = {};
+  for (let column of columns) {
+    const name = column.name;
+    result[name] = tuple[name];
+  }
+  return result;
+}
 
-  const [{ lsn }] = pgwireRows(await db.query(`SELECT pg_logical_emit_message(false, 'powersync', 'ping') as lsn`));
+export function getUuidReplicaIdentityBson(
+  tuple: sync_rules.ToastableSqliteRow,
+  columns: storage.ColumnDescriptor[]
+): bson.UUID {
+  if (columns.length == 0) {
+    // REPLICA IDENTITY NOTHING - generate random id
+    return new bson.UUID(uuid.v4());
+  }
+  const rawIdentity = getRawReplicaIdentity(tuple, columns);
 
-  // This old API needs a persisted checkpoint id.
-  // Since we don't use LSNs anymore, the only way to get that is to wait.
+  return uuidForRowBson(rawIdentity);
+}
 
-  const timeout = options?.timeout ?? 50_000;
+export function uuidForRowBson(row: sync_rules.SqliteRow): bson.UUID {
+  // Important: This must not change, since it will affect how ids are generated.
+  // Use BSON so that it's a well-defined format without encoding ambiguities.
+  const repr = bson.serialize(row);
+  const buffer = Buffer.alloc(16);
+  return new bson.UUID(uuid.v5(repr, ID_NAMESPACE, buffer));
+}
 
-  logger.info(`Waiting for LSN checkpoint: ${lsn}`);
-  while (Date.now() - start < timeout) {
-    const cp = await bucketStorage.getActiveCheckpoint();
-    if (!cp.hasSyncRules()) {
-      throw new Error('No sync rules available');
+export function hasToastedValues(row: sync_rules.ToastableSqliteRow) {
+  for (let key in row) {
+    if (typeof row[key] == 'undefined') {
+      return true;
     }
-    if (cp.lsn >= lsn) {
-      logger.info(`Got write checkpoint: ${lsn} : ${cp.checkpoint}`);
-      return cp.checkpoint;
-    }
-
-    await new Promise((resolve) => setTimeout(resolve, 30));
   }
+  return false;
+}
 
-  throw new Error('Timeout while waiting for checkpoint');
+export function isCompleteRow(row: sync_rules.ToastableSqliteRow): row is sync_rules.SqliteRow {
+  return !hasToastedValues(row);
 }
 
-export async function createWriteCheckpoint(
-  db: pgwire.PgClient,
-  bucketStorage: storage.BucketStorageFactory,
-  user_id: string
-): Promise<bigint> {
-  const [{ lsn }] = pgwireRows(
-    await retriedQuery(db, `SELECT pg_logical_emit_message(false, 'powersync', 'ping') as lsn`)
-  );
-
-  const id = await bucketStorage.createWriteCheckpoint(user_id, { '1': lsn });
-  logger.info(`Write checkpoint 2: ${JSON.stringify({ lsn, id: String(id) })}`);
-  return id;
+export function checkpointUserId(user_id: string | undefined, client_id: string | undefined) {
+  if (user_id == null) {
+    throw new Error('user_id is required');
+  }
+  if (client_id == null) {
+    return user_id;
+  }
+  return `${user_id}/${client_id}`;
 }

package/test/src/__snapshots__/sync.test.ts.snap

@@ -61,7 +61,7 @@ exports[`sync - mongodb > compacting data - invalidate checkpoint 2`] = `
           "object_type": "test",
           "op": "PUT",
           "op_id": "3",
-          "subkey": "6544e3899293153fa7b38331/117ab485-4b42-58a2-ab32-0053a22c3423",
+          "subkey": "e5aa2ddc-1328-58fa-a000-0b5ed31eaf1a",
         },
         {
           "checksum": 3028503153n,
@@ -70,7 +70,7 @@ exports[`sync - mongodb > compacting data - invalidate checkpoint 2`] = `
           "object_type": "test",
           "op": "PUT",
           "op_id": "4",
-          "subkey": "6544e3899293153fa7b38331/ec27c691-b47a-5d92-927a-9944feb89eee",
+          "subkey": "13423353-9f27-59b4-baf0-64a5e09f1769",
         },
       ],
       "has_more": false,
@@ -151,7 +151,7 @@ exports[`sync - mongodb > sync global data 1`] = `
           "object_type": "test",
           "op": "PUT",
           "op_id": "1",
-          "subkey": "6544e3899293153fa7b38331/117ab485-4b42-58a2-ab32-0053a22c3423",
+          "subkey": "e5aa2ddc-1328-58fa-a000-0b5ed31eaf1a",
         },
         {
           "checksum": 3280762209n,
@@ -160,7 +160,7 @@ exports[`sync - mongodb > sync global data 1`] = `
           "object_type": "test",
           "op": "PUT",
           "op_id": "2",
-          "subkey": "6544e3899293153fa7b38331/ec27c691-b47a-5d92-927a-9944feb89eee",
+          "subkey": "13423353-9f27-59b4-baf0-64a5e09f1769",
         },
       ],
       "has_more": false,
@@ -207,7 +207,7 @@ exports[`sync - mongodb > sync legacy non-raw data 1`] = `
           "object_type": "test",
           "op": "PUT",
           "op_id": "1",
-          "subkey": "6544e3899293153fa7b38331/117ab485-4b42-58a2-ab32-0053a22c3423",
+          "subkey": "e5aa2ddc-1328-58fa-a000-0b5ed31eaf1a",
         },
       ],
       "has_more": false,
@@ -273,7 +273,7 @@ exports[`sync - mongodb > sync updates to global data 2`] = `
           "object_type": "test",
           "op": "PUT",
           "op_id": "1",
-          "subkey": "6544e3899293153fa7b38331/117ab485-4b42-58a2-ab32-0053a22c3423",
+          "subkey": "e5aa2ddc-1328-58fa-a000-0b5ed31eaf1a",
         },
       ],
       "has_more": false,
@@ -316,7 +316,7 @@ exports[`sync - mongodb > sync updates to global data 3`] = `
           "object_type": "test",
           "op": "PUT",
           "op_id": "2",
-          "subkey": "6544e3899293153fa7b38331/ec27c691-b47a-5d92-927a-9944feb89eee",
+          "subkey": "13423353-9f27-59b4-baf0-64a5e09f1769",
         },
       ],
       "has_more": false,

package/test/src/auth.test.ts

@@ -1,11 +1,11 @@
-import { describe, expect, test } from 'vitest';
-import { StaticKeyCollector } from '../../src/auth/StaticKeyCollector.js';
+import { CachedKeyCollector } from '@/auth/CachedKeyCollector.js';
+import { KeyResult } from '@/auth/KeyCollector.js';
+import { KeySpec } from '@/auth/KeySpec.js';
+import { KeyStore } from '@/auth/KeyStore.js';
+import { RemoteJWKSCollector } from '@/auth/RemoteJWKSCollector.js';
+import { StaticKeyCollector } from '@/auth/StaticKeyCollector.js';
 import * as jose from 'jose';
-import { KeyStore } from '../../src/auth/KeyStore.js';
-import { KeySpec } from '../../src/auth/KeySpec.js';
-import { RemoteJWKSCollector } from '../../src/auth/RemoteJWKSCollector.js';
-import { KeyResult } from '../../src/auth/KeyCollector.js';
-import { CachedKeyCollector } from '../../src/auth/CachedKeyCollector.js';
+import { describe, expect, test } from 'vitest';
 
 const publicKey: jose.JWK = {
   use: 'sig',

package/test/src/broadcast_iterable.test.ts

@@ -1,3 +1,4 @@
+import { BroadcastIterable, IterableSource } from '@/sync/BroadcastIterable.js';
 import { AsyncIterableX, interval } from 'ix/asynciterable/index.js';
 import { delayEach } from 'ix/asynciterable/operators/delayeach.js';
 import { take } from 'ix/asynciterable/operators/take.js';
@@ -5,7 +6,6 @@ import { wrapWithAbort } from 'ix/asynciterable/operators/withabort.js';
 import { toArray } from 'ix/asynciterable/toarray.js';
 import * as timers from 'timers/promises';
 import { describe, expect, test } from 'vitest';
-import { BroadcastIterable, IterableSource } from '../../src/sync/BroadcastIterable.js';
 
 describe('BroadcastIterable', () => {
   test('should iterate', async () => {

package/test/src/checksum_cache.test.ts

@@ -1,8 +1,8 @@
-import { describe, expect, it } from 'vitest';
+import { ChecksumCache, FetchChecksums, FetchPartialBucketChecksum } from '@/storage/ChecksumCache.js';
 import { BucketChecksum, OpId } from '@/util/protocol-types.js';
-import * as crypto from 'node:crypto';
 import { addBucketChecksums } from '@/util/util-index.js';
-import { ChecksumCache, FetchChecksums, FetchPartialBucketChecksum } from '@/storage/ChecksumCache.js';
+import * as crypto from 'node:crypto';
+import { describe, expect, it } from 'vitest';
 
 /**
  * Create a deterministic BucketChecksum based on the bucket name and checkpoint for testing purposes.

package/test/src/compacting.test.ts

@@ -1,9 +1,11 @@
+import { MongoCompactOptions } from '@/storage/mongo/MongoCompactor.js';
 import { SqlSyncRules } from '@powersync/service-sync-rules';
 import { describe, expect, test } from 'vitest';
-import { makeTestTable, MONGO_STORAGE_FACTORY } from './util.js';
-import { oneFromAsync } from './wal_stream_utils.js';
-import { MongoCompactOptions } from '@/storage/mongo/MongoCompactor.js';
-import { reduceBucket, validateCompactedBucket, validateBucket } from './bucket_validation.js';
+import { validateCompactedBucket } from './bucket_validation.js';
+import { oneFromAsync } from './stream_utils.js';
+import { BATCH_OPTIONS, makeTestTable, MONGO_STORAGE_FACTORY, rid, testRules, ZERO_LSN } from './util.js';
+import { ParseSyncRulesOptions, PersistedSyncRulesContent, StartBatchOptions } from '@/storage/BucketStorage.js';
+import { getUuidReplicaIdentityBson } from '@/util/util-index.js';
 
 const TEST_TABLE = makeTestTable('test', ['id']);
 
@@ -18,21 +20,22 @@ function compactTests(compactOptions: MongoCompactOptions) {
   const factory = MONGO_STORAGE_FACTORY;
 
   test('compacting (1)', async () => {
-    const sync_rules = SqlSyncRules.fromYaml(`
+    const sync_rules = testRules(`
 bucket_definitions:
   global:
     data: [select * from test]
     `);
 
-    const storage = (await factory()).getInstance({ id: 1, sync_rules, slot_name: 'test' });
+    const storage = (await factory()).getInstance(sync_rules);
 
-    const result = await storage.startBatch({}, async (batch) => {
+    const result = await storage.startBatch(BATCH_OPTIONS, async (batch) => {
       await batch.save({
         sourceTable: TEST_TABLE,
         tag: 'insert',
         after: {
           id: 't1'
-        }
+        },
+        afterReplicaId: rid('t1')
       });
 
       await batch.save({
@@ -40,7 +43,8 @@ bucket_definitions:
         tag: 'insert',
         after: {
           id: 't2'
-        }
+        },
+        afterReplicaId: rid('t2')
       });
 
       await batch.save({
@@ -48,7 +52,8 @@ bucket_definitions:
         tag: 'update',
         after: {
           id: 't2'
-        }
+        },
+        afterReplicaId: rid('t2')
       });
     });
 
@@ -108,21 +113,22 @@ bucket_definitions:
   });
 
   test('compacting (2)', async () => {
-    const sync_rules = SqlSyncRules.fromYaml(`
+    const sync_rules = testRules(`
 bucket_definitions:
   global:
     data: [select * from test]
     `);
 
-    const storage = (await factory()).getInstance({ id: 1, sync_rules, slot_name: 'test' });
+    const storage = (await factory()).getInstance(sync_rules);
 
-    const result = await storage.startBatch({}, async (batch) => {
+    const result = await storage.startBatch(BATCH_OPTIONS, async (batch) => {
       await batch.save({
         sourceTable: TEST_TABLE,
         tag: 'insert',
         after: {
           id: 't1'
-        }
+        },
+        afterReplicaId: rid('t1')
       });
 
       await batch.save({
@@ -130,7 +136,8 @@ bucket_definitions:
         tag: 'insert',
         after: {
           id: 't2'
-        }
+        },
+        afterReplicaId: rid('t2')
       });
 
       await batch.save({
@@ -138,7 +145,8 @@ bucket_definitions:
         tag: 'delete',
         before: {
           id: 't1'
-        }
+        },
+        beforeReplicaId: rid('t1')
       });
 
       await batch.save({
@@ -146,7 +154,8 @@ bucket_definitions:
         tag: 'update',
         after: {
           id: 't2'
-        }
+        },
+        afterReplicaId: rid('t2')
       });
     });