@powersync/service-core 1.7.2 → 1.8.1

package/src/util/utils.ts

@@ -2,7 +2,7 @@ import * as sync_rules from '@powersync/service-sync-rules';
 import * as bson from 'bson';
 import crypto from 'crypto';
 import * as uuid from 'uuid';
-import { BucketChecksum, OpId, OplogEntry } from './protocol-types.js';
+import { BucketChecksum, ProtocolOpId, OplogEntry } from './protocol-types.js';
 
 import * as storage from '../storage/storage-index.js';
 
@@ -11,6 +11,13 @@ import { ServiceAssertionError } from '@powersync/lib-services-framework';
 
 export type ChecksumMap = Map<string, BucketChecksum>;
 
+/**
+ * op_id as used internally, for individual operations and checkpoints.
+ *
+ * This is just a type alias, but serves to document that we're working with an op_id.
+ */
+export type InternalOpId = bigint;
+
 export const ID_NAMESPACE = 'a396dd91-09fc-4017-a28d-3df722f651e9';
 
 export function escapeIdentifier(identifier: string) {
@@ -31,7 +38,11 @@ export function hashDelete(sourceKey: string) {
   return buffer.readUInt32LE(0);
 }
 
-export function timestampToOpId(ts: bigint): OpId {
+/**
+ * Internally we always use bigint for op_ids. Externally (in JSON) we use strings.
+ * This converts between the two.
+ */
+export function internalToExternalOpId(ts: InternalOpId): ProtocolOpId {
   // Dynamic values are passed in in some cases, so we make extra sure that the
   // number is a bigint and not number or Long.
   if (typeof ts != 'bigint') {

package/test/src/checksum_cache.test.ts

@@ -1,13 +1,12 @@
 import { ChecksumCache, FetchChecksums, FetchPartialBucketChecksum, PartialChecksum } from '@/storage/ChecksumCache.js';
-import { OpId } from '@/util/protocol-types.js';
-import { addChecksums } from '@/util/util-index.js';
+import { addChecksums, InternalOpId } from '@/util/util-index.js';
 import * as crypto from 'node:crypto';
 import { describe, expect, it } from 'vitest';
 
 /**
  * Create a deterministic BucketChecksum based on the bucket name and checkpoint for testing purposes.
  */
-function testHash(bucket: string, checkpoint: OpId) {
+function testHash(bucket: string, checkpoint: InternalOpId) {
   const key = `${checkpoint}/${bucket}`;
   const hash = crypto.createHash('sha256').update(key).digest().readInt32LE(0);
   return hash;
@@ -77,17 +76,17 @@ describe('checksum cache', function () {
       return fetchTestChecksums(batch);
     });
 
-    expect(await cache.getChecksums('123', ['test'])).toEqual([TEST_123]);
+    expect(await cache.getChecksums(123n, ['test'])).toEqual([TEST_123]);
 
-    expect(await cache.getChecksums('1234', ['test'])).toEqual([TEST_1234]);
+    expect(await cache.getChecksums(1234n, ['test'])).toEqual([TEST_1234]);
 
-    expect(await cache.getChecksums('123', ['test2'])).toEqual([TEST2_123]);
+    expect(await cache.getChecksums(123n, ['test2'])).toEqual([TEST2_123]);
 
     expect(lookups).toEqual([
-      [{ bucket: 'test', end: '123' }],
+      [{ bucket: 'test', end: 123n }],
       // This should use the previous lookup
-      [{ bucket: 'test', start: '123', end: '1234' }],
-      [{ bucket: 'test2', end: '123' }]
+      [{ bucket: 'test', start: 123n, end: 1234n }],
+      [{ bucket: 'test2', end: 123n }]
     ]);
   });
 
@@ -99,17 +98,17 @@ describe('checksum cache', function () {
       return fetchTestChecksums(batch);
     });
 
-    expect(await cache.getChecksums('123', ['test2'])).toEqual([TEST2_123]);
+    expect(await cache.getChecksums(123n, ['test2'])).toEqual([TEST2_123]);
 
-    expect(await cache.getChecksums('1234', ['test'])).toEqual([TEST_1234]);
+    expect(await cache.getChecksums(1234n, ['test'])).toEqual([TEST_1234]);
 
-    expect(await cache.getChecksums('123', ['test'])).toEqual([TEST_123]);
+    expect(await cache.getChecksums(123n, ['test'])).toEqual([TEST_123]);
 
     expect(lookups).toEqual([
       // With this order, there is no option for a partial lookup
-      [{ bucket: 'test2', end: '123' }],
-      [{ bucket: 'test', end: '1234' }],
-      [{ bucket: 'test', end: '123' }]
+      [{ bucket: 'test2', end: 123n }],
+      [{ bucket: 'test', end: 1234n }],
+      [{ bucket: 'test', end: 123n }]
     ]);
   });
 
@@ -120,9 +119,9 @@ describe('checksum cache', function () {
       return fetchTestChecksums(batch);
     });
 
-    const p1 = cache.getChecksums('123', ['test']);
-    const p2 = cache.getChecksums('1234', ['test']);
-    const p3 = cache.getChecksums('123', ['test2']);
+    const p1 = cache.getChecksums(123n, ['test']);
+    const p2 = cache.getChecksums(1234n, ['test']);
+    const p3 = cache.getChecksums(123n, ['test2']);
 
     expect(await p1).toEqual([TEST_123]);
     expect(await p2).toEqual([TEST_1234]);
@@ -130,9 +129,9 @@ describe('checksum cache', function () {
 
     // Concurrent requests, so we can't do a partial lookup for 123 -> 1234
     expect(lookups).toEqual([
-      [{ bucket: 'test', end: '123' }],
-      [{ bucket: 'test', end: '1234' }],
-      [{ bucket: 'test2', end: '123' }]
+      [{ bucket: 'test', end: 123n }],
+      [{ bucket: 'test', end: 1234n }],
+      [{ bucket: 'test2', end: 123n }]
     ]);
   });
 
@@ -143,15 +142,15 @@ describe('checksum cache', function () {
       return fetchTestChecksums(batch);
     });
 
-    const p1 = cache.getChecksums('123', ['test']);
-    const p2 = cache.getChecksums('123', ['test']);
+    const p1 = cache.getChecksums(123n, ['test']);
+    const p2 = cache.getChecksums(123n, ['test']);
 
     expect(await p1).toEqual([TEST_123]);
 
     expect(await p2).toEqual([TEST_123]);
 
     // The lookup should be deduplicated, even though it's in progress
-    expect(lookups).toEqual([[{ bucket: 'test', end: '123' }]]);
+    expect(lookups).toEqual([[{ bucket: 'test', end: 123n }]]);
   });
 
   it('should handle serial + concurrent lookups', async function () {
@@ -161,18 +160,18 @@ describe('checksum cache', function () {
       return fetchTestChecksums(batch);
     });
 
-    expect(await cache.getChecksums('123', ['test'])).toEqual([TEST_123]);
+    expect(await cache.getChecksums(123n, ['test'])).toEqual([TEST_123]);
 
-    const p2 = cache.getChecksums('1234', ['test']);
-    const p3 = cache.getChecksums('1234', ['test']);
+    const p2 = cache.getChecksums(1234n, ['test']);
+    const p3 = cache.getChecksums(1234n, ['test']);
 
     expect(await p2).toEqual([TEST_1234]);
     expect(await p3).toEqual([TEST_1234]);
 
     expect(lookups).toEqual([
-      [{ bucket: 'test', end: '123' }],
+      [{ bucket: 'test', end: 123n }],
       // This lookup is deduplicated
-      [{ bucket: 'test', start: '123', end: '1234' }]
+      [{ bucket: 'test', start: 123n, end: 1234n }]
     ]);
   });
 
@@ -183,13 +182,13 @@ describe('checksum cache', function () {
       return fetchTestChecksums(batch);
     });
 
-    expect(await cache.getChecksums('123', ['test', 'test2'])).toEqual([TEST_123, TEST2_123]);
+    expect(await cache.getChecksums(123n, ['test', 'test2'])).toEqual([TEST_123, TEST2_123]);
 
     expect(lookups).toEqual([
       [
         // Both lookups in the same request
-        { bucket: 'test', end: '123' },
-        { bucket: 'test2', end: '123' }
+        { bucket: 'test', end: 123n },
+        { bucket: 'test2', end: 123n }
       ]
     ]);
   });
@@ -201,14 +200,14 @@ describe('checksum cache', function () {
       return fetchTestChecksums(batch);
     });
 
-    expect(await cache.getChecksums('123', ['test'])).toEqual([TEST_123]);
-    expect(await cache.getChecksums('123', ['test', 'test2'])).toEqual([TEST_123, TEST2_123]);
+    expect(await cache.getChecksums(123n, ['test'])).toEqual([TEST_123]);
+    expect(await cache.getChecksums(123n, ['test', 'test2'])).toEqual([TEST_123, TEST2_123]);
 
     expect(lookups).toEqual([
       // Request 1
-      [{ bucket: 'test', end: '123' }],
+      [{ bucket: 'test', end: 123n }],
       // Request 2
-      [{ bucket: 'test2', end: '123' }]
+      [{ bucket: 'test2', end: 123n }]
     ]);
   });
 
@@ -219,8 +218,8 @@ describe('checksum cache', function () {
       return fetchTestChecksums(batch);
     });
 
-    const a = cache.getChecksums('123', ['test', 'test2']);
-    const b = cache.getChecksums('123', ['test2', 'test3']);
+    const a = cache.getChecksums(123n, ['test', 'test2']);
+    const b = cache.getChecksums(123n, ['test2', 'test3']);
 
     expect(await a).toEqual([TEST_123, TEST2_123]);
     expect(await b).toEqual([TEST2_123, TEST3_123]);
@@ -228,11 +227,11 @@ describe('checksum cache', function () {
     expect(lookups).toEqual([
       // Request A
       [
-        { bucket: 'test', end: '123' },
-        { bucket: 'test2', end: '123' }
+        { bucket: 'test', end: 123n },
+        { bucket: 'test2', end: 123n }
       ],
       // Request B (re-uses the checksum for test2 from request a)
-      [{ bucket: 'test3', end: '123' }]
+      [{ bucket: 'test3', end: 123n }]
     ]);
   });
 
@@ -243,9 +242,9 @@ describe('checksum cache', function () {
       return fetchTestChecksums(batch);
     });
 
-    expect(await cache.getChecksums('123', ['test'])).toEqual([TEST_123]);
+    expect(await cache.getChecksums(123n, ['test'])).toEqual([TEST_123]);
 
-    expect(await cache.getChecksums('125', ['test'])).toEqual([
+    expect(await cache.getChecksums(125n, ['test'])).toEqual([
       {
         bucket: 'test',
         checksum: -1865121912,
@@ -253,7 +252,7 @@ describe('checksum cache', function () {
       }
     ]);
 
-    expect(await cache.getChecksums('124', ['test'])).toEqual([
+    expect(await cache.getChecksums(124n, ['test'])).toEqual([
       {
         bucket: 'test',
         checksum: 1887460431,
@@ -261,9 +260,9 @@ describe('checksum cache', function () {
       }
     ]);
     expect(lookups).toEqual([
-      [{ bucket: 'test', end: '123' }],
-      [{ bucket: 'test', start: '123', end: '125' }],
-      [{ bucket: 'test', start: '123', end: '124' }]
+      [{ bucket: 'test', end: 123n }],
+      [{ bucket: 'test', start: 123n, end: 125n }],
+      [{ bucket: 'test', start: 123n, end: 124n }]
     ]);
   });
 
@@ -278,14 +277,14 @@ describe('checksum cache', function () {
       return fetchTestChecksums(batch);
     });
 
-    const a = cache.getChecksums('123', ['test', 'test2']);
-    const b = cache.getChecksums('123', ['test2', 'test3']);
+    const a = cache.getChecksums(123n, ['test', 'test2']);
+    const b = cache.getChecksums(123n, ['test2', 'test3']);
 
     await expect(a).rejects.toEqual(TEST_ERROR);
     await expect(b).rejects.toEqual(TEST_ERROR);
 
-    const a2 = cache.getChecksums('123', ['test', 'test2']);
-    const b2 = cache.getChecksums('123', ['test2', 'test3']);
+    const a2 = cache.getChecksums(123n, ['test', 'test2']);
+    const b2 = cache.getChecksums(123n, ['test2', 'test3']);
 
     expect(await a2).toEqual([TEST_123, TEST2_123]);
     expect(await b2).toEqual([TEST2_123, TEST3_123]);
@@ -293,16 +292,16 @@ describe('checksum cache', function () {
     expect(lookups).toEqual([
       // Request A (fails)
       [
-        { bucket: 'test', end: '123' },
-        { bucket: 'test2', end: '123' }
+        { bucket: 'test', end: 123n },
+        { bucket: 'test2', end: 123n }
       ],
       // Request B (re-uses the checksum for test2 from request a)
       // Even thought the full request fails, this batch succeeds
-      [{ bucket: 'test3', end: '123' }],
+      [{ bucket: 'test3', end: 123n }],
       // Retry request A
       [
-        { bucket: 'test', end: '123' },
-        { bucket: 'test2', end: '123' }
+        { bucket: 'test', end: 123n },
+        { bucket: 'test2', end: 123n }
       ]
     ]);
   });
@@ -314,8 +313,8 @@ describe('checksum cache', function () {
       return fetchTestChecksums(batch.filter((b) => b.bucket != 'test'));
     });
 
-    expect(await cache.getChecksums('123', ['test'])).toEqual([{ bucket: 'test', checksum: 0, count: 0 }]);
-    expect(await cache.getChecksums('123', ['test', 'test2'])).toEqual([
+    expect(await cache.getChecksums(123n, ['test'])).toEqual([{ bucket: 'test', checksum: 0, count: 0 }]);
+    expect(await cache.getChecksums(123n, ['test', 'test2'])).toEqual([
       { bucket: 'test', checksum: 0, count: 0 },
       TEST2_123
     ]);
@@ -325,11 +324,11 @@ describe('checksum cache', function () {
     let lookups: FetchPartialBucketChecksum[][] = [];
     const cache = factory(async (batch) => {
       lookups.push(batch);
-      return fetchTestChecksums(batch.filter((b) => b.bucket != 'test' || b.end != '123'));
+      return fetchTestChecksums(batch.filter((b) => b.bucket != 'test' || b.end != 123n));
     });
 
-    expect(await cache.getChecksums('123', ['test'])).toEqual([{ bucket: 'test', checksum: 0, count: 0 }]);
-    expect(await cache.getChecksums('1234', ['test'])).toEqual([
+    expect(await cache.getChecksums(123n, ['test'])).toEqual([{ bucket: 'test', checksum: 0, count: 0 }]);
+    expect(await cache.getChecksums(1234n, ['test'])).toEqual([
       {
         bucket: 'test',
         checksum: 1597020602,
@@ -337,7 +336,7 @@ describe('checksum cache', function () {
       }
     ]);
 
-    expect(lookups).toEqual([[{ bucket: 'test', end: '123' }], [{ bucket: 'test', start: '123', end: '1234' }]]);
+    expect(lookups).toEqual([[{ bucket: 'test', end: 123n }], [{ bucket: 'test', start: 123n, end: 1234n }]]);
   });
 
   it('should use maxSize', async function () {
@@ -350,8 +349,8 @@ describe('checksum cache', function () {
       maxSize: 2
     });
 
-    expect(await cache.getChecksums('123', ['test'])).toEqual([TEST_123]);
-    expect(await cache.getChecksums('124', ['test'])).toEqual([
+    expect(await cache.getChecksums(123n, ['test'])).toEqual([TEST_123]);
+    expect(await cache.getChecksums(124n, ['test'])).toEqual([
       {
         bucket: 'test',
         checksum: 1887460431,
@@ -359,36 +358,36 @@ describe('checksum cache', function () {
       }
     ]);
 
-    expect(await cache.getChecksums('125', ['test'])).toEqual([
+    expect(await cache.getChecksums(125n, ['test'])).toEqual([
       {
         bucket: 'test',
         checksum: -1865121912,
         count: 125
       }
     ]);
-    expect(await cache.getChecksums('126', ['test'])).toEqual([
+    expect(await cache.getChecksums(126n, ['test'])).toEqual([
       {
         bucket: 'test',
         checksum: -1720007310,
         count: 126
       }
     ]);
-    expect(await cache.getChecksums('124', ['test'])).toEqual([
+    expect(await cache.getChecksums(124n, ['test'])).toEqual([
       {
         bucket: 'test',
         checksum: 1887460431,
         count: 124
       }
     ]);
-    expect(await cache.getChecksums('123', ['test'])).toEqual([TEST_123]);
+    expect(await cache.getChecksums(123n, ['test'])).toEqual([TEST_123]);
 
     expect(lookups).toEqual([
-      [{ bucket: 'test', end: '123' }],
-      [{ bucket: 'test', start: '123', end: '124' }],
-      [{ bucket: 'test', start: '124', end: '125' }],
-      [{ bucket: 'test', start: '125', end: '126' }],
-      [{ bucket: 'test', end: '124' }],
-      [{ bucket: 'test', end: '123' }]
+      [{ bucket: 'test', end: 123n }],
+      [{ bucket: 'test', start: 123n, end: 124n }],
+      [{ bucket: 'test', start: 124n, end: 125n }],
+      [{ bucket: 'test', start: 125n, end: 126n }],
+      [{ bucket: 'test', end: 124n }],
+      [{ bucket: 'test', end: 123n }]
     ]);
   });
 
@@ -403,10 +402,10 @@ describe('checksum cache', function () {
       maxSize: 2
     });
 
-    const p3 = cache.getChecksums('123', ['test3']);
-    const p4 = cache.getChecksums('123', ['test4']);
-    const p1 = cache.getChecksums('123', ['test']);
-    const p2 = cache.getChecksums('123', ['test2']);
+    const p3 = cache.getChecksums(123n, ['test3']);
+    const p4 = cache.getChecksums(123n, ['test4']);
+    const p1 = cache.getChecksums(123n, ['test']);
+    const p2 = cache.getChecksums(123n, ['test2']);
 
     expect(await p1).toEqual([TEST_123]);
     expect(await p2).toEqual([TEST2_123]);
@@ -421,10 +420,10 @@ describe('checksum cache', function () {
 
     // The lookup should be deduplicated, even though it's in progress
     expect(lookups).toEqual([
-      [{ bucket: 'test3', end: '123' }],
-      [{ bucket: 'test4', end: '123' }],
-      [{ bucket: 'test', end: '123' }],
-      [{ bucket: 'test2', end: '123' }]
+      [{ bucket: 'test3', end: 123n }],
+      [{ bucket: 'test4', end: 123n }],
+      [{ bucket: 'test', end: 123n }],
+      [{ bucket: 'test2', end: 123n }]
     ]);
   });
 
@@ -437,7 +436,7 @@ describe('checksum cache', function () {
       return fetchTestChecksums(batch);
     });
 
-    expect(await cache.getChecksums('123', ['test'])).toEqual([TEST_123]);
-    expect(await cache.getChecksums('1234', ['test'])).toEqual([TEST_1234]);
+    expect(await cache.getChecksums(123n, ['test'])).toEqual([TEST_123]);
+    expect(await cache.getChecksums(1234n, ['test'])).toEqual([TEST_1234]);
   });
 });