@powersync/service-core-tests 0.0.0-dev-20260602073133 → 0.0.0-dev-20260803144812

package/src/tests/register-compacting-tests.ts

@@ -17,6 +17,7 @@ bucket_definitions:
     `)
     );
     const bucketStorage = factory.getInstance(syncRules);
+    const syncRulesContent = syncRules.syncConfigContent[0];
 
     await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
     const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
@@ -53,11 +54,13 @@ bucket_definitions:
 
     const checkpoint = writer.last_flushed_op!;
 
-    const request = bucketRequest(syncRules, 'global[]');
+    const request = bucketRequest(syncRulesContent, 'global[]');
 
-    const batchBefore = await test_utils.oneFromAsync(bucketStorage.getBucketDataBatch(checkpoint, [request]));
+    const batchBefore = await test_utils.getSingleBatchItem(
+      bucketStorage.getBucketDataBatch(test_utils.testCheckpoint(checkpoint), [request])
+    );
     const dataBefore = batchBefore.chunkData.data;
-    const checksumBefore = await bucketStorage.getChecksums(checkpoint, [request]);
+    const checksumBefore = await bucketStorage.getChecksums(test_utils.testCheckpoint(checkpoint), [request]);
 
     expect(dataBefore).toMatchObject([
       {
@@ -86,11 +89,13 @@ bucket_definitions:
       minChangeRatio: 0
     });
 
-    const batchAfter = await test_utils.oneFromAsync(bucketStorage.getBucketDataBatch(checkpoint, [request]));
+    const batchAfter = await test_utils.getSingleBatchItem(
+      bucketStorage.getBucketDataBatch(test_utils.testCheckpoint(checkpoint), [request])
+    );
     const dataAfter = batchAfter.chunkData.data;
-    const checksumAfter = await bucketStorage.getChecksums(checkpoint, [request]);
+    const checksumAfter = await bucketStorage.getChecksums(test_utils.testCheckpoint(checkpoint), [request]);
     bucketStorage.clearChecksumCache();
-    const checksumAfter2 = await bucketStorage.getChecksums(checkpoint, [request]);
+    const checksumAfter2 = await bucketStorage.getChecksums(test_utils.testCheckpoint(checkpoint), [request]);
 
     expect(batchAfter.targetOp).toEqual(3n);
     expect(dataAfter).toMatchObject([
@@ -124,6 +129,7 @@ bucket_definitions:
     `)
     );
     const bucketStorage = factory.getInstance(syncRules);
+    const syncRulesContent = syncRules.syncConfigContent[0];
 
     await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
     const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
@@ -168,11 +174,13 @@ bucket_definitions:
     await writer.flush();
 
     const checkpoint = writer.last_flushed_op!;
-    const request = bucketRequest(syncRules, 'global[]');
+    const request = bucketRequest(syncRulesContent, 'global[]');
 
-    const batchBefore = await test_utils.oneFromAsync(bucketStorage.getBucketDataBatch(checkpoint, [request]));
+    const batchBefore = await test_utils.getSingleBatchItem(
+      bucketStorage.getBucketDataBatch(test_utils.testCheckpoint(checkpoint), [request])
+    );
     const dataBefore = batchBefore.chunkData.data;
-    const checksumBefore = await bucketStorage.getChecksums(checkpoint, [request]);
+    const checksumBefore = await bucketStorage.getChecksums(test_utils.testCheckpoint(checkpoint), [request]);
 
     // op_id sequence depends on the storage implementation
     expect(dataBefore).toMatchObject([
@@ -202,10 +210,12 @@ bucket_definitions:
       minChangeRatio: 0
     });
 
-    const batchAfter = await test_utils.oneFromAsync(bucketStorage.getBucketDataBatch(checkpoint, [request]));
+    const batchAfter = await test_utils.getSingleBatchItem(
+      bucketStorage.getBucketDataBatch(test_utils.testCheckpoint(checkpoint), [request])
+    );
     const dataAfter = batchAfter.chunkData.data;
     bucketStorage.clearChecksumCache();
-    const checksumAfter = await bucketStorage.getChecksums(checkpoint, [request]);
+    const checksumAfter = await bucketStorage.getChecksums(test_utils.testCheckpoint(checkpoint), [request]);
 
     expect(batchAfter.targetOp).toBeLessThanOrEqual(checkpoint);
     expect(dataAfter).toMatchObject([
@@ -240,6 +250,7 @@ bucket_definitions:
     `)
     );
     const bucketStorage = factory.getInstance(syncRules);
+    const syncRulesContent = syncRules.syncConfigContent[0];
 
     await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
     const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
@@ -275,7 +286,7 @@ bucket_definitions:
     await writer.flush();
 
     const checkpoint1 = writer.last_flushed_op!;
-    const request = bucketRequest(syncRules, 'global[]');
+    const request = bucketRequest(syncRulesContent, 'global[]');
     await using writer2 = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
     const testTable2 = await test_utils.resolveTestTable(writer2, 'test', ['id'], config);
     await writer2.save({
@@ -298,10 +309,12 @@ bucket_definitions:
       minChangeRatio: 0
     });
 
-    const batchAfter = await test_utils.oneFromAsync(bucketStorage.getBucketDataBatch(checkpoint2, [request]));
+    const batchAfter = await test_utils.getSingleBatchItem(
+      bucketStorage.getBucketDataBatch(test_utils.testCheckpoint(checkpoint2), [request])
+    );
     const dataAfter = batchAfter.chunkData.data;
     await bucketStorage.clearChecksumCache();
-    const checksumAfter = await bucketStorage.getChecksums(checkpoint2, [request]);
+    const checksumAfter = await bucketStorage.getChecksums(test_utils.testCheckpoint(checkpoint2), [request]);
 
     expect(dataAfter).toMatchObject([
       {
@@ -328,6 +341,7 @@ bucket_definitions:
               - select * from test where b = bucket.b`)
     );
     const bucketStorage = factory.getInstance(syncRules);
+    const syncRulesContent = syncRules.syncConfigContent[0];
 
     await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
     const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
@@ -410,10 +424,10 @@ bucket_definitions:
       minChangeRatio: 0
     });
 
-    const batchAfter = await test_utils.fromAsync(
+    const batchAfter = await test_utils.getBatchArray(
       bucketStorage.getBucketDataBatch(
-        checkpoint,
-        bucketRequestMap(syncRules, [
+        test_utils.testCheckpoint(checkpoint),
+        bucketRequestMap(syncRulesContent, [
           ['grouped["b1"]', 0n],
           ['grouped["b2"]', 0n]
         ])
@@ -458,6 +472,7 @@ bucket_definitions:
     `)
     );
     const bucketStorage = factory.getInstance(syncRules);
+    const syncRulesContent = syncRules.syncConfigContent[0];
 
     await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
     const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
@@ -513,9 +528,9 @@ bucket_definitions:
     await writer2.commit('2/1');
     await writer2.flush();
     const checkpoint2 = writer2.last_flushed_op!;
-    const request = bucketRequest(syncRules, 'global[]');
+    const request = bucketRequest(syncRulesContent, 'global[]');
     await bucketStorage.clearChecksumCache();
-    const checksumAfter = await bucketStorage.getChecksums(checkpoint2, [request]);
+    const checksumAfter = await bucketStorage.getChecksums(test_utils.testCheckpoint(checkpoint2), [request]);
     const globalChecksum = checksumAfter.get(request.bucket);
     expect(globalChecksum).toMatchObject({
       bucket: request.bucket,
@@ -536,6 +551,7 @@ bucket_definitions:
     `)
     );
     const bucketStorage = factory.getInstance(syncRules);
+    const syncRulesContent = syncRules.syncConfigContent[0];
 
     await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
     const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
@@ -562,7 +578,10 @@ bucket_definitions:
     await writer.flush();
 
     // Get checksums here just to populate the cache
-    await bucketStorage.getChecksums(writer.last_flushed_op!, bucketRequests(syncRules, ['global[]']));
+    await bucketStorage.getChecksums(
+      test_utils.testCheckpoint(writer.last_flushed_op!),
+      bucketRequests(syncRulesContent, ['global[]'])
+    );
     await using writer2 = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
     const testTable2 = await test_utils.resolveTestTable(writer2, 'test', ['id'], config);
     await writer2.save({
@@ -585,9 +604,9 @@ bucket_definitions:
     });
 
     const checkpoint2 = writer2.last_flushed_op!;
-    const request = bucketRequest(syncRules, 'global[]');
+    const request = bucketRequest(syncRulesContent, 'global[]');
     // Check that the checksum was correctly updated with the clear operation after having a cached checksum
-    const checksumAfter = await bucketStorage.getChecksums(checkpoint2, [request]);
+    const checksumAfter = await bucketStorage.getChecksums(test_utils.testCheckpoint(checkpoint2), [request]);
     const globalChecksum = checksumAfter.get(request.bucket);
     expect(globalChecksum).toMatchObject({
       bucket: request.bucket,
@@ -607,6 +626,7 @@ bucket_definitions:
       `)
     );
     const bucketStorage = factory.getInstance(syncRules);
+    const syncRulesContent = syncRules.syncConfigContent[0];
 
     await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
     const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config);
@@ -645,8 +665,11 @@ bucket_definitions:
       minChangeRatio: 0
     });
 
-    const batchAfterDefaultCompact = await test_utils.oneFromAsync(
-      bucketStorage.getBucketDataBatch(checkpoint2, bucketRequestMap(syncRules, [['global[]', 0n]]))
+    const batchAfterDefaultCompact = await test_utils.getSingleBatchItem(
+      bucketStorage.getBucketDataBatch(
+        test_utils.testCheckpoint(checkpoint2),
+        bucketRequestMap(syncRulesContent, [['global[]', 0n]])
+      )
     );
 
     // Operation 1 should remain a PUT because op_id=2 is above the default maxOpId checkpoint.

package/src/tests/register-data-storage-checkpoint-tests.ts

@@ -42,7 +42,7 @@ bucket_definitions:
     await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
     await writer.markAllSnapshotDone('1/1');
 
-    const writeCheckpoint = await bucketStorage.createManagedWriteCheckpoint({
+    const writeCheckpoint = await createManagedWriteCheckpoint(bucketStorage, {
       heads: { '1': '5/0' },
       user_id: 'user1'
     });
@@ -102,12 +102,12 @@ bucket_definitions:
       }
     });
 
-    const writeCheckpoint = await bucketStorage.createManagedWriteCheckpoint({
+    const writeCheckpoint = await createManagedWriteCheckpoint(bucketStorage, {
       heads: { '1': '6/0' },
       user_id: 'user1'
     });
     // We have to trigger a new keepalive after the checkpoint, at least to cover postgres storage.
-    // This is what is effetively triggered with RouteAPI.createReplicationHead().
+    // This is what is effectively triggered by RouteAPI.createReplicationHead().
     // MongoDB storage doesn't explicitly need this anymore.
     await writer.keepalive('6/0');
 
@@ -128,6 +128,72 @@ bucket_definitions:
     });
   });
 
+  test('managed write checkpoints - client supplied ids are monotonic', async () => {
+    await using factory = await generateStorageFactory();
+    const r = await factory.configureSyncRules(
+      updateSyncRulesFromYaml(
+        `
+bucket_definitions:
+  mybucket:
+    data: []
+    `,
+        {
+          validate: false,
+          storageVersion
+        }
+      )
+    );
+    const bucketStorage = factory.getInstance(r.persisted_sync_rules!);
+
+    const first = await createManagedWriteCheckpointResult(bucketStorage, {
+      user_id: 'user1',
+      heads: { '1': '5/0' },
+      checkpoint_request_id: 42n
+    });
+    expect(first.writeCheckpoints.get('user1')).toEqual(42n);
+    expect(first.shouldAdvance).toBe(true);
+    await expect(bucketStorage.lastWriteCheckpoint({ user_id: 'user1', heads: { '1': '5/0' } })).resolves.toEqual(42n);
+
+    const stale = await createManagedWriteCheckpointResult(bucketStorage, {
+      user_id: 'user1',
+      heads: { '1': '8/0' },
+      checkpoint_request_id: 41n
+    });
+    expect(stale.writeCheckpoints.get('user1')).toEqual(42n);
+    // The stored checkpoint is unchanged but has not been processed by replication
+    // yet, so the source marker must still be forced (e.g. for a retry after a
+    // previous attempt failed to advance the marker).
+    expect(stale.shouldAdvance).toBe(true);
+    await expect(bucketStorage.lastWriteCheckpoint({ user_id: 'user1', heads: { '1': '6/0' } })).resolves.toEqual(42n);
+
+    const retried = await createManagedWriteCheckpointResult(bucketStorage, {
+      user_id: 'user1',
+      heads: { '1': '8/0' },
+      checkpoint_request_id: 42n
+    });
+    expect(retried.writeCheckpoints.get('user1')).toEqual(42n);
+    // Same as above: the duplicate request still points at a pending checkpoint.
+    expect(retried.shouldAdvance).toBe(true);
+    await expect(bucketStorage.lastWriteCheckpoint({ user_id: 'user1', heads: { '1': '6/0' } })).resolves.toEqual(42n);
+
+    const advanced = await createManagedWriteCheckpointResult(bucketStorage, {
+      user_id: 'user1',
+      heads: { '1': '8/0' },
+      checkpoint_request_id: 43n
+    });
+    expect(advanced.writeCheckpoints.get('user1')).toEqual(43n);
+    expect(advanced.shouldAdvance).toBe(true);
+    await expect(bucketStorage.lastWriteCheckpoint({ user_id: 'user1', heads: { '1': '7/0' } })).resolves.toBeNull();
+    await expect(bucketStorage.lastWriteCheckpoint({ user_id: 'user1', heads: { '1': '8/0' } })).resolves.toEqual(43n);
+
+    const generated = await createManagedWriteCheckpointResult(bucketStorage, {
+      user_id: 'user1',
+      heads: { '1': '9/0' }
+    });
+    expect(generated.writeCheckpoints.get('user1')).toEqual(44n);
+    expect(generated.shouldAdvance).toBe(true);
+  });
+
   test('custom write checkpoints - checkpoint after write', async (context) => {
     await using factory = await generateStorageFactory();
     const r = await factory.configureSyncRules(
@@ -301,4 +367,67 @@ bucket_definitions:
       }
     });
   });
+
+  test('custom write checkpoints - checkpoint request markers are temporary', async () => {
+    await using factory = await generateStorageFactory();
+    const r = await factory.configureSyncRules(
+      updateSyncRulesFromYaml(
+        `
+bucket_definitions:
+  mybucket:
+    data: []
+    `,
+        {
+          validate: false,
+          storageVersion
+        }
+      )
+    );
+    const bucketStorage = factory.getInstance(r.persisted_sync_rules!);
+    bucketStorage.setWriteCheckpointMode(storage.WriteCheckpointMode.CUSTOM);
+
+    await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
+    await writer.markAllSnapshotDone('1/1');
+
+    writer.addCustomWriteCheckpoint({
+      checkpoint: 5n,
+      user_id: 'persistent'
+    });
+    writer.addCustomWriteCheckpoint({
+      checkpoint: 6n,
+      user_id: 'temporary',
+      checkpoint_requested_at: new Date('2024-01-01T00:00:00.000Z')
+    });
+    await writer.flush();
+
+    await expect(bucketStorage.lastWriteCheckpoint({ user_id: 'persistent' })).resolves.toEqual(5n);
+    await expect(bucketStorage.lastWriteCheckpoint({ user_id: 'temporary' })).resolves.toEqual(6n);
+
+    await bucketStorage.compact({
+      compactBuckets: [],
+      deleteCheckpointRequestsBefore: new Date(Date.now() + 1_000)
+    });
+
+    await expect(bucketStorage.lastWriteCheckpoint({ user_id: 'persistent' })).resolves.toEqual(5n);
+    await expect(bucketStorage.lastWriteCheckpoint({ user_id: 'temporary' })).resolves.toBeNull();
+  });
+}
+
+async function createManagedWriteCheckpoint(
+  bucketStorage: storage.SyncRulesBucketStorage,
+  checkpoint: storage.ManagedWriteCheckpointOptions
+) {
+  const result = await createManagedWriteCheckpointResult(bucketStorage, checkpoint);
+  const writeCheckpoint = result.writeCheckpoints.get(checkpoint.user_id);
+  expect(writeCheckpoint).not.toBeUndefined();
+  return writeCheckpoint!;
+}
+
+async function createManagedWriteCheckpointResult(
+  bucketStorage: storage.SyncRulesBucketStorage,
+  checkpoint: storage.ManagedWriteCheckpointOptions
+) {
+  const result = await bucketStorage.createManagedWriteCheckpoints([checkpoint]);
+  expect(result.writeCheckpoints.get(checkpoint.user_id)).not.toBeUndefined();
+  return result;
 }