@powersync/service-core 0.8.4 → 0.8.5

package/src/storage/mongo/MongoSyncBucketStorage.ts

@@ -18,7 +18,7 @@ import {
   SyncRulesBucketStorage,
   SyncRuleStatus
 } from '../BucketStorage.js';
-import { ChecksumCache, FetchPartialBucketChecksum } from '../ChecksumCache.js';
+import { ChecksumCache, FetchPartialBucketChecksum, PartialChecksum, PartialChecksumMap } from '../ChecksumCache.js';
 import { MongoBucketStorage } from '../MongoBucketStorage.js';
 import { SourceTable } from '../SourceTable.js';
 import { PowerSyncMongo } from './db.js';
@@ -333,7 +333,7 @@ export class MongoSyncBucketStorage implements SyncRulesBucketStorage {
     return this.checksumCache.getChecksumMap(checkpoint, buckets);
   }
 
-  private async getChecksumsInternal(batch: FetchPartialBucketChecksum[]): Promise<util.ChecksumMap> {
+  private async getChecksumsInternal(batch: FetchPartialBucketChecksum[]): Promise<PartialChecksumMap> {
     if (batch.length == 0) {
       return new Map();
     }
@@ -365,22 +365,32 @@ export class MongoSyncBucketStorage implements SyncRulesBucketStorage {
             }
           },
           {
-            $group: { _id: '$_id.b', checksum_total: { $sum: '$checksum' }, count: { $sum: 1 } }
+            $group: {
+              _id: '$_id.b',
+              checksum_total: { $sum: '$checksum' },
+              count: { $sum: 1 },
+              has_clear_op: {
+                $max: {
+                  $cond: [{ $eq: ['$op', 'CLEAR'] }, 1, 0]
+                }
+              }
+            }
           }
         ],
-        { session: undefined }
+        { session: undefined, readConcern: 'snapshot' }
       )
       .toArray();
 
-    return new Map<string, util.BucketChecksum>(
+    return new Map<string, PartialChecksum>(
       aggregate.map((doc) => {
         return [
           doc._id,
           {
             bucket: doc._id,
-            count: doc.count,
-            checksum: Number(BigInt(doc.checksum_total) & 0xffffffffn) & 0xffffffff
-          } satisfies util.BucketChecksum
+            partialCount: doc.count,
+            partialChecksum: Number(BigInt(doc.checksum_total) & 0xffffffffn) & 0xffffffff,
+            isFullChecksum: doc.has_clear_op == 1
+          } satisfies PartialChecksum
         ];
       })
     );

package/src/storage/mongo/util.ts

@@ -1,10 +1,10 @@
 import { SqliteJsonValue } from '@powersync/service-sync-rules';
 import * as bson from 'bson';
-import * as mongo from 'mongodb';
 import * as crypto from 'crypto';
-import { BucketDataDocument } from './models.js';
-import { timestampToOpId } from '../../util/utils.js';
+import * as mongo from 'mongodb';
 import { OplogEntry } from '../../util/protocol-types.js';
+import { timestampToOpId } from '../../util/utils.js';
+import { BucketDataDocument } from './models.js';
 
 /**
  * Lookup serialization must be number-agnostic. I.e. normalize numbers, instead of preserving numbers.

package/src/util/protocol-types.ts

@@ -1,59 +1,6 @@
 import * as t from 'ts-codec';
 import { SqliteJsonValue } from '@powersync/service-sync-rules';
 
-/**
- * For sync2.json
- */
-export interface ContinueCheckpointRequest {
-  /**
-   * Existing bucket states. Only these buckets are synchronized.
-   */
-  buckets: BucketRequest[];
-
-  checkpoint_token: string;
-
-  limit?: number;
-}
-
-export interface SyncNewCheckpointRequest {
-  /**
-   * Existing bucket states. Used if include_data is specified.
-   */
-  buckets?: BucketRequest[];
-
-  request_checkpoint: {
-    /**
-     * Whether or not to include an initial data request.
-     */
-    include_data: boolean;
-
-    /**
-     * Whether or not to compute a checksum.
-     */
-    include_checksum: boolean;
-  };
-
-  limit?: number;
-}
-
-export type SyncRequest = ContinueCheckpointRequest | SyncNewCheckpointRequest;
-
-export interface SyncResponse {
-  /**
-   * Data for the buckets returned. May not have an an entry for each bucket in the request.
-   */
-  data?: SyncBucketData[];
-
-  /**
-   * True if the response limit has been reached, and another request must be made.
-   */
-  has_more: boolean;
-
-  checkpoint_token?: string;
-
-  checkpoint?: Checkpoint;
-}
-
 export const BucketRequest = t.object({
   name: t.string,
 
@@ -195,39 +142,3 @@ export interface BucketChecksum {
    */
   count: number;
 }
-
-export function isContinueCheckpointRequest(request: SyncRequest): request is ContinueCheckpointRequest {
-  return (
-    Array.isArray((request as ContinueCheckpointRequest).buckets) &&
-    typeof (request as ContinueCheckpointRequest).checkpoint_token == 'string'
-  );
-}
-
-export function isSyncNewCheckpointRequest(request: SyncRequest): request is SyncNewCheckpointRequest {
-  return typeof (request as SyncNewCheckpointRequest).request_checkpoint == 'object';
-}
-
-/**
- * For crud.json
- */
-export interface CrudRequest {
-  data: CrudEntry[];
-}
-
-export interface CrudEntry {
-  op: 'PUT' | 'PATCH' | 'DELETE';
-  type: string;
-  id: string;
-  data: string;
-}
-
-export interface CrudResponse {
-  /**
-   * A sync response with a checkpoint >= this checkpoint would contain all the changes in this request.
-   *
-   * Any earlier checkpoint may or may not contain these changes.
-   *
-   * May be empty when the request contains no ops.
-   */
-  checkpoint?: OpId;
-}

package/src/util/utils.ts

@@ -1,11 +1,12 @@
 import crypto from 'crypto';
+
+import { logger } from '@powersync/lib-services-framework';
 import * as pgwire from '@powersync/service-jpgwire';
 import { pgwireRows } from '@powersync/service-jpgwire';
-
+import { PartialChecksum } from '../storage/ChecksumCache.js';
 import * as storage from '../storage/storage-index.js';
-import { BucketChecksum, OpId } from './protocol-types.js';
 import { retriedQuery } from './pgwire_utils.js';
-import { logger } from '@powersync/lib-services-framework';
+import { BucketChecksum, OpId } from './protocol-types.js';
 
 export type ChecksumMap = Map<string, BucketChecksum>;
 
@@ -64,14 +65,20 @@ export function addChecksums(a: number, b: number) {
   return (a + b) & 0xffffffff;
 }
 
-export function addBucketChecksums(a: BucketChecksum, b: BucketChecksum | null): BucketChecksum {
+export function addBucketChecksums(a: BucketChecksum, b: PartialChecksum | null): BucketChecksum {
   if (b == null) {
     return a;
+  } else if (b.isFullChecksum) {
+    return {
+      bucket: b.bucket,
+      count: b.partialCount,
+      checksum: b.partialChecksum
+    };
   } else {
     return {
       bucket: a.bucket,
-      count: a.count + b.count,
-      checksum: addChecksums(a.checksum, b.checksum)
+      count: a.count + b.partialCount,
+      checksum: addChecksums(a.checksum, b.partialChecksum)
     };
   }
 }

package/test/src/checksum_cache.test.ts

@@ -1,8 +1,8 @@
-import { describe, expect, it } from 'vitest';
-import { BucketChecksum, OpId } from '@/util/protocol-types.js';
+import { ChecksumCache, FetchChecksums, FetchPartialBucketChecksum, PartialChecksum } from '@/storage/ChecksumCache.js';
+import { OpId } from '@/util/protocol-types.js';
+import { addChecksums } from '@/util/util-index.js';
 import * as crypto from 'node:crypto';
-import { addBucketChecksums } from '@/util/util-index.js';
-import { ChecksumCache, FetchChecksums, FetchPartialBucketChecksum } from '@/storage/ChecksumCache.js';
+import { describe, expect, it } from 'vitest';
 
 /**
  * Create a deterministic BucketChecksum based on the bucket name and checkpoint for testing purposes.
@@ -13,28 +13,22 @@ function testHash(bucket: string, checkpoint: OpId) {
   return hash;
 }
 
-function testPartialHash(request: FetchPartialBucketChecksum): BucketChecksum {
+function testPartialHash(request: FetchPartialBucketChecksum): PartialChecksum {
   if (request.start) {
     const a = testHash(request.bucket, request.start);
     const b = testHash(request.bucket, request.end);
-    return addBucketChecksums(
-      {
-        bucket: request.bucket,
-        checksum: b,
-        count: Number(request.end)
-      },
-      {
-        // Subtract a
-        bucket: request.bucket,
-        checksum: -a,
-        count: -Number(request.start)
-      }
-    );
+    return {
+      bucket: request.bucket,
+      partialCount: Number(request.end) - Number(request.start),
+      partialChecksum: addChecksums(b, -a),
+      isFullChecksum: false
+    };
   } else {
     return {
       bucket: request.bucket,
-      checksum: testHash(request.bucket, request.end),
-      count: Number(request.end)
+      partialChecksum: testHash(request.bucket, request.end),
+      partialCount: Number(request.end),
+      isFullChecksum: true
     };
   }
 }
@@ -433,4 +427,17 @@ describe('checksum cache', function () {
       [{ bucket: 'test2', end: '123' }]
     ]);
   });
+
+  it('should handle CLEAR/isFullChecksum checksums', async function () {
+    let lookups: FetchPartialBucketChecksum[][] = [];
+    const cache = factory(async (batch) => {
+      lookups.push(batch);
+      // This forces a `isFullChecksum: true` result
+      delete batch[0].start;
+      return fetchTestChecksums(batch);
+    });
+
+    expect(await cache.getChecksums('123', ['test'])).toEqual([TEST_123]);
+    expect(await cache.getChecksums('1234', ['test'])).toEqual([TEST_1234]);
+  });
 });

package/test/src/compacting.test.ts

@@ -56,6 +56,7 @@ bucket_definitions:
 
     const batchBefore = await oneFromAsync(storage.getBucketDataBatch(checkpoint, new Map([['global[]', '0']])));
     const dataBefore = batchBefore.batch.data;
+    const checksumBefore = await storage.getChecksums(checkpoint, ['global[]']);
 
     expect(dataBefore).toMatchObject([
       {
@@ -82,6 +83,7 @@ bucket_definitions:
 
     const batchAfter = await oneFromAsync(storage.getBucketDataBatch(checkpoint, new Map([['global[]', '0']])));
     const dataAfter = batchAfter.batch.data;
+    const checksumAfter = await storage.getChecksums(checkpoint, ['global[]']);
 
     expect(batchAfter.targetOp).toEqual(3n);
     expect(dataAfter).toMatchObject([
@@ -104,6 +106,8 @@ bucket_definitions:
       }
     ]);
 
+    expect(checksumBefore.get('global[]')).toEqual(checksumAfter.get('global[]'));
+
     validateCompactedBucket(dataBefore, dataAfter);
   });
 
@@ -154,6 +158,7 @@ bucket_definitions:
 
     const batchBefore = await oneFromAsync(storage.getBucketDataBatch(checkpoint, new Map([['global[]', '0']])));
     const dataBefore = batchBefore.batch.data;
+    const checksumBefore = await storage.getChecksums(checkpoint, ['global[]']);
 
     expect(dataBefore).toMatchObject([
       {
@@ -186,6 +191,7 @@ bucket_definitions:
 
     const batchAfter = await oneFromAsync(storage.getBucketDataBatch(checkpoint, new Map([['global[]', '0']])));
     const dataAfter = batchAfter.batch.data;
+    const checksumAfter = await storage.getChecksums(checkpoint, ['global[]']);
 
     expect(batchAfter.targetOp).toEqual(4n);
     expect(dataAfter).toMatchObject([
@@ -201,7 +207,79 @@ bucket_definitions:
         op_id: '4'
       }
     ]);
+    expect(checksumBefore.get('global[]')).toEqual(checksumAfter.get('global[]'));
 
     validateCompactedBucket(dataBefore, dataAfter);
   });
+
+  test('compacting (3)', async () => {
+    const sync_rules = SqlSyncRules.fromYaml(`
+bucket_definitions:
+  global:
+    data: [select * from test]
+    `);
+
+    const storage = (await factory()).getInstance({ id: 1, sync_rules, slot_name: 'test' });
+
+    const result = await storage.startBatch({}, async (batch) => {
+      await batch.save({
+        sourceTable: TEST_TABLE,
+        tag: 'insert',
+        after: {
+          id: 't1'
+        }
+      });
+
+      await batch.save({
+        sourceTable: TEST_TABLE,
+        tag: 'insert',
+        after: {
+          id: 't2'
+        }
+      });
+
+      await batch.save({
+        sourceTable: TEST_TABLE,
+        tag: 'delete',
+        before: {
+          id: 't1'
+        }
+      });
+    });
+
+    const checkpoint1 = result!.flushed_op;
+    const checksumBefore = await storage.getChecksums(checkpoint1, ['global[]']);
+    console.log('before', checksumBefore);
+
+    const result2 = await storage.startBatch({}, async (batch) => {
+      await batch.save({
+        sourceTable: TEST_TABLE,
+        tag: 'delete',
+        before: {
+          id: 't2'
+        }
+      });
+    });
+    const checkpoint2 = result2!.flushed_op;
+
+    await storage.compact(compactOptions);
+
+    const batchAfter = await oneFromAsync(storage.getBucketDataBatch(checkpoint2, new Map([['global[]', '0']])));
+    const dataAfter = batchAfter.batch.data;
+    const checksumAfter = await storage.getChecksums(checkpoint2, ['global[]']);
+
+    expect(batchAfter.targetOp).toEqual(4n);
+    expect(dataAfter).toMatchObject([
+      {
+        checksum: 857217610,
+        op: 'CLEAR',
+        op_id: '4'
+      }
+    ]);
+    expect(checksumAfter.get('global[]')).toEqual({
+      bucket: 'global[]',
+      count: 1,
+      checksum: 857217610
+    });
+  });
 }