@powersync/service-core 0.8.4 → 0.8.6

package/src/util/protocol-types.ts

@@ -1,59 +1,6 @@
 import * as t from 'ts-codec';
 import { SqliteJsonValue } from '@powersync/service-sync-rules';
 
-/**
- * For sync2.json
- */
-export interface ContinueCheckpointRequest {
-  /**
-   * Existing bucket states. Only these buckets are synchronized.
-   */
-  buckets: BucketRequest[];
-
-  checkpoint_token: string;
-
-  limit?: number;
-}
-
-export interface SyncNewCheckpointRequest {
-  /**
-   * Existing bucket states. Used if include_data is specified.
-   */
-  buckets?: BucketRequest[];
-
-  request_checkpoint: {
-    /**
-     * Whether or not to include an initial data request.
-     */
-    include_data: boolean;
-
-    /**
-     * Whether or not to compute a checksum.
-     */
-    include_checksum: boolean;
-  };
-
-  limit?: number;
-}
-
-export type SyncRequest = ContinueCheckpointRequest | SyncNewCheckpointRequest;
-
-export interface SyncResponse {
-  /**
-   * Data for the buckets returned. May not have an an entry for each bucket in the request.
-   */
-  data?: SyncBucketData[];
-
-  /**
-   * True if the response limit has been reached, and another request must be made.
-   */
-  has_more: boolean;
-
-  checkpoint_token?: string;
-
-  checkpoint?: Checkpoint;
-}
-
 export const BucketRequest = t.object({
   name: t.string,
 
@@ -195,39 +142,3 @@ export interface BucketChecksum {
    */
   count: number;
 }
-
-export function isContinueCheckpointRequest(request: SyncRequest): request is ContinueCheckpointRequest {
-  return (
-    Array.isArray((request as ContinueCheckpointRequest).buckets) &&
-    typeof (request as ContinueCheckpointRequest).checkpoint_token == 'string'
-  );
-}
-
-export function isSyncNewCheckpointRequest(request: SyncRequest): request is SyncNewCheckpointRequest {
-  return typeof (request as SyncNewCheckpointRequest).request_checkpoint == 'object';
-}
-
-/**
- * For crud.json
- */
-export interface CrudRequest {
-  data: CrudEntry[];
-}
-
-export interface CrudEntry {
-  op: 'PUT' | 'PATCH' | 'DELETE';
-  type: string;
-  id: string;
-  data: string;
-}
-
-export interface CrudResponse {
-  /**
-   * A sync response with a checkpoint >= this checkpoint would contain all the changes in this request.
-   *
-   * Any earlier checkpoint may or may not contain these changes.
-   *
-   * May be empty when the request contains no ops.
-   */
-  checkpoint?: OpId;
-}

package/src/util/utils.ts

@@ -1,11 +1,12 @@
 import crypto from 'crypto';
+
+import { logger } from '@powersync/lib-services-framework';
 import * as pgwire from '@powersync/service-jpgwire';
 import { pgwireRows } from '@powersync/service-jpgwire';
-
+import { PartialChecksum } from '../storage/ChecksumCache.js';
 import * as storage from '../storage/storage-index.js';
-import { BucketChecksum, OpId } from './protocol-types.js';
 import { retriedQuery } from './pgwire_utils.js';
-import { logger } from '@powersync/lib-services-framework';
+import { BucketChecksum, OpId } from './protocol-types.js';
 
 export type ChecksumMap = Map<string, BucketChecksum>;
 
@@ -64,14 +65,20 @@ export function addChecksums(a: number, b: number) {
   return (a + b) & 0xffffffff;
 }
 
-export function addBucketChecksums(a: BucketChecksum, b: BucketChecksum | null): BucketChecksum {
+export function addBucketChecksums(a: BucketChecksum, b: PartialChecksum | null): BucketChecksum {
   if (b == null) {
     return a;
+  } else if (b.isFullChecksum) {
+    return {
+      bucket: b.bucket,
+      count: b.partialCount,
+      checksum: b.partialChecksum
+    };
   } else {
     return {
       bucket: a.bucket,
-      count: a.count + b.count,
-      checksum: addChecksums(a.checksum, b.checksum)
+      count: a.count + b.partialCount,
+      checksum: addChecksums(a.checksum, b.partialChecksum)
     };
   }
 }

package/test/src/checksum_cache.test.ts

@@ -1,8 +1,8 @@
-import { describe, expect, it } from 'vitest';
-import { BucketChecksum, OpId } from '@/util/protocol-types.js';
+import { ChecksumCache, FetchChecksums, FetchPartialBucketChecksum, PartialChecksum } from '@/storage/ChecksumCache.js';
+import { OpId } from '@/util/protocol-types.js';
+import { addChecksums } from '@/util/util-index.js';
 import * as crypto from 'node:crypto';
-import { addBucketChecksums } from '@/util/util-index.js';
-import { ChecksumCache, FetchChecksums, FetchPartialBucketChecksum } from '@/storage/ChecksumCache.js';
+import { describe, expect, it } from 'vitest';
 
 /**
  * Create a deterministic BucketChecksum based on the bucket name and checkpoint for testing purposes.
@@ -13,28 +13,22 @@ function testHash(bucket: string, checkpoint: OpId) {
   return hash;
 }
 
-function testPartialHash(request: FetchPartialBucketChecksum): BucketChecksum {
+function testPartialHash(request: FetchPartialBucketChecksum): PartialChecksum {
   if (request.start) {
     const a = testHash(request.bucket, request.start);
     const b = testHash(request.bucket, request.end);
-    return addBucketChecksums(
-      {
-        bucket: request.bucket,
-        checksum: b,
-        count: Number(request.end)
-      },
-      {
-        // Subtract a
-        bucket: request.bucket,
-        checksum: -a,
-        count: -Number(request.start)
-      }
-    );
+    return {
+      bucket: request.bucket,
+      partialCount: Number(request.end) - Number(request.start),
+      partialChecksum: addChecksums(b, -a),
+      isFullChecksum: false
+    };
   } else {
     return {
       bucket: request.bucket,
-      checksum: testHash(request.bucket, request.end),
-      count: Number(request.end)
+      partialChecksum: testHash(request.bucket, request.end),
+      partialCount: Number(request.end),
+      isFullChecksum: true
     };
   }
 }
@@ -433,4 +427,17 @@ describe('checksum cache', function () {
       [{ bucket: 'test2', end: '123' }]
     ]);
   });
+
+  it('should handle CLEAR/isFullChecksum checksums', async function () {
+    let lookups: FetchPartialBucketChecksum[][] = [];
+    const cache = factory(async (batch) => {
+      lookups.push(batch);
+      // This forces a `isFullChecksum: true` result
+      delete batch[0].start;
+      return fetchTestChecksums(batch);
+    });
+
+    expect(await cache.getChecksums('123', ['test'])).toEqual([TEST_123]);
+    expect(await cache.getChecksums('1234', ['test'])).toEqual([TEST_1234]);
+  });
 });

package/test/src/compacting.test.ts

@@ -56,6 +56,7 @@ bucket_definitions:
 
     const batchBefore = await oneFromAsync(storage.getBucketDataBatch(checkpoint, new Map([['global[]', '0']])));
     const dataBefore = batchBefore.batch.data;
+    const checksumBefore = await storage.getChecksums(checkpoint, ['global[]']);
 
     expect(dataBefore).toMatchObject([
       {
@@ -82,6 +83,7 @@ bucket_definitions:
 
     const batchAfter = await oneFromAsync(storage.getBucketDataBatch(checkpoint, new Map([['global[]', '0']])));
     const dataAfter = batchAfter.batch.data;
+    const checksumAfter = await storage.getChecksums(checkpoint, ['global[]']);
 
     expect(batchAfter.targetOp).toEqual(3n);
     expect(dataAfter).toMatchObject([
@@ -104,6 +106,8 @@ bucket_definitions:
       }
     ]);
 
+    expect(checksumBefore.get('global[]')).toEqual(checksumAfter.get('global[]'));
+
     validateCompactedBucket(dataBefore, dataAfter);
   });
 
@@ -154,6 +158,7 @@ bucket_definitions:
 
     const batchBefore = await oneFromAsync(storage.getBucketDataBatch(checkpoint, new Map([['global[]', '0']])));
     const dataBefore = batchBefore.batch.data;
+    const checksumBefore = await storage.getChecksums(checkpoint, ['global[]']);
 
     expect(dataBefore).toMatchObject([
       {
@@ -186,6 +191,7 @@ bucket_definitions:
 
     const batchAfter = await oneFromAsync(storage.getBucketDataBatch(checkpoint, new Map([['global[]', '0']])));
     const dataAfter = batchAfter.batch.data;
+    const checksumAfter = await storage.getChecksums(checkpoint, ['global[]']);
 
     expect(batchAfter.targetOp).toEqual(4n);
     expect(dataAfter).toMatchObject([
@@ -201,7 +207,79 @@ bucket_definitions:
         op_id: '4'
       }
     ]);
+    expect(checksumBefore.get('global[]')).toEqual(checksumAfter.get('global[]'));
 
     validateCompactedBucket(dataBefore, dataAfter);
   });
+
+  test('compacting (3)', async () => {
+    const sync_rules = SqlSyncRules.fromYaml(`
+bucket_definitions:
+  global:
+    data: [select * from test]
+    `);
+
+    const storage = (await factory()).getInstance({ id: 1, sync_rules, slot_name: 'test' });
+
+    const result = await storage.startBatch({}, async (batch) => {
+      await batch.save({
+        sourceTable: TEST_TABLE,
+        tag: 'insert',
+        after: {
+          id: 't1'
+        }
+      });
+
+      await batch.save({
+        sourceTable: TEST_TABLE,
+        tag: 'insert',
+        after: {
+          id: 't2'
+        }
+      });
+
+      await batch.save({
+        sourceTable: TEST_TABLE,
+        tag: 'delete',
+        before: {
+          id: 't1'
+        }
+      });
+    });
+
+    const checkpoint1 = result!.flushed_op;
+    const checksumBefore = await storage.getChecksums(checkpoint1, ['global[]']);
+    console.log('before', checksumBefore);
+
+    const result2 = await storage.startBatch({}, async (batch) => {
+      await batch.save({
+        sourceTable: TEST_TABLE,
+        tag: 'delete',
+        before: {
+          id: 't2'
+        }
+      });
+    });
+    const checkpoint2 = result2!.flushed_op;
+
+    await storage.compact(compactOptions);
+
+    const batchAfter = await oneFromAsync(storage.getBucketDataBatch(checkpoint2, new Map([['global[]', '0']])));
+    const dataAfter = batchAfter.batch.data;
+    const checksumAfter = await storage.getChecksums(checkpoint2, ['global[]']);
+
+    expect(batchAfter.targetOp).toEqual(4n);
+    expect(dataAfter).toMatchObject([
+      {
+        checksum: 857217610,
+        op: 'CLEAR',
+        op_id: '4'
+      }
+    ]);
+    expect(checksumAfter.get('global[]')).toEqual({
+      bucket: 'global[]',
+      count: 1,
+      checksum: 857217610
+    });
+  });
 }

package/test/src/data_storage.test.ts

@@ -1294,4 +1294,26 @@ bucket_definitions:
 
     expect(getBatchMeta(batch3)).toEqual(null);
   });
+
+  test('empty storage metrics', async () => {
+    const f = await factory({ dropAll: true });
+
+    const metrics = await f.getStorageMetrics();
+    expect(metrics).toEqual({
+      operations_size_bytes: 0,
+      parameters_size_bytes: 0,
+      replication_size_bytes: 0
+    });
+
+    const r = await f.configureSyncRules('bucket_definitions: {}');
+    const storage = f.getInstance(r.persisted_sync_rules!.parsed());
+    await storage.autoActivate();
+
+    const metrics2 = await f.getStorageMetrics();
+    expect(metrics2).toEqual({
+      operations_size_bytes: 0,
+      parameters_size_bytes: 0,
+      replication_size_bytes: 0
+    });
+  });
 }

package/test/src/sync.test.ts

@@ -5,7 +5,6 @@ import { JSONBig } from '@powersync/service-jsonbig';
 import { RequestParameters } from '@powersync/service-sync-rules';
 import * as timers from 'timers/promises';
 import { describe, expect, test } from 'vitest';
-import { ZERO_LSN } from '../../src/replication/WalStream.js';
 import { streamResponse } from '../../src/sync/sync.js';
 import { makeTestTable, MONGO_STORAGE_FACTORY, StorageFactory } from './util.js';
 
@@ -33,7 +32,6 @@ function defineTests(factory: StorageFactory) {
     });
 
     const storage = await f.getInstance(syncRules.parsed());
-    await storage.setSnapshotDone(ZERO_LSN);
     await storage.autoActivate();
 
     const result = await storage.startBatch({}, async (batch) => {
@@ -82,7 +80,6 @@ function defineTests(factory: StorageFactory) {
     });
 
     const storage = await f.getInstance(syncRules.parsed());
-    await storage.setSnapshotDone(ZERO_LSN);
     await storage.autoActivate();
 
     const result = await storage.startBatch({}, async (batch) => {
@@ -125,7 +122,6 @@ function defineTests(factory: StorageFactory) {
     });
 
     const storage = await f.getInstance(syncRules.parsed());
-    await storage.setSnapshotDone(ZERO_LSN);
     await storage.autoActivate();
 
     const stream = streamResponse({
@@ -152,7 +148,6 @@ function defineTests(factory: StorageFactory) {
     });
 
     const storage = await f.getInstance(syncRules.parsed());
-    await storage.setSnapshotDone(ZERO_LSN);
     await storage.autoActivate();
 
     const stream = streamResponse({
@@ -211,7 +206,6 @@ function defineTests(factory: StorageFactory) {
     });
 
     const storage = await f.getInstance(syncRules.parsed());
-    await storage.setSnapshotDone(ZERO_LSN);
     await storage.autoActivate();
 
     const exp = Date.now() / 1000 + 0.1;
@@ -249,7 +243,6 @@ function defineTests(factory: StorageFactory) {
     });
 
     const storage = await f.getInstance(syncRules.parsed());
-    await storage.setSnapshotDone(ZERO_LSN);
     await storage.autoActivate();
 
     await storage.startBatch({}, async (batch) => {

package/test/src/util.ts

@@ -22,11 +22,22 @@ Metrics.getInstance().resetCounters();
 
 export const TEST_URI = env.PG_TEST_URL;
 
-export type StorageFactory = () => Promise<BucketStorageFactory>;
+export interface StorageOptions {
+  /**
+   * By default, collections are only cleared/
+   * Setting this to true will drop the collections completely.
+   */
+  dropAll?: boolean;
+}
+export type StorageFactory = (options?: StorageOptions) => Promise<BucketStorageFactory>;
 
-export const MONGO_STORAGE_FACTORY: StorageFactory = async () => {
+export const MONGO_STORAGE_FACTORY: StorageFactory = async (options?: StorageOptions) => {
   const db = await connectMongo();
-  await db.clear();
+  if (options?.dropAll) {
+    await db.drop();
+  } else {
+    await db.clear();
+  }
   return new MongoBucketStorage(db, { slot_name_prefix: 'test_' });
 };