@powersync/service-module-mongodb-storage 0.15.3 → 0.16.0

package/test/src/storage_compacting.test.ts

@@ -1,7 +1,7 @@
+import { VersionedPowerSyncMongoV3 } from '@module/storage/implementation/v3/VersionedPowerSyncMongoV3.js';
 import { storage, SyncRulesBucketStorage, updateSyncRulesFromYaml } from '@powersync/service-core';
 import { bucketRequest, register, test_utils } from '@powersync/service-core-tests';
 import { describe, expect, test } from 'vitest';
-import { MongoCompactor } from '../../src/storage/implementation/MongoCompactor.js';
 import { INITIALIZED_MONGO_STORAGE_FACTORY } from './util.js';
 
 describe('Mongo Sync Bucket Storage Compact', () => {
@@ -64,9 +64,16 @@ bucket_definitions:
 
     test('full compact', async () => {
       const { bucketStorage, checkpoint, factory, syncRules } = await setup();
+      const storageDb = bucketStorage.db;
 
       // Simulate bucket_state from old version not being available
-      await factory.db.bucket_state.deleteMany({});
+      if (storageDb.storageConfig.incrementalReprocessing) {
+        // This should actually never happen on V3, but we test this anyway.
+        // Can remove this if it causes issues in the future.
+        await (storageDb as VersionedPowerSyncMongoV3).bucketStateV3(bucketStorage.group_id).deleteMany({});
+      } else {
+        await factory.db.bucket_state.deleteMany({});
+      }
 
       await bucketStorage.compact({
         clearBatchLimit: 200,
@@ -95,10 +102,10 @@ bucket_definitions:
     });
 
     test('populatePersistentChecksumCache', async () => {
-      // Populate old sync rules version
+      // Populate old replication stream
       const { factory } = await setup();
 
-      // Now populate another version (bucket definition name changed)
+      // Now populate another replication stream (bucket definition name changed)
       const syncRules = await factory.updateSyncRules(
         updateSyncRulesFromYaml(`
 bucket_definitions:
@@ -108,6 +115,7 @@ bucket_definitions:
     `)
       );
       const bucketStorage = factory.getInstance(syncRules);
+      const storageDb = (bucketStorage as any).db;
 
       await populate(bucketStorage, 2);
       const { checkpoint } = await bucketStorage.getCheckpoint();
@@ -158,35 +166,54 @@ bucket_definitions:
     `)
       );
       const bucketStorage = factory.getInstance(syncRules);
+      const storageDb = bucketStorage.db;
 
       // This simulates bucket_state created using bigint bytes.
       // This typically happens when buckets get very large (> 2GiB). We don't want to create that much
       // data in the tests, so we directly insert the bucket_state here.
-      await factory.db.bucket_state.insertOne({
-        _id: {
-          g: bucketStorage.group_id,
-          b: 'global[]'
-        },
-        last_op: 5n,
-        compacted_state: {
-          op_id: 3n,
-          count: 3,
-          checksum: 0n,
-          bytes: 7n
-        },
-        estimate_since_compact: {
-          count: 2,
-          bytes: 5n
-        }
-      });
-
-      // This test uses a couple of internal APIs of the compactor - there is no simple way
-      // to test this using the current public APIs.
-      const compactor = new MongoCompactor(bucketStorage, (bucketStorage as any).db, {
-        maxOpId: 5n
-      });
-
-      const dirtyBuckets = (compactor as any).dirtyBucketBatches({
+      if (storageDb.storageConfig.incrementalReprocessing) {
+        const bucketStateCollection = (storageDb as VersionedPowerSyncMongoV3).bucketStateV3(bucketStorage.group_id);
+        await bucketStateCollection.insertOne({
+          _id: {
+            d: '1',
+            b: 'global[]'
+          },
+          last_op: 5n,
+          compacted_state: {
+            op_id: 3n,
+            count: 3,
+            checksum: 0n,
+            bytes: 7n
+          },
+          estimate_since_compact: {
+            count: 2,
+            bytes: 5n
+          }
+        });
+      } else {
+        await factory.db.bucket_state.insertOne({
+          _id: {
+            g: bucketStorage.group_id,
+            b: 'global[]'
+          },
+          last_op: 5n,
+          compacted_state: {
+            op_id: 3n,
+            count: 3,
+            checksum: 0n,
+            bytes: 7n
+          },
+          estimate_since_compact: {
+            count: 2,
+            bytes: 5n
+          }
+        });
+      }
+
+      // This test uses a couple of "internal" APIs of the compactor.
+      const compactor = bucketStorage.createMongoCompactor({ maxOpId: 5n });
+
+      const dirtyBuckets = compactor.dirtyBucketBatches({
         minBucketChanges: 1,
         minChangeRatio: 0.39
       });
@@ -205,6 +232,7 @@ bucket_definitions:
       expect(checksumBuckets).toEqual([
         {
           bucket: 'global[]',
+          definitionId: storageDb.storageConfig.incrementalReprocessing ? '1' : null,
           estimatedCount: 5
         }
       ]);

package/test/src/storage_sync.test.ts

@@ -1,6 +1,14 @@
-import { storage, updateSyncRulesFromYaml } from '@powersync/service-core';
+import { deserializeParameterLookup, JwtPayload, storage, updateSyncRulesFromYaml } from '@powersync/service-core';
 import { bucketRequest, register, test_utils } from '@powersync/service-core-tests';
+import { RequestParameters } from '@powersync/service-sync-rules';
+import * as bson from 'bson';
 import { describe, expect, test } from 'vitest';
+import { MongoBucketStorage } from '../../src/storage/MongoBucketStorage.js';
+import { MongoSyncBucketStorage } from '../../src/storage/implementation/createMongoSyncBucketStorage.js';
+import { SyncRuleDocument } from '../../src/storage/implementation/models.js';
+import { SourceRecordStoreV3 } from '../../src/storage/implementation/v3/SourceRecordStoreV3.js';
+import type { VersionedPowerSyncMongoV3 } from '../../src/storage/implementation/v3/VersionedPowerSyncMongoV3.js';
+import { CurrentBucketV3 } from '../../src/storage/implementation/v3/models.js';
 import { INITIALIZED_MONGO_STORAGE_FACTORY, TEST_STORAGE_VERSIONS } from './util.js';
 
 function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, storageVersion: number) {
@@ -126,12 +134,350 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor
 
     // Test that the checksum type is correct.
     // Specifically, test that it never persisted as double.
-    const mongoFactory = factory as any;
-    const checksumTypes = await mongoFactory.db.bucket_data
-      .aggregate([{ $group: { _id: { $type: '$checksum' }, count: { $sum: 1 } } }])
-      .toArray();
+    const mongoFactory = factory as MongoBucketStorage;
+    const checksumTypes =
+      storageVersion >= 3
+        ? (
+            await Promise.all(
+              (
+                await mongoFactory.db.db
+                  .listCollections({ name: new RegExp(`^bucket_data_${syncRules.id}_`) }, { nameOnly: true })
+                  .toArray()
+              ).map((collection: { name: string }) =>
+                mongoFactory.db.db
+                  .collection(collection.name)
+                  .aggregate([{ $group: { _id: { $type: '$checksum' }, count: { $sum: 1 } } }])
+                  .toArray()
+              )
+            )
+          ).flat()
+        : await mongoFactory.db.bucket_data
+            .aggregate([{ $group: { _id: { $type: '$checksum' }, count: { $sum: 1 } } }])
+            .toArray();
     expect(checksumTypes).toEqual([{ _id: 'long', count: 4 }]);
   });
+
+  test.runIf(storageVersion >= 3)('uses v3 mongodb model shapes', async () => {
+    await using factory = await storageConfig.factory();
+    const syncRules = await factory.updateSyncRules(
+      updateSyncRulesFromYaml(
+        `
+    bucket_definitions:
+      global:
+        parameters:
+          - SELECT owner_id FROM test WHERE id = token_parameters.test
+        data:
+          - SELECT id, description, owner_id FROM test WHERE id = bucket.owner_id
+    `,
+        { storageVersion }
+      )
+    );
+    const bucketStorage = factory.getInstance(syncRules);
+    const sync_rules = syncRules.parsed(test_utils.PARSE_OPTIONS).hydratedSyncRules();
+    await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
+    const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], INITIALIZED_MONGO_STORAGE_FACTORY);
+
+    await writer.save({
+      sourceTable,
+      tag: storage.SaveOperationTag.INSERT,
+      after: {
+        id: 'shape-check',
+        description: 'shape',
+        owner_id: 'user-1'
+      },
+      afterReplicaId: test_utils.rid('shape-check')
+    });
+    await writer.markAllSnapshotDone('1/1');
+    await writer.commit('1/1');
+
+    const checkpoint = await bucketStorage.getCheckpoint();
+    const parameters = new RequestParameters(new JwtPayload({ sub: 'u1', parameters: { test: 'shape-check' } }), {});
+    const querier = sync_rules.getBucketParameterQuerier(test_utils.querierOptions(parameters)).querier;
+    const buckets = await querier.queryDynamicBucketDescriptions({
+      async getParameterSets(lookups) {
+        expect(lookups.map((l) => l.indexKey)).toEqual([['shape-check']]);
+        expect(lookups[0].indexId).toEqual('1');
+
+        const parameter_sets = await checkpoint.getParameterSets(lookups, 1000);
+        expect(parameter_sets).toEqual([{ lookup: lookups[0], rows: [{ owner_id: 'user-1' }] }]);
+        return parameter_sets;
+      }
+    });
+    expect(buckets.map((b) => b.bucket)).toEqual([bucketRequest(syncRules, 'global["user-1"]').bucket]);
+
+    const mongoFactory = factory as MongoBucketStorage;
+    const db = (bucketStorage as MongoSyncBucketStorage).db as VersionedPowerSyncMongoV3;
+    const currentDataCollections = await db.listSourceRecordCollectionsV3(syncRules.id);
+    const currentData = await currentDataCollections[0]?.findOne({});
+    const firstBucket: CurrentBucketV3 | undefined = currentData?.buckets[0] as CurrentBucketV3 | undefined;
+    expect(firstBucket?.def).toMatch(/^[0-9a-f]+$/);
+
+    const bucketCollections = await mongoFactory.db.db
+      .listCollections({ name: new RegExp(`^bucket_data_${syncRules.id}_`) }, { nameOnly: true })
+      .toArray();
+    expect(
+      bucketCollections.some((collection) => collection.name === `bucket_data_${syncRules.id}_${firstBucket?.def}`)
+    ).toBe(true);
+
+    const syncRule = await mongoFactory.db.sync_rules.findOne({ _id: syncRules.id });
+    const ruleMapping: SyncRuleDocument['rule_mapping'] | undefined = syncRule?.rule_mapping;
+    expect(Object.keys(ruleMapping?.definitions ?? {})).not.toHaveLength(0);
+
+    const parameterIndexId = Object.values(ruleMapping?.parameter_indexes ?? {})[0] as string | undefined;
+    expect(parameterIndexId).toBeDefined();
+    const parameterEntry = await db.parameterIndexV3(syncRules.id, parameterIndexId!).findOne({});
+    expect(deserializeParameterLookup(parameterEntry!.lookup)).toEqual(['shape-check']);
+  });
+
+  test.runIf(storageVersion < 3)('uses a single current_data collection for v1 source records', async () => {
+    await using factory = await storageConfig.factory();
+    const syncRules = await factory.updateSyncRules(
+      updateSyncRulesFromYaml(
+        `
+    bucket_definitions:
+      global:
+        data:
+          - SELECT id, description FROM test
+    `,
+        { storageVersion }
+      )
+    );
+    const bucketStorage = factory.getInstance(syncRules);
+
+    await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
+    const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], INITIALIZED_MONGO_STORAGE_FACTORY);
+
+    await writer.save({
+      sourceTable,
+      tag: storage.SaveOperationTag.INSERT,
+      after: {
+        id: 'shape-check',
+        description: 'shape'
+      },
+      afterReplicaId: test_utils.rid('shape-check')
+    });
+    await writer.markAllSnapshotDone('1/1');
+    await writer.commit('1/1');
+
+    const mongoFactory = factory as MongoBucketStorage;
+    expect(await mongoFactory.db.current_data.countDocuments({ '_id.g': syncRules.id })).toBe(1);
+
+    const sourceRecordCollections = await mongoFactory.db.db
+      .listCollections({ name: new RegExp(`^source_records_${syncRules.id}_`) }, { nameOnly: true })
+      .toArray();
+    expect(sourceRecordCollections).toEqual([]);
+  });
+
+  test.runIf(storageVersion < 3)('clear removes v1 current_data rows', async () => {
+    await using factory = await storageConfig.factory();
+    const syncRules = await factory.updateSyncRules(
+      updateSyncRulesFromYaml(
+        `
+    bucket_definitions:
+      global:
+        data:
+          - SELECT id, description FROM test
+    `,
+        { storageVersion }
+      )
+    );
+    const bucketStorage = factory.getInstance(syncRules);
+
+    await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
+    const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], INITIALIZED_MONGO_STORAGE_FACTORY);
+
+    await writer.save({
+      sourceTable,
+      tag: storage.SaveOperationTag.INSERT,
+      after: {
+        id: 'clear-check',
+        description: 'shape'
+      },
+      afterReplicaId: test_utils.rid('clear-check')
+    });
+    await writer.markAllSnapshotDone('1/1');
+    await writer.commit('1/1');
+
+    const mongoFactory = factory as MongoBucketStorage;
+    expect(await mongoFactory.db.current_data.countDocuments({ '_id.g': syncRules.id })).toBe(1);
+
+    await bucketStorage.clear();
+
+    expect(await mongoFactory.db.current_data.countDocuments({ '_id.g': syncRules.id })).toBe(0);
+  });
+
+  test.runIf(storageVersion < 3)('storage metrics include v1 current_data', async () => {
+    await using factory = await storageConfig.factory();
+    const syncRules = await factory.updateSyncRules(
+      updateSyncRulesFromYaml(
+        `
+    bucket_definitions:
+      global:
+        data:
+          - SELECT id, description FROM test
+    `,
+        { storageVersion }
+      )
+    );
+    const bucketStorage = factory.getInstance(syncRules);
+    const metricsBefore = await factory.getStorageMetrics();
+
+    await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
+    const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], INITIALIZED_MONGO_STORAGE_FACTORY);
+
+    await writer.save({
+      sourceTable,
+      tag: storage.SaveOperationTag.INSERT,
+      after: {
+        id: 'metric-check',
+        description: 'shape'
+      },
+      afterReplicaId: test_utils.rid('metric-check')
+    });
+    await writer.markAllSnapshotDone('1/1');
+    await writer.commit('1/1');
+
+    const mongoFactory = factory as MongoBucketStorage;
+    expect(await mongoFactory.db.current_data.countDocuments({ '_id.g': syncRules.id })).toBe(1);
+
+    const metricsAfter = await factory.getStorageMetrics();
+    expect(metricsAfter.replication_size_bytes).toBeGreaterThan(metricsBefore.replication_size_bytes);
+  });
+
+  test.runIf(storageVersion >= 3)(
+    'loads parameter checkpoint changes across all v3 parameter index collections',
+    async () => {
+      await using factory = await storageConfig.factory();
+      const syncRules = await factory.updateSyncRules(
+        updateSyncRulesFromYaml(
+          `
+    bucket_definitions:
+      by_owner:
+        parameters:
+          - SELECT owner_id FROM test WHERE id = token_parameters.owner_lookup
+        data:
+          - SELECT id, owner_id FROM test WHERE owner_id = bucket.owner_id
+      by_category:
+        parameters:
+          - SELECT category_id FROM test WHERE id = token_parameters.category_lookup
+        data:
+          - SELECT id, category_id FROM test WHERE category_id = bucket.category_id
+    `,
+          { storageVersion }
+        )
+      );
+      const bucketStorage = factory.getInstance(syncRules) as MongoSyncBucketStorage;
+      const previousCheckpoint = await bucketStorage.getCheckpoint();
+
+      await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
+      const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], INITIALIZED_MONGO_STORAGE_FACTORY);
+
+      await writer.save({
+        sourceTable,
+        tag: storage.SaveOperationTag.INSERT,
+        after: {
+          id: 'shape-check',
+          owner_id: 'user-1',
+          category_id: 'cat-1'
+        },
+        afterReplicaId: test_utils.rid('shape-check')
+      });
+      await writer.markAllSnapshotDone('1/1');
+      await writer.commit('1/1');
+
+      const nextCheckpoint = await bucketStorage.getCheckpoint();
+      const changes = await bucketStorage.getCheckpointChanges({
+        lastCheckpoint: previousCheckpoint,
+        nextCheckpoint
+      });
+
+      expect(changes.invalidateParameterBuckets).toBe(false);
+      expect(changes.updatedParameterLookups).toEqual(new Set(['["1","","shape-check"]', '["2","","shape-check"]']));
+    }
+  );
+
+  test.runIf(storageVersion >= 3)('cleans pending deletes only for tracked v3 source tables', async () => {
+    await using factory = await storageConfig.factory();
+    const syncRules = await factory.updateSyncRules(
+      updateSyncRulesFromYaml(
+        `
+    bucket_definitions:
+      global:
+        data:
+          - SELECT id, description FROM test
+    `,
+        { storageVersion }
+      )
+    );
+
+    const mongoFactory = factory as MongoBucketStorage;
+    const bucketStorage = mongoFactory.getInstance(syncRules) as any;
+    const db = bucketStorage.db;
+    await db.initializeStreamStorage(syncRules.id);
+
+    const sourceTableA = new bson.ObjectId();
+    const sourceTableB = new bson.ObjectId();
+    await db.sourceTablesV3(syncRules.id).insertMany([
+      {
+        _id: sourceTableA,
+        connection_id: 1,
+        relation_id: 'a',
+        schema_name: 'public',
+        table_name: 'table_a',
+        replica_id_columns: null,
+        replica_id_columns2: [],
+        snapshot_done: true,
+        snapshot_status: undefined,
+        bucket_data_source_ids: [],
+        parameter_lookup_source_ids: [],
+        latest_pending_delete: 9n
+      },
+      {
+        _id: sourceTableB,
+        connection_id: 1,
+        relation_id: 'b',
+        schema_name: 'public',
+        table_name: 'table_b',
+        replica_id_columns: null,
+        replica_id_columns2: [],
+        snapshot_done: true,
+        snapshot_status: undefined,
+        bucket_data_source_ids: [],
+        parameter_lookup_source_ids: [],
+        latest_pending_delete: 12n
+      }
+    ]);
+
+    await db.sourceRecordsV3(syncRules.id, sourceTableA).insertMany([
+      { _id: 'deleted-1', data: null, buckets: [], lookups: [], pending_delete: 5n },
+      { _id: 'deleted-2', data: null, buckets: [], lookups: [], pending_delete: 9n },
+      { _id: 'active', data: null, buckets: [], lookups: [] }
+    ]);
+    await db
+      .sourceRecordsV3(syncRules.id, sourceTableB)
+      .insertMany([{ _id: 'later-delete', data: null, buckets: [], lookups: [], pending_delete: 12n }]);
+
+    const store = new SourceRecordStoreV3(db, syncRules.id, bucketStorage.sync_rules.mapping);
+    const logger = { info() {} } as any;
+
+    await store.postCommitCleanup(6n, logger);
+
+    expect(await db.sourceRecordsV3(syncRules.id, sourceTableA).countDocuments({ pending_delete: 5n })).toBe(0);
+    expect(await db.sourceRecordsV3(syncRules.id, sourceTableA).countDocuments({ pending_delete: 9n })).toBe(1);
+    expect(await db.sourceRecordsV3(syncRules.id, sourceTableB).countDocuments({ pending_delete: 12n })).toBe(1);
+    expect((await db.sourceTablesV3(syncRules.id).findOne({ _id: sourceTableA }))?.latest_pending_delete).toBe(9n);
+    expect((await db.sourceTablesV3(syncRules.id).findOne({ _id: sourceTableB }))?.latest_pending_delete).toBe(12n);
+
+    await store.postCommitCleanup(10n, logger);
+
+    expect(
+      await db.sourceRecordsV3(syncRules.id, sourceTableA).countDocuments({ pending_delete: { $exists: true } })
+    ).toBe(0);
+    expect(
+      (await db.sourceTablesV3(syncRules.id).findOne({ _id: sourceTableA }))?.latest_pending_delete
+    ).toBeUndefined();
+    expect((await db.sourceTablesV3(syncRules.id).findOne({ _id: sourceTableB }))?.latest_pending_delete).toBe(12n);
+  });
 }
 
 describe('sync - mongodb', () => {

package/test/tsconfig.json

@@ -1,7 +1,6 @@
 {
   "extends": "../../../tsconfig.tests.json",
   "compilerOptions": {
-    "baseUrl": "./",
     "paths": {
       "@/*": ["../../../packages/service-core/src/*"],
       "@module/*": ["../src/*"],