@powersync/service-module-mongodb-storage 0.18.2 → 0.19.0

package/test/src/__snapshots__/storage_sync.test.ts.snap

@@ -589,200 +589,6 @@ exports[`sync - mongodb > storage v1 > sync global data 1`] = `
 ]
 `;
 
-exports[`sync - mongodb > storage v1 > sync interrupts low-priority buckets on new checkpoints (2) 1`] = `
-[
-  {
-    "checkpoint": {
-      "buckets": [
-        {
-          "bucket": "b0a[]",
-          "checksum": -659831575,
-          "count": 2000,
-          "priority": 2,
-          "subscriptions": [
-            {
-              "default": 0,
-            },
-          ],
-        },
-        {
-          "bucket": "b0b[]",
-          "checksum": -659831575,
-          "count": 2000,
-          "priority": 2,
-          "subscriptions": [
-            {
-              "default": 1,
-            },
-          ],
-        },
-        {
-          "bucket": "b1[]",
-          "checksum": -1096116670,
-          "count": 1,
-          "priority": 1,
-          "subscriptions": [
-            {
-              "default": 2,
-            },
-          ],
-        },
-      ],
-      "last_op_id": "4001",
-      "streams": [
-        {
-          "errors": [],
-          "is_default": true,
-          "name": "b0a",
-        },
-        {
-          "errors": [],
-          "is_default": true,
-          "name": "b0b",
-        },
-        {
-          "errors": [],
-          "is_default": true,
-          "name": "b1",
-        },
-      ],
-      "write_checkpoint": undefined,
-    },
-  },
-  {
-    "data": {
-      "after": "0",
-      "bucket": "b1[]",
-      "data": undefined,
-      "has_more": false,
-      "next_after": "1",
-    },
-  },
-  {
-    "partial_checkpoint_complete": {
-      "last_op_id": "4001",
-      "priority": 1,
-    },
-  },
-  {
-    "data": {
-      "after": "0",
-      "bucket": "b0a[]",
-      "data": undefined,
-      "has_more": true,
-      "next_after": "2000",
-    },
-  },
-  {
-    "data": {
-      "after": "2000",
-      "bucket": "b0a[]",
-      "data": undefined,
-      "has_more": true,
-      "next_after": "4000",
-    },
-  },
-  {
-    "checkpoint_diff": {
-      "last_op_id": "4004",
-      "removed_buckets": [],
-      "updated_buckets": [
-        {
-          "bucket": "b0a[]",
-          "checksum": 883076828,
-          "count": 2001,
-          "priority": 2,
-          "subscriptions": [
-            {
-              "default": 0,
-            },
-          ],
-        },
-        {
-          "bucket": "b0b[]",
-          "checksum": 883076828,
-          "count": 2001,
-          "priority": 2,
-          "subscriptions": [
-            {
-              "default": 1,
-            },
-          ],
-        },
-        {
-          "bucket": "b1[]",
-          "checksum": 1841937527,
-          "count": 2,
-          "priority": 1,
-          "subscriptions": [
-            {
-              "default": 2,
-            },
-          ],
-        },
-      ],
-      "write_checkpoint": undefined,
-    },
-  },
-  {
-    "data": {
-      "after": "1",
-      "bucket": "b1[]",
-      "data": undefined,
-      "has_more": false,
-      "next_after": "4002",
-    },
-  },
-  {
-    "partial_checkpoint_complete": {
-      "last_op_id": "4004",
-      "priority": 1,
-    },
-  },
-  {
-    "data": {
-      "after": "4000",
-      "bucket": "b0a[]",
-      "data": undefined,
-      "has_more": false,
-      "next_after": "4003",
-    },
-  },
-  {
-    "data": {
-      "after": "0",
-      "bucket": "b0b[]",
-      "data": undefined,
-      "has_more": true,
-      "next_after": "1999",
-    },
-  },
-  {
-    "data": {
-      "after": "1999",
-      "bucket": "b0b[]",
-      "data": undefined,
-      "has_more": true,
-      "next_after": "3999",
-    },
-  },
-  {
-    "data": {
-      "after": "3999",
-      "bucket": "b0b[]",
-      "data": undefined,
-      "has_more": false,
-      "next_after": "4004",
-    },
-  },
-  {
-    "checkpoint_complete": {
-      "last_op_id": "4004",
-    },
-  },
-]
-`;
-
 exports[`sync - mongodb > storage v1 > sync legacy non-raw data 1`] = `
 [
   {
@@ -1778,200 +1584,6 @@ exports[`sync - mongodb > storage v2 > sync global data 1`] = `
 ]
 `;
 
-exports[`sync - mongodb > storage v2 > sync interrupts low-priority buckets on new checkpoints (2) 1`] = `
-[
-  {
-    "checkpoint": {
-      "buckets": [
-        {
-          "bucket": "1#b0a[]",
-          "checksum": -659831575,
-          "count": 2000,
-          "priority": 2,
-          "subscriptions": [
-            {
-              "default": 0,
-            },
-          ],
-        },
-        {
-          "bucket": "1#b0b[]",
-          "checksum": -659831575,
-          "count": 2000,
-          "priority": 2,
-          "subscriptions": [
-            {
-              "default": 1,
-            },
-          ],
-        },
-        {
-          "bucket": "1#b1[]",
-          "checksum": -1096116670,
-          "count": 1,
-          "priority": 1,
-          "subscriptions": [
-            {
-              "default": 2,
-            },
-          ],
-        },
-      ],
-      "last_op_id": "4001",
-      "streams": [
-        {
-          "errors": [],
-          "is_default": true,
-          "name": "b0a",
-        },
-        {
-          "errors": [],
-          "is_default": true,
-          "name": "b0b",
-        },
-        {
-          "errors": [],
-          "is_default": true,
-          "name": "b1",
-        },
-      ],
-      "write_checkpoint": undefined,
-    },
-  },
-  {
-    "data": {
-      "after": "0",
-      "bucket": "1#b1[]",
-      "data": undefined,
-      "has_more": false,
-      "next_after": "1",
-    },
-  },
-  {
-    "partial_checkpoint_complete": {
-      "last_op_id": "4001",
-      "priority": 1,
-    },
-  },
-  {
-    "data": {
-      "after": "0",
-      "bucket": "1#b0a[]",
-      "data": undefined,
-      "has_more": true,
-      "next_after": "2000",
-    },
-  },
-  {
-    "data": {
-      "after": "2000",
-      "bucket": "1#b0a[]",
-      "data": undefined,
-      "has_more": true,
-      "next_after": "4000",
-    },
-  },
-  {
-    "checkpoint_diff": {
-      "last_op_id": "4004",
-      "removed_buckets": [],
-      "updated_buckets": [
-        {
-          "bucket": "1#b0a[]",
-          "checksum": 883076828,
-          "count": 2001,
-          "priority": 2,
-          "subscriptions": [
-            {
-              "default": 0,
-            },
-          ],
-        },
-        {
-          "bucket": "1#b0b[]",
-          "checksum": 883076828,
-          "count": 2001,
-          "priority": 2,
-          "subscriptions": [
-            {
-              "default": 1,
-            },
-          ],
-        },
-        {
-          "bucket": "1#b1[]",
-          "checksum": 1841937527,
-          "count": 2,
-          "priority": 1,
-          "subscriptions": [
-            {
-              "default": 2,
-            },
-          ],
-        },
-      ],
-      "write_checkpoint": undefined,
-    },
-  },
-  {
-    "data": {
-      "after": "1",
-      "bucket": "1#b1[]",
-      "data": undefined,
-      "has_more": false,
-      "next_after": "4002",
-    },
-  },
-  {
-    "partial_checkpoint_complete": {
-      "last_op_id": "4004",
-      "priority": 1,
-    },
-  },
-  {
-    "data": {
-      "after": "4000",
-      "bucket": "1#b0a[]",
-      "data": undefined,
-      "has_more": false,
-      "next_after": "4003",
-    },
-  },
-  {
-    "data": {
-      "after": "0",
-      "bucket": "1#b0b[]",
-      "data": undefined,
-      "has_more": true,
-      "next_after": "1999",
-    },
-  },
-  {
-    "data": {
-      "after": "1999",
-      "bucket": "1#b0b[]",
-      "data": undefined,
-      "has_more": true,
-      "next_after": "3999",
-    },
-  },
-  {
-    "data": {
-      "after": "3999",
-      "bucket": "1#b0b[]",
-      "data": undefined,
-      "has_more": false,
-      "next_after": "4004",
-    },
-  },
-  {
-    "checkpoint_complete": {
-      "last_op_id": "4004",
-    },
-  },
-]
-`;
-
 exports[`sync - mongodb > storage v2 > sync legacy non-raw data 1`] = `
 [
   {
@@ -2967,200 +2579,6 @@ exports[`sync - mongodb > storage v3 > sync global data 1`] = `
 ]
 `;
 
-exports[`sync - mongodb > storage v3 > sync interrupts low-priority buckets on new checkpoints (2) 1`] = `
-[
-  {
-    "checkpoint": {
-      "buckets": [
-        {
-          "bucket": "b0a.1.1[]",
-          "checksum": -659831575,
-          "count": 2000,
-          "priority": 2,
-          "subscriptions": [
-            {
-              "default": 0,
-            },
-          ],
-        },
-        {
-          "bucket": "b0b.1.2[]",
-          "checksum": -659831575,
-          "count": 2000,
-          "priority": 2,
-          "subscriptions": [
-            {
-              "default": 1,
-            },
-          ],
-        },
-        {
-          "bucket": "b1.1.3[]",
-          "checksum": -1096116670,
-          "count": 1,
-          "priority": 1,
-          "subscriptions": [
-            {
-              "default": 2,
-            },
-          ],
-        },
-      ],
-      "last_op_id": "4001",
-      "streams": [
-        {
-          "errors": [],
-          "is_default": true,
-          "name": "b0a",
-        },
-        {
-          "errors": [],
-          "is_default": true,
-          "name": "b0b",
-        },
-        {
-          "errors": [],
-          "is_default": true,
-          "name": "b1",
-        },
-      ],
-      "write_checkpoint": undefined,
-    },
-  },
-  {
-    "data": {
-      "after": "0",
-      "bucket": "b1.1.3[]",
-      "data": undefined,
-      "has_more": false,
-      "next_after": "1",
-    },
-  },
-  {
-    "partial_checkpoint_complete": {
-      "last_op_id": "4001",
-      "priority": 1,
-    },
-  },
-  {
-    "data": {
-      "after": "0",
-      "bucket": "b0a.1.1[]",
-      "data": undefined,
-      "has_more": true,
-      "next_after": "2000",
-    },
-  },
-  {
-    "data": {
-      "after": "2000",
-      "bucket": "b0a.1.1[]",
-      "data": undefined,
-      "has_more": true,
-      "next_after": "4000",
-    },
-  },
-  {
-    "checkpoint_diff": {
-      "last_op_id": "4004",
-      "removed_buckets": [],
-      "updated_buckets": [
-        {
-          "bucket": "b0a.1.1[]",
-          "checksum": 883076828,
-          "count": 2001,
-          "priority": 2,
-          "subscriptions": [
-            {
-              "default": 0,
-            },
-          ],
-        },
-        {
-          "bucket": "b0b.1.2[]",
-          "checksum": 883076828,
-          "count": 2001,
-          "priority": 2,
-          "subscriptions": [
-            {
-              "default": 1,
-            },
-          ],
-        },
-        {
-          "bucket": "b1.1.3[]",
-          "checksum": 1841937527,
-          "count": 2,
-          "priority": 1,
-          "subscriptions": [
-            {
-              "default": 2,
-            },
-          ],
-        },
-      ],
-      "write_checkpoint": undefined,
-    },
-  },
-  {
-    "data": {
-      "after": "1",
-      "bucket": "b1.1.3[]",
-      "data": undefined,
-      "has_more": false,
-      "next_after": "4002",
-    },
-  },
-  {
-    "partial_checkpoint_complete": {
-      "last_op_id": "4004",
-      "priority": 1,
-    },
-  },
-  {
-    "data": {
-      "after": "4000",
-      "bucket": "b0a.1.1[]",
-      "data": undefined,
-      "has_more": false,
-      "next_after": "4003",
-    },
-  },
-  {
-    "data": {
-      "after": "0",
-      "bucket": "b0b.1.2[]",
-      "data": undefined,
-      "has_more": true,
-      "next_after": "1999",
-    },
-  },
-  {
-    "data": {
-      "after": "1999",
-      "bucket": "b0b.1.2[]",
-      "data": undefined,
-      "has_more": true,
-      "next_after": "3999",
-    },
-  },
-  {
-    "data": {
-      "after": "3999",
-      "bucket": "b0b.1.2[]",
-      "data": undefined,
-      "has_more": false,
-      "next_after": "4004",
-    },
-  },
-  {
-    "checkpoint_complete": {
-      "last_op_id": "4004",
-    },
-  },
-]
-`;
-
 exports[`sync - mongodb > storage v3 > sync legacy non-raw data 1`] = `
 [
   {
@@ -3281,7 +2699,7 @@ exports[`sync - mongodb > storage v3 > sync updates to data query only 2`] = `
   },
   {
     "data": {
-      "after": "0",
+      "after": "1",
       "bucket": "by_user.1.1["user1"]",
       "data": [
         {

package/test/src/cleanup-stopped-sync-configs.test.ts

@@ -1,10 +1,14 @@
 import { MongoSyncBucketStorageV3 } from '@module/storage/implementation/v3/MongoSyncBucketStorageV3.js';
+import { ObjectStorageLifecycle } from '@module/storage/implementation/v3/object-storage/ObjectStorageLifecycle.js';
+import { mongoTestStorageFactoryGenerator } from '@module/utils/test-utils.js';
 import { storage, updateSyncRulesFromYaml } from '@powersync/service-core';
 import { test_utils } from '@powersync/service-core-tests';
 import * as bson from 'bson';
 import { describe, expect, test } from 'vitest';
 import { VersionedPowerSyncMongoV3 } from '../../src/storage/implementation/v3/VersionedPowerSyncMongoV3.js';
 import { ReplicationStreamDocumentV3 } from '../../src/storage/implementation/v3/models.js';
+import { env } from './env.js';
+import { MemoryObjectStorage } from './helpers/MemoryObjectStorage.js';
 import { INITIALIZED_MONGO_STORAGE_FACTORY } from './util.js';
 
 function sourceDescriptor(name: string, options: { objectId?: string } = {}): storage.SourceEntityDescriptor {
@@ -47,7 +51,14 @@ async function cleanupStoppedSyncConfigs(bucketStorage: MongoSyncBucketStorageV3
 
 describe('cleanupStoppedSyncConfigs - mongodb', () => {
   test('cleans up unused stopped sync config storage', async () => {
-    await using factory = await INITIALIZED_MONGO_STORAGE_FACTORY.factory();
+    const objectStorage = new MemoryObjectStorage();
+    const storageFactory = mongoTestStorageFactoryGenerator({
+      url: env.MONGO_TEST_URL,
+      isCI: env.CI,
+      supportsMultipleSyncConfigs: true,
+      objectStorage
+    });
+    await using factory = await storageFactory.factory();
 
     const first = await factory.updateSyncRules(
       updateSyncRulesFromYaml(
@@ -149,6 +160,16 @@ streams:
     await secondWriter.markAllSnapshotDone('2/1');
     await secondWriter.commit('2/1');
 
+    const lifecycle = new ObjectStorageLifecycle(db, first.replicationStreamId, objectStorage);
+    const stoppedObject = `${lifecycle.definitionPrefix(stoppedBucketDefinitionId!)}stopped.bson`;
+    const similarDefinitionObject = `${lifecycle.definitionPrefix(`${stoppedBucketDefinitionId!}0`)}live.bson`;
+    for (const path of [stoppedObject, similarDefinitionObject]) {
+      await objectStorage.put(path, new Uint8Array(), {
+        contentType: 'application/bson',
+        contentEncoding: null
+      });
+    }
+
     const result = await cleanupStoppedSyncConfigs(
       (await factory.getActiveSyncConfig())!.storage as MongoSyncBucketStorageV3
     );
@@ -166,6 +187,8 @@ streams:
     expect(await collectionExists(db, parameterIndexCollection)).toBe(false);
     expect(await collectionExists(db, sceneRecordsCollection)).toBe(false);
     expect(await collectionExists(db, invitationRecordsCollection)).toBe(false);
+    expect(objectStorage.store.has(stoppedObject)).toBe(false);
+    expect(objectStorage.store.has(similarDefinitionObject)).toBe(true);
     expect(
       await db.sourceTables(first.replicationStreamId).countDocuments({ _id: sceneTable.id as bson.ObjectId })
     ).toBe(0);

package/test/src/helpers/MemoryObjectStorage.ts

@@ -0,0 +1,53 @@
+import {
+  ObjectStorage,
+  ObjectStoragePutMetadata
+} from '@module/storage/implementation/v3/object-storage/ObjectStorage.js';
+
+export class MemoryObjectStorage implements ObjectStorage {
+  /**
+   * Public for testing purposes.
+   */
+  public readonly store = new Map<string, { data: Uint8Array; metadata: ObjectStoragePutMetadata }>();
+
+  async put(path: string, data: Uint8Array, metadata: ObjectStoragePutMetadata): Promise<void> {
+    this.store.set(path, { data, metadata: metadata });
+  }
+
+  async get(
+    path: string,
+    options?: { signal?: AbortSignal }
+  ): Promise<{ data: Uint8Array; metadata: ObjectStoragePutMetadata }> {
+    const signal = options?.signal;
+    signal?.throwIfAborted();
+    const data = this.store.get(path);
+    if (!data) {
+      throw new Error(`NotFound: ${path}`);
+    }
+    return data;
+  }
+
+  async *list(prefix: string, options?: { signal?: AbortSignal }): AsyncIterable<string> {
+    const signal = options?.signal;
+    for (const path of this.store.keys()) {
+      signal?.throwIfAborted();
+      if (path.startsWith(prefix)) {
+        yield path;
+      }
+    }
+  }
+
+  async delete(paths: string[]): Promise<void> {
+    for (const p of paths) {
+      this.store.delete(p);
+    }
+  }
+
+  async deletePrefix(prefix: string, options?: { signal?: AbortSignal }): Promise<{ objectCount: number }> {
+    const paths: string[] = [];
+    for await (const path of this.list(prefix, options)) {
+      paths.push(path);
+    }
+    await this.delete(paths);
+    return { objectCount: paths.length };
+  }
+}