@powersync/service-module-mongodb-storage 0.24.1 → 0.25.0

package/src/storage/implementation/v1/VersionedPowerSyncMongoV1.ts

@@ -26,7 +26,9 @@ export class VersionedPowerSyncMongoV1 extends BaseVersionedPowerSyncMongo {
     return this.upstream.source_tables as any as mongo.Collection<SourceTableDocumentV1>;
   }
 
-  async initializeStreamStorage(_replicationStreamId: number): Promise<void> {}
+  async initializeStreamStorage(_replicationStreamId: number): Promise<void> {
+    await this.upstream.initializeV1StorageCollections();
+  }
 
   get bucketDataV1(): mongo.Collection<BucketDataDocumentV1> {
     return this.upstream.bucket_data;

package/src/types/types.ts

@@ -54,6 +54,12 @@ export const MongoStorageConfig = service_types.configFile.BaseStorageConfig.and
      * This is an experimental option, and may be removed in a future release.
      */
     bulk_read_preference: MongoStorageReadPreference.optional(),
+    /**
+     * Read preference for collection discovery and collection stats used by storage metrics.
+     * For example, use secondaryPreferred to offload collection stats from the primary.
+     * If unset, MongoDB driver defaults are used.
+     */
+    stats_read_preference: MongoStorageReadPreference.optional(),
     /**
      * Throttle clear operations after sync config deploys, by pausing between batches.
      * The rate produces a pause proportional to the previous batch duration.

package/src/utils/test-utils.ts

@@ -15,11 +15,6 @@ export function mongoTestStorageFactoryGenerator(factoryOptions: MongoTestStorag
     factory: async (options?: TestStorageOptions) => {
       const db = connectMongoForTests(factoryOptions.url, factoryOptions.isCI, factoryOptions.monitorCommands);
 
-      // None of the tests insert data into this collection, so it was never created
-      if (!(await db.db.listCollections({ name: db.bucket_parameters.collectionName }).hasNext())) {
-        await db.db.createCollection('bucket_parameters');
-      }
-
       // Full migrations are not currently run for tests, so we manually create this
       await db.createCheckpointEventsCollection();
 

package/test/src/storage_sync.test.ts

@@ -7,7 +7,13 @@ import {
   storage,
   updateSyncRulesFromYaml
 } from '@powersync/service-core';
-import { bucketRequest, getTestStorage, register, test_utils } from '@powersync/service-core-tests';
+import {
+  bucketRequest,
+  getTestStorage,
+  register,
+  releaseTestStorageLease,
+  test_utils
+} from '@powersync/service-core-tests';
 import { parseSyncConfig, RequestParameters, ScopedParameterLookup } from '@powersync/service-sync-rules';
 import * as bson from 'bson';
 import { describe, expect, test } from 'vitest';
@@ -2178,6 +2184,91 @@ streams:
 }
 
 describe('sync - mongodb', () => {
+  test.each([
+    [1, false],
+    [2, false],
+    [1, true],
+    [2, true]
+  ] as const)('v%i activation cleans up v4 streams (previously stranded: %s)', async (storageVersion, stranded) => {
+    await using factory = await INITIALIZED_MONGO_STORAGE_FACTORY.factory();
+    const yaml = /* yaml */ `
+      # sync config fixture.
+      config:
+        edition: 3
+      streams:
+        global:
+          query: SELECT id FROM test
+    `;
+
+    async function activate(version: number, lsn: string) {
+      const stream = await factory.updateSyncRules(updateSyncRulesFromYaml(yaml, { storageVersion: version }));
+      const bucketStorage = await getTestStorage(factory, stream);
+      await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
+      const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], INITIALIZED_MONGO_STORAGE_FACTORY);
+      await writer.save({
+        sourceTable,
+        tag: storage.SaveOperationTag.INSERT,
+        after: { id: 'item-1' },
+        afterReplicaId: test_utils.rid('item-1')
+      });
+      await writer.markAllSnapshotDone(lsn);
+      await writer.commit(lsn);
+      return stream;
+    }
+
+    const previous = await activate(storage.STORAGE_VERSION_4, '1/1');
+    await releaseTestStorageLease(factory, previous.replicationStreamId);
+    const oldCollections = await factory.db.db
+      .listCollections(
+        {
+          name: new RegExp(
+            `^(bucket_data|source_records|source_table|bucket_state|parameter_index)_${previous.replicationStreamId}(_|$)`
+          )
+        },
+        { nameOnly: true }
+      )
+      .toArray();
+    expect(oldCollections.some((collection) => collection.name.startsWith('bucket_data_'))).toBe(true);
+    expect(oldCollections.some((collection) => collection.name.startsWith('source_records_'))).toBe(true);
+
+    const replacement = await activate(storageVersion, '2/1');
+    const previousDoc = (await factory.db.sync_rules.findOne({
+      _id: previous.replicationStreamId
+    })) as ReplicationStreamDocumentV3;
+    expect(previousDoc.state).toBe(storage.SyncRuleState.STOP);
+    if (stranded) {
+      // Reproduce databases left behind by older services: only the stream was stopped.
+      await factory.db.sync_rules.updateOne(
+        { _id: previous.replicationStreamId },
+        { $set: { 'sync_configs.$[].state': storage.SyncRuleState.ACTIVE } }
+      );
+    } else {
+      expect(previousDoc.sync_configs.map((config) => config.state)).toEqual([storage.SyncRuleState.STOP]);
+    }
+
+    const stopped = await factory.getStoppedReplicationStreams();
+    expect(stopped.map((stream) => stream.replicationStreamId)).toEqual([previous.replicationStreamId]);
+    const lock = await stopped[0].lock();
+    try {
+      await factory.getInstance(stopped[0], { skipLifecycleHooks: true }).terminate({ clearStorage: true });
+    } finally {
+      await lock.release();
+    }
+
+    expect((await factory.db.sync_rules.findOne({ _id: previous.replicationStreamId }))?.state).toBe(
+      storage.SyncRuleState.TERMINATED
+    );
+    const remaining = new Set(
+      (await factory.db.db.listCollections({}, { nameOnly: true }).toArray()).map((c) => c.name)
+    );
+    for (const collection of oldCollections) expect(remaining.has(collection.name)).toBe(false);
+    expect(await factory.getStoppedReplicationStreams()).toEqual([]);
+    expect((await factory.getActiveSyncConfig())?.replicationStream.replicationStreamId).toBe(
+      replacement.replicationStreamId
+    );
+    expect(await factory.db.current_data.countDocuments({ '_id.g': replacement.replicationStreamId })).toBe(1);
+  });
+
   test('v3 activation stops legacy active sync rules', async () => {
     await using factory = await INITIALIZED_MONGO_STORAGE_FACTORY.factory();
     const mongoFactory = factory as MongoBucketStorage;

package/test/src/v1_collections.test.ts

@@ -0,0 +1,233 @@
+import { MongoBucketStorage } from '@module/storage/MongoBucketStorage.js';
+import { mongoTestStorageFactoryGenerator } from '@module/utils/test-utils.js';
+import { storage, updateSyncRulesFromYaml } from '@powersync/service-core';
+import { getTestStorage, releaseTestStorageLease, test_utils } from '@powersync/service-core-tests';
+import { describe, expect, test, vi } from 'vitest';
+import { up as parameterIndexes } from '../../src/migrations/db/migrations/1684951997326-init.js';
+import { up as bucketIndexes } from '../../src/migrations/db/migrations/1741697235857-bucket-state-index.js';
+import { up as dirtyBucketIndexes } from '../../src/migrations/db/migrations/1760433882550-bucket-state-index2.js';
+import {
+  up as removeUnusedV1Collections,
+  down as restoreV1Collections
+} from '../../src/migrations/db/migrations/1794268800000-v1-storage-collections.js';
+import { env } from './env.js';
+
+const collectionNames = ['current_data', 'bucket_data', 'bucket_parameters', 'bucket_state', 'source_tables'];
+const migrationContext = {
+  service_context: {
+    configuration: { storage: { type: 'mongodb', uri: env.MONGO_TEST_URL }, slot_name_prefix: 'test_' }
+  }
+} as unknown as Parameters<typeof parameterIndexes>[0];
+const testStorage = mongoTestStorageFactoryGenerator({
+  url: env.MONGO_TEST_URL,
+  isCI: env.CI,
+  monitorCommands: true,
+  supportsMultipleSyncConfigs: true
+});
+const yaml = /* yaml */ `
+  # sync config fixture.
+  config:
+    edition: 3
+  streams:
+    global:
+      query: SELECT id FROM test
+`;
+
+async function activate(factory: MongoBucketStorage, version: number, lsn: string) {
+  const stream = await factory.updateSyncRules(updateSyncRulesFromYaml(yaml, { storageVersion: version }));
+  const bucketStorage = await getTestStorage(factory, stream);
+  await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
+  const sourceTable = await test_utils.resolveTestTable(
+    writer,
+    'test',
+    ['id'],
+    testStorage,
+    stream.replicationStreamId
+  );
+  await writer.save({
+    sourceTable,
+    tag: storage.SaveOperationTag.INSERT,
+    after: { id: 'item-1' },
+    afterReplicaId: test_utils.rid('item-1')
+  });
+  await writer.markAllSnapshotDone(lsn);
+  await writer.commit(lsn);
+  return stream;
+}
+
+async function names(factory: MongoBucketStorage) {
+  return (await factory.db.db.listCollections({}, { nameOnly: true }).toArray()).map((c) => c.name);
+}
+
+async function terminateStopped(factory: MongoBucketStorage) {
+  for (const stream of await factory.getStoppedReplicationStreams()) {
+    const lock = await stream.lock();
+    try {
+      await factory.getInstance(stream, { skipLifecycleHooks: true }).terminate({ clearStorage: true });
+    } finally {
+      await lock.release();
+    }
+  }
+}
+
+describe('v1/v2 collection lifecycle', () => {
+  test('migrations and v4 initialization do not create v1/v2 collections', async () => {
+    await using factory = await testStorage.factory();
+    await factory.db.drop();
+    await factory.db.createCheckpointEventsCollection();
+    for (const migration of [parameterIndexes, bucketIndexes, dirtyBucketIndexes, removeUnusedV1Collections]) {
+      await migration(migrationContext);
+    }
+    for (const name of collectionNames) expect(await names(factory)).not.toContain(name);
+    await activate(factory, storage.STORAGE_VERSION_4, '1/1');
+    for (const name of collectionNames) expect(await names(factory)).not.toContain(name);
+  });
+
+  test.each([1, 2])('drops unused v%i collections instead of deleting individual records', async (version) => {
+    await using factory = await testStorage.factory();
+    const previous = await activate(factory, version, '1/1');
+    await releaseTestStorageLease(factory, previous.replicationStreamId);
+    expect(await names(factory)).toEqual(expect.arrayContaining(collectionNames));
+    const current = await activate(factory, storage.STORAGE_VERSION_4, '2/1');
+    const dropped: string[] = [];
+    const deleted: string[] = [];
+    factory.db.client.on('commandStarted', (event) => {
+      if (event.commandName === 'drop') dropped.push(event.command.drop);
+      if (event.commandName === 'delete') deleted.push(event.command.delete);
+    });
+    await terminateStopped(factory);
+    expect(dropped).toEqual(expect.arrayContaining(collectionNames));
+    expect(deleted.filter((name) => collectionNames.includes(name))).toEqual([]);
+    for (const name of collectionNames) expect(await names(factory)).not.toContain(name);
+    expect((await factory.db.sync_rules.findOne({ _id: previous.replicationStreamId }))?.state).toBe(
+      storage.SyncRuleState.TERMINATED
+    );
+    expect((await factory.getActiveSyncConfig())?.replicationStream.replicationStreamId).toBe(
+      current.replicationStreamId
+    );
+
+    const returning = await activate(factory, version, '3/1');
+    await releaseTestStorageLease(factory, current.replicationStreamId);
+    await terminateStopped(factory);
+    expect(await names(factory)).toEqual(expect.arrayContaining(collectionNames));
+    expect((await factory.db.bucket_parameters.listIndexes().toArray()).map((index) => index.name)).toContain(
+      'lookup1'
+    );
+    expect((await factory.db.bucket_state.listIndexes().toArray()).map((index) => index.name)).toEqual(
+      expect.arrayContaining(['bucket_updates', 'dirty_count'])
+    );
+    expect(await factory.db.current_data.countDocuments({ '_id.g': returning.replicationStreamId })).toBe(1);
+  });
+
+  test.each([storage.SyncRuleState.PROCESSING, storage.SyncRuleState.ACTIVE, storage.SyncRuleState.ERRORED])(
+    'retains shared collections for a stream in state %s and deletes only stopped stream records',
+    async (state) => {
+      await using factory = await testStorage.factory();
+      const previous = await activate(factory, storage.STORAGE_VERSION_1, '1/1');
+      await releaseTestStorageLease(factory, previous.replicationStreamId);
+      const current = await activate(factory, storage.STORAGE_VERSION_2, '2/1');
+      await factory.db.sync_rules.updateOne({ _id: current.replicationStreamId }, { $set: { state } });
+      await terminateStopped(factory);
+      expect(await names(factory)).toEqual(expect.arrayContaining(collectionNames));
+      expect(await factory.db.current_data.countDocuments({ '_id.g': previous.replicationStreamId })).toBe(0);
+      expect(await factory.db.current_data.countDocuments({ '_id.g': current.replicationStreamId })).toBe(1);
+      // A missing storage_version also identifies a v1 stream.
+      await factory.db.sync_rules.updateOne({ _id: current.replicationStreamId }, { $unset: { storage_version: 1 } });
+      expect(await factory.dropUnusedV1StorageCollections()).toBe(false);
+      await removeUnusedV1Collections(migrationContext);
+      expect(await names(factory)).toEqual(expect.arrayContaining(collectionNames));
+      expect(await factory.db.current_data.countDocuments({ '_id.g': current.replicationStreamId })).toBe(1);
+    }
+  );
+
+  test('restarting and clearing an active v4 stream do not drop v1 collections', async () => {
+    await using factory = await testStorage.factory();
+    const previous = await activate(factory, storage.STORAGE_VERSION_2, '1/1');
+    await releaseTestStorageLease(factory, previous.replicationStreamId);
+    const current = await activate(factory, storage.STORAGE_VERSION_4, '2/1');
+    await factory.db.sync_rules.updateOne(
+      { _id: previous.replicationStreamId },
+      { $set: { state: storage.SyncRuleState.TERMINATED } }
+    );
+    await using restarted = await testStorage.factory({ doNotClear: true });
+    const drop = vi.spyOn(restarted, 'dropUnusedV1StorageCollections');
+    expect(await restarted.getStoppedReplicationStreams()).toEqual([]);
+    expect(drop).not.toHaveBeenCalled();
+    expect(await names(restarted)).toEqual(expect.arrayContaining(collectionNames));
+    await factory.getInstance(current).clear();
+    expect(await names(factory)).toEqual(expect.arrayContaining(collectionNames));
+  });
+
+  test('migration removes unused v1 collections and rollback recreates collections and indexes', async () => {
+    await using factory = await testStorage.factory();
+    const previous = await activate(factory, storage.STORAGE_VERSION_2, '1/1');
+    await releaseTestStorageLease(factory, previous.replicationStreamId);
+    const current = await activate(factory, storage.STORAGE_VERSION_4, '2/1');
+    await factory.db.sync_rules.updateOne(
+      { _id: previous.replicationStreamId },
+      { $set: { state: storage.SyncRuleState.TERMINATED } }
+    );
+    await removeUnusedV1Collections(migrationContext);
+    for (const name of collectionNames) expect(await names(factory)).not.toContain(name);
+    // Both directions are idempotent, and rollback must retain any existing data.
+    await removeUnusedV1Collections(migrationContext);
+    await restoreV1Collections(migrationContext);
+    await factory.db.current_data.insertOne({ _id: { g: 99, t: 'test', k: 'item' } } as any);
+    await restoreV1Collections(migrationContext);
+    expect(await names(factory)).toEqual(expect.arrayContaining(collectionNames));
+    expect(await factory.db.current_data.countDocuments({ '_id.g': 99 })).toBe(1);
+    expect((await factory.db.bucket_parameters.listIndexes().toArray()).map((index) => index.name)).toEqual(
+      expect.arrayContaining(['_id_', 'lookup1'])
+    );
+    expect((await factory.db.bucket_state.listIndexes().toArray()).map((index) => index.name)).toEqual(
+      expect.arrayContaining(['_id_', 'bucket_updates', 'dirty_count'])
+    );
+    expect((await factory.getActiveSyncConfig())?.replicationStream.replicationStreamId).toBe(
+      current.replicationStreamId
+    );
+  });
+
+  test('retries a partially completed collection drop', async () => {
+    await using factory = await testStorage.factory();
+    const previous = await activate(factory, storage.STORAGE_VERSION_2, '1/1');
+    await releaseTestStorageLease(factory, previous.replicationStreamId);
+    await activate(factory, storage.STORAGE_VERSION_4, '2/1');
+    const drop = vi.spyOn(factory.db.bucket_parameters, 'drop').mockRejectedValueOnce(new Error('drop interrupted'));
+    await expect(terminateStopped(factory)).rejects.toThrow('drop interrupted');
+    expect((await factory.db.sync_rules.findOne({ _id: previous.replicationStreamId }))?.state).toBe(
+      storage.SyncRuleState.STOP
+    );
+    await terminateStopped(factory);
+    for (const name of collectionNames) expect(await names(factory)).not.toContain(name);
+    drop.mockRestore();
+  });
+
+  test('serializes collection drops with deployment by another worker', async () => {
+    await using factory = await testStorage.factory();
+    await using other = await testStorage.factory({ doNotClear: true });
+    const entered = Promise.withResolvers<void>();
+    const resume = Promise.withResolvers<void>();
+    const originalDrop = factory.db.current_data.drop.bind(factory.db.current_data);
+    const drop = vi.spyOn(factory.db.current_data, 'drop').mockImplementation(async (options) => {
+      entered.resolve();
+      await resume.promise;
+      return originalDrop(options);
+    });
+    const cleanup = factory.dropUnusedV1StorageCollections();
+    await entered.promise;
+    let deployed = false;
+    const deployment = other.updateSyncRules(updateSyncRulesFromYaml(yaml, { storageVersion: 2 })).then(() => {
+      deployed = true;
+    });
+    try {
+      await new Promise((resolve) => setTimeout(resolve, 100));
+      expect(deployed).toBe(false);
+    } finally {
+      resume.resolve();
+      await cleanup;
+      drop.mockRestore();
+    }
+    await deployment;
+    expect(deployed).toBe(true);
+  }, 15_000);
+});