@powersync/service-module-mongodb-storage 0.24.2 → 0.25.0

package/src/migrations/db/migrations/1684951997326-init.ts

@@ -1,39 +1,6 @@
-import * as lib_mongo from '@powersync/lib-service-mongodb';
 import { migrations } from '@powersync/service-core';
-import * as storage from '../../../storage/storage-index.js';
-import { MongoStorageConfig } from '../../../types/types.js';
 
-export const up: migrations.PowerSyncMigrationFunction = async (context) => {
-  const {
-    service_context: { configuration }
-  } = context;
-  const database = storage.createPowerSyncMongo(configuration.storage as MongoStorageConfig);
-  await lib_mongo.waitForAuth(database.db);
-  try {
-    await database.bucket_parameters.createIndex(
-      {
-        'key.g': 1,
-        lookup: 1,
-        _id: 1
-      },
-      { name: 'lookup1' }
-    );
-  } finally {
-    await database.client.close();
-  }
-};
+// Indexes for v1/v2 data collections are managed by stream initialization.
+export const up: migrations.PowerSyncMigrationFunction = async () => {};
 
-export const down: migrations.PowerSyncMigrationFunction = async (context) => {
-  const {
-    service_context: { configuration }
-  } = context;
-
-  const database = storage.createPowerSyncMongo(configuration.storage as MongoStorageConfig);
-  try {
-    if (await database.bucket_parameters.indexExists('lookup')) {
-      await database.bucket_parameters.dropIndex('lookup1');
-    }
-  } finally {
-    await database.client.close();
-  }
-};
+export const down: migrations.PowerSyncMigrationFunction = async () => {};

package/src/migrations/db/migrations/1741697235857-bucket-state-index.ts

@@ -1,34 +1,6 @@
 import { migrations } from '@powersync/service-core';
-import * as storage from '../../../storage/storage-index.js';
-import { MongoStorageConfig } from '../../../types/types.js';
 
-const INDEX_NAME = 'bucket_updates';
+// Indexes for v1/v2 data collections are managed by stream initialization.
+export const up: migrations.PowerSyncMigrationFunction = async () => {};
 
-export const up: migrations.PowerSyncMigrationFunction = async (context) => {
-  const {
-    service_context: { configuration }
-  } = context;
-  const db = storage.createPowerSyncMongo(configuration.storage as MongoStorageConfig);
-
-  try {
-    await db.createBucketStateIndex();
-  } finally {
-    await db.client.close();
-  }
-};
-
-export const down: migrations.PowerSyncMigrationFunction = async (context) => {
-  const {
-    service_context: { configuration }
-  } = context;
-
-  const db = storage.createPowerSyncMongo(configuration.storage as MongoStorageConfig);
-
-  try {
-    if (await db.bucket_state.indexExists(INDEX_NAME)) {
-      await db.bucket_state.dropIndex(INDEX_NAME);
-    }
-  } finally {
-    await db.client.close();
-  }
-};
+export const down: migrations.PowerSyncMigrationFunction = async () => {};

package/src/migrations/db/migrations/1760433882550-bucket-state-index2.ts

@@ -1,34 +1,6 @@
 import { migrations } from '@powersync/service-core';
-import * as storage from '../../../storage/storage-index.js';
-import { MongoStorageConfig } from '../../../types/types.js';
 
-const INDEX_NAME = 'dirty_buckets';
+// Indexes for v1/v2 data collections are managed by stream initialization.
+export const up: migrations.PowerSyncMigrationFunction = async () => {};
 
-export const up: migrations.PowerSyncMigrationFunction = async (context) => {
-  const {
-    service_context: { configuration }
-  } = context;
-  const db = storage.createPowerSyncMongo(configuration.storage as MongoStorageConfig);
-
-  try {
-    await db.createBucketStateIndex2();
-  } finally {
-    await db.client.close();
-  }
-};
-
-export const down: migrations.PowerSyncMigrationFunction = async (context) => {
-  const {
-    service_context: { configuration }
-  } = context;
-
-  const db = storage.createPowerSyncMongo(configuration.storage as MongoStorageConfig);
-
-  try {
-    if (await db.bucket_state.indexExists(INDEX_NAME)) {
-      await db.bucket_state.dropIndex(INDEX_NAME);
-    }
-  } finally {
-    await db.client.close();
-  }
-};
+export const down: migrations.PowerSyncMigrationFunction = async () => {};

package/src/migrations/db/migrations/1794268800000-v1-storage-collections.ts

@@ -0,0 +1,22 @@
+import { migrations } from '@powersync/service-core';
+import { MongoBucketStorage, createPowerSyncMongo } from '../../../storage/storage-index.js';
+import { MongoStorageConfig } from '../../../types/types.js';
+
+export const up: migrations.PowerSyncMigrationFunction = async ({ service_context: { configuration } }) => {
+  const db = createPowerSyncMongo(configuration.storage as MongoStorageConfig);
+  try {
+    const factory = new MongoBucketStorage(db, { replicationStreamNamePrefix: configuration.slot_name_prefix });
+    await factory.dropUnusedV1StorageCollections();
+  } finally {
+    await db.client.close();
+  }
+};
+
+export const down: migrations.PowerSyncMigrationFunction = async ({ service_context: { configuration } }) => {
+  const db = createPowerSyncMongo(configuration.storage as MongoStorageConfig);
+  try {
+    await db.initializeV1StorageCollections();
+  } finally {
+    await db.client.close();
+  }
+};

package/src/storage/MongoBucketStorage.ts

@@ -576,7 +576,67 @@ export class MongoBucketStorage extends storage.BucketStorageFactory {
   }
 
   private async withTransaction<T>(callback: (session: mongo.ClientSession) => Promise<T>): Promise<T> {
-    return this.client.withSession((session) => session.withTransaction(callback));
+    // Serialize deployment with collection drops so newly deployed streams retain their storage.
+    return this.withStorageSchemaLock((refresh) =>
+      this.client.withSession((session) =>
+        session.withTransaction(async () => {
+          await refresh();
+          return callback(session);
+        })
+      )
+    );
+  }
+
+  private async withStorageSchemaLock<T>(callback: (refresh: () => Promise<void>) => Promise<T>): Promise<T> {
+    const name = 'v1-storage-collections';
+    // Seed by unique _id so concurrent first acquisitions share one lock document.
+    await this.db.db
+      .collection<{ _id: string; name: string }>('locks')
+      .updateOne({ _id: name }, { $setOnInsert: { name } }, { upsert: true });
+    const manager = new lib_mongo.locks.MongoLockManager({ collection: this.db.locks, name, timeout: 300_000 });
+    let result!: T;
+    await manager.lock(
+      async (refresh) => {
+        let lastRefresh = Date.now();
+        result = await callback(async () => {
+          // Rate-limit refresh writes; identical timestamps are not a modified document.
+          if (Date.now() - lastRefresh >= 30_000) {
+            await refresh();
+            lastRefresh = Date.now();
+          }
+        });
+      },
+      { max_wait_ms: 60_000 }
+    );
+    return result;
+  }
+
+  /** Returns true when no v1/v2 stream needs the shared data collections. */
+  async dropUnusedV1StorageCollections(signal?: AbortSignal): Promise<boolean> {
+    return this.withStorageSchemaLock(async (refresh) => {
+      signal?.throwIfAborted();
+      const live = await this.db.sync_rules.findOne(
+        {
+          $or: [{ storage_version: { $lt: storage.STORAGE_VERSION_3 } }, { storage_version: { $exists: false } }],
+          // Processing streams write data; errored streams still serve clients.
+          state: { $nin: [storage.SyncRuleState.STOP, storage.SyncRuleState.TERMINATED] }
+        },
+        { projection: { _id: 1 } }
+      );
+      if (live != null) return false;
+      for (const collection of this.db.v1DataCollections) {
+        signal?.throwIfAborted();
+        await refresh();
+        const dropped = await collection
+          .drop({ maxTimeMS: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS })
+          .catch((error) => {
+            if (!lib_mongo.isMongoNamespaceNotFoundError(error)) throw error;
+            return false;
+          });
+        if (dropped) logger.info(`Dropped unused v1/v2 storage collection: ${collection.collectionName}`);
+      }
+      return true;
+    });
   }
 
   /**

package/src/storage/implementation/MongoSyncBucketStorage.ts

@@ -405,11 +405,18 @@ export abstract class MongoSyncBucketStorage
     this.factory.discardOpIdAllocator(this.replicationStreamId);
     await this.clearSyncRuleState();
 
-    await this.clearBucketData(signal);
-    await this.clearParameterIndexes(signal);
-    await this.clearSourceRecords(signal);
-    await this.clearBucketState(signal);
-    await this.clearSourceTables(signal);
+    // Only stopped-stream cleanup may remove shared collections. Clearing an
+    // active or processing stream for a snapshot must not change shared storage.
+    const droppedV1Collections =
+      this.replicationStream.state == storage.SyncRuleState.STOP &&
+      (await this.factory.dropUnusedV1StorageCollections(signal));
+    if (this.storageConfig.version >= storage.STORAGE_VERSION_3 || !droppedV1Collections) {
+      await this.clearBucketData(signal);
+      await this.clearParameterIndexes(signal);
+      await this.clearSourceRecords(signal);
+      await this.clearBucketState(signal);
+      await this.clearSourceTables(signal);
+    }
     await this.clearCustomCheckpointRequests(signal);
 
     this.#storageInitialized = false;

package/src/storage/implementation/db.ts

@@ -178,6 +178,21 @@ export class PowerSyncMongo {
     return collections.map((c) => this.db.collection<CommonSourceTableDocument>(c.name));
   }
 
+  get v1DataCollections() {
+    return [this.current_data, this.bucket_data, this.bucket_parameters, this.bucket_state, this.source_tables];
+  }
+
+  async initializeV1StorageCollections(): Promise<void> {
+    for (const collection of this.v1DataCollections) {
+      await this.db.createCollection(collection.collectionName).catch((error) => {
+        if (!lib_mongo.isMongoServerError(error) || error.codeName !== 'NamespaceExists') throw error;
+      });
+    }
+    await this.bucket_parameters.createIndex({ 'key.g': 1, lookup: 1, _id: 1 }, { name: 'lookup1' });
+    await this.createBucketStateIndex();
+    await this.createBucketStateIndex2();
+  }
+
   /**
    * Clear all collections.
    */

package/src/storage/implementation/v1/VersionedPowerSyncMongoV1.ts

@@ -26,7 +26,9 @@ export class VersionedPowerSyncMongoV1 extends BaseVersionedPowerSyncMongo {
     return this.upstream.source_tables as any as mongo.Collection<SourceTableDocumentV1>;
   }
 
-  async initializeStreamStorage(_replicationStreamId: number): Promise<void> {}
+  async initializeStreamStorage(_replicationStreamId: number): Promise<void> {
+    await this.upstream.initializeV1StorageCollections();
+  }
 
   get bucketDataV1(): mongo.Collection<BucketDataDocumentV1> {
     return this.upstream.bucket_data;

package/src/utils/test-utils.ts

@@ -15,11 +15,6 @@ export function mongoTestStorageFactoryGenerator(factoryOptions: MongoTestStorag
     factory: async (options?: TestStorageOptions) => {
       const db = connectMongoForTests(factoryOptions.url, factoryOptions.isCI, factoryOptions.monitorCommands);
 
-      // None of the tests insert data into this collection, so it was never created
-      if (!(await db.db.listCollections({ name: db.bucket_parameters.collectionName }).hasNext())) {
-        await db.db.createCollection('bucket_parameters');
-      }
-
       // Full migrations are not currently run for tests, so we manually create this
       await db.createCheckpointEventsCollection();
 

package/test/src/v1_collections.test.ts

@@ -0,0 +1,233 @@
+import { MongoBucketStorage } from '@module/storage/MongoBucketStorage.js';
+import { mongoTestStorageFactoryGenerator } from '@module/utils/test-utils.js';
+import { storage, updateSyncRulesFromYaml } from '@powersync/service-core';
+import { getTestStorage, releaseTestStorageLease, test_utils } from '@powersync/service-core-tests';
+import { describe, expect, test, vi } from 'vitest';
+import { up as parameterIndexes } from '../../src/migrations/db/migrations/1684951997326-init.js';
+import { up as bucketIndexes } from '../../src/migrations/db/migrations/1741697235857-bucket-state-index.js';
+import { up as dirtyBucketIndexes } from '../../src/migrations/db/migrations/1760433882550-bucket-state-index2.js';
+import {
+  up as removeUnusedV1Collections,
+  down as restoreV1Collections
+} from '../../src/migrations/db/migrations/1794268800000-v1-storage-collections.js';
+import { env } from './env.js';
+
+const collectionNames = ['current_data', 'bucket_data', 'bucket_parameters', 'bucket_state', 'source_tables'];
+const migrationContext = {
+  service_context: {
+    configuration: { storage: { type: 'mongodb', uri: env.MONGO_TEST_URL }, slot_name_prefix: 'test_' }
+  }
+} as unknown as Parameters<typeof parameterIndexes>[0];
+const testStorage = mongoTestStorageFactoryGenerator({
+  url: env.MONGO_TEST_URL,
+  isCI: env.CI,
+  monitorCommands: true,
+  supportsMultipleSyncConfigs: true
+});
+const yaml = /* yaml */ `
+  # sync config fixture.
+  config:
+    edition: 3
+  streams:
+    global:
+      query: SELECT id FROM test
+`;
+
+async function activate(factory: MongoBucketStorage, version: number, lsn: string) {
+  const stream = await factory.updateSyncRules(updateSyncRulesFromYaml(yaml, { storageVersion: version }));
+  const bucketStorage = await getTestStorage(factory, stream);
+  await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
+  const sourceTable = await test_utils.resolveTestTable(
+    writer,
+    'test',
+    ['id'],
+    testStorage,
+    stream.replicationStreamId
+  );
+  await writer.save({
+    sourceTable,
+    tag: storage.SaveOperationTag.INSERT,
+    after: { id: 'item-1' },
+    afterReplicaId: test_utils.rid('item-1')
+  });
+  await writer.markAllSnapshotDone(lsn);
+  await writer.commit(lsn);
+  return stream;
+}
+
+async function names(factory: MongoBucketStorage) {
+  return (await factory.db.db.listCollections({}, { nameOnly: true }).toArray()).map((c) => c.name);
+}
+
+async function terminateStopped(factory: MongoBucketStorage) {
+  for (const stream of await factory.getStoppedReplicationStreams()) {
+    const lock = await stream.lock();
+    try {
+      await factory.getInstance(stream, { skipLifecycleHooks: true }).terminate({ clearStorage: true });
+    } finally {
+      await lock.release();
+    }
+  }
+}
+
+describe('v1/v2 collection lifecycle', () => {
+  test('migrations and v4 initialization do not create v1/v2 collections', async () => {
+    await using factory = await testStorage.factory();
+    await factory.db.drop();
+    await factory.db.createCheckpointEventsCollection();
+    for (const migration of [parameterIndexes, bucketIndexes, dirtyBucketIndexes, removeUnusedV1Collections]) {
+      await migration(migrationContext);
+    }
+    for (const name of collectionNames) expect(await names(factory)).not.toContain(name);
+    await activate(factory, storage.STORAGE_VERSION_4, '1/1');
+    for (const name of collectionNames) expect(await names(factory)).not.toContain(name);
+  });
+
+  test.each([1, 2])('drops unused v%i collections instead of deleting individual records', async (version) => {
+    await using factory = await testStorage.factory();
+    const previous = await activate(factory, version, '1/1');
+    await releaseTestStorageLease(factory, previous.replicationStreamId);
+    expect(await names(factory)).toEqual(expect.arrayContaining(collectionNames));
+    const current = await activate(factory, storage.STORAGE_VERSION_4, '2/1');
+    const dropped: string[] = [];
+    const deleted: string[] = [];
+    factory.db.client.on('commandStarted', (event) => {
+      if (event.commandName === 'drop') dropped.push(event.command.drop);
+      if (event.commandName === 'delete') deleted.push(event.command.delete);
+    });
+    await terminateStopped(factory);
+    expect(dropped).toEqual(expect.arrayContaining(collectionNames));
+    expect(deleted.filter((name) => collectionNames.includes(name))).toEqual([]);
+    for (const name of collectionNames) expect(await names(factory)).not.toContain(name);
+    expect((await factory.db.sync_rules.findOne({ _id: previous.replicationStreamId }))?.state).toBe(
+      storage.SyncRuleState.TERMINATED
+    );
+    expect((await factory.getActiveSyncConfig())?.replicationStream.replicationStreamId).toBe(
+      current.replicationStreamId
+    );
+
+    const returning = await activate(factory, version, '3/1');
+    await releaseTestStorageLease(factory, current.replicationStreamId);
+    await terminateStopped(factory);
+    expect(await names(factory)).toEqual(expect.arrayContaining(collectionNames));
+    expect((await factory.db.bucket_parameters.listIndexes().toArray()).map((index) => index.name)).toContain(
+      'lookup1'
+    );
+    expect((await factory.db.bucket_state.listIndexes().toArray()).map((index) => index.name)).toEqual(
+      expect.arrayContaining(['bucket_updates', 'dirty_count'])
+    );
+    expect(await factory.db.current_data.countDocuments({ '_id.g': returning.replicationStreamId })).toBe(1);
+  });
+
+  test.each([storage.SyncRuleState.PROCESSING, storage.SyncRuleState.ACTIVE, storage.SyncRuleState.ERRORED])(
+    'retains shared collections for a stream in state %s and deletes only stopped stream records',
+    async (state) => {
+      await using factory = await testStorage.factory();
+      const previous = await activate(factory, storage.STORAGE_VERSION_1, '1/1');
+      await releaseTestStorageLease(factory, previous.replicationStreamId);
+      const current = await activate(factory, storage.STORAGE_VERSION_2, '2/1');
+      await factory.db.sync_rules.updateOne({ _id: current.replicationStreamId }, { $set: { state } });
+      await terminateStopped(factory);
+      expect(await names(factory)).toEqual(expect.arrayContaining(collectionNames));
+      expect(await factory.db.current_data.countDocuments({ '_id.g': previous.replicationStreamId })).toBe(0);
+      expect(await factory.db.current_data.countDocuments({ '_id.g': current.replicationStreamId })).toBe(1);
+      // A missing storage_version also identifies a v1 stream.
+      await factory.db.sync_rules.updateOne({ _id: current.replicationStreamId }, { $unset: { storage_version: 1 } });
+      expect(await factory.dropUnusedV1StorageCollections()).toBe(false);
+      await removeUnusedV1Collections(migrationContext);
+      expect(await names(factory)).toEqual(expect.arrayContaining(collectionNames));
+      expect(await factory.db.current_data.countDocuments({ '_id.g': current.replicationStreamId })).toBe(1);
+    }
+  );
+
+  test('restarting and clearing an active v4 stream do not drop v1 collections', async () => {
+    await using factory = await testStorage.factory();
+    const previous = await activate(factory, storage.STORAGE_VERSION_2, '1/1');
+    await releaseTestStorageLease(factory, previous.replicationStreamId);
+    const current = await activate(factory, storage.STORAGE_VERSION_4, '2/1');
+    await factory.db.sync_rules.updateOne(
+      { _id: previous.replicationStreamId },
+      { $set: { state: storage.SyncRuleState.TERMINATED } }
+    );
+    await using restarted = await testStorage.factory({ doNotClear: true });
+    const drop = vi.spyOn(restarted, 'dropUnusedV1StorageCollections');
+    expect(await restarted.getStoppedReplicationStreams()).toEqual([]);
+    expect(drop).not.toHaveBeenCalled();
+    expect(await names(restarted)).toEqual(expect.arrayContaining(collectionNames));
+    await factory.getInstance(current).clear();
+    expect(await names(factory)).toEqual(expect.arrayContaining(collectionNames));
+  });
+
+  test('migration removes unused v1 collections and rollback recreates collections and indexes', async () => {
+    await using factory = await testStorage.factory();
+    const previous = await activate(factory, storage.STORAGE_VERSION_2, '1/1');
+    await releaseTestStorageLease(factory, previous.replicationStreamId);
+    const current = await activate(factory, storage.STORAGE_VERSION_4, '2/1');
+    await factory.db.sync_rules.updateOne(
+      { _id: previous.replicationStreamId },
+      { $set: { state: storage.SyncRuleState.TERMINATED } }
+    );
+    await removeUnusedV1Collections(migrationContext);
+    for (const name of collectionNames) expect(await names(factory)).not.toContain(name);
+    // Both directions are idempotent, and rollback must retain any existing data.
+    await removeUnusedV1Collections(migrationContext);
+    await restoreV1Collections(migrationContext);
+    await factory.db.current_data.insertOne({ _id: { g: 99, t: 'test', k: 'item' } } as any);
+    await restoreV1Collections(migrationContext);
+    expect(await names(factory)).toEqual(expect.arrayContaining(collectionNames));
+    expect(await factory.db.current_data.countDocuments({ '_id.g': 99 })).toBe(1);
+    expect((await factory.db.bucket_parameters.listIndexes().toArray()).map((index) => index.name)).toEqual(
+      expect.arrayContaining(['_id_', 'lookup1'])
+    );
+    expect((await factory.db.bucket_state.listIndexes().toArray()).map((index) => index.name)).toEqual(
+      expect.arrayContaining(['_id_', 'bucket_updates', 'dirty_count'])
+    );
+    expect((await factory.getActiveSyncConfig())?.replicationStream.replicationStreamId).toBe(
+      current.replicationStreamId
+    );
+  });
+
+  test('retries a partially completed collection drop', async () => {
+    await using factory = await testStorage.factory();
+    const previous = await activate(factory, storage.STORAGE_VERSION_2, '1/1');
+    await releaseTestStorageLease(factory, previous.replicationStreamId);
+    await activate(factory, storage.STORAGE_VERSION_4, '2/1');
+    const drop = vi.spyOn(factory.db.bucket_parameters, 'drop').mockRejectedValueOnce(new Error('drop interrupted'));
+    await expect(terminateStopped(factory)).rejects.toThrow('drop interrupted');
+    expect((await factory.db.sync_rules.findOne({ _id: previous.replicationStreamId }))?.state).toBe(
+      storage.SyncRuleState.STOP
+    );
+    await terminateStopped(factory);
+    for (const name of collectionNames) expect(await names(factory)).not.toContain(name);
+    drop.mockRestore();
+  });
+
+  test('serializes collection drops with deployment by another worker', async () => {
+    await using factory = await testStorage.factory();
+    await using other = await testStorage.factory({ doNotClear: true });
+    const entered = Promise.withResolvers<void>();
+    const resume = Promise.withResolvers<void>();
+    const originalDrop = factory.db.current_data.drop.bind(factory.db.current_data);
+    const drop = vi.spyOn(factory.db.current_data, 'drop').mockImplementation(async (options) => {
+      entered.resolve();
+      await resume.promise;
+      return originalDrop(options);
+    });
+    const cleanup = factory.dropUnusedV1StorageCollections();
+    await entered.promise;
+    let deployed = false;
+    const deployment = other.updateSyncRules(updateSyncRulesFromYaml(yaml, { storageVersion: 2 })).then(() => {
+      deployed = true;
+    });
+    try {
+      await new Promise((resolve) => setTimeout(resolve, 100));
+      expect(deployed).toBe(false);
+    } finally {
+      resume.resolve();
+      await cleanup;
+      drop.mockRestore();
+    }
+    await deployment;
+    expect(deployed).toBe(true);
+  }, 15_000);
+});