@powersync/service-module-mongodb-storage 0.19.0 → 0.21.0

package/test/src/storage_sync.test.ts

@@ -16,6 +16,7 @@ import { MongoPersistedSyncConfigContentV3 } from '../../src/storage/implementat
 import { BucketDataDoc, BucketKey } from '../../src/storage/implementation/common/BucketDataDoc.js';
 import { MongoSyncBucketStorage } from '../../src/storage/implementation/createMongoSyncBucketStorage.js';
 import { getMongoStorageConfig } from '../../src/storage/implementation/models.js';
+import { loadBucketDataDocumentV1, serializeBucketDataV1 } from '../../src/storage/implementation/v1/models.js';
 import { SourceRecordStoreV3 } from '../../src/storage/implementation/v3/SourceRecordStoreV3.js';
 import type { VersionedPowerSyncMongoV3 } from '../../src/storage/implementation/v3/VersionedPowerSyncMongoV3.js';
 import { serializeBucketData } from '../../src/storage/implementation/v3/bucket-format.js';
@@ -114,11 +115,65 @@ function bucketDefinitionId(bucket: string) {
   return parseInt(match[1], 16);
 }
 
+test('V1 bucket data retains an optional persisted subkey', () => {
+  const sourceTable = new bson.ObjectId();
+  const document = serializeBucketDataV1({
+    bucketKey: {
+      replicationStreamId: 1,
+      definitionId: '0',
+      bucket: 'global[]'
+    },
+    o: 1n,
+    op: 'PUT',
+    source_table: sourceTable,
+    source_key: 'source-key',
+    subkey: 'persisted-subkey',
+    table: 'items',
+    row_id: 'item-1',
+    checksum: 1n,
+    data: '{"id":"item-1"}'
+  });
+
+  expect(loadBucketDataDocumentV1(document).subkey).toBe('persisted-subkey');
+
+  const { subkey: _subkey, ...legacyDocument } = document;
+  expect(loadBucketDataDocumentV1(legacyDocument).subkey).toBeUndefined();
+});
+
 function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, storageVersion: number) {
   register.registerSyncTests(storageConfig.factory, {
     storageVersion,
     tableIdStrings: storageConfig.tableIdStrings
   });
+
+  test('updates source metadata on an existing resolved table', async () => {
+    await using factory = await storageConfig.factory();
+    const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(MINIMAL_SYNC_RULES, { storageVersion }));
+    const bucketStorage = factory.getInstance(syncRules);
+    await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
+    const source = sourceDescriptor('test');
+
+    const initial = await writer.resolveTables({ connection_id: 1, source });
+    expect(initial.tables[0].sourceMetadata).toBeNull();
+
+    const sourceMetadata = { captureTableObjectId: 42 };
+    const updated = await writer.resolveTables({
+      connection_id: 1,
+      source,
+      reconcileSourceTables: ({ candidates }) => ({
+        compatibleTables: candidates.map((candidate) => candidate.withSourceMetadata(sourceMetadata)),
+        incompatibleTables: [],
+        newTableValues: { sourceMetadata }
+      })
+    });
+
+    expect(updated.tables.map((table) => table.id.toString())).toEqual(
+      initial.tables.map((table) => table.id.toString())
+    );
+    expect(updated.tables[0].sourceMetadata).toEqual(sourceMetadata);
+    expect((await writer.getSourceTableStatus(updated.tables[0]))?.sourceMetadata).toEqual(sourceMetadata);
+  });
+
   // The split of returned results can vary depending on storage drivers
   test('large batch (2)', async () => {
     // Test syncing a batch of data that is small in count,
@@ -504,6 +559,7 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor
     const source = sourceDescriptor('memberships', { objectId: 'memberships-relation' });
     const dataOnlyTableId = new bson.ObjectId('6544e3899293153fa7b38348');
     const addedParameterTableId = new bson.ObjectId('6544e3899293153fa7b38349');
+    const addedEventTableId = new bson.ObjectId('6544e3899293153fa7b3834a');
 
     const dataOnly = await writer.resolveTables({
       connection_id: 1,
@@ -561,15 +617,13 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor
     const eventOnly = await writer.resolveTables({
       connection_id: 1,
       source,
-      idGenerator: () => {
-        throw new Error('resolve should reuse existing v3 source table');
-      },
+      idGenerator: () => addedEventTableId,
       parsedSyncConfig: eventOnlyRules
     });
 
-    // Event-only table can re-use any existing table.
+    // A newly-added event gets its own source table so that its existing rows are snapshotted.
     expect(eventOnly.tables).toHaveLength(1);
-    expect([dataOnlyTableId.toString(), addedParameterTableId.toString()]).toContain(eventOnly.tables[0].id.toString());
+    expect(eventOnly.tables[0].id).toEqual(addedEventTableId);
     expect(eventOnly.dropTables.map((table) => table.id)).toEqual([]);
     expect(eventOnly.tables[0].bucketDataSources).toHaveLength(0);
     expect(eventOnly.tables[0].parameterLookupSources).toHaveLength(0);
@@ -578,13 +632,12 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor
     expect(eventOnly.tables[0].syncEvent).toBe(true);
   });
 
-  test.runIf(storageVersion >= 3)(
-    'resolveTables designates a single event carrier across split source tables',
-    async () => {
-      // When memberships are split over multiple SourceTables for the same ref, a row change
-      // is saved once per table. Only one table may fire events, otherwise the same event
-      // would fire once per table for every row change.
-      const dataOnlyEventYaml = `
+  test.runIf(storageVersion >= 3)('resolveTables assigns an event definition to one split source table', async () => {
+    // When memberships are split over multiple SourceTables for the same ref, a row change
+    // is saved once per table. Each event-definition id must belong to only one of those tables,
+    // otherwise that definition would be evaluated more than once for the same row change.
+    // Other tables may independently own and evaluate different event definitions.
+    const dataOnlyEventYaml = `
     bucket_definitions:
       by_owner:
         parameters:
@@ -597,7 +650,7 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor
         payloads:
           - SELECT id, owner_id FROM memberships
     `;
-      const fullEventYaml = `
+    const fullEventYaml = `
     bucket_definitions:
       by_owner:
         parameters:
@@ -611,42 +664,201 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor
           - SELECT id, owner_id FROM memberships
     `;
 
+    await using factory = await storageConfig.factory();
+    const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(dataOnlyEventYaml, { storageVersion }));
+    const bucketStorage = factory.getInstance(syncRules) as MongoSyncBucketStorage;
+    await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
+    const dataOnlyRules = parsedSyncConfigSetFor(dataOnlyEventYaml, storageVersion);
+    const fullRules = parsedSyncConfigSetFor(fullEventYaml, storageVersion);
+    const source = sourceDescriptor('memberships', { objectId: 'memberships-relation' });
+
+    const initial = await writer.resolveTables({
+      connection_id: 1,
+      source,
+      idGenerator: objectIdGenerator('6544e3899293153fa7b3834b'),
+      parsedSyncConfig: dataOnlyRules
+    });
+    expect(initial.tables).toHaveLength(1);
+    expect(initial.tables[0].syncEvent).toBe(true);
+
+    // Adding the table-based parameter lookup creates a second SourceTable for the same ref.
+    const split = await writer.resolveTables({
+      connection_id: 1,
+      source,
+      idGenerator: objectIdGenerator('6544e3899293153fa7b3834c'),
+      parsedSyncConfig: fullRules
+    });
+    expect(split.tables).toHaveLength(2);
+    // Both tables match the event by ref, but this event id belongs to only one of them.
+    const eventOwners = split.tables.filter((table) => table.syncEvent);
+    expect(eventOwners).toHaveLength(1);
+
+    // getSourceTableStatus rehydrates the persisted event memberships rather than recomputing
+    // them from the ref, so refreshing the other table does not assign this event to it.
+    const nonOwner = split.tables.find((table) => !table.syncEvent)!;
+    const refreshed = await writer.getSourceTableStatus(nonOwner);
+    expect(refreshed!.syncEvent).toBe(false);
+  });
+
+  test.runIf(storageVersion >= 3)(
+    'reuses an unchanged event definition without resnapshotting or duplicate evaluation',
+    async () => {
+      const firstYaml = `
+config:
+  edition: 3
+
+streams:
+  by_owner:
+    query: SELECT * FROM todos WHERE owner_id = subscription.parameter('owner_id')
+
+event_definitions:
+  write_checkpoints:
+    payloads:
+      - SELECT user_id, checkpoint FROM checkpoints WHERE active = true AND checkpoint > 0
+`;
+      const secondYaml = `
+config:
+  edition: 3
+
+streams:
+  by_owner:
+    query: SELECT * FROM todos WHERE owner_id = subscription.parameter('owner_id')
+  by_project:
+    query: SELECT * FROM todos WHERE project_id = subscription.parameter('project_id')
+
+event_definitions:
+  write_checkpoints:
+    payloads:
+      - select "user_id", "checkpoint" from "checkpoints" where "active" = true and "checkpoint" > 0
+`;
+
       await using factory = await storageConfig.factory();
-      const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(dataOnlyEventYaml, { storageVersion }));
-      const bucketStorage = factory.getInstance(syncRules) as MongoSyncBucketStorage;
-      await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
-      const dataOnlyRules = parsedSyncConfigSetFor(dataOnlyEventYaml, storageVersion);
-      const fullRules = parsedSyncConfigSetFor(fullEventYaml, storageVersion);
-      const source = sourceDescriptor('memberships', { objectId: 'memberships-relation' });
+      const emittedEvents: { name: string; eventId: string | undefined }[] = [];
+      const disposeListener = factory.registerListener({
+        replicationEvent: ({ event, event_id }) => emittedEvents.push({ name: event.name, eventId: event_id })
+      });
+      try {
+        const first = await factory.updateSyncRules(updateSyncRulesFromYaml(firstYaml, { storageVersion }));
+        const firstStorage = factory.getInstance(first) as MongoSyncBucketStorage;
+        await using firstWriter = await firstStorage.createWriter(test_utils.BATCH_OPTIONS);
+        const source = sourceDescriptor('checkpoints', { objectId: 'checkpoints-relation' });
+        const firstResolved = await firstWriter.resolveTables({
+          connection_id: 1,
+          source,
+          idGenerator: objectIdGenerator('6544e3899293153fa7b38351')
+        });
+        const eventId = [...firstResolved.tables[0].eventDefinitionIds!][0];
+        expect(eventId).toBeDefined();
+        await firstWriter.markTableSnapshotDone(firstResolved.tables, '1/1');
+        await firstWriter.markAllSnapshotDone('1/1');
+        await firstWriter.commit('1/1');
+
+        const second = await factory.updateSyncRules(updateSyncRulesFromYaml(secondYaml, { storageVersion }));
+        expect(second.replicationStreamId).toBe(first.replicationStreamId);
+        const secondStorage = factory.getInstance(second) as MongoSyncBucketStorage;
+        await using secondWriter = await secondStorage.createWriter(test_utils.BATCH_OPTIONS);
+        const resolved = await secondWriter.resolveTables({
+          connection_id: 1,
+          source,
+          idGenerator: () => {
+            throw new Error('unchanged event definition should reuse the snapshotted source table');
+          }
+        });
+
+        expect(resolved.tables).toHaveLength(1);
+        expect(resolved.tables[0].id).toEqual(firstResolved.tables[0].id);
+        expect(resolved.tables[0].snapshotComplete).toBe(true);
+        expect([...resolved.tables[0].eventDefinitionIds!]).toEqual([eventId]);
+
+        emittedEvents.length = 0;
+        await secondWriter.save({
+          sourceTable: resolved.tables[0],
+          tag: storage.SaveOperationTag.INSERT,
+          after: { id: 'checkpoint-1', user_id: 'user-1', checkpoint: 1n, active: 1 },
+          afterReplicaId: test_utils.rid('checkpoint-1')
+        });
+        expect(emittedEvents).toEqual([{ name: 'write_checkpoints', eventId }]);
+      } finally {
+        disposeListener();
+      }
+    }
+  );
 
-      const initial = await writer.resolveTables({
+  test.runIf(storageVersion >= 3)('creates snapshot work for a changed event definition', async () => {
+    const yaml = (active: boolean) => `
+config:
+  edition: 3
+
+streams:
+  by_owner:
+    query: SELECT * FROM todos WHERE owner_id = subscription.parameter('owner_id')
+
+event_definitions:
+  write_checkpoints:
+    payloads:
+      - SELECT user_id, checkpoint FROM checkpoints WHERE active = ${active}
+`;
+
+    await using factory = await storageConfig.factory();
+    const emittedEvents: { name: string; eventId: string | undefined }[] = [];
+    const disposeListener = factory.registerListener({
+      replicationEvent: ({ event, event_id }) => emittedEvents.push({ name: event.name, eventId: event_id })
+    });
+    try {
+      const first = await factory.updateSyncRules(updateSyncRulesFromYaml(yaml(true), { storageVersion }));
+      const firstStorage = factory.getInstance(first) as MongoSyncBucketStorage;
+      await using firstWriter = await firstStorage.createWriter(test_utils.BATCH_OPTIONS);
+      const source = sourceDescriptor('checkpoints', { objectId: 'checkpoints-relation' });
+      const firstResolved = await firstWriter.resolveTables({
         connection_id: 1,
         source,
-        idGenerator: objectIdGenerator('6544e3899293153fa7b3834b'),
-        parsedSyncConfig: dataOnlyRules
+        idGenerator: objectIdGenerator('6544e3899293153fa7b38352')
       });
-      expect(initial.tables).toHaveLength(1);
-      expect(initial.tables[0].syncEvent).toBe(true);
+      const oldEventId = [...firstResolved.tables[0].eventDefinitionIds!][0];
+      await firstWriter.markTableSnapshotDone(firstResolved.tables, '1/1');
+      await firstWriter.markAllSnapshotDone('1/1');
+      await firstWriter.commit('1/1');
 
-      // Adding the table-based parameter lookup creates a second SourceTable for the same ref.
-      const split = await writer.resolveTables({
+      const second = await factory.updateSyncRules(updateSyncRulesFromYaml(yaml(false), { storageVersion }));
+      expect(second.replicationStreamId).toBe(first.replicationStreamId);
+      const secondStorage = factory.getInstance(second) as MongoSyncBucketStorage;
+      await using secondWriter = await secondStorage.createWriter(test_utils.BATCH_OPTIONS);
+      const resolved = await secondWriter.resolveTables({
         connection_id: 1,
         source,
-        idGenerator: objectIdGenerator('6544e3899293153fa7b3834c'),
-        parsedSyncConfig: fullRules
+        idGenerator: objectIdGenerator('6544e3899293153fa7b38353')
       });
-      expect(split.tables).toHaveLength(2);
-      // Both tables match the event by ref, but only one may carry it.
-      const carriers = split.tables.filter((table) => table.syncEvent);
-      expect(carriers).toHaveLength(1);
-
-      // getSourceTableStatus preserves the carrier designation rather than recomputing it
-      // from the ref, so refreshing a non-carrier table does not make it fire events.
-      const nonCarrier = split.tables.find((table) => !table.syncEvent)!;
-      const refreshed = await writer.getSourceTableStatus(nonCarrier);
-      expect(refreshed!.syncEvent).toBe(false);
+
+      expect(resolved.tables).toHaveLength(2);
+      const oldTable = resolved.tables.find((table) => table.eventDefinitionIds?.has(oldEventId))!;
+      const newTable = resolved.tables.find((table) => !table.eventDefinitionIds?.has(oldEventId))!;
+      const newEventId = [...newTable.eventDefinitionIds!][0];
+      expect(newEventId).toBeDefined();
+      expect(newEventId).not.toBe(oldEventId);
+      expect(oldTable.snapshotComplete).toBe(true);
+      expect(newTable.snapshotComplete).toBe(false);
+      expect(newTable.bucketDataSources).toEqual([]);
+      expect(newTable.parameterLookupSources).toEqual([]);
+
+      await expect(secondWriter.markSnapshotDone('2/1')).rejects.toThrow(
+        'Cannot mark snapshot done while source tables still require snapshotting'
+      );
+
+      emittedEvents.length = 0;
+      await secondWriter.save({
+        sourceTable: newTable,
+        tag: storage.SaveOperationTag.INSERT,
+        after: { id: 'checkpoint-2', user_id: 'user-2', checkpoint: 2n, active: 0 },
+        afterReplicaId: test_utils.rid('checkpoint-2')
+      });
+      expect(emittedEvents).toEqual([{ name: 'write_checkpoints', eventId: newEventId }]);
+
+      await secondWriter.markTableSnapshotDone([newTable], '2/1');
+      await secondWriter.markSnapshotDone('2/1');
+    } finally {
+      disposeListener();
     }
-  );
+  });
 
   test.runIf(storageVersion >= 3)('uses v3 mongodb model shapes', async () => {
     await using factory = await storageConfig.factory();
@@ -1130,6 +1342,100 @@ streams:
     }
   );
 
+  test.runIf(storageVersion >= 3)(
+    'restart creates a replacement stream instead of appending to the active stream',
+    async () => {
+      await using factory = await storageConfig.factory();
+
+      const active = await factory.updateSyncRules(
+        updateSyncRulesFromYaml(
+          `
+config:
+  edition: 3
+
+streams:
+  by_owner:
+    query: SELECT * FROM todos WHERE owner_id = subscription.parameter('owner_id')
+`,
+          { storageVersion }
+        )
+      );
+      const activeStorage = factory.getInstance(active) as MongoSyncBucketStorage;
+      await using activeWriter = await activeStorage.createWriter(test_utils.BATCH_OPTIONS);
+      await activeWriter.markAllSnapshotDone('1/1');
+      await activeWriter.commit('1/1');
+
+      await factory.restartReplication(active.replicationStreamId);
+
+      const replicatingStreams = await factory.getReplicatingReplicationStreams();
+      expect(replicatingStreams).toHaveLength(1);
+      expect(replicatingStreams[0].replicationStreamId).not.toEqual(active.replicationStreamId);
+      expect(replicatingStreams[0].state).toBe(storage.SyncRuleState.PROCESSING);
+
+      const oldStream = (await (factory as MongoBucketStorage).db.sync_rules.findOne({
+        _id: active.replicationStreamId
+      })) as ReplicationStreamDocumentV3;
+      expect(oldStream.state).toBe(storage.SyncRuleState.ERRORED);
+      expect(oldStream.sync_configs.map((config) => config.state)).toEqual([storage.SyncRuleState.ERRORED]);
+    }
+  );
+
+  test.runIf(storageVersion >= 3)('restart errors an active stream with an embedded deploying config', async () => {
+    await using factory = await storageConfig.factory();
+
+    const active = await factory.updateSyncRules(
+      updateSyncRulesFromYaml(
+        `
+config:
+  edition: 3
+
+streams:
+  by_owner:
+    query: SELECT * FROM todos WHERE owner_id = subscription.parameter('owner_id')
+`,
+        { storageVersion }
+      )
+    );
+    const activeStorage = factory.getInstance(active) as MongoSyncBucketStorage;
+    await using activeWriter = await activeStorage.createWriter(test_utils.BATCH_OPTIONS);
+    await activeWriter.markAllSnapshotDone('1/1');
+    await activeWriter.commit('1/1');
+
+    const deploying = await factory.updateSyncRules(
+      updateSyncRulesFromYaml(
+        `
+config:
+  edition: 3
+
+streams:
+  by_project:
+    query: SELECT * FROM todos WHERE project_id = subscription.parameter('project_id')
+`,
+        { storageVersion }
+      )
+    );
+    expect(deploying.replicationStreamId).toEqual(active.replicationStreamId);
+
+    await factory.restartReplication(active.replicationStreamId);
+
+    const replicatingStreams = await factory.getReplicatingReplicationStreams();
+    expect(replicatingStreams).toHaveLength(1);
+    expect(replicatingStreams[0].replicationStreamId).not.toEqual(active.replicationStreamId);
+
+    const oldStream = (await (factory as MongoBucketStorage).db.sync_rules.findOne({
+      _id: active.replicationStreamId
+    })) as ReplicationStreamDocumentV3;
+    expect(oldStream.state).toBe(storage.SyncRuleState.ERRORED);
+    expect(oldStream.sync_configs.map((config) => config.state)).toEqual([
+      storage.SyncRuleState.ERRORED,
+      storage.SyncRuleState.STOP
+    ]);
+    const oldActive = await factory.getActiveSyncConfig();
+    expect(oldActive).not.toBeNull();
+    expect(oldActive!.replicationStream.syncConfigContent).toHaveLength(1);
+    await expect(oldActive!.storage.getCheckpoint()).resolves.toBeDefined();
+  });
+
   test.runIf(storageVersion >= 3)(
     'appended compatible config adopts the stream checkpoint instead of regressing to 0',
     async () => {
@@ -1595,6 +1901,22 @@ streams:
     expect(metricsAfter.replication_size_bytes).toBeGreaterThan(metricsBefore.replication_size_bytes);
   });
 
+  test.runIf(storageVersion >= 3)('storage metrics include the sync config version label', async () => {
+    await using factory = await storageConfig.factory();
+    const syncRules = await factory.updateSyncRules(
+      updateSyncRulesFromYaml(MINIMAL_SYNC_RULES, { storageVersion, version_label: 'metrics-test' })
+    );
+
+    const metrics = await factory.getStorageMetrics();
+
+    expect(metrics.sync_config_metrics).toContainEqual(
+      expect.objectContaining({
+        sync_config_id: syncRules.syncConfigContent[0].syncConfigId,
+        version_label: 'metrics-test'
+      })
+    );
+  });
+
   test.runIf(storageVersion >= 3)(
     'loads parameter checkpoint changes across all v3 parameter index collections',
     async () => {
@@ -1647,7 +1969,7 @@ streams:
     }
   );
 
-  test.runIf(storageVersion == 3)('cleans pending deletes only for tracked v3 source tables', async () => {
+  test.runIf(storageVersion >= 3)('cleans pending deletes only for tracked v3 source tables', async () => {
     await using factory = await storageConfig.factory();
     const syncRules = await factory.updateSyncRules(
       updateSyncRulesFromYaml(
@@ -1680,6 +2002,7 @@ streams:
         snapshot_status: undefined,
         bucket_data_source_ids: [],
         parameter_lookup_source_ids: [],
+        event_definition_ids: [],
         latest_pending_delete: 9n
       },
       {
@@ -1693,6 +2016,7 @@ streams:
         snapshot_status: undefined,
         bucket_data_source_ids: [],
         parameter_lookup_source_ids: [],
+        event_definition_ids: [],
         latest_pending_delete: 12n
       }
     ]);
@@ -1781,7 +2105,7 @@ describe('sync - mongodb', () => {
     describe(`storage v${storageVersion}`, () => {
       registerSyncStorageTests(INITIALIZED_MONGO_STORAGE_FACTORY, storageVersion);
 
-      describe.runIf(storageVersion == 3)('V3 read filtering boundaries', () => {
+      describe.runIf(storageVersion >= 3)('V3 read filtering boundaries', () => {
         async function setupFilteringTest() {
           await using factory = await INITIALIZED_MONGO_STORAGE_FACTORY.factory();
           const syncRules = await factory.updateSyncRules(

package/test/src/util.ts

@@ -13,4 +13,4 @@ export const INITIALIZED_MONGO_REPORT_STORAGE_FACTORY = mongoTestReportStorageFa
   isCI: env.CI
 });
 
-export const TEST_STORAGE_VERSIONS = SUPPORTED_STORAGE_VERSIONS;
+export const TEST_STORAGE_VERSIONS = SUPPORTED_STORAGE_VERSIONS.filter((v) => v != 3);