@powersync/service-module-mongodb-storage 0.16.0 → 0.17.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (102) hide show
  1. package/CHANGELOG.md +34 -0
  2. package/dist/storage/MongoBucketStorage.d.ts +6 -4
  3. package/dist/storage/MongoBucketStorage.js +110 -36
  4. package/dist/storage/MongoBucketStorage.js.map +1 -1
  5. package/dist/storage/implementation/BucketDefinitionMapping.d.ts +4 -6
  6. package/dist/storage/implementation/BucketDefinitionMapping.js +3 -3
  7. package/dist/storage/implementation/BucketDefinitionMapping.js.map +1 -1
  8. package/dist/storage/implementation/CheckpointState.d.ts +20 -0
  9. package/dist/storage/implementation/CheckpointState.js +31 -0
  10. package/dist/storage/implementation/CheckpointState.js.map +1 -0
  11. package/dist/storage/implementation/MongoBucketBatch.d.ts +33 -22
  12. package/dist/storage/implementation/MongoBucketBatch.js +45 -271
  13. package/dist/storage/implementation/MongoBucketBatch.js.map +1 -1
  14. package/dist/storage/implementation/MongoChecksums.d.ts +2 -1
  15. package/dist/storage/implementation/MongoChecksums.js.map +1 -1
  16. package/dist/storage/implementation/MongoCompactor.d.ts +1 -1
  17. package/dist/storage/implementation/MongoPersistedSyncRules.d.ts +4 -4
  18. package/dist/storage/implementation/MongoPersistedSyncRules.js +11 -8
  19. package/dist/storage/implementation/MongoPersistedSyncRules.js.map +1 -1
  20. package/dist/storage/implementation/MongoPersistedSyncRulesContent.d.ts +19 -5
  21. package/dist/storage/implementation/MongoPersistedSyncRulesContent.js +53 -19
  22. package/dist/storage/implementation/MongoPersistedSyncRulesContent.js.map +1 -1
  23. package/dist/storage/implementation/MongoSyncBucketStorage.d.ts +21 -10
  24. package/dist/storage/implementation/MongoSyncBucketStorage.js +18 -163
  25. package/dist/storage/implementation/MongoSyncBucketStorage.js.map +1 -1
  26. package/dist/storage/implementation/MongoSyncRulesLock.d.ts +5 -1
  27. package/dist/storage/implementation/MongoSyncRulesLock.js +7 -3
  28. package/dist/storage/implementation/MongoSyncRulesLock.js.map +1 -1
  29. package/dist/storage/implementation/SyncRuleStateUpdate.d.ts +14 -0
  30. package/dist/storage/implementation/SyncRuleStateUpdate.js +36 -0
  31. package/dist/storage/implementation/SyncRuleStateUpdate.js.map +1 -0
  32. package/dist/storage/implementation/common/BucketDataDoc.d.ts +1 -1
  33. package/dist/storage/implementation/common/PersistedBatch.d.ts +2 -2
  34. package/dist/storage/implementation/common/SourceRecordStore.d.ts +1 -2
  35. package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.d.ts +1 -1
  36. package/dist/storage/implementation/createMongoSyncBucketStorage.d.ts +2 -2
  37. package/dist/storage/implementation/createMongoSyncBucketStorage.js.map +1 -1
  38. package/dist/storage/implementation/db.d.ts +10 -2
  39. package/dist/storage/implementation/db.js.map +1 -1
  40. package/dist/storage/implementation/models.d.ts +31 -47
  41. package/dist/storage/implementation/models.js.map +1 -1
  42. package/dist/storage/implementation/v1/MongoBucketBatchV1.d.ts +15 -1
  43. package/dist/storage/implementation/v1/MongoBucketBatchV1.js +385 -0
  44. package/dist/storage/implementation/v1/MongoBucketBatchV1.js.map +1 -1
  45. package/dist/storage/implementation/v1/MongoCompactorV1.d.ts +1 -1
  46. package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.d.ts +16 -7
  47. package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js +77 -6
  48. package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js.map +1 -1
  49. package/dist/storage/implementation/v1/PersistedBatchV1.d.ts +1 -2
  50. package/dist/storage/implementation/v1/PersistedBatchV1.js.map +1 -1
  51. package/dist/storage/implementation/v1/models.d.ts +12 -1
  52. package/dist/storage/implementation/v1/models.js.map +1 -1
  53. package/dist/storage/implementation/v3/MongoBucketBatchV3.d.ts +17 -0
  54. package/dist/storage/implementation/v3/MongoBucketBatchV3.js +429 -0
  55. package/dist/storage/implementation/v3/MongoBucketBatchV3.js.map +1 -1
  56. package/dist/storage/implementation/v3/MongoCompactorV3.d.ts +1 -1
  57. package/dist/storage/implementation/v3/MongoParameterLookupV3.d.ts +1 -2
  58. package/dist/storage/implementation/v3/MongoParameterLookupV3.js.map +1 -1
  59. package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.d.ts +29 -7
  60. package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js +117 -16
  61. package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js.map +1 -1
  62. package/dist/storage/implementation/v3/PersistedBatchV3.d.ts +1 -2
  63. package/dist/storage/implementation/v3/PersistedBatchV3.js.map +1 -1
  64. package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.d.ts +3 -2
  65. package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.js +3 -0
  66. package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.js.map +1 -1
  67. package/dist/storage/implementation/v3/models.d.ts +61 -3
  68. package/dist/storage/implementation/v3/models.js.map +1 -1
  69. package/package.json +6 -6
  70. package/src/migrations/db/migrations/1702295701188-sync-rule-state.ts +1 -1
  71. package/src/storage/MongoBucketStorage.ts +166 -44
  72. package/src/storage/implementation/BucketDefinitionMapping.ts +12 -9
  73. package/src/storage/implementation/CheckpointState.ts +59 -0
  74. package/src/storage/implementation/MongoBucketBatch.ts +81 -355
  75. package/src/storage/implementation/MongoChecksums.ts +2 -1
  76. package/src/storage/implementation/MongoCompactor.ts +1 -1
  77. package/src/storage/implementation/MongoPersistedSyncRules.ts +13 -7
  78. package/src/storage/implementation/MongoPersistedSyncRulesContent.ts +69 -24
  79. package/src/storage/implementation/MongoSyncBucketStorage.ts +40 -215
  80. package/src/storage/implementation/MongoSyncRulesLock.ts +9 -3
  81. package/src/storage/implementation/SyncRuleStateUpdate.ts +38 -0
  82. package/src/storage/implementation/common/BucketDataDoc.ts +1 -1
  83. package/src/storage/implementation/common/PersistedBatch.ts +2 -2
  84. package/src/storage/implementation/common/SourceRecordStore.ts +1 -2
  85. package/src/storage/implementation/createMongoSyncBucketStorage.ts +2 -2
  86. package/src/storage/implementation/db.ts +5 -2
  87. package/src/storage/implementation/models.ts +35 -58
  88. package/src/storage/implementation/v1/MongoBucketBatchV1.ts +478 -1
  89. package/src/storage/implementation/v1/MongoCompactorV1.ts +1 -1
  90. package/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts +111 -16
  91. package/src/storage/implementation/v1/PersistedBatchV1.ts +1 -2
  92. package/src/storage/implementation/v1/models.ts +15 -0
  93. package/src/storage/implementation/v3/MongoBucketBatchV3.ts +564 -1
  94. package/src/storage/implementation/v3/MongoCompactorV3.ts +1 -1
  95. package/src/storage/implementation/v3/MongoParameterLookupV3.ts +1 -2
  96. package/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts +150 -22
  97. package/src/storage/implementation/v3/PersistedBatchV3.ts +1 -2
  98. package/src/storage/implementation/v3/VersionedPowerSyncMongoV3.ts +7 -2
  99. package/src/storage/implementation/v3/models.ts +70 -2
  100. package/test/src/storage_sync.test.ts +422 -6
  101. package/test/src/storeCurrentData.test.ts +211 -0
  102. package/tsconfig.tsbuildinfo +1 -1
@@ -1,16 +1,64 @@
1
1
  import { deserializeParameterLookup, JwtPayload, storage, updateSyncRulesFromYaml } from '@powersync/service-core';
2
2
  import { bucketRequest, register, test_utils } from '@powersync/service-core-tests';
3
- import { RequestParameters } from '@powersync/service-sync-rules';
3
+ import { DEFAULT_HYDRATION_STATE, nodeSqlite, RequestParameters, SqlSyncRules } from '@powersync/service-sync-rules';
4
4
  import * as bson from 'bson';
5
+ import * as sqlite from 'node:sqlite';
5
6
  import { describe, expect, test } from 'vitest';
6
7
  import { MongoBucketStorage } from '../../src/storage/MongoBucketStorage.js';
7
8
  import { MongoSyncBucketStorage } from '../../src/storage/implementation/createMongoSyncBucketStorage.js';
8
- import { SyncRuleDocument } from '../../src/storage/implementation/models.js';
9
9
  import { SourceRecordStoreV3 } from '../../src/storage/implementation/v3/SourceRecordStoreV3.js';
10
10
  import type { VersionedPowerSyncMongoV3 } from '../../src/storage/implementation/v3/VersionedPowerSyncMongoV3.js';
11
- import { CurrentBucketV3 } from '../../src/storage/implementation/v3/models.js';
11
+ import {
12
+ CurrentBucketV3,
13
+ ReplicationStreamDocumentV3,
14
+ SyncConfigDefinition
15
+ } from '../../src/storage/implementation/v3/models.js';
12
16
  import { INITIALIZED_MONGO_STORAGE_FACTORY, TEST_STORAGE_VERSIONS } from './util.js';
13
17
 
18
+ const MINIMAL_SYNC_RULES = `
19
+ bucket_definitions:
20
+ global:
21
+ data:
22
+ - SELECT id FROM test
23
+ `;
24
+
25
+ function sourceDescriptor(
26
+ name: string,
27
+ options: {
28
+ objectId?: string;
29
+ replicaIdColumns?: string[];
30
+ } = {}
31
+ ): storage.SourceEntityDescriptor {
32
+ return {
33
+ connectionTag: storage.SourceTable.DEFAULT_TAG,
34
+ objectId: options.objectId ?? name,
35
+ schema: 'public',
36
+ name,
37
+ replicaIdColumns: (options.replicaIdColumns ?? ['id']).map((column) => ({
38
+ name: column,
39
+ type: 'VARCHAR',
40
+ typeId: 25
41
+ }))
42
+ };
43
+ }
44
+
45
+ function objectIdGenerator(id: string) {
46
+ let used = false;
47
+ return () => {
48
+ if (used) {
49
+ throw new Error(`Can only generate a single id using ${id}`);
50
+ }
51
+ used = true;
52
+ return new bson.ObjectId(id);
53
+ };
54
+ }
55
+
56
+ function hydratedRulesFor(yaml: string) {
57
+ const parsed = SqlSyncRules.fromYaml(yaml, test_utils.PARSE_OPTIONS);
58
+ expect(parsed.errors).toEqual([]);
59
+ return parsed.config.hydrate({ hydrationState: DEFAULT_HYDRATION_STATE, sqlite: nodeSqlite(sqlite) });
60
+ }
61
+
14
62
  function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, storageVersion: number) {
15
63
  register.registerSyncTests(storageConfig.factory, {
16
64
  storageVersion,
@@ -157,6 +205,316 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor
157
205
  expect(checksumTypes).toEqual([{ _id: 'long', count: 4 }]);
158
206
  });
159
207
 
208
+ test('resolveTables populates matching data and parameter sources', async () => {
209
+ await using factory = await storageConfig.factory();
210
+ const syncRules = await factory.updateSyncRules(
211
+ updateSyncRulesFromYaml(
212
+ `
213
+ bucket_definitions:
214
+ by_owner:
215
+ parameters:
216
+ - SELECT owner_id FROM test WHERE id = token_parameters.test_id
217
+ data:
218
+ - SELECT id, owner_id FROM test WHERE owner_id = bucket.owner_id
219
+ `,
220
+ { storageVersion }
221
+ )
222
+ );
223
+ const bucketStorage = factory.getInstance(syncRules);
224
+
225
+ await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
226
+ const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], INITIALIZED_MONGO_STORAGE_FACTORY);
227
+
228
+ expect(sourceTable.bucketDataSources).toHaveLength(1);
229
+ expect(sourceTable.parameterLookupSources).toHaveLength(1);
230
+ expect(sourceTable.syncData).toBe(true);
231
+ expect(sourceTable.syncParameters).toBe(true);
232
+ expect(sourceTable.syncEvent).toBe(false);
233
+ });
234
+
235
+ test('resolveTables drops old table when table name changes for the same objectId', async () => {
236
+ await using factory = await storageConfig.factory();
237
+ const syncRules = await factory.updateSyncRules(
238
+ updateSyncRulesFromYaml(
239
+ `
240
+ bucket_definitions:
241
+ global:
242
+ data:
243
+ - SELECT id FROM "%"
244
+ `,
245
+ { storageVersion }
246
+ )
247
+ );
248
+ const bucketStorage = factory.getInstance(syncRules);
249
+
250
+ await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
251
+ const before = await writer.resolveTables({
252
+ connection_id: 1,
253
+ source: sourceDescriptor('orders', { objectId: 'orders-relation' }),
254
+ idGenerator: objectIdGenerator('6544e3899293153fa7b38342')
255
+ });
256
+ const after = await writer.resolveTables({
257
+ connection_id: 1,
258
+ source: sourceDescriptor('renamed_orders', { objectId: 'orders-relation' }),
259
+ idGenerator: objectIdGenerator('6544e3899293153fa7b38343')
260
+ });
261
+
262
+ expect(after.tables).toHaveLength(1);
263
+ expect(after.tables[0].id).not.toEqual(before.tables[0].id);
264
+ expect(after.tables[0].bucketDataSources).toHaveLength(1);
265
+ expect(after.tables[0].parameterLookupSources).toHaveLength(0);
266
+ expect(after.dropTables.map((table) => ({ id: table.id, name: table.name }))).toEqual([
267
+ { id: before.tables[0].id, name: 'orders' }
268
+ ]);
269
+ expect(after.dropTables[0].bucketDataSources).toHaveLength(1);
270
+ expect(after.dropTables[0].parameterLookupSources).toHaveLength(0);
271
+ });
272
+
273
+ test('resolveTables drops old table when objectId changes for the same table name', async () => {
274
+ await using factory = await storageConfig.factory();
275
+ const syncRules = await factory.updateSyncRules(
276
+ updateSyncRulesFromYaml(
277
+ `
278
+ bucket_definitions:
279
+ global:
280
+ data:
281
+ - SELECT id FROM "%"
282
+ `,
283
+ { storageVersion }
284
+ )
285
+ );
286
+ const bucketStorage = factory.getInstance(syncRules);
287
+
288
+ await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
289
+ const before = await writer.resolveTables({
290
+ connection_id: 1,
291
+ source: sourceDescriptor('accounts', { objectId: 'accounts-relation-old' }),
292
+ idGenerator: objectIdGenerator('6544e3899293153fa7b38344')
293
+ });
294
+ const after = await writer.resolveTables({
295
+ connection_id: 1,
296
+ source: sourceDescriptor('accounts', { objectId: 'accounts-relation-new' }),
297
+ idGenerator: objectIdGenerator('6544e3899293153fa7b38345')
298
+ });
299
+
300
+ expect(after.tables).toHaveLength(1);
301
+ expect(after.tables[0].id).not.toEqual(before.tables[0].id);
302
+ expect(after.tables[0].bucketDataSources).toHaveLength(1);
303
+ expect(after.dropTables.map((table) => ({ id: table.id, objectId: table.objectId }))).toEqual([
304
+ { id: before.tables[0].id, objectId: 'accounts-relation-old' }
305
+ ]);
306
+ expect(after.dropTables[0].bucketDataSources).toHaveLength(1);
307
+ });
308
+
309
+ test('resolveTables drops old table when replica id columns change', async () => {
310
+ await using factory = await storageConfig.factory();
311
+ const syncRules = await factory.updateSyncRules(
312
+ updateSyncRulesFromYaml(
313
+ `
314
+ bucket_definitions:
315
+ global:
316
+ data:
317
+ - SELECT id FROM "%"
318
+ `,
319
+ { storageVersion }
320
+ )
321
+ );
322
+ const bucketStorage = factory.getInstance(syncRules);
323
+
324
+ await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
325
+ const before = await writer.resolveTables({
326
+ connection_id: 1,
327
+ source: sourceDescriptor('items', { objectId: 'items-relation', replicaIdColumns: ['id'] }),
328
+ idGenerator: objectIdGenerator('6544e3899293153fa7b38346')
329
+ });
330
+ const after = await writer.resolveTables({
331
+ connection_id: 1,
332
+ source: sourceDescriptor('items', { objectId: 'items-relation', replicaIdColumns: ['tenant_id', 'id'] }),
333
+ idGenerator: objectIdGenerator('6544e3899293153fa7b38347')
334
+ });
335
+
336
+ expect(after.tables).toHaveLength(1);
337
+ expect(after.tables[0].id).not.toEqual(before.tables[0].id);
338
+ expect(after.tables[0].replicaIdColumns.map((column) => column.name)).toEqual(['tenant_id', 'id']);
339
+ expect(after.tables[0].bucketDataSources).toHaveLength(1);
340
+ expect(
341
+ after.dropTables.map((table) => ({ id: table.id, columns: table.replicaIdColumns.map((c) => c.name) }))
342
+ ).toEqual([{ id: before.tables[0].id, columns: ['id'] }]);
343
+ expect(after.dropTables[0].bucketDataSources).toHaveLength(1);
344
+ });
345
+
346
+ test.runIf(storageVersion >= 3)(
347
+ 'resolveTables resolves v3 event-only tables without source memberships',
348
+ async () => {
349
+ await using factory = await storageConfig.factory();
350
+ const syncRules = await factory.updateSyncRules(
351
+ updateSyncRulesFromYaml(
352
+ `
353
+ bucket_definitions:
354
+ by_owner:
355
+ data:
356
+ - SELECT id FROM users
357
+
358
+ event_definitions:
359
+ write_checkpoints:
360
+ payloads:
361
+ - SELECT user_id, checkpoint FROM checkpoints
362
+ `,
363
+ { storageVersion }
364
+ )
365
+ );
366
+ const bucketStorage = factory.getInstance(syncRules);
367
+
368
+ await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
369
+ const resolved = await writer.resolveTables({
370
+ connection_id: 1,
371
+ source: {
372
+ connectionTag: storage.SourceTable.DEFAULT_TAG,
373
+ objectId: 'checkpoints',
374
+ schema: 'public',
375
+ name: 'checkpoints',
376
+ replicaIdColumns: [{ name: 'id', type: 'VARCHAR', typeId: 25 }]
377
+ },
378
+ idGenerator: () => new bson.ObjectId('6544e3899293153fa7b38341')
379
+ });
380
+
381
+ expect(resolved.tables).toHaveLength(1);
382
+ expect(resolved.dropTables).toHaveLength(0);
383
+ expect(resolved.tables[0].bucketDataSources).toEqual([]);
384
+ expect(resolved.tables[0].parameterLookupSources).toEqual([]);
385
+ expect(resolved.tables[0].syncData).toBe(false);
386
+ expect(resolved.tables[0].syncParameters).toBe(false);
387
+ expect(resolved.tables[0].syncEvent).toBe(true);
388
+ }
389
+ );
390
+
391
+ test.runIf(storageVersion >= 3)('resolveTables handles v3 source membership additions and removals', async () => {
392
+ // Tests the behavior of resolveTables when bucket data sources and parameter index creators are added or removed.
393
+ // These are not end-to-end tests yet, since we don't have a full incremental reprocessing implementation.
394
+ // This just tests the specific resolveTables behavior.
395
+
396
+ // The same tests should work with sync streams, but legacy bucket_definitions make it easy
397
+ // to see the distinction between the parameter index queries and the data sources.
398
+ const fullRulesYaml = `
399
+ bucket_definitions:
400
+ by_owner:
401
+ parameters:
402
+ - SELECT owner_id FROM memberships WHERE id = token_parameters.test_id
403
+ data:
404
+ - SELECT id, owner_id FROM memberships WHERE owner_id = bucket.owner_id
405
+ `;
406
+ const dataOnlyRulesYaml = `
407
+ bucket_definitions:
408
+ by_owner:
409
+ parameters:
410
+ - SELECT token_parameters.owner_id as owner_id
411
+ data:
412
+ - SELECT id, owner_id FROM memberships WHERE owner_id = bucket.owner_id
413
+ `;
414
+ const parameterOnlyRulesYaml = `
415
+ bucket_definitions:
416
+ by_owner:
417
+ parameters:
418
+ - SELECT owner_id FROM memberships WHERE id = token_parameters.test_id
419
+ data: []
420
+ `;
421
+ const eventOnlyRulesYaml = `
422
+ bucket_definitions: {}
423
+
424
+ event_definitions:
425
+ write_checkpoints:
426
+ payloads:
427
+ - SELECT id, owner_id FROM memberships
428
+ `;
429
+
430
+ await using factory = await storageConfig.factory();
431
+ // This does not quite match what actual API usage would look like.
432
+ // Here we're persisting one sync config, then resolving tables with others.
433
+ // We're also using the default hydration state for them all.
434
+ const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(fullRulesYaml, { storageVersion }));
435
+ const bucketStorage = factory.getInstance(syncRules) as MongoSyncBucketStorage;
436
+ await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
437
+ const fullRules = hydratedRulesFor(fullRulesYaml);
438
+ const dataOnlyRules = hydratedRulesFor(dataOnlyRulesYaml);
439
+ const parameterOnlyRules = hydratedRulesFor(parameterOnlyRulesYaml);
440
+ const eventOnlyRules = hydratedRulesFor(eventOnlyRulesYaml);
441
+ const source = sourceDescriptor('memberships', { objectId: 'memberships-relation' });
442
+ const dataOnlyTableId = new bson.ObjectId('6544e3899293153fa7b38348');
443
+ const addedParameterTableId = new bson.ObjectId('6544e3899293153fa7b38349');
444
+ const removedDataTableId = new bson.ObjectId('6544e3899293153fa7b3834a');
445
+
446
+ const dataOnly = await writer.resolveTables({
447
+ connection_id: 1,
448
+ source,
449
+ idGenerator: () => dataOnlyTableId,
450
+ syncRules: dataOnlyRules
451
+ });
452
+ expect(dataOnly.tables.map((table) => table.id)).toEqual([dataOnlyTableId]);
453
+ expect(dataOnly.dropTables.map((table) => table.id)).toEqual([]);
454
+ expect(dataOnly.tables[0].bucketDataSources).toHaveLength(1);
455
+ expect(dataOnly.tables[0].parameterLookupSources).toHaveLength(0);
456
+
457
+ const addedParameter = await writer.resolveTables({
458
+ connection_id: 1,
459
+ source,
460
+ idGenerator: () => addedParameterTableId,
461
+ syncRules: fullRules
462
+ });
463
+ // Adding a definition always creates a new SourceTable
464
+ expect(addedParameter.tables.map((table) => table.id)).toEqual([dataOnlyTableId, addedParameterTableId]);
465
+ expect(addedParameter.tables.map((table) => table.bucketDataSources.length).sort()).toEqual([0, 1]);
466
+ expect(addedParameter.tables.map((table) => table.parameterLookupSources.length).sort()).toEqual([0, 1]);
467
+ expect(addedParameter.dropTables.map((table) => table.id)).toEqual([]);
468
+
469
+ const removedParameter = await writer.resolveTables({
470
+ connection_id: 1,
471
+ source,
472
+ idGenerator: () => {
473
+ throw new Error('data-only resolve should reuse existing v3 source table');
474
+ },
475
+ syncRules: dataOnlyRules
476
+ });
477
+ expect(removedParameter.tables.map((table) => table.id)).toEqual([dataOnlyTableId]);
478
+ // Now this sourceTable is unused & dropped
479
+ expect(removedParameter.dropTables.map((table) => table.id)).toEqual([addedParameterTableId]);
480
+ expect(removedParameter.tables[0].bucketDataSources).toHaveLength(1);
481
+ expect(removedParameter.tables[0].parameterLookupSources).toHaveLength(0);
482
+ await writer.drop(removedParameter.dropTables);
483
+
484
+ const removedData = await writer.resolveTables({
485
+ connection_id: 1,
486
+ source,
487
+ idGenerator: () => removedDataTableId,
488
+ syncRules: parameterOnlyRules
489
+ });
490
+
491
+ // This goes from dataOnlyRules -> parameterOnlyRules, which adds one definition and removes another.
492
+ // This generates a new SourceTable again, and removes all others.
493
+ expect(removedData.tables.map((table) => table.id)).toEqual([removedDataTableId]);
494
+ expect(removedData.dropTables.map((table) => table.id)).toEqual([dataOnlyTableId]);
495
+ expect(removedData.tables[0].bucketDataSources).toHaveLength(0);
496
+ expect(removedData.tables[0].parameterLookupSources).toHaveLength(1);
497
+ await writer.drop(removedData.dropTables);
498
+
499
+ const eventOnly = await writer.resolveTables({
500
+ connection_id: 1,
501
+ source,
502
+ idGenerator: () => {
503
+ throw new Error('resolve should reuse existing v3 source table');
504
+ },
505
+ syncRules: eventOnlyRules
506
+ });
507
+
508
+ // Event-only table can re-use any existing table.
509
+ expect(eventOnly.tables.map((table) => table.id)).toEqual([removedDataTableId]);
510
+ expect(eventOnly.dropTables.map((table) => table.id)).toEqual([]);
511
+ expect(eventOnly.tables[0].bucketDataSources).toHaveLength(0);
512
+ expect(eventOnly.tables[0].parameterLookupSources).toHaveLength(0);
513
+ expect(eventOnly.tables[0].syncData).toBe(false);
514
+ expect(eventOnly.tables[0].syncParameters).toBe(false);
515
+ expect(eventOnly.tables[0].syncEvent).toBe(true);
516
+ });
517
+
160
518
  test.runIf(storageVersion >= 3)('uses v3 mongodb model shapes', async () => {
161
519
  await using factory = await storageConfig.factory();
162
520
  const syncRules = await factory.updateSyncRules(
@@ -173,7 +531,7 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor
173
531
  )
174
532
  );
175
533
  const bucketStorage = factory.getInstance(syncRules);
176
- const sync_rules = syncRules.parsed(test_utils.PARSE_OPTIONS).hydratedSyncRules();
534
+ const sync_rules = syncRules.parsed(test_utils.PARSE_OPTIONS).hydratedSyncConfig();
177
535
  await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
178
536
  const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], INITIALIZED_MONGO_STORAGE_FACTORY);
179
537
 
@@ -219,8 +577,9 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor
219
577
  bucketCollections.some((collection) => collection.name === `bucket_data_${syncRules.id}_${firstBucket?.def}`)
220
578
  ).toBe(true);
221
579
 
222
- const syncRule = await mongoFactory.db.sync_rules.findOne({ _id: syncRules.id });
223
- const ruleMapping: SyncRuleDocument['rule_mapping'] | undefined = syncRule?.rule_mapping;
580
+ const syncRule = (await mongoFactory.db.sync_rules.findOne({ _id: syncRules.id })) as ReplicationStreamDocumentV3;
581
+ const syncConfig = await db.syncConfigDefinitions.findOne({ _id: syncRule.sync_configs[0]._id });
582
+ const ruleMapping: SyncConfigDefinition['rule_mapping'] | undefined = syncConfig?.rule_mapping;
224
583
  expect(Object.keys(ruleMapping?.definitions ?? {})).not.toHaveLength(0);
225
584
 
226
585
  const parameterIndexId = Object.values(ruleMapping?.parameter_indexes ?? {})[0] as string | undefined;
@@ -229,6 +588,34 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor
229
588
  expect(deserializeParameterLookup(parameterEntry!.lookup)).toEqual(['shape-check']);
230
589
  });
231
590
 
591
+ test.runIf(storageVersion < 3)('can replace processing legacy sync rules', async () => {
592
+ await using factory = await storageConfig.factory();
593
+
594
+ const firstSyncRules = await factory.updateSyncRules(
595
+ updateSyncRulesFromYaml(MINIMAL_SYNC_RULES, { storageVersion })
596
+ );
597
+
598
+ await expect(
599
+ factory.updateSyncRules(updateSyncRulesFromYaml(MINIMAL_SYNC_RULES, { storageVersion }))
600
+ ).resolves.toBeDefined();
601
+
602
+ const mongoFactory = factory as MongoBucketStorage;
603
+ expect((await mongoFactory.db.sync_rules.findOne({ _id: firstSyncRules.id }))?.state).toBe(
604
+ storage.SyncRuleState.STOP
605
+ );
606
+ });
607
+
608
+ test('can lock newly-created sync rules', async () => {
609
+ await using factory = await storageConfig.factory();
610
+
611
+ const syncRules = await factory.updateSyncRules(
612
+ updateSyncRulesFromYaml(MINIMAL_SYNC_RULES, { storageVersion, lock: true })
613
+ );
614
+
615
+ expect(syncRules.current_lock?.sync_rules_id).toBe(syncRules.id);
616
+ await syncRules.current_lock?.release();
617
+ });
618
+
232
619
  test.runIf(storageVersion < 3)('uses a single current_data collection for v1 source records', async () => {
233
620
  await using factory = await storageConfig.factory();
234
621
  const syncRules = await factory.updateSyncRules(
@@ -481,6 +868,35 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor
481
868
  }
482
869
 
483
870
  describe('sync - mongodb', () => {
871
+ test('v3 activation stops legacy active sync rules', async () => {
872
+ await using factory = await INITIALIZED_MONGO_STORAGE_FACTORY.factory();
873
+ const mongoFactory = factory as MongoBucketStorage;
874
+
875
+ const legacySyncRules = await factory.updateSyncRules(
876
+ updateSyncRulesFromYaml(MINIMAL_SYNC_RULES, { storageVersion: storage.LEGACY_STORAGE_VERSION })
877
+ );
878
+ const legacyStorage = factory.getInstance(legacySyncRules);
879
+ await using legacyWriter = await legacyStorage.createWriter(test_utils.BATCH_OPTIONS);
880
+ await legacyWriter.markAllSnapshotDone('1/1');
881
+ await legacyWriter.commit('1/1');
882
+
883
+ expect((await mongoFactory.db.sync_rules.findOne({ _id: legacySyncRules.id }))?.state).toBe(
884
+ storage.SyncRuleState.ACTIVE
885
+ );
886
+
887
+ const v3SyncRules = await factory.updateSyncRules(
888
+ updateSyncRulesFromYaml(MINIMAL_SYNC_RULES, { storageVersion: storage.STORAGE_VERSION_3 })
889
+ );
890
+ const v3Storage = factory.getInstance(v3SyncRules);
891
+ await using v3Writer = await v3Storage.createWriter(test_utils.BATCH_OPTIONS);
892
+ await v3Writer.markAllSnapshotDone('2/1');
893
+ await v3Writer.commit('2/1');
894
+
895
+ expect((await mongoFactory.db.sync_rules.findOne({ _id: legacySyncRules.id }))?.state).toBe(
896
+ storage.SyncRuleState.STOP
897
+ );
898
+ });
899
+
484
900
  for (const storageVersion of TEST_STORAGE_VERSIONS) {
485
901
  describe(`storage v${storageVersion}`, () => {
486
902
  registerSyncStorageTests(INITIALIZED_MONGO_STORAGE_FACTORY, storageVersion);
@@ -0,0 +1,211 @@
1
+ import { storage, updateSyncRulesFromYaml } from '@powersync/service-core';
2
+ import { bucketRequest, test_utils } from '@powersync/service-core-tests';
3
+ import * as bson from 'bson';
4
+ import { describe, expect, test } from 'vitest';
5
+ import { MongoBucketStorage } from '../../src/storage/MongoBucketStorage.js';
6
+ import { INITIALIZED_MONGO_STORAGE_FACTORY, TEST_STORAGE_VERSIONS } from './util.js';
7
+
8
+ /**
9
+ * Tests for the per-table `storeCurrentData` flag in MongoDB storage (both v1 and v3).
10
+ *
11
+ * Two things are exercised:
12
+ * 1. `resolveTables` derives `storeCurrentData = !sendsCompleteRows` in memory on every call
13
+ * (it is not persisted; an unreported source defaults to keeping a copy).
14
+ * 2. A batch honours the flag: when `storeCurrentData` is false, the row payload is NOT kept in
15
+ * the current_data collection, while the bucket data is still produced as usual.
16
+ *
17
+ * The mapping from a source's replica identity to `sendsCompleteRows` is a Postgres concern and is
18
+ * covered in module-postgres; here we only exercise the storage layer's boolean handling.
19
+ */
20
+
21
+ const SYNC_RULES = `
22
+ bucket_definitions:
23
+ global:
24
+ data:
25
+ - SELECT id, description FROM "%"
26
+ `;
27
+
28
+ function descriptor(name: string, sendsCompleteRows?: boolean): storage.SourceEntityDescriptor {
29
+ return {
30
+ connectionTag: storage.SourceTable.DEFAULT_TAG,
31
+ objectId: name,
32
+ schema: 'public',
33
+ name,
34
+ replicaIdColumns: [{ name: 'id', type: 'VARCHAR', typeId: 25 }],
35
+ sendsCompleteRows
36
+ };
37
+ }
38
+
39
+ function singleUseIdGenerator(hex: string) {
40
+ let used = false;
41
+ return () => {
42
+ if (used) {
43
+ throw new Error(`Can only generate a single id using ${hex}`);
44
+ }
45
+ used = true;
46
+ return new bson.ObjectId(hex);
47
+ };
48
+ }
49
+
50
+ /**
51
+ * Returns the row payload stored in current_data for the (single) record of `table`, or null if no
52
+ * payload is stored. v1 stores an empty BSON object ({}) when storeCurrentData is false; v3 stores
53
+ * null. Both are normalised to null here so the assertions are version-agnostic.
54
+ */
55
+ async function storedRowPayload(
56
+ storageVersion: number,
57
+ factory: storage.BucketStorageFactory,
58
+ bucketStorage: storage.SyncRulesBucketStorage,
59
+ syncRulesId: number,
60
+ table: storage.SourceTable
61
+ ): Promise<Record<string, any> | null> {
62
+ let data: bson.Binary | null | undefined;
63
+ if (storageVersion < 3) {
64
+ const db = (factory as MongoBucketStorage).db;
65
+ const doc = await db.current_data.findOne({ '_id.g': syncRulesId });
66
+ data = doc?.data;
67
+ } else {
68
+ const db = (bucketStorage as any).db;
69
+ const doc = await db.sourceRecordsV3(syncRulesId, table.id).findOne({});
70
+ data = doc?.data;
71
+ }
72
+ if (data == null) {
73
+ return null;
74
+ }
75
+ const decoded = bson.deserialize(data.buffer);
76
+ return Object.keys(decoded).length === 0 ? null : decoded;
77
+ }
78
+
79
+ function registerStoreCurrentDataTests(storageVersion: number) {
80
+ test('resolveTables derives storeCurrentData fresh each call, with no persisted memory', async () => {
81
+ await using factory = await INITIALIZED_MONGO_STORAGE_FACTORY.factory();
82
+ const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(SYNC_RULES, { storageVersion }));
83
+ const bucketStorage = factory.getInstance(syncRules);
84
+
85
+ await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
86
+
87
+ // Sources that always send complete rows (e.g. Postgres REPLICA IDENTITY FULL) don't need current_data.
88
+ const complete = await writer.resolveTables({
89
+ connection_id: 1,
90
+ source: descriptor('test_complete', true),
91
+ idGenerator: singleUseIdGenerator('6544e3899293153fa7b38301')
92
+ });
93
+ expect(complete.tables[0].storeCurrentData).toBe(false);
94
+
95
+ // Sources that may send partial/key data still need current_data.
96
+ const partial = await writer.resolveTables({
97
+ connection_id: 1,
98
+ source: descriptor('test_partial', false),
99
+ idGenerator: singleUseIdGenerator('6544e3899293153fa7b38302')
100
+ });
101
+ expect(partial.tables[0].storeCurrentData).toBe(true);
102
+
103
+ // Re-resolving an already-resolved table with no completeness info (sendsCompleteRows undefined)
104
+ // does not remember the earlier value: nothing is persisted, so it falls back to the default
105
+ // (keep a copy).
106
+ const reResolved = await writer.resolveTables({
107
+ connection_id: 1,
108
+ source: descriptor('test_complete')
109
+ });
110
+ expect(reResolved.tables[0].storeCurrentData).toBe(true);
111
+ });
112
+
113
+ test('storeCurrentData=false omits the row payload from current_data, data still syncs', async () => {
114
+ await using factory = await INITIALIZED_MONGO_STORAGE_FACTORY.factory();
115
+ const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(SYNC_RULES, { storageVersion }));
116
+ const bucketStorage = factory.getInstance(syncRules);
117
+
118
+ await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
119
+ const table = await test_utils.resolveTestTable(writer, 'test_data', ['id'], INITIALIZED_MONGO_STORAGE_FACTORY);
120
+ table.storeCurrentData = false;
121
+
122
+ await writer.save({
123
+ sourceTable: table,
124
+ tag: storage.SaveOperationTag.INSERT,
125
+ after: { id: 'test1', description: 'test data' },
126
+ afterReplicaId: test_utils.rid('test1')
127
+ });
128
+ const flushResult = await writer.flush();
129
+ const checkpoint = flushResult!.flushed_op;
130
+
131
+ const batch = await test_utils.fromAsync(
132
+ bucketStorage.getBucketDataBatch(checkpoint, [bucketRequest(syncRules, 'global[]', 0n)])
133
+ );
134
+ expect(test_utils.getBatchData(batch)).toMatchObject([{ op: 'PUT', object_id: 'test1' }]);
135
+
136
+ // No row payload retained in current_data.
137
+ expect(await storedRowPayload(storageVersion, factory, bucketStorage, syncRules.id, table)).toBeNull();
138
+ });
139
+
140
+ test('storeCurrentData=true retains the row payload in current_data', async () => {
141
+ await using factory = await INITIALIZED_MONGO_STORAGE_FACTORY.factory();
142
+ const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(SYNC_RULES, { storageVersion }));
143
+ const bucketStorage = factory.getInstance(syncRules);
144
+
145
+ await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
146
+ const table = await test_utils.resolveTestTable(writer, 'test_data', ['id'], INITIALIZED_MONGO_STORAGE_FACTORY);
147
+ table.storeCurrentData = true;
148
+
149
+ await writer.save({
150
+ sourceTable: table,
151
+ tag: storage.SaveOperationTag.INSERT,
152
+ after: { id: 'test1', description: 'test data' },
153
+ afterReplicaId: test_utils.rid('test1')
154
+ });
155
+ const flushResult = await writer.flush();
156
+ const checkpoint = flushResult!.flushed_op;
157
+
158
+ const batch = await test_utils.fromAsync(
159
+ bucketStorage.getBucketDataBatch(checkpoint, [bucketRequest(syncRules, 'global[]', 0n)])
160
+ );
161
+ expect(test_utils.getBatchData(batch)).toMatchObject([{ op: 'PUT', object_id: 'test1' }]);
162
+
163
+ expect(await storedRowPayload(storageVersion, factory, bucketStorage, syncRules.id, table)).toMatchObject({
164
+ id: 'test1'
165
+ });
166
+ });
167
+
168
+ test('storeCurrentData=false processes UPDATE without a stored copy', async () => {
169
+ await using factory = await INITIALIZED_MONGO_STORAGE_FACTORY.factory();
170
+ const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(SYNC_RULES, { storageVersion }));
171
+ const bucketStorage = factory.getInstance(syncRules);
172
+
173
+ await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
174
+ const table = await test_utils.resolveTestTable(writer, 'test_data', ['id'], INITIALIZED_MONGO_STORAGE_FACTORY);
175
+ table.storeCurrentData = false;
176
+
177
+ // With REPLICA IDENTITY FULL the UPDATE carries the full row, so it is applied from `after`
178
+ // rather than a stored copy of the previous row.
179
+ await writer.save({
180
+ sourceTable: table,
181
+ tag: storage.SaveOperationTag.INSERT,
182
+ after: { id: 'test1', description: 'original' },
183
+ afterReplicaId: test_utils.rid('test1')
184
+ });
185
+ await writer.save({
186
+ sourceTable: table,
187
+ tag: storage.SaveOperationTag.UPDATE,
188
+ after: { id: 'test1', description: 'updated' },
189
+ afterReplicaId: test_utils.rid('test1')
190
+ });
191
+ const flushResult = await writer.flush();
192
+ const checkpoint = flushResult!.flushed_op;
193
+
194
+ const batch = await test_utils.fromAsync(
195
+ bucketStorage.getBucketDataBatch(checkpoint, [bucketRequest(syncRules, 'global[]', 0n)])
196
+ );
197
+ const data = test_utils.getBatchData(batch);
198
+ expect(data.length).toBeGreaterThan(0);
199
+ expect(data.at(-1)).toMatchObject({ op: 'PUT', object_id: 'test1' });
200
+
201
+ expect(await storedRowPayload(storageVersion, factory, bucketStorage, syncRules.id, table)).toBeNull();
202
+ });
203
+ }
204
+
205
+ describe('MongoDB storage - storeCurrentData', () => {
206
+ for (const storageVersion of TEST_STORAGE_VERSIONS) {
207
+ describe(`storage v${storageVersion}`, () => {
208
+ registerStoreCurrentDataTests(storageVersion);
209
+ });
210
+ }
211
+ });