@powersync/service-module-mongodb-storage 0.0.0-dev-20260803144812 → 0.0.0-dev-20260929095716

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (213) hide show
  1. package/CHANGELOG.md +129 -7
  2. package/dist/storage/MongoBucketStorage.d.ts +42 -5
  3. package/dist/storage/MongoBucketStorage.js +404 -215
  4. package/dist/storage/MongoBucketStorage.js.map +1 -1
  5. package/dist/storage/implementation/MongoBucketBatch.d.ts +39 -14
  6. package/dist/storage/implementation/MongoBucketBatch.js +169 -98
  7. package/dist/storage/implementation/MongoBucketBatch.js.map +1 -1
  8. package/dist/storage/implementation/MongoCompactor.d.ts +21 -79
  9. package/dist/storage/implementation/MongoCompactor.js +27 -238
  10. package/dist/storage/implementation/MongoCompactor.js.map +1 -1
  11. package/dist/storage/implementation/MongoIdSequence.d.ts +10 -1
  12. package/dist/storage/implementation/MongoIdSequence.js +14 -1
  13. package/dist/storage/implementation/MongoIdSequence.js.map +1 -1
  14. package/dist/storage/implementation/MongoOpIdAllocator.d.ts +27 -0
  15. package/dist/storage/implementation/MongoOpIdAllocator.js +97 -0
  16. package/dist/storage/implementation/MongoOpIdAllocator.js.map +1 -0
  17. package/dist/storage/implementation/MongoParameterCompactor.d.ts +83 -10
  18. package/dist/storage/implementation/MongoParameterCompactor.js +309 -96
  19. package/dist/storage/implementation/MongoParameterCompactor.js.map +1 -1
  20. package/dist/storage/implementation/MongoParsedSyncConfigSet.d.ts +9 -1
  21. package/dist/storage/implementation/MongoParsedSyncConfigSet.js +35 -2
  22. package/dist/storage/implementation/MongoParsedSyncConfigSet.js.map +1 -1
  23. package/dist/storage/implementation/MongoPersistedReplicationStream.d.ts +2 -1
  24. package/dist/storage/implementation/MongoPersistedReplicationStream.js +7 -4
  25. package/dist/storage/implementation/MongoPersistedReplicationStream.js.map +1 -1
  26. package/dist/storage/implementation/MongoPersistedSyncConfigContent.d.ts +3 -3
  27. package/dist/storage/implementation/MongoPersistedSyncConfigContent.js +10 -8
  28. package/dist/storage/implementation/MongoPersistedSyncConfigContent.js.map +1 -1
  29. package/dist/storage/implementation/MongoStorageProvider.js +8 -4
  30. package/dist/storage/implementation/MongoStorageProvider.js.map +1 -1
  31. package/dist/storage/implementation/MongoSyncBucketStorage.d.ts +32 -11
  32. package/dist/storage/implementation/MongoSyncBucketStorage.js +95 -31
  33. package/dist/storage/implementation/MongoSyncBucketStorage.js.map +1 -1
  34. package/dist/storage/implementation/MongoSyncRulesLock.d.ts +26 -2
  35. package/dist/storage/implementation/MongoSyncRulesLock.js +59 -10
  36. package/dist/storage/implementation/MongoSyncRulesLock.js.map +1 -1
  37. package/dist/storage/implementation/MongoWriteBatch.d.ts +49 -0
  38. package/dist/storage/implementation/MongoWriteBatch.js +149 -0
  39. package/dist/storage/implementation/MongoWriteBatch.js.map +1 -0
  40. package/dist/storage/implementation/MongoWriteCheckpointAPI.d.ts +9 -7
  41. package/dist/storage/implementation/MongoWriteCheckpointAPI.js +9 -30
  42. package/dist/storage/implementation/MongoWriteCheckpointAPI.js.map +1 -1
  43. package/dist/storage/implementation/SyncRuleStateUpdate.d.ts +11 -5
  44. package/dist/storage/implementation/SyncRuleStateUpdate.js +14 -6
  45. package/dist/storage/implementation/SyncRuleStateUpdate.js.map +1 -1
  46. package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.d.ts +22 -2
  47. package/dist/storage/implementation/common/PersistedBatch.d.ts +32 -4
  48. package/dist/storage/implementation/common/PersistedBatch.js +57 -14
  49. package/dist/storage/implementation/common/PersistedBatch.js.map +1 -1
  50. package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.d.ts +3 -0
  51. package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.js +3 -0
  52. package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.js.map +1 -1
  53. package/dist/storage/implementation/db.d.ts +9 -1
  54. package/dist/storage/implementation/db.js +28 -1
  55. package/dist/storage/implementation/db.js.map +1 -1
  56. package/dist/storage/implementation/models.d.ts +35 -0
  57. package/dist/storage/implementation/models.js.map +1 -1
  58. package/dist/storage/implementation/v1/MongoBucketBatchV1.d.ts +11 -0
  59. package/dist/storage/implementation/v1/MongoBucketBatchV1.js +227 -221
  60. package/dist/storage/implementation/v1/MongoBucketBatchV1.js.map +1 -1
  61. package/dist/storage/implementation/v1/MongoCompactorV1.d.ts +38 -6
  62. package/dist/storage/implementation/v1/MongoCompactorV1.js +247 -20
  63. package/dist/storage/implementation/v1/MongoCompactorV1.js.map +1 -1
  64. package/dist/storage/implementation/v1/MongoParameterCompactorV1.d.ts +16 -2
  65. package/dist/storage/implementation/v1/MongoParameterCompactorV1.js +20 -9
  66. package/dist/storage/implementation/v1/MongoParameterCompactorV1.js.map +1 -1
  67. package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.d.ts +18 -11
  68. package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js +66 -17
  69. package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js.map +1 -1
  70. package/dist/storage/implementation/v1/PersistedBatchV1.d.ts +7 -5
  71. package/dist/storage/implementation/v1/PersistedBatchV1.js +22 -28
  72. package/dist/storage/implementation/v1/PersistedBatchV1.js.map +1 -1
  73. package/dist/storage/implementation/v1/models.js +1 -0
  74. package/dist/storage/implementation/v1/models.js.map +1 -1
  75. package/dist/storage/implementation/v3/CompactionLease.d.ts +50 -0
  76. package/dist/storage/implementation/v3/CompactionLease.js +131 -0
  77. package/dist/storage/implementation/v3/CompactionLease.js.map +1 -0
  78. package/dist/storage/implementation/v3/MongoBucketBatchV3.d.ts +14 -2
  79. package/dist/storage/implementation/v3/MongoBucketBatchV3.js +261 -194
  80. package/dist/storage/implementation/v3/MongoBucketBatchV3.js.map +1 -1
  81. package/dist/storage/implementation/v3/MongoCompactorV3.d.ts +83 -25
  82. package/dist/storage/implementation/v3/MongoCompactorV3.js +702 -159
  83. package/dist/storage/implementation/v3/MongoCompactorV3.js.map +1 -1
  84. package/dist/storage/implementation/v3/MongoParameterCompactorV3.d.ts +18 -0
  85. package/dist/storage/implementation/v3/MongoParameterCompactorV3.js +23 -0
  86. package/dist/storage/implementation/v3/MongoParameterCompactorV3.js.map +1 -0
  87. package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.d.ts +2 -15
  88. package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.js +60 -92
  89. package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.js.map +1 -1
  90. package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.d.ts +19 -9
  91. package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js +92 -32
  92. package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js.map +1 -1
  93. package/dist/storage/implementation/v3/MongoWriteCheckpointAPIV3.d.ts +26 -0
  94. package/dist/storage/implementation/v3/MongoWriteCheckpointAPIV3.js +82 -0
  95. package/dist/storage/implementation/v3/MongoWriteCheckpointAPIV3.js.map +1 -0
  96. package/dist/storage/implementation/v3/PersistedBatchV3.d.ts +14 -8
  97. package/dist/storage/implementation/v3/PersistedBatchV3.js +77 -45
  98. package/dist/storage/implementation/v3/PersistedBatchV3.js.map +1 -1
  99. package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.d.ts +12 -2
  100. package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.js +42 -3
  101. package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.js.map +1 -1
  102. package/dist/storage/implementation/v3/bucket-format.js +2 -0
  103. package/dist/storage/implementation/v3/bucket-format.js.map +1 -1
  104. package/dist/storage/implementation/v3/compact-utils.d.ts +108 -0
  105. package/dist/storage/implementation/v3/compact-utils.js +223 -0
  106. package/dist/storage/implementation/v3/compact-utils.js.map +1 -0
  107. package/dist/storage/implementation/v3/compaction-constants.d.ts +5 -0
  108. package/dist/storage/implementation/v3/compaction-constants.js +6 -0
  109. package/dist/storage/implementation/v3/compaction-constants.js.map +1 -0
  110. package/dist/storage/implementation/v3/models.d.ts +90 -6
  111. package/dist/storage/implementation/v3/models.js.map +1 -1
  112. package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.d.ts +7 -8
  113. package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.js +16 -5
  114. package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.js.map +1 -1
  115. package/dist/storage/implementation/v3/object-storage/ObjectStorage.d.ts +14 -11
  116. package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.d.ts +6 -8
  117. package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.js +17 -11
  118. package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.js.map +1 -1
  119. package/dist/storage/implementation/v3/object-storage/ObjectStorageUsage.d.ts +38 -0
  120. package/dist/storage/implementation/v3/object-storage/ObjectStorageUsage.js +164 -0
  121. package/dist/storage/implementation/v3/object-storage/ObjectStorageUsage.js.map +1 -0
  122. package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.d.ts +37 -12
  123. package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.js +197 -25
  124. package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.js.map +1 -1
  125. package/dist/storage/implementation/v3/source-table-utils.d.ts +15 -7
  126. package/dist/storage/implementation/v3/source-table-utils.js +49 -57
  127. package/dist/storage/implementation/v3/source-table-utils.js.map +1 -1
  128. package/dist/storage/storage-index.d.ts +2 -1
  129. package/dist/storage/storage-index.js +1 -0
  130. package/dist/storage/storage-index.js.map +1 -1
  131. package/dist/types/types.d.ts +28 -1
  132. package/dist/types/types.js +28 -7
  133. package/dist/types/types.js.map +1 -1
  134. package/dist/utils/test-utils.d.ts +4 -2
  135. package/dist/utils/test-utils.js +7 -4
  136. package/dist/utils/test-utils.js.map +1 -1
  137. package/dist/utils/util.js +3 -1
  138. package/dist/utils/util.js.map +1 -1
  139. package/package.json +10 -9
  140. package/src/storage/MongoBucketStorage.ts +545 -277
  141. package/src/storage/implementation/MongoBucketBatch.ts +258 -124
  142. package/src/storage/implementation/MongoCompactor.ts +37 -346
  143. package/src/storage/implementation/MongoIdSequence.ts +21 -1
  144. package/src/storage/implementation/MongoOpIdAllocator.ts +105 -0
  145. package/src/storage/implementation/MongoParameterCompactor.ts +388 -102
  146. package/src/storage/implementation/MongoParsedSyncConfigSet.ts +46 -1
  147. package/src/storage/implementation/MongoPersistedReplicationStream.ts +12 -4
  148. package/src/storage/implementation/MongoPersistedSyncConfigContent.ts +48 -29
  149. package/src/storage/implementation/MongoStorageProvider.ts +16 -4
  150. package/src/storage/implementation/MongoSyncBucketStorage.ts +133 -40
  151. package/src/storage/implementation/MongoSyncRulesLock.ts +78 -12
  152. package/src/storage/implementation/MongoWriteBatch.ts +172 -0
  153. package/src/storage/implementation/MongoWriteCheckpointAPI.ts +16 -46
  154. package/src/storage/implementation/SyncRuleStateUpdate.ts +13 -6
  155. package/src/storage/implementation/common/MongoSyncBucketStorageCheckpoint.ts +24 -2
  156. package/src/storage/implementation/common/PersistedBatch.ts +76 -20
  157. package/src/storage/implementation/common/VersionedPowerSyncMongoBase.ts +4 -0
  158. package/src/storage/implementation/db.ts +36 -2
  159. package/src/storage/implementation/models.ts +39 -0
  160. package/src/storage/implementation/v1/MongoBucketBatchV1.ts +272 -254
  161. package/src/storage/implementation/v1/MongoCompactorV1.ts +327 -40
  162. package/src/storage/implementation/v1/MongoParameterCompactorV1.ts +25 -9
  163. package/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts +88 -28
  164. package/src/storage/implementation/v1/PersistedBatchV1.ts +25 -30
  165. package/src/storage/implementation/v1/models.ts +1 -0
  166. package/src/storage/implementation/v3/CompactionLease.ts +152 -0
  167. package/src/storage/implementation/v3/MongoBucketBatchV3.ts +339 -247
  168. package/src/storage/implementation/v3/MongoCompactorV3.ts +955 -225
  169. package/src/storage/implementation/v3/MongoParameterCompactorV3.ts +34 -0
  170. package/src/storage/implementation/v3/MongoStoppedSyncConfigCleanup.ts +100 -109
  171. package/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts +112 -42
  172. package/src/storage/implementation/v3/MongoWriteCheckpointAPIV3.ts +111 -0
  173. package/src/storage/implementation/v3/PersistedBatchV3.ts +94 -51
  174. package/src/storage/implementation/v3/VersionedPowerSyncMongoV3.ts +55 -4
  175. package/src/storage/implementation/v3/bucket-format.ts +2 -0
  176. package/src/storage/implementation/v3/compact-utils.ts +329 -0
  177. package/src/storage/implementation/v3/compaction-constants.ts +5 -0
  178. package/src/storage/implementation/v3/models.ts +97 -4
  179. package/src/storage/implementation/v3/object-storage/BucketDataObjectStorage.ts +24 -9
  180. package/src/storage/implementation/v3/object-storage/ObjectStorage.ts +20 -5
  181. package/src/storage/implementation/v3/object-storage/ObjectStorageLifecycle.ts +26 -14
  182. package/src/storage/implementation/v3/object-storage/ObjectStorageUsage.ts +229 -0
  183. package/src/storage/implementation/v3/object-storage/S3ObjectStorage.ts +279 -29
  184. package/src/storage/implementation/v3/source-table-utils.ts +82 -76
  185. package/src/storage/storage-index.ts +2 -0
  186. package/src/types/types.ts +42 -7
  187. package/src/utils/test-utils.ts +9 -5
  188. package/src/utils/util.ts +3 -1
  189. package/test/src/__snapshots__/storage.test.ts.snap +25 -0
  190. package/test/src/__snapshots__/storage_sync.test.ts.snap +794 -205
  191. package/test/src/chunk_compaction_config.test.ts +38 -0
  192. package/test/src/cleanup-stopped-sync-configs.test.ts +59 -26
  193. package/test/src/compact-utils.test.ts +251 -0
  194. package/test/src/concurrent_writers.test.ts +599 -0
  195. package/test/src/connection_config.test.ts +159 -0
  196. package/test/src/helpers/MemoryObjectStorage.ts +14 -6
  197. package/test/src/helpers/concurrentWriter.mjs +29 -0
  198. package/test/src/helpers/s3TestFactory.ts +12 -12
  199. package/test/src/object_storage_usage.test.ts +268 -0
  200. package/test/src/parameter_compacting_v1.test.ts +120 -0
  201. package/test/src/parameter_compacting_v3.test.ts +192 -0
  202. package/test/src/parameter_compaction_fence.test.ts +276 -0
  203. package/test/src/storage.test.ts +280 -24
  204. package/test/src/storage_compacting.test.ts +1097 -253
  205. package/test/src/storage_object_storage_inline_threshold.test.ts +3 -3
  206. package/test/src/storage_s3_checksums.test.ts +6 -7
  207. package/test/src/storage_s3_compaction_lifecycle.test.ts +143 -17
  208. package/test/src/storage_s3_reading.test.ts +294 -6
  209. package/test/src/storage_s3_writing.test.ts +4 -3
  210. package/test/src/storage_sync.test.ts +412 -80
  211. package/test/src/storeCurrentData.test.ts +5 -5
  212. package/test/src/util.ts +1 -1
  213. package/tsconfig.tsbuildinfo +1 -1
@@ -0,0 +1,159 @@
1
+ import { mongoTestStorageFactoryGenerator } from '@module/utils/test-utils.js';
2
+ import { SqlSyncConfigParser, updateSyncRulesFromConfig } from '@powersync/service-core';
3
+ import { test_utils } from '@powersync/service-core-tests';
4
+ import {
5
+ AdditionalSyncConfigParser,
6
+ normalizeConnectionConfig,
7
+ PrecompiledSyncConfig
8
+ } from '@powersync/service-sync-rules';
9
+ import { describe, expect, test } from 'vitest';
10
+ import { env } from './env.js';
11
+ import { TEST_STORAGE_VERSIONS } from './util.js';
12
+
13
+ const STREAMS = /* yaml */ `
14
+ config:
15
+ edition: 3
16
+ streams:
17
+ orders:
18
+ query: SELECT * FROM orders
19
+ `;
20
+ const CONFIGURED = /* yaml */ `
21
+ config:
22
+ edition: 3
23
+ connections:
24
+ default:
25
+ type: example
26
+ tables:
27
+ orders:
28
+ sample: 10
29
+ streams:
30
+ orders:
31
+ query: SELECT * FROM orders
32
+ `;
33
+
34
+ // A small module-owned option exercises persistence without introducing MongoDB filter syntax into core.
35
+ const TABLE_OPTIONS: AdditionalSyncConfigParser = {
36
+ id: 'example.tables',
37
+ extendJsonSchema({ schema }) {
38
+ const map = (schema.properties as any).config.properties.connections;
39
+ map.additionalProperties = {
40
+ if: { type: 'object', required: ['type'], properties: { type: { const: 'example' } } },
41
+ then: {
42
+ type: 'object',
43
+ additionalProperties: false,
44
+ required: ['type'],
45
+ properties: {
46
+ type: { const: 'example' },
47
+ tables: {
48
+ type: 'object',
49
+ additionalProperties: {
50
+ type: 'object',
51
+ additionalProperties: false,
52
+ properties: { sample: { type: 'number' } }
53
+ }
54
+ }
55
+ }
56
+ },
57
+ else: map.additionalProperties
58
+ };
59
+ },
60
+ parse({ config, context }) {
61
+ const connections = normalizeConnectionConfig(
62
+ (config as { config?: { connections?: unknown } }).config?.connections
63
+ );
64
+ for (const [tag, connection] of Object.entries(connections)) {
65
+ if (connection?.type != 'example') continue;
66
+ context.parsedConfig.connectionConfig = { ...context.parsedConfig.connectionConfig, [tag]: connection };
67
+ const { plan } = context.parsedConfig as PrecompiledSyncConfig;
68
+ plan.moduleData = { ...plan.moduleData, ['example.tables']: null };
69
+ }
70
+ }
71
+ };
72
+
73
+ function storageFactory(withModule: boolean) {
74
+ const syncConfigParser = new SqlSyncConfigParser(withModule ? [TABLE_OPTIONS] : []);
75
+ return mongoTestStorageFactoryGenerator({
76
+ url: env.MONGO_TEST_URL,
77
+ isCI: env.CI,
78
+ supportsMultipleSyncConfigs: true,
79
+ syncConfigParser
80
+ });
81
+ }
82
+
83
+ describe.for(TEST_STORAGE_VERSIONS)('connection config with storage v%i', (storageVersion) => {
84
+ test('preserves module options across closing storage and loading its compiled plan', async () => {
85
+ let expected;
86
+ {
87
+ await using factory = await storageFactory(true).factory();
88
+ const parsed = factory.syncConfigParser.parseContent(CONFIGURED, test_utils.PARSE_OPTIONS);
89
+ expected = parsed.config.connectionConfig;
90
+ await factory.updateSyncRules(updateSyncRulesFromConfig(parsed, { storageVersion }));
91
+ }
92
+
93
+ // Reopen the same database without clearing it, as a service restart would.
94
+ await using reopened = await storageFactory(true).factory({ doNotClear: true });
95
+ const deploying = (await reopened.getDeployingSyncConfig())!;
96
+ expect(deploying.content.compiled_plan?.plan.version).toBe(3);
97
+ const parsed = deploying.content.parsed(test_utils.PARSE_OPTIONS);
98
+ expect(parsed.syncConfigs[0].config.connectionConfig).toEqual(expected);
99
+ expect(parsed.hydratedSyncConfig.connectionConfig).toEqual(expected);
100
+ });
101
+
102
+ test('rejects saved options if the module is absent after restart', async () => {
103
+ {
104
+ await using factory = await storageFactory(true).factory();
105
+ await factory.updateSyncRules(
106
+ updateSyncRulesFromConfig(factory.syncConfigParser.parseContent(CONFIGURED, test_utils.PARSE_OPTIONS), {
107
+ storageVersion
108
+ })
109
+ );
110
+ }
111
+ await using reopened = await storageFactory(false).factory({ doNotClear: true });
112
+ const deploying = (await reopened.getDeployingSyncConfig())!;
113
+ expect(() => deploying.content.parsed(test_utils.PARSE_OPTIONS)).toThrow(
114
+ 'Missing required sync config parsers: example.tables'
115
+ );
116
+ });
117
+
118
+ test('continues loading unfiltered plans without an additional parser', async () => {
119
+ await using factory = await storageFactory(false).factory();
120
+ await factory.updateSyncRules(
121
+ updateSyncRulesFromConfig(factory.syncConfigParser.parseContent(STREAMS, test_utils.PARSE_OPTIONS), {
122
+ storageVersion
123
+ })
124
+ );
125
+ const deploying = (await factory.getDeployingSyncConfig())!;
126
+ expect(deploying.content.compiled_plan?.plan.version).toBe(1);
127
+ expect(deploying.content.parsed(test_utils.PARSE_OPTIONS).hydratedSyncConfig.connectionConfig).toEqual({});
128
+ });
129
+
130
+ test('starts replacement processing when connection options change', async () => {
131
+ await using factory = await storageFactory(true).factory();
132
+ const deploy = (yaml: string) =>
133
+ factory.updateSyncRules(
134
+ updateSyncRulesFromConfig(factory.syncConfigParser.parseContent(yaml, test_utils.PARSE_OPTIONS), {
135
+ storageVersion
136
+ })
137
+ );
138
+ const first = await deploy(CONFIGURED);
139
+ {
140
+ // Activate the first config so compatibility is checked against data already being served.
141
+ const storage = await test_utils.getTestStorage(factory, first);
142
+ await using writer = await storage.createWriter(test_utils.BATCH_OPTIONS);
143
+ await writer.markAllSnapshotDone('1/1');
144
+ await writer.commit('1/1');
145
+ }
146
+ if (storageVersion >= 3) {
147
+ // SQL-only edits can share source data when the connection options are identical.
148
+ const sameOptions = await deploy(CONFIGURED.replace('SELECT * FROM orders', 'SELECT id FROM orders'));
149
+ expect(sameOptions.replicationStreamId).toBe(first.replicationStreamId);
150
+ const storage = await test_utils.getTestStorage(factory, sameOptions);
151
+ await using writer = await storage.createWriter(test_utils.BATCH_OPTIONS);
152
+ await writer.markAllSnapshotDone('2/1');
153
+ await writer.commit('2/1');
154
+ }
155
+ const changed = await deploy(CONFIGURED.replace('sample: 10', 'sample: 20'));
156
+ expect(changed.replicationStreamId).not.toBe(first.replicationStreamId);
157
+ expect((await factory.getActiveSyncConfig())?.content.replicationStreamId).toBe(first.replicationStreamId);
158
+ });
159
+ });
@@ -1,5 +1,6 @@
1
1
  import {
2
2
  ObjectStorage,
3
+ ObjectStorageOperationOptions,
3
4
  ObjectStoragePutMetadata
4
5
  } from '@module/storage/implementation/v3/object-storage/ObjectStorage.js';
5
6
 
@@ -9,13 +10,19 @@ export class MemoryObjectStorage implements ObjectStorage {
9
10
  */
10
11
  public readonly store = new Map<string, { data: Uint8Array; metadata: ObjectStoragePutMetadata }>();
11
12
 
12
- async put(path: string, data: Uint8Array, metadata: ObjectStoragePutMetadata): Promise<void> {
13
+ async put(
14
+ path: string,
15
+ data: Uint8Array,
16
+ metadata: ObjectStoragePutMetadata,
17
+ options?: ObjectStorageOperationOptions
18
+ ): Promise<void> {
19
+ options?.signal?.throwIfAborted();
13
20
  this.store.set(path, { data, metadata: metadata });
14
21
  }
15
22
 
16
23
  async get(
17
24
  path: string,
18
- options?: { signal?: AbortSignal }
25
+ options?: ObjectStorageOperationOptions
19
26
  ): Promise<{ data: Uint8Array; metadata: ObjectStoragePutMetadata }> {
20
27
  const signal = options?.signal;
21
28
  signal?.throwIfAborted();
@@ -26,7 +33,7 @@ export class MemoryObjectStorage implements ObjectStorage {
26
33
  return data;
27
34
  }
28
35
 
29
- async *list(prefix: string, options?: { signal?: AbortSignal }): AsyncIterable<string> {
36
+ async *list(prefix: string, options?: ObjectStorageOperationOptions): AsyncIterable<string> {
30
37
  const signal = options?.signal;
31
38
  for (const path of this.store.keys()) {
32
39
  signal?.throwIfAborted();
@@ -36,18 +43,19 @@ export class MemoryObjectStorage implements ObjectStorage {
36
43
  }
37
44
  }
38
45
 
39
- async delete(paths: string[]): Promise<void> {
46
+ async delete(paths: string[], options?: ObjectStorageOperationOptions): Promise<void> {
47
+ options?.signal?.throwIfAborted();
40
48
  for (const p of paths) {
41
49
  this.store.delete(p);
42
50
  }
43
51
  }
44
52
 
45
- async deletePrefix(prefix: string, options?: { signal?: AbortSignal }): Promise<{ objectCount: number }> {
53
+ async deletePrefix(prefix: string, options?: ObjectStorageOperationOptions): Promise<{ objectCount: number }> {
46
54
  const paths: string[] = [];
47
55
  for await (const path of this.list(prefix, options)) {
48
56
  paths.push(path);
49
57
  }
50
- await this.delete(paths);
58
+ await this.delete(paths, options);
51
59
  return { objectCount: paths.length };
52
60
  }
53
61
  }
@@ -0,0 +1,29 @@
1
+ import { storage } from '@powersync/service-core';
2
+ import { test_utils } from '@powersync/service-core-tests';
3
+ import { mongoTestStorageFactoryGenerator } from '../../../dist/utils/test-utils.js';
4
+
5
+ // A separate process exercises the database protocol without sharing any mutex,
6
+ // allocator, MongoClient, or lease object with the test runner.
7
+ async function main() {
8
+ const factoryGen = mongoTestStorageFactoryGenerator({ url: process.argv[2], isCI: true });
9
+ await using factory = await factoryGen.factory({ doNotClear: true });
10
+ const streamId = Number(process.argv[3]);
11
+ const stream = (await factory.getReplicatingReplicationStreams()).find((s) => s.replicationStreamId === streamId);
12
+ const lock = await stream.lock();
13
+ await using lockLifetime = { [Symbol.asyncDispose]: () => lock.release() };
14
+ const bucketStorage = factory.getInstance(stream, { replicationLock: lock });
15
+ await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
16
+ const table = await test_utils.resolveTestTable(writer, 'items', ['id'], factoryGen, streamId);
17
+ await writer.save({
18
+ sourceTable: table,
19
+ tag: storage.SaveOperationTag.INSERT,
20
+ after: { id: 'child', description: 'child' },
21
+ afterReplicaId: test_utils.rid('child')
22
+ });
23
+ await writer.commit('1/2');
24
+ console.log(`checkpoint=${(await bucketStorage.getCheckpoint()).checkpoint}`);
25
+ }
26
+
27
+ await main();
28
+ // Workspace test utilities start background timers. Exit after all resources are disposed.
29
+ process.exit(0);
@@ -4,7 +4,7 @@ import { mongoTestStorageFactoryGenerator } from '@module/utils/test-utils.js';
4
4
  import { randomUUID } from 'node:crypto';
5
5
  import { MemoryObjectStorage } from './MemoryObjectStorage.js';
6
6
 
7
- const minioStorages: S3ObjectStorage[] = [];
7
+ const rustfsStorages: S3ObjectStorage[] = [];
8
8
 
9
9
  export interface S3TestFactoryOptions {
10
10
  url: string;
@@ -26,24 +26,24 @@ function createTestStorageSuite(options: S3TestFactoryOptions, objectStorage: Ob
26
26
 
27
27
  /**
28
28
  * Creates an ObjectStorage instance for S3 tests.
29
- * Set MINIO_ENDPOINT to switch all S3 tests from MemoryObjectStorage
30
- * to a real MinIO/S3 endpoint.
31
- * MINIO_ENDPOINT=http://localhost:9000
29
+ * Set RUSTFS_ENDPOINT to switch all S3 tests from MemoryObjectStorage
30
+ * to a real RustFS/S3 endpoint.
31
+ * RUSTFS_ENDPOINT=http://localhost:9000
32
32
  */
33
33
  export function createS3TestStorageSuite(options: S3TestFactoryOptions) {
34
- const minioEndpoint = process.env.MINIO_ENDPOINT;
34
+ const rustfsEndpoint = process.env.RUSTFS_ENDPOINT;
35
35
  let objectStorage: ObjectStorage;
36
- if (minioEndpoint) {
36
+ if (rustfsEndpoint) {
37
37
  const s3 = new S3ObjectStorage({
38
38
  bucket: 'powersync-s3-test',
39
39
  region: 'us-east-1',
40
40
  prefix: `test-${process.pid}-${randomUUID()}`,
41
- endpoint: minioEndpoint,
41
+ endpoint: rustfsEndpoint,
42
42
  forcePathStyle: true,
43
- accessKeyId: process.env.MINIO_ACCESS_KEY ?? 'minioadmin',
44
- secretAccessKey: process.env.MINIO_SECRET_KEY ?? 'minioadmin'
43
+ accessKeyId: process.env.RUSTFS_ACCESS_KEY ?? 'rustfsadmin',
44
+ secretAccessKey: process.env.RUSTFS_SECRET_KEY ?? 'rustfsadmin'
45
45
  });
46
- minioStorages.push(s3);
46
+ rustfsStorages.push(s3);
47
47
  objectStorage = s3;
48
48
  } else {
49
49
  objectStorage = new MemoryObjectStorage();
@@ -52,9 +52,9 @@ export function createS3TestStorageSuite(options: S3TestFactoryOptions) {
52
52
  return createTestStorageSuite(options, objectStorage);
53
53
  }
54
54
 
55
- /** Remove all objects created by MinIO-backed suites in the current test. */
55
+ /** Remove all objects created by RustFS-backed suites in the current test. */
56
56
  export async function cleanupS3TestStorage(): Promise<void> {
57
- for (const storage of minioStorages.splice(0)) {
57
+ for (const storage of rustfsStorages.splice(0)) {
58
58
  await storage.deletePrefix('bucket-data/');
59
59
  }
60
60
  }
@@ -0,0 +1,268 @@
1
+ import { MongoSyncBucketStorage } from '@module/storage/implementation/createMongoSyncBucketStorage.js';
2
+ import { BucketDataDocumentV3, ObjectStorageUsageDocument } from '@module/storage/implementation/v3/models.js';
3
+ import {
4
+ OBJECT_STORAGE_USAGE_BASE_WRITER_ID,
5
+ ObjectStorageUsage
6
+ } from '@module/storage/implementation/v3/object-storage/ObjectStorageUsage.js';
7
+ import { VersionedPowerSyncMongoV3 } from '@module/storage/implementation/v3/VersionedPowerSyncMongoV3.js';
8
+ import * as lib_mongo from '@powersync/lib-service-mongodb';
9
+ import { mongo } from '@powersync/lib-service-mongodb';
10
+ import { updateSyncRulesFromYaml } from '@powersync/service-core';
11
+ import { getTestStorage } from '@powersync/service-core-tests';
12
+ import { BucketDefinitionId } from '@powersync/service-sync-rules';
13
+ import { describe, expect, test } from 'vitest';
14
+ import { INITIALIZED_MONGO_STORAGE_FACTORY } from './util.js';
15
+
16
+ const SYNC_RULES_YAML = `
17
+ bucket_definitions:
18
+ global:
19
+ data:
20
+ - SELECT id FROM items
21
+ `;
22
+
23
+ type UsageContext = {
24
+ db: VersionedPowerSyncMongoV3;
25
+ bucketStorage: MongoSyncBucketStorage;
26
+ definitionId: BucketDefinitionId;
27
+ };
28
+
29
+ type UsageEntry = {
30
+ definition_id: BucketDefinitionId;
31
+ active_bytes: bigint;
32
+ };
33
+
34
+ async function withUsageContext<T>(callback: (context: UsageContext) => Promise<T>): Promise<T> {
35
+ await using factory = await INITIALIZED_MONGO_STORAGE_FACTORY.factory();
36
+ const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(SYNC_RULES_YAML, { storageVersion: 3 }));
37
+ const bucketStorage = (await getTestStorage(factory, syncRules)) as MongoSyncBucketStorage;
38
+ const db = bucketStorage.db as VersionedPowerSyncMongoV3;
39
+ const definitionId = syncRules.syncConfigContent[0].mapping.allBucketDefinitionIds()[0];
40
+ return callback({ db, bucketStorage, definitionId });
41
+ }
42
+
43
+ async function withTransaction<T>(
44
+ db: VersionedPowerSyncMongoV3,
45
+ callback: (session: mongo.ClientSession) => Promise<T>
46
+ ): Promise<T> {
47
+ const session = db.client.startSession();
48
+ try {
49
+ return await session.withTransaction(() => callback(session));
50
+ } finally {
51
+ await session.endSession();
52
+ }
53
+ }
54
+
55
+ async function readUsageEntries(
56
+ db: VersionedPowerSyncMongoV3,
57
+ replicationStreamId: number,
58
+ session?: mongo.ClientSession
59
+ ): Promise<UsageEntry[]> {
60
+ const entries = await db.objectStorageUsage
61
+ .aggregate<{ _id: BucketDefinitionId; active_bytes: bigint }>(
62
+ [
63
+ { $match: { '_id.g': replicationStreamId } },
64
+ { $project: { definitions: { $objectToArray: '$definitions' } } },
65
+ { $unwind: '$definitions' },
66
+ { $group: { _id: '$definitions.k', active_bytes: { $sum: '$definitions.v' } } }
67
+ ],
68
+ { session, readConcern: 'snapshot' }
69
+ )
70
+ .toArray()
71
+ .catch((error) => {
72
+ if (lib_mongo.isMongoNamespaceNotFoundError(error)) {
73
+ return [];
74
+ }
75
+ throw error;
76
+ });
77
+
78
+ return entries.map((entry) => ({
79
+ definition_id: entry._id,
80
+ active_bytes: BigInt(entry.active_bytes ?? 0)
81
+ }));
82
+ }
83
+
84
+ function usageDocument(
85
+ replicationStreamId: number,
86
+ writerId: string,
87
+ definitions: Record<string, bigint>,
88
+ updatedAt = new Date(0)
89
+ ): ObjectStorageUsageDocument {
90
+ return {
91
+ _id: { g: replicationStreamId, w: writerId },
92
+ updated_at: updatedAt,
93
+ definitions
94
+ };
95
+ }
96
+
97
+ function bucketDataDocument(
98
+ replicationStreamId: number,
99
+ definitionId: BucketDefinitionId,
100
+ fileSize: number
101
+ ): BucketDataDocumentV3 {
102
+ return {
103
+ _id: { b: `${replicationStreamId}-${definitionId}`, o: 1n },
104
+ min_op: 1n,
105
+ checksum: 1n,
106
+ count: 1,
107
+ size: fileSize,
108
+ storage_ref: {
109
+ path: `bucket-data/${replicationStreamId}/${definitionId}/1.bson`,
110
+ file_size: fileSize
111
+ }
112
+ };
113
+ }
114
+
115
+ describe('ObjectStorageUsage', () => {
116
+ test('counts a committed reference exactly once after an aborted transaction is retried', async () => {
117
+ await withUsageContext(async ({ db, bucketStorage, definitionId }) => {
118
+ const replicationStreamId = bucketStorage.replicationStreamId;
119
+ const writerId = 'replication-writer';
120
+ const usage = new ObjectStorageUsage(db, replicationStreamId, writerId);
121
+ const bucketData = db.bucketData(replicationStreamId, definitionId);
122
+ const document = bucketDataDocument(replicationStreamId, definitionId, 123);
123
+
124
+ await expect(
125
+ withTransaction(db, async (session) => {
126
+ await bucketData.insertOne(document, { session });
127
+ const writes = db.createWriteBatch(session, { ordered: false });
128
+ usage.applyDelta(definitionId, 123n, writes);
129
+ await writes.execute();
130
+ throw new Error('abort this attempt');
131
+ })
132
+ ).rejects.toThrow('abort this attempt');
133
+
134
+ await withTransaction(db, async (session) => {
135
+ await bucketData.insertOne(document, { session });
136
+ const writes = db.createWriteBatch(session, { ordered: false });
137
+ usage.applyDelta(definitionId, 123n, writes);
138
+ await writes.execute();
139
+ });
140
+
141
+ expect(await bucketData.countDocuments({ storage_ref: { $exists: true } })).toBe(1);
142
+ await expect(readUsageEntries(db, replicationStreamId)).resolves.toEqual([
143
+ { definition_id: definitionId, active_bytes: 123n }
144
+ ]);
145
+ });
146
+ });
147
+
148
+ test('snapshot usage reads observe a complete pre-fold view', async () => {
149
+ await withUsageContext(async ({ db, bucketStorage, definitionId }) => {
150
+ const replicationStreamId = bucketStorage.replicationStreamId;
151
+ const usage = new ObjectStorageUsage(db, replicationStreamId);
152
+ await db.objectStorageUsage.insertMany([
153
+ usageDocument(replicationStreamId, OBJECT_STORAGE_USAGE_BASE_WRITER_ID, { [definitionId]: 100n }),
154
+ usageDocument(replicationStreamId, 'replication', { [definitionId]: 20n }),
155
+ usageDocument(replicationStreamId, 'compactor', { [definitionId]: -5n })
156
+ ]);
157
+
158
+ const readSession = db.client.startSession();
159
+ try {
160
+ readSession.startTransaction({ readConcern: { level: 'snapshot' } });
161
+ await db.objectStorageUsage.findOne(
162
+ { _id: { g: replicationStreamId, w: OBJECT_STORAGE_USAGE_BASE_WRITER_ID } },
163
+ { session: readSession }
164
+ );
165
+
166
+ await usage.foldStaleWriterDeltas({ staleWriterMs: 0 });
167
+ const entries = await readUsageEntries(db, replicationStreamId, readSession);
168
+
169
+ await readSession.commitTransaction();
170
+ expect(entries).toEqual([{ definition_id: definitionId, active_bytes: 115n }]);
171
+ } catch (error) {
172
+ await readSession.abortTransaction().catch(() => undefined);
173
+ throw error;
174
+ } finally {
175
+ await readSession.endSession();
176
+ }
177
+
178
+ await expect(readUsageEntries(db, replicationStreamId)).resolves.toEqual([
179
+ { definition_id: definitionId, active_bytes: 115n }
180
+ ]);
181
+ });
182
+ });
183
+
184
+ test('folding preserves totals across both writer/folder commit orders', async () => {
185
+ await withUsageContext(async ({ db, bucketStorage, definitionId }) => {
186
+ const replicationStreamId = bucketStorage.replicationStreamId;
187
+ const writerId = 'reused-writer';
188
+ const usage = new ObjectStorageUsage(db, replicationStreamId, writerId);
189
+
190
+ await withTransaction(db, async (session) => {
191
+ const writes = db.createWriteBatch(session, { ordered: false });
192
+ usage.applyDelta(definitionId, 10n, writes);
193
+ await writes.execute();
194
+ });
195
+ await db.objectStorageUsage.updateOne(
196
+ { _id: { g: replicationStreamId, w: writerId } },
197
+ { $set: { updated_at: new Date(0) } }
198
+ );
199
+ await usage.foldStaleWriterDeltas({ staleWriterMs: 0 });
200
+
201
+ // The folder committed first and deleted the writer shard. The writer's
202
+ // next transaction must recreate it through the upsert.
203
+ await withTransaction(db, async (session) => {
204
+ const writes = db.createWriteBatch(session, { ordered: false });
205
+ usage.applyDelta(definitionId, 4n, writes);
206
+ await writes.execute();
207
+ });
208
+ expect((await readUsageEntries(db, replicationStreamId))[0].active_bytes).toBe(14n);
209
+
210
+ await db.objectStorageUsage.updateOne(
211
+ { _id: { g: replicationStreamId, w: writerId } },
212
+ { $set: { updated_at: new Date(0) } }
213
+ );
214
+ await usage.foldStaleWriterDeltas({ staleWriterMs: 0 });
215
+
216
+ // The writer committed before the folder. Folding the recreated shard
217
+ // must preserve the same visible total.
218
+ expect((await readUsageEntries(db, replicationStreamId))[0].active_bytes).toBe(14n);
219
+ });
220
+ });
221
+
222
+ test('definition cleanup racing a fold removes only that definition, and stream clear is isolated', async () => {
223
+ await withUsageContext(async ({ db, bucketStorage, definitionId }) => {
224
+ const replicationStreamId = bucketStorage.replicationStreamId;
225
+ const otherStreamId = replicationStreamId + 1;
226
+ const otherDefinitionId = 'other_definition';
227
+ const usage = new ObjectStorageUsage(db, replicationStreamId);
228
+
229
+ await db.objectStorageUsage.insertMany([
230
+ usageDocument(replicationStreamId, OBJECT_STORAGE_USAGE_BASE_WRITER_ID, {
231
+ [definitionId]: 3n,
232
+ [otherDefinitionId]: 7n
233
+ }),
234
+ usageDocument(replicationStreamId, 'writer', { [definitionId]: 20n, [otherDefinitionId]: 11n }),
235
+ usageDocument(otherStreamId, 'other-writer', { [definitionId]: 50n })
236
+ ]);
237
+
238
+ await Promise.all([
239
+ usage.foldStaleWriterDeltas({ staleWriterMs: 0 }),
240
+ withTransaction(db, (session) => usage.removeDefinition(definitionId, session))
241
+ ]);
242
+
243
+ await expect(readUsageEntries(db, replicationStreamId)).resolves.toEqual([
244
+ { definition_id: otherDefinitionId, active_bytes: 18n }
245
+ ]);
246
+
247
+ await withTransaction(db, async (session) => {
248
+ await new ObjectStorageUsage(db, otherStreamId).removeStream(session);
249
+ });
250
+ await db.objectStorageUsage.insertOne(usageDocument(otherStreamId, 'other-writer', { [definitionId]: 50n }));
251
+
252
+ await withTransaction(db, async (session) => {
253
+ await usage.removeStream(session);
254
+ });
255
+
256
+ await expect(ObjectStorageUsage.readAllDefinitionUsage(db)).resolves.toEqual([
257
+ { replication_stream_id: otherStreamId, definition_id: definitionId, active_bytes: 50n }
258
+ ]);
259
+ });
260
+ });
261
+
262
+ test('returns no entries for an absent usage collection', async () => {
263
+ await withUsageContext(async ({ db, bucketStorage }) => {
264
+ await db.objectStorageUsage.drop().catch(() => undefined);
265
+ await expect(readUsageEntries(db, bucketStorage.replicationStreamId)).resolves.toEqual([]);
266
+ });
267
+ });
268
+ });