@powersync/service-module-mongodb-storage 0.18.0 → 0.18.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (70) hide show
  1. package/CHANGELOG.md +28 -0
  2. package/dist/storage/MongoBucketStorage.d.ts +3 -0
  3. package/dist/storage/MongoBucketStorage.js +5 -3
  4. package/dist/storage/MongoBucketStorage.js.map +1 -1
  5. package/dist/storage/implementation/MongoBucketBatch.d.ts +2 -2
  6. package/dist/storage/implementation/MongoBucketBatch.js +19 -10
  7. package/dist/storage/implementation/MongoBucketBatch.js.map +1 -1
  8. package/dist/storage/implementation/MongoChecksums.d.ts +29 -4
  9. package/dist/storage/implementation/MongoChecksums.js +32 -10
  10. package/dist/storage/implementation/MongoChecksums.js.map +1 -1
  11. package/dist/storage/implementation/MongoCompactor.d.ts +1 -1
  12. package/dist/storage/implementation/MongoCompactor.js +1 -1
  13. package/dist/storage/implementation/MongoParameterCompactor.d.ts +1 -1
  14. package/dist/storage/implementation/MongoParameterCompactor.js +1 -1
  15. package/dist/storage/implementation/MongoPersistedSyncConfigContent.js +4 -2
  16. package/dist/storage/implementation/MongoPersistedSyncConfigContent.js.map +1 -1
  17. package/dist/storage/implementation/MongoStorageProvider.js +9 -0
  18. package/dist/storage/implementation/MongoStorageProvider.js.map +1 -1
  19. package/dist/storage/implementation/MongoSyncBucketStorage.d.ts +7 -4
  20. package/dist/storage/implementation/MongoSyncBucketStorage.js +11 -4
  21. package/dist/storage/implementation/MongoSyncBucketStorage.js.map +1 -1
  22. package/dist/storage/implementation/MongoWriteCheckpointAPI.d.ts +1 -1
  23. package/dist/storage/implementation/MongoWriteCheckpointAPI.js +47 -12
  24. package/dist/storage/implementation/MongoWriteCheckpointAPI.js.map +1 -1
  25. package/dist/storage/implementation/db.js +8 -1
  26. package/dist/storage/implementation/db.js.map +1 -1
  27. package/dist/storage/implementation/v1/MongoBucketBatchV1.js +17 -2
  28. package/dist/storage/implementation/v1/MongoBucketBatchV1.js.map +1 -1
  29. package/dist/storage/implementation/v1/MongoChecksumsV1.d.ts +4 -4
  30. package/dist/storage/implementation/v1/MongoChecksumsV1.js +7 -6
  31. package/dist/storage/implementation/v1/MongoChecksumsV1.js.map +1 -1
  32. package/dist/storage/implementation/v1/MongoCompactorV1.js +1 -1
  33. package/dist/storage/implementation/v1/MongoCompactorV1.js.map +1 -1
  34. package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.d.ts +4 -3
  35. package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js +17 -4
  36. package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js.map +1 -1
  37. package/dist/storage/implementation/v3/MongoBucketBatchV3.js +20 -2
  38. package/dist/storage/implementation/v3/MongoBucketBatchV3.js.map +1 -1
  39. package/dist/storage/implementation/v3/MongoChecksumsV3.d.ts +3 -3
  40. package/dist/storage/implementation/v3/MongoChecksumsV3.js +8 -7
  41. package/dist/storage/implementation/v3/MongoChecksumsV3.js.map +1 -1
  42. package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.d.ts +4 -3
  43. package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js +16 -3
  44. package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js.map +1 -1
  45. package/dist/types/types.d.ts +12 -1
  46. package/dist/types/types.js +14 -1
  47. package/dist/types/types.js.map +1 -1
  48. package/package.json +7 -7
  49. package/src/storage/MongoBucketStorage.ts +7 -3
  50. package/src/storage/implementation/MongoBucketBatch.ts +23 -13
  51. package/src/storage/implementation/MongoChecksums.ts +84 -15
  52. package/src/storage/implementation/MongoCompactor.ts +1 -1
  53. package/src/storage/implementation/MongoParameterCompactor.ts +1 -1
  54. package/src/storage/implementation/MongoPersistedSyncConfigContent.ts +4 -2
  55. package/src/storage/implementation/MongoStorageProvider.ts +10 -0
  56. package/src/storage/implementation/MongoSyncBucketStorage.ts +18 -8
  57. package/src/storage/implementation/MongoWriteCheckpointAPI.ts +62 -15
  58. package/src/storage/implementation/db.ts +8 -1
  59. package/src/storage/implementation/v1/MongoBucketBatchV1.ts +18 -2
  60. package/src/storage/implementation/v1/MongoChecksumsV1.ts +20 -11
  61. package/src/storage/implementation/v1/MongoCompactorV1.ts +2 -1
  62. package/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts +23 -6
  63. package/src/storage/implementation/v3/MongoBucketBatchV3.ts +20 -2
  64. package/src/storage/implementation/v3/MongoChecksumsV3.ts +23 -11
  65. package/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts +22 -5
  66. package/src/types/types.ts +17 -1
  67. package/test/src/storage_compacting.test.ts +4 -4
  68. package/test/src/storage_sync.test.ts +51 -5
  69. package/test/src/storeCurrentData.test.ts +9 -3
  70. package/tsconfig.tsbuildinfo +1 -1
@@ -1,5 +1,7 @@
1
1
  import * as service_types from '@powersync/service-types';
2
2
  import * as t from 'ts-codec';
3
+ export declare const MongoStorageReadPreference: t.Union<t.Codec<"primary" | "primaryPreferred" | "secondary" | "secondaryPreferred", "primary" | "primaryPreferred" | "secondary" | "secondaryPreferred", string, t.CodecProps>, t.LiteralCodec<"nearest">>;
4
+ export type MongoStorageReadPreference = t.Encoded<typeof MongoStorageReadPreference>;
3
5
  export declare const MongoStorageConfig: t.Intersection<t.Codec<{
4
6
  type: "mongodb";
5
7
  uri: string;
@@ -14,7 +16,16 @@ export declare const MongoStorageConfig: t.Intersection<t.Codec<{
14
16
  username?: string | undefined;
15
17
  password?: string | undefined;
16
18
  reject_ip_ranges?: string[] | undefined;
17
- }, string, t.CodecProps>, t.ObjectCodec<{}>>;
19
+ }, string, t.CodecProps>, t.ObjectCodec<{
20
+ /**
21
+ * Read preference for bulk checksum and bucket data reads.
22
+ *
23
+ * If unset, MongoDB driver defaults are used for backwards compatibility.
24
+ *
25
+ * This is an experimental option, and may be removed in a future release.
26
+ */
27
+ bulk_read_preference: t.OptionalCodec<t.Codec<"primary" | "primaryPreferred" | "secondary" | "secondaryPreferred" | "nearest", "primary" | "primaryPreferred" | "secondary" | "secondaryPreferred" | "nearest", string, t.CodecProps>>;
28
+ }>>;
18
29
  export type MongoStorageConfig = t.Encoded<typeof MongoStorageConfig>;
19
30
  export type MongoStorageConfigDecoded = t.Decoded<typeof MongoStorageConfig>;
20
31
  export declare function isMongoStorageConfig(config: service_types.configFile.GenericStorageConfig): config is MongoStorageConfig;
@@ -1,7 +1,20 @@
1
1
  import * as lib_mongo from '@powersync/lib-service-mongodb';
2
2
  import * as t from 'ts-codec';
3
+ export const MongoStorageReadPreference = t
4
+ .literal('primary')
5
+ .or(t.literal('primaryPreferred'))
6
+ .or(t.literal('secondary'))
7
+ .or(t.literal('secondaryPreferred'))
8
+ .or(t.literal('nearest'));
3
9
  export const MongoStorageConfig = lib_mongo.BaseMongoConfig.and(t.object({
4
- // Add any mongo specific storage settings here in future
10
+ /**
11
+ * Read preference for bulk checksum and bucket data reads.
12
+ *
13
+ * If unset, MongoDB driver defaults are used for backwards compatibility.
14
+ *
15
+ * This is an experimental option, and may be removed in a future release.
16
+ */
17
+ bulk_read_preference: MongoStorageReadPreference.optional()
5
18
  }));
6
19
  export function isMongoStorageConfig(config) {
7
20
  return config.type == lib_mongo.MONGO_CONNECTION_TYPE;
@@ -1 +1 @@
1
- {"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/types/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,SAAS,MAAM,gCAAgC,CAAC;AAE5D,OAAO,KAAK,CAAC,MAAM,UAAU,CAAC;AAE9B,MAAM,CAAC,MAAM,kBAAkB,GAAG,SAAS,CAAC,eAAe,CAAC,GAAG,CAC7D,CAAC,CAAC,MAAM,CAAC;AACP,yDAAyD;CAC1D,CAAC,CACH,CAAC;AAKF,MAAM,UAAU,oBAAoB,CAClC,MAAqD;IAErD,OAAO,MAAM,CAAC,IAAI,IAAI,SAAS,CAAC,qBAAqB,CAAC;AACxD,CAAC"}
1
+ {"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/types/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,SAAS,MAAM,gCAAgC,CAAC;AAE5D,OAAO,KAAK,CAAC,MAAM,UAAU,CAAC;AAE9B,MAAM,CAAC,MAAM,0BAA0B,GAAG,CAAC;KACxC,OAAO,CAAC,SAAS,CAAC;KAClB,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,kBAAkB,CAAC,CAAC;KACjC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;KAC1B,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,oBAAoB,CAAC,CAAC;KACnC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC;AAI5B,MAAM,CAAC,MAAM,kBAAkB,GAAG,SAAS,CAAC,eAAe,CAAC,GAAG,CAC7D,CAAC,CAAC,MAAM,CAAC;IACP;;;;;;OAMG;IACH,oBAAoB,EAAE,0BAA0B,CAAC,QAAQ,EAAE;CAC5D,CAAC,CACH,CAAC;AAKF,MAAM,UAAU,oBAAoB,CAClC,MAAqD;IAErD,OAAO,MAAM,CAAC,IAAI,IAAI,SAAS,CAAC,qBAAqB,CAAC;AACxD,CAAC"}
package/package.json CHANGED
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "name": "@powersync/service-module-mongodb-storage",
3
3
  "repository": "https://github.com/powersync-ja/powersync-service",
4
- "version": "0.18.0",
4
+ "version": "0.18.2",
5
5
  "license": "FSL-1.1-ALv2",
6
6
  "type": "module",
7
7
  "publishConfig": {
@@ -17,15 +17,15 @@
17
17
  "lru-cache": "^10.2.2",
18
18
  "ts-codec": "^1.3.0",
19
19
  "uuid": "^14.0.0",
20
- "@powersync/lib-service-mongodb": "0.6.27",
21
- "@powersync/lib-services-framework": "0.9.6",
22
- "@powersync/service-core": "1.23.0",
20
+ "@powersync/lib-service-mongodb": "0.6.28",
21
+ "@powersync/lib-services-framework": "0.9.7",
22
+ "@powersync/service-core": "1.23.2",
23
23
  "@powersync/service-jsonbig": "0.17.13",
24
- "@powersync/service-sync-rules": "0.38.0",
25
- "@powersync/service-types": "0.16.0"
24
+ "@powersync/service-sync-rules": "0.38.1",
25
+ "@powersync/service-types": "0.16.1"
26
26
  },
27
27
  "devDependencies": {
28
- "@powersync/service-core-tests": "0.17.0"
28
+ "@powersync/service-core-tests": "0.17.1"
29
29
  },
30
30
  "scripts": {
31
31
  "build": "tsc -b",
@@ -34,7 +34,9 @@ export interface MongoBucketStorageOptions {
34
34
  * Prefix for replication stream name and Postgres logical replication slot name.
35
35
  */
36
36
  replicationStreamNamePrefix: string;
37
+ readPreference?: mongo.ReadPreference;
37
38
  checksumOptions?: Omit<MongoChecksumOptions, 'storageConfig'>;
39
+ checksumCacheTtlMs?: number;
38
40
  /**
39
41
  * Reuse a compatible active replication stream by appending a new sync config.
40
42
  *
@@ -91,6 +93,8 @@ export class MongoBucketStorage extends storage.BucketStorageFactory {
91
93
  undefined,
92
94
  {
93
95
  checksumOptions: this.options.checksumOptions,
96
+ readPreference: this.options.readPreference,
97
+ checksumCacheTtlMs: this.options.checksumCacheTtlMs,
94
98
  storageConfig
95
99
  }
96
100
  );
@@ -458,9 +462,9 @@ export class MongoBucketStorage extends storage.BucketStorageFactory {
458
462
  },
459
463
  [...existingConfigDocs, syncConfigDoc]
460
464
  );
461
- if (updateOptions.lock) {
462
- await stream.lock(session);
463
- }
465
+ // The stream already exists, so an active replication job may already hold the stream lock.
466
+ // Deployment only persists the appended sync config; replication job locking is handled by
467
+ // the replicator.
464
468
  return stream;
465
469
  }
466
470
 
@@ -107,7 +107,7 @@ export abstract class MongoBucketBatch
107
107
  private hooks: storage.StorageHooks | undefined;
108
108
  private clearedError = false;
109
109
 
110
- private tracer: PerformanceTracer<'storage' | 'evaluate'>;
110
+ protected tracer: PerformanceTracer<'storage' | 'evaluate'>;
111
111
 
112
112
  /**
113
113
  * Last written op, if any. This may not reflect a consistent checkpoint.
@@ -198,14 +198,18 @@ export abstract class MongoBucketBatch
198
198
  const batch = this.batch;
199
199
  let last_op: InternalOpId | null = null;
200
200
  let resumeBatch: OperationBatch | null = null;
201
+ let clearedError = false;
201
202
 
202
203
  using _ = this.tracer.span('storage', 'flush');
203
204
 
204
205
  await this.hooks?.beforeBatchFlush?.(this);
205
206
 
206
207
  await this.withReplicationTransaction(`Flushing ${batch?.length ?? 0} ops`, async (session, opSeq) => {
208
+ clearedError = false;
207
209
  if (batch != null) {
208
- resumeBatch = await this.replicateBatch(session, batch, opSeq, options);
210
+ const result = await this.replicateBatch(session, batch, opSeq, options);
211
+ resumeBatch = result.resumeBatch;
212
+ clearedError ||= result.clearedError;
209
213
  }
210
214
 
211
215
  if (this.write_checkpoint_batch.length > 0) {
@@ -217,6 +221,10 @@ export abstract class MongoBucketBatch
217
221
  last_op = opSeq.last();
218
222
  });
219
223
 
224
+ if (clearedError) {
225
+ this.clearedError = true;
226
+ }
227
+
220
228
  // null if done, set if we need another flush
221
229
  this.batch = resumeBatch;
222
230
 
@@ -235,7 +243,7 @@ export abstract class MongoBucketBatch
235
243
  batch: OperationBatch,
236
244
  op_seq: MongoIdSequence,
237
245
  options?: storage.BucketBatchCommitOptions
238
- ): Promise<OperationBatch | null> {
246
+ ): Promise<{ resumeBatch: OperationBatch | null; clearedError: boolean }> {
239
247
  let sizes: Map<string, number> | undefined = undefined;
240
248
  using _ = this.tracer.span('storage', 'replicate_batch');
241
249
  // Only look up current_data sizes if the batch stores current_data and at least one
@@ -340,12 +348,17 @@ export abstract class MongoBucketBatch
340
348
  }
341
349
  }
342
350
 
343
- if (didFlush) {
351
+ const clearedError = didFlush && !this.clearedError;
352
+ if (clearedError) {
353
+ // No need to clear an error more than once per batch, since an error would always result in restarting the batch.
344
354
  using _ = this.tracer.span('storage', 'clear_error');
345
- await this.clearError();
355
+ await this.clearError(this.session);
346
356
  }
347
357
 
348
- return resumeBatch?.hasData() ? resumeBatch : null;
358
+ return {
359
+ resumeBatch: resumeBatch?.hasData() ? resumeBatch : null,
360
+ clearedError
361
+ };
349
362
  }
350
363
 
351
364
  private saveOperation(
@@ -899,12 +912,7 @@ export abstract class MongoBucketBatch
899
912
  return copy;
900
913
  }
901
914
 
902
- protected async clearError(): Promise<void> {
903
- // No need to clear an error more than once per batch, since an error would always result in restarting the batch.
904
- if (this.clearedError) {
905
- return;
906
- }
907
-
915
+ protected async clearError(session: mongo.ClientSession): Promise<void> {
908
916
  await this.db.sync_rules.updateOne(
909
917
  {
910
918
  _id: this.replicationStreamId
@@ -914,9 +922,11 @@ export abstract class MongoBucketBatch
914
922
  last_fatal_error: null,
915
923
  last_fatal_error_ts: null
916
924
  }
925
+ },
926
+ {
927
+ session
917
928
  }
918
929
  );
919
- this.clearedError = true;
920
930
  }
921
931
 
922
932
  /**
@@ -1,3 +1,4 @@
1
+ import { mongo } from '@powersync/lib-service-mongodb';
1
2
  import {
2
3
  addPartialChecksums,
3
4
  bson,
@@ -13,7 +14,9 @@ import {
13
14
  } from '@powersync/service-core';
14
15
  import type { VersionedPowerSyncMongo } from './db.js';
15
16
 
17
+ import { ServiceAssertionError } from '@powersync/lib-services-framework';
16
18
  import { BucketDefinitionId } from '@powersync/service-sync-rules';
19
+ import { setSessionSnapshotTime } from '../../utils/util.js';
17
20
  import { StorageConfig } from './models.js';
18
21
 
19
22
  export interface FetchPartialBucketChecksumByDefinition {
@@ -43,14 +46,40 @@ export interface MongoChecksumOptions {
43
46
  */
44
47
  operationBatchLimit?: number;
45
48
 
49
+ checksumCacheTtlMs?: number;
50
+
46
51
  storageConfig: StorageConfig;
47
52
  }
48
53
 
54
+ /**
55
+ * Read options used in requests to the cache, used to construct a session.
56
+ *
57
+ * readConcern is always 'snapshot'.
58
+ */
59
+ interface MongoChecksumCacheReadContext {
60
+ /**
61
+ * (Optional) Specify a specific Timestamp for the snapshot read.
62
+ */
63
+ snapshotTime?: mongo.Timestamp;
64
+
65
+ /**
66
+ * Read preference. Use undefined for the default,
67
+ */
68
+ readPreference: mongo.ReadPreference | undefined;
69
+ }
70
+
71
+ /**
72
+ * Read options passed into individual methods.
73
+ */
74
+ export interface MongoChecksumSessionContext {
75
+ readOptions: Pick<mongo.CommandOperationOptions, 'session' | 'readPreference' | 'readConcern'>;
76
+ }
77
+
49
78
  const DEFAULT_BUCKET_BATCH_LIMIT = 200;
50
79
  export const DEFAULT_OPERATION_BATCH_LIMIT = 50_000;
51
80
 
52
81
  export abstract class MongoChecksums {
53
- private _cache: ChecksumCache | undefined;
82
+ private _cache: ChecksumCache<MongoChecksumCacheReadContext> | undefined;
54
83
  protected readonly storageConfig: StorageConfig;
55
84
 
56
85
  constructor(
@@ -66,10 +95,11 @@ export abstract class MongoChecksums {
66
95
  *
67
96
  * This means the cache only allocates memory once it is used for the first time.
68
97
  */
69
- private get cache(): ChecksumCache {
70
- this._cache ??= new ChecksumCache({
71
- fetchChecksums: (batch) => {
72
- return this.computePartialChecksums(batch);
98
+ private get cache(): ChecksumCache<MongoChecksumCacheReadContext> {
99
+ this._cache ??= new ChecksumCache<MongoChecksumCacheReadContext>({
100
+ ttlMs: this.options.checksumCacheTtlMs,
101
+ fetchChecksums: (batch, context) => {
102
+ return this.computePartialChecksums(batch, context);
73
103
  }
74
104
  });
75
105
  return this._cache;
@@ -79,8 +109,12 @@ export abstract class MongoChecksums {
79
109
  * Calculate checksums, utilizing the cache for partial checkums, and querying the remainder from
80
110
  * the database (bucket_state + bucket_data).
81
111
  */
82
- async getChecksums(checkpoint: InternalOpId, buckets: BucketChecksumRequest[]): Promise<ChecksumMap> {
83
- return this.cache.getChecksumMap(checkpoint, buckets);
112
+ async getChecksums(
113
+ checkpoint: InternalOpId,
114
+ buckets: BucketChecksumRequest[],
115
+ options: MongoChecksumCacheReadContext
116
+ ): Promise<ChecksumMap> {
117
+ return this.cache.getChecksumMap(checkpoint, buckets, options);
84
118
  }
85
119
 
86
120
  clearCache() {
@@ -96,11 +130,39 @@ export abstract class MongoChecksums {
96
130
  * As long as data is compacted regularly, this should be fast. Large buckets without pre-compacted bucket_state
97
131
  * can be slow.
98
132
  */
99
- private async computePartialChecksums(batch: FetchPartialBucketChecksum[]): Promise<PartialChecksumMap> {
133
+ private async computePartialChecksums(
134
+ batch: FetchPartialBucketChecksum[],
135
+ context: MongoChecksumCacheReadContext | undefined
136
+ ): Promise<PartialChecksumMap> {
100
137
  if (batch.length == 0) {
101
138
  return new Map();
102
139
  }
103
- const preStates = await this.fetchPreStates(batch);
140
+
141
+ if (context == null) {
142
+ throw new ServiceAssertionError(`context is required`);
143
+ }
144
+
145
+ const { snapshotTime, readPreference } = context;
146
+ return this.db.client.withSession({ snapshot: true }, async (session) => {
147
+ if (snapshotTime != null) {
148
+ setSessionSnapshotTime(session, snapshotTime);
149
+ }
150
+ return this.computePartialChecksumsWithSession(batch, {
151
+ readOptions: {
152
+ // This is set on the session, but we set it on each operation as well just in case
153
+ readConcern: 'snapshot',
154
+ readPreference,
155
+ session
156
+ }
157
+ });
158
+ });
159
+ }
160
+
161
+ private async computePartialChecksumsWithSession(
162
+ batch: FetchPartialBucketChecksum[],
163
+ context: MongoChecksumSessionContext
164
+ ): Promise<PartialChecksumMap> {
165
+ const preStates = await this.fetchPreStates(batch, context);
104
166
 
105
167
  const mappedRequests = batch.map((request) => {
106
168
  let start = request.start;
@@ -116,7 +178,7 @@ export abstract class MongoChecksums {
116
178
  };
117
179
  });
118
180
 
119
- const queriedChecksums = await this.computePartialChecksumsDirect(mappedRequests);
181
+ const queriedChecksums = await this.computePartialChecksumsDirect(mappedRequests, context);
120
182
 
121
183
  return new Map<string, PartialOrFullChecksum>(
122
184
  batch.map((request) => {
@@ -142,19 +204,22 @@ export abstract class MongoChecksums {
142
204
  * For large buckets, this can be slow, but should not time out as the underlying queries are performed in
143
205
  * smaller batches.
144
206
  */
145
- public async computePartialChecksumsDirect(batch: FetchPartialBucketChecksum[]): Promise<PartialChecksumMap> {
207
+ private async computePartialChecksumsDirect(
208
+ batch: FetchPartialBucketChecksum[],
209
+ context: MongoChecksumSessionContext
210
+ ): Promise<PartialChecksumMap> {
146
211
  // Limit the number of buckets we query for at a time.
147
212
  const bucketBatchLimit = this.options?.bucketBatchLimit ?? DEFAULT_BUCKET_BATCH_LIMIT;
148
213
 
149
214
  if (batch.length <= bucketBatchLimit) {
150
215
  // Single batch - no need for splitting the batch and merging results
151
- return await this.computePartialChecksumsInternal(batch);
216
+ return await this.computePartialChecksumsInternal(batch, context);
152
217
  }
153
218
  // Split the batch and merge results
154
219
  let results = new Map<string, PartialOrFullChecksum>();
155
220
  for (let i = 0; i < batch.length; i += bucketBatchLimit) {
156
221
  const bucketBatch = batch.slice(i, i + bucketBatchLimit);
157
- const batchResults = await this.computePartialChecksumsInternal(bucketBatch);
222
+ const batchResults = await this.computePartialChecksumsInternal(bucketBatch, context);
158
223
  for (let r of batchResults.values()) {
159
224
  results.set(r.bucket, r);
160
225
  }
@@ -169,10 +234,14 @@ export abstract class MongoChecksums {
169
234
  *
170
235
  * `batch` must be limited to DEFAULT_BUCKET_BATCH_LIMIT buckets before calling this.
171
236
  */
172
- protected abstract computePartialChecksumsInternal(batch: FetchPartialBucketChecksum[]): Promise<PartialChecksumMap>;
237
+ protected abstract computePartialChecksumsInternal(
238
+ batch: FetchPartialBucketChecksum[],
239
+ context: MongoChecksumSessionContext
240
+ ): Promise<PartialChecksumMap>;
173
241
 
174
242
  protected abstract fetchPreStates(
175
- batch: FetchPartialBucketChecksum[]
243
+ batch: FetchPartialBucketChecksum[],
244
+ context: MongoChecksumSessionContext
176
245
  ): Promise<Map<string, { opId: InternalOpId; checksum: BucketChecksum }>>;
177
246
  }
178
247
 
@@ -120,7 +120,7 @@ export abstract class MongoCompactor {
120
120
  /**
121
121
  * Compact buckets by converting operations into MOVE and/or CLEAR operations.
122
122
  *
123
- * See /docs/compacting-operations.md for details.
123
+ * See /docs/storage/compacting-operations.md for details.
124
124
  */
125
125
  async compact() {
126
126
  if (this.buckets) {
@@ -16,7 +16,7 @@ type ParameterCompactionReadDocument = {
16
16
  *
17
17
  * This scans through the entire collection to find data to compact.
18
18
  *
19
- * For background, see the `/docs/parameters-lookups.md` file.
19
+ * For background, see the `/docs/storage/parameter-lookups.md` file.
20
20
  */
21
21
  export class MongoParameterCompactor {
22
22
  constructor(
@@ -69,7 +69,8 @@ export class MongoPersistedSyncConfigContentV1 extends MongoPersistedSyncConfigC
69
69
  replicationStreamName: doc.slot_name ?? `powersync_${doc._id}`,
70
70
  storageVersion: doc.storage_version ?? storage.LEGACY_STORAGE_VERSION,
71
71
  mapping: new SingleSyncConfigBucketDefinitionMapping(),
72
- syncConfigId: null
72
+ syncConfigId: null,
73
+ syncConfigState: doc.state
73
74
  });
74
75
  }
75
76
 
@@ -98,7 +99,8 @@ export class MongoPersistedSyncConfigContentV3 extends MongoPersistedSyncConfigC
98
99
  replicationStreamName: doc.slot_name ?? `powersync_${doc._id}`,
99
100
  storageVersion: doc.storage_version,
100
101
  mapping: SingleSyncConfigBucketDefinitionMapping.fromPersistedMapping(config.rule_mapping),
101
- syncConfigId: config._id
102
+ syncConfigId: config._id,
103
+ syncConfigState: state.state
102
104
  });
103
105
  }
104
106
 
@@ -38,8 +38,18 @@ export class MongoStorageProvider implements storage.StorageProvider {
38
38
  await client.connect();
39
39
 
40
40
  const database = new PowerSyncMongo(client, { database: resolvedConfig.storage.database });
41
+ const readPreference =
42
+ decodedConfig.bulk_read_preference == null
43
+ ? undefined
44
+ : new lib_mongo.mongo.ReadPreference(decodedConfig.bulk_read_preference, undefined, {
45
+ // maxStalenessSeconds is relevant for all modes except 'primary'.
46
+ // 90 is the minimum value.
47
+ maxStalenessSeconds: decodedConfig.bulk_read_preference == 'primary' ? undefined : 90
48
+ });
41
49
  const syncStorageFactory = new MongoBucketStorage(database, {
42
50
  replicationStreamNamePrefix: resolvedConfig.slot_name_prefix,
51
+ readPreference,
52
+ checksumCacheTtlMs: resolvedConfig.api_parameters.bucket_count_cache_ttl_minutes * 60_000,
43
53
  // Right now, only MongoDB source databases supports incremental reprocessing.
44
54
  // Remove this filter when we support it for other source databases.
45
55
  // This assumes a single source connection - revisit if we ever support multiple connections.
@@ -40,6 +40,8 @@ import { MongoWriteCheckpointAPI } from './MongoWriteCheckpointAPI.js';
40
40
 
41
41
  export interface MongoSyncBucketStorageOptions {
42
42
  checksumOptions?: Omit<MongoChecksumOptions, 'storageConfig'>;
43
+ readPreference?: mongo.ReadPreference;
44
+ checksumCacheTtlMs?: number;
43
45
  storageConfig: StorageConfig;
44
46
  }
45
47
 
@@ -80,6 +82,7 @@ export abstract class MongoSyncBucketStorage
80
82
  private writeCheckpointAPI: MongoWriteCheckpointAPI;
81
83
  public readonly logger: Logger;
82
84
  public readonly storageConfig: StorageConfig;
85
+ public readonly readPreference: mongo.ReadPreference | undefined;
83
86
  #storageInitialized = false;
84
87
 
85
88
  constructor(
@@ -92,6 +95,7 @@ export abstract class MongoSyncBucketStorage
92
95
  ) {
93
96
  super();
94
97
  this.storageConfig = options.storageConfig;
98
+ this.readPreference = options.readPreference;
95
99
  this.db = factory.db.versioned(this.storageConfig);
96
100
  this.checksums = this.createMongoChecksums(options);
97
101
  this.writeCheckpointAPI = new MongoWriteCheckpointAPI({
@@ -130,8 +134,8 @@ export abstract class MongoSyncBucketStorage
130
134
  this.writeCheckpointAPI.setWriteCheckpointMode(mode);
131
135
  }
132
136
 
133
- createManagedWriteCheckpoint(checkpoint: storage.ManagedWriteCheckpointOptions): Promise<bigint> {
134
- return this.writeCheckpointAPI.createManagedWriteCheckpoint(checkpoint);
137
+ createManagedWriteCheckpoints(checkpoints: storage.ManagedWriteCheckpointOptions[]): Promise<Map<string, bigint>> {
138
+ return this.writeCheckpointAPI.createManagedWriteCheckpoints(checkpoints);
135
139
  }
136
140
 
137
141
  lastWriteCheckpoint(filters: storage.SyncStorageLastWriteCheckpointFilters): Promise<bigint | null> {
@@ -247,24 +251,30 @@ export abstract class MongoSyncBucketStorage
247
251
  }
248
252
 
249
253
  protected abstract getBucketDataBatchImpl(
250
- checkpoint: utils.InternalOpId,
254
+ checkpoint: MongoReplicationCheckpoint,
251
255
  dataBuckets: storage.BucketDataRequest[],
252
256
  options?: storage.BucketDataBatchOptions
253
257
  ): AsyncIterable<storage.SyncBucketDataChunk>;
254
258
 
255
259
  async *getBucketDataBatch(
256
- checkpoint: utils.InternalOpId,
260
+ checkpoint: storage.ReplicationCheckpoint,
257
261
  dataBuckets: storage.BucketDataRequest[],
258
262
  options?: storage.BucketDataBatchOptions
259
263
  ): AsyncIterable<storage.SyncBucketDataChunk> {
260
- yield* this.getBucketDataBatchImpl(checkpoint, dataBuckets, options);
264
+ yield* this.getBucketDataBatchImpl(checkpoint as MongoReplicationCheckpoint, dataBuckets, options);
261
265
  }
262
266
 
263
267
  async getChecksums(
264
- checkpoint: utils.InternalOpId,
265
- buckets: storage.BucketChecksumRequest[]
268
+ checkpoint: storage.ReplicationCheckpoint,
269
+ buckets: storage.BucketChecksumRequest[],
270
+ options?: storage.BucketChecksumOptions
266
271
  ): Promise<utils.ChecksumMap> {
267
- return this.checksums.getChecksums(checkpoint, buckets);
272
+ const mongoCheckpoint = checkpoint as MongoReplicationCheckpoint;
273
+ const snapshotTime = mongoCheckpoint.snapshotTime; // May be undefined in tests
274
+ return this.checksums.getChecksums(checkpoint.checkpoint, buckets, {
275
+ snapshotTime,
276
+ readPreference: options?.requestHint == 'bulk' ? this.readPreference : undefined
277
+ });
268
278
  }
269
279
 
270
280
  clearChecksumCache() {
@@ -26,30 +26,77 @@ export class MongoWriteCheckpointAPI implements storage.WriteCheckpointAPI {
26
26
  this._mode = mode;
27
27
  }
28
28
 
29
- async createManagedWriteCheckpoint(checkpoint: storage.ManagedWriteCheckpointOptions): Promise<bigint> {
29
+ async createManagedWriteCheckpoints(
30
+ checkpoints: storage.ManagedWriteCheckpointOptions[]
31
+ ): Promise<Map<string, bigint>> {
30
32
  if (this.writeCheckpointMode !== storage.WriteCheckpointMode.MANAGED) {
31
33
  throw new framework.ServiceAssertionError(
32
34
  `Attempting to create a managed Write Checkpoint when the current Write Checkpoint mode is set to "${this.writeCheckpointMode}"`
33
35
  );
34
36
  }
35
37
 
36
- const { user_id, heads: lsns } = checkpoint;
37
- const doc = await this.db.write_checkpoints.findOneAndUpdate(
38
- {
39
- user_id: user_id
40
- },
41
- {
42
- $set: {
43
- lsns,
44
- processed_at_lsn: null
38
+ const uniqueCheckpoints = [...new Map(checkpoints.map((checkpoint) => [checkpoint.user_id, checkpoint])).values()];
39
+ if (uniqueCheckpoints.length == 0) {
40
+ return new Map();
41
+ }
42
+
43
+ if (uniqueCheckpoints.length == 1) {
44
+ // For the common case of a single checkpoint, we can do this in a single request.
45
+ const { user_id, heads: lsns } = uniqueCheckpoints[0];
46
+ const doc = await this.db.write_checkpoints.findOneAndUpdate(
47
+ {
48
+ user_id
45
49
  },
46
- $inc: {
47
- client_id: 1n
50
+ {
51
+ $set: {
52
+ lsns,
53
+ processed_at_lsn: null
54
+ },
55
+ $inc: {
56
+ client_id: 1n
57
+ }
58
+ },
59
+ { upsert: true, returnDocument: 'after' }
60
+ );
61
+
62
+ return new Map([[doc!.user_id, doc!.client_id]]);
63
+ }
64
+
65
+ // For more than one checkpoint, this gives a constant 2 requests
66
+ await this.db.write_checkpoints.bulkWrite(
67
+ uniqueCheckpoints.map(({ user_id, heads: lsns }) => ({
68
+ updateOne: {
69
+ filter: { user_id },
70
+ update: {
71
+ $set: {
72
+ lsns,
73
+ processed_at_lsn: null
74
+ },
75
+ $inc: {
76
+ client_id: 1n
77
+ }
78
+ },
79
+ upsert: true
48
80
  }
49
- },
50
- { upsert: true, returnDocument: 'after' }
81
+ }))
51
82
  );
52
- return doc!.client_id;
83
+
84
+ const userIds = uniqueCheckpoints.map((checkpoint) => checkpoint.user_id);
85
+ const docs = await this.db.write_checkpoints
86
+ .find(
87
+ {
88
+ user_id: { $in: userIds }
89
+ },
90
+ {
91
+ projection: {
92
+ user_id: 1,
93
+ client_id: 1
94
+ }
95
+ }
96
+ )
97
+ .toArray();
98
+
99
+ return new Map(docs.map((doc) => [doc.user_id, doc.client_id]));
53
100
  }
54
101
 
55
102
  async lastWriteCheckpoint(filters: storage.LastWriteCheckpointFilters): Promise<bigint | null> {
@@ -195,7 +195,14 @@ export class PowerSyncMongo {
195
195
  * This is used in a similar way to the Postgres NOTIFY functionality.
196
196
  */
197
197
  async notifyCheckpoint() {
198
- await this.checkpoint_events.insertOne({} as any, { forceServerObjectId: true });
198
+ await this.checkpoint_events.insertOne({} as any, {
199
+ forceServerObjectId: true,
200
+ // No correctness issue if this write is missed / rolled back.
201
+ // It could cause some sync delays, that will resolve in at most a minute, but no consistency issues.
202
+ writeConcern: {
203
+ w: 1
204
+ }
205
+ });
199
206
  }
200
207
 
201
208
  /**