@powersync/service-module-mongodb-storage 0.18.1 → 0.18.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (69) hide show
  1. package/CHANGELOG.md +34 -0
  2. package/dist/storage/MongoBucketStorage.d.ts +3 -0
  3. package/dist/storage/MongoBucketStorage.js +2 -0
  4. package/dist/storage/MongoBucketStorage.js.map +1 -1
  5. package/dist/storage/implementation/MongoBucketBatch.d.ts +2 -2
  6. package/dist/storage/implementation/MongoBucketBatch.js +19 -10
  7. package/dist/storage/implementation/MongoBucketBatch.js.map +1 -1
  8. package/dist/storage/implementation/MongoChecksums.d.ts +31 -4
  9. package/dist/storage/implementation/MongoChecksums.js +33 -10
  10. package/dist/storage/implementation/MongoChecksums.js.map +1 -1
  11. package/dist/storage/implementation/MongoCompactor.d.ts +1 -1
  12. package/dist/storage/implementation/MongoCompactor.js +1 -1
  13. package/dist/storage/implementation/MongoParameterCompactor.d.ts +1 -1
  14. package/dist/storage/implementation/MongoParameterCompactor.js +1 -1
  15. package/dist/storage/implementation/MongoStorageProvider.js +9 -0
  16. package/dist/storage/implementation/MongoStorageProvider.js.map +1 -1
  17. package/dist/storage/implementation/MongoSyncBucketStorage.d.ts +9 -5
  18. package/dist/storage/implementation/MongoSyncBucketStorage.js +23 -6
  19. package/dist/storage/implementation/MongoSyncBucketStorage.js.map +1 -1
  20. package/dist/storage/implementation/MongoWriteCheckpointAPI.d.ts +1 -1
  21. package/dist/storage/implementation/MongoWriteCheckpointAPI.js +47 -12
  22. package/dist/storage/implementation/MongoWriteCheckpointAPI.js.map +1 -1
  23. package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.d.ts +2 -0
  24. package/dist/storage/implementation/db.js +8 -1
  25. package/dist/storage/implementation/db.js.map +1 -1
  26. package/dist/storage/implementation/v1/MongoBucketBatchV1.js +17 -2
  27. package/dist/storage/implementation/v1/MongoBucketBatchV1.js.map +1 -1
  28. package/dist/storage/implementation/v1/MongoChecksumsV1.d.ts +4 -4
  29. package/dist/storage/implementation/v1/MongoChecksumsV1.js +7 -6
  30. package/dist/storage/implementation/v1/MongoChecksumsV1.js.map +1 -1
  31. package/dist/storage/implementation/v1/MongoCompactorV1.js +1 -1
  32. package/dist/storage/implementation/v1/MongoCompactorV1.js.map +1 -1
  33. package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.d.ts +4 -3
  34. package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js +18 -4
  35. package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js.map +1 -1
  36. package/dist/storage/implementation/v3/MongoBucketBatchV3.js +20 -2
  37. package/dist/storage/implementation/v3/MongoBucketBatchV3.js.map +1 -1
  38. package/dist/storage/implementation/v3/MongoChecksumsV3.d.ts +3 -3
  39. package/dist/storage/implementation/v3/MongoChecksumsV3.js +8 -7
  40. package/dist/storage/implementation/v3/MongoChecksumsV3.js.map +1 -1
  41. package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.d.ts +4 -3
  42. package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js +17 -3
  43. package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js.map +1 -1
  44. package/dist/types/types.d.ts +12 -1
  45. package/dist/types/types.js +14 -1
  46. package/dist/types/types.js.map +1 -1
  47. package/package.json +7 -7
  48. package/src/storage/MongoBucketStorage.ts +4 -0
  49. package/src/storage/implementation/MongoBucketBatch.ts +23 -13
  50. package/src/storage/implementation/MongoChecksums.ts +87 -15
  51. package/src/storage/implementation/MongoCompactor.ts +1 -1
  52. package/src/storage/implementation/MongoParameterCompactor.ts +1 -1
  53. package/src/storage/implementation/MongoStorageProvider.ts +10 -0
  54. package/src/storage/implementation/MongoSyncBucketStorage.ts +29 -10
  55. package/src/storage/implementation/MongoWriteCheckpointAPI.ts +62 -15
  56. package/src/storage/implementation/common/MongoSyncBucketStorageCheckpoint.ts +2 -0
  57. package/src/storage/implementation/db.ts +8 -1
  58. package/src/storage/implementation/v1/MongoBucketBatchV1.ts +18 -2
  59. package/src/storage/implementation/v1/MongoChecksumsV1.ts +20 -11
  60. package/src/storage/implementation/v1/MongoCompactorV1.ts +2 -1
  61. package/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts +24 -6
  62. package/src/storage/implementation/v3/MongoBucketBatchV3.ts +20 -2
  63. package/src/storage/implementation/v3/MongoChecksumsV3.ts +23 -11
  64. package/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts +23 -5
  65. package/src/types/types.ts +17 -1
  66. package/test/src/storage_compacting.test.ts +4 -4
  67. package/test/src/storage_sync.test.ts +13 -5
  68. package/test/src/storeCurrentData.test.ts +9 -3
  69. package/tsconfig.tsbuildinfo +1 -1
@@ -1,5 +1,7 @@
1
1
  import * as service_types from '@powersync/service-types';
2
2
  import * as t from 'ts-codec';
3
+ export declare const MongoStorageReadPreference: t.Union<t.Codec<"primary" | "primaryPreferred" | "secondary" | "secondaryPreferred", "primary" | "primaryPreferred" | "secondary" | "secondaryPreferred", string, t.CodecProps>, t.LiteralCodec<"nearest">>;
4
+ export type MongoStorageReadPreference = t.Encoded<typeof MongoStorageReadPreference>;
3
5
  export declare const MongoStorageConfig: t.Intersection<t.Codec<{
4
6
  type: "mongodb";
5
7
  uri: string;
@@ -14,7 +16,16 @@ export declare const MongoStorageConfig: t.Intersection<t.Codec<{
14
16
  username?: string | undefined;
15
17
  password?: string | undefined;
16
18
  reject_ip_ranges?: string[] | undefined;
17
- }, string, t.CodecProps>, t.ObjectCodec<{}>>;
19
+ }, string, t.CodecProps>, t.ObjectCodec<{
20
+ /**
21
+ * Read preference for bulk checksum and bucket data reads.
22
+ *
23
+ * If unset, MongoDB driver defaults are used for backwards compatibility.
24
+ *
25
+ * This is an experimental option, and may be removed in a future release.
26
+ */
27
+ bulk_read_preference: t.OptionalCodec<t.Codec<"primary" | "primaryPreferred" | "secondary" | "secondaryPreferred" | "nearest", "primary" | "primaryPreferred" | "secondary" | "secondaryPreferred" | "nearest", string, t.CodecProps>>;
28
+ }>>;
18
29
  export type MongoStorageConfig = t.Encoded<typeof MongoStorageConfig>;
19
30
  export type MongoStorageConfigDecoded = t.Decoded<typeof MongoStorageConfig>;
20
31
  export declare function isMongoStorageConfig(config: service_types.configFile.GenericStorageConfig): config is MongoStorageConfig;
@@ -1,7 +1,20 @@
1
1
  import * as lib_mongo from '@powersync/lib-service-mongodb';
2
2
  import * as t from 'ts-codec';
3
+ export const MongoStorageReadPreference = t
4
+ .literal('primary')
5
+ .or(t.literal('primaryPreferred'))
6
+ .or(t.literal('secondary'))
7
+ .or(t.literal('secondaryPreferred'))
8
+ .or(t.literal('nearest'));
3
9
  export const MongoStorageConfig = lib_mongo.BaseMongoConfig.and(t.object({
4
- // Add any mongo specific storage settings here in future
10
+ /**
11
+ * Read preference for bulk checksum and bucket data reads.
12
+ *
13
+ * If unset, MongoDB driver defaults are used for backwards compatibility.
14
+ *
15
+ * This is an experimental option, and may be removed in a future release.
16
+ */
17
+ bulk_read_preference: MongoStorageReadPreference.optional()
5
18
  }));
6
19
  export function isMongoStorageConfig(config) {
7
20
  return config.type == lib_mongo.MONGO_CONNECTION_TYPE;
@@ -1 +1 @@
1
- {"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/types/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,SAAS,MAAM,gCAAgC,CAAC;AAE5D,OAAO,KAAK,CAAC,MAAM,UAAU,CAAC;AAE9B,MAAM,CAAC,MAAM,kBAAkB,GAAG,SAAS,CAAC,eAAe,CAAC,GAAG,CAC7D,CAAC,CAAC,MAAM,CAAC;AACP,yDAAyD;CAC1D,CAAC,CACH,CAAC;AAKF,MAAM,UAAU,oBAAoB,CAClC,MAAqD;IAErD,OAAO,MAAM,CAAC,IAAI,IAAI,SAAS,CAAC,qBAAqB,CAAC;AACxD,CAAC"}
1
+ {"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/types/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,SAAS,MAAM,gCAAgC,CAAC;AAE5D,OAAO,KAAK,CAAC,MAAM,UAAU,CAAC;AAE9B,MAAM,CAAC,MAAM,0BAA0B,GAAG,CAAC;KACxC,OAAO,CAAC,SAAS,CAAC;KAClB,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,kBAAkB,CAAC,CAAC;KACjC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;KAC1B,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,oBAAoB,CAAC,CAAC;KACnC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC;AAI5B,MAAM,CAAC,MAAM,kBAAkB,GAAG,SAAS,CAAC,eAAe,CAAC,GAAG,CAC7D,CAAC,CAAC,MAAM,CAAC;IACP;;;;;;OAMG;IACH,oBAAoB,EAAE,0BAA0B,CAAC,QAAQ,EAAE;CAC5D,CAAC,CACH,CAAC;AAKF,MAAM,UAAU,oBAAoB,CAClC,MAAqD;IAErD,OAAO,MAAM,CAAC,IAAI,IAAI,SAAS,CAAC,qBAAqB,CAAC;AACxD,CAAC"}
package/package.json CHANGED
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "name": "@powersync/service-module-mongodb-storage",
3
3
  "repository": "https://github.com/powersync-ja/powersync-service",
4
- "version": "0.18.1",
4
+ "version": "0.18.3",
5
5
  "license": "FSL-1.1-ALv2",
6
6
  "type": "module",
7
7
  "publishConfig": {
@@ -17,15 +17,15 @@
17
17
  "lru-cache": "^10.2.2",
18
18
  "ts-codec": "^1.3.0",
19
19
  "uuid": "^14.0.0",
20
- "@powersync/lib-service-mongodb": "0.6.27",
21
- "@powersync/lib-services-framework": "0.9.6",
22
- "@powersync/service-core": "1.23.1",
20
+ "@powersync/lib-service-mongodb": "0.6.29",
21
+ "@powersync/lib-services-framework": "0.9.8",
22
+ "@powersync/service-core": "1.23.3",
23
23
  "@powersync/service-jsonbig": "0.17.13",
24
- "@powersync/service-sync-rules": "0.38.0",
25
- "@powersync/service-types": "0.16.0"
24
+ "@powersync/service-sync-rules": "0.39.0",
25
+ "@powersync/service-types": "0.16.1"
26
26
  },
27
27
  "devDependencies": {
28
- "@powersync/service-core-tests": "0.17.0"
28
+ "@powersync/service-core-tests": "0.17.2"
29
29
  },
30
30
  "scripts": {
31
31
  "build": "tsc -b",
@@ -34,7 +34,9 @@ export interface MongoBucketStorageOptions {
34
34
  * Prefix for replication stream name and Postgres logical replication slot name.
35
35
  */
36
36
  replicationStreamNamePrefix: string;
37
+ readPreference?: mongo.ReadPreference;
37
38
  checksumOptions?: Omit<MongoChecksumOptions, 'storageConfig'>;
39
+ checksumCacheTtlMs?: number;
38
40
  /**
39
41
  * Reuse a compatible active replication stream by appending a new sync config.
40
42
  *
@@ -91,6 +93,8 @@ export class MongoBucketStorage extends storage.BucketStorageFactory {
91
93
  undefined,
92
94
  {
93
95
  checksumOptions: this.options.checksumOptions,
96
+ readPreference: this.options.readPreference,
97
+ checksumCacheTtlMs: this.options.checksumCacheTtlMs,
94
98
  storageConfig
95
99
  }
96
100
  );
@@ -107,7 +107,7 @@ export abstract class MongoBucketBatch
107
107
  private hooks: storage.StorageHooks | undefined;
108
108
  private clearedError = false;
109
109
 
110
- private tracer: PerformanceTracer<'storage' | 'evaluate'>;
110
+ protected tracer: PerformanceTracer<'storage' | 'evaluate'>;
111
111
 
112
112
  /**
113
113
  * Last written op, if any. This may not reflect a consistent checkpoint.
@@ -198,14 +198,18 @@ export abstract class MongoBucketBatch
198
198
  const batch = this.batch;
199
199
  let last_op: InternalOpId | null = null;
200
200
  let resumeBatch: OperationBatch | null = null;
201
+ let clearedError = false;
201
202
 
202
203
  using _ = this.tracer.span('storage', 'flush');
203
204
 
204
205
  await this.hooks?.beforeBatchFlush?.(this);
205
206
 
206
207
  await this.withReplicationTransaction(`Flushing ${batch?.length ?? 0} ops`, async (session, opSeq) => {
208
+ clearedError = false;
207
209
  if (batch != null) {
208
- resumeBatch = await this.replicateBatch(session, batch, opSeq, options);
210
+ const result = await this.replicateBatch(session, batch, opSeq, options);
211
+ resumeBatch = result.resumeBatch;
212
+ clearedError ||= result.clearedError;
209
213
  }
210
214
 
211
215
  if (this.write_checkpoint_batch.length > 0) {
@@ -217,6 +221,10 @@ export abstract class MongoBucketBatch
217
221
  last_op = opSeq.last();
218
222
  });
219
223
 
224
+ if (clearedError) {
225
+ this.clearedError = true;
226
+ }
227
+
220
228
  // null if done, set if we need another flush
221
229
  this.batch = resumeBatch;
222
230
 
@@ -235,7 +243,7 @@ export abstract class MongoBucketBatch
235
243
  batch: OperationBatch,
236
244
  op_seq: MongoIdSequence,
237
245
  options?: storage.BucketBatchCommitOptions
238
- ): Promise<OperationBatch | null> {
246
+ ): Promise<{ resumeBatch: OperationBatch | null; clearedError: boolean }> {
239
247
  let sizes: Map<string, number> | undefined = undefined;
240
248
  using _ = this.tracer.span('storage', 'replicate_batch');
241
249
  // Only look up current_data sizes if the batch stores current_data and at least one
@@ -340,12 +348,17 @@ export abstract class MongoBucketBatch
340
348
  }
341
349
  }
342
350
 
343
- if (didFlush) {
351
+ const clearedError = didFlush && !this.clearedError;
352
+ if (clearedError) {
353
+ // No need to clear an error more than once per batch, since an error would always result in restarting the batch.
344
354
  using _ = this.tracer.span('storage', 'clear_error');
345
- await this.clearError();
355
+ await this.clearError(this.session);
346
356
  }
347
357
 
348
- return resumeBatch?.hasData() ? resumeBatch : null;
358
+ return {
359
+ resumeBatch: resumeBatch?.hasData() ? resumeBatch : null,
360
+ clearedError
361
+ };
349
362
  }
350
363
 
351
364
  private saveOperation(
@@ -899,12 +912,7 @@ export abstract class MongoBucketBatch
899
912
  return copy;
900
913
  }
901
914
 
902
- protected async clearError(): Promise<void> {
903
- // No need to clear an error more than once per batch, since an error would always result in restarting the batch.
904
- if (this.clearedError) {
905
- return;
906
- }
907
-
915
+ protected async clearError(session: mongo.ClientSession): Promise<void> {
908
916
  await this.db.sync_rules.updateOne(
909
917
  {
910
918
  _id: this.replicationStreamId
@@ -914,9 +922,11 @@ export abstract class MongoBucketBatch
914
922
  last_fatal_error: null,
915
923
  last_fatal_error_ts: null
916
924
  }
925
+ },
926
+ {
927
+ session
917
928
  }
918
929
  );
919
- this.clearedError = true;
920
930
  }
921
931
 
922
932
  /**
@@ -1,3 +1,4 @@
1
+ import { mongo } from '@powersync/lib-service-mongodb';
1
2
  import {
2
3
  addPartialChecksums,
3
4
  bson,
@@ -13,7 +14,9 @@ import {
13
14
  } from '@powersync/service-core';
14
15
  import type { VersionedPowerSyncMongo } from './db.js';
15
16
 
17
+ import { ServiceAssertionError } from '@powersync/lib-services-framework';
16
18
  import { BucketDefinitionId } from '@powersync/service-sync-rules';
19
+ import { setSessionSnapshotTime } from '../../utils/util.js';
17
20
  import { StorageConfig } from './models.js';
18
21
 
19
22
  export interface FetchPartialBucketChecksumByDefinition {
@@ -43,14 +46,42 @@ export interface MongoChecksumOptions {
43
46
  */
44
47
  operationBatchLimit?: number;
45
48
 
49
+ checksumCacheTtlMs?: number;
50
+
46
51
  storageConfig: StorageConfig;
47
52
  }
48
53
 
54
+ /**
55
+ * Read options used in requests to the cache, used to construct a session.
56
+ *
57
+ * readConcern is always 'snapshot'.
58
+ */
59
+ interface MongoChecksumCacheReadContext {
60
+ /**
61
+ * (Optional) Specify a specific Timestamp for the snapshot read.
62
+ * Must include clusterTime as well.
63
+ */
64
+ snapshotTime?: mongo.Timestamp;
65
+ clusterTime?: mongo.ClusterTime;
66
+
67
+ /**
68
+ * Read preference. Use undefined for the default,
69
+ */
70
+ readPreference: mongo.ReadPreference | undefined;
71
+ }
72
+
73
+ /**
74
+ * Read options passed into individual methods.
75
+ */
76
+ export interface MongoChecksumSessionContext {
77
+ readOptions: Pick<mongo.CommandOperationOptions, 'session' | 'readPreference' | 'readConcern'>;
78
+ }
79
+
49
80
  const DEFAULT_BUCKET_BATCH_LIMIT = 200;
50
81
  export const DEFAULT_OPERATION_BATCH_LIMIT = 50_000;
51
82
 
52
83
  export abstract class MongoChecksums {
53
- private _cache: ChecksumCache | undefined;
84
+ private _cache: ChecksumCache<MongoChecksumCacheReadContext> | undefined;
54
85
  protected readonly storageConfig: StorageConfig;
55
86
 
56
87
  constructor(
@@ -66,10 +97,11 @@ export abstract class MongoChecksums {
66
97
  *
67
98
  * This means the cache only allocates memory once it is used for the first time.
68
99
  */
69
- private get cache(): ChecksumCache {
70
- this._cache ??= new ChecksumCache({
71
- fetchChecksums: (batch) => {
72
- return this.computePartialChecksums(batch);
100
+ private get cache(): ChecksumCache<MongoChecksumCacheReadContext> {
101
+ this._cache ??= new ChecksumCache<MongoChecksumCacheReadContext>({
102
+ ttlMs: this.options.checksumCacheTtlMs,
103
+ fetchChecksums: (batch, context) => {
104
+ return this.computePartialChecksums(batch, context);
73
105
  }
74
106
  });
75
107
  return this._cache;
@@ -79,8 +111,12 @@ export abstract class MongoChecksums {
79
111
  * Calculate checksums, utilizing the cache for partial checkums, and querying the remainder from
80
112
  * the database (bucket_state + bucket_data).
81
113
  */
82
- async getChecksums(checkpoint: InternalOpId, buckets: BucketChecksumRequest[]): Promise<ChecksumMap> {
83
- return this.cache.getChecksumMap(checkpoint, buckets);
114
+ async getChecksums(
115
+ checkpoint: InternalOpId,
116
+ buckets: BucketChecksumRequest[],
117
+ options: MongoChecksumCacheReadContext
118
+ ): Promise<ChecksumMap> {
119
+ return this.cache.getChecksumMap(checkpoint, buckets, options);
84
120
  }
85
121
 
86
122
  clearCache() {
@@ -96,11 +132,40 @@ export abstract class MongoChecksums {
96
132
  * As long as data is compacted regularly, this should be fast. Large buckets without pre-compacted bucket_state
97
133
  * can be slow.
98
134
  */
99
- private async computePartialChecksums(batch: FetchPartialBucketChecksum[]): Promise<PartialChecksumMap> {
135
+ private async computePartialChecksums(
136
+ batch: FetchPartialBucketChecksum[],
137
+ context: MongoChecksumCacheReadContext | undefined
138
+ ): Promise<PartialChecksumMap> {
100
139
  if (batch.length == 0) {
101
140
  return new Map();
102
141
  }
103
- const preStates = await this.fetchPreStates(batch);
142
+
143
+ if (context == null) {
144
+ throw new ServiceAssertionError(`context is required`);
145
+ }
146
+
147
+ const { snapshotTime, clusterTime, readPreference } = context;
148
+ return this.db.client.withSession({ snapshot: true }, async (session) => {
149
+ if (snapshotTime != null && clusterTime != null) {
150
+ setSessionSnapshotTime(session, snapshotTime);
151
+ session.advanceClusterTime(clusterTime);
152
+ }
153
+ return this.computePartialChecksumsWithSession(batch, {
154
+ readOptions: {
155
+ // This is set on the session, but we set it on each operation as well just in case
156
+ readConcern: 'snapshot',
157
+ readPreference,
158
+ session
159
+ }
160
+ });
161
+ });
162
+ }
163
+
164
+ private async computePartialChecksumsWithSession(
165
+ batch: FetchPartialBucketChecksum[],
166
+ context: MongoChecksumSessionContext
167
+ ): Promise<PartialChecksumMap> {
168
+ const preStates = await this.fetchPreStates(batch, context);
104
169
 
105
170
  const mappedRequests = batch.map((request) => {
106
171
  let start = request.start;
@@ -116,7 +181,7 @@ export abstract class MongoChecksums {
116
181
  };
117
182
  });
118
183
 
119
- const queriedChecksums = await this.computePartialChecksumsDirect(mappedRequests);
184
+ const queriedChecksums = await this.computePartialChecksumsDirect(mappedRequests, context);
120
185
 
121
186
  return new Map<string, PartialOrFullChecksum>(
122
187
  batch.map((request) => {
@@ -142,19 +207,22 @@ export abstract class MongoChecksums {
142
207
  * For large buckets, this can be slow, but should not time out as the underlying queries are performed in
143
208
  * smaller batches.
144
209
  */
145
- public async computePartialChecksumsDirect(batch: FetchPartialBucketChecksum[]): Promise<PartialChecksumMap> {
210
+ private async computePartialChecksumsDirect(
211
+ batch: FetchPartialBucketChecksum[],
212
+ context: MongoChecksumSessionContext
213
+ ): Promise<PartialChecksumMap> {
146
214
  // Limit the number of buckets we query for at a time.
147
215
  const bucketBatchLimit = this.options?.bucketBatchLimit ?? DEFAULT_BUCKET_BATCH_LIMIT;
148
216
 
149
217
  if (batch.length <= bucketBatchLimit) {
150
218
  // Single batch - no need for splitting the batch and merging results
151
- return await this.computePartialChecksumsInternal(batch);
219
+ return await this.computePartialChecksumsInternal(batch, context);
152
220
  }
153
221
  // Split the batch and merge results
154
222
  let results = new Map<string, PartialOrFullChecksum>();
155
223
  for (let i = 0; i < batch.length; i += bucketBatchLimit) {
156
224
  const bucketBatch = batch.slice(i, i + bucketBatchLimit);
157
- const batchResults = await this.computePartialChecksumsInternal(bucketBatch);
225
+ const batchResults = await this.computePartialChecksumsInternal(bucketBatch, context);
158
226
  for (let r of batchResults.values()) {
159
227
  results.set(r.bucket, r);
160
228
  }
@@ -169,10 +237,14 @@ export abstract class MongoChecksums {
169
237
  *
170
238
  * `batch` must be limited to DEFAULT_BUCKET_BATCH_LIMIT buckets before calling this.
171
239
  */
172
- protected abstract computePartialChecksumsInternal(batch: FetchPartialBucketChecksum[]): Promise<PartialChecksumMap>;
240
+ protected abstract computePartialChecksumsInternal(
241
+ batch: FetchPartialBucketChecksum[],
242
+ context: MongoChecksumSessionContext
243
+ ): Promise<PartialChecksumMap>;
173
244
 
174
245
  protected abstract fetchPreStates(
175
- batch: FetchPartialBucketChecksum[]
246
+ batch: FetchPartialBucketChecksum[],
247
+ context: MongoChecksumSessionContext
176
248
  ): Promise<Map<string, { opId: InternalOpId; checksum: BucketChecksum }>>;
177
249
  }
178
250
 
@@ -120,7 +120,7 @@ export abstract class MongoCompactor {
120
120
  /**
121
121
  * Compact buckets by converting operations into MOVE and/or CLEAR operations.
122
122
  *
123
- * See /docs/compacting-operations.md for details.
123
+ * See /docs/storage/compacting-operations.md for details.
124
124
  */
125
125
  async compact() {
126
126
  if (this.buckets) {
@@ -16,7 +16,7 @@ type ParameterCompactionReadDocument = {
16
16
  *
17
17
  * This scans through the entire collection to find data to compact.
18
18
  *
19
- * For background, see the `/docs/parameters-lookups.md` file.
19
+ * For background, see the `/docs/storage/parameter-lookups.md` file.
20
20
  */
21
21
  export class MongoParameterCompactor {
22
22
  constructor(
@@ -38,8 +38,18 @@ export class MongoStorageProvider implements storage.StorageProvider {
38
38
  await client.connect();
39
39
 
40
40
  const database = new PowerSyncMongo(client, { database: resolvedConfig.storage.database });
41
+ const readPreference =
42
+ decodedConfig.bulk_read_preference == null
43
+ ? undefined
44
+ : new lib_mongo.mongo.ReadPreference(decodedConfig.bulk_read_preference, undefined, {
45
+ // maxStalenessSeconds is relevant for all modes except 'primary'.
46
+ // 90 is the minimum value.
47
+ maxStalenessSeconds: decodedConfig.bulk_read_preference == 'primary' ? undefined : 90
48
+ });
41
49
  const syncStorageFactory = new MongoBucketStorage(database, {
42
50
  replicationStreamNamePrefix: resolvedConfig.slot_name_prefix,
51
+ readPreference,
52
+ checksumCacheTtlMs: resolvedConfig.api_parameters.bucket_count_cache_ttl_minutes * 60_000,
43
53
  // Right now, only MongoDB source databases supports incremental reprocessing.
44
54
  // Remove this filter when we support it for other source databases.
45
55
  // This assumes a single source connection - revisit if we ever support multiple connections.
@@ -40,6 +40,8 @@ import { MongoWriteCheckpointAPI } from './MongoWriteCheckpointAPI.js';
40
40
 
41
41
  export interface MongoSyncBucketStorageOptions {
42
42
  checksumOptions?: Omit<MongoChecksumOptions, 'storageConfig'>;
43
+ readPreference?: mongo.ReadPreference;
44
+ checksumCacheTtlMs?: number;
43
45
  storageConfig: StorageConfig;
44
46
  }
45
47
 
@@ -80,6 +82,7 @@ export abstract class MongoSyncBucketStorage
80
82
  private writeCheckpointAPI: MongoWriteCheckpointAPI;
81
83
  public readonly logger: Logger;
82
84
  public readonly storageConfig: StorageConfig;
85
+ public readonly readPreference: mongo.ReadPreference | undefined;
83
86
  #storageInitialized = false;
84
87
 
85
88
  constructor(
@@ -92,6 +95,7 @@ export abstract class MongoSyncBucketStorage
92
95
  ) {
93
96
  super();
94
97
  this.storageConfig = options.storageConfig;
98
+ this.readPreference = options.readPreference;
95
99
  this.db = factory.db.versioned(this.storageConfig);
96
100
  this.checksums = this.createMongoChecksums(options);
97
101
  this.writeCheckpointAPI = new MongoWriteCheckpointAPI({
@@ -130,8 +134,8 @@ export abstract class MongoSyncBucketStorage
130
134
  this.writeCheckpointAPI.setWriteCheckpointMode(mode);
131
135
  }
132
136
 
133
- createManagedWriteCheckpoint(checkpoint: storage.ManagedWriteCheckpointOptions): Promise<bigint> {
134
- return this.writeCheckpointAPI.createManagedWriteCheckpoint(checkpoint);
137
+ createManagedWriteCheckpoints(checkpoints: storage.ManagedWriteCheckpointOptions[]): Promise<Map<string, bigint>> {
138
+ return this.writeCheckpointAPI.createManagedWriteCheckpoints(checkpoints);
135
139
  }
136
140
 
137
141
  lastWriteCheckpoint(filters: storage.SyncStorageLastWriteCheckpointFilters): Promise<bigint | null> {
@@ -170,10 +174,14 @@ export abstract class MongoSyncBucketStorage
170
174
  }
171
175
 
172
176
  const snapshotTime = (session as any).snapshotTime as bson.Timestamp | undefined;
177
+ const clusterTime = session.clusterTime;
173
178
  if (snapshotTime == null) {
174
179
  throw new ServiceAssertionError('Missing snapshotTime in getCheckpoint()');
175
180
  }
176
- return new MongoReplicationCheckpoint(this, state.checkpoint, state.lsn, snapshotTime);
181
+ if (clusterTime == null) {
182
+ throw new ServiceAssertionError('Missing clusterTime in getCheckpoint()');
183
+ }
184
+ return new MongoReplicationCheckpoint(this, state.checkpoint, state.lsn, snapshotTime, clusterTime);
177
185
  });
178
186
  }
179
187
 
@@ -247,24 +255,31 @@ export abstract class MongoSyncBucketStorage
247
255
  }
248
256
 
249
257
  protected abstract getBucketDataBatchImpl(
250
- checkpoint: utils.InternalOpId,
258
+ checkpoint: MongoReplicationCheckpoint,
251
259
  dataBuckets: storage.BucketDataRequest[],
252
260
  options?: storage.BucketDataBatchOptions
253
261
  ): AsyncIterable<storage.SyncBucketDataChunk>;
254
262
 
255
263
  async *getBucketDataBatch(
256
- checkpoint: utils.InternalOpId,
264
+ checkpoint: storage.ReplicationCheckpoint,
257
265
  dataBuckets: storage.BucketDataRequest[],
258
266
  options?: storage.BucketDataBatchOptions
259
267
  ): AsyncIterable<storage.SyncBucketDataChunk> {
260
- yield* this.getBucketDataBatchImpl(checkpoint, dataBuckets, options);
268
+ yield* this.getBucketDataBatchImpl(checkpoint as MongoReplicationCheckpoint, dataBuckets, options);
261
269
  }
262
270
 
263
271
  async getChecksums(
264
- checkpoint: utils.InternalOpId,
265
- buckets: storage.BucketChecksumRequest[]
272
+ checkpoint: storage.ReplicationCheckpoint,
273
+ buckets: storage.BucketChecksumRequest[],
274
+ options?: storage.BucketChecksumOptions
266
275
  ): Promise<utils.ChecksumMap> {
267
- return this.checksums.getChecksums(checkpoint, buckets);
276
+ const mongoCheckpoint = checkpoint as MongoReplicationCheckpoint;
277
+ const snapshotTime = mongoCheckpoint.snapshotTime; // May be undefined in tests
278
+ return this.checksums.getChecksums(checkpoint.checkpoint, buckets, {
279
+ snapshotTime,
280
+ clusterTime: mongoCheckpoint.clusterTime,
281
+ readPreference: options?.requestHint == 'bulk' ? this.readPreference : undefined
282
+ });
268
283
  }
269
284
 
270
285
  clearChecksumCache() {
@@ -580,6 +595,9 @@ export abstract class MongoSyncBucketStorage
580
595
  >({
581
596
  max: 50,
582
597
  maxSize: 12 * 1024 * 1024,
598
+ // When we have more fetches than the cache size, complete the fetches instead
599
+ // of failing with Error('evicted').
600
+ ignoreFetchAbort: true,
583
601
  sizeCalculation: (value: InternalCheckpointChanges) => {
584
602
  const paramSize = [...value.updatedParameterLookups].reduce<number>((a, b) => a + b.length, 0);
585
603
  const bucketSize = [...value.updatedDataBuckets].reduce<number>((a, b) => a + b.length, 0);
@@ -617,7 +635,8 @@ class MongoReplicationCheckpoint implements ReplicationCheckpoint {
617
635
  storage: MongoSyncBucketStorage,
618
636
  public readonly checkpoint: InternalOpId,
619
637
  public readonly lsn: string | null,
620
- public snapshotTime: mongo.Timestamp
638
+ public snapshotTime: mongo.Timestamp,
639
+ public clusterTime: mongo.ClusterTime
621
640
  ) {
622
641
  this.#storage = storage;
623
642
  }
@@ -26,30 +26,77 @@ export class MongoWriteCheckpointAPI implements storage.WriteCheckpointAPI {
26
26
  this._mode = mode;
27
27
  }
28
28
 
29
- async createManagedWriteCheckpoint(checkpoint: storage.ManagedWriteCheckpointOptions): Promise<bigint> {
29
+ async createManagedWriteCheckpoints(
30
+ checkpoints: storage.ManagedWriteCheckpointOptions[]
31
+ ): Promise<Map<string, bigint>> {
30
32
  if (this.writeCheckpointMode !== storage.WriteCheckpointMode.MANAGED) {
31
33
  throw new framework.ServiceAssertionError(
32
34
  `Attempting to create a managed Write Checkpoint when the current Write Checkpoint mode is set to "${this.writeCheckpointMode}"`
33
35
  );
34
36
  }
35
37
 
36
- const { user_id, heads: lsns } = checkpoint;
37
- const doc = await this.db.write_checkpoints.findOneAndUpdate(
38
- {
39
- user_id: user_id
40
- },
41
- {
42
- $set: {
43
- lsns,
44
- processed_at_lsn: null
38
+ const uniqueCheckpoints = [...new Map(checkpoints.map((checkpoint) => [checkpoint.user_id, checkpoint])).values()];
39
+ if (uniqueCheckpoints.length == 0) {
40
+ return new Map();
41
+ }
42
+
43
+ if (uniqueCheckpoints.length == 1) {
44
+ // For the common case of a single checkpoint, we can do this in a single request.
45
+ const { user_id, heads: lsns } = uniqueCheckpoints[0];
46
+ const doc = await this.db.write_checkpoints.findOneAndUpdate(
47
+ {
48
+ user_id
45
49
  },
46
- $inc: {
47
- client_id: 1n
50
+ {
51
+ $set: {
52
+ lsns,
53
+ processed_at_lsn: null
54
+ },
55
+ $inc: {
56
+ client_id: 1n
57
+ }
58
+ },
59
+ { upsert: true, returnDocument: 'after' }
60
+ );
61
+
62
+ return new Map([[doc!.user_id, doc!.client_id]]);
63
+ }
64
+
65
+ // For more than one checkpoint, this gives a constant 2 requests
66
+ await this.db.write_checkpoints.bulkWrite(
67
+ uniqueCheckpoints.map(({ user_id, heads: lsns }) => ({
68
+ updateOne: {
69
+ filter: { user_id },
70
+ update: {
71
+ $set: {
72
+ lsns,
73
+ processed_at_lsn: null
74
+ },
75
+ $inc: {
76
+ client_id: 1n
77
+ }
78
+ },
79
+ upsert: true
48
80
  }
49
- },
50
- { upsert: true, returnDocument: 'after' }
81
+ }))
51
82
  );
52
- return doc!.client_id;
83
+
84
+ const userIds = uniqueCheckpoints.map((checkpoint) => checkpoint.user_id);
85
+ const docs = await this.db.write_checkpoints
86
+ .find(
87
+ {
88
+ user_id: { $in: userIds }
89
+ },
90
+ {
91
+ projection: {
92
+ user_id: 1,
93
+ client_id: 1
94
+ }
95
+ }
96
+ )
97
+ .toArray();
98
+
99
+ return new Map(docs.map((doc) => [doc.user_id, doc.client_id]));
53
100
  }
54
101
 
55
102
  async lastWriteCheckpoint(filters: storage.LastWriteCheckpointFilters): Promise<bigint | null> {
@@ -1,7 +1,9 @@
1
+ import { mongo } from '@powersync/lib-service-mongodb';
1
2
  import { InternalOpId } from '@powersync/service-core';
2
3
  import * as bson from 'bson';
3
4
 
4
5
  export interface MongoSyncBucketStorageCheckpoint {
5
6
  checkpoint: InternalOpId;
6
7
  snapshotTime: bson.Timestamp;
8
+ clusterTime: mongo.ClusterTime;
7
9
  }
@@ -195,7 +195,14 @@ export class PowerSyncMongo {
195
195
  * This is used in a similar way to the Postgres NOTIFY functionality.
196
196
  */
197
197
  async notifyCheckpoint() {
198
- await this.checkpoint_events.insertOne({} as any, { forceServerObjectId: true });
198
+ await this.checkpoint_events.insertOne({} as any, {
199
+ forceServerObjectId: true,
200
+ // No correctness issue if this write is missed / rolled back.
201
+ // It could cause some sync delays, that will resolve in at most a minute, but no consistency issues.
202
+ writeConcern: {
203
+ w: 1
204
+ }
205
+ });
199
206
  }
200
207
 
201
208
  /**