@powersync/service-module-mongodb-storage 0.18.1 → 0.18.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +34 -0
- package/dist/storage/MongoBucketStorage.d.ts +3 -0
- package/dist/storage/MongoBucketStorage.js +2 -0
- package/dist/storage/MongoBucketStorage.js.map +1 -1
- package/dist/storage/implementation/MongoBucketBatch.d.ts +2 -2
- package/dist/storage/implementation/MongoBucketBatch.js +19 -10
- package/dist/storage/implementation/MongoBucketBatch.js.map +1 -1
- package/dist/storage/implementation/MongoChecksums.d.ts +31 -4
- package/dist/storage/implementation/MongoChecksums.js +33 -10
- package/dist/storage/implementation/MongoChecksums.js.map +1 -1
- package/dist/storage/implementation/MongoCompactor.d.ts +1 -1
- package/dist/storage/implementation/MongoCompactor.js +1 -1
- package/dist/storage/implementation/MongoParameterCompactor.d.ts +1 -1
- package/dist/storage/implementation/MongoParameterCompactor.js +1 -1
- package/dist/storage/implementation/MongoStorageProvider.js +9 -0
- package/dist/storage/implementation/MongoStorageProvider.js.map +1 -1
- package/dist/storage/implementation/MongoSyncBucketStorage.d.ts +9 -5
- package/dist/storage/implementation/MongoSyncBucketStorage.js +23 -6
- package/dist/storage/implementation/MongoSyncBucketStorage.js.map +1 -1
- package/dist/storage/implementation/MongoWriteCheckpointAPI.d.ts +1 -1
- package/dist/storage/implementation/MongoWriteCheckpointAPI.js +47 -12
- package/dist/storage/implementation/MongoWriteCheckpointAPI.js.map +1 -1
- package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.d.ts +2 -0
- package/dist/storage/implementation/db.js +8 -1
- package/dist/storage/implementation/db.js.map +1 -1
- package/dist/storage/implementation/v1/MongoBucketBatchV1.js +17 -2
- package/dist/storage/implementation/v1/MongoBucketBatchV1.js.map +1 -1
- package/dist/storage/implementation/v1/MongoChecksumsV1.d.ts +4 -4
- package/dist/storage/implementation/v1/MongoChecksumsV1.js +7 -6
- package/dist/storage/implementation/v1/MongoChecksumsV1.js.map +1 -1
- package/dist/storage/implementation/v1/MongoCompactorV1.js +1 -1
- package/dist/storage/implementation/v1/MongoCompactorV1.js.map +1 -1
- package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.d.ts +4 -3
- package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js +18 -4
- package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js.map +1 -1
- package/dist/storage/implementation/v3/MongoBucketBatchV3.js +20 -2
- package/dist/storage/implementation/v3/MongoBucketBatchV3.js.map +1 -1
- package/dist/storage/implementation/v3/MongoChecksumsV3.d.ts +3 -3
- package/dist/storage/implementation/v3/MongoChecksumsV3.js +8 -7
- package/dist/storage/implementation/v3/MongoChecksumsV3.js.map +1 -1
- package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.d.ts +4 -3
- package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js +17 -3
- package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js.map +1 -1
- package/dist/types/types.d.ts +12 -1
- package/dist/types/types.js +14 -1
- package/dist/types/types.js.map +1 -1
- package/package.json +7 -7
- package/src/storage/MongoBucketStorage.ts +4 -0
- package/src/storage/implementation/MongoBucketBatch.ts +23 -13
- package/src/storage/implementation/MongoChecksums.ts +87 -15
- package/src/storage/implementation/MongoCompactor.ts +1 -1
- package/src/storage/implementation/MongoParameterCompactor.ts +1 -1
- package/src/storage/implementation/MongoStorageProvider.ts +10 -0
- package/src/storage/implementation/MongoSyncBucketStorage.ts +29 -10
- package/src/storage/implementation/MongoWriteCheckpointAPI.ts +62 -15
- package/src/storage/implementation/common/MongoSyncBucketStorageCheckpoint.ts +2 -0
- package/src/storage/implementation/db.ts +8 -1
- package/src/storage/implementation/v1/MongoBucketBatchV1.ts +18 -2
- package/src/storage/implementation/v1/MongoChecksumsV1.ts +20 -11
- package/src/storage/implementation/v1/MongoCompactorV1.ts +2 -1
- package/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts +24 -6
- package/src/storage/implementation/v3/MongoBucketBatchV3.ts +20 -2
- package/src/storage/implementation/v3/MongoChecksumsV3.ts +23 -11
- package/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts +23 -5
- package/src/types/types.ts +17 -1
- package/test/src/storage_compacting.test.ts +4 -4
- package/test/src/storage_sync.test.ts +13 -5
- package/test/src/storeCurrentData.test.ts +9 -3
- package/tsconfig.tsbuildinfo +1 -1
package/dist/types/types.d.ts
CHANGED
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
import * as service_types from '@powersync/service-types';
|
|
2
2
|
import * as t from 'ts-codec';
|
|
3
|
+
export declare const MongoStorageReadPreference: t.Union<t.Codec<"primary" | "primaryPreferred" | "secondary" | "secondaryPreferred", "primary" | "primaryPreferred" | "secondary" | "secondaryPreferred", string, t.CodecProps>, t.LiteralCodec<"nearest">>;
|
|
4
|
+
export type MongoStorageReadPreference = t.Encoded<typeof MongoStorageReadPreference>;
|
|
3
5
|
export declare const MongoStorageConfig: t.Intersection<t.Codec<{
|
|
4
6
|
type: "mongodb";
|
|
5
7
|
uri: string;
|
|
@@ -14,7 +16,16 @@ export declare const MongoStorageConfig: t.Intersection<t.Codec<{
|
|
|
14
16
|
username?: string | undefined;
|
|
15
17
|
password?: string | undefined;
|
|
16
18
|
reject_ip_ranges?: string[] | undefined;
|
|
17
|
-
}, string, t.CodecProps>, t.ObjectCodec<{
|
|
19
|
+
}, string, t.CodecProps>, t.ObjectCodec<{
|
|
20
|
+
/**
|
|
21
|
+
* Read preference for bulk checksum and bucket data reads.
|
|
22
|
+
*
|
|
23
|
+
* If unset, MongoDB driver defaults are used for backwards compatibility.
|
|
24
|
+
*
|
|
25
|
+
* This is an experimental option, and may be removed in a future release.
|
|
26
|
+
*/
|
|
27
|
+
bulk_read_preference: t.OptionalCodec<t.Codec<"primary" | "primaryPreferred" | "secondary" | "secondaryPreferred" | "nearest", "primary" | "primaryPreferred" | "secondary" | "secondaryPreferred" | "nearest", string, t.CodecProps>>;
|
|
28
|
+
}>>;
|
|
18
29
|
export type MongoStorageConfig = t.Encoded<typeof MongoStorageConfig>;
|
|
19
30
|
export type MongoStorageConfigDecoded = t.Decoded<typeof MongoStorageConfig>;
|
|
20
31
|
export declare function isMongoStorageConfig(config: service_types.configFile.GenericStorageConfig): config is MongoStorageConfig;
|
package/dist/types/types.js
CHANGED
|
@@ -1,7 +1,20 @@
|
|
|
1
1
|
import * as lib_mongo from '@powersync/lib-service-mongodb';
|
|
2
2
|
import * as t from 'ts-codec';
|
|
3
|
+
export const MongoStorageReadPreference = t
|
|
4
|
+
.literal('primary')
|
|
5
|
+
.or(t.literal('primaryPreferred'))
|
|
6
|
+
.or(t.literal('secondary'))
|
|
7
|
+
.or(t.literal('secondaryPreferred'))
|
|
8
|
+
.or(t.literal('nearest'));
|
|
3
9
|
export const MongoStorageConfig = lib_mongo.BaseMongoConfig.and(t.object({
|
|
4
|
-
|
|
10
|
+
/**
|
|
11
|
+
* Read preference for bulk checksum and bucket data reads.
|
|
12
|
+
*
|
|
13
|
+
* If unset, MongoDB driver defaults are used for backwards compatibility.
|
|
14
|
+
*
|
|
15
|
+
* This is an experimental option, and may be removed in a future release.
|
|
16
|
+
*/
|
|
17
|
+
bulk_read_preference: MongoStorageReadPreference.optional()
|
|
5
18
|
}));
|
|
6
19
|
export function isMongoStorageConfig(config) {
|
|
7
20
|
return config.type == lib_mongo.MONGO_CONNECTION_TYPE;
|
package/dist/types/types.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/types/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,SAAS,MAAM,gCAAgC,CAAC;AAE5D,OAAO,KAAK,CAAC,MAAM,UAAU,CAAC;AAE9B,MAAM,CAAC,MAAM,kBAAkB,GAAG,SAAS,CAAC,eAAe,CAAC,GAAG,CAC7D,CAAC,CAAC,MAAM,CAAC;
|
|
1
|
+
{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/types/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,SAAS,MAAM,gCAAgC,CAAC;AAE5D,OAAO,KAAK,CAAC,MAAM,UAAU,CAAC;AAE9B,MAAM,CAAC,MAAM,0BAA0B,GAAG,CAAC;KACxC,OAAO,CAAC,SAAS,CAAC;KAClB,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,kBAAkB,CAAC,CAAC;KACjC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;KAC1B,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,oBAAoB,CAAC,CAAC;KACnC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC;AAI5B,MAAM,CAAC,MAAM,kBAAkB,GAAG,SAAS,CAAC,eAAe,CAAC,GAAG,CAC7D,CAAC,CAAC,MAAM,CAAC;IACP;;;;;;OAMG;IACH,oBAAoB,EAAE,0BAA0B,CAAC,QAAQ,EAAE;CAC5D,CAAC,CACH,CAAC;AAKF,MAAM,UAAU,oBAAoB,CAClC,MAAqD;IAErD,OAAO,MAAM,CAAC,IAAI,IAAI,SAAS,CAAC,qBAAqB,CAAC;AACxD,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@powersync/service-module-mongodb-storage",
|
|
3
3
|
"repository": "https://github.com/powersync-ja/powersync-service",
|
|
4
|
-
"version": "0.18.
|
|
4
|
+
"version": "0.18.3",
|
|
5
5
|
"license": "FSL-1.1-ALv2",
|
|
6
6
|
"type": "module",
|
|
7
7
|
"publishConfig": {
|
|
@@ -17,15 +17,15 @@
|
|
|
17
17
|
"lru-cache": "^10.2.2",
|
|
18
18
|
"ts-codec": "^1.3.0",
|
|
19
19
|
"uuid": "^14.0.0",
|
|
20
|
-
"@powersync/lib-service-mongodb": "0.6.
|
|
21
|
-
"@powersync/lib-services-framework": "0.9.
|
|
22
|
-
"@powersync/service-core": "1.23.
|
|
20
|
+
"@powersync/lib-service-mongodb": "0.6.29",
|
|
21
|
+
"@powersync/lib-services-framework": "0.9.8",
|
|
22
|
+
"@powersync/service-core": "1.23.3",
|
|
23
23
|
"@powersync/service-jsonbig": "0.17.13",
|
|
24
|
-
"@powersync/service-sync-rules": "0.
|
|
25
|
-
"@powersync/service-types": "0.16.
|
|
24
|
+
"@powersync/service-sync-rules": "0.39.0",
|
|
25
|
+
"@powersync/service-types": "0.16.1"
|
|
26
26
|
},
|
|
27
27
|
"devDependencies": {
|
|
28
|
-
"@powersync/service-core-tests": "0.17.
|
|
28
|
+
"@powersync/service-core-tests": "0.17.2"
|
|
29
29
|
},
|
|
30
30
|
"scripts": {
|
|
31
31
|
"build": "tsc -b",
|
|
@@ -34,7 +34,9 @@ export interface MongoBucketStorageOptions {
|
|
|
34
34
|
* Prefix for replication stream name and Postgres logical replication slot name.
|
|
35
35
|
*/
|
|
36
36
|
replicationStreamNamePrefix: string;
|
|
37
|
+
readPreference?: mongo.ReadPreference;
|
|
37
38
|
checksumOptions?: Omit<MongoChecksumOptions, 'storageConfig'>;
|
|
39
|
+
checksumCacheTtlMs?: number;
|
|
38
40
|
/**
|
|
39
41
|
* Reuse a compatible active replication stream by appending a new sync config.
|
|
40
42
|
*
|
|
@@ -91,6 +93,8 @@ export class MongoBucketStorage extends storage.BucketStorageFactory {
|
|
|
91
93
|
undefined,
|
|
92
94
|
{
|
|
93
95
|
checksumOptions: this.options.checksumOptions,
|
|
96
|
+
readPreference: this.options.readPreference,
|
|
97
|
+
checksumCacheTtlMs: this.options.checksumCacheTtlMs,
|
|
94
98
|
storageConfig
|
|
95
99
|
}
|
|
96
100
|
);
|
|
@@ -107,7 +107,7 @@ export abstract class MongoBucketBatch
|
|
|
107
107
|
private hooks: storage.StorageHooks | undefined;
|
|
108
108
|
private clearedError = false;
|
|
109
109
|
|
|
110
|
-
|
|
110
|
+
protected tracer: PerformanceTracer<'storage' | 'evaluate'>;
|
|
111
111
|
|
|
112
112
|
/**
|
|
113
113
|
* Last written op, if any. This may not reflect a consistent checkpoint.
|
|
@@ -198,14 +198,18 @@ export abstract class MongoBucketBatch
|
|
|
198
198
|
const batch = this.batch;
|
|
199
199
|
let last_op: InternalOpId | null = null;
|
|
200
200
|
let resumeBatch: OperationBatch | null = null;
|
|
201
|
+
let clearedError = false;
|
|
201
202
|
|
|
202
203
|
using _ = this.tracer.span('storage', 'flush');
|
|
203
204
|
|
|
204
205
|
await this.hooks?.beforeBatchFlush?.(this);
|
|
205
206
|
|
|
206
207
|
await this.withReplicationTransaction(`Flushing ${batch?.length ?? 0} ops`, async (session, opSeq) => {
|
|
208
|
+
clearedError = false;
|
|
207
209
|
if (batch != null) {
|
|
208
|
-
|
|
210
|
+
const result = await this.replicateBatch(session, batch, opSeq, options);
|
|
211
|
+
resumeBatch = result.resumeBatch;
|
|
212
|
+
clearedError ||= result.clearedError;
|
|
209
213
|
}
|
|
210
214
|
|
|
211
215
|
if (this.write_checkpoint_batch.length > 0) {
|
|
@@ -217,6 +221,10 @@ export abstract class MongoBucketBatch
|
|
|
217
221
|
last_op = opSeq.last();
|
|
218
222
|
});
|
|
219
223
|
|
|
224
|
+
if (clearedError) {
|
|
225
|
+
this.clearedError = true;
|
|
226
|
+
}
|
|
227
|
+
|
|
220
228
|
// null if done, set if we need another flush
|
|
221
229
|
this.batch = resumeBatch;
|
|
222
230
|
|
|
@@ -235,7 +243,7 @@ export abstract class MongoBucketBatch
|
|
|
235
243
|
batch: OperationBatch,
|
|
236
244
|
op_seq: MongoIdSequence,
|
|
237
245
|
options?: storage.BucketBatchCommitOptions
|
|
238
|
-
): Promise<OperationBatch | null> {
|
|
246
|
+
): Promise<{ resumeBatch: OperationBatch | null; clearedError: boolean }> {
|
|
239
247
|
let sizes: Map<string, number> | undefined = undefined;
|
|
240
248
|
using _ = this.tracer.span('storage', 'replicate_batch');
|
|
241
249
|
// Only look up current_data sizes if the batch stores current_data and at least one
|
|
@@ -340,12 +348,17 @@ export abstract class MongoBucketBatch
|
|
|
340
348
|
}
|
|
341
349
|
}
|
|
342
350
|
|
|
343
|
-
|
|
351
|
+
const clearedError = didFlush && !this.clearedError;
|
|
352
|
+
if (clearedError) {
|
|
353
|
+
// No need to clear an error more than once per batch, since an error would always result in restarting the batch.
|
|
344
354
|
using _ = this.tracer.span('storage', 'clear_error');
|
|
345
|
-
await this.clearError();
|
|
355
|
+
await this.clearError(this.session);
|
|
346
356
|
}
|
|
347
357
|
|
|
348
|
-
return
|
|
358
|
+
return {
|
|
359
|
+
resumeBatch: resumeBatch?.hasData() ? resumeBatch : null,
|
|
360
|
+
clearedError
|
|
361
|
+
};
|
|
349
362
|
}
|
|
350
363
|
|
|
351
364
|
private saveOperation(
|
|
@@ -899,12 +912,7 @@ export abstract class MongoBucketBatch
|
|
|
899
912
|
return copy;
|
|
900
913
|
}
|
|
901
914
|
|
|
902
|
-
protected async clearError(): Promise<void> {
|
|
903
|
-
// No need to clear an error more than once per batch, since an error would always result in restarting the batch.
|
|
904
|
-
if (this.clearedError) {
|
|
905
|
-
return;
|
|
906
|
-
}
|
|
907
|
-
|
|
915
|
+
protected async clearError(session: mongo.ClientSession): Promise<void> {
|
|
908
916
|
await this.db.sync_rules.updateOne(
|
|
909
917
|
{
|
|
910
918
|
_id: this.replicationStreamId
|
|
@@ -914,9 +922,11 @@ export abstract class MongoBucketBatch
|
|
|
914
922
|
last_fatal_error: null,
|
|
915
923
|
last_fatal_error_ts: null
|
|
916
924
|
}
|
|
925
|
+
},
|
|
926
|
+
{
|
|
927
|
+
session
|
|
917
928
|
}
|
|
918
929
|
);
|
|
919
|
-
this.clearedError = true;
|
|
920
930
|
}
|
|
921
931
|
|
|
922
932
|
/**
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { mongo } from '@powersync/lib-service-mongodb';
|
|
1
2
|
import {
|
|
2
3
|
addPartialChecksums,
|
|
3
4
|
bson,
|
|
@@ -13,7 +14,9 @@ import {
|
|
|
13
14
|
} from '@powersync/service-core';
|
|
14
15
|
import type { VersionedPowerSyncMongo } from './db.js';
|
|
15
16
|
|
|
17
|
+
import { ServiceAssertionError } from '@powersync/lib-services-framework';
|
|
16
18
|
import { BucketDefinitionId } from '@powersync/service-sync-rules';
|
|
19
|
+
import { setSessionSnapshotTime } from '../../utils/util.js';
|
|
17
20
|
import { StorageConfig } from './models.js';
|
|
18
21
|
|
|
19
22
|
export interface FetchPartialBucketChecksumByDefinition {
|
|
@@ -43,14 +46,42 @@ export interface MongoChecksumOptions {
|
|
|
43
46
|
*/
|
|
44
47
|
operationBatchLimit?: number;
|
|
45
48
|
|
|
49
|
+
checksumCacheTtlMs?: number;
|
|
50
|
+
|
|
46
51
|
storageConfig: StorageConfig;
|
|
47
52
|
}
|
|
48
53
|
|
|
54
|
+
/**
|
|
55
|
+
* Read options used in requests to the cache, used to construct a session.
|
|
56
|
+
*
|
|
57
|
+
* readConcern is always 'snapshot'.
|
|
58
|
+
*/
|
|
59
|
+
interface MongoChecksumCacheReadContext {
|
|
60
|
+
/**
|
|
61
|
+
* (Optional) Specify a specific Timestamp for the snapshot read.
|
|
62
|
+
* Must include clusterTime as well.
|
|
63
|
+
*/
|
|
64
|
+
snapshotTime?: mongo.Timestamp;
|
|
65
|
+
clusterTime?: mongo.ClusterTime;
|
|
66
|
+
|
|
67
|
+
/**
|
|
68
|
+
* Read preference. Use undefined for the default,
|
|
69
|
+
*/
|
|
70
|
+
readPreference: mongo.ReadPreference | undefined;
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
/**
|
|
74
|
+
* Read options passed into individual methods.
|
|
75
|
+
*/
|
|
76
|
+
export interface MongoChecksumSessionContext {
|
|
77
|
+
readOptions: Pick<mongo.CommandOperationOptions, 'session' | 'readPreference' | 'readConcern'>;
|
|
78
|
+
}
|
|
79
|
+
|
|
49
80
|
const DEFAULT_BUCKET_BATCH_LIMIT = 200;
|
|
50
81
|
export const DEFAULT_OPERATION_BATCH_LIMIT = 50_000;
|
|
51
82
|
|
|
52
83
|
export abstract class MongoChecksums {
|
|
53
|
-
private _cache: ChecksumCache | undefined;
|
|
84
|
+
private _cache: ChecksumCache<MongoChecksumCacheReadContext> | undefined;
|
|
54
85
|
protected readonly storageConfig: StorageConfig;
|
|
55
86
|
|
|
56
87
|
constructor(
|
|
@@ -66,10 +97,11 @@ export abstract class MongoChecksums {
|
|
|
66
97
|
*
|
|
67
98
|
* This means the cache only allocates memory once it is used for the first time.
|
|
68
99
|
*/
|
|
69
|
-
private get cache(): ChecksumCache {
|
|
70
|
-
this._cache ??= new ChecksumCache({
|
|
71
|
-
|
|
72
|
-
|
|
100
|
+
private get cache(): ChecksumCache<MongoChecksumCacheReadContext> {
|
|
101
|
+
this._cache ??= new ChecksumCache<MongoChecksumCacheReadContext>({
|
|
102
|
+
ttlMs: this.options.checksumCacheTtlMs,
|
|
103
|
+
fetchChecksums: (batch, context) => {
|
|
104
|
+
return this.computePartialChecksums(batch, context);
|
|
73
105
|
}
|
|
74
106
|
});
|
|
75
107
|
return this._cache;
|
|
@@ -79,8 +111,12 @@ export abstract class MongoChecksums {
|
|
|
79
111
|
* Calculate checksums, utilizing the cache for partial checkums, and querying the remainder from
|
|
80
112
|
* the database (bucket_state + bucket_data).
|
|
81
113
|
*/
|
|
82
|
-
async getChecksums(
|
|
83
|
-
|
|
114
|
+
async getChecksums(
|
|
115
|
+
checkpoint: InternalOpId,
|
|
116
|
+
buckets: BucketChecksumRequest[],
|
|
117
|
+
options: MongoChecksumCacheReadContext
|
|
118
|
+
): Promise<ChecksumMap> {
|
|
119
|
+
return this.cache.getChecksumMap(checkpoint, buckets, options);
|
|
84
120
|
}
|
|
85
121
|
|
|
86
122
|
clearCache() {
|
|
@@ -96,11 +132,40 @@ export abstract class MongoChecksums {
|
|
|
96
132
|
* As long as data is compacted regularly, this should be fast. Large buckets without pre-compacted bucket_state
|
|
97
133
|
* can be slow.
|
|
98
134
|
*/
|
|
99
|
-
private async computePartialChecksums(
|
|
135
|
+
private async computePartialChecksums(
|
|
136
|
+
batch: FetchPartialBucketChecksum[],
|
|
137
|
+
context: MongoChecksumCacheReadContext | undefined
|
|
138
|
+
): Promise<PartialChecksumMap> {
|
|
100
139
|
if (batch.length == 0) {
|
|
101
140
|
return new Map();
|
|
102
141
|
}
|
|
103
|
-
|
|
142
|
+
|
|
143
|
+
if (context == null) {
|
|
144
|
+
throw new ServiceAssertionError(`context is required`);
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
const { snapshotTime, clusterTime, readPreference } = context;
|
|
148
|
+
return this.db.client.withSession({ snapshot: true }, async (session) => {
|
|
149
|
+
if (snapshotTime != null && clusterTime != null) {
|
|
150
|
+
setSessionSnapshotTime(session, snapshotTime);
|
|
151
|
+
session.advanceClusterTime(clusterTime);
|
|
152
|
+
}
|
|
153
|
+
return this.computePartialChecksumsWithSession(batch, {
|
|
154
|
+
readOptions: {
|
|
155
|
+
// This is set on the session, but we set it on each operation as well just in case
|
|
156
|
+
readConcern: 'snapshot',
|
|
157
|
+
readPreference,
|
|
158
|
+
session
|
|
159
|
+
}
|
|
160
|
+
});
|
|
161
|
+
});
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
private async computePartialChecksumsWithSession(
|
|
165
|
+
batch: FetchPartialBucketChecksum[],
|
|
166
|
+
context: MongoChecksumSessionContext
|
|
167
|
+
): Promise<PartialChecksumMap> {
|
|
168
|
+
const preStates = await this.fetchPreStates(batch, context);
|
|
104
169
|
|
|
105
170
|
const mappedRequests = batch.map((request) => {
|
|
106
171
|
let start = request.start;
|
|
@@ -116,7 +181,7 @@ export abstract class MongoChecksums {
|
|
|
116
181
|
};
|
|
117
182
|
});
|
|
118
183
|
|
|
119
|
-
const queriedChecksums = await this.computePartialChecksumsDirect(mappedRequests);
|
|
184
|
+
const queriedChecksums = await this.computePartialChecksumsDirect(mappedRequests, context);
|
|
120
185
|
|
|
121
186
|
return new Map<string, PartialOrFullChecksum>(
|
|
122
187
|
batch.map((request) => {
|
|
@@ -142,19 +207,22 @@ export abstract class MongoChecksums {
|
|
|
142
207
|
* For large buckets, this can be slow, but should not time out as the underlying queries are performed in
|
|
143
208
|
* smaller batches.
|
|
144
209
|
*/
|
|
145
|
-
|
|
210
|
+
private async computePartialChecksumsDirect(
|
|
211
|
+
batch: FetchPartialBucketChecksum[],
|
|
212
|
+
context: MongoChecksumSessionContext
|
|
213
|
+
): Promise<PartialChecksumMap> {
|
|
146
214
|
// Limit the number of buckets we query for at a time.
|
|
147
215
|
const bucketBatchLimit = this.options?.bucketBatchLimit ?? DEFAULT_BUCKET_BATCH_LIMIT;
|
|
148
216
|
|
|
149
217
|
if (batch.length <= bucketBatchLimit) {
|
|
150
218
|
// Single batch - no need for splitting the batch and merging results
|
|
151
|
-
return await this.computePartialChecksumsInternal(batch);
|
|
219
|
+
return await this.computePartialChecksumsInternal(batch, context);
|
|
152
220
|
}
|
|
153
221
|
// Split the batch and merge results
|
|
154
222
|
let results = new Map<string, PartialOrFullChecksum>();
|
|
155
223
|
for (let i = 0; i < batch.length; i += bucketBatchLimit) {
|
|
156
224
|
const bucketBatch = batch.slice(i, i + bucketBatchLimit);
|
|
157
|
-
const batchResults = await this.computePartialChecksumsInternal(bucketBatch);
|
|
225
|
+
const batchResults = await this.computePartialChecksumsInternal(bucketBatch, context);
|
|
158
226
|
for (let r of batchResults.values()) {
|
|
159
227
|
results.set(r.bucket, r);
|
|
160
228
|
}
|
|
@@ -169,10 +237,14 @@ export abstract class MongoChecksums {
|
|
|
169
237
|
*
|
|
170
238
|
* `batch` must be limited to DEFAULT_BUCKET_BATCH_LIMIT buckets before calling this.
|
|
171
239
|
*/
|
|
172
|
-
protected abstract computePartialChecksumsInternal(
|
|
240
|
+
protected abstract computePartialChecksumsInternal(
|
|
241
|
+
batch: FetchPartialBucketChecksum[],
|
|
242
|
+
context: MongoChecksumSessionContext
|
|
243
|
+
): Promise<PartialChecksumMap>;
|
|
173
244
|
|
|
174
245
|
protected abstract fetchPreStates(
|
|
175
|
-
batch: FetchPartialBucketChecksum[]
|
|
246
|
+
batch: FetchPartialBucketChecksum[],
|
|
247
|
+
context: MongoChecksumSessionContext
|
|
176
248
|
): Promise<Map<string, { opId: InternalOpId; checksum: BucketChecksum }>>;
|
|
177
249
|
}
|
|
178
250
|
|
|
@@ -120,7 +120,7 @@ export abstract class MongoCompactor {
|
|
|
120
120
|
/**
|
|
121
121
|
* Compact buckets by converting operations into MOVE and/or CLEAR operations.
|
|
122
122
|
*
|
|
123
|
-
* See /docs/compacting-operations.md for details.
|
|
123
|
+
* See /docs/storage/compacting-operations.md for details.
|
|
124
124
|
*/
|
|
125
125
|
async compact() {
|
|
126
126
|
if (this.buckets) {
|
|
@@ -16,7 +16,7 @@ type ParameterCompactionReadDocument = {
|
|
|
16
16
|
*
|
|
17
17
|
* This scans through the entire collection to find data to compact.
|
|
18
18
|
*
|
|
19
|
-
* For background, see the `/docs/
|
|
19
|
+
* For background, see the `/docs/storage/parameter-lookups.md` file.
|
|
20
20
|
*/
|
|
21
21
|
export class MongoParameterCompactor {
|
|
22
22
|
constructor(
|
|
@@ -38,8 +38,18 @@ export class MongoStorageProvider implements storage.StorageProvider {
|
|
|
38
38
|
await client.connect();
|
|
39
39
|
|
|
40
40
|
const database = new PowerSyncMongo(client, { database: resolvedConfig.storage.database });
|
|
41
|
+
const readPreference =
|
|
42
|
+
decodedConfig.bulk_read_preference == null
|
|
43
|
+
? undefined
|
|
44
|
+
: new lib_mongo.mongo.ReadPreference(decodedConfig.bulk_read_preference, undefined, {
|
|
45
|
+
// maxStalenessSeconds is relevant for all modes except 'primary'.
|
|
46
|
+
// 90 is the minimum value.
|
|
47
|
+
maxStalenessSeconds: decodedConfig.bulk_read_preference == 'primary' ? undefined : 90
|
|
48
|
+
});
|
|
41
49
|
const syncStorageFactory = new MongoBucketStorage(database, {
|
|
42
50
|
replicationStreamNamePrefix: resolvedConfig.slot_name_prefix,
|
|
51
|
+
readPreference,
|
|
52
|
+
checksumCacheTtlMs: resolvedConfig.api_parameters.bucket_count_cache_ttl_minutes * 60_000,
|
|
43
53
|
// Right now, only MongoDB source databases supports incremental reprocessing.
|
|
44
54
|
// Remove this filter when we support it for other source databases.
|
|
45
55
|
// This assumes a single source connection - revisit if we ever support multiple connections.
|
|
@@ -40,6 +40,8 @@ import { MongoWriteCheckpointAPI } from './MongoWriteCheckpointAPI.js';
|
|
|
40
40
|
|
|
41
41
|
export interface MongoSyncBucketStorageOptions {
|
|
42
42
|
checksumOptions?: Omit<MongoChecksumOptions, 'storageConfig'>;
|
|
43
|
+
readPreference?: mongo.ReadPreference;
|
|
44
|
+
checksumCacheTtlMs?: number;
|
|
43
45
|
storageConfig: StorageConfig;
|
|
44
46
|
}
|
|
45
47
|
|
|
@@ -80,6 +82,7 @@ export abstract class MongoSyncBucketStorage
|
|
|
80
82
|
private writeCheckpointAPI: MongoWriteCheckpointAPI;
|
|
81
83
|
public readonly logger: Logger;
|
|
82
84
|
public readonly storageConfig: StorageConfig;
|
|
85
|
+
public readonly readPreference: mongo.ReadPreference | undefined;
|
|
83
86
|
#storageInitialized = false;
|
|
84
87
|
|
|
85
88
|
constructor(
|
|
@@ -92,6 +95,7 @@ export abstract class MongoSyncBucketStorage
|
|
|
92
95
|
) {
|
|
93
96
|
super();
|
|
94
97
|
this.storageConfig = options.storageConfig;
|
|
98
|
+
this.readPreference = options.readPreference;
|
|
95
99
|
this.db = factory.db.versioned(this.storageConfig);
|
|
96
100
|
this.checksums = this.createMongoChecksums(options);
|
|
97
101
|
this.writeCheckpointAPI = new MongoWriteCheckpointAPI({
|
|
@@ -130,8 +134,8 @@ export abstract class MongoSyncBucketStorage
|
|
|
130
134
|
this.writeCheckpointAPI.setWriteCheckpointMode(mode);
|
|
131
135
|
}
|
|
132
136
|
|
|
133
|
-
|
|
134
|
-
return this.writeCheckpointAPI.
|
|
137
|
+
createManagedWriteCheckpoints(checkpoints: storage.ManagedWriteCheckpointOptions[]): Promise<Map<string, bigint>> {
|
|
138
|
+
return this.writeCheckpointAPI.createManagedWriteCheckpoints(checkpoints);
|
|
135
139
|
}
|
|
136
140
|
|
|
137
141
|
lastWriteCheckpoint(filters: storage.SyncStorageLastWriteCheckpointFilters): Promise<bigint | null> {
|
|
@@ -170,10 +174,14 @@ export abstract class MongoSyncBucketStorage
|
|
|
170
174
|
}
|
|
171
175
|
|
|
172
176
|
const snapshotTime = (session as any).snapshotTime as bson.Timestamp | undefined;
|
|
177
|
+
const clusterTime = session.clusterTime;
|
|
173
178
|
if (snapshotTime == null) {
|
|
174
179
|
throw new ServiceAssertionError('Missing snapshotTime in getCheckpoint()');
|
|
175
180
|
}
|
|
176
|
-
|
|
181
|
+
if (clusterTime == null) {
|
|
182
|
+
throw new ServiceAssertionError('Missing clusterTime in getCheckpoint()');
|
|
183
|
+
}
|
|
184
|
+
return new MongoReplicationCheckpoint(this, state.checkpoint, state.lsn, snapshotTime, clusterTime);
|
|
177
185
|
});
|
|
178
186
|
}
|
|
179
187
|
|
|
@@ -247,24 +255,31 @@ export abstract class MongoSyncBucketStorage
|
|
|
247
255
|
}
|
|
248
256
|
|
|
249
257
|
protected abstract getBucketDataBatchImpl(
|
|
250
|
-
checkpoint:
|
|
258
|
+
checkpoint: MongoReplicationCheckpoint,
|
|
251
259
|
dataBuckets: storage.BucketDataRequest[],
|
|
252
260
|
options?: storage.BucketDataBatchOptions
|
|
253
261
|
): AsyncIterable<storage.SyncBucketDataChunk>;
|
|
254
262
|
|
|
255
263
|
async *getBucketDataBatch(
|
|
256
|
-
checkpoint:
|
|
264
|
+
checkpoint: storage.ReplicationCheckpoint,
|
|
257
265
|
dataBuckets: storage.BucketDataRequest[],
|
|
258
266
|
options?: storage.BucketDataBatchOptions
|
|
259
267
|
): AsyncIterable<storage.SyncBucketDataChunk> {
|
|
260
|
-
yield* this.getBucketDataBatchImpl(checkpoint, dataBuckets, options);
|
|
268
|
+
yield* this.getBucketDataBatchImpl(checkpoint as MongoReplicationCheckpoint, dataBuckets, options);
|
|
261
269
|
}
|
|
262
270
|
|
|
263
271
|
async getChecksums(
|
|
264
|
-
checkpoint:
|
|
265
|
-
buckets: storage.BucketChecksumRequest[]
|
|
272
|
+
checkpoint: storage.ReplicationCheckpoint,
|
|
273
|
+
buckets: storage.BucketChecksumRequest[],
|
|
274
|
+
options?: storage.BucketChecksumOptions
|
|
266
275
|
): Promise<utils.ChecksumMap> {
|
|
267
|
-
|
|
276
|
+
const mongoCheckpoint = checkpoint as MongoReplicationCheckpoint;
|
|
277
|
+
const snapshotTime = mongoCheckpoint.snapshotTime; // May be undefined in tests
|
|
278
|
+
return this.checksums.getChecksums(checkpoint.checkpoint, buckets, {
|
|
279
|
+
snapshotTime,
|
|
280
|
+
clusterTime: mongoCheckpoint.clusterTime,
|
|
281
|
+
readPreference: options?.requestHint == 'bulk' ? this.readPreference : undefined
|
|
282
|
+
});
|
|
268
283
|
}
|
|
269
284
|
|
|
270
285
|
clearChecksumCache() {
|
|
@@ -580,6 +595,9 @@ export abstract class MongoSyncBucketStorage
|
|
|
580
595
|
>({
|
|
581
596
|
max: 50,
|
|
582
597
|
maxSize: 12 * 1024 * 1024,
|
|
598
|
+
// When we have more fetches than the cache size, complete the fetches instead
|
|
599
|
+
// of failing with Error('evicted').
|
|
600
|
+
ignoreFetchAbort: true,
|
|
583
601
|
sizeCalculation: (value: InternalCheckpointChanges) => {
|
|
584
602
|
const paramSize = [...value.updatedParameterLookups].reduce<number>((a, b) => a + b.length, 0);
|
|
585
603
|
const bucketSize = [...value.updatedDataBuckets].reduce<number>((a, b) => a + b.length, 0);
|
|
@@ -617,7 +635,8 @@ class MongoReplicationCheckpoint implements ReplicationCheckpoint {
|
|
|
617
635
|
storage: MongoSyncBucketStorage,
|
|
618
636
|
public readonly checkpoint: InternalOpId,
|
|
619
637
|
public readonly lsn: string | null,
|
|
620
|
-
public snapshotTime: mongo.Timestamp
|
|
638
|
+
public snapshotTime: mongo.Timestamp,
|
|
639
|
+
public clusterTime: mongo.ClusterTime
|
|
621
640
|
) {
|
|
622
641
|
this.#storage = storage;
|
|
623
642
|
}
|
|
@@ -26,30 +26,77 @@ export class MongoWriteCheckpointAPI implements storage.WriteCheckpointAPI {
|
|
|
26
26
|
this._mode = mode;
|
|
27
27
|
}
|
|
28
28
|
|
|
29
|
-
async
|
|
29
|
+
async createManagedWriteCheckpoints(
|
|
30
|
+
checkpoints: storage.ManagedWriteCheckpointOptions[]
|
|
31
|
+
): Promise<Map<string, bigint>> {
|
|
30
32
|
if (this.writeCheckpointMode !== storage.WriteCheckpointMode.MANAGED) {
|
|
31
33
|
throw new framework.ServiceAssertionError(
|
|
32
34
|
`Attempting to create a managed Write Checkpoint when the current Write Checkpoint mode is set to "${this.writeCheckpointMode}"`
|
|
33
35
|
);
|
|
34
36
|
}
|
|
35
37
|
|
|
36
|
-
const
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
38
|
+
const uniqueCheckpoints = [...new Map(checkpoints.map((checkpoint) => [checkpoint.user_id, checkpoint])).values()];
|
|
39
|
+
if (uniqueCheckpoints.length == 0) {
|
|
40
|
+
return new Map();
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
if (uniqueCheckpoints.length == 1) {
|
|
44
|
+
// For the common case of a single checkpoint, we can do this in a single request.
|
|
45
|
+
const { user_id, heads: lsns } = uniqueCheckpoints[0];
|
|
46
|
+
const doc = await this.db.write_checkpoints.findOneAndUpdate(
|
|
47
|
+
{
|
|
48
|
+
user_id
|
|
45
49
|
},
|
|
46
|
-
|
|
47
|
-
|
|
50
|
+
{
|
|
51
|
+
$set: {
|
|
52
|
+
lsns,
|
|
53
|
+
processed_at_lsn: null
|
|
54
|
+
},
|
|
55
|
+
$inc: {
|
|
56
|
+
client_id: 1n
|
|
57
|
+
}
|
|
58
|
+
},
|
|
59
|
+
{ upsert: true, returnDocument: 'after' }
|
|
60
|
+
);
|
|
61
|
+
|
|
62
|
+
return new Map([[doc!.user_id, doc!.client_id]]);
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
// For more than one checkpoint, this gives a constant 2 requests
|
|
66
|
+
await this.db.write_checkpoints.bulkWrite(
|
|
67
|
+
uniqueCheckpoints.map(({ user_id, heads: lsns }) => ({
|
|
68
|
+
updateOne: {
|
|
69
|
+
filter: { user_id },
|
|
70
|
+
update: {
|
|
71
|
+
$set: {
|
|
72
|
+
lsns,
|
|
73
|
+
processed_at_lsn: null
|
|
74
|
+
},
|
|
75
|
+
$inc: {
|
|
76
|
+
client_id: 1n
|
|
77
|
+
}
|
|
78
|
+
},
|
|
79
|
+
upsert: true
|
|
48
80
|
}
|
|
49
|
-
}
|
|
50
|
-
{ upsert: true, returnDocument: 'after' }
|
|
81
|
+
}))
|
|
51
82
|
);
|
|
52
|
-
|
|
83
|
+
|
|
84
|
+
const userIds = uniqueCheckpoints.map((checkpoint) => checkpoint.user_id);
|
|
85
|
+
const docs = await this.db.write_checkpoints
|
|
86
|
+
.find(
|
|
87
|
+
{
|
|
88
|
+
user_id: { $in: userIds }
|
|
89
|
+
},
|
|
90
|
+
{
|
|
91
|
+
projection: {
|
|
92
|
+
user_id: 1,
|
|
93
|
+
client_id: 1
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
)
|
|
97
|
+
.toArray();
|
|
98
|
+
|
|
99
|
+
return new Map(docs.map((doc) => [doc.user_id, doc.client_id]));
|
|
53
100
|
}
|
|
54
101
|
|
|
55
102
|
async lastWriteCheckpoint(filters: storage.LastWriteCheckpointFilters): Promise<bigint | null> {
|
|
@@ -1,7 +1,9 @@
|
|
|
1
|
+
import { mongo } from '@powersync/lib-service-mongodb';
|
|
1
2
|
import { InternalOpId } from '@powersync/service-core';
|
|
2
3
|
import * as bson from 'bson';
|
|
3
4
|
|
|
4
5
|
export interface MongoSyncBucketStorageCheckpoint {
|
|
5
6
|
checkpoint: InternalOpId;
|
|
6
7
|
snapshotTime: bson.Timestamp;
|
|
8
|
+
clusterTime: mongo.ClusterTime;
|
|
7
9
|
}
|
|
@@ -195,7 +195,14 @@ export class PowerSyncMongo {
|
|
|
195
195
|
* This is used in a similar way to the Postgres NOTIFY functionality.
|
|
196
196
|
*/
|
|
197
197
|
async notifyCheckpoint() {
|
|
198
|
-
await this.checkpoint_events.insertOne({} as any, {
|
|
198
|
+
await this.checkpoint_events.insertOne({} as any, {
|
|
199
|
+
forceServerObjectId: true,
|
|
200
|
+
// No correctness issue if this write is missed / rolled back.
|
|
201
|
+
// It could cause some sync delays, that will resolve in at most a minute, but no consistency issues.
|
|
202
|
+
writeConcern: {
|
|
203
|
+
w: 1
|
|
204
|
+
}
|
|
205
|
+
});
|
|
199
206
|
}
|
|
200
207
|
|
|
201
208
|
/**
|