@powersync/service-module-mongodb-storage 0.0.0-dev-20250804065653 → 0.0.0-dev-20250811145516
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +24 -10
- package/LICENSE +3 -3
- package/dist/storage/MongoReportStorage.js +16 -5
- package/dist/storage/MongoReportStorage.js.map +1 -1
- package/dist/storage/implementation/MongoParameterCompactor.d.ts +17 -0
- package/dist/storage/implementation/MongoParameterCompactor.js +92 -0
- package/dist/storage/implementation/MongoParameterCompactor.js.map +1 -0
- package/dist/storage/implementation/MongoSyncBucketStorage.d.ts +12 -3
- package/dist/storage/implementation/MongoSyncBucketStorage.js +114 -68
- package/dist/storage/implementation/MongoSyncBucketStorage.js.map +1 -1
- package/dist/storage/implementation/util.d.ts +1 -0
- package/dist/storage/implementation/util.js +13 -0
- package/dist/storage/implementation/util.js.map +1 -1
- package/package.json +9 -9
- package/src/storage/MongoReportStorage.ts +24 -10
- package/src/storage/implementation/MongoParameterCompactor.ts +105 -0
- package/src/storage/implementation/MongoSyncBucketStorage.ts +128 -86
- package/src/storage/implementation/util.ts +13 -0
- package/test/src/__snapshots__/sdk-report-storage.test.ts.snap +188 -0
- package/test/src/sdk-report-storage.test.ts +12 -9
- package/test/src/storage_compacting.test.ts +2 -0
- package/tsconfig.tsbuildinfo +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -1,9 +1,27 @@
|
|
|
1
1
|
# @powersync/service-module-mongodb-storage
|
|
2
2
|
|
|
3
|
-
## 0.0.0-dev-
|
|
3
|
+
## 0.0.0-dev-20250811145516
|
|
4
|
+
|
|
5
|
+
### Patch Changes
|
|
6
|
+
|
|
7
|
+
- f1d187b: sdk reporting
|
|
8
|
+
- a700ec9: Reporting mongo storage added to storage engine.
|
|
9
|
+
- 060b829: Update license abbreviation to FSL-1.1-ALv2.
|
|
10
|
+
- Updated dependencies [f1d187b]
|
|
11
|
+
- Updated dependencies [a700ec9]
|
|
12
|
+
- Updated dependencies [060b829]
|
|
13
|
+
- @powersync/service-core@0.0.0-dev-20250811145516
|
|
14
|
+
- @powersync/service-types@0.0.0-dev-20250811145516
|
|
15
|
+
- @powersync/lib-services-framework@0.0.0-dev-20250811145516
|
|
16
|
+
- @powersync/service-sync-rules@0.0.0-dev-20250811145516
|
|
17
|
+
- @powersync/lib-service-mongodb@0.0.0-dev-20250811145516
|
|
18
|
+
- @powersync/service-jsonbig@0.0.0-dev-20250811145516
|
|
19
|
+
|
|
20
|
+
## 0.11.0
|
|
4
21
|
|
|
5
22
|
### Minor Changes
|
|
6
23
|
|
|
24
|
+
- b1add5a: [MongoDB Storage] Compact action now also compacts parameter lookup storage.
|
|
7
25
|
- d56eeb9: Delay switching over to new sync rules until we have a consistent checkpoint.
|
|
8
26
|
- d4db4e2: MySQL:
|
|
9
27
|
- Added schema change handling
|
|
@@ -18,22 +36,18 @@
|
|
|
18
36
|
|
|
19
37
|
### Patch Changes
|
|
20
38
|
|
|
21
|
-
-
|
|
22
|
-
- a700ec9: Reporting mongo storage added to storage engine.
|
|
39
|
+
- Updated dependencies [b1add5a]
|
|
23
40
|
- Updated dependencies [2378e36]
|
|
24
41
|
- Updated dependencies [4a34a51]
|
|
25
|
-
- Updated dependencies [f1d187b]
|
|
26
42
|
- Updated dependencies [4ebc3bf]
|
|
27
43
|
- Updated dependencies [2378e36]
|
|
28
44
|
- Updated dependencies [1aafdaf]
|
|
29
45
|
- Updated dependencies [d56eeb9]
|
|
30
|
-
- Updated dependencies [a700ec9]
|
|
31
46
|
- Updated dependencies [d4db4e2]
|
|
32
|
-
- @powersync/service-core@
|
|
33
|
-
- @powersync/service-
|
|
34
|
-
- @powersync/lib-services-framework@0.
|
|
35
|
-
- @powersync/service-
|
|
36
|
-
- @powersync/lib-service-mongodb@0.0.0-dev-20250804065653
|
|
47
|
+
- @powersync/service-core@1.14.0
|
|
48
|
+
- @powersync/service-sync-rules@0.28.0
|
|
49
|
+
- @powersync/lib-services-framework@0.7.2
|
|
50
|
+
- @powersync/lib-service-mongodb@0.6.3
|
|
37
51
|
|
|
38
52
|
## 0.10.4
|
|
39
53
|
|
package/LICENSE
CHANGED
|
@@ -1,12 +1,12 @@
|
|
|
1
|
-
# Functional Source License, Version 1.1,
|
|
1
|
+
# Functional Source License, Version 1.1, ALv2 Future License
|
|
2
2
|
|
|
3
3
|
## Abbreviation
|
|
4
4
|
|
|
5
|
-
FSL-1.1-
|
|
5
|
+
FSL-1.1-ALv2
|
|
6
6
|
|
|
7
7
|
## Notice
|
|
8
8
|
|
|
9
|
-
Copyright 2023-
|
|
9
|
+
Copyright 2023-2025 Journey Mobile, Inc.
|
|
10
10
|
|
|
11
11
|
## Terms and Conditions
|
|
12
12
|
|
|
@@ -48,6 +48,11 @@ export class MongoReportStorage {
|
|
|
48
48
|
users: { $size: '$user_ids' },
|
|
49
49
|
clients: { $size: '$client_ids' }
|
|
50
50
|
}
|
|
51
|
+
},
|
|
52
|
+
{
|
|
53
|
+
$sort: {
|
|
54
|
+
sdk: 1
|
|
55
|
+
}
|
|
51
56
|
}
|
|
52
57
|
]
|
|
53
58
|
}
|
|
@@ -82,8 +87,10 @@ export class MongoReportStorage {
|
|
|
82
87
|
const endDate = data.range?.end_date ? new Date(data.range.end_date) : new Date();
|
|
83
88
|
const startDate = new Date(range.start_date);
|
|
84
89
|
return {
|
|
85
|
-
|
|
86
|
-
|
|
90
|
+
connect_at: {
|
|
91
|
+
$lte: endDate,
|
|
92
|
+
$gt: startDate
|
|
93
|
+
}
|
|
87
94
|
};
|
|
88
95
|
}
|
|
89
96
|
timeFrameQuery(timeframe, interval = 1) {
|
|
@@ -183,8 +190,12 @@ export class MongoReportStorage {
|
|
|
183
190
|
});
|
|
184
191
|
}
|
|
185
192
|
async reportSdkDisconnect(data) {
|
|
186
|
-
const
|
|
187
|
-
await this.db.sdk_report_events.findOneAndUpdate(
|
|
193
|
+
const { connect_at, user_id, client_id } = data;
|
|
194
|
+
await this.db.sdk_report_events.findOneAndUpdate({
|
|
195
|
+
client_id,
|
|
196
|
+
user_id,
|
|
197
|
+
connect_at
|
|
198
|
+
}, {
|
|
188
199
|
$set: {
|
|
189
200
|
disconnect_at: data.disconnect_at
|
|
190
201
|
},
|
|
@@ -201,7 +212,7 @@ export class MongoReportStorage {
|
|
|
201
212
|
$match: {
|
|
202
213
|
disconnect_at: { $exists: false },
|
|
203
214
|
jwt_exp: { $gt: new Date() },
|
|
204
|
-
|
|
215
|
+
...timeframeFilter
|
|
205
216
|
}
|
|
206
217
|
},
|
|
207
218
|
this.sdkFacetPipeline(),
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"MongoReportStorage.js","sourceRoot":"","sources":["../../src/storage/MongoReportStorage.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,MAAM,EAAE,MAAM,mCAAmC,CAAC;AAE3D,MAAM,OAAO,kBAAkB;IACZ,MAAM,CAAoB;IAC3B,EAAE,CAAiB;IAEnC,YAAY,EAAkB;QAC5B,IAAI,CAAC,MAAM,GAAG,EAAE,CAAC,MAAM,CAAC;QACxB,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC;IACf,CAAC;IAEO,WAAW,CAAC,IAAU;QAC5B,MAAM,IAAI,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC;QAChC,MAAM,KAAK,GAAG,IAAI,CAAC,QAAQ,EAAE,CAAC;QAC9B,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC;QAC7B,MAAM,GAAG,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;QAC1B,OAAO;YACL,IAAI;YACJ,KAAK;YACL,KAAK;YACL,GAAG;YACH,UAAU,EAAE,IAAI;SACjB,CAAC;IACJ,CAAC;IAEO,gBAAgB;QACtB,OAAO;YACL,MAAM,EAAE;gBACN,YAAY,EAAE;oBACZ;wBACE,MAAM,EAAE;4BACN,GAAG,EAAE,UAAU;yBAChB;qBACF;oBACD;wBACE,MAAM,EAAE,OAAO;qBAChB;iBACF;gBACD,kBAAkB,EAAE;oBAClB;wBACE,MAAM,EAAE;4BACN,GAAG,EAAE,MAAM;4BACX,KAAK,EAAE,EAAE,IAAI,EAAE,CAAC,EAAE;4BAClB,UAAU,EAAE,EAAE,SAAS,EAAE,YAAY,EAAE;4BACvC,QAAQ,EAAE,EAAE,SAAS,EAAE,UAAU,EAAE;yBACpC;qBACF;oBACD;wBACE,QAAQ,EAAE;4BACR,GAAG,EAAE,CAAC;4BACN,GAAG,EAAE,MAAM;4BACX,KAAK,EAAE,EAAE,KAAK,EAAE,WAAW,EAAE;4BAC7B,OAAO,EAAE,EAAE,KAAK,EAAE,aAAa,EAAE;yBAClC;qBACF;iBACF;aACF;SACF,CAAC;IACJ,CAAC;IAEO,kBAAkB;QACxB,OAAO;YACL,QAAQ,EAAE;gBACR,KAAK,EAAE,EAAE,OAAO,EAAE,CAAC,EAAE,YAAY,EAAE,CAAC,qBAAqB,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE;gBACrE,IAAI,EAAE,qBAAqB;aAC5B;SACF,CAAC;IACJ,CAAC;IAEO,eAAe,CAAC,MAAc,EAAE,QAAgB;QACtD,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,GAAG,IAAI,CAAC,WAAW,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC;QAC5D,MAAM,OAAO,GAAG,KAAK,GAAG,CAAC,CAAC;QAC1B,OAAO;YACL,OAAO,EAAE,MAAM;YACf,SAAS,EAAE,QAAQ;YACnB,UAAU,EAAE;gBACV,8DAA8D;gBAC9D,IAAI,EAAE,IAAI,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE,KAAK,CAAC;gBAClC,GAAG,EAAE,IAAI,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE,OAAO,CAAC;aACpC;SACF,CAAC;IACJ,CAAC;IAEO,wBAAwB,CAAC,IAA+C;QAC9E,MAAM,EAAE,KAAK,EAAE,GAAG,IAAI,CAAC;QACvB,IAAI,CAAC,KAAK,EAAE,CAAC;YACX,OAAO,SAAS,CAAC;QACnB,CAAC;QACD,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC;QAClF,MAAM,SAAS,GAAG,IAAI,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;QAC7C,OAAO;YACL,IAAI,EAAE,OAAO;
|
|
1
|
+
{"version":3,"file":"MongoReportStorage.js","sourceRoot":"","sources":["../../src/storage/MongoReportStorage.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,MAAM,EAAE,MAAM,mCAAmC,CAAC;AAE3D,MAAM,OAAO,kBAAkB;IACZ,MAAM,CAAoB;IAC3B,EAAE,CAAiB;IAEnC,YAAY,EAAkB;QAC5B,IAAI,CAAC,MAAM,GAAG,EAAE,CAAC,MAAM,CAAC;QACxB,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC;IACf,CAAC;IAEO,WAAW,CAAC,IAAU;QAC5B,MAAM,IAAI,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC;QAChC,MAAM,KAAK,GAAG,IAAI,CAAC,QAAQ,EAAE,CAAC;QAC9B,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC;QAC7B,MAAM,GAAG,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;QAC1B,OAAO;YACL,IAAI;YACJ,KAAK;YACL,KAAK;YACL,GAAG;YACH,UAAU,EAAE,IAAI;SACjB,CAAC;IACJ,CAAC;IAEO,gBAAgB;QACtB,OAAO;YACL,MAAM,EAAE;gBACN,YAAY,EAAE;oBACZ;wBACE,MAAM,EAAE;4BACN,GAAG,EAAE,UAAU;yBAChB;qBACF;oBACD;wBACE,MAAM,EAAE,OAAO;qBAChB;iBACF;gBACD,kBAAkB,EAAE;oBAClB;wBACE,MAAM,EAAE;4BACN,GAAG,EAAE,MAAM;4BACX,KAAK,EAAE,EAAE,IAAI,EAAE,CAAC,EAAE;4BAClB,UAAU,EAAE,EAAE,SAAS,EAAE,YAAY,EAAE;4BACvC,QAAQ,EAAE,EAAE,SAAS,EAAE,UAAU,EAAE;yBACpC;qBACF;oBACD;wBACE,QAAQ,EAAE;4BACR,GAAG,EAAE,CAAC;4BACN,GAAG,EAAE,MAAM;4BACX,KAAK,EAAE,EAAE,KAAK,EAAE,WAAW,EAAE;4BAC7B,OAAO,EAAE,EAAE,KAAK,EAAE,aAAa,EAAE;yBAClC;qBACF;oBACD;wBACE,KAAK,EAAE;4BACL,GAAG,EAAE,CAAC;yBACP;qBACF;iBACF;aACF;SACF,CAAC;IACJ,CAAC;IAEO,kBAAkB;QACxB,OAAO;YACL,QAAQ,EAAE;gBACR,KAAK,EAAE,EAAE,OAAO,EAAE,CAAC,EAAE,YAAY,EAAE,CAAC,qBAAqB,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE;gBACrE,IAAI,EAAE,qBAAqB;aAC5B;SACF,CAAC;IACJ,CAAC;IAEO,eAAe,CAAC,MAAc,EAAE,QAAgB;QACtD,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,GAAG,IAAI,CAAC,WAAW,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC;QAC5D,MAAM,OAAO,GAAG,KAAK,GAAG,CAAC,CAAC;QAC1B,OAAO;YACL,OAAO,EAAE,MAAM;YACf,SAAS,EAAE,QAAQ;YACnB,UAAU,EAAE;gBACV,8DAA8D;gBAC9D,IAAI,EAAE,IAAI,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE,KAAK,CAAC;gBAClC,GAAG,EAAE,IAAI,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE,OAAO,CAAC;aACpC;SACF,CAAC;IACJ,CAAC;IAEO,wBAAwB,CAAC,IAA+C;QAC9E,MAAM,EAAE,KAAK,EAAE,GAAG,IAAI,CAAC;QACvB,IAAI,CAAC,KAAK,EAAE,CAAC;YACX,OAAO,SAAS,CAAC;QACnB,CAAC;QACD,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC;QAClF,MAAM,SAAS,GAAG,IAAI,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;QAC7C,OAAO;YACL,UAAU,EAAE;gBACV,IAAI,EAAE,OAAO;gBACb,GAAG,EAAE,SAAS;aACf;SACF,CAAC;IACJ,CAAC;IAEO,cAAc,CAAC,SAAiC,EAAE,WAAmB,CAAC;QAC5E,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,UAAU,EAAE,GAAG,IAAI,CAAC,WAAW,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC;QACxE,QAAQ,SAAS,EAAE,CAAC;YAClB,KAAK,OAAO,CAAC,CAAC,CAAC;gBACb,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,GAAG,EAAE,IAAI,IAAI,CAAC,IAAI,EAAE,UAAU,CAAC,QAAQ,EAAE,GAAG,QAAQ,CAAC,EAAE,CAAC;YACrF,CAAC;YACD,KAAK,MAAM,CAAC,CAAC,CAAC;gBACZ,MAAM,aAAa,GAAG,IAAI,IAAI,CAAC,UAAU,CAAC,CAAC;gBAC3C,aAAa,CAAC,OAAO,CAAC,aAAa,CAAC,OAAO,EAAE,GAAG,CAAC,GAAG,QAAQ,CAAC,CAAC;gBAC9D,MAAM,SAAS,GAAG,IAAI,CAAC,WAAW,CAAC,aAAa,CAAC,CAAC;gBAClD,OAAO;oBACL,IAAI,EAAE,UAAU;oBAChB,GAAG,EAAE,IAAI,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,SAAS,CAAC,KAAK,EAAE,SAAS,CAAC,KAAK,CAAC;iBAChE,CAAC;YACJ,CAAC;YACD,KAAK,MAAM,CAAC,CAAC,CAAC;gBACZ,0CAA0C;gBAC1C,MAAM,YAAY,GAAG,UAAU,CAAC,QAAQ,EAAE,GAAG,QAAQ,CAAC;gBACtD,OAAO;oBACL,GAAG,EAAE,IAAI,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,YAAY,CAAC;oBAC/C,IAAI,EAAE,IAAI,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,UAAU,CAAC,QAAQ,EAAE,CAAC;iBAC1D,CAAC;YACJ,CAAC;YACD,OAAO,CAAC,CAAC,CAAC;gBACR,OAAO;oBACL,IAAI,EAAE,UAAU;oBAChB,GAAG,EAAE,IAAI,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE,KAAK,GAAG,QAAQ,CAAC;iBAC7C,CAAC;YACJ,CAAC;QACH,CAAC;IACH,CAAC;IAEO,oBAAoB,CAAC,SAAiC,EAAE,WAAmB,CAAC;QAClF,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,UAAU,EAAE,GAAG,IAAI,CAAC,WAAW,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC;QACxE,QAAQ,SAAS,EAAE,CAAC;YAClB,KAAK,OAAO,CAAC,CAAC,CAAC;gBACb,OAAO,EAAE,GAAG,EAAE,IAAI,IAAI,CAAC,IAAI,EAAE,UAAU,CAAC,QAAQ,EAAE,GAAG,QAAQ,CAAC,EAAE,CAAC;YACnE,CAAC;YACD,KAAK,MAAM,CAAC,CAAC,CAAC;gBACZ,MAAM,aAAa,GAAG,IAAI,IAAI,CAAC,UAAU,CAAC,CAAC;gBAC3C,aAAa,CAAC,OAAO,CAAC,aAAa,CAAC,OAAO,EAAE,GAAG,CAAC,GAAG,QAAQ,CAAC,CAAC;gBAC9D,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,IAAI,CAAC,WAAW,CAAC,aAAa,CAAC,CAAC;gBAC/D,OAAO;oBACL,GAAG,EAAE,IAAI,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE,KAAK,CAAC;iBAClC,CAAC;YACJ,CAAC;YACD,KAAK,MAAM,CAAC,CAAC,CAAC;gBACZ,MAAM,YAAY,GAAG,UAAU,CAAC,QAAQ,EAAE,GAAG,QAAQ,CAAC;gBACtD,OAAO;oBACL,GAAG,EAAE,IAAI,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,YAAY,CAAC;iBAChD,CAAC;YACJ,CAAC;YACD,OAAO,CAAC,CAAC,CAAC;gBACR,OAAO;oBACL,GAAG,EAAE,IAAI,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE,KAAK,GAAG,QAAQ,CAAC;iBAC7C,CAAC;YACJ,CAAC;QACH,CAAC;IACH,CAAC;IAED,KAAK,CAAC,gBAAgB,CAAC,IAAkC;QACvD,MAAM,EAAE,QAAQ,EAAE,SAAS,EAAE,GAAG,IAAI,CAAC;QACrC,MAAM,eAAe,GAAG,IAAI,CAAC,oBAAoB,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;QACvE,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,iBAAiB,CAAC,UAAU,CAAC;YACxD,UAAU,EAAE,eAAe;YAC3B,GAAG,EAAE,CAAC,EAAE,aAAa,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,EAAE,EAAE,EAAE,OAAO,EAAE,EAAE,GAAG,EAAE,IAAI,IAAI,EAAE,EAAE,EAAE,aAAa,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,EAAE,CAAC;SACjH,CAAC,CAAC;QACH,IAAI,MAAM,CAAC,YAAY,GAAG,CAAC,EAAE,CAAC;YAC5B,MAAM,CAAC,IAAI,CACT,OAAO,QAAQ,IAAI,SAAS,KAAK,MAAM,CAAC,YAAY,8DAA8D,CACnH,CAAC;QACJ,CAAC;IACH,CAAC;IAED,KAAK,CAAC,aAAa,CAAC,IAAsC;QACxD,MAAM,EAAE,QAAQ,EAAE,SAAS,EAAE,GAAG,IAAI,CAAC;QACrC,MAAM,eAAe,GAAG,IAAI,CAAC,cAAc,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;QACjE,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,iBAAiB;aAC3C,SAAS,CAAqC;YAC7C;gBACE,MAAM,EAAE;oBACN,UAAU,EAAE,eAAe;iBAC5B;aACF;YACD,IAAI,CAAC,gBAAgB,EAAE;YACvB,IAAI,CAAC,kBAAkB,EAAE;SAC1B,CAAC;aACD,OAAO,EAAE,CAAC;QACb,OAAO,MAAM,CAAC,CAAC,CAAC,CAAC;IACnB,CAAC;IAED,KAAK,CAAC,gBAAgB,CAAC,IAAsC;QAC3D,MAAM,YAAY,GAAG,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,SAAU,CAAC,CAAC;QACzE,MAAM,IAAI,CAAC,EAAE,CAAC,iBAAiB,CAAC,gBAAgB,CAC9C,YAAY,EACZ;YACE,IAAI,EAAE,IAAI;YACV,MAAM,EAAE;gBACN,aAAa,EAAE,EAAE;aAClB;SACF,EACD;YACE,MAAM,EAAE,IAAI;SACb,CACF,CAAC;IACJ,CAAC;IACD,KAAK,CAAC,mBAAmB,CAAC,IAAwC;QAChE,MAAM,EAAE,UAAU,EAAE,OAAO,EAAE,SAAS,EAAE,GAAG,IAAI,CAAC;QAChD,MAAM,IAAI,CAAC,EAAE,CAAC,iBAAiB,CAAC,gBAAgB,CAC9C;YACE,SAAS;YACT,OAAO;YACP,UAAU;SACX,EACD;YACE,IAAI,EAAE;gBACJ,aAAa,EAAE,IAAI,CAAC,aAAa;aAClC;YACD,MAAM,EAAE;gBACN,OAAO,EAAE,EAAE;aACZ;SACF,CACF,CAAC;IACJ,CAAC;IACD,KAAK,CAAC,sBAAsB,CAC1B,IAA+C;QAE/C,MAAM,eAAe,GAAG,IAAI,CAAC,wBAAwB,CAAC,IAAI,CAAC,CAAC;QAC5D,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,iBAAiB;aAC3C,SAAS,CAAqC;YAC7C;gBACE,MAAM,EAAE;oBACN,aAAa,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE;oBACjC,OAAO,EAAE,EAAE,GAAG,EAAE,IAAI,IAAI,EAAE,EAAE;oBAC5B,GAAG,eAAe;iBACnB;aACF;YACD,IAAI,CAAC,gBAAgB,EAAE;YACvB,IAAI,CAAC,kBAAkB,EAAE;SAC1B,CAAC;aACD,OAAO,EAAE,CAAC;QACb,OAAO,MAAM,CAAC,CAAC,CAAC,CAAC;IACnB,CAAC;IAED,KAAK,CAAC,CAAC,MAAM,CAAC,YAAY,CAAC;QACzB,QAAQ;IACV,CAAC;CACF"}
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import { CompactOptions, InternalOpId } from '@powersync/service-core';
|
|
2
|
+
import { PowerSyncMongo } from './db.js';
|
|
3
|
+
/**
|
|
4
|
+
* Compacts parameter lookup data (the bucket_parameters collection).
|
|
5
|
+
*
|
|
6
|
+
* This scans through the entire collection to find data to compact.
|
|
7
|
+
*
|
|
8
|
+
* For background, see the `/docs/parameters-lookups.md` file.
|
|
9
|
+
*/
|
|
10
|
+
export declare class MongoParameterCompactor {
|
|
11
|
+
private db;
|
|
12
|
+
private group_id;
|
|
13
|
+
private checkpoint;
|
|
14
|
+
private options;
|
|
15
|
+
constructor(db: PowerSyncMongo, group_id: number, checkpoint: InternalOpId, options: CompactOptions);
|
|
16
|
+
compact(): Promise<void>;
|
|
17
|
+
}
|
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
import { logger } from '@powersync/lib-services-framework';
|
|
2
|
+
import { bson } from '@powersync/service-core';
|
|
3
|
+
import { LRUCache } from 'lru-cache';
|
|
4
|
+
/**
|
|
5
|
+
* Compacts parameter lookup data (the bucket_parameters collection).
|
|
6
|
+
*
|
|
7
|
+
* This scans through the entire collection to find data to compact.
|
|
8
|
+
*
|
|
9
|
+
* For background, see the `/docs/parameters-lookups.md` file.
|
|
10
|
+
*/
|
|
11
|
+
export class MongoParameterCompactor {
|
|
12
|
+
db;
|
|
13
|
+
group_id;
|
|
14
|
+
checkpoint;
|
|
15
|
+
options;
|
|
16
|
+
constructor(db, group_id, checkpoint, options) {
|
|
17
|
+
this.db = db;
|
|
18
|
+
this.group_id = group_id;
|
|
19
|
+
this.checkpoint = checkpoint;
|
|
20
|
+
this.options = options;
|
|
21
|
+
}
|
|
22
|
+
async compact() {
|
|
23
|
+
logger.info(`Compacting parameters for group ${this.group_id} up to checkpoint ${this.checkpoint}`);
|
|
24
|
+
// This is the currently-active checkpoint.
|
|
25
|
+
// We do not remove any data that may be used by this checkpoint.
|
|
26
|
+
// snapshot queries ensure that if any clients are still using older checkpoints, they would
|
|
27
|
+
// not be affected by this compaction.
|
|
28
|
+
const checkpoint = this.checkpoint;
|
|
29
|
+
// Index on {'key.g': 1, lookup: 1, _id: 1}
|
|
30
|
+
// In theory, we could let MongoDB do more of the work here, by grouping by (key, lookup)
|
|
31
|
+
// in MongoDB already. However, that risks running into cases where MongoDB needs to process
|
|
32
|
+
// very large amounts of data before returning results, which could lead to timeouts.
|
|
33
|
+
const cursor = this.db.bucket_parameters.find({
|
|
34
|
+
'key.g': this.group_id
|
|
35
|
+
}, {
|
|
36
|
+
sort: { lookup: 1, _id: 1 },
|
|
37
|
+
batchSize: 10_000,
|
|
38
|
+
projection: { _id: 1, key: 1, lookup: 1, bucket_parameters: 1 }
|
|
39
|
+
});
|
|
40
|
+
// The index doesn't cover sorting by key, so we keep our own cache of the last seen key.
|
|
41
|
+
let lastByKey = new LRUCache({
|
|
42
|
+
max: this.options.compactParameterCacheLimit ?? 10_000
|
|
43
|
+
});
|
|
44
|
+
let removeIds = [];
|
|
45
|
+
let removeDeleted = [];
|
|
46
|
+
const flush = async (force) => {
|
|
47
|
+
if (removeIds.length >= 1000 || (force && removeIds.length > 0)) {
|
|
48
|
+
const results = await this.db.bucket_parameters.deleteMany({ _id: { $in: removeIds } });
|
|
49
|
+
logger.info(`Removed ${results.deletedCount} (${removeIds.length}) superseded parameter entries`);
|
|
50
|
+
removeIds = [];
|
|
51
|
+
}
|
|
52
|
+
if (removeDeleted.length > 10 || (force && removeDeleted.length > 0)) {
|
|
53
|
+
const results = await this.db.bucket_parameters.bulkWrite(removeDeleted);
|
|
54
|
+
logger.info(`Removed ${results.deletedCount} (${removeDeleted.length}) deleted parameter entries`);
|
|
55
|
+
removeDeleted = [];
|
|
56
|
+
}
|
|
57
|
+
};
|
|
58
|
+
while (await cursor.hasNext()) {
|
|
59
|
+
const batch = cursor.readBufferedDocuments();
|
|
60
|
+
for (let doc of batch) {
|
|
61
|
+
if (doc._id >= checkpoint) {
|
|
62
|
+
continue;
|
|
63
|
+
}
|
|
64
|
+
const uniqueKey = bson.serialize({
|
|
65
|
+
k: doc.key,
|
|
66
|
+
l: doc.lookup
|
|
67
|
+
}).toString('base64');
|
|
68
|
+
const previous = lastByKey.get(uniqueKey);
|
|
69
|
+
if (previous != null && previous < doc._id) {
|
|
70
|
+
// We have a newer entry for the same key, so we can remove the old one.
|
|
71
|
+
removeIds.push(previous);
|
|
72
|
+
}
|
|
73
|
+
lastByKey.set(uniqueKey, doc._id);
|
|
74
|
+
if (doc.bucket_parameters?.length == 0) {
|
|
75
|
+
// This is a delete operation, so we can remove it completely.
|
|
76
|
+
// For this we cannot remove the operation itself only: There is a possibility that
|
|
77
|
+
// there is still an earlier operation with the same key and lookup, that we don't have
|
|
78
|
+
// in the cache due to cache size limits. So we need to explicitly remove all earlier operations.
|
|
79
|
+
removeDeleted.push({
|
|
80
|
+
deleteMany: {
|
|
81
|
+
filter: { 'key.g': doc.key.g, lookup: doc.lookup, _id: { $lte: doc._id }, key: doc.key }
|
|
82
|
+
}
|
|
83
|
+
});
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
await flush(false);
|
|
87
|
+
}
|
|
88
|
+
await flush(true);
|
|
89
|
+
logger.info('Parameter compaction completed');
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
//# sourceMappingURL=MongoParameterCompactor.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"MongoParameterCompactor.js","sourceRoot":"","sources":["../../../src/storage/implementation/MongoParameterCompactor.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,mCAAmC,CAAC;AAC3D,OAAO,EAAE,IAAI,EAAgC,MAAM,yBAAyB,CAAC;AAC7E,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AAKrC;;;;;;GAMG;AACH,MAAM,OAAO,uBAAuB;IAExB;IACA;IACA;IACA;IAJV,YACU,EAAkB,EAClB,QAAgB,EAChB,UAAwB,EACxB,OAAuB;QAHvB,OAAE,GAAF,EAAE,CAAgB;QAClB,aAAQ,GAAR,QAAQ,CAAQ;QAChB,eAAU,GAAV,UAAU,CAAc;QACxB,YAAO,GAAP,OAAO,CAAgB;IAC9B,CAAC;IAEJ,KAAK,CAAC,OAAO;QACX,MAAM,CAAC,IAAI,CAAC,mCAAmC,IAAI,CAAC,QAAQ,qBAAqB,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC;QACpG,2CAA2C;QAC3C,iEAAiE;QACjE,4FAA4F;QAC5F,sCAAsC;QACtC,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC;QAEnC,2CAA2C;QAC3C,yFAAyF;QACzF,4FAA4F;QAC5F,qFAAqF;QACrF,MAAM,MAAM,GAAG,IAAI,CAAC,EAAE,CAAC,iBAAiB,CAAC,IAAI,CAC3C;YACE,OAAO,EAAE,IAAI,CAAC,QAAQ;SACvB,EACD;YACE,IAAI,EAAE,EAAE,MAAM,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE;YAC3B,SAAS,EAAE,MAAM;YACjB,UAAU,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,iBAAiB,EAAE,CAAC,EAAE;SAChE,CACF,CAAC;QAEF,yFAAyF;QACzF,IAAI,SAAS,GAAG,IAAI,QAAQ,CAAuB;YACjD,GAAG,EAAE,IAAI,CAAC,OAAO,CAAC,0BAA0B,IAAI,MAAM;SACvD,CAAC,CAAC;QACH,IAAI,SAAS,GAAmB,EAAE,CAAC;QACnC,IAAI,aAAa,GAA2D,EAAE,CAAC;QAE/E,MAAM,KAAK,GAAG,KAAK,EAAE,KAAc,EAAE,EAAE;YACrC,IAAI,SAAS,CAAC,MAAM,IAAI,IAAI,IAAI,CAAC,KAAK,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,EAAE,CAAC;gBAChE,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,iBAAiB,CAAC,UAAU,CAAC,EAAE,GAAG,EAAE,EAAE,GAAG,EAAE,SAAS,EAAE,EAAE,CAAC,CAAC;gBACxF,MAAM,CAAC,IAAI,CAAC,WAAW,OAAO,CAAC,YAAY,KAAK,SAAS,CAAC,MAAM,gCAAgC,CAAC,CAAC;gBAClG,SAAS,GAAG,EAAE,CAAC;YACjB,CAAC;YAED,IAAI,aAAa,CAAC,MAAM,GAAG,EAAE,IAAI,CAAC,KAAK,IAAI,aAAa,CAAC,MAAM,GAAG,CAAC,CAAC,EAAE,CAAC;gBACrE,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,iBAAiB,CAAC,SAAS,CAAC,aAAa,CAAC,CAAC;gBACzE,MAAM,CAAC,IAAI,CAAC,WAAW,OAAO,CAAC,YAAY,KAAK,aAAa,CAAC,MAAM,6BAA6B,CAAC,CAAC;gBACnG,aAAa,GAAG,EAAE,CAAC;YACrB,CAAC;QACH,CAAC,CAAC;QAEF,OAAO,MAAM,MAAM,CAAC,OAAO,EAAE,EAAE,CAAC;YAC9B,MAAM,KAAK,GAAG,MAAM,CAAC,qBAAqB,EAAE,CAAC;YAC7C,KAAK,IAAI,GAAG,IAAI,KAAK,EAAE,CAAC;gBACtB,IAAI,GAAG,CAAC,GAAG,IAAI,UAAU,EAAE,CAAC;oBAC1B,SAAS;gBACX,CAAC;gBACD,MAAM,SAAS,GACb,IAAI,CAAC,SAAS,CAAC;oBACb,CAAC,EAAE,GAAG,CAAC,GAAG;oBACV,CAAC,EAAE,GAAG,CAAC,MAAM;iBACd,CACF,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;gBACrB,MAAM,QAAQ,GAAG,SAAS,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;gBAC1C,IAAI,QAAQ,IAAI,IAAI,IAAI,QAAQ,GAAG,GAAG,CAAC,GAAG,EAAE,CAAC;oBAC3C,wEAAwE;oBACxE,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;gBAC3B,CAAC;gBACD,SAAS,CAAC,GAAG,CAAC,SAAS,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC;gBAElC,IAAI,GAAG,CAAC,iBAAiB,EAAE,MAAM,IAAI,CAAC,EAAE,CAAC;oBACvC,8DAA8D;oBAC9D,mFAAmF;oBACnF,uFAAuF;oBACvF,iGAAiG;oBACjG,aAAa,CAAC,IAAI,CAAC;wBACjB,UAAU,EAAE;4BACV,MAAM,EAAE,EAAE,OAAO,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,GAAG,EAAE,EAAE,IAAI,EAAE,GAAG,CAAC,GAAG,EAAE,EAAE,GAAG,EAAE,GAAG,CAAC,GAAG,EAAE;yBACzF;qBACF,CAAC,CAAC;gBACL,CAAC;YACH,CAAC;YAED,MAAM,KAAK,CAAC,KAAK,CAAC,CAAC;QACrB,CAAC;QAED,MAAM,KAAK,CAAC,IAAI,CAAC,CAAC;QAClB,MAAM,CAAC,IAAI,CAAC,gCAAgC,CAAC,CAAC;IAChD,CAAC;CACF"}
|
|
@@ -1,5 +1,6 @@
|
|
|
1
|
+
import { mongo } from '@powersync/lib-service-mongodb';
|
|
1
2
|
import { BaseObserver } from '@powersync/lib-services-framework';
|
|
2
|
-
import { CheckpointChanges, GetCheckpointChangesOptions, InternalOpId, storage, utils, WatchWriteCheckpointOptions } from '@powersync/service-core';
|
|
3
|
+
import { CheckpointChanges, GetCheckpointChangesOptions, InternalOpId, ReplicationCheckpoint, storage, utils, WatchWriteCheckpointOptions } from '@powersync/service-core';
|
|
3
4
|
import { ParameterLookup, SqliteJsonRow, SqlSyncRules } from '@powersync/service-sync-rules';
|
|
4
5
|
import { MongoBucketStorage } from '../MongoBucketStorage.js';
|
|
5
6
|
export declare class MongoSyncBucketStorage extends BaseObserver<storage.SyncRulesBucketStorageListener> implements storage.SyncRulesBucketStorage {
|
|
@@ -18,9 +19,10 @@ export declare class MongoSyncBucketStorage extends BaseObserver<storage.SyncRul
|
|
|
18
19
|
lastWriteCheckpoint(filters: storage.SyncStorageLastWriteCheckpointFilters): Promise<bigint | null>;
|
|
19
20
|
getParsedSyncRules(options: storage.ParseSyncRulesOptions): SqlSyncRules;
|
|
20
21
|
getCheckpoint(): Promise<storage.ReplicationCheckpoint>;
|
|
22
|
+
getCheckpointInternal(): Promise<storage.ReplicationCheckpoint | null>;
|
|
21
23
|
startBatch(options: storage.StartBatchOptions, callback: (batch: storage.BucketStorageBatch) => Promise<void>): Promise<storage.FlushedResult | null>;
|
|
22
24
|
resolveTable(options: storage.ResolveTableOptions): Promise<storage.ResolveTableResult>;
|
|
23
|
-
getParameterSets(checkpoint:
|
|
25
|
+
getParameterSets(checkpoint: MongoReplicationCheckpoint, lookups: ParameterLookup[]): Promise<SqliteJsonRow[]>;
|
|
24
26
|
getBucketDataBatch(checkpoint: utils.InternalOpId, dataBuckets: Map<string, InternalOpId>, options?: storage.BucketDataBatchOptions): AsyncIterable<storage.SyncBucketDataChunk>;
|
|
25
27
|
getChecksums(checkpoint: utils.InternalOpId, buckets: string[]): Promise<utils.ChecksumMap>;
|
|
26
28
|
private getChecksumsInternal;
|
|
@@ -30,7 +32,6 @@ export declare class MongoSyncBucketStorage extends BaseObserver<storage.SyncRul
|
|
|
30
32
|
private clearIteration;
|
|
31
33
|
reportError(e: any): Promise<void>;
|
|
32
34
|
compact(options?: storage.CompactOptions): Promise<void>;
|
|
33
|
-
private makeActiveCheckpoint;
|
|
34
35
|
/**
|
|
35
36
|
* Instance-wide watch on the latest available checkpoint (op_id + lsn).
|
|
36
37
|
*/
|
|
@@ -57,4 +58,12 @@ interface InternalCheckpointChanges extends CheckpointChanges {
|
|
|
57
58
|
updatedWriteCheckpoints: Map<string, bigint>;
|
|
58
59
|
invalidateWriteCheckpoints: boolean;
|
|
59
60
|
}
|
|
61
|
+
declare class MongoReplicationCheckpoint implements ReplicationCheckpoint {
|
|
62
|
+
private storage;
|
|
63
|
+
readonly checkpoint: InternalOpId;
|
|
64
|
+
readonly lsn: string | null;
|
|
65
|
+
snapshotTime: mongo.Timestamp;
|
|
66
|
+
constructor(storage: MongoSyncBucketStorage, checkpoint: InternalOpId, lsn: string | null, snapshotTime: mongo.Timestamp);
|
|
67
|
+
getParameterSets(lookups: ParameterLookup[]): Promise<SqliteJsonRow[]>;
|
|
68
|
+
}
|
|
60
69
|
export {};
|
|
@@ -51,7 +51,7 @@ var __disposeResources = (this && this.__disposeResources) || (function (Suppres
|
|
|
51
51
|
return e.name = "SuppressedError", e.error = error, e.suppressed = suppressed, e;
|
|
52
52
|
});
|
|
53
53
|
import * as lib_mongo from '@powersync/lib-service-mongodb';
|
|
54
|
-
import { BaseObserver,
|
|
54
|
+
import { BaseObserver, logger, ReplicationAbortedError, ServiceAssertionError } from '@powersync/lib-services-framework';
|
|
55
55
|
import { BroadcastIterable, CHECKPOINT_INVALIDATE_ALL, deserializeParameterLookup, internalToExternalOpId, maxLsn, storage } from '@powersync/service-core';
|
|
56
56
|
import { JSONBig } from '@powersync/service-jsonbig';
|
|
57
57
|
import * as bson from 'bson';
|
|
@@ -60,7 +60,8 @@ import * as timers from 'timers/promises';
|
|
|
60
60
|
import { MongoBucketBatch } from './MongoBucketBatch.js';
|
|
61
61
|
import { MongoCompactor } from './MongoCompactor.js';
|
|
62
62
|
import { MongoWriteCheckpointAPI } from './MongoWriteCheckpointAPI.js';
|
|
63
|
-
import { idPrefixFilter, mapOpEntry, readSingleBatch } from './util.js';
|
|
63
|
+
import { idPrefixFilter, mapOpEntry, readSingleBatch, setSessionSnapshotTime } from './util.js';
|
|
64
|
+
import { MongoParameterCompactor } from './MongoParameterCompactor.js';
|
|
64
65
|
export class MongoSyncBucketStorage extends BaseObserver {
|
|
65
66
|
factory;
|
|
66
67
|
group_id;
|
|
@@ -114,19 +115,35 @@ export class MongoSyncBucketStorage extends BaseObserver {
|
|
|
114
115
|
return this.parsedSyncRulesCache.parsed;
|
|
115
116
|
}
|
|
116
117
|
async getCheckpoint() {
|
|
117
|
-
|
|
118
|
-
|
|
118
|
+
return (await this.getCheckpointInternal()) ?? new EmptyReplicationCheckpoint();
|
|
119
|
+
}
|
|
120
|
+
async getCheckpointInternal() {
|
|
121
|
+
return await this.db.client.withSession({ snapshot: true }, async (session) => {
|
|
122
|
+
const doc = await this.db.sync_rules.findOne({ _id: this.group_id }, {
|
|
123
|
+
session,
|
|
124
|
+
projection: { _id: 1, state: 1, last_checkpoint: 1, last_checkpoint_lsn: 1, snapshot_done: 1 }
|
|
125
|
+
});
|
|
126
|
+
if (!doc?.snapshot_done || !['ACTIVE', 'ERRORED'].includes(doc.state)) {
|
|
127
|
+
// Sync rules not active - return null
|
|
128
|
+
return null;
|
|
129
|
+
}
|
|
130
|
+
// Specifically using operationTime instead of clusterTime
|
|
131
|
+
// There are 3 fields in the response:
|
|
132
|
+
// 1. operationTime, not exposed for snapshot sessions (used for causal consistency)
|
|
133
|
+
// 2. clusterTime (used for connection management)
|
|
134
|
+
// 3. atClusterTime, which is session.snapshotTime
|
|
135
|
+
// We use atClusterTime, to match the driver's internal snapshot handling.
|
|
136
|
+
// There are cases where clusterTime > operationTime and atClusterTime,
|
|
137
|
+
// which could cause snapshot queries using this as the snapshotTime to timeout.
|
|
138
|
+
// This was specifically observed on MongoDB 6.0 and 7.0.
|
|
139
|
+
const snapshotTime = session.snapshotTime;
|
|
140
|
+
if (snapshotTime == null) {
|
|
141
|
+
throw new ServiceAssertionError('Missing snapshotTime in getCheckpoint()');
|
|
142
|
+
}
|
|
143
|
+
return new MongoReplicationCheckpoint(this,
|
|
144
|
+
// null/0n is a valid checkpoint in some cases, for example if the initial snapshot was empty
|
|
145
|
+
doc.last_checkpoint ?? 0n, doc.last_checkpoint_lsn ?? null, snapshotTime);
|
|
119
146
|
});
|
|
120
|
-
if (!doc?.snapshot_done) {
|
|
121
|
-
return {
|
|
122
|
-
checkpoint: 0n,
|
|
123
|
-
lsn: null
|
|
124
|
-
};
|
|
125
|
-
}
|
|
126
|
-
return {
|
|
127
|
-
checkpoint: doc?.last_checkpoint ?? 0n,
|
|
128
|
-
lsn: doc?.last_checkpoint_lsn ?? null
|
|
129
|
-
};
|
|
130
147
|
}
|
|
131
148
|
async startBatch(options, callback) {
|
|
132
149
|
const env_1 = { stack: [], error: void 0, hasError: false };
|
|
@@ -258,37 +275,63 @@ export class MongoSyncBucketStorage extends BaseObserver {
|
|
|
258
275
|
return result;
|
|
259
276
|
}
|
|
260
277
|
async getParameterSets(checkpoint, lookups) {
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
},
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
278
|
+
return this.db.client.withSession({ snapshot: true }, async (session) => {
|
|
279
|
+
// Set the session's snapshot time to the checkpoint's snapshot time.
|
|
280
|
+
// An alternative would be to create the session when the checkpoint is created, but managing
|
|
281
|
+
// the session lifetime would become more complex.
|
|
282
|
+
// Starting and ending sessions are cheap (synchronous when no transactions are used),
|
|
283
|
+
// so this should be fine.
|
|
284
|
+
// This is a roundabout way of setting {readConcern: {atClusterTime: clusterTime}}, since
|
|
285
|
+
// that is not exposed directly by the driver.
|
|
286
|
+
// Future versions of the driver may change the snapshotTime behavior, so we need tests to
|
|
287
|
+
// validate that this works as expected. We test this in the compacting tests.
|
|
288
|
+
setSessionSnapshotTime(session, checkpoint.snapshotTime);
|
|
289
|
+
const lookupFilter = lookups.map((lookup) => {
|
|
290
|
+
return storage.serializeLookup(lookup);
|
|
291
|
+
});
|
|
292
|
+
// This query does not use indexes super efficiently, apart from the lookup filter.
|
|
293
|
+
// From some experimentation I could do individual lookups more efficient using an index
|
|
294
|
+
// on {'key.g': 1, lookup: 1, 'key.t': 1, 'key.k': 1, _id: -1},
|
|
295
|
+
// but could not do the same using $group.
|
|
296
|
+
// For now, just rely on compacting to remove extraneous data.
|
|
297
|
+
// For a description of the data format, see the `/docs/parameters-lookups.md` file.
|
|
298
|
+
const rows = await this.db.bucket_parameters
|
|
299
|
+
.aggregate([
|
|
300
|
+
{
|
|
301
|
+
$match: {
|
|
302
|
+
'key.g': this.group_id,
|
|
303
|
+
lookup: { $in: lookupFilter },
|
|
304
|
+
_id: { $lte: checkpoint.checkpoint }
|
|
305
|
+
}
|
|
306
|
+
},
|
|
307
|
+
{
|
|
308
|
+
$sort: {
|
|
309
|
+
_id: -1
|
|
310
|
+
}
|
|
311
|
+
},
|
|
312
|
+
{
|
|
313
|
+
$group: {
|
|
314
|
+
_id: { key: '$key', lookup: '$lookup' },
|
|
315
|
+
bucket_parameters: {
|
|
316
|
+
$first: '$bucket_parameters'
|
|
317
|
+
}
|
|
283
318
|
}
|
|
284
319
|
}
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
320
|
+
], {
|
|
321
|
+
session,
|
|
322
|
+
readConcern: 'snapshot',
|
|
323
|
+
// Limit the time for the operation to complete, to avoid getting connection timeouts
|
|
324
|
+
maxTimeMS: lib_mongo.db.MONGO_OPERATION_TIMEOUT_MS
|
|
325
|
+
})
|
|
326
|
+
.toArray()
|
|
327
|
+
.catch((e) => {
|
|
328
|
+
throw lib_mongo.mapQueryError(e, 'while evaluating parameter queries');
|
|
329
|
+
});
|
|
330
|
+
const groupedParameters = rows.map((row) => {
|
|
331
|
+
return row.bucket_parameters;
|
|
332
|
+
});
|
|
333
|
+
return groupedParameters.flat();
|
|
290
334
|
});
|
|
291
|
-
return groupedParameters.flat();
|
|
292
335
|
}
|
|
293
336
|
async *getBucketDataBatch(checkpoint, dataBuckets, options) {
|
|
294
337
|
if (dataBuckets.size == 0) {
|
|
@@ -579,13 +622,11 @@ export class MongoSyncBucketStorage extends BaseObserver {
|
|
|
579
622
|
await this.db.notifyCheckpoint();
|
|
580
623
|
}
|
|
581
624
|
async compact(options) {
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
lsn: doc?.last_checkpoint_lsn ?? null
|
|
588
|
-
};
|
|
625
|
+
const checkpoint = await this.getCheckpointInternal();
|
|
626
|
+
await new MongoCompactor(this.db, this.group_id, options).compact();
|
|
627
|
+
if (checkpoint != null && options?.compactParameterData) {
|
|
628
|
+
await new MongoParameterCompactor(this.db, this.group_id, checkpoint.checkpoint, options).compact();
|
|
629
|
+
}
|
|
589
630
|
}
|
|
590
631
|
/**
|
|
591
632
|
* Instance-wide watch on the latest available checkpoint (op_id + lsn).
|
|
@@ -602,29 +643,12 @@ export class MongoSyncBucketStorage extends BaseObserver {
|
|
|
602
643
|
if (signal.aborted) {
|
|
603
644
|
break;
|
|
604
645
|
}
|
|
605
|
-
const
|
|
606
|
-
|
|
607
|
-
state: { $in: [storage.SyncRuleState.ACTIVE, storage.SyncRuleState.ERRORED] }
|
|
608
|
-
}, {
|
|
609
|
-
limit: 1,
|
|
610
|
-
projection: {
|
|
611
|
-
_id: 1,
|
|
612
|
-
state: 1,
|
|
613
|
-
last_checkpoint: 1,
|
|
614
|
-
last_checkpoint_lsn: 1
|
|
615
|
-
}
|
|
616
|
-
});
|
|
617
|
-
if (doc == null) {
|
|
618
|
-
// Sync rules not present or not active.
|
|
619
|
-
// Abort the connections - clients will have to retry later.
|
|
620
|
-
throw new ServiceError(ErrorCode.PSYNC_S2302, 'No active sync rules available');
|
|
621
|
-
}
|
|
622
|
-
else if (doc.state != storage.SyncRuleState.ACTIVE && doc.state != storage.SyncRuleState.ERRORED) {
|
|
646
|
+
const op = await this.getCheckpointInternal();
|
|
647
|
+
if (op == null) {
|
|
623
648
|
// Sync rules have changed - abort and restart.
|
|
624
649
|
// We do a soft close of the stream here - no error
|
|
625
650
|
break;
|
|
626
651
|
}
|
|
627
|
-
const op = this.makeActiveCheckpoint(doc);
|
|
628
652
|
// Check for LSN / checkpoint changes - ignore other metadata changes
|
|
629
653
|
if (lastOp == null || op.lsn != lastOp.lsn || op.checkpoint != lastOp.checkpoint) {
|
|
630
654
|
lastOp = op;
|
|
@@ -846,4 +870,26 @@ export class MongoSyncBucketStorage extends BaseObserver {
|
|
|
846
870
|
};
|
|
847
871
|
}
|
|
848
872
|
}
|
|
873
|
+
class MongoReplicationCheckpoint {
|
|
874
|
+
storage;
|
|
875
|
+
checkpoint;
|
|
876
|
+
lsn;
|
|
877
|
+
snapshotTime;
|
|
878
|
+
constructor(storage, checkpoint, lsn, snapshotTime) {
|
|
879
|
+
this.storage = storage;
|
|
880
|
+
this.checkpoint = checkpoint;
|
|
881
|
+
this.lsn = lsn;
|
|
882
|
+
this.snapshotTime = snapshotTime;
|
|
883
|
+
}
|
|
884
|
+
async getParameterSets(lookups) {
|
|
885
|
+
return this.storage.getParameterSets(this, lookups);
|
|
886
|
+
}
|
|
887
|
+
}
|
|
888
|
+
class EmptyReplicationCheckpoint {
|
|
889
|
+
checkpoint = 0n;
|
|
890
|
+
lsn = null;
|
|
891
|
+
async getParameterSets(lookups) {
|
|
892
|
+
return [];
|
|
893
|
+
}
|
|
894
|
+
}
|
|
849
895
|
//# sourceMappingURL=MongoSyncBucketStorage.js.map
|