@powersync/service-core 0.12.0 → 0.12.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +6 -0
- package/dist/storage/mongo/MongoCompactor.js +2 -1
- package/dist/storage/mongo/MongoCompactor.js.map +1 -1
- package/dist/storage/mongo/MongoWriteCheckpointAPI.js +3 -2
- package/dist/storage/mongo/MongoWriteCheckpointAPI.js.map +1 -1
- package/dist/storage/mongo/PersistedBatch.js +15 -5
- package/dist/storage/mongo/PersistedBatch.js.map +1 -1
- package/dist/storage/mongo/util.d.ts +14 -0
- package/dist/storage/mongo/util.js +39 -0
- package/dist/storage/mongo/util.js.map +1 -1
- package/package.json +2 -2
- package/src/storage/mongo/MongoCompactor.ts +2 -1
- package/src/storage/mongo/MongoWriteCheckpointAPI.ts +5 -2
- package/src/storage/mongo/PersistedBatch.ts +18 -5
- package/src/storage/mongo/util.ts +45 -0
- package/tsconfig.tsbuildinfo +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -2,6 +2,7 @@ import { logger } from '@powersync/lib-services-framework';
|
|
|
2
2
|
import { MaxKey, MinKey } from 'mongodb';
|
|
3
3
|
import { addChecksums } from '../../util/utils.js';
|
|
4
4
|
import { cacheKey } from './OperationBatch.js';
|
|
5
|
+
import { safeBulkWrite } from './util.js';
|
|
5
6
|
const DEFAULT_CLEAR_BATCH_LIMIT = 5000;
|
|
6
7
|
const DEFAULT_MOVE_BATCH_LIMIT = 2000;
|
|
7
8
|
const DEFAULT_MOVE_BATCH_QUERY_LIMIT = 10000;
|
|
@@ -192,7 +193,7 @@ export class MongoCompactor {
|
|
|
192
193
|
async flush() {
|
|
193
194
|
if (this.updates.length > 0) {
|
|
194
195
|
logger.info(`Compacting ${this.updates.length} ops`);
|
|
195
|
-
await this.db.bucket_data
|
|
196
|
+
await safeBulkWrite(this.db.bucket_data, this.updates, {
|
|
196
197
|
// Order is not important.
|
|
197
198
|
// Since checksums are not affected, these operations can happen in any order,
|
|
198
199
|
// and it's fine if the operations are partially applied.
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"MongoCompactor.js","sourceRoot":"","sources":["../../../src/storage/mongo/MongoCompactor.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,mCAAmC,CAAC;AAC3D,OAAO,EAAyB,MAAM,EAAE,MAAM,EAAE,MAAM,SAAS,CAAC;AAChE,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AAInD,OAAO,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAC;
|
|
1
|
+
{"version":3,"file":"MongoCompactor.js","sourceRoot":"","sources":["../../../src/storage/mongo/MongoCompactor.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,mCAAmC,CAAC;AAC3D,OAAO,EAAyB,MAAM,EAAE,MAAM,EAAE,MAAM,SAAS,CAAC;AAChE,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AAInD,OAAO,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAC;AAC/C,OAAO,EAAE,aAAa,EAAE,MAAM,WAAW,CAAC;AAqC1C,MAAM,yBAAyB,GAAG,IAAI,CAAC;AACvC,MAAM,wBAAwB,GAAG,IAAI,CAAC;AACtC,MAAM,8BAA8B,GAAG,KAAM,CAAC;AAE9C,2CAA2C;AAC3C,MAAM,uBAAuB,GAAG,EAAE,CAAC;AAEnC,MAAM,OAAO,cAAc;IAUzB,YACU,EAAkB,EAClB,QAAgB,EACxB,OAA6B;QAFrB,OAAE,GAAF,EAAE,CAAgB;QAClB,aAAQ,GAAR,QAAQ,CAAQ;QAXlB,YAAO,GAAgD,EAAE,CAAC;QAchE,IAAI,CAAC,YAAY,GAAG,CAAC,OAAO,EAAE,aAAa,IAAI,uBAAuB,CAAC,GAAG,IAAI,GAAG,IAAI,CAAC;QACtF,IAAI,CAAC,cAAc,GAAG,OAAO,EAAE,cAAc,IAAI,wBAAwB,CAAC;QAC1E,IAAI,CAAC,mBAAmB,GAAG,OAAO,EAAE,mBAAmB,IAAI,8BAA8B,CAAC;QAC1F,IAAI,CAAC,eAAe,GAAG,OAAO,EAAE,eAAe,IAAI,yBAAyB,CAAC;QAC7E,IAAI,CAAC,OAAO,GAAG,OAAO,EAAE,OAAO,CAAC;QAChC,IAAI,CAAC,OAAO,GAAG,OAAO,EAAE,cAAc,CAAC;IACzC,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,OAAO;QACX,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACjB,KAAK,IAAI,MAAM,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;gBAChC,wDAAwD;gBACxD,yCAAyC;gBACzC,0DAA0D;gBAC1D,MAAM,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC;YACrC,CAAC;QACH,CAAC;aAAM,CAAC;YACN,MAAM,IAAI,CAAC,eAAe,CAAC,SAAS,CAAC,CAAC;QACxC,CAAC;IACH,CAAC;IAED,KAAK,CAAC,eAAe,CAAC,MAA0B;QAC9C,MAAM,YAAY,GAAG,IAAI,CAAC,YAAY,CAAC;QAEvC,IAAI,YAAY,GAA8B,IAAI,CAAC;QAEnD,IAAI,WAA4B,CAAC;QACjC,IAAI,WAA4B,CAAC;QAEjC,IAAI,MAAM,IAAI,IAAI,EAAE,CAAC;YACnB,WAAW,GAAG,IAAI,MAAM,EAAE,CAAC;YAC3B,WAAW,GAAG,IAAI,MAAM,EAAE,CAAC;QAC7B,CAAC;aAAM,IAAI,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;YAChC,oBAAoB;YACpB,WAAW,GAAG,MAAM,CAAC;YACrB,WAAW,GAAG,MAAM,CAAC;QACvB,CAAC;aAAM,CAAC;YACN,yBAAyB;YACzB,WAAW,GAAG,GAAG,MAAM,GAAG,CAAC;YAC3B,WAAW,GAAG,GAAG,MAAM,SAAS,CAAC;QACnC,CAAC;QAED,uBAAuB;QACvB,MAAM,UAAU,GAAkB;YAChC,CAAC,EAAE,IAAI,CAAC,QAAQ;YAChB,CAAC,EAAE,WAAqB;YACxB,CAAC,EAAE,IAAI,MAAM,EAAS;SACvB,CAAC;QAEF,yCAAyC;QACzC,IAAI,UAAU,GAAkB;YAC9B,CAAC,EAAE,IAAI,CAAC,QAAQ;YAChB,CAAC,EAAE,WAAqB;YACxB,CAAC,EAAE,IAAI,MAAM,EAAS;SACvB,CAAC;QAEF,OAAO,IAAI,EAAE,CAAC;YACZ,sDAAsD;YACtD,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,WAAW;iBACpC,IAAI,CACH;gBACE,GAAG,EAAE;oBACH,IAAI,EAAE,UAAU;oBAChB,GAAG,EAAE,UAAU;iBAChB;aACF,EACD;gBACE,UAAU,EAAE;oBACV,GAAG,EAAE,CAAC;oBACN,EAAE,EAAE,CAAC;oBACL,KAAK,EAAE,CAAC;oBACR,MAAM,EAAE,CAAC;oBACT,YAAY,EAAE,CAAC;oBACf,UAAU,EAAE,CAAC;iBACd;gBACD,KAAK,EAAE,IAAI,CAAC,mBAAmB;gBAC/B,IAAI,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC,EAAE;gBACjB,WAAW,EAAE,IAAI;aAClB,CACF;iBACA,OAAO,EAAE,CAAC;YAEb,IAAI,KAAK,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;gBACtB,wBAAwB;gBACxB,MAAM;YACR,CAAC;YAED,oCAAoC;YACpC,UAAU,GAAG,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC;YAEzC,KAAK,IAAI,GAAG,IAAI,KAAK,EAAE,CAAC;gBACtB,IAAI,YAAY,IAAI,IAAI,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,IAAI,YAAY,CAAC,MAAM,EAAE,CAAC;oBAC7D,IAAI,YAAY,IAAI,IAAI,IAAI,YAAY,CAAC,UAAU,IAAI,IAAI,IAAI,YAAY,CAAC,WAAW,IAAI,CAAC,EAAE,CAAC;wBAC7F,0CAA0C;wBAC1C,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;wBACnB,MAAM,CAAC,IAAI,CACT,sBAAsB,IAAI,CAAC,QAAQ,IAAI,YAAY,CAAC,MAAM,IAAI,YAAY,CAAC,UAAU,cAAc,YAAY,CAAC,WAAW,aAAa,CACzI,CAAC;wBAEF,MAAM,MAAM,GAAG,YAAY,CAAC,MAAM,CAAC;wBACnC,MAAM,OAAO,GAAG,YAAY,CAAC,UAAU,CAAC;wBACxC,qCAAqC;wBACrC,YAAY,GAAG,IAAI,CAAC;wBACpB,MAAM,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;oBAC1C,CAAC;oBACD,YAAY,GAAG;wBACb,MAAM,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC;wBACjB,IAAI,EAAE,IAAI,GAAG,EAAE;wBACf,YAAY,EAAE,CAAC;wBACf,UAAU,EAAE,IAAI;wBAChB,WAAW,EAAE,CAAC;qBACf,CAAC;gBACJ,CAAC;gBAED,IAAI,IAAI,CAAC,OAAO,IAAI,IAAI,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC;oBACrD,SAAS;gBACX,CAAC;gBAED,IAAI,eAAe,GAAG,GAAG,CAAC,EAAE,IAAI,KAAK,CAAC;gBAEtC,IAAI,GAAG,CAAC,EAAE,IAAI,QAAQ,IAAI,GAAG,CAAC,EAAE,IAAI,KAAK,EAAE,CAAC;oBAC1C,MAAM,GAAG,GAAG,GAAG,GAAG,CAAC,KAAK,IAAI,GAAG,CAAC,MAAM,IAAI,QAAQ,CAAC,GAAG,CAAC,YAAa,EAAE,GAAG,CAAC,UAAW,CAAC,EAAE,CAAC;oBACzF,MAAM,QAAQ,GAAG,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;oBAC5C,IAAI,QAAQ,EAAE,CAAC;wBACb,8CAA8C;wBAC9C,eAAe,GAAG,KAAK,CAAC;wBAExB,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC;4BAChB,SAAS,EAAE;gCACT,MAAM,EAAE;oCACN,GAAG,EAAE,GAAG,CAAC,GAAG;iCACb;gCACD,MAAM,EAAE;oCACN,IAAI,EAAE;wCACJ,EAAE,EAAE,MAAM;wCACV,SAAS,EAAE,QAAQ;qCACpB;oCACD,MAAM,EAAE;wCACN,YAAY,EAAE,CAAC;wCACf,UAAU,EAAE,CAAC;wCACb,KAAK,EAAE,CAAC;wCACR,MAAM,EAAE,CAAC;wCACT,IAAI,EAAE,CAAC;qCACR;iCACF;6BACF;yBACF,CAAC,CAAC;oBACL,CAAC;yBAAM,CAAC;wBACN,IAAI,YAAY,CAAC,YAAY,IAAI,YAAY,EAAE,CAAC;4BAC9C,wBAAwB;4BACxB,6CAA6C;wBAC/C,CAAC;6BAAM,CAAC;4BACN,4DAA4D;4BAC5D,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;4BAC/C,6BAA6B;4BAC7B,oBAAoB;4BACpB,sBAAsB;4BACtB,6BAA6B;4BAC7B,YAAY,CAAC,YAAY,IAAI,GAAG,CAAC,MAAM,GAAG,GAAG,CAAC;wBAChD,CAAC;oBACH,CAAC;gBACH,CAAC;gBAED,IAAI,eAAe,EAAE,CAAC;oBACpB,YAAY,CAAC,UAAU,GAAG,IAAI,CAAC;oBAC/B,YAAY,CAAC,WAAW,GAAG,CAAC,CAAC;gBAC/B,CAAC;qBAAM,IAAI,GAAG,CAAC,EAAE,IAAI,OAAO,EAAE,CAAC;oBAC7B,IAAI,YAAY,CAAC,UAAU,IAAI,IAAI,EAAE,CAAC;wBACpC,YAAY,CAAC,UAAU,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;oBACtC,CAAC;oBACD,YAAY,CAAC,WAAW,IAAI,CAAC,CAAC;gBAChC,CAAC;gBAED,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;oBAC/C,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;gBACrB,CAAC;YACH,CAAC;QACH,CAAC;QAED,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;QACnB,YAAY,EAAE,IAAI,CAAC,KAAK,EAAE,CAAC;QAC3B,IAAI,YAAY,EAAE,UAAU,IAAI,IAAI,IAAI,YAAY,EAAE,WAAW,GAAG,CAAC,EAAE,CAAC;YACtE,MAAM,CAAC,IAAI,CACT,sBAAsB,IAAI,CAAC,QAAQ,IAAI,YAAY,CAAC,MAAM,IAAI,YAAY,CAAC,UAAU,cAAc,YAAY,CAAC,WAAW,aAAa,CACzI,CAAC;YACF,MAAM,MAAM,GAAG,YAAY,CAAC,MAAM,CAAC;YACnC,MAAM,OAAO,GAAG,YAAY,CAAC,UAAU,CAAC;YACxC,qCAAqC;YACrC,YAAY,GAAG,IAAI,CAAC;YACpB,MAAM,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QAC1C,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,KAAK;QACjB,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC5B,MAAM,CAAC,IAAI,CAAC,cAAc,IAAI,CAAC,OAAO,CAAC,MAAM,MAAM,CAAC,CAAC;YACrD,MAAM,aAAa,CAAC,IAAI,CAAC,EAAE,CAAC,WAAW,EAAE,IAAI,CAAC,OAAO,EAAE;gBACrD,0BAA0B;gBAC1B,8EAA8E;gBAC9E,yDAAyD;gBACzD,uCAAuC;gBACvC,OAAO,EAAE,KAAK;aACf,CAAC,CAAC;YACH,IAAI,CAAC,OAAO,GAAG,EAAE,CAAC;QACpB,CAAC;IACH,CAAC;IAED;;;;;OAKG;IACK,KAAK,CAAC,WAAW,CAAC,MAAc,EAAE,EAAU;QAClD,MAAM,QAAQ,GAAG;YACf,GAAG,EAAE;gBACH,IAAI,EAAE;oBACJ,CAAC,EAAE,IAAI,CAAC,QAAQ;oBAChB,CAAC,EAAE,MAAM;oBACT,CAAC,EAAE,IAAI,MAAM,EAAS;iBACvB;gBACD,IAAI,EAAE;oBACJ,CAAC,EAAE,IAAI,CAAC,QAAQ;oBAChB,CAAC,EAAE,MAAM;oBACT,CAAC,EAAE,EAAE;iBACN;aACF;SACF,CAAC;QAEF,MAAM,OAAO,GAAG,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,YAAY,EAAE,CAAC;QAC9C,IAAI,CAAC;YACH,IAAI,IAAI,GAAG,KAAK,CAAC;YACjB,OAAO,CAAC,IAAI,EAAE,CAAC;gBACb,6EAA6E;gBAC7E,kDAAkD;gBAClD,0EAA0E;gBAC1E,MAAM,OAAO,CAAC,eAAe,CAC3B,KAAK,IAAI,EAAE;oBACT,MAAM,KAAK,GAAG,IAAI,CAAC,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,QAAQ,EAAE;wBAC/C,OAAO;wBACP,IAAI,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE;wBAChB,UAAU,EAAE;4BACV,GAAG,EAAE,CAAC;4BACN,EAAE,EAAE,CAAC;4BACL,QAAQ,EAAE,CAAC;4BACX,SAAS,EAAE,CAAC;yBACb;wBACD,KAAK,EAAE,IAAI,CAAC,eAAe;qBAC5B,CAAC,CAAC;oBACH,IAAI,QAAQ,GAAG,CAAC,CAAC;oBACjB,IAAI,QAAQ,GAAyB,IAAI,CAAC;oBAC1C,IAAI,QAAQ,GAAkB,IAAI,CAAC;oBACnC,IAAI,OAAO,GAAG,KAAK,CAAC;oBACpB,IAAI,KAAK,EAAE,IAAI,EAAE,IAAI,KAAK,CAAC,MAAM,EAAE,EAAE,CAAC;wBACpC,IAAI,EAAE,CAAC,EAAE,IAAI,MAAM,IAAI,EAAE,CAAC,EAAE,IAAI,QAAQ,IAAI,EAAE,CAAC,EAAE,IAAI,OAAO,EAAE,CAAC;4BAC7D,QAAQ,GAAG,YAAY,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,CAAC;4BAC/C,QAAQ,GAAG,EAAE,CAAC,GAAG,CAAC;4BAClB,IAAI,EAAE,CAAC,EAAE,IAAI,OAAO,EAAE,CAAC;gCACrB,OAAO,GAAG,IAAI,CAAC;4BACjB,CAAC;4BACD,IAAI,EAAE,CAAC,SAAS,IAAI,IAAI,EAAE,CAAC;gCACzB,IAAI,QAAQ,IAAI,IAAI,IAAI,EAAE,CAAC,SAAS,GAAG,QAAQ,EAAE,CAAC;oCAChD,QAAQ,GAAG,EAAE,CAAC,SAAS,CAAC;gCAC1B,CAAC;4BACH,CAAC;wBACH,CAAC;6BAAM,CAAC;4BACN,MAAM,IAAI,KAAK,CAAC,cAAc,EAAE,CAAC,EAAE,iBAAiB,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC;wBAC1F,CAAC;oBACH,CAAC;oBACD,IAAI,CAAC,OAAO,EAAE,CAAC;wBACb,IAAI,GAAG,IAAI,CAAC;wBACZ,OAAO;oBACT,CAAC;oBAED,MAAM,CAAC,IAAI,CAAC,qBAAqB,QAAQ,EAAE,CAAC,EAAE,CAAC,CAAC;oBAChD,MAAM,IAAI,CAAC,EAAE,CAAC,WAAW,CAAC,UAAU,CAClC;wBACE,GAAG,EAAE;4BACH,IAAI,EAAE;gCACJ,CAAC,EAAE,IAAI,CAAC,QAAQ;gCAChB,CAAC,EAAE,MAAM;gCACT,CAAC,EAAE,IAAI,MAAM,EAAS;6BACvB;4BACD,IAAI,EAAE,QAAS;yBAChB;qBACF,EACD,EAAE,OAAO,EAAE,CACZ,CAAC;oBAEF,MAAM,IAAI,CAAC,EAAE,CAAC,WAAW,CAAC,SAAS,CACjC;wBACE,GAAG,EAAE,QAAS;wBACd,EAAE,EAAE,OAAO;wBACX,QAAQ,EAAE,QAAQ;wBAClB,IAAI,EAAE,IAAI;wBACV,SAAS,EAAE,QAAQ;qBACpB,EACD,EAAE,OAAO,EAAE,CACZ,CAAC;gBACJ,CAAC,EACD;oBACE,YAAY,EAAE,EAAE,CAAC,EAAE,UAAU,EAAE;oBAC/B,WAAW,EAAE,EAAE,KAAK,EAAE,UAAU,EAAE;iBACnC,CACF,CAAC;YACJ,CAAC;QACH,CAAC;gBAAS,CAAC;YACT,MAAM,OAAO,CAAC,UAAU,EAAE,CAAC;QAC7B,CAAC;IACH,CAAC;CACF;AAED;;;;;GAKG;AACH,SAAS,OAAO,CAAC,CAAS;IACxB,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IACf,OAAO,CAAC,CAAC;AACX,CAAC"}
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import * as framework from '@powersync/lib-services-framework';
|
|
2
2
|
import { WriteCheckpointMode } from '../WriteCheckpointAPI.js';
|
|
3
|
+
import { safeBulkWrite } from './util.js';
|
|
3
4
|
export class MongoWriteCheckpointAPI {
|
|
4
5
|
constructor(options) {
|
|
5
6
|
this.db = options.db;
|
|
@@ -87,7 +88,7 @@ export async function batchCreateCustomWriteCheckpoints(db, checkpoints) {
|
|
|
87
88
|
if (!checkpoints.length) {
|
|
88
89
|
return;
|
|
89
90
|
}
|
|
90
|
-
await db.custom_write_checkpoints
|
|
91
|
+
await safeBulkWrite(db.custom_write_checkpoints, checkpoints.map((checkpointOptions) => ({
|
|
91
92
|
updateOne: {
|
|
92
93
|
filter: { user_id: checkpointOptions.user_id, sync_rules_id: checkpointOptions.sync_rules_id },
|
|
93
94
|
update: {
|
|
@@ -98,6 +99,6 @@ export async function batchCreateCustomWriteCheckpoints(db, checkpoints) {
|
|
|
98
99
|
},
|
|
99
100
|
upsert: true
|
|
100
101
|
}
|
|
101
|
-
})));
|
|
102
|
+
})), {});
|
|
102
103
|
}
|
|
103
104
|
//# sourceMappingURL=MongoWriteCheckpointAPI.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"MongoWriteCheckpointAPI.js","sourceRoot":"","sources":["../../../src/storage/mongo/MongoWriteCheckpointAPI.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,SAAS,MAAM,mCAAmC,CAAC;AAC/D,OAAO,EAOL,mBAAmB,EACpB,MAAM,0BAA0B,CAAC;
|
|
1
|
+
{"version":3,"file":"MongoWriteCheckpointAPI.js","sourceRoot":"","sources":["../../../src/storage/mongo/MongoWriteCheckpointAPI.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,SAAS,MAAM,mCAAmC,CAAC;AAC/D,OAAO,EAOL,mBAAmB,EACpB,MAAM,0BAA0B,CAAC;AAElC,OAAO,EAAE,aAAa,EAAE,MAAM,WAAW,CAAC;AAO1C,MAAM,OAAO,uBAAuB;IAIlC,YAAY,OAAkC;QAC5C,IAAI,CAAC,EAAE,GAAG,OAAO,CAAC,EAAE,CAAC;QACrB,IAAI,CAAC,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC;IAC5B,CAAC;IAED,IAAI,mBAAmB;QACrB,OAAO,IAAI,CAAC,KAAK,CAAC;IACpB,CAAC;IAED,sBAAsB,CAAC,IAAyB;QAC9C,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC;IACpB,CAAC;IAED,KAAK,CAAC,iCAAiC,CAAC,WAA2C;QACjF,OAAO,iCAAiC,CAAC,IAAI,CAAC,EAAE,EAAE,WAAW,CAAC,CAAC;IACjE,CAAC;IAED,KAAK,CAAC,2BAA2B,CAAC,OAAqC;QACrE,IAAI,IAAI,CAAC,mBAAmB,KAAK,mBAAmB,CAAC,MAAM,EAAE,CAAC;YAC5D,MAAM,IAAI,SAAS,CAAC,MAAM,CAAC,eAAe,CACxC,wFAAwF,IAAI,CAAC,mBAAmB,GAAG,CACpH,CAAC;QACJ,CAAC;QAED,MAAM,EAAE,UAAU,EAAE,OAAO,EAAE,aAAa,EAAE,GAAG,OAAO,CAAC;QACvD,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,wBAAwB,CAAC,gBAAgB,CACjE;YACE,OAAO,EAAE,OAAO;YAChB,aAAa;SACd,EACD;YACE,IAAI,EAAE;gBACJ,UAAU;aACX;SACF,EACD,EAAE,MAAM,EAAE,IAAI,EAAE,cAAc,EAAE,OAAO,EAAE,CAC1C,CAAC;QACF,OAAO,GAAI,CAAC,UAAU,CAAC;IACzB,CAAC;IAED,KAAK,CAAC,4BAA4B,CAAC,UAAyC;QAC1E,IAAI,IAAI,CAAC,mBAAmB,KAAK,mBAAmB,CAAC,OAAO,EAAE,CAAC;YAC7D,MAAM,IAAI,SAAS,CAAC,MAAM,CAAC,eAAe,CACxC,qGAAqG,IAAI,CAAC,mBAAmB,GAAG,CACjI,CAAC;QACJ,CAAC;QAED,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,GAAG,UAAU,CAAC;QAC5C,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,iBAAiB,CAAC,gBAAgB,CAC1D;YACE,OAAO,EAAE,OAAO;SACjB,EACD;YACE,IAAI,EAAE;gBACJ,IAAI;aACL;YACD,IAAI,EAAE;gBACJ,SAAS,EAAE,EAAE;aACd;SACF,EACD,EAAE,MAAM,EAAE,IAAI,EAAE,cAAc,EAAE,OAAO,EAAE,CAC1C,CAAC;QACF,OAAO,GAAI,CAAC,SAAS,CAAC;IACxB,CAAC;IAED,KAAK,CAAC,mBAAmB,CAAC,OAAmC;QAC3D,QAAQ,IAAI,CAAC,mBAAmB,EAAE,CAAC;YACjC,KAAK,mBAAmB,CAAC,MAAM;gBAC7B,IAAI,KAAK,IAAI,eAAe,IAAI,OAAO,EAAE,CAAC;oBACxC,MAAM,IAAI,SAAS,CAAC,MAAM,CAAC,eAAe,CAAC,iEAAiE,CAAC,CAAC;gBAChH,CAAC;gBACD,OAAO,IAAI,CAAC,yBAAyB,CAAC,OAAO,CAAC,CAAC;YACjD,KAAK,mBAAmB,CAAC,OAAO;gBAC9B,IAAI,KAAK,IAAI,OAAO,IAAI,OAAO,EAAE,CAAC;oBAChC,MAAM,IAAI,SAAS,CAAC,MAAM,CAAC,eAAe,CACxC,qEAAqE,CACtE,CAAC;gBACJ,CAAC;gBACD,OAAO,IAAI,CAAC,0BAA0B,CAAC,OAAO,CAAC,CAAC;QACpD,CAAC;IACH,CAAC;IAES,KAAK,CAAC,yBAAyB,CAAC,OAAqC;QAC7E,MAAM,EAAE,OAAO,EAAE,aAAa,EAAE,GAAG,OAAO,CAAC;QAC3C,MAAM,mBAAmB,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,wBAAwB,CAAC,OAAO,CAAC;YACzE,OAAO;YACP,aAAa;SACd,CAAC,CAAC;QACH,OAAO,mBAAmB,EAAE,UAAU,IAAI,IAAI,CAAC;IACjD,CAAC;IAES,KAAK,CAAC,0BAA0B,CAAC,OAAsC;QAC/E,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,GAAG,OAAO,CAAC;QACnC,4EAA4E;QAC5E,MAAM,GAAG,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC;QACvB,IAAI,GAAG,IAAI,IAAI,EAAE,CAAC;YAChB,oDAAoD;YACpD,OAAO,IAAI,CAAC;QACd,CAAC;QACD,MAAM,mBAAmB,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,iBAAiB,CAAC,OAAO,CAAC;YAClE,OAAO,EAAE,OAAO;YAChB,QAAQ,EAAE,EAAE,IAAI,EAAE,GAAG,EAAE;SACxB,CAAC,CAAC;QACH,OAAO,mBAAmB,EAAE,SAAS,IAAI,IAAI,CAAC;IAChD,CAAC;CACF;AAED,MAAM,CAAC,KAAK,UAAU,iCAAiC,CACrD,EAAkB,EAClB,WAA2C;IAE3C,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,CAAC;QACxB,OAAO;IACT,CAAC;IAED,MAAM,aAAa,CACjB,EAAE,CAAC,wBAAwB,EAC3B,WAAW,CAAC,GAAG,CAAC,CAAC,iBAAiB,EAAE,EAAE,CAAC,CAAC;QACtC,SAAS,EAAE;YACT,MAAM,EAAE,EAAE,OAAO,EAAE,iBAAiB,CAAC,OAAO,EAAE,aAAa,EAAE,iBAAiB,CAAC,aAAa,EAAE;YAC9F,MAAM,EAAE;gBACN,IAAI,EAAE;oBACJ,UAAU,EAAE,iBAAiB,CAAC,UAAU;oBACxC,aAAa,EAAE,iBAAiB,CAAC,aAAa;iBAC/C;aACF;YACD,MAAM,EAAE,IAAI;SACb;KACF,CAAC,CAAC,EACH,EAAE,CACH,CAAC;AACJ,CAAC"}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { JSONBig } from '@powersync/service-jsonbig';
|
|
2
2
|
import * as util from '../../util/util-index.js';
|
|
3
3
|
import { currentBucketKey } from './MongoBucketBatch.js';
|
|
4
|
-
import { replicaIdToSubkey, serializeLookup } from './util.js';
|
|
4
|
+
import { replicaIdToSubkey, safeBulkWrite, serializeLookup } from './util.js';
|
|
5
5
|
import { logger } from '@powersync/lib-services-framework';
|
|
6
6
|
/**
|
|
7
7
|
* Maximum size of operations we write in a single transaction.
|
|
@@ -16,6 +16,12 @@ import { logger } from '@powersync/lib-services-framework';
|
|
|
16
16
|
* When we reach this threshold, we commit the transaction and start a new one.
|
|
17
17
|
*/
|
|
18
18
|
const MAX_TRANSACTION_BATCH_SIZE = 30000000;
|
|
19
|
+
/**
|
|
20
|
+
* Limit number of documents to write in a single transaction.
|
|
21
|
+
*
|
|
22
|
+
* This has an effect on error message size in some cases.
|
|
23
|
+
*/
|
|
24
|
+
const MAX_TRANSACTION_DOC_COUNT = 2000;
|
|
19
25
|
/**
|
|
20
26
|
* Keeps track of bulkwrite operations within a transaction.
|
|
21
27
|
*
|
|
@@ -178,25 +184,29 @@ export class PersistedBatch {
|
|
|
178
184
|
this.currentSize += (values.data?.length() ?? 0) + 100;
|
|
179
185
|
}
|
|
180
186
|
shouldFlushTransaction() {
|
|
181
|
-
return this.currentSize >= MAX_TRANSACTION_BATCH_SIZE
|
|
187
|
+
return (this.currentSize >= MAX_TRANSACTION_BATCH_SIZE ||
|
|
188
|
+
this.bucketData.length >= MAX_TRANSACTION_DOC_COUNT ||
|
|
189
|
+
this.currentData.length >= MAX_TRANSACTION_DOC_COUNT ||
|
|
190
|
+
this.bucketParameters.length >= MAX_TRANSACTION_DOC_COUNT);
|
|
182
191
|
}
|
|
183
192
|
async flush(db, session) {
|
|
184
193
|
if (this.bucketData.length > 0) {
|
|
185
|
-
|
|
194
|
+
// calculate total size
|
|
195
|
+
await safeBulkWrite(db.bucket_data, this.bucketData, {
|
|
186
196
|
session,
|
|
187
197
|
// inserts only - order doesn't matter
|
|
188
198
|
ordered: false
|
|
189
199
|
});
|
|
190
200
|
}
|
|
191
201
|
if (this.bucketParameters.length > 0) {
|
|
192
|
-
await db.bucket_parameters
|
|
202
|
+
await safeBulkWrite(db.bucket_parameters, this.bucketParameters, {
|
|
193
203
|
session,
|
|
194
204
|
// inserts only - order doesn't matter
|
|
195
205
|
ordered: false
|
|
196
206
|
});
|
|
197
207
|
}
|
|
198
208
|
if (this.currentData.length > 0) {
|
|
199
|
-
await db.current_data
|
|
209
|
+
await safeBulkWrite(db.current_data, this.currentData, {
|
|
200
210
|
session,
|
|
201
211
|
// may update and delete data within the same batch - order matters
|
|
202
212
|
ordered: true
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"PersistedBatch.js","sourceRoot":"","sources":["../../../src/storage/mongo/PersistedBatch.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,4BAA4B,CAAC;AAKrD,OAAO,KAAK,IAAI,MAAM,0BAA0B,CAAC;AAEjD,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AAWzD,OAAO,EAAE,iBAAiB,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;
|
|
1
|
+
{"version":3,"file":"PersistedBatch.js","sourceRoot":"","sources":["../../../src/storage/mongo/PersistedBatch.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,4BAA4B,CAAC;AAKrD,OAAO,KAAK,IAAI,MAAM,0BAA0B,CAAC;AAEjD,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AAWzD,OAAO,EAAE,iBAAiB,EAAE,aAAa,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;AAC9E,OAAO,EAAE,MAAM,EAAE,MAAM,mCAAmC,CAAC;AAE3D;;;;;;;;;;;GAWG;AACH,MAAM,0BAA0B,GAAG,QAAU,CAAC;AAE9C;;;;GAIG;AACH,MAAM,yBAAyB,GAAG,IAAK,CAAC;AAExC;;;;;GAKG;AACH,MAAM,OAAO,cAAc;IAezB,YACU,QAAgB,EACxB,WAAmB;QADX,aAAQ,GAAR,QAAQ,CAAQ;QAf1B,eAAU,GAAsD,EAAE,CAAC;QACnE,qBAAgB,GAA2D,EAAE,CAAC;QAC9E,gBAAW,GAAuD,EAAE,CAAC;QAErE;;WAEG;QACH,kBAAa,GAAkB,IAAI,CAAC;QAEpC;;WAEG;QACH,gBAAW,GAAG,CAAC,CAAC;QAMd,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC;IACjC,CAAC;IAED,cAAc,CAAC,OAMd;QACC,MAAM,iBAAiB,GAAG,IAAI,GAAG,EAAyB,CAAC;QAC3D,KAAK,IAAI,CAAC,IAAI,OAAO,CAAC,cAAc,EAAE,CAAC;YACrC,MAAM,GAAG,GAAG,gBAAgB,CAAC,CAAC,CAAC,CAAC;YAChC,iBAAiB,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;QAChC,CAAC;QAED,MAAM,SAAS,GAAG,IAAI,CAAC,UAAU,CAAC,iBAAiB,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,EAAE,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC;QAE1F,KAAK,IAAI,CAAC,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC;YAChC,MAAM,GAAG,GAAG,gBAAgB,CAAC,CAAC,CAAC,CAAC;YAChC,iBAAiB,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YAE9B,SAAS;YACT,MAAM,UAAU,GAAG,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;YAC7C,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,EAAE,EAAE,UAAU,CAAC,CAAC;YAC1D,IAAI,CAAC,WAAW,IAAI,UAAU,CAAC,MAAM,GAAG,GAAG,CAAC;YAE5C,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;YACpC,IAAI,CAAC,aAAa,GAAG,KAAK,CAAC;YAE3B,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;gBACnB,SAAS,EAAE;oBACT,QAAQ,EAAE;wBACR,GAAG,EAAE;4BACH,CAAC,EAAE,IAAI,CAAC,QAAQ;4BAChB,CAAC,EAAE,CAAC,CAAC,MAAM;4BACX,CAAC,EAAE,KAAK;yBACT;wBACD,EAAE,EAAE,KAAK;wBACT,YAAY,EAAE,OAAO,CAAC,KAAK,CAAC,EAAE;wBAC9B,UAAU,EAAE,OAAO,CAAC,SAAS;wBAC7B,KAAK,EAAE,CAAC,CAAC,KAAK;wBACd,MAAM,EAAE,CAAC,CAAC,EAAE;wBACZ,QAAQ,EAAE,QAAQ;wBAClB,IAAI,EAAE,UAAU;qBACjB;iBACF;aACF,CAAC,CAAC;QACL,CAAC;QAED,KAAK,IAAI,EAAE,IAAI,iBAAiB,CAAC,MAAM,EAAE,EAAE,CAAC;YAC1C,SAAS;YAET,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;YACpC,IAAI,CAAC,aAAa,GAAG,KAAK,CAAC;YAE3B,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;gBACnB,SAAS,EAAE;oBACT,QAAQ,EAAE;wBACR,GAAG,EAAE;4BACH,CAAC,EAAE,IAAI,CAAC,QAAQ;4BAChB,CAAC,EAAE,EAAE,CAAC,MAAM;4BACZ,CAAC,EAAE,KAAK;yBACT;wBACD,EAAE,EAAE,QAAQ;wBACZ,YAAY,EAAE,OAAO,CAAC,KAAK,CAAC,EAAE;wBAC9B,UAAU,EAAE,OAAO,CAAC,SAAS;wBAC7B,KAAK,EAAE,EAAE,CAAC,KAAK;wBACf,MAAM,EAAE,EAAE,CAAC,EAAE;wBACb,QAAQ,EAAE,SAAS;wBACnB,IAAI,EAAE,IAAI;qBACX;iBACF;aACF,CAAC,CAAC;YACH,IAAI,CAAC,WAAW,IAAI,GAAG,CAAC;QAC1B,CAAC;IACH,CAAC;IAED,iBAAiB,CAAC,IAMjB;QACC,yCAAyC;QACzC,qEAAqE;QACrE,0HAA0H;QAC1H,2DAA2D;QAC3D,8GAA8G;QAC9G,+BAA+B;QAC/B,6CAA6C;QAC7C,MAAM,EAAE,WAAW,EAAE,SAAS,EAAE,SAAS,EAAE,GAAG,IAAI,CAAC;QAEnD,MAAM,iBAAiB,GAAG,IAAI,GAAG,EAAuB,CAAC;QACzD,KAAK,IAAI,CAAC,IAAI,IAAI,CAAC,gBAAgB,EAAE,CAAC;YACpC,iBAAiB,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC;QACjD,CAAC;QAED,wBAAwB;QACxB,KAAK,IAAI,MAAM,IAAI,SAAS,EAAE,CAAC;YAC7B,MAAM,SAAS,GAAG,eAAe,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;YACjD,MAAM,GAAG,GAAG,SAAS,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;YACzC,iBAAiB,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YAE9B,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;YACjC,IAAI,CAAC,aAAa,GAAG,KAAK,CAAC;YAC3B,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC;gBACzB,SAAS,EAAE;oBACT,QAAQ,EAAE;wBACR,GAAG,EAAE,KAAK;wBACV,GAAG,EAAE;4BACH,CAAC,EAAE,IAAI,CAAC,QAAQ;4BAChB,CAAC,EAAE,WAAW,CAAC,EAAE;4BACjB,CAAC,EAAE,SAAS;yBACb;wBACD,MAAM,EAAE,SAAS;wBACjB,iBAAiB,EAAE,MAAM,CAAC,iBAAiB;qBAC5C;iBACF;aACF,CAAC,CAAC;YAEH,IAAI,CAAC,WAAW,IAAI,GAAG,CAAC;QAC1B,CAAC;QAED,kDAAkD;QAClD,KAAK,IAAI,MAAM,IAAI,iBAAiB,CAAC,MAAM,EAAE,EAAE,CAAC;YAC9C,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;YACjC,IAAI,CAAC,aAAa,GAAG,KAAK,CAAC;YAC3B,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC;gBACzB,SAAS,EAAE;oBACT,QAAQ,EAAE;wBACR,GAAG,EAAE,KAAK;wBACV,GAAG,EAAE;4BACH,CAAC,EAAE,IAAI,CAAC,QAAQ;4BAChB,CAAC,EAAE,WAAW,CAAC,EAAE;4BACjB,CAAC,EAAE,SAAS;yBACb;wBACD,MAAM,EAAE,MAAM;wBACd,iBAAiB,EAAE,EAAE;qBACtB;iBACF;aACF,CAAC,CAAC;YAEH,IAAI,CAAC,WAAW,IAAI,GAAG,CAAC;QAC1B,CAAC;IACH,CAAC;IAED,iBAAiB,CAAC,EAAa;QAC7B,MAAM,EAAE,GAAqD;YAC3D,SAAS,EAAE;gBACT,MAAM,EAAE,EAAE,GAAG,EAAE,EAAE,EAAE;aACpB;SACF,CAAC;QACF,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAC1B,IAAI,CAAC,WAAW,IAAI,EAAE,CAAC;IACzB,CAAC;IAED,iBAAiB,CAAC,EAAa,EAAE,MAAoC;QACnE,MAAM,EAAE,GAAqD;YAC3D,SAAS,EAAE;gBACT,MAAM,EAAE,EAAE,GAAG,EAAE,EAAE,EAAE;gBACnB,MAAM,EAAE;oBACN,IAAI,EAAE,MAAM;iBACb;gBACD,MAAM,EAAE,IAAI;aACb;SACF,CAAC;QACF,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAC1B,IAAI,CAAC,WAAW,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,CAAC,GAAG,GAAG,CAAC;IACzD,CAAC;IAED,sBAAsB;QACpB,OAAO,CACL,IAAI,CAAC,WAAW,IAAI,0BAA0B;YAC9C,IAAI,CAAC,UAAU,CAAC,MAAM,IAAI,yBAAyB;YACnD,IAAI,CAAC,WAAW,CAAC,MAAM,IAAI,yBAAyB;YACpD,IAAI,CAAC,gBAAgB,CAAC,MAAM,IAAI,yBAAyB,CAC1D,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,EAAkB,EAAE,OAA4B;QAC1D,IAAI,IAAI,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC/B,uBAAuB;YACvB,MAAM,aAAa,CAAC,EAAE,CAAC,WAAW,EAAE,IAAI,CAAC,UAAU,EAAE;gBACnD,OAAO;gBACP,sCAAsC;gBACtC,OAAO,EAAE,KAAK;aACf,CAAC,CAAC;QACL,CAAC;QACD,IAAI,IAAI,CAAC,gBAAgB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACrC,MAAM,aAAa,CAAC,EAAE,CAAC,iBAAiB,EAAE,IAAI,CAAC,gBAAgB,EAAE;gBAC/D,OAAO;gBACP,sCAAsC;gBACtC,OAAO,EAAE,KAAK;aACf,CAAC,CAAC;QACL,CAAC;QACD,IAAI,IAAI,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAChC,MAAM,aAAa,CAAC,EAAE,CAAC,YAAY,EAAE,IAAI,CAAC,WAAW,EAAE;gBACrD,OAAO;gBACP,mEAAmE;gBACnE,OAAO,EAAE,IAAI;aACd,CAAC,CAAC;QACL,CAAC;QAED,MAAM,CAAC,IAAI,CACT,aAAa,IAAI,CAAC,QAAQ,YAAY,IAAI,CAAC,UAAU,CAAC,MAAM,MAAM,IAAI,CAAC,gBAAgB,CAAC,MAAM,MAC5F,IAAI,CAAC,WAAW,CAAC,MACnB,aAAa,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,mBAAmB,IAAI,CAAC,aAAa,EAAE,CACxF,CAAC;QAEF,IAAI,CAAC,UAAU,GAAG,EAAE,CAAC;QACrB,IAAI,CAAC,gBAAgB,GAAG,EAAE,CAAC;QAC3B,IAAI,CAAC,WAAW,GAAG,EAAE,CAAC;QACtB,IAAI,CAAC,WAAW,GAAG,CAAC,CAAC;QACrB,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC;IAC5B,CAAC;CACF"}
|
|
@@ -38,3 +38,17 @@ export declare function replicaIdToSubkey(table: bson.ObjectId, id: ReplicaId):
|
|
|
38
38
|
* Works even with multiple copies of the bson package.
|
|
39
39
|
*/
|
|
40
40
|
export declare function isUUID(value: any): value is bson.UUID;
|
|
41
|
+
/**
|
|
42
|
+
* MongoDB bulkWrite internally splits the operations into batches
|
|
43
|
+
* so that no batch exceeds 16MB. However, there are cases where
|
|
44
|
+
* the batch size is very close to 16MB, where additional metadata
|
|
45
|
+
* on the server pushes it over the limit, resulting in this error
|
|
46
|
+
* from the server:
|
|
47
|
+
*
|
|
48
|
+
* > MongoBulkWriteError: BSONObj size: 16814023 (0x1008FC7) is invalid. Size must be between 0 and 16793600(16MB) First element: insert: "bucket_data"
|
|
49
|
+
*
|
|
50
|
+
* We work around the issue by doing our own batching, limiting the
|
|
51
|
+
* batch size to 15MB. This does add additional overhead with
|
|
52
|
+
* BSON.calculateObjectSize.
|
|
53
|
+
*/
|
|
54
|
+
export declare function safeBulkWrite<T extends mongo.Document>(collection: mongo.Collection<T>, operations: mongo.AnyBulkWriteOperation<T>[], options: mongo.BulkWriteOptions): Promise<void>;
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import * as bson from 'bson';
|
|
2
2
|
import * as crypto from 'crypto';
|
|
3
|
+
import * as mongo from 'mongodb';
|
|
3
4
|
import * as uuid from 'uuid';
|
|
4
5
|
import { ID_NAMESPACE, timestampToOpId } from '../../util/utils.js';
|
|
5
6
|
/**
|
|
@@ -148,4 +149,42 @@ export function isUUID(value) {
|
|
|
148
149
|
const uuid = value;
|
|
149
150
|
return uuid._bsontype == 'Binary' && uuid.sub_type == bson.Binary.SUBTYPE_UUID;
|
|
150
151
|
}
|
|
152
|
+
/**
|
|
153
|
+
* MongoDB bulkWrite internally splits the operations into batches
|
|
154
|
+
* so that no batch exceeds 16MB. However, there are cases where
|
|
155
|
+
* the batch size is very close to 16MB, where additional metadata
|
|
156
|
+
* on the server pushes it over the limit, resulting in this error
|
|
157
|
+
* from the server:
|
|
158
|
+
*
|
|
159
|
+
* > MongoBulkWriteError: BSONObj size: 16814023 (0x1008FC7) is invalid. Size must be between 0 and 16793600(16MB) First element: insert: "bucket_data"
|
|
160
|
+
*
|
|
161
|
+
* We work around the issue by doing our own batching, limiting the
|
|
162
|
+
* batch size to 15MB. This does add additional overhead with
|
|
163
|
+
* BSON.calculateObjectSize.
|
|
164
|
+
*/
|
|
165
|
+
export async function safeBulkWrite(collection, operations, options) {
|
|
166
|
+
// Must be below 16MB.
|
|
167
|
+
// We could probably go a little closer, but 15MB is a safe threshold.
|
|
168
|
+
const BULK_WRITE_LIMIT = 15 * 1024 * 1024;
|
|
169
|
+
let batch = [];
|
|
170
|
+
let currentSize = 0;
|
|
171
|
+
// Estimated overhead per operation, should be smaller in reality.
|
|
172
|
+
const keySize = 8;
|
|
173
|
+
for (let op of operations) {
|
|
174
|
+
const bsonSize = mongo.BSON.calculateObjectSize(op, {
|
|
175
|
+
checkKeys: false,
|
|
176
|
+
ignoreUndefined: true
|
|
177
|
+
}) + keySize;
|
|
178
|
+
if (batch.length > 0 && currentSize + bsonSize > BULK_WRITE_LIMIT) {
|
|
179
|
+
await collection.bulkWrite(batch, options);
|
|
180
|
+
currentSize = 0;
|
|
181
|
+
batch = [];
|
|
182
|
+
}
|
|
183
|
+
batch.push(op);
|
|
184
|
+
currentSize += bsonSize;
|
|
185
|
+
}
|
|
186
|
+
if (batch.length > 0) {
|
|
187
|
+
await collection.bulkWrite(batch, options);
|
|
188
|
+
}
|
|
189
|
+
}
|
|
151
190
|
//# sourceMappingURL=util.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"util.js","sourceRoot":"","sources":["../../../src/storage/mongo/util.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAC7B,OAAO,KAAK,MAAM,MAAM,QAAQ,CAAC;
|
|
1
|
+
{"version":3,"file":"util.js","sourceRoot":"","sources":["../../../src/storage/mongo/util.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAC7B,OAAO,KAAK,MAAM,MAAM,QAAQ,CAAC;AACjC,OAAO,KAAK,KAAK,MAAM,SAAS,CAAC;AACjC,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAE7B,OAAO,EAAE,YAAY,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAGpE;;;GAGG;AAEH,MAAM,UAAU,eAAe,CAAC,MAAyB;IACvD,MAAM,UAAU,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE;QACtC,IAAI,OAAO,KAAK,IAAI,QAAQ,IAAI,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,CAAC;YACxD,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC;QACvB,CAAC;aAAM,CAAC;YACN,OAAO,KAAK,CAAC;QACf,CAAC;IACH,CAAC,CAAC,CAAC;IACH,OAAO,IAAI,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC,EAAE,UAAU,EAAE,CAAC,CAAC,CAAC;AAC5D,CAAC;AAED,MAAM,UAAU,cAAc,CAAI,MAAkB,EAAE,IAAiB;IACrE,IAAI,MAAM,GAAG;QACX,IAAI,EAAE;YACJ,GAAG,MAAM;SACH;QACR,GAAG,EAAE;YACH,GAAG,MAAM;SACH;KACT,CAAC;IAEF,KAAK,IAAI,GAAG,IAAI,IAAI,EAAE,CAAC;QACrB,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;QACrC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;IACtC,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,MAAc,EAAE,aAAqB;IACpE,MAAM,WAAW,GAAG,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;IAC1D,OAAO,GAAG,MAAM,GAAG,aAAa,IAAI,WAAW,EAAE,CAAC;AACpD,CAAC;AAED;;;;;;;;;;GAUG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CAAI,MAA2B;IAClE,IAAI,CAAC;QACH,IAAI,IAAS,CAAC;QACd,IAAI,OAAO,GAAG,IAAI,CAAC;QACnB,2CAA2C;QAC3C,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,OAAO,EAAE,CAAC;QACtC,yCAAyC;QACzC,IAAI,GAAG,MAAM,CAAC,qBAAqB,EAAE,CAAC;QACtC,IAAI,CAAC,MAAM,IAAI,MAAM,CAAC,EAAE,EAAE,MAAM,EAAE,EAAE,CAAC;YACnC,0CAA0C;YAC1C,wEAAwE;YACxE,uEAAuE;YACvE,oCAAoC;YACpC,EAAE;YACF,4EAA4E;YAC5E,2DAA2D;YAC3D,gCAAgC;YAChC,OAAO,GAAG,KAAK,CAAC;QAClB,CAAC;QACD,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC;IAC3B,CAAC;YAAS,CAAC;QACT,iDAAiD;QACjD,uIAAuI;QACvI,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;YACnB,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC;QACvB,CAAC;IACH,CAAC;AACH,CAAC;AAED,MAAM,CAAC,MAAM,wBAAwB,GAA4B;IAC/D,6BAA6B;IAC7B,WAAW,EAAE,IAAI;CAClB,CAAC;AAEF,MAAM,UAAU,UAAU,CAAC,GAAuB;IAChD,IAAI,GAAG,CAAC,EAAE,IAAI,KAAK,IAAI,GAAG,CAAC,EAAE,IAAI,QAAQ,EAAE,CAAC;QAC1C,OAAO;YACL,KAAK,EAAE,eAAe,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;YACjC,EAAE,EAAE,GAAG,CAAC,EAAE;YACV,WAAW,EAAE,GAAG,CAAC,KAAK;YACtB,SAAS,EAAE,GAAG,CAAC,MAAM;YACrB,QAAQ,EAAE,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;YAC9B,MAAM,EAAE,iBAAiB,CAAC,GAAG,CAAC,YAAa,EAAE,GAAG,CAAC,UAAW,CAAC;YAC7D,IAAI,EAAE,GAAG,CAAC,IAAI;SACf,CAAC;IACJ,CAAC;SAAM,CAAC;QACN,cAAc;QAEd,OAAO;YACL,KAAK,EAAE,eAAe,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;YACjC,EAAE,EAAE,GAAG,CAAC,EAAE;YACV,QAAQ,EAAE,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;SAC/B,CAAC;IACJ,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,eAAe,CAAC,CAAY,EAAE,CAAY;IACxD,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QACZ,OAAO,IAAI,CAAC;IACd,CAAC;SAAM,IAAI,OAAO,CAAC,IAAI,QAAQ,IAAI,OAAO,CAAC,IAAI,QAAQ,EAAE,CAAC;QACxD,OAAO,CAAC,IAAI,CAAC,CAAC;IAChB,CAAC;SAAM,IAAI,MAAM,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC;QAClC,OAAO,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;IACrB,CAAC;SAAM,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC;QAClC,OAAO,IAAI,CAAC;IACd,CAAC;SAAM,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC;QAClC,OAAO,KAAK,CAAC;IACf,CAAC;SAAM,CAAC;QACN,iEAAiE;QACjE,OAAQ,IAAI,CAAC,SAAS,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAY,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;IACjF,CAAC;AACH,CAAC;AAED,MAAM,UAAU,iBAAiB,CAAC,KAAoB,EAAE,EAAa;IACnE,IAAI,MAAM,CAAC,EAAE,CAAC,EAAE,CAAC;QACf,mDAAmD;QACnD,OAAO,GAAG,KAAK,CAAC,WAAW,EAAE,IAAI,EAAE,CAAC,WAAW,EAAE,EAAE,CAAC;IACtD,CAAC;SAAM,CAAC;QACN,oCAAoC;QACpC,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC,CAAC;QAC3C,OAAO,IAAI,CAAC,EAAE,CAAC,IAAI,EAAE,YAAY,CAAC,CAAC;IACrC,CAAC;AACH,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,MAAM,CAAC,KAAU;IAC/B,IAAI,KAAK,IAAI,IAAI,IAAI,OAAO,KAAK,IAAI,QAAQ,EAAE,CAAC;QAC9C,OAAO,KAAK,CAAC;IACf,CAAC;IACD,MAAM,IAAI,GAAG,KAAkB,CAAC;IAChC,OAAO,IAAI,CAAC,SAAS,IAAI,QAAQ,IAAI,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC;AACjF,CAAC;AAED;;;;;;;;;;;;GAYG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,UAA+B,EAC/B,UAA4C,EAC5C,OAA+B;IAE/B,sBAAsB;IACtB,sEAAsE;IACtE,MAAM,gBAAgB,GAAG,EAAE,GAAG,IAAI,GAAG,IAAI,CAAC;IAE1C,IAAI,KAAK,GAAqC,EAAE,CAAC;IACjD,IAAI,WAAW,GAAG,CAAC,CAAC;IACpB,kEAAkE;IAClE,MAAM,OAAO,GAAG,CAAC,CAAC;IAClB,KAAK,IAAI,EAAE,IAAI,UAAU,EAAE,CAAC;QAC1B,MAAM,QAAQ,GACZ,KAAK,CAAC,IAAI,CAAC,mBAAmB,CAAC,EAAE,EAAE;YACjC,SAAS,EAAE,KAAK;YAChB,eAAe,EAAE,IAAI;SACf,CAAC,GAAG,OAAO,CAAC;QACtB,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,IAAI,WAAW,GAAG,QAAQ,GAAG,gBAAgB,EAAE,CAAC;YAClE,MAAM,UAAU,CAAC,SAAS,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;YAC3C,WAAW,GAAG,CAAC,CAAC;YAChB,KAAK,GAAG,EAAE,CAAC;QACb,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACf,WAAW,IAAI,QAAQ,CAAC;IAC1B,CAAC;IACD,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACrB,MAAM,UAAU,CAAC,SAAS,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;IAC7C,CAAC;AACH,CAAC"}
|
package/package.json
CHANGED
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
"publishConfig": {
|
|
6
6
|
"access": "public"
|
|
7
7
|
},
|
|
8
|
-
"version": "0.12.
|
|
8
|
+
"version": "0.12.1",
|
|
9
9
|
"main": "dist/index.js",
|
|
10
10
|
"license": "FSL-1.1-Apache-2.0",
|
|
11
11
|
"type": "module",
|
|
@@ -26,7 +26,7 @@
|
|
|
26
26
|
"jose": "^4.15.1",
|
|
27
27
|
"lodash": "^4.17.21",
|
|
28
28
|
"lru-cache": "^10.2.2",
|
|
29
|
-
"mongodb": "^6.
|
|
29
|
+
"mongodb": "^6.11.0",
|
|
30
30
|
"node-fetch": "^3.3.2",
|
|
31
31
|
"ts-codec": "^1.2.2",
|
|
32
32
|
"uri-js": "^4.4.1",
|
|
@@ -5,6 +5,7 @@ import { PowerSyncMongo } from './db.js';
|
|
|
5
5
|
import { BucketDataDocument, BucketDataKey } from './models.js';
|
|
6
6
|
import { CompactOptions } from '../BucketStorage.js';
|
|
7
7
|
import { cacheKey } from './OperationBatch.js';
|
|
8
|
+
import { safeBulkWrite } from './util.js';
|
|
8
9
|
|
|
9
10
|
interface CurrentBucketState {
|
|
10
11
|
/** Bucket name */
|
|
@@ -264,7 +265,7 @@ export class MongoCompactor {
|
|
|
264
265
|
private async flush() {
|
|
265
266
|
if (this.updates.length > 0) {
|
|
266
267
|
logger.info(`Compacting ${this.updates.length} ops`);
|
|
267
|
-
await this.db.bucket_data
|
|
268
|
+
await safeBulkWrite(this.db.bucket_data, this.updates, {
|
|
268
269
|
// Order is not important.
|
|
269
270
|
// Since checksums are not affected, these operations can happen in any order,
|
|
270
271
|
// and it's fine if the operations are partially applied.
|
|
@@ -9,6 +9,7 @@ import {
|
|
|
9
9
|
WriteCheckpointMode
|
|
10
10
|
} from '../WriteCheckpointAPI.js';
|
|
11
11
|
import { PowerSyncMongo } from './db.js';
|
|
12
|
+
import { safeBulkWrite } from './util.js';
|
|
12
13
|
|
|
13
14
|
export type MongoCheckpointAPIOptions = {
|
|
14
15
|
db: PowerSyncMongo;
|
|
@@ -134,7 +135,8 @@ export async function batchCreateCustomWriteCheckpoints(
|
|
|
134
135
|
return;
|
|
135
136
|
}
|
|
136
137
|
|
|
137
|
-
await
|
|
138
|
+
await safeBulkWrite(
|
|
139
|
+
db.custom_write_checkpoints,
|
|
138
140
|
checkpoints.map((checkpointOptions) => ({
|
|
139
141
|
updateOne: {
|
|
140
142
|
filter: { user_id: checkpointOptions.user_id, sync_rules_id: checkpointOptions.sync_rules_id },
|
|
@@ -146,6 +148,7 @@ export async function batchCreateCustomWriteCheckpoints(
|
|
|
146
148
|
},
|
|
147
149
|
upsert: true
|
|
148
150
|
}
|
|
149
|
-
}))
|
|
151
|
+
})),
|
|
152
|
+
{}
|
|
150
153
|
);
|
|
151
154
|
}
|
|
@@ -16,7 +16,7 @@ import {
|
|
|
16
16
|
SourceKey,
|
|
17
17
|
ReplicaId
|
|
18
18
|
} from './models.js';
|
|
19
|
-
import { replicaIdToSubkey, serializeLookup } from './util.js';
|
|
19
|
+
import { replicaIdToSubkey, safeBulkWrite, serializeLookup } from './util.js';
|
|
20
20
|
import { logger } from '@powersync/lib-services-framework';
|
|
21
21
|
|
|
22
22
|
/**
|
|
@@ -33,6 +33,13 @@ import { logger } from '@powersync/lib-services-framework';
|
|
|
33
33
|
*/
|
|
34
34
|
const MAX_TRANSACTION_BATCH_SIZE = 30_000_000;
|
|
35
35
|
|
|
36
|
+
/**
|
|
37
|
+
* Limit number of documents to write in a single transaction.
|
|
38
|
+
*
|
|
39
|
+
* This has an effect on error message size in some cases.
|
|
40
|
+
*/
|
|
41
|
+
const MAX_TRANSACTION_DOC_COUNT = 2_000;
|
|
42
|
+
|
|
36
43
|
/**
|
|
37
44
|
* Keeps track of bulkwrite operations within a transaction.
|
|
38
45
|
*
|
|
@@ -231,26 +238,32 @@ export class PersistedBatch {
|
|
|
231
238
|
}
|
|
232
239
|
|
|
233
240
|
shouldFlushTransaction() {
|
|
234
|
-
return
|
|
241
|
+
return (
|
|
242
|
+
this.currentSize >= MAX_TRANSACTION_BATCH_SIZE ||
|
|
243
|
+
this.bucketData.length >= MAX_TRANSACTION_DOC_COUNT ||
|
|
244
|
+
this.currentData.length >= MAX_TRANSACTION_DOC_COUNT ||
|
|
245
|
+
this.bucketParameters.length >= MAX_TRANSACTION_DOC_COUNT
|
|
246
|
+
);
|
|
235
247
|
}
|
|
236
248
|
|
|
237
249
|
async flush(db: PowerSyncMongo, session: mongo.ClientSession) {
|
|
238
250
|
if (this.bucketData.length > 0) {
|
|
239
|
-
|
|
251
|
+
// calculate total size
|
|
252
|
+
await safeBulkWrite(db.bucket_data, this.bucketData, {
|
|
240
253
|
session,
|
|
241
254
|
// inserts only - order doesn't matter
|
|
242
255
|
ordered: false
|
|
243
256
|
});
|
|
244
257
|
}
|
|
245
258
|
if (this.bucketParameters.length > 0) {
|
|
246
|
-
await db.bucket_parameters
|
|
259
|
+
await safeBulkWrite(db.bucket_parameters, this.bucketParameters, {
|
|
247
260
|
session,
|
|
248
261
|
// inserts only - order doesn't matter
|
|
249
262
|
ordered: false
|
|
250
263
|
});
|
|
251
264
|
}
|
|
252
265
|
if (this.currentData.length > 0) {
|
|
253
|
-
await db.current_data
|
|
266
|
+
await safeBulkWrite(db.current_data, this.currentData, {
|
|
254
267
|
session,
|
|
255
268
|
// may update and delete data within the same batch - order matters
|
|
256
269
|
ordered: true
|
|
@@ -156,3 +156,48 @@ export function isUUID(value: any): value is bson.UUID {
|
|
|
156
156
|
const uuid = value as bson.UUID;
|
|
157
157
|
return uuid._bsontype == 'Binary' && uuid.sub_type == bson.Binary.SUBTYPE_UUID;
|
|
158
158
|
}
|
|
159
|
+
|
|
160
|
+
/**
|
|
161
|
+
* MongoDB bulkWrite internally splits the operations into batches
|
|
162
|
+
* so that no batch exceeds 16MB. However, there are cases where
|
|
163
|
+
* the batch size is very close to 16MB, where additional metadata
|
|
164
|
+
* on the server pushes it over the limit, resulting in this error
|
|
165
|
+
* from the server:
|
|
166
|
+
*
|
|
167
|
+
* > MongoBulkWriteError: BSONObj size: 16814023 (0x1008FC7) is invalid. Size must be between 0 and 16793600(16MB) First element: insert: "bucket_data"
|
|
168
|
+
*
|
|
169
|
+
* We work around the issue by doing our own batching, limiting the
|
|
170
|
+
* batch size to 15MB. This does add additional overhead with
|
|
171
|
+
* BSON.calculateObjectSize.
|
|
172
|
+
*/
|
|
173
|
+
export async function safeBulkWrite<T extends mongo.Document>(
|
|
174
|
+
collection: mongo.Collection<T>,
|
|
175
|
+
operations: mongo.AnyBulkWriteOperation<T>[],
|
|
176
|
+
options: mongo.BulkWriteOptions
|
|
177
|
+
) {
|
|
178
|
+
// Must be below 16MB.
|
|
179
|
+
// We could probably go a little closer, but 15MB is a safe threshold.
|
|
180
|
+
const BULK_WRITE_LIMIT = 15 * 1024 * 1024;
|
|
181
|
+
|
|
182
|
+
let batch: mongo.AnyBulkWriteOperation<T>[] = [];
|
|
183
|
+
let currentSize = 0;
|
|
184
|
+
// Estimated overhead per operation, should be smaller in reality.
|
|
185
|
+
const keySize = 8;
|
|
186
|
+
for (let op of operations) {
|
|
187
|
+
const bsonSize =
|
|
188
|
+
mongo.BSON.calculateObjectSize(op, {
|
|
189
|
+
checkKeys: false,
|
|
190
|
+
ignoreUndefined: true
|
|
191
|
+
} as any) + keySize;
|
|
192
|
+
if (batch.length > 0 && currentSize + bsonSize > BULK_WRITE_LIMIT) {
|
|
193
|
+
await collection.bulkWrite(batch, options);
|
|
194
|
+
currentSize = 0;
|
|
195
|
+
batch = [];
|
|
196
|
+
}
|
|
197
|
+
batch.push(op);
|
|
198
|
+
currentSize += bsonSize;
|
|
199
|
+
}
|
|
200
|
+
if (batch.length > 0) {
|
|
201
|
+
await collection.bulkWrite(batch, options);
|
|
202
|
+
}
|
|
203
|
+
}
|