@powersync/service-module-mongodb-storage 0.10.4 → 0.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +65 -0
- package/LICENSE +3 -3
- package/dist/storage/implementation/MongoBucketBatch.d.ts +21 -2
- package/dist/storage/implementation/MongoBucketBatch.js +66 -7
- package/dist/storage/implementation/MongoBucketBatch.js.map +1 -1
- package/dist/storage/implementation/MongoCompactor.d.ts +7 -0
- package/dist/storage/implementation/MongoCompactor.js +122 -44
- package/dist/storage/implementation/MongoCompactor.js.map +1 -1
- package/dist/storage/implementation/MongoParameterCompactor.d.ts +17 -0
- package/dist/storage/implementation/MongoParameterCompactor.js +92 -0
- package/dist/storage/implementation/MongoParameterCompactor.js.map +1 -0
- package/dist/storage/implementation/MongoSyncBucketStorage.d.ts +14 -4
- package/dist/storage/implementation/MongoSyncBucketStorage.js +229 -115
- package/dist/storage/implementation/MongoSyncBucketStorage.js.map +1 -1
- package/dist/storage/implementation/PersistedBatch.d.ts +1 -0
- package/dist/storage/implementation/PersistedBatch.js +12 -5
- package/dist/storage/implementation/PersistedBatch.js.map +1 -1
- package/dist/storage/implementation/models.d.ts +20 -0
- package/dist/storage/implementation/util.d.ts +2 -1
- package/dist/storage/implementation/util.js +13 -0
- package/dist/storage/implementation/util.js.map +1 -1
- package/package.json +9 -9
- package/src/storage/implementation/MongoBucketBatch.ts +82 -8
- package/src/storage/implementation/MongoCompactor.ts +147 -47
- package/src/storage/implementation/MongoParameterCompactor.ts +105 -0
- package/src/storage/implementation/MongoSyncBucketStorage.ts +257 -157
- package/src/storage/implementation/PersistedBatch.ts +13 -5
- package/src/storage/implementation/models.ts +21 -0
- package/src/storage/implementation/util.ts +14 -1
- package/test/src/__snapshots__/storage_sync.test.ts.snap +319 -11
- package/test/src/storage_compacting.test.ts +2 -0
- package/tsconfig.tsbuildinfo +1 -1
|
@@ -48,16 +48,18 @@ export class PersistedBatch {
|
|
|
48
48
|
this.currentSize = writtenSize;
|
|
49
49
|
this.logger = options?.logger ?? defaultLogger;
|
|
50
50
|
}
|
|
51
|
-
incrementBucket(bucket, op_id) {
|
|
51
|
+
incrementBucket(bucket, op_id, bytes) {
|
|
52
52
|
let existingState = this.bucketStates.get(bucket);
|
|
53
53
|
if (existingState) {
|
|
54
54
|
existingState.lastOp = op_id;
|
|
55
55
|
existingState.incrementCount += 1;
|
|
56
|
+
existingState.incrementBytes += bytes;
|
|
56
57
|
}
|
|
57
58
|
else {
|
|
58
59
|
this.bucketStates.set(bucket, {
|
|
59
60
|
lastOp: op_id,
|
|
60
|
-
incrementCount: 1
|
|
61
|
+
incrementCount: 1,
|
|
62
|
+
incrementBytes: bytes
|
|
61
63
|
});
|
|
62
64
|
}
|
|
63
65
|
}
|
|
@@ -82,7 +84,8 @@ export class PersistedBatch {
|
|
|
82
84
|
continue;
|
|
83
85
|
}
|
|
84
86
|
remaining_buckets.delete(key);
|
|
85
|
-
|
|
87
|
+
const byteEstimate = recordData.length + 200;
|
|
88
|
+
this.currentSize += byteEstimate;
|
|
86
89
|
const op_id = options.op_seq.next();
|
|
87
90
|
this.debugLastOpId = op_id;
|
|
88
91
|
this.bucketData.push({
|
|
@@ -103,7 +106,7 @@ export class PersistedBatch {
|
|
|
103
106
|
}
|
|
104
107
|
}
|
|
105
108
|
});
|
|
106
|
-
this.incrementBucket(k.bucket, op_id);
|
|
109
|
+
this.incrementBucket(k.bucket, op_id, byteEstimate);
|
|
107
110
|
}
|
|
108
111
|
for (let bd of remaining_buckets.values()) {
|
|
109
112
|
// REMOVE
|
|
@@ -128,7 +131,7 @@ export class PersistedBatch {
|
|
|
128
131
|
}
|
|
129
132
|
});
|
|
130
133
|
this.currentSize += 200;
|
|
131
|
-
this.incrementBucket(bd.bucket, op_id);
|
|
134
|
+
this.incrementBucket(bd.bucket, op_id, 200);
|
|
132
135
|
}
|
|
133
136
|
}
|
|
134
137
|
saveParameterData(data) {
|
|
@@ -298,6 +301,10 @@ export class PersistedBatch {
|
|
|
298
301
|
update: {
|
|
299
302
|
$set: {
|
|
300
303
|
last_op: state.lastOp
|
|
304
|
+
},
|
|
305
|
+
$inc: {
|
|
306
|
+
'estimate_since_compact.count': state.incrementCount,
|
|
307
|
+
'estimate_since_compact.bytes': state.incrementBytes
|
|
301
308
|
}
|
|
302
309
|
},
|
|
303
310
|
upsert: true
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"PersistedBatch.js","sourceRoot":"","sources":["../../../src/storage/implementation/PersistedBatch.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,OAAO,EAAE,MAAM,4BAA4B,CAAC;AAIrD,OAAO,EAAU,MAAM,IAAI,aAAa,EAAE,MAAM,mCAAmC,CAAC;AACpF,OAAO,EAAgB,OAAO,EAAE,KAAK,EAAE,MAAM,yBAAyB,CAAC;AACvE,OAAO,EAAE,gBAAgB,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAC;AAWvE,OAAO,EAAE,iBAAiB,EAAE,MAAM,WAAW,CAAC;AAE9C;;;;;;;;;;;GAWG;AACH,MAAM,0BAA0B,GAAG,UAAU,CAAC;AAE9C;;;;GAIG;AACH,MAAM,yBAAyB,GAAG,KAAK,CAAC;AAExC;;;;;GAKG;AACH,MAAM,OAAO,cAAc;IAkBf;IAjBV,MAAM,CAAS;IACf,UAAU,GAAsD,EAAE,CAAC;IACnE,gBAAgB,GAA2D,EAAE,CAAC;IAC9E,WAAW,GAAuD,EAAE,CAAC;IACrE,YAAY,GAAmC,IAAI,GAAG,EAAE,CAAC;IAEzD;;OAEG;IACH,aAAa,GAAwB,IAAI,CAAC;IAE1C;;OAEG;IACH,WAAW,GAAG,CAAC,CAAC;IAEhB,YACU,QAAgB,EACxB,WAAmB,EACnB,OAA6B;QAFrB,aAAQ,GAAR,QAAQ,CAAQ;QAIxB,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC;QAC/B,IAAI,CAAC,MAAM,GAAG,OAAO,EAAE,MAAM,IAAI,aAAa,CAAC;IACjD,CAAC;IAEO,eAAe,CAAC,MAAc,EAAE,KAAmB;
|
|
1
|
+
{"version":3,"file":"PersistedBatch.js","sourceRoot":"","sources":["../../../src/storage/implementation/PersistedBatch.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,OAAO,EAAE,MAAM,4BAA4B,CAAC;AAIrD,OAAO,EAAU,MAAM,IAAI,aAAa,EAAE,MAAM,mCAAmC,CAAC;AACpF,OAAO,EAAgB,OAAO,EAAE,KAAK,EAAE,MAAM,yBAAyB,CAAC;AACvE,OAAO,EAAE,gBAAgB,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAC;AAWvE,OAAO,EAAE,iBAAiB,EAAE,MAAM,WAAW,CAAC;AAE9C;;;;;;;;;;;GAWG;AACH,MAAM,0BAA0B,GAAG,UAAU,CAAC;AAE9C;;;;GAIG;AACH,MAAM,yBAAyB,GAAG,KAAK,CAAC;AAExC;;;;;GAKG;AACH,MAAM,OAAO,cAAc;IAkBf;IAjBV,MAAM,CAAS;IACf,UAAU,GAAsD,EAAE,CAAC;IACnE,gBAAgB,GAA2D,EAAE,CAAC;IAC9E,WAAW,GAAuD,EAAE,CAAC;IACrE,YAAY,GAAmC,IAAI,GAAG,EAAE,CAAC;IAEzD;;OAEG;IACH,aAAa,GAAwB,IAAI,CAAC;IAE1C;;OAEG;IACH,WAAW,GAAG,CAAC,CAAC;IAEhB,YACU,QAAgB,EACxB,WAAmB,EACnB,OAA6B;QAFrB,aAAQ,GAAR,QAAQ,CAAQ;QAIxB,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC;QAC/B,IAAI,CAAC,MAAM,GAAG,OAAO,EAAE,MAAM,IAAI,aAAa,CAAC;IACjD,CAAC;IAEO,eAAe,CAAC,MAAc,EAAE,KAAmB,EAAE,KAAa;QACxE,IAAI,aAAa,GAAG,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QAClD,IAAI,aAAa,EAAE,CAAC;YAClB,aAAa,CAAC,MAAM,GAAG,KAAK,CAAC;YAC7B,aAAa,CAAC,cAAc,IAAI,CAAC,CAAC;YAClC,aAAa,CAAC,cAAc,IAAI,KAAK,CAAC;QACxC,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,MAAM,EAAE;gBAC5B,MAAM,EAAE,KAAK;gBACb,cAAc,EAAE,CAAC;gBACjB,cAAc,EAAE,KAAK;aACtB,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,cAAc,CAAC,OAMd;QACC,MAAM,iBAAiB,GAAG,IAAI,GAAG,EAAyB,CAAC;QAC3D,KAAK,IAAI,CAAC,IAAI,OAAO,CAAC,cAAc,EAAE,CAAC;YACrC,MAAM,GAAG,GAAG,gBAAgB,CAAC,CAAC,CAAC,CAAC;YAChC,iBAAiB,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;QAChC,CAAC;QAED,MAAM,SAAS,GAAG,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,iBAAiB,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,EAAE,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC;QAEnG,KAAK,MAAM,CAAC,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC;YAClC,MAAM,GAAG,GAAG,gBAAgB,CAAC,CAAC,CAAC,CAAC;YAEhC,SAAS;YACT,MAAM,UAAU,GAAG,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;YAC7C,MAAM,QAAQ,GAAG,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,EAAE,EAAE,UAAU,CAAC,CAAC;YAC3D,IAAI,UAAU,CAAC,MAAM,GAAG,YAAY,EAAE,CAAC;gBACrC,gGAAgG;gBAChG,iEAAiE;gBACjE,oGAAoG;gBACpG,yBAAyB;gBACzB,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,GAAG,eAAe,UAAU,CAAC,MAAM,mBAAmB,CAAC,CAAC;gBACjF,SAAS;YACX,CAAC;YAED,iBAAiB,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YAC9B,MAAM,YAAY,GAAG,UAAU,CAAC,MAAM,GAAG,GAAG,CAAC;YAC7C,IAAI,CAAC,WAAW,IAAI,YAAY,CAAC;YAEjC,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;YACpC,IAAI,CAAC,aAAa,GAAG,KAAK,CAAC;YAE3B,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;gBACnB,SAAS,EAAE;oBACT,QAAQ,EAAE;wBACR,GAAG,EAAE;4BACH,CAAC,EAAE,IAAI,CAAC,QAAQ;4BAChB,CAAC,EAAE,CAAC,CAAC,MAAM;4BACX,CAAC,EAAE,KAAK;yBACT;wBACD,EAAE,EAAE,KAAK;wBACT,YAAY,EAAE,OAAO,CAAC,KAAK,CAAC,EAAE;wBAC9B,UAAU,EAAE,OAAO,CAAC,SAAS;wBAC7B,KAAK,EAAE,CAAC,CAAC,KAAK;wBACd,MAAM,EAAE,CAAC,CAAC,EAAE;wBACZ,QAAQ,EAAE,MAAM,CAAC,QAAQ,CAAC;wBAC1B,IAAI,EAAE,UAAU;qBACjB;iBACF;aACF,CAAC,CAAC;YACH,IAAI,CAAC,eAAe,CAAC,CAAC,CAAC,MAAM,EAAE,KAAK,EAAE,YAAY,CAAC,CAAC;QACtD,CAAC;QAED,KAAK,IAAI,EAAE,IAAI,iBAAiB,CAAC,MAAM,EAAE,EAAE,CAAC;YAC1C,SAAS;YAET,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;YACpC,IAAI,CAAC,aAAa,GAAG,KAAK,CAAC;YAE3B,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;gBACnB,SAAS,EAAE;oBACT,QAAQ,EAAE;wBACR,GAAG,EAAE;4BACH,CAAC,EAAE,IAAI,CAAC,QAAQ;4BAChB,CAAC,EAAE,EAAE,CAAC,MAAM;4BACZ,CAAC,EAAE,KAAK;yBACT;wBACD,EAAE,EAAE,QAAQ;wBACZ,YAAY,EAAE,OAAO,CAAC,KAAK,CAAC,EAAE;wBAC9B,UAAU,EAAE,OAAO,CAAC,SAAS;wBAC7B,KAAK,EAAE,EAAE,CAAC,KAAK;wBACf,MAAM,EAAE,EAAE,CAAC,EAAE;wBACb,QAAQ,EAAE,SAAS;wBACnB,IAAI,EAAE,IAAI;qBACX;iBACF;aACF,CAAC,CAAC;YACH,IAAI,CAAC,WAAW,IAAI,GAAG,CAAC;YACxB,IAAI,CAAC,eAAe,CAAC,EAAE,CAAC,MAAM,EAAE,KAAK,EAAE,GAAG,CAAC,CAAC;QAC9C,CAAC;IACH,CAAC;IAED,iBAAiB,CAAC,IAMjB;QACC,yCAAyC;QACzC,qEAAqE;QACrE,0HAA0H;QAC1H,2DAA2D;QAC3D,8GAA8G;QAC9G,+BAA+B;QAC/B,6CAA6C;QAC7C,MAAM,EAAE,WAAW,EAAE,SAAS,EAAE,SAAS,EAAE,GAAG,IAAI,CAAC;QAEnD,MAAM,iBAAiB,GAAG,IAAI,GAAG,EAAuB,CAAC;QACzD,KAAK,IAAI,CAAC,IAAI,IAAI,CAAC,gBAAgB,EAAE,CAAC;YACpC,iBAAiB,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC;QACjD,CAAC;QAED,wBAAwB;QACxB,KAAK,IAAI,MAAM,IAAI,SAAS,EAAE,CAAC;YAC7B,MAAM,SAAS,GAAG,OAAO,CAAC,eAAe,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;YACzD,MAAM,GAAG,GAAG,SAAS,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;YACzC,iBAAiB,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YAE9B,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;YACjC,IAAI,CAAC,aAAa,GAAG,KAAK,CAAC;YAC3B,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC;gBACzB,SAAS,EAAE;oBACT,QAAQ,EAAE;wBACR,GAAG,EAAE,KAAK;wBACV,GAAG,EAAE;4BACH,CAAC,EAAE,IAAI,CAAC,QAAQ;4BAChB,CAAC,EAAE,WAAW,CAAC,EAAE;4BACjB,CAAC,EAAE,SAAS;yBACb;wBACD,MAAM,EAAE,SAAS;wBACjB,iBAAiB,EAAE,MAAM,CAAC,gBAAgB;qBAC3C;iBACF;aACF,CAAC,CAAC;YAEH,IAAI,CAAC,WAAW,IAAI,GAAG,CAAC;QAC1B,CAAC;QAED,kDAAkD;QAClD,KAAK,IAAI,MAAM,IAAI,iBAAiB,CAAC,MAAM,EAAE,EAAE,CAAC;YAC9C,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;YACjC,IAAI,CAAC,aAAa,GAAG,KAAK,CAAC;YAC3B,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC;gBACzB,SAAS,EAAE;oBACT,QAAQ,EAAE;wBACR,GAAG,EAAE,KAAK;wBACV,GAAG,EAAE;4BACH,CAAC,EAAE,IAAI,CAAC,QAAQ;4BAChB,CAAC,EAAE,WAAW,CAAC,EAAE;4BACjB,CAAC,EAAE,SAAS;yBACb;wBACD,MAAM,EAAE,MAAM;wBACd,iBAAiB,EAAE,EAAE;qBACtB;iBACF;aACF,CAAC,CAAC;YAEH,IAAI,CAAC,WAAW,IAAI,GAAG,CAAC;QAC1B,CAAC;IACH,CAAC;IAED,iBAAiB,CAAC,EAAa;QAC7B,MAAM,EAAE,GAAqD;YAC3D,SAAS,EAAE;gBACT,MAAM,EAAE,EAAE,GAAG,EAAE,EAAE,EAAE;aACpB;SACF,CAAC;QACF,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAC1B,IAAI,CAAC,WAAW,IAAI,EAAE,CAAC;IACzB,CAAC;IAED,iBAAiB,CAAC,EAAa,EAAE,MAAoC;QACnE,MAAM,EAAE,GAAqD;YAC3D,SAAS,EAAE;gBACT,MAAM,EAAE,EAAE,GAAG,EAAE,EAAE,EAAE;gBACnB,MAAM,EAAE;oBACN,IAAI,EAAE,MAAM;iBACb;gBACD,MAAM,EAAE,IAAI;aACb;SACF,CAAC;QACF,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAC1B,IAAI,CAAC,WAAW,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,CAAC,GAAG,GAAG,CAAC;IACzD,CAAC;IAED,sBAAsB;QACpB,OAAO,CACL,IAAI,CAAC,WAAW,IAAI,0BAA0B;YAC9C,IAAI,CAAC,UAAU,CAAC,MAAM,IAAI,yBAAyB;YACnD,IAAI,CAAC,WAAW,CAAC,MAAM,IAAI,yBAAyB;YACpD,IAAI,CAAC,gBAAgB,CAAC,MAAM,IAAI,yBAAyB,CAC1D,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,EAAkB,EAAE,OAA4B,EAAE,OAA0C;QACtG,MAAM,OAAO,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;QAClC,IAAI,gBAAgB,GAAG,KAAK,CAAC;QAC7B,IAAI,IAAI,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC/B,gBAAgB,GAAG,IAAI,CAAC;YACxB,MAAM,EAAE,CAAC,WAAW,CAAC,SAAS,CAAC,IAAI,CAAC,UAAU,EAAE;gBAC9C,OAAO;gBACP,sCAAsC;gBACtC,OAAO,EAAE,KAAK;aACf,CAAC,CAAC;QACL,CAAC;QACD,IAAI,IAAI,CAAC,gBAAgB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACrC,gBAAgB,GAAG,IAAI,CAAC;YACxB,MAAM,EAAE,CAAC,iBAAiB,CAAC,SAAS,CAAC,IAAI,CAAC,gBAAgB,EAAE;gBAC1D,OAAO;gBACP,sCAAsC;gBACtC,OAAO,EAAE,KAAK;aACf,CAAC,CAAC;QACL,CAAC;QACD,IAAI,IAAI,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAChC,gBAAgB,GAAG,IAAI,CAAC;YACxB,MAAM,EAAE,CAAC,YAAY,CAAC,SAAS,CAAC,IAAI,CAAC,WAAW,EAAE;gBAChD,OAAO;gBACP,mEAAmE;gBACnE,OAAO,EAAE,IAAI;aACd,CAAC,CAAC;QACL,CAAC;QAED,IAAI,IAAI,CAAC,YAAY,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;YAC/B,gBAAgB,GAAG,IAAI,CAAC;YACxB,MAAM,EAAE,CAAC,YAAY,CAAC,SAAS,CAAC,IAAI,CAAC,qBAAqB,EAAE,EAAE;gBAC5D,OAAO;gBACP,8CAA8C;gBAC9C,OAAO,EAAE,KAAK;aACf,CAAC,CAAC;QACL,CAAC;QAED,IAAI,gBAAgB,EAAE,CAAC;YACrB,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC,CAAC;YACzD,IAAI,OAAO,EAAE,uBAAuB,IAAI,IAAI,EAAE,CAAC;gBAC7C,MAAM,cAAc,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC,uBAAuB,CAAC,OAAO,EAAE,CAAC,GAAG,IAAI,CAAC,CAAC;gBAEnG,IAAI,CAAC,MAAM,CAAC,IAAI,CACd,WAAW,IAAI,CAAC,UAAU,CAAC,MAAM,MAAM,IAAI,CAAC,gBAAgB,CAAC,MAAM,MACjE,IAAI,CAAC,WAAW,CAAC,MACnB,aAAa,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,SAAS,QAAQ,mBAAmB,IAAI,CAAC,aAAa,sBAAsB,cAAc,GAAG,EAC7I;oBACE,OAAO,EAAE;wBACP,QAAQ,EAAE,QAAQ;wBAClB,IAAI,EAAE,IAAI,CAAC,WAAW;wBACtB,iBAAiB,EAAE,IAAI,CAAC,UAAU,CAAC,MAAM;wBACzC,oBAAoB,EAAE,IAAI,CAAC,gBAAgB,CAAC,MAAM;wBAClD,kBAAkB,EAAE,IAAI,CAAC,WAAW,CAAC,MAAM;wBAC3C,uBAAuB,EAAE,cAAc;qBACxC;iBACF,CACF,CAAC;YACJ,CAAC;iBAAM,CAAC;gBACN,IAAI,CAAC,MAAM,CAAC,IAAI,CACd,WAAW,IAAI,CAAC,UAAU,CAAC,MAAM,MAAM,IAAI,CAAC,gBAAgB,CAAC,MAAM,MACjE,IAAI,CAAC,WAAW,CAAC,MACnB,aAAa,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,SAAS,QAAQ,mBAAmB,IAAI,CAAC,aAAa,EAAE,EACxG;oBACE,OAAO,EAAE;wBACP,QAAQ,EAAE,QAAQ;wBAClB,IAAI,EAAE,IAAI,CAAC,WAAW;wBACtB,iBAAiB,EAAE,IAAI,CAAC,UAAU,CAAC,MAAM;wBACzC,oBAAoB,EAAE,IAAI,CAAC,gBAAgB,CAAC,MAAM;wBAClD,kBAAkB,EAAE,IAAI,CAAC,WAAW,CAAC,MAAM;qBAC5C;iBACF,CACF,CAAC;YACJ,CAAC;QACH,CAAC;QAED,IAAI,CAAC,UAAU,GAAG,EAAE,CAAC;QACrB,IAAI,CAAC,gBAAgB,GAAG,EAAE,CAAC;QAC3B,IAAI,CAAC,WAAW,GAAG,EAAE,CAAC;QACtB,IAAI,CAAC,YAAY,CAAC,KAAK,EAAE,CAAC;QAC1B,IAAI,CAAC,WAAW,GAAG,CAAC,CAAC;QACrB,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC;IAC5B,CAAC;IAEO,qBAAqB;QAC3B,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,OAAO,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,EAAE,KAAK,CAAC,EAAE,EAAE;YACrE,OAAO;gBACL,SAAS,EAAE;oBACT,MAAM,EAAE;wBACN,GAAG,EAAE;4BACH,CAAC,EAAE,IAAI,CAAC,QAAQ;4BAChB,CAAC,EAAE,MAAM;yBACV;qBACF;oBACD,MAAM,EAAE;wBACN,IAAI,EAAE;4BACJ,OAAO,EAAE,KAAK,CAAC,MAAM;yBACtB;wBACD,IAAI,EAAE;4BACJ,8BAA8B,EAAE,KAAK,CAAC,cAAc;4BACpD,8BAA8B,EAAE,KAAK,CAAC,cAAc;yBACrD;qBACF;oBACD,MAAM,EAAE,IAAI;iBACb;aACyD,CAAC;QAC/D,CAAC,CAAC,CAAC;IACL,CAAC;CACF"}
|
|
@@ -91,6 +91,20 @@ export interface BucketStateDocument {
|
|
|
91
91
|
b: string;
|
|
92
92
|
};
|
|
93
93
|
last_op: bigint;
|
|
94
|
+
/**
|
|
95
|
+
* If set, this can be treated as "cache" of a checksum at a specific point.
|
|
96
|
+
* Can be updated periodically, for example by the compact job.
|
|
97
|
+
*/
|
|
98
|
+
compacted_state?: {
|
|
99
|
+
op_id: InternalOpId;
|
|
100
|
+
count: number;
|
|
101
|
+
checksum: bigint;
|
|
102
|
+
bytes: number;
|
|
103
|
+
};
|
|
104
|
+
estimate_since_compact?: {
|
|
105
|
+
count: number;
|
|
106
|
+
bytes: number;
|
|
107
|
+
};
|
|
94
108
|
}
|
|
95
109
|
export interface IdSequenceDocument {
|
|
96
110
|
_id: string;
|
|
@@ -106,9 +120,15 @@ export interface SyncRuleDocument {
|
|
|
106
120
|
*/
|
|
107
121
|
snapshot_done: boolean;
|
|
108
122
|
/**
|
|
123
|
+
* This is now used for "resumeLsn".
|
|
124
|
+
*
|
|
109
125
|
* If snapshot_done = false, this may be the lsn at which we started the snapshot.
|
|
110
126
|
*
|
|
111
127
|
* This can be used for resuming the snapshot after a restart.
|
|
128
|
+
*
|
|
129
|
+
* If snapshot_done is true, this is treated as the point to restart replication from.
|
|
130
|
+
*
|
|
131
|
+
* More specifically, we resume replication from max(snapshot_lsn, last_checkpoint_lsn).
|
|
112
132
|
*/
|
|
113
133
|
snapshot_lsn: string | undefined;
|
|
114
134
|
/**
|
|
@@ -16,7 +16,7 @@ export declare function generateSlotName(prefix: string, sync_rules_id: number):
|
|
|
16
16
|
*
|
|
17
17
|
* For this to be effective, set batchSize = limit in the find command.
|
|
18
18
|
*/
|
|
19
|
-
export declare function readSingleBatch<T>(cursor: mongo.
|
|
19
|
+
export declare function readSingleBatch<T>(cursor: mongo.AbstractCursor<T>): Promise<{
|
|
20
20
|
data: T[];
|
|
21
21
|
hasMore: boolean;
|
|
22
22
|
}>;
|
|
@@ -26,3 +26,4 @@ export declare function replicaIdToSubkey(table: bson.ObjectId, id: storage.Repl
|
|
|
26
26
|
* Helper for unit tests
|
|
27
27
|
*/
|
|
28
28
|
export declare const connectMongoForTests: (url: string, isCI: boolean) => PowerSyncMongo;
|
|
29
|
+
export declare function setSessionSnapshotTime(session: mongo.ClientSession, time: bson.Timestamp): void;
|
|
@@ -4,6 +4,7 @@ import * as uuid from 'uuid';
|
|
|
4
4
|
import { mongo } from '@powersync/lib-service-mongodb';
|
|
5
5
|
import { storage, utils } from '@powersync/service-core';
|
|
6
6
|
import { PowerSyncMongo } from './db.js';
|
|
7
|
+
import { ServiceAssertionError } from '@powersync/lib-services-framework';
|
|
7
8
|
export function idPrefixFilter(prefix, rest) {
|
|
8
9
|
let filter = {
|
|
9
10
|
$gte: {
|
|
@@ -108,4 +109,16 @@ export const connectMongoForTests = (url, isCI) => {
|
|
|
108
109
|
});
|
|
109
110
|
return new PowerSyncMongo(client);
|
|
110
111
|
};
|
|
112
|
+
export function setSessionSnapshotTime(session, time) {
|
|
113
|
+
// This is a workaround for the lack of direct support for snapshot reads in the MongoDB driver.
|
|
114
|
+
if (!session.snapshotEnabled) {
|
|
115
|
+
throw new ServiceAssertionError(`Session must be a snapshot session`);
|
|
116
|
+
}
|
|
117
|
+
if (session.snapshotTime == null) {
|
|
118
|
+
session.snapshotTime = time;
|
|
119
|
+
}
|
|
120
|
+
else {
|
|
121
|
+
throw new ServiceAssertionError(`Session snapshotTime is already set`);
|
|
122
|
+
}
|
|
123
|
+
}
|
|
111
124
|
//# sourceMappingURL=util.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"util.js","sourceRoot":"","sources":["../../../src/storage/implementation/util.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAC7B,OAAO,KAAK,MAAM,MAAM,QAAQ,CAAC;AACjC,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAE7B,OAAO,EAAE,KAAK,EAAE,MAAM,gCAAgC,CAAC;AACvD,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,yBAAyB,CAAC;AAEzD,OAAO,EAAE,cAAc,EAAE,MAAM,SAAS,CAAC;
|
|
1
|
+
{"version":3,"file":"util.js","sourceRoot":"","sources":["../../../src/storage/implementation/util.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAC7B,OAAO,KAAK,MAAM,MAAM,QAAQ,CAAC;AACjC,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAE7B,OAAO,EAAE,KAAK,EAAE,MAAM,gCAAgC,CAAC;AACvD,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,yBAAyB,CAAC;AAEzD,OAAO,EAAE,cAAc,EAAE,MAAM,SAAS,CAAC;AAEzC,OAAO,EAAE,qBAAqB,EAAE,MAAM,mCAAmC,CAAC;AAE1E,MAAM,UAAU,cAAc,CAAI,MAAkB,EAAE,IAAiB;IACrE,IAAI,MAAM,GAAG;QACX,IAAI,EAAE;YACJ,GAAG,MAAM;SACH;QACR,GAAG,EAAE;YACH,GAAG,MAAM;SACH;KACT,CAAC;IAEF,KAAK,IAAI,GAAG,IAAI,IAAI,EAAE,CAAC;QACrB,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;QACrC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;IACtC,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,MAAc,EAAE,aAAqB;IACpE,MAAM,WAAW,GAAG,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;IAC1D,OAAO,GAAG,MAAM,GAAG,aAAa,IAAI,WAAW,EAAE,CAAC;AACpD,CAAC;AAED;;;;;;;;;;GAUG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CAAI,MAA+B;IACtE,IAAI,CAAC;QACH,IAAI,IAAS,CAAC;QACd,IAAI,OAAO,GAAG,IAAI,CAAC;QACnB,2CAA2C;QAC3C,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,OAAO,EAAE,CAAC;QACtC,yCAAyC;QACzC,IAAI,GAAG,MAAM,CAAC,qBAAqB,EAAE,CAAC;QACtC,IAAI,CAAC,MAAM,IAAI,MAAM,CAAC,EAAE,EAAE,MAAM,EAAE,EAAE,CAAC;YACnC,0CAA0C;YAC1C,wEAAwE;YACxE,uEAAuE;YACvE,oCAAoC;YACpC,EAAE;YACF,4EAA4E;YAC5E,2DAA2D;YAC3D,gCAAgC;YAChC,OAAO,GAAG,KAAK,CAAC;QAClB,CAAC;QACD,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC;IAC3B,CAAC;YAAS,CAAC;QACT,iDAAiD;QACjD,uIAAuI;QACvI,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;YACnB,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC;QACvB,CAAC;IACH,CAAC;AACH,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,GAAuB;IAChD,IAAI,GAAG,CAAC,EAAE,IAAI,KAAK,IAAI,GAAG,CAAC,EAAE,IAAI,QAAQ,EAAE,CAAC;QAC1C,OAAO;YACL,KAAK,EAAE,KAAK,CAAC,sBAAsB,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;YAC9C,EAAE,EAAE,GAAG,CAAC,EAAE;YACV,WAAW,EAAE,GAAG,CAAC,KAAK;YACtB,SAAS,EAAE,GAAG,CAAC,MAAM;YACrB,QAAQ,EAAE,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;YAC9B,MAAM,EAAE,iBAAiB,CAAC,GAAG,CAAC,YAAa,EAAE,GAAG,CAAC,UAAW,CAAC;YAC7D,IAAI,EAAE,GAAG,CAAC,IAAI;SACf,CAAC;IACJ,CAAC;SAAM,CAAC;QACN,cAAc;QAEd,OAAO;YACL,KAAK,EAAE,KAAK,CAAC,sBAAsB,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;YAC9C,EAAE,EAAE,GAAG,CAAC,EAAE;YACV,QAAQ,EAAE,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;SAC/B,CAAC;IACJ,CAAC;AACH,CAAC;AAED,MAAM,UAAU,iBAAiB,CAAC,KAAoB,EAAE,EAAqB;IAC3E,IAAI,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,CAAC;QACvB,mDAAmD;QACnD,OAAO,GAAG,KAAK,CAAC,WAAW,EAAE,IAAI,EAAE,CAAC,WAAW,EAAE,EAAE,CAAC;IACtD,CAAC;SAAM,CAAC;QACN,oCAAoC;QACpC,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC,CAAC;QAC3C,OAAO,IAAI,CAAC,EAAE,CAAC,IAAI,EAAE,KAAK,CAAC,YAAY,CAAC,CAAC;IAC3C,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,MAAM,oBAAoB,GAAG,CAAC,GAAW,EAAE,IAAa,EAAE,EAAE;IACjE,0EAA0E;IAC1E,oEAAoE;IACpE,MAAM,MAAM,GAAG,IAAI,KAAK,CAAC,WAAW,CAAC,GAAG,EAAE;QACxC,gBAAgB,EAAE,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK;QACvC,eAAe,EAAE,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK;QACtC,wBAAwB,EAAE,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK;KAChD,CAAC,CAAC;IACH,OAAO,IAAI,cAAc,CAAC,MAAM,CAAC,CAAC;AACpC,CAAC,CAAC;AAEF,MAAM,UAAU,sBAAsB,CAAC,OAA4B,EAAE,IAAoB;IACvF,gGAAgG;IAChG,IAAI,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;QAC7B,MAAM,IAAI,qBAAqB,CAAC,oCAAoC,CAAC,CAAC;IACxE,CAAC;IACD,IAAK,OAAe,CAAC,YAAY,IAAI,IAAI,EAAE,CAAC;QACzC,OAAe,CAAC,YAAY,GAAG,IAAI,CAAC;IACvC,CAAC;SAAM,CAAC;QACN,MAAM,IAAI,qBAAqB,CAAC,qCAAqC,CAAC,CAAC;IACzE,CAAC;AACH,CAAC"}
|
package/package.json
CHANGED
|
@@ -2,9 +2,9 @@
|
|
|
2
2
|
"name": "@powersync/service-module-mongodb-storage",
|
|
3
3
|
"repository": "https://github.com/powersync-ja/powersync-service",
|
|
4
4
|
"types": "dist/index.d.ts",
|
|
5
|
-
"version": "0.
|
|
5
|
+
"version": "0.12.0",
|
|
6
6
|
"main": "dist/index.js",
|
|
7
|
-
"license": "FSL-1.1-
|
|
7
|
+
"license": "FSL-1.1-ALv2",
|
|
8
8
|
"type": "module",
|
|
9
9
|
"publishConfig": {
|
|
10
10
|
"access": "public"
|
|
@@ -27,15 +27,15 @@
|
|
|
27
27
|
"lru-cache": "^10.2.2",
|
|
28
28
|
"ts-codec": "^1.3.0",
|
|
29
29
|
"uuid": "^11.1.0",
|
|
30
|
-
"@powersync/lib-service-mongodb": "0.6.
|
|
31
|
-
"@powersync/lib-services-framework": "0.7.
|
|
32
|
-
"@powersync/service-core": "1.
|
|
33
|
-
"@powersync/service-jsonbig": "0.17.
|
|
34
|
-
"@powersync/service-sync-rules": "0.
|
|
35
|
-
"@powersync/service-types": "0.
|
|
30
|
+
"@powersync/lib-service-mongodb": "0.6.4",
|
|
31
|
+
"@powersync/lib-services-framework": "0.7.3",
|
|
32
|
+
"@powersync/service-core": "1.15.0",
|
|
33
|
+
"@powersync/service-jsonbig": "0.17.11",
|
|
34
|
+
"@powersync/service-sync-rules": "0.29.0",
|
|
35
|
+
"@powersync/service-types": "0.13.0"
|
|
36
36
|
},
|
|
37
37
|
"devDependencies": {
|
|
38
|
-
"@powersync/service-core-tests": "0.
|
|
38
|
+
"@powersync/service-core-tests": "0.12.0"
|
|
39
39
|
},
|
|
40
40
|
"scripts": {
|
|
41
41
|
"build": "tsc -b",
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { mongo } from '@powersync/lib-service-mongodb';
|
|
2
|
-
import { SqlEventDescriptor, SqliteRow, SqlSyncRules } from '@powersync/service-sync-rules';
|
|
2
|
+
import { SqlEventDescriptor, SqliteRow, SqliteValue, SqlSyncRules } from '@powersync/service-sync-rules';
|
|
3
3
|
import * as bson from 'bson';
|
|
4
4
|
|
|
5
5
|
import {
|
|
@@ -16,6 +16,7 @@ import {
|
|
|
16
16
|
BucketStorageMarkRecordUnavailable,
|
|
17
17
|
deserializeBson,
|
|
18
18
|
InternalOpId,
|
|
19
|
+
isCompleteRow,
|
|
19
20
|
SaveOperationTag,
|
|
20
21
|
storage,
|
|
21
22
|
utils
|
|
@@ -49,6 +50,7 @@ export interface MongoBucketBatchOptions {
|
|
|
49
50
|
lastCheckpointLsn: string | null;
|
|
50
51
|
keepaliveOp: InternalOpId | null;
|
|
51
52
|
noCheckpointBeforeLsn: string;
|
|
53
|
+
resumeFromLsn: string | null;
|
|
52
54
|
storeCurrentData: boolean;
|
|
53
55
|
/**
|
|
54
56
|
* Set to true for initial replication.
|
|
@@ -95,10 +97,24 @@ export class MongoBucketBatch
|
|
|
95
97
|
private persisted_op: InternalOpId | null = null;
|
|
96
98
|
|
|
97
99
|
/**
|
|
98
|
-
*
|
|
100
|
+
* Last written op, if any. This may not reflect a consistent checkpoint.
|
|
99
101
|
*/
|
|
100
102
|
public last_flushed_op: InternalOpId | null = null;
|
|
101
103
|
|
|
104
|
+
/**
|
|
105
|
+
* lastCheckpointLsn is the last consistent commit.
|
|
106
|
+
*
|
|
107
|
+
* While that is generally a "safe" point to resume from, there are cases where we may want to resume from a different point:
|
|
108
|
+
* 1. After an initial snapshot, we don't have a consistent commit yet, but need to resume from the snapshot LSN.
|
|
109
|
+
* 2. If "no_checkpoint_before_lsn" is set far in advance, it may take a while to reach that point. We
|
|
110
|
+
* may want to resume at incremental points before that.
|
|
111
|
+
*
|
|
112
|
+
* This is set when creating the batch, but may not be updated afterwards.
|
|
113
|
+
*/
|
|
114
|
+
public resumeFromLsn: string | null = null;
|
|
115
|
+
|
|
116
|
+
private needsActivation = true;
|
|
117
|
+
|
|
102
118
|
constructor(options: MongoBucketBatchOptions) {
|
|
103
119
|
super();
|
|
104
120
|
this.logger = options.logger ?? defaultLogger;
|
|
@@ -107,6 +123,7 @@ export class MongoBucketBatch
|
|
|
107
123
|
this.group_id = options.groupId;
|
|
108
124
|
this.last_checkpoint_lsn = options.lastCheckpointLsn;
|
|
109
125
|
this.no_checkpoint_before_lsn = options.noCheckpointBeforeLsn;
|
|
126
|
+
this.resumeFromLsn = options.resumeFromLsn;
|
|
110
127
|
this.session = this.client.startSession();
|
|
111
128
|
this.slot_name = options.slotName;
|
|
112
129
|
this.sync_rules = options.syncRules;
|
|
@@ -302,7 +319,8 @@ export class MongoBucketBatch
|
|
|
302
319
|
const record = operation.record;
|
|
303
320
|
const beforeId = operation.beforeId;
|
|
304
321
|
const afterId = operation.afterId;
|
|
305
|
-
let
|
|
322
|
+
let sourceAfter = record.after;
|
|
323
|
+
let after = sourceAfter && this.sync_rules.applyRowContext(sourceAfter);
|
|
306
324
|
const sourceTable = record.sourceTable;
|
|
307
325
|
|
|
308
326
|
let existing_buckets: CurrentBucket[] = [];
|
|
@@ -332,7 +350,7 @@ export class MongoBucketBatch
|
|
|
332
350
|
// Not an error if we re-apply a transaction
|
|
333
351
|
existing_buckets = [];
|
|
334
352
|
existing_lookups = [];
|
|
335
|
-
if (this.storeCurrentData) {
|
|
353
|
+
if (!isCompleteRow(this.storeCurrentData, after!)) {
|
|
336
354
|
if (this.markRecordUnavailable != null) {
|
|
337
355
|
// This will trigger a "resnapshot" of the record.
|
|
338
356
|
// This is not relevant if storeCurrentData is false, since we'll get the full row
|
|
@@ -350,7 +368,7 @@ export class MongoBucketBatch
|
|
|
350
368
|
existing_lookups = result.lookups;
|
|
351
369
|
if (this.storeCurrentData) {
|
|
352
370
|
const data = deserializeBson((result.data as mongo.Binary).buffer) as SqliteRow;
|
|
353
|
-
after = storage.mergeToast(after!, data);
|
|
371
|
+
after = storage.mergeToast<SqliteValue>(after!, data);
|
|
354
372
|
}
|
|
355
373
|
}
|
|
356
374
|
} else if (record.tag == SaveOperationTag.DELETE) {
|
|
@@ -444,7 +462,8 @@ export class MongoBucketBatch
|
|
|
444
462
|
if (sourceTable.syncData) {
|
|
445
463
|
const { results: evaluated, errors: syncErrors } = this.sync_rules.evaluateRowWithErrors({
|
|
446
464
|
record: after,
|
|
447
|
-
sourceTable
|
|
465
|
+
sourceTable,
|
|
466
|
+
bucketIdTransformer: SqlSyncRules.versionedBucketIdTransformer(`${this.group_id}`)
|
|
448
467
|
});
|
|
449
468
|
|
|
450
469
|
for (let error of syncErrors) {
|
|
@@ -685,6 +704,7 @@ export class MongoBucketBatch
|
|
|
685
704
|
|
|
686
705
|
if (!createEmptyCheckpoints && this.persisted_op == null) {
|
|
687
706
|
// Nothing to commit - also return true
|
|
707
|
+
await this.autoActivate(lsn);
|
|
688
708
|
return true;
|
|
689
709
|
}
|
|
690
710
|
|
|
@@ -729,14 +749,67 @@ export class MongoBucketBatch
|
|
|
729
749
|
},
|
|
730
750
|
{ session: this.session }
|
|
731
751
|
);
|
|
752
|
+
await this.autoActivate(lsn);
|
|
732
753
|
await this.db.notifyCheckpoint();
|
|
733
754
|
this.persisted_op = null;
|
|
734
755
|
this.last_checkpoint_lsn = lsn;
|
|
735
756
|
return true;
|
|
736
757
|
}
|
|
737
758
|
|
|
759
|
+
/**
|
|
760
|
+
* Switch from processing -> active if relevant.
|
|
761
|
+
*
|
|
762
|
+
* Called on new commits.
|
|
763
|
+
*/
|
|
764
|
+
private async autoActivate(lsn: string) {
|
|
765
|
+
if (!this.needsActivation) {
|
|
766
|
+
return;
|
|
767
|
+
}
|
|
768
|
+
|
|
769
|
+
// Activate the batch, so it can start processing.
|
|
770
|
+
// This is done automatically when the first save() is called.
|
|
771
|
+
|
|
772
|
+
const session = this.session;
|
|
773
|
+
let activated = false;
|
|
774
|
+
await session.withTransaction(async () => {
|
|
775
|
+
const doc = await this.db.sync_rules.findOne({ _id: this.group_id }, { session });
|
|
776
|
+
if (doc && doc.state == 'PROCESSING') {
|
|
777
|
+
await this.db.sync_rules.updateOne(
|
|
778
|
+
{
|
|
779
|
+
_id: this.group_id
|
|
780
|
+
},
|
|
781
|
+
{
|
|
782
|
+
$set: {
|
|
783
|
+
state: storage.SyncRuleState.ACTIVE
|
|
784
|
+
}
|
|
785
|
+
},
|
|
786
|
+
{ session }
|
|
787
|
+
);
|
|
788
|
+
|
|
789
|
+
await this.db.sync_rules.updateMany(
|
|
790
|
+
{
|
|
791
|
+
_id: { $ne: this.group_id },
|
|
792
|
+
state: { $in: [storage.SyncRuleState.ACTIVE, storage.SyncRuleState.ERRORED] }
|
|
793
|
+
},
|
|
794
|
+
{
|
|
795
|
+
$set: {
|
|
796
|
+
state: storage.SyncRuleState.STOP
|
|
797
|
+
}
|
|
798
|
+
},
|
|
799
|
+
{ session }
|
|
800
|
+
);
|
|
801
|
+
activated = true;
|
|
802
|
+
}
|
|
803
|
+
});
|
|
804
|
+
if (activated) {
|
|
805
|
+
this.logger.info(`Activated new sync rules at ${lsn}`);
|
|
806
|
+
await this.db.notifyCheckpoint();
|
|
807
|
+
}
|
|
808
|
+
this.needsActivation = false;
|
|
809
|
+
}
|
|
810
|
+
|
|
738
811
|
async keepalive(lsn: string): Promise<boolean> {
|
|
739
|
-
if (this.last_checkpoint_lsn != null && lsn
|
|
812
|
+
if (this.last_checkpoint_lsn != null && lsn < this.last_checkpoint_lsn) {
|
|
740
813
|
// No-op
|
|
741
814
|
return false;
|
|
742
815
|
}
|
|
@@ -782,13 +855,14 @@ export class MongoBucketBatch
|
|
|
782
855
|
},
|
|
783
856
|
{ session: this.session }
|
|
784
857
|
);
|
|
858
|
+
await this.autoActivate(lsn);
|
|
785
859
|
await this.db.notifyCheckpoint();
|
|
786
860
|
this.last_checkpoint_lsn = lsn;
|
|
787
861
|
|
|
788
862
|
return true;
|
|
789
863
|
}
|
|
790
864
|
|
|
791
|
-
async
|
|
865
|
+
async setResumeLsn(lsn: string): Promise<void> {
|
|
792
866
|
const update: Partial<SyncRuleDocument> = {
|
|
793
867
|
snapshot_lsn: lsn
|
|
794
868
|
};
|