@powersync/service-core 0.2.0 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +8 -0
- package/dist/storage/BucketStorage.d.ts +6 -1
- package/dist/storage/BucketStorage.js.map +1 -1
- package/dist/storage/ChecksumCache.d.ts +50 -0
- package/dist/storage/ChecksumCache.js +234 -0
- package/dist/storage/ChecksumCache.js.map +1 -0
- package/dist/storage/mongo/MongoSyncBucketStorage.d.ts +3 -1
- package/dist/storage/mongo/MongoSyncBucketStorage.js +26 -14
- package/dist/storage/mongo/MongoSyncBucketStorage.js.map +1 -1
- package/dist/sync/sync.js +32 -21
- package/dist/sync/sync.js.map +1 -1
- package/dist/util/utils.d.ts +6 -3
- package/dist/util/utils.js +32 -15
- package/dist/util/utils.js.map +1 -1
- package/package.json +6 -5
- package/src/storage/BucketStorage.ts +6 -1
- package/src/storage/ChecksumCache.ts +294 -0
- package/src/storage/mongo/MongoSyncBucketStorage.ts +31 -15
- package/src/sync/sync.ts +44 -37
- package/src/util/utils.ts +36 -16
- package/test/src/__snapshots__/sync.test.ts.snap +14 -2
- package/test/src/checksum_cache.test.ts +436 -0
- package/test/src/data_storage.test.ts +3 -3
- package/test/src/large_batch.test.ts +4 -4
- package/tsconfig.tsbuildinfo +1 -1
package/dist/sync/sync.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"sync.js","sourceRoot":"","sources":["../../src/sync/sync.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,KAAK,MAAM,6BAA6B,CAAC;AACrD,OAAO,EAAE,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAC;AAEpE,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AACxC,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAI9C,OAAO,KAAK,IAAI,MAAM,sBAAsB,CAAC;AAE7C,OAAO,EAAE,mBAAmB,EAAE,MAAM,YAAY,CAAC;AACjD,OAAO,EAAsB,WAAW,EAAE,MAAM,WAAW,CAAC;AAC5D,OAAO,EAAE,OAAO,EAAE,MAAM,sBAAsB,CAAC;AAE/C;;GAEG;AACH,MAAM,sBAAsB,GAAG,EAAE,CAAC;AAClC,MAAM,aAAa,GAAG,IAAI,SAAS,CAAC,sBAAsB,CAAC,CAAC;AAc5D,MAAM,CAAC,KAAK,SAAS,CAAC,CAAC,cAAc,CACnC,OAA6B;IAE7B,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,KAAK,EAAE,kBAAkB,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC;IACnF,qEAAqE;IACrE,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;IACzC,IAAI,MAAM,EAAE,CAAC;QACX,MAAM,CAAC,gBAAgB,CACrB,OAAO,EACP,GAAG,EAAE;YACH,UAAU,CAAC,KAAK,EAAE,CAAC;QACrB,CAAC,EACD,EAAE,IAAI,EAAE,IAAI,EAAE,CACf,CAAC;QACF,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;YACnB,UAAU,CAAC,KAAK,EAAE,CAAC;QACrB,CAAC;IACH,CAAC;IACD,MAAM,EAAE,GAAG,WAAW,CAAC,KAAK,EAAE,UAAU,CAAC,MAAM,EAAE,kBAAkB,CAAC,CAAC;IACrE,MAAM,MAAM,GAAG,mBAAmB,CAAC,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,UAAU,CAAC,MAAM,CAAC,CAAC;IACnF,sEAAsE;IACtE,MAAM,MAAM,GAAG,mBAAmB,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,EAAE,UAAU,CAAC,MAAM,CAAC,CAAC;IAEpE,IAAI,CAAC;QACH,KAAK,CAAC,CAAC,MAAM,CAAC;IAChB,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,IAAI,CAAC,YAAY,UAAU,EAAE,CAAC;YAC5B,OAAO;QACT,CAAC;aAAM,CAAC;YACN,MAAM,CAAC,CAAC;QACV,CAAC;IACH,CAAC;YAAS,CAAC;QACT,iFAAiF;QACjF,qBAAqB;QACrB,UAAU,CAAC,KAAK,EAAE,CAAC;IACrB,CAAC;AACH,CAAC;AAED,KAAK,SAAS,CAAC,CAAC,mBAAmB,CACjC,OAAqC,EACrC,MAAiC,EACjC,UAA0B,EAC1B,MAAmB;IAEnB,sCAAsC;IACtC,iHAAiH;IACjH,IAAI,WAAW,GAAG,IAAI,GAAG,EAAkB,CAAC;IAE5C,IAAI,
|
|
1
|
+
{"version":3,"file":"sync.js","sourceRoot":"","sources":["../../src/sync/sync.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,KAAK,MAAM,6BAA6B,CAAC;AACrD,OAAO,EAAE,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAC;AAEpE,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AACxC,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAI9C,OAAO,KAAK,IAAI,MAAM,sBAAsB,CAAC;AAE7C,OAAO,EAAE,mBAAmB,EAAE,MAAM,YAAY,CAAC;AACjD,OAAO,EAAsB,WAAW,EAAE,MAAM,WAAW,CAAC;AAC5D,OAAO,EAAE,OAAO,EAAE,MAAM,sBAAsB,CAAC;AAE/C;;GAEG;AACH,MAAM,sBAAsB,GAAG,EAAE,CAAC;AAClC,MAAM,aAAa,GAAG,IAAI,SAAS,CAAC,sBAAsB,CAAC,CAAC;AAc5D,MAAM,CAAC,KAAK,SAAS,CAAC,CAAC,cAAc,CACnC,OAA6B;IAE7B,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,KAAK,EAAE,kBAAkB,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC;IACnF,qEAAqE;IACrE,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;IACzC,IAAI,MAAM,EAAE,CAAC;QACX,MAAM,CAAC,gBAAgB,CACrB,OAAO,EACP,GAAG,EAAE;YACH,UAAU,CAAC,KAAK,EAAE,CAAC;QACrB,CAAC,EACD,EAAE,IAAI,EAAE,IAAI,EAAE,CACf,CAAC;QACF,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;YACnB,UAAU,CAAC,KAAK,EAAE,CAAC;QACrB,CAAC;IACH,CAAC;IACD,MAAM,EAAE,GAAG,WAAW,CAAC,KAAK,EAAE,UAAU,CAAC,MAAM,EAAE,kBAAkB,CAAC,CAAC;IACrE,MAAM,MAAM,GAAG,mBAAmB,CAAC,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,UAAU,CAAC,MAAM,CAAC,CAAC;IACnF,sEAAsE;IACtE,MAAM,MAAM,GAAG,mBAAmB,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,EAAE,UAAU,CAAC,MAAM,CAAC,CAAC;IAEpE,IAAI,CAAC;QACH,KAAK,CAAC,CAAC,MAAM,CAAC;IAChB,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,IAAI,CAAC,YAAY,UAAU,EAAE,CAAC;YAC5B,OAAO;QACT,CAAC;aAAM,CAAC;YACN,MAAM,CAAC,CAAC;QACV,CAAC;IACH,CAAC;YAAS,CAAC;QACT,iFAAiF;QACjF,qBAAqB;QACrB,UAAU,CAAC,KAAK,EAAE,CAAC;IACrB,CAAC;AACH,CAAC;AAED,KAAK,SAAS,CAAC,CAAC,mBAAmB,CACjC,OAAqC,EACrC,MAAiC,EACjC,UAA0B,EAC1B,MAAmB;IAEnB,sCAAsC;IACtC,iHAAiH;IACjH,IAAI,WAAW,GAAG,IAAI,GAAG,EAAkB,CAAC;IAE5C,IAAI,aAAa,GAA4B,IAAI,CAAC;IAClD,IAAI,mBAAmB,GAAkB,IAAI,CAAC;IAE9C,MAAM,EAAE,QAAQ,EAAE,WAAW,EAAE,GAAG,MAAM,CAAC;IAEzC,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;QACnB,KAAK,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;YAClD,WAAW,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;QAC/B,CAAC;IACH,CAAC;IAED,MAAM,MAAM,GAAG,OAAO,CAAC,oBAAoB,CAAC,UAAU,CAAC,gBAAgB,CAAC,OAAiB,EAAE,MAAM,CAAC,CAAC;IACnG,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,MAAM,EAAE,CAAC;QAChC,MAAM,EAAE,IAAI,EAAE,eAAe,EAAE,GAAG,IAAI,CAAC;QACvC,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC;QAEnC,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,gBAAgB,EAAE,CAAC;QAC9C,IAAI,OAAO,IAAI,IAAI,EAAE,CAAC;YACpB,2EAA2E;YAC3E,SAAS;QACX,CAAC;QACD,MAAM,UAAU,GAAG,OAAO,CAAC,UAAU,CAAC;QAEtC,MAAM,UAAU,GAAG,MAAM,UAAU,CAAC,cAAc,CAAC;YACjD,gBAAgB,CAAC,OAAO;gBACtB,OAAO,OAAO,CAAC,gBAAgB,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;YACvD,CAAC;YACD,UAAU,EAAE,UAAU;SACvB,CAAC,CAAC;QAEH,IAAI,UAAU,CAAC,MAAM,GAAG,IAAI,EAAE,CAAC;YAC7B,iEAAiE;YACjE,MAAM,IAAI,KAAK,CAAC,qBAAqB,UAAU,CAAC,MAAM,EAAE,CAAC,CAAC;QAC5D,CAAC;QAED,IAAI,cAAc,GAAG,IAAI,GAAG,EAAkB,CAAC;QAC/C,KAAK,IAAI,MAAM,IAAI,UAAU,EAAE,CAAC;YAC9B,cAAc,CAAC,GAAG,CAAC,MAAM,EAAE,WAAW,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,GAAG,CAAC,CAAC;QAC7D,CAAC;QACD,WAAW,GAAG,cAAc,CAAC;QAE7B,MAAM,UAAU,GAAG,CAAC,GAAG,WAAW,CAAC,IAAI,EAAE,CAAC,CAAC;QAC3C,MAAM,WAAW,GAAG,MAAM,OAAO,CAAC,YAAY,CAAC,UAAU,EAAE,UAAU,CAAC,CAAC;QACvE,mEAAmE;QACnE,IAAI,cAAwB,CAAC;QAE7B,IAAI,aAAa,EAAE,CAAC;YAClB,MAAM,IAAI,GAAG,IAAI,CAAC,aAAa,CAAC,aAAa,EAAE,WAAW,CAAC,CAAC;YAE5D,IACE,mBAAmB,IAAI,eAAe;gBACtC,IAAI,CAAC,cAAc,CAAC,MAAM,IAAI,CAAC;gBAC/B,IAAI,CAAC,cAAc,CAAC,MAAM,IAAI,CAAC,EAC/B,CAAC;gBACD,iDAAiD;gBACjD,SAAS;YACX,CAAC;YACD,cAAc,GAAG,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;YAE1D,IAAI,OAAO,GAAG,uBAAuB,UAAU,aAAa,eAAe,KAAK,CAAC;YACjF,OAAO,IAAI,YAAY,UAAU,CAAC,MAAM,KAAK,CAAC;YAC9C,OAAO,IAAI,YAAY,cAAc,CAAC,IAAI,CAAC,cAAc,EAAE,EAAE,CAAC,KAAK,CAAC;YACpE,OAAO,IAAI,YAAY,cAAc,CAAC,IAAI,CAAC,cAAc,EAAE,EAAE,CAAC,KAAK,CAAC;YACpE,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YAE3B,MAAM,aAAa,GAAqC;gBACtD,eAAe,EAAE;oBACf,UAAU,EAAE,UAAU;oBACtB,gBAAgB,EAAE,eAAe,CAAC,CAAC,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC,SAAS;oBACvE,eAAe,EAAE,IAAI,CAAC,cAAc;oBACpC,eAAe,EAAE,IAAI,CAAC,cAAc;iBACrC;aACF,CAAC;YAEF,MAAM,aAAa,CAAC;QACtB,CAAC;aAAM,CAAC;YACN,IAAI,OAAO,GAAG,mBAAmB,UAAU,aAAa,eAAe,KAAK,CAAC;YAC7E,OAAO,IAAI,YAAY,UAAU,CAAC,MAAM,IAAI,cAAc,CAAC,UAAU,EAAE,EAAE,CAAC,EAAE,CAAC;YAC7E,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YAC3B,cAAc,GAAG,UAAU,CAAC;YAC5B,MAAM,aAAa,GAAiC;gBAClD,UAAU,EAAE;oBACV,UAAU,EAAE,UAAU;oBACtB,gBAAgB,EAAE,eAAe,CAAC,CAAC,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC,SAAS;oBACvE,OAAO,EAAE,CAAC,GAAG,WAAW,CAAC,MAAM,EAAE,CAAC;iBACnC;aACF,CAAC;YACF,MAAM,aAAa,CAAC;QACtB,CAAC;QACD,aAAa,GAAG,WAAW,CAAC;QAC5B,mBAAmB,GAAG,eAAe,CAAC;QAEtC,+EAA+E;QAC/E,sFAAsF;QACtF,KAAK,CAAC,CAAC,mBAAmB,CAAC,EAAE,OAAO,EAAE,UAAU,EAAE,cAAc,EAAE,WAAW,EAAE,QAAQ,EAAE,WAAW,EAAE,MAAM,EAAE,CAAC,CAAC;QAEhH,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;IAC1D,CAAC;AACH,CAAC;AAaD,KAAK,SAAS,CAAC,CAAC,mBAAmB,CAAC,OAA0B;IAC5D,IAAI,MAAM,GAAG,KAAK,CAAC;IACnB,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,IAAI,CAAC,MAAM,EAAE,CAAC;QAC1C,wEAAwE;QACxE,6EAA6E;QAC7E,4GAA4G;QAC5G,gBAAgB;QAChB,gBAAgB;QAChB,qBAAqB;QACrB,MAAM;QACN,WAAW;QACX,IAAI;QACJ,MAAM,IAAI,GAAG,eAAe,CAAC,OAAO,CAAC,CAAC;QACtC,IAAI,CAAC;YACH,OAAO,IAAI,EAAE,CAAC;gBACZ,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;gBACpD,IAAI,QAAQ,EAAE,CAAC;oBACb,MAAM;gBACR,CAAC;qBAAM,CAAC;oBACN,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,KAAK,CAAC;oBAC7B,MAAM,IAAI,CAAC;oBACX,IAAI,IAAI,EAAE,CAAC;wBACT,MAAM,GAAG,IAAI,CAAC;oBAChB,CAAC;gBACH,CAAC;YACH,CAAC;QACH,CAAC;gBAAS,CAAC;YACT,MAAM,IAAI,CAAC,MAAM,EAAE,CAAC;QACtB,CAAC;IACH,CAAC;AACH,CAAC;AAED;;GAEG;AACH,KAAK,SAAS,CAAC,CAAC,eAAe,CAAC,OAA0B;IACxD,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE,cAAc,EAAE,WAAW,EAAE,QAAQ,EAAE,WAAW,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC;IAEpG,MAAM,CAAC,CAAC,EAAE,OAAO,CAAC,GAAG,MAAM,aAAa,CAAC,OAAO,EAAE,CAAC;IACnD,IAAI,CAAC;QACH,kGAAkG;QAClG,iDAAiD;QACjD,MAAM,eAAe,GAAG,IAAI,GAAG,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,WAAW,CAAC,GAAG,CAAC,MAAM,CAAE,CAAC,CAAC,CAAC,CAAC;QACpG,MAAM,IAAI,GAAG,OAAO,CAAC,kBAAkB,CAAC,UAAU,EAAE,eAAe,CAAC,CAAC;QAErE,IAAI,QAAQ,GAAG,KAAK,CAAC;QAErB,IAAI,KAAK,EAAE,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC;YACzB,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;gBACnB,OAAO;YACT,CAAC;YACD,IAAI,CAAC,CAAC,QAAQ,EAAE,CAAC;gBACf,QAAQ,GAAG,IAAI,CAAC;YAClB,CAAC;YACD,IAAI,CAAC,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;gBACvB,SAAS;YACX,CAAC;YACD,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,oBAAoB,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC;YAEnD,IAAI,SAAc,CAAC;YACnB,IAAI,WAAW,EAAE,CAAC;gBAChB,wEAAwE;gBACxE,SAAS,GAAG,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC;YAC1B,CAAC;iBAAM,IAAI,QAAQ,EAAE,CAAC;gBACpB,uEAAuE;gBACvE,MAAM,QAAQ,GAA2B;oBACvC,IAAI,EAAE,CAAC;iBACR,CAAC;gBACF,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;YACvC,CAAC;iBAAM,CAAC;gBACN,8EAA8E;gBAC9E,4BAA4B;gBAC5B,MAAM,QAAQ,GAA2B;oBACvC,IAAI,EAAE,uBAAuB,CAAC,CAAC,CAAC;iBACjC,CAAC;gBACF,SAAS,GAAG,OAAO,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;YAC1C,CAAC;YACD,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC;YACvC,IAAI,SAAS,CAAC,MAAM,GAAG,KAAM,EAAE,CAAC;gBAC9B,0EAA0E;gBAC1E,8CAA8C;gBAC9C,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC;YACpC,CAAC;YACD,OAAO,CAAC,WAAW,EAAE,CAAC,uBAAuB,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAEjE,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,UAAU,CAAC,CAAC;QAC1C,CAAC;QAED,IAAI,CAAC,QAAQ,EAAE,CAAC;YACd,MAAM,IAAI,GAAyC;gBACjD,mBAAmB,EAAE;oBACnB,UAAU,EAAE,UAAU;iBACvB;aACF,CAAC;YACF,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;QACnC,CAAC;IACH,CAAC;YAAS,CAAC;QACT,OAAO,EAAE,CAAC;IACZ,CAAC;AACH,CAAC;AAED,SAAS,uBAAuB,CAAC,UAA+B;IAC9D,OAAO;QACL,GAAG,UAAU;QACb,IAAI,EAAE,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE;YAClC,OAAO;gBACL,GAAG,KAAK;gBACR,IAAI,EAAE,KAAK,CAAC,IAAI,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,aAAa,CAAC,KAAK,CAAC,IAAc,CAAC;gBACzE,QAAQ,EAAE,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC;aACjC,CAAC;QACJ,CAAC,CAAC;KACH,CAAC;AACJ,CAAC;AAED,SAAS,cAAc,CAAC,OAAyC,EAAE,KAAa;IAC9E,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;QAC1B,IAAI,OAAO,CAAC,IAAI,QAAQ,EAAE,CAAC;YACzB,OAAO,CAAC,CAAC,MAAM,CAAC;QAClB,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,CAAC;QACX,CAAC;IACH,CAAC,CAAC,CAAC;IACH,IAAI,OAAO,CAAC,MAAM,IAAI,KAAK,EAAE,CAAC;QAC5B,OAAO,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;IACjC,CAAC;IACD,MAAM,OAAO,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;IACxC,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,KAAK,CAAC;AACzC,CAAC"}
|
package/dist/util/utils.d.ts
CHANGED
|
@@ -1,13 +1,16 @@
|
|
|
1
1
|
import * as pgwire from '@powersync/service-jpgwire';
|
|
2
2
|
import * as storage from '../storage/storage-index.js';
|
|
3
3
|
import { BucketChecksum, OpId } from './protocol-types.js';
|
|
4
|
+
export type ChecksumMap = Map<string, BucketChecksum>;
|
|
4
5
|
export declare function hashData(type: string, id: string, data: string): number;
|
|
5
6
|
export declare function hashDelete(sourceKey: string): number;
|
|
6
7
|
export declare function timestampToOpId(ts: bigint): OpId;
|
|
7
|
-
export declare function checksumsDiff(previous:
|
|
8
|
-
|
|
9
|
-
|
|
8
|
+
export declare function checksumsDiff(previous: ChecksumMap, current: ChecksumMap): {
|
|
9
|
+
updatedBuckets: BucketChecksum[];
|
|
10
|
+
removedBuckets: string[];
|
|
10
11
|
};
|
|
12
|
+
export declare function addChecksums(a: number, b: number): number;
|
|
13
|
+
export declare function addBucketChecksums(a: BucketChecksum, b: BucketChecksum | null): BucketChecksum;
|
|
11
14
|
export declare function getClientCheckpoint(db: pgwire.PgClient, bucketStorage: storage.BucketStorageFactory, options?: {
|
|
12
15
|
timeout?: number;
|
|
13
16
|
}): Promise<OpId>;
|
package/dist/util/utils.js
CHANGED
|
@@ -23,29 +23,46 @@ export function timestampToOpId(ts) {
|
|
|
23
23
|
return ts.toString(10);
|
|
24
24
|
}
|
|
25
25
|
export function checksumsDiff(previous, current) {
|
|
26
|
-
|
|
27
|
-
const
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
26
|
+
// All changed ones
|
|
27
|
+
const updatedBuckets = new Map();
|
|
28
|
+
const toRemove = new Set(previous.keys());
|
|
29
|
+
for (let checksum of current.values()) {
|
|
30
|
+
const p = previous.get(checksum.bucket);
|
|
31
|
+
if (p == null) {
|
|
32
|
+
// Added
|
|
33
|
+
updatedBuckets.set(checksum.bucket, checksum);
|
|
34
34
|
}
|
|
35
35
|
else {
|
|
36
|
-
|
|
37
|
-
if (
|
|
38
|
-
|
|
36
|
+
toRemove.delete(checksum.bucket);
|
|
37
|
+
if (checksum.checksum != p.checksum || checksum.count != p.count) {
|
|
38
|
+
// Updated
|
|
39
|
+
updatedBuckets.set(checksum.bucket, checksum);
|
|
40
|
+
}
|
|
41
|
+
else {
|
|
42
|
+
// No change
|
|
39
43
|
}
|
|
40
|
-
previousBuckets.delete(checksum.bucket);
|
|
41
44
|
}
|
|
42
45
|
}
|
|
43
|
-
const removed_buckets = [...previousBuckets.keys()];
|
|
44
46
|
return {
|
|
45
|
-
|
|
46
|
-
|
|
47
|
+
updatedBuckets: [...updatedBuckets.values()],
|
|
48
|
+
removedBuckets: [...toRemove]
|
|
47
49
|
};
|
|
48
50
|
}
|
|
51
|
+
export function addChecksums(a, b) {
|
|
52
|
+
return (a + b) & 0xffffffff;
|
|
53
|
+
}
|
|
54
|
+
export function addBucketChecksums(a, b) {
|
|
55
|
+
if (b == null) {
|
|
56
|
+
return a;
|
|
57
|
+
}
|
|
58
|
+
else {
|
|
59
|
+
return {
|
|
60
|
+
bucket: a.bucket,
|
|
61
|
+
count: a.count + b.count,
|
|
62
|
+
checksum: addChecksums(a.checksum, b.checksum)
|
|
63
|
+
};
|
|
64
|
+
}
|
|
65
|
+
}
|
|
49
66
|
export async function getClientCheckpoint(db, bucketStorage, options) {
|
|
50
67
|
const start = Date.now();
|
|
51
68
|
const [{ lsn }] = pgwireRows(await db.query(`SELECT pg_logical_emit_message(false, 'powersync', 'ping') as lsn`));
|
package/dist/util/utils.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"utils.js","sourceRoot":"","sources":["../../src/util/utils.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,QAAQ,CAAC;AAE5B,OAAO,EAAE,UAAU,EAAE,MAAM,4BAA4B,CAAC;AACxD,OAAO,KAAK,KAAK,MAAM,6BAA6B,CAAC;AAIrD,OAAO,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;
|
|
1
|
+
{"version":3,"file":"utils.js","sourceRoot":"","sources":["../../src/util/utils.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,QAAQ,CAAC;AAE5B,OAAO,EAAE,UAAU,EAAE,MAAM,4BAA4B,CAAC;AACxD,OAAO,KAAK,KAAK,MAAM,6BAA6B,CAAC;AAIrD,OAAO,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AAIjD,MAAM,UAAU,QAAQ,CAAC,IAAY,EAAE,EAAU,EAAE,IAAY;IAC7D,MAAM,IAAI,GAAG,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;IACzC,IAAI,CAAC,MAAM,CAAC,OAAO,IAAI,IAAI,EAAE,IAAI,IAAI,EAAE,CAAC,CAAC;IACzC,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;IAC7B,OAAO,MAAM,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;AAChC,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,SAAiB;IAC1C,MAAM,IAAI,GAAG,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;IACzC,IAAI,CAAC,MAAM,CAAC,UAAU,SAAS,EAAE,CAAC,CAAC;IACnC,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;IAC7B,OAAO,MAAM,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;AAChC,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,EAAU;IACxC,6EAA6E;IAC7E,6CAA6C;IAC7C,IAAI,OAAO,EAAE,IAAI,QAAQ,EAAE,CAAC;QAC1B,MAAM,IAAI,KAAK,CAAC,yBAAyB,EAAE,KAAK,OAAO,EAAE,GAAG,CAAC,CAAC;IAChE,CAAC;IACD,OAAO,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;AACzB,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,QAAqB,EAAE,OAAoB;IACvE,mBAAmB;IACnB,MAAM,cAAc,GAAG,IAAI,GAAG,EAA0B,CAAC;IAEzD,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAS,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC;IAElD,KAAK,IAAI,QAAQ,IAAI,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;QACtC,MAAM,CAAC,GAAG,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QACxC,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC;YACd,QAAQ;YACR,cAAc,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;QAChD,CAAC;aAAM,CAAC;YACN,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;YACjC,IAAI,QAAQ,CAAC,QAAQ,IAAI,CAAC,CAAC,QAAQ,IAAI,QAAQ,CAAC,KAAK,IAAI,CAAC,CAAC,KAAK,EAAE,CAAC;gBACjE,UAAU;gBACV,cAAc,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;YAChD,CAAC;iBAAM,CAAC;gBACN,YAAY;YACd,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO;QACL,cAAc,EAAE,CAAC,GAAG,cAAc,CAAC,MAAM,EAAE,CAAC;QAC5C,cAAc,EAAE,CAAC,GAAG,QAAQ,CAAC;KAC9B,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,CAAS,EAAE,CAAS;IAC/C,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,UAAU,CAAC;AAC9B,CAAC;AAED,MAAM,UAAU,kBAAkB,CAAC,CAAiB,EAAE,CAAwB;IAC5E,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC;QACd,OAAO,CAAC,CAAC;IACX,CAAC;SAAM,CAAC;QACN,OAAO;YACL,MAAM,EAAE,CAAC,CAAC,MAAM;YAChB,KAAK,EAAE,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK;YACxB,QAAQ,EAAE,YAAY,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,QAAQ,CAAC;SAC/C,CAAC;IACJ,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,mBAAmB,CACvC,EAAmB,EACnB,aAA2C,EAC3C,OAA8B;IAE9B,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAEzB,MAAM,CAAC,EAAE,GAAG,EAAE,CAAC,GAAG,UAAU,CAAC,MAAM,EAAE,CAAC,KAAK,CAAC,mEAAmE,CAAC,CAAC,CAAC;IAElH,gDAAgD;IAChD,wEAAwE;IAExE,MAAM,OAAO,GAAG,OAAO,EAAE,OAAO,IAAI,KAAM,CAAC;IAE3C,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,+BAA+B,GAAG,EAAE,CAAC,CAAC;IACxD,OAAO,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,GAAG,OAAO,EAAE,CAAC;QACpC,MAAM,EAAE,GAAG,MAAM,aAAa,CAAC,mBAAmB,EAAE,CAAC;QACrD,IAAI,CAAC,EAAE,CAAC,YAAY,EAAE,EAAE,CAAC;YACvB,MAAM,IAAI,KAAK,CAAC,yBAAyB,CAAC,CAAC;QAC7C,CAAC;QACD,IAAI,EAAE,CAAC,GAAG,IAAI,GAAG,EAAE,CAAC;YAClB,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,yBAAyB,GAAG,MAAM,EAAE,CAAC,UAAU,EAAE,CAAC,CAAC;YACrE,OAAO,EAAE,CAAC,UAAU,CAAC;QACvB,CAAC;QAED,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;IAC1D,CAAC;IAED,MAAM,IAAI,KAAK,CAAC,sCAAsC,CAAC,CAAC;AAC1D,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,qBAAqB,CACzC,EAAmB,EACnB,aAA2C,EAC3C,OAAe;IAEf,MAAM,CAAC,EAAE,GAAG,EAAE,CAAC,GAAG,UAAU,CAC1B,MAAM,YAAY,CAAC,EAAE,EAAE,mEAAmE,CAAC,CAC5F,CAAC;IAEF,MAAM,EAAE,GAAG,MAAM,aAAa,CAAC,qBAAqB,CAAC,OAAO,EAAE,EAAE,GAAG,EAAE,GAAG,EAAE,CAAC,CAAC;IAC5E,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,uBAAuB,IAAI,CAAC,SAAS,CAAC,EAAE,GAAG,EAAE,EAAE,EAAE,MAAM,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;IACpF,OAAO,EAAE,CAAC;AACZ,CAAC"}
|
package/package.json
CHANGED
|
@@ -5,17 +5,18 @@
|
|
|
5
5
|
"publishConfig": {
|
|
6
6
|
"access": "public"
|
|
7
7
|
},
|
|
8
|
-
"version": "0.2.
|
|
8
|
+
"version": "0.2.1",
|
|
9
9
|
"main": "dist/index.js",
|
|
10
10
|
"license": "FSL-1.1-Apache-2.0",
|
|
11
11
|
"type": "module",
|
|
12
12
|
"dependencies": {
|
|
13
13
|
"@journeyapps-platform/micro": "^17.0.1",
|
|
14
14
|
"@journeyapps-platform/micro-migrate": "^4.0.1",
|
|
15
|
+
"@js-sdsl/ordered-set": "^4.4.2",
|
|
15
16
|
"@opentelemetry/api": "~1.8.0",
|
|
16
|
-
"@opentelemetry/resources": "^1.24.1",
|
|
17
|
-
"@opentelemetry/exporter-prometheus": "^0.51.1",
|
|
18
17
|
"@opentelemetry/exporter-metrics-otlp-http": "^0.51.1",
|
|
18
|
+
"@opentelemetry/exporter-prometheus": "^0.51.1",
|
|
19
|
+
"@opentelemetry/resources": "^1.24.1",
|
|
19
20
|
"@opentelemetry/sdk-metrics": "1.24.1",
|
|
20
21
|
"async-mutex": "^0.5.0",
|
|
21
22
|
"bson": "^6.6.0",
|
|
@@ -33,10 +34,10 @@
|
|
|
33
34
|
"uuid": "^9.0.1",
|
|
34
35
|
"yaml": "^2.3.2",
|
|
35
36
|
"@powersync/service-jpgwire": "0.17.12",
|
|
36
|
-
"@powersync/service-types": "0.1.0",
|
|
37
37
|
"@powersync/service-jsonbig": "0.17.10",
|
|
38
38
|
"@powersync/service-rsocket-router": "0.0.7",
|
|
39
|
-
"@powersync/service-sync-rules": "0.17.10"
|
|
39
|
+
"@powersync/service-sync-rules": "0.17.10",
|
|
40
|
+
"@powersync/service-types": "0.1.0"
|
|
40
41
|
},
|
|
41
42
|
"devDependencies": {
|
|
42
43
|
"@journeyapps-platform/micro-dev": "^1.6.7",
|
|
@@ -230,7 +230,12 @@ export interface SyncRulesBucketStorage {
|
|
|
230
230
|
options?: BucketDataBatchOptions
|
|
231
231
|
): AsyncIterable<util.SyncBucketData>;
|
|
232
232
|
|
|
233
|
-
|
|
233
|
+
/**
|
|
234
|
+
* Compute checksums for a given list of buckets.
|
|
235
|
+
*
|
|
236
|
+
* Returns zero checksums for any buckets not found.
|
|
237
|
+
*/
|
|
238
|
+
getChecksums(checkpoint: util.OpId, buckets: string[]): Promise<util.ChecksumMap>;
|
|
234
239
|
|
|
235
240
|
/**
|
|
236
241
|
* Terminate the sync rules.
|
|
@@ -0,0 +1,294 @@
|
|
|
1
|
+
import { BucketChecksum, OpId } from '@/util/protocol-types.js';
|
|
2
|
+
import { ChecksumMap, addBucketChecksums } from '@/util/utils.js';
|
|
3
|
+
import { LRUCache } from 'lru-cache/min';
|
|
4
|
+
import { OrderedSet } from '@js-sdsl/ordered-set';
|
|
5
|
+
|
|
6
|
+
interface ChecksumFetchContext {
|
|
7
|
+
fetch(bucket: string): Promise<BucketChecksum>;
|
|
8
|
+
checkpoint: bigint;
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
export interface FetchPartialBucketChecksum {
|
|
12
|
+
bucket: string;
|
|
13
|
+
start?: OpId;
|
|
14
|
+
end: OpId;
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
export type FetchChecksums = (batch: FetchPartialBucketChecksum[]) => Promise<ChecksumMap>;
|
|
18
|
+
|
|
19
|
+
export interface ChecksumCacheOptions {
|
|
20
|
+
/**
|
|
21
|
+
* Upstream checksum implementation.
|
|
22
|
+
*
|
|
23
|
+
* This fetches a batch of either entire bucket checksums, or a partial range.
|
|
24
|
+
*/
|
|
25
|
+
fetchChecksums: FetchChecksums;
|
|
26
|
+
|
|
27
|
+
/**
|
|
28
|
+
* Maximum number of cached checksums.
|
|
29
|
+
*/
|
|
30
|
+
maxSize?: number;
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
// Approximately 5MB of memory, if we assume 50 bytes per entry
|
|
34
|
+
const DEFAULT_MAX_SIZE = 100_000;
|
|
35
|
+
|
|
36
|
+
/**
|
|
37
|
+
* Implement a LRU cache for checksum requests. Each (bucket, checkpoint) request is cached separately,
|
|
38
|
+
* while the lookups occur in batches.
|
|
39
|
+
*
|
|
40
|
+
* For each bucket, we keep a separate OrderedSet of cached checkpoints.
|
|
41
|
+
* This allows us to do incrementally update checksums by using the last cached checksum for the same bucket.
|
|
42
|
+
*
|
|
43
|
+
* We use the LRUCache fetchMethod to deduplicate in-progress requests.
|
|
44
|
+
*/
|
|
45
|
+
export class ChecksumCache {
|
|
46
|
+
/**
|
|
47
|
+
* The primary checksum cache, with key of `${checkpoint}/${bucket}`.
|
|
48
|
+
*/
|
|
49
|
+
private cache: LRUCache<string, BucketChecksum, ChecksumFetchContext>;
|
|
50
|
+
/**
|
|
51
|
+
* For each bucket, an ordered set of cached checkpoints.
|
|
52
|
+
*/
|
|
53
|
+
private bucketCheckpoints = new Map<string, OrderedSet<bigint>>();
|
|
54
|
+
|
|
55
|
+
private fetchChecksums: FetchChecksums;
|
|
56
|
+
|
|
57
|
+
constructor(options: ChecksumCacheOptions) {
|
|
58
|
+
this.fetchChecksums = options.fetchChecksums;
|
|
59
|
+
|
|
60
|
+
this.cache = new LRUCache<string, BucketChecksum, ChecksumFetchContext>({
|
|
61
|
+
max: options.maxSize ?? DEFAULT_MAX_SIZE,
|
|
62
|
+
fetchMethod: async (cacheKey, _staleValue, options) => {
|
|
63
|
+
// Called when this checksum hasn't been cached yet.
|
|
64
|
+
// Pass the call back to the request, which implements batch fetching.
|
|
65
|
+
const { bucket } = parseCacheKey(cacheKey);
|
|
66
|
+
const result = await options.context.fetch(bucket);
|
|
67
|
+
|
|
68
|
+
// Add to the set of cached checkpoints for the bucket.
|
|
69
|
+
let checkpointSet = this.bucketCheckpoints.get(bucket);
|
|
70
|
+
if (checkpointSet == null) {
|
|
71
|
+
checkpointSet = new OrderedSet();
|
|
72
|
+
this.bucketCheckpoints.set(bucket, checkpointSet);
|
|
73
|
+
}
|
|
74
|
+
checkpointSet.insert(options.context.checkpoint);
|
|
75
|
+
return result;
|
|
76
|
+
},
|
|
77
|
+
|
|
78
|
+
dispose: (value, key) => {
|
|
79
|
+
// Remove from the set of cached checkpoints for the bucket
|
|
80
|
+
const { checkpointString } = parseCacheKey(key);
|
|
81
|
+
const checkpoint = BigInt(checkpointString);
|
|
82
|
+
const checkpointSet = this.bucketCheckpoints.get(value.bucket);
|
|
83
|
+
if (checkpointSet == null) {
|
|
84
|
+
return;
|
|
85
|
+
}
|
|
86
|
+
checkpointSet.eraseElementByKey(checkpoint);
|
|
87
|
+
if (checkpointSet.length == 0) {
|
|
88
|
+
this.bucketCheckpoints.delete(value.bucket);
|
|
89
|
+
}
|
|
90
|
+
},
|
|
91
|
+
|
|
92
|
+
noDisposeOnSet: true,
|
|
93
|
+
|
|
94
|
+
// When we have more fetches than the cache size, complete the fetches instead
|
|
95
|
+
// of failing with Error('evicted').
|
|
96
|
+
ignoreFetchAbort: true
|
|
97
|
+
});
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
async getChecksums(checkpoint: OpId, buckets: string[]): Promise<BucketChecksum[]> {
|
|
101
|
+
const checksums = await this.getChecksumMap(checkpoint, buckets);
|
|
102
|
+
// Return results in the same order as the request
|
|
103
|
+
return buckets.map((bucket) => checksums.get(bucket)!);
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
/**
|
|
107
|
+
* Get bucket checksums for a checkpoint.
|
|
108
|
+
*
|
|
109
|
+
* Any checksums not found upstream are returned as zero checksums.
|
|
110
|
+
*
|
|
111
|
+
* @returns a Map with exactly one entry for each bucket requested
|
|
112
|
+
*/
|
|
113
|
+
async getChecksumMap(checkpoint: OpId, buckets: string[]): Promise<ChecksumMap> {
|
|
114
|
+
// Buckets that don't have a cached checksum for this checkpoint yet
|
|
115
|
+
let toFetch = new Set<string>();
|
|
116
|
+
|
|
117
|
+
// Newly fetched results
|
|
118
|
+
let fetchResults = new Map<string, BucketChecksum>();
|
|
119
|
+
|
|
120
|
+
// Promise for the bactch new fetch requests
|
|
121
|
+
let resolveFetch!: () => void;
|
|
122
|
+
let rejectFetch!: (err: any) => void;
|
|
123
|
+
let fetchPromise = new Promise<void>((resolve, reject) => {
|
|
124
|
+
resolveFetch = resolve;
|
|
125
|
+
rejectFetch = reject;
|
|
126
|
+
});
|
|
127
|
+
|
|
128
|
+
// Accumulated results - both from cached checksums, and fetched checksums
|
|
129
|
+
let finalResults = new Map<string, BucketChecksum>();
|
|
130
|
+
|
|
131
|
+
const context: ChecksumFetchContext = {
|
|
132
|
+
async fetch(bucket) {
|
|
133
|
+
await fetchPromise;
|
|
134
|
+
if (!toFetch.has(bucket)) {
|
|
135
|
+
// Should never happen
|
|
136
|
+
throw new Error(`Expected to fetch ${bucket}`);
|
|
137
|
+
}
|
|
138
|
+
const checksum = fetchResults.get(bucket);
|
|
139
|
+
if (checksum == null) {
|
|
140
|
+
// Should never happen
|
|
141
|
+
throw new Error(`Failed to fetch checksum for bucket ${bucket}`);
|
|
142
|
+
}
|
|
143
|
+
return checksum;
|
|
144
|
+
},
|
|
145
|
+
checkpoint: BigInt(checkpoint)
|
|
146
|
+
};
|
|
147
|
+
|
|
148
|
+
// One promise to await to ensure all fetch requests completed.
|
|
149
|
+
let settledPromise: Promise<PromiseSettledResult<void>[]> | null = null;
|
|
150
|
+
|
|
151
|
+
try {
|
|
152
|
+
// Individual cache fetch promises
|
|
153
|
+
let cacheFetchPromises: Promise<void>[] = [];
|
|
154
|
+
|
|
155
|
+
for (let bucket of buckets) {
|
|
156
|
+
const cacheKey = makeCacheKey(checkpoint, bucket);
|
|
157
|
+
let status: LRUCache.Status<BucketChecksum> = {};
|
|
158
|
+
const p = this.cache.fetch(cacheKey, { context: context, status: status }).then((checksums) => {
|
|
159
|
+
if (checksums == null) {
|
|
160
|
+
// Should never happen
|
|
161
|
+
throw new Error(`Failed to get checksums for ${cacheKey}`);
|
|
162
|
+
}
|
|
163
|
+
finalResults.set(bucket, checksums);
|
|
164
|
+
});
|
|
165
|
+
cacheFetchPromises.push(p);
|
|
166
|
+
if (status.fetch == 'hit' || status.fetch == 'inflight') {
|
|
167
|
+
// The checksums is either cached already (hit), or another request is busy
|
|
168
|
+
// fetching (inflight).
|
|
169
|
+
// In either case, we don't need to fetch a new checksum.
|
|
170
|
+
} else {
|
|
171
|
+
// We need a new request for this checksum.
|
|
172
|
+
toFetch.add(bucket);
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
// We do this directly after creating the promises, otherwise
|
|
176
|
+
// we could end up with weird uncaught rejection errors.
|
|
177
|
+
settledPromise = Promise.allSettled(cacheFetchPromises);
|
|
178
|
+
|
|
179
|
+
if (toFetch.size == 0) {
|
|
180
|
+
// Nothing to fetch, but resolve in case
|
|
181
|
+
resolveFetch();
|
|
182
|
+
} else {
|
|
183
|
+
let bucketRequests: FetchPartialBucketChecksum[] = [];
|
|
184
|
+
// Partial checksum (previously cached) to add to the partial fetch
|
|
185
|
+
let add = new Map<string, BucketChecksum>();
|
|
186
|
+
|
|
187
|
+
for (let bucket of toFetch) {
|
|
188
|
+
let bucketRequest: FetchPartialBucketChecksum | null = null;
|
|
189
|
+
const checkpointSet = this.bucketCheckpoints.get(bucket);
|
|
190
|
+
if (checkpointSet != null) {
|
|
191
|
+
// Find smaller checkpoints, sorted in descending order
|
|
192
|
+
let iter = checkpointSet.reverseUpperBound(context.checkpoint);
|
|
193
|
+
const begin = checkpointSet.begin();
|
|
194
|
+
while (iter.isAccessible()) {
|
|
195
|
+
const cp = iter.pointer;
|
|
196
|
+
const cacheKey = makeCacheKey(cp, bucket);
|
|
197
|
+
// peek to avoid refreshing the key
|
|
198
|
+
const cached = this.cache.peek(cacheKey);
|
|
199
|
+
// As long as dispose() works correctly, the checkpointset should
|
|
200
|
+
// match up with the cache, and `cached` should also have a value here.
|
|
201
|
+
// However, we handle caces where it's not present either way.
|
|
202
|
+
// Test by disabling the `dispose()` callback.
|
|
203
|
+
if (cached != null) {
|
|
204
|
+
// Partial checksum found - make a partial checksum request
|
|
205
|
+
bucketRequest = {
|
|
206
|
+
bucket,
|
|
207
|
+
start: cp.toString(),
|
|
208
|
+
end: checkpoint
|
|
209
|
+
};
|
|
210
|
+
add.set(bucket, cached);
|
|
211
|
+
break;
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
if (iter.equals(begin)) {
|
|
215
|
+
// Cannot iterate further
|
|
216
|
+
break;
|
|
217
|
+
}
|
|
218
|
+
// Iterate backwards
|
|
219
|
+
iter = iter.pre();
|
|
220
|
+
}
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
if (bucketRequest == null) {
|
|
224
|
+
// No partial checksum found - make a new full checksum request
|
|
225
|
+
bucketRequest = {
|
|
226
|
+
bucket,
|
|
227
|
+
end: checkpoint
|
|
228
|
+
};
|
|
229
|
+
add.set(bucket, {
|
|
230
|
+
bucket,
|
|
231
|
+
checksum: 0,
|
|
232
|
+
count: 0
|
|
233
|
+
});
|
|
234
|
+
}
|
|
235
|
+
bucketRequests.push(bucketRequest);
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
// Fetch partial checksums from upstream
|
|
239
|
+
const results = await this.fetchChecksums(bucketRequests);
|
|
240
|
+
|
|
241
|
+
for (let bucket of toFetch) {
|
|
242
|
+
const result = results.get(bucket);
|
|
243
|
+
const toAdd = add.get(bucket);
|
|
244
|
+
if (toAdd == null) {
|
|
245
|
+
// Should never happen
|
|
246
|
+
throw new Error(`toAdd null for ${bucket}`);
|
|
247
|
+
}
|
|
248
|
+
// Compute the full checksum from the two partials.
|
|
249
|
+
// No results returned are treated the same as a zero result.
|
|
250
|
+
const added = addBucketChecksums(toAdd, result ?? null);
|
|
251
|
+
fetchResults.set(bucket, added);
|
|
252
|
+
}
|
|
253
|
+
|
|
254
|
+
// fetchResults is fully populated, so we resolve the Promise
|
|
255
|
+
resolveFetch();
|
|
256
|
+
}
|
|
257
|
+
} catch (e) {
|
|
258
|
+
// Failure when fetching checksums - reject the Promise.
|
|
259
|
+
// This will reject all individual cache fetch requests, and each will be retried
|
|
260
|
+
// on the next request.
|
|
261
|
+
rejectFetch(e);
|
|
262
|
+
|
|
263
|
+
// Wait for the above rejection to propagate, otherwise we end up with "uncaught" errors.
|
|
264
|
+
// This promise never throws.
|
|
265
|
+
await settledPromise;
|
|
266
|
+
|
|
267
|
+
throw e;
|
|
268
|
+
}
|
|
269
|
+
|
|
270
|
+
// Wait for all cache fetch reqeusts to complete
|
|
271
|
+
const settledResults = (await settledPromise) ?? [];
|
|
272
|
+
// Check if any of them failed
|
|
273
|
+
for (let result of settledResults) {
|
|
274
|
+
if (result.status == 'rejected') {
|
|
275
|
+
throw result.reason;
|
|
276
|
+
}
|
|
277
|
+
}
|
|
278
|
+
|
|
279
|
+
if (finalResults.size != buckets.length) {
|
|
280
|
+
// Should not happen
|
|
281
|
+
throw new Error(`Bucket results mismatch: ${finalResults.size} != ${buckets.length}`);
|
|
282
|
+
}
|
|
283
|
+
return finalResults;
|
|
284
|
+
}
|
|
285
|
+
}
|
|
286
|
+
|
|
287
|
+
function makeCacheKey(checkpoint: bigint | string, bucket: string) {
|
|
288
|
+
return `${checkpoint}/${bucket}`;
|
|
289
|
+
}
|
|
290
|
+
|
|
291
|
+
function parseCacheKey(key: string) {
|
|
292
|
+
const index = key.indexOf('/');
|
|
293
|
+
return { checkpointString: key.substring(0, index), bucket: key.substring(index + 1) };
|
|
294
|
+
}
|
|
@@ -22,9 +22,15 @@ import { PowerSyncMongo } from './db.js';
|
|
|
22
22
|
import { BucketDataDocument, BucketDataKey, SourceKey, SyncRuleState } from './models.js';
|
|
23
23
|
import { MongoBucketBatch } from './MongoBucketBatch.js';
|
|
24
24
|
import { BSON_DESERIALIZE_OPTIONS, idPrefixFilter, readSingleBatch, serializeLookup } from './util.js';
|
|
25
|
+
import { ChecksumCache, FetchPartialBucketChecksum } from '../ChecksumCache.js';
|
|
25
26
|
|
|
26
27
|
export class MongoSyncBucketStorage implements SyncRulesBucketStorage {
|
|
27
28
|
private readonly db: PowerSyncMongo;
|
|
29
|
+
private checksumCache = new ChecksumCache({
|
|
30
|
+
fetchChecksums: (batch) => {
|
|
31
|
+
return this.getChecksumsInternal(batch);
|
|
32
|
+
}
|
|
33
|
+
});
|
|
28
34
|
|
|
29
35
|
constructor(
|
|
30
36
|
public readonly factory: MongoBucketStorage,
|
|
@@ -316,23 +322,28 @@ export class MongoSyncBucketStorage implements SyncRulesBucketStorage {
|
|
|
316
322
|
}
|
|
317
323
|
}
|
|
318
324
|
|
|
319
|
-
async getChecksums(checkpoint: util.OpId, buckets: string[]): Promise<util.
|
|
320
|
-
|
|
321
|
-
|
|
325
|
+
async getChecksums(checkpoint: util.OpId, buckets: string[]): Promise<util.ChecksumMap> {
|
|
326
|
+
return this.checksumCache.getChecksumMap(checkpoint, buckets);
|
|
327
|
+
}
|
|
328
|
+
|
|
329
|
+
private async getChecksumsInternal(batch: FetchPartialBucketChecksum[]): Promise<util.ChecksumMap> {
|
|
330
|
+
if (batch.length == 0) {
|
|
331
|
+
return new Map();
|
|
322
332
|
}
|
|
333
|
+
|
|
323
334
|
const filters: any[] = [];
|
|
324
|
-
for (let
|
|
335
|
+
for (let request of batch) {
|
|
325
336
|
filters.push({
|
|
326
337
|
_id: {
|
|
327
338
|
$gt: {
|
|
328
339
|
g: this.group_id,
|
|
329
|
-
b:
|
|
330
|
-
o: new bson.MinKey()
|
|
340
|
+
b: request.bucket,
|
|
341
|
+
o: request.start ? BigInt(request.start) : new bson.MinKey()
|
|
331
342
|
},
|
|
332
343
|
$lte: {
|
|
333
344
|
g: this.group_id,
|
|
334
|
-
b:
|
|
335
|
-
o: BigInt(
|
|
345
|
+
b: request.bucket,
|
|
346
|
+
o: BigInt(request.end)
|
|
336
347
|
}
|
|
337
348
|
}
|
|
338
349
|
});
|
|
@@ -354,13 +365,18 @@ export class MongoSyncBucketStorage implements SyncRulesBucketStorage {
|
|
|
354
365
|
)
|
|
355
366
|
.toArray();
|
|
356
367
|
|
|
357
|
-
return
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
368
|
+
return new Map<string, util.BucketChecksum>(
|
|
369
|
+
aggregate.map((doc) => {
|
|
370
|
+
return [
|
|
371
|
+
doc._id,
|
|
372
|
+
{
|
|
373
|
+
bucket: doc._id,
|
|
374
|
+
count: doc.count,
|
|
375
|
+
checksum: Number(BigInt(doc.checksum_total) & 0xffffffffn) & 0xffffffff
|
|
376
|
+
} satisfies util.BucketChecksum
|
|
377
|
+
];
|
|
378
|
+
})
|
|
379
|
+
);
|
|
364
380
|
}
|
|
365
381
|
|
|
366
382
|
async terminate() {
|