@powersync/service-module-mongodb-storage 0.0.0-dev-20250122110924 → 0.0.0-dev-20250227082606

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (53) hide show
  1. package/CHANGELOG.md +101 -10
  2. package/dist/migrations/MongoMigrationAgent.js +3 -0
  3. package/dist/migrations/MongoMigrationAgent.js.map +1 -1
  4. package/dist/storage/MongoBucketStorage.d.ts +9 -20
  5. package/dist/storage/MongoBucketStorage.js +86 -199
  6. package/dist/storage/MongoBucketStorage.js.map +1 -1
  7. package/dist/storage/implementation/MongoBucketBatch.d.ts +3 -3
  8. package/dist/storage/implementation/MongoBucketBatch.js +37 -24
  9. package/dist/storage/implementation/MongoBucketBatch.js.map +1 -1
  10. package/dist/storage/implementation/MongoCompactor.d.ts +0 -6
  11. package/dist/storage/implementation/MongoCompactor.js +11 -4
  12. package/dist/storage/implementation/MongoCompactor.js.map +1 -1
  13. package/dist/storage/implementation/MongoIdSequence.js +1 -0
  14. package/dist/storage/implementation/MongoIdSequence.js.map +1 -1
  15. package/dist/storage/implementation/MongoPersistedSyncRules.js +4 -0
  16. package/dist/storage/implementation/MongoPersistedSyncRules.js.map +1 -1
  17. package/dist/storage/implementation/MongoPersistedSyncRulesContent.js +9 -1
  18. package/dist/storage/implementation/MongoPersistedSyncRulesContent.js.map +1 -1
  19. package/dist/storage/implementation/MongoStorageProvider.js +3 -1
  20. package/dist/storage/implementation/MongoStorageProvider.js.map +1 -1
  21. package/dist/storage/implementation/MongoSyncBucketStorage.d.ts +16 -3
  22. package/dist/storage/implementation/MongoSyncBucketStorage.js +225 -22
  23. package/dist/storage/implementation/MongoSyncBucketStorage.js.map +1 -1
  24. package/dist/storage/implementation/MongoSyncRulesLock.js +5 -1
  25. package/dist/storage/implementation/MongoSyncRulesLock.js.map +1 -1
  26. package/dist/storage/implementation/MongoWriteCheckpointAPI.js +3 -2
  27. package/dist/storage/implementation/MongoWriteCheckpointAPI.js.map +1 -1
  28. package/dist/storage/implementation/OperationBatch.js +10 -6
  29. package/dist/storage/implementation/OperationBatch.js.map +1 -1
  30. package/dist/storage/implementation/PersistedBatch.js +18 -18
  31. package/dist/storage/implementation/PersistedBatch.js.map +1 -1
  32. package/dist/storage/implementation/db.d.ts +1 -1
  33. package/dist/storage/implementation/db.js +15 -3
  34. package/dist/storage/implementation/db.js.map +1 -1
  35. package/dist/storage/implementation/models.d.ts +1 -0
  36. package/dist/storage/implementation/util.d.ts +0 -14
  37. package/dist/storage/implementation/util.js +3 -41
  38. package/dist/storage/implementation/util.js.map +1 -1
  39. package/package.json +8 -8
  40. package/src/storage/MongoBucketStorage.ts +88 -232
  41. package/src/storage/implementation/MongoBucketBatch.ts +12 -10
  42. package/src/storage/implementation/MongoCompactor.ts +2 -10
  43. package/src/storage/implementation/MongoStorageProvider.ts +3 -1
  44. package/src/storage/implementation/MongoSyncBucketStorage.ts +292 -37
  45. package/src/storage/implementation/MongoWriteCheckpointAPI.ts +1 -3
  46. package/src/storage/implementation/PersistedBatch.ts +4 -5
  47. package/src/storage/implementation/db.ts +3 -3
  48. package/src/storage/implementation/models.ts +5 -0
  49. package/src/storage/implementation/util.ts +0 -45
  50. package/test/src/__snapshots__/storage_sync.test.ts.snap +138 -0
  51. package/test/src/storage_compacting.test.ts +1 -7
  52. package/test/src/storage_sync.test.ts +1 -1
  53. package/tsconfig.tsbuildinfo +1 -1
@@ -1 +1 @@
1
- {"version":3,"file":"PersistedBatch.js","sourceRoot":"","sources":["../../../src/storage/implementation/PersistedBatch.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,OAAO,EAAE,MAAM,4BAA4B,CAAC;AAIrD,OAAO,EAAE,MAAM,EAAE,MAAM,mCAAmC,CAAC;AAC3D,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,yBAAyB,CAAC;AACzD,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AAUzD,OAAO,EAAE,iBAAiB,EAAE,aAAa,EAAE,MAAM,WAAW,CAAC;AAE7D;;;;;;;;;;;GAWG;AACH,MAAM,0BAA0B,GAAG,QAAU,CAAC;AAE9C;;;;GAIG;AACH,MAAM,yBAAyB,GAAG,IAAK,CAAC;AAExC;;;;;GAKG;AACH,MAAM,OAAO,cAAc;IAezB,YACU,QAAgB,EACxB,WAAmB;QADX,aAAQ,GAAR,QAAQ,CAAQ;QAf1B,eAAU,GAAsD,EAAE,CAAC;QACnE,qBAAgB,GAA2D,EAAE,CAAC;QAC9E,gBAAW,GAAuD,EAAE,CAAC;QAErE;;WAEG;QACH,kBAAa,GAAkB,IAAI,CAAC;QAEpC;;WAEG;QACH,gBAAW,GAAG,CAAC,CAAC;QAMd,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC;IACjC,CAAC;IAED,cAAc,CAAC,OAMd;QACC,MAAM,iBAAiB,GAAG,IAAI,GAAG,EAAyB,CAAC;QAC3D,KAAK,IAAI,CAAC,IAAI,OAAO,CAAC,cAAc,EAAE,CAAC;YACrC,MAAM,GAAG,GAAG,gBAAgB,CAAC,CAAC,CAAC,CAAC;YAChC,iBAAiB,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;QAChC,CAAC;QAED,MAAM,SAAS,GAAG,KAAK,CAAC,UAAU,CAAC,iBAAiB,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,EAAE,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC;QAE3F,KAAK,MAAM,CAAC,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC;YAClC,MAAM,GAAG,GAAG,gBAAgB,CAAC,CAAC,CAAC,CAAC;YAChC,iBAAiB,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YAE9B,SAAS;YACT,MAAM,UAAU,GAAG,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;YAC7C,MAAM,QAAQ,GAAG,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,EAAE,EAAE,UAAU,CAAC,CAAC;YAC3D,IAAI,CAAC,WAAW,IAAI,UAAU,CAAC,MAAM,GAAG,GAAG,CAAC;YAE5C,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;YACpC,IAAI,CAAC,aAAa,GAAG,KAAK,CAAC;YAE3B,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;gBACnB,SAAS,EAAE;oBACT,QAAQ,EAAE;wBACR,GAAG,EAAE;4BACH,CAAC,EAAE,IAAI,CAAC,QAAQ;4BAChB,CAAC,EAAE,CAAC,CAAC,MAAM;4BACX,CAAC,EAAE,KAAK;yBACT;wBACD,EAAE,EAAE,KAAK;wBACT,YAAY,EAAE,OAAO,CAAC,KAAK,CAAC,EAAE;wBAC9B,UAAU,EAAE,OAAO,CAAC,SAAS;wBAC7B,KAAK,EAAE,CAAC,CAAC,KAAK;wBACd,MAAM,EAAE,CAAC,CAAC,EAAE;wBACZ,QAAQ,EAAE,QAAQ;wBAClB,IAAI,EAAE,UAAU;qBACjB;iBACF;aACF,CAAC,CAAC;QACL,CAAC;QAED,KAAK,IAAI,EAAE,IAAI,iBAAiB,CAAC,MAAM,EAAE,EAAE,CAAC;YAC1C,SAAS;YAET,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;YACpC,IAAI,CAAC,aAAa,GAAG,KAAK,CAAC;YAE3B,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;gBACnB,SAAS,EAAE;oBACT,QAAQ,EAAE;wBACR,GAAG,EAAE;4BACH,CAAC,EAAE,IAAI,CAAC,QAAQ;4BAChB,CAAC,EAAE,EAAE,CAAC,MAAM;4BACZ,CAAC,EAAE,KAAK;yBACT;wBACD,EAAE,EAAE,QAAQ;wBACZ,YAAY,EAAE,OAAO,CAAC,KAAK,CAAC,EAAE;wBAC9B,UAAU,EAAE,OAAO,CAAC,SAAS;wBAC7B,KAAK,EAAE,EAAE,CAAC,KAAK;wBACf,MAAM,EAAE,EAAE,CAAC,EAAE;wBACb,QAAQ,EAAE,SAAS;wBACnB,IAAI,EAAE,IAAI;qBACX;iBACF;aACF,CAAC,CAAC;YACH,IAAI,CAAC,WAAW,IAAI,GAAG,CAAC;QAC1B,CAAC;IACH,CAAC;IAED,iBAAiB,CAAC,IAMjB;QACC,yCAAyC;QACzC,qEAAqE;QACrE,0HAA0H;QAC1H,2DAA2D;QAC3D,8GAA8G;QAC9G,+BAA+B;QAC/B,6CAA6C;QAC7C,MAAM,EAAE,WAAW,EAAE,SAAS,EAAE,SAAS,EAAE,GAAG,IAAI,CAAC;QAEnD,MAAM,iBAAiB,GAAG,IAAI,GAAG,EAAuB,CAAC;QACzD,KAAK,IAAI,CAAC,IAAI,IAAI,CAAC,gBAAgB,EAAE,CAAC;YACpC,iBAAiB,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC;QACjD,CAAC;QAED,wBAAwB;QACxB,KAAK,IAAI,MAAM,IAAI,SAAS,EAAE,CAAC;YAC7B,MAAM,SAAS,GAAG,OAAO,CAAC,eAAe,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;YACzD,MAAM,GAAG,GAAG,SAAS,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;YACzC,iBAAiB,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YAE9B,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;YACjC,IAAI,CAAC,aAAa,GAAG,KAAK,CAAC;YAC3B,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC;gBACzB,SAAS,EAAE;oBACT,QAAQ,EAAE;wBACR,GAAG,EAAE,KAAK;wBACV,GAAG,EAAE;4BACH,CAAC,EAAE,IAAI,CAAC,QAAQ;4BAChB,CAAC,EAAE,WAAW,CAAC,EAAE;4BACjB,CAAC,EAAE,SAAS;yBACb;wBACD,MAAM,EAAE,SAAS;wBACjB,iBAAiB,EAAE,MAAM,CAAC,iBAAiB;qBAC5C;iBACF;aACF,CAAC,CAAC;YAEH,IAAI,CAAC,WAAW,IAAI,GAAG,CAAC;QAC1B,CAAC;QAED,kDAAkD;QAClD,KAAK,IAAI,MAAM,IAAI,iBAAiB,CAAC,MAAM,EAAE,EAAE,CAAC;YAC9C,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;YACjC,IAAI,CAAC,aAAa,GAAG,KAAK,CAAC;YAC3B,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC;gBACzB,SAAS,EAAE;oBACT,QAAQ,EAAE;wBACR,GAAG,EAAE,KAAK;wBACV,GAAG,EAAE;4BACH,CAAC,EAAE,IAAI,CAAC,QAAQ;4BAChB,CAAC,EAAE,WAAW,CAAC,EAAE;4BACjB,CAAC,EAAE,SAAS;yBACb;wBACD,MAAM,EAAE,MAAM;wBACd,iBAAiB,EAAE,EAAE;qBACtB;iBACF;aACF,CAAC,CAAC;YAEH,IAAI,CAAC,WAAW,IAAI,GAAG,CAAC;QAC1B,CAAC;IACH,CAAC;IAED,iBAAiB,CAAC,EAAa;QAC7B,MAAM,EAAE,GAAqD;YAC3D,SAAS,EAAE;gBACT,MAAM,EAAE,EAAE,GAAG,EAAE,EAAE,EAAE;aACpB;SACF,CAAC;QACF,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAC1B,IAAI,CAAC,WAAW,IAAI,EAAE,CAAC;IACzB,CAAC;IAED,iBAAiB,CAAC,EAAa,EAAE,MAAoC;QACnE,MAAM,EAAE,GAAqD;YAC3D,SAAS,EAAE;gBACT,MAAM,EAAE,EAAE,GAAG,EAAE,EAAE,EAAE;gBACnB,MAAM,EAAE;oBACN,IAAI,EAAE,MAAM;iBACb;gBACD,MAAM,EAAE,IAAI;aACb;SACF,CAAC;QACF,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAC1B,IAAI,CAAC,WAAW,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,CAAC,GAAG,GAAG,CAAC;IACzD,CAAC;IAED,sBAAsB;QACpB,OAAO,CACL,IAAI,CAAC,WAAW,IAAI,0BAA0B;YAC9C,IAAI,CAAC,UAAU,CAAC,MAAM,IAAI,yBAAyB;YACnD,IAAI,CAAC,WAAW,CAAC,MAAM,IAAI,yBAAyB;YACpD,IAAI,CAAC,gBAAgB,CAAC,MAAM,IAAI,yBAAyB,CAC1D,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,EAAkB,EAAE,OAA4B;QAC1D,IAAI,IAAI,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC/B,uBAAuB;YACvB,MAAM,aAAa,CAAC,EAAE,CAAC,WAAW,EAAE,IAAI,CAAC,UAAU,EAAE;gBACnD,OAAO;gBACP,sCAAsC;gBACtC,OAAO,EAAE,KAAK;aACf,CAAC,CAAC;QACL,CAAC;QACD,IAAI,IAAI,CAAC,gBAAgB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACrC,MAAM,aAAa,CAAC,EAAE,CAAC,iBAAiB,EAAE,IAAI,CAAC,gBAAgB,EAAE;gBAC/D,OAAO;gBACP,sCAAsC;gBACtC,OAAO,EAAE,KAAK;aACf,CAAC,CAAC;QACL,CAAC;QACD,IAAI,IAAI,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAChC,MAAM,aAAa,CAAC,EAAE,CAAC,YAAY,EAAE,IAAI,CAAC,WAAW,EAAE;gBACrD,OAAO;gBACP,mEAAmE;gBACnE,OAAO,EAAE,IAAI;aACd,CAAC,CAAC;QACL,CAAC;QAED,MAAM,CAAC,IAAI,CACT,aAAa,IAAI,CAAC,QAAQ,YAAY,IAAI,CAAC,UAAU,CAAC,MAAM,MAAM,IAAI,CAAC,gBAAgB,CAAC,MAAM,MAC5F,IAAI,CAAC,WAAW,CAAC,MACnB,aAAa,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,mBAAmB,IAAI,CAAC,aAAa,EAAE,CACxF,CAAC;QAEF,IAAI,CAAC,UAAU,GAAG,EAAE,CAAC;QACrB,IAAI,CAAC,gBAAgB,GAAG,EAAE,CAAC;QAC3B,IAAI,CAAC,WAAW,GAAG,EAAE,CAAC;QACtB,IAAI,CAAC,WAAW,GAAG,CAAC,CAAC;QACrB,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC;IAC5B,CAAC;CACF"}
1
+ {"version":3,"file":"PersistedBatch.js","sourceRoot":"","sources":["../../../src/storage/implementation/PersistedBatch.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,OAAO,EAAE,MAAM,4BAA4B,CAAC;AAIrD,OAAO,EAAE,MAAM,EAAE,MAAM,mCAAmC,CAAC;AAC3D,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,yBAAyB,CAAC;AACzD,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AAUzD,OAAO,EAAE,iBAAiB,EAAE,MAAM,WAAW,CAAC;AAE9C;;;;;;;;;;;GAWG;AACH,MAAM,0BAA0B,GAAG,UAAU,CAAC;AAE9C;;;;GAIG;AACH,MAAM,yBAAyB,GAAG,KAAK,CAAC;AAExC;;;;;GAKG;AACH,MAAM,OAAO,cAAc;IAgBf;IAfV,UAAU,GAAsD,EAAE,CAAC;IACnE,gBAAgB,GAA2D,EAAE,CAAC;IAC9E,WAAW,GAAuD,EAAE,CAAC;IAErE;;OAEG;IACH,aAAa,GAAkB,IAAI,CAAC;IAEpC;;OAEG;IACH,WAAW,GAAG,CAAC,CAAC;IAEhB,YACU,QAAgB,EACxB,WAAmB;QADX,aAAQ,GAAR,QAAQ,CAAQ;QAGxB,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC;IACjC,CAAC;IAED,cAAc,CAAC,OAMd;QACC,MAAM,iBAAiB,GAAG,IAAI,GAAG,EAAyB,CAAC;QAC3D,KAAK,IAAI,CAAC,IAAI,OAAO,CAAC,cAAc,EAAE,CAAC;YACrC,MAAM,GAAG,GAAG,gBAAgB,CAAC,CAAC,CAAC,CAAC;YAChC,iBAAiB,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;QAChC,CAAC;QAED,MAAM,SAAS,GAAG,KAAK,CAAC,UAAU,CAAC,iBAAiB,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,EAAE,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC;QAE3F,KAAK,MAAM,CAAC,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC;YAClC,MAAM,GAAG,GAAG,gBAAgB,CAAC,CAAC,CAAC,CAAC;YAChC,iBAAiB,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YAE9B,SAAS;YACT,MAAM,UAAU,GAAG,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;YAC7C,MAAM,QAAQ,GAAG,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,EAAE,EAAE,UAAU,CAAC,CAAC;YAC3D,IAAI,CAAC,WAAW,IAAI,UAAU,CAAC,MAAM,GAAG,GAAG,CAAC;YAE5C,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;YACpC,IAAI,CAAC,aAAa,GAAG,KAAK,CAAC;YAE3B,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;gBACnB,SAAS,EAAE;oBACT,QAAQ,EAAE;wBACR,GAAG,EAAE;4BACH,CAAC,EAAE,IAAI,CAAC,QAAQ;4BAChB,CAAC,EAAE,CAAC,CAAC,MAAM;4BACX,CAAC,EAAE,KAAK;yBACT;wBACD,EAAE,EAAE,KAAK;wBACT,YAAY,EAAE,OAAO,CAAC,KAAK,CAAC,EAAE;wBAC9B,UAAU,EAAE,OAAO,CAAC,SAAS;wBAC7B,KAAK,EAAE,CAAC,CAAC,KAAK;wBACd,MAAM,EAAE,CAAC,CAAC,EAAE;wBACZ,QAAQ,EAAE,QAAQ;wBAClB,IAAI,EAAE,UAAU;qBACjB;iBACF;aACF,CAAC,CAAC;QACL,CAAC;QAED,KAAK,IAAI,EAAE,IAAI,iBAAiB,CAAC,MAAM,EAAE,EAAE,CAAC;YAC1C,SAAS;YAET,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;YACpC,IAAI,CAAC,aAAa,GAAG,KAAK,CAAC;YAE3B,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;gBACnB,SAAS,EAAE;oBACT,QAAQ,EAAE;wBACR,GAAG,EAAE;4BACH,CAAC,EAAE,IAAI,CAAC,QAAQ;4BAChB,CAAC,EAAE,EAAE,CAAC,MAAM;4BACZ,CAAC,EAAE,KAAK;yBACT;wBACD,EAAE,EAAE,QAAQ;wBACZ,YAAY,EAAE,OAAO,CAAC,KAAK,CAAC,EAAE;wBAC9B,UAAU,EAAE,OAAO,CAAC,SAAS;wBAC7B,KAAK,EAAE,EAAE,CAAC,KAAK;wBACf,MAAM,EAAE,EAAE,CAAC,EAAE;wBACb,QAAQ,EAAE,SAAS;wBACnB,IAAI,EAAE,IAAI;qBACX;iBACF;aACF,CAAC,CAAC;YACH,IAAI,CAAC,WAAW,IAAI,GAAG,CAAC;QAC1B,CAAC;IACH,CAAC;IAED,iBAAiB,CAAC,IAMjB;QACC,yCAAyC;QACzC,qEAAqE;QACrE,0HAA0H;QAC1H,2DAA2D;QAC3D,8GAA8G;QAC9G,+BAA+B;QAC/B,6CAA6C;QAC7C,MAAM,EAAE,WAAW,EAAE,SAAS,EAAE,SAAS,EAAE,GAAG,IAAI,CAAC;QAEnD,MAAM,iBAAiB,GAAG,IAAI,GAAG,EAAuB,CAAC;QACzD,KAAK,IAAI,CAAC,IAAI,IAAI,CAAC,gBAAgB,EAAE,CAAC;YACpC,iBAAiB,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC;QACjD,CAAC;QAED,wBAAwB;QACxB,KAAK,IAAI,MAAM,IAAI,SAAS,EAAE,CAAC;YAC7B,MAAM,SAAS,GAAG,OAAO,CAAC,eAAe,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;YACzD,MAAM,GAAG,GAAG,SAAS,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;YACzC,iBAAiB,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YAE9B,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;YACjC,IAAI,CAAC,aAAa,GAAG,KAAK,CAAC;YAC3B,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC;gBACzB,SAAS,EAAE;oBACT,QAAQ,EAAE;wBACR,GAAG,EAAE,KAAK;wBACV,GAAG,EAAE;4BACH,CAAC,EAAE,IAAI,CAAC,QAAQ;4BAChB,CAAC,EAAE,WAAW,CAAC,EAAE;4BACjB,CAAC,EAAE,SAAS;yBACb;wBACD,MAAM,EAAE,SAAS;wBACjB,iBAAiB,EAAE,MAAM,CAAC,iBAAiB;qBAC5C;iBACF;aACF,CAAC,CAAC;YAEH,IAAI,CAAC,WAAW,IAAI,GAAG,CAAC;QAC1B,CAAC;QAED,kDAAkD;QAClD,KAAK,IAAI,MAAM,IAAI,iBAAiB,CAAC,MAAM,EAAE,EAAE,CAAC;YAC9C,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;YACjC,IAAI,CAAC,aAAa,GAAG,KAAK,CAAC;YAC3B,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC;gBACzB,SAAS,EAAE;oBACT,QAAQ,EAAE;wBACR,GAAG,EAAE,KAAK;wBACV,GAAG,EAAE;4BACH,CAAC,EAAE,IAAI,CAAC,QAAQ;4BAChB,CAAC,EAAE,WAAW,CAAC,EAAE;4BACjB,CAAC,EAAE,SAAS;yBACb;wBACD,MAAM,EAAE,MAAM;wBACd,iBAAiB,EAAE,EAAE;qBACtB;iBACF;aACF,CAAC,CAAC;YAEH,IAAI,CAAC,WAAW,IAAI,GAAG,CAAC;QAC1B,CAAC;IACH,CAAC;IAED,iBAAiB,CAAC,EAAa;QAC7B,MAAM,EAAE,GAAqD;YAC3D,SAAS,EAAE;gBACT,MAAM,EAAE,EAAE,GAAG,EAAE,EAAE,EAAE;aACpB;SACF,CAAC;QACF,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAC1B,IAAI,CAAC,WAAW,IAAI,EAAE,CAAC;IACzB,CAAC;IAED,iBAAiB,CAAC,EAAa,EAAE,MAAoC;QACnE,MAAM,EAAE,GAAqD;YAC3D,SAAS,EAAE;gBACT,MAAM,EAAE,EAAE,GAAG,EAAE,EAAE,EAAE;gBACnB,MAAM,EAAE;oBACN,IAAI,EAAE,MAAM;iBACb;gBACD,MAAM,EAAE,IAAI;aACb;SACF,CAAC;QACF,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAC1B,IAAI,CAAC,WAAW,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,CAAC,GAAG,GAAG,CAAC;IACzD,CAAC;IAED,sBAAsB;QACpB,OAAO,CACL,IAAI,CAAC,WAAW,IAAI,0BAA0B;YAC9C,IAAI,CAAC,UAAU,CAAC,MAAM,IAAI,yBAAyB;YACnD,IAAI,CAAC,WAAW,CAAC,MAAM,IAAI,yBAAyB;YACpD,IAAI,CAAC,gBAAgB,CAAC,MAAM,IAAI,yBAAyB,CAC1D,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,EAAkB,EAAE,OAA4B;QAC1D,IAAI,IAAI,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC/B,MAAM,EAAE,CAAC,WAAW,CAAC,SAAS,CAAC,IAAI,CAAC,UAAU,EAAE;gBAC9C,OAAO;gBACP,sCAAsC;gBACtC,OAAO,EAAE,KAAK;aACf,CAAC,CAAC;QACL,CAAC;QACD,IAAI,IAAI,CAAC,gBAAgB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACrC,MAAM,EAAE,CAAC,iBAAiB,CAAC,SAAS,CAAC,IAAI,CAAC,gBAAgB,EAAE;gBAC1D,OAAO;gBACP,sCAAsC;gBACtC,OAAO,EAAE,KAAK;aACf,CAAC,CAAC;QACL,CAAC;QACD,IAAI,IAAI,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAChC,MAAM,EAAE,CAAC,YAAY,CAAC,SAAS,CAAC,IAAI,CAAC,WAAW,EAAE;gBAChD,OAAO;gBACP,mEAAmE;gBACnE,OAAO,EAAE,IAAI;aACd,CAAC,CAAC;QACL,CAAC;QAED,MAAM,CAAC,IAAI,CACT,aAAa,IAAI,CAAC,QAAQ,YAAY,IAAI,CAAC,UAAU,CAAC,MAAM,MAAM,IAAI,CAAC,gBAAgB,CAAC,MAAM,MAC5F,IAAI,CAAC,WAAW,CAAC,MACnB,aAAa,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,mBAAmB,IAAI,CAAC,aAAa,EAAE,CACxF,CAAC;QAEF,IAAI,CAAC,UAAU,GAAG,EAAE,CAAC;QACrB,IAAI,CAAC,gBAAgB,GAAG,EAAE,CAAC;QAC3B,IAAI,CAAC,WAAW,GAAG,EAAE,CAAC;QACtB,IAAI,CAAC,WAAW,GAAG,CAAC,CAAC;QACrB,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC;IAC5B,CAAC;CACF"}
@@ -33,4 +33,4 @@ export declare class PowerSyncMongo {
33
33
  */
34
34
  drop(): Promise<void>;
35
35
  }
36
- export declare function createPowerSyncMongo(config: MongoStorageConfig): PowerSyncMongo;
36
+ export declare function createPowerSyncMongo(config: MongoStorageConfig, options?: lib_mongo.MongoConnectionOptions): PowerSyncMongo;
@@ -1,10 +1,22 @@
1
1
  import * as lib_mongo from '@powersync/lib-service-mongodb';
2
2
  import { storage } from '@powersync/service-core';
3
3
  export class PowerSyncMongo {
4
+ current_data;
5
+ bucket_data;
6
+ bucket_parameters;
7
+ op_id_sequence;
8
+ sync_rules;
9
+ source_tables;
10
+ custom_write_checkpoints;
11
+ write_checkpoints;
12
+ instance;
13
+ locks;
14
+ client;
15
+ db;
4
16
  constructor(client, options) {
5
17
  this.client = client;
6
18
  const db = client.db(options?.database, {
7
- ...storage.BSON_DESERIALIZE_OPTIONS
19
+ ...storage.BSON_DESERIALIZE_INTERNAL_OPTIONS
8
20
  });
9
21
  this.db = db;
10
22
  this.current_data = db.collection('current_data');
@@ -41,7 +53,7 @@ export class PowerSyncMongo {
41
53
  await this.db.dropDatabase();
42
54
  }
43
55
  }
44
- export function createPowerSyncMongo(config) {
45
- return new PowerSyncMongo(lib_mongo.createMongoClient(config), { database: config.database });
56
+ export function createPowerSyncMongo(config, options) {
57
+ return new PowerSyncMongo(lib_mongo.createMongoClient(config, options), { database: config.database });
46
58
  }
47
59
  //# sourceMappingURL=db.js.map
@@ -1 +1 @@
1
- {"version":3,"file":"db.js","sourceRoot":"","sources":["../../../src/storage/implementation/db.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,SAAS,MAAM,gCAAgC,CAAC;AAE5D,OAAO,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAsBlD,MAAM,OAAO,cAAc;IAezB,YAAY,MAAyB,EAAE,OAA+B;QACpE,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QAErB,MAAM,EAAE,GAAG,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,QAAQ,EAAE;YACtC,GAAG,OAAO,CAAC,wBAAwB;SACpC,CAAC,CAAC;QACH,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC;QAEb,IAAI,CAAC,YAAY,GAAG,EAAE,CAAC,UAAU,CAAsB,cAAc,CAAC,CAAC;QACvE,IAAI,CAAC,WAAW,GAAG,EAAE,CAAC,UAAU,CAAC,aAAa,CAAC,CAAC;QAChD,IAAI,CAAC,iBAAiB,GAAG,EAAE,CAAC,UAAU,CAAC,mBAAmB,CAAC,CAAC;QAC5D,IAAI,CAAC,cAAc,GAAG,EAAE,CAAC,UAAU,CAAC,gBAAgB,CAAC,CAAC;QACtD,IAAI,CAAC,UAAU,GAAG,EAAE,CAAC,UAAU,CAAC,YAAY,CAAC,CAAC;QAC9C,IAAI,CAAC,aAAa,GAAG,EAAE,CAAC,UAAU,CAAC,eAAe,CAAC,CAAC;QACpD,IAAI,CAAC,wBAAwB,GAAG,EAAE,CAAC,UAAU,CAAC,0BAA0B,CAAC,CAAC;QAC1E,IAAI,CAAC,iBAAiB,GAAG,EAAE,CAAC,UAAU,CAAC,mBAAmB,CAAC,CAAC;QAC5D,IAAI,CAAC,QAAQ,GAAG,EAAE,CAAC,UAAU,CAAC,UAAU,CAAC,CAAC;QAC1C,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;IAC3C,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,KAAK;QACT,MAAM,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QACvC,MAAM,IAAI,CAAC,WAAW,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QACtC,MAAM,IAAI,CAAC,iBAAiB,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QAC5C,MAAM,IAAI,CAAC,cAAc,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QACzC,MAAM,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QACrC,MAAM,IAAI,CAAC,aAAa,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QACxC,MAAM,IAAI,CAAC,iBAAiB,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QAC5C,MAAM,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;QAClC,MAAM,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;IAClC,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,IAAI;QACR,MAAM,IAAI,CAAC,EAAE,CAAC,YAAY,EAAE,CAAC;IAC/B,CAAC;CACF;AAED,MAAM,UAAU,oBAAoB,CAAC,MAA0B;IAC7D,OAAO,IAAI,cAAc,CAAC,SAAS,CAAC,iBAAiB,CAAC,MAAM,CAAC,EAAE,EAAE,QAAQ,EAAE,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC;AAChG,CAAC"}
1
+ {"version":3,"file":"db.js","sourceRoot":"","sources":["../../../src/storage/implementation/db.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,SAAS,MAAM,gCAAgC,CAAC;AAE5D,OAAO,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAsBlD,MAAM,OAAO,cAAc;IAChB,YAAY,CAAwC;IACpD,WAAW,CAAuC;IAClD,iBAAiB,CAA4C;IAC7D,cAAc,CAAuC;IACrD,UAAU,CAAqC;IAC/C,aAAa,CAAwC;IACrD,wBAAwB,CAAkD;IAC1E,iBAAiB,CAA4C;IAC7D,QAAQ,CAAqC;IAC7C,KAAK,CAAyC;IAE9C,MAAM,CAAoB;IAC1B,EAAE,CAAW;IAEtB,YAAY,MAAyB,EAAE,OAA+B;QACpE,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QAErB,MAAM,EAAE,GAAG,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,QAAQ,EAAE;YACtC,GAAG,OAAO,CAAC,iCAAiC;SAC7C,CAAC,CAAC;QACH,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC;QAEb,IAAI,CAAC,YAAY,GAAG,EAAE,CAAC,UAAU,CAAsB,cAAc,CAAC,CAAC;QACvE,IAAI,CAAC,WAAW,GAAG,EAAE,CAAC,UAAU,CAAC,aAAa,CAAC,CAAC;QAChD,IAAI,CAAC,iBAAiB,GAAG,EAAE,CAAC,UAAU,CAAC,mBAAmB,CAAC,CAAC;QAC5D,IAAI,CAAC,cAAc,GAAG,EAAE,CAAC,UAAU,CAAC,gBAAgB,CAAC,CAAC;QACtD,IAAI,CAAC,UAAU,GAAG,EAAE,CAAC,UAAU,CAAC,YAAY,CAAC,CAAC;QAC9C,IAAI,CAAC,aAAa,GAAG,EAAE,CAAC,UAAU,CAAC,eAAe,CAAC,CAAC;QACpD,IAAI,CAAC,wBAAwB,GAAG,EAAE,CAAC,UAAU,CAAC,0BAA0B,CAAC,CAAC;QAC1E,IAAI,CAAC,iBAAiB,GAAG,EAAE,CAAC,UAAU,CAAC,mBAAmB,CAAC,CAAC;QAC5D,IAAI,CAAC,QAAQ,GAAG,EAAE,CAAC,UAAU,CAAC,UAAU,CAAC,CAAC;QAC1C,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;IAC3C,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,KAAK;QACT,MAAM,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QACvC,MAAM,IAAI,CAAC,WAAW,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QACtC,MAAM,IAAI,CAAC,iBAAiB,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QAC5C,MAAM,IAAI,CAAC,cAAc,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QACzC,MAAM,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QACrC,MAAM,IAAI,CAAC,aAAa,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QACxC,MAAM,IAAI,CAAC,iBAAiB,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QAC5C,MAAM,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;QAClC,MAAM,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;IAClC,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,IAAI;QACR,MAAM,IAAI,CAAC,EAAE,CAAC,YAAY,EAAE,CAAC;IAC/B,CAAC;CACF;AAED,MAAM,UAAU,oBAAoB,CAAC,MAA0B,EAAE,OAA0C;IACzG,OAAO,IAAI,cAAc,CAAC,SAAS,CAAC,iBAAiB,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,EAAE,QAAQ,EAAE,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC;AACzG,CAAC"}
@@ -122,6 +122,7 @@ export interface SyncRuleDocument {
122
122
  last_fatal_error: string | null;
123
123
  content: string;
124
124
  }
125
+ export type SyncRuleCheckpointState = Pick<SyncRuleDocument, 'last_checkpoint' | 'last_checkpoint_lsn' | '_id' | 'state'>;
125
126
  export interface CustomWriteCheckpointDocument {
126
127
  _id: bson.ObjectId;
127
128
  user_id: string;
@@ -30,17 +30,3 @@ export declare const createMongoClient: (url: string, options?: mongo.MongoClien
30
30
  * Helper for unit tests
31
31
  */
32
32
  export declare const connectMongoForTests: (url: string, isCI: boolean) => PowerSyncMongo;
33
- /**
34
- * MongoDB bulkWrite internally splits the operations into batches
35
- * so that no batch exceeds 16MB. However, there are cases where
36
- * the batch size is very close to 16MB, where additional metadata
37
- * on the server pushes it over the limit, resulting in this error
38
- * from the server:
39
- *
40
- * > MongoBulkWriteError: BSONObj size: 16814023 (0x1008FC7) is invalid. Size must be between 0 and 16793600(16MB) First element: insert: "bucket_data"
41
- *
42
- * We work around the issue by doing our own batching, limiting the
43
- * batch size to 15MB. This does add additional overhead with
44
- * BSON.calculateObjectSize.
45
- */
46
- export declare function safeBulkWrite<T extends mongo.Document>(collection: mongo.Collection<T>, operations: mongo.AnyBulkWriteOperation<T>[], options: mongo.BulkWriteOptions): Promise<void>;
@@ -108,48 +108,10 @@ export const connectMongoForTests = (url, isCI) => {
108
108
  // Short timeout for tests, to fail fast when the server is not available.
109
109
  // Slightly longer timeouts for CI, to avoid arbitrary test failures
110
110
  const client = createMongoClient(url, {
111
- connectTimeoutMS: isCI ? 15000 : 5000,
112
- socketTimeoutMS: isCI ? 15000 : 5000,
113
- serverSelectionTimeoutMS: isCI ? 15000 : 2500
111
+ connectTimeoutMS: isCI ? 15_000 : 5_000,
112
+ socketTimeoutMS: isCI ? 15_000 : 5_000,
113
+ serverSelectionTimeoutMS: isCI ? 15_000 : 2_500
114
114
  });
115
115
  return new PowerSyncMongo(client);
116
116
  };
117
- /**
118
- * MongoDB bulkWrite internally splits the operations into batches
119
- * so that no batch exceeds 16MB. However, there are cases where
120
- * the batch size is very close to 16MB, where additional metadata
121
- * on the server pushes it over the limit, resulting in this error
122
- * from the server:
123
- *
124
- * > MongoBulkWriteError: BSONObj size: 16814023 (0x1008FC7) is invalid. Size must be between 0 and 16793600(16MB) First element: insert: "bucket_data"
125
- *
126
- * We work around the issue by doing our own batching, limiting the
127
- * batch size to 15MB. This does add additional overhead with
128
- * BSON.calculateObjectSize.
129
- */
130
- export async function safeBulkWrite(collection, operations, options) {
131
- // Must be below 16MB.
132
- // We could probably go a little closer, but 15MB is a safe threshold.
133
- const BULK_WRITE_LIMIT = 15 * 1024 * 1024;
134
- let batch = [];
135
- let currentSize = 0;
136
- // Estimated overhead per operation, should be smaller in reality.
137
- const keySize = 8;
138
- for (let op of operations) {
139
- const bsonSize = mongo.BSON.calculateObjectSize(op, {
140
- checkKeys: false,
141
- ignoreUndefined: true
142
- }) + keySize;
143
- if (batch.length > 0 && currentSize + bsonSize > BULK_WRITE_LIMIT) {
144
- await collection.bulkWrite(batch, options);
145
- currentSize = 0;
146
- batch = [];
147
- }
148
- batch.push(op);
149
- currentSize += bsonSize;
150
- }
151
- if (batch.length > 0) {
152
- await collection.bulkWrite(batch, options);
153
- }
154
- }
155
117
  //# sourceMappingURL=util.js.map
@@ -1 +1 @@
1
- {"version":3,"file":"util.js","sourceRoot":"","sources":["../../../src/storage/implementation/util.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAC7B,OAAO,KAAK,MAAM,MAAM,QAAQ,CAAC;AACjC,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAE7B,OAAO,EAAE,KAAK,EAAE,MAAM,gCAAgC,CAAC;AACvD,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,yBAAyB,CAAC;AAEzD,OAAO,EAAE,cAAc,EAAE,MAAM,SAAS,CAAC;AAGzC,MAAM,UAAU,cAAc,CAAI,MAAkB,EAAE,IAAiB;IACrE,IAAI,MAAM,GAAG;QACX,IAAI,EAAE;YACJ,GAAG,MAAM;SACH;QACR,GAAG,EAAE;YACH,GAAG,MAAM;SACH;KACT,CAAC;IAEF,KAAK,IAAI,GAAG,IAAI,IAAI,EAAE,CAAC;QACrB,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;QACrC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;IACtC,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,MAAc,EAAE,aAAqB;IACpE,MAAM,WAAW,GAAG,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;IAC1D,OAAO,GAAG,MAAM,GAAG,aAAa,IAAI,WAAW,EAAE,CAAC;AACpD,CAAC;AAED;;;;;;;;;;GAUG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CAAI,MAA2B;IAClE,IAAI,CAAC;QACH,IAAI,IAAS,CAAC;QACd,IAAI,OAAO,GAAG,IAAI,CAAC;QACnB,2CAA2C;QAC3C,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,OAAO,EAAE,CAAC;QACtC,yCAAyC;QACzC,IAAI,GAAG,MAAM,CAAC,qBAAqB,EAAE,CAAC;QACtC,IAAI,CAAC,MAAM,IAAI,MAAM,CAAC,EAAE,EAAE,MAAM,EAAE,EAAE,CAAC;YACnC,0CAA0C;YAC1C,wEAAwE;YACxE,uEAAuE;YACvE,oCAAoC;YACpC,EAAE;YACF,4EAA4E;YAC5E,2DAA2D;YAC3D,gCAAgC;YAChC,OAAO,GAAG,KAAK,CAAC;QAClB,CAAC;QACD,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC;IAC3B,CAAC;YAAS,CAAC;QACT,iDAAiD;QACjD,uIAAuI;QACvI,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;YACnB,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC;QACvB,CAAC;IACH,CAAC;AACH,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,GAAuB;IAChD,IAAI,GAAG,CAAC,EAAE,IAAI,KAAK,IAAI,GAAG,CAAC,EAAE,IAAI,QAAQ,EAAE,CAAC;QAC1C,OAAO;YACL,KAAK,EAAE,KAAK,CAAC,eAAe,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;YACvC,EAAE,EAAE,GAAG,CAAC,EAAE;YACV,WAAW,EAAE,GAAG,CAAC,KAAK;YACtB,SAAS,EAAE,GAAG,CAAC,MAAM;YACrB,QAAQ,EAAE,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;YAC9B,MAAM,EAAE,iBAAiB,CAAC,GAAG,CAAC,YAAa,EAAE,GAAG,CAAC,UAAW,CAAC;YAC7D,IAAI,EAAE,GAAG,CAAC,IAAI;SACf,CAAC;IACJ,CAAC;SAAM,CAAC;QACN,cAAc;QAEd,OAAO;YACL,KAAK,EAAE,KAAK,CAAC,eAAe,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;YACvC,EAAE,EAAE,GAAG,CAAC,EAAE;YACV,QAAQ,EAAE,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;SAC/B,CAAC;IACJ,CAAC;AACH,CAAC;AAED,MAAM,UAAU,iBAAiB,CAAC,KAAoB,EAAE,EAAqB;IAC3E,IAAI,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,CAAC;QACvB,mDAAmD;QACnD,OAAO,GAAG,KAAK,CAAC,WAAW,EAAE,IAAI,EAAE,CAAC,WAAW,EAAE,EAAE,CAAC;IACtD,CAAC;SAAM,CAAC;QACN,oCAAoC;QACpC,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC,CAAC;QAC3C,OAAO,IAAI,CAAC,EAAE,CAAC,IAAI,EAAE,KAAK,CAAC,YAAY,CAAC,CAAC;IAC3C,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,MAAM,iBAAiB,GAAG,CAAC,GAAW,EAAE,OAAkC,EAAE,EAAE;IACnF,OAAO,IAAI,KAAK,CAAC,WAAW,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;AAC7C,CAAC,CAAC;AAEF;;GAEG;AACH,MAAM,CAAC,MAAM,oBAAoB,GAAG,CAAC,GAAW,EAAE,IAAa,EAAE,EAAE;IACjE,0EAA0E;IAC1E,oEAAoE;IACpE,MAAM,MAAM,GAAG,iBAAiB,CAAC,GAAG,EAAE;QACpC,gBAAgB,EAAE,IAAI,CAAC,CAAC,CAAC,KAAM,CAAC,CAAC,CAAC,IAAK;QACvC,eAAe,EAAE,IAAI,CAAC,CAAC,CAAC,KAAM,CAAC,CAAC,CAAC,IAAK;QACtC,wBAAwB,EAAE,IAAI,CAAC,CAAC,CAAC,KAAM,CAAC,CAAC,CAAC,IAAK;KAChD,CAAC,CAAC;IACH,OAAO,IAAI,cAAc,CAAC,MAAM,CAAC,CAAC;AACpC,CAAC,CAAC;AAEF;;;;;;;;;;;;GAYG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,UAA+B,EAC/B,UAA4C,EAC5C,OAA+B;IAE/B,sBAAsB;IACtB,sEAAsE;IACtE,MAAM,gBAAgB,GAAG,EAAE,GAAG,IAAI,GAAG,IAAI,CAAC;IAE1C,IAAI,KAAK,GAAqC,EAAE,CAAC;IACjD,IAAI,WAAW,GAAG,CAAC,CAAC;IACpB,kEAAkE;IAClE,MAAM,OAAO,GAAG,CAAC,CAAC;IAClB,KAAK,IAAI,EAAE,IAAI,UAAU,EAAE,CAAC;QAC1B,MAAM,QAAQ,GACZ,KAAK,CAAC,IAAI,CAAC,mBAAmB,CAAC,EAAE,EAAE;YACjC,SAAS,EAAE,KAAK;YAChB,eAAe,EAAE,IAAI;SACf,CAAC,GAAG,OAAO,CAAC;QACtB,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,IAAI,WAAW,GAAG,QAAQ,GAAG,gBAAgB,EAAE,CAAC;YAClE,MAAM,UAAU,CAAC,SAAS,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;YAC3C,WAAW,GAAG,CAAC,CAAC;YAChB,KAAK,GAAG,EAAE,CAAC;QACb,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACf,WAAW,IAAI,QAAQ,CAAC;IAC1B,CAAC;IACD,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACrB,MAAM,UAAU,CAAC,SAAS,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;IAC7C,CAAC;AACH,CAAC"}
1
+ {"version":3,"file":"util.js","sourceRoot":"","sources":["../../../src/storage/implementation/util.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAC7B,OAAO,KAAK,MAAM,MAAM,QAAQ,CAAC;AACjC,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAE7B,OAAO,EAAE,KAAK,EAAE,MAAM,gCAAgC,CAAC;AACvD,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,yBAAyB,CAAC;AAEzD,OAAO,EAAE,cAAc,EAAE,MAAM,SAAS,CAAC;AAGzC,MAAM,UAAU,cAAc,CAAI,MAAkB,EAAE,IAAiB;IACrE,IAAI,MAAM,GAAG;QACX,IAAI,EAAE;YACJ,GAAG,MAAM;SACH;QACR,GAAG,EAAE;YACH,GAAG,MAAM;SACH;KACT,CAAC;IAEF,KAAK,IAAI,GAAG,IAAI,IAAI,EAAE,CAAC;QACrB,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;QACrC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;IACtC,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,MAAc,EAAE,aAAqB;IACpE,MAAM,WAAW,GAAG,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;IAC1D,OAAO,GAAG,MAAM,GAAG,aAAa,IAAI,WAAW,EAAE,CAAC;AACpD,CAAC;AAED;;;;;;;;;;GAUG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CAAI,MAA2B;IAClE,IAAI,CAAC;QACH,IAAI,IAAS,CAAC;QACd,IAAI,OAAO,GAAG,IAAI,CAAC;QACnB,2CAA2C;QAC3C,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,OAAO,EAAE,CAAC;QACtC,yCAAyC;QACzC,IAAI,GAAG,MAAM,CAAC,qBAAqB,EAAE,CAAC;QACtC,IAAI,CAAC,MAAM,IAAI,MAAM,CAAC,EAAE,EAAE,MAAM,EAAE,EAAE,CAAC;YACnC,0CAA0C;YAC1C,wEAAwE;YACxE,uEAAuE;YACvE,oCAAoC;YACpC,EAAE;YACF,4EAA4E;YAC5E,2DAA2D;YAC3D,gCAAgC;YAChC,OAAO,GAAG,KAAK,CAAC;QAClB,CAAC;QACD,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC;IAC3B,CAAC;YAAS,CAAC;QACT,iDAAiD;QACjD,uIAAuI;QACvI,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;YACnB,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC;QACvB,CAAC;IACH,CAAC;AACH,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,GAAuB;IAChD,IAAI,GAAG,CAAC,EAAE,IAAI,KAAK,IAAI,GAAG,CAAC,EAAE,IAAI,QAAQ,EAAE,CAAC;QAC1C,OAAO;YACL,KAAK,EAAE,KAAK,CAAC,eAAe,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;YACvC,EAAE,EAAE,GAAG,CAAC,EAAE;YACV,WAAW,EAAE,GAAG,CAAC,KAAK;YACtB,SAAS,EAAE,GAAG,CAAC,MAAM;YACrB,QAAQ,EAAE,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;YAC9B,MAAM,EAAE,iBAAiB,CAAC,GAAG,CAAC,YAAa,EAAE,GAAG,CAAC,UAAW,CAAC;YAC7D,IAAI,EAAE,GAAG,CAAC,IAAI;SACf,CAAC;IACJ,CAAC;SAAM,CAAC;QACN,cAAc;QAEd,OAAO;YACL,KAAK,EAAE,KAAK,CAAC,eAAe,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;YACvC,EAAE,EAAE,GAAG,CAAC,EAAE;YACV,QAAQ,EAAE,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;SAC/B,CAAC;IACJ,CAAC;AACH,CAAC;AAED,MAAM,UAAU,iBAAiB,CAAC,KAAoB,EAAE,EAAqB;IAC3E,IAAI,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,CAAC;QACvB,mDAAmD;QACnD,OAAO,GAAG,KAAK,CAAC,WAAW,EAAE,IAAI,EAAE,CAAC,WAAW,EAAE,EAAE,CAAC;IACtD,CAAC;SAAM,CAAC;QACN,oCAAoC;QACpC,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC,CAAC;QAC3C,OAAO,IAAI,CAAC,EAAE,CAAC,IAAI,EAAE,KAAK,CAAC,YAAY,CAAC,CAAC;IAC3C,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,MAAM,iBAAiB,GAAG,CAAC,GAAW,EAAE,OAAkC,EAAE,EAAE;IACnF,OAAO,IAAI,KAAK,CAAC,WAAW,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;AAC7C,CAAC,CAAC;AAEF;;GAEG;AACH,MAAM,CAAC,MAAM,oBAAoB,GAAG,CAAC,GAAW,EAAE,IAAa,EAAE,EAAE;IACjE,0EAA0E;IAC1E,oEAAoE;IACpE,MAAM,MAAM,GAAG,iBAAiB,CAAC,GAAG,EAAE;QACpC,gBAAgB,EAAE,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK;QACvC,eAAe,EAAE,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK;QACtC,wBAAwB,EAAE,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK;KAChD,CAAC,CAAC;IACH,OAAO,IAAI,cAAc,CAAC,MAAM,CAAC,CAAC;AACpC,CAAC,CAAC"}
package/package.json CHANGED
@@ -2,7 +2,7 @@
2
2
  "name": "@powersync/service-module-mongodb-storage",
3
3
  "repository": "https://github.com/powersync-ja/powersync-service",
4
4
  "types": "dist/index.d.ts",
5
- "version": "0.0.0-dev-20250122110924",
5
+ "version": "0.0.0-dev-20250227082606",
6
6
  "main": "dist/index.js",
7
7
  "license": "FSL-1.1-Apache-2.0",
8
8
  "type": "module",
@@ -22,21 +22,21 @@
22
22
  }
23
23
  },
24
24
  "dependencies": {
25
- "bson": "^6.8.0",
25
+ "bson": "^6.10.3",
26
26
  "ts-codec": "^1.3.0",
27
27
  "ix": "^5.0.0",
28
28
  "lru-cache": "^10.2.2",
29
29
  "uuid": "^9.0.1",
30
- "@powersync/lib-services-framework": "0.0.0-dev-20250122110924",
31
- "@powersync/service-core": "0.0.0-dev-20250122110924",
30
+ "@powersync/lib-services-framework": "0.5.3",
31
+ "@powersync/service-core": "0.0.0-dev-20250227082606",
32
32
  "@powersync/service-jsonbig": "0.17.10",
33
- "@powersync/service-sync-rules": "0.0.0-dev-20250122110924",
34
- "@powersync/service-types": "0.0.0-dev-20250122110924",
35
- "@powersync/lib-service-mongodb": "0.0.0-dev-20250122110924"
33
+ "@powersync/service-sync-rules": "0.24.0",
34
+ "@powersync/service-types": "0.0.0-dev-20250227082606",
35
+ "@powersync/lib-service-mongodb": "0.0.0-dev-20250227082606"
36
36
  },
37
37
  "devDependencies": {
38
38
  "@types/uuid": "^9.0.4",
39
- "@powersync/service-core-tests": "0.0.0-dev-20250122110924"
39
+ "@powersync/service-core-tests": "0.0.0-dev-20250227082606"
40
40
  },
41
41
  "scripts": {
42
42
  "build": "tsc -b",
@@ -1,11 +1,8 @@
1
1
  import { SqlSyncRules } from '@powersync/service-sync-rules';
2
- import { wrapWithAbort } from 'ix/asynciterable/operators/withabort.js';
3
- import { LRUCache } from 'lru-cache/min';
4
- import * as timers from 'timers/promises';
5
2
 
6
- import { storage, sync, utils } from '@powersync/service-core';
3
+ import { GetIntanceOptions, storage } from '@powersync/service-core';
7
4
 
8
- import { DisposableObserver, ErrorCode, logger, ServiceError } from '@powersync/lib-services-framework';
5
+ import { BaseObserver, ErrorCode, logger, ServiceError } from '@powersync/lib-services-framework';
9
6
  import { v4 as uuid } from 'uuid';
10
7
 
11
8
  import * as lib_mongo from '@powersync/lib-service-mongodb';
@@ -18,7 +15,7 @@ import { MongoSyncBucketStorage } from './implementation/MongoSyncBucketStorage.
18
15
  import { generateSlotName } from './implementation/util.js';
19
16
 
20
17
  export class MongoBucketStorage
21
- extends DisposableObserver<storage.BucketStorageFactoryListener>
18
+ extends BaseObserver<storage.BucketStorageFactoryListener>
22
19
  implements storage.BucketStorageFactory
23
20
  {
24
21
  private readonly client: mongo.MongoClient;
@@ -26,26 +23,7 @@ export class MongoBucketStorage
26
23
  // TODO: This is still Postgres specific and needs to be reworked
27
24
  public readonly slot_name_prefix: string;
28
25
 
29
- private readonly storageCache = new LRUCache<number, MongoSyncBucketStorage>({
30
- max: 3,
31
- fetchMethod: async (id) => {
32
- const doc2 = await this.db.sync_rules.findOne(
33
- {
34
- _id: id
35
- },
36
- { limit: 1 }
37
- );
38
- if (doc2 == null) {
39
- // Deleted in the meantime?
40
- return undefined;
41
- }
42
- const rules = new MongoPersistedSyncRulesContent(this.db, doc2);
43
- return this.getInstance(rules);
44
- },
45
- dispose: (storage) => {
46
- storage[Symbol.dispose]();
47
- }
48
- });
26
+ private activeStorageCache: MongoSyncBucketStorage | undefined;
49
27
 
50
28
  public readonly db: PowerSyncMongo;
51
29
 
@@ -63,20 +41,21 @@ export class MongoBucketStorage
63
41
  }
64
42
 
65
43
  async [Symbol.asyncDispose]() {
66
- super[Symbol.dispose]();
44
+ // No-op
67
45
  }
68
46
 
69
- getInstance(options: storage.PersistedSyncRulesContent): MongoSyncBucketStorage {
70
- let { id, slot_name } = options;
47
+ getInstance(syncRules: storage.PersistedSyncRulesContent, options?: GetIntanceOptions): MongoSyncBucketStorage {
48
+ let { id, slot_name } = syncRules;
71
49
  if ((typeof id as any) == 'bigint') {
72
50
  id = Number(id);
73
51
  }
74
- const storage = new MongoSyncBucketStorage(this, id, options, slot_name);
75
- this.iterateListeners((cb) => cb.syncStorageCreated?.(storage));
52
+ const storage = new MongoSyncBucketStorage(this, id, syncRules, slot_name);
53
+ if (!options?.skipLifecycleHooks) {
54
+ this.iterateListeners((cb) => cb.syncStorageCreated?.(storage));
55
+ }
76
56
  storage.registerListener({
77
57
  batchStarted: (batch) => {
78
- // This nested listener will be automatically disposed when the storage is disposed
79
- batch.registerManagedListener(storage, {
58
+ batch.registerListener({
80
59
  replicationEvent: (payload) => this.iterateListeners((cb) => cb.replicationEvent?.(payload))
81
60
  });
82
61
  }
@@ -84,36 +63,49 @@ export class MongoBucketStorage
84
63
  return storage;
85
64
  }
86
65
 
87
- async configureSyncRules(sync_rules: string, options?: { lock?: boolean }) {
66
+ async getSystemIdentifier(): Promise<storage.BucketStorageSystemIdentifier> {
67
+ const { setName: id } = await this.db.db.command({
68
+ hello: 1
69
+ });
70
+ if (id == null) {
71
+ throw new ServiceError(
72
+ ErrorCode.PSYNC_S1342,
73
+ 'Standalone MongoDB instances are not supported - use a replicaset.'
74
+ );
75
+ }
76
+
77
+ return {
78
+ id,
79
+ type: lib_mongo.MONGO_CONNECTION_TYPE
80
+ };
81
+ }
82
+
83
+ async configureSyncRules(options: storage.UpdateSyncRulesOptions) {
88
84
  const next = await this.getNextSyncRulesContent();
89
85
  const active = await this.getActiveSyncRulesContent();
90
86
 
91
- if (next?.sync_rules_content == sync_rules) {
87
+ if (next?.sync_rules_content == options.content) {
92
88
  logger.info('Sync rules from configuration unchanged');
93
89
  return { updated: false };
94
- } else if (next == null && active?.sync_rules_content == sync_rules) {
90
+ } else if (next == null && active?.sync_rules_content == options.content) {
95
91
  logger.info('Sync rules from configuration unchanged');
96
92
  return { updated: false };
97
93
  } else {
98
94
  logger.info('Sync rules updated from configuration');
99
- const persisted_sync_rules = await this.updateSyncRules({
100
- content: sync_rules,
101
- lock: options?.lock
102
- });
95
+ const persisted_sync_rules = await this.updateSyncRules(options);
103
96
  return { updated: true, persisted_sync_rules, lock: persisted_sync_rules.current_lock ?? undefined };
104
97
  }
105
98
  }
106
99
 
107
- async slotRemoved(slot_name: string) {
100
+ async restartReplication(sync_rules_group_id: number) {
108
101
  const next = await this.getNextSyncRulesContent();
109
102
  const active = await this.getActiveSyncRulesContent();
110
103
 
111
- // In both the below cases, we create a new sync rules instance.
112
- // The current one will continue erroring until the next one has finished processing.
113
- if (next != null && next.slot_name == slot_name) {
104
+ if (next != null && next.id == sync_rules_group_id) {
114
105
  // We need to redo the "next" sync rules
115
106
  await this.updateSyncRules({
116
- content: next.sync_rules_content
107
+ content: next.sync_rules_content,
108
+ validate: false
117
109
  });
118
110
  // Pro-actively stop replicating
119
111
  await this.db.sync_rules.updateOne(
@@ -127,13 +119,17 @@ export class MongoBucketStorage
127
119
  }
128
120
  }
129
121
  );
130
- } else if (next == null && active?.slot_name == slot_name) {
122
+ } else if (next == null && active?.id == sync_rules_group_id) {
131
123
  // Slot removed for "active" sync rules, while there is no "next" one.
132
124
  await this.updateSyncRules({
133
- content: active.sync_rules_content
125
+ content: active.sync_rules_content,
126
+ validate: false
134
127
  });
135
128
 
136
- // Pro-actively stop replicating
129
+ // In this case we keep the old one as active for clients, so that that existing clients
130
+ // can still get the latest data while we replicate the new ones.
131
+ // It will however not replicate anymore.
132
+
137
133
  await this.db.sync_rules.updateOne(
138
134
  {
139
135
  _id: active.id,
@@ -141,7 +137,21 @@ export class MongoBucketStorage
141
137
  },
142
138
  {
143
139
  $set: {
144
- state: storage.SyncRuleState.STOP
140
+ state: storage.SyncRuleState.ERRORED
141
+ }
142
+ }
143
+ );
144
+ } else if (next != null && active?.id == sync_rules_group_id) {
145
+ // Already have next sync rules, but need to stop replicating the active one.
146
+
147
+ await this.db.sync_rules.updateOne(
148
+ {
149
+ _id: active.id,
150
+ state: storage.SyncRuleState.ACTIVE
151
+ },
152
+ {
153
+ $set: {
154
+ state: storage.SyncRuleState.ERRORED
145
155
  }
146
156
  }
147
157
  );
@@ -149,13 +159,18 @@ export class MongoBucketStorage
149
159
  }
150
160
 
151
161
  async updateSyncRules(options: storage.UpdateSyncRulesOptions): Promise<MongoPersistedSyncRulesContent> {
152
- // Parse and validate before applying any changes
153
- const parsed = SqlSyncRules.fromYaml(options.content, {
154
- // No schema-based validation at this point
155
- schema: undefined,
156
- defaultSchema: 'not_applicable', // Not needed for validation
157
- throwOnError: true
158
- });
162
+ if (options.validate) {
163
+ // Parse and validate before applying any changes
164
+ SqlSyncRules.fromYaml(options.content, {
165
+ // No schema-based validation at this point
166
+ schema: undefined,
167
+ defaultSchema: 'not_applicable', // Not needed for validation
168
+ throwOnError: true
169
+ });
170
+ } else {
171
+ // We do not validate sync rules at this point.
172
+ // That is done when using the sync rules, so that the diagnostics API can report the errors.
173
+ }
159
174
 
160
175
  let rules: MongoPersistedSyncRulesContent | undefined = undefined;
161
176
 
@@ -213,7 +228,7 @@ export class MongoBucketStorage
213
228
  async getActiveSyncRulesContent(): Promise<MongoPersistedSyncRulesContent | null> {
214
229
  const doc = await this.db.sync_rules.findOne(
215
230
  {
216
- state: storage.SyncRuleState.ACTIVE
231
+ state: { $in: [storage.SyncRuleState.ACTIVE, storage.SyncRuleState.ERRORED] }
217
232
  },
218
233
  { sort: { _id: -1 }, limit: 1 }
219
234
  );
@@ -251,7 +266,7 @@ export class MongoBucketStorage
251
266
  async getReplicatingSyncRules(): Promise<storage.PersistedSyncRulesContent[]> {
252
267
  const docs = await this.db.sync_rules
253
268
  .find({
254
- $or: [{ state: storage.SyncRuleState.ACTIVE }, { state: storage.SyncRuleState.PROCESSING }]
269
+ state: { $in: [storage.SyncRuleState.PROCESSING, storage.SyncRuleState.ACTIVE] }
255
270
  })
256
271
  .toArray();
257
272
 
@@ -272,19 +287,22 @@ export class MongoBucketStorage
272
287
  });
273
288
  }
274
289
 
275
- async getActiveCheckpoint(): Promise<storage.ActiveCheckpoint> {
276
- const doc = await this.db.sync_rules.findOne(
277
- {
278
- state: storage.SyncRuleState.ACTIVE
279
- },
280
- {
281
- sort: { _id: -1 },
282
- limit: 1,
283
- projection: { _id: 1, last_checkpoint: 1, last_checkpoint_lsn: 1 }
284
- }
285
- );
290
+ async getActiveStorage(): Promise<MongoSyncBucketStorage | null> {
291
+ const content = await this.getActiveSyncRulesContent();
292
+ if (content == null) {
293
+ return null;
294
+ }
286
295
 
287
- return this.makeActiveCheckpoint(doc);
296
+ // It is important that this instance is cached.
297
+ // Not for the instance construction itself, but to ensure that internal caches on the instance
298
+ // are re-used properly.
299
+ if (this.activeStorageCache?.group_id == content.id) {
300
+ return this.activeStorageCache;
301
+ } else {
302
+ const instance = this.getInstance(content);
303
+ this.activeStorageCache = instance;
304
+ return instance;
305
+ }
288
306
  }
289
307
 
290
308
  async getStorageMetrics(): Promise<storage.StorageMetrics> {
@@ -370,166 +388,4 @@ export class MongoBucketStorage
370
388
 
371
389
  return instance!._id;
372
390
  }
373
-
374
- private makeActiveCheckpoint(doc: SyncRuleDocument | null) {
375
- return {
376
- checkpoint: utils.timestampToOpId(doc?.last_checkpoint ?? 0n),
377
- lsn: doc?.last_checkpoint_lsn ?? null,
378
- hasSyncRules() {
379
- return doc != null;
380
- },
381
- getBucketStorage: async () => {
382
- if (doc == null) {
383
- return null;
384
- }
385
- return (await this.storageCache.fetch(doc._id)) ?? null;
386
- }
387
- } satisfies storage.ActiveCheckpoint;
388
- }
389
-
390
- /**
391
- * Instance-wide watch on the latest available checkpoint (op_id + lsn).
392
- */
393
- private async *watchActiveCheckpoint(signal: AbortSignal): AsyncIterable<storage.ActiveCheckpoint> {
394
- const pipeline: mongo.Document[] = [
395
- {
396
- $match: {
397
- 'fullDocument.state': 'ACTIVE',
398
- operationType: { $in: ['insert', 'update'] }
399
- }
400
- },
401
- {
402
- $project: {
403
- operationType: 1,
404
- 'fullDocument._id': 1,
405
- 'fullDocument.last_checkpoint': 1,
406
- 'fullDocument.last_checkpoint_lsn': 1
407
- }
408
- }
409
- ];
410
-
411
- // Use this form instead of (doc: SyncRuleDocument | null = null),
412
- // otherwise we get weird "doc: never" issues.
413
- let doc = null as SyncRuleDocument | null;
414
- let clusterTime = null as mongo.Timestamp | null;
415
-
416
- await this.client.withSession(async (session) => {
417
- doc = await this.db.sync_rules.findOne(
418
- {
419
- state: storage.SyncRuleState.ACTIVE
420
- },
421
- {
422
- session,
423
- sort: { _id: -1 },
424
- limit: 1,
425
- projection: {
426
- _id: 1,
427
- last_checkpoint: 1,
428
- last_checkpoint_lsn: 1
429
- }
430
- }
431
- );
432
- const time = session.clusterTime?.clusterTime ?? null;
433
- clusterTime = time;
434
- });
435
- if (clusterTime == null) {
436
- throw new ServiceError(ErrorCode.PSYNC_S2401, 'Could not get clusterTime');
437
- }
438
-
439
- if (signal.aborted) {
440
- return;
441
- }
442
-
443
- if (doc) {
444
- yield this.makeActiveCheckpoint(doc);
445
- }
446
-
447
- const stream = this.db.sync_rules.watch(pipeline, {
448
- fullDocument: 'updateLookup',
449
- // Start at the cluster time where we got the initial doc, to make sure
450
- // we don't skip any updates.
451
- // This may result in the first operation being a duplicate, but we filter
452
- // it out anyway.
453
- startAtOperationTime: clusterTime
454
- });
455
-
456
- signal.addEventListener(
457
- 'abort',
458
- () => {
459
- stream.close();
460
- },
461
- { once: true }
462
- );
463
-
464
- let lastOp: storage.ActiveCheckpoint | null = null;
465
-
466
- for await (const update of stream.stream()) {
467
- if (signal.aborted) {
468
- break;
469
- }
470
- if (update.operationType != 'insert' && update.operationType != 'update') {
471
- continue;
472
- }
473
- const doc = update.fullDocument!;
474
- if (doc == null) {
475
- continue;
476
- }
477
-
478
- const op = this.makeActiveCheckpoint(doc);
479
- // Check for LSN / checkpoint changes - ignore other metadata changes
480
- if (lastOp == null || op.lsn != lastOp.lsn || op.checkpoint != lastOp.checkpoint) {
481
- lastOp = op;
482
- yield op;
483
- }
484
- }
485
- }
486
-
487
- // Nothing is done here until a subscriber starts to iterate
488
- private readonly sharedIter = new sync.BroadcastIterable((signal) => {
489
- return this.watchActiveCheckpoint(signal);
490
- });
491
-
492
- /**
493
- * User-specific watch on the latest checkpoint and/or write checkpoint.
494
- */
495
- async *watchWriteCheckpoint(user_id: string, signal: AbortSignal): AsyncIterable<storage.WriteCheckpoint> {
496
- let lastCheckpoint: utils.OpId | null = null;
497
- let lastWriteCheckpoint: bigint | null = null;
498
-
499
- const iter = wrapWithAbort(this.sharedIter, signal);
500
- for await (const cp of iter) {
501
- const { checkpoint, lsn } = cp;
502
-
503
- // lsn changes are not important by itself.
504
- // What is important is:
505
- // 1. checkpoint (op_id) changes.
506
- // 2. write checkpoint changes for the specific user
507
- const bucketStorage = await cp.getBucketStorage();
508
- if (!bucketStorage) {
509
- continue;
510
- }
511
-
512
- const lsnFilters: Record<string, string> = lsn ? { 1: lsn } : {};
513
-
514
- const currentWriteCheckpoint = await bucketStorage.lastWriteCheckpoint({
515
- user_id,
516
- heads: {
517
- ...lsnFilters
518
- }
519
- });
520
-
521
- if (currentWriteCheckpoint == lastWriteCheckpoint && checkpoint == lastCheckpoint) {
522
- // No change - wait for next one
523
- // In some cases, many LSNs may be produced in a short time.
524
- // Add a delay to throttle the write checkpoint lookup a bit.
525
- await timers.setTimeout(20 + 10 * Math.random());
526
- continue;
527
- }
528
-
529
- lastWriteCheckpoint = currentWriteCheckpoint;
530
- lastCheckpoint = checkpoint;
531
-
532
- yield { base: cp, writeCheckpoint: currentWriteCheckpoint };
533
- }
534
- }
535
391
  }