@powersync/service-core 0.0.0-dev-20241016113053 → 0.0.0-dev-20241016143203

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (37) hide show
  1. package/CHANGELOG.md +16 -8
  2. package/dist/db/mongo.d.ts +6 -0
  3. package/dist/db/mongo.js +6 -0
  4. package/dist/db/mongo.js.map +1 -1
  5. package/dist/storage/BucketStorage.d.ts +0 -6
  6. package/dist/storage/BucketStorage.js.map +1 -1
  7. package/dist/storage/MongoBucketStorage.d.ts +1 -9
  8. package/dist/storage/MongoBucketStorage.js +32 -30
  9. package/dist/storage/MongoBucketStorage.js.map +1 -1
  10. package/dist/storage/mongo/MongoSyncBucketStorage.d.ts +1 -1
  11. package/dist/storage/mongo/MongoSyncBucketStorage.js +26 -17
  12. package/dist/storage/mongo/MongoSyncBucketStorage.js.map +1 -1
  13. package/dist/storage/mongo/MongoWriteCheckpointAPI.d.ts +1 -3
  14. package/dist/storage/mongo/MongoWriteCheckpointAPI.js +4 -12
  15. package/dist/storage/mongo/MongoWriteCheckpointAPI.js.map +1 -1
  16. package/dist/storage/mongo/db.d.ts +9 -0
  17. package/dist/storage/mongo/db.js +11 -0
  18. package/dist/storage/mongo/db.js.map +1 -1
  19. package/dist/storage/write-checkpoint.d.ts +0 -1
  20. package/dist/storage/write-checkpoint.js.map +1 -1
  21. package/dist/util/config/compound-config-collector.js +1 -2
  22. package/dist/util/config/compound-config-collector.js.map +1 -1
  23. package/dist/util/config/types.d.ts +0 -1
  24. package/package.json +5 -5
  25. package/src/db/mongo.ts +7 -0
  26. package/src/storage/BucketStorage.ts +0 -9
  27. package/src/storage/MongoBucketStorage.ts +32 -37
  28. package/src/storage/mongo/MongoSyncBucketStorage.ts +26 -21
  29. package/src/storage/mongo/MongoWriteCheckpointAPI.ts +8 -15
  30. package/src/storage/mongo/db.ts +12 -0
  31. package/src/storage/write-checkpoint.ts +0 -2
  32. package/src/util/config/compound-config-collector.ts +1 -2
  33. package/src/util/config/types.ts +0 -1
  34. package/test/src/data_storage.test.ts +22 -0
  35. package/test/src/sync.test.ts +2 -16
  36. package/test/src/util.ts +14 -3
  37. package/tsconfig.tsbuildinfo +1 -1
@@ -75,8 +75,7 @@ export class CompoundConfigCollector {
75
75
  },
76
76
  // TODO maybe move this out of the connection or something
77
77
  // slot_name_prefix: connections[0]?.slot_name_prefix ?? 'powersync_'
78
- slot_name_prefix: 'powersync_',
79
- parameters: baseConfig.parameters ?? {}
78
+ slot_name_prefix: 'powersync_'
80
79
  };
81
80
  return config;
82
81
  }
@@ -1 +1 @@
1
- {"version":3,"file":"compound-config-collector.js","sourceRoot":"","sources":["../../../src/util/config/compound-config-collector.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,mCAAmC,CAAC;AAE3D,OAAO,KAAK,IAAI,MAAM,0BAA0B,CAAC;AAEjD,OAAO,EAAE,qBAAqB,EAAE,MAAM,8CAA8C,CAAC;AACrF,OAAO,EAAE,uBAAuB,EAAE,MAAM,gDAAgD,CAAC;AACzF,OAAO,EAAE,yBAAyB,EAAE,MAAM,kDAAkD,CAAC;AAC7F,OAAO,EAAE,wBAAwB,EAAE,MAAM,kDAAkD,CAAC;AAC5F,OAAO,EAAE,4BAA4B,EAAE,MAAM,sDAAsD,CAAC;AACpG,OAAO,EAAE,wBAAwB,EAAE,MAAM,kDAAkD,CAAC;AA4B5F,MAAM,iBAAiB,GAAG,eAAe,CAAC;AAE1C,MAAM,yBAAyB,GAAmC;IAChE,gBAAgB,EAAE,CAAC,IAAI,qBAAqB,EAAE,EAAE,IAAI,yBAAyB,EAAE,EAAE,IAAI,uBAAuB,EAAE,CAAC;IAC/G,mBAAmB,EAAE;QACnB,IAAI,wBAAwB,EAAE;QAC9B,IAAI,4BAA4B,EAAE;QAClC,IAAI,wBAAwB,EAAE;KAC/B;CACF,CAAC;AAEF,MAAM,OAAO,uBAAuB;IAClC,YAAsB,UAA0C,yBAAyB;QAAnE,YAAO,GAAP,OAAO,CAA4D;IAAG,CAAC;IAE7F;;OAEG;IACH,KAAK,CAAC,aAAa,CAAC,eAA6B,EAAE;QACjD,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,YAAY,CAAC,CAAC;QAE9D,MAAM,WAAW,GAAG,UAAU,CAAC,WAAW,EAAE,WAAW,IAAI,EAAE,CAAC;QAC9D,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC3B,MAAM,IAAI,KAAK,CAAC,8DAA8D,CAAC,CAAC;QAClF,CAAC;QAED,MAAM,UAAU,GAAG,IAAI,IAAI,CAAC,oBAAoB,EAAE,CAAC;QACnD,MAAM,QAAQ,GAAG,IAAI,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;QAE/C,MAAM,SAAS,GAAG,UAAU,CAAC,WAAW,EAAE,IAAI,EAAE,IAAI,IAAI,EAAE,CAAC;QAC3D,MAAM,eAAe,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC;QAE5E,UAAU,CAAC,GAAG,CAAC,eAAe,CAAC,CAAC;QAEhC,IAAI,SAAS,GAAG,UAAU,CAAC,WAAW,EAAE,QAAQ,IAAI,EAAE,CAAC;QACvD,IAAI,OAAO,SAAS,IAAI,QAAQ,EAAE,CAAC;YACjC,SAAS,GAAG,CAAC,SAAS,CAAC,CAAC;QAC1B,CAAC;QAED,KAAK,IAAI,GAAG,IAAI,SAAS,EAAE,CAAC;YAC1B,UAAU,CAAC,GAAG,CACZ,IAAI,IAAI,CAAC,kBAAkB,CACzB,IAAI,IAAI,CAAC,mBAAmB,CAAC,GAAG,EAAE,EAAE,cAAc,EAAE,CAAC,CAAC,UAAU,CAAC,WAAW,EAAE,gBAAgB,EAAE,CAAC,CAClG,CACF,CAAC;QACJ,CAAC;QAED,MAAM,UAAU,GAAG,CAAC,UAAU,CAAC,WAAW,EAAE,IAAI,EAAE,IAAI,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,GAAG,IAAI,iBAAiB,CAAC,CAAC;QAE1G,IAAI,MAAgC,CAAC;QACrC,IAAI,UAAU,CAAC,GAAG,EAAE,SAAS,IAAI,UAAU,IAAI,IAAI,IAAI,UAAU,CAAC,GAAG,IAAI,KAAK,EAAE,CAAC;YAC/E,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC;QACpD,CAAC;QAED,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,UAAU,EAAE,YAAY,CAAC,CAAC;QAEzE,IAAI,aAAa,GAAa,UAAU,CAAC,WAAW,EAAE,QAAQ,IAAI,EAAE,CAAC;QAErE,IAAI,MAAM,GAA4B;YACpC,WAAW,EAAE,UAAU;YACvB,WAAW,EAAE,UAAU,CAAC,WAAW,EAAE,WAAW,IAAI,EAAE;YACtD,OAAO,EAAE,UAAU,CAAC,OAAO;YAC3B,eAAe,EAAE,QAAQ;YACzB,+DAA+D;YAC/D,mEAAmE;YACnE,mBAAmB,EAAE,IAAI,IAAI,CAAC,QAAQ,CAAC,eAAe,CAAC;YACvD,UAAU,EAAE,UAAU,CAAC,GAAG,EAAE,MAAM,IAAI,EAAE;YACxC,GAAG,EAAE;gBACH,SAAS,EAAE,UAAU,CAAC,GAAG,EAAE,SAAS,IAAI,KAAK;gBAC7C,WAAW,EAAE,UAAU,CAAC,GAAG,EAAE,WAAW,IAAI,KAAK;gBACjD,aAAa,EAAE,UAAU,CAAC,GAAG,EAAE,aAAa;gBAC5C,QAAQ,EAAE,UAAU,CAAC,GAAG,EAAE,QAAQ,IAAI,KAAK;gBAC3C,OAAO,EAAE,MAAM;aAChB;YACD,IAAI,EAAE,UAAU,CAAC,IAAI,IAAI,IAAI;YAC7B,UAAU;YACV,aAAa;YAEb,oBAAoB,EAAE,IAAI,EAAE,QAAQ;YACpC,QAAQ,EAAE,UAAU,CAAC,QAAQ,IAAI,EAAE;YACnC,UAAU,EAAE,UAAU,CAAC,UAAU;YACjC,SAAS,EAAE;gBACT,yBAAyB,EAAE,UAAU,CAAC,SAAS,EAAE,yBAAyB,IAAI,KAAK;gBACnF,yBAAyB,EACvB,UAAU,CAAC,SAAS,EAAE,yBAAyB,IAAI,0CAA0C;aAChG;YACD,0DAA0D;YAC1D,qEAAqE;YACrE,gBAAgB,EAAE,YAAY;YAC9B,UAAU,EAAE,UAAU,CAAC,UAAU,IAAI,EAAE;SACxC,CAAC;QAEF,OAAO,MAAM,CAAC;IAChB,CAAC;IAED;;;OAGG;IACO,KAAK,CAAC,iBAAiB,CAAC,aAA2B;QAC3D,KAAK,MAAM,SAAS,IAAI,IAAI,CAAC,OAAO,CAAC,gBAAgB,EAAE,CAAC;YACtD,IAAI,CAAC;gBACH,MAAM,UAAU,GAAG,MAAM,SAAS,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC;gBAC1D,IAAI,UAAU,EAAE,CAAC;oBACf,OAAO,UAAU,CAAC;gBACpB,CAAC;gBACD,MAAM,CAAC,KAAK,CACV,2CAA2C,SAAS,CAAC,IAAI,iDAAiD,CAC3G,CAAC;YACJ,CAAC;YAAC,OAAO,EAAE,EAAE,CAAC;gBACZ,yCAAyC;gBACzC,MAAM,IAAI,KAAK,CAAC,kCAAkC,SAAS,CAAC,IAAI,8BAA8B,EAAE,EAAE,CAAC,CAAC;YACtG,CAAC;QACH,CAAC;QACD,MAAM,IAAI,KAAK,CAAC,uFAAuF,CAAC,CAAC;IAC3G,CAAC;IAES,KAAK,CAAC,gBAAgB,CAC9B,UAAsC,EACtC,YAA0B;QAE1B,KAAK,MAAM,SAAS,IAAI,IAAI,CAAC,OAAO,CAAC,mBAAmB,EAAE,CAAC;YACzD,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,OAAO,CAAC,UAAU,EAAE,YAAY,CAAC,CAAC;gBACjE,IAAI,MAAM,EAAE,CAAC;oBACX,OAAO,MAAM,CAAC;gBAChB,CAAC;gBACD,MAAM,CAAC,KAAK,CACV,qCAAqC,SAAS,CAAC,IAAI,iDAAiD,CACrG,CAAC;YACJ,CAAC;YAAC,OAAO,EAAE,EAAE,CAAC;gBACZ,yCAAyC;gBACzC,MAAM,IAAI,KAAK,CAAC,sCAAsC,SAAS,CAAC,IAAI,8BAA8B,EAAE,EAAE,CAAC,CAAC;YAC1G,CAAC;QACH,CAAC;QACD,OAAO;YACL,OAAO,EAAE,KAAK;SACf,CAAC;IACJ,CAAC;CACF"}
1
+ {"version":3,"file":"compound-config-collector.js","sourceRoot":"","sources":["../../../src/util/config/compound-config-collector.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,mCAAmC,CAAC;AAE3D,OAAO,KAAK,IAAI,MAAM,0BAA0B,CAAC;AAEjD,OAAO,EAAE,qBAAqB,EAAE,MAAM,8CAA8C,CAAC;AACrF,OAAO,EAAE,uBAAuB,EAAE,MAAM,gDAAgD,CAAC;AACzF,OAAO,EAAE,yBAAyB,EAAE,MAAM,kDAAkD,CAAC;AAC7F,OAAO,EAAE,wBAAwB,EAAE,MAAM,kDAAkD,CAAC;AAC5F,OAAO,EAAE,4BAA4B,EAAE,MAAM,sDAAsD,CAAC;AACpG,OAAO,EAAE,wBAAwB,EAAE,MAAM,kDAAkD,CAAC;AA4B5F,MAAM,iBAAiB,GAAG,eAAe,CAAC;AAE1C,MAAM,yBAAyB,GAAmC;IAChE,gBAAgB,EAAE,CAAC,IAAI,qBAAqB,EAAE,EAAE,IAAI,yBAAyB,EAAE,EAAE,IAAI,uBAAuB,EAAE,CAAC;IAC/G,mBAAmB,EAAE;QACnB,IAAI,wBAAwB,EAAE;QAC9B,IAAI,4BAA4B,EAAE;QAClC,IAAI,wBAAwB,EAAE;KAC/B;CACF,CAAC;AAEF,MAAM,OAAO,uBAAuB;IAClC,YAAsB,UAA0C,yBAAyB;QAAnE,YAAO,GAAP,OAAO,CAA4D;IAAG,CAAC;IAE7F;;OAEG;IACH,KAAK,CAAC,aAAa,CAAC,eAA6B,EAAE;QACjD,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,YAAY,CAAC,CAAC;QAE9D,MAAM,WAAW,GAAG,UAAU,CAAC,WAAW,EAAE,WAAW,IAAI,EAAE,CAAC;QAC9D,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC3B,MAAM,IAAI,KAAK,CAAC,8DAA8D,CAAC,CAAC;QAClF,CAAC;QAED,MAAM,UAAU,GAAG,IAAI,IAAI,CAAC,oBAAoB,EAAE,CAAC;QACnD,MAAM,QAAQ,GAAG,IAAI,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;QAE/C,MAAM,SAAS,GAAG,UAAU,CAAC,WAAW,EAAE,IAAI,EAAE,IAAI,IAAI,EAAE,CAAC;QAC3D,MAAM,eAAe,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC;QAE5E,UAAU,CAAC,GAAG,CAAC,eAAe,CAAC,CAAC;QAEhC,IAAI,SAAS,GAAG,UAAU,CAAC,WAAW,EAAE,QAAQ,IAAI,EAAE,CAAC;QACvD,IAAI,OAAO,SAAS,IAAI,QAAQ,EAAE,CAAC;YACjC,SAAS,GAAG,CAAC,SAAS,CAAC,CAAC;QAC1B,CAAC;QAED,KAAK,IAAI,GAAG,IAAI,SAAS,EAAE,CAAC;YAC1B,UAAU,CAAC,GAAG,CACZ,IAAI,IAAI,CAAC,kBAAkB,CACzB,IAAI,IAAI,CAAC,mBAAmB,CAAC,GAAG,EAAE,EAAE,cAAc,EAAE,CAAC,CAAC,UAAU,CAAC,WAAW,EAAE,gBAAgB,EAAE,CAAC,CAClG,CACF,CAAC;QACJ,CAAC;QAED,MAAM,UAAU,GAAG,CAAC,UAAU,CAAC,WAAW,EAAE,IAAI,EAAE,IAAI,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,GAAG,IAAI,iBAAiB,CAAC,CAAC;QAE1G,IAAI,MAAgC,CAAC;QACrC,IAAI,UAAU,CAAC,GAAG,EAAE,SAAS,IAAI,UAAU,IAAI,IAAI,IAAI,UAAU,CAAC,GAAG,IAAI,KAAK,EAAE,CAAC;YAC/E,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC;QACpD,CAAC;QAED,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,UAAU,EAAE,YAAY,CAAC,CAAC;QAEzE,IAAI,aAAa,GAAa,UAAU,CAAC,WAAW,EAAE,QAAQ,IAAI,EAAE,CAAC;QAErE,IAAI,MAAM,GAA4B;YACpC,WAAW,EAAE,UAAU;YACvB,WAAW,EAAE,UAAU,CAAC,WAAW,EAAE,WAAW,IAAI,EAAE;YACtD,OAAO,EAAE,UAAU,CAAC,OAAO;YAC3B,eAAe,EAAE,QAAQ;YACzB,+DAA+D;YAC/D,mEAAmE;YACnE,mBAAmB,EAAE,IAAI,IAAI,CAAC,QAAQ,CAAC,eAAe,CAAC;YACvD,UAAU,EAAE,UAAU,CAAC,GAAG,EAAE,MAAM,IAAI,EAAE;YACxC,GAAG,EAAE;gBACH,SAAS,EAAE,UAAU,CAAC,GAAG,EAAE,SAAS,IAAI,KAAK;gBAC7C,WAAW,EAAE,UAAU,CAAC,GAAG,EAAE,WAAW,IAAI,KAAK;gBACjD,aAAa,EAAE,UAAU,CAAC,GAAG,EAAE,aAAa;gBAC5C,QAAQ,EAAE,UAAU,CAAC,GAAG,EAAE,QAAQ,IAAI,KAAK;gBAC3C,OAAO,EAAE,MAAM;aAChB;YACD,IAAI,EAAE,UAAU,CAAC,IAAI,IAAI,IAAI;YAC7B,UAAU;YACV,aAAa;YAEb,oBAAoB,EAAE,IAAI,EAAE,QAAQ;YACpC,QAAQ,EAAE,UAAU,CAAC,QAAQ,IAAI,EAAE;YACnC,UAAU,EAAE,UAAU,CAAC,UAAU;YACjC,SAAS,EAAE;gBACT,yBAAyB,EAAE,UAAU,CAAC,SAAS,EAAE,yBAAyB,IAAI,KAAK;gBACnF,yBAAyB,EACvB,UAAU,CAAC,SAAS,EAAE,yBAAyB,IAAI,0CAA0C;aAChG;YACD,0DAA0D;YAC1D,qEAAqE;YACrE,gBAAgB,EAAE,YAAY;SAC/B,CAAC;QAEF,OAAO,MAAM,CAAC;IAChB,CAAC;IAED;;;OAGG;IACO,KAAK,CAAC,iBAAiB,CAAC,aAA2B;QAC3D,KAAK,MAAM,SAAS,IAAI,IAAI,CAAC,OAAO,CAAC,gBAAgB,EAAE,CAAC;YACtD,IAAI,CAAC;gBACH,MAAM,UAAU,GAAG,MAAM,SAAS,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC;gBAC1D,IAAI,UAAU,EAAE,CAAC;oBACf,OAAO,UAAU,CAAC;gBACpB,CAAC;gBACD,MAAM,CAAC,KAAK,CACV,2CAA2C,SAAS,CAAC,IAAI,iDAAiD,CAC3G,CAAC;YACJ,CAAC;YAAC,OAAO,EAAE,EAAE,CAAC;gBACZ,yCAAyC;gBACzC,MAAM,IAAI,KAAK,CAAC,kCAAkC,SAAS,CAAC,IAAI,8BAA8B,EAAE,EAAE,CAAC,CAAC;YACtG,CAAC;QACH,CAAC;QACD,MAAM,IAAI,KAAK,CAAC,uFAAuF,CAAC,CAAC;IAC3G,CAAC;IAES,KAAK,CAAC,gBAAgB,CAC9B,UAAsC,EACtC,YAA0B;QAE1B,KAAK,MAAM,SAAS,IAAI,IAAI,CAAC,OAAO,CAAC,mBAAmB,EAAE,CAAC;YACzD,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,OAAO,CAAC,UAAU,EAAE,YAAY,CAAC,CAAC;gBACjE,IAAI,MAAM,EAAE,CAAC;oBACX,OAAO,MAAM,CAAC;gBAChB,CAAC;gBACD,MAAM,CAAC,KAAK,CACV,qCAAqC,SAAS,CAAC,IAAI,iDAAiD,CACrG,CAAC;YACJ,CAAC;YAAC,OAAO,EAAE,EAAE,CAAC;gBACZ,yCAAyC;gBACzC,MAAM,IAAI,KAAK,CAAC,sCAAsC,SAAS,CAAC,IAAI,8BAA8B,EAAE,EAAE,CAAC,CAAC;YAC1G,CAAC;QACH,CAAC;QACD,OAAO;YACL,OAAO,EAAE,KAAK;SACf,CAAC;IACJ,CAAC;CACF"}
@@ -56,5 +56,4 @@ export type ResolvedPowerSyncConfig = {
56
56
  };
57
57
  /** Prefix for postgres replication slot names. May eventually be connection-specific. */
58
58
  slot_name_prefix: string;
59
- parameters: Record<string, number | string | boolean | null>;
60
59
  };
package/package.json CHANGED
@@ -5,7 +5,7 @@
5
5
  "publishConfig": {
6
6
  "access": "public"
7
7
  },
8
- "version": "0.0.0-dev-20241016113053",
8
+ "version": "0.0.0-dev-20241016143203",
9
9
  "main": "dist/index.js",
10
10
  "license": "FSL-1.1-Apache-2.0",
11
11
  "type": "module",
@@ -33,11 +33,11 @@
33
33
  "uuid": "^9.0.1",
34
34
  "winston": "^3.13.0",
35
35
  "yaml": "^2.3.2",
36
- "@powersync/lib-services-framework": "0.0.0-dev-20241016113053",
36
+ "@powersync/lib-services-framework": "0.0.0-dev-20241016143203",
37
37
  "@powersync/service-jsonbig": "0.17.10",
38
- "@powersync/service-rsocket-router": "0.0.0-dev-20241016113053",
39
- "@powersync/service-sync-rules": "0.0.0-dev-20241016113053",
40
- "@powersync/service-types": "0.0.0-dev-20241016113053"
38
+ "@powersync/service-rsocket-router": "0.0.0-dev-20241016143203",
39
+ "@powersync/service-sync-rules": "0.0.0-dev-20241016143203",
40
+ "@powersync/service-types": "0.0.0-dev-20241016143203"
41
41
  },
42
42
  "devDependencies": {
43
43
  "@types/async": "^3.2.24",
package/src/db/mongo.ts CHANGED
@@ -23,6 +23,13 @@ export const MONGO_SOCKET_TIMEOUT_MS = 60_000;
23
23
  */
24
24
  export const MONGO_OPERATION_TIMEOUT_MS = 30_000;
25
25
 
26
+ /**
27
+ * Same as above, but specifically for clear operations.
28
+ *
29
+ * These are retried when reaching the timeout.
30
+ */
31
+ export const MONGO_CLEAR_OPERATION_TIMEOUT_MS = 5_000;
32
+
26
33
  export function createMongoClient(config: configFile.PowerSyncConfig['storage']) {
27
34
  const normalized = normalizeMongoConfig(config);
28
35
  return new mongo.MongoClient(normalized.uri, {
@@ -90,11 +90,6 @@ export interface BucketStorageFactory
90
90
  */
91
91
  getActiveCheckpoint(): Promise<ActiveCheckpoint>;
92
92
 
93
- /**
94
- * Yields the latest sync checkpoint.
95
- */
96
- watchActiveCheckpoint(signal: AbortSignal): AsyncIterable<ActiveCheckpoint>;
97
-
98
93
  /**
99
94
  * Yields the latest user write checkpoint whenever the sync checkpoint updates.
100
95
  */
@@ -123,8 +118,6 @@ export interface ActiveCheckpoint {
123
118
  hasSyncRules(): boolean;
124
119
 
125
120
  getBucketStorage(): Promise<SyncRulesBucketStorage | null>;
126
-
127
- syncRules: PersistedSyncRules | null;
128
121
  }
129
122
 
130
123
  export interface StorageMetrics {
@@ -272,8 +265,6 @@ export interface SyncRulesBucketStorage extends DisposableObserverClient<SyncRul
272
265
  */
273
266
  clear(): Promise<void>;
274
267
 
275
- setSnapshotDone(lsn: string): Promise<void>;
276
-
277
268
  autoActivate(): Promise<void>;
278
269
 
279
270
  /**
@@ -78,9 +78,6 @@ export class MongoBucketStorage
78
78
  db: PowerSyncMongo,
79
79
  options: {
80
80
  slot_name_prefix: string;
81
- /**
82
- * Initial Write Checkpoint Mode
83
- */
84
81
  write_checkpoint_mode?: WriteCheckpointMode;
85
82
  }
86
83
  ) {
@@ -306,10 +303,6 @@ export class MongoBucketStorage
306
303
  return this.writeCheckpointAPI.batchCreateCustomWriteCheckpoints(checkpoints);
307
304
  }
308
305
 
309
- setWriteCheckpointMode(mode: WriteCheckpointMode): void {
310
- return this.writeCheckpointAPI.setWriteCheckpointMode(mode);
311
- }
312
-
313
306
  async createCustomWriteCheckpoint(options: CustomWriteCheckpointOptions): Promise<bigint> {
314
307
  return this.writeCheckpointAPI.createCustomWriteCheckpoint(options);
315
308
  }
@@ -338,39 +331,56 @@ export class MongoBucketStorage
338
331
  }
339
332
 
340
333
  async getStorageMetrics(): Promise<StorageMetrics> {
334
+ const ignoreNotExiting = (e: unknown) => {
335
+ if (e instanceof mongo.MongoServerError && e.codeName == 'NamespaceNotFound') {
336
+ // Collection doesn't exist - return 0
337
+ return [{ storageStats: { size: 0 } }];
338
+ } else {
339
+ return Promise.reject(e);
340
+ }
341
+ };
342
+
343
+ const active_sync_rules = await this.getActiveSyncRules({ defaultSchema: 'public' });
344
+ if (active_sync_rules == null) {
345
+ return {
346
+ operations_size_bytes: 0,
347
+ parameters_size_bytes: 0,
348
+ replication_size_bytes: 0
349
+ };
350
+ }
341
351
  const operations_aggregate = await this.db.bucket_data
342
352
 
343
353
  .aggregate([
344
354
  {
345
355
  $collStats: {
346
- storageStats: {},
347
- count: {}
356
+ storageStats: {}
348
357
  }
349
358
  }
350
359
  ])
351
- .toArray();
360
+ .toArray()
361
+ .catch(ignoreNotExiting);
352
362
 
353
363
  const parameters_aggregate = await this.db.bucket_parameters
354
364
  .aggregate([
355
365
  {
356
366
  $collStats: {
357
- storageStats: {},
358
- count: {}
367
+ storageStats: {}
359
368
  }
360
369
  }
361
370
  ])
362
- .toArray();
371
+ .toArray()
372
+ .catch(ignoreNotExiting);
363
373
 
364
374
  const replication_aggregate = await this.db.current_data
365
375
  .aggregate([
366
376
  {
367
377
  $collStats: {
368
- storageStats: {},
369
- count: {}
378
+ storageStats: {}
370
379
  }
371
380
  }
372
381
  ])
373
- .toArray();
382
+ .toArray()
383
+ .catch(ignoreNotExiting);
374
384
 
375
385
  return {
376
386
  operations_size_bytes: operations_aggregate[0].storageStats.size,
@@ -415,19 +425,14 @@ export class MongoBucketStorage
415
425
  return null;
416
426
  }
417
427
  return (await this.storageCache.fetch(doc._id)) ?? null;
418
- },
419
- syncRules: doc
420
- ? new MongoPersistedSyncRulesContent(this.db, doc).parsed({
421
- defaultSchema: ''
422
- })
423
- : null
424
- } satisfies ActiveCheckpoint;
428
+ }
429
+ };
425
430
  }
426
431
 
427
432
  /**
428
433
  * Instance-wide watch on the latest available checkpoint (op_id + lsn).
429
434
  */
430
- private async *_watchActiveCheckpoint(signal: AbortSignal): AsyncIterable<ActiveCheckpoint> {
435
+ private async *watchActiveCheckpoint(signal: AbortSignal): AsyncIterable<ActiveCheckpoint> {
431
436
  const pipeline: mongo.Document[] = [
432
437
  {
433
438
  $match: {
@@ -440,8 +445,7 @@ export class MongoBucketStorage
440
445
  operationType: 1,
441
446
  'fullDocument._id': 1,
442
447
  'fullDocument.last_checkpoint': 1,
443
- 'fullDocument.last_checkpoint_lsn': 1,
444
- 'fullDocument.content': 1
448
+ 'fullDocument.last_checkpoint_lsn': 1
445
449
  }
446
450
  }
447
451
  ];
@@ -463,8 +467,7 @@ export class MongoBucketStorage
463
467
  projection: {
464
468
  _id: 1,
465
469
  last_checkpoint: 1,
466
- last_checkpoint_lsn: 1,
467
- content: 1
470
+ last_checkpoint_lsn: 1
468
471
  }
469
472
  }
470
473
  );
@@ -513,7 +516,6 @@ export class MongoBucketStorage
513
516
  if (doc == null) {
514
517
  continue;
515
518
  }
516
-
517
519
  const op = this.makeActiveCheckpoint(doc);
518
520
  // Check for LSN / checkpoint changes - ignore other metadata changes
519
521
  if (lastOp == null || op.lsn != lastOp.lsn || op.checkpoint != lastOp.checkpoint) {
@@ -525,16 +527,9 @@ export class MongoBucketStorage
525
527
 
526
528
  // Nothing is done here until a subscriber starts to iterate
527
529
  private readonly sharedIter = new sync.BroadcastIterable((signal) => {
528
- return this._watchActiveCheckpoint(signal);
530
+ return this.watchActiveCheckpoint(signal);
529
531
  });
530
532
 
531
- /**
532
- * Watch changes to the active sync rules and checkpoint.
533
- */
534
- watchActiveCheckpoint(signal: AbortSignal): AsyncIterable<ActiveCheckpoint> {
535
- return wrapWithAbort(this.sharedIter, signal);
536
- }
537
-
538
533
  /**
539
534
  * User-specific watch on the latest checkpoint and/or write checkpoint.
540
535
  */
@@ -31,6 +31,8 @@ import { BucketDataDocument, BucketDataKey, SourceKey, SyncRuleState } from './m
31
31
  import { MongoBucketBatch } from './MongoBucketBatch.js';
32
32
  import { MongoCompactor } from './MongoCompactor.js';
33
33
  import { BSON_DESERIALIZE_OPTIONS, idPrefixFilter, mapOpEntry, readSingleBatch, serializeLookup } from './util.js';
34
+ import { logger } from '@powersync/lib-services-framework';
35
+ import * as timers from 'timers/promises';
34
36
 
35
37
  export class MongoSyncBucketStorage
36
38
  extends DisposableObserver<SyncRulesBucketStorageListener>
@@ -459,10 +461,28 @@ export class MongoSyncBucketStorage
459
461
  }
460
462
 
461
463
  async clear(): Promise<void> {
464
+ while (true) {
465
+ try {
466
+ await this.clearIteration();
467
+ return;
468
+ } catch (e: unknown) {
469
+ if (e instanceof mongo.MongoServerError && e.codeName == 'MaxTimeMSExpired') {
470
+ logger.info(
471
+ `Clearing took longer than ${db.mongo.MONGO_CLEAR_OPERATION_TIMEOUT_MS}ms, waiting and triggering another iteration.`
472
+ );
473
+ await timers.setTimeout(db.mongo.MONGO_CLEAR_OPERATION_TIMEOUT_MS / 5);
474
+ continue;
475
+ } else {
476
+ throw e;
477
+ }
478
+ }
479
+ }
480
+ }
481
+
482
+ private async clearIteration(): Promise<void> {
462
483
  // Individual operations here may time out with the maxTimeMS option.
463
484
  // It is expected to still make progress, and continue on the next try.
464
485
 
465
- // TODO: Transactional?
466
486
  await this.db.sync_rules.updateOne(
467
487
  {
468
488
  _id: this.group_id
@@ -476,48 +496,33 @@ export class MongoSyncBucketStorage
476
496
  no_checkpoint_before: null
477
497
  }
478
498
  },
479
- { maxTimeMS: db.mongo.MONGO_OPERATION_TIMEOUT_MS }
499
+ { maxTimeMS: db.mongo.MONGO_CLEAR_OPERATION_TIMEOUT_MS }
480
500
  );
481
501
  await this.db.bucket_data.deleteMany(
482
502
  {
483
503
  _id: idPrefixFilter<BucketDataKey>({ g: this.group_id }, ['b', 'o'])
484
504
  },
485
- { maxTimeMS: db.mongo.MONGO_OPERATION_TIMEOUT_MS }
505
+ { maxTimeMS: db.mongo.MONGO_CLEAR_OPERATION_TIMEOUT_MS }
486
506
  );
487
507
  await this.db.bucket_parameters.deleteMany(
488
508
  {
489
509
  key: idPrefixFilter<SourceKey>({ g: this.group_id }, ['t', 'k'])
490
510
  },
491
- { maxTimeMS: db.mongo.MONGO_OPERATION_TIMEOUT_MS }
511
+ { maxTimeMS: db.mongo.MONGO_CLEAR_OPERATION_TIMEOUT_MS }
492
512
  );
493
513
 
494
514
  await this.db.current_data.deleteMany(
495
515
  {
496
516
  _id: idPrefixFilter<SourceKey>({ g: this.group_id }, ['t', 'k'])
497
517
  },
498
- { maxTimeMS: db.mongo.MONGO_OPERATION_TIMEOUT_MS }
518
+ { maxTimeMS: db.mongo.MONGO_CLEAR_OPERATION_TIMEOUT_MS }
499
519
  );
500
520
 
501
521
  await this.db.source_tables.deleteMany(
502
522
  {
503
523
  group_id: this.group_id
504
524
  },
505
- { maxTimeMS: db.mongo.MONGO_OPERATION_TIMEOUT_MS }
506
- );
507
- }
508
-
509
- async setSnapshotDone(lsn: string): Promise<void> {
510
- await this.db.sync_rules.updateOne(
511
- {
512
- _id: this.group_id
513
- },
514
- {
515
- $set: {
516
- snapshot_done: true,
517
- persisted_lsn: lsn,
518
- last_checkpoint_ts: new Date()
519
- }
520
- }
525
+ { maxTimeMS: db.mongo.MONGO_CLEAR_OPERATION_TIMEOUT_MS }
521
526
  );
522
527
  }
523
528
 
@@ -17,19 +17,11 @@ export type MongoCheckpointAPIOptions = {
17
17
 
18
18
  export class MongoWriteCheckpointAPI implements WriteCheckpointAPI {
19
19
  readonly db: PowerSyncMongo;
20
- private _mode: WriteCheckpointMode;
20
+ readonly mode: WriteCheckpointMode;
21
21
 
22
22
  constructor(options: MongoCheckpointAPIOptions) {
23
23
  this.db = options.db;
24
- this._mode = options.mode;
25
- }
26
-
27
- get mode() {
28
- return this._mode;
29
- }
30
-
31
- setWriteCheckpointMode(mode: WriteCheckpointMode): void {
32
- this._mode = mode;
24
+ this.mode = options.mode;
33
25
  }
34
26
 
35
27
  async batchCreateCustomWriteCheckpoints(checkpoints: CustomWriteCheckpointOptions[]): Promise<void> {
@@ -37,11 +29,12 @@ export class MongoWriteCheckpointAPI implements WriteCheckpointAPI {
37
29
  }
38
30
 
39
31
  async createCustomWriteCheckpoint(options: CustomWriteCheckpointOptions): Promise<bigint> {
40
- /**
41
- * Allow creating custom checkpoints even if the current mode is not `custom`.
42
- * There might be a state where the next sync rules rely on replicating custom
43
- * write checkpoints, but the current active sync rules uses managed checkpoints.
44
- */
32
+ if (this.mode !== WriteCheckpointMode.CUSTOM) {
33
+ throw new framework.errors.ValidationError(
34
+ `Creating a custom Write Checkpoint when the current Write Checkpoint mode is set to "${this.mode}"`
35
+ );
36
+ }
37
+
45
38
  const { checkpoint, user_id, sync_rules_id } = options;
46
39
  const doc = await this.db.custom_write_checkpoints.findOneAndUpdate(
47
40
  {
@@ -62,6 +62,9 @@ export class PowerSyncMongo {
62
62
  this.locks = this.db.collection('locks');
63
63
  }
64
64
 
65
+ /**
66
+ * Clear all collections.
67
+ */
65
68
  async clear() {
66
69
  await this.current_data.deleteMany({});
67
70
  await this.bucket_data.deleteMany({});
@@ -73,4 +76,13 @@ export class PowerSyncMongo {
73
76
  await this.instance.deleteOne({});
74
77
  await this.locks.deleteMany({});
75
78
  }
79
+
80
+ /**
81
+ * Drop the entire database.
82
+ *
83
+ * Primarily for tests.
84
+ */
85
+ async drop() {
86
+ await this.db.dropDatabase();
87
+ }
76
88
  }
@@ -55,8 +55,6 @@ export type ManagedWriteCheckpointOptions = ManagedWriteCheckpointFilters;
55
55
  export type LastWriteCheckpointFilters = CustomWriteCheckpointFilters | ManagedWriteCheckpointFilters;
56
56
 
57
57
  export interface WriteCheckpointAPI {
58
- setWriteCheckpointMode(mode: WriteCheckpointMode): void;
59
-
60
58
  batchCreateCustomWriteCheckpoints(checkpoints: CustomWriteCheckpointOptions[]): Promise<void>;
61
59
 
62
60
  createCustomWriteCheckpoint(checkpoint: CustomWriteCheckpointOptions): Promise<bigint>;
@@ -122,8 +122,7 @@ export class CompoundConfigCollector {
122
122
  },
123
123
  // TODO maybe move this out of the connection or something
124
124
  // slot_name_prefix: connections[0]?.slot_name_prefix ?? 'powersync_'
125
- slot_name_prefix: 'powersync_',
126
- parameters: baseConfig.parameters ?? {}
125
+ slot_name_prefix: 'powersync_'
127
126
  };
128
127
 
129
128
  return config;
@@ -64,5 +64,4 @@ export type ResolvedPowerSyncConfig = {
64
64
 
65
65
  /** Prefix for postgres replication slot names. May eventually be connection-specific. */
66
66
  slot_name_prefix: string;
67
- parameters: Record<string, number | string | boolean | null>;
68
67
  };
@@ -1438,4 +1438,26 @@ bucket_definitions:
1438
1438
  expect(errorCaught).true;
1439
1439
  expect(isDisposed).true;
1440
1440
  });
1441
+
1442
+ test('empty storage metrics', async () => {
1443
+ const f = await factory({ dropAll: true });
1444
+
1445
+ const metrics = await f.getStorageMetrics();
1446
+ expect(metrics).toEqual({
1447
+ operations_size_bytes: 0,
1448
+ parameters_size_bytes: 0,
1449
+ replication_size_bytes: 0
1450
+ });
1451
+
1452
+ const r = await f.configureSyncRules('bucket_definitions: {}');
1453
+ const storage = f.getInstance(r.persisted_sync_rules!);
1454
+ await storage.autoActivate();
1455
+
1456
+ const metrics2 = await f.getStorageMetrics();
1457
+ expect(metrics2).toEqual({
1458
+ operations_size_bytes: 0,
1459
+ parameters_size_bytes: 0,
1460
+ replication_size_bytes: 0
1461
+ });
1462
+ });
1441
1463
  }
@@ -5,15 +5,7 @@ import { JSONBig } from '@powersync/service-jsonbig';
5
5
  import { RequestParameters } from '@powersync/service-sync-rules';
6
6
  import * as timers from 'timers/promises';
7
7
  import { describe, expect, test } from 'vitest';
8
- import {
9
- BATCH_OPTIONS,
10
- makeTestTable,
11
- MONGO_STORAGE_FACTORY,
12
- PARSE_OPTIONS,
13
- StorageFactory,
14
- ZERO_LSN
15
- } from './util.js';
16
- import { ParseSyncRulesOptions, StartBatchOptions } from '@/storage/BucketStorage.js';
8
+ import { BATCH_OPTIONS, makeTestTable, MONGO_STORAGE_FACTORY, PARSE_OPTIONS, StorageFactory } from './util.js';
17
9
 
18
10
  describe('sync - mongodb', function () {
19
11
  defineTests(MONGO_STORAGE_FACTORY);
@@ -38,8 +30,7 @@ function defineTests(factory: StorageFactory) {
38
30
  content: BASIC_SYNC_RULES
39
31
  });
40
32
 
41
- const storage = await f.getInstance(syncRules);
42
- await storage.setSnapshotDone(ZERO_LSN);
33
+ const storage = f.getInstance(syncRules);
43
34
  await storage.autoActivate();
44
35
 
45
36
  const result = await storage.startBatch(BATCH_OPTIONS, async (batch) => {
@@ -91,7 +82,6 @@ function defineTests(factory: StorageFactory) {
91
82
  });
92
83
 
93
84
  const storage = await f.getInstance(syncRules);
94
- await storage.setSnapshotDone(ZERO_LSN);
95
85
  await storage.autoActivate();
96
86
 
97
87
  const result = await storage.startBatch(BATCH_OPTIONS, async (batch) => {
@@ -136,7 +126,6 @@ function defineTests(factory: StorageFactory) {
136
126
  });
137
127
 
138
128
  const storage = await f.getInstance(syncRules);
139
- await storage.setSnapshotDone(ZERO_LSN);
140
129
  await storage.autoActivate();
141
130
 
142
131
  const stream = streamResponse({
@@ -164,7 +153,6 @@ function defineTests(factory: StorageFactory) {
164
153
  });
165
154
 
166
155
  const storage = await f.getInstance(syncRules);
167
- await storage.setSnapshotDone(ZERO_LSN);
168
156
  await storage.autoActivate();
169
157
 
170
158
  const stream = streamResponse({
@@ -226,7 +214,6 @@ function defineTests(factory: StorageFactory) {
226
214
  });
227
215
 
228
216
  const storage = await f.getInstance(syncRules);
229
- await storage.setSnapshotDone(ZERO_LSN);
230
217
  await storage.autoActivate();
231
218
 
232
219
  const exp = Date.now() / 1000 + 0.1;
@@ -265,7 +252,6 @@ function defineTests(factory: StorageFactory) {
265
252
  });
266
253
 
267
254
  const storage = await f.getInstance(syncRules);
268
- await storage.setSnapshotDone(ZERO_LSN);
269
255
  await storage.autoActivate();
270
256
 
271
257
  await storage.startBatch(BATCH_OPTIONS, async (batch) => {
package/test/src/util.ts CHANGED
@@ -24,11 +24,22 @@ await Metrics.initialise({
24
24
  });
25
25
  Metrics.getInstance().resetCounters();
26
26
 
27
- export type StorageFactory = () => Promise<BucketStorageFactory>;
27
+ export interface StorageOptions {
28
+ /**
29
+ * By default, collections are only cleared/
30
+ * Setting this to true will drop the collections completely.
31
+ */
32
+ dropAll?: boolean;
33
+ }
34
+ export type StorageFactory = (options?: StorageOptions) => Promise<BucketStorageFactory>;
28
35
 
29
- export const MONGO_STORAGE_FACTORY: StorageFactory = async () => {
36
+ export const MONGO_STORAGE_FACTORY: StorageFactory = async (options?: StorageOptions) => {
30
37
  const db = await connectMongo();
31
- await db.clear();
38
+ if (options?.dropAll) {
39
+ await db.drop();
40
+ } else {
41
+ await db.clear();
42
+ }
32
43
  return new MongoBucketStorage(db, { slot_name_prefix: 'test_' });
33
44
  };
34
45