@powersync/service-core 1.23.3 → 1.25.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (90) hide show
  1. package/CHANGELOG.md +45 -0
  2. package/dist/api/RouteAPI.d.ts +17 -3
  3. package/dist/entry/commands/compact-action.js +4 -0
  4. package/dist/entry/commands/compact-action.js.map +1 -1
  5. package/dist/replication/AbstractReplicator.d.ts +9 -4
  6. package/dist/replication/AbstractReplicator.js +35 -11
  7. package/dist/replication/AbstractReplicator.js.map +1 -1
  8. package/dist/routes/configure-fastify.d.ts +31 -0
  9. package/dist/routes/endpoints/checkpointing.d.ts +62 -0
  10. package/dist/routes/endpoints/checkpointing.js +63 -3
  11. package/dist/routes/endpoints/checkpointing.js.map +1 -1
  12. package/dist/storage/BucketStorageBatch.d.ts +4 -1
  13. package/dist/storage/BucketStorageBatch.js.map +1 -1
  14. package/dist/storage/BucketStorageFactory.d.ts +2 -2
  15. package/dist/storage/CheckpointChecksumInvalidatedError.d.ts +12 -0
  16. package/dist/storage/CheckpointChecksumInvalidatedError.js +17 -0
  17. package/dist/storage/CheckpointChecksumInvalidatedError.js.map +1 -0
  18. package/dist/storage/SourceEntity.d.ts +7 -2
  19. package/dist/storage/SourceTable.d.ts +24 -1
  20. package/dist/storage/SourceTable.js +34 -6
  21. package/dist/storage/SourceTable.js.map +1 -1
  22. package/dist/storage/SourceTableReconciler.d.ts +83 -0
  23. package/dist/storage/SourceTableReconciler.js +95 -0
  24. package/dist/storage/SourceTableReconciler.js.map +1 -0
  25. package/dist/storage/SyncRulesBucketStorage.d.ts +32 -1
  26. package/dist/storage/SyncRulesBucketStorage.js +3 -0
  27. package/dist/storage/SyncRulesBucketStorage.js.map +1 -1
  28. package/dist/storage/WriteCheckpointAPI.d.ts +60 -3
  29. package/dist/storage/WriteCheckpointAPI.js +33 -0
  30. package/dist/storage/WriteCheckpointAPI.js.map +1 -1
  31. package/dist/storage/implementation/BucketDefinitionMapping.d.ts +4 -4
  32. package/dist/storage/implementation/BucketDefinitionMapping.js.map +1 -1
  33. package/dist/storage/storage-index.d.ts +2 -0
  34. package/dist/storage/storage-index.js +2 -0
  35. package/dist/storage/storage-index.js.map +1 -1
  36. package/dist/sync/BucketChecksumState.d.ts +7 -0
  37. package/dist/sync/BucketChecksumState.js +33 -13
  38. package/dist/sync/BucketChecksumState.js.map +1 -1
  39. package/dist/sync/sync.js +34 -6
  40. package/dist/sync/sync.js.map +1 -1
  41. package/dist/sync/util.d.ts +5 -0
  42. package/dist/sync/util.js +9 -0
  43. package/dist/sync/util.js.map +1 -1
  44. package/dist/util/checkpointing.d.ts +1 -0
  45. package/dist/util/checkpointing.js +1 -1
  46. package/dist/util/checkpointing.js.map +1 -1
  47. package/dist/util/config/compound-config-collector.js +9 -1
  48. package/dist/util/config/compound-config-collector.js.map +1 -1
  49. package/dist/util/config/defaults.d.ts +1 -0
  50. package/dist/util/config/defaults.js +1 -0
  51. package/dist/util/config/defaults.js.map +1 -1
  52. package/dist/util/config/types.d.ts +1 -0
  53. package/dist/util/utils.d.ts +1 -1
  54. package/dist/util/utils.js +1 -1
  55. package/dist/util/utils.js.map +1 -1
  56. package/dist/util/write-checkpoint-batcher.d.ts +1 -1
  57. package/dist/util/write-checkpoint-batcher.js +18 -8
  58. package/dist/util/write-checkpoint-batcher.js.map +1 -1
  59. package/package.json +5 -5
  60. package/src/api/RouteAPI.ts +18 -3
  61. package/src/entry/commands/compact-action.ts +6 -0
  62. package/src/replication/AbstractReplicator.ts +42 -12
  63. package/src/routes/endpoints/checkpointing.ts +77 -3
  64. package/src/storage/BucketStorageBatch.ts +4 -1
  65. package/src/storage/BucketStorageFactory.ts +2 -2
  66. package/src/storage/CheckpointChecksumInvalidatedError.ts +17 -0
  67. package/src/storage/SourceEntity.ts +6 -2
  68. package/src/storage/SourceTable.ts +51 -7
  69. package/src/storage/SourceTableReconciler.ts +188 -0
  70. package/src/storage/SyncRulesBucketStorage.ts +38 -1
  71. package/src/storage/WriteCheckpointAPI.ts +108 -3
  72. package/src/storage/implementation/BucketDefinitionMapping.ts +4 -4
  73. package/src/storage/storage-index.ts +2 -0
  74. package/src/sync/BucketChecksumState.ts +38 -13
  75. package/src/sync/sync.ts +33 -7
  76. package/src/sync/util.ts +12 -0
  77. package/src/util/checkpointing.ts +2 -1
  78. package/src/util/config/compound-config-collector.ts +12 -0
  79. package/src/util/config/defaults.ts +1 -0
  80. package/src/util/config/types.ts +1 -0
  81. package/src/util/utils.ts +1 -1
  82. package/src/util/write-checkpoint-batcher.ts +30 -10
  83. package/test/src/AbstractReplicator.test.ts +147 -0
  84. package/test/src/config.test.ts +60 -0
  85. package/test/src/routes/checkpointing.test.ts +54 -0
  86. package/test/src/source-table-reconciler.test.ts +244 -0
  87. package/test/src/sync/BucketChecksumState.test.ts +57 -11
  88. package/test/src/sync/util.test.ts +13 -1
  89. package/test/src/util/checkpointing.test.ts +99 -20
  90. package/tsconfig.tsbuildinfo +1 -1
@@ -1 +1 @@
1
- {"version":3,"file":"utils.js","sourceRoot":"","sources":["../../src/util/utils.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAC7B,OAAO,MAAM,MAAM,QAAQ,CAAC;AAC5B,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAK7B,OAAO,EAAE,qBAAqB,EAAE,MAAM,mCAAmC,CAAC;AA2B1E,MAAM,CAAC,MAAM,YAAY,GAAG,sCAAsC,CAAC;AAEnE,MAAM,UAAU,gBAAgB,CAAC,UAAkB;IACjD,OAAO,IAAI,UAAU,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,KAAK,CAAC,GAAG,CAAC;AACrE,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,aAAa,CAAC,GAA8B;IAC1D,OAAO,GAAG,gBAAgB,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,gBAAgB,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;AACzE,CAAC;AAED,MAAM,UAAU,QAAQ,CAAC,IAAY,EAAE,EAAU,EAAE,IAAY;IAC7D,MAAM,IAAI,GAAG,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;IACzC,IAAI,CAAC,MAAM,CAAC,OAAO,IAAI,IAAI,EAAE,IAAI,IAAI,EAAE,CAAC,CAAC;IACzC,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;IAC7B,OAAO,MAAM,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;AAChC,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,SAAiB;IAC1C,MAAM,IAAI,GAAG,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;IACzC,IAAI,CAAC,MAAM,CAAC,UAAU,SAAS,EAAE,CAAC,CAAC;IACnC,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;IAC7B,OAAO,MAAM,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;AAChC,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,sBAAsB,CAAC,EAAgB;IACrD,6EAA6E;IAC7E,6CAA6C;IAC7C,IAAI,OAAO,EAAE,IAAI,QAAQ,EAAE,CAAC;QAC1B,MAAM,IAAI,qBAAqB,CAAC,yBAAyB,EAAE,KAAK,OAAO,EAAE,GAAG,CAAC,CAAC;IAChF,CAAC;IACD,OAAO,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;AACzB,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,QAAqB,EAAE,OAAoB;IACvE,mBAAmB;IACnB,MAAM,cAAc,GAAG,IAAI,GAAG,EAA0B,CAAC;IAEzD,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAS,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC;IAElD,KAAK,IAAI,QAAQ,IAAI,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;QACtC,MAAM,CAAC,GAAG,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QACxC,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC;YACd,QAAQ;YACR,cAAc,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;QAChD,CAAC;aAAM,CAAC;YACN,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;YACjC,IAAI,QAAQ,CAAC,QAAQ,IAAI,CAAC,CAAC,QAAQ,IAAI,QAAQ,CAAC,KAAK,IAAI,CAAC,CAAC,KAAK,EAAE,CAAC;gBACjE,UAAU;gBACV,cAAc,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;YAChD,CAAC;iBAAM,CAAC;gBACN,YAAY;YACd,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO;QACL,cAAc,EAAE,CAAC,GAAG,cAAc,CAAC,MAAM,EAAE,CAAC;QAC5C,cAAc,EAAE,CAAC,GAAG,QAAQ,CAAC;KAC9B,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,CAAS,EAAE,CAAS;IAC/C,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,UAAU,CAAC;AAC9B,CAAC;AAED,MAAM,UAAU,iBAAiB,CAAC,CAAmC;IACnE,OAAO,iBAAiB,IAAI,CAAC,CAAC;AAChC,CAAC;AAED,MAAM,UAAU,kBAAkB,CAAC,CAAiB,EAAE,CAA0C;IAC9F,MAAM,QAAQ,GAAG,mBAAmB,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;IACrD,IAAI,iBAAiB,CAAC,QAAQ,CAAC,EAAE,CAAC;QAChC,oCAAoC;QACpC,MAAM,IAAI,qBAAqB,CAAC,wBAAwB,CAAC,CAAC;IAC5D,CAAC;IACD,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,MAAM,UAAU,mBAAmB,CACjC,MAAc,EACd,CAA0C,EAC1C,CAA0C;IAE1C,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC;QAC3B,IAAI,CAAC,iBAAiB,CAAC,CAAC,CAAC,EAAE,CAAC;YAC1B,aAAa;YACb,OAAO,CAAC,CAAC;QACX,CAAC;QACD,QAAQ;QACR,IAAI,CAAC,iBAAiB,CAAC,CAAC,CAAC,EAAE,CAAC;YAC1B,OAAO;gBACL,MAAM;gBACN,QAAQ,EAAE,YAAY,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,eAAe,CAAC;gBACrD,KAAK,EAAE,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,YAAY;aAChC,CAAC;QACJ,CAAC;aAAM,CAAC;YACN,OAAO;gBACL,MAAM;gBACN,eAAe,EAAE,YAAY,CAAC,CAAC,CAAC,eAAe,EAAE,CAAC,CAAC,eAAe,CAAC;gBACnE,YAAY,EAAE,CAAC,CAAC,YAAY,GAAG,CAAC,CAAC,YAAY;aAC9C,CAAC;QACJ,CAAC;IACH,CAAC;SAAM,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC;QACrB,OAAO,CAAC,CAAC;IACX,CAAC;SAAM,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC;QACrB,OAAO,CAAC,CAAC;IACX,CAAC;SAAM,CAAC;QACN,+DAA+D;QAC/D,OAAO;YACL,MAAM;YACN,eAAe,EAAE,CAAC;YAClB,YAAY,EAAE,CAAC;SAChB,CAAC;IACJ,CAAC;AACH,CAAC;AAED,SAAS,qBAAqB,CAC5B,KAAoC,EACpC,OAAmC;IAEnC,IAAI,MAAM,GAAwB,EAAE,CAAC;IACrC,KAAK,IAAI,MAAM,IAAI,OAAO,EAAE,CAAC;QAC3B,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC;QACzB,MAAM,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC;IAC7B,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,UAAU,0BAA0B,CACxC,KAAoC,EACpC,OAAmC;IAEnC,IAAI,OAAO,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;QACxB,gDAAgD;QAChD,OAAO,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC;IAClC,CAAC;IACD,MAAM,WAAW,GAAG,qBAAqB,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;IAE1D,OAAO,cAAc,CAAC,WAAW,CAAC,CAAC;AACrC,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,GAAyB;IACtD,+EAA+E;IAC/E,4EAA4E;IAC5E,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;IACjC,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;IAChC,OAAO,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,EAAE,YAAY,EAAE,MAAM,CAAC,CAAC,CAAC;AAC5D,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAI,GAAqC;IACvE,KAAK,IAAI,GAAG,IAAI,GAAG,EAAE,CAAC;QACpB,IAAI,OAAO,GAAG,CAAC,GAAG,CAAC,IAAI,WAAW,EAAE,CAAC;YACnC,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,aAAa,CAC3B,SAAkB,EAClB,GAAqC;IAErC,IAAI,CAAC,SAAS,EAAE,CAAC;QACf,gDAAgD;QAChD,OAAO,IAAI,CAAC;IACd,CAAC;IACD,OAAO,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC;AAChC,CAAC;AAED;;;;;;;;;;;GAWG;AACH,MAAM,UAAU,YAAY,CAAC,UAAwB;IACnD,IAAI,QAAQ,GAAG,IAAI,GAAG,EAAsB,CAAC;IAC7C,IAAI,aAAa,GAAG,CAAC,CAAC;IAEtB,KAAK,IAAI,EAAE,IAAI,UAAU,EAAE,CAAC;QAC1B,MAAM,GAAG,GAAG,MAAM,CAAC,EAAE,CAAC,CAAC;QACvB,IAAI,EAAE,CAAC,EAAE,IAAI,KAAK,EAAE,CAAC;YACnB,MAAM,QAAQ,GAAG,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YACnC,IAAI,QAAQ,EAAE,CAAC;gBACb,aAAa,GAAG,YAAY,CAAC,aAAa,EAAE,QAAQ,CAAC,QAAkB,CAAC,CAAC;YAC3E,CAAC;YACD,QAAQ,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;QACxB,CAAC;aAAM,IAAI,EAAE,CAAC,EAAE,IAAI,QAAQ,EAAE,CAAC;YAC7B,MAAM,QAAQ,GAAG,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YACnC,IAAI,QAAQ,EAAE,CAAC;gBACb,aAAa,GAAG,YAAY,CAAC,aAAa,EAAE,QAAQ,CAAC,QAAkB,CAAC,CAAC;YAC3E,CAAC;YACD,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YACrB,aAAa,GAAG,YAAY,CAAC,aAAa,EAAE,EAAE,CAAC,QAAkB,CAAC,CAAC;QACrE,CAAC;aAAM,IAAI,EAAE,CAAC,EAAE,IAAI,OAAO,EAAE,CAAC;YAC5B,QAAQ,CAAC,KAAK,EAAE,CAAC;YACjB,aAAa,GAAG,EAAE,CAAC,QAAkB,CAAC;QACxC,CAAC;aAAM,IAAI,EAAE,CAAC,EAAE,IAAI,MAAM,EAAE,CAAC;YAC3B,aAAa,GAAG,YAAY,CAAC,aAAa,EAAE,EAAE,CAAC,QAAkB,CAAC,CAAC;QACrE,CAAC;aAAM,CAAC;YACN,MAAM,IAAI,KAAK,CAAC,qBAAqB,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;QAChD,CAAC;IACH,CAAC;IAED,MAAM,IAAI,GAAG,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;QAChD,OAAO,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;IACnD,CAAC,CAAC,CAAC;IAEH,IAAI,UAAU,GAAiB;QAC7B,wDAAwD;QACxD,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE,aAAa,EAAE;QACpD,GAAG,IAAI;KACR,CAAC;IAEF,OAAO,UAAU,CAAC;AACpB,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,OAAO,CAAC,CAAS;IAC/B,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IACf,OAAO,CAAC,CAAC;AACX,CAAC;AAED,SAAS,MAAM,CAAC,KAAiB;IAC/B,OAAO,GAAG,KAAK,CAAC,WAAW,IAAI,KAAK,CAAC,SAAS,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;AACnE,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,eAAe,CAAC,MAAiD;IAC/E,IAAI,MAAM,IAAI,IAAI,EAAE,CAAC;QACnB,OAAO,EAAE,CAAC;IACZ,CAAC;IACD,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,KAAK,IAAI,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;QAChD,IAAI,IAAI,EAAE,GAAG,GAAG,CAAC,MAAM,CAAC;QACxB,+CAA+C;QAC/C,IAAI,KAAK,IAAI,IAAI,EAAE,CAAC;YAClB,IAAI,IAAI,CAAC,CAAC;QACZ,CAAC;aAAM,IAAI,OAAO,KAAK,IAAI,QAAQ,EAAE,CAAC;YACpC,IAAI,IAAI,CAAC,CAAC;QACZ,CAAC;aAAM,IAAI,OAAO,KAAK,IAAI,QAAQ,EAAE,CAAC;YACpC,IAAI,IAAI,CAAC,CAAC;QACZ,CAAC;aAAM,IAAI,OAAO,KAAK,IAAI,QAAQ,EAAE,CAAC;YACpC,IAAI,IAAI,KAAK,CAAC,MAAM,CAAC;QACvB,CAAC;aAAM,IAAI,KAAK,YAAY,UAAU,EAAE,CAAC;YACvC,IAAI,IAAI,KAAK,CAAC,UAAU,CAAC;QAC3B,CAAC;IACH,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC"}
1
+ {"version":3,"file":"utils.js","sourceRoot":"","sources":["../../src/util/utils.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAC7B,OAAO,MAAM,MAAM,QAAQ,CAAC;AAC5B,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAK7B,OAAO,EAAE,qBAAqB,EAAE,MAAM,mCAAmC,CAAC;AA2B1E,MAAM,CAAC,MAAM,YAAY,GAAG,IAAI,CAAC,KAAK,CAAC,sCAAsC,CAAC,CAAC;AAE/E,MAAM,UAAU,gBAAgB,CAAC,UAAkB;IACjD,OAAO,IAAI,UAAU,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,KAAK,CAAC,GAAG,CAAC;AACrE,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,aAAa,CAAC,GAA8B;IAC1D,OAAO,GAAG,gBAAgB,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,gBAAgB,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;AACzE,CAAC;AAED,MAAM,UAAU,QAAQ,CAAC,IAAY,EAAE,EAAU,EAAE,IAAY;IAC7D,MAAM,IAAI,GAAG,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;IACzC,IAAI,CAAC,MAAM,CAAC,OAAO,IAAI,IAAI,EAAE,IAAI,IAAI,EAAE,CAAC,CAAC;IACzC,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;IAC7B,OAAO,MAAM,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;AAChC,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,SAAiB;IAC1C,MAAM,IAAI,GAAG,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;IACzC,IAAI,CAAC,MAAM,CAAC,UAAU,SAAS,EAAE,CAAC,CAAC;IACnC,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;IAC7B,OAAO,MAAM,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;AAChC,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,sBAAsB,CAAC,EAAgB;IACrD,6EAA6E;IAC7E,6CAA6C;IAC7C,IAAI,OAAO,EAAE,IAAI,QAAQ,EAAE,CAAC;QAC1B,MAAM,IAAI,qBAAqB,CAAC,yBAAyB,EAAE,KAAK,OAAO,EAAE,GAAG,CAAC,CAAC;IAChF,CAAC;IACD,OAAO,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;AACzB,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,QAAqB,EAAE,OAAoB;IACvE,mBAAmB;IACnB,MAAM,cAAc,GAAG,IAAI,GAAG,EAA0B,CAAC;IAEzD,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAS,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC;IAElD,KAAK,IAAI,QAAQ,IAAI,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;QACtC,MAAM,CAAC,GAAG,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QACxC,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC;YACd,QAAQ;YACR,cAAc,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;QAChD,CAAC;aAAM,CAAC;YACN,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;YACjC,IAAI,QAAQ,CAAC,QAAQ,IAAI,CAAC,CAAC,QAAQ,IAAI,QAAQ,CAAC,KAAK,IAAI,CAAC,CAAC,KAAK,EAAE,CAAC;gBACjE,UAAU;gBACV,cAAc,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;YAChD,CAAC;iBAAM,CAAC;gBACN,YAAY;YACd,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO;QACL,cAAc,EAAE,CAAC,GAAG,cAAc,CAAC,MAAM,EAAE,CAAC;QAC5C,cAAc,EAAE,CAAC,GAAG,QAAQ,CAAC;KAC9B,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,CAAS,EAAE,CAAS;IAC/C,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,UAAU,CAAC;AAC9B,CAAC;AAED,MAAM,UAAU,iBAAiB,CAAC,CAAmC;IACnE,OAAO,iBAAiB,IAAI,CAAC,CAAC;AAChC,CAAC;AAED,MAAM,UAAU,kBAAkB,CAAC,CAAiB,EAAE,CAA0C;IAC9F,MAAM,QAAQ,GAAG,mBAAmB,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;IACrD,IAAI,iBAAiB,CAAC,QAAQ,CAAC,EAAE,CAAC;QAChC,oCAAoC;QACpC,MAAM,IAAI,qBAAqB,CAAC,wBAAwB,CAAC,CAAC;IAC5D,CAAC;IACD,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,MAAM,UAAU,mBAAmB,CACjC,MAAc,EACd,CAA0C,EAC1C,CAA0C;IAE1C,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC;QAC3B,IAAI,CAAC,iBAAiB,CAAC,CAAC,CAAC,EAAE,CAAC;YAC1B,aAAa;YACb,OAAO,CAAC,CAAC;QACX,CAAC;QACD,QAAQ;QACR,IAAI,CAAC,iBAAiB,CAAC,CAAC,CAAC,EAAE,CAAC;YAC1B,OAAO;gBACL,MAAM;gBACN,QAAQ,EAAE,YAAY,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,eAAe,CAAC;gBACrD,KAAK,EAAE,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,YAAY;aAChC,CAAC;QACJ,CAAC;aAAM,CAAC;YACN,OAAO;gBACL,MAAM;gBACN,eAAe,EAAE,YAAY,CAAC,CAAC,CAAC,eAAe,EAAE,CAAC,CAAC,eAAe,CAAC;gBACnE,YAAY,EAAE,CAAC,CAAC,YAAY,GAAG,CAAC,CAAC,YAAY;aAC9C,CAAC;QACJ,CAAC;IACH,CAAC;SAAM,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC;QACrB,OAAO,CAAC,CAAC;IACX,CAAC;SAAM,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC;QACrB,OAAO,CAAC,CAAC;IACX,CAAC;SAAM,CAAC;QACN,+DAA+D;QAC/D,OAAO;YACL,MAAM;YACN,eAAe,EAAE,CAAC;YAClB,YAAY,EAAE,CAAC;SAChB,CAAC;IACJ,CAAC;AACH,CAAC;AAED,SAAS,qBAAqB,CAC5B,KAAoC,EACpC,OAAmC;IAEnC,IAAI,MAAM,GAAwB,EAAE,CAAC;IACrC,KAAK,IAAI,MAAM,IAAI,OAAO,EAAE,CAAC;QAC3B,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC;QACzB,MAAM,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC;IAC7B,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,UAAU,0BAA0B,CACxC,KAAoC,EACpC,OAAmC;IAEnC,IAAI,OAAO,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;QACxB,gDAAgD;QAChD,OAAO,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC;IAClC,CAAC;IACD,MAAM,WAAW,GAAG,qBAAqB,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;IAE1D,OAAO,cAAc,CAAC,WAAW,CAAC,CAAC;AACrC,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,GAAyB;IACtD,+EAA+E;IAC/E,4EAA4E;IAC5E,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;IACjC,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;IAChC,OAAO,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,EAAE,YAAY,EAAE,MAAM,CAAC,CAAC,CAAC;AAC5D,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAI,GAAqC;IACvE,KAAK,IAAI,GAAG,IAAI,GAAG,EAAE,CAAC;QACpB,IAAI,OAAO,GAAG,CAAC,GAAG,CAAC,IAAI,WAAW,EAAE,CAAC;YACnC,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,aAAa,CAC3B,SAAkB,EAClB,GAAqC;IAErC,IAAI,CAAC,SAAS,EAAE,CAAC;QACf,gDAAgD;QAChD,OAAO,IAAI,CAAC;IACd,CAAC;IACD,OAAO,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC;AAChC,CAAC;AAED;;;;;;;;;;;GAWG;AACH,MAAM,UAAU,YAAY,CAAC,UAAwB;IACnD,IAAI,QAAQ,GAAG,IAAI,GAAG,EAAsB,CAAC;IAC7C,IAAI,aAAa,GAAG,CAAC,CAAC;IAEtB,KAAK,IAAI,EAAE,IAAI,UAAU,EAAE,CAAC;QAC1B,MAAM,GAAG,GAAG,MAAM,CAAC,EAAE,CAAC,CAAC;QACvB,IAAI,EAAE,CAAC,EAAE,IAAI,KAAK,EAAE,CAAC;YACnB,MAAM,QAAQ,GAAG,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YACnC,IAAI,QAAQ,EAAE,CAAC;gBACb,aAAa,GAAG,YAAY,CAAC,aAAa,EAAE,QAAQ,CAAC,QAAkB,CAAC,CAAC;YAC3E,CAAC;YACD,QAAQ,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;QACxB,CAAC;aAAM,IAAI,EAAE,CAAC,EAAE,IAAI,QAAQ,EAAE,CAAC;YAC7B,MAAM,QAAQ,GAAG,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YACnC,IAAI,QAAQ,EAAE,CAAC;gBACb,aAAa,GAAG,YAAY,CAAC,aAAa,EAAE,QAAQ,CAAC,QAAkB,CAAC,CAAC;YAC3E,CAAC;YACD,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YACrB,aAAa,GAAG,YAAY,CAAC,aAAa,EAAE,EAAE,CAAC,QAAkB,CAAC,CAAC;QACrE,CAAC;aAAM,IAAI,EAAE,CAAC,EAAE,IAAI,OAAO,EAAE,CAAC;YAC5B,QAAQ,CAAC,KAAK,EAAE,CAAC;YACjB,aAAa,GAAG,EAAE,CAAC,QAAkB,CAAC;QACxC,CAAC;aAAM,IAAI,EAAE,CAAC,EAAE,IAAI,MAAM,EAAE,CAAC;YAC3B,aAAa,GAAG,YAAY,CAAC,aAAa,EAAE,EAAE,CAAC,QAAkB,CAAC,CAAC;QACrE,CAAC;aAAM,CAAC;YACN,MAAM,IAAI,KAAK,CAAC,qBAAqB,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;QAChD,CAAC;IACH,CAAC;IAED,MAAM,IAAI,GAAG,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;QAChD,OAAO,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;IACnD,CAAC,CAAC,CAAC;IAEH,IAAI,UAAU,GAAiB;QAC7B,wDAAwD;QACxD,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE,aAAa,EAAE;QACpD,GAAG,IAAI;KACR,CAAC;IAEF,OAAO,UAAU,CAAC;AACpB,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,OAAO,CAAC,CAAS;IAC/B,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IACf,OAAO,CAAC,CAAC;AACX,CAAC;AAED,SAAS,MAAM,CAAC,KAAiB;IAC/B,OAAO,GAAG,KAAK,CAAC,WAAW,IAAI,KAAK,CAAC,SAAS,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;AACnE,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,eAAe,CAAC,MAAiD;IAC/E,IAAI,MAAM,IAAI,IAAI,EAAE,CAAC;QACnB,OAAO,EAAE,CAAC;IACZ,CAAC;IACD,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,KAAK,IAAI,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;QAChD,IAAI,IAAI,EAAE,GAAG,GAAG,CAAC,MAAM,CAAC;QACxB,+CAA+C;QAC/C,IAAI,KAAK,IAAI,IAAI,EAAE,CAAC;YAClB,IAAI,IAAI,CAAC,CAAC;QACZ,CAAC;aAAM,IAAI,OAAO,KAAK,IAAI,QAAQ,EAAE,CAAC;YACpC,IAAI,IAAI,CAAC,CAAC;QACZ,CAAC;aAAM,IAAI,OAAO,KAAK,IAAI,QAAQ,EAAE,CAAC;YACpC,IAAI,IAAI,CAAC,CAAC;QACZ,CAAC;aAAM,IAAI,OAAO,KAAK,IAAI,QAAQ,EAAE,CAAC;YACpC,IAAI,IAAI,KAAK,CAAC,MAAM,CAAC;QACvB,CAAC;aAAM,IAAI,KAAK,YAAY,UAAU,EAAE,CAAC;YACvC,IAAI,IAAI,KAAK,CAAC,UAAU,CAAC;QAC3B,CAAC;IACH,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC"}
@@ -11,7 +11,7 @@ export declare class WriteCheckpointBatcher {
11
11
  private inFlight;
12
12
  private scheduledPump;
13
13
  constructor(getAPI: () => RouteAPI, getStorage: () => BucketStorageFactory);
14
- enqueue(userId: string): Promise<CreateWriteCheckpointResult>;
14
+ enqueue(userId: string, checkpointRequestId?: bigint): Promise<CreateWriteCheckpointResult>;
15
15
  private schedulePump;
16
16
  private pump;
17
17
  private executeBatch;
@@ -15,9 +15,9 @@ export class WriteCheckpointBatcher {
15
15
  this.getAPI = getAPI;
16
16
  this.getStorage = getStorage;
17
17
  }
18
- enqueue(userId) {
18
+ enqueue(userId, checkpointRequestId) {
19
19
  const resolvers = Promise.withResolvers();
20
- this.pending.push({ userId, resolvers });
20
+ this.pending.push({ userId, checkpointRequestId, resolvers });
21
21
  this.schedulePump();
22
22
  return resolvers.promise;
23
23
  }
@@ -54,9 +54,13 @@ export class WriteCheckpointBatcher {
54
54
  if (!syncBucketStorage) {
55
55
  throw new ServiceError(ErrorCode.PSYNC_S2302, `Cannot create Write Checkpoint since no sync config is active.`);
56
56
  }
57
+ // The source adapter reads the head, hands it to this callback to persist the
58
+ // write-checkpoint mapping, then forces a source marker only when storage
59
+ // reports an advance. Keeping the marker inside the callback means it is
60
+ // causally ordered after the head within a single source session.
57
61
  const { writeCheckpoints, currentCheckpoint } = await this.getAPI().createReplicationHead(async (currentCheckpoint) => {
58
- const writeCheckpoints = await this.createBatchWriteCheckpoints(syncBucketStorage, batch, currentCheckpoint);
59
- return { writeCheckpoints, currentCheckpoint };
62
+ const { writeCheckpoints, shouldAdvance } = await this.createBatchWriteCheckpoints(syncBucketStorage, batch, currentCheckpoint);
63
+ return { response: { writeCheckpoints, currentCheckpoint }, shouldAdvance };
60
64
  });
61
65
  const results = batch.map((request) => {
62
66
  const writeCheckpoint = writeCheckpoints.get(request.userId);
@@ -81,10 +85,16 @@ export class WriteCheckpointBatcher {
81
85
  }
82
86
  }
83
87
  async createBatchWriteCheckpoints(syncBucketStorage, batch, currentCheckpoint) {
84
- return syncBucketStorage.createManagedWriteCheckpoints(batch.map((request) => ({
85
- user_id: request.userId,
86
- heads: { '1': currentCheckpoint }
87
- })));
88
+ return syncBucketStorage.createManagedWriteCheckpoints(batch.map((request) => {
89
+ const checkpoint = {
90
+ user_id: request.userId,
91
+ heads: { '1': currentCheckpoint }
92
+ };
93
+ if (request.checkpointRequestId != null) {
94
+ checkpoint.checkpoint_request_id = request.checkpointRequestId;
95
+ }
96
+ return checkpoint;
97
+ }));
88
98
  }
89
99
  }
90
100
  //# sourceMappingURL=write-checkpoint-batcher.js.map
@@ -1 +1 @@
1
- {"version":3,"file":"write-checkpoint-batcher.js","sourceRoot":"","sources":["../../src/util/write-checkpoint-batcher.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,qBAAqB,EAAE,YAAY,EAAE,MAAM,mCAAmC,CAAC;AAInG,kFAAkF;AAClF,iFAAiF;AACjF,mEAAmE;AACnE,iHAAiH;AACjH,qHAAqH;AACrH,MAAM,sCAAsC,GAAG,CAAC,CAAC;AAYjD,MAAM,OAAO,sBAAsB;IAMd;IACA;IANX,OAAO,GAA4B,EAAE,CAAC;IACtC,QAAQ,GAAG,CAAC,CAAC;IACb,aAAa,CAA6B;IAElD,YACmB,MAAsB,EACtB,UAAsC;QADtC,WAAM,GAAN,MAAM,CAAgB;QACtB,eAAU,GAAV,UAAU,CAA4B;IACtD,CAAC;IAEJ,OAAO,CAAC,MAAc;QACpB,MAAM,SAAS,GAAG,OAAO,CAAC,aAAa,EAA+B,CAAC;QACvE,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC,CAAC;QACzC,IAAI,CAAC,YAAY,EAAE,CAAC;QACpB,OAAO,SAAS,CAAC,OAAO,CAAC;IAC3B,CAAC;IAEO,YAAY;QAClB,IAAI,IAAI,CAAC,aAAa,IAAI,IAAI,EAAE,CAAC;YAC/B,OAAO;QACT,CAAC;QAED,2EAA2E;QAC3E,+CAA+C;QAC/C,uGAAuG;QACvG,yGAAyG;QACzG,gHAAgH;QAChH,+GAA+G;QAC/G,wEAAwE;QACxE,IAAI,CAAC,aAAa,GAAG,UAAU,CAAC,GAAG,EAAE;YACnC,IAAI,CAAC,aAAa,GAAG,SAAS,CAAC;YAC/B,IAAI,CAAC,IAAI,EAAE,CAAC;QACd,CAAC,EAAE,CAAC,CAAC,CAAC;IACR,CAAC;IAEO,IAAI;QACV,OAAO,IAAI,CAAC,QAAQ,GAAG,sCAAsC,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACzF,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC;YAC3B,IAAI,CAAC,OAAO,GAAG,EAAE,CAAC;YAClB,IAAI,CAAC,QAAQ,EAAE,CAAC;YAEhB,KAAK,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,GAAG,EAAE;gBACzC,IAAI,CAAC,QAAQ,EAAE,CAAC;gBAChB,IAAI,CAAC,YAAY,EAAE,CAAC;YACtB,CAAC,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,YAAY,CAAC,KAA8B;QACvD,IAAI,CAAC;YACH,MAAM,iBAAiB,GAAG,CAAC,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC,mBAAmB,EAAE,CAAC,EAAE,OAAO,CAAC;YACnF,IAAI,CAAC,iBAAiB,EAAE,CAAC;gBACvB,MAAM,IAAI,YAAY,CAAC,SAAS,CAAC,WAAW,EAAE,gEAAgE,CAAC,CAAC;YAClH,CAAC;YAED,MAAM,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,GAAG,MAAM,IAAI,CAAC,MAAM,EAAE,CAAC,qBAAqB,CACvF,KAAK,EAAE,iBAAiB,EAAE,EAAE;gBAC1B,MAAM,gBAAgB,GAAG,MAAM,IAAI,CAAC,2BAA2B,CAAC,iBAAiB,EAAE,KAAK,EAAE,iBAAiB,CAAC,CAAC;gBAC7G,OAAO,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,CAAC;YACjD,CAAC,CACF,CAAC;YAEF,MAAM,OAAO,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE;gBACpC,MAAM,eAAe,GAAG,gBAAgB,CAAC,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;gBAC7D,IAAI,eAAe,IAAI,IAAI,EAAE,CAAC;oBAC5B,MAAM,IAAI,qBAAqB,CAC7B,iEAAiE,OAAO,CAAC,MAAM,EAAE,CAClF,CAAC;gBACJ,CAAC;gBAED,OAAO,EAAE,OAAO,EAAE,eAAe,EAAE,CAAC;YACtC,CAAC,CAAC,CAAC;YAEH,KAAK,MAAM,EAAE,OAAO,EAAE,eAAe,EAAE,IAAI,OAAO,EAAE,CAAC;gBACnD,OAAO,CAAC,SAAS,CAAC,OAAO,CAAC;oBACxB,eAAe,EAAE,MAAM,CAAC,eAAe,CAAC;oBACxC,eAAe,EAAE,iBAAiB;iBACnC,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,2EAA2E;YAC3E,oDAAoD;YACpD,KAAK,MAAM,OAAO,IAAI,KAAK,EAAE,CAAC;gBAC5B,OAAO,CAAC,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YAClC,CAAC;QACH,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,2BAA2B,CACvC,iBAAyC,EACzC,KAA8B,EAC9B,iBAAyB;QAEzB,OAAO,iBAAiB,CAAC,6BAA6B,CACpD,KAAK,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;YACtB,OAAO,EAAE,OAAO,CAAC,MAAM;YACvB,KAAK,EAAE,EAAE,GAAG,EAAE,iBAAiB,EAAE;SAClC,CAAC,CAAC,CACJ,CAAC;IACJ,CAAC;CACF"}
1
+ {"version":3,"file":"write-checkpoint-batcher.js","sourceRoot":"","sources":["../../src/util/write-checkpoint-batcher.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,qBAAqB,EAAE,YAAY,EAAE,MAAM,mCAAmC,CAAC;AASnG,kFAAkF;AAClF,iFAAiF;AACjF,mEAAmE;AACnE,iHAAiH;AACjH,qHAAqH;AACrH,MAAM,sCAAsC,GAAG,CAAC,CAAC;AAajD,MAAM,OAAO,sBAAsB;IAMd;IACA;IANX,OAAO,GAA4B,EAAE,CAAC;IACtC,QAAQ,GAAG,CAAC,CAAC;IACb,aAAa,CAA6B;IAElD,YACmB,MAAsB,EACtB,UAAsC;QADtC,WAAM,GAAN,MAAM,CAAgB;QACtB,eAAU,GAAV,UAAU,CAA4B;IACtD,CAAC;IAEJ,OAAO,CAAC,MAAc,EAAE,mBAA4B;QAClD,MAAM,SAAS,GAAG,OAAO,CAAC,aAAa,EAA+B,CAAC;QACvE,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,mBAAmB,EAAE,SAAS,EAAE,CAAC,CAAC;QAC9D,IAAI,CAAC,YAAY,EAAE,CAAC;QACpB,OAAO,SAAS,CAAC,OAAO,CAAC;IAC3B,CAAC;IAEO,YAAY;QAClB,IAAI,IAAI,CAAC,aAAa,IAAI,IAAI,EAAE,CAAC;YAC/B,OAAO;QACT,CAAC;QAED,2EAA2E;QAC3E,+CAA+C;QAC/C,uGAAuG;QACvG,yGAAyG;QACzG,gHAAgH;QAChH,+GAA+G;QAC/G,wEAAwE;QACxE,IAAI,CAAC,aAAa,GAAG,UAAU,CAAC,GAAG,EAAE;YACnC,IAAI,CAAC,aAAa,GAAG,SAAS,CAAC;YAC/B,IAAI,CAAC,IAAI,EAAE,CAAC;QACd,CAAC,EAAE,CAAC,CAAC,CAAC;IACR,CAAC;IAEO,IAAI;QACV,OAAO,IAAI,CAAC,QAAQ,GAAG,sCAAsC,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACzF,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC;YAC3B,IAAI,CAAC,OAAO,GAAG,EAAE,CAAC;YAClB,IAAI,CAAC,QAAQ,EAAE,CAAC;YAEhB,KAAK,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,GAAG,EAAE;gBACzC,IAAI,CAAC,QAAQ,EAAE,CAAC;gBAChB,IAAI,CAAC,YAAY,EAAE,CAAC;YACtB,CAAC,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,YAAY,CAAC,KAA8B;QACvD,IAAI,CAAC;YACH,MAAM,iBAAiB,GAAG,CAAC,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC,mBAAmB,EAAE,CAAC,EAAE,OAAO,CAAC;YACnF,IAAI,CAAC,iBAAiB,EAAE,CAAC;gBACvB,MAAM,IAAI,YAAY,CAAC,SAAS,CAAC,WAAW,EAAE,gEAAgE,CAAC,CAAC;YAClH,CAAC;YAED,8EAA8E;YAC9E,0EAA0E;YAC1E,yEAAyE;YACzE,kEAAkE;YAClE,MAAM,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,GAAG,MAAM,IAAI,CAAC,MAAM,EAAE,CAAC,qBAAqB,CACvF,KAAK,EAAE,iBAAiB,EAAE,EAAE;gBAC1B,MAAM,EAAE,gBAAgB,EAAE,aAAa,EAAE,GAAG,MAAM,IAAI,CAAC,2BAA2B,CAChF,iBAAiB,EACjB,KAAK,EACL,iBAAiB,CAClB,CAAC;gBACF,OAAO,EAAE,QAAQ,EAAE,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,EAAE,aAAa,EAAE,CAAC;YAC9E,CAAC,CACF,CAAC;YAEF,MAAM,OAAO,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE;gBACpC,MAAM,eAAe,GAAG,gBAAgB,CAAC,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;gBAC7D,IAAI,eAAe,IAAI,IAAI,EAAE,CAAC;oBAC5B,MAAM,IAAI,qBAAqB,CAC7B,iEAAiE,OAAO,CAAC,MAAM,EAAE,CAClF,CAAC;gBACJ,CAAC;gBAED,OAAO,EAAE,OAAO,EAAE,eAAe,EAAE,CAAC;YACtC,CAAC,CAAC,CAAC;YAEH,KAAK,MAAM,EAAE,OAAO,EAAE,eAAe,EAAE,IAAI,OAAO,EAAE,CAAC;gBACnD,OAAO,CAAC,SAAS,CAAC,OAAO,CAAC;oBACxB,eAAe,EAAE,MAAM,CAAC,eAAe,CAAC;oBACxC,eAAe,EAAE,iBAAiB;iBACnC,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,2EAA2E;YAC3E,oDAAoD;YACpD,KAAK,MAAM,OAAO,IAAI,KAAK,EAAE,CAAC;gBAC5B,OAAO,CAAC,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YAClC,CAAC;QACH,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,2BAA2B,CACvC,iBAAyC,EACzC,KAA8B,EAC9B,iBAAyB;QAEzB,OAAO,iBAAiB,CAAC,6BAA6B,CACpD,KAAK,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE;YACpB,MAAM,UAAU,GAAkC;gBAChD,OAAO,EAAE,OAAO,CAAC,MAAM;gBACvB,KAAK,EAAE,EAAE,GAAG,EAAE,iBAAiB,EAAE;aAClC,CAAC;YACF,IAAI,OAAO,CAAC,mBAAmB,IAAI,IAAI,EAAE,CAAC;gBACxC,UAAU,CAAC,qBAAqB,GAAG,OAAO,CAAC,mBAAmB,CAAC;YACjE,CAAC;YACD,OAAO,UAAU,CAAC;QACpB,CAAC,CAAC,CACH,CAAC;IACJ,CAAC;CACF"}
package/package.json CHANGED
@@ -4,7 +4,7 @@
4
4
  "publishConfig": {
5
5
  "access": "public"
6
6
  },
7
- "version": "1.23.3",
7
+ "version": "1.25.0",
8
8
  "license": "FSL-1.1-ALv2",
9
9
  "type": "module",
10
10
  "exports": "./dist/index.js",
@@ -32,11 +32,11 @@
32
32
  "uuid": "^14.0.0",
33
33
  "winston": "^3.13.0",
34
34
  "yaml": "^2.8.3",
35
- "@powersync/lib-services-framework": "0.9.8",
35
+ "@powersync/lib-services-framework": "0.10.0",
36
36
  "@powersync/service-jsonbig": "0.17.13",
37
- "@powersync/service-rsocket-router": "0.2.24",
38
- "@powersync/service-sync-rules": "0.39.0",
39
- "@powersync/service-types": "0.16.1"
37
+ "@powersync/service-rsocket-router": "0.2.25",
38
+ "@powersync/service-sync-rules": "0.40.0",
39
+ "@powersync/service-types": "0.17.0"
40
40
  },
41
41
  "devDependencies": {
42
42
  "@types/async": "^3.2.24",
@@ -73,9 +73,15 @@ export interface RouteAPI {
73
73
  *
74
74
  * The position is provided to the callback so the caller can persist its
75
75
  * write-checkpoint mapping before the source adapter forces any required
76
- * source-side marker or keepalive. After the callback returns, the adapter
76
+ * source-side marker or keepalive. The callback returns whether storage
77
+ * actually needs a source marker: when `shouldAdvance` is true the adapter
77
78
  * must ensure that the replication stream will observe this position or a
78
- * greater one, even when the source is otherwise idle.
79
+ * greater one, even when the source is otherwise idle. When it is false, the
80
+ * adapter may skip the source marker.
81
+ *
82
+ * Reading the head and forcing the marker happen within a single source
83
+ * session/connection where applicable, so the marker is causally ordered
84
+ * after the head that was handed to the callback.
79
85
  */
80
86
  createReplicationHead<T>(callback: ReplicationHeadCallback<T>): Promise<T>;
81
87
 
@@ -111,4 +117,13 @@ export interface RouteAPI {
111
117
  getParseSyncRulesOptions(): ParseSyncConfigOptions;
112
118
  }
113
119
 
114
- export type ReplicationHeadCallback<T> = (head: string) => Promise<T>;
120
+ export interface ReplicationHeadResult<T> {
121
+ response: T;
122
+ /**
123
+ * True when storage needs the source to force a later observable position.
124
+ * False when storage can prove no marker is needed.
125
+ */
126
+ shouldAdvance: boolean;
127
+ }
128
+
129
+ export type ReplicationHeadCallback<T> = (head: string) => Promise<ReplicationHeadResult<T>>;
@@ -21,6 +21,7 @@ const HEAP_LIMIT = v8.getHeapStatistics().heap_size_limit;
21
21
  * Limit to 1024MB overall.
22
22
  */
23
23
  const COMPACT_MEMORY_LIMIT_MB = Math.min(HEAP_LIMIT / 1024 / 1024 - 128, 1024);
24
+ const MINUTE_MS = 60 * 1000;
24
25
 
25
26
  export function registerCompactAction(program: Command) {
26
27
  const compactCommand = program
@@ -82,6 +83,9 @@ export function registerCompactAction(program: Command) {
82
83
  // Start the storage engine in order to create the appropriate BucketStorage
83
84
  await serviceContext.lifeCycleEngine.start();
84
85
  const bucketStorage = serviceContext.storageEngine.activeBucketStorage;
86
+ const deleteCheckpointRequestsBefore = new Date(
87
+ Date.now() - config.api_parameters.checkpoint_request_retention_minutes * MINUTE_MS
88
+ );
85
89
 
86
90
  const active = (await bucketStorage.getActiveSyncConfig())?.storage;
87
91
  if (active == null) {
@@ -94,12 +98,14 @@ export function registerCompactAction(program: Command) {
94
98
  memoryLimitMB: COMPACT_MEMORY_LIMIT_MB,
95
99
  compactBuckets: buckets,
96
100
  compactParameterData: compactParameters ?? false,
101
+ deleteCheckpointRequestsBefore,
97
102
  signal: abortController.signal
98
103
  });
99
104
  } else {
100
105
  await active.compact({
101
106
  memoryLimitMB: COMPACT_MEMORY_LIMIT_MB,
102
107
  compactParameterData: compactParameters ?? true,
108
+ deleteCheckpointRequestsBefore,
103
109
  signal: abortController.signal
104
110
  });
105
111
  }
@@ -1,4 +1,4 @@
1
- import { container, ErrorCode, logger } from '@powersync/lib-services-framework';
1
+ import { container, ErrorCode, logger, ReplicationAbortedError } from '@powersync/lib-services-framework';
2
2
  import { ReplicationMetric } from '@powersync/service-types';
3
3
  import { hrtime } from 'node:process';
4
4
  import { setTimeout as sleep } from 'node:timers/promises';
@@ -11,8 +11,8 @@ import { AbstractReplicationJob } from './AbstractReplicationJob.js';
11
11
  import { ErrorRateLimiter } from './ErrorRateLimiter.js';
12
12
  import { ConnectionTestResult } from './ReplicationModule.js';
13
13
 
14
- // 1 minute
15
- const PING_INTERVAL = 1_000_000_000n * 60n;
14
+ // Default to 1 minute when no source-specific interval is configured.
15
+ const DEFAULT_HEARTBEAT_INTERVAL_SECONDS = 60;
16
16
 
17
17
  // In the initial startup period, we use a short refresh interval. This helps to take over replication quickly in the case
18
18
  // of rolling deploys. After the initial period, we switch to a longer refresh interval to reduce load.
@@ -34,6 +34,10 @@ export interface AbstractReplicatorOptions {
34
34
  * This limits the effect of retries when there is a persistent issue.
35
35
  */
36
36
  rateLimiter: ErrorRateLimiter;
37
+ /**
38
+ * Interval in seconds between source connection pings. Null or undefined uses the default; 0 disables pings.
39
+ */
40
+ heartbeatIntervalSeconds?: number | null;
37
41
  }
38
42
 
39
43
  /**
@@ -51,25 +55,29 @@ export abstract class AbstractReplicator<T extends AbstractReplicationJob = Abst
51
55
  private replicationJobs = new Map<string, T>();
52
56
 
53
57
  /**
54
- * Map of replciation stream ids to promises that are clearing the replication stream.
58
+ * Map of replication stream ids to promises that are clearing the replication stream.
55
59
  *
56
- * We primarily do this to keep track of what we're currently clearing, but don't currently
57
- * use the Promise value.
60
+ * We primarily do this to keep track of what we're currently clearing.
58
61
  */
59
- private clearingJobs = new Map<number, Promise<void>>();
62
+ protected readonly clearingJobs = new Map<number, Promise<void>>();
60
63
 
61
64
  /**
62
65
  * Used for replication lag computation.
63
66
  */
64
67
  private activeReplicationJob: T | undefined = undefined;
65
68
 
66
- // First ping is only after 5 minutes, not when starting
69
+ // The first ping is only after the configured interval, not when starting.
67
70
  private lastPing = hrtime.bigint();
68
71
 
72
+ private readonly heartbeatIntervalNanos: bigint | null;
73
+
69
74
  private abortController: AbortController | undefined;
70
75
 
71
76
  protected constructor(private options: AbstractReplicatorOptions) {
72
77
  this.logger = logger.child({ name: `Replicator:${options.id}` });
78
+ const heartbeatIntervalSeconds = options.heartbeatIntervalSeconds ?? DEFAULT_HEARTBEAT_INTERVAL_SECONDS;
79
+ this.heartbeatIntervalNanos =
80
+ heartbeatIntervalSeconds === 0 ? null : BigInt(Math.round(heartbeatIntervalSeconds * 1_000_000_000));
73
81
  }
74
82
 
75
83
  /**
@@ -139,6 +147,7 @@ export abstract class AbstractReplicator<T extends AbstractReplicationJob = Abst
139
147
  for (const job of this.replicationJobs.values()) {
140
148
  promises.push(job.stop());
141
149
  }
150
+ promises.push(...this.clearingJobs.values());
142
151
  await Promise.all(promises);
143
152
  }
144
153
 
@@ -184,9 +193,9 @@ export abstract class AbstractReplicator<T extends AbstractReplicationJob = Abst
184
193
 
185
194
  // Ensure that the replication jobs' connections are kept alive.
186
195
  // We don't ping while in error retry back-off, to avoid having too failures.
187
- if (this.rateLimiter.mayPing()) {
196
+ if (this.heartbeatIntervalNanos != null && this.rateLimiter.mayPing()) {
188
197
  const now = hrtime.bigint();
189
- if (now - this.lastPing >= PING_INTERVAL) {
198
+ if (now - this.lastPing >= this.heartbeatIntervalNanos) {
190
199
  for (const activeJob of this.replicationJobs.values()) {
191
200
  await activeJob.keepAlive();
192
201
  }
@@ -312,9 +321,17 @@ export abstract class AbstractReplicator<T extends AbstractReplicationJob = Abst
312
321
  // It is important to be able to continue running the refresh loop, otherwise we cannot
313
322
  // retry locked replication stream, for example.
314
323
  const syncRuleStorage = this.storage.getInstance(replicationStream, { skipLifecycleHooks: true });
315
- const promise = this.terminateSyncRules(syncRuleStorage)
324
+ const promise = this.terminateStoppedReplicationStream(replicationStream, syncRuleStorage)
316
325
  .catch((e) => {
317
- syncRuleStorage.logger.warn(`Failed clean up replication config`, e);
326
+ if (e instanceof ReplicationAbortedError) {
327
+ // Expected when shutdown aborts an in-progress cleanup.
328
+ } else if (e?.errorData?.code === ErrorCode.PSYNC_S1003) {
329
+ this.logReplicationStreamInfoOnce(replicationStream, 'replication-stream-cleanup-locked', () => {
330
+ replicationStream.logger.info(`[${e.errorData.code}] ${e.errorData.description}`);
331
+ });
332
+ } else {
333
+ syncRuleStorage.logger.warn(`Failed clean up replication config`, e);
334
+ }
318
335
  })
319
336
  .finally(() => {
320
337
  this.clearingJobs.delete(replicationStream.replicationStreamId);
@@ -370,6 +387,19 @@ export abstract class AbstractReplicator<T extends AbstractReplicationJob = Abst
370
387
  return `${this.id}-${syncRuleId}`;
371
388
  }
372
389
 
390
+ protected async terminateStoppedReplicationStream(
391
+ replicationStream: storage.PersistedReplicationStream,
392
+ syncRuleStorage: storage.SyncRulesBucketStorage
393
+ ) {
394
+ const lock = await replicationStream.lock();
395
+ this.clearReplicationStreamInfoLog(replicationStream, 'replication-stream-cleanup-locked');
396
+ try {
397
+ await this.terminateSyncRules(syncRuleStorage);
398
+ } finally {
399
+ await lock.release();
400
+ }
401
+ }
402
+
373
403
  protected async terminateSyncRules(syncRuleStorage: storage.SyncRulesBucketStorage) {
374
404
  syncRuleStorage.logger.info(`Terminating replication stream...`);
375
405
  // This deletes postgres replication slots - should complete quickly.
@@ -1,10 +1,49 @@
1
- import { logger, router, schema } from '@powersync/lib-services-framework';
1
+ import { codecs, logger, router, schema } from '@powersync/lib-services-framework';
2
2
  import * as t from 'ts-codec';
3
3
 
4
4
  import * as util from '../../util/util-index.js';
5
5
  import { authUser } from '../auth.js';
6
6
  import { routeDefinition } from '../router.js';
7
7
 
8
+ const CHECKPOINT_REQUEST_ID_MAX = 9_223_372_036_854_775_807n;
9
+
10
+ function isValidCheckpointRequestId(value: bigint) {
11
+ return value > 0n && value <= CHECKPOINT_REQUEST_ID_MAX;
12
+ }
13
+
14
+ export const CheckpointRequestPayload = t.object({
15
+ client_id: t.string,
16
+ // Positive int64, matching the managed write checkpoint id. Clients should
17
+ // send values larger than Number.MAX_SAFE_INTEGER as strings, since JSON
18
+ // numbers are only validated up to the safe-integer range.
19
+ checkpoint_request_id: codecs.bigint
20
+ });
21
+
22
+ const CheckpointRequestPayloadShapeValidator = schema.createTsCodecValidator(CheckpointRequestPayload, {
23
+ allowAdditional: true
24
+ });
25
+
26
+ const CheckpointRequestPayloadValidator = {
27
+ validate(params: t.Encoded<typeof CheckpointRequestPayload>) {
28
+ const shapeValidation = CheckpointRequestPayloadShapeValidator.validate(params);
29
+ if (!shapeValidation.valid) {
30
+ return shapeValidation;
31
+ }
32
+
33
+ const decodedParams = CheckpointRequestPayload.decode(params);
34
+ if (!isValidCheckpointRequestId(decodedParams.checkpoint_request_id)) {
35
+ return {
36
+ valid: false as const,
37
+ errors: [`Expected checkpoint_request_id between 1 and ${CHECKPOINT_REQUEST_ID_MAX}`]
38
+ };
39
+ }
40
+
41
+ return {
42
+ valid: true as const
43
+ };
44
+ }
45
+ };
46
+
8
47
  const WriteCheckpointRequest = t.object({
9
48
  client_id: t.string.optional()
10
49
  });
@@ -24,7 +63,7 @@ export const writeCheckpoint = routeDefinition({
24
63
  // Since we don't use LSNs anymore, the only way to get that is to wait.
25
64
  const start = Date.now();
26
65
 
27
- const head = await apiHandler.createReplicationHead(async (head) => head);
66
+ const head = await apiHandler.createReplicationHead(async (head) => ({ response: head, shouldAdvance: true }));
28
67
 
29
68
  const timeout = 50_000;
30
69
 
@@ -70,4 +109,39 @@ export const writeCheckpoint2 = routeDefinition({
70
109
  }
71
110
  });
72
111
 
73
- export const CHECKPOINT_ROUTES = [writeCheckpoint, writeCheckpoint2];
112
+ export const checkpointRequest = routeDefinition({
113
+ path: '/sync/checkpoint-request',
114
+ method: router.HTTPMethod.POST,
115
+ authorize: authUser,
116
+ validator: CheckpointRequestPayloadValidator,
117
+ handler: async (request) => {
118
+ const { token_payload, service_context } = request.context;
119
+ const { params } = request;
120
+
121
+ const decodedParams = CheckpointRequestPayload.decode(params);
122
+
123
+ // Storage only applies supplied request ids that advance the stored managed checkpoint.
124
+ // Stale or duplicate ids return the stored checkpoint; storage decides whether
125
+ // the matched checkpoint still needs a source marker.
126
+ const { replicationHead, writeCheckpoint } = await util.createWriteCheckpoint({
127
+ userId: token_payload!.userIdString,
128
+ clientId: decodedParams.client_id,
129
+ batcher: service_context.writeCheckpointBatcher,
130
+ checkpointRequestId: decodedParams.checkpoint_request_id
131
+ });
132
+
133
+ logger.info(
134
+ `Requested checkpoint for ${token_payload!.userIdString}/${decodedParams.client_id}: ${writeCheckpoint} | ${replicationHead}`
135
+ );
136
+
137
+ // Return the checkpoint request id storage is actually at after this request.
138
+ // When an earlier request has already advanced the stored value beyond the
139
+ // supplied checkpoint_request_id, this returns that larger previous value so
140
+ // the client can treat the response as a stale-request acknowledgement.
141
+ return {
142
+ checkpoint_request_id: String(writeCheckpoint)
143
+ };
144
+ }
145
+ });
146
+
147
+ export const CHECKPOINT_ROUTES = [checkpointRequest, writeCheckpoint, writeCheckpoint2];
@@ -173,7 +173,10 @@ export interface BucketStorageBatch extends ObserverClient<BucketBatchStorageLis
173
173
  resolveTables(options: ResolveTablesOptions): Promise<ResolveTablesResult>;
174
174
 
175
175
  /**
176
- * Queue a custom write checkpoint to be persisted after operations are flushed.
176
+ * Queue a custom checkpoint request to be persisted after operations are
177
+ * flushed. Set `checkpoint_requested_at` when the custom checkpoint came from
178
+ * a client checkpoint request and should be cleaned up by request-retention
179
+ * compaction; omit it for persistent source-owned checkpoints.
177
180
  */
178
181
  addCustomWriteCheckpoint(checkpoint: BatchedCustomWriteCheckpointOptions): void;
179
182
  }
@@ -1,8 +1,8 @@
1
1
  import { BaseObserver, logger } from '@powersync/lib-services-framework';
2
2
  import {
3
3
  PrecompiledSyncConfig,
4
+ SerializedSyncPlan as RawSerializedSyncPlan,
4
5
  SerializedCompatibilityContext,
5
- SerializedSyncPlanV1,
6
6
  serializeSyncPlan,
7
7
  SqlSyncRules,
8
8
  SyncConfigWithErrors
@@ -171,7 +171,7 @@ export interface SerializedSyncPlan {
171
171
  /**
172
172
  * The serialized plan, from {@link serializeSyncPlan}.
173
173
  */
174
- plan: SerializedSyncPlanV1;
174
+ plan: RawSerializedSyncPlan;
175
175
  compatibility: SerializedCompatibilityContext;
176
176
  /**
177
177
  * Event descriptors are not currently represented in the sync plan because they don't use the sync streams compiler
@@ -0,0 +1,17 @@
1
+ import { InternalOpId } from '../util/util-index.js';
2
+
3
+ /**
4
+ * A checkpoint cannot be served because compaction rewrote a bucket-data
5
+ * document across its end boundary.
6
+ *
7
+ * The sync loop must skip this checkpoint before it sends its checkpoint line.
8
+ */
9
+ export class CheckpointChecksumInvalidatedError extends Error {
10
+ constructor(
11
+ public readonly checkpoint: InternalOpId,
12
+ public readonly bucket: string
13
+ ) {
14
+ super(`Checkpoint ${checkpoint} was invalidated by compaction in bucket ${bucket}`);
15
+ this.name = 'CheckpointChecksumInvalidatedError';
16
+ }
17
+ }
@@ -1,5 +1,11 @@
1
1
  import { SourceTableRef } from '@powersync/service-sync-rules';
2
2
 
3
+ /**
4
+ * Source-specific JSON metadata. Storage does not interpret it. Source-table APIs use null to
5
+ * represent the absence of metadata.
6
+ */
7
+ export type JsonValue = string | number | boolean | null | JsonValue[] | { [key: string]: JsonValue };
8
+
3
9
  export interface ColumnDescriptor {
4
10
  name: string;
5
11
  /**
@@ -19,8 +25,6 @@ export interface SourceEntityDescriptor extends SourceTableRef {
19
25
  * If specified, this is specifically used to detect renames.
20
26
  */
21
27
  objectId: number | string | undefined;
22
- schema: string;
23
- name: string;
24
28
  /**
25
29
  * The columns that are used to uniquely identify a record in the source entity.
26
30
  */
@@ -8,13 +8,23 @@ import {
8
8
  } from '@powersync/service-sync-rules';
9
9
  import { bson } from '../index.js';
10
10
  import * as util from '../util/util-index.js';
11
- import { ColumnDescriptor } from './SourceEntity.js';
11
+ import { ColumnDescriptor, JsonValue } from './SourceEntity.js';
12
12
 
13
13
  /**
14
14
  * Format of the id depends on the bucket storage module. It should be consistent within the module.
15
15
  */
16
16
  export type SourceTableId = string | bson.ObjectId;
17
17
 
18
+ /**
19
+ * Compare source-table ids without coercing between storage-specific types.
20
+ */
21
+ export function sourceTableIdEquals(left: SourceTableId, right: SourceTableId): boolean {
22
+ if (typeof left === 'string' || typeof right === 'string') {
23
+ return typeof left === 'string' && typeof right === 'string' && left === right;
24
+ }
25
+ return left.equals(right);
26
+ }
27
+
18
28
  export interface SourceTableOptions {
19
29
  id: SourceTableId;
20
30
  ref: SourceTableRef;
@@ -25,6 +35,10 @@ export interface SourceTableOptions {
25
35
  parameterLookupSources: ParameterIndexLookupCreator[];
26
36
  bucketDataSourceIds?: Set<BucketDefinitionId>;
27
37
  parameterLookupSourceIds?: Set<ParameterIndexId>;
38
+ /**
39
+ * Source-specific metadata. Null when no metadata has been recorded.
40
+ */
41
+ sourceMetadata?: JsonValue;
28
42
  }
29
43
 
30
44
  export interface TableSnapshotStatus {
@@ -136,6 +150,10 @@ export class SourceTable {
136
150
  return this.options.parameterLookupSourceIds;
137
151
  }
138
152
 
153
+ get sourceMetadata() {
154
+ return this.options.sourceMetadata ?? null;
155
+ }
156
+
139
157
  /**
140
158
  * Sanitized name of the entity in the format of "{schema}.{entity name}".
141
159
  * Suitable for safe use in Postgres queries.
@@ -152,23 +170,41 @@ export class SourceTable {
152
170
  * In-memory clone of the table status.
153
171
  */
154
172
  clone() {
173
+ return this.copyWithSourceMetadata(this.sourceMetadata);
174
+ }
175
+
176
+ /**
177
+ * Copy this table with different source metadata, preserving its resolved state.
178
+ */
179
+ withSourceMetadata(sourceMetadata: JsonValue) {
180
+ return this.copyWithSourceMetadata(sourceMetadata);
181
+ }
182
+
183
+ private copyWithSourceMetadata(sourceMetadata: JsonValue) {
155
184
  const copy = new SourceTable({
156
185
  id: this.id,
157
- ref: this.options.ref,
186
+ ref: { ...this.options.ref },
158
187
  objectId: this.objectId,
159
- replicaIdColumns: this.replicaIdColumns,
188
+ replicaIdColumns: this.replicaIdColumns.map((column) => ({ ...column })),
160
189
  snapshotComplete: this.snapshotComplete,
161
- bucketDataSources: this.bucketDataSources,
162
- parameterLookupSources: this.parameterLookupSources,
190
+ bucketDataSources: [...this.bucketDataSources],
191
+ parameterLookupSources: [...this.parameterLookupSources],
163
192
  bucketDataSourceIds: this.bucketDataSourceIds == null ? undefined : new Set(this.bucketDataSourceIds),
164
193
  parameterLookupSourceIds:
165
- this.parameterLookupSourceIds == null ? undefined : new Set(this.parameterLookupSourceIds)
194
+ this.parameterLookupSourceIds == null ? undefined : new Set(this.parameterLookupSourceIds),
195
+ sourceMetadata: structuredClone(sourceMetadata)
166
196
  });
167
197
  copy.syncData = this.syncData;
168
198
  copy.syncParameters = this.syncParameters;
169
199
  copy.syncEvent = this.syncEvent;
170
200
  copy.storeCurrentData = this.storeCurrentData;
171
- copy.snapshotStatus = this.snapshotStatus;
201
+ copy.snapshotStatus =
202
+ this.snapshotStatus == null
203
+ ? undefined
204
+ : {
205
+ ...this.snapshotStatus,
206
+ lastKey: this.snapshotStatus.lastKey?.slice() ?? null
207
+ };
172
208
  return copy;
173
209
  }
174
210
 
@@ -183,3 +219,11 @@ export class SourceTable {
183
219
  }
184
220
  }
185
221
  }
222
+
223
+ /**
224
+ * A cloned SourceTable exposed to reconciliation with public fields typed as read-only.
225
+ * `options` is omitted so callers cannot mutate the underlying option bag without an explicit cast.
226
+ */
227
+ export type SourceTableCandidate = Omit<Readonly<SourceTable>, 'options' | 'withSourceMetadata'> & {
228
+ withSourceMetadata(sourceMetadata: JsonValue): SourceTableCandidate;
229
+ };