@powersync/service-module-mssql 0.5.0 → 0.6.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. package/CHANGELOG.md +32 -0
  2. package/dist/common/CaptureInstance.d.ts +14 -0
  3. package/dist/common/CaptureInstance.js +2 -0
  4. package/dist/common/CaptureInstance.js.map +1 -0
  5. package/dist/common/MSSQLSourceTable.d.ts +16 -14
  6. package/dist/common/MSSQLSourceTable.js +35 -16
  7. package/dist/common/MSSQLSourceTable.js.map +1 -1
  8. package/dist/replication/CDCPoller.d.ts +42 -20
  9. package/dist/replication/CDCPoller.js +200 -60
  10. package/dist/replication/CDCPoller.js.map +1 -1
  11. package/dist/replication/CDCReplicationJob.js +9 -1
  12. package/dist/replication/CDCReplicationJob.js.map +1 -1
  13. package/dist/replication/CDCStream.d.ts +35 -4
  14. package/dist/replication/CDCStream.js +181 -74
  15. package/dist/replication/CDCStream.js.map +1 -1
  16. package/dist/replication/MSSQLConnectionManager.js +16 -5
  17. package/dist/replication/MSSQLConnectionManager.js.map +1 -1
  18. package/dist/types/types.d.ts +4 -56
  19. package/dist/types/types.js +5 -24
  20. package/dist/types/types.js.map +1 -1
  21. package/dist/utils/deadlock.d.ts +9 -0
  22. package/dist/utils/deadlock.js +40 -0
  23. package/dist/utils/deadlock.js.map +1 -0
  24. package/dist/utils/mssql.d.ts +33 -15
  25. package/dist/utils/mssql.js +101 -99
  26. package/dist/utils/mssql.js.map +1 -1
  27. package/dist/utils/schema.d.ts +9 -0
  28. package/dist/utils/schema.js +34 -0
  29. package/dist/utils/schema.js.map +1 -1
  30. package/package.json +8 -8
  31. package/src/common/CaptureInstance.ts +15 -0
  32. package/src/common/MSSQLSourceTable.ts +33 -24
  33. package/src/replication/CDCPoller.ts +272 -72
  34. package/src/replication/CDCReplicationJob.ts +8 -1
  35. package/src/replication/CDCStream.ts +237 -90
  36. package/src/replication/MSSQLConnectionManager.ts +15 -5
  37. package/src/types/types.ts +5 -28
  38. package/src/utils/deadlock.ts +44 -0
  39. package/src/utils/mssql.ts +159 -124
  40. package/src/utils/schema.ts +43 -0
  41. package/test/src/CDCStreamTestContext.ts +9 -2
  42. package/test/src/env.ts +1 -1
  43. package/test/src/mssql-to-sqlite.test.ts +18 -10
  44. package/test/src/schema-changes.test.ts +470 -0
  45. package/test/src/util.ts +75 -12
  46. package/tsconfig.tsbuildinfo +1 -1
@@ -1 +1 @@
1
- {"version":3,"file":"CDCPoller.js","sourceRoot":"","sources":["../../src/replication/CDCPoller.ts"],"names":[],"mappings":"AAAA,OAAO,EAAU,MAAM,IAAI,aAAa,EAAE,yBAAyB,EAAE,MAAM,mCAAmC,CAAC;AAC/G,OAAO,MAAM,MAAM,iBAAiB,CAAC;AAGrC,OAAO,EAAE,GAAG,EAAE,MAAM,kBAAkB,CAAC;AACvC,OAAO,GAAG,MAAM,OAAO,CAAC;AACxB,OAAO,EAAE,SAAS,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AAG5D,IAAK,SAKJ;AALD,WAAK,SAAS;IACZ,6CAAU,CAAA;IACV,6CAAU,CAAA;IACV,2DAAiB,CAAA;IACjB,yDAAgB,CAAA;AAClB,CAAC,EALI,SAAS,KAAT,SAAS,QAKb;AACD;;;;GAIG;AACH,MAAM,CAAN,IAAY,gBAMX;AAND,WAAY,gBAAgB;IAC1B,iDAA6B,CAAA;IAC7B,6CAAyB,CAAA;IACzB,qDAAiC,CAAA;IACjC,6DAAyC,CAAA;IACzC,uEAAmD,CAAA;AACrD,CAAC,EANW,gBAAgB,KAAhB,gBAAgB,QAM3B;AAgCD;;GAEG;AACH,MAAM,OAAO,SAAS;IAWD;IAVX,iBAAiB,CAAyB;IAC1C,YAAY,CAAkB;IAC9B,UAAU,CAAM;IAChB,MAAM,CAAS;IACf,aAAa,CAAe;IAE5B,SAAS,GAAY,KAAK,CAAC;IAC3B,UAAU,GAAY,KAAK,CAAC;IAC5B,SAAS,GAAY,KAAK,CAAC;IAEnC,YAAmB,OAAyB;QAAzB,YAAO,GAAP,OAAO,CAAkB;QAC1C,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,aAAa,CAAC;QAC9C,IAAI,CAAC,iBAAiB,GAAG,OAAO,CAAC,iBAAiB,CAAC;QACnD,IAAI,CAAC,YAAY,GAAG,OAAO,CAAC,YAAY,CAAC;QACzC,IAAI,CAAC,UAAU,GAAG,OAAO,CAAC,QAAQ,CAAC;QACnC,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC;IAC5B,CAAC;IAED,IAAY,gBAAgB;QAC1B,OAAO,IAAI,CAAC,OAAO,CAAC,gBAAgB,CAAC,gBAAgB,CAAC;IACxD,CAAC;IAED,IAAY,iBAAiB;QAC3B,OAAO,IAAI,CAAC,OAAO,CAAC,gBAAgB,CAAC,iBAAiB,CAAC;IACzD,CAAC;IAED,IAAY,YAAY;QACtB,OAAO,IAAI,CAAC,OAAO,CAAC,YAAY,CAAC;IACnC,CAAC;IAEM,KAAK,CAAC,IAAI;QACf,IAAI,CAAC,CAAC,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,UAAU,CAAC,EAAE,CAAC;YACzC,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC;YACvB,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;QACxB,CAAC;IACH,CAAC;IAEM,KAAK,CAAC,qBAAqB;QAChC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,wCAAwC,IAAI,CAAC,iBAAiB,OAAO,CAAC,CAAC;QACxF,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,wBAAwB,IAAI,CAAC,gBAAgB,kCAAkC,CAAC,CAAC;QAClG,OAAO,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC;YACvB,oDAAoD;YACpD,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;gBACnB,MAAM,IAAI,yBAAyB,CAAC,yCAAyC,CAAC,CAAC;YACjF,CAAC;YAED,IAAI,CAAC;gBACH,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;gBACrC,IAAI,CAAC,UAAU,EAAE,CAAC;oBAChB,0CAA0C;oBAC1C,MAAM,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;gBAClD,CAAC;gBACD,0DAA0D;YAC5D,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,IAAI,CAAC,CAAC,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,UAAU,CAAC,EAAE,CAAC;oBACzC,IAAI,CAAC,aAAa,GAAG,KAAc,CAAC;oBACpC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,2BAA2B,EAAE,KAAK,CAAC,CAAC;oBACtD,IAAI,CAAC,IAAI,EAAE,CAAC;gBACd,CAAC;gBACD,MAAM;YACR,CAAC;QACH,CAAC;QAED,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;YACvB,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,0CAA0C,EAAE,IAAI,CAAC,aAAa,CAAC,CAAC;YAClF,MAAM,IAAI,CAAC,aAAa,CAAC;QAC3B,CAAC;QAED,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,wBAAwB,CAAC,CAAC;IAC7C,CAAC;IAEO,KAAK,CAAC,IAAI;QAChB,wDAAwD;QACxD,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;QAEtB,IAAI,CAAC;YACH,0CAA0C;YAC1C,mFAAmF;YACnF,MAAM,QAAQ,GAAG,MAAM,YAAY,CAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,iBAAiB,CAAC,CAAC;YAE7E,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,KAAK,CAC/D,eAAe,IAAI,CAAC,gBAAgB;;;;SAInC,EACD,CAAC,EAAE,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,GAAG,CAAC,SAAS,EAAE,KAAK,EAAE,QAAQ,CAAC,QAAQ,EAAE,EAAE,CAAC,CACxE,CAAC;YAEF,2CAA2C;YAC3C,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACzB,OAAO,KAAK,CAAC;YACf,CAAC;YAED,kDAAkD;YAClD,MAAM,MAAM,GAAG,GAAG,CAAC,UAAU,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;YAErE,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,sBAAsB,QAAQ,OAAO,MAAM,aAAa,OAAO,CAAC,MAAM,kBAAkB,CAAC,CAAC;YAE3G,IAAI,gBAAgB,GAAG,CAAC,CAAC;YACzB,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;gBACtC,MAAM,qBAAqB,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAC;gBAChF,2GAA2G;gBAC3G,yKAAyK;gBACzK,IAAI,qBAAqB,GAAG,gBAAgB,EAAE,CAAC;oBAC7C,gBAAgB,GAAG,qBAAqB,CAAC;gBAC3C,CAAC;YACH,CAAC;YAED,IAAI,CAAC,MAAM,CAAC,IAAI,CACd,aAAa,OAAO,CAAC,MAAM,8BAA8B,gBAAgB,+BAA+B,CACzG,CAAC;YACF,sEAAsE;YACtE,MAAM,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ,EAAE,EAAE,gBAAgB,CAAC,CAAC;YAEtE,IAAI,CAAC,UAAU,GAAG,MAAM,CAAC;YAEzB,OAAO,IAAI,CAAC;QACd,CAAC;gBAAS,CAAC;YACT,2CAA2C;YAC3C,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC;QACzB,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,SAAS,CAAC,KAAuB,EAAE,MAAsC;QACrF,uEAAuE;QACvE,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,IAAI,CAAC,iBAAiB,EAAE,KAAK,CAAC,eAAe,CAAC,CAAC;QAC9E,IAAI,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC;YAC3B,OAAO,CAAC,CAAC;QACX,CAAC;aAAM,IAAI,MAAM,IAAI,MAAM,CAAC,QAAQ,EAAE,CAAC;YACrC,MAAM,CAAC,QAAQ,GAAG,MAAM,CAAC;QAC3B,CAAC;QACD,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,KAAK,CAC/D;wBACkB,KAAK,CAAC,kBAAkB;KAC3C,EACC;YACE,EAAE,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,GAAG,CAAC,SAAS,EAAE,KAAK,EAAE,MAAM,CAAC,QAAQ,CAAC,QAAQ,EAAE,EAAE;YAC5E,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,GAAG,CAAC,SAAS,EAAE,KAAK,EAAE,MAAM,CAAC,MAAM,CAAC,QAAQ,EAAE,EAAE;SACzE,CACF,CAAC;QAEF,IAAI,gBAAgB,GAAG,CAAC,CAAC;QACzB,IAAI,YAAY,GAAQ,IAAI,CAAC;QAC7B,IAAI,kBAAkB,GAAe,IAAI,CAAC;QAC1C,KAAK,MAAM,GAAG,IAAI,OAAO,EAAE,CAAC;YAC1B,MAAM,cAAc,GAAG,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;YACxD,QAAQ,GAAG,CAAC,YAAY,EAAE,CAAC;gBACzB,KAAK,SAAS,CAAC,MAAM;oBACnB,MAAM,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC,GAAG,EAAE,KAAK,EAAE,OAAO,CAAC,OAAO,CAAC,CAAC;oBAC9D,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,6BAA6B,cAAc,EAAE,CAAC,CAAC;oBAChE,MAAM;gBACR,KAAK,SAAS,CAAC,MAAM;oBACnB,MAAM,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC,GAAG,EAAE,KAAK,EAAE,OAAO,CAAC,OAAO,CAAC,CAAC;oBAC9D,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,6BAA6B,cAAc,EAAE,CAAC,CAAC;oBAChE,MAAM;gBACR,KAAK,SAAS,CAAC,aAAa;oBAC1B,YAAY,GAAG,GAAG,CAAC;oBACnB,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,qCAAqC,cAAc,EAAE,CAAC,CAAC;oBACzE,MAAM;gBACR,KAAK,SAAS,CAAC,YAAY;oBACzB,IAAI,YAAY,KAAK,IAAI,EAAE,CAAC;wBAC1B,MAAM,IAAI,yBAAyB,CAAC,wCAAwC,CAAC,CAAC;oBAChF,CAAC;oBACD,MAAM,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC,GAAG,EAAE,YAAY,EAAE,KAAK,EAAE,OAAO,CAAC,OAAO,CAAC,CAAC;oBAC5E,YAAY,GAAG,IAAI,CAAC;oBACpB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,6BAA6B,cAAc,EAAE,CAAC,CAAC;oBAChE,MAAM;gBACR;oBACE,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,2BAA2B,GAAG,CAAC,YAAY,+BAA+B,CAAC,CAAC;YACjG,CAAC;YAED,sGAAsG;YACtG,IAAI,cAAc,IAAI,kBAAkB,EAAE,CAAC;gBACzC,kBAAkB,GAAG,cAAc,CAAC;gBACpC,IAAI,GAAG,CAAC,YAAY,KAAK,SAAS,CAAC,aAAa,EAAE,CAAC;oBACjD,gBAAgB,EAAE,CAAC;gBACrB,CAAC;YACH,CAAC;QACH,CAAC;QAED,OAAO,gBAAgB,CAAC;IAC1B,CAAC;CACF"}
1
+ {"version":3,"file":"CDCPoller.js","sourceRoot":"","sources":["../../src/replication/CDCPoller.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,kBAAkB,EAClB,SAAS,EAET,MAAM,IAAI,aAAa,EACvB,yBAAyB,EAC1B,MAAM,mCAAmC,CAAC;AAC3C,OAAO,MAAM,MAAM,iBAAiB,CAAC;AAGrC,OAAO,EAAE,GAAG,EAAE,MAAM,kBAAkB,CAAC;AACvC,OAAO,GAAG,MAAM,OAAO,CAAC;AACxB,OAAO,EAA0B,mBAAmB,EAAE,YAAY,EAAE,oBAAoB,EAAE,MAAM,mBAAmB,CAAC;AACpH,OAAO,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AAEvD,OAAO,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AAKjD,IAAK,SAKJ;AALD,WAAK,SAAS;IACZ,6CAAU,CAAA;IACV,6CAAU,CAAA;IACV,2DAAiB,CAAA;IACjB,yDAAgB,CAAA;AAClB,CAAC,EALI,SAAS,KAAT,SAAS,QAKb;AAED,MAAM,CAAN,IAAY,gBAOX;AAPD,WAAY,gBAAgB;IAC1B,iDAA6B,CAAA;IAC7B,6CAAyB,CAAA;IACzB,iDAA6B,CAAA;IAC7B,iEAA6C,CAAA;IAC7C,iEAA6C,CAAA;IAC7C,yEAAqD,CAAA;AACvD,CAAC,EAPW,gBAAgB,KAAhB,gBAAgB,QAO3B;AAwBD,MAAM,CAAC,MAAM,gCAAgC,GAAG,MAAM,CAAC;AAsBvD;;GAEG;AACH,MAAM,OAAO,SAAS;IAaD;IAZX,iBAAiB,CAAyB;IAC1C,YAAY,CAAkB;IAC9B,UAAU,CAAM;IAChB,MAAM,CAAS;IACf,aAAa,CAAe;IAC5B,gBAAgB,CAAsC;IAEtD,SAAS,GAAY,KAAK,CAAC;IAC3B,UAAU,GAAY,KAAK,CAAC;IAC5B,SAAS,GAAY,KAAK,CAAC;IAC3B,mBAAmB,GAAW,CAAC,CAAC;IAExC,YAAmB,OAAyB;QAAzB,YAAO,GAAP,OAAO,CAAkB;QAC1C,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,aAAa,CAAC;QAC9C,IAAI,CAAC,iBAAiB,GAAG,OAAO,CAAC,iBAAiB,CAAC;QACnD,IAAI,CAAC,YAAY,GAAG,OAAO,CAAC,YAAY,CAAC;QACzC,IAAI,CAAC,UAAU,GAAG,OAAO,CAAC,QAAQ,CAAC;QACnC,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC;QAC1B,IAAI,CAAC,gBAAgB,GAAG,IAAI,GAAG,EAAkC,CAAC;IACpE,CAAC;IAED,IAAY,gBAAgB;QAC1B,OAAO,IAAI,CAAC,OAAO,CAAC,gBAAgB,CAAC,gBAAgB,CAAC;IACxD,CAAC;IAED,IAAY,iBAAiB;QAC3B,OAAO,IAAI,CAAC,OAAO,CAAC,gBAAgB,CAAC,iBAAiB,CAAC;IACzD,CAAC;IAED,IAAY,qBAAqB;QAC/B,OAAO,IAAI,CAAC,OAAO,CAAC,qBAAqB,IAAI,gCAAgC,CAAC;IAChF,CAAC;IAED,IAAY,gBAAgB;QAC1B,OAAO,IAAI,CAAC,OAAO,CAAC,mBAAmB,EAAE,CAAC;IAC5C,CAAC;IAEM,KAAK,CAAC,IAAI;QACf,IAAI,CAAC,CAAC,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,UAAU,CAAC,EAAE,CAAC;YACzC,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC;YACvB,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;QACxB,CAAC;IACH,CAAC;IAEM,KAAK,CAAC,qBAAqB;QAChC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,wCAAwC,IAAI,CAAC,iBAAiB,OAAO,CAAC,CAAC;QACxF,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,wBAAwB,IAAI,CAAC,gBAAgB,kCAAkC,CAAC,CAAC;QAClG,OAAO,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC;YACvB,oDAAoD;YACpD,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;gBACnB,MAAM,IAAI,yBAAyB,CAAC,yCAAyC,CAAC,CAAC;YACjF,CAAC;YAED,IAAI,CAAC;gBACH,IAAI,IAAI,CAAC,iBAAiB,EAAE,EAAE,CAAC;oBAC7B,IAAI,CAAC,gBAAgB,GAAG,MAAM,mBAAmB,CAAC,EAAE,iBAAiB,EAAE,IAAI,CAAC,iBAAiB,EAAE,CAAC,CAAC;oBACjG,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,qBAAqB,EAAE,CAAC;oBACzD,KAAK,MAAM,YAAY,IAAI,aAAa,EAAE,CAAC;wBACzC,MAAM,IAAI,CAAC,YAAY,CAAC,cAAc,CAAC,YAAY,CAAC,CAAC;oBACvD,CAAC;oBACD,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;oBAEtC,IAAI,CAAC,MAAM,CAAC,KAAK,CACf,uDAAuD,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CACrG,CAAC;gBACJ,CAAC;gBAED,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;gBACrC,IAAI,CAAC,UAAU,EAAE,CAAC;oBAChB,8CAA8C;oBAC9C,MAAM,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;gBAClD,CAAC;gBAED,0DAA0D;YAC5D,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,IAAI,CAAC,CAAC,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,UAAU,CAAC,EAAE,CAAC;oBACzC,qBAAqB;oBACrB,IAAI,KAAK,YAAY,kBAAkB,EAAE,CAAC;wBACxC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;wBAChC,kEAAkE;wBAClE,IAAI,CAAC,mBAAmB,GAAG,CAAC,CAAC;wBAC7B,SAAS;oBACX,CAAC;oBACD,kFAAkF;oBAClF,sFAAsF;oBACtF,IAAI,eAAe,CAAC,KAAK,CAAC,EAAE,CAAC;wBAC3B,IAAI,CAAC,MAAM,CAAC,IAAI,CACd,mGAAoG,KAAe,CAAC,OAAO,EAAE,CAC9H,CAAC;wBACF,SAAS;oBACX,CAAC;oBAED,yBAAyB;oBACzB,IAAI,CAAC,aAAa,GAAG,KAAc,CAAC;oBACpC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,2BAA2B,EAAE,KAAK,CAAC,CAAC;oBACtD,IAAI,CAAC,IAAI,EAAE,CAAC;gBACd,CAAC;gBACD,MAAM;YACR,CAAC;QACH,CAAC;QAED,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;YACvB,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,0CAA0C,EAAE,IAAI,CAAC,aAAa,CAAC,CAAC;YAClF,MAAM,IAAI,CAAC,aAAa,CAAC;QAC3B,CAAC;QAED,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,wBAAwB,CAAC,CAAC;IAC7C,CAAC;IAEO,KAAK,CAAC,IAAI;QAChB,wDAAwD;QACxD,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;QAEtB,IAAI,CAAC;YACH,0CAA0C;YAC1C,mFAAmF;YACnF,MAAM,QAAQ,GAAG,MAAM,YAAY,CAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,iBAAiB,CAAC,CAAC;YAE7E,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,KAAK,CAC/D,eAAe,IAAI,CAAC,gBAAgB;;;;SAInC,EACD,CAAC,EAAE,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,GAAG,CAAC,SAAS,EAAE,KAAK,EAAE,QAAQ,CAAC,QAAQ,EAAE,EAAE,CAAC,CACxE,CAAC;YAEF,2CAA2C;YAC3C,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACzB,OAAO,KAAK,CAAC;YACf,CAAC;YAED,kDAAkD;YAClD,MAAM,MAAM,GAAG,GAAG,CAAC,UAAU,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;YAErE,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,sBAAsB,QAAQ,OAAO,MAAM,aAAa,OAAO,CAAC,MAAM,kBAAkB,CAAC,CAAC;YAE3G,IAAI,gBAAgB,GAAG,CAAC,CAAC;YACzB,IAAI,CAAC,MAAM,CAAC,KAAK,CACf,iCAAiC,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,eAAe,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAC5G,CAAC;YACF,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,gBAAgB,EAAE,CAAC;gBAC1C,IAAI,KAAK,CAAC,aAAa,EAAE,EAAE,CAAC;oBAC1B,MAAM,qBAAqB,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAC;oBAChF,2GAA2G;oBAC3G,yKAAyK;oBACzK,IAAI,qBAAqB,GAAG,gBAAgB,EAAE,CAAC;wBAC7C,gBAAgB,GAAG,qBAAqB,CAAC;oBAC3C,CAAC;gBACH,CAAC;YACH,CAAC;YAED,IAAI,CAAC,MAAM,CAAC,IAAI,CACd,aAAa,OAAO,CAAC,MAAM,8BAA8B,gBAAgB,+CAA+C,MAAM,CAAC,QAAQ,EAAE,EAAE,CAC5I,CAAC;YACF,sEAAsE;YACtE,MAAM,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ,EAAE,EAAE,gBAAgB,CAAC,CAAC;YAEtE,IAAI,CAAC,UAAU,GAAG,MAAM,CAAC;YAEzB,OAAO,IAAI,CAAC;QACd,CAAC;gBAAS,CAAC;YACT,2CAA2C;YAC3C,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC;QACzB,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,SAAS,CAAC,KAAuB,EAAE,MAAsC;QACrF,uEAAuE;QACvE,MAAM,MAAM,GAAG,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,KAAK,CAAC,QAAQ,CAAE,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;QAC9E,IAAI,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC;YAC3B,OAAO,CAAC,CAAC;QACX,CAAC;aAAM,IAAI,MAAM,IAAI,MAAM,CAAC,QAAQ,EAAE,CAAC;YACrC,MAAM,CAAC,QAAQ,GAAG,MAAM,CAAC;QAC3B,CAAC;QAED,IAAI,CAAC;YACH,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,KAAK,CAC/D;wBACgB,KAAK,CAAC,kBAAkB;KAC3C,EACG;gBACE,EAAE,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,GAAG,CAAC,SAAS,EAAE,KAAK,EAAE,MAAM,CAAC,QAAQ,CAAC,QAAQ,EAAE,EAAE;gBAC5E,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,GAAG,CAAC,SAAS,EAAE,KAAK,EAAE,MAAM,CAAC,MAAM,CAAC,QAAQ,EAAE,EAAE;aACzE,CACF,CAAC;YAEF,IAAI,gBAAgB,GAAG,CAAC,CAAC;YACzB,IAAI,YAAY,GAAQ,IAAI,CAAC;YAC7B,IAAI,kBAAkB,GAAe,IAAI,CAAC;YAC1C,KAAK,MAAM,GAAG,IAAI,OAAO,EAAE,CAAC;gBAC1B,MAAM,cAAc,GAAG,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;gBACxD,QAAQ,GAAG,CAAC,YAAY,EAAE,CAAC;oBACzB,KAAK,SAAS,CAAC,MAAM;wBACnB,MAAM,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC,GAAG,EAAE,KAAK,EAAE,OAAO,CAAC,OAAO,CAAC,CAAC;wBAC9D,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,6BAA6B,cAAc,EAAE,CAAC,CAAC;wBAChE,MAAM;oBACR,KAAK,SAAS,CAAC,MAAM;wBACnB,MAAM,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC,GAAG,EAAE,KAAK,EAAE,OAAO,CAAC,OAAO,CAAC,CAAC;wBAC9D,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,6BAA6B,cAAc,EAAE,CAAC,CAAC;wBAChE,MAAM;oBACR,KAAK,SAAS,CAAC,aAAa;wBAC1B,YAAY,GAAG,GAAG,CAAC;wBACnB,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,qCAAqC,cAAc,EAAE,CAAC,CAAC;wBACzE,MAAM;oBACR,KAAK,SAAS,CAAC,YAAY;wBACzB,IAAI,YAAY,KAAK,IAAI,EAAE,CAAC;4BAC1B,MAAM,IAAI,yBAAyB,CAAC,wCAAwC,CAAC,CAAC;wBAChF,CAAC;wBACD,MAAM,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC,GAAG,EAAE,YAAY,EAAE,KAAK,EAAE,OAAO,CAAC,OAAO,CAAC,CAAC;wBAC5E,YAAY,GAAG,IAAI,CAAC;wBACpB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,6BAA6B,cAAc,EAAE,CAAC,CAAC;wBAChE,MAAM;oBACR;wBACE,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,2BAA2B,GAAG,CAAC,YAAY,+BAA+B,CAAC,CAAC;gBACjG,CAAC;gBAED,sGAAsG;gBACtG,IAAI,cAAc,IAAI,kBAAkB,EAAE,CAAC;oBACzC,kBAAkB,GAAG,cAAc,CAAC;oBACpC,IAAI,GAAG,CAAC,YAAY,KAAK,SAAS,CAAC,aAAa,EAAE,CAAC;wBACjD,gBAAgB,EAAE,CAAC;oBACrB,CAAC;gBACH,CAAC;YACH,CAAC;YAED,OAAO,gBAAgB,CAAC;QAC1B,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,wDAAwD;YACxD,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,qBAAqB,CAAC,EAAE,CAAC;gBAClD,MAAM,IAAI,kBAAkB,CAC1B,SAAS,CAAC,WAAW,EACrB,8BAA8B,KAAK,CAAC,eAAe,EAAE,0BAA0B,EAC/E,KAAK,CACN,CAAC;YACJ,CAAC;YACD,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;IAEO,iBAAiB;QACvB,OAAO,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,mBAAmB,IAAI,IAAI,CAAC,qBAAqB,CAAC;IAC7E,CAAC;IAED;;;OAGG;IACK,KAAK,CAAC,qBAAqB;QACjC,MAAM,aAAa,GAAmB,EAAE,CAAC;QAEzC,MAAM,SAAS,GAAG,IAAI,CAAC,iBAAiB,EAAE,CAAC;QAC3C,KAAK,MAAM,KAAK,IAAI,SAAS,EAAE,CAAC;YAC9B,IAAI,CAAC,MAAM,CAAC,IAAI,CACd,aAAa,oBAAoB,CAAC,KAAK,CAAC,WAAW,CAAC,MAAM,EAAE,KAAK,CAAC,WAAW,CAAC,IAAI,CAAC,sEAAsE,CAC1J,CAAC;YACF,aAAa,CAAC,IAAI,CAAC;gBACjB,IAAI,EAAE,gBAAgB,CAAC,YAAY;gBACnC,QAAQ,EAAE;oBACR,IAAI,EAAE,KAAK,CAAC,WAAW,CAAC,IAAI;oBAC5B,MAAM,EAAE,KAAK,CAAC,WAAW,CAAC,MAAM;oBAChC,QAAQ,EAAE,KAAK,CAAC,WAAW,CAAC,QAAQ;iBACrC;gBACD,kBAAkB,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC;aACvC,CAAC,CAAC;QACL,CAAC;QAED,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,gBAAgB,EAAE,CAAC;YAC1C,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC,KAAK,CAAC,QAAQ,EAAE,IAAI,CAAC,iBAAiB,CAAC,CAAC;YACzE,IAAI,CAAC,MAAM,EAAE,CAAC;gBACZ,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,KAAK,CAAC,eAAe,EAAE,8CAA8C,CAAC,CAAC;gBACjG,aAAa,CAAC,IAAI,CAAC;oBACjB,IAAI,EAAE,gBAAgB,CAAC,UAAU;oBACjC,KAAK;iBACN,CAAC,CAAC;gBACH,SAAS;YACX,CAAC;YAED,MAAM,sBAAsB,GAAG,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;YACzE,IAAI,CAAC,sBAAsB,EAAE,CAAC;gBAC5B,IAAI,KAAK,CAAC,aAAa,EAAE,EAAE,CAAC;oBAC1B,mDAAmD;oBACnD,aAAa,CAAC,IAAI,CAAC;wBACjB,IAAI,EAAE,gBAAgB,CAAC,wBAAwB;wBAC/C,KAAK;qBACN,CAAC,CAAC;gBACL,CAAC;gBACD,SAAS;YACX,CAAC;YAED,MAAM,qBAAqB,GAAG,sBAAsB,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC;YAClE,oHAAoH;YACpH,IAAI,CAAC,KAAK,CAAC,aAAa,EAAE,IAAI,KAAK,CAAC,eAAgB,CAAC,QAAQ,KAAK,qBAAqB,CAAC,QAAQ,EAAE,CAAC;gBACjG,aAAa,CAAC,IAAI,CAAC;oBACjB,IAAI,EAAE,gBAAgB,CAAC,oBAAoB;oBAC3C,KAAK;oBACL,kBAAkB,EAAE,qBAAqB;iBAC1C,CAAC,CAAC;gBACH,SAAS;YACX,CAAC;YAED,gDAAgD;YAChD,IAAI,KAAK,CAAC,WAAW,CAAC,IAAI,KAAK,sBAAsB,CAAC,WAAW,CAAC,IAAI,EAAE,CAAC;gBACvE,MAAM,QAAQ,GAAG,IAAI,CAAC,qBAAqB,CACzC,sBAAsB,CAAC,WAAW,CAAC,MAAM,EACzC,sBAAsB,CAAC,WAAW,CAAC,IAAI,CACxC;oBACC,CAAC,CAAC;wBACE,IAAI,EAAE,sBAAsB,CAAC,WAAW,CAAC,IAAI;wBAC7C,MAAM,EAAE,sBAAsB,CAAC,WAAW,CAAC,MAAM;wBACjD,QAAQ,EAAE,sBAAsB,CAAC,WAAW,CAAC,QAAQ;qBACtD;oBACH,CAAC,CAAC,SAAS,CAAC;gBAEd,aAAa,CAAC,IAAI,CAAC;oBACjB,IAAI,EAAE,gBAAgB,CAAC,YAAY;oBACnC,KAAK;oBACL,QAAQ;oBACR,kBAAkB,EAAE,qBAAqB;iBAC1C,CAAC,CAAC;gBACH,SAAS;YACX,CAAC;YAED,IAAI,qBAAqB,CAAC,oBAAoB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC1D,aAAa,CAAC,IAAI,CAAC;oBACjB,IAAI,EAAE,gBAAgB,CAAC,oBAAoB;oBAC3C,KAAK;oBACL,kBAAkB,EAAE,qBAAqB;iBAC1C,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAED,OAAO,aAAa,CAAC;IACvB,CAAC;IAEO,iBAAiB;QACvB,MAAM,SAAS,GAA6B,EAAE,CAAC;QAC/C,KAAK,MAAM,CAAC,QAAQ,EAAE,sBAAsB,CAAC,IAAI,IAAI,CAAC,gBAAgB,CAAC,OAAO,EAAE,EAAE,CAAC;YACjF,6IAA6I;YAC7I,IAAI,CAAC,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,QAAQ,KAAK,QAAQ,CAAC,EAAE,CAAC;gBACxE,sEAAsE;gBACtE,IACE,IAAI,CAAC,qBAAqB,CAAC,sBAAsB,CAAC,WAAW,CAAC,MAAM,EAAE,sBAAsB,CAAC,WAAW,CAAC,IAAI,CAAC,EAC9G,CAAC;oBACD,SAAS,CAAC,IAAI,CAAC,sBAAsB,CAAC,CAAC;gBACzC,CAAC;YACH,CAAC;QACH,CAAC;QAED,OAAO,SAAS,CAAC;IACnB,CAAC;IAEO,qBAAqB,CAAC,MAAc,EAAE,SAAiB;QAC7D,OAAO,IAAI,CAAC,OAAO,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,YAAY,EAAE,EAAE,CACrD,YAAY,CAAC,OAAO,CAAC;YACnB,aAAa,EAAE,IAAI,CAAC,iBAAiB,CAAC,aAAa;YACnD,MAAM,EAAE,MAAM;YACd,IAAI,EAAE,SAAS;SAChB,CAAC,CACH,CAAC;IACJ,CAAC;CACF"}
@@ -1,6 +1,7 @@
1
1
  import { replication } from '@powersync/service-core';
2
2
  import { container, logger as defaultLogger } from '@powersync/lib-services-framework';
3
3
  import { CDCDataExpiredError, CDCStream } from './CDCStream.js';
4
+ import { POWERSYNC_CHECKPOINTS_TABLE } from '../utils/mssql.js';
4
5
  export class CDCReplicationJob extends replication.AbstractReplicationJob {
5
6
  connectionFactory;
6
7
  lastStream = null;
@@ -12,7 +13,14 @@ export class CDCReplicationJob extends replication.AbstractReplicationJob {
12
13
  this.cdcReplicationJobOptions = options;
13
14
  }
14
15
  async keepAlive() {
15
- // TODO Might need to leverage checkpoints table as a keepAlive
16
+ if (this.lastStream) {
17
+ try {
18
+ await this.lastStream.keepAlive();
19
+ }
20
+ catch (e) {
21
+ this.logger.warn(`KeepAlive failed, unable to write an update to the ${POWERSYNC_CHECKPOINTS_TABLE} table`, e);
22
+ }
23
+ }
16
24
  }
17
25
  async replicate() {
18
26
  try {
@@ -1 +1 @@
1
- {"version":3,"file":"CDCReplicationJob.js","sourceRoot":"","sources":["../../src/replication/CDCReplicationJob.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,yBAAyB,CAAC;AAEtD,OAAO,EAAE,SAAS,EAAE,MAAM,IAAI,aAAa,EAAE,MAAM,mCAAmC,CAAC;AACvF,OAAO,EAAE,mBAAmB,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAQhE,MAAM,OAAO,iBAAkB,SAAQ,WAAW,CAAC,sBAAsB;IAC/D,iBAAiB,CAAgC;IACjD,UAAU,GAAqB,IAAI,CAAC;IACpC,wBAAwB,CAA2B;IAE3D,YAAY,OAAiC;QAC3C,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,MAAM,GAAG,aAAa,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,cAAc,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC;QAC/F,IAAI,CAAC,iBAAiB,GAAG,OAAO,CAAC,iBAAiB,CAAC;QACnD,IAAI,CAAC,wBAAwB,GAAG,OAAO,CAAC;IAC1C,CAAC;IAED,KAAK,CAAC,SAAS;QACb,+DAA+D;IACjE,CAAC;IAED,KAAK,CAAC,SAAS;QACb,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,aAAa,EAAE,CAAC;QAC7B,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,kBAAkB;YAClB,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC;gBACpB,wBAAwB;gBACxB,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,mBAAmB,EAAE,CAAC,CAAC,CAAC;gBAC1C,IAAI,CAAC,CAAC,KAAK,IAAI,IAAI,EAAE,CAAC;oBACpB,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC;gBACtC,CAAC;gBAED,SAAS,CAAC,QAAQ,CAAC,gBAAgB,CAAC,CAAC,EAAE;oBACrC,QAAQ,EAAE,EAAE;iBACb,CAAC,CAAC;gBAEH,4BAA4B;gBAC5B,IAAI,CAAC,WAAW,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC;YAClC,CAAC;YACD,IAAI,CAAC,YAAY,mBAAmB,EAAE,CAAC;gBACrC,0DAA0D;gBAC1D,MAAM,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,kBAAkB,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;YAC/E,CAAC;QACH,CAAC;gBAAS,CAAC;YACT,IAAI,CAAC,eAAe,CAAC,KAAK,EAAE,CAAC;QAC/B,CAAC;IACH,CAAC;IAED,KAAK,CAAC,aAAa;QACjB,+DAA+D;QAC/D,gEAAgE;QAChE,uCAAuC;QACvC,MAAM,iBAAiB,GAAG,IAAI,CAAC,iBAAiB,CAAC,MAAM,CAAC;YACtD,iBAAiB,EAAE,MAAM;YACzB,GAAG,EAAE,CAAC;SACP,CAAC,CAAC;QACH,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,WAAW,EAAE,gBAAgB,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,eAAe,CAAC,MAAM,EAAE,CAAC,CAAC;YAClF,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;gBACnB,OAAO;YACT,CAAC;YACD,MAAM,MAAM,GAAG,IAAI,SAAS,CAAC;gBAC3B,MAAM,EAAE,IAAI,CAAC,MAAM;gBACnB,WAAW,EAAE,IAAI,CAAC,eAAe,CAAC,MAAM;gBACxC,OAAO,EAAE,IAAI,CAAC,OAAO,CAAC,OAAO;gBAC7B,OAAO,EAAE,IAAI,CAAC,OAAO,CAAC,OAAO;gBAC7B,WAAW,EAAE,iBAAiB;gBAC9B,gBAAgB,EAAE,IAAI,CAAC,wBAAwB,CAAC,gBAAgB;aACjE,CAAC,CAAC;YACH,IAAI,CAAC,UAAU,GAAG,MAAM,CAAC;YACzB,MAAM,MAAM,CAAC,SAAS,EAAE,CAAC;QAC3B,CAAC;gBAAS,CAAC;YACT,MAAM,iBAAiB,CAAC,GAAG,EAAE,CAAC;QAChC,CAAC;IACH,CAAC;IAED,KAAK,CAAC,uBAAuB;QAC3B,OAAO,IAAI,CAAC,UAAU,EAAE,uBAAuB,EAAE,CAAC;IACpD,CAAC;CACF"}
1
+ {"version":3,"file":"CDCReplicationJob.js","sourceRoot":"","sources":["../../src/replication/CDCReplicationJob.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,yBAAyB,CAAC;AAEtD,OAAO,EAAE,SAAS,EAAE,MAAM,IAAI,aAAa,EAAE,MAAM,mCAAmC,CAAC;AACvF,OAAO,EAAE,mBAAmB,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAEhE,OAAO,EAAE,2BAA2B,EAAE,MAAM,mBAAmB,CAAC;AAOhE,MAAM,OAAO,iBAAkB,SAAQ,WAAW,CAAC,sBAAsB;IAC/D,iBAAiB,CAAgC;IACjD,UAAU,GAAqB,IAAI,CAAC;IACpC,wBAAwB,CAA2B;IAE3D,YAAY,OAAiC;QAC3C,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,MAAM,GAAG,aAAa,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,cAAc,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC;QAC/F,IAAI,CAAC,iBAAiB,GAAG,OAAO,CAAC,iBAAiB,CAAC;QACnD,IAAI,CAAC,wBAAwB,GAAG,OAAO,CAAC;IAC1C,CAAC;IAED,KAAK,CAAC,SAAS;QACb,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YACpB,IAAI,CAAC;gBACH,MAAM,IAAI,CAAC,UAAU,CAAC,SAAS,EAAE,CAAC;YACpC,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACX,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,sDAAsD,2BAA2B,QAAQ,EAAE,CAAC,CAAC,CAAC;YACjH,CAAC;QACH,CAAC;IACH,CAAC;IAED,KAAK,CAAC,SAAS;QACb,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,aAAa,EAAE,CAAC;QAC7B,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,kBAAkB;YAClB,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC;gBACpB,wBAAwB;gBACxB,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,mBAAmB,EAAE,CAAC,CAAC,CAAC;gBAC1C,IAAI,CAAC,CAAC,KAAK,IAAI,IAAI,EAAE,CAAC;oBACpB,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC;gBACtC,CAAC;gBAED,SAAS,CAAC,QAAQ,CAAC,gBAAgB,CAAC,CAAC,EAAE;oBACrC,QAAQ,EAAE,EAAE;iBACb,CAAC,CAAC;gBAEH,4BAA4B;gBAC5B,IAAI,CAAC,WAAW,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC;YAClC,CAAC;YACD,IAAI,CAAC,YAAY,mBAAmB,EAAE,CAAC;gBACrC,0DAA0D;gBAC1D,MAAM,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,kBAAkB,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;YAC/E,CAAC;QACH,CAAC;gBAAS,CAAC;YACT,IAAI,CAAC,eAAe,CAAC,KAAK,EAAE,CAAC;QAC/B,CAAC;IACH,CAAC;IAED,KAAK,CAAC,aAAa;QACjB,+DAA+D;QAC/D,gEAAgE;QAChE,uCAAuC;QACvC,MAAM,iBAAiB,GAAG,IAAI,CAAC,iBAAiB,CAAC,MAAM,CAAC;YACtD,iBAAiB,EAAE,MAAM;YACzB,GAAG,EAAE,CAAC;SACP,CAAC,CAAC;QACH,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,WAAW,EAAE,gBAAgB,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,eAAe,CAAC,MAAM,EAAE,CAAC,CAAC;YAClF,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;gBACnB,OAAO;YACT,CAAC;YACD,MAAM,MAAM,GAAG,IAAI,SAAS,CAAC;gBAC3B,MAAM,EAAE,IAAI,CAAC,MAAM;gBACnB,WAAW,EAAE,IAAI,CAAC,eAAe,CAAC,MAAM;gBACxC,OAAO,EAAE,IAAI,CAAC,OAAO,CAAC,OAAO;gBAC7B,OAAO,EAAE,IAAI,CAAC,OAAO,CAAC,OAAO;gBAC7B,WAAW,EAAE,iBAAiB;gBAC9B,gBAAgB,EAAE,IAAI,CAAC,wBAAwB,CAAC,gBAAgB;aACjE,CAAC,CAAC;YACH,IAAI,CAAC,UAAU,GAAG,MAAM,CAAC;YACzB,MAAM,MAAM,CAAC,SAAS,EAAE,CAAC;QAC3B,CAAC;gBAAS,CAAC;YACT,MAAM,iBAAiB,CAAC,GAAG,EAAE,CAAC;QAChC,CAAC;IACH,CAAC;IAED,KAAK,CAAC,uBAAuB;QAC3B,OAAO,IAAI,CAAC,UAAU,EAAE,uBAAuB,EAAE,CAAC;IACpD,CAAC;CACF"}
@@ -4,7 +4,10 @@ import { TablePattern } from '@powersync/service-sync-rules';
4
4
  import { MSSQLConnectionManager } from './MSSQLConnectionManager.js';
5
5
  import sql from 'mssql';
6
6
  import { MSSQLSourceTable } from '../common/MSSQLSourceTable.js';
7
+ import { MSSQLSourceTableCache } from '../common/MSSQLSourceTableCache.js';
8
+ import { SchemaChange } from './CDCPoller.js';
7
9
  import { AdditionalConfig } from '../types/types.js';
10
+ import { CaptureInstance } from '../common/CaptureInstance.js';
8
11
  export interface CDCStreamOptions {
9
12
  connections: MSSQLConnectionManager;
10
13
  storage: storage.SyncRulesBucketStorage;
@@ -18,15 +21,27 @@ export interface CDCStreamOptions {
18
21
  */
19
22
  snapshotBatchSize?: number;
20
23
  additionalConfig: AdditionalConfig;
24
+ /**
25
+ * Override schema check interval, defaults is 60 seconds.
26
+ */
27
+ schemaCheckIntervalMs?: number;
21
28
  }
22
29
  export declare enum SnapshotStatus {
23
- IN_PROGRESS = "in-progress",
30
+ INITIAL = "initial",
31
+ RESUME = "resume",
24
32
  DONE = "done",
25
- RESTART_REQUIRED = "restart-required"
33
+ LIMITED_RESNAPSHOT = "limited-resnapshot"
26
34
  }
27
35
  export interface SnapshotStatusResult {
28
36
  status: SnapshotStatus;
29
37
  snapshotLSN: string | null;
38
+ /**
39
+ * Under certain circumstances it may be necessary to re-snapshot specific tables:
40
+ * * A new capture instance has been created for a table.
41
+ * * A table has been renamed.
42
+ * * The retention threshold has been exceeded for a table.
43
+ */
44
+ specificTablesToResnapshot?: MSSQLSourceTable[];
30
45
  }
31
46
  export declare class CDCConfigurationError extends Error {
32
47
  constructor(message: string);
@@ -48,7 +63,7 @@ export declare class CDCStream {
48
63
  private readonly connections;
49
64
  private readonly abortSignal;
50
65
  private readonly logger;
51
- private tableCache;
66
+ tableCache: MSSQLSourceTableCache;
52
67
  /**
53
68
  * Time of the oldest uncommitted change, according to the source db.
54
69
  * This is used to determine the replication lag.
@@ -70,7 +85,7 @@ export declare class CDCStream {
70
85
  replicate(): Promise<void>;
71
86
  populateTableCache(): Promise<void>;
72
87
  getQualifiedTableNames(batch: storage.BucketStorageBatch, tablePattern: TablePattern): Promise<MSSQLSourceTable[]>;
73
- processTable(batch: storage.BucketStorageBatch, table: SourceEntityDescriptor, snapshot: boolean): Promise<MSSQLSourceTable>;
88
+ processTable(batch: storage.BucketStorageBatch, table: SourceEntityDescriptor, captureInstance: CaptureInstance | null, snapshot: boolean): Promise<MSSQLSourceTable>;
74
89
  private snapshotTableInTx;
75
90
  private snapshotTable;
76
91
  /**
@@ -95,6 +110,18 @@ export declare class CDCStream {
95
110
  private checkSnapshotStatus;
96
111
  streamChanges(): Promise<void>;
97
112
  private createEventHandler;
113
+ handleSchemaChange(batch: storage.BucketStorageBatch, change: SchemaChange): Promise<void>;
114
+ private handleCreateOrUpdateTable;
115
+ /**
116
+ * There is very little that can be automatically done to handle the column changes other than to warn the user about the schema drift.
117
+ *
118
+ * Due to the way CDC works, users are prevented from making column schema changes that affect the replication identities of a table.
119
+ * If changes like that are required, CDC has to be disabled and re-enabled for the table. This would then be handled by the detection of the new
120
+ * capture instance.
121
+ * @param table
122
+ * @param captureInstance
123
+ */
124
+ private handleColumnChanges;
98
125
  /**
99
126
  * Convert CDC row data to SqliteRow format.
100
127
  * CDC rows include table columns plus CDC metadata columns (__$operation, __$start_lsn, etc.).
@@ -103,4 +130,8 @@ export declare class CDCStream {
103
130
  private toSqliteRow;
104
131
  getReplicationLagMillis(): Promise<number | undefined>;
105
132
  private touch;
133
+ /**
134
+ * Creates an update in the source database to ensure regular checkpoints via the CDC
135
+ */
136
+ keepAlive(): Promise<void>;
106
137
  }
@@ -1,20 +1,21 @@
1
- import { container, DatabaseConnectionError, ErrorCode, logger as defaultLogger, ReplicationAbortedError, ReplicationAssertionError, ServiceAssertionError } from '@powersync/lib-services-framework';
1
+ import { container, DatabaseConnectionError, ErrorCode, logger as defaultLogger, ReplicationAbortedError, ReplicationAssertionError } from '@powersync/lib-services-framework';
2
2
  import { getUuidReplicaIdentityBson, storage } from '@powersync/service-core';
3
3
  import { ReplicationMetric } from '@powersync/service-types';
4
4
  import { BatchedSnapshotQuery, SimpleSnapshotQuery } from './MSSQLSnapshotQuery.js';
5
5
  import { getReplicationIdentityColumns, getTablesFromPattern } from '../utils/schema.js';
6
- import { checkSourceConfiguration, createCheckpoint, getCaptureInstance, getLatestLSN, getLatestReplicatedLSN, isIColumnMetadata, isTableEnabledForCDC, isWithinRetentionThreshold, toQualifiedTableName } from '../utils/mssql.js';
6
+ import { checkRetentionThresholds, checkSourceConfiguration, createCheckpoint, getCaptureInstances, getLatestLSN, getLatestReplicatedLSN, isIColumnMetadata } from '../utils/mssql.js';
7
7
  import sql from 'mssql';
8
8
  import { CDCToSqliteRow, toSqliteInputRow } from '../common/mssqls-to-sqlite.js';
9
9
  import { LSN } from '../common/LSN.js';
10
10
  import { MSSQLSourceTable } from '../common/MSSQLSourceTable.js';
11
11
  import { MSSQLSourceTableCache } from '../common/MSSQLSourceTableCache.js';
12
- import { CDCPoller } from './CDCPoller.js';
12
+ import { CDCPoller, SchemaChangeType } from './CDCPoller.js';
13
13
  export var SnapshotStatus;
14
14
  (function (SnapshotStatus) {
15
- SnapshotStatus["IN_PROGRESS"] = "in-progress";
15
+ SnapshotStatus["INITIAL"] = "initial";
16
+ SnapshotStatus["RESUME"] = "resume";
16
17
  SnapshotStatus["DONE"] = "done";
17
- SnapshotStatus["RESTART_REQUIRED"] = "restart-required";
18
+ SnapshotStatus["LIMITED_RESNAPSHOT"] = "limited-resnapshot";
18
19
  })(SnapshotStatus || (SnapshotStatus = {}));
19
20
  export class CDCConfigurationError extends Error {
20
21
  constructor(message) {
@@ -108,13 +109,10 @@ export class CDCStream {
108
109
  logger: this.logger,
109
110
  zeroLSN: LSN.ZERO,
110
111
  defaultSchema: this.defaultSchema,
111
- storeCurrentData: true
112
+ storeCurrentData: false
112
113
  }, async (batch) => {
113
114
  for (let tablePattern of sourceTables) {
114
- const tables = await this.getQualifiedTableNames(batch, tablePattern);
115
- for (const table of tables) {
116
- this.tableCache.set(table);
117
- }
115
+ await this.getQualifiedTableNames(batch, tablePattern);
118
116
  }
119
117
  });
120
118
  }
@@ -122,18 +120,11 @@ export class CDCStream {
122
120
  if (tablePattern.connectionTag != this.connections.connectionTag) {
123
121
  return [];
124
122
  }
123
+ const captureInstances = await getCaptureInstances({ connectionManager: this.connections });
125
124
  const matchedTables = await getTablesFromPattern(this.connections, tablePattern);
126
125
  const tables = [];
127
126
  for (const matchedTable of matchedTables) {
128
- const isEnabled = await isTableEnabledForCDC({
129
- connectionManager: this.connections,
130
- table: matchedTable.name,
131
- schema: matchedTable.schema
132
- });
133
- if (!isEnabled) {
134
- this.logger.info(`Skipping ${matchedTable.schema}.${matchedTable.name} - table is not enabled for CDC.`);
135
- continue;
136
- }
127
+ const captureInstanceDetails = captureInstances.get(matchedTable.objectId);
137
128
  // TODO: Check RLS settings for table
138
129
  const replicaIdColumns = await getReplicationIdentityColumns({
139
130
  connectionManager: this.connections,
@@ -145,12 +136,12 @@ export class CDCStream {
145
136
  schema: matchedTable.schema,
146
137
  objectId: matchedTable.objectId,
147
138
  replicaIdColumns: replicaIdColumns.columns
148
- }, false);
139
+ }, captureInstanceDetails?.instances[0] ?? null, false);
149
140
  tables.push(table);
150
141
  }
151
142
  return tables;
152
143
  }
153
- async processTable(batch, table, snapshot) {
144
+ async processTable(batch, table, captureInstance, snapshot) {
154
145
  if (!table.objectId && typeof table.objectId != 'number') {
155
146
  throw new ReplicationAssertionError(`objectId expected, got ${typeof table.objectId}`);
156
147
  }
@@ -161,35 +152,25 @@ export class CDCStream {
161
152
  entity_descriptor: table,
162
153
  sync_rules: this.syncRules
163
154
  });
164
- const captureInstance = await getCaptureInstance({
165
- connectionManager: this.connections,
166
- tableName: resolved.table.name,
167
- schema: resolved.table.schema
168
- });
155
+ const resolvedTable = new MSSQLSourceTable(resolved.table);
169
156
  if (!captureInstance) {
170
- throw new ServiceAssertionError(`Missing capture instance for table ${toQualifiedTableName(resolved.table.schema, resolved.table.name)}`);
157
+ this.logger.warn(`Missing capture instance for table ${resolvedTable.toQualifiedName()}. This table will not be replicated until CDC is enabled for it.`);
171
158
  }
172
- const resolvedTable = new MSSQLSourceTable({
173
- sourceTable: resolved.table,
174
- captureInstance: captureInstance
175
- });
159
+ else {
160
+ resolvedTable.setCaptureInstance(captureInstance);
161
+ }
162
+ this.tableCache.set(resolvedTable);
176
163
  // Drop conflicting tables. This includes for example renamed tables.
177
164
  await batch.drop(resolved.dropTables);
178
165
  // Snapshot if:
179
- // 1. Snapshot is requested (false for initial snapshot, since that process handles it elsewhere)
180
- // 2. Snapshot is not already done, AND:
181
- // 3. The table is used in sync rules.
182
- const shouldSnapshot = snapshot && !resolved.table.snapshotComplete && resolved.table.syncAny;
166
+ // 1. The table is in the sync rules and snapshot is requested, or not already done.
167
+ // 2. AND the table is enabled for CDC with a valid capture instance.
168
+ const shouldSnapshot = snapshot && !resolved.table.snapshotComplete && resolved.table.syncAny && resolvedTable.enabledForCDC();
183
169
  if (shouldSnapshot) {
184
170
  // Truncate this table in case a previous snapshot was interrupted.
185
171
  await batch.truncate([resolved.table]);
186
172
  // Start the snapshot inside a transaction.
187
- try {
188
- await this.snapshotTableInTx(batch, resolvedTable);
189
- }
190
- finally {
191
- // TODO Cleanup?
192
- }
173
+ await this.snapshotTableInTx(batch, resolvedTable);
193
174
  }
194
175
  return resolvedTable;
195
176
  }
@@ -217,6 +198,7 @@ export class CDCStream {
217
198
  // Side note: A ROLLBACK would probably also be fine here, since we only read in this transaction.
218
199
  await transaction.commit();
219
200
  const [updatedSourceTable] = await batch.markTableSnapshotDone([table.sourceTable], postSnapshotLSN.toString());
201
+ this.logger.info(`Snapshot of ${table.toQualifiedName()} completed. Post-snapshot LSN: ${postSnapshotLSN.toString()}`);
220
202
  this.tableCache.updateSourceTable(updatedSourceTable);
221
203
  }
222
204
  catch (e) {
@@ -237,15 +219,15 @@ export class CDCStream {
237
219
  const orderByKey = table.sourceTable.replicaIdColumns[0];
238
220
  query = new BatchedSnapshotQuery(transaction, table, this.snapshotBatchSize, table.sourceTable.snapshotStatus?.lastKey ?? null);
239
221
  if (table.sourceTable.snapshotStatus?.lastKey != null) {
240
- this.logger.info(`Replicating ${table.toQualifiedName()} ${table.sourceTable.formatSnapshotProgress()} - resuming from ${orderByKey.name} > ${query.lastKey}`);
222
+ this.logger.info(`Snapshotting ${table.toQualifiedName()} ${table.sourceTable.formatSnapshotProgress()} - resuming from ${orderByKey.name} > ${query.lastKey}`);
241
223
  }
242
224
  else {
243
- this.logger.info(`Replicating ${table.toQualifiedName()} ${table.sourceTable.formatSnapshotProgress()} - resumable`);
225
+ this.logger.info(`Snapshotting ${table.toQualifiedName()} ${table.sourceTable.formatSnapshotProgress()} - resumable`);
244
226
  }
245
227
  }
246
228
  else {
247
229
  // Fallback case - query the entire table
248
- this.logger.info(`Replicating ${table.toQualifiedName()} ${table.sourceTable.formatSnapshotProgress()} - not resumable`);
230
+ this.logger.info(`Snapshotting ${table.toQualifiedName()} ${table.sourceTable.formatSnapshotProgress()} - not resumable`);
249
231
  query = new SimpleSnapshotQuery(transaction, table);
250
232
  replicatedCount = 0;
251
233
  }
@@ -305,7 +287,6 @@ export class CDCStream {
305
287
  totalEstimatedCount: totalEstimatedCount
306
288
  });
307
289
  this.tableCache.updateSourceTable(updatedSourceTable);
308
- this.logger.info(`Replicating ${table.toQualifiedName()} ${table.sourceTable.formatSnapshotProgress()}`);
309
290
  if (this.abortSignal.aborted) {
310
291
  // We only abort after flushing
311
292
  throw new ReplicationAbortedError(`Initial replication interrupted`);
@@ -314,6 +295,9 @@ export class CDCStream {
314
295
  if (batchReplicatedCount < this.snapshotBatchSize) {
315
296
  hasRemainingData = false;
316
297
  }
298
+ else {
299
+ this.logger.info(`Snapshotting ${table.toQualifiedName()} ${table.sourceTable.formatSnapshotProgress()}`);
300
+ }
317
301
  }
318
302
  }
319
303
  /**
@@ -337,12 +321,7 @@ export class CDCStream {
337
321
  * and starts again from scratch.
338
322
  */
339
323
  async startInitialReplication(snapshotStatus) {
340
- let { status, snapshotLSN } = snapshotStatus;
341
- if (status === SnapshotStatus.RESTART_REQUIRED) {
342
- this.logger.info(`Snapshot restart required, clearing state.`);
343
- // This happens if the last replicated checkpoint LSN is no longer available in the CDC tables.
344
- await this.storage.clear({ signal: this.abortSignal });
345
- }
324
+ let { status, snapshotLSN, specificTablesToResnapshot } = snapshotStatus;
346
325
  await this.storage.startBatch({
347
326
  logger: this.logger,
348
327
  zeroLSN: LSN.ZERO,
@@ -350,15 +329,26 @@ export class CDCStream {
350
329
  storeCurrentData: false,
351
330
  skipExistingRows: true
352
331
  }, async (batch) => {
353
- if (snapshotLSN == null) {
354
- // First replication attempt - set the snapshot LSN to the current LSN before starting
355
- snapshotLSN = (await getLatestReplicatedLSN(this.connections)).toString();
356
- await batch.setResumeLsn(snapshotLSN);
357
- const latestLSN = (await getLatestLSN(this.connections)).toString();
358
- this.logger.info(`Marking snapshot at ${snapshotLSN}, Latest DB LSN ${latestLSN}.`);
359
- }
360
- else {
361
- this.logger.info(`Resuming snapshot at ${snapshotLSN}.`);
332
+ switch (status) {
333
+ case SnapshotStatus.INITIAL:
334
+ // First replication attempt - set the snapshot LSN to the current LSN before starting
335
+ snapshotLSN = (await getLatestReplicatedLSN(this.connections)).toString();
336
+ await batch.setResumeLsn(snapshotLSN);
337
+ const latestLSN = (await getLatestLSN(this.connections)).toString();
338
+ this.logger.info(`Marking snapshot at ${snapshotLSN}, Latest DB LSN ${latestLSN}.`);
339
+ break;
340
+ case SnapshotStatus.RESUME:
341
+ this.logger.info(`Resuming snapshot at ${snapshotLSN}.`);
342
+ break;
343
+ case SnapshotStatus.LIMITED_RESNAPSHOT:
344
+ for (const table of specificTablesToResnapshot) {
345
+ await batch.drop([table.sourceTable]);
346
+ // Update table in the table cache
347
+ await this.processTable(batch, table.sourceTable, table.captureInstance, false);
348
+ }
349
+ break;
350
+ default:
351
+ throw new ReplicationAssertionError(`Unsupported snapshot status: ${status}`);
362
352
  }
363
353
  const tablesToSnapshot = [];
364
354
  for (const table of this.tableCache.getAll()) {
@@ -366,13 +356,16 @@ export class CDCStream {
366
356
  this.logger.info(`Skipping table [${table.toQualifiedName()}] - snapshot already done.`);
367
357
  continue;
368
358
  }
359
+ if (!table.enabledForCDC()) {
360
+ this.logger.info(`Skipping table [${table.toQualifiedName()}] - not enabled for CDC.`);
361
+ continue;
362
+ }
369
363
  const count = await this.estimatedCountNumber(table);
370
364
  const updatedSourceTable = await batch.updateTableProgress(table.sourceTable, {
371
365
  totalEstimatedCount: count
372
366
  });
373
367
  this.tableCache.updateSourceTable(updatedSourceTable);
374
368
  tablesToSnapshot.push(table);
375
- this.logger.info(`To replicate: ${table.toQualifiedName()} ${table.sourceTable.formatSnapshotProgress()}`);
376
369
  }
377
370
  for (const table of tablesToSnapshot) {
378
371
  await this.snapshotTableInTx(batch, table);
@@ -383,7 +376,12 @@ export class CDCStream {
383
376
  const postSnapshotLSN = await getLatestLSN(this.connections);
384
377
  await batch.markAllSnapshotDone(postSnapshotLSN.toString());
385
378
  await batch.commit(snapshotLSN);
386
- this.logger.info(`Snapshot done. Need to replicate from ${snapshotLSN} to ${postSnapshotLSN} to be consistent`);
379
+ if (tablesToSnapshot.length > 0) {
380
+ this.logger.info(`All snapshots done. Need to replicate from ${snapshotLSN} to ${postSnapshotLSN} to be consistent.`);
381
+ }
382
+ else {
383
+ this.logger.info(`No tables to snapshot. Need to replicate from ${snapshotLSN}.`);
384
+ }
387
385
  });
388
386
  }
389
387
  async initReplication() {
@@ -396,6 +394,9 @@ export class CDCStream {
396
394
  if (snapshotStatus.status !== SnapshotStatus.DONE) {
397
395
  await this.startInitialReplication(snapshotStatus);
398
396
  }
397
+ else {
398
+ this.logger.info(`Initial replication already done`);
399
+ }
399
400
  }
400
401
  /**
401
402
  * Checks if the initial sync has already been completed and if updates from the last checkpoint are still available
@@ -404,23 +405,44 @@ export class CDCStream {
404
405
  async checkSnapshotStatus() {
405
406
  const status = await this.storage.getStatus();
406
407
  if (status.snapshot_done && status.checkpoint_lsn) {
408
+ const additionalTablesToSnapshot = new Set();
409
+ const newTables = this.tableCache.getAll().filter((table) => !table.sourceTable.snapshotComplete);
410
+ if (newTables.length > 0) {
411
+ this.logger.info(`Detected new table(s) [${newTables.map((table) => table.toQualifiedName()).join(', ')}] that have not been snapshotted yet.`);
412
+ newTables.forEach((table) => additionalTablesToSnapshot.add(table));
413
+ }
407
414
  // Snapshot is done, but we still need to check that the last known checkpoint LSN is still
408
- // within the threshold of the CDC tables
409
- this.logger.info(`Initial replication already done`);
415
+ // within the retention threshold of the CDC tables
410
416
  const lastCheckpointLSN = LSN.fromString(status.checkpoint_lsn);
411
417
  // Check that the CDC tables still have valid data
412
- const isAvailable = await isWithinRetentionThreshold({
418
+ const tablesOutsideRetentionThreshold = await checkRetentionThresholds({
413
419
  checkpointLSN: lastCheckpointLSN,
414
420
  tables: this.tableCache.getAll(),
415
421
  connectionManager: this.connections
416
422
  });
417
- if (!isAvailable) {
418
- this.logger.warn(`Updates from the last checkpoint are no longer available in the CDC instance, starting initial replication again.`);
423
+ if (tablesOutsideRetentionThreshold.length > 0) {
424
+ this.logger.warn(`Updates from the last checkpoint are no longer available in the CDC instances of the following table(s): ${tablesOutsideRetentionThreshold.map((table) => table.toQualifiedName()).join(', ')}.`);
425
+ tablesOutsideRetentionThreshold.forEach((table) => additionalTablesToSnapshot.add(table));
426
+ }
427
+ if (additionalTablesToSnapshot.size > 0) {
428
+ return {
429
+ status: SnapshotStatus.LIMITED_RESNAPSHOT,
430
+ snapshotLSN: status.checkpoint_lsn,
431
+ specificTablesToResnapshot: Array.from(additionalTablesToSnapshot)
432
+ };
433
+ }
434
+ else {
435
+ return {
436
+ status: SnapshotStatus.DONE,
437
+ snapshotLSN: null
438
+ };
419
439
  }
420
- return { status: isAvailable ? SnapshotStatus.DONE : SnapshotStatus.RESTART_REQUIRED, snapshotLSN: null };
421
440
  }
422
441
  else {
423
- return { status: SnapshotStatus.IN_PROGRESS, snapshotLSN: status.snapshot_lsn };
442
+ return {
443
+ status: status.snapshot_lsn != null ? SnapshotStatus.RESUME : SnapshotStatus.INITIAL,
444
+ snapshotLSN: status.snapshot_lsn
445
+ };
424
446
  }
425
447
  }
426
448
  async streamChanges() {
@@ -435,15 +457,16 @@ export class CDCStream {
435
457
  throw new ReplicationAssertionError(`No LSN found to resume replication from.`);
436
458
  }
437
459
  const startLSN = LSN.fromString(batch.resumeFromLsn);
438
- const sourceTables = this.tableCache.getAll();
439
460
  const eventHandler = this.createEventHandler(batch);
440
461
  const poller = new CDCPoller({
441
462
  connectionManager: this.connections,
442
463
  eventHandler,
443
- sourceTables,
464
+ getReplicatedTables: () => this.tableCache.getAll(),
465
+ sourceTables: this.syncRules.getSourceTables(),
444
466
  startLSN,
445
467
  logger: this.logger,
446
- additionalConfig: this.options.additionalConfig
468
+ additionalConfig: this.options.additionalConfig,
469
+ schemaCheckIntervalMs: this.options.schemaCheckIntervalMs
447
470
  });
448
471
  this.abortSignal.addEventListener('abort', async () => {
449
472
  await poller.stop();
@@ -499,11 +522,87 @@ export class CDCStream {
499
522
  this.isStartingReplication = false;
500
523
  }
501
524
  },
502
- onSchemaChange: async () => {
503
- // TODO: Handle schema changes
525
+ onSchemaChange: async (schemaChange) => {
526
+ await this.handleSchemaChange(batch, schemaChange);
504
527
  }
505
528
  };
506
529
  }
530
+ async handleSchemaChange(batch, change) {
531
+ let actionedSchemaChange = true;
532
+ switch (change.type) {
533
+ case SchemaChangeType.TABLE_RENAME:
534
+ const fromTable = change.table;
535
+ this.logger.info(`Table ${fromTable.toQualifiedName()} has been renamed ${change.newTable ? `to [${change.newTable.name}].` : '.'}`);
536
+ // Old table needs to be cleaned up
537
+ await batch.drop([fromTable.sourceTable]);
538
+ this.tableCache.delete(fromTable.objectId);
539
+ if (change.newTable) {
540
+ await this.handleCreateOrUpdateTable(batch, change.newTable, change.newCaptureInstance);
541
+ }
542
+ break;
543
+ case SchemaChangeType.TABLE_CREATE:
544
+ await this.handleCreateOrUpdateTable(batch, change.newTable, change.newCaptureInstance);
545
+ break;
546
+ case SchemaChangeType.TABLE_COLUMN_CHANGES:
547
+ await this.handleColumnChanges(change.table, change.newCaptureInstance);
548
+ actionedSchemaChange = false;
549
+ break;
550
+ case SchemaChangeType.NEW_CAPTURE_INSTANCE:
551
+ this.logger.info(`New CDC capture instance detected for table ${change.table.toQualifiedName()}. Re-snapshotting table...`);
552
+ await batch.drop([change.table.sourceTable]);
553
+ this.tableCache.delete(change.table.objectId);
554
+ await this.handleCreateOrUpdateTable(batch, change.table.sourceTable, change.newCaptureInstance);
555
+ break;
556
+ case SchemaChangeType.TABLE_DROP:
557
+ await batch.drop([change.table.sourceTable]);
558
+ this.tableCache.delete(change.table.objectId);
559
+ break;
560
+ case SchemaChangeType.MISSING_CAPTURE_INSTANCE:
561
+ // Stop replication for this table until CDC is re-enabled.
562
+ this.logger.warn(`Table ${change.table.toQualifiedName()} has been disabled for CDC. Re-enable CDC to continue replication.`);
563
+ change.table.clearCaptureInstance();
564
+ actionedSchemaChange = false;
565
+ break;
566
+ default:
567
+ throw new ReplicationAssertionError(`Unknown schema change type: ${change.type}`);
568
+ }
569
+ // Create a new checkpoint after the schema change
570
+ if (actionedSchemaChange) {
571
+ await createCheckpoint(this.connections);
572
+ }
573
+ }
574
+ async handleCreateOrUpdateTable(batch, table, captureInstance) {
575
+ const replicaIdColumns = await getReplicationIdentityColumns({
576
+ connectionManager: this.connections,
577
+ tableName: table.name,
578
+ schema: table.schema
579
+ });
580
+ await this.processTable(batch, {
581
+ name: table.name,
582
+ schema: table.schema,
583
+ objectId: table.objectId,
584
+ replicaIdColumns: replicaIdColumns.columns
585
+ }, captureInstance, true);
586
+ }
587
+ /**
588
+ * There is very little that can be automatically done to handle the column changes other than to warn the user about the schema drift.
589
+ *
590
+ * Due to the way CDC works, users are prevented from making column schema changes that affect the replication identities of a table.
591
+ * If changes like that are required, CDC has to be disabled and re-enabled for the table. This would then be handled by the detection of the new
592
+ * capture instance.
593
+ * @param table
594
+ * @param captureInstance
595
+ */
596
+ async handleColumnChanges(table, captureInstance) {
597
+ // Check there are any new pending schema changes
598
+ if (table.captureInstance?.objectId === captureInstance.objectId &&
599
+ table.captureInstance?.pendingSchemaChanges.length === captureInstance.pendingSchemaChanges.length) {
600
+ return;
601
+ }
602
+ // New pending schema changes were detected - warn about those as well.
603
+ this.logger.warn(`Schema drift detected for table ${table.toQualifiedName()}. To ensure consistency, disable and re-enable CDC for this table.\n Pending schema changes:\n ${captureInstance.pendingSchemaChanges.join(', \n')}`);
604
+ table.captureInstance = captureInstance;
605
+ }
507
606
  /**
508
607
  * Convert CDC row data to SqliteRow format.
509
608
  * CDC rows include table columns plus CDC metadata columns (__$operation, __$start_lsn, etc.).
@@ -531,5 +630,13 @@ export class CDCStream {
531
630
  this.logger.error(`Error touching probe`, e);
532
631
  });
533
632
  }
633
+ /**
634
+ * Creates an update in the source database to ensure regular checkpoints via the CDC
635
+ */
636
+ async keepAlive() {
637
+ if (!this.isStartingReplication && !this.stopped) {
638
+ await createCheckpoint(this.connections);
639
+ }
640
+ }
534
641
  }
535
642
  //# sourceMappingURL=CDCStream.js.map