@powersync/service-module-mssql 0.5.0 → 0.6.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +32 -0
- package/dist/common/CaptureInstance.d.ts +14 -0
- package/dist/common/CaptureInstance.js +2 -0
- package/dist/common/CaptureInstance.js.map +1 -0
- package/dist/common/MSSQLSourceTable.d.ts +16 -14
- package/dist/common/MSSQLSourceTable.js +35 -16
- package/dist/common/MSSQLSourceTable.js.map +1 -1
- package/dist/replication/CDCPoller.d.ts +42 -20
- package/dist/replication/CDCPoller.js +200 -60
- package/dist/replication/CDCPoller.js.map +1 -1
- package/dist/replication/CDCReplicationJob.js +9 -1
- package/dist/replication/CDCReplicationJob.js.map +1 -1
- package/dist/replication/CDCStream.d.ts +35 -4
- package/dist/replication/CDCStream.js +181 -74
- package/dist/replication/CDCStream.js.map +1 -1
- package/dist/replication/MSSQLConnectionManager.js +16 -5
- package/dist/replication/MSSQLConnectionManager.js.map +1 -1
- package/dist/types/types.d.ts +4 -56
- package/dist/types/types.js +5 -24
- package/dist/types/types.js.map +1 -1
- package/dist/utils/deadlock.d.ts +9 -0
- package/dist/utils/deadlock.js +40 -0
- package/dist/utils/deadlock.js.map +1 -0
- package/dist/utils/mssql.d.ts +33 -15
- package/dist/utils/mssql.js +101 -99
- package/dist/utils/mssql.js.map +1 -1
- package/dist/utils/schema.d.ts +9 -0
- package/dist/utils/schema.js +34 -0
- package/dist/utils/schema.js.map +1 -1
- package/package.json +8 -8
- package/src/common/CaptureInstance.ts +15 -0
- package/src/common/MSSQLSourceTable.ts +33 -24
- package/src/replication/CDCPoller.ts +272 -72
- package/src/replication/CDCReplicationJob.ts +8 -1
- package/src/replication/CDCStream.ts +237 -90
- package/src/replication/MSSQLConnectionManager.ts +15 -5
- package/src/types/types.ts +5 -28
- package/src/utils/deadlock.ts +44 -0
- package/src/utils/mssql.ts +159 -124
- package/src/utils/schema.ts +43 -0
- package/test/src/CDCStreamTestContext.ts +9 -2
- package/test/src/env.ts +1 -1
- package/test/src/mssql-to-sqlite.test.ts +18 -10
- package/test/src/schema-changes.test.ts +470 -0
- package/test/src/util.ts +75 -12
- package/tsconfig.tsbuildinfo +1 -1
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"CDCPoller.js","sourceRoot":"","sources":["../../src/replication/CDCPoller.ts"],"names":[],"mappings":"AAAA,OAAO,EAAU,MAAM,IAAI,aAAa,EAAE,yBAAyB,EAAE,MAAM,mCAAmC,CAAC;AAC/G,OAAO,MAAM,MAAM,iBAAiB,CAAC;AAGrC,OAAO,EAAE,GAAG,EAAE,MAAM,kBAAkB,CAAC;AACvC,OAAO,GAAG,MAAM,OAAO,CAAC;AACxB,OAAO,EAAE,SAAS,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AAG5D,IAAK,SAKJ;AALD,WAAK,SAAS;IACZ,6CAAU,CAAA;IACV,6CAAU,CAAA;IACV,2DAAiB,CAAA;IACjB,yDAAgB,CAAA;AAClB,CAAC,EALI,SAAS,KAAT,SAAS,QAKb;AACD;;;;GAIG;AACH,MAAM,CAAN,IAAY,gBAMX;AAND,WAAY,gBAAgB;IAC1B,iDAA6B,CAAA;IAC7B,6CAAyB,CAAA;IACzB,qDAAiC,CAAA;IACjC,6DAAyC,CAAA;IACzC,uEAAmD,CAAA;AACrD,CAAC,EANW,gBAAgB,KAAhB,gBAAgB,QAM3B;AAgCD;;GAEG;AACH,MAAM,OAAO,SAAS;IAWD;IAVX,iBAAiB,CAAyB;IAC1C,YAAY,CAAkB;IAC9B,UAAU,CAAM;IAChB,MAAM,CAAS;IACf,aAAa,CAAe;IAE5B,SAAS,GAAY,KAAK,CAAC;IAC3B,UAAU,GAAY,KAAK,CAAC;IAC5B,SAAS,GAAY,KAAK,CAAC;IAEnC,YAAmB,OAAyB;QAAzB,YAAO,GAAP,OAAO,CAAkB;QAC1C,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,aAAa,CAAC;QAC9C,IAAI,CAAC,iBAAiB,GAAG,OAAO,CAAC,iBAAiB,CAAC;QACnD,IAAI,CAAC,YAAY,GAAG,OAAO,CAAC,YAAY,CAAC;QACzC,IAAI,CAAC,UAAU,GAAG,OAAO,CAAC,QAAQ,CAAC;QACnC,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC;IAC5B,CAAC;IAED,IAAY,gBAAgB;QAC1B,OAAO,IAAI,CAAC,OAAO,CAAC,gBAAgB,CAAC,gBAAgB,CAAC;IACxD,CAAC;IAED,IAAY,iBAAiB;QAC3B,OAAO,IAAI,CAAC,OAAO,CAAC,gBAAgB,CAAC,iBAAiB,CAAC;IACzD,CAAC;IAED,IAAY,YAAY;QACtB,OAAO,IAAI,CAAC,OAAO,CAAC,YAAY,CAAC;IACnC,CAAC;IAEM,KAAK,CAAC,IAAI;QACf,IAAI,CAAC,CAAC,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,UAAU,CAAC,EAAE,CAAC;YACzC,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC;YACvB,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;QACxB,CAAC;IACH,CAAC;IAEM,KAAK,CAAC,qBAAqB;QAChC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,wCAAwC,IAAI,CAAC,iBAAiB,OAAO,CAAC,CAAC;QACxF,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,wBAAwB,IAAI,CAAC,gBAAgB,kCAAkC,CAAC,CAAC;QAClG,OAAO,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC;YACvB,oDAAoD;YACpD,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;gBACnB,MAAM,IAAI,yBAAyB,CAAC,yCAAyC,CAAC,CAAC;YACjF,CAAC;YAED,IAAI,CAAC;gBACH,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;gBACrC,IAAI,CAAC,UAAU,EAAE,CAAC;oBAChB,0CAA0C;oBAC1C,MAAM,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;gBAClD,CAAC;gBACD,0DAA0D;YAC5D,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,IAAI,CAAC,CAAC,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,UAAU,CAAC,EAAE,CAAC;oBACzC,IAAI,CAAC,aAAa,GAAG,KAAc,CAAC;oBACpC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,2BAA2B,EAAE,KAAK,CAAC,CAAC;oBACtD,IAAI,CAAC,IAAI,EAAE,CAAC;gBACd,CAAC;gBACD,MAAM;YACR,CAAC;QACH,CAAC;QAED,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;YACvB,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,0CAA0C,EAAE,IAAI,CAAC,aAAa,CAAC,CAAC;YAClF,MAAM,IAAI,CAAC,aAAa,CAAC;QAC3B,CAAC;QAED,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,wBAAwB,CAAC,CAAC;IAC7C,CAAC;IAEO,KAAK,CAAC,IAAI;QAChB,wDAAwD;QACxD,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;QAEtB,IAAI,CAAC;YACH,0CAA0C;YAC1C,mFAAmF;YACnF,MAAM,QAAQ,GAAG,MAAM,YAAY,CAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,iBAAiB,CAAC,CAAC;YAE7E,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,KAAK,CAC/D,eAAe,IAAI,CAAC,gBAAgB;;;;SAInC,EACD,CAAC,EAAE,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,GAAG,CAAC,SAAS,EAAE,KAAK,EAAE,QAAQ,CAAC,QAAQ,EAAE,EAAE,CAAC,CACxE,CAAC;YAEF,2CAA2C;YAC3C,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACzB,OAAO,KAAK,CAAC;YACf,CAAC;YAED,kDAAkD;YAClD,MAAM,MAAM,GAAG,GAAG,CAAC,UAAU,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;YAErE,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,sBAAsB,QAAQ,OAAO,MAAM,aAAa,OAAO,CAAC,MAAM,kBAAkB,CAAC,CAAC;YAE3G,IAAI,gBAAgB,GAAG,CAAC,CAAC;YACzB,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;gBACtC,MAAM,qBAAqB,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAC;gBAChF,2GAA2G;gBAC3G,yKAAyK;gBACzK,IAAI,qBAAqB,GAAG,gBAAgB,EAAE,CAAC;oBAC7C,gBAAgB,GAAG,qBAAqB,CAAC;gBAC3C,CAAC;YACH,CAAC;YAED,IAAI,CAAC,MAAM,CAAC,IAAI,CACd,aAAa,OAAO,CAAC,MAAM,8BAA8B,gBAAgB,+BAA+B,CACzG,CAAC;YACF,sEAAsE;YACtE,MAAM,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ,EAAE,EAAE,gBAAgB,CAAC,CAAC;YAEtE,IAAI,CAAC,UAAU,GAAG,MAAM,CAAC;YAEzB,OAAO,IAAI,CAAC;QACd,CAAC;gBAAS,CAAC;YACT,2CAA2C;YAC3C,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC;QACzB,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,SAAS,CAAC,KAAuB,EAAE,MAAsC;QACrF,uEAAuE;QACvE,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,IAAI,CAAC,iBAAiB,EAAE,KAAK,CAAC,eAAe,CAAC,CAAC;QAC9E,IAAI,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC;YAC3B,OAAO,CAAC,CAAC;QACX,CAAC;aAAM,IAAI,MAAM,IAAI,MAAM,CAAC,QAAQ,EAAE,CAAC;YACrC,MAAM,CAAC,QAAQ,GAAG,MAAM,CAAC;QAC3B,CAAC;QACD,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,KAAK,CAC/D;wBACkB,KAAK,CAAC,kBAAkB;KAC3C,EACC;YACE,EAAE,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,GAAG,CAAC,SAAS,EAAE,KAAK,EAAE,MAAM,CAAC,QAAQ,CAAC,QAAQ,EAAE,EAAE;YAC5E,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,GAAG,CAAC,SAAS,EAAE,KAAK,EAAE,MAAM,CAAC,MAAM,CAAC,QAAQ,EAAE,EAAE;SACzE,CACF,CAAC;QAEF,IAAI,gBAAgB,GAAG,CAAC,CAAC;QACzB,IAAI,YAAY,GAAQ,IAAI,CAAC;QAC7B,IAAI,kBAAkB,GAAe,IAAI,CAAC;QAC1C,KAAK,MAAM,GAAG,IAAI,OAAO,EAAE,CAAC;YAC1B,MAAM,cAAc,GAAG,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;YACxD,QAAQ,GAAG,CAAC,YAAY,EAAE,CAAC;gBACzB,KAAK,SAAS,CAAC,MAAM;oBACnB,MAAM,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC,GAAG,EAAE,KAAK,EAAE,OAAO,CAAC,OAAO,CAAC,CAAC;oBAC9D,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,6BAA6B,cAAc,EAAE,CAAC,CAAC;oBAChE,MAAM;gBACR,KAAK,SAAS,CAAC,MAAM;oBACnB,MAAM,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC,GAAG,EAAE,KAAK,EAAE,OAAO,CAAC,OAAO,CAAC,CAAC;oBAC9D,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,6BAA6B,cAAc,EAAE,CAAC,CAAC;oBAChE,MAAM;gBACR,KAAK,SAAS,CAAC,aAAa;oBAC1B,YAAY,GAAG,GAAG,CAAC;oBACnB,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,qCAAqC,cAAc,EAAE,CAAC,CAAC;oBACzE,MAAM;gBACR,KAAK,SAAS,CAAC,YAAY;oBACzB,IAAI,YAAY,KAAK,IAAI,EAAE,CAAC;wBAC1B,MAAM,IAAI,yBAAyB,CAAC,wCAAwC,CAAC,CAAC;oBAChF,CAAC;oBACD,MAAM,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC,GAAG,EAAE,YAAY,EAAE,KAAK,EAAE,OAAO,CAAC,OAAO,CAAC,CAAC;oBAC5E,YAAY,GAAG,IAAI,CAAC;oBACpB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,6BAA6B,cAAc,EAAE,CAAC,CAAC;oBAChE,MAAM;gBACR;oBACE,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,2BAA2B,GAAG,CAAC,YAAY,+BAA+B,CAAC,CAAC;YACjG,CAAC;YAED,sGAAsG;YACtG,IAAI,cAAc,IAAI,kBAAkB,EAAE,CAAC;gBACzC,kBAAkB,GAAG,cAAc,CAAC;gBACpC,IAAI,GAAG,CAAC,YAAY,KAAK,SAAS,CAAC,aAAa,EAAE,CAAC;oBACjD,gBAAgB,EAAE,CAAC;gBACrB,CAAC;YACH,CAAC;QACH,CAAC;QAED,OAAO,gBAAgB,CAAC;IAC1B,CAAC;CACF"}
|
|
1
|
+
{"version":3,"file":"CDCPoller.js","sourceRoot":"","sources":["../../src/replication/CDCPoller.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,kBAAkB,EAClB,SAAS,EAET,MAAM,IAAI,aAAa,EACvB,yBAAyB,EAC1B,MAAM,mCAAmC,CAAC;AAC3C,OAAO,MAAM,MAAM,iBAAiB,CAAC;AAGrC,OAAO,EAAE,GAAG,EAAE,MAAM,kBAAkB,CAAC;AACvC,OAAO,GAAG,MAAM,OAAO,CAAC;AACxB,OAAO,EAA0B,mBAAmB,EAAE,YAAY,EAAE,oBAAoB,EAAE,MAAM,mBAAmB,CAAC;AACpH,OAAO,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AAEvD,OAAO,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AAKjD,IAAK,SAKJ;AALD,WAAK,SAAS;IACZ,6CAAU,CAAA;IACV,6CAAU,CAAA;IACV,2DAAiB,CAAA;IACjB,yDAAgB,CAAA;AAClB,CAAC,EALI,SAAS,KAAT,SAAS,QAKb;AAED,MAAM,CAAN,IAAY,gBAOX;AAPD,WAAY,gBAAgB;IAC1B,iDAA6B,CAAA;IAC7B,6CAAyB,CAAA;IACzB,iDAA6B,CAAA;IAC7B,iEAA6C,CAAA;IAC7C,iEAA6C,CAAA;IAC7C,yEAAqD,CAAA;AACvD,CAAC,EAPW,gBAAgB,KAAhB,gBAAgB,QAO3B;AAwBD,MAAM,CAAC,MAAM,gCAAgC,GAAG,MAAM,CAAC;AAsBvD;;GAEG;AACH,MAAM,OAAO,SAAS;IAaD;IAZX,iBAAiB,CAAyB;IAC1C,YAAY,CAAkB;IAC9B,UAAU,CAAM;IAChB,MAAM,CAAS;IACf,aAAa,CAAe;IAC5B,gBAAgB,CAAsC;IAEtD,SAAS,GAAY,KAAK,CAAC;IAC3B,UAAU,GAAY,KAAK,CAAC;IAC5B,SAAS,GAAY,KAAK,CAAC;IAC3B,mBAAmB,GAAW,CAAC,CAAC;IAExC,YAAmB,OAAyB;QAAzB,YAAO,GAAP,OAAO,CAAkB;QAC1C,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,aAAa,CAAC;QAC9C,IAAI,CAAC,iBAAiB,GAAG,OAAO,CAAC,iBAAiB,CAAC;QACnD,IAAI,CAAC,YAAY,GAAG,OAAO,CAAC,YAAY,CAAC;QACzC,IAAI,CAAC,UAAU,GAAG,OAAO,CAAC,QAAQ,CAAC;QACnC,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC;QAC1B,IAAI,CAAC,gBAAgB,GAAG,IAAI,GAAG,EAAkC,CAAC;IACpE,CAAC;IAED,IAAY,gBAAgB;QAC1B,OAAO,IAAI,CAAC,OAAO,CAAC,gBAAgB,CAAC,gBAAgB,CAAC;IACxD,CAAC;IAED,IAAY,iBAAiB;QAC3B,OAAO,IAAI,CAAC,OAAO,CAAC,gBAAgB,CAAC,iBAAiB,CAAC;IACzD,CAAC;IAED,IAAY,qBAAqB;QAC/B,OAAO,IAAI,CAAC,OAAO,CAAC,qBAAqB,IAAI,gCAAgC,CAAC;IAChF,CAAC;IAED,IAAY,gBAAgB;QAC1B,OAAO,IAAI,CAAC,OAAO,CAAC,mBAAmB,EAAE,CAAC;IAC5C,CAAC;IAEM,KAAK,CAAC,IAAI;QACf,IAAI,CAAC,CAAC,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,UAAU,CAAC,EAAE,CAAC;YACzC,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC;YACvB,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;QACxB,CAAC;IACH,CAAC;IAEM,KAAK,CAAC,qBAAqB;QAChC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,wCAAwC,IAAI,CAAC,iBAAiB,OAAO,CAAC,CAAC;QACxF,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,wBAAwB,IAAI,CAAC,gBAAgB,kCAAkC,CAAC,CAAC;QAClG,OAAO,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC;YACvB,oDAAoD;YACpD,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;gBACnB,MAAM,IAAI,yBAAyB,CAAC,yCAAyC,CAAC,CAAC;YACjF,CAAC;YAED,IAAI,CAAC;gBACH,IAAI,IAAI,CAAC,iBAAiB,EAAE,EAAE,CAAC;oBAC7B,IAAI,CAAC,gBAAgB,GAAG,MAAM,mBAAmB,CAAC,EAAE,iBAAiB,EAAE,IAAI,CAAC,iBAAiB,EAAE,CAAC,CAAC;oBACjG,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,qBAAqB,EAAE,CAAC;oBACzD,KAAK,MAAM,YAAY,IAAI,aAAa,EAAE,CAAC;wBACzC,MAAM,IAAI,CAAC,YAAY,CAAC,cAAc,CAAC,YAAY,CAAC,CAAC;oBACvD,CAAC;oBACD,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;oBAEtC,IAAI,CAAC,MAAM,CAAC,KAAK,CACf,uDAAuD,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CACrG,CAAC;gBACJ,CAAC;gBAED,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;gBACrC,IAAI,CAAC,UAAU,EAAE,CAAC;oBAChB,8CAA8C;oBAC9C,MAAM,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;gBAClD,CAAC;gBAED,0DAA0D;YAC5D,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,IAAI,CAAC,CAAC,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,UAAU,CAAC,EAAE,CAAC;oBACzC,qBAAqB;oBACrB,IAAI,KAAK,YAAY,kBAAkB,EAAE,CAAC;wBACxC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;wBAChC,kEAAkE;wBAClE,IAAI,CAAC,mBAAmB,GAAG,CAAC,CAAC;wBAC7B,SAAS;oBACX,CAAC;oBACD,kFAAkF;oBAClF,sFAAsF;oBACtF,IAAI,eAAe,CAAC,KAAK,CAAC,EAAE,CAAC;wBAC3B,IAAI,CAAC,MAAM,CAAC,IAAI,CACd,mGAAoG,KAAe,CAAC,OAAO,EAAE,CAC9H,CAAC;wBACF,SAAS;oBACX,CAAC;oBAED,yBAAyB;oBACzB,IAAI,CAAC,aAAa,GAAG,KAAc,CAAC;oBACpC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,2BAA2B,EAAE,KAAK,CAAC,CAAC;oBACtD,IAAI,CAAC,IAAI,EAAE,CAAC;gBACd,CAAC;gBACD,MAAM;YACR,CAAC;QACH,CAAC;QAED,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;YACvB,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,0CAA0C,EAAE,IAAI,CAAC,aAAa,CAAC,CAAC;YAClF,MAAM,IAAI,CAAC,aAAa,CAAC;QAC3B,CAAC;QAED,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,wBAAwB,CAAC,CAAC;IAC7C,CAAC;IAEO,KAAK,CAAC,IAAI;QAChB,wDAAwD;QACxD,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;QAEtB,IAAI,CAAC;YACH,0CAA0C;YAC1C,mFAAmF;YACnF,MAAM,QAAQ,GAAG,MAAM,YAAY,CAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,iBAAiB,CAAC,CAAC;YAE7E,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,KAAK,CAC/D,eAAe,IAAI,CAAC,gBAAgB;;;;SAInC,EACD,CAAC,EAAE,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,GAAG,CAAC,SAAS,EAAE,KAAK,EAAE,QAAQ,CAAC,QAAQ,EAAE,EAAE,CAAC,CACxE,CAAC;YAEF,2CAA2C;YAC3C,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACzB,OAAO,KAAK,CAAC;YACf,CAAC;YAED,kDAAkD;YAClD,MAAM,MAAM,GAAG,GAAG,CAAC,UAAU,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;YAErE,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,sBAAsB,QAAQ,OAAO,MAAM,aAAa,OAAO,CAAC,MAAM,kBAAkB,CAAC,CAAC;YAE3G,IAAI,gBAAgB,GAAG,CAAC,CAAC;YACzB,IAAI,CAAC,MAAM,CAAC,KAAK,CACf,iCAAiC,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,eAAe,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAC5G,CAAC;YACF,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,gBAAgB,EAAE,CAAC;gBAC1C,IAAI,KAAK,CAAC,aAAa,EAAE,EAAE,CAAC;oBAC1B,MAAM,qBAAqB,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAC;oBAChF,2GAA2G;oBAC3G,yKAAyK;oBACzK,IAAI,qBAAqB,GAAG,gBAAgB,EAAE,CAAC;wBAC7C,gBAAgB,GAAG,qBAAqB,CAAC;oBAC3C,CAAC;gBACH,CAAC;YACH,CAAC;YAED,IAAI,CAAC,MAAM,CAAC,IAAI,CACd,aAAa,OAAO,CAAC,MAAM,8BAA8B,gBAAgB,+CAA+C,MAAM,CAAC,QAAQ,EAAE,EAAE,CAC5I,CAAC;YACF,sEAAsE;YACtE,MAAM,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ,EAAE,EAAE,gBAAgB,CAAC,CAAC;YAEtE,IAAI,CAAC,UAAU,GAAG,MAAM,CAAC;YAEzB,OAAO,IAAI,CAAC;QACd,CAAC;gBAAS,CAAC;YACT,2CAA2C;YAC3C,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC;QACzB,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,SAAS,CAAC,KAAuB,EAAE,MAAsC;QACrF,uEAAuE;QACvE,MAAM,MAAM,GAAG,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,KAAK,CAAC,QAAQ,CAAE,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;QAC9E,IAAI,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC;YAC3B,OAAO,CAAC,CAAC;QACX,CAAC;aAAM,IAAI,MAAM,IAAI,MAAM,CAAC,QAAQ,EAAE,CAAC;YACrC,MAAM,CAAC,QAAQ,GAAG,MAAM,CAAC;QAC3B,CAAC;QAED,IAAI,CAAC;YACH,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,KAAK,CAC/D;wBACgB,KAAK,CAAC,kBAAkB;KAC3C,EACG;gBACE,EAAE,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,GAAG,CAAC,SAAS,EAAE,KAAK,EAAE,MAAM,CAAC,QAAQ,CAAC,QAAQ,EAAE,EAAE;gBAC5E,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,GAAG,CAAC,SAAS,EAAE,KAAK,EAAE,MAAM,CAAC,MAAM,CAAC,QAAQ,EAAE,EAAE;aACzE,CACF,CAAC;YAEF,IAAI,gBAAgB,GAAG,CAAC,CAAC;YACzB,IAAI,YAAY,GAAQ,IAAI,CAAC;YAC7B,IAAI,kBAAkB,GAAe,IAAI,CAAC;YAC1C,KAAK,MAAM,GAAG,IAAI,OAAO,EAAE,CAAC;gBAC1B,MAAM,cAAc,GAAG,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;gBACxD,QAAQ,GAAG,CAAC,YAAY,EAAE,CAAC;oBACzB,KAAK,SAAS,CAAC,MAAM;wBACnB,MAAM,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC,GAAG,EAAE,KAAK,EAAE,OAAO,CAAC,OAAO,CAAC,CAAC;wBAC9D,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,6BAA6B,cAAc,EAAE,CAAC,CAAC;wBAChE,MAAM;oBACR,KAAK,SAAS,CAAC,MAAM;wBACnB,MAAM,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC,GAAG,EAAE,KAAK,EAAE,OAAO,CAAC,OAAO,CAAC,CAAC;wBAC9D,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,6BAA6B,cAAc,EAAE,CAAC,CAAC;wBAChE,MAAM;oBACR,KAAK,SAAS,CAAC,aAAa;wBAC1B,YAAY,GAAG,GAAG,CAAC;wBACnB,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,qCAAqC,cAAc,EAAE,CAAC,CAAC;wBACzE,MAAM;oBACR,KAAK,SAAS,CAAC,YAAY;wBACzB,IAAI,YAAY,KAAK,IAAI,EAAE,CAAC;4BAC1B,MAAM,IAAI,yBAAyB,CAAC,wCAAwC,CAAC,CAAC;wBAChF,CAAC;wBACD,MAAM,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC,GAAG,EAAE,YAAY,EAAE,KAAK,EAAE,OAAO,CAAC,OAAO,CAAC,CAAC;wBAC5E,YAAY,GAAG,IAAI,CAAC;wBACpB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,6BAA6B,cAAc,EAAE,CAAC,CAAC;wBAChE,MAAM;oBACR;wBACE,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,2BAA2B,GAAG,CAAC,YAAY,+BAA+B,CAAC,CAAC;gBACjG,CAAC;gBAED,sGAAsG;gBACtG,IAAI,cAAc,IAAI,kBAAkB,EAAE,CAAC;oBACzC,kBAAkB,GAAG,cAAc,CAAC;oBACpC,IAAI,GAAG,CAAC,YAAY,KAAK,SAAS,CAAC,aAAa,EAAE,CAAC;wBACjD,gBAAgB,EAAE,CAAC;oBACrB,CAAC;gBACH,CAAC;YACH,CAAC;YAED,OAAO,gBAAgB,CAAC;QAC1B,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,wDAAwD;YACxD,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,qBAAqB,CAAC,EAAE,CAAC;gBAClD,MAAM,IAAI,kBAAkB,CAC1B,SAAS,CAAC,WAAW,EACrB,8BAA8B,KAAK,CAAC,eAAe,EAAE,0BAA0B,EAC/E,KAAK,CACN,CAAC;YACJ,CAAC;YACD,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;IAEO,iBAAiB;QACvB,OAAO,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,mBAAmB,IAAI,IAAI,CAAC,qBAAqB,CAAC;IAC7E,CAAC;IAED;;;OAGG;IACK,KAAK,CAAC,qBAAqB;QACjC,MAAM,aAAa,GAAmB,EAAE,CAAC;QAEzC,MAAM,SAAS,GAAG,IAAI,CAAC,iBAAiB,EAAE,CAAC;QAC3C,KAAK,MAAM,KAAK,IAAI,SAAS,EAAE,CAAC;YAC9B,IAAI,CAAC,MAAM,CAAC,IAAI,CACd,aAAa,oBAAoB,CAAC,KAAK,CAAC,WAAW,CAAC,MAAM,EAAE,KAAK,CAAC,WAAW,CAAC,IAAI,CAAC,sEAAsE,CAC1J,CAAC;YACF,aAAa,CAAC,IAAI,CAAC;gBACjB,IAAI,EAAE,gBAAgB,CAAC,YAAY;gBACnC,QAAQ,EAAE;oBACR,IAAI,EAAE,KAAK,CAAC,WAAW,CAAC,IAAI;oBAC5B,MAAM,EAAE,KAAK,CAAC,WAAW,CAAC,MAAM;oBAChC,QAAQ,EAAE,KAAK,CAAC,WAAW,CAAC,QAAQ;iBACrC;gBACD,kBAAkB,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC;aACvC,CAAC,CAAC;QACL,CAAC;QAED,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,gBAAgB,EAAE,CAAC;YAC1C,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC,KAAK,CAAC,QAAQ,EAAE,IAAI,CAAC,iBAAiB,CAAC,CAAC;YACzE,IAAI,CAAC,MAAM,EAAE,CAAC;gBACZ,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,KAAK,CAAC,eAAe,EAAE,8CAA8C,CAAC,CAAC;gBACjG,aAAa,CAAC,IAAI,CAAC;oBACjB,IAAI,EAAE,gBAAgB,CAAC,UAAU;oBACjC,KAAK;iBACN,CAAC,CAAC;gBACH,SAAS;YACX,CAAC;YAED,MAAM,sBAAsB,GAAG,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;YACzE,IAAI,CAAC,sBAAsB,EAAE,CAAC;gBAC5B,IAAI,KAAK,CAAC,aAAa,EAAE,EAAE,CAAC;oBAC1B,mDAAmD;oBACnD,aAAa,CAAC,IAAI,CAAC;wBACjB,IAAI,EAAE,gBAAgB,CAAC,wBAAwB;wBAC/C,KAAK;qBACN,CAAC,CAAC;gBACL,CAAC;gBACD,SAAS;YACX,CAAC;YAED,MAAM,qBAAqB,GAAG,sBAAsB,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC;YAClE,oHAAoH;YACpH,IAAI,CAAC,KAAK,CAAC,aAAa,EAAE,IAAI,KAAK,CAAC,eAAgB,CAAC,QAAQ,KAAK,qBAAqB,CAAC,QAAQ,EAAE,CAAC;gBACjG,aAAa,CAAC,IAAI,CAAC;oBACjB,IAAI,EAAE,gBAAgB,CAAC,oBAAoB;oBAC3C,KAAK;oBACL,kBAAkB,EAAE,qBAAqB;iBAC1C,CAAC,CAAC;gBACH,SAAS;YACX,CAAC;YAED,gDAAgD;YAChD,IAAI,KAAK,CAAC,WAAW,CAAC,IAAI,KAAK,sBAAsB,CAAC,WAAW,CAAC,IAAI,EAAE,CAAC;gBACvE,MAAM,QAAQ,GAAG,IAAI,CAAC,qBAAqB,CACzC,sBAAsB,CAAC,WAAW,CAAC,MAAM,EACzC,sBAAsB,CAAC,WAAW,CAAC,IAAI,CACxC;oBACC,CAAC,CAAC;wBACE,IAAI,EAAE,sBAAsB,CAAC,WAAW,CAAC,IAAI;wBAC7C,MAAM,EAAE,sBAAsB,CAAC,WAAW,CAAC,MAAM;wBACjD,QAAQ,EAAE,sBAAsB,CAAC,WAAW,CAAC,QAAQ;qBACtD;oBACH,CAAC,CAAC,SAAS,CAAC;gBAEd,aAAa,CAAC,IAAI,CAAC;oBACjB,IAAI,EAAE,gBAAgB,CAAC,YAAY;oBACnC,KAAK;oBACL,QAAQ;oBACR,kBAAkB,EAAE,qBAAqB;iBAC1C,CAAC,CAAC;gBACH,SAAS;YACX,CAAC;YAED,IAAI,qBAAqB,CAAC,oBAAoB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC1D,aAAa,CAAC,IAAI,CAAC;oBACjB,IAAI,EAAE,gBAAgB,CAAC,oBAAoB;oBAC3C,KAAK;oBACL,kBAAkB,EAAE,qBAAqB;iBAC1C,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAED,OAAO,aAAa,CAAC;IACvB,CAAC;IAEO,iBAAiB;QACvB,MAAM,SAAS,GAA6B,EAAE,CAAC;QAC/C,KAAK,MAAM,CAAC,QAAQ,EAAE,sBAAsB,CAAC,IAAI,IAAI,CAAC,gBAAgB,CAAC,OAAO,EAAE,EAAE,CAAC;YACjF,6IAA6I;YAC7I,IAAI,CAAC,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,QAAQ,KAAK,QAAQ,CAAC,EAAE,CAAC;gBACxE,sEAAsE;gBACtE,IACE,IAAI,CAAC,qBAAqB,CAAC,sBAAsB,CAAC,WAAW,CAAC,MAAM,EAAE,sBAAsB,CAAC,WAAW,CAAC,IAAI,CAAC,EAC9G,CAAC;oBACD,SAAS,CAAC,IAAI,CAAC,sBAAsB,CAAC,CAAC;gBACzC,CAAC;YACH,CAAC;QACH,CAAC;QAED,OAAO,SAAS,CAAC;IACnB,CAAC;IAEO,qBAAqB,CAAC,MAAc,EAAE,SAAiB;QAC7D,OAAO,IAAI,CAAC,OAAO,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,YAAY,EAAE,EAAE,CACrD,YAAY,CAAC,OAAO,CAAC;YACnB,aAAa,EAAE,IAAI,CAAC,iBAAiB,CAAC,aAAa;YACnD,MAAM,EAAE,MAAM;YACd,IAAI,EAAE,SAAS;SAChB,CAAC,CACH,CAAC;IACJ,CAAC;CACF"}
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { replication } from '@powersync/service-core';
|
|
2
2
|
import { container, logger as defaultLogger } from '@powersync/lib-services-framework';
|
|
3
3
|
import { CDCDataExpiredError, CDCStream } from './CDCStream.js';
|
|
4
|
+
import { POWERSYNC_CHECKPOINTS_TABLE } from '../utils/mssql.js';
|
|
4
5
|
export class CDCReplicationJob extends replication.AbstractReplicationJob {
|
|
5
6
|
connectionFactory;
|
|
6
7
|
lastStream = null;
|
|
@@ -12,7 +13,14 @@ export class CDCReplicationJob extends replication.AbstractReplicationJob {
|
|
|
12
13
|
this.cdcReplicationJobOptions = options;
|
|
13
14
|
}
|
|
14
15
|
async keepAlive() {
|
|
15
|
-
|
|
16
|
+
if (this.lastStream) {
|
|
17
|
+
try {
|
|
18
|
+
await this.lastStream.keepAlive();
|
|
19
|
+
}
|
|
20
|
+
catch (e) {
|
|
21
|
+
this.logger.warn(`KeepAlive failed, unable to write an update to the ${POWERSYNC_CHECKPOINTS_TABLE} table`, e);
|
|
22
|
+
}
|
|
23
|
+
}
|
|
16
24
|
}
|
|
17
25
|
async replicate() {
|
|
18
26
|
try {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"CDCReplicationJob.js","sourceRoot":"","sources":["../../src/replication/CDCReplicationJob.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,yBAAyB,CAAC;AAEtD,OAAO,EAAE,SAAS,EAAE,MAAM,IAAI,aAAa,EAAE,MAAM,mCAAmC,CAAC;AACvF,OAAO,EAAE,mBAAmB,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;
|
|
1
|
+
{"version":3,"file":"CDCReplicationJob.js","sourceRoot":"","sources":["../../src/replication/CDCReplicationJob.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,yBAAyB,CAAC;AAEtD,OAAO,EAAE,SAAS,EAAE,MAAM,IAAI,aAAa,EAAE,MAAM,mCAAmC,CAAC;AACvF,OAAO,EAAE,mBAAmB,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAEhE,OAAO,EAAE,2BAA2B,EAAE,MAAM,mBAAmB,CAAC;AAOhE,MAAM,OAAO,iBAAkB,SAAQ,WAAW,CAAC,sBAAsB;IAC/D,iBAAiB,CAAgC;IACjD,UAAU,GAAqB,IAAI,CAAC;IACpC,wBAAwB,CAA2B;IAE3D,YAAY,OAAiC;QAC3C,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,MAAM,GAAG,aAAa,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,cAAc,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC;QAC/F,IAAI,CAAC,iBAAiB,GAAG,OAAO,CAAC,iBAAiB,CAAC;QACnD,IAAI,CAAC,wBAAwB,GAAG,OAAO,CAAC;IAC1C,CAAC;IAED,KAAK,CAAC,SAAS;QACb,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YACpB,IAAI,CAAC;gBACH,MAAM,IAAI,CAAC,UAAU,CAAC,SAAS,EAAE,CAAC;YACpC,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACX,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,sDAAsD,2BAA2B,QAAQ,EAAE,CAAC,CAAC,CAAC;YACjH,CAAC;QACH,CAAC;IACH,CAAC;IAED,KAAK,CAAC,SAAS;QACb,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,aAAa,EAAE,CAAC;QAC7B,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,kBAAkB;YAClB,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC;gBACpB,wBAAwB;gBACxB,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,mBAAmB,EAAE,CAAC,CAAC,CAAC;gBAC1C,IAAI,CAAC,CAAC,KAAK,IAAI,IAAI,EAAE,CAAC;oBACpB,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC;gBACtC,CAAC;gBAED,SAAS,CAAC,QAAQ,CAAC,gBAAgB,CAAC,CAAC,EAAE;oBACrC,QAAQ,EAAE,EAAE;iBACb,CAAC,CAAC;gBAEH,4BAA4B;gBAC5B,IAAI,CAAC,WAAW,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC;YAClC,CAAC;YACD,IAAI,CAAC,YAAY,mBAAmB,EAAE,CAAC;gBACrC,0DAA0D;gBAC1D,MAAM,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,kBAAkB,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;YAC/E,CAAC;QACH,CAAC;gBAAS,CAAC;YACT,IAAI,CAAC,eAAe,CAAC,KAAK,EAAE,CAAC;QAC/B,CAAC;IACH,CAAC;IAED,KAAK,CAAC,aAAa;QACjB,+DAA+D;QAC/D,gEAAgE;QAChE,uCAAuC;QACvC,MAAM,iBAAiB,GAAG,IAAI,CAAC,iBAAiB,CAAC,MAAM,CAAC;YACtD,iBAAiB,EAAE,MAAM;YACzB,GAAG,EAAE,CAAC;SACP,CAAC,CAAC;QACH,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,WAAW,EAAE,gBAAgB,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,eAAe,CAAC,MAAM,EAAE,CAAC,CAAC;YAClF,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;gBACnB,OAAO;YACT,CAAC;YACD,MAAM,MAAM,GAAG,IAAI,SAAS,CAAC;gBAC3B,MAAM,EAAE,IAAI,CAAC,MAAM;gBACnB,WAAW,EAAE,IAAI,CAAC,eAAe,CAAC,MAAM;gBACxC,OAAO,EAAE,IAAI,CAAC,OAAO,CAAC,OAAO;gBAC7B,OAAO,EAAE,IAAI,CAAC,OAAO,CAAC,OAAO;gBAC7B,WAAW,EAAE,iBAAiB;gBAC9B,gBAAgB,EAAE,IAAI,CAAC,wBAAwB,CAAC,gBAAgB;aACjE,CAAC,CAAC;YACH,IAAI,CAAC,UAAU,GAAG,MAAM,CAAC;YACzB,MAAM,MAAM,CAAC,SAAS,EAAE,CAAC;QAC3B,CAAC;gBAAS,CAAC;YACT,MAAM,iBAAiB,CAAC,GAAG,EAAE,CAAC;QAChC,CAAC;IACH,CAAC;IAED,KAAK,CAAC,uBAAuB;QAC3B,OAAO,IAAI,CAAC,UAAU,EAAE,uBAAuB,EAAE,CAAC;IACpD,CAAC;CACF"}
|
|
@@ -4,7 +4,10 @@ import { TablePattern } from '@powersync/service-sync-rules';
|
|
|
4
4
|
import { MSSQLConnectionManager } from './MSSQLConnectionManager.js';
|
|
5
5
|
import sql from 'mssql';
|
|
6
6
|
import { MSSQLSourceTable } from '../common/MSSQLSourceTable.js';
|
|
7
|
+
import { MSSQLSourceTableCache } from '../common/MSSQLSourceTableCache.js';
|
|
8
|
+
import { SchemaChange } from './CDCPoller.js';
|
|
7
9
|
import { AdditionalConfig } from '../types/types.js';
|
|
10
|
+
import { CaptureInstance } from '../common/CaptureInstance.js';
|
|
8
11
|
export interface CDCStreamOptions {
|
|
9
12
|
connections: MSSQLConnectionManager;
|
|
10
13
|
storage: storage.SyncRulesBucketStorage;
|
|
@@ -18,15 +21,27 @@ export interface CDCStreamOptions {
|
|
|
18
21
|
*/
|
|
19
22
|
snapshotBatchSize?: number;
|
|
20
23
|
additionalConfig: AdditionalConfig;
|
|
24
|
+
/**
|
|
25
|
+
* Override schema check interval, defaults is 60 seconds.
|
|
26
|
+
*/
|
|
27
|
+
schemaCheckIntervalMs?: number;
|
|
21
28
|
}
|
|
22
29
|
export declare enum SnapshotStatus {
|
|
23
|
-
|
|
30
|
+
INITIAL = "initial",
|
|
31
|
+
RESUME = "resume",
|
|
24
32
|
DONE = "done",
|
|
25
|
-
|
|
33
|
+
LIMITED_RESNAPSHOT = "limited-resnapshot"
|
|
26
34
|
}
|
|
27
35
|
export interface SnapshotStatusResult {
|
|
28
36
|
status: SnapshotStatus;
|
|
29
37
|
snapshotLSN: string | null;
|
|
38
|
+
/**
|
|
39
|
+
* Under certain circumstances it may be necessary to re-snapshot specific tables:
|
|
40
|
+
* * A new capture instance has been created for a table.
|
|
41
|
+
* * A table has been renamed.
|
|
42
|
+
* * The retention threshold has been exceeded for a table.
|
|
43
|
+
*/
|
|
44
|
+
specificTablesToResnapshot?: MSSQLSourceTable[];
|
|
30
45
|
}
|
|
31
46
|
export declare class CDCConfigurationError extends Error {
|
|
32
47
|
constructor(message: string);
|
|
@@ -48,7 +63,7 @@ export declare class CDCStream {
|
|
|
48
63
|
private readonly connections;
|
|
49
64
|
private readonly abortSignal;
|
|
50
65
|
private readonly logger;
|
|
51
|
-
|
|
66
|
+
tableCache: MSSQLSourceTableCache;
|
|
52
67
|
/**
|
|
53
68
|
* Time of the oldest uncommitted change, according to the source db.
|
|
54
69
|
* This is used to determine the replication lag.
|
|
@@ -70,7 +85,7 @@ export declare class CDCStream {
|
|
|
70
85
|
replicate(): Promise<void>;
|
|
71
86
|
populateTableCache(): Promise<void>;
|
|
72
87
|
getQualifiedTableNames(batch: storage.BucketStorageBatch, tablePattern: TablePattern): Promise<MSSQLSourceTable[]>;
|
|
73
|
-
processTable(batch: storage.BucketStorageBatch, table: SourceEntityDescriptor, snapshot: boolean): Promise<MSSQLSourceTable>;
|
|
88
|
+
processTable(batch: storage.BucketStorageBatch, table: SourceEntityDescriptor, captureInstance: CaptureInstance | null, snapshot: boolean): Promise<MSSQLSourceTable>;
|
|
74
89
|
private snapshotTableInTx;
|
|
75
90
|
private snapshotTable;
|
|
76
91
|
/**
|
|
@@ -95,6 +110,18 @@ export declare class CDCStream {
|
|
|
95
110
|
private checkSnapshotStatus;
|
|
96
111
|
streamChanges(): Promise<void>;
|
|
97
112
|
private createEventHandler;
|
|
113
|
+
handleSchemaChange(batch: storage.BucketStorageBatch, change: SchemaChange): Promise<void>;
|
|
114
|
+
private handleCreateOrUpdateTable;
|
|
115
|
+
/**
|
|
116
|
+
* There is very little that can be automatically done to handle the column changes other than to warn the user about the schema drift.
|
|
117
|
+
*
|
|
118
|
+
* Due to the way CDC works, users are prevented from making column schema changes that affect the replication identities of a table.
|
|
119
|
+
* If changes like that are required, CDC has to be disabled and re-enabled for the table. This would then be handled by the detection of the new
|
|
120
|
+
* capture instance.
|
|
121
|
+
* @param table
|
|
122
|
+
* @param captureInstance
|
|
123
|
+
*/
|
|
124
|
+
private handleColumnChanges;
|
|
98
125
|
/**
|
|
99
126
|
* Convert CDC row data to SqliteRow format.
|
|
100
127
|
* CDC rows include table columns plus CDC metadata columns (__$operation, __$start_lsn, etc.).
|
|
@@ -103,4 +130,8 @@ export declare class CDCStream {
|
|
|
103
130
|
private toSqliteRow;
|
|
104
131
|
getReplicationLagMillis(): Promise<number | undefined>;
|
|
105
132
|
private touch;
|
|
133
|
+
/**
|
|
134
|
+
* Creates an update in the source database to ensure regular checkpoints via the CDC
|
|
135
|
+
*/
|
|
136
|
+
keepAlive(): Promise<void>;
|
|
106
137
|
}
|
|
@@ -1,20 +1,21 @@
|
|
|
1
|
-
import { container, DatabaseConnectionError, ErrorCode, logger as defaultLogger, ReplicationAbortedError, ReplicationAssertionError
|
|
1
|
+
import { container, DatabaseConnectionError, ErrorCode, logger as defaultLogger, ReplicationAbortedError, ReplicationAssertionError } from '@powersync/lib-services-framework';
|
|
2
2
|
import { getUuidReplicaIdentityBson, storage } from '@powersync/service-core';
|
|
3
3
|
import { ReplicationMetric } from '@powersync/service-types';
|
|
4
4
|
import { BatchedSnapshotQuery, SimpleSnapshotQuery } from './MSSQLSnapshotQuery.js';
|
|
5
5
|
import { getReplicationIdentityColumns, getTablesFromPattern } from '../utils/schema.js';
|
|
6
|
-
import { checkSourceConfiguration, createCheckpoint,
|
|
6
|
+
import { checkRetentionThresholds, checkSourceConfiguration, createCheckpoint, getCaptureInstances, getLatestLSN, getLatestReplicatedLSN, isIColumnMetadata } from '../utils/mssql.js';
|
|
7
7
|
import sql from 'mssql';
|
|
8
8
|
import { CDCToSqliteRow, toSqliteInputRow } from '../common/mssqls-to-sqlite.js';
|
|
9
9
|
import { LSN } from '../common/LSN.js';
|
|
10
10
|
import { MSSQLSourceTable } from '../common/MSSQLSourceTable.js';
|
|
11
11
|
import { MSSQLSourceTableCache } from '../common/MSSQLSourceTableCache.js';
|
|
12
|
-
import { CDCPoller } from './CDCPoller.js';
|
|
12
|
+
import { CDCPoller, SchemaChangeType } from './CDCPoller.js';
|
|
13
13
|
export var SnapshotStatus;
|
|
14
14
|
(function (SnapshotStatus) {
|
|
15
|
-
SnapshotStatus["
|
|
15
|
+
SnapshotStatus["INITIAL"] = "initial";
|
|
16
|
+
SnapshotStatus["RESUME"] = "resume";
|
|
16
17
|
SnapshotStatus["DONE"] = "done";
|
|
17
|
-
SnapshotStatus["
|
|
18
|
+
SnapshotStatus["LIMITED_RESNAPSHOT"] = "limited-resnapshot";
|
|
18
19
|
})(SnapshotStatus || (SnapshotStatus = {}));
|
|
19
20
|
export class CDCConfigurationError extends Error {
|
|
20
21
|
constructor(message) {
|
|
@@ -108,13 +109,10 @@ export class CDCStream {
|
|
|
108
109
|
logger: this.logger,
|
|
109
110
|
zeroLSN: LSN.ZERO,
|
|
110
111
|
defaultSchema: this.defaultSchema,
|
|
111
|
-
storeCurrentData:
|
|
112
|
+
storeCurrentData: false
|
|
112
113
|
}, async (batch) => {
|
|
113
114
|
for (let tablePattern of sourceTables) {
|
|
114
|
-
|
|
115
|
-
for (const table of tables) {
|
|
116
|
-
this.tableCache.set(table);
|
|
117
|
-
}
|
|
115
|
+
await this.getQualifiedTableNames(batch, tablePattern);
|
|
118
116
|
}
|
|
119
117
|
});
|
|
120
118
|
}
|
|
@@ -122,18 +120,11 @@ export class CDCStream {
|
|
|
122
120
|
if (tablePattern.connectionTag != this.connections.connectionTag) {
|
|
123
121
|
return [];
|
|
124
122
|
}
|
|
123
|
+
const captureInstances = await getCaptureInstances({ connectionManager: this.connections });
|
|
125
124
|
const matchedTables = await getTablesFromPattern(this.connections, tablePattern);
|
|
126
125
|
const tables = [];
|
|
127
126
|
for (const matchedTable of matchedTables) {
|
|
128
|
-
const
|
|
129
|
-
connectionManager: this.connections,
|
|
130
|
-
table: matchedTable.name,
|
|
131
|
-
schema: matchedTable.schema
|
|
132
|
-
});
|
|
133
|
-
if (!isEnabled) {
|
|
134
|
-
this.logger.info(`Skipping ${matchedTable.schema}.${matchedTable.name} - table is not enabled for CDC.`);
|
|
135
|
-
continue;
|
|
136
|
-
}
|
|
127
|
+
const captureInstanceDetails = captureInstances.get(matchedTable.objectId);
|
|
137
128
|
// TODO: Check RLS settings for table
|
|
138
129
|
const replicaIdColumns = await getReplicationIdentityColumns({
|
|
139
130
|
connectionManager: this.connections,
|
|
@@ -145,12 +136,12 @@ export class CDCStream {
|
|
|
145
136
|
schema: matchedTable.schema,
|
|
146
137
|
objectId: matchedTable.objectId,
|
|
147
138
|
replicaIdColumns: replicaIdColumns.columns
|
|
148
|
-
}, false);
|
|
139
|
+
}, captureInstanceDetails?.instances[0] ?? null, false);
|
|
149
140
|
tables.push(table);
|
|
150
141
|
}
|
|
151
142
|
return tables;
|
|
152
143
|
}
|
|
153
|
-
async processTable(batch, table, snapshot) {
|
|
144
|
+
async processTable(batch, table, captureInstance, snapshot) {
|
|
154
145
|
if (!table.objectId && typeof table.objectId != 'number') {
|
|
155
146
|
throw new ReplicationAssertionError(`objectId expected, got ${typeof table.objectId}`);
|
|
156
147
|
}
|
|
@@ -161,35 +152,25 @@ export class CDCStream {
|
|
|
161
152
|
entity_descriptor: table,
|
|
162
153
|
sync_rules: this.syncRules
|
|
163
154
|
});
|
|
164
|
-
const
|
|
165
|
-
connectionManager: this.connections,
|
|
166
|
-
tableName: resolved.table.name,
|
|
167
|
-
schema: resolved.table.schema
|
|
168
|
-
});
|
|
155
|
+
const resolvedTable = new MSSQLSourceTable(resolved.table);
|
|
169
156
|
if (!captureInstance) {
|
|
170
|
-
|
|
157
|
+
this.logger.warn(`Missing capture instance for table ${resolvedTable.toQualifiedName()}. This table will not be replicated until CDC is enabled for it.`);
|
|
171
158
|
}
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
159
|
+
else {
|
|
160
|
+
resolvedTable.setCaptureInstance(captureInstance);
|
|
161
|
+
}
|
|
162
|
+
this.tableCache.set(resolvedTable);
|
|
176
163
|
// Drop conflicting tables. This includes for example renamed tables.
|
|
177
164
|
await batch.drop(resolved.dropTables);
|
|
178
165
|
// Snapshot if:
|
|
179
|
-
// 1.
|
|
180
|
-
// 2.
|
|
181
|
-
|
|
182
|
-
const shouldSnapshot = snapshot && !resolved.table.snapshotComplete && resolved.table.syncAny;
|
|
166
|
+
// 1. The table is in the sync rules and snapshot is requested, or not already done.
|
|
167
|
+
// 2. AND the table is enabled for CDC with a valid capture instance.
|
|
168
|
+
const shouldSnapshot = snapshot && !resolved.table.snapshotComplete && resolved.table.syncAny && resolvedTable.enabledForCDC();
|
|
183
169
|
if (shouldSnapshot) {
|
|
184
170
|
// Truncate this table in case a previous snapshot was interrupted.
|
|
185
171
|
await batch.truncate([resolved.table]);
|
|
186
172
|
// Start the snapshot inside a transaction.
|
|
187
|
-
|
|
188
|
-
await this.snapshotTableInTx(batch, resolvedTable);
|
|
189
|
-
}
|
|
190
|
-
finally {
|
|
191
|
-
// TODO Cleanup?
|
|
192
|
-
}
|
|
173
|
+
await this.snapshotTableInTx(batch, resolvedTable);
|
|
193
174
|
}
|
|
194
175
|
return resolvedTable;
|
|
195
176
|
}
|
|
@@ -217,6 +198,7 @@ export class CDCStream {
|
|
|
217
198
|
// Side note: A ROLLBACK would probably also be fine here, since we only read in this transaction.
|
|
218
199
|
await transaction.commit();
|
|
219
200
|
const [updatedSourceTable] = await batch.markTableSnapshotDone([table.sourceTable], postSnapshotLSN.toString());
|
|
201
|
+
this.logger.info(`Snapshot of ${table.toQualifiedName()} completed. Post-snapshot LSN: ${postSnapshotLSN.toString()}`);
|
|
220
202
|
this.tableCache.updateSourceTable(updatedSourceTable);
|
|
221
203
|
}
|
|
222
204
|
catch (e) {
|
|
@@ -237,15 +219,15 @@ export class CDCStream {
|
|
|
237
219
|
const orderByKey = table.sourceTable.replicaIdColumns[0];
|
|
238
220
|
query = new BatchedSnapshotQuery(transaction, table, this.snapshotBatchSize, table.sourceTable.snapshotStatus?.lastKey ?? null);
|
|
239
221
|
if (table.sourceTable.snapshotStatus?.lastKey != null) {
|
|
240
|
-
this.logger.info(`
|
|
222
|
+
this.logger.info(`Snapshotting ${table.toQualifiedName()} ${table.sourceTable.formatSnapshotProgress()} - resuming from ${orderByKey.name} > ${query.lastKey}`);
|
|
241
223
|
}
|
|
242
224
|
else {
|
|
243
|
-
this.logger.info(`
|
|
225
|
+
this.logger.info(`Snapshotting ${table.toQualifiedName()} ${table.sourceTable.formatSnapshotProgress()} - resumable`);
|
|
244
226
|
}
|
|
245
227
|
}
|
|
246
228
|
else {
|
|
247
229
|
// Fallback case - query the entire table
|
|
248
|
-
this.logger.info(`
|
|
230
|
+
this.logger.info(`Snapshotting ${table.toQualifiedName()} ${table.sourceTable.formatSnapshotProgress()} - not resumable`);
|
|
249
231
|
query = new SimpleSnapshotQuery(transaction, table);
|
|
250
232
|
replicatedCount = 0;
|
|
251
233
|
}
|
|
@@ -305,7 +287,6 @@ export class CDCStream {
|
|
|
305
287
|
totalEstimatedCount: totalEstimatedCount
|
|
306
288
|
});
|
|
307
289
|
this.tableCache.updateSourceTable(updatedSourceTable);
|
|
308
|
-
this.logger.info(`Replicating ${table.toQualifiedName()} ${table.sourceTable.formatSnapshotProgress()}`);
|
|
309
290
|
if (this.abortSignal.aborted) {
|
|
310
291
|
// We only abort after flushing
|
|
311
292
|
throw new ReplicationAbortedError(`Initial replication interrupted`);
|
|
@@ -314,6 +295,9 @@ export class CDCStream {
|
|
|
314
295
|
if (batchReplicatedCount < this.snapshotBatchSize) {
|
|
315
296
|
hasRemainingData = false;
|
|
316
297
|
}
|
|
298
|
+
else {
|
|
299
|
+
this.logger.info(`Snapshotting ${table.toQualifiedName()} ${table.sourceTable.formatSnapshotProgress()}`);
|
|
300
|
+
}
|
|
317
301
|
}
|
|
318
302
|
}
|
|
319
303
|
/**
|
|
@@ -337,12 +321,7 @@ export class CDCStream {
|
|
|
337
321
|
* and starts again from scratch.
|
|
338
322
|
*/
|
|
339
323
|
async startInitialReplication(snapshotStatus) {
|
|
340
|
-
let { status, snapshotLSN } = snapshotStatus;
|
|
341
|
-
if (status === SnapshotStatus.RESTART_REQUIRED) {
|
|
342
|
-
this.logger.info(`Snapshot restart required, clearing state.`);
|
|
343
|
-
// This happens if the last replicated checkpoint LSN is no longer available in the CDC tables.
|
|
344
|
-
await this.storage.clear({ signal: this.abortSignal });
|
|
345
|
-
}
|
|
324
|
+
let { status, snapshotLSN, specificTablesToResnapshot } = snapshotStatus;
|
|
346
325
|
await this.storage.startBatch({
|
|
347
326
|
logger: this.logger,
|
|
348
327
|
zeroLSN: LSN.ZERO,
|
|
@@ -350,15 +329,26 @@ export class CDCStream {
|
|
|
350
329
|
storeCurrentData: false,
|
|
351
330
|
skipExistingRows: true
|
|
352
331
|
}, async (batch) => {
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
332
|
+
switch (status) {
|
|
333
|
+
case SnapshotStatus.INITIAL:
|
|
334
|
+
// First replication attempt - set the snapshot LSN to the current LSN before starting
|
|
335
|
+
snapshotLSN = (await getLatestReplicatedLSN(this.connections)).toString();
|
|
336
|
+
await batch.setResumeLsn(snapshotLSN);
|
|
337
|
+
const latestLSN = (await getLatestLSN(this.connections)).toString();
|
|
338
|
+
this.logger.info(`Marking snapshot at ${snapshotLSN}, Latest DB LSN ${latestLSN}.`);
|
|
339
|
+
break;
|
|
340
|
+
case SnapshotStatus.RESUME:
|
|
341
|
+
this.logger.info(`Resuming snapshot at ${snapshotLSN}.`);
|
|
342
|
+
break;
|
|
343
|
+
case SnapshotStatus.LIMITED_RESNAPSHOT:
|
|
344
|
+
for (const table of specificTablesToResnapshot) {
|
|
345
|
+
await batch.drop([table.sourceTable]);
|
|
346
|
+
// Update table in the table cache
|
|
347
|
+
await this.processTable(batch, table.sourceTable, table.captureInstance, false);
|
|
348
|
+
}
|
|
349
|
+
break;
|
|
350
|
+
default:
|
|
351
|
+
throw new ReplicationAssertionError(`Unsupported snapshot status: ${status}`);
|
|
362
352
|
}
|
|
363
353
|
const tablesToSnapshot = [];
|
|
364
354
|
for (const table of this.tableCache.getAll()) {
|
|
@@ -366,13 +356,16 @@ export class CDCStream {
|
|
|
366
356
|
this.logger.info(`Skipping table [${table.toQualifiedName()}] - snapshot already done.`);
|
|
367
357
|
continue;
|
|
368
358
|
}
|
|
359
|
+
if (!table.enabledForCDC()) {
|
|
360
|
+
this.logger.info(`Skipping table [${table.toQualifiedName()}] - not enabled for CDC.`);
|
|
361
|
+
continue;
|
|
362
|
+
}
|
|
369
363
|
const count = await this.estimatedCountNumber(table);
|
|
370
364
|
const updatedSourceTable = await batch.updateTableProgress(table.sourceTable, {
|
|
371
365
|
totalEstimatedCount: count
|
|
372
366
|
});
|
|
373
367
|
this.tableCache.updateSourceTable(updatedSourceTable);
|
|
374
368
|
tablesToSnapshot.push(table);
|
|
375
|
-
this.logger.info(`To replicate: ${table.toQualifiedName()} ${table.sourceTable.formatSnapshotProgress()}`);
|
|
376
369
|
}
|
|
377
370
|
for (const table of tablesToSnapshot) {
|
|
378
371
|
await this.snapshotTableInTx(batch, table);
|
|
@@ -383,7 +376,12 @@ export class CDCStream {
|
|
|
383
376
|
const postSnapshotLSN = await getLatestLSN(this.connections);
|
|
384
377
|
await batch.markAllSnapshotDone(postSnapshotLSN.toString());
|
|
385
378
|
await batch.commit(snapshotLSN);
|
|
386
|
-
|
|
379
|
+
if (tablesToSnapshot.length > 0) {
|
|
380
|
+
this.logger.info(`All snapshots done. Need to replicate from ${snapshotLSN} to ${postSnapshotLSN} to be consistent.`);
|
|
381
|
+
}
|
|
382
|
+
else {
|
|
383
|
+
this.logger.info(`No tables to snapshot. Need to replicate from ${snapshotLSN}.`);
|
|
384
|
+
}
|
|
387
385
|
});
|
|
388
386
|
}
|
|
389
387
|
async initReplication() {
|
|
@@ -396,6 +394,9 @@ export class CDCStream {
|
|
|
396
394
|
if (snapshotStatus.status !== SnapshotStatus.DONE) {
|
|
397
395
|
await this.startInitialReplication(snapshotStatus);
|
|
398
396
|
}
|
|
397
|
+
else {
|
|
398
|
+
this.logger.info(`Initial replication already done`);
|
|
399
|
+
}
|
|
399
400
|
}
|
|
400
401
|
/**
|
|
401
402
|
* Checks if the initial sync has already been completed and if updates from the last checkpoint are still available
|
|
@@ -404,23 +405,44 @@ export class CDCStream {
|
|
|
404
405
|
async checkSnapshotStatus() {
|
|
405
406
|
const status = await this.storage.getStatus();
|
|
406
407
|
if (status.snapshot_done && status.checkpoint_lsn) {
|
|
408
|
+
const additionalTablesToSnapshot = new Set();
|
|
409
|
+
const newTables = this.tableCache.getAll().filter((table) => !table.sourceTable.snapshotComplete);
|
|
410
|
+
if (newTables.length > 0) {
|
|
411
|
+
this.logger.info(`Detected new table(s) [${newTables.map((table) => table.toQualifiedName()).join(', ')}] that have not been snapshotted yet.`);
|
|
412
|
+
newTables.forEach((table) => additionalTablesToSnapshot.add(table));
|
|
413
|
+
}
|
|
407
414
|
// Snapshot is done, but we still need to check that the last known checkpoint LSN is still
|
|
408
|
-
// within the threshold of the CDC tables
|
|
409
|
-
this.logger.info(`Initial replication already done`);
|
|
415
|
+
// within the retention threshold of the CDC tables
|
|
410
416
|
const lastCheckpointLSN = LSN.fromString(status.checkpoint_lsn);
|
|
411
417
|
// Check that the CDC tables still have valid data
|
|
412
|
-
const
|
|
418
|
+
const tablesOutsideRetentionThreshold = await checkRetentionThresholds({
|
|
413
419
|
checkpointLSN: lastCheckpointLSN,
|
|
414
420
|
tables: this.tableCache.getAll(),
|
|
415
421
|
connectionManager: this.connections
|
|
416
422
|
});
|
|
417
|
-
if (
|
|
418
|
-
this.logger.warn(`Updates from the last checkpoint are no longer available in the CDC
|
|
423
|
+
if (tablesOutsideRetentionThreshold.length > 0) {
|
|
424
|
+
this.logger.warn(`Updates from the last checkpoint are no longer available in the CDC instances of the following table(s): ${tablesOutsideRetentionThreshold.map((table) => table.toQualifiedName()).join(', ')}.`);
|
|
425
|
+
tablesOutsideRetentionThreshold.forEach((table) => additionalTablesToSnapshot.add(table));
|
|
426
|
+
}
|
|
427
|
+
if (additionalTablesToSnapshot.size > 0) {
|
|
428
|
+
return {
|
|
429
|
+
status: SnapshotStatus.LIMITED_RESNAPSHOT,
|
|
430
|
+
snapshotLSN: status.checkpoint_lsn,
|
|
431
|
+
specificTablesToResnapshot: Array.from(additionalTablesToSnapshot)
|
|
432
|
+
};
|
|
433
|
+
}
|
|
434
|
+
else {
|
|
435
|
+
return {
|
|
436
|
+
status: SnapshotStatus.DONE,
|
|
437
|
+
snapshotLSN: null
|
|
438
|
+
};
|
|
419
439
|
}
|
|
420
|
-
return { status: isAvailable ? SnapshotStatus.DONE : SnapshotStatus.RESTART_REQUIRED, snapshotLSN: null };
|
|
421
440
|
}
|
|
422
441
|
else {
|
|
423
|
-
return {
|
|
442
|
+
return {
|
|
443
|
+
status: status.snapshot_lsn != null ? SnapshotStatus.RESUME : SnapshotStatus.INITIAL,
|
|
444
|
+
snapshotLSN: status.snapshot_lsn
|
|
445
|
+
};
|
|
424
446
|
}
|
|
425
447
|
}
|
|
426
448
|
async streamChanges() {
|
|
@@ -435,15 +457,16 @@ export class CDCStream {
|
|
|
435
457
|
throw new ReplicationAssertionError(`No LSN found to resume replication from.`);
|
|
436
458
|
}
|
|
437
459
|
const startLSN = LSN.fromString(batch.resumeFromLsn);
|
|
438
|
-
const sourceTables = this.tableCache.getAll();
|
|
439
460
|
const eventHandler = this.createEventHandler(batch);
|
|
440
461
|
const poller = new CDCPoller({
|
|
441
462
|
connectionManager: this.connections,
|
|
442
463
|
eventHandler,
|
|
443
|
-
|
|
464
|
+
getReplicatedTables: () => this.tableCache.getAll(),
|
|
465
|
+
sourceTables: this.syncRules.getSourceTables(),
|
|
444
466
|
startLSN,
|
|
445
467
|
logger: this.logger,
|
|
446
|
-
additionalConfig: this.options.additionalConfig
|
|
468
|
+
additionalConfig: this.options.additionalConfig,
|
|
469
|
+
schemaCheckIntervalMs: this.options.schemaCheckIntervalMs
|
|
447
470
|
});
|
|
448
471
|
this.abortSignal.addEventListener('abort', async () => {
|
|
449
472
|
await poller.stop();
|
|
@@ -499,11 +522,87 @@ export class CDCStream {
|
|
|
499
522
|
this.isStartingReplication = false;
|
|
500
523
|
}
|
|
501
524
|
},
|
|
502
|
-
onSchemaChange: async () => {
|
|
503
|
-
|
|
525
|
+
onSchemaChange: async (schemaChange) => {
|
|
526
|
+
await this.handleSchemaChange(batch, schemaChange);
|
|
504
527
|
}
|
|
505
528
|
};
|
|
506
529
|
}
|
|
530
|
+
async handleSchemaChange(batch, change) {
|
|
531
|
+
let actionedSchemaChange = true;
|
|
532
|
+
switch (change.type) {
|
|
533
|
+
case SchemaChangeType.TABLE_RENAME:
|
|
534
|
+
const fromTable = change.table;
|
|
535
|
+
this.logger.info(`Table ${fromTable.toQualifiedName()} has been renamed ${change.newTable ? `to [${change.newTable.name}].` : '.'}`);
|
|
536
|
+
// Old table needs to be cleaned up
|
|
537
|
+
await batch.drop([fromTable.sourceTable]);
|
|
538
|
+
this.tableCache.delete(fromTable.objectId);
|
|
539
|
+
if (change.newTable) {
|
|
540
|
+
await this.handleCreateOrUpdateTable(batch, change.newTable, change.newCaptureInstance);
|
|
541
|
+
}
|
|
542
|
+
break;
|
|
543
|
+
case SchemaChangeType.TABLE_CREATE:
|
|
544
|
+
await this.handleCreateOrUpdateTable(batch, change.newTable, change.newCaptureInstance);
|
|
545
|
+
break;
|
|
546
|
+
case SchemaChangeType.TABLE_COLUMN_CHANGES:
|
|
547
|
+
await this.handleColumnChanges(change.table, change.newCaptureInstance);
|
|
548
|
+
actionedSchemaChange = false;
|
|
549
|
+
break;
|
|
550
|
+
case SchemaChangeType.NEW_CAPTURE_INSTANCE:
|
|
551
|
+
this.logger.info(`New CDC capture instance detected for table ${change.table.toQualifiedName()}. Re-snapshotting table...`);
|
|
552
|
+
await batch.drop([change.table.sourceTable]);
|
|
553
|
+
this.tableCache.delete(change.table.objectId);
|
|
554
|
+
await this.handleCreateOrUpdateTable(batch, change.table.sourceTable, change.newCaptureInstance);
|
|
555
|
+
break;
|
|
556
|
+
case SchemaChangeType.TABLE_DROP:
|
|
557
|
+
await batch.drop([change.table.sourceTable]);
|
|
558
|
+
this.tableCache.delete(change.table.objectId);
|
|
559
|
+
break;
|
|
560
|
+
case SchemaChangeType.MISSING_CAPTURE_INSTANCE:
|
|
561
|
+
// Stop replication for this table until CDC is re-enabled.
|
|
562
|
+
this.logger.warn(`Table ${change.table.toQualifiedName()} has been disabled for CDC. Re-enable CDC to continue replication.`);
|
|
563
|
+
change.table.clearCaptureInstance();
|
|
564
|
+
actionedSchemaChange = false;
|
|
565
|
+
break;
|
|
566
|
+
default:
|
|
567
|
+
throw new ReplicationAssertionError(`Unknown schema change type: ${change.type}`);
|
|
568
|
+
}
|
|
569
|
+
// Create a new checkpoint after the schema change
|
|
570
|
+
if (actionedSchemaChange) {
|
|
571
|
+
await createCheckpoint(this.connections);
|
|
572
|
+
}
|
|
573
|
+
}
|
|
574
|
+
async handleCreateOrUpdateTable(batch, table, captureInstance) {
|
|
575
|
+
const replicaIdColumns = await getReplicationIdentityColumns({
|
|
576
|
+
connectionManager: this.connections,
|
|
577
|
+
tableName: table.name,
|
|
578
|
+
schema: table.schema
|
|
579
|
+
});
|
|
580
|
+
await this.processTable(batch, {
|
|
581
|
+
name: table.name,
|
|
582
|
+
schema: table.schema,
|
|
583
|
+
objectId: table.objectId,
|
|
584
|
+
replicaIdColumns: replicaIdColumns.columns
|
|
585
|
+
}, captureInstance, true);
|
|
586
|
+
}
|
|
587
|
+
/**
|
|
588
|
+
* There is very little that can be automatically done to handle the column changes other than to warn the user about the schema drift.
|
|
589
|
+
*
|
|
590
|
+
* Due to the way CDC works, users are prevented from making column schema changes that affect the replication identities of a table.
|
|
591
|
+
* If changes like that are required, CDC has to be disabled and re-enabled for the table. This would then be handled by the detection of the new
|
|
592
|
+
* capture instance.
|
|
593
|
+
* @param table
|
|
594
|
+
* @param captureInstance
|
|
595
|
+
*/
|
|
596
|
+
async handleColumnChanges(table, captureInstance) {
|
|
597
|
+
// Check there are any new pending schema changes
|
|
598
|
+
if (table.captureInstance?.objectId === captureInstance.objectId &&
|
|
599
|
+
table.captureInstance?.pendingSchemaChanges.length === captureInstance.pendingSchemaChanges.length) {
|
|
600
|
+
return;
|
|
601
|
+
}
|
|
602
|
+
// New pending schema changes were detected - warn about those as well.
|
|
603
|
+
this.logger.warn(`Schema drift detected for table ${table.toQualifiedName()}. To ensure consistency, disable and re-enable CDC for this table.\n Pending schema changes:\n ${captureInstance.pendingSchemaChanges.join(', \n')}`);
|
|
604
|
+
table.captureInstance = captureInstance;
|
|
605
|
+
}
|
|
507
606
|
/**
|
|
508
607
|
* Convert CDC row data to SqliteRow format.
|
|
509
608
|
* CDC rows include table columns plus CDC metadata columns (__$operation, __$start_lsn, etc.).
|
|
@@ -531,5 +630,13 @@ export class CDCStream {
|
|
|
531
630
|
this.logger.error(`Error touching probe`, e);
|
|
532
631
|
});
|
|
533
632
|
}
|
|
633
|
+
/**
|
|
634
|
+
* Creates an update in the source database to ensure regular checkpoints via the CDC
|
|
635
|
+
*/
|
|
636
|
+
async keepAlive() {
|
|
637
|
+
if (!this.isStartingReplication && !this.stopped) {
|
|
638
|
+
await createCheckpoint(this.connections);
|
|
639
|
+
}
|
|
640
|
+
}
|
|
534
641
|
}
|
|
535
642
|
//# sourceMappingURL=CDCStream.js.map
|