@powersync/service-module-mssql 0.4.0 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +45 -0
- package/dist/common/CaptureInstance.d.ts +14 -0
- package/dist/common/CaptureInstance.js +2 -0
- package/dist/common/CaptureInstance.js.map +1 -0
- package/dist/common/MSSQLSourceTable.d.ts +16 -14
- package/dist/common/MSSQLSourceTable.js +35 -16
- package/dist/common/MSSQLSourceTable.js.map +1 -1
- package/dist/replication/CDCPoller.d.ts +42 -20
- package/dist/replication/CDCPoller.js +200 -60
- package/dist/replication/CDCPoller.js.map +1 -1
- package/dist/replication/CDCReplicationJob.js +9 -1
- package/dist/replication/CDCReplicationJob.js.map +1 -1
- package/dist/replication/CDCStream.d.ts +35 -4
- package/dist/replication/CDCStream.js +188 -77
- package/dist/replication/CDCStream.js.map +1 -1
- package/dist/replication/MSSQLConnectionManager.js +16 -5
- package/dist/replication/MSSQLConnectionManager.js.map +1 -1
- package/dist/types/types.d.ts +4 -56
- package/dist/types/types.js +5 -24
- package/dist/types/types.js.map +1 -1
- package/dist/utils/deadlock.d.ts +9 -0
- package/dist/utils/deadlock.js +40 -0
- package/dist/utils/deadlock.js.map +1 -0
- package/dist/utils/mssql.d.ts +33 -15
- package/dist/utils/mssql.js +101 -99
- package/dist/utils/mssql.js.map +1 -1
- package/dist/utils/schema.d.ts +9 -0
- package/dist/utils/schema.js +34 -0
- package/dist/utils/schema.js.map +1 -1
- package/package.json +8 -8
- package/src/common/CaptureInstance.ts +15 -0
- package/src/common/MSSQLSourceTable.ts +33 -24
- package/src/replication/CDCPoller.ts +272 -72
- package/src/replication/CDCReplicationJob.ts +8 -1
- package/src/replication/CDCStream.ts +245 -96
- package/src/replication/MSSQLConnectionManager.ts +15 -5
- package/src/types/types.ts +5 -28
- package/src/utils/deadlock.ts +44 -0
- package/src/utils/mssql.ts +159 -124
- package/src/utils/schema.ts +43 -0
- package/test/src/CDCStream.test.ts +3 -1
- package/test/src/CDCStreamTestContext.ts +28 -7
- package/test/src/CDCStream_resumable_snapshot.test.ts +9 -7
- package/test/src/env.ts +1 -1
- package/test/src/mssql-to-sqlite.test.ts +18 -10
- package/test/src/schema-changes.test.ts +470 -0
- package/test/src/util.ts +84 -15
- package/tsconfig.tsbuildinfo +1 -1
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"CDCPoller.js","sourceRoot":"","sources":["../../src/replication/CDCPoller.ts"],"names":[],"mappings":"AAAA,OAAO,EAAU,MAAM,IAAI,aAAa,EAAE,yBAAyB,EAAE,MAAM,mCAAmC,CAAC;AAC/G,OAAO,MAAM,MAAM,iBAAiB,CAAC;AAGrC,OAAO,EAAE,GAAG,EAAE,MAAM,kBAAkB,CAAC;AACvC,OAAO,GAAG,MAAM,OAAO,CAAC;AACxB,OAAO,EAAE,SAAS,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AAG5D,IAAK,SAKJ;AALD,WAAK,SAAS;IACZ,6CAAU,CAAA;IACV,6CAAU,CAAA;IACV,2DAAiB,CAAA;IACjB,yDAAgB,CAAA;AAClB,CAAC,EALI,SAAS,KAAT,SAAS,QAKb;AACD;;;;GAIG;AACH,MAAM,CAAN,IAAY,gBAMX;AAND,WAAY,gBAAgB;IAC1B,iDAA6B,CAAA;IAC7B,6CAAyB,CAAA;IACzB,qDAAiC,CAAA;IACjC,6DAAyC,CAAA;IACzC,uEAAmD,CAAA;AACrD,CAAC,EANW,gBAAgB,KAAhB,gBAAgB,QAM3B;AAgCD;;GAEG;AACH,MAAM,OAAO,SAAS;IAWD;IAVX,iBAAiB,CAAyB;IAC1C,YAAY,CAAkB;IAC9B,UAAU,CAAM;IAChB,MAAM,CAAS;IACf,aAAa,CAAe;IAE5B,SAAS,GAAY,KAAK,CAAC;IAC3B,UAAU,GAAY,KAAK,CAAC;IAC5B,SAAS,GAAY,KAAK,CAAC;IAEnC,YAAmB,OAAyB;QAAzB,YAAO,GAAP,OAAO,CAAkB;QAC1C,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,aAAa,CAAC;QAC9C,IAAI,CAAC,iBAAiB,GAAG,OAAO,CAAC,iBAAiB,CAAC;QACnD,IAAI,CAAC,YAAY,GAAG,OAAO,CAAC,YAAY,CAAC;QACzC,IAAI,CAAC,UAAU,GAAG,OAAO,CAAC,QAAQ,CAAC;QACnC,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC;IAC5B,CAAC;IAED,IAAY,gBAAgB;QAC1B,OAAO,IAAI,CAAC,OAAO,CAAC,gBAAgB,CAAC,gBAAgB,CAAC;IACxD,CAAC;IAED,IAAY,iBAAiB;QAC3B,OAAO,IAAI,CAAC,OAAO,CAAC,gBAAgB,CAAC,iBAAiB,CAAC;IACzD,CAAC;IAED,IAAY,YAAY;QACtB,OAAO,IAAI,CAAC,OAAO,CAAC,YAAY,CAAC;IACnC,CAAC;IAEM,KAAK,CAAC,IAAI;QACf,IAAI,CAAC,CAAC,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,UAAU,CAAC,EAAE,CAAC;YACzC,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC;YACvB,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;QACxB,CAAC;IACH,CAAC;IAEM,KAAK,CAAC,qBAAqB;QAChC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,wCAAwC,IAAI,CAAC,iBAAiB,OAAO,CAAC,CAAC;QACxF,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,wBAAwB,IAAI,CAAC,gBAAgB,kCAAkC,CAAC,CAAC;QAClG,OAAO,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC;YACvB,oDAAoD;YACpD,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;gBACnB,MAAM,IAAI,yBAAyB,CAAC,yCAAyC,CAAC,CAAC;YACjF,CAAC;YAED,IAAI,CAAC;gBACH,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;gBACrC,IAAI,CAAC,UAAU,EAAE,CAAC;oBAChB,0CAA0C;oBAC1C,MAAM,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;gBAClD,CAAC;gBACD,0DAA0D;YAC5D,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,IAAI,CAAC,CAAC,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,UAAU,CAAC,EAAE,CAAC;oBACzC,IAAI,CAAC,aAAa,GAAG,KAAc,CAAC;oBACpC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,2BAA2B,EAAE,KAAK,CAAC,CAAC;oBACtD,IAAI,CAAC,IAAI,EAAE,CAAC;gBACd,CAAC;gBACD,MAAM;YACR,CAAC;QACH,CAAC;QAED,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;YACvB,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,0CAA0C,EAAE,IAAI,CAAC,aAAa,CAAC,CAAC;YAClF,MAAM,IAAI,CAAC,aAAa,CAAC;QAC3B,CAAC;QAED,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,wBAAwB,CAAC,CAAC;IAC7C,CAAC;IAEO,KAAK,CAAC,IAAI;QAChB,wDAAwD;QACxD,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;QAEtB,IAAI,CAAC;YACH,0CAA0C;YAC1C,mFAAmF;YACnF,MAAM,QAAQ,GAAG,MAAM,YAAY,CAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,iBAAiB,CAAC,CAAC;YAE7E,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,KAAK,CAC/D,eAAe,IAAI,CAAC,gBAAgB;;;;SAInC,EACD,CAAC,EAAE,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,GAAG,CAAC,SAAS,EAAE,KAAK,EAAE,QAAQ,CAAC,QAAQ,EAAE,EAAE,CAAC,CACxE,CAAC;YAEF,2CAA2C;YAC3C,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACzB,OAAO,KAAK,CAAC;YACf,CAAC;YAED,kDAAkD;YAClD,MAAM,MAAM,GAAG,GAAG,CAAC,UAAU,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;YAErE,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,sBAAsB,QAAQ,OAAO,MAAM,aAAa,OAAO,CAAC,MAAM,kBAAkB,CAAC,CAAC;YAE3G,IAAI,gBAAgB,GAAG,CAAC,CAAC;YACzB,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;gBACtC,MAAM,qBAAqB,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAC;gBAChF,2GAA2G;gBAC3G,yKAAyK;gBACzK,IAAI,qBAAqB,GAAG,gBAAgB,EAAE,CAAC;oBAC7C,gBAAgB,GAAG,qBAAqB,CAAC;gBAC3C,CAAC;YACH,CAAC;YAED,IAAI,CAAC,MAAM,CAAC,IAAI,CACd,aAAa,OAAO,CAAC,MAAM,8BAA8B,gBAAgB,+BAA+B,CACzG,CAAC;YACF,sEAAsE;YACtE,MAAM,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ,EAAE,EAAE,gBAAgB,CAAC,CAAC;YAEtE,IAAI,CAAC,UAAU,GAAG,MAAM,CAAC;YAEzB,OAAO,IAAI,CAAC;QACd,CAAC;gBAAS,CAAC;YACT,2CAA2C;YAC3C,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC;QACzB,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,SAAS,CAAC,KAAuB,EAAE,MAAsC;QACrF,uEAAuE;QACvE,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,IAAI,CAAC,iBAAiB,EAAE,KAAK,CAAC,eAAe,CAAC,CAAC;QAC9E,IAAI,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC;YAC3B,OAAO,CAAC,CAAC;QACX,CAAC;aAAM,IAAI,MAAM,IAAI,MAAM,CAAC,QAAQ,EAAE,CAAC;YACrC,MAAM,CAAC,QAAQ,GAAG,MAAM,CAAC;QAC3B,CAAC;QACD,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,KAAK,CAC/D;wBACkB,KAAK,CAAC,kBAAkB;KAC3C,EACC;YACE,EAAE,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,GAAG,CAAC,SAAS,EAAE,KAAK,EAAE,MAAM,CAAC,QAAQ,CAAC,QAAQ,EAAE,EAAE;YAC5E,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,GAAG,CAAC,SAAS,EAAE,KAAK,EAAE,MAAM,CAAC,MAAM,CAAC,QAAQ,EAAE,EAAE;SACzE,CACF,CAAC;QAEF,IAAI,gBAAgB,GAAG,CAAC,CAAC;QACzB,IAAI,YAAY,GAAQ,IAAI,CAAC;QAC7B,IAAI,kBAAkB,GAAe,IAAI,CAAC;QAC1C,KAAK,MAAM,GAAG,IAAI,OAAO,EAAE,CAAC;YAC1B,MAAM,cAAc,GAAG,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;YACxD,QAAQ,GAAG,CAAC,YAAY,EAAE,CAAC;gBACzB,KAAK,SAAS,CAAC,MAAM;oBACnB,MAAM,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC,GAAG,EAAE,KAAK,EAAE,OAAO,CAAC,OAAO,CAAC,CAAC;oBAC9D,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,6BAA6B,cAAc,EAAE,CAAC,CAAC;oBAChE,MAAM;gBACR,KAAK,SAAS,CAAC,MAAM;oBACnB,MAAM,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC,GAAG,EAAE,KAAK,EAAE,OAAO,CAAC,OAAO,CAAC,CAAC;oBAC9D,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,6BAA6B,cAAc,EAAE,CAAC,CAAC;oBAChE,MAAM;gBACR,KAAK,SAAS,CAAC,aAAa;oBAC1B,YAAY,GAAG,GAAG,CAAC;oBACnB,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,qCAAqC,cAAc,EAAE,CAAC,CAAC;oBACzE,MAAM;gBACR,KAAK,SAAS,CAAC,YAAY;oBACzB,IAAI,YAAY,KAAK,IAAI,EAAE,CAAC;wBAC1B,MAAM,IAAI,yBAAyB,CAAC,wCAAwC,CAAC,CAAC;oBAChF,CAAC;oBACD,MAAM,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC,GAAG,EAAE,YAAY,EAAE,KAAK,EAAE,OAAO,CAAC,OAAO,CAAC,CAAC;oBAC5E,YAAY,GAAG,IAAI,CAAC;oBACpB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,6BAA6B,cAAc,EAAE,CAAC,CAAC;oBAChE,MAAM;gBACR;oBACE,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,2BAA2B,GAAG,CAAC,YAAY,+BAA+B,CAAC,CAAC;YACjG,CAAC;YAED,sGAAsG;YACtG,IAAI,cAAc,IAAI,kBAAkB,EAAE,CAAC;gBACzC,kBAAkB,GAAG,cAAc,CAAC;gBACpC,IAAI,GAAG,CAAC,YAAY,KAAK,SAAS,CAAC,aAAa,EAAE,CAAC;oBACjD,gBAAgB,EAAE,CAAC;gBACrB,CAAC;YACH,CAAC;QACH,CAAC;QAED,OAAO,gBAAgB,CAAC;IAC1B,CAAC;CACF"}
|
|
1
|
+
{"version":3,"file":"CDCPoller.js","sourceRoot":"","sources":["../../src/replication/CDCPoller.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,kBAAkB,EAClB,SAAS,EAET,MAAM,IAAI,aAAa,EACvB,yBAAyB,EAC1B,MAAM,mCAAmC,CAAC;AAC3C,OAAO,MAAM,MAAM,iBAAiB,CAAC;AAGrC,OAAO,EAAE,GAAG,EAAE,MAAM,kBAAkB,CAAC;AACvC,OAAO,GAAG,MAAM,OAAO,CAAC;AACxB,OAAO,EAA0B,mBAAmB,EAAE,YAAY,EAAE,oBAAoB,EAAE,MAAM,mBAAmB,CAAC;AACpH,OAAO,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AAEvD,OAAO,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AAKjD,IAAK,SAKJ;AALD,WAAK,SAAS;IACZ,6CAAU,CAAA;IACV,6CAAU,CAAA;IACV,2DAAiB,CAAA;IACjB,yDAAgB,CAAA;AAClB,CAAC,EALI,SAAS,KAAT,SAAS,QAKb;AAED,MAAM,CAAN,IAAY,gBAOX;AAPD,WAAY,gBAAgB;IAC1B,iDAA6B,CAAA;IAC7B,6CAAyB,CAAA;IACzB,iDAA6B,CAAA;IAC7B,iEAA6C,CAAA;IAC7C,iEAA6C,CAAA;IAC7C,yEAAqD,CAAA;AACvD,CAAC,EAPW,gBAAgB,KAAhB,gBAAgB,QAO3B;AAwBD,MAAM,CAAC,MAAM,gCAAgC,GAAG,MAAM,CAAC;AAsBvD;;GAEG;AACH,MAAM,OAAO,SAAS;IAaD;IAZX,iBAAiB,CAAyB;IAC1C,YAAY,CAAkB;IAC9B,UAAU,CAAM;IAChB,MAAM,CAAS;IACf,aAAa,CAAe;IAC5B,gBAAgB,CAAsC;IAEtD,SAAS,GAAY,KAAK,CAAC;IAC3B,UAAU,GAAY,KAAK,CAAC;IAC5B,SAAS,GAAY,KAAK,CAAC;IAC3B,mBAAmB,GAAW,CAAC,CAAC;IAExC,YAAmB,OAAyB;QAAzB,YAAO,GAAP,OAAO,CAAkB;QAC1C,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,aAAa,CAAC;QAC9C,IAAI,CAAC,iBAAiB,GAAG,OAAO,CAAC,iBAAiB,CAAC;QACnD,IAAI,CAAC,YAAY,GAAG,OAAO,CAAC,YAAY,CAAC;QACzC,IAAI,CAAC,UAAU,GAAG,OAAO,CAAC,QAAQ,CAAC;QACnC,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC;QAC1B,IAAI,CAAC,gBAAgB,GAAG,IAAI,GAAG,EAAkC,CAAC;IACpE,CAAC;IAED,IAAY,gBAAgB;QAC1B,OAAO,IAAI,CAAC,OAAO,CAAC,gBAAgB,CAAC,gBAAgB,CAAC;IACxD,CAAC;IAED,IAAY,iBAAiB;QAC3B,OAAO,IAAI,CAAC,OAAO,CAAC,gBAAgB,CAAC,iBAAiB,CAAC;IACzD,CAAC;IAED,IAAY,qBAAqB;QAC/B,OAAO,IAAI,CAAC,OAAO,CAAC,qBAAqB,IAAI,gCAAgC,CAAC;IAChF,CAAC;IAED,IAAY,gBAAgB;QAC1B,OAAO,IAAI,CAAC,OAAO,CAAC,mBAAmB,EAAE,CAAC;IAC5C,CAAC;IAEM,KAAK,CAAC,IAAI;QACf,IAAI,CAAC,CAAC,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,UAAU,CAAC,EAAE,CAAC;YACzC,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC;YACvB,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;QACxB,CAAC;IACH,CAAC;IAEM,KAAK,CAAC,qBAAqB;QAChC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,wCAAwC,IAAI,CAAC,iBAAiB,OAAO,CAAC,CAAC;QACxF,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,wBAAwB,IAAI,CAAC,gBAAgB,kCAAkC,CAAC,CAAC;QAClG,OAAO,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC;YACvB,oDAAoD;YACpD,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;gBACnB,MAAM,IAAI,yBAAyB,CAAC,yCAAyC,CAAC,CAAC;YACjF,CAAC;YAED,IAAI,CAAC;gBACH,IAAI,IAAI,CAAC,iBAAiB,EAAE,EAAE,CAAC;oBAC7B,IAAI,CAAC,gBAAgB,GAAG,MAAM,mBAAmB,CAAC,EAAE,iBAAiB,EAAE,IAAI,CAAC,iBAAiB,EAAE,CAAC,CAAC;oBACjG,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,qBAAqB,EAAE,CAAC;oBACzD,KAAK,MAAM,YAAY,IAAI,aAAa,EAAE,CAAC;wBACzC,MAAM,IAAI,CAAC,YAAY,CAAC,cAAc,CAAC,YAAY,CAAC,CAAC;oBACvD,CAAC;oBACD,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;oBAEtC,IAAI,CAAC,MAAM,CAAC,KAAK,CACf,uDAAuD,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CACrG,CAAC;gBACJ,CAAC;gBAED,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;gBACrC,IAAI,CAAC,UAAU,EAAE,CAAC;oBAChB,8CAA8C;oBAC9C,MAAM,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;gBAClD,CAAC;gBAED,0DAA0D;YAC5D,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,IAAI,CAAC,CAAC,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,UAAU,CAAC,EAAE,CAAC;oBACzC,qBAAqB;oBACrB,IAAI,KAAK,YAAY,kBAAkB,EAAE,CAAC;wBACxC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;wBAChC,kEAAkE;wBAClE,IAAI,CAAC,mBAAmB,GAAG,CAAC,CAAC;wBAC7B,SAAS;oBACX,CAAC;oBACD,kFAAkF;oBAClF,sFAAsF;oBACtF,IAAI,eAAe,CAAC,KAAK,CAAC,EAAE,CAAC;wBAC3B,IAAI,CAAC,MAAM,CAAC,IAAI,CACd,mGAAoG,KAAe,CAAC,OAAO,EAAE,CAC9H,CAAC;wBACF,SAAS;oBACX,CAAC;oBAED,yBAAyB;oBACzB,IAAI,CAAC,aAAa,GAAG,KAAc,CAAC;oBACpC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,2BAA2B,EAAE,KAAK,CAAC,CAAC;oBACtD,IAAI,CAAC,IAAI,EAAE,CAAC;gBACd,CAAC;gBACD,MAAM;YACR,CAAC;QACH,CAAC;QAED,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;YACvB,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,0CAA0C,EAAE,IAAI,CAAC,aAAa,CAAC,CAAC;YAClF,MAAM,IAAI,CAAC,aAAa,CAAC;QAC3B,CAAC;QAED,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,wBAAwB,CAAC,CAAC;IAC7C,CAAC;IAEO,KAAK,CAAC,IAAI;QAChB,wDAAwD;QACxD,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;QAEtB,IAAI,CAAC;YACH,0CAA0C;YAC1C,mFAAmF;YACnF,MAAM,QAAQ,GAAG,MAAM,YAAY,CAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,iBAAiB,CAAC,CAAC;YAE7E,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,KAAK,CAC/D,eAAe,IAAI,CAAC,gBAAgB;;;;SAInC,EACD,CAAC,EAAE,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,GAAG,CAAC,SAAS,EAAE,KAAK,EAAE,QAAQ,CAAC,QAAQ,EAAE,EAAE,CAAC,CACxE,CAAC;YAEF,2CAA2C;YAC3C,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACzB,OAAO,KAAK,CAAC;YACf,CAAC;YAED,kDAAkD;YAClD,MAAM,MAAM,GAAG,GAAG,CAAC,UAAU,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;YAErE,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,sBAAsB,QAAQ,OAAO,MAAM,aAAa,OAAO,CAAC,MAAM,kBAAkB,CAAC,CAAC;YAE3G,IAAI,gBAAgB,GAAG,CAAC,CAAC;YACzB,IAAI,CAAC,MAAM,CAAC,KAAK,CACf,iCAAiC,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,eAAe,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAC5G,CAAC;YACF,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,gBAAgB,EAAE,CAAC;gBAC1C,IAAI,KAAK,CAAC,aAAa,EAAE,EAAE,CAAC;oBAC1B,MAAM,qBAAqB,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAC;oBAChF,2GAA2G;oBAC3G,yKAAyK;oBACzK,IAAI,qBAAqB,GAAG,gBAAgB,EAAE,CAAC;wBAC7C,gBAAgB,GAAG,qBAAqB,CAAC;oBAC3C,CAAC;gBACH,CAAC;YACH,CAAC;YAED,IAAI,CAAC,MAAM,CAAC,IAAI,CACd,aAAa,OAAO,CAAC,MAAM,8BAA8B,gBAAgB,+CAA+C,MAAM,CAAC,QAAQ,EAAE,EAAE,CAC5I,CAAC;YACF,sEAAsE;YACtE,MAAM,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ,EAAE,EAAE,gBAAgB,CAAC,CAAC;YAEtE,IAAI,CAAC,UAAU,GAAG,MAAM,CAAC;YAEzB,OAAO,IAAI,CAAC;QACd,CAAC;gBAAS,CAAC;YACT,2CAA2C;YAC3C,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC;QACzB,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,SAAS,CAAC,KAAuB,EAAE,MAAsC;QACrF,uEAAuE;QACvE,MAAM,MAAM,GAAG,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,KAAK,CAAC,QAAQ,CAAE,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;QAC9E,IAAI,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC;YAC3B,OAAO,CAAC,CAAC;QACX,CAAC;aAAM,IAAI,MAAM,IAAI,MAAM,CAAC,QAAQ,EAAE,CAAC;YACrC,MAAM,CAAC,QAAQ,GAAG,MAAM,CAAC;QAC3B,CAAC;QAED,IAAI,CAAC;YACH,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,KAAK,CAC/D;wBACgB,KAAK,CAAC,kBAAkB;KAC3C,EACG;gBACE,EAAE,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,GAAG,CAAC,SAAS,EAAE,KAAK,EAAE,MAAM,CAAC,QAAQ,CAAC,QAAQ,EAAE,EAAE;gBAC5E,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,GAAG,CAAC,SAAS,EAAE,KAAK,EAAE,MAAM,CAAC,MAAM,CAAC,QAAQ,EAAE,EAAE;aACzE,CACF,CAAC;YAEF,IAAI,gBAAgB,GAAG,CAAC,CAAC;YACzB,IAAI,YAAY,GAAQ,IAAI,CAAC;YAC7B,IAAI,kBAAkB,GAAe,IAAI,CAAC;YAC1C,KAAK,MAAM,GAAG,IAAI,OAAO,EAAE,CAAC;gBAC1B,MAAM,cAAc,GAAG,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;gBACxD,QAAQ,GAAG,CAAC,YAAY,EAAE,CAAC;oBACzB,KAAK,SAAS,CAAC,MAAM;wBACnB,MAAM,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC,GAAG,EAAE,KAAK,EAAE,OAAO,CAAC,OAAO,CAAC,CAAC;wBAC9D,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,6BAA6B,cAAc,EAAE,CAAC,CAAC;wBAChE,MAAM;oBACR,KAAK,SAAS,CAAC,MAAM;wBACnB,MAAM,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC,GAAG,EAAE,KAAK,EAAE,OAAO,CAAC,OAAO,CAAC,CAAC;wBAC9D,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,6BAA6B,cAAc,EAAE,CAAC,CAAC;wBAChE,MAAM;oBACR,KAAK,SAAS,CAAC,aAAa;wBAC1B,YAAY,GAAG,GAAG,CAAC;wBACnB,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,qCAAqC,cAAc,EAAE,CAAC,CAAC;wBACzE,MAAM;oBACR,KAAK,SAAS,CAAC,YAAY;wBACzB,IAAI,YAAY,KAAK,IAAI,EAAE,CAAC;4BAC1B,MAAM,IAAI,yBAAyB,CAAC,wCAAwC,CAAC,CAAC;wBAChF,CAAC;wBACD,MAAM,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC,GAAG,EAAE,YAAY,EAAE,KAAK,EAAE,OAAO,CAAC,OAAO,CAAC,CAAC;wBAC5E,YAAY,GAAG,IAAI,CAAC;wBACpB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,6BAA6B,cAAc,EAAE,CAAC,CAAC;wBAChE,MAAM;oBACR;wBACE,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,2BAA2B,GAAG,CAAC,YAAY,+BAA+B,CAAC,CAAC;gBACjG,CAAC;gBAED,sGAAsG;gBACtG,IAAI,cAAc,IAAI,kBAAkB,EAAE,CAAC;oBACzC,kBAAkB,GAAG,cAAc,CAAC;oBACpC,IAAI,GAAG,CAAC,YAAY,KAAK,SAAS,CAAC,aAAa,EAAE,CAAC;wBACjD,gBAAgB,EAAE,CAAC;oBACrB,CAAC;gBACH,CAAC;YACH,CAAC;YAED,OAAO,gBAAgB,CAAC;QAC1B,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,wDAAwD;YACxD,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,qBAAqB,CAAC,EAAE,CAAC;gBAClD,MAAM,IAAI,kBAAkB,CAC1B,SAAS,CAAC,WAAW,EACrB,8BAA8B,KAAK,CAAC,eAAe,EAAE,0BAA0B,EAC/E,KAAK,CACN,CAAC;YACJ,CAAC;YACD,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;IAEO,iBAAiB;QACvB,OAAO,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,mBAAmB,IAAI,IAAI,CAAC,qBAAqB,CAAC;IAC7E,CAAC;IAED;;;OAGG;IACK,KAAK,CAAC,qBAAqB;QACjC,MAAM,aAAa,GAAmB,EAAE,CAAC;QAEzC,MAAM,SAAS,GAAG,IAAI,CAAC,iBAAiB,EAAE,CAAC;QAC3C,KAAK,MAAM,KAAK,IAAI,SAAS,EAAE,CAAC;YAC9B,IAAI,CAAC,MAAM,CAAC,IAAI,CACd,aAAa,oBAAoB,CAAC,KAAK,CAAC,WAAW,CAAC,MAAM,EAAE,KAAK,CAAC,WAAW,CAAC,IAAI,CAAC,sEAAsE,CAC1J,CAAC;YACF,aAAa,CAAC,IAAI,CAAC;gBACjB,IAAI,EAAE,gBAAgB,CAAC,YAAY;gBACnC,QAAQ,EAAE;oBACR,IAAI,EAAE,KAAK,CAAC,WAAW,CAAC,IAAI;oBAC5B,MAAM,EAAE,KAAK,CAAC,WAAW,CAAC,MAAM;oBAChC,QAAQ,EAAE,KAAK,CAAC,WAAW,CAAC,QAAQ;iBACrC;gBACD,kBAAkB,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC;aACvC,CAAC,CAAC;QACL,CAAC;QAED,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,gBAAgB,EAAE,CAAC;YAC1C,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC,KAAK,CAAC,QAAQ,EAAE,IAAI,CAAC,iBAAiB,CAAC,CAAC;YACzE,IAAI,CAAC,MAAM,EAAE,CAAC;gBACZ,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,KAAK,CAAC,eAAe,EAAE,8CAA8C,CAAC,CAAC;gBACjG,aAAa,CAAC,IAAI,CAAC;oBACjB,IAAI,EAAE,gBAAgB,CAAC,UAAU;oBACjC,KAAK;iBACN,CAAC,CAAC;gBACH,SAAS;YACX,CAAC;YAED,MAAM,sBAAsB,GAAG,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;YACzE,IAAI,CAAC,sBAAsB,EAAE,CAAC;gBAC5B,IAAI,KAAK,CAAC,aAAa,EAAE,EAAE,CAAC;oBAC1B,mDAAmD;oBACnD,aAAa,CAAC,IAAI,CAAC;wBACjB,IAAI,EAAE,gBAAgB,CAAC,wBAAwB;wBAC/C,KAAK;qBACN,CAAC,CAAC;gBACL,CAAC;gBACD,SAAS;YACX,CAAC;YAED,MAAM,qBAAqB,GAAG,sBAAsB,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC;YAClE,oHAAoH;YACpH,IAAI,CAAC,KAAK,CAAC,aAAa,EAAE,IAAI,KAAK,CAAC,eAAgB,CAAC,QAAQ,KAAK,qBAAqB,CAAC,QAAQ,EAAE,CAAC;gBACjG,aAAa,CAAC,IAAI,CAAC;oBACjB,IAAI,EAAE,gBAAgB,CAAC,oBAAoB;oBAC3C,KAAK;oBACL,kBAAkB,EAAE,qBAAqB;iBAC1C,CAAC,CAAC;gBACH,SAAS;YACX,CAAC;YAED,gDAAgD;YAChD,IAAI,KAAK,CAAC,WAAW,CAAC,IAAI,KAAK,sBAAsB,CAAC,WAAW,CAAC,IAAI,EAAE,CAAC;gBACvE,MAAM,QAAQ,GAAG,IAAI,CAAC,qBAAqB,CACzC,sBAAsB,CAAC,WAAW,CAAC,MAAM,EACzC,sBAAsB,CAAC,WAAW,CAAC,IAAI,CACxC;oBACC,CAAC,CAAC;wBACE,IAAI,EAAE,sBAAsB,CAAC,WAAW,CAAC,IAAI;wBAC7C,MAAM,EAAE,sBAAsB,CAAC,WAAW,CAAC,MAAM;wBACjD,QAAQ,EAAE,sBAAsB,CAAC,WAAW,CAAC,QAAQ;qBACtD;oBACH,CAAC,CAAC,SAAS,CAAC;gBAEd,aAAa,CAAC,IAAI,CAAC;oBACjB,IAAI,EAAE,gBAAgB,CAAC,YAAY;oBACnC,KAAK;oBACL,QAAQ;oBACR,kBAAkB,EAAE,qBAAqB;iBAC1C,CAAC,CAAC;gBACH,SAAS;YACX,CAAC;YAED,IAAI,qBAAqB,CAAC,oBAAoB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC1D,aAAa,CAAC,IAAI,CAAC;oBACjB,IAAI,EAAE,gBAAgB,CAAC,oBAAoB;oBAC3C,KAAK;oBACL,kBAAkB,EAAE,qBAAqB;iBAC1C,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAED,OAAO,aAAa,CAAC;IACvB,CAAC;IAEO,iBAAiB;QACvB,MAAM,SAAS,GAA6B,EAAE,CAAC;QAC/C,KAAK,MAAM,CAAC,QAAQ,EAAE,sBAAsB,CAAC,IAAI,IAAI,CAAC,gBAAgB,CAAC,OAAO,EAAE,EAAE,CAAC;YACjF,6IAA6I;YAC7I,IAAI,CAAC,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,QAAQ,KAAK,QAAQ,CAAC,EAAE,CAAC;gBACxE,sEAAsE;gBACtE,IACE,IAAI,CAAC,qBAAqB,CAAC,sBAAsB,CAAC,WAAW,CAAC,MAAM,EAAE,sBAAsB,CAAC,WAAW,CAAC,IAAI,CAAC,EAC9G,CAAC;oBACD,SAAS,CAAC,IAAI,CAAC,sBAAsB,CAAC,CAAC;gBACzC,CAAC;YACH,CAAC;QACH,CAAC;QAED,OAAO,SAAS,CAAC;IACnB,CAAC;IAEO,qBAAqB,CAAC,MAAc,EAAE,SAAiB;QAC7D,OAAO,IAAI,CAAC,OAAO,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,YAAY,EAAE,EAAE,CACrD,YAAY,CAAC,OAAO,CAAC;YACnB,aAAa,EAAE,IAAI,CAAC,iBAAiB,CAAC,aAAa;YACnD,MAAM,EAAE,MAAM;YACd,IAAI,EAAE,SAAS;SAChB,CAAC,CACH,CAAC;IACJ,CAAC;CACF"}
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { replication } from '@powersync/service-core';
|
|
2
2
|
import { container, logger as defaultLogger } from '@powersync/lib-services-framework';
|
|
3
3
|
import { CDCDataExpiredError, CDCStream } from './CDCStream.js';
|
|
4
|
+
import { POWERSYNC_CHECKPOINTS_TABLE } from '../utils/mssql.js';
|
|
4
5
|
export class CDCReplicationJob extends replication.AbstractReplicationJob {
|
|
5
6
|
connectionFactory;
|
|
6
7
|
lastStream = null;
|
|
@@ -12,7 +13,14 @@ export class CDCReplicationJob extends replication.AbstractReplicationJob {
|
|
|
12
13
|
this.cdcReplicationJobOptions = options;
|
|
13
14
|
}
|
|
14
15
|
async keepAlive() {
|
|
15
|
-
|
|
16
|
+
if (this.lastStream) {
|
|
17
|
+
try {
|
|
18
|
+
await this.lastStream.keepAlive();
|
|
19
|
+
}
|
|
20
|
+
catch (e) {
|
|
21
|
+
this.logger.warn(`KeepAlive failed, unable to write an update to the ${POWERSYNC_CHECKPOINTS_TABLE} table`, e);
|
|
22
|
+
}
|
|
23
|
+
}
|
|
16
24
|
}
|
|
17
25
|
async replicate() {
|
|
18
26
|
try {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"CDCReplicationJob.js","sourceRoot":"","sources":["../../src/replication/CDCReplicationJob.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,yBAAyB,CAAC;AAEtD,OAAO,EAAE,SAAS,EAAE,MAAM,IAAI,aAAa,EAAE,MAAM,mCAAmC,CAAC;AACvF,OAAO,EAAE,mBAAmB,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;
|
|
1
|
+
{"version":3,"file":"CDCReplicationJob.js","sourceRoot":"","sources":["../../src/replication/CDCReplicationJob.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,yBAAyB,CAAC;AAEtD,OAAO,EAAE,SAAS,EAAE,MAAM,IAAI,aAAa,EAAE,MAAM,mCAAmC,CAAC;AACvF,OAAO,EAAE,mBAAmB,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAEhE,OAAO,EAAE,2BAA2B,EAAE,MAAM,mBAAmB,CAAC;AAOhE,MAAM,OAAO,iBAAkB,SAAQ,WAAW,CAAC,sBAAsB;IAC/D,iBAAiB,CAAgC;IACjD,UAAU,GAAqB,IAAI,CAAC;IACpC,wBAAwB,CAA2B;IAE3D,YAAY,OAAiC;QAC3C,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,MAAM,GAAG,aAAa,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,cAAc,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC;QAC/F,IAAI,CAAC,iBAAiB,GAAG,OAAO,CAAC,iBAAiB,CAAC;QACnD,IAAI,CAAC,wBAAwB,GAAG,OAAO,CAAC;IAC1C,CAAC;IAED,KAAK,CAAC,SAAS;QACb,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YACpB,IAAI,CAAC;gBACH,MAAM,IAAI,CAAC,UAAU,CAAC,SAAS,EAAE,CAAC;YACpC,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACX,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,sDAAsD,2BAA2B,QAAQ,EAAE,CAAC,CAAC,CAAC;YACjH,CAAC;QACH,CAAC;IACH,CAAC;IAED,KAAK,CAAC,SAAS;QACb,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,aAAa,EAAE,CAAC;QAC7B,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,kBAAkB;YAClB,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC;gBACpB,wBAAwB;gBACxB,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,mBAAmB,EAAE,CAAC,CAAC,CAAC;gBAC1C,IAAI,CAAC,CAAC,KAAK,IAAI,IAAI,EAAE,CAAC;oBACpB,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC;gBACtC,CAAC;gBAED,SAAS,CAAC,QAAQ,CAAC,gBAAgB,CAAC,CAAC,EAAE;oBACrC,QAAQ,EAAE,EAAE;iBACb,CAAC,CAAC;gBAEH,4BAA4B;gBAC5B,IAAI,CAAC,WAAW,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC;YAClC,CAAC;YACD,IAAI,CAAC,YAAY,mBAAmB,EAAE,CAAC;gBACrC,0DAA0D;gBAC1D,MAAM,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,kBAAkB,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;YAC/E,CAAC;QACH,CAAC;gBAAS,CAAC;YACT,IAAI,CAAC,eAAe,CAAC,KAAK,EAAE,CAAC;QAC/B,CAAC;IACH,CAAC;IAED,KAAK,CAAC,aAAa;QACjB,+DAA+D;QAC/D,gEAAgE;QAChE,uCAAuC;QACvC,MAAM,iBAAiB,GAAG,IAAI,CAAC,iBAAiB,CAAC,MAAM,CAAC;YACtD,iBAAiB,EAAE,MAAM;YACzB,GAAG,EAAE,CAAC;SACP,CAAC,CAAC;QACH,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,WAAW,EAAE,gBAAgB,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,eAAe,CAAC,MAAM,EAAE,CAAC,CAAC;YAClF,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;gBACnB,OAAO;YACT,CAAC;YACD,MAAM,MAAM,GAAG,IAAI,SAAS,CAAC;gBAC3B,MAAM,EAAE,IAAI,CAAC,MAAM;gBACnB,WAAW,EAAE,IAAI,CAAC,eAAe,CAAC,MAAM;gBACxC,OAAO,EAAE,IAAI,CAAC,OAAO,CAAC,OAAO;gBAC7B,OAAO,EAAE,IAAI,CAAC,OAAO,CAAC,OAAO;gBAC7B,WAAW,EAAE,iBAAiB;gBAC9B,gBAAgB,EAAE,IAAI,CAAC,wBAAwB,CAAC,gBAAgB;aACjE,CAAC,CAAC;YACH,IAAI,CAAC,UAAU,GAAG,MAAM,CAAC;YACzB,MAAM,MAAM,CAAC,SAAS,EAAE,CAAC;QAC3B,CAAC;gBAAS,CAAC;YACT,MAAM,iBAAiB,CAAC,GAAG,EAAE,CAAC;QAChC,CAAC;IACH,CAAC;IAED,KAAK,CAAC,uBAAuB;QAC3B,OAAO,IAAI,CAAC,UAAU,EAAE,uBAAuB,EAAE,CAAC;IACpD,CAAC;CACF"}
|
|
@@ -4,7 +4,10 @@ import { TablePattern } from '@powersync/service-sync-rules';
|
|
|
4
4
|
import { MSSQLConnectionManager } from './MSSQLConnectionManager.js';
|
|
5
5
|
import sql from 'mssql';
|
|
6
6
|
import { MSSQLSourceTable } from '../common/MSSQLSourceTable.js';
|
|
7
|
+
import { MSSQLSourceTableCache } from '../common/MSSQLSourceTableCache.js';
|
|
8
|
+
import { SchemaChange } from './CDCPoller.js';
|
|
7
9
|
import { AdditionalConfig } from '../types/types.js';
|
|
10
|
+
import { CaptureInstance } from '../common/CaptureInstance.js';
|
|
8
11
|
export interface CDCStreamOptions {
|
|
9
12
|
connections: MSSQLConnectionManager;
|
|
10
13
|
storage: storage.SyncRulesBucketStorage;
|
|
@@ -18,15 +21,27 @@ export interface CDCStreamOptions {
|
|
|
18
21
|
*/
|
|
19
22
|
snapshotBatchSize?: number;
|
|
20
23
|
additionalConfig: AdditionalConfig;
|
|
24
|
+
/**
|
|
25
|
+
* Override schema check interval, defaults is 60 seconds.
|
|
26
|
+
*/
|
|
27
|
+
schemaCheckIntervalMs?: number;
|
|
21
28
|
}
|
|
22
29
|
export declare enum SnapshotStatus {
|
|
23
|
-
|
|
30
|
+
INITIAL = "initial",
|
|
31
|
+
RESUME = "resume",
|
|
24
32
|
DONE = "done",
|
|
25
|
-
|
|
33
|
+
LIMITED_RESNAPSHOT = "limited-resnapshot"
|
|
26
34
|
}
|
|
27
35
|
export interface SnapshotStatusResult {
|
|
28
36
|
status: SnapshotStatus;
|
|
29
37
|
snapshotLSN: string | null;
|
|
38
|
+
/**
|
|
39
|
+
* Under certain circumstances it may be necessary to re-snapshot specific tables:
|
|
40
|
+
* * A new capture instance has been created for a table.
|
|
41
|
+
* * A table has been renamed.
|
|
42
|
+
* * The retention threshold has been exceeded for a table.
|
|
43
|
+
*/
|
|
44
|
+
specificTablesToResnapshot?: MSSQLSourceTable[];
|
|
30
45
|
}
|
|
31
46
|
export declare class CDCConfigurationError extends Error {
|
|
32
47
|
constructor(message: string);
|
|
@@ -48,7 +63,7 @@ export declare class CDCStream {
|
|
|
48
63
|
private readonly connections;
|
|
49
64
|
private readonly abortSignal;
|
|
50
65
|
private readonly logger;
|
|
51
|
-
|
|
66
|
+
tableCache: MSSQLSourceTableCache;
|
|
52
67
|
/**
|
|
53
68
|
* Time of the oldest uncommitted change, according to the source db.
|
|
54
69
|
* This is used to determine the replication lag.
|
|
@@ -70,7 +85,7 @@ export declare class CDCStream {
|
|
|
70
85
|
replicate(): Promise<void>;
|
|
71
86
|
populateTableCache(): Promise<void>;
|
|
72
87
|
getQualifiedTableNames(batch: storage.BucketStorageBatch, tablePattern: TablePattern): Promise<MSSQLSourceTable[]>;
|
|
73
|
-
processTable(batch: storage.BucketStorageBatch, table: SourceEntityDescriptor, snapshot: boolean): Promise<MSSQLSourceTable>;
|
|
88
|
+
processTable(batch: storage.BucketStorageBatch, table: SourceEntityDescriptor, captureInstance: CaptureInstance | null, snapshot: boolean): Promise<MSSQLSourceTable>;
|
|
74
89
|
private snapshotTableInTx;
|
|
75
90
|
private snapshotTable;
|
|
76
91
|
/**
|
|
@@ -95,6 +110,18 @@ export declare class CDCStream {
|
|
|
95
110
|
private checkSnapshotStatus;
|
|
96
111
|
streamChanges(): Promise<void>;
|
|
97
112
|
private createEventHandler;
|
|
113
|
+
handleSchemaChange(batch: storage.BucketStorageBatch, change: SchemaChange): Promise<void>;
|
|
114
|
+
private handleCreateOrUpdateTable;
|
|
115
|
+
/**
|
|
116
|
+
* There is very little that can be automatically done to handle the column changes other than to warn the user about the schema drift.
|
|
117
|
+
*
|
|
118
|
+
* Due to the way CDC works, users are prevented from making column schema changes that affect the replication identities of a table.
|
|
119
|
+
* If changes like that are required, CDC has to be disabled and re-enabled for the table. This would then be handled by the detection of the new
|
|
120
|
+
* capture instance.
|
|
121
|
+
* @param table
|
|
122
|
+
* @param captureInstance
|
|
123
|
+
*/
|
|
124
|
+
private handleColumnChanges;
|
|
98
125
|
/**
|
|
99
126
|
* Convert CDC row data to SqliteRow format.
|
|
100
127
|
* CDC rows include table columns plus CDC metadata columns (__$operation, __$start_lsn, etc.).
|
|
@@ -103,4 +130,8 @@ export declare class CDCStream {
|
|
|
103
130
|
private toSqliteRow;
|
|
104
131
|
getReplicationLagMillis(): Promise<number | undefined>;
|
|
105
132
|
private touch;
|
|
133
|
+
/**
|
|
134
|
+
* Creates an update in the source database to ensure regular checkpoints via the CDC
|
|
135
|
+
*/
|
|
136
|
+
keepAlive(): Promise<void>;
|
|
106
137
|
}
|
|
@@ -1,20 +1,21 @@
|
|
|
1
|
-
import { container, DatabaseConnectionError, ErrorCode, logger as defaultLogger, ReplicationAbortedError, ReplicationAssertionError
|
|
1
|
+
import { container, DatabaseConnectionError, ErrorCode, logger as defaultLogger, ReplicationAbortedError, ReplicationAssertionError } from '@powersync/lib-services-framework';
|
|
2
2
|
import { getUuidReplicaIdentityBson, storage } from '@powersync/service-core';
|
|
3
3
|
import { ReplicationMetric } from '@powersync/service-types';
|
|
4
4
|
import { BatchedSnapshotQuery, SimpleSnapshotQuery } from './MSSQLSnapshotQuery.js';
|
|
5
5
|
import { getReplicationIdentityColumns, getTablesFromPattern } from '../utils/schema.js';
|
|
6
|
-
import { checkSourceConfiguration, createCheckpoint,
|
|
6
|
+
import { checkRetentionThresholds, checkSourceConfiguration, createCheckpoint, getCaptureInstances, getLatestLSN, getLatestReplicatedLSN, isIColumnMetadata } from '../utils/mssql.js';
|
|
7
7
|
import sql from 'mssql';
|
|
8
8
|
import { CDCToSqliteRow, toSqliteInputRow } from '../common/mssqls-to-sqlite.js';
|
|
9
9
|
import { LSN } from '../common/LSN.js';
|
|
10
10
|
import { MSSQLSourceTable } from '../common/MSSQLSourceTable.js';
|
|
11
11
|
import { MSSQLSourceTableCache } from '../common/MSSQLSourceTableCache.js';
|
|
12
|
-
import { CDCPoller } from './CDCPoller.js';
|
|
12
|
+
import { CDCPoller, SchemaChangeType } from './CDCPoller.js';
|
|
13
13
|
export var SnapshotStatus;
|
|
14
14
|
(function (SnapshotStatus) {
|
|
15
|
-
SnapshotStatus["
|
|
15
|
+
SnapshotStatus["INITIAL"] = "initial";
|
|
16
|
+
SnapshotStatus["RESUME"] = "resume";
|
|
16
17
|
SnapshotStatus["DONE"] = "done";
|
|
17
|
-
SnapshotStatus["
|
|
18
|
+
SnapshotStatus["LIMITED_RESNAPSHOT"] = "limited-resnapshot";
|
|
18
19
|
})(SnapshotStatus || (SnapshotStatus = {}));
|
|
19
20
|
export class CDCConfigurationError extends Error {
|
|
20
21
|
constructor(message) {
|
|
@@ -108,13 +109,10 @@ export class CDCStream {
|
|
|
108
109
|
logger: this.logger,
|
|
109
110
|
zeroLSN: LSN.ZERO,
|
|
110
111
|
defaultSchema: this.defaultSchema,
|
|
111
|
-
storeCurrentData:
|
|
112
|
+
storeCurrentData: false
|
|
112
113
|
}, async (batch) => {
|
|
113
114
|
for (let tablePattern of sourceTables) {
|
|
114
|
-
|
|
115
|
-
for (const table of tables) {
|
|
116
|
-
this.tableCache.set(table);
|
|
117
|
-
}
|
|
115
|
+
await this.getQualifiedTableNames(batch, tablePattern);
|
|
118
116
|
}
|
|
119
117
|
});
|
|
120
118
|
}
|
|
@@ -122,18 +120,11 @@ export class CDCStream {
|
|
|
122
120
|
if (tablePattern.connectionTag != this.connections.connectionTag) {
|
|
123
121
|
return [];
|
|
124
122
|
}
|
|
123
|
+
const captureInstances = await getCaptureInstances({ connectionManager: this.connections });
|
|
125
124
|
const matchedTables = await getTablesFromPattern(this.connections, tablePattern);
|
|
126
125
|
const tables = [];
|
|
127
126
|
for (const matchedTable of matchedTables) {
|
|
128
|
-
const
|
|
129
|
-
connectionManager: this.connections,
|
|
130
|
-
table: matchedTable.name,
|
|
131
|
-
schema: matchedTable.schema
|
|
132
|
-
});
|
|
133
|
-
if (!isEnabled) {
|
|
134
|
-
this.logger.info(`Skipping ${matchedTable.schema}.${matchedTable.name} - table is not enabled for CDC.`);
|
|
135
|
-
continue;
|
|
136
|
-
}
|
|
127
|
+
const captureInstanceDetails = captureInstances.get(matchedTable.objectId);
|
|
137
128
|
// TODO: Check RLS settings for table
|
|
138
129
|
const replicaIdColumns = await getReplicationIdentityColumns({
|
|
139
130
|
connectionManager: this.connections,
|
|
@@ -145,12 +136,12 @@ export class CDCStream {
|
|
|
145
136
|
schema: matchedTable.schema,
|
|
146
137
|
objectId: matchedTable.objectId,
|
|
147
138
|
replicaIdColumns: replicaIdColumns.columns
|
|
148
|
-
}, false);
|
|
139
|
+
}, captureInstanceDetails?.instances[0] ?? null, false);
|
|
149
140
|
tables.push(table);
|
|
150
141
|
}
|
|
151
142
|
return tables;
|
|
152
143
|
}
|
|
153
|
-
async processTable(batch, table, snapshot) {
|
|
144
|
+
async processTable(batch, table, captureInstance, snapshot) {
|
|
154
145
|
if (!table.objectId && typeof table.objectId != 'number') {
|
|
155
146
|
throw new ReplicationAssertionError(`objectId expected, got ${typeof table.objectId}`);
|
|
156
147
|
}
|
|
@@ -161,35 +152,25 @@ export class CDCStream {
|
|
|
161
152
|
entity_descriptor: table,
|
|
162
153
|
sync_rules: this.syncRules
|
|
163
154
|
});
|
|
164
|
-
const
|
|
165
|
-
connectionManager: this.connections,
|
|
166
|
-
tableName: resolved.table.name,
|
|
167
|
-
schema: resolved.table.schema
|
|
168
|
-
});
|
|
155
|
+
const resolvedTable = new MSSQLSourceTable(resolved.table);
|
|
169
156
|
if (!captureInstance) {
|
|
170
|
-
|
|
157
|
+
this.logger.warn(`Missing capture instance for table ${resolvedTable.toQualifiedName()}. This table will not be replicated until CDC is enabled for it.`);
|
|
171
158
|
}
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
159
|
+
else {
|
|
160
|
+
resolvedTable.setCaptureInstance(captureInstance);
|
|
161
|
+
}
|
|
162
|
+
this.tableCache.set(resolvedTable);
|
|
176
163
|
// Drop conflicting tables. This includes for example renamed tables.
|
|
177
164
|
await batch.drop(resolved.dropTables);
|
|
178
165
|
// Snapshot if:
|
|
179
|
-
// 1.
|
|
180
|
-
// 2.
|
|
181
|
-
|
|
182
|
-
const shouldSnapshot = snapshot && !resolved.table.snapshotComplete && resolved.table.syncAny;
|
|
166
|
+
// 1. The table is in the sync rules and snapshot is requested, or not already done.
|
|
167
|
+
// 2. AND the table is enabled for CDC with a valid capture instance.
|
|
168
|
+
const shouldSnapshot = snapshot && !resolved.table.snapshotComplete && resolved.table.syncAny && resolvedTable.enabledForCDC();
|
|
183
169
|
if (shouldSnapshot) {
|
|
184
170
|
// Truncate this table in case a previous snapshot was interrupted.
|
|
185
171
|
await batch.truncate([resolved.table]);
|
|
186
172
|
// Start the snapshot inside a transaction.
|
|
187
|
-
|
|
188
|
-
await this.snapshotTableInTx(batch, resolvedTable);
|
|
189
|
-
}
|
|
190
|
-
finally {
|
|
191
|
-
// TODO Cleanup?
|
|
192
|
-
}
|
|
173
|
+
await this.snapshotTableInTx(batch, resolvedTable);
|
|
193
174
|
}
|
|
194
175
|
return resolvedTable;
|
|
195
176
|
}
|
|
@@ -216,7 +197,8 @@ export class CDCStream {
|
|
|
216
197
|
const postSnapshotLSN = await getLatestLSN(this.connections);
|
|
217
198
|
// Side note: A ROLLBACK would probably also be fine here, since we only read in this transaction.
|
|
218
199
|
await transaction.commit();
|
|
219
|
-
const [updatedSourceTable] = await batch.
|
|
200
|
+
const [updatedSourceTable] = await batch.markTableSnapshotDone([table.sourceTable], postSnapshotLSN.toString());
|
|
201
|
+
this.logger.info(`Snapshot of ${table.toQualifiedName()} completed. Post-snapshot LSN: ${postSnapshotLSN.toString()}`);
|
|
220
202
|
this.tableCache.updateSourceTable(updatedSourceTable);
|
|
221
203
|
}
|
|
222
204
|
catch (e) {
|
|
@@ -237,15 +219,15 @@ export class CDCStream {
|
|
|
237
219
|
const orderByKey = table.sourceTable.replicaIdColumns[0];
|
|
238
220
|
query = new BatchedSnapshotQuery(transaction, table, this.snapshotBatchSize, table.sourceTable.snapshotStatus?.lastKey ?? null);
|
|
239
221
|
if (table.sourceTable.snapshotStatus?.lastKey != null) {
|
|
240
|
-
this.logger.info(`
|
|
222
|
+
this.logger.info(`Snapshotting ${table.toQualifiedName()} ${table.sourceTable.formatSnapshotProgress()} - resuming from ${orderByKey.name} > ${query.lastKey}`);
|
|
241
223
|
}
|
|
242
224
|
else {
|
|
243
|
-
this.logger.info(`
|
|
225
|
+
this.logger.info(`Snapshotting ${table.toQualifiedName()} ${table.sourceTable.formatSnapshotProgress()} - resumable`);
|
|
244
226
|
}
|
|
245
227
|
}
|
|
246
228
|
else {
|
|
247
229
|
// Fallback case - query the entire table
|
|
248
|
-
this.logger.info(`
|
|
230
|
+
this.logger.info(`Snapshotting ${table.toQualifiedName()} ${table.sourceTable.formatSnapshotProgress()} - not resumable`);
|
|
249
231
|
query = new SimpleSnapshotQuery(transaction, table);
|
|
250
232
|
replicatedCount = 0;
|
|
251
233
|
}
|
|
@@ -305,7 +287,6 @@ export class CDCStream {
|
|
|
305
287
|
totalEstimatedCount: totalEstimatedCount
|
|
306
288
|
});
|
|
307
289
|
this.tableCache.updateSourceTable(updatedSourceTable);
|
|
308
|
-
this.logger.info(`Replicating ${table.toQualifiedName()} ${table.sourceTable.formatSnapshotProgress()}`);
|
|
309
290
|
if (this.abortSignal.aborted) {
|
|
310
291
|
// We only abort after flushing
|
|
311
292
|
throw new ReplicationAbortedError(`Initial replication interrupted`);
|
|
@@ -314,6 +295,9 @@ export class CDCStream {
|
|
|
314
295
|
if (batchReplicatedCount < this.snapshotBatchSize) {
|
|
315
296
|
hasRemainingData = false;
|
|
316
297
|
}
|
|
298
|
+
else {
|
|
299
|
+
this.logger.info(`Snapshotting ${table.toQualifiedName()} ${table.sourceTable.formatSnapshotProgress()}`);
|
|
300
|
+
}
|
|
317
301
|
}
|
|
318
302
|
}
|
|
319
303
|
/**
|
|
@@ -337,12 +321,7 @@ export class CDCStream {
|
|
|
337
321
|
* and starts again from scratch.
|
|
338
322
|
*/
|
|
339
323
|
async startInitialReplication(snapshotStatus) {
|
|
340
|
-
let { status, snapshotLSN } = snapshotStatus;
|
|
341
|
-
if (status === SnapshotStatus.RESTART_REQUIRED) {
|
|
342
|
-
this.logger.info(`Snapshot restart required, clearing state.`);
|
|
343
|
-
// This happens if the last replicated checkpoint LSN is no longer available in the CDC tables.
|
|
344
|
-
await this.storage.clear({ signal: this.abortSignal });
|
|
345
|
-
}
|
|
324
|
+
let { status, snapshotLSN, specificTablesToResnapshot } = snapshotStatus;
|
|
346
325
|
await this.storage.startBatch({
|
|
347
326
|
logger: this.logger,
|
|
348
327
|
zeroLSN: LSN.ZERO,
|
|
@@ -350,15 +329,26 @@ export class CDCStream {
|
|
|
350
329
|
storeCurrentData: false,
|
|
351
330
|
skipExistingRows: true
|
|
352
331
|
}, async (batch) => {
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
332
|
+
switch (status) {
|
|
333
|
+
case SnapshotStatus.INITIAL:
|
|
334
|
+
// First replication attempt - set the snapshot LSN to the current LSN before starting
|
|
335
|
+
snapshotLSN = (await getLatestReplicatedLSN(this.connections)).toString();
|
|
336
|
+
await batch.setResumeLsn(snapshotLSN);
|
|
337
|
+
const latestLSN = (await getLatestLSN(this.connections)).toString();
|
|
338
|
+
this.logger.info(`Marking snapshot at ${snapshotLSN}, Latest DB LSN ${latestLSN}.`);
|
|
339
|
+
break;
|
|
340
|
+
case SnapshotStatus.RESUME:
|
|
341
|
+
this.logger.info(`Resuming snapshot at ${snapshotLSN}.`);
|
|
342
|
+
break;
|
|
343
|
+
case SnapshotStatus.LIMITED_RESNAPSHOT:
|
|
344
|
+
for (const table of specificTablesToResnapshot) {
|
|
345
|
+
await batch.drop([table.sourceTable]);
|
|
346
|
+
// Update table in the table cache
|
|
347
|
+
await this.processTable(batch, table.sourceTable, table.captureInstance, false);
|
|
348
|
+
}
|
|
349
|
+
break;
|
|
350
|
+
default:
|
|
351
|
+
throw new ReplicationAssertionError(`Unsupported snapshot status: ${status}`);
|
|
362
352
|
}
|
|
363
353
|
const tablesToSnapshot = [];
|
|
364
354
|
for (const table of this.tableCache.getAll()) {
|
|
@@ -366,13 +356,16 @@ export class CDCStream {
|
|
|
366
356
|
this.logger.info(`Skipping table [${table.toQualifiedName()}] - snapshot already done.`);
|
|
367
357
|
continue;
|
|
368
358
|
}
|
|
359
|
+
if (!table.enabledForCDC()) {
|
|
360
|
+
this.logger.info(`Skipping table [${table.toQualifiedName()}] - not enabled for CDC.`);
|
|
361
|
+
continue;
|
|
362
|
+
}
|
|
369
363
|
const count = await this.estimatedCountNumber(table);
|
|
370
364
|
const updatedSourceTable = await batch.updateTableProgress(table.sourceTable, {
|
|
371
365
|
totalEstimatedCount: count
|
|
372
366
|
});
|
|
373
367
|
this.tableCache.updateSourceTable(updatedSourceTable);
|
|
374
368
|
tablesToSnapshot.push(table);
|
|
375
|
-
this.logger.info(`To replicate: ${table.toQualifiedName()} ${table.sourceTable.formatSnapshotProgress()}`);
|
|
376
369
|
}
|
|
377
370
|
for (const table of tablesToSnapshot) {
|
|
378
371
|
await this.snapshotTableInTx(batch, table);
|
|
@@ -380,8 +373,15 @@ export class CDCStream {
|
|
|
380
373
|
}
|
|
381
374
|
// This will not create a consistent checkpoint yet, but will persist the op.
|
|
382
375
|
// Actual checkpoint will be created when streaming replication caught up.
|
|
376
|
+
const postSnapshotLSN = await getLatestLSN(this.connections);
|
|
377
|
+
await batch.markAllSnapshotDone(postSnapshotLSN.toString());
|
|
383
378
|
await batch.commit(snapshotLSN);
|
|
384
|
-
|
|
379
|
+
if (tablesToSnapshot.length > 0) {
|
|
380
|
+
this.logger.info(`All snapshots done. Need to replicate from ${snapshotLSN} to ${postSnapshotLSN} to be consistent.`);
|
|
381
|
+
}
|
|
382
|
+
else {
|
|
383
|
+
this.logger.info(`No tables to snapshot. Need to replicate from ${snapshotLSN}.`);
|
|
384
|
+
}
|
|
385
385
|
});
|
|
386
386
|
}
|
|
387
387
|
async initReplication() {
|
|
@@ -394,6 +394,9 @@ export class CDCStream {
|
|
|
394
394
|
if (snapshotStatus.status !== SnapshotStatus.DONE) {
|
|
395
395
|
await this.startInitialReplication(snapshotStatus);
|
|
396
396
|
}
|
|
397
|
+
else {
|
|
398
|
+
this.logger.info(`Initial replication already done`);
|
|
399
|
+
}
|
|
397
400
|
}
|
|
398
401
|
/**
|
|
399
402
|
* Checks if the initial sync has already been completed and if updates from the last checkpoint are still available
|
|
@@ -402,23 +405,44 @@ export class CDCStream {
|
|
|
402
405
|
async checkSnapshotStatus() {
|
|
403
406
|
const status = await this.storage.getStatus();
|
|
404
407
|
if (status.snapshot_done && status.checkpoint_lsn) {
|
|
408
|
+
const additionalTablesToSnapshot = new Set();
|
|
409
|
+
const newTables = this.tableCache.getAll().filter((table) => !table.sourceTable.snapshotComplete);
|
|
410
|
+
if (newTables.length > 0) {
|
|
411
|
+
this.logger.info(`Detected new table(s) [${newTables.map((table) => table.toQualifiedName()).join(', ')}] that have not been snapshotted yet.`);
|
|
412
|
+
newTables.forEach((table) => additionalTablesToSnapshot.add(table));
|
|
413
|
+
}
|
|
405
414
|
// Snapshot is done, but we still need to check that the last known checkpoint LSN is still
|
|
406
|
-
// within the threshold of the CDC tables
|
|
407
|
-
this.logger.info(`Initial replication already done`);
|
|
415
|
+
// within the retention threshold of the CDC tables
|
|
408
416
|
const lastCheckpointLSN = LSN.fromString(status.checkpoint_lsn);
|
|
409
417
|
// Check that the CDC tables still have valid data
|
|
410
|
-
const
|
|
418
|
+
const tablesOutsideRetentionThreshold = await checkRetentionThresholds({
|
|
411
419
|
checkpointLSN: lastCheckpointLSN,
|
|
412
420
|
tables: this.tableCache.getAll(),
|
|
413
421
|
connectionManager: this.connections
|
|
414
422
|
});
|
|
415
|
-
if (
|
|
416
|
-
this.logger.warn(`Updates from the last checkpoint are no longer available in the CDC
|
|
423
|
+
if (tablesOutsideRetentionThreshold.length > 0) {
|
|
424
|
+
this.logger.warn(`Updates from the last checkpoint are no longer available in the CDC instances of the following table(s): ${tablesOutsideRetentionThreshold.map((table) => table.toQualifiedName()).join(', ')}.`);
|
|
425
|
+
tablesOutsideRetentionThreshold.forEach((table) => additionalTablesToSnapshot.add(table));
|
|
426
|
+
}
|
|
427
|
+
if (additionalTablesToSnapshot.size > 0) {
|
|
428
|
+
return {
|
|
429
|
+
status: SnapshotStatus.LIMITED_RESNAPSHOT,
|
|
430
|
+
snapshotLSN: status.checkpoint_lsn,
|
|
431
|
+
specificTablesToResnapshot: Array.from(additionalTablesToSnapshot)
|
|
432
|
+
};
|
|
433
|
+
}
|
|
434
|
+
else {
|
|
435
|
+
return {
|
|
436
|
+
status: SnapshotStatus.DONE,
|
|
437
|
+
snapshotLSN: null
|
|
438
|
+
};
|
|
417
439
|
}
|
|
418
|
-
return { status: isAvailable ? SnapshotStatus.DONE : SnapshotStatus.RESTART_REQUIRED, snapshotLSN: null };
|
|
419
440
|
}
|
|
420
441
|
else {
|
|
421
|
-
return {
|
|
442
|
+
return {
|
|
443
|
+
status: status.snapshot_lsn != null ? SnapshotStatus.RESUME : SnapshotStatus.INITIAL,
|
|
444
|
+
snapshotLSN: status.snapshot_lsn
|
|
445
|
+
};
|
|
422
446
|
}
|
|
423
447
|
}
|
|
424
448
|
async streamChanges() {
|
|
@@ -433,15 +457,16 @@ export class CDCStream {
|
|
|
433
457
|
throw new ReplicationAssertionError(`No LSN found to resume replication from.`);
|
|
434
458
|
}
|
|
435
459
|
const startLSN = LSN.fromString(batch.resumeFromLsn);
|
|
436
|
-
const sourceTables = this.tableCache.getAll();
|
|
437
460
|
const eventHandler = this.createEventHandler(batch);
|
|
438
461
|
const poller = new CDCPoller({
|
|
439
462
|
connectionManager: this.connections,
|
|
440
463
|
eventHandler,
|
|
441
|
-
|
|
464
|
+
getReplicatedTables: () => this.tableCache.getAll(),
|
|
465
|
+
sourceTables: this.syncRules.getSourceTables(),
|
|
442
466
|
startLSN,
|
|
443
467
|
logger: this.logger,
|
|
444
|
-
additionalConfig: this.options.additionalConfig
|
|
468
|
+
additionalConfig: this.options.additionalConfig,
|
|
469
|
+
schemaCheckIntervalMs: this.options.schemaCheckIntervalMs
|
|
445
470
|
});
|
|
446
471
|
this.abortSignal.addEventListener('abort', async () => {
|
|
447
472
|
await poller.stop();
|
|
@@ -491,15 +516,93 @@ export class CDCStream {
|
|
|
491
516
|
this.metrics.getCounter(ReplicationMetric.ROWS_REPLICATED).add(1);
|
|
492
517
|
},
|
|
493
518
|
onCommit: async (lsn, transactionCount) => {
|
|
494
|
-
await batch.commit(lsn);
|
|
519
|
+
const { checkpointBlocked } = await batch.commit(lsn);
|
|
495
520
|
this.metrics.getCounter(ReplicationMetric.TRANSACTIONS_REPLICATED).add(transactionCount);
|
|
496
|
-
|
|
521
|
+
if (!checkpointBlocked) {
|
|
522
|
+
this.isStartingReplication = false;
|
|
523
|
+
}
|
|
497
524
|
},
|
|
498
|
-
onSchemaChange: async () => {
|
|
499
|
-
|
|
525
|
+
onSchemaChange: async (schemaChange) => {
|
|
526
|
+
await this.handleSchemaChange(batch, schemaChange);
|
|
500
527
|
}
|
|
501
528
|
};
|
|
502
529
|
}
|
|
530
|
+
async handleSchemaChange(batch, change) {
|
|
531
|
+
let actionedSchemaChange = true;
|
|
532
|
+
switch (change.type) {
|
|
533
|
+
case SchemaChangeType.TABLE_RENAME:
|
|
534
|
+
const fromTable = change.table;
|
|
535
|
+
this.logger.info(`Table ${fromTable.toQualifiedName()} has been renamed ${change.newTable ? `to [${change.newTable.name}].` : '.'}`);
|
|
536
|
+
// Old table needs to be cleaned up
|
|
537
|
+
await batch.drop([fromTable.sourceTable]);
|
|
538
|
+
this.tableCache.delete(fromTable.objectId);
|
|
539
|
+
if (change.newTable) {
|
|
540
|
+
await this.handleCreateOrUpdateTable(batch, change.newTable, change.newCaptureInstance);
|
|
541
|
+
}
|
|
542
|
+
break;
|
|
543
|
+
case SchemaChangeType.TABLE_CREATE:
|
|
544
|
+
await this.handleCreateOrUpdateTable(batch, change.newTable, change.newCaptureInstance);
|
|
545
|
+
break;
|
|
546
|
+
case SchemaChangeType.TABLE_COLUMN_CHANGES:
|
|
547
|
+
await this.handleColumnChanges(change.table, change.newCaptureInstance);
|
|
548
|
+
actionedSchemaChange = false;
|
|
549
|
+
break;
|
|
550
|
+
case SchemaChangeType.NEW_CAPTURE_INSTANCE:
|
|
551
|
+
this.logger.info(`New CDC capture instance detected for table ${change.table.toQualifiedName()}. Re-snapshotting table...`);
|
|
552
|
+
await batch.drop([change.table.sourceTable]);
|
|
553
|
+
this.tableCache.delete(change.table.objectId);
|
|
554
|
+
await this.handleCreateOrUpdateTable(batch, change.table.sourceTable, change.newCaptureInstance);
|
|
555
|
+
break;
|
|
556
|
+
case SchemaChangeType.TABLE_DROP:
|
|
557
|
+
await batch.drop([change.table.sourceTable]);
|
|
558
|
+
this.tableCache.delete(change.table.objectId);
|
|
559
|
+
break;
|
|
560
|
+
case SchemaChangeType.MISSING_CAPTURE_INSTANCE:
|
|
561
|
+
// Stop replication for this table until CDC is re-enabled.
|
|
562
|
+
this.logger.warn(`Table ${change.table.toQualifiedName()} has been disabled for CDC. Re-enable CDC to continue replication.`);
|
|
563
|
+
change.table.clearCaptureInstance();
|
|
564
|
+
actionedSchemaChange = false;
|
|
565
|
+
break;
|
|
566
|
+
default:
|
|
567
|
+
throw new ReplicationAssertionError(`Unknown schema change type: ${change.type}`);
|
|
568
|
+
}
|
|
569
|
+
// Create a new checkpoint after the schema change
|
|
570
|
+
if (actionedSchemaChange) {
|
|
571
|
+
await createCheckpoint(this.connections);
|
|
572
|
+
}
|
|
573
|
+
}
|
|
574
|
+
async handleCreateOrUpdateTable(batch, table, captureInstance) {
|
|
575
|
+
const replicaIdColumns = await getReplicationIdentityColumns({
|
|
576
|
+
connectionManager: this.connections,
|
|
577
|
+
tableName: table.name,
|
|
578
|
+
schema: table.schema
|
|
579
|
+
});
|
|
580
|
+
await this.processTable(batch, {
|
|
581
|
+
name: table.name,
|
|
582
|
+
schema: table.schema,
|
|
583
|
+
objectId: table.objectId,
|
|
584
|
+
replicaIdColumns: replicaIdColumns.columns
|
|
585
|
+
}, captureInstance, true);
|
|
586
|
+
}
|
|
587
|
+
/**
|
|
588
|
+
* There is very little that can be automatically done to handle the column changes other than to warn the user about the schema drift.
|
|
589
|
+
*
|
|
590
|
+
* Due to the way CDC works, users are prevented from making column schema changes that affect the replication identities of a table.
|
|
591
|
+
* If changes like that are required, CDC has to be disabled and re-enabled for the table. This would then be handled by the detection of the new
|
|
592
|
+
* capture instance.
|
|
593
|
+
* @param table
|
|
594
|
+
* @param captureInstance
|
|
595
|
+
*/
|
|
596
|
+
async handleColumnChanges(table, captureInstance) {
|
|
597
|
+
// Check there are any new pending schema changes
|
|
598
|
+
if (table.captureInstance?.objectId === captureInstance.objectId &&
|
|
599
|
+
table.captureInstance?.pendingSchemaChanges.length === captureInstance.pendingSchemaChanges.length) {
|
|
600
|
+
return;
|
|
601
|
+
}
|
|
602
|
+
// New pending schema changes were detected - warn about those as well.
|
|
603
|
+
this.logger.warn(`Schema drift detected for table ${table.toQualifiedName()}. To ensure consistency, disable and re-enable CDC for this table.\n Pending schema changes:\n ${captureInstance.pendingSchemaChanges.join(', \n')}`);
|
|
604
|
+
table.captureInstance = captureInstance;
|
|
605
|
+
}
|
|
503
606
|
/**
|
|
504
607
|
* Convert CDC row data to SqliteRow format.
|
|
505
608
|
* CDC rows include table columns plus CDC metadata columns (__$operation, __$start_lsn, etc.).
|
|
@@ -527,5 +630,13 @@ export class CDCStream {
|
|
|
527
630
|
this.logger.error(`Error touching probe`, e);
|
|
528
631
|
});
|
|
529
632
|
}
|
|
633
|
+
/**
|
|
634
|
+
* Creates an update in the source database to ensure regular checkpoints via the CDC
|
|
635
|
+
*/
|
|
636
|
+
async keepAlive() {
|
|
637
|
+
if (!this.isStartingReplication && !this.stopped) {
|
|
638
|
+
await createCheckpoint(this.connections);
|
|
639
|
+
}
|
|
640
|
+
}
|
|
530
641
|
}
|
|
531
642
|
//# sourceMappingURL=CDCStream.js.map
|