@powersync/service-module-mongodb-storage 0.19.0 → 0.20.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +19 -0
- package/dist/storage/implementation/models.d.ts +4 -0
- package/dist/storage/implementation/models.js.map +1 -1
- package/dist/storage/implementation/v1/MongoBucketBatchV1.d.ts +4 -0
- package/dist/storage/implementation/v1/MongoBucketBatchV1.js +58 -84
- package/dist/storage/implementation/v1/MongoBucketBatchV1.js.map +1 -1
- package/dist/storage/implementation/v3/MongoBucketBatchV3.js +32 -22
- package/dist/storage/implementation/v3/MongoBucketBatchV3.js.map +1 -1
- package/dist/storage/implementation/v3/models.d.ts +5 -1
- package/dist/storage/implementation/v3/models.js.map +1 -1
- package/dist/storage/implementation/v3/source-table-utils.d.ts +9 -1
- package/dist/storage/implementation/v3/source-table-utils.js +20 -21
- package/dist/storage/implementation/v3/source-table-utils.js.map +1 -1
- package/package.json +2 -2
- package/src/storage/implementation/models.ts +4 -0
- package/src/storage/implementation/v1/MongoBucketBatchV1.ts +76 -96
- package/src/storage/implementation/v3/MongoBucketBatchV3.ts +37 -22
- package/src/storage/implementation/v3/models.ts +5 -0
- package/src/storage/implementation/v3/source-table-utils.ts +43 -35
- package/test/src/storage_sync.test.ts +29 -0
- package/tsconfig.tsbuildinfo +1 -1
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"source-table-utils.js","sourceRoot":"","sources":["../../../../src/storage/implementation/v3/source-table-utils.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,yBAAyB,EAAE,MAAM,mCAAmC,CAAC;AAC9E,OAAO,
|
|
1
|
+
{"version":3,"file":"source-table-utils.js","sourceRoot":"","sources":["../../../../src/storage/implementation/v3/source-table-utils.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,yBAAyB,EAAE,MAAM,mCAAmC,CAAC;AAC9E,OAAO,EAAwD,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAyExG,MAAM,UAAU,4BAA4B,CAC1C,UAA8B,EAC9B,GAAmB,EACnB,OAAgC;IAEhC,MAAM,eAAe,GAAG,UAAU,CAAC,kBAAkB,CAAC,GAAG,CAAC,CAAC;IAC3D,OAAO;QACL,gBAAgB,EAAE,IAAI,GAAG,CACvB,eAAe,CAAC,iBAAiB,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,cAAc,CAAC,MAAM,CAAC,EAAE,MAAM,CAAU,CAAC,CACrG;QACD,yBAAyB,EAAE,IAAI,GAAG,CAChC,eAAe,CAAC,sBAAsB,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,iBAAiB,CAAC,MAAM,CAAC,EAAE,MAAM,CAAU,CAAC,CAC7G;QACD,aAAa,EAAE,UAAU,CAAC,kBAAkB,CAAC,GAAG,CAAC;KAClD,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,6BAA6B,CAC3C,aAAsC,EACtC,OAAyC;IAEzC,OAAO,IAAI,gCAAgC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;AAC3E,CAAC;AAED,SAAS,eAAe,CAAC,IAAc,EAAE,KAAe;IACtD,OAAO,IAAI,CAAC,MAAM,IAAI,KAAK,CAAC,MAAM,IAAI,IAAI,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE,CAAC,KAAK,IAAI,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC;AAC5F,CAAC;AAED,SAAS,kBAAkB,CACzB,OAAqC,EACrC,WAAqC;IAErC,OAAO;QACL,iBAAiB,EAAE,WAAW,CAAC,mBAAmB,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,OAAO,CAAC,gBAAgB,CAAC,GAAG,CAAC,EAAE,CAAE,CAAC;QACjG,sBAAsB,EAAE,WAAW,CAAC,wBAAwB,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,OAAO,CAAC,yBAAyB,CAAC,GAAG,CAAC,EAAE,CAAE,CAAC;KACrH,CAAC;AACJ,CAAC;AAED,SAAS,sBAAsB,CAC7B,GAA0B,EAC1B,OAAqC;IAErC,OAAO;QACL,mBAAmB,EAAE,GAAG,CAAC,sBAAsB,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,OAAO,CAAC,gBAAgB,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAChG,wBAAwB,EAAE,GAAG,CAAC,2BAA2B,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,OAAO,CAAC,yBAAyB,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;KACpH,CAAC;AACJ,CAAC;AAED,SAAS,gBAAgB,CAAC,WAAqC;IAC7D,OAAO,WAAW,CAAC,mBAAmB,CAAC,MAAM,GAAG,CAAC,IAAI,WAAW,CAAC,wBAAwB,CAAC,MAAM,GAAG,CAAC,CAAC;AACvG,CAAC;AAED,SAAS,iBAAiB,CAAC,GAA0B,EAAE,WAAqC;IAC1F,OAAO,CACL,eAAe,CAAC,GAAG,CAAC,sBAAsB,EAAE,WAAW,CAAC,mBAAmB,CAAC;QAC5E,eAAe,CAAC,GAAG,CAAC,2BAA2B,EAAE,WAAW,CAAC,wBAAwB,CAAC,CACvF,CAAC;AACJ,CAAC;AAED,MAAM,gCAAgC;IAQP;IAPZ,0BAA0B,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/C,+BAA+B,GAAG,IAAI,GAAG,EAAU,CAAC;IACpD,cAAc,GAAG,IAAI,GAAG,EAAU,CAAC;IACnC,MAAM,GAA0B,EAAE,CAAC;IACnC,gBAAgB,GAAkC,EAAE,CAAC;IAC9D,sBAAsB,GAAG,KAAK,CAAC;IAEvC,YAA6B,OAAyC;QAAzC,YAAO,GAAP,OAAO,CAAkC;IAAG,CAAC;IAE1E,IAAI,CAAC,aAAsC;QACzC,KAAK,MAAM,GAAG,IAAI,aAAa,EAAE,CAAC;YAChC,IAAI,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,EAAE,CAAC;gBAC3B,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;YACtB,CAAC;QACH,CAAC;QAED,OAAO;YACL,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,gBAAgB,EAAE,IAAI,CAAC,gBAAgB;YACvC,mBAAmB,EAAE,IAAI,CAAC,mBAAmB,EAAE;YAC/C,qFAAqF;YACrF,QAAQ,EAAE,aAAa,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC;SACjE,CAAC;IACJ,CAAC;IAEO,YAAY,CAAC,GAA0B;QAC7C,OAAO,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC;IAC9C,CAAC;IAEO,kBAAkB,CAAC,GAA0B;QACnD,OAAO,IAAI,CAAC,OAAO,CAAC,sBAAsB,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,OAAO,CAAC,mBAAmB,CAAC,KAAK,CAAC,EAAE,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;IAC7G,CAAC;IAEO,SAAS,CAAC,GAA0B;QAC1C,MAAM,WAAW,GAAG,sBAAsB,CAAC,GAAG,EAAE,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QACtE,MAAM,uBAAuB,GAAG,gBAAgB,CAAC,WAAW,CAAC,CAAC;QAC9D,MAAM,oBAAoB,GACxB,CAAC,IAAI,CAAC,oBAAoB,EAAE,IAAI,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,aAAa,IAAI,CAAC,IAAI,CAAC,sBAAsB,CAAC;QAErG,IAAI,CAAC,cAAc,CAAC,GAAG,EAAE,WAAW,CAAC,CAAC;QACtC,IAAI,CAAC,mBAAmB,CAAC,GAAG,EAAE,WAAW,EAAE,uBAAuB,EAAE,oBAAoB,CAAC,CAAC;QAE1F,IAAI,uBAAuB,IAAI,oBAAoB,EAAE,CAAC;YACpD,IAAI,CAAC,sBAAsB,KAAK,oBAAoB,CAAC;YACrD,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,WAAW,EAAE,CAAC,CAAC;YAC/C,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,GAAG,EAAE,WAAW,CAAC,CAAC,CAAC;QAC1D,CAAC;IACH,CAAC;IAEO,oBAAoB;QAC1B,MAAM,EAAE,OAAO,EAAE,GAAG,IAAI,CAAC,OAAO,CAAC;QACjC,OAAO,OAAO,CAAC,gBAAgB,CAAC,IAAI,GAAG,CAAC,IAAI,OAAO,CAAC,yBAAyB,CAAC,IAAI,GAAG,CAAC,CAAC;IACzF,CAAC;IAEO,cAAc,CAAC,GAA0B,EAAE,WAAqC;QACtF,IAAI,CAAC,WAAW,CAAC,GAAG,EAAE,oBAAoB,EAAE,IAAI,CAAC,0BAA0B,EAAE,WAAW,CAAC,mBAAmB,CAAC,CAAC;QAC9G,IAAI,CAAC,WAAW,CACd,GAAG,EACH,yBAAyB,EACzB,IAAI,CAAC,+BAA+B,EACpC,WAAW,CAAC,wBAAwB,CACrC,CAAC;IACJ,CAAC;IAED,mFAAmF;IACnF,kFAAkF;IAClF,mFAAmF;IACnF,8EAA8E;IACtE,WAAW,CAAC,GAA0B,EAAE,IAAY,EAAE,OAAoB,EAAE,GAAa;QAC/F,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC;QAC/C,KAAK,MAAM,EAAE,IAAI,GAAG,EAAE,CAAC;YACrB,IAAI,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC;gBACpB,MAAM,IAAI,yBAAyB,CACjC,gBAAgB,GAAG,CAAC,GAAG,2BAA2B,IAAI,IAAI,EAAE,QAAQ,MAAM,IAAI,IAAI,EAAE,CACrF,CAAC;YACJ,CAAC;YACD,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAClB,CAAC;IACH,CAAC;IAEO,mBAAmB,CACzB,GAA0B,EAC1B,WAAqC,EACrC,uBAAgC,EAChC,oBAA6B;QAE7B,MAAM,YAAY,GAChB,CAAC,uBAAuB,IAAI,oBAAoB,CAAC,IAAI,CAAC,GAAG,CAAC,aAAa,IAAI,CAAC,iBAAiB,CAAC,GAAG,EAAE,WAAW,CAAC,CAAC;QAElH,IAAI,CAAC,YAAY,EAAE,CAAC;YAClB,OAAO;QACT,CAAC;QAED,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC;YACzB,EAAE,EAAE,GAAG,CAAC,GAAG;YACX,WAAW;SACZ,CAAC,CAAC;IACL,CAAC;IAEO,mBAAmB;QACzB,MAAM,EAAE,OAAO,EAAE,GAAG,IAAI,CAAC,OAAO,CAAC;QACjC,MAAM,SAAS,GAA6B;YAC1C,mBAAmB,EAAE,CAAC,GAAG,OAAO,CAAC,gBAAgB,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAC9D,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,0BAA0B,CAAC,GAAG,CAAC,EAAE,CAAC,CACjD;YACD,wBAAwB,EAAE,CAAC,GAAG,OAAO,CAAC,yBAAyB,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAC5E,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,+BAA+B,CAAC,GAAG,CAAC,EAAE,CAAC,CACtD;SACF,CAAC;QACF,IAAI,gBAAgB,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,aAAa,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM,IAAI,CAAC,CAAC,EAAE,CAAC;YACtF,OAAO,SAAS,CAAC;QACnB,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IAEO,cAAc,CAAC,GAA0B,EAAE,WAAqC;QACtF,MAAM,EAAE,aAAa,EAAE,UAAU,EAAE,OAAO,EAAE,OAAO,EAAE,gBAAgB,EAAE,GAAG,IAAI,CAAC,OAAO,CAAC;QACvF,MAAM,KAAK,GAAG,uBAAuB,CACnC,GAAG,EACH,aAAa,EACb,UAAU,EACV,OAAO,EACP,kBAAkB,CAAC,OAAO,EAAE,WAAW,CAAC,EACxC,WAAW,CACZ,CAAC;QACF,sDAAsD;QACtD,MAAM,QAAQ,GAAG,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,CAAC;QAC9C,MAAM,KAAK,GAAG,QAAQ,IAAI,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,kBAAkB,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAC;QAC3F,KAAK,CAAC,gBAAgB,GAAG,gBAAgB,CAAC;QAC1C,OAAO,KAAK,CAAC;IACf,CAAC;CACF;AAED,MAAM,UAAU,4BAA4B,CAC1C,YAAoB,EACpB,QAAmE;IAEnE,MAAM,OAAO,GAAG,CAAC,EAAE,WAAW,EAAE,QAAQ,CAAC,MAAM,EAAE,UAAU,EAAE,QAAQ,CAAC,IAAI,EAAE,CAA8B,CAAC;IAC3G,IAAI,QAAQ,CAAC,QAAQ,IAAI,IAAI,EAAE,CAAC;QAC9B,OAAO,CAAC,IAAI,CAAC,EAAE,WAAW,EAAE,QAAQ,CAAC,QAAQ,EAAE,CAAC,CAAC;IACnD,CAAC;IAED,OAAO;QACL,aAAa,EAAE,YAAY;QAC3B,GAAG,EAAE,OAAO;KACb,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,oBAAoB,CAClC,EAAiB,EACjB,WAAqC,EACrC,OAAyC;IAEzC,MAAM,EACJ,YAAY,EACZ,aAAa,EACb,QAAQ,EACR,UAAU,EACV,OAAO,EACP,OAAO,EACP,gBAAgB,EAChB,sBAAsB,EACvB,GAAG,OAAO,CAAC;IACZ,MAAM,GAAG,GAA0B;QACjC,GAAG,EAAE,EAAE;QACP,aAAa,EAAE,YAAY;QAC3B,WAAW,EAAE,QAAQ,CAAC,QAAQ;QAC9B,WAAW,EAAE,QAAQ,CAAC,MAAM;QAC5B,UAAU,EAAE,QAAQ,CAAC,IAAI;QACzB,kBAAkB,EAAE,QAAQ,CAAC,gBAAgB;QAC7C,aAAa,EAAE,KAAK;QACpB,eAAe,EAAE,SAAS;QAC1B,sBAAsB,EAAE,WAAW,CAAC,mBAAmB;QACvD,2BAA2B,EAAE,WAAW,CAAC,wBAAwB;QACjE,2DAA2D;QAC3D,eAAe,EAAE,sBAAsB;KACxC,CAAC;IACF,MAAM,KAAK,GAAG,uBAAuB,CACnC,GAAG,EACH,aAAa,EACb,UAAU,EACV,OAAO,EACP,kBAAkB,CAAC,OAAO,EAAE,WAAW,CAAC,EACxC,WAAW,CACZ,CAAC;IACF,KAAK,CAAC,gBAAgB,GAAG,gBAAgB,CAAC;IAE1C,OAAO,EAAE,GAAG,EAAE,KAAK,EAAE,CAAC;AACxB,CAAC;AAED,MAAM,UAAU,qBAAqB,CAAC,MAA6B,EAAE,aAAsB;IACzF,IAAI,CAAC,aAAa,EAAE,CAAC;QACnB,OAAO;IACT,CAAC;IAED,MAAM,YAAY,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,gBAAgB,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,CAAC;IACjF,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,KAAK,CAAC,SAAS,GAAG,KAAK,KAAK,YAAY,CAAC;IAC3C,CAAC;AACH,CAAC;AAED,SAAS,kCAAkC,CACzC,GAA0B,EAC1B,UAA8B,EAC9B,OAAgC;IAEhC,MAAM,mBAAmB,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,sBAAsB,CAAC,CAAC;IAChE,MAAM,wBAAwB,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,2BAA2B,CAAC,CAAC;IAE1E,OAAO;QACL,iBAAiB,EAAE,UAAU,CAAC,iBAAiB,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAChE,mBAAmB,CAAC,GAAG,CAAC,OAAO,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC,CACxD;QACD,sBAAsB,EAAE,UAAU,CAAC,4BAA4B,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAChF,wBAAwB,CAAC,GAAG,CAAC,OAAO,CAAC,iBAAiB,CAAC,MAAM,CAAC,CAAC,CAChE;KACF,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,uBAAuB,CACrC,GAA0B,EAC1B,aAAqB,EACrB,UAA8B,EAC9B,OAAgC,EAChC,WAA6B,EAC7B,aAAwC;IAExC,MAAM,mBAAmB,GAAG,WAAW,IAAI,kCAAkC,CAAC,GAAG,EAAE,UAAU,EAAE,OAAO,CAAC,CAAC;IACxG,MAAM,qBAAqB,GAAG,aAAa,IAAI;QAC7C,mBAAmB,EAAE,mBAAmB,CAAC,iBAAiB,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,OAAO,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC;QAC1G,wBAAwB,EAAE,mBAAmB,CAAC,sBAAsB,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAClF,OAAO,CAAC,iBAAiB,CAAC,MAAM,CAAC,CAClC;KACF,CAAC;IACF,MAAM,KAAK,GAAG,IAAI,OAAO,CAAC,WAAW,CAAC;QACpC,EAAE,EAAE,GAAG,CAAC,GAAG;QACX,GAAG,EAAE;YACH,aAAa;YACb,MAAM,EAAE,GAAG,CAAC,WAAW;YACvB,IAAI,EAAE,GAAG,CAAC,UAAU;SACrB;QACD,QAAQ,EAAE,GAAG,CAAC,WAAW;QACzB,gBAAgB,EACd,GAAG,CAAC,kBAAkB,EAAE,GAAG,CACzB,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,CAAC,QAAQ,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,EAAE,CAA4B,CAC3G,IAAI,EAAE;QACT,gBAAgB,EAAE,GAAG,CAAC,aAAa;QACnC,iBAAiB,EAAE,mBAAmB,CAAC,iBAAiB;QACxD,sBAAsB,EAAE,mBAAmB,CAAC,sBAAsB;QAClE,mBAAmB,EAAE,IAAI,GAAG,CAAC,qBAAqB,CAAC,mBAAmB,CAAC;QACvE,wBAAwB,EAAE,IAAI,GAAG,CAAC,qBAAqB,CAAC,wBAAwB,CAAC;QACjF,cAAc,EAAE,GAAG,CAAC,eAAe,IAAI,IAAI;KAC5C,CAAC,CAAC;IACH,KAAK,CAAC,QAAQ,GAAG,KAAK,CAAC,iBAAiB,CAAC,MAAM,GAAG,CAAC,CAAC;IACpD,KAAK,CAAC,cAAc,GAAG,KAAK,CAAC,sBAAsB,CAAC,MAAM,GAAG,CAAC,CAAC;IAC/D,KAAK,CAAC,SAAS,GAAG,UAAU,CAAC,kBAAkB,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAC3D,KAAK,CAAC,cAAc;QAClB,GAAG,CAAC,eAAe,IAAI,IAAI;YACzB,CAAC,CAAC,SAAS;YACX,CAAC,CAAC;gBACE,OAAO,EAAE,GAAG,CAAC,eAAe,CAAC,QAAQ,EAAE,MAAM,IAAI,IAAI;gBACrD,mBAAmB,EAAE,GAAG,CAAC,eAAe,CAAC,qBAAqB;gBAC9D,eAAe,EAAE,GAAG,CAAC,eAAe,CAAC,gBAAgB;aACtD,CAAC;IACR,OAAO,KAAK,CAAC;AACf,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@powersync/service-module-mongodb-storage",
|
|
3
3
|
"repository": "https://github.com/powersync-ja/powersync-service",
|
|
4
|
-
"version": "0.
|
|
4
|
+
"version": "0.20.0",
|
|
5
5
|
"license": "FSL-1.1-ALv2",
|
|
6
6
|
"type": "module",
|
|
7
7
|
"publishConfig": {
|
|
@@ -22,7 +22,7 @@
|
|
|
22
22
|
"uuid": "^14.0.0",
|
|
23
23
|
"@powersync/lib-service-mongodb": "0.6.30",
|
|
24
24
|
"@powersync/lib-services-framework": "0.10.0",
|
|
25
|
-
"@powersync/service-core": "1.
|
|
25
|
+
"@powersync/service-core": "1.25.0",
|
|
26
26
|
"@powersync/service-jsonbig": "0.17.13",
|
|
27
27
|
"@powersync/service-sync-rules": "0.40.0",
|
|
28
28
|
"@powersync/service-types": "0.17.0"
|
|
@@ -110,6 +110,10 @@ export interface SourceTableDocument {
|
|
|
110
110
|
replica_id_columns2: { name: string; type_oid?: number; type?: string }[] | undefined;
|
|
111
111
|
snapshot_done: boolean | undefined;
|
|
112
112
|
snapshot_status: SourceTableDocumentSnapshotStatus | undefined;
|
|
113
|
+
/**
|
|
114
|
+
* Source-specific metadata. Absent for legacy records.
|
|
115
|
+
*/
|
|
116
|
+
source_metadata?: storage.JsonValue;
|
|
113
117
|
}
|
|
114
118
|
|
|
115
119
|
export interface SourceTableDocumentSnapshotStatus {
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { ReplicationAssertionError } from '@powersync/lib-services-framework';
|
|
2
2
|
import { ColumnDescriptor, InternalOpId, SourceTable, storage } from '@powersync/service-core';
|
|
3
|
+
import { HydratedSyncConfig } from '@powersync/service-sync-rules';
|
|
3
4
|
import * as bson from 'bson';
|
|
4
5
|
import { mongoTableId } from '../../../utils/util.js';
|
|
5
6
|
import { calculateCheckpointState } from '../CheckpointState.js';
|
|
@@ -9,7 +10,7 @@ import { SourceRecordStore } from '../common/SourceRecordStore.js';
|
|
|
9
10
|
import { PersistedBatchV1 } from './PersistedBatchV1.js';
|
|
10
11
|
import { SourceRecordStoreV1 } from './SourceRecordStoreV1.js';
|
|
11
12
|
import { VersionedPowerSyncMongoV1 } from './VersionedPowerSyncMongoV1.js';
|
|
12
|
-
import { SyncRuleDocumentV1 } from './models.js';
|
|
13
|
+
import { SourceTableDocumentV1, SyncRuleDocumentV1 } from './models.js';
|
|
13
14
|
|
|
14
15
|
export class MongoBucketBatchV1 extends MongoBucketBatch {
|
|
15
16
|
declare readonly db: VersionedPowerSyncMongoV1;
|
|
@@ -51,6 +52,7 @@ export class MongoBucketBatchV1 extends MongoBucketBatch {
|
|
|
51
52
|
|
|
52
53
|
async resolveTables(options: storage.ResolveTablesOptions): Promise<storage.ResolveTablesResult> {
|
|
53
54
|
const syncRules = options.parsedSyncConfig?.hydratedSyncConfig ?? this.sync_rules;
|
|
55
|
+
const reconcile = options.reconcileSourceTables ?? storage.defaultSourceTableReconciler;
|
|
54
56
|
const { connection_id, source } = options;
|
|
55
57
|
const { schema, name, objectId, replicaIdColumns, connectionTag, sendsCompleteRows } = source;
|
|
56
58
|
|
|
@@ -63,20 +65,38 @@ export class MongoBucketBatchV1 extends MongoBucketBatch {
|
|
|
63
65
|
let result: storage.ResolveTablesResult | null = null;
|
|
64
66
|
await this.db.client.withSession(async (session) => {
|
|
65
67
|
const col = this.db.sourceTablesV1(this.replicationStreamId);
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
schema_name: schema,
|
|
70
|
-
table_name: name,
|
|
71
|
-
replica_id_columns2: normalizedReplicaIdColumns
|
|
72
|
-
};
|
|
68
|
+
|
|
69
|
+
// Find records that overlap by name or relation id.
|
|
70
|
+
const orClauses: Record<string, unknown>[] = [{ schema_name: schema, table_name: name }];
|
|
73
71
|
if (objectId != null) {
|
|
74
|
-
|
|
72
|
+
orClauses.push({ relation_id: objectId });
|
|
73
|
+
}
|
|
74
|
+
const candidateDocs = await col
|
|
75
|
+
.find({ group_id: this.replicationStreamId, connection_id, $or: orClauses }, { session })
|
|
76
|
+
.toArray();
|
|
77
|
+
|
|
78
|
+
const candidateTables = candidateDocs.map((doc) =>
|
|
79
|
+
this.hydrateSourceTable(doc, connectionTag, syncRules, sendsCompleteRows)
|
|
80
|
+
);
|
|
81
|
+
const candidates = candidateTables.map((table) => table.clone());
|
|
82
|
+
const resolution = await reconcile({ source, candidates });
|
|
83
|
+
storage.validateSourceTableCandidateResolution(candidates, resolution);
|
|
84
|
+
|
|
85
|
+
for (const { id, sourceMetadata } of storage.diffSourceTableUpdates(candidateTables, resolution)) {
|
|
86
|
+
await col.updateOne({ _id: mongoTableId(id) }, { $set: { source_metadata: sourceMetadata } }, { session });
|
|
75
87
|
}
|
|
76
88
|
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
89
|
+
const materializedResolution = storage.materializeSourceTableResolution(candidateTables, resolution);
|
|
90
|
+
const compatibleTables = materializedResolution.compatibleTables;
|
|
91
|
+
|
|
92
|
+
// V1 keeps one record, preferring one that has already been snapshotted.
|
|
93
|
+
const reuse = compatibleTables.find((table) => table.snapshotComplete) ?? compatibleTables[0] ?? null;
|
|
94
|
+
|
|
95
|
+
let sourceTable: storage.SourceTable;
|
|
96
|
+
if (reuse != null) {
|
|
97
|
+
sourceTable = reuse;
|
|
98
|
+
} else {
|
|
99
|
+
const doc: SourceTableDocumentV1 = {
|
|
80
100
|
_id: options.idGenerator ? (options.idGenerator() as bson.ObjectId) : new bson.ObjectId(),
|
|
81
101
|
group_id: this.replicationStreamId,
|
|
82
102
|
connection_id,
|
|
@@ -86,69 +106,18 @@ export class MongoBucketBatchV1 extends MongoBucketBatch {
|
|
|
86
106
|
replica_id_columns: null,
|
|
87
107
|
replica_id_columns2: normalizedReplicaIdColumns,
|
|
88
108
|
snapshot_done: false,
|
|
89
|
-
snapshot_status: undefined
|
|
109
|
+
snapshot_status: undefined,
|
|
110
|
+
source_metadata: resolution.newTableValues.sourceMetadata
|
|
90
111
|
};
|
|
91
112
|
await col.insertOne(doc, { session });
|
|
113
|
+
sourceTable = this.hydrateSourceTable(doc, connectionTag, syncRules, sendsCompleteRows);
|
|
92
114
|
}
|
|
93
115
|
|
|
94
|
-
const
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
snapshotComplete: doc.snapshot_done ?? true,
|
|
100
|
-
...syncRules.getMatchingSources(source)
|
|
101
|
-
});
|
|
102
|
-
sourceTable.syncEvent = syncRules.tableTriggersEvent(source);
|
|
103
|
-
sourceTable.syncData = sourceTable.bucketDataSources.length > 0;
|
|
104
|
-
sourceTable.syncParameters = sourceTable.parameterLookupSources.length > 0;
|
|
105
|
-
sourceTable.storeCurrentData = sendsCompleteRows !== true;
|
|
106
|
-
sourceTable.snapshotStatus =
|
|
107
|
-
doc.snapshot_status == null
|
|
108
|
-
? undefined
|
|
109
|
-
: {
|
|
110
|
-
lastKey: doc.snapshot_status.last_key?.buffer ?? null,
|
|
111
|
-
totalEstimatedCount: doc.snapshot_status.total_estimated_count,
|
|
112
|
-
replicatedCount: doc.snapshot_status.replicated_count
|
|
113
|
-
};
|
|
114
|
-
|
|
115
|
-
const truncateFilter = [{ schema_name: schema, table_name: name }] as any[];
|
|
116
|
-
if (objectId != null) {
|
|
117
|
-
truncateFilter.push({ relation_id: objectId });
|
|
118
|
-
}
|
|
119
|
-
const truncate = await col
|
|
120
|
-
.find(
|
|
121
|
-
{
|
|
122
|
-
group_id: this.replicationStreamId,
|
|
123
|
-
connection_id,
|
|
124
|
-
_id: { $ne: doc._id },
|
|
125
|
-
$or: truncateFilter
|
|
126
|
-
},
|
|
127
|
-
{ session }
|
|
128
|
-
)
|
|
129
|
-
.toArray();
|
|
130
|
-
const dropTables = truncate.map((dropDoc) => {
|
|
131
|
-
const ref = {
|
|
132
|
-
connectionTag,
|
|
133
|
-
schema: dropDoc.schema_name,
|
|
134
|
-
name: dropDoc.table_name
|
|
135
|
-
};
|
|
136
|
-
const table = new storage.SourceTable({
|
|
137
|
-
id: dropDoc._id,
|
|
138
|
-
ref,
|
|
139
|
-
objectId: dropDoc.relation_id,
|
|
140
|
-
replicaIdColumns:
|
|
141
|
-
dropDoc.replica_id_columns2?.map(
|
|
142
|
-
(c) => ({ name: c.name, typeId: c.type_oid, type: c.type }) satisfies ColumnDescriptor
|
|
143
|
-
) ?? [],
|
|
144
|
-
snapshotComplete: dropDoc.snapshot_done ?? true,
|
|
145
|
-
...syncRules.getMatchingSources(ref)
|
|
146
|
-
});
|
|
147
|
-
table.syncEvent = syncRules.tableTriggersEvent(ref);
|
|
148
|
-
table.syncData = table.bucketDataSources.length > 0;
|
|
149
|
-
table.syncParameters = table.parameterLookupSources.length > 0;
|
|
150
|
-
return table;
|
|
151
|
-
});
|
|
116
|
+
const dropTables = [
|
|
117
|
+
...materializedResolution.incompatibleTables,
|
|
118
|
+
// V1 only keeps one SourceTable per physical table.
|
|
119
|
+
...compatibleTables.filter((table) => !storage.sourceTableIdEquals(table.id, sourceTable.id))
|
|
120
|
+
];
|
|
152
121
|
|
|
153
122
|
result = { tables: [sourceTable], dropTables };
|
|
154
123
|
});
|
|
@@ -156,38 +125,33 @@ export class MongoBucketBatchV1 extends MongoBucketBatch {
|
|
|
156
125
|
return result!;
|
|
157
126
|
}
|
|
158
127
|
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
const ref = {
|
|
172
|
-
connectionTag: table.ref.connectionTag,
|
|
173
|
-
schema: doc.schema_name,
|
|
174
|
-
name: doc.table_name
|
|
175
|
-
};
|
|
176
|
-
const sourceTable = new storage.SourceTable({
|
|
128
|
+
/**
|
|
129
|
+
* Build a SourceTable from its persisted v1 record.
|
|
130
|
+
*/
|
|
131
|
+
private hydrateSourceTable(
|
|
132
|
+
doc: SourceTableDocumentV1,
|
|
133
|
+
connectionTag: string,
|
|
134
|
+
syncRules: HydratedSyncConfig,
|
|
135
|
+
sendsCompleteRows: boolean | undefined
|
|
136
|
+
): storage.SourceTable {
|
|
137
|
+
const ref = { connectionTag, schema: doc.schema_name, name: doc.table_name };
|
|
138
|
+
const table = new storage.SourceTable({
|
|
177
139
|
id: doc._id,
|
|
178
140
|
ref,
|
|
179
141
|
objectId: doc.relation_id,
|
|
180
142
|
replicaIdColumns:
|
|
181
143
|
doc.replica_id_columns2?.map(
|
|
182
|
-
(
|
|
144
|
+
(column) => ({ name: column.name, typeId: column.type_oid, type: column.type }) satisfies ColumnDescriptor
|
|
183
145
|
) ?? [],
|
|
184
146
|
snapshotComplete: doc.snapshot_done ?? true,
|
|
185
|
-
|
|
147
|
+
sourceMetadata: doc.source_metadata ?? null,
|
|
148
|
+
...syncRules.getMatchingSources(ref)
|
|
186
149
|
});
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
150
|
+
table.syncEvent = syncRules.tableTriggersEvent(ref);
|
|
151
|
+
table.syncData = table.bucketDataSources.length > 0;
|
|
152
|
+
table.syncParameters = table.parameterLookupSources.length > 0;
|
|
153
|
+
table.storeCurrentData = sendsCompleteRows !== true;
|
|
154
|
+
table.snapshotStatus =
|
|
191
155
|
doc.snapshot_status == null
|
|
192
156
|
? undefined
|
|
193
157
|
: {
|
|
@@ -195,7 +159,23 @@ export class MongoBucketBatchV1 extends MongoBucketBatch {
|
|
|
195
159
|
totalEstimatedCount: doc.snapshot_status.total_estimated_count,
|
|
196
160
|
replicatedCount: doc.snapshot_status.replicated_count
|
|
197
161
|
};
|
|
198
|
-
return
|
|
162
|
+
return table;
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
async getSourceTableStatus(table: storage.SourceTable): Promise<storage.SourceTable | null> {
|
|
166
|
+
const doc = await this.db.sourceTablesV1(this.replicationStreamId).findOne(
|
|
167
|
+
{
|
|
168
|
+
group_id: this.replicationStreamId,
|
|
169
|
+
_id: mongoTableId(table.id)
|
|
170
|
+
},
|
|
171
|
+
{ session: this.session }
|
|
172
|
+
);
|
|
173
|
+
if (doc == null) {
|
|
174
|
+
return null;
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
// Use the default storeCurrentData value when reading outside resolution.
|
|
178
|
+
return this.hydrateSourceTable(doc, table.ref.connectionTag, this.sync_rules, undefined);
|
|
199
179
|
}
|
|
200
180
|
|
|
201
181
|
async commit(lsn: string, options?: storage.BucketBatchCommitOptions): Promise<storage.CheckpointResult> {
|
|
@@ -148,23 +148,16 @@ export class MongoBucketBatchV3 extends MongoBucketBatch {
|
|
|
148
148
|
const mapping = parsedOverride?.mapping ?? this.mapping;
|
|
149
149
|
|
|
150
150
|
const { connection_id, source } = options;
|
|
151
|
-
const
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
type_oid: column.typeId
|
|
162
|
-
}))
|
|
163
|
-
},
|
|
164
|
-
storeCurrentData: source.sendsCompleteRows !== true,
|
|
165
|
-
syncConfig,
|
|
166
|
-
mapping,
|
|
167
|
-
desired: sourceTableDesiredResolution(syncConfig, source, mapping)
|
|
151
|
+
const reconcile = options.reconcileSourceTables ?? storage.defaultSourceTableReconciler;
|
|
152
|
+
const identity = {
|
|
153
|
+
schema: source.schema,
|
|
154
|
+
name: source.name,
|
|
155
|
+
objectId: source.objectId,
|
|
156
|
+
replicaIdColumns: source.replicaIdColumns.map((column) => ({
|
|
157
|
+
name: column.name,
|
|
158
|
+
type: column.type,
|
|
159
|
+
type_oid: column.typeId
|
|
160
|
+
}))
|
|
168
161
|
};
|
|
169
162
|
|
|
170
163
|
let result: storage.ResolveTablesResult | null = null;
|
|
@@ -174,14 +167,36 @@ export class MongoBucketBatchV3 extends MongoBucketBatch {
|
|
|
174
167
|
await session.withTransaction(async () => {
|
|
175
168
|
const col = this.db.sourceTables(this.replicationStreamId);
|
|
176
169
|
|
|
177
|
-
//
|
|
178
|
-
// Exact-identity docs are candidates for reuse; non-exact overlaps are possible drops.
|
|
170
|
+
// Find records that overlap by name or relation id.
|
|
179
171
|
const candidateDocs = await col
|
|
180
|
-
.find(overlappingSourceTableFilter(connection_id,
|
|
172
|
+
.find(overlappingSourceTableFilter(connection_id, identity), { session })
|
|
181
173
|
.toArray();
|
|
182
174
|
|
|
183
|
-
|
|
184
|
-
|
|
175
|
+
const candidateTables = candidateDocs.map((doc) =>
|
|
176
|
+
sourceTableFromDocument(doc, source.connectionTag, syncConfig, mapping)
|
|
177
|
+
);
|
|
178
|
+
const candidates = candidateTables.map((table) => table.clone());
|
|
179
|
+
const resolution = await reconcile({ source, candidates });
|
|
180
|
+
storage.validateSourceTableCandidateResolution(candidates, resolution);
|
|
181
|
+
|
|
182
|
+
// Persist metadata from the reconciler without mutating the queried documents.
|
|
183
|
+
for (const { id, sourceMetadata } of storage.diffSourceTableUpdates(candidateTables, resolution)) {
|
|
184
|
+
await col.updateOne({ _id: mongoTableId(id) }, { $set: { source_metadata: sourceMetadata } }, { session });
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
const context: SourceTableReconciliationContext = {
|
|
188
|
+
connectionId: connection_id,
|
|
189
|
+
connectionTag: source.connectionTag,
|
|
190
|
+
identity,
|
|
191
|
+
storeCurrentData: source.sendsCompleteRows !== true,
|
|
192
|
+
syncConfig,
|
|
193
|
+
mapping,
|
|
194
|
+
desired: sourceTableDesiredResolution(syncConfig, source, mapping),
|
|
195
|
+
sourceCompatibleTables: resolution.compatibleTables,
|
|
196
|
+
newTableSourceMetadata: resolution.newTableValues.sourceMetadata
|
|
197
|
+
};
|
|
198
|
+
|
|
199
|
+
// Plan record reuse, membership changes, creation, and removal.
|
|
185
200
|
const plan = planSourceTableReconciliation(candidateDocs, context);
|
|
186
201
|
|
|
187
202
|
// Persist narrowing for incomplete snapshots only. Snapshot-complete docs keep stale
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import {
|
|
2
2
|
InternalOpId,
|
|
3
|
+
JsonValue,
|
|
3
4
|
PersistedDefinitionMapping,
|
|
4
5
|
SerializedSyncPlan,
|
|
5
6
|
SyncRuleState,
|
|
@@ -157,6 +158,10 @@ export interface SourceTableDocumentV3 {
|
|
|
157
158
|
bucket_data_source_ids: BucketDefinitionId[];
|
|
158
159
|
parameter_lookup_source_ids: ParameterIndexId[];
|
|
159
160
|
latest_pending_delete?: InternalOpId | undefined;
|
|
161
|
+
/**
|
|
162
|
+
* Source-specific metadata. Absent for legacy records.
|
|
163
|
+
*/
|
|
164
|
+
source_metadata?: JsonValue;
|
|
160
165
|
}
|
|
161
166
|
|
|
162
167
|
export interface BucketStateDocumentV3 extends BucketStateDocumentBase {
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { ReplicationAssertionError } from '@powersync/lib-services-framework';
|
|
2
|
-
import { BucketDefinitionMapping, ColumnDescriptor, storage } from '@powersync/service-core';
|
|
2
|
+
import { BucketDefinitionMapping, ColumnDescriptor, JsonValue, storage } from '@powersync/service-core';
|
|
3
3
|
import {
|
|
4
4
|
BucketDataSource,
|
|
5
5
|
BucketDefinitionId,
|
|
@@ -38,6 +38,14 @@ export interface SourceTableReconciliationContext {
|
|
|
38
38
|
syncConfig: HydratedSyncConfig;
|
|
39
39
|
mapping: BucketDefinitionMapping;
|
|
40
40
|
desired: SourceTableDesiredResolution;
|
|
41
|
+
/**
|
|
42
|
+
* Candidates the source connector considers safe to reuse.
|
|
43
|
+
*/
|
|
44
|
+
sourceCompatibleTables: readonly storage.SourceTableCandidate[];
|
|
45
|
+
/**
|
|
46
|
+
* Source metadata for records created by this resolution.
|
|
47
|
+
*/
|
|
48
|
+
newTableSourceMetadata: JsonValue;
|
|
41
49
|
}
|
|
42
50
|
|
|
43
51
|
export interface SourceTableReconciliationPlan {
|
|
@@ -134,9 +142,8 @@ class SourceTableReconciliationPlanner {
|
|
|
134
142
|
constructor(private readonly context: SourceTableReconciliationContext) {}
|
|
135
143
|
|
|
136
144
|
plan(candidateDocs: SourceTableDocumentV3[]): SourceTableReconciliationPlan {
|
|
137
|
-
const { identity } = this.context;
|
|
138
145
|
for (const doc of candidateDocs) {
|
|
139
|
-
if (
|
|
146
|
+
if (this.isCompatible(doc)) {
|
|
140
147
|
this.retainDoc(doc);
|
|
141
148
|
}
|
|
142
149
|
}
|
|
@@ -145,14 +152,19 @@ class SourceTableReconciliationPlanner {
|
|
|
145
152
|
tables: this.tables,
|
|
146
153
|
narrowingUpdates: this.narrowingUpdates,
|
|
147
154
|
newTableMemberships: this.newTableMemberships(),
|
|
148
|
-
//
|
|
149
|
-
|
|
150
|
-
dropDocs: candidateDocs.filter(
|
|
151
|
-
(doc) => !this.retainedDocIds.has(doc._id.toHexString()) && !matchingSourceTableIdentity(doc, identity)
|
|
152
|
-
)
|
|
155
|
+
// Compatible records remain available for reuse even when not needed by this config.
|
|
156
|
+
dropDocs: candidateDocs.filter((doc) => !this.isCompatible(doc))
|
|
153
157
|
};
|
|
154
158
|
}
|
|
155
159
|
|
|
160
|
+
private isCompatible(doc: SourceTableDocumentV3) {
|
|
161
|
+
return this.compatibleTableFor(doc) != null;
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
private compatibleTableFor(doc: SourceTableDocumentV3): storage.SourceTableCandidate | undefined {
|
|
165
|
+
return this.context.sourceCompatibleTables.find((table) => storage.sourceTableIdEquals(table.id, doc._id));
|
|
166
|
+
}
|
|
167
|
+
|
|
156
168
|
private retainDoc(doc: SourceTableDocumentV3) {
|
|
157
169
|
const memberships = intersectMembershipIds(doc, this.context.desired);
|
|
158
170
|
const coversDesiredMembership = hasMembershipIds(memberships);
|
|
@@ -237,7 +249,7 @@ class SourceTableReconciliationPlanner {
|
|
|
237
249
|
|
|
238
250
|
private sourceTableFor(doc: SourceTableDocumentV3, memberships: SourceTableMembershipIds): storage.SourceTable {
|
|
239
251
|
const { connectionTag, syncConfig, mapping, desired, storeCurrentData } = this.context;
|
|
240
|
-
const
|
|
252
|
+
const built = sourceTableFromDocument(
|
|
241
253
|
doc,
|
|
242
254
|
connectionTag,
|
|
243
255
|
syncConfig,
|
|
@@ -245,31 +257,14 @@ class SourceTableReconciliationPlanner {
|
|
|
245
257
|
matchingSourcesFor(desired, memberships),
|
|
246
258
|
memberships
|
|
247
259
|
);
|
|
260
|
+
// Use any metadata update returned by the reconciler.
|
|
261
|
+
const resolved = this.compatibleTableFor(doc);
|
|
262
|
+
const table = resolved == null ? built : built.withSourceMetadata(resolved.sourceMetadata);
|
|
248
263
|
table.storeCurrentData = storeCurrentData;
|
|
249
264
|
return table;
|
|
250
265
|
}
|
|
251
266
|
}
|
|
252
267
|
|
|
253
|
-
function sameReplicaIdColumns(left: ReplicaIdColumn[] | undefined, right: ReplicaIdColumn[]) {
|
|
254
|
-
return (
|
|
255
|
-
left != null &&
|
|
256
|
-
left.length == right.length &&
|
|
257
|
-
left.every(
|
|
258
|
-
(column, index) =>
|
|
259
|
-
column.name == right[index].name && column.type == right[index].type && column.type_oid == right[index].type_oid
|
|
260
|
-
)
|
|
261
|
-
);
|
|
262
|
-
}
|
|
263
|
-
|
|
264
|
-
function matchingSourceTableIdentity(doc: SourceTableDocumentV3, identity: SourceTableIdentity) {
|
|
265
|
-
return (
|
|
266
|
-
doc.schema_name == identity.schema &&
|
|
267
|
-
doc.table_name == identity.name &&
|
|
268
|
-
(identity.objectId == null || doc.relation_id == identity.objectId) &&
|
|
269
|
-
sameReplicaIdColumns(doc.replica_id_columns, identity.replicaIdColumns)
|
|
270
|
-
);
|
|
271
|
-
}
|
|
272
|
-
|
|
273
268
|
export function overlappingSourceTableFilter(
|
|
274
269
|
connectionId: number,
|
|
275
270
|
identity: Pick<SourceTableIdentity, 'schema' | 'name' | 'objectId'>
|
|
@@ -290,7 +285,16 @@ export function createNewSourceTable(
|
|
|
290
285
|
memberships: SourceTableMembershipIds,
|
|
291
286
|
context: SourceTableReconciliationContext
|
|
292
287
|
): NewSourceTable {
|
|
293
|
-
const {
|
|
288
|
+
const {
|
|
289
|
+
connectionId,
|
|
290
|
+
connectionTag,
|
|
291
|
+
identity,
|
|
292
|
+
syncConfig,
|
|
293
|
+
mapping,
|
|
294
|
+
desired,
|
|
295
|
+
storeCurrentData,
|
|
296
|
+
newTableSourceMetadata
|
|
297
|
+
} = context;
|
|
294
298
|
const doc: SourceTableDocumentV3 = {
|
|
295
299
|
_id: id,
|
|
296
300
|
connection_id: connectionId,
|
|
@@ -301,7 +305,9 @@ export function createNewSourceTable(
|
|
|
301
305
|
snapshot_done: false,
|
|
302
306
|
snapshot_status: undefined,
|
|
303
307
|
bucket_data_source_ids: memberships.bucketDataSourceIds,
|
|
304
|
-
parameter_lookup_source_ids: memberships.parameterLookupSourceIds
|
|
308
|
+
parameter_lookup_source_ids: memberships.parameterLookupSourceIds,
|
|
309
|
+
// Records created together share the same source metadata.
|
|
310
|
+
source_metadata: newTableSourceMetadata
|
|
305
311
|
};
|
|
306
312
|
const table = sourceTableFromDocument(
|
|
307
313
|
doc,
|
|
@@ -368,14 +374,16 @@ export function sourceTableFromDocument(
|
|
|
368
374
|
name: doc.table_name
|
|
369
375
|
},
|
|
370
376
|
objectId: doc.relation_id,
|
|
371
|
-
replicaIdColumns:
|
|
372
|
-
(
|
|
373
|
-
|
|
377
|
+
replicaIdColumns:
|
|
378
|
+
doc.replica_id_columns?.map(
|
|
379
|
+
(column) => ({ name: column.name, typeId: column.type_oid, type: column.type }) satisfies ColumnDescriptor
|
|
380
|
+
) ?? [],
|
|
374
381
|
snapshotComplete: doc.snapshot_done,
|
|
375
382
|
bucketDataSources: resolvedMemberships.bucketDataSources,
|
|
376
383
|
parameterLookupSources: resolvedMemberships.parameterLookupSources,
|
|
377
384
|
bucketDataSourceIds: new Set(resolvedMembershipIds.bucketDataSourceIds),
|
|
378
|
-
parameterLookupSourceIds: new Set(resolvedMembershipIds.parameterLookupSourceIds)
|
|
385
|
+
parameterLookupSourceIds: new Set(resolvedMembershipIds.parameterLookupSourceIds),
|
|
386
|
+
sourceMetadata: doc.source_metadata ?? null
|
|
379
387
|
});
|
|
380
388
|
table.syncData = table.bucketDataSources.length > 0;
|
|
381
389
|
table.syncParameters = table.parameterLookupSources.length > 0;
|
|
@@ -119,6 +119,35 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor
|
|
|
119
119
|
storageVersion,
|
|
120
120
|
tableIdStrings: storageConfig.tableIdStrings
|
|
121
121
|
});
|
|
122
|
+
|
|
123
|
+
test('updates source metadata on an existing resolved table', async () => {
|
|
124
|
+
await using factory = await storageConfig.factory();
|
|
125
|
+
const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(MINIMAL_SYNC_RULES, { storageVersion }));
|
|
126
|
+
const bucketStorage = factory.getInstance(syncRules);
|
|
127
|
+
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
128
|
+
const source = sourceDescriptor('test');
|
|
129
|
+
|
|
130
|
+
const initial = await writer.resolveTables({ connection_id: 1, source });
|
|
131
|
+
expect(initial.tables[0].sourceMetadata).toBeNull();
|
|
132
|
+
|
|
133
|
+
const sourceMetadata = { captureTableObjectId: 42 };
|
|
134
|
+
const updated = await writer.resolveTables({
|
|
135
|
+
connection_id: 1,
|
|
136
|
+
source,
|
|
137
|
+
reconcileSourceTables: ({ candidates }) => ({
|
|
138
|
+
compatibleTables: candidates.map((candidate) => candidate.withSourceMetadata(sourceMetadata)),
|
|
139
|
+
incompatibleTables: [],
|
|
140
|
+
newTableValues: { sourceMetadata }
|
|
141
|
+
})
|
|
142
|
+
});
|
|
143
|
+
|
|
144
|
+
expect(updated.tables.map((table) => table.id.toString())).toEqual(
|
|
145
|
+
initial.tables.map((table) => table.id.toString())
|
|
146
|
+
);
|
|
147
|
+
expect(updated.tables[0].sourceMetadata).toEqual(sourceMetadata);
|
|
148
|
+
expect((await writer.getSourceTableStatus(updated.tables[0]))?.sourceMetadata).toEqual(sourceMetadata);
|
|
149
|
+
});
|
|
150
|
+
|
|
122
151
|
// The split of returned results can vary depending on storage drivers
|
|
123
152
|
test('large batch (2)', async () => {
|
|
124
153
|
// Test syncing a batch of data that is small in count,
|