@powersync/service-core 0.0.0-dev-20250820110726 → 0.0.0-dev-20250827072023
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +10 -6
- package/dist/events/EventsEngine.d.ts +4 -1
- package/dist/events/EventsEngine.js +4 -3
- package/dist/events/EventsEngine.js.map +1 -1
- package/dist/routes/configure-fastify.d.ts +0 -5
- package/dist/routes/configure-fastify.js.map +1 -1
- package/dist/routes/endpoints/socket-route.js +6 -5
- package/dist/routes/endpoints/socket-route.js.map +1 -1
- package/dist/routes/endpoints/sync-stream.d.ts +0 -10
- package/dist/routes/endpoints/sync-stream.js +8 -5
- package/dist/routes/endpoints/sync-stream.js.map +1 -1
- package/dist/routes/router.d.ts +3 -3
- package/dist/storage/BucketStorageBatch.d.ts +4 -4
- package/dist/storage/BucketStorageBatch.js.map +1 -1
- package/dist/storage/BucketStorageFactory.d.ts +2 -0
- package/dist/storage/ReplicationEventPayload.d.ts +2 -2
- package/dist/storage/ReportStorage.d.ts +30 -2
- package/dist/sync/sync.d.ts +1 -0
- package/dist/sync/sync.js +18 -35
- package/dist/sync/sync.js.map +1 -1
- package/dist/util/protocol-types.d.ts +0 -4
- package/dist/util/protocol-types.js +0 -4
- package/dist/util/protocol-types.js.map +1 -1
- package/dist/util/utils.d.ts +1 -1
- package/dist/util/utils.js.map +1 -1
- package/package.json +6 -6
- package/src/events/EventsEngine.ts +4 -4
- package/src/routes/configure-fastify.ts +0 -1
- package/src/routes/endpoints/socket-route.ts +6 -5
- package/src/routes/endpoints/sync-stream.ts +8 -6
- package/src/routes/router.ts +3 -3
- package/src/storage/BucketStorageBatch.ts +10 -4
- package/src/storage/BucketStorageFactory.ts +2 -0
- package/src/storage/ReplicationEventPayload.ts +2 -2
- package/src/storage/ReportStorage.ts +30 -2
- package/src/sync/sync.ts +36 -36
- package/src/util/protocol-types.ts +0 -5
- package/src/util/utils.ts +4 -1
- package/test/src/sync/BucketChecksumState.test.ts +36 -92
- package/tsconfig.tsbuildinfo +1 -1
|
@@ -58,10 +58,6 @@ export const StreamingSyncRequest = t.object({
|
|
|
58
58
|
* True to keep `data` as a string, instead of nested JSON.
|
|
59
59
|
*/
|
|
60
60
|
raw_data: t.boolean.optional(),
|
|
61
|
-
/**
|
|
62
|
-
* Data is received in a serialized BSON Buffer
|
|
63
|
-
*/
|
|
64
|
-
binary_data: t.boolean.optional(),
|
|
65
61
|
/**
|
|
66
62
|
* Client parameters to be passed to the sync rules.
|
|
67
63
|
*/
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"protocol-types.js","sourceRoot":"","sources":["../../src/util/protocol-types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,CAAC,MAAM,UAAU,CAAC;AAI9B,MAAM,CAAC,MAAM,aAAa,GAAG,CAAC,CAAC,MAAM,CAAC;IACpC,IAAI,EAAE,CAAC,CAAC,MAAM;IAEd;;OAEG;IACH,KAAK,EAAE,CAAC,CAAC,MAAM;CAChB,CAAC,CAAC;AAIH;;GAEG;AACH,MAAM,CAAC,MAAM,2BAA2B,GAAG,CAAC,CAAC,MAAM,CAAC;IAClD;;OAEG;IACH,MAAM,EAAE,CAAC,CAAC,MAAM;IAChB;;OAEG;IACH,UAAU,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE;IACtC;;;;;OAKG;IACH,iBAAiB,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC;CAC7C,CAAC,CAAC;AAIH;;GAEG;AACH,MAAM,CAAC,MAAM,yBAAyB,GAAG,CAAC,CAAC,MAAM,CAAC;IAChD;;;;OAIG;IACH,gBAAgB,EAAE,CAAC,CAAC,OAAO,CAAC,QAAQ,EAAE;IAEtC;;OAEG;IACH,aAAa,EAAE,CAAC,CAAC,KAAK,CAAC,2BAA2B,CAAC;CACpD,CAAC,CAAC;AAIH,MAAM,CAAC,MAAM,oBAAoB,GAAG,CAAC,CAAC,MAAM,CAAC;IAC3C;;OAEG;IACH,OAAO,EAAE,CAAC,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC,QAAQ,EAAE;IAE1C;;OAEG;IACH,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE;IAElC;;OAEG;IACH,gBAAgB,EAAE,CAAC,CAAC,OAAO,CAAC,QAAQ,EAAE;IAEtC;;OAEG;IACH,QAAQ,EAAE,CAAC,CAAC,OAAO,CAAC,QAAQ,EAAE;IAE9B;;OAEG;IACH,
|
|
1
|
+
{"version":3,"file":"protocol-types.js","sourceRoot":"","sources":["../../src/util/protocol-types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,CAAC,MAAM,UAAU,CAAC;AAI9B,MAAM,CAAC,MAAM,aAAa,GAAG,CAAC,CAAC,MAAM,CAAC;IACpC,IAAI,EAAE,CAAC,CAAC,MAAM;IAEd;;OAEG;IACH,KAAK,EAAE,CAAC,CAAC,MAAM;CAChB,CAAC,CAAC;AAIH;;GAEG;AACH,MAAM,CAAC,MAAM,2BAA2B,GAAG,CAAC,CAAC,MAAM,CAAC;IAClD;;OAEG;IACH,MAAM,EAAE,CAAC,CAAC,MAAM;IAChB;;OAEG;IACH,UAAU,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE;IACtC;;;;;OAKG;IACH,iBAAiB,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC;CAC7C,CAAC,CAAC;AAIH;;GAEG;AACH,MAAM,CAAC,MAAM,yBAAyB,GAAG,CAAC,CAAC,MAAM,CAAC;IAChD;;;;OAIG;IACH,gBAAgB,EAAE,CAAC,CAAC,OAAO,CAAC,QAAQ,EAAE;IAEtC;;OAEG;IACH,aAAa,EAAE,CAAC,CAAC,KAAK,CAAC,2BAA2B,CAAC;CACpD,CAAC,CAAC;AAIH,MAAM,CAAC,MAAM,oBAAoB,GAAG,CAAC,CAAC,MAAM,CAAC;IAC3C;;OAEG;IACH,OAAO,EAAE,CAAC,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC,QAAQ,EAAE;IAE1C;;OAEG;IACH,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE;IAElC;;OAEG;IACH,gBAAgB,EAAE,CAAC,CAAC,OAAO,CAAC,QAAQ,EAAE;IAEtC;;OAEG;IACH,QAAQ,EAAE,CAAC,CAAC,OAAO,CAAC,QAAQ,EAAE;IAE9B;;OAEG;IACH,UAAU,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE;IAEtC;;OAEG;IACH,SAAS,EAAE,CAAC,CAAC,MAAM,CAAC,QAAQ,EAAE;IAE9B;;OAEG;IACH,OAAO,EAAE,yBAAyB,CAAC,QAAQ,EAAE;CAC9C,CAAC,CAAC"}
|
package/dist/util/utils.d.ts
CHANGED
|
@@ -33,7 +33,7 @@ export declare function hasToastedValues(row: sync_rules.ToastableSqliteRow): bo
|
|
|
33
33
|
*
|
|
34
34
|
* If we don't store data, we assume we always have a complete row.
|
|
35
35
|
*/
|
|
36
|
-
export declare function isCompleteRow(storeData: boolean, row: sync_rules.ToastableSqliteRow): row is sync_rules.
|
|
36
|
+
export declare function isCompleteRow(storeData: boolean, row: sync_rules.ToastableSqliteRow): row is sync_rules.SqliteInputRow;
|
|
37
37
|
/**
|
|
38
38
|
* Reduce a bucket to the final state as stored on the client.
|
|
39
39
|
*
|
package/dist/util/utils.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"utils.js","sourceRoot":"","sources":["../../src/util/utils.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAC7B,OAAO,MAAM,MAAM,QAAQ,CAAC;AAC5B,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAM7B,OAAO,EAAE,qBAAqB,EAAE,MAAM,mCAAmC,CAAC;AAW1E,MAAM,CAAC,MAAM,YAAY,GAAG,sCAAsC,CAAC;AAEnE,MAAM,UAAU,gBAAgB,CAAC,UAAkB;IACjD,OAAO,IAAI,UAAU,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,KAAK,CAAC,GAAG,CAAC;AACrE,CAAC;AAED,MAAM,UAAU,QAAQ,CAAC,IAAY,EAAE,EAAU,EAAE,IAAY;IAC7D,MAAM,IAAI,GAAG,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;IACzC,IAAI,CAAC,MAAM,CAAC,OAAO,IAAI,IAAI,EAAE,IAAI,IAAI,EAAE,CAAC,CAAC;IACzC,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;IAC7B,OAAO,MAAM,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;AAChC,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,SAAiB;IAC1C,MAAM,IAAI,GAAG,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;IACzC,IAAI,CAAC,MAAM,CAAC,UAAU,SAAS,EAAE,CAAC,CAAC;IACnC,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;IAC7B,OAAO,MAAM,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;AAChC,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,sBAAsB,CAAC,EAAgB;IACrD,6EAA6E;IAC7E,6CAA6C;IAC7C,IAAI,OAAO,EAAE,IAAI,QAAQ,EAAE,CAAC;QAC1B,MAAM,IAAI,qBAAqB,CAAC,yBAAyB,EAAE,KAAK,OAAO,EAAE,GAAG,CAAC,CAAC;IAChF,CAAC;IACD,OAAO,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;AACzB,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,QAAqB,EAAE,OAAoB;IACvE,mBAAmB;IACnB,MAAM,cAAc,GAAG,IAAI,GAAG,EAA0B,CAAC;IAEzD,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAS,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC;IAElD,KAAK,IAAI,QAAQ,IAAI,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;QACtC,MAAM,CAAC,GAAG,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QACxC,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC;YACd,QAAQ;YACR,cAAc,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;QAChD,CAAC;aAAM,CAAC;YACN,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;YACjC,IAAI,QAAQ,CAAC,QAAQ,IAAI,CAAC,CAAC,QAAQ,IAAI,QAAQ,CAAC,KAAK,IAAI,CAAC,CAAC,KAAK,EAAE,CAAC;gBACjE,UAAU;gBACV,cAAc,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;YAChD,CAAC;iBAAM,CAAC;gBACN,YAAY;YACd,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO;QACL,cAAc,EAAE,CAAC,GAAG,cAAc,CAAC,MAAM,EAAE,CAAC;QAC5C,cAAc,EAAE,CAAC,GAAG,QAAQ,CAAC;KAC9B,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,CAAS,EAAE,CAAS;IAC/C,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,UAAU,CAAC;AAC9B,CAAC;AAED,MAAM,UAAU,kBAAkB,CAAC,CAAiB,EAAE,CAAyB;IAC7E,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC;QACd,OAAO,CAAC,CAAC;IACX,CAAC;SAAM,IAAI,CAAC,CAAC,cAAc,EAAE,CAAC;QAC5B,OAAO;YACL,MAAM,EAAE,CAAC,CAAC,MAAM;YAChB,KAAK,EAAE,CAAC,CAAC,YAAY;YACrB,QAAQ,EAAE,CAAC,CAAC,eAAe;SAC5B,CAAC;IACJ,CAAC;SAAM,CAAC;QACN,OAAO;YACL,MAAM,EAAE,CAAC,CAAC,MAAM;YAChB,KAAK,EAAE,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,YAAY;YAC/B,QAAQ,EAAE,YAAY,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,eAAe,CAAC;SACtD,CAAC;IACJ,CAAC;AACH,CAAC;AAED,SAAS,qBAAqB,CAC5B,KAAoC,EACpC,OAAmC;IAEnC,IAAI,MAAM,GAAwB,EAAE,CAAC;IACrC,KAAK,IAAI,MAAM,IAAI,OAAO,EAAE,CAAC;QAC3B,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC;QACzB,MAAM,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC;IAC7B,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,UAAU,0BAA0B,CACxC,KAAoC,EACpC,OAAmC;IAEnC,IAAI,OAAO,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;QACxB,gDAAgD;QAChD,OAAO,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC;IAClC,CAAC;IACD,MAAM,WAAW,GAAG,qBAAqB,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;IAE1D,OAAO,cAAc,CAAC,WAAW,CAAC,CAAC;AACrC,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,GAAyB;IACtD,+EAA+E;IAC/E,4EAA4E;IAC5E,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;IACjC,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;IAChC,OAAO,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,EAAE,YAAY,EAAE,MAAM,CAAC,CAAC,CAAC;AAC5D,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,GAAkC;IACjE,KAAK,IAAI,GAAG,IAAI,GAAG,EAAE,CAAC;QACpB,IAAI,OAAO,GAAG,CAAC,GAAG,CAAC,IAAI,WAAW,EAAE,CAAC;YACnC,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,aAAa,
|
|
1
|
+
{"version":3,"file":"utils.js","sourceRoot":"","sources":["../../src/util/utils.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAC7B,OAAO,MAAM,MAAM,QAAQ,CAAC;AAC5B,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAM7B,OAAO,EAAE,qBAAqB,EAAE,MAAM,mCAAmC,CAAC;AAW1E,MAAM,CAAC,MAAM,YAAY,GAAG,sCAAsC,CAAC;AAEnE,MAAM,UAAU,gBAAgB,CAAC,UAAkB;IACjD,OAAO,IAAI,UAAU,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,KAAK,CAAC,GAAG,CAAC;AACrE,CAAC;AAED,MAAM,UAAU,QAAQ,CAAC,IAAY,EAAE,EAAU,EAAE,IAAY;IAC7D,MAAM,IAAI,GAAG,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;IACzC,IAAI,CAAC,MAAM,CAAC,OAAO,IAAI,IAAI,EAAE,IAAI,IAAI,EAAE,CAAC,CAAC;IACzC,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;IAC7B,OAAO,MAAM,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;AAChC,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,SAAiB;IAC1C,MAAM,IAAI,GAAG,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;IACzC,IAAI,CAAC,MAAM,CAAC,UAAU,SAAS,EAAE,CAAC,CAAC;IACnC,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;IAC7B,OAAO,MAAM,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;AAChC,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,sBAAsB,CAAC,EAAgB;IACrD,6EAA6E;IAC7E,6CAA6C;IAC7C,IAAI,OAAO,EAAE,IAAI,QAAQ,EAAE,CAAC;QAC1B,MAAM,IAAI,qBAAqB,CAAC,yBAAyB,EAAE,KAAK,OAAO,EAAE,GAAG,CAAC,CAAC;IAChF,CAAC;IACD,OAAO,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;AACzB,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,QAAqB,EAAE,OAAoB;IACvE,mBAAmB;IACnB,MAAM,cAAc,GAAG,IAAI,GAAG,EAA0B,CAAC;IAEzD,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAS,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC;IAElD,KAAK,IAAI,QAAQ,IAAI,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;QACtC,MAAM,CAAC,GAAG,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QACxC,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC;YACd,QAAQ;YACR,cAAc,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;QAChD,CAAC;aAAM,CAAC;YACN,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;YACjC,IAAI,QAAQ,CAAC,QAAQ,IAAI,CAAC,CAAC,QAAQ,IAAI,QAAQ,CAAC,KAAK,IAAI,CAAC,CAAC,KAAK,EAAE,CAAC;gBACjE,UAAU;gBACV,cAAc,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;YAChD,CAAC;iBAAM,CAAC;gBACN,YAAY;YACd,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO;QACL,cAAc,EAAE,CAAC,GAAG,cAAc,CAAC,MAAM,EAAE,CAAC;QAC5C,cAAc,EAAE,CAAC,GAAG,QAAQ,CAAC;KAC9B,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,CAAS,EAAE,CAAS;IAC/C,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,UAAU,CAAC;AAC9B,CAAC;AAED,MAAM,UAAU,kBAAkB,CAAC,CAAiB,EAAE,CAAyB;IAC7E,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC;QACd,OAAO,CAAC,CAAC;IACX,CAAC;SAAM,IAAI,CAAC,CAAC,cAAc,EAAE,CAAC;QAC5B,OAAO;YACL,MAAM,EAAE,CAAC,CAAC,MAAM;YAChB,KAAK,EAAE,CAAC,CAAC,YAAY;YACrB,QAAQ,EAAE,CAAC,CAAC,eAAe;SAC5B,CAAC;IACJ,CAAC;SAAM,CAAC;QACN,OAAO;YACL,MAAM,EAAE,CAAC,CAAC,MAAM;YAChB,KAAK,EAAE,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,YAAY;YAC/B,QAAQ,EAAE,YAAY,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,eAAe,CAAC;SACtD,CAAC;IACJ,CAAC;AACH,CAAC;AAED,SAAS,qBAAqB,CAC5B,KAAoC,EACpC,OAAmC;IAEnC,IAAI,MAAM,GAAwB,EAAE,CAAC;IACrC,KAAK,IAAI,MAAM,IAAI,OAAO,EAAE,CAAC;QAC3B,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC;QACzB,MAAM,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC;IAC7B,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,UAAU,0BAA0B,CACxC,KAAoC,EACpC,OAAmC;IAEnC,IAAI,OAAO,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;QACxB,gDAAgD;QAChD,OAAO,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC;IAClC,CAAC;IACD,MAAM,WAAW,GAAG,qBAAqB,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;IAE1D,OAAO,cAAc,CAAC,WAAW,CAAC,CAAC;AACrC,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,GAAyB;IACtD,+EAA+E;IAC/E,4EAA4E;IAC5E,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;IACjC,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;IAChC,OAAO,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,EAAE,YAAY,EAAE,MAAM,CAAC,CAAC,CAAC;AAC5D,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,GAAkC;IACjE,KAAK,IAAI,GAAG,IAAI,GAAG,EAAE,CAAC;QACpB,IAAI,OAAO,GAAG,CAAC,GAAG,CAAC,IAAI,WAAW,EAAE,CAAC;YACnC,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,aAAa,CAC3B,SAAkB,EAClB,GAAkC;IAElC,IAAI,CAAC,SAAS,EAAE,CAAC;QACf,gDAAgD;QAChD,OAAO,IAAI,CAAC;IACd,CAAC;IACD,OAAO,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC;AAChC,CAAC;AAED;;;;;;;;;;;GAWG;AACH,MAAM,UAAU,YAAY,CAAC,UAAwB;IACnD,IAAI,QAAQ,GAAG,IAAI,GAAG,EAAsB,CAAC;IAC7C,IAAI,aAAa,GAAG,CAAC,CAAC;IAEtB,KAAK,IAAI,EAAE,IAAI,UAAU,EAAE,CAAC;QAC1B,MAAM,GAAG,GAAG,MAAM,CAAC,EAAE,CAAC,CAAC;QACvB,IAAI,EAAE,CAAC,EAAE,IAAI,KAAK,EAAE,CAAC;YACnB,MAAM,QAAQ,GAAG,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YACnC,IAAI,QAAQ,EAAE,CAAC;gBACb,aAAa,GAAG,YAAY,CAAC,aAAa,EAAE,QAAQ,CAAC,QAAkB,CAAC,CAAC;YAC3E,CAAC;YACD,QAAQ,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;QACxB,CAAC;aAAM,IAAI,EAAE,CAAC,EAAE,IAAI,QAAQ,EAAE,CAAC;YAC7B,MAAM,QAAQ,GAAG,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YACnC,IAAI,QAAQ,EAAE,CAAC;gBACb,aAAa,GAAG,YAAY,CAAC,aAAa,EAAE,QAAQ,CAAC,QAAkB,CAAC,CAAC;YAC3E,CAAC;YACD,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YACrB,aAAa,GAAG,YAAY,CAAC,aAAa,EAAE,EAAE,CAAC,QAAkB,CAAC,CAAC;QACrE,CAAC;aAAM,IAAI,EAAE,CAAC,EAAE,IAAI,OAAO,EAAE,CAAC;YAC5B,QAAQ,CAAC,KAAK,EAAE,CAAC;YACjB,aAAa,GAAG,EAAE,CAAC,QAAkB,CAAC;QACxC,CAAC;aAAM,IAAI,EAAE,CAAC,EAAE,IAAI,MAAM,EAAE,CAAC;YAC3B,aAAa,GAAG,YAAY,CAAC,aAAa,EAAE,EAAE,CAAC,QAAkB,CAAC,CAAC;QACrE,CAAC;aAAM,CAAC;YACN,MAAM,IAAI,KAAK,CAAC,qBAAqB,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;QAChD,CAAC;IACH,CAAC;IAED,MAAM,IAAI,GAAG,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;QAChD,OAAO,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;IACnD,CAAC,CAAC,CAAC;IAEH,IAAI,UAAU,GAAiB;QAC7B,wDAAwD;QACxD,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE,aAAa,EAAE;QACpD,GAAG,IAAI;KACR,CAAC;IAEF,OAAO,UAAU,CAAC;AACpB,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,OAAO,CAAC,CAAS;IAC/B,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IACf,OAAO,CAAC,CAAC;AACX,CAAC;AAED,SAAS,MAAM,CAAC,KAAiB;IAC/B,OAAO,GAAG,KAAK,CAAC,WAAW,IAAI,KAAK,CAAC,SAAS,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;AACnE,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,eAAe,CAAC,MAAiD;IAC/E,IAAI,MAAM,IAAI,IAAI,EAAE,CAAC;QACnB,OAAO,EAAE,CAAC;IACZ,CAAC;IACD,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,KAAK,IAAI,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;QAChD,IAAI,IAAI,EAAE,GAAG,GAAG,CAAC,MAAM,CAAC;QACxB,+CAA+C;QAC/C,IAAI,KAAK,IAAI,IAAI,EAAE,CAAC;YAClB,IAAI,IAAI,CAAC,CAAC;QACZ,CAAC;aAAM,IAAI,OAAO,KAAK,IAAI,QAAQ,EAAE,CAAC;YACpC,IAAI,IAAI,CAAC,CAAC;QACZ,CAAC;aAAM,IAAI,OAAO,KAAK,IAAI,QAAQ,EAAE,CAAC;YACpC,IAAI,IAAI,CAAC,CAAC;QACZ,CAAC;aAAM,IAAI,OAAO,KAAK,IAAI,QAAQ,EAAE,CAAC;YACpC,IAAI,IAAI,KAAK,CAAC,MAAM,CAAC;QACvB,CAAC;aAAM,IAAI,KAAK,YAAY,UAAU,EAAE,CAAC;YACvC,IAAI,IAAI,KAAK,CAAC,UAAU,CAAC;QAC3B,CAAC;IACH,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC"}
|
package/package.json
CHANGED
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
"publishConfig": {
|
|
6
6
|
"access": "public"
|
|
7
7
|
},
|
|
8
|
-
"version": "0.0.0-dev-
|
|
8
|
+
"version": "0.0.0-dev-20250827072023",
|
|
9
9
|
"main": "dist/index.js",
|
|
10
10
|
"license": "FSL-1.1-ALv2",
|
|
11
11
|
"type": "module",
|
|
@@ -33,11 +33,11 @@
|
|
|
33
33
|
"uuid": "^11.1.0",
|
|
34
34
|
"winston": "^3.13.0",
|
|
35
35
|
"yaml": "^2.3.2",
|
|
36
|
-
"@powersync/lib-services-framework": "0.0.0-dev-
|
|
37
|
-
"@powersync/service-jsonbig": "0.0.0-dev-
|
|
38
|
-
"@powersync/service-rsocket-router": "0.0.0-dev-
|
|
39
|
-
"@powersync/service-sync-rules": "0.0.0-dev-
|
|
40
|
-
"@powersync/service-types": "0.0.0-dev-
|
|
36
|
+
"@powersync/lib-services-framework": "0.0.0-dev-20250827072023",
|
|
37
|
+
"@powersync/service-jsonbig": "0.0.0-dev-20250827072023",
|
|
38
|
+
"@powersync/service-rsocket-router": "0.0.0-dev-20250827072023",
|
|
39
|
+
"@powersync/service-sync-rules": "0.0.0-dev-20250827072023",
|
|
40
|
+
"@powersync/service-types": "0.0.0-dev-20250827072023"
|
|
41
41
|
},
|
|
42
42
|
"devDependencies": {
|
|
43
43
|
"@types/async": "^3.2.24",
|
|
@@ -12,6 +12,10 @@ export class EventsEngine {
|
|
|
12
12
|
});
|
|
13
13
|
}
|
|
14
14
|
|
|
15
|
+
/**
|
|
16
|
+
* All new events added need to be subscribed to be used.
|
|
17
|
+
* @example engine.subscribe(new MyNewEvent(storageEngine));
|
|
18
|
+
*/
|
|
15
19
|
subscribe<K extends event_types.EventsEngineEventType>(event: event_types.EmitterEvent<K>): void {
|
|
16
20
|
if (!this.events.has(event.event)) {
|
|
17
21
|
this.events.add(event.event);
|
|
@@ -23,10 +27,6 @@ export class EventsEngine {
|
|
|
23
27
|
return Array.from(this.events.values());
|
|
24
28
|
}
|
|
25
29
|
|
|
26
|
-
countListeners(eventName: event_types.EventsEngineEventType): number {
|
|
27
|
-
return this.emitter.listenerCount(eventName);
|
|
28
|
-
}
|
|
29
|
-
|
|
30
30
|
emit<K extends keyof event_types.SubscribeEvents>(event: K, data: event_types.SubscribeEvents[K]): void {
|
|
31
31
|
this.emitter.emit(event, data);
|
|
32
32
|
}
|
|
@@ -23,10 +23,11 @@ export const syncStreamReactive: SocketRouteGenerator = (router) =>
|
|
|
23
23
|
};
|
|
24
24
|
|
|
25
25
|
const sdkData: event_types.ConnectedUserData & event_types.ClientConnectionEventData = {
|
|
26
|
-
client_id: params.client_id,
|
|
26
|
+
client_id: params.client_id ?? '',
|
|
27
27
|
user_id: context.user_id!,
|
|
28
28
|
user_agent: context.user_agent,
|
|
29
|
-
|
|
29
|
+
// At this point the token_payload is guaranteed to be present
|
|
30
|
+
jwt_exp: new Date(context.token_payload!.exp * 1000),
|
|
30
31
|
connected_at: new Date(streamStart)
|
|
31
32
|
};
|
|
32
33
|
|
|
@@ -98,8 +99,7 @@ export const syncStreamReactive: SocketRouteGenerator = (router) =>
|
|
|
98
99
|
bucketStorage: bucketStorage,
|
|
99
100
|
syncRules: syncRules,
|
|
100
101
|
params: {
|
|
101
|
-
...params
|
|
102
|
-
binary_data: true // always true for web sockets
|
|
102
|
+
...params
|
|
103
103
|
},
|
|
104
104
|
token: context!.token_payload!,
|
|
105
105
|
tokenStreamOptions: {
|
|
@@ -108,7 +108,8 @@ export const syncStreamReactive: SocketRouteGenerator = (router) =>
|
|
|
108
108
|
},
|
|
109
109
|
tracker,
|
|
110
110
|
signal,
|
|
111
|
-
logger
|
|
111
|
+
logger,
|
|
112
|
+
isEncodingAsBson: true
|
|
112
113
|
})) {
|
|
113
114
|
if (signal.aborted) {
|
|
114
115
|
break;
|
|
@@ -30,9 +30,9 @@ export const syncStreamed = routeDefinition({
|
|
|
30
30
|
const clientId = payload.params.client_id;
|
|
31
31
|
const streamStart = Date.now();
|
|
32
32
|
// This falls back to JSON unless there's preference for the bson-stream in the Accept header.
|
|
33
|
-
const useBson =
|
|
34
|
-
payload.request.
|
|
35
|
-
|
|
33
|
+
const useBson = payload.request.headers.accept
|
|
34
|
+
? new Negotiator(payload.request).mediaType(supportedContentTypes) == concatenatedBsonContentType
|
|
35
|
+
: false;
|
|
36
36
|
|
|
37
37
|
logger.defaultMeta = {
|
|
38
38
|
...logger.defaultMeta,
|
|
@@ -42,10 +42,11 @@ export const syncStreamed = routeDefinition({
|
|
|
42
42
|
bson: useBson
|
|
43
43
|
};
|
|
44
44
|
const sdkData: event_types.ConnectedUserData & event_types.ClientConnectionEventData = {
|
|
45
|
-
client_id: clientId,
|
|
45
|
+
client_id: clientId ?? '',
|
|
46
46
|
user_id: payload.context.user_id!,
|
|
47
47
|
user_agent: userAgent as string,
|
|
48
|
-
|
|
48
|
+
// At this point the token_payload is guaranteed to be present
|
|
49
|
+
jwt_exp: new Date(token_payload!.exp * 1000),
|
|
49
50
|
connected_at: new Date(streamStart)
|
|
50
51
|
};
|
|
51
52
|
|
|
@@ -82,7 +83,8 @@ export const syncStreamed = routeDefinition({
|
|
|
82
83
|
token: payload.context.token_payload!,
|
|
83
84
|
tracker,
|
|
84
85
|
signal: controller.signal,
|
|
85
|
-
logger
|
|
86
|
+
logger,
|
|
87
|
+
isEncodingAsBson: useBson
|
|
86
88
|
});
|
|
87
89
|
|
|
88
90
|
const byteContents = useBson ? sync.bsonLines(syncLines) : sync.ndjson(syncLines);
|
package/src/routes/router.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { Logger, router } from '@powersync/lib-services-framework';
|
|
2
2
|
import type { JwtPayload } from '../auth/auth-index.js';
|
|
3
3
|
import { ServiceContext } from '../system/ServiceContext.js';
|
|
4
4
|
import { RouterEngine } from './RouterEngine.js';
|
|
@@ -31,11 +31,11 @@ export type BasicRouterRequest = {
|
|
|
31
31
|
hostname: string;
|
|
32
32
|
};
|
|
33
33
|
|
|
34
|
-
export type
|
|
34
|
+
export type ContextProviderOptions = {
|
|
35
35
|
logger: Logger;
|
|
36
36
|
};
|
|
37
37
|
|
|
38
|
-
export type ContextProvider = (request: BasicRouterRequest, options:
|
|
38
|
+
export type ContextProvider = (request: BasicRouterRequest, options: ContextProviderOptions) => Promise<Context>;
|
|
39
39
|
|
|
40
40
|
export type RequestEndpoint<
|
|
41
41
|
I,
|
|
@@ -1,5 +1,11 @@
|
|
|
1
1
|
import { ObserverClient } from '@powersync/lib-services-framework';
|
|
2
|
-
import {
|
|
2
|
+
import {
|
|
3
|
+
EvaluatedParameters,
|
|
4
|
+
EvaluatedRow,
|
|
5
|
+
SqliteInputRow,
|
|
6
|
+
SqliteRow,
|
|
7
|
+
ToastableSqliteRow
|
|
8
|
+
} from '@powersync/service-sync-rules';
|
|
3
9
|
import { BSON } from 'bson';
|
|
4
10
|
import { ReplicationEventPayload } from './ReplicationEventPayload.js';
|
|
5
11
|
import { SourceTable, TableSnapshotStatus } from './SourceTable.js';
|
|
@@ -132,7 +138,7 @@ export interface SaveInsert {
|
|
|
132
138
|
sourceTable: SourceTable;
|
|
133
139
|
before?: undefined;
|
|
134
140
|
beforeReplicaId?: undefined;
|
|
135
|
-
after:
|
|
141
|
+
after: SqliteInputRow;
|
|
136
142
|
afterReplicaId: ReplicaId;
|
|
137
143
|
}
|
|
138
144
|
|
|
@@ -143,7 +149,7 @@ export interface SaveUpdate {
|
|
|
143
149
|
/**
|
|
144
150
|
* This is only present when the id has changed, and will only contain replica identity columns.
|
|
145
151
|
*/
|
|
146
|
-
before?:
|
|
152
|
+
before?: SqliteInputRow;
|
|
147
153
|
beforeReplicaId?: ReplicaId;
|
|
148
154
|
|
|
149
155
|
/**
|
|
@@ -158,7 +164,7 @@ export interface SaveUpdate {
|
|
|
158
164
|
export interface SaveDelete {
|
|
159
165
|
tag: SaveOperationTag.DELETE;
|
|
160
166
|
sourceTable: SourceTable;
|
|
161
|
-
before?:
|
|
167
|
+
before?: SqliteInputRow;
|
|
162
168
|
beforeReplicaId: ReplicaId;
|
|
163
169
|
after?: undefined;
|
|
164
170
|
afterReplicaId?: undefined;
|
|
@@ -3,6 +3,7 @@ import { ParseSyncRulesOptions, PersistedSyncRules, PersistedSyncRulesContent }
|
|
|
3
3
|
import { ReplicationEventPayload } from './ReplicationEventPayload.js';
|
|
4
4
|
import { ReplicationLock } from './ReplicationLock.js';
|
|
5
5
|
import { SyncRulesBucketStorage } from './SyncRulesBucketStorage.js';
|
|
6
|
+
import { ReportStorage } from './ReportStorage.js';
|
|
6
7
|
|
|
7
8
|
/**
|
|
8
9
|
* Represents a configured storage provider.
|
|
@@ -164,3 +165,4 @@ export interface TestStorageOptions {
|
|
|
164
165
|
doNotClear?: boolean;
|
|
165
166
|
}
|
|
166
167
|
export type TestStorageFactory = (options?: TestStorageOptions) => Promise<BucketStorageFactory>;
|
|
168
|
+
export type TestReportStorageFactory = (options?: TestStorageOptions) => Promise<ReportStorage>;
|
|
@@ -4,8 +4,8 @@ import { BucketStorageBatch, SaveOp } from './BucketStorageBatch.js';
|
|
|
4
4
|
|
|
5
5
|
export type EventData = {
|
|
6
6
|
op: SaveOp;
|
|
7
|
-
before?: sync_rules.
|
|
8
|
-
after?: sync_rules.
|
|
7
|
+
before?: sync_rules.SqliteInputRow;
|
|
8
|
+
after?: sync_rules.SqliteInputRow;
|
|
9
9
|
};
|
|
10
10
|
|
|
11
11
|
export type ReplicationEventPayload = {
|
|
@@ -1,11 +1,39 @@
|
|
|
1
1
|
import { event_types } from '@powersync/service-types';
|
|
2
2
|
|
|
3
|
+
/**
|
|
4
|
+
* Represents a configured report storage.
|
|
5
|
+
*
|
|
6
|
+
* Report storage is used for storing localized data for the instance.
|
|
7
|
+
* Data can then be used for reporting purposes.
|
|
8
|
+
*
|
|
9
|
+
*/
|
|
3
10
|
export interface ReportStorage extends AsyncDisposable {
|
|
11
|
+
/**
|
|
12
|
+
* Report a client connection.
|
|
13
|
+
*/
|
|
4
14
|
reportClientConnection(data: event_types.ClientConnectionBucketData): Promise<void>;
|
|
15
|
+
/**
|
|
16
|
+
* Report a client disconnection.
|
|
17
|
+
*/
|
|
5
18
|
reportClientDisconnection(data: event_types.ClientDisconnectionEventData): Promise<void>;
|
|
6
|
-
|
|
19
|
+
/**
|
|
20
|
+
* Get currently connected clients.
|
|
21
|
+
* This will return any short or long term connected clients.
|
|
22
|
+
* Clients that have no disconnected_at timestamp and that have a valid jwt_exp timestamp are considered connected.
|
|
23
|
+
*/
|
|
24
|
+
getConnectedClients(): Promise<event_types.ClientConnectionReportResponse>;
|
|
25
|
+
/**
|
|
26
|
+
* Get a report of client connections over a day, week or month.
|
|
27
|
+
* This is internally used to generate reports over it always returns the previous day, week or month.
|
|
28
|
+
* Usually this is call on the start of the new day, week or month. It will return all unique completed connections
|
|
29
|
+
* as well as uniques currently connected clients.
|
|
30
|
+
*/
|
|
7
31
|
getClientConnectionReports(
|
|
8
32
|
data: event_types.ClientConnectionReportRequest
|
|
9
|
-
): Promise<event_types.
|
|
33
|
+
): Promise<event_types.ClientConnectionReportResponse>;
|
|
34
|
+
/**
|
|
35
|
+
* Delete old connection data based on a specific date.
|
|
36
|
+
* This is used to clean up old connection data that is no longer needed.
|
|
37
|
+
*/
|
|
10
38
|
deleteOldConnectionData(data: event_types.DeleteOldConnectionData): Promise<void>;
|
|
11
39
|
}
|
package/src/sync/sync.ts
CHANGED
|
@@ -27,6 +27,7 @@ export interface SyncStreamParameters {
|
|
|
27
27
|
params: util.StreamingSyncRequest;
|
|
28
28
|
token: auth.JwtPayload;
|
|
29
29
|
logger?: Logger;
|
|
30
|
+
isEncodingAsBson: boolean;
|
|
30
31
|
/**
|
|
31
32
|
* If this signal is aborted, the stream response ends as soon as possible, without error.
|
|
32
33
|
*/
|
|
@@ -39,7 +40,17 @@ export interface SyncStreamParameters {
|
|
|
39
40
|
export async function* streamResponse(
|
|
40
41
|
options: SyncStreamParameters
|
|
41
42
|
): AsyncIterable<util.StreamingSyncLine | string | null> {
|
|
42
|
-
const {
|
|
43
|
+
const {
|
|
44
|
+
syncContext,
|
|
45
|
+
bucketStorage,
|
|
46
|
+
syncRules,
|
|
47
|
+
params,
|
|
48
|
+
token,
|
|
49
|
+
tokenStreamOptions,
|
|
50
|
+
tracker,
|
|
51
|
+
signal,
|
|
52
|
+
isEncodingAsBson
|
|
53
|
+
} = options;
|
|
43
54
|
const logger = options.logger ?? defaultLogger;
|
|
44
55
|
|
|
45
56
|
// We also need to be able to abort, so we create our own controller.
|
|
@@ -65,7 +76,8 @@ export async function* streamResponse(
|
|
|
65
76
|
token,
|
|
66
77
|
tracker,
|
|
67
78
|
controller.signal,
|
|
68
|
-
logger
|
|
79
|
+
logger,
|
|
80
|
+
isEncodingAsBson
|
|
69
81
|
);
|
|
70
82
|
// Merge the two streams, and abort as soon as one of the streams end.
|
|
71
83
|
const merged = mergeAsyncIterables([stream, ki], controller.signal);
|
|
@@ -93,9 +105,10 @@ async function* streamResponseInner(
|
|
|
93
105
|
tokenPayload: RequestJwtPayload,
|
|
94
106
|
tracker: RequestTracker,
|
|
95
107
|
signal: AbortSignal,
|
|
96
|
-
logger: Logger
|
|
108
|
+
logger: Logger,
|
|
109
|
+
isEncodingAsBson: boolean
|
|
97
110
|
): AsyncGenerator<util.StreamingSyncLine | string | null> {
|
|
98
|
-
const { raw_data
|
|
111
|
+
const { raw_data } = params;
|
|
99
112
|
|
|
100
113
|
const userId = tokenPayload.sub;
|
|
101
114
|
const checkpointUserId = util.checkpointUserId(userId as string, params.client_id);
|
|
@@ -225,8 +238,7 @@ async function* streamResponseInner(
|
|
|
225
238
|
checkpoint: next.value.value.checkpoint,
|
|
226
239
|
bucketsToFetch: buckets,
|
|
227
240
|
checkpointLine: line,
|
|
228
|
-
raw_data,
|
|
229
|
-
binary_data,
|
|
241
|
+
legacyDataLines: !isEncodingAsBson && params.raw_data != true,
|
|
230
242
|
onRowsSent: markOperationsSent,
|
|
231
243
|
abort_connection: signal,
|
|
232
244
|
abort_batch: abortCheckpointSignal,
|
|
@@ -255,8 +267,8 @@ interface BucketDataRequest {
|
|
|
255
267
|
checkpointLine: CheckpointLine;
|
|
256
268
|
/** Subset of checkpointLine.bucketsToFetch, filtered by priority. */
|
|
257
269
|
bucketsToFetch: BucketDescription[];
|
|
258
|
-
|
|
259
|
-
|
|
270
|
+
/** Whether data lines should be encoded in a legacy format where {@link util.OplogEntry.data} is a nested object. */
|
|
271
|
+
legacyDataLines: boolean;
|
|
260
272
|
/** Signals that the connection was aborted and that streaming should stop ASAP. */
|
|
261
273
|
abort_connection: AbortSignal;
|
|
262
274
|
/**
|
|
@@ -317,8 +329,7 @@ async function* bucketDataBatch(request: BucketDataRequest): AsyncGenerator<Buck
|
|
|
317
329
|
checkpoint,
|
|
318
330
|
bucketsToFetch,
|
|
319
331
|
checkpointLine,
|
|
320
|
-
|
|
321
|
-
binary_data,
|
|
332
|
+
legacyDataLines,
|
|
322
333
|
abort_connection,
|
|
323
334
|
abort_batch,
|
|
324
335
|
onRowsSent,
|
|
@@ -368,32 +379,21 @@ async function* bucketDataBatch(request: BucketDataRequest): AsyncGenerator<Buck
|
|
|
368
379
|
}
|
|
369
380
|
logger.debug(`Sending data for ${r.bucket}`);
|
|
370
381
|
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
const response: util.StreamingSyncData = {
|
|
387
|
-
data: transformLegacyResponse(r)
|
|
388
|
-
};
|
|
389
|
-
send_data = JSONBig.stringify(response);
|
|
390
|
-
}
|
|
391
|
-
yield { data: send_data, done: false };
|
|
392
|
-
if (send_data.length > 50_000) {
|
|
393
|
-
// IMPORTANT: This does not affect the output stream, but is used to flush
|
|
394
|
-
// iterator memory in case if large data sent.
|
|
395
|
-
yield { data: null, done: false };
|
|
396
|
-
}
|
|
382
|
+
const line = legacyDataLines
|
|
383
|
+
? // We need to preserve the embedded data exactly, so this uses a JsonContainer
|
|
384
|
+
// and JSONBig to stringify.
|
|
385
|
+
JSONBig.stringify({
|
|
386
|
+
data: transformLegacyResponse(r)
|
|
387
|
+
} satisfies util.StreamingSyncData)
|
|
388
|
+
: // We can send the object as-is, which will be converted to JSON or BSON by a downstream transformer.
|
|
389
|
+
({ data: r } satisfies util.StreamingSyncData);
|
|
390
|
+
|
|
391
|
+
yield { data: line, done: false };
|
|
392
|
+
|
|
393
|
+
// IMPORTANT: This does not affect the output stream, but is used to flush
|
|
394
|
+
// iterator memory in case if large data sent.
|
|
395
|
+
yield { data: null, done: false };
|
|
396
|
+
|
|
397
397
|
onRowsSent(statsForBatch(r));
|
|
398
398
|
|
|
399
399
|
checkpointLine.updateBucketPosition({ bucket: r.bucket, nextAfter: BigInt(r.next_after), hasMore: r.has_more });
|
|
@@ -76,11 +76,6 @@ export const StreamingSyncRequest = t.object({
|
|
|
76
76
|
*/
|
|
77
77
|
raw_data: t.boolean.optional(),
|
|
78
78
|
|
|
79
|
-
/**
|
|
80
|
-
* Data is received in a serialized BSON Buffer
|
|
81
|
-
*/
|
|
82
|
-
binary_data: t.boolean.optional(),
|
|
83
|
-
|
|
84
79
|
/**
|
|
85
80
|
* Client parameters to be passed to the sync rules.
|
|
86
81
|
*/
|
package/src/util/utils.ts
CHANGED
|
@@ -148,7 +148,10 @@ export function hasToastedValues(row: sync_rules.ToastableSqliteRow) {
|
|
|
148
148
|
*
|
|
149
149
|
* If we don't store data, we assume we always have a complete row.
|
|
150
150
|
*/
|
|
151
|
-
export function isCompleteRow(
|
|
151
|
+
export function isCompleteRow(
|
|
152
|
+
storeData: boolean,
|
|
153
|
+
row: sync_rules.ToastableSqliteRow
|
|
154
|
+
): row is sync_rules.SqliteInputRow {
|
|
152
155
|
if (!storeData) {
|
|
153
156
|
// Assume the row is complete - no need to check
|
|
154
157
|
return true;
|