@powersync/service-core 0.4.1 → 0.4.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +10 -0
- package/dist/routes/endpoints/socket-route.js +9 -1
- package/dist/routes/endpoints/socket-route.js.map +1 -1
- package/dist/routes/endpoints/sync-stream.js +9 -1
- package/dist/routes/endpoints/sync-stream.js.map +1 -1
- package/dist/sync/RequestTracker.d.ts +9 -0
- package/dist/sync/RequestTracker.js +19 -0
- package/dist/sync/RequestTracker.js.map +1 -0
- package/dist/sync/sync.d.ts +2 -0
- package/dist/sync/sync.js +31 -11
- package/dist/sync/sync.js.map +1 -1
- package/dist/sync/util.d.ts +2 -1
- package/dist/sync/util.js +2 -3
- package/dist/sync/util.js.map +1 -1
- package/package.json +4 -4
- package/src/routes/endpoints/socket-route.ts +9 -1
- package/src/routes/endpoints/sync-stream.ts +10 -1
- package/src/sync/RequestTracker.ts +21 -0
- package/src/sync/sync.ts +41 -10
- package/src/sync/util.ts +6 -2
- package/test/src/sync.test.ts +8 -0
- package/tsconfig.tsbuildinfo +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,15 @@
|
|
|
1
1
|
# @powersync/service-core
|
|
2
2
|
|
|
3
|
+
## 0.4.2
|
|
4
|
+
|
|
5
|
+
### Patch Changes
|
|
6
|
+
|
|
7
|
+
- bdbf95c: Log user_id and sync stats for each connection
|
|
8
|
+
- Updated dependencies [876f4a0]
|
|
9
|
+
- Updated dependencies [9bff878]
|
|
10
|
+
- @powersync/service-sync-rules@0.18.1
|
|
11
|
+
- @powersync/service-rsocket-router@0.0.10
|
|
12
|
+
|
|
3
13
|
## 0.4.1
|
|
4
14
|
|
|
5
15
|
### Patch Changes
|
|
@@ -5,6 +5,7 @@ import { Metrics } from '../../metrics/Metrics.js';
|
|
|
5
5
|
import { streamResponse } from '../../sync/sync.js';
|
|
6
6
|
import * as util from '../../util/util-index.js';
|
|
7
7
|
import { SyncRoutes } from './sync-stream.js';
|
|
8
|
+
import { RequestTracker } from '../../sync/RequestTracker.js';
|
|
8
9
|
export const syncStreamReactive = (router) => router.reactiveStream(SyncRoutes.STREAM, {
|
|
9
10
|
authorize: ({ context }) => {
|
|
10
11
|
return {
|
|
@@ -51,6 +52,7 @@ export const syncStreamReactive = (router) => router.reactiveStream(SyncRoutes.S
|
|
|
51
52
|
observer.triggerCancel();
|
|
52
53
|
});
|
|
53
54
|
Metrics.getInstance().concurrent_connections.add(1);
|
|
55
|
+
const tracker = new RequestTracker();
|
|
54
56
|
try {
|
|
55
57
|
for await (const data of streamResponse({
|
|
56
58
|
storage,
|
|
@@ -64,6 +66,7 @@ export const syncStreamReactive = (router) => router.reactiveStream(SyncRoutes.S
|
|
|
64
66
|
// RSocket handles keepalive events by default
|
|
65
67
|
keep_alive: false
|
|
66
68
|
},
|
|
69
|
+
tracker,
|
|
67
70
|
signal: controller.signal
|
|
68
71
|
})) {
|
|
69
72
|
if (data == null) {
|
|
@@ -79,7 +82,7 @@ export const syncStreamReactive = (router) => router.reactiveStream(SyncRoutes.S
|
|
|
79
82
|
const serialized = serialize(data);
|
|
80
83
|
responder.onNext({ data: serialized }, false);
|
|
81
84
|
requestedN--;
|
|
82
|
-
|
|
85
|
+
tracker.addDataSynced(serialized.length);
|
|
83
86
|
}
|
|
84
87
|
if (requestedN <= 0) {
|
|
85
88
|
await new Promise((resolve) => {
|
|
@@ -112,6 +115,11 @@ export const syncStreamReactive = (router) => router.reactiveStream(SyncRoutes.S
|
|
|
112
115
|
responder.onComplete();
|
|
113
116
|
removeStopHandler();
|
|
114
117
|
disposer();
|
|
118
|
+
logger.info(`Sync stream complete`, {
|
|
119
|
+
user_id: syncParams.user_id,
|
|
120
|
+
operations_synced: tracker.operationsSynced,
|
|
121
|
+
data_synced_bytes: tracker.dataSyncedBytes
|
|
122
|
+
});
|
|
115
123
|
Metrics.getInstance().concurrent_connections.add(-1);
|
|
116
124
|
}
|
|
117
125
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"socket-route.js","sourceRoot":"","sources":["../../../src/routes/endpoints/socket-route.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,mCAAmC,CAAC;AAC3E,OAAO,EAAE,iBAAiB,EAAE,MAAM,+BAA+B,CAAC;AAClE,OAAO,EAAE,SAAS,EAAE,MAAM,MAAM,CAAC;AAEjC,OAAO,EAAE,OAAO,EAAE,MAAM,0BAA0B,CAAC;AACnD,OAAO,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AACpD,OAAO,KAAK,IAAI,MAAM,0BAA0B,CAAC;AAEjD,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;
|
|
1
|
+
{"version":3,"file":"socket-route.js","sourceRoot":"","sources":["../../../src/routes/endpoints/socket-route.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,mCAAmC,CAAC;AAC3E,OAAO,EAAE,iBAAiB,EAAE,MAAM,+BAA+B,CAAC;AAClE,OAAO,EAAE,SAAS,EAAE,MAAM,MAAM,CAAC;AAEjC,OAAO,EAAE,OAAO,EAAE,MAAM,0BAA0B,CAAC;AACnD,OAAO,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AACpD,OAAO,KAAK,IAAI,MAAM,0BAA0B,CAAC;AAEjD,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAC9C,OAAO,EAAE,cAAc,EAAE,MAAM,8BAA8B,CAAC;AAE9D,MAAM,CAAC,MAAM,kBAAkB,GAAyB,CAAC,MAAM,EAAE,EAAE,CACjE,MAAM,CAAC,cAAc,CAAiC,UAAU,CAAC,MAAM,EAAE;IACvE,SAAS,EAAE,CAAC,EAAE,OAAO,EAAE,EAAE,EAAE;QACzB,OAAO;YACL,UAAU,EAAE,CAAC,CAAC,OAAO,CAAC,aAAa;YACnC,MAAM,EAAE,CAAC,yBAAyB,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,YAAY,IAAI,EAAE,CAAC;SACvE,CAAC;IACJ,CAAC;IACD,SAAS,EAAE,MAAM,CAAC,sBAAsB,CAAC,IAAI,CAAC,oBAAoB,EAAE,EAAE,eAAe,EAAE,IAAI,EAAE,CAAC;IAC9F,OAAO,EAAE,KAAK,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,QAAQ,EAAE,QAAQ,EAAE,EAAE,EAAE;QACpE,MAAM,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC;QAE3B,IAAI,MAAM,CAAC,MAAM,EAAE;YACjB,SAAS,CAAC,OAAO,CACf,IAAI,MAAM,CAAC,YAAY,CAAC;gBACtB,MAAM,EAAE,GAAG;gBACX,IAAI,EAAE,qBAAqB;gBAC3B,WAAW,EAAE,iCAAiC;aAC/C,CAAC,CACH,CAAC;YACF,SAAS,CAAC,UAAU,EAAE,CAAC;YACvB,OAAO;SACR;QAED,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;QAEzC,MAAM,UAAU,GAAG,IAAI,iBAAiB,CAAC,OAAO,CAAC,aAAc,EAAE,MAAM,CAAC,UAAU,IAAI,EAAE,CAAC,CAAC;QAE1F,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC;QAC/B,0CAA0C;QAC1C,MAAM,EAAE,GAAG,MAAM,OAAO,CAAC,mBAAmB,EAAE,CAAC;QAC/C,IAAI,CAAC,EAAE,CAAC,YAAY,EAAE,EAAE;YACtB,SAAS,CAAC,OAAO,CACf,IAAI,MAAM,CAAC,YAAY,CAAC;gBACtB,MAAM,EAAE,GAAG;gBACX,IAAI,EAAE,eAAe;gBACrB,WAAW,EAAE,yBAAyB;aACvC,CAAC,CACH,CAAC;YACF,SAAS,CAAC,UAAU,EAAE,CAAC;YACvB,OAAO;SACR;QAED,IAAI,UAAU,GAAG,QAAQ,CAAC;QAC1B,MAAM,QAAQ,GAAG,QAAQ,CAAC,gBAAgB,CAAC;YACzC,OAAO,CAAC,CAAC;gBACP,UAAU,IAAI,CAAC,CAAC;YAClB,CAAC;YACD,MAAM,EAAE,GAAG,EAAE;gBACX,UAAU,CAAC,KAAK,EAAE,CAAC;YACrB,CAAC;SACF,CAAC,CAAC;QAEH,MAAM,iBAAiB,GAAG,MAAM,CAAC,cAAc,CAAC,GAAG,EAAE;YACnD,QAAQ,CAAC,aAAa,EAAE,CAAC;QAC3B,CAAC,CAAC,CAAC;QAEH,OAAO,CAAC,WAAW,EAAE,CAAC,sBAAsB,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;QACpD,MAAM,OAAO,GAAG,IAAI,cAAc,EAAE,CAAC;QACrC,IAAI;YACF,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,cAAc,CAAC;gBACtC,OAAO;gBACP,MAAM,EAAE;oBACN,GAAG,MAAM;oBACT,WAAW,EAAE,IAAI,CAAC,8BAA8B;iBACjD;gBACD,UAAU;gBACV,KAAK,EAAE,OAAQ,CAAC,aAAc;gBAC9B,kBAAkB,EAAE;oBAClB,8CAA8C;oBAC9C,UAAU,EAAE,KAAK;iBAClB;gBACD,OAAO;gBACP,MAAM,EAAE,UAAU,CAAC,MAAM;aAC1B,CAAC,EAAE;gBACF,IAAI,IAAI,IAAI,IAAI,EAAE;oBAChB,4CAA4C;oBAC5C,SAAS;iBACV;qBAAM,IAAI,OAAO,IAAI,IAAI,QAAQ,EAAE;oBAClC,2CAA2C;oBAC3C,MAAM,IAAI,KAAK,CAAC,2BAA2B,IAAI,EAAE,CAAC,CAAC;iBACpD;gBAED;oBACE,+CAA+C;oBAC/C,MAAM,UAAU,GAAG,SAAS,CAAC,IAAI,CAAW,CAAC;oBAC7C,SAAS,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,UAAU,EAAE,EAAE,KAAK,CAAC,CAAC;oBAC9C,UAAU,EAAE,CAAC;oBACb,OAAO,CAAC,aAAa,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;iBAC1C;gBAED,IAAI,UAAU,IAAI,CAAC,EAAE;oBACnB,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;wBAClC,MAAM,CAAC,GAAG,QAAQ,CAAC,gBAAgB,CAAC;4BAClC,OAAO;gCACL,IAAI,UAAU,GAAG,CAAC,EAAE;oCAClB,iEAAiE;oCACjE,OAAO,EAAE,CAAC;oCACV,CAAC,EAAE,CAAC;iCACL;4BACH,CAAC;4BACD,MAAM,EAAE,GAAG,EAAE;gCACX,8CAA8C;gCAC9C,OAAO,EAAE,CAAC;gCACV,CAAC,EAAE,CAAC;4BACN,CAAC;yBACF,CAAC,CAAC;oBACL,CAAC,CAAC,CAAC;iBACJ;aACF;SACF;QAAC,OAAO,EAAE,EAAE;YACX,kDAAkD;YAClD,4CAA4C;YAC5C,MAAM,KAAK,GAAG,IAAI,MAAM,CAAC,mBAAmB,CAAC,EAAE,CAAC,CAAC;YACjD,MAAM,CAAC,KAAK,CAAC,mBAAmB,EAAE,KAAK,CAAC,CAAC;YACzC,SAAS,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;SAC1B;gBAAS;YACR,SAAS,CAAC,UAAU,EAAE,CAAC;YACvB,iBAAiB,EAAE,CAAC;YACpB,QAAQ,EAAE,CAAC;YACX,MAAM,CAAC,IAAI,CAAC,sBAAsB,EAAE;gBAClC,OAAO,EAAE,UAAU,CAAC,OAAO;gBAC3B,iBAAiB,EAAE,OAAO,CAAC,gBAAgB;gBAC3C,iBAAiB,EAAE,OAAO,CAAC,eAAe;aAC3C,CAAC,CAAC;YACH,OAAO,CAAC,WAAW,EAAE,CAAC,sBAAsB,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;SACtD;IACH,CAAC;CACF,CAAC,CAAC"}
|
|
@@ -6,6 +6,7 @@ import * as util from '../../util/util-index.js';
|
|
|
6
6
|
import { Metrics } from '../../metrics/Metrics.js';
|
|
7
7
|
import { authUser } from '../auth.js';
|
|
8
8
|
import { routeDefinition } from '../router.js';
|
|
9
|
+
import { RequestTracker } from '../../sync/RequestTracker.js';
|
|
9
10
|
export var SyncRoutes;
|
|
10
11
|
(function (SyncRoutes) {
|
|
11
12
|
SyncRoutes["STREAM"] = "/sync/stream";
|
|
@@ -37,6 +38,7 @@ export const syncStreamed = routeDefinition({
|
|
|
37
38
|
});
|
|
38
39
|
}
|
|
39
40
|
const controller = new AbortController();
|
|
41
|
+
const tracker = new RequestTracker();
|
|
40
42
|
try {
|
|
41
43
|
Metrics.getInstance().concurrent_connections.add(1);
|
|
42
44
|
const stream = Readable.from(sync.transformToBytesTracked(sync.ndjson(sync.streamResponse({
|
|
@@ -44,8 +46,9 @@ export const syncStreamed = routeDefinition({
|
|
|
44
46
|
params,
|
|
45
47
|
syncParams,
|
|
46
48
|
token: payload.context.token_payload,
|
|
49
|
+
tracker,
|
|
47
50
|
signal: controller.signal
|
|
48
|
-
}))), { objectMode: false, highWaterMark: 16 * 1024 });
|
|
51
|
+
})), tracker), { objectMode: false, highWaterMark: 16 * 1024 });
|
|
49
52
|
const deregister = system.addStopHandler(() => {
|
|
50
53
|
// This error is not currently propagated to the client
|
|
51
54
|
controller.abort();
|
|
@@ -70,6 +73,11 @@ export const syncStreamed = routeDefinition({
|
|
|
70
73
|
afterSend: async () => {
|
|
71
74
|
controller.abort();
|
|
72
75
|
Metrics.getInstance().concurrent_connections.add(-1);
|
|
76
|
+
logger.info(`Sync stream complete`, {
|
|
77
|
+
user_id: syncParams.user_id,
|
|
78
|
+
operations_synced: tracker.operationsSynced,
|
|
79
|
+
data_synced_bytes: tracker.dataSyncedBytes
|
|
80
|
+
});
|
|
73
81
|
}
|
|
74
82
|
});
|
|
75
83
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"sync-stream.js","sourceRoot":"","sources":["../../../src/routes/endpoints/sync-stream.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,mCAAmC,CAAC;AACnF,OAAO,EAAE,iBAAiB,EAAE,MAAM,+BAA+B,CAAC;AAClE,OAAO,EAAE,QAAQ,EAAE,MAAM,QAAQ,CAAC;AAElC,OAAO,KAAK,IAAI,MAAM,0BAA0B,CAAC;AACjD,OAAO,KAAK,IAAI,MAAM,0BAA0B,CAAC;AAEjD,OAAO,EAAE,OAAO,EAAE,MAAM,0BAA0B,CAAC;AACnD,OAAO,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AACtC,OAAO,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;
|
|
1
|
+
{"version":3,"file":"sync-stream.js","sourceRoot":"","sources":["../../../src/routes/endpoints/sync-stream.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,mCAAmC,CAAC;AACnF,OAAO,EAAE,iBAAiB,EAAE,MAAM,+BAA+B,CAAC;AAClE,OAAO,EAAE,QAAQ,EAAE,MAAM,QAAQ,CAAC;AAElC,OAAO,KAAK,IAAI,MAAM,0BAA0B,CAAC;AACjD,OAAO,KAAK,IAAI,MAAM,0BAA0B,CAAC;AAEjD,OAAO,EAAE,OAAO,EAAE,MAAM,0BAA0B,CAAC;AACnD,OAAO,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AACtC,OAAO,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AAC/C,OAAO,EAAE,cAAc,EAAE,MAAM,8BAA8B,CAAC;AAE9D,MAAM,CAAN,IAAY,UAEX;AAFD,WAAY,UAAU;IACpB,qCAAuB,CAAA;AACzB,CAAC,EAFW,UAAU,KAAV,UAAU,QAErB;AAED,MAAM,CAAC,MAAM,YAAY,GAAG,eAAe,CAAC;IAC1C,IAAI,EAAE,UAAU,CAAC,MAAM;IACvB,MAAM,EAAE,MAAM,CAAC,UAAU,CAAC,IAAI;IAC9B,SAAS,EAAE,QAAQ;IACnB,SAAS,EAAE,MAAM,CAAC,sBAAsB,CAAC,IAAI,CAAC,oBAAoB,EAAE,EAAE,eAAe,EAAE,IAAI,EAAE,CAAC;IAC9F,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE;QACzB,MAAM,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC;QAEtC,IAAI,MAAM,CAAC,MAAM,EAAE;YACjB,MAAM,IAAI,MAAM,CAAC,YAAY,CAAC;gBAC5B,MAAM,EAAE,GAAG;gBACX,IAAI,EAAE,qBAAqB;gBAC3B,WAAW,EAAE,iCAAiC;aAC/C,CAAC,CAAC;SACJ;QAED,MAAM,MAAM,GAA8B,OAAO,CAAC,MAAM,CAAC;QACzD,MAAM,UAAU,GAAG,IAAI,iBAAiB,CAAC,OAAO,CAAC,OAAO,CAAC,aAAc,EAAE,OAAO,CAAC,MAAM,CAAC,UAAU,IAAI,EAAE,CAAC,CAAC;QAE1G,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC;QAC/B,0CAA0C;QAC1C,MAAM,EAAE,GAAG,MAAM,OAAO,CAAC,mBAAmB,EAAE,CAAC;QAC/C,IAAI,CAAC,EAAE,CAAC,YAAY,EAAE,EAAE;YACtB,MAAM,IAAI,MAAM,CAAC,YAAY,CAAC;gBAC5B,MAAM,EAAE,GAAG;gBACX,IAAI,EAAE,eAAe;gBACrB,WAAW,EAAE,yBAAyB;aACvC,CAAC,CAAC;SACJ;QACD,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;QACzC,MAAM,OAAO,GAAG,IAAI,cAAc,EAAE,CAAC;QACrC,IAAI;YACF,OAAO,CAAC,WAAW,EAAE,CAAC,sBAAsB,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;YACpD,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,CAC1B,IAAI,CAAC,uBAAuB,CAC1B,IAAI,CAAC,MAAM,CACT,IAAI,CAAC,cAAc,CAAC;gBAClB,OAAO;gBACP,MAAM;gBACN,UAAU;gBACV,KAAK,EAAE,OAAO,CAAC,OAAO,CAAC,aAAc;gBACrC,OAAO;gBACP,MAAM,EAAE,UAAU,CAAC,MAAM;aAC1B,CAAC,CACH,EACD,OAAO,CACR,EACD,EAAE,UAAU,EAAE,KAAK,EAAE,aAAa,EAAE,EAAE,GAAG,IAAI,EAAE,CAChD,CAAC;YAEF,MAAM,UAAU,GAAG,MAAM,CAAC,cAAc,CAAC,GAAG,EAAE;gBAC5C,uDAAuD;gBACvD,UAAU,CAAC,KAAK,EAAE,CAAC;gBACnB,MAAM,CAAC,OAAO,CAAC,IAAI,KAAK,CAAC,sBAAsB,CAAC,CAAC,CAAC;YACpD,CAAC,CAAC,CAAC;YACH,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE;gBACtB,UAAU,EAAE,CAAC;YACf,CAAC,CAAC,CAAC;YAEH,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,EAAE;gBAC3B,UAAU,CAAC,KAAK,EAAE,CAAC;gBACnB,oDAAoD;gBACpD,IAAI,KAAK,CAAC,OAAO,IAAI,sBAAsB,EAAE;oBAC3C,MAAM,CAAC,KAAK,CAAC,+BAA+B,EAAE,KAAK,CAAC,CAAC;iBACtD;YACH,CAAC,CAAC,CAAC;YAEH,OAAO,IAAI,MAAM,CAAC,cAAc,CAAC;gBAC/B,MAAM,EAAE,GAAG;gBACX,OAAO,EAAE;oBACP,cAAc,EAAE,sBAAsB;iBACvC;gBACD,IAAI,EAAE,MAAM;gBACZ,SAAS,EAAE,KAAK,IAAI,EAAE;oBACpB,UAAU,CAAC,KAAK,EAAE,CAAC;oBACnB,OAAO,CAAC,WAAW,EAAE,CAAC,sBAAsB,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;oBACrD,MAAM,CAAC,IAAI,CAAC,sBAAsB,EAAE;wBAClC,OAAO,EAAE,UAAU,CAAC,OAAO;wBAC3B,iBAAiB,EAAE,OAAO,CAAC,gBAAgB;wBAC3C,iBAAiB,EAAE,OAAO,CAAC,eAAe;qBAC3C,CAAC,CAAC;gBACL,CAAC;aACF,CAAC,CAAC;SACJ;QAAC,OAAO,EAAE,EAAE;YACX,UAAU,CAAC,KAAK,EAAE,CAAC;YACnB,OAAO,CAAC,WAAW,EAAE,CAAC,sBAAsB,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;SACtD;IACH,CAAC;CACF,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,kBAAkB,GAAG,CAAC,YAAY,CAAC,CAAC"}
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
import { Metrics } from '../metrics/Metrics.js';
|
|
2
|
+
/**
|
|
3
|
+
* Record sync stats per request stream.
|
|
4
|
+
*/
|
|
5
|
+
export class RequestTracker {
|
|
6
|
+
constructor() {
|
|
7
|
+
this.operationsSynced = 0;
|
|
8
|
+
this.dataSyncedBytes = 0;
|
|
9
|
+
}
|
|
10
|
+
addOperationsSynced(operations) {
|
|
11
|
+
this.operationsSynced += operations;
|
|
12
|
+
Metrics.getInstance().operations_synced_total.add(operations);
|
|
13
|
+
}
|
|
14
|
+
addDataSynced(bytes) {
|
|
15
|
+
this.dataSyncedBytes += bytes;
|
|
16
|
+
Metrics.getInstance().data_synced_bytes.add(bytes);
|
|
17
|
+
}
|
|
18
|
+
}
|
|
19
|
+
//# sourceMappingURL=RequestTracker.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"RequestTracker.js","sourceRoot":"","sources":["../../src/sync/RequestTracker.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,uBAAuB,CAAC;AAEhD;;GAEG;AACH,MAAM,OAAO,cAAc;IAA3B;QACE,qBAAgB,GAAG,CAAC,CAAC;QACrB,oBAAe,GAAG,CAAC,CAAC;IAatB,CAAC;IAXC,mBAAmB,CAAC,UAAkB;QACpC,IAAI,CAAC,gBAAgB,IAAI,UAAU,CAAC;QAEpC,OAAO,CAAC,WAAW,EAAE,CAAC,uBAAuB,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;IAChE,CAAC;IAED,aAAa,CAAC,KAAa;QACzB,IAAI,CAAC,eAAe,IAAI,KAAK,CAAC;QAE9B,OAAO,CAAC,WAAW,EAAE,CAAC,iBAAiB,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;IACrD,CAAC;CACF"}
|
package/dist/sync/sync.d.ts
CHANGED
|
@@ -4,6 +4,7 @@ import * as auth from '../auth/auth-index.js';
|
|
|
4
4
|
import * as storage from '../storage/storage-index.js';
|
|
5
5
|
import * as util from '../util/util-index.js';
|
|
6
6
|
import { TokenStreamOptions } from './util.js';
|
|
7
|
+
import { RequestTracker } from './RequestTracker.js';
|
|
7
8
|
export interface SyncStreamParameters {
|
|
8
9
|
storage: storage.BucketStorageFactory;
|
|
9
10
|
params: util.StreamingSyncRequest;
|
|
@@ -14,5 +15,6 @@ export interface SyncStreamParameters {
|
|
|
14
15
|
*/
|
|
15
16
|
signal?: AbortSignal;
|
|
16
17
|
tokenStreamOptions?: Partial<TokenStreamOptions>;
|
|
18
|
+
tracker: RequestTracker;
|
|
17
19
|
}
|
|
18
20
|
export declare function streamResponse(options: SyncStreamParameters): AsyncIterable<util.StreamingSyncLine | string | null>;
|
package/dist/sync/sync.js
CHANGED
|
@@ -3,7 +3,6 @@ import { Semaphore } from 'async-mutex';
|
|
|
3
3
|
import { AbortError } from 'ix/aborterror.js';
|
|
4
4
|
import * as util from '../util/util-index.js';
|
|
5
5
|
import { logger } from '@powersync/lib-services-framework';
|
|
6
|
-
import { Metrics } from '../metrics/Metrics.js';
|
|
7
6
|
import { mergeAsyncIterables } from './merge.js';
|
|
8
7
|
import { tokenStream } from './util.js';
|
|
9
8
|
/**
|
|
@@ -12,7 +11,7 @@ import { tokenStream } from './util.js';
|
|
|
12
11
|
const MAX_ACTIVE_CONNECTIONS = 10;
|
|
13
12
|
const syncSemaphore = new Semaphore(MAX_ACTIVE_CONNECTIONS);
|
|
14
13
|
export async function* streamResponse(options) {
|
|
15
|
-
const { storage, params, syncParams, token, tokenStreamOptions, signal } = options;
|
|
14
|
+
const { storage, params, syncParams, token, tokenStreamOptions, tracker, signal } = options;
|
|
16
15
|
// We also need to be able to abort, so we create our own controller.
|
|
17
16
|
const controller = new AbortController();
|
|
18
17
|
if (signal) {
|
|
@@ -24,7 +23,7 @@ export async function* streamResponse(options) {
|
|
|
24
23
|
}
|
|
25
24
|
}
|
|
26
25
|
const ki = tokenStream(token, controller.signal, tokenStreamOptions);
|
|
27
|
-
const stream = streamResponseInner(storage, params, syncParams, controller.signal);
|
|
26
|
+
const stream = streamResponseInner(storage, params, syncParams, tracker, controller.signal);
|
|
28
27
|
// Merge the two streams, and abort as soon as one of the streams end.
|
|
29
28
|
const merged = mergeAsyncIterables([stream, ki], controller.signal);
|
|
30
29
|
try {
|
|
@@ -44,7 +43,7 @@ export async function* streamResponse(options) {
|
|
|
44
43
|
controller.abort();
|
|
45
44
|
}
|
|
46
45
|
}
|
|
47
|
-
async function* streamResponseInner(storage, params, syncParams, signal) {
|
|
46
|
+
async function* streamResponseInner(storage, params, syncParams, tracker, signal) {
|
|
48
47
|
// Bucket state of bucket id -> op_id.
|
|
49
48
|
// This starts with the state from the client. May contain buckets that the user do not have access to (anymore).
|
|
50
49
|
let dataBuckets = new Map();
|
|
@@ -73,6 +72,11 @@ async function* streamResponseInner(storage, params, syncParams, signal) {
|
|
|
73
72
|
parameters: syncParams
|
|
74
73
|
});
|
|
75
74
|
if (allBuckets.length > 1000) {
|
|
75
|
+
logger.error(`Too many buckets`, {
|
|
76
|
+
checkpoint,
|
|
77
|
+
user_id: syncParams.user_id,
|
|
78
|
+
buckets: allBuckets.length
|
|
79
|
+
});
|
|
76
80
|
// TODO: Limit number of buckets even before we get to this point
|
|
77
81
|
throw new Error(`Too many buckets: ${allBuckets.length}`);
|
|
78
82
|
}
|
|
@@ -94,11 +98,18 @@ async function* streamResponseInner(storage, params, syncParams, signal) {
|
|
|
94
98
|
continue;
|
|
95
99
|
}
|
|
96
100
|
bucketsToFetch = diff.updatedBuckets.map((c) => c.bucket);
|
|
97
|
-
let message = `Updated checkpoint: ${checkpoint} |
|
|
101
|
+
let message = `Updated checkpoint: ${checkpoint} | `;
|
|
102
|
+
message += `write: ${writeCheckpoint} | `;
|
|
98
103
|
message += `buckets: ${allBuckets.length} | `;
|
|
99
104
|
message += `updated: ${limitedBuckets(diff.updatedBuckets, 20)} | `;
|
|
100
|
-
message += `removed: ${limitedBuckets(diff.removedBuckets, 20)}
|
|
101
|
-
logger.info(message
|
|
105
|
+
message += `removed: ${limitedBuckets(diff.removedBuckets, 20)}`;
|
|
106
|
+
logger.info(message, {
|
|
107
|
+
checkpoint,
|
|
108
|
+
user_id: syncParams.user_id,
|
|
109
|
+
buckets: allBuckets.length,
|
|
110
|
+
updated: diff.updatedBuckets.length,
|
|
111
|
+
removed: diff.removedBuckets.length
|
|
112
|
+
});
|
|
102
113
|
const checksum_line = {
|
|
103
114
|
checkpoint_diff: {
|
|
104
115
|
last_op_id: checkpoint,
|
|
@@ -112,7 +123,7 @@ async function* streamResponseInner(storage, params, syncParams, signal) {
|
|
|
112
123
|
else {
|
|
113
124
|
let message = `New checkpoint: ${checkpoint} | write: ${writeCheckpoint} | `;
|
|
114
125
|
message += `buckets: ${allBuckets.length} ${limitedBuckets(allBuckets, 20)}`;
|
|
115
|
-
logger.info(message);
|
|
126
|
+
logger.info(message, { checkpoint, user_id: syncParams.user_id, buckets: allBuckets.length });
|
|
116
127
|
bucketsToFetch = allBuckets;
|
|
117
128
|
const checksum_line = {
|
|
118
129
|
checkpoint: {
|
|
@@ -127,7 +138,16 @@ async function* streamResponseInner(storage, params, syncParams, signal) {
|
|
|
127
138
|
lastWriteCheckpoint = writeCheckpoint;
|
|
128
139
|
// This incrementally updates dataBuckets with each individual bucket position.
|
|
129
140
|
// At the end of this, we can be sure that all buckets have data up to the checkpoint.
|
|
130
|
-
yield* bucketDataInBatches({
|
|
141
|
+
yield* bucketDataInBatches({
|
|
142
|
+
storage,
|
|
143
|
+
checkpoint,
|
|
144
|
+
bucketsToFetch,
|
|
145
|
+
dataBuckets,
|
|
146
|
+
raw_data,
|
|
147
|
+
binary_data,
|
|
148
|
+
signal,
|
|
149
|
+
tracker
|
|
150
|
+
});
|
|
131
151
|
await new Promise((resolve) => setTimeout(resolve, 10));
|
|
132
152
|
}
|
|
133
153
|
}
|
|
@@ -168,7 +188,7 @@ async function* bucketDataInBatches(request) {
|
|
|
168
188
|
* Extracted as a separate internal function just to avoid memory leaks.
|
|
169
189
|
*/
|
|
170
190
|
async function* bucketDataBatch(request) {
|
|
171
|
-
const { storage, checkpoint, bucketsToFetch, dataBuckets, raw_data, binary_data, signal } = request;
|
|
191
|
+
const { storage, checkpoint, bucketsToFetch, dataBuckets, raw_data, binary_data, tracker, signal } = request;
|
|
172
192
|
const [_, release] = await syncSemaphore.acquire();
|
|
173
193
|
try {
|
|
174
194
|
// Optimization: Only fetch buckets for which the checksums have changed since the last checkpoint
|
|
@@ -213,7 +233,7 @@ async function* bucketDataBatch(request) {
|
|
|
213
233
|
// iterator memory in case if large data sent.
|
|
214
234
|
yield { data: null, done: false };
|
|
215
235
|
}
|
|
216
|
-
|
|
236
|
+
tracker.addOperationsSynced(r.data.length);
|
|
217
237
|
dataBuckets.set(r.bucket, r.next_after);
|
|
218
238
|
}
|
|
219
239
|
if (!has_more) {
|
package/dist/sync/sync.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"sync.js","sourceRoot":"","sources":["../../src/sync/sync.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAC;AAEpE,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AACxC,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAI9C,OAAO,KAAK,IAAI,MAAM,uBAAuB,CAAC;AAE9C,OAAO,EAAE,MAAM,EAAE,MAAM,mCAAmC,CAAC;
|
|
1
|
+
{"version":3,"file":"sync.js","sourceRoot":"","sources":["../../src/sync/sync.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAC;AAEpE,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AACxC,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAI9C,OAAO,KAAK,IAAI,MAAM,uBAAuB,CAAC;AAE9C,OAAO,EAAE,MAAM,EAAE,MAAM,mCAAmC,CAAC;AAE3D,OAAO,EAAE,mBAAmB,EAAE,MAAM,YAAY,CAAC;AACjD,OAAO,EAAsB,WAAW,EAAE,MAAM,WAAW,CAAC;AAG5D;;GAEG;AACH,MAAM,sBAAsB,GAAG,EAAE,CAAC;AAClC,MAAM,aAAa,GAAG,IAAI,SAAS,CAAC,sBAAsB,CAAC,CAAC;AAgB5D,MAAM,CAAC,KAAK,SAAS,CAAC,CAAC,cAAc,CACnC,OAA6B;IAE7B,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,KAAK,EAAE,kBAAkB,EAAE,OAAO,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC;IAC5F,qEAAqE;IACrE,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;IACzC,IAAI,MAAM,EAAE;QACV,MAAM,CAAC,gBAAgB,CACrB,OAAO,EACP,GAAG,EAAE;YACH,UAAU,CAAC,KAAK,EAAE,CAAC;QACrB,CAAC,EACD,EAAE,IAAI,EAAE,IAAI,EAAE,CACf,CAAC;QACF,IAAI,MAAM,CAAC,OAAO,EAAE;YAClB,UAAU,CAAC,KAAK,EAAE,CAAC;SACpB;KACF;IACD,MAAM,EAAE,GAAG,WAAW,CAAC,KAAK,EAAE,UAAU,CAAC,MAAM,EAAE,kBAAkB,CAAC,CAAC;IACrE,MAAM,MAAM,GAAG,mBAAmB,CAAC,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,OAAO,EAAE,UAAU,CAAC,MAAM,CAAC,CAAC;IAC5F,sEAAsE;IACtE,MAAM,MAAM,GAAG,mBAAmB,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,EAAE,UAAU,CAAC,MAAM,CAAC,CAAC;IAEpE,IAAI;QACF,KAAK,CAAC,CAAC,MAAM,CAAC;KACf;IAAC,OAAO,CAAC,EAAE;QACV,IAAI,CAAC,YAAY,UAAU,EAAE;YAC3B,OAAO;SACR;aAAM;YACL,MAAM,CAAC,CAAC;SACT;KACF;YAAS;QACR,iFAAiF;QACjF,qBAAqB;QACrB,UAAU,CAAC,KAAK,EAAE,CAAC;KACpB;AACH,CAAC;AAED,KAAK,SAAS,CAAC,CAAC,mBAAmB,CACjC,OAAqC,EACrC,MAAiC,EACjC,UAA6B,EAC7B,OAAuB,EACvB,MAAmB;IAEnB,sCAAsC;IACtC,iHAAiH;IACjH,IAAI,WAAW,GAAG,IAAI,GAAG,EAAkB,CAAC;IAE5C,IAAI,aAAa,GAA4B,IAAI,CAAC;IAClD,IAAI,mBAAmB,GAAkB,IAAI,CAAC;IAE9C,MAAM,EAAE,QAAQ,EAAE,WAAW,EAAE,GAAG,MAAM,CAAC;IAEzC,IAAI,MAAM,CAAC,OAAO,EAAE;QAClB,KAAK,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,IAAI,MAAM,CAAC,OAAO,EAAE;YACjD,WAAW,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;SAC9B;KACF;IAED,MAAM,MAAM,GAAG,OAAO,CAAC,oBAAoB,CAAC,UAAU,CAAC,gBAAgB,CAAC,OAAiB,EAAE,MAAM,CAAC,CAAC;IACnG,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,MAAM,EAAE;QAC/B,MAAM,EAAE,IAAI,EAAE,eAAe,EAAE,GAAG,IAAI,CAAC;QACvC,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC;QAEnC,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,gBAAgB,EAAE,CAAC;QAC9C,IAAI,OAAO,IAAI,IAAI,EAAE;YACnB,2EAA2E;YAC3E,SAAS;SACV;QACD,MAAM,UAAU,GAAG,OAAO,CAAC,UAAU,CAAC;QAEtC,MAAM,UAAU,GAAG,MAAM,UAAU,CAAC,cAAc,CAAC;YACjD,gBAAgB,CAAC,OAAO;gBACtB,OAAO,OAAO,CAAC,gBAAgB,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;YACvD,CAAC;YACD,UAAU,EAAE,UAAU;SACvB,CAAC,CAAC;QAEH,IAAI,UAAU,CAAC,MAAM,GAAG,IAAI,EAAE;YAC5B,MAAM,CAAC,KAAK,CAAC,kBAAkB,EAAE;gBAC/B,UAAU;gBACV,OAAO,EAAE,UAAU,CAAC,OAAO;gBAC3B,OAAO,EAAE,UAAU,CAAC,MAAM;aAC3B,CAAC,CAAC;YACH,iEAAiE;YACjE,MAAM,IAAI,KAAK,CAAC,qBAAqB,UAAU,CAAC,MAAM,EAAE,CAAC,CAAC;SAC3D;QAED,IAAI,cAAc,GAAG,IAAI,GAAG,EAAkB,CAAC;QAC/C,KAAK,IAAI,MAAM,IAAI,UAAU,EAAE;YAC7B,cAAc,CAAC,GAAG,CAAC,MAAM,EAAE,WAAW,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,GAAG,CAAC,CAAC;SAC5D;QACD,WAAW,GAAG,cAAc,CAAC;QAE7B,MAAM,UAAU,GAAG,CAAC,GAAG,WAAW,CAAC,IAAI,EAAE,CAAC,CAAC;QAC3C,MAAM,WAAW,GAAG,MAAM,OAAO,CAAC,YAAY,CAAC,UAAU,EAAE,UAAU,CAAC,CAAC;QACvE,mEAAmE;QACnE,IAAI,cAAwB,CAAC;QAE7B,IAAI,aAAa,EAAE;YACjB,MAAM,IAAI,GAAG,IAAI,CAAC,aAAa,CAAC,aAAa,EAAE,WAAW,CAAC,CAAC;YAE5D,IACE,mBAAmB,IAAI,eAAe;gBACtC,IAAI,CAAC,cAAc,CAAC,MAAM,IAAI,CAAC;gBAC/B,IAAI,CAAC,cAAc,CAAC,MAAM,IAAI,CAAC,EAC/B;gBACA,iDAAiD;gBACjD,SAAS;aACV;YACD,cAAc,GAAG,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;YAE1D,IAAI,OAAO,GAAG,uBAAuB,UAAU,KAAK,CAAC;YACrD,OAAO,IAAI,UAAU,eAAe,KAAK,CAAC;YAC1C,OAAO,IAAI,YAAY,UAAU,CAAC,MAAM,KAAK,CAAC;YAC9C,OAAO,IAAI,YAAY,cAAc,CAAC,IAAI,CAAC,cAAc,EAAE,EAAE,CAAC,KAAK,CAAC;YACpE,OAAO,IAAI,YAAY,cAAc,CAAC,IAAI,CAAC,cAAc,EAAE,EAAE,CAAC,EAAE,CAAC;YACjE,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE;gBACnB,UAAU;gBACV,OAAO,EAAE,UAAU,CAAC,OAAO;gBAC3B,OAAO,EAAE,UAAU,CAAC,MAAM;gBAC1B,OAAO,EAAE,IAAI,CAAC,cAAc,CAAC,MAAM;gBACnC,OAAO,EAAE,IAAI,CAAC,cAAc,CAAC,MAAM;aACpC,CAAC,CAAC;YAEH,MAAM,aAAa,GAAqC;gBACtD,eAAe,EAAE;oBACf,UAAU,EAAE,UAAU;oBACtB,gBAAgB,EAAE,eAAe,CAAC,CAAC,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC,SAAS;oBACvE,eAAe,EAAE,IAAI,CAAC,cAAc;oBACpC,eAAe,EAAE,IAAI,CAAC,cAAc;iBACrC;aACF,CAAC;YAEF,MAAM,aAAa,CAAC;SACrB;aAAM;YACL,IAAI,OAAO,GAAG,mBAAmB,UAAU,aAAa,eAAe,KAAK,CAAC;YAC7E,OAAO,IAAI,YAAY,UAAU,CAAC,MAAM,IAAI,cAAc,CAAC,UAAU,EAAE,EAAE,CAAC,EAAE,CAAC;YAC7E,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,UAAU,EAAE,OAAO,EAAE,UAAU,CAAC,OAAO,EAAE,OAAO,EAAE,UAAU,CAAC,MAAM,EAAE,CAAC,CAAC;YAC9F,cAAc,GAAG,UAAU,CAAC;YAC5B,MAAM,aAAa,GAAiC;gBAClD,UAAU,EAAE;oBACV,UAAU,EAAE,UAAU;oBACtB,gBAAgB,EAAE,eAAe,CAAC,CAAC,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC,SAAS;oBACvE,OAAO,EAAE,CAAC,GAAG,WAAW,CAAC,MAAM,EAAE,CAAC;iBACnC;aACF,CAAC;YACF,MAAM,aAAa,CAAC;SACrB;QACD,aAAa,GAAG,WAAW,CAAC;QAC5B,mBAAmB,GAAG,eAAe,CAAC;QAEtC,+EAA+E;QAC/E,sFAAsF;QACtF,KAAK,CAAC,CAAC,mBAAmB,CAAC;YACzB,OAAO;YACP,UAAU;YACV,cAAc;YACd,WAAW;YACX,QAAQ;YACR,WAAW;YACX,MAAM;YACN,OAAO;SACR,CAAC,CAAC;QAEH,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;KACzD;AACH,CAAC;AAcD,KAAK,SAAS,CAAC,CAAC,mBAAmB,CAAC,OAA0B;IAC5D,IAAI,MAAM,GAAG,KAAK,CAAC;IACnB,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,IAAI,CAAC,MAAM,EAAE;QACzC,wEAAwE;QACxE,6EAA6E;QAC7E,4GAA4G;QAC5G,gBAAgB;QAChB,gBAAgB;QAChB,qBAAqB;QACrB,MAAM;QACN,WAAW;QACX,IAAI;QACJ,MAAM,IAAI,GAAG,eAAe,CAAC,OAAO,CAAC,CAAC;QACtC,IAAI;YACF,OAAO,IAAI,EAAE;gBACX,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;gBACpD,IAAI,QAAQ,EAAE;oBACZ,MAAM;iBACP;qBAAM;oBACL,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,KAAK,CAAC;oBAC7B,MAAM,IAAI,CAAC;oBACX,IAAI,IAAI,EAAE;wBACR,MAAM,GAAG,IAAI,CAAC;qBACf;iBACF;aACF;SACF;gBAAS;YACR,MAAM,IAAI,CAAC,MAAM,EAAE,CAAC;SACrB;KACF;AACH,CAAC;AAOD;;GAEG;AACH,KAAK,SAAS,CAAC,CAAC,eAAe,CAAC,OAA0B;IACxD,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE,cAAc,EAAE,WAAW,EAAE,QAAQ,EAAE,WAAW,EAAE,OAAO,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC;IAE7G,MAAM,CAAC,CAAC,EAAE,OAAO,CAAC,GAAG,MAAM,aAAa,CAAC,OAAO,EAAE,CAAC;IACnD,IAAI;QACF,kGAAkG;QAClG,iDAAiD;QACjD,MAAM,eAAe,GAAG,IAAI,GAAG,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,WAAW,CAAC,GAAG,CAAC,MAAM,CAAE,CAAC,CAAC,CAAC,CAAC;QACpG,MAAM,IAAI,GAAG,OAAO,CAAC,kBAAkB,CAAC,UAAU,EAAE,eAAe,CAAC,CAAC;QAErE,IAAI,QAAQ,GAAG,KAAK,CAAC;QAErB,IAAI,KAAK,EAAE,IAAI,CAAC,IAAI,IAAI,EAAE;YACxB,IAAI,MAAM,CAAC,OAAO,EAAE;gBAClB,OAAO;aACR;YACD,IAAI,CAAC,CAAC,QAAQ,EAAE;gBACd,QAAQ,GAAG,IAAI,CAAC;aACjB;YACD,IAAI,CAAC,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,EAAE;gBACtB,SAAS;aACV;YACD,MAAM,CAAC,KAAK,CAAC,oBAAoB,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC;YAE7C,IAAI,SAAc,CAAC;YACnB,IAAI,WAAW,EAAE;gBACf,wEAAwE;gBACxE,SAAS,GAAG,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC;aACzB;iBAAM,IAAI,QAAQ,EAAE;gBACnB,uEAAuE;gBACvE,MAAM,QAAQ,GAA2B;oBACvC,IAAI,EAAE,CAAC;iBACR,CAAC;gBACF,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;aACtC;iBAAM;gBACL,8EAA8E;gBAC9E,4BAA4B;gBAC5B,MAAM,QAAQ,GAA2B;oBACvC,IAAI,EAAE,uBAAuB,CAAC,CAAC,CAAC;iBACjC,CAAC;gBACF,SAAS,GAAG,OAAO,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;aACzC;YACD,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC;YACvC,IAAI,SAAS,CAAC,MAAM,GAAG,KAAM,EAAE;gBAC7B,0EAA0E;gBAC1E,8CAA8C;gBAC9C,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC;aACnC;YACD,OAAO,CAAC,mBAAmB,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAE3C,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,UAAU,CAAC,CAAC;SACzC;QAED,IAAI,CAAC,QAAQ,EAAE;YACb,MAAM,IAAI,GAAyC;gBACjD,mBAAmB,EAAE;oBACnB,UAAU,EAAE,UAAU;iBACvB;aACF,CAAC;YACF,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;SAClC;KACF;YAAS;QACR,OAAO,EAAE,CAAC;KACX;AACH,CAAC;AAED,SAAS,uBAAuB,CAAC,UAA+B;IAC9D,OAAO;QACL,GAAG,UAAU;QACb,IAAI,EAAE,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE;YAClC,OAAO;gBACL,GAAG,KAAK;gBACR,IAAI,EAAE,KAAK,CAAC,IAAI,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,aAAa,CAAC,KAAK,CAAC,IAAc,CAAC;gBACzE,QAAQ,EAAE,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC;aACjC,CAAC;QACJ,CAAC,CAAC;KACH,CAAC;AACJ,CAAC;AAED,SAAS,cAAc,CAAC,OAAyC,EAAE,KAAa;IAC9E,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;QAC1B,IAAI,OAAO,CAAC,IAAI,QAAQ,EAAE;YACxB,OAAO,CAAC,CAAC,MAAM,CAAC;SACjB;aAAM;YACL,OAAO,CAAC,CAAC;SACV;IACH,CAAC,CAAC,CAAC;IACH,IAAI,OAAO,CAAC,MAAM,IAAI,KAAK,EAAE;QAC3B,OAAO,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;KAChC;IACD,MAAM,OAAO,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;IACxC,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,KAAK,CAAC;AACzC,CAAC"}
|
package/dist/sync/util.d.ts
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
/// <reference types="node" resolution-mode="require"/>
|
|
2
2
|
/// <reference types="node" resolution-mode="require"/>
|
|
3
3
|
import * as util from '../util/util-index.js';
|
|
4
|
+
import { RequestTracker } from './RequestTracker.js';
|
|
4
5
|
export type TokenStreamOptions = {
|
|
5
6
|
/**
|
|
6
7
|
* Adds periodic keepalive events
|
|
@@ -23,4 +24,4 @@ export declare function tokenStream(token: {
|
|
|
23
24
|
exp: number;
|
|
24
25
|
}, signal: AbortSignal, options?: Partial<TokenStreamOptions>): AsyncGenerator<util.StreamingSyncKeepalive>;
|
|
25
26
|
export declare function ndjson(iterator: AsyncIterable<string | null | Record<string, any>>): AsyncGenerator<string>;
|
|
26
|
-
export declare function transformToBytesTracked(iterator: AsyncIterable<string
|
|
27
|
+
export declare function transformToBytesTracked(iterator: AsyncIterable<string>, tracker: RequestTracker): AsyncGenerator<Buffer>;
|
package/dist/sync/util.js
CHANGED
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
import * as timers from 'timers/promises';
|
|
2
|
-
import { Metrics } from '../metrics/Metrics.js';
|
|
3
2
|
const KEEPALIVE_INTERVAL = 20000;
|
|
4
3
|
const DEFAULT_TOKEN_STREAM_OPTIONS = {
|
|
5
4
|
keep_alive: true,
|
|
@@ -63,10 +62,10 @@ export async function* ndjson(iterator) {
|
|
|
63
62
|
}
|
|
64
63
|
}
|
|
65
64
|
}
|
|
66
|
-
export async function* transformToBytesTracked(iterator) {
|
|
65
|
+
export async function* transformToBytesTracked(iterator, tracker) {
|
|
67
66
|
for await (let data of iterator) {
|
|
68
67
|
const encoded = Buffer.from(data, 'utf8');
|
|
69
|
-
|
|
68
|
+
tracker.addDataSynced(encoded.length);
|
|
70
69
|
yield encoded;
|
|
71
70
|
}
|
|
72
71
|
}
|
package/dist/sync/util.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"util.js","sourceRoot":"","sources":["../../src/sync/util.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,MAAM,iBAAiB,CAAC;
|
|
1
|
+
{"version":3,"file":"util.js","sourceRoot":"","sources":["../../src/sync/util.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,MAAM,iBAAiB,CAAC;AAiB1C,MAAM,kBAAkB,GAAG,KAAM,CAAC;AAElC,MAAM,4BAA4B,GAAuB;IACvD,UAAU,EAAE,IAAI;IAChB,qBAAqB,EAAE,KAAM;CAC9B,CAAC;AAEF;;;;;;;GAOG;AACH,MAAM,CAAC,KAAK,SAAS,CAAC,CAAC,WAAW,CAChC,KAAsB,EACtB,MAAmB,EACnB,OAAqC;IAErC,MAAM,gBAAgB,GAAuB;QAC3C,GAAG,4BAA4B;QAC/B,GAAG,CAAC,OAAO,IAAI,EAAE,CAAC;KACnB,CAAC;IAEF,MAAM,EAAE,UAAU,EAAE,qBAAqB,EAAE,GAAG,gBAAgB,CAAC;IAE/D,8CAA8C;IAC9C,+DAA+D;IAC/D,+BAA+B;IAC/B,MAAM,UAAU,GAAG,KAAK,CAAC,GAAG,GAAG,IAAI,CAAC;IACpC,MAAM,iBAAiB,GAAG,UAAU,GAAG,qBAAqB,CAAC;IAE7D,IAAI,iBAAiB,GAAG,IAAI,CAAC;IAE7B,OAAO,CAAC,MAAM,CAAC,OAAO,EAAE;QACtB,MAAM,gBAAgB,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,CAAC,CAAC;QAC/E,IAAI,iBAAiB,IAAI,gBAAgB,GAAG,CAAC,EAAE;YAC7C,iBAAiB,GAAG,KAAK,CAAC;SAC3B;aAAM;YACL,MAAM,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,CAAC;YAC7C,IAAI,gBAAgB,IAAI,CAAC,EAAE;gBACzB,OAAO;aACR;SACF;QAED,MAAM,gBAAgB,GAAG,kBAAkB,GAAG,CAAC,GAAG,GAAG,IAAI,CAAC,MAAM,EAAE,GAAG,GAAG,CAAC,CAAC;QAE1E,4CAA4C;QAC5C,MAAM,YAAY,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC;QAC9D,MAAM,oBAAoB,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,iBAAiB,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC;QAC7E,6CAA6C;QAC7C,MAAM,qBAAqB,GAAG,oBAAoB,IAAI,CAAC,CAAC,CAAC,CAAC,oBAAoB,CAAC,CAAC,CAAC,YAAY,CAAC;QAE9F,MAAM,KAAK,GAAG,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,qBAAqB,EAAE,gBAAgB,CAAC,CAAC,CAAC,CAAC,qBAAqB,CAAC;QACrG,MAAM,MAAM,CAAC,UAAU,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE;YAC1D,oBAAoB;QACtB,CAAC,CAAC,CAAC;KACJ;AACH,CAAC;AAED,MAAM,CAAC,KAAK,SAAS,CAAC,CAAC,MAAM,CAAC,QAA4D;IACxF,IAAI,KAAK,EAAE,IAAI,IAAI,IAAI,QAAQ,EAAE;QAC/B,IAAI,IAAI,IAAI,IAAI,EAAE;YAChB,uCAAuC;YACvC,SAAS;SACV;aAAM,IAAI,OAAO,IAAI,IAAI,QAAQ,EAAE;YAClC,uBAAuB;YACvB,MAAM,IAAI,GAAG,IAAI,CAAC;SACnB;aAAM;YACL,MAAM,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC;SACnC;KACF;AACH,CAAC;AAED,MAAM,CAAC,KAAK,SAAS,CAAC,CAAC,uBAAuB,CAC5C,QAA+B,EAC/B,OAAuB;IAEvB,IAAI,KAAK,EAAE,IAAI,IAAI,IAAI,QAAQ,EAAE;QAC/B,MAAM,OAAO,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QAC1C,OAAO,CAAC,aAAa,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QACtC,MAAM,OAAO,CAAC;KACf;AACH,CAAC"}
|
package/package.json
CHANGED
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
"publishConfig": {
|
|
6
6
|
"access": "public"
|
|
7
7
|
},
|
|
8
|
-
"version": "0.4.
|
|
8
|
+
"version": "0.4.2",
|
|
9
9
|
"main": "dist/index.js",
|
|
10
10
|
"license": "FSL-1.1-Apache-2.0",
|
|
11
11
|
"type": "module",
|
|
@@ -33,11 +33,11 @@
|
|
|
33
33
|
"uuid": "^9.0.1",
|
|
34
34
|
"winston": "^3.13.0",
|
|
35
35
|
"yaml": "^2.3.2",
|
|
36
|
+
"@powersync/service-jsonbig": "0.17.10",
|
|
36
37
|
"@powersync/service-jpgwire": "0.17.13",
|
|
37
38
|
"@powersync/lib-services-framework": "0.1.1",
|
|
38
|
-
"@powersync/service-
|
|
39
|
-
"@powersync/service-rsocket-router": "0.0.
|
|
40
|
-
"@powersync/service-sync-rules": "0.18.0",
|
|
39
|
+
"@powersync/service-sync-rules": "0.18.1",
|
|
40
|
+
"@powersync/service-rsocket-router": "0.0.10",
|
|
41
41
|
"@powersync/service-types": "0.1.0"
|
|
42
42
|
},
|
|
43
43
|
"devDependencies": {
|
|
@@ -7,6 +7,7 @@ import { streamResponse } from '../../sync/sync.js';
|
|
|
7
7
|
import * as util from '../../util/util-index.js';
|
|
8
8
|
import { SocketRouteGenerator } from '../router-socket.js';
|
|
9
9
|
import { SyncRoutes } from './sync-stream.js';
|
|
10
|
+
import { RequestTracker } from '../../sync/RequestTracker.js';
|
|
10
11
|
|
|
11
12
|
export const syncStreamReactive: SocketRouteGenerator = (router) =>
|
|
12
13
|
router.reactiveStream<util.StreamingSyncRequest, any>(SyncRoutes.STREAM, {
|
|
@@ -66,6 +67,7 @@ export const syncStreamReactive: SocketRouteGenerator = (router) =>
|
|
|
66
67
|
});
|
|
67
68
|
|
|
68
69
|
Metrics.getInstance().concurrent_connections.add(1);
|
|
70
|
+
const tracker = new RequestTracker();
|
|
69
71
|
try {
|
|
70
72
|
for await (const data of streamResponse({
|
|
71
73
|
storage,
|
|
@@ -79,6 +81,7 @@ export const syncStreamReactive: SocketRouteGenerator = (router) =>
|
|
|
79
81
|
// RSocket handles keepalive events by default
|
|
80
82
|
keep_alive: false
|
|
81
83
|
},
|
|
84
|
+
tracker,
|
|
82
85
|
signal: controller.signal
|
|
83
86
|
})) {
|
|
84
87
|
if (data == null) {
|
|
@@ -94,7 +97,7 @@ export const syncStreamReactive: SocketRouteGenerator = (router) =>
|
|
|
94
97
|
const serialized = serialize(data) as Buffer;
|
|
95
98
|
responder.onNext({ data: serialized }, false);
|
|
96
99
|
requestedN--;
|
|
97
|
-
|
|
100
|
+
tracker.addDataSynced(serialized.length);
|
|
98
101
|
}
|
|
99
102
|
|
|
100
103
|
if (requestedN <= 0) {
|
|
@@ -126,6 +129,11 @@ export const syncStreamReactive: SocketRouteGenerator = (router) =>
|
|
|
126
129
|
responder.onComplete();
|
|
127
130
|
removeStopHandler();
|
|
128
131
|
disposer();
|
|
132
|
+
logger.info(`Sync stream complete`, {
|
|
133
|
+
user_id: syncParams.user_id,
|
|
134
|
+
operations_synced: tracker.operationsSynced,
|
|
135
|
+
data_synced_bytes: tracker.dataSyncedBytes
|
|
136
|
+
});
|
|
129
137
|
Metrics.getInstance().concurrent_connections.add(-1);
|
|
130
138
|
}
|
|
131
139
|
}
|
|
@@ -8,6 +8,7 @@ import * as util from '../../util/util-index.js';
|
|
|
8
8
|
import { Metrics } from '../../metrics/Metrics.js';
|
|
9
9
|
import { authUser } from '../auth.js';
|
|
10
10
|
import { routeDefinition } from '../router.js';
|
|
11
|
+
import { RequestTracker } from '../../sync/RequestTracker.js';
|
|
11
12
|
|
|
12
13
|
export enum SyncRoutes {
|
|
13
14
|
STREAM = '/sync/stream'
|
|
@@ -43,6 +44,7 @@ export const syncStreamed = routeDefinition({
|
|
|
43
44
|
});
|
|
44
45
|
}
|
|
45
46
|
const controller = new AbortController();
|
|
47
|
+
const tracker = new RequestTracker();
|
|
46
48
|
try {
|
|
47
49
|
Metrics.getInstance().concurrent_connections.add(1);
|
|
48
50
|
const stream = Readable.from(
|
|
@@ -53,9 +55,11 @@ export const syncStreamed = routeDefinition({
|
|
|
53
55
|
params,
|
|
54
56
|
syncParams,
|
|
55
57
|
token: payload.context.token_payload!,
|
|
58
|
+
tracker,
|
|
56
59
|
signal: controller.signal
|
|
57
60
|
})
|
|
58
|
-
)
|
|
61
|
+
),
|
|
62
|
+
tracker
|
|
59
63
|
),
|
|
60
64
|
{ objectMode: false, highWaterMark: 16 * 1024 }
|
|
61
65
|
);
|
|
@@ -86,6 +90,11 @@ export const syncStreamed = routeDefinition({
|
|
|
86
90
|
afterSend: async () => {
|
|
87
91
|
controller.abort();
|
|
88
92
|
Metrics.getInstance().concurrent_connections.add(-1);
|
|
93
|
+
logger.info(`Sync stream complete`, {
|
|
94
|
+
user_id: syncParams.user_id,
|
|
95
|
+
operations_synced: tracker.operationsSynced,
|
|
96
|
+
data_synced_bytes: tracker.dataSyncedBytes
|
|
97
|
+
});
|
|
89
98
|
}
|
|
90
99
|
});
|
|
91
100
|
} catch (ex) {
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import { Metrics } from '../metrics/Metrics.js';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Record sync stats per request stream.
|
|
5
|
+
*/
|
|
6
|
+
export class RequestTracker {
|
|
7
|
+
operationsSynced = 0;
|
|
8
|
+
dataSyncedBytes = 0;
|
|
9
|
+
|
|
10
|
+
addOperationsSynced(operations: number) {
|
|
11
|
+
this.operationsSynced += operations;
|
|
12
|
+
|
|
13
|
+
Metrics.getInstance().operations_synced_total.add(operations);
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
addDataSynced(bytes: number) {
|
|
17
|
+
this.dataSyncedBytes += bytes;
|
|
18
|
+
|
|
19
|
+
Metrics.getInstance().data_synced_bytes.add(bytes);
|
|
20
|
+
}
|
|
21
|
+
}
|