@powersync/service-core 0.7.0 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +19 -0
- package/dist/routes/configure-fastify.d.ts +33 -8
- package/dist/routes/configure-rsocket.js +2 -1
- package/dist/routes/configure-rsocket.js.map +1 -1
- package/dist/routes/endpoints/checkpointing.d.ts +56 -16
- package/dist/routes/endpoints/checkpointing.js +6 -2
- package/dist/routes/endpoints/checkpointing.js.map +1 -1
- package/dist/routes/endpoints/socket-route.js +1 -0
- package/dist/routes/endpoints/socket-route.js.map +1 -1
- package/dist/routes/endpoints/sync-stream.d.ts +10 -0
- package/dist/routes/endpoints/sync-stream.js +5 -0
- package/dist/routes/endpoints/sync-stream.js.map +1 -1
- package/dist/routes/router-socket.d.ts +1 -0
- package/dist/routes/router-socket.js +2 -1
- package/dist/routes/router-socket.js.map +1 -1
- package/dist/routes/router.d.ts +4 -0
- package/dist/routes/router.js.map +1 -1
- package/dist/sync/sync.js +23 -3
- package/dist/sync/sync.js.map +1 -1
- package/dist/util/protocol-types.d.ts +4 -0
- package/dist/util/protocol-types.js +5 -1
- package/dist/util/protocol-types.js.map +1 -1
- package/dist/util/utils.d.ts +1 -0
- package/dist/util/utils.js +9 -0
- package/dist/util/utils.js.map +1 -1
- package/package.json +3 -3
- package/src/routes/configure-rsocket.ts +2 -1
- package/src/routes/endpoints/checkpointing.ts +6 -2
- package/src/routes/endpoints/socket-route.ts +1 -0
- package/src/routes/endpoints/sync-stream.ts +5 -0
- package/src/routes/router-socket.ts +2 -1
- package/src/routes/router.ts +4 -0
- package/src/sync/sync.ts +24 -3
- package/src/util/protocol-types.ts +6 -1
- package/src/util/utils.ts +10 -0
- package/tsconfig.tsbuildinfo +1 -1
package/dist/util/utils.d.ts
CHANGED
|
@@ -15,3 +15,4 @@ export declare function getClientCheckpoint(db: pgwire.PgClient, bucketStorage:
|
|
|
15
15
|
timeout?: number;
|
|
16
16
|
}): Promise<OpId>;
|
|
17
17
|
export declare function createWriteCheckpoint(db: pgwire.PgClient, bucketStorage: storage.BucketStorageFactory, user_id: string): Promise<bigint>;
|
|
18
|
+
export declare function checkpointUserId(user_id: string | undefined, client_id: string | undefined): string;
|
package/dist/util/utils.js
CHANGED
|
@@ -89,4 +89,13 @@ export async function createWriteCheckpoint(db, bucketStorage, user_id) {
|
|
|
89
89
|
logger.info(`Write checkpoint 2: ${JSON.stringify({ lsn, id: String(id) })}`);
|
|
90
90
|
return id;
|
|
91
91
|
}
|
|
92
|
+
export function checkpointUserId(user_id, client_id) {
|
|
93
|
+
if (user_id == null) {
|
|
94
|
+
throw new Error('user_id is required');
|
|
95
|
+
}
|
|
96
|
+
if (client_id == null) {
|
|
97
|
+
return user_id;
|
|
98
|
+
}
|
|
99
|
+
return `${user_id}/${client_id}`;
|
|
100
|
+
}
|
|
92
101
|
//# sourceMappingURL=utils.js.map
|
package/dist/util/utils.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"utils.js","sourceRoot":"","sources":["../../src/util/utils.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,QAAQ,CAAC;AAE5B,OAAO,EAAE,UAAU,EAAE,MAAM,4BAA4B,CAAC;AAIxD,OAAO,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AACjD,OAAO,EAAE,MAAM,EAAE,MAAM,mCAAmC,CAAC;AAI3D,MAAM,UAAU,QAAQ,CAAC,IAAY,EAAE,EAAU,EAAE,IAAY;IAC7D,MAAM,IAAI,GAAG,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;IACzC,IAAI,CAAC,MAAM,CAAC,OAAO,IAAI,IAAI,EAAE,IAAI,IAAI,EAAE,CAAC,CAAC;IACzC,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;IAC7B,OAAO,MAAM,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;AAChC,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,SAAiB;IAC1C,MAAM,IAAI,GAAG,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;IACzC,IAAI,CAAC,MAAM,CAAC,UAAU,SAAS,EAAE,CAAC,CAAC;IACnC,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;IAC7B,OAAO,MAAM,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;AAChC,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,EAAU;IACxC,6EAA6E;IAC7E,6CAA6C;IAC7C,IAAI,OAAO,EAAE,IAAI,QAAQ,EAAE;QACzB,MAAM,IAAI,KAAK,CAAC,yBAAyB,EAAE,KAAK,OAAO,EAAE,GAAG,CAAC,CAAC;KAC/D;IACD,OAAO,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;AACzB,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,QAAqB,EAAE,OAAoB;IACvE,mBAAmB;IACnB,MAAM,cAAc,GAAG,IAAI,GAAG,EAA0B,CAAC;IAEzD,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAS,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC;IAElD,KAAK,IAAI,QAAQ,IAAI,OAAO,CAAC,MAAM,EAAE,EAAE;QACrC,MAAM,CAAC,GAAG,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QACxC,IAAI,CAAC,IAAI,IAAI,EAAE;YACb,QAAQ;YACR,cAAc,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;SAC/C;aAAM;YACL,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;YACjC,IAAI,QAAQ,CAAC,QAAQ,IAAI,CAAC,CAAC,QAAQ,IAAI,QAAQ,CAAC,KAAK,IAAI,CAAC,CAAC,KAAK,EAAE;gBAChE,UAAU;gBACV,cAAc,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;aAC/C;iBAAM;gBACL,YAAY;aACb;SACF;KACF;IAED,OAAO;QACL,cAAc,EAAE,CAAC,GAAG,cAAc,CAAC,MAAM,EAAE,CAAC;QAC5C,cAAc,EAAE,CAAC,GAAG,QAAQ,CAAC;KAC9B,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,CAAS,EAAE,CAAS;IAC/C,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,UAAU,CAAC;AAC9B,CAAC;AAED,MAAM,UAAU,kBAAkB,CAAC,CAAiB,EAAE,CAAwB;IAC5E,IAAI,CAAC,IAAI,IAAI,EAAE;QACb,OAAO,CAAC,CAAC;KACV;SAAM;QACL,OAAO;YACL,MAAM,EAAE,CAAC,CAAC,MAAM;YAChB,KAAK,EAAE,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK;YACxB,QAAQ,EAAE,YAAY,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,QAAQ,CAAC;SAC/C,CAAC;KACH;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,mBAAmB,CACvC,EAAmB,EACnB,aAA2C,EAC3C,OAA8B;IAE9B,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAEzB,MAAM,CAAC,EAAE,GAAG,EAAE,CAAC,GAAG,UAAU,CAAC,MAAM,EAAE,CAAC,KAAK,CAAC,mEAAmE,CAAC,CAAC,CAAC;IAElH,gDAAgD;IAChD,wEAAwE;IAExE,MAAM,OAAO,GAAG,OAAO,EAAE,OAAO,IAAI,KAAM,CAAC;IAE3C,MAAM,CAAC,IAAI,CAAC,+BAA+B,GAAG,EAAE,CAAC,CAAC;IAClD,OAAO,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,GAAG,OAAO,EAAE;QACnC,MAAM,EAAE,GAAG,MAAM,aAAa,CAAC,mBAAmB,EAAE,CAAC;QACrD,IAAI,CAAC,EAAE,CAAC,YAAY,EAAE,EAAE;YACtB,MAAM,IAAI,KAAK,CAAC,yBAAyB,CAAC,CAAC;SAC5C;QACD,IAAI,EAAE,CAAC,GAAG,IAAI,GAAG,EAAE;YACjB,MAAM,CAAC,IAAI,CAAC,yBAAyB,GAAG,MAAM,EAAE,CAAC,UAAU,EAAE,CAAC,CAAC;YAC/D,OAAO,EAAE,CAAC,UAAU,CAAC;SACtB;QAED,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;KACzD;IAED,MAAM,IAAI,KAAK,CAAC,sCAAsC,CAAC,CAAC;AAC1D,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,qBAAqB,CACzC,EAAmB,EACnB,aAA2C,EAC3C,OAAe;IAEf,MAAM,CAAC,EAAE,GAAG,EAAE,CAAC,GAAG,UAAU,CAC1B,MAAM,YAAY,CAAC,EAAE,EAAE,mEAAmE,CAAC,CAC5F,CAAC;IAEF,MAAM,EAAE,GAAG,MAAM,aAAa,CAAC,qBAAqB,CAAC,OAAO,EAAE,EAAE,GAAG,EAAE,GAAG,EAAE,CAAC,CAAC;IAC5E,MAAM,CAAC,IAAI,CAAC,uBAAuB,IAAI,CAAC,SAAS,CAAC,EAAE,GAAG,EAAE,EAAE,EAAE,MAAM,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;IAC9E,OAAO,EAAE,CAAC;AACZ,CAAC"}
|
|
1
|
+
{"version":3,"file":"utils.js","sourceRoot":"","sources":["../../src/util/utils.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,QAAQ,CAAC;AAE5B,OAAO,EAAE,UAAU,EAAE,MAAM,4BAA4B,CAAC;AAIxD,OAAO,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AACjD,OAAO,EAAE,MAAM,EAAE,MAAM,mCAAmC,CAAC;AAI3D,MAAM,UAAU,QAAQ,CAAC,IAAY,EAAE,EAAU,EAAE,IAAY;IAC7D,MAAM,IAAI,GAAG,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;IACzC,IAAI,CAAC,MAAM,CAAC,OAAO,IAAI,IAAI,EAAE,IAAI,IAAI,EAAE,CAAC,CAAC;IACzC,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;IAC7B,OAAO,MAAM,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;AAChC,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,SAAiB;IAC1C,MAAM,IAAI,GAAG,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;IACzC,IAAI,CAAC,MAAM,CAAC,UAAU,SAAS,EAAE,CAAC,CAAC;IACnC,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;IAC7B,OAAO,MAAM,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;AAChC,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,EAAU;IACxC,6EAA6E;IAC7E,6CAA6C;IAC7C,IAAI,OAAO,EAAE,IAAI,QAAQ,EAAE;QACzB,MAAM,IAAI,KAAK,CAAC,yBAAyB,EAAE,KAAK,OAAO,EAAE,GAAG,CAAC,CAAC;KAC/D;IACD,OAAO,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;AACzB,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,QAAqB,EAAE,OAAoB;IACvE,mBAAmB;IACnB,MAAM,cAAc,GAAG,IAAI,GAAG,EAA0B,CAAC;IAEzD,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAS,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC;IAElD,KAAK,IAAI,QAAQ,IAAI,OAAO,CAAC,MAAM,EAAE,EAAE;QACrC,MAAM,CAAC,GAAG,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QACxC,IAAI,CAAC,IAAI,IAAI,EAAE;YACb,QAAQ;YACR,cAAc,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;SAC/C;aAAM;YACL,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;YACjC,IAAI,QAAQ,CAAC,QAAQ,IAAI,CAAC,CAAC,QAAQ,IAAI,QAAQ,CAAC,KAAK,IAAI,CAAC,CAAC,KAAK,EAAE;gBAChE,UAAU;gBACV,cAAc,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;aAC/C;iBAAM;gBACL,YAAY;aACb;SACF;KACF;IAED,OAAO;QACL,cAAc,EAAE,CAAC,GAAG,cAAc,CAAC,MAAM,EAAE,CAAC;QAC5C,cAAc,EAAE,CAAC,GAAG,QAAQ,CAAC;KAC9B,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,CAAS,EAAE,CAAS;IAC/C,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,UAAU,CAAC;AAC9B,CAAC;AAED,MAAM,UAAU,kBAAkB,CAAC,CAAiB,EAAE,CAAwB;IAC5E,IAAI,CAAC,IAAI,IAAI,EAAE;QACb,OAAO,CAAC,CAAC;KACV;SAAM;QACL,OAAO;YACL,MAAM,EAAE,CAAC,CAAC,MAAM;YAChB,KAAK,EAAE,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK;YACxB,QAAQ,EAAE,YAAY,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,QAAQ,CAAC;SAC/C,CAAC;KACH;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,mBAAmB,CACvC,EAAmB,EACnB,aAA2C,EAC3C,OAA8B;IAE9B,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAEzB,MAAM,CAAC,EAAE,GAAG,EAAE,CAAC,GAAG,UAAU,CAAC,MAAM,EAAE,CAAC,KAAK,CAAC,mEAAmE,CAAC,CAAC,CAAC;IAElH,gDAAgD;IAChD,wEAAwE;IAExE,MAAM,OAAO,GAAG,OAAO,EAAE,OAAO,IAAI,KAAM,CAAC;IAE3C,MAAM,CAAC,IAAI,CAAC,+BAA+B,GAAG,EAAE,CAAC,CAAC;IAClD,OAAO,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,GAAG,OAAO,EAAE;QACnC,MAAM,EAAE,GAAG,MAAM,aAAa,CAAC,mBAAmB,EAAE,CAAC;QACrD,IAAI,CAAC,EAAE,CAAC,YAAY,EAAE,EAAE;YACtB,MAAM,IAAI,KAAK,CAAC,yBAAyB,CAAC,CAAC;SAC5C;QACD,IAAI,EAAE,CAAC,GAAG,IAAI,GAAG,EAAE;YACjB,MAAM,CAAC,IAAI,CAAC,yBAAyB,GAAG,MAAM,EAAE,CAAC,UAAU,EAAE,CAAC,CAAC;YAC/D,OAAO,EAAE,CAAC,UAAU,CAAC;SACtB;QAED,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;KACzD;IAED,MAAM,IAAI,KAAK,CAAC,sCAAsC,CAAC,CAAC;AAC1D,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,qBAAqB,CACzC,EAAmB,EACnB,aAA2C,EAC3C,OAAe;IAEf,MAAM,CAAC,EAAE,GAAG,EAAE,CAAC,GAAG,UAAU,CAC1B,MAAM,YAAY,CAAC,EAAE,EAAE,mEAAmE,CAAC,CAC5F,CAAC;IAEF,MAAM,EAAE,GAAG,MAAM,aAAa,CAAC,qBAAqB,CAAC,OAAO,EAAE,EAAE,GAAG,EAAE,GAAG,EAAE,CAAC,CAAC;IAC5E,MAAM,CAAC,IAAI,CAAC,uBAAuB,IAAI,CAAC,SAAS,CAAC,EAAE,GAAG,EAAE,EAAE,EAAE,MAAM,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;IAC9E,OAAO,EAAE,CAAC;AACZ,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,OAA2B,EAAE,SAA6B;IACzF,IAAI,OAAO,IAAI,IAAI,EAAE;QACnB,MAAM,IAAI,KAAK,CAAC,qBAAqB,CAAC,CAAC;KACxC;IACD,IAAI,SAAS,IAAI,IAAI,EAAE;QACrB,OAAO,OAAO,CAAC;KAChB;IACD,OAAO,GAAG,OAAO,IAAI,SAAS,EAAE,CAAC;AACnC,CAAC"}
|
package/package.json
CHANGED
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
"publishConfig": {
|
|
6
6
|
"access": "public"
|
|
7
7
|
},
|
|
8
|
-
"version": "0.
|
|
8
|
+
"version": "0.8.0",
|
|
9
9
|
"main": "dist/index.js",
|
|
10
10
|
"license": "FSL-1.1-Apache-2.0",
|
|
11
11
|
"type": "module",
|
|
@@ -36,8 +36,8 @@
|
|
|
36
36
|
"@powersync/lib-services-framework": "0.1.1",
|
|
37
37
|
"@powersync/service-jpgwire": "0.17.14",
|
|
38
38
|
"@powersync/service-jsonbig": "0.17.10",
|
|
39
|
-
"@powersync/service-rsocket-router": "0.0.
|
|
40
|
-
"@powersync/service-sync-rules": "0.18.
|
|
39
|
+
"@powersync/service-rsocket-router": "0.0.11",
|
|
40
|
+
"@powersync/service-sync-rules": "0.18.2",
|
|
41
41
|
"@powersync/service-types": "0.2.0"
|
|
42
42
|
},
|
|
43
43
|
"devDependencies": {
|
|
@@ -23,7 +23,7 @@ export function configureRSocket(router: ReactiveSocketRouter<Context>, options:
|
|
|
23
23
|
|
|
24
24
|
router.applyWebSocketEndpoints(server, {
|
|
25
25
|
contextProvider: async (data: Buffer) => {
|
|
26
|
-
const { token } = RSocketContextMeta.decode(deserialize(data) as any);
|
|
26
|
+
const { token, user_agent } = RSocketContextMeta.decode(deserialize(data) as any);
|
|
27
27
|
|
|
28
28
|
if (!token) {
|
|
29
29
|
throw new errors.AuthorizationError('No token provided');
|
|
@@ -38,6 +38,7 @@ export function configureRSocket(router: ReactiveSocketRouter<Context>, options:
|
|
|
38
38
|
}
|
|
39
39
|
return {
|
|
40
40
|
token,
|
|
41
|
+
user_agent,
|
|
41
42
|
...context,
|
|
42
43
|
token_errors: token_errors,
|
|
43
44
|
system
|
|
@@ -5,7 +5,9 @@ import * as util from '../../util/util-index.js';
|
|
|
5
5
|
import { authUser } from '../auth.js';
|
|
6
6
|
import { routeDefinition } from '../router.js';
|
|
7
7
|
|
|
8
|
-
const WriteCheckpointRequest = t.object({
|
|
8
|
+
const WriteCheckpointRequest = t.object({
|
|
9
|
+
client_id: t.string.optional()
|
|
10
|
+
});
|
|
9
11
|
|
|
10
12
|
export const writeCheckpoint = routeDefinition({
|
|
11
13
|
path: '/write-checkpoint.json',
|
|
@@ -30,8 +32,10 @@ export const writeCheckpoint2 = routeDefinition({
|
|
|
30
32
|
validator: schema.createTsCodecValidator(WriteCheckpointRequest, { allowAdditional: true }),
|
|
31
33
|
handler: async (payload) => {
|
|
32
34
|
const { user_id, system } = payload.context;
|
|
35
|
+
const client_id = payload.params.client_id;
|
|
36
|
+
const full_user_id = util.checkpointUserId(user_id, client_id);
|
|
33
37
|
const storage = system.storage;
|
|
34
|
-
const write_checkpoint = await util.createWriteCheckpoint(system.requirePgPool(), storage,
|
|
38
|
+
const write_checkpoint = await util.createWriteCheckpoint(system.requirePgPool(), storage, full_user_id);
|
|
35
39
|
return {
|
|
36
40
|
write_checkpoint: String(write_checkpoint)
|
|
37
41
|
};
|
|
@@ -124,6 +124,7 @@ export const syncStreamReactive: SocketRouteGenerator = (router) =>
|
|
|
124
124
|
disposer();
|
|
125
125
|
logger.info(`Sync stream complete`, {
|
|
126
126
|
user_id: syncParams.user_id,
|
|
127
|
+
user_agent: context.user_agent,
|
|
127
128
|
operations_synced: tracker.operationsSynced,
|
|
128
129
|
data_synced_bytes: tracker.dataSyncedBytes
|
|
129
130
|
});
|
|
@@ -21,6 +21,9 @@ export const syncStreamed = routeDefinition({
|
|
|
21
21
|
validator: schema.createTsCodecValidator(util.StreamingSyncRequest, { allowAdditional: true }),
|
|
22
22
|
handler: async (payload) => {
|
|
23
23
|
const system = payload.context.system;
|
|
24
|
+
const headers = payload.request.headers;
|
|
25
|
+
const userAgent = headers['x-user-agent'] ?? headers['user-agent'];
|
|
26
|
+
const clientId = payload.params.client_id;
|
|
24
27
|
|
|
25
28
|
if (system.closed) {
|
|
26
29
|
throw new errors.JourneyError({
|
|
@@ -92,6 +95,8 @@ export const syncStreamed = routeDefinition({
|
|
|
92
95
|
Metrics.getInstance().concurrent_connections.add(-1);
|
|
93
96
|
logger.info(`Sync stream complete`, {
|
|
94
97
|
user_id: syncParams.user_id,
|
|
98
|
+
client_id: clientId,
|
|
99
|
+
user_agent: userAgent,
|
|
95
100
|
operations_synced: tracker.operationsSynced,
|
|
96
101
|
data_synced_bytes: tracker.dataSyncedBytes
|
|
97
102
|
});
|
|
@@ -9,5 +9,6 @@ import { Context } from './router.js';
|
|
|
9
9
|
export type SocketRouteGenerator = (router: ReactiveSocketRouter<Context>) => IReactiveStream;
|
|
10
10
|
|
|
11
11
|
export const RSocketContextMeta = t.object({
|
|
12
|
-
token: t.string
|
|
12
|
+
token: t.string,
|
|
13
|
+
user_agent: t.string.optional()
|
|
13
14
|
});
|
package/src/routes/router.ts
CHANGED
package/src/sync/sync.ts
CHANGED
|
@@ -93,7 +93,8 @@ async function* streamResponseInner(
|
|
|
93
93
|
}
|
|
94
94
|
}
|
|
95
95
|
|
|
96
|
-
const
|
|
96
|
+
const checkpointUserId = util.checkpointUserId(syncParams.token_parameters.user_id as string, params.client_id);
|
|
97
|
+
const stream = storage.watchWriteCheckpoint(checkpointUserId, signal);
|
|
97
98
|
for await (const next of stream) {
|
|
98
99
|
const { base, writeCheckpoint } = next;
|
|
99
100
|
const checkpoint = base.checkpoint;
|
|
@@ -196,7 +197,8 @@ async function* streamResponseInner(
|
|
|
196
197
|
raw_data,
|
|
197
198
|
binary_data,
|
|
198
199
|
signal,
|
|
199
|
-
tracker
|
|
200
|
+
tracker,
|
|
201
|
+
user_id: syncParams.user_id
|
|
200
202
|
});
|
|
201
203
|
|
|
202
204
|
await new Promise((resolve) => setTimeout(resolve, 10));
|
|
@@ -213,6 +215,7 @@ interface BucketDataRequest {
|
|
|
213
215
|
binary_data: boolean | undefined;
|
|
214
216
|
tracker: RequestTracker;
|
|
215
217
|
signal: AbortSignal;
|
|
218
|
+
user_id?: string;
|
|
216
219
|
}
|
|
217
220
|
|
|
218
221
|
async function* bucketDataInBatches(request: BucketDataRequest) {
|
|
@@ -261,8 +264,19 @@ async function* bucketDataBatch(request: BucketDataRequest): AsyncGenerator<Buck
|
|
|
261
264
|
const checkpointOp = BigInt(checkpoint);
|
|
262
265
|
let checkpointInvalidated = false;
|
|
263
266
|
|
|
264
|
-
|
|
267
|
+
if (syncSemaphore.isLocked()) {
|
|
268
|
+
logger.info('Sync concurrency limit reached, waiting for lock', { user_id: request.user_id });
|
|
269
|
+
}
|
|
270
|
+
const [value, release] = await syncSemaphore.acquire();
|
|
265
271
|
try {
|
|
272
|
+
if (value <= 3) {
|
|
273
|
+
// This can be noisy, so we only log when we get close to the
|
|
274
|
+
// concurrency limit.
|
|
275
|
+
logger.info(`Got sync lock. Slots available: ${value - 1}`, {
|
|
276
|
+
user_id: request.user_id,
|
|
277
|
+
sync_data_slots: value - 1
|
|
278
|
+
});
|
|
279
|
+
}
|
|
266
280
|
// Optimization: Only fetch buckets for which the checksums have changed since the last checkpoint
|
|
267
281
|
// For the first batch, this will be all buckets.
|
|
268
282
|
const filteredBuckets = new Map(bucketsToFetch.map((bucket) => [bucket, dataBuckets.get(bucket)!]));
|
|
@@ -330,6 +344,13 @@ async function* bucketDataBatch(request: BucketDataRequest): AsyncGenerator<Buck
|
|
|
330
344
|
}
|
|
331
345
|
}
|
|
332
346
|
} finally {
|
|
347
|
+
if (value <= 3) {
|
|
348
|
+
// This can be noisy, so we only log when we get close to the
|
|
349
|
+
// concurrency limit.
|
|
350
|
+
logger.info(`Releasing sync lock`, {
|
|
351
|
+
user_id: request.user_id
|
|
352
|
+
});
|
|
353
|
+
}
|
|
333
354
|
release();
|
|
334
355
|
}
|
|
335
356
|
}
|
|
@@ -94,7 +94,12 @@ export const StreamingSyncRequest = t.object({
|
|
|
94
94
|
/**
|
|
95
95
|
* Client parameters to be passed to the sync rules.
|
|
96
96
|
*/
|
|
97
|
-
parameters: t.record(t.any).optional()
|
|
97
|
+
parameters: t.record(t.any).optional(),
|
|
98
|
+
|
|
99
|
+
/**
|
|
100
|
+
* Unique client id.
|
|
101
|
+
*/
|
|
102
|
+
client_id: t.string.optional()
|
|
98
103
|
});
|
|
99
104
|
|
|
100
105
|
export type StreamingSyncRequest = t.Decoded<typeof StreamingSyncRequest>;
|
package/src/util/utils.ts
CHANGED
|
@@ -120,3 +120,13 @@ export async function createWriteCheckpoint(
|
|
|
120
120
|
logger.info(`Write checkpoint 2: ${JSON.stringify({ lsn, id: String(id) })}`);
|
|
121
121
|
return id;
|
|
122
122
|
}
|
|
123
|
+
|
|
124
|
+
export function checkpointUserId(user_id: string | undefined, client_id: string | undefined) {
|
|
125
|
+
if (user_id == null) {
|
|
126
|
+
throw new Error('user_id is required');
|
|
127
|
+
}
|
|
128
|
+
if (client_id == null) {
|
|
129
|
+
return user_id;
|
|
130
|
+
}
|
|
131
|
+
return `${user_id}/${client_id}`;
|
|
132
|
+
}
|