@powersync/service-core 1.14.0 → 1.15.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +32 -0
- package/LICENSE +3 -3
- package/dist/api/api-metrics.js +5 -0
- package/dist/api/api-metrics.js.map +1 -1
- package/dist/api/diagnostics.js +1 -1
- package/dist/api/diagnostics.js.map +1 -1
- package/dist/metrics/open-telemetry/util.d.ts +0 -3
- package/dist/metrics/open-telemetry/util.js +18 -13
- package/dist/metrics/open-telemetry/util.js.map +1 -1
- package/dist/routes/compression.d.ts +19 -0
- package/dist/routes/compression.js +70 -0
- package/dist/routes/compression.js.map +1 -0
- package/dist/routes/configure-fastify.d.ts +40 -5
- package/dist/routes/endpoints/socket-route.js +24 -9
- package/dist/routes/endpoints/socket-route.js.map +1 -1
- package/dist/routes/endpoints/sync-rules.js +1 -27
- package/dist/routes/endpoints/sync-rules.js.map +1 -1
- package/dist/routes/endpoints/sync-stream.d.ts +80 -10
- package/dist/routes/endpoints/sync-stream.js +17 -12
- package/dist/routes/endpoints/sync-stream.js.map +1 -1
- package/dist/storage/BucketStorage.d.ts +1 -1
- package/dist/storage/BucketStorage.js.map +1 -1
- package/dist/storage/BucketStorageBatch.d.ts +4 -4
- package/dist/storage/BucketStorageBatch.js.map +1 -1
- package/dist/storage/ChecksumCache.d.ts +4 -19
- package/dist/storage/ChecksumCache.js +4 -0
- package/dist/storage/ChecksumCache.js.map +1 -1
- package/dist/storage/ReplicationEventPayload.d.ts +2 -2
- package/dist/storage/SyncRulesBucketStorage.d.ts +9 -0
- package/dist/storage/SyncRulesBucketStorage.js.map +1 -1
- package/dist/sync/BucketChecksumState.d.ts +40 -10
- package/dist/sync/BucketChecksumState.js +154 -18
- package/dist/sync/BucketChecksumState.js.map +1 -1
- package/dist/sync/RequestTracker.d.ts +7 -1
- package/dist/sync/RequestTracker.js +22 -2
- package/dist/sync/RequestTracker.js.map +1 -1
- package/dist/sync/sync.d.ts +3 -3
- package/dist/sync/sync.js +23 -42
- package/dist/sync/sync.js.map +1 -1
- package/dist/sync/util.js +1 -1
- package/dist/sync/util.js.map +1 -1
- package/dist/util/protocol-types.d.ts +153 -9
- package/dist/util/protocol-types.js +41 -6
- package/dist/util/protocol-types.js.map +1 -1
- package/dist/util/utils.d.ts +18 -3
- package/dist/util/utils.js +33 -9
- package/dist/util/utils.js.map +1 -1
- package/package.json +14 -14
- package/src/api/api-metrics.ts +6 -0
- package/src/api/diagnostics.ts +1 -1
- package/src/metrics/open-telemetry/util.ts +22 -21
- package/src/routes/compression.ts +75 -0
- package/src/routes/endpoints/socket-route.ts +24 -9
- package/src/routes/endpoints/sync-rules.ts +1 -28
- package/src/routes/endpoints/sync-stream.ts +18 -15
- package/src/storage/BucketStorage.ts +2 -2
- package/src/storage/BucketStorageBatch.ts +10 -4
- package/src/storage/ChecksumCache.ts +8 -22
- package/src/storage/ReplicationEventPayload.ts +2 -2
- package/src/storage/SyncRulesBucketStorage.ts +12 -0
- package/src/sync/BucketChecksumState.ts +192 -29
- package/src/sync/RequestTracker.ts +27 -2
- package/src/sync/sync.ts +53 -51
- package/src/sync/util.ts +1 -1
- package/src/util/protocol-types.ts +138 -10
- package/src/util/utils.ts +59 -12
- package/test/src/checksum_cache.test.ts +6 -8
- package/test/src/routes/mocks.ts +59 -0
- package/test/src/routes/stream.test.ts +84 -0
- package/test/src/sync/BucketChecksumState.test.ts +340 -42
- package/tsconfig.tsbuildinfo +1 -1
|
@@ -12,8 +12,15 @@ export declare const syncStreamed: router.Endpoint<{
|
|
|
12
12
|
only?: string[] | undefined;
|
|
13
13
|
include_checksum?: boolean | undefined;
|
|
14
14
|
raw_data?: boolean | undefined;
|
|
15
|
-
binary_data?: boolean | undefined;
|
|
16
15
|
client_id?: string | undefined;
|
|
16
|
+
streams?: {
|
|
17
|
+
subscriptions: {
|
|
18
|
+
stream: string;
|
|
19
|
+
override_priority: number | null;
|
|
20
|
+
parameters?: Record<string, any> | undefined;
|
|
21
|
+
}[];
|
|
22
|
+
include_defaults?: boolean | undefined;
|
|
23
|
+
} | undefined;
|
|
17
24
|
}, router.RouterResponse<Readable> | undefined, import("../router.js").Context, import("../router.js").RequestEndpointHandlerPayload<{
|
|
18
25
|
parameters?: Record<string, any> | undefined;
|
|
19
26
|
buckets?: {
|
|
@@ -23,8 +30,15 @@ export declare const syncStreamed: router.Endpoint<{
|
|
|
23
30
|
only?: string[] | undefined;
|
|
24
31
|
include_checksum?: boolean | undefined;
|
|
25
32
|
raw_data?: boolean | undefined;
|
|
26
|
-
binary_data?: boolean | undefined;
|
|
27
33
|
client_id?: string | undefined;
|
|
34
|
+
streams?: {
|
|
35
|
+
subscriptions: {
|
|
36
|
+
stream: string;
|
|
37
|
+
override_priority: number | null;
|
|
38
|
+
parameters?: Record<string, any> | undefined;
|
|
39
|
+
}[];
|
|
40
|
+
include_defaults?: boolean | undefined;
|
|
41
|
+
} | undefined;
|
|
28
42
|
}, import("../router.js").Context, import("../router.js").BasicRouterRequest>, router.EndpointHandler<import("../router.js").RequestEndpointHandlerPayload<{
|
|
29
43
|
parameters?: Record<string, any> | undefined;
|
|
30
44
|
buckets?: {
|
|
@@ -34,8 +48,15 @@ export declare const syncStreamed: router.Endpoint<{
|
|
|
34
48
|
only?: string[] | undefined;
|
|
35
49
|
include_checksum?: boolean | undefined;
|
|
36
50
|
raw_data?: boolean | undefined;
|
|
37
|
-
binary_data?: boolean | undefined;
|
|
38
51
|
client_id?: string | undefined;
|
|
52
|
+
streams?: {
|
|
53
|
+
subscriptions: {
|
|
54
|
+
stream: string;
|
|
55
|
+
override_priority: number | null;
|
|
56
|
+
parameters?: Record<string, any> | undefined;
|
|
57
|
+
}[];
|
|
58
|
+
include_defaults?: boolean | undefined;
|
|
59
|
+
} | undefined;
|
|
39
60
|
}, import("../router.js").Context, import("../router.js").BasicRouterRequest>, router.RouterResponse<Readable> | undefined>> & {
|
|
40
61
|
path: SyncRoutes;
|
|
41
62
|
method: router.HTTPMethod.POST;
|
|
@@ -49,8 +70,15 @@ export declare const syncStreamed: router.Endpoint<{
|
|
|
49
70
|
only?: string[] | undefined;
|
|
50
71
|
include_checksum?: boolean | undefined;
|
|
51
72
|
raw_data?: boolean | undefined;
|
|
52
|
-
binary_data?: boolean | undefined;
|
|
53
73
|
client_id?: string | undefined;
|
|
74
|
+
streams?: {
|
|
75
|
+
subscriptions: {
|
|
76
|
+
stream: string;
|
|
77
|
+
override_priority: number | null;
|
|
78
|
+
parameters?: Record<string, any> | undefined;
|
|
79
|
+
}[];
|
|
80
|
+
include_defaults?: boolean | undefined;
|
|
81
|
+
} | undefined;
|
|
54
82
|
}, string[]>;
|
|
55
83
|
handler: (payload: import("../router.js").RequestEndpointHandlerPayload<{
|
|
56
84
|
parameters?: Record<string, any> | undefined;
|
|
@@ -61,8 +89,15 @@ export declare const syncStreamed: router.Endpoint<{
|
|
|
61
89
|
only?: string[] | undefined;
|
|
62
90
|
include_checksum?: boolean | undefined;
|
|
63
91
|
raw_data?: boolean | undefined;
|
|
64
|
-
binary_data?: boolean | undefined;
|
|
65
92
|
client_id?: string | undefined;
|
|
93
|
+
streams?: {
|
|
94
|
+
subscriptions: {
|
|
95
|
+
stream: string;
|
|
96
|
+
override_priority: number | null;
|
|
97
|
+
parameters?: Record<string, any> | undefined;
|
|
98
|
+
}[];
|
|
99
|
+
include_defaults?: boolean | undefined;
|
|
100
|
+
} | undefined;
|
|
66
101
|
}, import("../router.js").Context, import("../router.js").BasicRouterRequest>) => Promise<router.RouterResponse<Readable> | undefined>;
|
|
67
102
|
};
|
|
68
103
|
export declare const SYNC_STREAM_ROUTES: (router.Endpoint<{
|
|
@@ -74,8 +109,15 @@ export declare const SYNC_STREAM_ROUTES: (router.Endpoint<{
|
|
|
74
109
|
only?: string[] | undefined;
|
|
75
110
|
include_checksum?: boolean | undefined;
|
|
76
111
|
raw_data?: boolean | undefined;
|
|
77
|
-
binary_data?: boolean | undefined;
|
|
78
112
|
client_id?: string | undefined;
|
|
113
|
+
streams?: {
|
|
114
|
+
subscriptions: {
|
|
115
|
+
stream: string;
|
|
116
|
+
override_priority: number | null;
|
|
117
|
+
parameters?: Record<string, any> | undefined;
|
|
118
|
+
}[];
|
|
119
|
+
include_defaults?: boolean | undefined;
|
|
120
|
+
} | undefined;
|
|
79
121
|
}, router.RouterResponse<Readable> | undefined, import("../router.js").Context, import("../router.js").RequestEndpointHandlerPayload<{
|
|
80
122
|
parameters?: Record<string, any> | undefined;
|
|
81
123
|
buckets?: {
|
|
@@ -85,8 +127,15 @@ export declare const SYNC_STREAM_ROUTES: (router.Endpoint<{
|
|
|
85
127
|
only?: string[] | undefined;
|
|
86
128
|
include_checksum?: boolean | undefined;
|
|
87
129
|
raw_data?: boolean | undefined;
|
|
88
|
-
binary_data?: boolean | undefined;
|
|
89
130
|
client_id?: string | undefined;
|
|
131
|
+
streams?: {
|
|
132
|
+
subscriptions: {
|
|
133
|
+
stream: string;
|
|
134
|
+
override_priority: number | null;
|
|
135
|
+
parameters?: Record<string, any> | undefined;
|
|
136
|
+
}[];
|
|
137
|
+
include_defaults?: boolean | undefined;
|
|
138
|
+
} | undefined;
|
|
90
139
|
}, import("../router.js").Context, import("../router.js").BasicRouterRequest>, router.EndpointHandler<import("../router.js").RequestEndpointHandlerPayload<{
|
|
91
140
|
parameters?: Record<string, any> | undefined;
|
|
92
141
|
buckets?: {
|
|
@@ -96,8 +145,15 @@ export declare const SYNC_STREAM_ROUTES: (router.Endpoint<{
|
|
|
96
145
|
only?: string[] | undefined;
|
|
97
146
|
include_checksum?: boolean | undefined;
|
|
98
147
|
raw_data?: boolean | undefined;
|
|
99
|
-
binary_data?: boolean | undefined;
|
|
100
148
|
client_id?: string | undefined;
|
|
149
|
+
streams?: {
|
|
150
|
+
subscriptions: {
|
|
151
|
+
stream: string;
|
|
152
|
+
override_priority: number | null;
|
|
153
|
+
parameters?: Record<string, any> | undefined;
|
|
154
|
+
}[];
|
|
155
|
+
include_defaults?: boolean | undefined;
|
|
156
|
+
} | undefined;
|
|
101
157
|
}, import("../router.js").Context, import("../router.js").BasicRouterRequest>, router.RouterResponse<Readable> | undefined>> & {
|
|
102
158
|
path: SyncRoutes;
|
|
103
159
|
method: router.HTTPMethod.POST;
|
|
@@ -111,8 +167,15 @@ export declare const SYNC_STREAM_ROUTES: (router.Endpoint<{
|
|
|
111
167
|
only?: string[] | undefined;
|
|
112
168
|
include_checksum?: boolean | undefined;
|
|
113
169
|
raw_data?: boolean | undefined;
|
|
114
|
-
binary_data?: boolean | undefined;
|
|
115
170
|
client_id?: string | undefined;
|
|
171
|
+
streams?: {
|
|
172
|
+
subscriptions: {
|
|
173
|
+
stream: string;
|
|
174
|
+
override_priority: number | null;
|
|
175
|
+
parameters?: Record<string, any> | undefined;
|
|
176
|
+
}[];
|
|
177
|
+
include_defaults?: boolean | undefined;
|
|
178
|
+
} | undefined;
|
|
116
179
|
}, string[]>;
|
|
117
180
|
handler: (payload: import("../router.js").RequestEndpointHandlerPayload<{
|
|
118
181
|
parameters?: Record<string, any> | undefined;
|
|
@@ -123,7 +186,14 @@ export declare const SYNC_STREAM_ROUTES: (router.Endpoint<{
|
|
|
123
186
|
only?: string[] | undefined;
|
|
124
187
|
include_checksum?: boolean | undefined;
|
|
125
188
|
raw_data?: boolean | undefined;
|
|
126
|
-
binary_data?: boolean | undefined;
|
|
127
189
|
client_id?: string | undefined;
|
|
190
|
+
streams?: {
|
|
191
|
+
subscriptions: {
|
|
192
|
+
stream: string;
|
|
193
|
+
override_priority: number | null;
|
|
194
|
+
parameters?: Record<string, any> | undefined;
|
|
195
|
+
}[];
|
|
196
|
+
include_defaults?: boolean | undefined;
|
|
197
|
+
} | undefined;
|
|
128
198
|
}, import("../router.js").Context, import("../router.js").BasicRouterRequest>) => Promise<router.RouterResponse<Readable> | undefined>;
|
|
129
199
|
})[];
|
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
import { ErrorCode, errors, router, schema } from '@powersync/lib-services-framework';
|
|
2
|
-
import { RequestParameters } from '@powersync/service-sync-rules';
|
|
3
|
-
import { Readable } from 'stream';
|
|
4
2
|
import Negotiator from 'negotiator';
|
|
3
|
+
import { Readable } from 'stream';
|
|
5
4
|
import * as sync from '../../sync/sync-index.js';
|
|
6
5
|
import * as util from '../../util/util-index.js';
|
|
7
6
|
import { authUser } from '../auth.js';
|
|
8
7
|
import { routeDefinition } from '../router.js';
|
|
9
8
|
import { APIMetric } from '@powersync/service-types';
|
|
9
|
+
import { maybeCompressResponseStream } from '../compression.js';
|
|
10
10
|
export var SyncRoutes;
|
|
11
11
|
(function (SyncRoutes) {
|
|
12
12
|
SyncRoutes["STREAM"] = "/sync/stream";
|
|
@@ -26,9 +26,11 @@ export const syncStreamed = routeDefinition({
|
|
|
26
26
|
const userAgent = headers['x-user-agent'] ?? headers['user-agent'];
|
|
27
27
|
const clientId = payload.params.client_id;
|
|
28
28
|
const streamStart = Date.now();
|
|
29
|
+
const negotiator = new Negotiator(payload.request);
|
|
29
30
|
// This falls back to JSON unless there's preference for the bson-stream in the Accept header.
|
|
30
|
-
const useBson = payload.request.headers.accept
|
|
31
|
-
|
|
31
|
+
const useBson = payload.request.headers.accept
|
|
32
|
+
? negotiator.mediaType(supportedContentTypes) == concatenatedBsonContentType
|
|
33
|
+
: false;
|
|
32
34
|
logger.defaultMeta = {
|
|
33
35
|
...logger.defaultMeta,
|
|
34
36
|
user_agent: userAgent,
|
|
@@ -43,8 +45,6 @@ export const syncStreamed = routeDefinition({
|
|
|
43
45
|
description: 'Service temporarily unavailable'
|
|
44
46
|
});
|
|
45
47
|
}
|
|
46
|
-
const params = payload.params;
|
|
47
|
-
const syncParams = new RequestParameters(payload.context.token_payload, payload.params.parameters ?? {});
|
|
48
48
|
const bucketStorage = await storageEngine.activeBucketStorage.getActiveStorage();
|
|
49
49
|
if (bucketStorage == null) {
|
|
50
50
|
throw new errors.ServiceError({
|
|
@@ -61,19 +61,23 @@ export const syncStreamed = routeDefinition({
|
|
|
61
61
|
const syncLines = sync.streamResponse({
|
|
62
62
|
syncContext: syncContext,
|
|
63
63
|
bucketStorage,
|
|
64
|
-
syncRules:
|
|
65
|
-
|
|
66
|
-
|
|
64
|
+
syncRules: {
|
|
65
|
+
syncRules,
|
|
66
|
+
version: bucketStorage.group_id
|
|
67
|
+
},
|
|
68
|
+
params: payload.params,
|
|
67
69
|
token: payload.context.token_payload,
|
|
68
70
|
tracker,
|
|
69
71
|
signal: controller.signal,
|
|
70
|
-
logger
|
|
72
|
+
logger,
|
|
73
|
+
isEncodingAsBson: useBson
|
|
71
74
|
});
|
|
72
75
|
const byteContents = useBson ? sync.bsonLines(syncLines) : sync.ndjson(syncLines);
|
|
73
|
-
const
|
|
76
|
+
const plainStream = Readable.from(sync.transformToBytesTracked(byteContents, tracker), {
|
|
74
77
|
objectMode: false,
|
|
75
78
|
highWaterMark: 16 * 1024
|
|
76
79
|
});
|
|
80
|
+
const { stream, encodingHeaders } = maybeCompressResponseStream(negotiator, plainStream, tracker);
|
|
77
81
|
// Best effort guess on why the stream was closed.
|
|
78
82
|
// We use the `??=` operator everywhere, so that we catch the first relevant
|
|
79
83
|
// event, which is usually the most specific.
|
|
@@ -102,7 +106,8 @@ export const syncStreamed = routeDefinition({
|
|
|
102
106
|
return new router.RouterResponse({
|
|
103
107
|
status: 200,
|
|
104
108
|
headers: {
|
|
105
|
-
'Content-Type': useBson ? concatenatedBsonContentType : ndJsonContentType
|
|
109
|
+
'Content-Type': useBson ? concatenatedBsonContentType : ndJsonContentType,
|
|
110
|
+
...encodingHeaders
|
|
106
111
|
},
|
|
107
112
|
data: stream,
|
|
108
113
|
afterSend: async (details) => {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"sync-stream.js","sourceRoot":"","sources":["../../../src/routes/endpoints/sync-stream.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,
|
|
1
|
+
{"version":3,"file":"sync-stream.js","sourceRoot":"","sources":["../../../src/routes/endpoints/sync-stream.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,mCAAmC,CAAC;AACtF,OAAO,UAAU,MAAM,YAAY,CAAC;AACpC,OAAO,EAAE,QAAQ,EAAE,MAAM,QAAQ,CAAC;AAElC,OAAO,KAAK,IAAI,MAAM,0BAA0B,CAAC;AACjD,OAAO,KAAK,IAAI,MAAM,0BAA0B,CAAC;AAEjD,OAAO,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AACtC,OAAO,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AAE/C,OAAO,EAAE,SAAS,EAAE,MAAM,0BAA0B,CAAC;AACrD,OAAO,EAAE,2BAA2B,EAAE,MAAM,mBAAmB,CAAC;AAEhE,MAAM,CAAN,IAAY,UAEX;AAFD,WAAY,UAAU;IACpB,qCAAuB,CAAA;AACzB,CAAC,EAFW,UAAU,KAAV,UAAU,QAErB;AAED,MAAM,iBAAiB,GAAG,sBAAsB,CAAC;AACjD,MAAM,2BAA2B,GAAG,uCAAuC,CAAC;AAC5E,MAAM,qBAAqB,GAAG,CAAC,iBAAiB,EAAE,2BAA2B,CAAC,CAAC;AAE/E,MAAM,CAAC,MAAM,YAAY,GAAG,eAAe,CAAC;IAC1C,IAAI,EAAE,UAAU,CAAC,MAAM;IACvB,MAAM,EAAE,MAAM,CAAC,UAAU,CAAC,IAAI;IAC9B,SAAS,EAAE,QAAQ;IACnB,SAAS,EAAE,MAAM,CAAC,sBAAsB,CAAC,IAAI,CAAC,oBAAoB,EAAE,EAAE,eAAe,EAAE,IAAI,EAAE,CAAC;IAC9F,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE;QACzB,MAAM,EAAE,eAAe,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,OAAO,CAAC;QACpD,MAAM,EAAE,YAAY,EAAE,aAAa,EAAE,aAAa,EAAE,WAAW,EAAE,GAAG,eAAe,CAAC;QACpF,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC;QACxC,MAAM,SAAS,GAAG,OAAO,CAAC,cAAc,CAAC,IAAI,OAAO,CAAC,YAAY,CAAC,CAAC;QACnE,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,SAAS,CAAC;QAC1C,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC/B,MAAM,UAAU,GAAG,IAAI,UAAU,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QACnD,8FAA8F;QAC9F,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM;YAC5C,CAAC,CAAC,UAAU,CAAC,SAAS,CAAC,qBAAqB,CAAC,IAAI,2BAA2B;YAC5E,CAAC,CAAC,KAAK,CAAC;QAEV,MAAM,CAAC,WAAW,GAAG;YACnB,GAAG,MAAM,CAAC,WAAW;YACrB,UAAU,EAAE,SAAS;YACrB,SAAS,EAAE,QAAQ;YACnB,OAAO,EAAE,OAAO,CAAC,OAAO,CAAC,OAAO;YAChC,IAAI,EAAE,OAAO;SACd,CAAC;QAEF,IAAI,YAAY,CAAC,MAAM,EAAE,CAAC;YACxB,MAAM,IAAI,MAAM,CAAC,YAAY,CAAC;gBAC5B,MAAM,EAAE,GAAG;gBACX,IAAI,EAAE,SAAS,CAAC,WAAW;gBAC3B,WAAW,EAAE,iCAAiC;aAC/C,CAAC,CAAC;QACL,CAAC;QAED,MAAM,aAAa,GAAG,MAAM,aAAa,CAAC,mBAAmB,CAAC,gBAAgB,EAAE,CAAC;QAEjF,IAAI,aAAa,IAAI,IAAI,EAAE,CAAC;YAC1B,MAAM,IAAI,MAAM,CAAC,YAAY,CAAC;gBAC5B,MAAM,EAAE,GAAG;gBACX,IAAI,EAAE,SAAS,CAAC,WAAW;gBAC3B,WAAW,EAAE,yBAAyB;aACvC,CAAC,CAAC;QACL,CAAC;QAED,MAAM,SAAS,GAAG,aAAa,CAAC,kBAAkB,CAAC,YAAY,CAAC,MAAM,EAAE,CAAC,wBAAwB,EAAE,CAAC,CAAC;QAErG,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;QACzC,MAAM,OAAO,GAAG,IAAI,IAAI,CAAC,cAAc,CAAC,aAAa,CAAC,CAAC;QACvD,IAAI,CAAC;YACH,aAAa,CAAC,gBAAgB,CAAC,SAAS,CAAC,sBAAsB,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;YACxE,MAAM,SAAS,GAAG,IAAI,CAAC,cAAc,CAAC;gBACpC,WAAW,EAAE,WAAW;gBACxB,aAAa;gBACb,SAAS,EAAE;oBACT,SAAS;oBACT,OAAO,EAAE,aAAa,CAAC,QAAQ;iBAChC;gBACD,MAAM,EAAE,OAAO,CAAC,MAAM;gBACtB,KAAK,EAAE,OAAO,CAAC,OAAO,CAAC,aAAc;gBACrC,OAAO;gBACP,MAAM,EAAE,UAAU,CAAC,MAAM;gBACzB,MAAM;gBACN,gBAAgB,EAAE,OAAO;aAC1B,CAAC,CAAC;YAEH,MAAM,YAAY,GAAG,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;YAClF,MAAM,WAAW,GAAG,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,uBAAuB,CAAC,YAAY,EAAE,OAAO,CAAC,EAAE;gBACrF,UAAU,EAAE,KAAK;gBACjB,aAAa,EAAE,EAAE,GAAG,IAAI;aACzB,CAAC,CAAC;YACH,MAAM,EAAE,MAAM,EAAE,eAAe,EAAE,GAAG,2BAA2B,CAAC,UAAU,EAAE,WAAW,EAAE,OAAO,CAAC,CAAC;YAElG,kDAAkD;YAClD,4EAA4E;YAC5E,6CAA6C;YAC7C,IAAI,WAAW,GAAuB,SAAS,CAAC;YAEhD,MAAM,UAAU,GAAG,YAAY,CAAC,cAAc,CAAC,GAAG,EAAE;gBAClD,uDAAuD;gBACvD,UAAU,CAAC,KAAK,EAAE,CAAC;gBACnB,WAAW,KAAK,kBAAkB,CAAC;gBACnC,MAAM,CAAC,OAAO,CAAC,IAAI,KAAK,CAAC,sBAAsB,CAAC,CAAC,CAAC;YACpD,CAAC,CAAC,CAAC;YAEH,MAAM,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE;gBACpB,2CAA2C;gBAC3C,WAAW,KAAK,wBAAwB,CAAC;YAC3C,CAAC,CAAC,CAAC;YAEH,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE;gBACtB,UAAU,EAAE,CAAC;YACf,CAAC,CAAC,CAAC;YAEH,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,EAAE;gBAC3B,WAAW,KAAK,cAAc,CAAC;gBAC/B,UAAU,CAAC,KAAK,EAAE,CAAC;gBACnB,oDAAoD;gBACpD,IAAI,KAAK,CAAC,OAAO,IAAI,sBAAsB,EAAE,CAAC;oBAC5C,MAAM,CAAC,KAAK,CAAC,+BAA+B,EAAE,KAAK,CAAC,CAAC;gBACvD,CAAC;YACH,CAAC,CAAC,CAAC;YAEH,OAAO,IAAI,MAAM,CAAC,cAAc,CAAC;gBAC/B,MAAM,EAAE,GAAG;gBACX,OAAO,EAAE;oBACP,cAAc,EAAE,OAAO,CAAC,CAAC,CAAC,2BAA2B,CAAC,CAAC,CAAC,iBAAiB;oBACzE,GAAG,eAAe;iBACnB;gBACD,IAAI,EAAE,MAAM;gBACZ,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE;oBAC3B,IAAI,OAAO,CAAC,YAAY,EAAE,CAAC;wBACzB,WAAW,KAAK,uBAAuB,CAAC;oBAC1C,CAAC;oBACD,UAAU,CAAC,KAAK,EAAE,CAAC;oBACnB,aAAa,CAAC,gBAAgB,CAAC,SAAS,CAAC,sBAAsB,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;oBACzE,MAAM,CAAC,IAAI,CAAC,sBAAsB,EAAE;wBAClC,GAAG,OAAO,CAAC,UAAU,EAAE;wBACvB,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,WAAW;wBACnC,YAAY,EAAE,WAAW,IAAI,SAAS;qBACvC,CAAC,CAAC;gBACL,CAAC;aACF,CAAC,CAAC;QACL,CAAC;QAAC,OAAO,EAAE,EAAE,CAAC;YACZ,UAAU,CAAC,KAAK,EAAE,CAAC;YACnB,aAAa,CAAC,gBAAgB,CAAC,SAAS,CAAC,sBAAsB,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QAC3E,CAAC;IACH,CAAC;CACF,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,kBAAkB,GAAG,CAAC,YAAY,CAAC,CAAC"}
|
|
@@ -34,4 +34,4 @@ export declare enum SyncRuleState {
|
|
|
34
34
|
}
|
|
35
35
|
export declare const DEFAULT_DOCUMENT_BATCH_LIMIT = 1000;
|
|
36
36
|
export declare const DEFAULT_DOCUMENT_CHUNK_LIMIT_BYTES: number;
|
|
37
|
-
export declare function mergeToast(record: ToastableSqliteRow
|
|
37
|
+
export declare function mergeToast<V>(record: ToastableSqliteRow<V>, persisted: ToastableSqliteRow<V>): ToastableSqliteRow<V>;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"BucketStorage.js","sourceRoot":"","sources":["../../src/storage/BucketStorage.ts"],"names":[],"mappings":"AAEA,MAAM,CAAN,IAAY,aAkCX;AAlCD,WAAY,aAAa;IACvB;;;;;OAKG;IACH,0CAAyB,CAAA;IAEzB;;;;OAIG;IACH,kCAAiB,CAAA;IACjB;;;OAGG;IACH,8BAAa,CAAA;IACb;;;OAGG;IACH,0CAAyB,CAAA;IAEzB;;;;;;OAMG;IACH,oCAAmB,CAAA;AACrB,CAAC,EAlCW,aAAa,KAAb,aAAa,QAkCxB;AAED,MAAM,CAAC,MAAM,4BAA4B,GAAG,IAAI,CAAC;AACjD,MAAM,CAAC,MAAM,kCAAkC,GAAG,CAAC,GAAG,IAAI,GAAG,IAAI,CAAC;AAElE,MAAM,UAAU,UAAU,
|
|
1
|
+
{"version":3,"file":"BucketStorage.js","sourceRoot":"","sources":["../../src/storage/BucketStorage.ts"],"names":[],"mappings":"AAEA,MAAM,CAAN,IAAY,aAkCX;AAlCD,WAAY,aAAa;IACvB;;;;;OAKG;IACH,0CAAyB,CAAA;IAEzB;;;;OAIG;IACH,kCAAiB,CAAA;IACjB;;;OAGG;IACH,8BAAa,CAAA;IACb;;;OAGG;IACH,0CAAyB,CAAA;IAEzB;;;;;;OAMG;IACH,oCAAmB,CAAA;AACrB,CAAC,EAlCW,aAAa,KAAb,aAAa,QAkCxB;AAED,MAAM,CAAC,MAAM,4BAA4B,GAAG,IAAI,CAAC;AACjD,MAAM,CAAC,MAAM,kCAAkC,GAAG,CAAC,GAAG,IAAI,GAAG,IAAI,CAAC;AAElE,MAAM,UAAU,UAAU,CAAI,MAA6B,EAAE,SAAgC;IAC3F,MAAM,SAAS,GAA0B,EAAE,CAAC;IAC5C,KAAK,IAAI,GAAG,IAAI,MAAM,EAAE,CAAC;QACvB,IAAI,OAAO,MAAM,CAAC,GAAG,CAAC,IAAI,WAAW,EAAE,CAAC;YACtC,SAAS,CAAC,GAAG,CAAC,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC;QAClC,CAAC;aAAM,CAAC;YACN,SAAS,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC;QAC/B,CAAC;IACH,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC"}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { ObserverClient } from '@powersync/lib-services-framework';
|
|
2
|
-
import { EvaluatedParameters, EvaluatedRow,
|
|
2
|
+
import { EvaluatedParameters, EvaluatedRow, SqliteInputRow, ToastableSqliteRow } from '@powersync/service-sync-rules';
|
|
3
3
|
import { BSON } from 'bson';
|
|
4
4
|
import { ReplicationEventPayload } from './ReplicationEventPayload.js';
|
|
5
5
|
import { SourceTable, TableSnapshotStatus } from './SourceTable.js';
|
|
@@ -108,7 +108,7 @@ export interface SaveInsert {
|
|
|
108
108
|
sourceTable: SourceTable;
|
|
109
109
|
before?: undefined;
|
|
110
110
|
beforeReplicaId?: undefined;
|
|
111
|
-
after:
|
|
111
|
+
after: SqliteInputRow;
|
|
112
112
|
afterReplicaId: ReplicaId;
|
|
113
113
|
}
|
|
114
114
|
export interface SaveUpdate {
|
|
@@ -117,7 +117,7 @@ export interface SaveUpdate {
|
|
|
117
117
|
/**
|
|
118
118
|
* This is only present when the id has changed, and will only contain replica identity columns.
|
|
119
119
|
*/
|
|
120
|
-
before?:
|
|
120
|
+
before?: SqliteInputRow;
|
|
121
121
|
beforeReplicaId?: ReplicaId;
|
|
122
122
|
/**
|
|
123
123
|
* A null value means null column.
|
|
@@ -130,7 +130,7 @@ export interface SaveUpdate {
|
|
|
130
130
|
export interface SaveDelete {
|
|
131
131
|
tag: SaveOperationTag.DELETE;
|
|
132
132
|
sourceTable: SourceTable;
|
|
133
|
-
before?:
|
|
133
|
+
before?: SqliteInputRow;
|
|
134
134
|
beforeReplicaId: ReplicaId;
|
|
135
135
|
after?: undefined;
|
|
136
136
|
afterReplicaId?: undefined;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"BucketStorageBatch.js","sourceRoot":"","sources":["../../src/storage/BucketStorageBatch.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"BucketStorageBatch.js","sourceRoot":"","sources":["../../src/storage/BucketStorageBatch.ts"],"names":[],"mappings":"AAcA,MAAM,CAAC,MAAM,mCAAmC,GAAqC;IACnF,sBAAsB,EAAE,IAAI;IAC5B,uBAAuB,EAAE,IAAI;CAC9B,CAAC;AAgHF,MAAM,CAAN,IAAY,gBAIX;AAJD,WAAY,gBAAgB;IAC1B,qCAAiB,CAAA;IACjB,qCAAiB,CAAA;IACjB,qCAAiB,CAAA;AACnB,CAAC,EAJW,gBAAgB,KAAhB,gBAAgB,QAI3B"}
|
|
@@ -1,28 +1,12 @@
|
|
|
1
1
|
import { BucketChecksum } from '../util/protocol-types.js';
|
|
2
|
-
import { ChecksumMap, InternalOpId } from '../util/utils.js';
|
|
3
|
-
export interface PartialChecksum {
|
|
4
|
-
bucket: string;
|
|
5
|
-
/**
|
|
6
|
-
* 32-bit unsigned hash.
|
|
7
|
-
*/
|
|
8
|
-
partialChecksum: number;
|
|
9
|
-
/**
|
|
10
|
-
* Count of operations - informational only.
|
|
11
|
-
*/
|
|
12
|
-
partialCount: number;
|
|
13
|
-
/**
|
|
14
|
-
* True if the queried operations contains (starts with) a CLEAR
|
|
15
|
-
* operation, indicating that the partial checksum is the full
|
|
16
|
-
* checksum, and must not be added to a previously-cached checksum.
|
|
17
|
-
*/
|
|
18
|
-
isFullChecksum: boolean;
|
|
19
|
-
}
|
|
2
|
+
import { ChecksumMap, InternalOpId, PartialChecksum } from '../util/utils.js';
|
|
20
3
|
export interface FetchPartialBucketChecksum {
|
|
21
4
|
bucket: string;
|
|
22
5
|
start?: InternalOpId;
|
|
23
6
|
end: InternalOpId;
|
|
24
7
|
}
|
|
25
|
-
export type
|
|
8
|
+
export type PartialOrFullChecksum = PartialChecksum | BucketChecksum;
|
|
9
|
+
export type PartialChecksumMap = Map<string, PartialOrFullChecksum>;
|
|
26
10
|
export type FetchChecksums = (batch: FetchPartialBucketChecksum[]) => Promise<PartialChecksumMap>;
|
|
27
11
|
export interface ChecksumCacheOptions {
|
|
28
12
|
/**
|
|
@@ -56,6 +40,7 @@ export declare class ChecksumCache {
|
|
|
56
40
|
private bucketCheckpoints;
|
|
57
41
|
private fetchChecksums;
|
|
58
42
|
constructor(options: ChecksumCacheOptions);
|
|
43
|
+
clear(): void;
|
|
59
44
|
getChecksums(checkpoint: InternalOpId, buckets: string[]): Promise<BucketChecksum[]>;
|
|
60
45
|
/**
|
|
61
46
|
* Get bucket checksums for a checkpoint.
|
|
@@ -65,6 +65,10 @@ export class ChecksumCache {
|
|
|
65
65
|
allowStale: false
|
|
66
66
|
});
|
|
67
67
|
}
|
|
68
|
+
clear() {
|
|
69
|
+
this.cache.clear();
|
|
70
|
+
this.bucketCheckpoints.clear();
|
|
71
|
+
}
|
|
68
72
|
async getChecksums(checkpoint, buckets) {
|
|
69
73
|
const checksums = await this.getChecksumMap(checkpoint, buckets);
|
|
70
74
|
// Return results in the same order as the request
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ChecksumCache.js","sourceRoot":"","sources":["../../src/storage/ChecksumCache.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAC;AAClD,OAAO,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AAEzC,OAAO,EAAE,kBAAkB,
|
|
1
|
+
{"version":3,"file":"ChecksumCache.js","sourceRoot":"","sources":["../../src/storage/ChecksumCache.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAC;AAClD,OAAO,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AAEzC,OAAO,EAAE,kBAAkB,EAA8C,MAAM,kBAAkB,CAAC;AAgClG,+DAA+D;AAC/D,MAAM,gBAAgB,GAAG,OAAO,CAAC;AAEjC,MAAM,MAAM,GAAG,SAAS,CAAC;AAEzB;;;;;;;;GAQG;AACH,MAAM,OAAO,aAAa;IACxB;;OAEG;IACK,KAAK,CAAyD;IACtE;;OAEG;IACK,iBAAiB,GAAG,IAAI,GAAG,EAA8B,CAAC;IAE1D,cAAc,CAAiB;IAEvC,YAAY,OAA6B;QACvC,IAAI,CAAC,cAAc,GAAG,OAAO,CAAC,cAAc,CAAC;QAE7C,IAAI,CAAC,KAAK,GAAG,IAAI,QAAQ,CAA+C;YACtE,GAAG,EAAE,OAAO,CAAC,OAAO,IAAI,gBAAgB;YACxC,WAAW,EAAE,KAAK,EAAE,QAAQ,EAAE,WAAW,EAAE,OAAO,EAAE,EAAE;gBACpD,oDAAoD;gBACpD,sEAAsE;gBACtE,MAAM,EAAE,MAAM,EAAE,GAAG,aAAa,CAAC,QAAQ,CAAC,CAAC;gBAC3C,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;gBAEnD,uDAAuD;gBACvD,IAAI,aAAa,GAAG,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;gBACvD,IAAI,aAAa,IAAI,IAAI,EAAE,CAAC;oBAC1B,aAAa,GAAG,IAAI,UAAU,EAAE,CAAC;oBACjC,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC,MAAM,EAAE,aAAa,CAAC,CAAC;gBACpD,CAAC;gBACD,aAAa,CAAC,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;gBACjD,OAAO,MAAM,CAAC;YAChB,CAAC;YAED,OAAO,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE,EAAE;gBACtB,2DAA2D;gBAC3D,MAAM,EAAE,UAAU,EAAE,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC;gBAC1C,MAAM,aAAa,GAAG,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;gBAC/D,IAAI,aAAa,IAAI,IAAI,EAAE,CAAC;oBAC1B,OAAO;gBACT,CAAC;gBACD,aAAa,CAAC,iBAAiB,CAAC,UAAU,CAAC,CAAC;gBAC5C,IAAI,aAAa,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;oBAC9B,IAAI,CAAC,iBAAiB,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;gBAC9C,CAAC;YACH,CAAC;YAED,cAAc,EAAE,IAAI;YAEpB,8EAA8E;YAC9E,oCAAoC;YACpC,gBAAgB,EAAE,IAAI;YAEtB,0DAA0D;YAC1D,0DAA0D;YAC1D,oCAAoC;YACpC,GAAG,EAAE,MAAM;YACX,aAAa,EAAE,KAAK;YACpB,UAAU,EAAE,KAAK;SAClB,CAAC,CAAC;IACL,CAAC;IAED,KAAK;QACH,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;QACnB,IAAI,CAAC,iBAAiB,CAAC,KAAK,EAAE,CAAC;IACjC,CAAC;IAED,KAAK,CAAC,YAAY,CAAC,UAAwB,EAAE,OAAiB;QAC5D,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;QACjE,kDAAkD;QAClD,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,SAAS,CAAC,GAAG,CAAC,MAAM,CAAE,CAAC,CAAC;IACzD,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,cAAc,CAAC,UAAwB,EAAE,OAAiB;QAC9D,oEAAoE;QACpE,IAAI,OAAO,GAAG,IAAI,GAAG,EAAU,CAAC;QAEhC,wBAAwB;QACxB,IAAI,YAAY,GAAG,IAAI,GAAG,EAA0B,CAAC;QAErD,4CAA4C;QAC5C,IAAI,YAAyB,CAAC;QAC9B,IAAI,WAAgC,CAAC;QACrC,IAAI,YAAY,GAAG,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACvD,YAAY,GAAG,OAAO,CAAC;YACvB,WAAW,GAAG,MAAM,CAAC;QACvB,CAAC,CAAC,CAAC;QAEH,0EAA0E;QAC1E,IAAI,YAAY,GAAG,IAAI,GAAG,EAA0B,CAAC;QAErD,MAAM,OAAO,GAAyB;YACpC,KAAK,CAAC,KAAK,CAAC,MAAM;gBAChB,MAAM,YAAY,CAAC;gBACnB,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;oBACzB,sBAAsB;oBACtB,MAAM,IAAI,KAAK,CAAC,qBAAqB,MAAM,EAAE,CAAC,CAAC;gBACjD,CAAC;gBACD,MAAM,QAAQ,GAAG,YAAY,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;gBAC1C,IAAI,QAAQ,IAAI,IAAI,EAAE,CAAC;oBACrB,sBAAsB;oBACtB,MAAM,IAAI,KAAK,CAAC,uCAAuC,MAAM,EAAE,CAAC,CAAC;gBACnE,CAAC;gBACD,OAAO,QAAQ,CAAC;YAClB,CAAC;YACD,UAAU,EAAE,MAAM,CAAC,UAAU,CAAC;SAC/B,CAAC;QAEF,+DAA+D;QAC/D,IAAI,cAAc,GAAiD,IAAI,CAAC;QAExE,IAAI,CAAC;YACH,kCAAkC;YAClC,IAAI,kBAAkB,GAAoB,EAAE,CAAC;YAE7C,KAAK,IAAI,MAAM,IAAI,OAAO,EAAE,CAAC;gBAC3B,MAAM,QAAQ,GAAG,YAAY,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;gBAClD,IAAI,MAAM,GAAoC,EAAE,CAAC;gBACjD,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,QAAQ,EAAE,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,EAAE;oBAC5F,IAAI,SAAS,IAAI,IAAI,EAAE,CAAC;wBACtB,sBAAsB;wBACtB,MAAM,IAAI,KAAK,CAAC,+BAA+B,QAAQ,EAAE,CAAC,CAAC;oBAC7D,CAAC;oBACD,YAAY,CAAC,GAAG,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;gBACtC,CAAC,CAAC,CAAC;gBACH,kBAAkB,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;gBAC3B,IAAI,MAAM,CAAC,KAAK,IAAI,KAAK,IAAI,MAAM,CAAC,KAAK,IAAI,UAAU,EAAE,CAAC;oBACxD,2EAA2E;oBAC3E,uBAAuB;oBACvB,yDAAyD;gBAC3D,CAAC;qBAAM,CAAC;oBACN,2CAA2C;oBAC3C,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;gBACtB,CAAC;YACH,CAAC;YACD,6DAA6D;YAC7D,wDAAwD;YACxD,cAAc,GAAG,OAAO,CAAC,UAAU,CAAC,kBAAkB,CAAC,CAAC;YAExD,IAAI,OAAO,CAAC,IAAI,IAAI,CAAC,EAAE,CAAC;gBACtB,wCAAwC;gBACxC,YAAY,EAAE,CAAC;YACjB,CAAC;iBAAM,CAAC;gBACN,IAAI,cAAc,GAAiC,EAAE,CAAC;gBACtD,mEAAmE;gBACnE,IAAI,GAAG,GAAG,IAAI,GAAG,EAA0B,CAAC;gBAE5C,KAAK,IAAI,MAAM,IAAI,OAAO,EAAE,CAAC;oBAC3B,IAAI,aAAa,GAAsC,IAAI,CAAC;oBAC5D,MAAM,aAAa,GAAG,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;oBACzD,IAAI,aAAa,IAAI,IAAI,EAAE,CAAC;wBAC1B,uDAAuD;wBACvD,IAAI,IAAI,GAAG,aAAa,CAAC,iBAAiB,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;wBAC/D,MAAM,KAAK,GAAG,aAAa,CAAC,KAAK,EAAE,CAAC;wBACpC,OAAO,IAAI,CAAC,YAAY,EAAE,EAAE,CAAC;4BAC3B,MAAM,EAAE,GAAG,IAAI,CAAC,OAAO,CAAC;4BACxB,MAAM,QAAQ,GAAG,YAAY,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;4BAC1C,mCAAmC;4BACnC,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;4BACzC,iEAAiE;4BACjE,uEAAuE;4BACvE,8DAA8D;4BAC9D,8CAA8C;4BAC9C,IAAI,MAAM,IAAI,IAAI,EAAE,CAAC;gCACnB,2DAA2D;gCAC3D,aAAa,GAAG;oCACd,MAAM;oCACN,KAAK,EAAE,EAAE;oCACT,GAAG,EAAE,UAAU;iCAChB,CAAC;gCACF,GAAG,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;gCACxB,MAAM;4BACR,CAAC;4BAED,IAAI,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;gCACvB,yBAAyB;gCACzB,MAAM;4BACR,CAAC;4BACD,oBAAoB;4BACpB,IAAI,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;wBACpB,CAAC;oBACH,CAAC;oBAED,IAAI,aAAa,IAAI,IAAI,EAAE,CAAC;wBAC1B,+DAA+D;wBAC/D,aAAa,GAAG;4BACd,MAAM;4BACN,GAAG,EAAE,UAAU;yBAChB,CAAC;wBACF,GAAG,CAAC,GAAG,CAAC,MAAM,EAAE;4BACd,MAAM;4BACN,QAAQ,EAAE,CAAC;4BACX,KAAK,EAAE,CAAC;yBACT,CAAC,CAAC;oBACL,CAAC;oBACD,cAAc,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;gBACrC,CAAC;gBAED,wCAAwC;gBACxC,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,cAAc,CAAC,CAAC;gBAE1D,KAAK,IAAI,MAAM,IAAI,OAAO,EAAE,CAAC;oBAC3B,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;oBACnC,MAAM,KAAK,GAAG,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;oBAC9B,IAAI,KAAK,IAAI,IAAI,EAAE,CAAC;wBAClB,sBAAsB;wBACtB,MAAM,IAAI,KAAK,CAAC,kBAAkB,MAAM,EAAE,CAAC,CAAC;oBAC9C,CAAC;oBACD,mDAAmD;oBACnD,6DAA6D;oBAC7D,MAAM,KAAK,GAAG,kBAAkB,CAAC,KAAK,EAAE,MAAM,IAAI,IAAI,CAAC,CAAC;oBACxD,YAAY,CAAC,GAAG,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;gBAClC,CAAC;gBAED,6DAA6D;gBAC7D,YAAY,EAAE,CAAC;YACjB,CAAC;QACH,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,wDAAwD;YACxD,iFAAiF;YACjF,uBAAuB;YACvB,WAAW,CAAC,CAAC,CAAC,CAAC;YAEf,yFAAyF;YACzF,6BAA6B;YAC7B,MAAM,cAAc,CAAC;YAErB,MAAM,CAAC,CAAC;QACV,CAAC;QAED,gDAAgD;QAChD,MAAM,cAAc,GAAG,CAAC,MAAM,cAAc,CAAC,IAAI,EAAE,CAAC;QACpD,8BAA8B;QAC9B,KAAK,IAAI,MAAM,IAAI,cAAc,EAAE,CAAC;YAClC,IAAI,MAAM,CAAC,MAAM,IAAI,UAAU,EAAE,CAAC;gBAChC,MAAM,MAAM,CAAC,MAAM,CAAC;YACtB,CAAC;QACH,CAAC;QAED,IAAI,YAAY,CAAC,IAAI,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;YACxC,oBAAoB;YACpB,MAAM,IAAI,KAAK,CAAC,4BAA4B,YAAY,CAAC,IAAI,OAAO,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;QACxF,CAAC;QACD,OAAO,YAAY,CAAC;IACtB,CAAC;CACF;AAED,SAAS,YAAY,CAAC,UAAiC,EAAE,MAAc;IACrE,OAAO,GAAG,UAAU,IAAI,MAAM,EAAE,CAAC;AACnC,CAAC;AAED,SAAS,aAAa,CAAC,GAAW;IAChC,MAAM,KAAK,GAAG,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IAC/B,OAAO,EAAE,UAAU,EAAE,MAAM,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,EAAE,MAAM,EAAE,GAAG,CAAC,SAAS,CAAC,KAAK,GAAG,CAAC,CAAC,EAAE,CAAC;AAC3F,CAAC"}
|
|
@@ -3,8 +3,8 @@ import { SourceTable } from './SourceTable.js';
|
|
|
3
3
|
import { BucketStorageBatch, SaveOp } from './BucketStorageBatch.js';
|
|
4
4
|
export type EventData = {
|
|
5
5
|
op: SaveOp;
|
|
6
|
-
before?: sync_rules.
|
|
7
|
-
after?: sync_rules.
|
|
6
|
+
before?: sync_rules.SqliteInputRow;
|
|
7
|
+
after?: sync_rules.SqliteInputRow;
|
|
8
8
|
};
|
|
9
9
|
export type ReplicationEventPayload = {
|
|
10
10
|
batch: BucketStorageBatch;
|
|
@@ -46,6 +46,10 @@ export interface SyncRulesBucketStorage extends ObserverClient<SyncRulesBucketSt
|
|
|
46
46
|
*/
|
|
47
47
|
reportError(e: any): Promise<void>;
|
|
48
48
|
compact(options?: CompactOptions): Promise<void>;
|
|
49
|
+
/**
|
|
50
|
+
* Lightweight "compact" process to populate the checksum cache, if any.
|
|
51
|
+
*/
|
|
52
|
+
populatePersistentChecksumCache(options?: Pick<CompactOptions, 'signal' | 'maxOpId'>): Promise<void>;
|
|
49
53
|
getCheckpoint(): Promise<ReplicationCheckpoint>;
|
|
50
54
|
/**
|
|
51
55
|
* Given two checkpoints, return the changes in bucket data and parameters that may have occurred
|
|
@@ -82,6 +86,10 @@ export interface SyncRulesBucketStorage extends ObserverClient<SyncRulesBucketSt
|
|
|
82
86
|
* Returns zero checksums for any buckets not found.
|
|
83
87
|
*/
|
|
84
88
|
getChecksums(checkpoint: util.InternalOpId, buckets: string[]): Promise<util.ChecksumMap>;
|
|
89
|
+
/**
|
|
90
|
+
* Clear checksum cache. Primarily intended for tests.
|
|
91
|
+
*/
|
|
92
|
+
clearChecksumCache(): void;
|
|
85
93
|
}
|
|
86
94
|
export interface SyncRulesBucketStorageListener {
|
|
87
95
|
batchStarted: (batch: BucketStorageBatch) => void;
|
|
@@ -166,6 +174,7 @@ export interface CompactOptions {
|
|
|
166
174
|
* Internal/testing use: Cache size for compacting parameters.
|
|
167
175
|
*/
|
|
168
176
|
compactParameterCacheLimit?: number;
|
|
177
|
+
signal?: AbortSignal;
|
|
169
178
|
}
|
|
170
179
|
export interface ClearStorageOptions {
|
|
171
180
|
signal?: AbortSignal;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"SyncRulesBucketStorage.js","sourceRoot":"","sources":["../../src/storage/SyncRulesBucketStorage.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"SyncRulesBucketStorage.js","sourceRoot":"","sources":["../../src/storage/SyncRulesBucketStorage.ts"],"names":[],"mappings":"AA+SA,MAAM,CAAC,MAAM,yBAAyB,GAAsB;IAC1D,kBAAkB,EAAE,IAAI,GAAG,EAAU;IACrC,qBAAqB,EAAE,IAAI;IAC3B,uBAAuB,EAAE,IAAI,GAAG,EAAU;IAC1C,0BAA0B,EAAE,IAAI;CACjC,CAAC"}
|
|
@@ -1,18 +1,20 @@
|
|
|
1
|
-
import { BucketDescription, RequestParameters, SqlSyncRules } from '@powersync/service-sync-rules';
|
|
1
|
+
import { BucketDescription, BucketSource, RequestJwtPayload, RequestParameters, ResolvedBucket, SqlSyncRules } from '@powersync/service-sync-rules';
|
|
2
2
|
import * as storage from '../storage/storage-index.js';
|
|
3
3
|
import * as util from '../util/util-index.js';
|
|
4
4
|
import { Logger } from '@powersync/lib-services-framework';
|
|
5
|
+
import { QuerierError } from '@powersync/service-sync-rules/src/BucketParameterQuerier.js';
|
|
5
6
|
import { SyncContext } from './SyncContext.js';
|
|
7
|
+
export interface VersionedSyncRules {
|
|
8
|
+
syncRules: SqlSyncRules;
|
|
9
|
+
version: number;
|
|
10
|
+
}
|
|
6
11
|
export interface BucketChecksumStateOptions {
|
|
7
12
|
syncContext: SyncContext;
|
|
8
13
|
bucketStorage: BucketChecksumStateStorage;
|
|
9
|
-
syncRules:
|
|
10
|
-
|
|
14
|
+
syncRules: VersionedSyncRules;
|
|
15
|
+
tokenPayload: RequestJwtPayload;
|
|
16
|
+
syncRequest: util.StreamingSyncRequest;
|
|
11
17
|
logger?: Logger;
|
|
12
|
-
initialBucketPositions?: {
|
|
13
|
-
name: string;
|
|
14
|
-
after: util.InternalOpId;
|
|
15
|
-
}[];
|
|
16
18
|
}
|
|
17
19
|
type BucketSyncState = {
|
|
18
20
|
start_op_id: util.InternalOpId;
|
|
@@ -37,6 +39,17 @@ export declare class BucketChecksumState {
|
|
|
37
39
|
*/
|
|
38
40
|
private lastChecksums;
|
|
39
41
|
private lastWriteCheckpoint;
|
|
42
|
+
/**
|
|
43
|
+
* Once we've sent the first full checkpoint line including all {@link util.Checkpoint.streams} that the user is
|
|
44
|
+
* subscribed to, we keep an index of the stream names to their index in that array.
|
|
45
|
+
*
|
|
46
|
+
* This is used to compress the representation of buckets in `checkpoint` and `checkpoint_diff` lines: For buckets
|
|
47
|
+
* that are part of sync rules or default streams, we need to include the name of the defining sync rule or definition
|
|
48
|
+
* yielding that bucket (so that clients can track progress for default streams).
|
|
49
|
+
* But instead of sending the name for each bucket, we use the fact that it's part of the streams array and only send
|
|
50
|
+
* their index, reducing the size of those messages.
|
|
51
|
+
*/
|
|
52
|
+
private streamNameToIndex;
|
|
40
53
|
private readonly parameterState;
|
|
41
54
|
/**
|
|
42
55
|
* Keep track of buckets that need to be downloaded. This is specifically relevant when
|
|
@@ -63,7 +76,7 @@ export interface CheckpointUpdate {
|
|
|
63
76
|
/**
|
|
64
77
|
* All buckets forming part of the checkpoint.
|
|
65
78
|
*/
|
|
66
|
-
buckets:
|
|
79
|
+
buckets: ResolvedBucket[];
|
|
67
80
|
/**
|
|
68
81
|
* If present, a set of buckets that have been updated since the last checkpoint.
|
|
69
82
|
*
|
|
@@ -74,15 +87,32 @@ export interface CheckpointUpdate {
|
|
|
74
87
|
export declare class BucketParameterState {
|
|
75
88
|
private readonly context;
|
|
76
89
|
readonly bucketStorage: BucketChecksumStateStorage;
|
|
77
|
-
readonly syncRules:
|
|
90
|
+
readonly syncRules: VersionedSyncRules;
|
|
78
91
|
readonly syncParams: RequestParameters;
|
|
79
92
|
private readonly querier;
|
|
93
|
+
/**
|
|
94
|
+
* Static buckets. This map is guaranteed not to change during a request, since resolving static buckets can only
|
|
95
|
+
* take request parameters into account,
|
|
96
|
+
*/
|
|
80
97
|
private readonly staticBuckets;
|
|
98
|
+
private readonly includeDefaultStreams;
|
|
99
|
+
private readonly explicitStreamSubscriptions;
|
|
100
|
+
readonly streamErrors: Record<string, QuerierError[]>;
|
|
101
|
+
private readonly subscribedStreamNames;
|
|
81
102
|
private readonly logger;
|
|
82
103
|
private cachedDynamicBuckets;
|
|
83
104
|
private cachedDynamicBucketSet;
|
|
84
105
|
private readonly lookups;
|
|
85
|
-
constructor(context: SyncContext, bucketStorage: BucketChecksumStateStorage, syncRules:
|
|
106
|
+
constructor(context: SyncContext, bucketStorage: BucketChecksumStateStorage, syncRules: VersionedSyncRules, tokenPayload: RequestJwtPayload, request: util.StreamingSyncRequest, logger: Logger);
|
|
107
|
+
/**
|
|
108
|
+
* Translates an internal sync-rules {@link ResolvedBucket} instance to the public
|
|
109
|
+
* {@link util.ClientBucketDescription}.
|
|
110
|
+
*
|
|
111
|
+
* @param lookupIndex A map from stream names to their index in {@link util.Checkpoint.streams}. These are used to
|
|
112
|
+
* reference default buckets by their stream index instead of duplicating the name on wire.
|
|
113
|
+
*/
|
|
114
|
+
translateResolvedBucket(description: ResolvedBucket, lookupIndex: Map<string, number>): util.ClientBucketDescription;
|
|
115
|
+
isSubscribedToStream(desc: BucketSource): boolean;
|
|
86
116
|
getCheckpointUpdate(checkpoint: storage.StorageCheckpointUpdate): Promise<CheckpointUpdate>;
|
|
87
117
|
/**
|
|
88
118
|
* For static buckets, we can keep track of which buckets have been updated.
|