@powersync/service-core 1.27.2 → 1.28.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +30 -0
- package/LICENSE +201 -67
- package/dist/api/RouteAPI.d.ts +6 -0
- package/dist/api/api-metrics.js +4 -0
- package/dist/api/api-metrics.js.map +1 -1
- package/dist/api/diagnostics.d.ts +5 -0
- package/dist/api/diagnostics.js +17 -2
- package/dist/api/diagnostics.js.map +1 -1
- package/dist/metrics/connection-metrics.d.ts +28 -0
- package/dist/metrics/connection-metrics.js +55 -0
- package/dist/metrics/connection-metrics.js.map +1 -0
- package/dist/metrics/metrics-index.d.ts +1 -0
- package/dist/metrics/metrics-index.js +1 -0
- package/dist/metrics/metrics-index.js.map +1 -1
- package/dist/metrics/metrics-interfaces.d.ts +7 -1
- package/dist/metrics/metrics-interfaces.js.map +1 -1
- package/dist/modules/loader.js +8 -2
- package/dist/modules/loader.js.map +1 -1
- package/dist/replication/AbstractReplicationJob.d.ts +2 -0
- package/dist/replication/AbstractReplicationJob.js +17 -1
- package/dist/replication/AbstractReplicationJob.js.map +1 -1
- package/dist/replication/AbstractReplicator.d.ts +8 -18
- package/dist/replication/AbstractReplicator.js +63 -64
- package/dist/replication/AbstractReplicator.js.map +1 -1
- package/dist/replication/ReplicationModule.js +0 -6
- package/dist/replication/ReplicationModule.js.map +1 -1
- package/dist/routes/configure-fastify.js +5 -1
- package/dist/routes/configure-fastify.js.map +1 -1
- package/dist/routes/endpoints/admin.js +24 -22
- package/dist/routes/endpoints/admin.js.map +1 -1
- package/dist/routes/endpoints/socket-route.js +19 -27
- package/dist/routes/endpoints/socket-route.js.map +1 -1
- package/dist/routes/endpoints/sync-rules.js +17 -40
- package/dist/routes/endpoints/sync-rules.js.map +1 -1
- package/dist/routes/endpoints/sync-stream.js +25 -23
- package/dist/routes/endpoints/sync-stream.js.map +1 -1
- package/dist/routes/hooks.d.ts +1 -1
- package/dist/routes/hooks.js +2 -1
- package/dist/routes/hooks.js.map +1 -1
- package/dist/routes/sync-connection.d.ts +7 -0
- package/dist/routes/sync-connection.js +34 -0
- package/dist/routes/sync-connection.js.map +1 -0
- package/dist/storage/BucketStorageFactory.d.ts +30 -22
- package/dist/storage/BucketStorageFactory.js +14 -14
- package/dist/storage/BucketStorageFactory.js.map +1 -1
- package/dist/storage/PersistedSyncConfigContent.js +2 -2
- package/dist/storage/PersistedSyncConfigContent.js.map +1 -1
- package/dist/storage/SyncRulesBucketStorage.d.ts +23 -2
- package/dist/storage/SyncRulesBucketStorage.js.map +1 -1
- package/dist/streams/BroadcastIterable.js +8 -7
- package/dist/streams/BroadcastIterable.js.map +1 -1
- package/dist/streams/LastValueSink.d.ts +2 -2
- package/dist/streams/LastValueSink.js +8 -13
- package/dist/streams/LastValueSink.js.map +1 -1
- package/dist/streams/merge.d.ts +15 -31
- package/dist/streams/merge.js +136 -152
- package/dist/streams/merge.js.map +1 -1
- package/dist/sync/BucketChecksumState.d.ts +3 -2
- package/dist/sync/BucketChecksumState.js +12 -5
- package/dist/sync/BucketChecksumState.js.map +1 -1
- package/dist/sync/sync.js +48 -38
- package/dist/sync/sync.js.map +1 -1
- package/dist/sync/util.d.ts +1 -2
- package/dist/sync/util.js +2 -5
- package/dist/sync/util.js.map +1 -1
- package/dist/system/ServiceContext.d.ts +7 -0
- package/dist/system/ServiceContext.js +3 -3
- package/dist/system/ServiceContext.js.map +1 -1
- package/dist/system/SyncConfigEngine.d.ts +135 -0
- package/dist/system/SyncConfigEngine.js +186 -0
- package/dist/system/SyncConfigEngine.js.map +1 -0
- package/dist/system/system-index.d.ts +1 -0
- package/dist/system/system-index.js +1 -0
- package/dist/system/system-index.js.map +1 -1
- package/dist/util/config/types.d.ts +4 -0
- package/package.json +8 -9
- package/src/api/RouteAPI.ts +6 -0
- package/src/api/api-metrics.ts +5 -0
- package/src/api/diagnostics.ts +22 -2
- package/src/metrics/connection-metrics.ts +75 -0
- package/src/metrics/metrics-index.ts +1 -0
- package/src/metrics/metrics-interfaces.ts +8 -1
- package/src/modules/loader.ts +16 -2
- package/src/replication/AbstractReplicationJob.ts +17 -1
- package/src/replication/AbstractReplicator.ts +83 -81
- package/src/replication/ReplicationModule.ts +0 -5
- package/src/routes/configure-fastify.ts +7 -1
- package/src/routes/endpoints/admin.ts +27 -20
- package/src/routes/endpoints/socket-route.ts +26 -38
- package/src/routes/endpoints/sync-rules.ts +22 -46
- package/src/routes/endpoints/sync-stream.ts +31 -27
- package/src/routes/hooks.ts +5 -1
- package/src/routes/sync-connection.ts +39 -0
- package/src/storage/BucketStorageFactory.ts +45 -29
- package/src/storage/PersistedSyncConfigContent.ts +2 -2
- package/src/storage/SyncRulesBucketStorage.ts +23 -2
- package/src/streams/BroadcastIterable.ts +8 -7
- package/src/streams/LastValueSink.ts +10 -16
- package/src/streams/merge.ts +149 -169
- package/src/sync/BucketChecksumState.ts +17 -5
- package/src/sync/sync.ts +71 -57
- package/src/sync/util.ts +2 -7
- package/src/system/ServiceContext.ts +8 -3
- package/src/system/SyncConfigEngine.ts +247 -0
- package/src/system/system-index.ts +1 -0
- package/src/util/config/types.ts +4 -0
- package/test/src/AbstractReplicator.test.ts +375 -45
- package/test/src/PersistedSyncConfigContent.test.ts +5 -5
- package/test/src/SyncConfigEngine.test.ts +208 -0
- package/test/src/broadcast_iterable.test.ts +59 -31
- package/test/src/config.test.ts +65 -3
- package/test/src/connection-metrics-export.test.ts +76 -0
- package/test/src/connection-metrics.test.ts +184 -0
- package/test/src/diagnostics.test.ts +3 -3
- package/test/src/last_value_sink.test.ts +34 -0
- package/test/src/merge_iterable.test.ts +304 -55
- package/test/src/module-loader.test.ts +14 -1
- package/test/src/recording-metrics.ts +36 -0
- package/test/src/routes/admin.test.ts +2 -2
- package/test/src/routes/concurrency-limit.test.ts +108 -0
- package/test/src/routes/connection-metrics.test.ts +269 -0
- package/test/src/routes/invalid-config.test.ts +184 -0
- package/test/src/routes/mocks.ts +26 -13
- package/test/src/routes/stream-disconnect.integration.test.ts +129 -0
- package/test/src/routes/stream.test.ts +200 -107
- package/test/src/routes/sync-rules.test.ts +19 -1
- package/test/src/source_table_config.test.ts +7 -7
- package/test/src/sync/BucketChecksumState.test.ts +10 -10
- package/test/src/sync/stream_response.test.ts +389 -0
- package/test/src/sync/util.test.ts +14 -3
- package/tsconfig.tsbuildinfo +1 -1
|
@@ -1,9 +1,11 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { errors, schema } from '@powersync/lib-services-framework';
|
|
2
2
|
import * as sync from '../../sync/sync-index.js';
|
|
3
3
|
import * as util from '../../util/util-index.js';
|
|
4
4
|
import { SyncRoutes } from './sync-stream.js';
|
|
5
5
|
import { APIMetric, event_types } from '@powersync/service-types';
|
|
6
|
+
import { recordSyncConnection, SyncCloseReason, syncConnectionCloseReasonLogText, SyncTransport } from '../../metrics/connection-metrics.js';
|
|
6
7
|
import { limitParamsForLogging } from '../../util/param-logging.js';
|
|
8
|
+
import { resolveSyncConnectionSetup } from '../sync-connection.js';
|
|
7
9
|
export const syncStreamReactive = (router) => router.reactiveStream(SyncRoutes.STREAM, {
|
|
8
10
|
validator: schema.createTsCodecValidator(util.StreamingSyncRequest, { allowAdditional: true }),
|
|
9
11
|
handler: async ({ context, params, responder, observer, initialN, signal: upstreamSignal, connection }) => {
|
|
@@ -27,17 +29,20 @@ export const syncStreamReactive = (router) => router.reactiveStream(SyncRoutes.S
|
|
|
27
29
|
jwt_exp: new Date(context.token_payload.exp * 1000),
|
|
28
30
|
connected_at: new Date(streamStart)
|
|
29
31
|
};
|
|
32
|
+
const { bucketStorage, syncRules } = await resolveSyncConnectionSetup(service_context, SyncTransport.RSocket);
|
|
30
33
|
// Best effort guess on why the stream was closed.
|
|
31
34
|
// We use the `??=` operator everywhere, so that we catch the first relevant
|
|
32
35
|
// event, which is usually the most specific.
|
|
33
36
|
let closeReason = undefined;
|
|
37
|
+
let connectionError;
|
|
34
38
|
// Create our own controller that we can abort directly
|
|
35
39
|
const controller = new AbortController();
|
|
36
40
|
upstreamSignal.addEventListener('abort', () => {
|
|
37
|
-
closeReason ??=
|
|
41
|
+
closeReason ??= SyncCloseReason.ClientClosed;
|
|
38
42
|
controller.abort();
|
|
39
43
|
});
|
|
40
44
|
if (upstreamSignal.aborted) {
|
|
45
|
+
closeReason ??= SyncCloseReason.ClientClosed;
|
|
41
46
|
controller.abort();
|
|
42
47
|
}
|
|
43
48
|
const signal = controller.signal;
|
|
@@ -47,29 +52,8 @@ export const syncStreamReactive = (router) => router.reactiveStream(SyncRoutes.S
|
|
|
47
52
|
requestedN += n;
|
|
48
53
|
}
|
|
49
54
|
});
|
|
50
|
-
if (routerEngine.closed) {
|
|
51
|
-
responder.onError(new errors.ServiceError({
|
|
52
|
-
status: 503,
|
|
53
|
-
code: ErrorCode.PSYNC_S2003,
|
|
54
|
-
description: 'Service temporarily unavailable'
|
|
55
|
-
}));
|
|
56
|
-
responder.onComplete();
|
|
57
|
-
return;
|
|
58
|
-
}
|
|
59
|
-
const { storageEngine: { activeBucketStorage } } = service_context;
|
|
60
|
-
const bucketStorage = (await activeBucketStorage.getActiveSyncConfig())?.storage;
|
|
61
|
-
if (bucketStorage == null) {
|
|
62
|
-
responder.onError(new errors.ServiceError({
|
|
63
|
-
status: 500,
|
|
64
|
-
code: ErrorCode.PSYNC_S2302,
|
|
65
|
-
description: 'No sync config available'
|
|
66
|
-
}));
|
|
67
|
-
responder.onComplete();
|
|
68
|
-
return;
|
|
69
|
-
}
|
|
70
|
-
const syncRules = bucketStorage.getParsedSyncRules(routerEngine.getAPI().getParseSyncRulesOptions());
|
|
71
55
|
const removeStopHandler = routerEngine.addStopHandler(() => {
|
|
72
|
-
closeReason ??=
|
|
56
|
+
closeReason ??= SyncCloseReason.ProcessShutdown;
|
|
73
57
|
controller.abort();
|
|
74
58
|
});
|
|
75
59
|
metricsEngine.getUpDownCounter(APIMetric.CONCURRENT_CONNECTIONS).add(1);
|
|
@@ -135,7 +119,7 @@ export const syncStreamReactive = (router) => router.reactiveStream(SyncRoutes.S
|
|
|
135
119
|
});
|
|
136
120
|
}
|
|
137
121
|
}
|
|
138
|
-
closeReason ??=
|
|
122
|
+
closeReason ??= SyncCloseReason.ServiceClosed;
|
|
139
123
|
}
|
|
140
124
|
catch (ex) {
|
|
141
125
|
// Convert to our standard form before responding.
|
|
@@ -143,7 +127,10 @@ export const syncStreamReactive = (router) => router.reactiveStream(SyncRoutes.S
|
|
|
143
127
|
// However, use the original error for the logs, so that we have the stack trace.
|
|
144
128
|
const error = new errors.InternalServerError(ex);
|
|
145
129
|
logger.error('Sync stream error', ex);
|
|
146
|
-
closeReason
|
|
130
|
+
if (closeReason == null) {
|
|
131
|
+
closeReason = SyncCloseReason.StreamError;
|
|
132
|
+
connectionError = ex;
|
|
133
|
+
}
|
|
147
134
|
responder.onError(error);
|
|
148
135
|
}
|
|
149
136
|
finally {
|
|
@@ -165,9 +152,14 @@ export const syncStreamReactive = (router) => router.reactiveStream(SyncRoutes.S
|
|
|
165
152
|
...tracker.getLogMeta(),
|
|
166
153
|
app_metadata: formattedAppMetadata,
|
|
167
154
|
stream_ms: Date.now() - streamStart,
|
|
168
|
-
close_reason: closeReason
|
|
155
|
+
close_reason: syncConnectionCloseReasonLogText(closeReason)
|
|
169
156
|
});
|
|
170
157
|
metricsEngine.getUpDownCounter(APIMetric.CONCURRENT_CONNECTIONS).add(-1);
|
|
158
|
+
recordSyncConnection(metricsEngine, {
|
|
159
|
+
transport: SyncTransport.RSocket,
|
|
160
|
+
closeReason: closeReason ?? SyncCloseReason.Unknown,
|
|
161
|
+
error: connectionError
|
|
162
|
+
});
|
|
171
163
|
service_context.eventsEngine.emit(event_types.EventsEngineEventType.SDK_DISCONNECT_EVENT, {
|
|
172
164
|
...sdkData,
|
|
173
165
|
disconnected_at: new Date()
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"socket-route.js","sourceRoot":"","sources":["../../../src/routes/endpoints/socket-route.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"socket-route.js","sourceRoot":"","sources":["../../../src/routes/endpoints/socket-route.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,mCAAmC,CAAC;AAEnE,OAAO,KAAK,IAAI,MAAM,0BAA0B,CAAC;AACjD,OAAO,KAAK,IAAI,MAAM,0BAA0B,CAAC;AAEjD,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAE9C,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,0BAA0B,CAAC;AAClE,OAAO,EACL,oBAAoB,EACpB,eAAe,EACf,gCAAgC,EAChC,aAAa,EACd,MAAM,qCAAqC,CAAC;AAC7C,OAAO,EAAE,qBAAqB,EAAE,MAAM,6BAA6B,CAAC;AACpE,OAAO,EAAE,0BAA0B,EAAE,MAAM,uBAAuB,CAAC;AAEnE,MAAM,CAAC,MAAM,kBAAkB,GAAyB,CAAC,MAAM,EAAE,EAAE,CACjE,MAAM,CAAC,cAAc,CAAiC,UAAU,CAAC,MAAM,EAAE;IACvE,SAAS,EAAE,MAAM,CAAC,sBAAsB,CAAC,IAAI,CAAC,oBAAoB,EAAE,EAAE,eAAe,EAAE,IAAI,EAAE,CAAC;IAC9F,OAAO,EAAE,KAAK,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,EAAE,cAAc,EAAE,UAAU,EAAE,EAAE,EAAE;QACxG,MAAM,EAAE,eAAe,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC;QAC5C,MAAM,EAAE,YAAY,EAAE,aAAa,EAAE,WAAW,EAAE,GAAG,eAAe,CAAC;QACrE,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAE/B,2EAA2E;QAC3E,MAAM,oBAAoB,GAAG,MAAM,CAAC,YAAY,CAAC,CAAC,CAAC,qBAAqB,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;QAC1G,MAAM,qBAAqB,GAAG,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,qBAAqB,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;QAEvG,MAAM,CAAC,WAAW,GAAG;YACnB,GAAG,MAAM,CAAC,WAAW;YACrB,OAAO,EAAE,OAAO,CAAC,aAAc,CAAC,UAAU;YAC1C,SAAS,EAAE,MAAM,CAAC,SAAS;YAC3B,UAAU,EAAE,OAAO,CAAC,UAAU;SAC/B,CAAC;QAEF,MAAM,OAAO,GAA0E;YACrF,SAAS,EAAE,MAAM,CAAC,SAAS,IAAI,EAAE;YACjC,OAAO,EAAE,OAAO,CAAC,aAAc,CAAC,YAAY;YAC5C,UAAU,EAAE,OAAO,CAAC,UAAU;YAC9B,8DAA8D;YAC9D,OAAO,EAAE,IAAI,IAAI,CAAC,OAAO,CAAC,aAAc,CAAC,GAAG,GAAG,IAAI,CAAC;YACpD,YAAY,EAAE,IAAI,IAAI,CAAC,WAAW,CAAC;SACpC,CAAC;QAEF,MAAM,EAAE,aAAa,EAAE,SAAS,EAAE,GAAG,MAAM,0BAA0B,CAAC,eAAe,EAAE,aAAa,CAAC,OAAO,CAAC,CAAC;QAE9G,kDAAkD;QAClD,4EAA4E;QAC5E,6CAA6C;QAC7C,IAAI,WAAW,GAAgC,SAAS,CAAC;QACzD,IAAI,eAAwB,CAAC;QAE7B,uDAAuD;QACvD,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;QACzC,cAAc,CAAC,gBAAgB,CAAC,OAAO,EAAE,GAAG,EAAE;YAC5C,WAAW,KAAK,eAAe,CAAC,YAAY,CAAC;YAC7C,UAAU,CAAC,KAAK,EAAE,CAAC;QACrB,CAAC,CAAC,CAAC;QACH,IAAI,cAAc,CAAC,OAAO,EAAE,CAAC;YAC3B,WAAW,KAAK,eAAe,CAAC,YAAY,CAAC;YAC7C,UAAU,CAAC,KAAK,EAAE,CAAC;QACrB,CAAC;QACD,MAAM,MAAM,GAAG,UAAU,CAAC,MAAM,CAAC;QAEjC,IAAI,UAAU,GAAG,QAAQ,CAAC;QAC1B,MAAM,QAAQ,GAAG,QAAQ,CAAC,gBAAgB,CAAC;YACzC,OAAO,CAAC,CAAC;gBACP,UAAU,IAAI,CAAC,CAAC;YAClB,CAAC;SACF,CAAC,CAAC;QAEH,MAAM,iBAAiB,GAAG,YAAY,CAAC,cAAc,CAAC,GAAG,EAAE;YACzD,WAAW,KAAK,eAAe,CAAC,eAAe,CAAC;YAChD,UAAU,CAAC,KAAK,EAAE,CAAC;QACrB,CAAC,CAAC,CAAC;QAEH,aAAa,CAAC,gBAAgB,CAAC,SAAS,CAAC,sBAAsB,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;QACxE,eAAe,CAAC,YAAY,CAAC,IAAI,CAAC,WAAW,CAAC,qBAAqB,CAAC,iBAAiB,EAAE,OAAO,CAAC,CAAC;QAChG,MAAM,OAAO,GAAG,IAAI,IAAI,CAAC,cAAc,CAAC,aAAa,CAAC,CAAC;QACvD,IAAI,UAAU,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC;YAChC,yCAAyC;YACzC,OAAO,CAAC,aAAa,CAAC,UAAU,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;QACrD,CAAC;QAED,IAAI,CAAC;YACH,MAAM,CAAC,IAAI,CAAC,qBAAqB,EAAE;gBACjC,YAAY,EAAE,oBAAoB;gBAClC,aAAa,EAAE,qBAAqB;aACrC,CAAC,CAAC;YAEH,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,IAAI,CAAC,cAAc,CAAC;gBAC3C,WAAW,EAAE,WAAW;gBACxB,aAAa,EAAE,aAAa;gBAC5B,SAAS;gBACT,MAAM,EAAE;oBACN,GAAG,MAAM;iBACV;gBACD,KAAK,EAAE,OAAQ,CAAC,aAAc;gBAC9B,kBAAkB,EAAE;oBAClB,8CAA8C;oBAC9C,UAAU,EAAE,KAAK;iBAClB;gBACD,OAAO;gBACP,MAAM;gBACN,MAAM;gBACN,gBAAgB,EAAE,IAAI;aACvB,CAAC,EAAE,CAAC;gBACH,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;oBACnB,MAAM;gBACR,CAAC;gBACD,IAAI,IAAI,IAAI,IAAI,EAAE,CAAC;oBACjB,SAAS;gBACX,CAAC;gBAED,CAAC;oBACC,MAAM,UAAU,GAAG,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC;oBAC7C,SAAS,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,UAAU,EAAE,EAAE,KAAK,CAAC,CAAC;oBAC9C,UAAU,EAAE,CAAC;oBACb,OAAO,CAAC,sBAAsB,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;gBACpD,CAAC;gBAED,IAAI,UAAU,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;oBACvC,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;wBAClC,MAAM,CAAC,GAAG,QAAQ,CAAC,gBAAgB,CAAC;4BAClC,OAAO;gCACL,IAAI,UAAU,GAAG,CAAC,EAAE,CAAC;oCACnB,iEAAiE;oCACjE,OAAO,EAAE,CAAC;oCACV,CAAC,EAAE,CAAC;oCACJ,MAAM,CAAC,mBAAmB,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;gCAC/C,CAAC;4BACH,CAAC;yBACF,CAAC,CAAC;wBACH,MAAM,OAAO,GAAG,GAAG,EAAE;4BACnB,8CAA8C;4BAC9C,OAAO,EAAE,CAAC;4BACV,CAAC,EAAE,CAAC;4BACJ,MAAM,CAAC,mBAAmB,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;wBAC/C,CAAC,CAAC;wBACF,MAAM,CAAC,gBAAgB,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;oBAC5C,CAAC,CAAC,CAAC;gBACL,CAAC;YACH,CAAC;YACD,WAAW,KAAK,eAAe,CAAC,aAAa,CAAC;QAChD,CAAC;QAAC,OAAO,EAAE,EAAE,CAAC;YACZ,kDAAkD;YAClD,4CAA4C;YAC5C,iFAAiF;YACjF,MAAM,KAAK,GAAG,IAAI,MAAM,CAAC,mBAAmB,CAAC,EAAE,CAAC,CAAC;YACjD,MAAM,CAAC,KAAK,CAAC,mBAAmB,EAAE,EAAE,CAAC,CAAC;YACtC,IAAI,WAAW,IAAI,IAAI,EAAE,CAAC;gBACxB,WAAW,GAAG,eAAe,CAAC,WAAW,CAAC;gBAC1C,eAAe,GAAG,EAAE,CAAC;YACvB,CAAC;YACD,SAAS,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;QAC3B,CAAC;gBAAS,CAAC;YACT,SAAS,CAAC,UAAU,EAAE,CAAC;YACvB,iBAAiB,EAAE,CAAC;YACpB,QAAQ,EAAE,CAAC;YACX,IAAI,UAAU,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC;gBAChC,6FAA6F;gBAC7F,kGAAkG;gBAClG,kGAAkG;gBAClG,iBAAiB;gBACjB,+FAA+F;gBAC/F,2BAA2B;gBAC3B,kEAAkE;gBAClE,MAAM,WAAW,GAAG,UAAU,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;gBACzD,OAAO,CAAC,qBAAqB,CAAC,WAAW,CAAC,CAAC;YAC7C,CAAC;YACD,MAAM,CAAC,IAAI,CAAC,sBAAsB,EAAE;gBAClC,GAAG,OAAO,CAAC,UAAU,EAAE;gBACvB,YAAY,EAAE,oBAAoB;gBAClC,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,WAAW;gBACnC,YAAY,EAAE,gCAAgC,CAAC,WAAW,CAAC;aAC5D,CAAC,CAAC;YACH,aAAa,CAAC,gBAAgB,CAAC,SAAS,CAAC,sBAAsB,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;YACzE,oBAAoB,CAAC,aAAa,EAAE;gBAClC,SAAS,EAAE,aAAa,CAAC,OAAO;gBAChC,WAAW,EAAE,WAAW,IAAI,eAAe,CAAC,OAAO;gBACnD,KAAK,EAAE,eAAe;aACvB,CAAC,CAAC;YACH,eAAe,CAAC,YAAY,CAAC,IAAI,CAAC,WAAW,CAAC,qBAAqB,CAAC,oBAAoB,EAAE;gBACxF,GAAG,OAAO;gBACV,eAAe,EAAE,IAAI,IAAI,EAAE;aAC5B,CAAC,CAAC;QACL,CAAC;IACH,CAAC;CACF,CAAC,CAAC"}
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
import { ErrorCode, errors, router, schema } from '@powersync/lib-services-framework';
|
|
2
|
-
import {
|
|
2
|
+
import { parseSyncConfig, SyncConfigErrors } from '@powersync/service-sync-rules';
|
|
3
3
|
import * as t from 'ts-codec';
|
|
4
|
-
import {
|
|
5
|
-
import { logSyncConfigErrors
|
|
4
|
+
import { assertAcceptedSyncConfig, assertConfiguredSyncConfigValid } from '../../system/SyncConfigEngine.js';
|
|
5
|
+
import { logSyncConfigErrors } from '../../storage/BucketStorageFactory.js';
|
|
6
6
|
import { authApi } from '../auth.js';
|
|
7
7
|
import { routeDefinition } from '../router.js';
|
|
8
8
|
const DeploySyncRulesRequest = t.object({
|
|
@@ -44,28 +44,9 @@ export const deploySyncRules = routeDefinition({
|
|
|
44
44
|
});
|
|
45
45
|
}
|
|
46
46
|
const content = payload.params.content;
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
const apiHandler = service_context.routerEngine.getAPI();
|
|
51
|
-
syncConfig = SqlSyncRules.fromYaml(content, {
|
|
52
|
-
...apiHandler.getParseSyncRulesOptions(),
|
|
53
|
-
// We don't do any schema-level validation at this point
|
|
54
|
-
schema: undefined
|
|
55
|
-
});
|
|
56
|
-
}
|
|
57
|
-
catch (e) {
|
|
58
|
-
throw new errors.ServiceError({
|
|
59
|
-
status: 422,
|
|
60
|
-
code: ErrorCode.PSYNC_R0001,
|
|
61
|
-
description: 'Sync config parsing failed',
|
|
62
|
-
details: e.message
|
|
63
|
-
});
|
|
64
|
-
}
|
|
65
|
-
logSyncConfigErrors(syncConfig);
|
|
66
|
-
// Validate if the current sources support the configuration specified in the sync config
|
|
67
|
-
await assertSourceCapabilities(service_context.routerEngine.getAPI(), syncConfig.config);
|
|
68
|
-
const sync_rules = await storageEngine.activeBucketStorage.updateSyncRules(updateSyncRulesFromConfig(syncConfig));
|
|
47
|
+
const compiled = assertAcceptedSyncConfig(await service_context.syncConfigEngine.validate(content));
|
|
48
|
+
logSyncConfigErrors(compiled.parsed);
|
|
49
|
+
const sync_rules = await storageEngine.activeBucketStorage.updateSyncRules({ config: compiled });
|
|
69
50
|
return {
|
|
70
51
|
slot_name: sync_rules.replicationStreamName
|
|
71
52
|
};
|
|
@@ -133,7 +114,8 @@ export const reprocessSyncRules = routeDefinition({
|
|
|
133
114
|
authorize: authApi,
|
|
134
115
|
validator: schema.createTsCodecValidator(ReprocessSyncRulesRequest),
|
|
135
116
|
handler: async (payload) => {
|
|
136
|
-
const
|
|
117
|
+
const service_context = payload.context.service_context;
|
|
118
|
+
const { storageEngine: { activeBucketStorage } } = service_context;
|
|
137
119
|
const next = await activeBucketStorage.getDeployingSyncConfig();
|
|
138
120
|
if (next != null) {
|
|
139
121
|
throw new errors.ServiceError({
|
|
@@ -142,6 +124,7 @@ export const reprocessSyncRules = routeDefinition({
|
|
|
142
124
|
description: 'Busy processing sync config - cannot reprocess'
|
|
143
125
|
});
|
|
144
126
|
}
|
|
127
|
+
await assertConfiguredSyncConfigValid(service_context.syncConfigEngine);
|
|
145
128
|
const active = await activeBucketStorage.getActiveSyncConfig();
|
|
146
129
|
if (active == null) {
|
|
147
130
|
throw new errors.ServiceError({
|
|
@@ -151,19 +134,13 @@ export const reprocessSyncRules = routeDefinition({
|
|
|
151
134
|
});
|
|
152
135
|
}
|
|
153
136
|
const sync_rules = active.content;
|
|
154
|
-
const
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
// to a service change, we do want to report the error here.
|
|
159
|
-
throwOnError: true
|
|
160
|
-
});
|
|
161
|
-
logSyncConfigErrors(parsed);
|
|
162
|
-
await assertSourceCapabilities(payload.context.service_context.routerEngine.getAPI(), parsed.config);
|
|
163
|
-
const new_rules = await activeBucketStorage.updateSyncRules(updateSyncRulesFromConfig(parsed, {
|
|
137
|
+
const compiled = assertAcceptedSyncConfig(await service_context.syncConfigEngine.validate(sync_rules.sync_rules_content));
|
|
138
|
+
logSyncConfigErrors(compiled.parsed);
|
|
139
|
+
const new_rules = await activeBucketStorage.updateSyncRules({
|
|
140
|
+
config: compiled,
|
|
164
141
|
version_label: sync_rules.version_label,
|
|
165
142
|
forceNewReplicationStream: true
|
|
166
|
-
})
|
|
143
|
+
});
|
|
167
144
|
return {
|
|
168
145
|
slot_name: new_rules.replicationStreamName
|
|
169
146
|
};
|
|
@@ -179,7 +156,7 @@ function replyPrettyJson(payload) {
|
|
|
179
156
|
}
|
|
180
157
|
async function debugSyncRules(apiHandler, sync_rules) {
|
|
181
158
|
try {
|
|
182
|
-
const rules =
|
|
159
|
+
const rules = parseSyncConfig(sync_rules, {
|
|
183
160
|
...apiHandler.getParseSyncRulesOptions(),
|
|
184
161
|
// No schema-based validation at this point
|
|
185
162
|
schema: undefined
|
|
@@ -187,7 +164,7 @@ async function debugSyncRules(apiHandler, sync_rules) {
|
|
|
187
164
|
const errors = [];
|
|
188
165
|
const warnings = rules.errors.filter((error) => error.type === 'warning').map((error) => error.message);
|
|
189
166
|
const recordError = (error) => {
|
|
190
|
-
errors.push(...(error instanceof
|
|
167
|
+
errors.push(...(error instanceof SyncConfigErrors ? error.errors.map((e) => e.message) : [error.message]));
|
|
191
168
|
};
|
|
192
169
|
try {
|
|
193
170
|
const diagnostics = (await apiHandler.validateSourceCapabilities?.(rules.config)) ?? [];
|
|
@@ -216,7 +193,7 @@ async function debugSyncRules(apiHandler, sync_rules) {
|
|
|
216
193
|
};
|
|
217
194
|
}
|
|
218
195
|
catch (e) {
|
|
219
|
-
if (e instanceof
|
|
196
|
+
if (e instanceof SyncConfigErrors) {
|
|
220
197
|
return {
|
|
221
198
|
valid: false,
|
|
222
199
|
errors: e.errors.map((e) => e.message)
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"sync-rules.js","sourceRoot":"","sources":["../../../src/routes/endpoints/sync-rules.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,mCAAmC,CAAC;AACtF,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"sync-rules.js","sourceRoot":"","sources":["../../../src/routes/endpoints/sync-rules.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,mCAAmC,CAAC;AACtF,OAAO,EAAE,eAAe,EAAE,gBAAgB,EAAE,MAAM,+BAA+B,CAAC;AAElF,OAAO,KAAK,CAAC,MAAM,UAAU,CAAC;AAC9B,OAAO,EAAE,wBAAwB,EAAE,+BAA+B,EAAE,MAAM,kCAAkC,CAAC;AAG7G,OAAO,EAAE,mBAAmB,EAAE,MAAM,uCAAuC,CAAC;AAC5E,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AACrC,OAAO,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AAE/C,MAAM,sBAAsB,GAAG,CAAC,CAAC,MAAM,CAAC;IACtC,OAAO,EAAE,CAAC,CAAC,MAAM;CAClB,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,UAAU,GAAuB,KAAK,EAAE,OAAO,EAAE,EAAE;IAC9D,OAAO,CAAC,oBAAoB,CAAC,kBAAkB,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,EAAE,EAAE,EAAE;QAC9E,MAAM,IAAI,GAAU,EAAE,CAAC;QACvB,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;YAClC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACnB,CAAC;QAED,OAAO,CAAC,MAAM,GAAG,EAAE,OAAO,EAAE,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;IACrE,CAAC,CAAC,CAAC;AACL,CAAC,CAAC;AAEF;;;;GAIG;AACH,kBAAkB;AAClB,UAAU,CAAC,MAAM,CAAC,GAAG,CAAC,eAAe,CAAC,CAAC,GAAG,IAAI,CAAC;AAE/C,MAAM,CAAC,MAAM,eAAe,GAAG,eAAe,CAAC;IAC7C,IAAI,EAAE,2BAA2B;IACjC,MAAM,EAAE,MAAM,CAAC,UAAU,CAAC,IAAI;IAC9B,SAAS,EAAE,OAAO;IAClB,KAAK,EAAE,IAAI;IACX,OAAO,EAAE,CAAC,UAAU,CAAC;IACrB,SAAS,EAAE,MAAM,CAAC,sBAAsB,CAAC,sBAAsB,EAAE,EAAE,eAAe,EAAE,IAAI,EAAE,CAAC;IAC3F,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE;QACzB,MAAM,EAAE,eAAe,EAAE,GAAG,OAAO,CAAC,OAAO,CAAC;QAC5C,MAAM,EAAE,aAAa,EAAE,GAAG,eAAe,CAAC;QAE1C,IAAI,eAAe,CAAC,aAAa,CAAC,UAAU,CAAC,OAAO,EAAE,CAAC;YACrD,mFAAmF;YACnF,MAAM,IAAI,MAAM,CAAC,YAAY,CAAC;gBAC5B,MAAM,EAAE,GAAG;gBACX,IAAI,EAAE,SAAS,CAAC,WAAW;gBAC3B,WAAW,EAAE,0BAA0B;gBACvC,OAAO,EAAE,iDAAiD;aAC3D,CAAC,CAAC;QACL,CAAC;QACD,MAAM,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC;QACvC,MAAM,QAAQ,GAAG,wBAAwB,CAAC,MAAM,eAAe,CAAC,gBAAgB,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC;QACpG,mBAAmB,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QACrC,MAAM,UAAU,GAAG,MAAM,aAAa,CAAC,mBAAmB,CAAC,eAAe,CAAC,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC,CAAC;QAEjG,OAAO;YACL,SAAS,EAAE,UAAU,CAAC,qBAAqB;SAC5C,CAAC;IACJ,CAAC;CACF,CAAC,CAAC;AAEH,MAAM,wBAAwB,GAAG,CAAC,CAAC,MAAM,CAAC;IACxC,OAAO,EAAE,CAAC,CAAC,MAAM;CAClB,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,iBAAiB,GAAG,eAAe,CAAC;IAC/C,IAAI,EAAE,6BAA6B;IACnC,MAAM,EAAE,MAAM,CAAC,UAAU,CAAC,IAAI;IAC9B,SAAS,EAAE,OAAO;IAClB,KAAK,EAAE,IAAI;IACX,OAAO,EAAE,CAAC,UAAU,CAAC;IACrB,SAAS,EAAE,MAAM,CAAC,sBAAsB,CAAC,wBAAwB,EAAE,EAAE,eAAe,EAAE,IAAI,EAAE,CAAC;IAC7F,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE;QACzB,MAAM,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC;QACvC,MAAM,EAAE,eAAe,EAAE,GAAG,OAAO,CAAC,OAAO,CAAC;QAC5C,MAAM,UAAU,GAAG,eAAe,CAAC,YAAY,CAAC,MAAM,EAAE,CAAC;QAEzD,MAAM,IAAI,GAAG,MAAM,cAAc,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;QAEvD,OAAO,eAAe,CAAC,IAAI,CAAC,CAAC;IAC/B,CAAC;CACF,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,gBAAgB,GAAG,eAAe,CAAC;IAC9C,IAAI,EAAE,4BAA4B;IAClC,MAAM,EAAE,MAAM,CAAC,UAAU,CAAC,GAAG;IAC7B,SAAS,EAAE,OAAO;IAClB,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE;QACzB,MAAM,EAAE,eAAe,EAAE,GAAG,OAAO,CAAC,OAAO,CAAC;QAC5C,MAAM,EACJ,aAAa,EAAE,EAAE,mBAAmB,EAAE,EACvC,GAAG,eAAe,CAAC;QAEpB,MAAM,MAAM,GAAG,MAAM,mBAAmB,CAAC,mBAAmB,EAAE,CAAC;QAC/D,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,MAAM,IAAI,MAAM,CAAC,YAAY,CAAC;gBAC5B,MAAM,EAAE,GAAG;gBACX,IAAI,EAAE,SAAS,CAAC,WAAW;gBAC3B,WAAW,EAAE,uBAAuB;aACrC,CAAC,CAAC;QACL,CAAC;QAED,MAAM,UAAU,GAAG,MAAM,CAAC,OAAO,CAAC;QAClC,MAAM,UAAU,GAAG,eAAe,CAAC,YAAY,CAAC,MAAM,EAAE,CAAC;QACzD,MAAM,IAAI,GAAG,MAAM,cAAc,CAAC,UAAU,EAAE,UAAU,CAAC,kBAAkB,CAAC,CAAC;QAC7E,MAAM,IAAI,GAAG,MAAM,mBAAmB,CAAC,sBAAsB,EAAE,CAAC;QAEhE,MAAM,SAAS,GAAG,IAAI,CAAC,CAAC,CAAC,MAAM,cAAc,CAAC,UAAU,EAAE,IAAI,CAAC,OAAO,CAAC,kBAAkB,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QAElG,MAAM,QAAQ,GAAG;YACf,OAAO,EAAE;gBACP,SAAS,EAAE,UAAU,CAAC,qBAAqB;gBAC3C,OAAO,EAAE,UAAU,CAAC,kBAAkB;gBACtC,GAAG,IAAI;aACR;YACD,IAAI,EACF,IAAI,IAAI,IAAI;gBACV,CAAC,CAAC,IAAI;gBACN,CAAC,CAAC;oBACE,SAAS,EAAE,IAAI,CAAC,OAAO,CAAC,qBAAqB;oBAC7C,OAAO,EAAE,IAAI,CAAC,OAAO,CAAC,kBAAkB;oBACxC,GAAG,SAAS;iBACb;SACR,CAAC;QAEF,OAAO,eAAe,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC,CAAC;IAC7C,CAAC;CACF,CAAC,CAAC;AAEH,MAAM,yBAAyB,GAAG,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;AAE/C,MAAM,CAAC,MAAM,kBAAkB,GAAG,eAAe,CAAC;IAChD,IAAI,EAAE,8BAA8B;IACpC,MAAM,EAAE,MAAM,CAAC,UAAU,CAAC,IAAI;IAC9B,SAAS,EAAE,OAAO;IAClB,SAAS,EAAE,MAAM,CAAC,sBAAsB,CAAC,yBAAyB,CAAC;IACnE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE;QACzB,MAAM,eAAe,GAAG,OAAO,CAAC,OAAO,CAAC,eAAe,CAAC;QACxD,MAAM,EACJ,aAAa,EAAE,EAAE,mBAAmB,EAAE,EACvC,GAAG,eAAe,CAAC;QACpB,MAAM,IAAI,GAAG,MAAM,mBAAmB,CAAC,sBAAsB,EAAE,CAAC;QAChE,IAAI,IAAI,IAAI,IAAI,EAAE,CAAC;YACjB,MAAM,IAAI,MAAM,CAAC,YAAY,CAAC;gBAC5B,MAAM,EAAE,GAAG;gBACX,IAAI,EAAE,SAAS,CAAC,WAAW;gBAC3B,WAAW,EAAE,gDAAgD;aAC9D,CAAC,CAAC;QACL,CAAC;QAED,MAAM,+BAA+B,CAAC,eAAe,CAAC,gBAAgB,CAAC,CAAC;QAExE,MAAM,MAAM,GAAG,MAAM,mBAAmB,CAAC,mBAAmB,EAAE,CAAC;QAC/D,IAAI,MAAM,IAAI,IAAI,EAAE,CAAC;YACnB,MAAM,IAAI,MAAM,CAAC,YAAY,CAAC;gBAC5B,MAAM,EAAE,GAAG;gBACX,IAAI,EAAE,SAAS,CAAC,WAAW;gBAC3B,WAAW,EAAE,uBAAuB;aACrC,CAAC,CAAC;QACL,CAAC;QAED,MAAM,UAAU,GAAG,MAAM,CAAC,OAAO,CAAC;QAClC,MAAM,QAAQ,GAAG,wBAAwB,CACvC,MAAM,eAAe,CAAC,gBAAgB,CAAC,QAAQ,CAAC,UAAU,CAAC,kBAAkB,CAAC,CAC/E,CAAC;QAEF,mBAAmB,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QACrC,MAAM,SAAS,GAAG,MAAM,mBAAmB,CAAC,eAAe,CAAC;YAC1D,MAAM,EAAE,QAAQ;YAChB,aAAa,EAAE,UAAU,CAAC,aAAa;YACvC,yBAAyB,EAAE,IAAI;SAChC,CAAC,CAAC;QACH,OAAO;YACL,SAAS,EAAE,SAAS,CAAC,qBAAqB;SAC3C,CAAC;IACJ,CAAC;CACF,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,iBAAiB,GAAG,CAAC,iBAAiB,EAAE,eAAe,EAAE,kBAAkB,EAAE,gBAAgB,CAAC,CAAC;AAE5G,SAAS,eAAe,CAAC,OAAY;IACnC,OAAO,IAAI,MAAM,CAAC,cAAc,CAAC;QAC/B,MAAM,EAAE,GAAG;QACX,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,GAAG,IAAI;QAC7C,OAAO,EAAE,EAAE,cAAc,EAAE,kBAAkB,EAAE;KAChD,CAAC,CAAC;AACL,CAAC;AAED,KAAK,UAAU,cAAc,CAAC,UAAoB,EAAE,UAAkB;IACpE,IAAI,CAAC;QACH,MAAM,KAAK,GAAG,eAAe,CAAC,UAAU,EAAE;YACxC,GAAG,UAAU,CAAC,wBAAwB,EAAE;YACxC,2CAA2C;YAC3C,MAAM,EAAE,SAAS;SAClB,CAAC,CAAC;QACH,MAAM,MAAM,GAAa,EAAE,CAAC;QAC5B,MAAM,QAAQ,GAAG,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QACxG,MAAM,WAAW,GAAG,CAAC,KAAU,EAAE,EAAE;YACjC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,YAAY,gBAAgB,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;QAC7G,CAAC,CAAC;QACF,IAAI,CAAC;YACH,MAAM,WAAW,GAAG,CAAC,MAAM,UAAU,CAAC,0BAA0B,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC;YACxF,KAAK,MAAM,UAAU,IAAI,WAAW,EAAE,CAAC;gBACrC,CAAC,UAAU,CAAC,KAAK,KAAK,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;YAC9E,CAAC;QACH,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,WAAW,CAAC,CAAC,CAAC,CAAC;QACjB,CAAC;QACD,IAAI,eAAe,GAAoB,EAAE,CAAC;QAC1C,IAAI,CAAC;YACH,MAAM,qBAAqB,GAAG,KAAK,CAAC,MAAM,CAAC,eAAe,EAAE,CAAC;YAC7D,eAAe,GAAG,MAAM,UAAU,CAAC,kBAAkB,CAAC,qBAAqB,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;QAC7F,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,WAAW,CAAC,CAAC,CAAC,CAAC;QACjB,CAAC;QAED,OAAO;YACL,KAAK,EAAE,MAAM,CAAC,MAAM,IAAI,CAAC;YACzB,GAAG,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC5C,GAAG,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACxC,kBAAkB,EAAE,KAAK,CAAC,MAAM,CAAC,mBAAmB,EAAE;YACtD,aAAa,EAAE,eAAe;YAC9B,WAAW,EAAE,KAAK,CAAC,MAAM,CAAC,oBAAoB,EAAE;SACjD,CAAC;IACJ,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,IAAI,CAAC,YAAY,gBAAgB,EAAE,CAAC;YAClC,OAAO;gBACL,KAAK,EAAE,KAAK;gBACZ,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC;aACvC,CAAC;QACJ,CAAC;QACD,OAAO;YACL,KAAK,EAAE,KAAK;YACZ,MAAM,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC;SACpB,CAAC;IACJ,CAAC;AACH,CAAC"}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { router, schema } from '@powersync/lib-services-framework';
|
|
2
2
|
import Negotiator from 'negotiator';
|
|
3
3
|
import { Readable } from 'stream';
|
|
4
4
|
import * as sync from '../../sync/sync-index.js';
|
|
@@ -6,8 +6,10 @@ import * as util from '../../util/util-index.js';
|
|
|
6
6
|
import { APIMetric, event_types } from '@powersync/service-types';
|
|
7
7
|
import { authUser } from '../auth.js';
|
|
8
8
|
import { routeDefinition } from '../router.js';
|
|
9
|
+
import { recordSyncConnection, SyncCloseReason, syncConnectionCloseReasonLogText, SyncTransport } from '../../metrics/connection-metrics.js';
|
|
9
10
|
import { limitParamsForLogging } from '../../util/param-logging.js';
|
|
10
11
|
import { maybeCompressResponseStream } from '../compression.js';
|
|
12
|
+
import { resolveSyncConnectionSetup } from '../sync-connection.js';
|
|
11
13
|
export var SyncRoutes;
|
|
12
14
|
(function (SyncRoutes) {
|
|
13
15
|
SyncRoutes["STREAM"] = "/sync/stream";
|
|
@@ -22,7 +24,7 @@ export const syncStreamed = routeDefinition({
|
|
|
22
24
|
validator: schema.createTsCodecValidator(util.StreamingSyncRequest, { allowAdditional: true }),
|
|
23
25
|
handler: async (payload) => {
|
|
24
26
|
const { service_context, logger, token_payload } = payload.context;
|
|
25
|
-
const { routerEngine,
|
|
27
|
+
const { routerEngine, metricsEngine, syncContext } = service_context;
|
|
26
28
|
const headers = payload.request.headers;
|
|
27
29
|
const userAgent = headers['x-user-agent'] ?? headers['user-agent'];
|
|
28
30
|
const clientId = payload.params.client_id;
|
|
@@ -47,22 +49,7 @@ export const syncStreamed = routeDefinition({
|
|
|
47
49
|
jwt_exp: new Date(token_payload.exp * 1000),
|
|
48
50
|
connected_at: new Date(streamStart)
|
|
49
51
|
};
|
|
50
|
-
|
|
51
|
-
throw new errors.ServiceError({
|
|
52
|
-
status: 503,
|
|
53
|
-
code: ErrorCode.PSYNC_S2003,
|
|
54
|
-
description: 'Service temporarily unavailable'
|
|
55
|
-
});
|
|
56
|
-
}
|
|
57
|
-
const bucketStorage = (await storageEngine.activeBucketStorage.getActiveSyncConfig())?.storage;
|
|
58
|
-
if (bucketStorage == null) {
|
|
59
|
-
throw new errors.ServiceError({
|
|
60
|
-
status: 500,
|
|
61
|
-
code: ErrorCode.PSYNC_S2302,
|
|
62
|
-
description: 'No sync config available'
|
|
63
|
-
});
|
|
64
|
-
}
|
|
65
|
-
const syncRules = bucketStorage.getParsedSyncRules(routerEngine.getAPI().getParseSyncRulesOptions());
|
|
52
|
+
const { bucketStorage, syncRules } = await resolveSyncConnectionSetup(service_context, SyncTransport.HttpStream);
|
|
66
53
|
const controller = new AbortController();
|
|
67
54
|
const tracker = new sync.RequestTracker(metricsEngine);
|
|
68
55
|
const formattedAppMetadata = payload.params.app_metadata
|
|
@@ -96,21 +83,25 @@ export const syncStreamed = routeDefinition({
|
|
|
96
83
|
// We use the `??=` operator everywhere, so that we catch the first relevant
|
|
97
84
|
// event, which is usually the most specific.
|
|
98
85
|
let closeReason = undefined;
|
|
86
|
+
let connectionError;
|
|
99
87
|
const deregister = routerEngine.addStopHandler(() => {
|
|
100
88
|
// This error is not currently propagated to the client
|
|
101
89
|
controller.abort();
|
|
102
|
-
closeReason ??=
|
|
90
|
+
closeReason ??= SyncCloseReason.ProcessShutdown;
|
|
103
91
|
stream.destroy(new Error('Shutting down system'));
|
|
104
92
|
});
|
|
105
93
|
stream.on('end', () => {
|
|
106
94
|
// Auth failure or switch to new sync config
|
|
107
|
-
closeReason ??=
|
|
95
|
+
closeReason ??= SyncCloseReason.ServiceClosed;
|
|
108
96
|
});
|
|
109
97
|
stream.on('close', () => {
|
|
110
98
|
deregister();
|
|
111
99
|
});
|
|
112
100
|
stream.on('error', (error) => {
|
|
113
|
-
closeReason
|
|
101
|
+
if (closeReason == null) {
|
|
102
|
+
closeReason = SyncCloseReason.StreamError;
|
|
103
|
+
connectionError = error;
|
|
104
|
+
}
|
|
114
105
|
controller.abort();
|
|
115
106
|
// Note: This appears as a 200 response in the logs.
|
|
116
107
|
if (error.message != 'Shutting down system') {
|
|
@@ -130,11 +121,17 @@ export const syncStreamed = routeDefinition({
|
|
|
130
121
|
},
|
|
131
122
|
data: stream,
|
|
132
123
|
afterSend: async (details) => {
|
|
124
|
+
// A client disconnect must not overwrite an earlier error or server close.
|
|
133
125
|
if (details.clientClosed) {
|
|
134
|
-
closeReason ??=
|
|
126
|
+
closeReason ??= SyncCloseReason.ClientClosed;
|
|
135
127
|
}
|
|
136
128
|
controller.abort();
|
|
137
129
|
metricsEngine.getUpDownCounter(APIMetric.CONCURRENT_CONNECTIONS).add(-1);
|
|
130
|
+
recordSyncConnection(metricsEngine, {
|
|
131
|
+
transport: SyncTransport.HttpStream,
|
|
132
|
+
closeReason: closeReason ?? SyncCloseReason.Unknown,
|
|
133
|
+
error: connectionError
|
|
134
|
+
});
|
|
138
135
|
service_context.eventsEngine.emit(event_types.EventsEngineEventType.SDK_DISCONNECT_EVENT, {
|
|
139
136
|
...sdkData,
|
|
140
137
|
disconnected_at: new Date()
|
|
@@ -143,7 +140,7 @@ export const syncStreamed = routeDefinition({
|
|
|
143
140
|
...tracker.getLogMeta(),
|
|
144
141
|
app_metadata: formattedAppMetadata,
|
|
145
142
|
stream_ms: Date.now() - streamStart,
|
|
146
|
-
close_reason: closeReason
|
|
143
|
+
close_reason: syncConnectionCloseReasonLogText(closeReason)
|
|
147
144
|
});
|
|
148
145
|
}
|
|
149
146
|
});
|
|
@@ -151,6 +148,11 @@ export const syncStreamed = routeDefinition({
|
|
|
151
148
|
catch (ex) {
|
|
152
149
|
controller.abort();
|
|
153
150
|
metricsEngine.getUpDownCounter(APIMetric.CONCURRENT_CONNECTIONS).add(-1);
|
|
151
|
+
recordSyncConnection(metricsEngine, {
|
|
152
|
+
transport: SyncTransport.HttpStream,
|
|
153
|
+
closeReason: SyncCloseReason.StreamError,
|
|
154
|
+
error: ex
|
|
155
|
+
});
|
|
154
156
|
service_context.eventsEngine.emit(event_types.EventsEngineEventType.SDK_DISCONNECT_EVENT, {
|
|
155
157
|
...sdkData,
|
|
156
158
|
disconnected_at: new Date()
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"sync-stream.js","sourceRoot":"","sources":["../../../src/routes/endpoints/sync-stream.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"sync-stream.js","sourceRoot":"","sources":["../../../src/routes/endpoints/sync-stream.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,mCAAmC,CAAC;AACnE,OAAO,UAAU,MAAM,YAAY,CAAC;AACpC,OAAO,EAAE,QAAQ,EAAE,MAAM,QAAQ,CAAC;AAElC,OAAO,KAAK,IAAI,MAAM,0BAA0B,CAAC;AACjD,OAAO,KAAK,IAAI,MAAM,0BAA0B,CAAC;AAEjD,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,0BAA0B,CAAC;AAClE,OAAO,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AACtC,OAAO,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AAE/C,OAAO,EACL,oBAAoB,EACpB,eAAe,EACf,gCAAgC,EAChC,aAAa,EACd,MAAM,qCAAqC,CAAC;AAC7C,OAAO,EAAE,qBAAqB,EAAE,MAAM,6BAA6B,CAAC;AACpE,OAAO,EAAE,2BAA2B,EAAE,MAAM,mBAAmB,CAAC;AAChE,OAAO,EAAE,0BAA0B,EAAE,MAAM,uBAAuB,CAAC;AAEnE,MAAM,CAAN,IAAY,UAEX;AAFD,WAAY,UAAU;IACpB,qCAAuB,CAAA;AACzB,CAAC,EAFW,UAAU,KAAV,UAAU,QAErB;AAED,MAAM,iBAAiB,GAAG,sBAAsB,CAAC;AACjD,MAAM,2BAA2B,GAAG,uCAAuC,CAAC;AAC5E,MAAM,qBAAqB,GAAG,CAAC,iBAAiB,EAAE,2BAA2B,CAAC,CAAC;AAE/E,MAAM,CAAC,MAAM,YAAY,GAAG,eAAe,CAAC;IAC1C,IAAI,EAAE,UAAU,CAAC,MAAM;IACvB,MAAM,EAAE,MAAM,CAAC,UAAU,CAAC,IAAI;IAC9B,SAAS,EAAE,QAAQ;IACnB,SAAS,EAAE,MAAM,CAAC,sBAAsB,CAAC,IAAI,CAAC,oBAAoB,EAAE,EAAE,eAAe,EAAE,IAAI,EAAE,CAAC;IAC9F,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE;QACzB,MAAM,EAAE,eAAe,EAAE,MAAM,EAAE,aAAa,EAAE,GAAG,OAAO,CAAC,OAAO,CAAC;QACnE,MAAM,EAAE,YAAY,EAAE,aAAa,EAAE,WAAW,EAAE,GAAG,eAAe,CAAC;QACrE,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC;QACxC,MAAM,SAAS,GAAG,OAAO,CAAC,cAAc,CAAC,IAAI,OAAO,CAAC,YAAY,CAAC,CAAC;QACnE,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,SAAS,CAAC;QAC1C,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC/B,MAAM,UAAU,GAAG,IAAI,UAAU,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QACnD,8FAA8F;QAC9F,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM;YAC5C,CAAC,CAAC,UAAU,CAAC,SAAS,CAAC,qBAAqB,CAAC,IAAI,2BAA2B;YAC5E,CAAC,CAAC,KAAK,CAAC;QAEV,MAAM,CAAC,WAAW,GAAG;YACnB,GAAG,MAAM,CAAC,WAAW;YACrB,UAAU,EAAE,SAAS;YACrB,SAAS,EAAE,QAAQ;YACnB,OAAO,EAAE,OAAO,CAAC,OAAO,CAAC,aAAc,CAAC,UAAU;YAClD,IAAI,EAAE,OAAO;SACd,CAAC;QACF,MAAM,OAAO,GAA0E;YACrF,SAAS,EAAE,QAAQ,IAAI,EAAE;YACzB,OAAO,EAAE,OAAO,CAAC,OAAO,CAAC,aAAc,CAAC,YAAY;YACpD,UAAU,EAAE,SAAmB;YAC/B,8DAA8D;YAC9D,OAAO,EAAE,IAAI,IAAI,CAAC,aAAc,CAAC,GAAG,GAAG,IAAI,CAAC;YAC5C,YAAY,EAAE,IAAI,IAAI,CAAC,WAAW,CAAC;SACpC,CAAC;QAEF,MAAM,EAAE,aAAa,EAAE,SAAS,EAAE,GAAG,MAAM,0BAA0B,CAAC,eAAe,EAAE,aAAa,CAAC,UAAU,CAAC,CAAC;QAEjH,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;QACzC,MAAM,OAAO,GAAG,IAAI,IAAI,CAAC,cAAc,CAAC,aAAa,CAAC,CAAC;QAEvD,MAAM,oBAAoB,GAAG,OAAO,CAAC,MAAM,CAAC,YAAY;YACtD,CAAC,CAAC,qBAAqB,CAAC,OAAO,CAAC,MAAM,CAAC,YAAY,CAAC;YACpD,CAAC,CAAC,SAAS,CAAC;QAEd,MAAM,CAAC,IAAI,CAAC,qBAAqB,EAAE;YACjC,YAAY,EAAE,oBAAoB;YAClC,aAAa,EAAE,OAAO,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,qBAAqB,CAAC,OAAO,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,SAAS;SACxG,CAAC,CAAC;QAEH,IAAI,CAAC;YACH,aAAa,CAAC,gBAAgB,CAAC,SAAS,CAAC,sBAAsB,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;YACxE,eAAe,CAAC,YAAY,CAAC,IAAI,CAAC,WAAW,CAAC,qBAAqB,CAAC,iBAAiB,EAAE,OAAO,CAAC,CAAC;YAChG,MAAM,SAAS,GAAG,IAAI,CAAC,cAAc,CAAC;gBACpC,WAAW,EAAE,WAAW;gBACxB,aAAa;gBACb,SAAS;gBACT,MAAM,EAAE,OAAO,CAAC,MAAM;gBACtB,KAAK,EAAE,OAAO,CAAC,OAAO,CAAC,aAAc;gBACrC,OAAO;gBACP,MAAM,EAAE,UAAU,CAAC,MAAM;gBACzB,MAAM;gBACN,gBAAgB,EAAE,OAAO;aAC1B,CAAC,CAAC;YAEH,MAAM,YAAY,GAAG,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;YAClF,MAAM,WAAW,GAAG,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,uBAAuB,CAAC,YAAY,EAAE,OAAO,CAAC,EAAE;gBACrF,UAAU,EAAE,KAAK;gBACjB,aAAa,EAAE,EAAE,GAAG,IAAI;aACzB,CAAC,CAAC;YACH,MAAM,EAAE,MAAM,EAAE,eAAe,EAAE,GAAG,2BAA2B,CAAC,UAAU,EAAE,WAAW,EAAE,OAAO,CAAC,CAAC;YAElG,kDAAkD;YAClD,4EAA4E;YAC5E,6CAA6C;YAC7C,IAAI,WAAW,GAAgC,SAAS,CAAC;YACzD,IAAI,eAAwB,CAAC;YAE7B,MAAM,UAAU,GAAG,YAAY,CAAC,cAAc,CAAC,GAAG,EAAE;gBAClD,uDAAuD;gBACvD,UAAU,CAAC,KAAK,EAAE,CAAC;gBACnB,WAAW,KAAK,eAAe,CAAC,eAAe,CAAC;gBAChD,MAAM,CAAC,OAAO,CAAC,IAAI,KAAK,CAAC,sBAAsB,CAAC,CAAC,CAAC;YACpD,CAAC,CAAC,CAAC;YAEH,MAAM,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE;gBACpB,4CAA4C;gBAC5C,WAAW,KAAK,eAAe,CAAC,aAAa,CAAC;YAChD,CAAC,CAAC,CAAC;YAEH,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE;gBACtB,UAAU,EAAE,CAAC;YACf,CAAC,CAAC,CAAC;YAEH,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,EAAE;gBAC3B,IAAI,WAAW,IAAI,IAAI,EAAE,CAAC;oBACxB,WAAW,GAAG,eAAe,CAAC,WAAW,CAAC;oBAC1C,eAAe,GAAG,KAAK,CAAC;gBAC1B,CAAC;gBACD,UAAU,CAAC,KAAK,EAAE,CAAC;gBACnB,oDAAoD;gBACpD,IAAI,KAAK,CAAC,OAAO,IAAI,sBAAsB,EAAE,CAAC;oBAC5C,MAAM,CAAC,KAAK,CAAC,+BAA+B,EAAE,KAAK,CAAC,CAAC;gBACvD,CAAC;YACH,CAAC,CAAC,CAAC;YAEH,OAAO,IAAI,MAAM,CAAC,cAAc,CAAC;gBAC/B,MAAM,EAAE,GAAG;gBACX,OAAO,EAAE;oBACP,cAAc,EAAE,OAAO,CAAC,CAAC,CAAC,2BAA2B,CAAC,CAAC,CAAC,iBAAiB;oBACzE,GAAG,eAAe;oBAClB,8GAA8G;oBAC9G,wGAAwG;oBACxG,yGAAyG;oBACzG,0EAA0E;oBAC1E,mBAAmB,EAAE,IAAI;iBAC1B;gBACD,IAAI,EAAE,MAAM;gBACZ,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE;oBAC3B,2EAA2E;oBAC3E,IAAI,OAAO,CAAC,YAAY,EAAE,CAAC;wBACzB,WAAW,KAAK,eAAe,CAAC,YAAY,CAAC;oBAC/C,CAAC;oBACD,UAAU,CAAC,KAAK,EAAE,CAAC;oBACnB,aAAa,CAAC,gBAAgB,CAAC,SAAS,CAAC,sBAAsB,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;oBACzE,oBAAoB,CAAC,aAAa,EAAE;wBAClC,SAAS,EAAE,aAAa,CAAC,UAAU;wBACnC,WAAW,EAAE,WAAW,IAAI,eAAe,CAAC,OAAO;wBACnD,KAAK,EAAE,eAAe;qBACvB,CAAC,CAAC;oBACH,eAAe,CAAC,YAAY,CAAC,IAAI,CAAC,WAAW,CAAC,qBAAqB,CAAC,oBAAoB,EAAE;wBACxF,GAAG,OAAO;wBACV,eAAe,EAAE,IAAI,IAAI,EAAE;qBAC5B,CAAC,CAAC;oBACH,MAAM,CAAC,IAAI,CAAC,sBAAsB,EAAE;wBAClC,GAAG,OAAO,CAAC,UAAU,EAAE;wBACvB,YAAY,EAAE,oBAAoB;wBAClC,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,WAAW;wBACnC,YAAY,EAAE,gCAAgC,CAAC,WAAW,CAAC;qBAC5D,CAAC,CAAC;gBACL,CAAC;aACF,CAAC,CAAC;QACL,CAAC;QAAC,OAAO,EAAE,EAAE,CAAC;YACZ,UAAU,CAAC,KAAK,EAAE,CAAC;YACnB,aAAa,CAAC,gBAAgB,CAAC,SAAS,CAAC,sBAAsB,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;YACzE,oBAAoB,CAAC,aAAa,EAAE;gBAClC,SAAS,EAAE,aAAa,CAAC,UAAU;gBACnC,WAAW,EAAE,eAAe,CAAC,WAAW;gBACxC,KAAK,EAAE,EAAE;aACV,CAAC,CAAC;YACH,eAAe,CAAC,YAAY,CAAC,IAAI,CAAC,WAAW,CAAC,qBAAqB,CAAC,oBAAoB,EAAE;gBACxF,GAAG,OAAO;gBACV,eAAe,EAAE,IAAI,IAAI,EAAE;aAC5B,CAAC,CAAC;QACL,CAAC;IACH,CAAC;CACF,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,kBAAkB,GAAG,CAAC,YAAY,CAAC,CAAC"}
|
package/dist/routes/hooks.d.ts
CHANGED
|
@@ -7,4 +7,4 @@ export type CreateRequestQueueParams = {
|
|
|
7
7
|
* Creates a request queue which limits the amount of concurrent connections which
|
|
8
8
|
* are active at any time.
|
|
9
9
|
*/
|
|
10
|
-
export declare const createRequestQueueHook: (params: CreateRequestQueueParams) => fastify.onRequestHookHandler;
|
|
10
|
+
export declare const createRequestQueueHook: (params: CreateRequestQueueParams, onRejected?: () => void) => fastify.onRequestHookHandler;
|
package/dist/routes/hooks.js
CHANGED
|
@@ -4,7 +4,7 @@ import a from 'async';
|
|
|
4
4
|
* Creates a request queue which limits the amount of concurrent connections which
|
|
5
5
|
* are active at any time.
|
|
6
6
|
*/
|
|
7
|
-
export const createRequestQueueHook = (params) => {
|
|
7
|
+
export const createRequestQueueHook = (params, onRejected) => {
|
|
8
8
|
const request_queue = a.queue((event, done) => {
|
|
9
9
|
event().finally(done);
|
|
10
10
|
}, params.concurrency);
|
|
@@ -17,6 +17,7 @@ export const createRequestQueueHook = (params) => {
|
|
|
17
17
|
path: request.url,
|
|
18
18
|
queue_overflow: true
|
|
19
19
|
});
|
|
20
|
+
onRejected?.();
|
|
20
21
|
return reply.status(429).send();
|
|
21
22
|
}
|
|
22
23
|
const finished = new Promise((resolve) => {
|
package/dist/routes/hooks.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"hooks.js","sourceRoot":"","sources":["../../src/routes/hooks.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,mCAAmC,CAAC;AAC3D,OAAO,CAAC,MAAM,OAAO,CAAC;AAQtB;;;GAGG;AACH,MAAM,CAAC,MAAM,sBAAsB,GAAG,
|
|
1
|
+
{"version":3,"file":"hooks.js","sourceRoot":"","sources":["../../src/routes/hooks.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,mCAAmC,CAAC;AAC3D,OAAO,CAAC,MAAM,OAAO,CAAC;AAQtB;;;GAGG;AACH,MAAM,CAAC,MAAM,sBAAsB,GAAG,CACpC,MAAgC,EAChC,UAAuB,EACO,EAAE;IAChC,MAAM,aAAa,GAAG,CAAC,CAAC,KAAK,CAAsB,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE;QACjE,KAAK,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IACxB,CAAC,EAAE,MAAM,CAAC,WAAW,CAAC,CAAC;IAEvB,OAAO,CAAC,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,EAAE;QAC9B,IACE,CAAC,MAAM,CAAC,eAAe,IAAI,CAAC,IAAI,aAAa,CAAC,OAAO,EAAE,IAAI,MAAM,CAAC,WAAW,CAAC;YAC9E,CAAC,MAAM,CAAC,eAAe,GAAG,CAAC,IAAI,aAAa,CAAC,MAAM,EAAE,IAAI,MAAM,CAAC,eAAe,CAAC,EAChF,CAAC;YACD,MAAM,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,MAAM,IAAI,OAAO,CAAC,GAAG,EAAE,EAAE;gBAC9C,MAAM,EAAE,GAAG;gBACX,MAAM,EAAE,OAAO,CAAC,MAAM;gBACtB,IAAI,EAAE,OAAO,CAAC,GAAG;gBACjB,cAAc,EAAE,IAAI;aACrB,CAAC,CAAC;YACH,UAAU,EAAE,EAAE,CAAC;YACf,OAAO,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;QAClC,CAAC;QAED,MAAM,QAAQ,GAAG,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;YAC7C,KAAK,CAAC,IAAI,CACR,GAAG,EAAE,CAAC,OAAO,EAAE,EACf,GAAG,EAAE,CAAC,OAAO,EAAE,CAChB,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,aAAa,CAAC,IAAI,CAAC,GAAG,EAAE;YACtB,IAAI,EAAE,CAAC;YACP,OAAO,QAAQ,CAAC;QAClB,CAAC,CAAC,CAAC;IACL,CAAC,CAAC;AACJ,CAAC,CAAC"}
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
import { SyncTransport } from '../metrics/connection-metrics.js';
|
|
2
|
+
import type { RouterServiceContext } from './router.js';
|
|
3
|
+
/** Resolves stream resources, counting setup failures as rejections before rethrowing. */
|
|
4
|
+
export declare function resolveSyncConnectionSetup(serviceContext: RouterServiceContext, transport: SyncTransport): Promise<{
|
|
5
|
+
bucketStorage: import("../index.js").SyncRulesBucketStorage;
|
|
6
|
+
syncRules: import("@powersync/service-sync-rules").HydratedSyncConfig;
|
|
7
|
+
}>;
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
import { ErrorCode, errors } from '@powersync/lib-services-framework';
|
|
2
|
+
import { recordSyncConnection, SyncCloseReason } from '../metrics/connection-metrics.js';
|
|
3
|
+
/** Resolves stream resources, counting setup failures as rejections before rethrowing. */
|
|
4
|
+
export async function resolveSyncConnectionSetup(serviceContext, transport) {
|
|
5
|
+
const { routerEngine, storageEngine, metricsEngine } = serviceContext;
|
|
6
|
+
let closeReason = SyncCloseReason.ServiceUnavailable;
|
|
7
|
+
try {
|
|
8
|
+
if (routerEngine.closed) {
|
|
9
|
+
throw new errors.ServiceError({
|
|
10
|
+
status: 503,
|
|
11
|
+
code: ErrorCode.PSYNC_S2003,
|
|
12
|
+
description: 'Service temporarily unavailable'
|
|
13
|
+
});
|
|
14
|
+
}
|
|
15
|
+
closeReason = SyncCloseReason.StorageError;
|
|
16
|
+
const bucketStorage = (await storageEngine.activeBucketStorage.getActiveSyncConfig())?.storage;
|
|
17
|
+
closeReason = SyncCloseReason.NoSyncConfig;
|
|
18
|
+
if (bucketStorage == null) {
|
|
19
|
+
throw new errors.ServiceError({
|
|
20
|
+
status: 500,
|
|
21
|
+
code: ErrorCode.PSYNC_S2302,
|
|
22
|
+
description: 'No sync config available'
|
|
23
|
+
});
|
|
24
|
+
}
|
|
25
|
+
closeReason = SyncCloseReason.SyncConfigError;
|
|
26
|
+
const syncRules = bucketStorage.getParsedSyncRules(routerEngine.getAPI().getParseSyncRulesOptions());
|
|
27
|
+
return { bucketStorage, syncRules };
|
|
28
|
+
}
|
|
29
|
+
catch (error) {
|
|
30
|
+
recordSyncConnection(metricsEngine, { transport, closeReason, error });
|
|
31
|
+
throw error;
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
//# sourceMappingURL=sync-connection.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"sync-connection.js","sourceRoot":"","sources":["../../src/routes/sync-connection.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,mCAAmC,CAAC;AAEtE,OAAO,EAAE,oBAAoB,EAAE,eAAe,EAAiB,MAAM,kCAAkC,CAAC;AAGxG,0FAA0F;AAC1F,MAAM,CAAC,KAAK,UAAU,0BAA0B,CAAC,cAAoC,EAAE,SAAwB;IAC7G,MAAM,EAAE,YAAY,EAAE,aAAa,EAAE,aAAa,EAAE,GAAG,cAAc,CAAC;IACtE,IAAI,WAAW,GAAG,eAAe,CAAC,kBAAkB,CAAC;IAErD,IAAI,CAAC;QACH,IAAI,YAAY,CAAC,MAAM,EAAE,CAAC;YACxB,MAAM,IAAI,MAAM,CAAC,YAAY,CAAC;gBAC5B,MAAM,EAAE,GAAG;gBACX,IAAI,EAAE,SAAS,CAAC,WAAW;gBAC3B,WAAW,EAAE,iCAAiC;aAC/C,CAAC,CAAC;QACL,CAAC;QAED,WAAW,GAAG,eAAe,CAAC,YAAY,CAAC;QAC3C,MAAM,aAAa,GAAG,CAAC,MAAM,aAAa,CAAC,mBAAmB,CAAC,mBAAmB,EAAE,CAAC,EAAE,OAAO,CAAC;QAE/F,WAAW,GAAG,eAAe,CAAC,YAAY,CAAC;QAC3C,IAAI,aAAa,IAAI,IAAI,EAAE,CAAC;YAC1B,MAAM,IAAI,MAAM,CAAC,YAAY,CAAC;gBAC5B,MAAM,EAAE,GAAG;gBACX,IAAI,EAAE,SAAS,CAAC,WAAW;gBAC3B,WAAW,EAAE,0BAA0B;aACxC,CAAC,CAAC;QACL,CAAC;QAED,WAAW,GAAG,eAAe,CAAC,eAAe,CAAC;QAC9C,MAAM,SAAS,GAAG,aAAa,CAAC,kBAAkB,CAAC,YAAY,CAAC,MAAM,EAAE,CAAC,wBAAwB,EAAE,CAAC,CAAC;QACrG,OAAO,EAAE,aAAa,EAAE,SAAS,EAAE,CAAC;IACtC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,oBAAoB,CAAC,aAAa,EAAE,EAAE,SAAS,EAAE,WAAW,EAAE,KAAK,EAAE,CAAC,CAAC;QACvE,MAAM,KAAK,CAAC;IACd,CAAC;AACH,CAAC"}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { BaseObserver, logger } from '@powersync/lib-services-framework';
|
|
2
2
|
import { SerializedSyncPlan as RawSerializedSyncPlan, SerializedCompatibilityContext, SyncConfigWithErrors } from '@powersync/service-sync-rules';
|
|
3
|
-
import { ReplicationError } from '@powersync/service-types';
|
|
3
|
+
import { ReplicationError, ValidationDiagnostic } from '@powersync/service-types';
|
|
4
4
|
import { PersistedReplicationStream } from './PersistedReplicationStream.js';
|
|
5
5
|
import { PersistedSyncConfigContent } from './PersistedSyncConfigContent.js';
|
|
6
6
|
import { ReplicationEventPayload } from './ReplicationEventPayload.js';
|
|
@@ -32,6 +32,10 @@ export declare abstract class BucketStorageFactory extends BaseObserver<BucketSt
|
|
|
32
32
|
* Deploy new sync config.
|
|
33
33
|
*/
|
|
34
34
|
abstract updateSyncRules(options: UpdateSyncRulesOptions): Promise<PersistedReplicationStream>;
|
|
35
|
+
/**
|
|
36
|
+
* Stop pending processing atomically, preserving active configs. Used for rejected startup submissions.
|
|
37
|
+
*/
|
|
38
|
+
abstract stopProcessingSyncConfigs(): Promise<void>;
|
|
35
39
|
/**
|
|
36
40
|
* Indicate that a slot was removed, and we should re-sync by creating
|
|
37
41
|
* a new replication stream.
|
|
@@ -40,6 +44,8 @@ export declare abstract class BucketStorageFactory extends BaseObserver<BucketSt
|
|
|
40
44
|
* config, but also accounts for cases where the current sync config is not
|
|
41
45
|
* the latest one.
|
|
42
46
|
*
|
|
47
|
+
* Recovery of the active config must atomically recheck pending work and create its replacement,
|
|
48
|
+
* preserving any newer pending deployment rather than superseding it with the active config.
|
|
43
49
|
* Replication should be restarted after this.
|
|
44
50
|
*/
|
|
45
51
|
abstract restartReplication(replicationStreamId: number): Promise<void>;
|
|
@@ -118,22 +124,27 @@ export interface StorageSyncConfigMetrics {
|
|
|
118
124
|
attributed_source_records_bytes: number;
|
|
119
125
|
attributed_object_storage_bytes: number;
|
|
120
126
|
}
|
|
127
|
+
/**
|
|
128
|
+
* Sync config content and its compiled representation, shared by startup validation and deployment.
|
|
129
|
+
* Validation errors are retained in parsed even when the config cannot be deployed.
|
|
130
|
+
*/
|
|
131
|
+
export interface CompiledSyncConfig {
|
|
132
|
+
yaml: string;
|
|
133
|
+
/**
|
|
134
|
+
* The serialized sync plan for the sync configuration, or `null` for configurations not using the Sync Streams
|
|
135
|
+
* compiler.
|
|
136
|
+
*/
|
|
137
|
+
plan: SerializedSyncPlan | null;
|
|
138
|
+
/**
|
|
139
|
+
* Parsed sync config, primarily to generate a definition mapping.
|
|
140
|
+
* Not persisted, and the defaultSchema used for parsing is not relevant.
|
|
141
|
+
*/
|
|
142
|
+
parsed: SyncConfigWithErrors;
|
|
143
|
+
}
|
|
121
144
|
export interface UpdateSyncRulesOptions {
|
|
122
145
|
/** Optional operator-supplied label identifying this deployed sync config version. */
|
|
123
146
|
version_label?: string;
|
|
124
|
-
config:
|
|
125
|
-
yaml: string;
|
|
126
|
-
/**
|
|
127
|
-
* The serialized sync plan for the sync configuration, or `null` for configurations not using the Sync Streams
|
|
128
|
-
* compiler.
|
|
129
|
-
*/
|
|
130
|
-
plan: SerializedSyncPlan | null;
|
|
131
|
-
/**
|
|
132
|
-
* Parsed sync config, primarily to generate a definition mapping.
|
|
133
|
-
* Not persisted, and the defaultSchema used for parsing is not relevant.
|
|
134
|
-
*/
|
|
135
|
-
parsed: SyncConfigWithErrors;
|
|
136
|
-
};
|
|
147
|
+
config: CompiledSyncConfig;
|
|
137
148
|
lock?: boolean;
|
|
138
149
|
storageVersion?: number;
|
|
139
150
|
/**
|
|
@@ -166,15 +177,12 @@ export declare function updateSyncRulesFromYaml(content: string, options?: Omit<
|
|
|
166
177
|
}): UpdateSyncRulesOptions;
|
|
167
178
|
export declare function updateSyncRulesFromConfig(parsed: SyncConfigWithErrors, options?: Omit<UpdateSyncRulesOptions, 'config'>): UpdateSyncRulesOptions;
|
|
168
179
|
/**
|
|
169
|
-
*
|
|
170
|
-
*
|
|
171
|
-
*
|
|
172
|
-
*
|
|
173
|
-
*
|
|
174
|
-
* Fatal errors usually throw during parsing before this is reached. They can reach this when the sync config is parsed
|
|
175
|
-
* without validation, such as at startup with `exit_on_error: false`.
|
|
180
|
+
* Log parser errors and warnings from a parsed config, or combined parsing and source-capability
|
|
181
|
+
* diagnostics from SyncConfigEngine. Use when loading or deploying a sync config so operators see
|
|
182
|
+
* diagnostics in service logs as well as API responses. Fatal diagnostics are logged at error level;
|
|
183
|
+
* warnings remain advisory. Uses the supplied logger or the service logger by default.
|
|
176
184
|
*/
|
|
177
|
-
export declare function logSyncConfigErrors(
|
|
185
|
+
export declare function logSyncConfigErrors(result: SyncConfigWithErrors | ValidationDiagnostic[], log?: typeof logger): void;
|
|
178
186
|
export interface GetIntanceOptions {
|
|
179
187
|
/**
|
|
180
188
|
* The job lease, including a lease acquired during initial configuration.
|