@powersync/service-core 1.27.1 → 1.28.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (131) hide show
  1. package/CHANGELOG.md +32 -0
  2. package/LICENSE +201 -67
  3. package/dist/api/RouteAPI.d.ts +6 -0
  4. package/dist/api/api-metrics.js +4 -0
  5. package/dist/api/api-metrics.js.map +1 -1
  6. package/dist/api/diagnostics.d.ts +5 -0
  7. package/dist/api/diagnostics.js +17 -2
  8. package/dist/api/diagnostics.js.map +1 -1
  9. package/dist/metrics/connection-metrics.d.ts +28 -0
  10. package/dist/metrics/connection-metrics.js +55 -0
  11. package/dist/metrics/connection-metrics.js.map +1 -0
  12. package/dist/metrics/metrics-index.d.ts +1 -0
  13. package/dist/metrics/metrics-index.js +1 -0
  14. package/dist/metrics/metrics-index.js.map +1 -1
  15. package/dist/metrics/metrics-interfaces.d.ts +7 -1
  16. package/dist/metrics/metrics-interfaces.js.map +1 -1
  17. package/dist/modules/loader.js +8 -2
  18. package/dist/modules/loader.js.map +1 -1
  19. package/dist/replication/AbstractReplicationJob.d.ts +2 -0
  20. package/dist/replication/AbstractReplicationJob.js +17 -1
  21. package/dist/replication/AbstractReplicationJob.js.map +1 -1
  22. package/dist/replication/AbstractReplicator.d.ts +8 -18
  23. package/dist/replication/AbstractReplicator.js +63 -64
  24. package/dist/replication/AbstractReplicator.js.map +1 -1
  25. package/dist/replication/ReplicationModule.js +0 -6
  26. package/dist/replication/ReplicationModule.js.map +1 -1
  27. package/dist/routes/configure-fastify.js +5 -1
  28. package/dist/routes/configure-fastify.js.map +1 -1
  29. package/dist/routes/endpoints/admin.js +24 -22
  30. package/dist/routes/endpoints/admin.js.map +1 -1
  31. package/dist/routes/endpoints/socket-route.js +19 -27
  32. package/dist/routes/endpoints/socket-route.js.map +1 -1
  33. package/dist/routes/endpoints/sync-rules.js +17 -40
  34. package/dist/routes/endpoints/sync-rules.js.map +1 -1
  35. package/dist/routes/endpoints/sync-stream.js +25 -23
  36. package/dist/routes/endpoints/sync-stream.js.map +1 -1
  37. package/dist/routes/hooks.d.ts +1 -1
  38. package/dist/routes/hooks.js +2 -1
  39. package/dist/routes/hooks.js.map +1 -1
  40. package/dist/routes/sync-connection.d.ts +7 -0
  41. package/dist/routes/sync-connection.js +34 -0
  42. package/dist/routes/sync-connection.js.map +1 -0
  43. package/dist/storage/BucketStorageFactory.d.ts +30 -22
  44. package/dist/storage/BucketStorageFactory.js +14 -14
  45. package/dist/storage/BucketStorageFactory.js.map +1 -1
  46. package/dist/storage/PersistedSyncConfigContent.js +2 -2
  47. package/dist/storage/PersistedSyncConfigContent.js.map +1 -1
  48. package/dist/storage/SyncRulesBucketStorage.d.ts +23 -2
  49. package/dist/storage/SyncRulesBucketStorage.js.map +1 -1
  50. package/dist/streams/BroadcastIterable.js +8 -7
  51. package/dist/streams/BroadcastIterable.js.map +1 -1
  52. package/dist/streams/LastValueSink.d.ts +2 -2
  53. package/dist/streams/LastValueSink.js +8 -13
  54. package/dist/streams/LastValueSink.js.map +1 -1
  55. package/dist/streams/merge.d.ts +15 -31
  56. package/dist/streams/merge.js +136 -152
  57. package/dist/streams/merge.js.map +1 -1
  58. package/dist/sync/BucketChecksumState.d.ts +3 -2
  59. package/dist/sync/BucketChecksumState.js +12 -5
  60. package/dist/sync/BucketChecksumState.js.map +1 -1
  61. package/dist/sync/sync.js +48 -38
  62. package/dist/sync/sync.js.map +1 -1
  63. package/dist/sync/util.d.ts +1 -2
  64. package/dist/sync/util.js +2 -5
  65. package/dist/sync/util.js.map +1 -1
  66. package/dist/system/ServiceContext.d.ts +7 -0
  67. package/dist/system/ServiceContext.js +3 -3
  68. package/dist/system/ServiceContext.js.map +1 -1
  69. package/dist/system/SyncConfigEngine.d.ts +135 -0
  70. package/dist/system/SyncConfigEngine.js +186 -0
  71. package/dist/system/SyncConfigEngine.js.map +1 -0
  72. package/dist/system/system-index.d.ts +1 -0
  73. package/dist/system/system-index.js +1 -0
  74. package/dist/system/system-index.js.map +1 -1
  75. package/dist/util/config/types.d.ts +4 -0
  76. package/package.json +8 -9
  77. package/src/api/RouteAPI.ts +6 -0
  78. package/src/api/api-metrics.ts +5 -0
  79. package/src/api/diagnostics.ts +22 -2
  80. package/src/metrics/connection-metrics.ts +75 -0
  81. package/src/metrics/metrics-index.ts +1 -0
  82. package/src/metrics/metrics-interfaces.ts +8 -1
  83. package/src/modules/loader.ts +16 -2
  84. package/src/replication/AbstractReplicationJob.ts +17 -1
  85. package/src/replication/AbstractReplicator.ts +83 -81
  86. package/src/replication/ReplicationModule.ts +0 -5
  87. package/src/routes/configure-fastify.ts +7 -1
  88. package/src/routes/endpoints/admin.ts +27 -20
  89. package/src/routes/endpoints/socket-route.ts +26 -38
  90. package/src/routes/endpoints/sync-rules.ts +22 -46
  91. package/src/routes/endpoints/sync-stream.ts +31 -27
  92. package/src/routes/hooks.ts +5 -1
  93. package/src/routes/sync-connection.ts +39 -0
  94. package/src/storage/BucketStorageFactory.ts +45 -29
  95. package/src/storage/PersistedSyncConfigContent.ts +2 -2
  96. package/src/storage/SyncRulesBucketStorage.ts +23 -2
  97. package/src/streams/BroadcastIterable.ts +8 -7
  98. package/src/streams/LastValueSink.ts +10 -16
  99. package/src/streams/merge.ts +149 -169
  100. package/src/sync/BucketChecksumState.ts +17 -5
  101. package/src/sync/sync.ts +71 -57
  102. package/src/sync/util.ts +2 -7
  103. package/src/system/ServiceContext.ts +8 -3
  104. package/src/system/SyncConfigEngine.ts +247 -0
  105. package/src/system/system-index.ts +1 -0
  106. package/src/util/config/types.ts +4 -0
  107. package/test/src/AbstractReplicator.test.ts +375 -45
  108. package/test/src/PersistedSyncConfigContent.test.ts +5 -5
  109. package/test/src/SyncConfigEngine.test.ts +208 -0
  110. package/test/src/broadcast_iterable.test.ts +59 -31
  111. package/test/src/config.test.ts +65 -3
  112. package/test/src/connection-metrics-export.test.ts +76 -0
  113. package/test/src/connection-metrics.test.ts +184 -0
  114. package/test/src/diagnostics.test.ts +3 -3
  115. package/test/src/last_value_sink.test.ts +34 -0
  116. package/test/src/merge_iterable.test.ts +304 -55
  117. package/test/src/module-loader.test.ts +14 -1
  118. package/test/src/recording-metrics.ts +36 -0
  119. package/test/src/routes/admin.test.ts +2 -2
  120. package/test/src/routes/concurrency-limit.test.ts +108 -0
  121. package/test/src/routes/connection-metrics.test.ts +269 -0
  122. package/test/src/routes/invalid-config.test.ts +184 -0
  123. package/test/src/routes/mocks.ts +26 -13
  124. package/test/src/routes/stream-disconnect.integration.test.ts +129 -0
  125. package/test/src/routes/stream.test.ts +200 -107
  126. package/test/src/routes/sync-rules.test.ts +19 -1
  127. package/test/src/source_table_config.test.ts +7 -7
  128. package/test/src/sync/BucketChecksumState.test.ts +10 -10
  129. package/test/src/sync/stream_response.test.ts +389 -0
  130. package/test/src/sync/util.test.ts +14 -3
  131. package/tsconfig.tsbuildinfo +1 -1
@@ -11,9 +11,13 @@ import {
11
11
  SyncContext,
12
12
  SyncRulesBucketStorage
13
13
  } from '@/index.js';
14
+ import { SyncConfigEngine } from '@/system/SyncConfigEngine.js';
14
15
  import { MeterProvider } from '@opentelemetry/sdk-metrics';
15
16
 
16
- export function mockServiceContext(storage: Partial<SyncRulesBucketStorage> | null) {
17
+ export function mockServiceContext(
18
+ storage: Partial<SyncRulesBucketStorage> | null,
19
+ overrideMetricsEngine?: MetricsEngine
20
+ ) {
17
21
  // This is very incomplete - just enough to get the current tests passing.
18
22
 
19
23
  const storageEngine: StorageEngine = {
@@ -28,21 +32,28 @@ export function mockServiceContext(storage: Partial<SyncRulesBucketStorage> | nu
28
32
  } as Partial<BucketStorageFactory>
29
33
  } as any;
30
34
 
31
- const meterProvider = new MeterProvider({
32
- readers: []
33
- });
34
- const meter = meterProvider.getMeter('powersync-tests');
35
- const metricsEngine = new MetricsEngine({
36
- disable_telemetry_sharing: true,
37
- factory: new OpenTelemetryMetricsFactory(meter)
38
- });
39
- createCoreAPIMetrics(metricsEngine);
35
+ let metricsEngine = overrideMetricsEngine;
36
+ if (metricsEngine == null) {
37
+ const meterProvider = new MeterProvider({
38
+ readers: []
39
+ });
40
+ const meter = meterProvider.getMeter('powersync-tests');
41
+ metricsEngine = new MetricsEngine({
42
+ disable_telemetry_sharing: true,
43
+ factory: new OpenTelemetryMetricsFactory(meter)
44
+ });
45
+ createCoreAPIMetrics(metricsEngine);
46
+ }
47
+ let routeAPI: RouteAPI | undefined;
40
48
  const service_context: Partial<ServiceContext> = {
49
+ configuration: {
50
+ sync_rules: { present: false, exit_on_error: true }
51
+ } as ServiceContext['configuration'],
41
52
  syncContext: new SyncContext({ maxBuckets: 1, maxDataFetchConcurrency: 1, maxParameterQueryResults: 1 }),
42
53
  eventsEngine: new EventsEngine(),
43
54
  routerEngine: {
44
55
  getAPI() {
45
- return {
56
+ return (routeAPI ??= {
46
57
  getParseSyncRulesOptions() {
47
58
  return { defaultSchema: 'public' };
48
59
  },
@@ -67,7 +78,7 @@ export function mockServiceContext(storage: Partial<SyncRulesBucketStorage> | nu
67
78
  async getDebugTablesInfo() {
68
79
  return [];
69
80
  }
70
- } satisfies Partial<RouteAPI> as unknown as RouteAPI;
81
+ } satisfies Partial<RouteAPI> as unknown as RouteAPI);
71
82
  },
72
83
  addStopHandler() {
73
84
  return () => {};
@@ -76,11 +87,13 @@ export function mockServiceContext(storage: Partial<SyncRulesBucketStorage> | nu
76
87
  storageEngine,
77
88
  metricsEngine: metricsEngine,
78
89
  // Not used
79
- configuration: null as any,
80
90
  lifeCycleEngine: null as any,
81
91
  migrations: null as any,
82
92
  replicationEngine: null as any,
83
93
  serviceMode: null as any
84
94
  };
95
+ service_context.syncConfigEngine = new SyncConfigEngine(service_context.configuration!.sync_rules, () =>
96
+ service_context.routerEngine!.getAPI()
97
+ );
85
98
  return service_context as ServiceContext;
86
99
  }
@@ -0,0 +1,129 @@
1
+ import { CHECKPOINT_INVALIDATE_ALL, ContextProvider, JwtPayload, SyncRulesBucketStorage } from '@/index.js';
2
+ import { DEFAULT_HYDRATION_STATE, nodeSqlite, parseSyncConfig } from '@powersync/service-sync-rules';
3
+ import { APIMetric } from '@powersync/service-types';
4
+ import Fastify from 'fastify';
5
+ import * as http from 'node:http';
6
+ import * as sqlite from 'node:sqlite';
7
+ import { describe, expect, it, onTestFinished } from 'vitest';
8
+ import { syncStreamed } from '../../../src/routes/endpoints/sync-stream.js';
9
+ import { registerFastifyErrorHandler, registerFastifyRoutes } from '../../../src/routes/route-register.js';
10
+ import { recordingMetricsEngine } from '../recording-metrics.js';
11
+ import { mockServiceContext } from './mocks.js';
12
+
13
+ // Use real HTTP disconnects to verify that transport teardown isn't counted as a stream error.
14
+ describe('Sync stream client disconnect', () => {
15
+ /** A stream still writing operations when the client disconnects, as on an initial sync. */
16
+ function busyStorage() {
17
+ return {
18
+ getParsedSyncRules: () =>
19
+ parseSyncConfig(
20
+ `
21
+ config:
22
+ edition: 3
23
+ streams:
24
+ global:
25
+ auto_subscribe: true
26
+ query: SELECT * FROM test
27
+ `,
28
+ { defaultSchema: 'public' }
29
+ ).config.hydrate({ hydrationState: DEFAULT_HYDRATION_STATE, sqlite: nodeSqlite(sqlite) }),
30
+ getChecksums: async (_checkpoint, buckets) =>
31
+ new Map(buckets.map(({ bucket }) => [bucket, { bucket, checksum: 1, count: 100_000 }])),
32
+ getBucketDataBatch: async function* (_checkpoint, buckets) {
33
+ // Bounded so an abort that fails to stop the stream cannot run away.
34
+ for (let batch = 0; batch < 100; batch++) {
35
+ yield {
36
+ chunkData: {
37
+ bucket: buckets[0].bucket,
38
+ data: Array.from({ length: 1000 }, () => ({ op: 'PUT' })),
39
+ has_more: true,
40
+ after: `${batch * 1000}`,
41
+ next_after: `${(batch + 1) * 1000}`
42
+ },
43
+ targetOp: null
44
+ };
45
+ }
46
+ yield { hasMore: true };
47
+ },
48
+ watchCheckpointChanges: async function* ({ signal }) {
49
+ yield {
50
+ base: { checkpoint: 1n, lsn: '1', getParameterSets: async () => [] },
51
+ writeCheckpoint: null,
52
+ update: CHECKPOINT_INVALIDATE_ALL
53
+ };
54
+ await new Promise<void>((resolve) => signal.addEventListener('abort', () => resolve(), { once: true }));
55
+ }
56
+ } as Partial<SyncRulesBucketStorage>;
57
+ }
58
+
59
+ async function abortClientMidStream(acceptEncoding: string) {
60
+ const recorder = recordingMetricsEngine('stream-disconnect-test');
61
+ onTestFinished(() => recorder.shutdown());
62
+ const service_context = mockServiceContext(busyStorage(), recorder.engine);
63
+ const contextProvider: ContextProvider = async (_request, options) => ({
64
+ logger: options.logger,
65
+ service_context,
66
+ token_payload: new JwtPayload({
67
+ exp: Date.now() / 1000 + 10_000,
68
+ iat: Date.now() / 1000 - 10_000,
69
+ sub: 'test-user'
70
+ })
71
+ });
72
+
73
+ const app = Fastify();
74
+ registerFastifyErrorHandler(app);
75
+ registerFastifyRoutes(app, contextProvider, [{ ...syncStreamed, authorize: async () => ({ authorized: true }) }]);
76
+
77
+ try {
78
+ const address = await app.listen({ port: 0, host: '127.0.0.1' });
79
+ const url = new URL(syncStreamed.path, address);
80
+
81
+ await new Promise<void>((resolve, reject) => {
82
+ const clientRequest = http.request(
83
+ {
84
+ host: url.hostname,
85
+ port: url.port,
86
+ path: url.pathname,
87
+ method: 'POST',
88
+ headers: { 'content-type': 'application/json', 'accept-encoding': acceptEncoding }
89
+ },
90
+ (clientResponse) => {
91
+ clientResponse.once('data', () => {
92
+ clientRequest.destroy();
93
+ resolve();
94
+ });
95
+ clientResponse.on('error', () => {});
96
+ }
97
+ );
98
+ clientRequest.on('error', (error) => reject(error));
99
+ clientRequest.end(JSON.stringify({ raw_data: true }));
100
+ });
101
+
102
+ // Wait for afterSend to record the close.
103
+ await expect
104
+ .poll(() =>
105
+ recorder.seriesValue(APIMetric.SYNC_CONNECTIONS, {
106
+ outcome: 'ok',
107
+ close_reason: 'client_closed',
108
+ transport: 'http_stream'
109
+ })
110
+ )
111
+ .toBe(1);
112
+
113
+ expect(await recorder.seriesValue(APIMetric.SYNC_CONNECTIONS, { outcome: 'error' })).toBeUndefined();
114
+ expect(await recorder.seriesValue(APIMetric.SYNC_CONNECTIONS, {})).toBe(1);
115
+ expect(await recorder.seriesValue(APIMetric.CONCURRENT_CONNECTIONS, {})).toBe(0);
116
+ } finally {
117
+ await app.close();
118
+ }
119
+ }
120
+
121
+ it('counts a real client hangup as an ok close', async () => {
122
+ await abortClientMidStream('identity');
123
+ });
124
+
125
+ it('counts a real client hangup on a compressed response as an ok close', async () => {
126
+ // Gzip adds a separate teardown path.
127
+ await abortClientMidStream('gzip');
128
+ });
129
+ });
@@ -7,20 +7,23 @@ import {
7
7
  streamResponse,
8
8
  SyncRulesBucketStorage
9
9
  } from '@/index.js';
10
- import { logger, RouterResponse, ServiceError } from '@powersync/lib-services-framework';
10
+ import { ErrorCode, logger, RouterResponse, ServiceError } from '@powersync/lib-services-framework';
11
11
  import {
12
12
  DEFAULT_HYDRATION_STATE,
13
13
  HydrateSyncConfigParams,
14
14
  nodeSqlite,
15
+ parseSyncConfig,
15
16
  SqlSyncRules
16
17
  } from '@powersync/service-sync-rules';
18
+ import { APIMetric } from '@powersync/service-types';
17
19
  import * as sqlite from 'node:sqlite';
18
20
  import { Readable, Writable } from 'stream';
19
21
  import { pipeline } from 'stream/promises';
20
- import { describe, expect, it, vi } from 'vitest';
22
+ import { describe, expect, it, onTestFinished, vi } from 'vitest';
21
23
  import winston from 'winston';
22
24
  import { syncStreamed } from '../../../src/routes/endpoints/sync-stream.js';
23
25
  import { DEFAULT_PARAM_LOGGING_FORMAT_OPTIONS, limitParamsForLogging } from '../../../src/util/param-logging.js';
26
+ import { recordingMetricsEngine } from '../recording-metrics.js';
24
27
  import { mockServiceContext } from './mocks.js';
25
28
 
26
29
  describe('Stream Route', () => {
@@ -189,120 +192,210 @@ describe('Stream Route', () => {
189
192
  DEFAULT_PARAM_LOGGING_FORMAT_OPTIONS.maxStringLength
190
193
  );
191
194
  });
195
+
196
+ // Fail the stream before reporting a client disconnect.
197
+ const runClientDisconnectAfter = async (streamError: Error) => {
198
+ const recorder = recordingMetricsEngine('stream-route-test');
199
+ onTestFinished(() => recorder.shutdown());
200
+ const storage = {
201
+ getParsedSyncRules() {
202
+ return parseSyncConfig('config:\n edition: 3\nstreams: {}', {
203
+ defaultSchema: 'public'
204
+ }).config.hydrate(defaultHydrationOptions);
205
+ },
206
+ watchCheckpointChanges: async function* (options) {
207
+ throw streamError;
208
+ }
209
+ } as Partial<SyncRulesBucketStorage>;
210
+
211
+ const context: Context = {
212
+ logger: logger,
213
+ service_context: mockServiceContext(storage, recorder.engine),
214
+ token_payload: new JwtPayload({
215
+ exp: new Date().getTime() / 1000 + 10000,
216
+ iat: new Date().getTime() / 1000 - 10000,
217
+ sub: 'test-user'
218
+ })
219
+ };
220
+ const request: BasicRouterRequest = { headers: {}, hostname: '', protocol: 'http' };
221
+
222
+ const response = await (syncStreamed.handler({ context, params: {}, request }) as Promise<RouterResponse>);
223
+ await drainWithTimeout(response.data as Readable).catch((error) => error);
224
+ await response.afterSend({ clientClosed: true });
225
+ expect(await recorder.seriesValue(APIMetric.SYNC_CONNECTIONS, {})).toBe(1);
226
+ expect(await recorder.seriesValue(APIMetric.CONCURRENT_CONNECTIONS, {})).toBe(0);
227
+ return recorder;
228
+ };
229
+
230
+ it('keeps a genuine mid-stream failure as an error even when the client then disconnects', async () => {
231
+ const recorder = await runClientDisconnectAfter(new Error('Simulated storage error'));
232
+
233
+ expect(
234
+ await recorder.seriesValue(APIMetric.SYNC_CONNECTIONS, {
235
+ outcome: 'error',
236
+ close_reason: 'stream_error',
237
+ error_code: 'other',
238
+ transport: 'http_stream'
239
+ })
240
+ ).toEqual(1);
241
+ expect(
242
+ await recorder.seriesValue(APIMetric.SYNC_CONNECTIONS, {
243
+ outcome: 'ok',
244
+ close_reason: 'client_closed',
245
+ transport: 'http_stream'
246
+ })
247
+ ).toBeUndefined();
248
+ });
249
+
250
+ it('reports the PowerSync code for a ServiceError mid-stream', async () => {
251
+ const recorder = await runClientDisconnectAfter(
252
+ new ServiceError({ status: 500, code: ErrorCode.PSYNC_S2305, description: 'Too many buckets' })
253
+ );
254
+
255
+ expect(
256
+ await recorder.seriesValue(APIMetric.SYNC_CONNECTIONS, {
257
+ outcome: 'error',
258
+ close_reason: 'stream_error',
259
+ error_code: ErrorCode.PSYNC_S2305,
260
+ transport: 'http_stream'
261
+ })
262
+ ).toEqual(1);
263
+ });
192
264
  });
193
265
 
194
- it('closes after a data fetch error while waiting for a new checkpoint', async () => {
195
- // After 1,000 operations, the sync stream concurrently calls next() on the
196
- // checkpoint watcher. This test then fails the next data fetch without
197
- // producing another checkpoint. Before the fix, awaiting watcher.return()
198
- // during cleanup queued it behind that pending next(), leaving the response
199
- // open until an unrelated checkpoint was written.
200
- const syncRules = SqlSyncRules.fromYaml(
201
- `
266
+ it.each([false, true])(
267
+ 'closes after a data fetch error while waiting for a new checkpoint (cleanupFails=%s)',
268
+ async (cleanupFails) => {
269
+ // After 1,000 operations, the sync stream concurrently calls next() on the
270
+ // checkpoint watcher. This test then fails the next data fetch without
271
+ // producing another checkpoint. Before the fix, awaiting watcher.return()
272
+ // during cleanup queued it behind that pending next(), leaving the response
273
+ // open until an unrelated checkpoint was written.
274
+ const syncRules = parseSyncConfig(
275
+ `
202
276
  bucket_definitions:
203
277
  global:
204
278
  data: []
205
279
  `,
206
- { defaultSchema: 'public' }
207
- ).config.hydrate(defaultHydrationOptions);
208
-
209
- let checkpointWatcherAborted = false;
210
- let dataFetches = 0;
211
- const storage = {
212
- watchCheckpointChanges: async function* ({ signal }) {
213
- yield {
214
- base: {
215
- checkpoint: 1n,
216
- lsn: '1',
217
- getParameterSets: async () => []
218
- },
219
- writeCheckpoint: null,
220
- update: CHECKPOINT_INVALIDATE_ALL
221
- };
222
-
223
- await new Promise<void>((_resolve, reject) => {
224
- // There is intentionally no next checkpoint: cleanup must abort this
225
- // pending wait rather than wait for it to complete naturally.
226
- signal.addEventListener(
227
- 'abort',
228
- () => {
229
- checkpointWatcherAborted = true;
230
- reject(new Error('Checkpoint watcher aborted'));
280
+ { defaultSchema: 'public' }
281
+ ).config.hydrate(defaultHydrationOptions);
282
+
283
+ let checkpointWatcherAborted = false;
284
+ let checkpointWatcherClosed = false;
285
+ const cleanupError = new Error('Checkpoint watcher cleanup failed');
286
+ let dataFetches = 0;
287
+ const storage = {
288
+ watchCheckpointChanges: async function* ({ signal }) {
289
+ try {
290
+ yield {
291
+ base: {
292
+ checkpoint: 1n,
293
+ lsn: '1',
294
+ getParameterSets: async () => []
295
+ },
296
+ writeCheckpoint: null,
297
+ update: CHECKPOINT_INVALIDATE_ALL
298
+ };
299
+
300
+ await new Promise<void>((_resolve, reject) => {
301
+ // There is intentionally no next checkpoint: cleanup must abort this
302
+ // pending wait rather than wait for it to complete naturally.
303
+ signal.addEventListener(
304
+ 'abort',
305
+ () => {
306
+ checkpointWatcherAborted = true;
307
+ reject(signal.reason);
308
+ },
309
+ { once: true }
310
+ );
311
+ });
312
+ } finally {
313
+ await new Promise<void>((resolve) => setImmediate(resolve));
314
+ checkpointWatcherClosed = true;
315
+ if (cleanupFails) throw cleanupError;
316
+ }
317
+ },
318
+ getChecksums: async (_checkpoint, buckets) =>
319
+ new Map(buckets.map(({ bucket }) => [bucket, { bucket, checksum: 1, count: 1000 }])),
320
+ getBucketDataBatch: async function* () {
321
+ if (dataFetches++ > 0) {
322
+ throw new Error('Simulated data fetch failure');
323
+ }
324
+
325
+ yield {
326
+ chunkData: {
327
+ bucket: 'global[]',
328
+ data: Array.from({ length: 1000 }, () => ({ op: 'PUT' })),
329
+ has_more: true,
330
+ after: '0',
331
+ next_after: '1000'
231
332
  },
232
- { once: true }
233
- );
234
- });
235
- },
236
- getChecksums: async (_checkpoint, buckets) =>
237
- new Map(buckets.map(({ bucket }) => [bucket, { bucket, checksum: 1, count: 1000 }])),
238
- getBucketDataBatch: async function* () {
239
- if (dataFetches++ > 0) {
240
- throw new Error('Simulated data fetch failure');
333
+ targetOp: null
334
+ };
335
+ yield { hasMore: true };
241
336
  }
242
-
243
- yield {
244
- chunkData: {
245
- bucket: 'global[]',
246
- data: Array.from({ length: 1000 }, () => ({ op: 'PUT' })),
247
- has_more: true,
248
- after: '0',
249
- next_after: '1000'
250
- },
251
- targetOp: null
252
- };
253
- yield { hasMore: true };
254
- }
255
- } as Partial<SyncRulesBucketStorage>;
256
- const serviceContext = mockServiceContext(storage);
257
- const controller = new AbortController();
258
- const checkpointLogger = logger.child({});
259
- const checkpointLogSpy = vi.spyOn(checkpointLogger, 'info');
260
-
261
- const response = (async () => {
262
- for await (const _line of streamResponse({
263
- syncContext: serviceContext.syncContext,
264
- bucketStorage: storage as SyncRulesBucketStorage,
265
- syncRules,
266
- params: { client_id: 'test-client', raw_data: true },
267
- token: new JwtPayload({ sub: 'test-user', exp: Date.now() / 1000 + 10_000 }),
268
- tracker: new RequestTracker(serviceContext.metricsEngine),
269
- isEncodingAsBson: false,
270
- signal: controller.signal,
271
- logger: checkpointLogger
272
- })) {
273
- // Consume the response until the simulated data fetch error is propagated.
337
+ } as Partial<SyncRulesBucketStorage>;
338
+ const serviceContext = mockServiceContext(storage);
339
+ const controller = new AbortController();
340
+ const checkpointLogger = logger.child({});
341
+ const checkpointLogSpy = vi.spyOn(checkpointLogger, 'info');
342
+
343
+ const response = (async () => {
344
+ for await (const _line of streamResponse({
345
+ syncContext: serviceContext.syncContext,
346
+ bucketStorage: storage as SyncRulesBucketStorage,
347
+ syncRules,
348
+ params: { client_id: 'test-client', raw_data: true },
349
+ token: new JwtPayload({ sub: 'test-user', exp: Date.now() / 1000 + 10_000 }),
350
+ tracker: new RequestTracker(serviceContext.metricsEngine),
351
+ isEncodingAsBson: false,
352
+ signal: controller.signal,
353
+ logger: checkpointLogger
354
+ })) {
355
+ // Consume the response until the simulated data fetch error is propagated.
356
+ }
357
+ })();
358
+
359
+ let timeout: NodeJS.Timeout | undefined;
360
+ const outcome = await Promise.race([
361
+ response.then(
362
+ () => new Error('Sync stream unexpectedly completed'),
363
+ (error) => error
364
+ ),
365
+ new Promise((resolve) => {
366
+ timeout = setTimeout(() => resolve(new Error('Sync stream did not close')), 500);
367
+ })
368
+ ]);
369
+ clearTimeout(timeout);
370
+
371
+ // Capture this before aborting below, which would set it regardless of the stream cleanup.
372
+ const abortedByCleanup = checkpointWatcherAborted;
373
+
374
+ controller.abort();
375
+ await response.catch(() => {});
376
+
377
+ expect(outcome).toBeInstanceOf(Error);
378
+ if (cleanupFails) {
379
+ expect(outcome).toBeInstanceOf(AggregateError);
380
+ expect((outcome as AggregateError).errors).toEqual([
381
+ expect.objectContaining({ message: 'Simulated data fetch failure' }),
382
+ cleanupError
383
+ ]);
384
+ expect((outcome as AggregateError).cause).toMatchObject({ message: 'Simulated data fetch failure' });
385
+ } else {
386
+ expect((outcome as Error).message).toContain('Simulated data fetch failure');
274
387
  }
275
- })();
276
-
277
- let timeout: NodeJS.Timeout | undefined;
278
- const outcome = await Promise.race([
279
- response.then(
280
- () => new Error('Sync stream unexpectedly completed'),
281
- (error) => error
282
- ),
283
- new Promise((resolve) => {
284
- timeout = setTimeout(() => resolve(new Error('Sync stream did not close')), 500);
285
- })
286
- ]);
287
- clearTimeout(timeout);
288
-
289
- // Capture this before aborting below, which would set it regardless of the stream cleanup.
290
- const abortedByCleanup = checkpointWatcherAborted;
291
-
292
- controller.abort();
293
- await response.catch(() => {});
294
-
295
- expect(outcome).toBeInstanceOf(Error);
296
- expect((outcome as Error).message).toContain('Simulated data fetch failure');
297
- expect(abortedByCleanup).toBe(true);
298
- expect(checkpointLogSpy).toHaveBeenCalledWith('checkpoint_interrupted: 1', {
299
- checkpoint: 1n,
300
- user_id: 'test-user',
301
- operations_synced: 1000,
302
- data_synced_bytes: 0,
303
- ms: expect.any(Object)
304
- });
305
- });
388
+ expect(checkpointWatcherClosed).toBe(true);
389
+ expect(abortedByCleanup).toBe(true);
390
+ expect(checkpointLogSpy).toHaveBeenCalledWith('checkpoint_interrupted: 1', {
391
+ checkpoint: 1n,
392
+ user_id: 'test-user',
393
+ operations_synced: 1000,
394
+ data_synced_bytes: 0,
395
+ ms: expect.any(Object)
396
+ });
397
+ }
398
+ );
306
399
  });
307
400
 
308
401
  export async function drainWithTimeout(readable: Readable, ms = 2_000) {
@@ -17,7 +17,6 @@ describe('sync-rules routes', () => {
17
17
  updateSyncRules: vi.fn(async () => ({ replicationStreamName: 'new_slot' }))
18
18
  };
19
19
  // Deploying through the API requires the sync config to not be set in the service config.
20
- (service_context as any).configuration = { sync_rules: { present: false } };
21
20
 
22
21
  return {
23
22
  logger: logger,
@@ -79,6 +78,25 @@ describe('sync-rules routes', () => {
79
78
  });
80
79
 
81
80
  describe('deploy', () => {
81
+ it.each([
82
+ {
83
+ inputState: 'invalid YAML',
84
+ configuration: { present: true, content: 'streams: [', exit_on_error: false }
85
+ },
86
+ { inputState: 'unavailable input', configuration: { present: true, exit_on_error: false } }
87
+ ])(
88
+ 'blocks deployment when a config is supplied regardless of the load result: $inputState',
89
+ async ({ configuration }) => {
90
+ const context = makeContext();
91
+ // Configured mode disables API deployment even if its YAML is rejected or unavailable.
92
+ context.service_context.configuration.sync_rules = configuration;
93
+ await expect(deploySyncRules.handler({ context, params: { content: '' }, request })).rejects.toMatchObject({
94
+ message: expect.stringContaining('Sync config API disabled')
95
+ });
96
+ expect(context.service_context.storageEngine.activeBucketStorage.updateSyncRules).not.toHaveBeenCalled();
97
+ }
98
+ );
99
+
82
100
  it('logs that bucket_definitions are deprecated', async () => {
83
101
  const warnSpy = vi.spyOn(logger, 'warn');
84
102
 
@@ -1,6 +1,6 @@
1
1
  import { isCompatible, parsePersistedSyncConfigContent, updateSyncRulesFromConfig } from '@/storage/storage-index.js';
2
2
  import { logger } from '@powersync/lib-services-framework';
3
- import { PrecompiledSyncConfig, SqlSyncRules } from '@powersync/service-sync-rules';
3
+ import { PrecompiledSyncConfig, parseSyncConfig } from '@powersync/service-sync-rules';
4
4
  import { describe, expect, test } from 'vitest';
5
5
 
6
6
  const CONFIGURED = /* yaml */ `
@@ -17,7 +17,7 @@ const CONFIGURED = /* yaml */ `
17
17
 
18
18
  describe('source-table options', () => {
19
19
  test('preserves filters through compiled storage', () => {
20
- const parsed = SqlSyncRules.fromYaml(CONFIGURED, { defaultSchema: 'app' });
20
+ const parsed = parseSyncConfig(CONFIGURED, { defaultSchema: 'app' });
21
21
  const update = updateSyncRulesFromConfig(parsed);
22
22
  const restored = parsePersistedSyncConfigContent({
23
23
  content: CONFIGURED,
@@ -31,7 +31,7 @@ describe('source-table options', () => {
31
31
  });
32
32
  test('reports invalid expressions before storage', () => {
33
33
  expect(() =>
34
- SqlSyncRules.fromYaml(CONFIGURED.replace('$$doc.active', '$doc.active'), {
34
+ parseSyncConfig(CONFIGURED.replace('$$doc.active', '$doc.active'), {
35
35
  defaultSchema: 'app'
36
36
  })
37
37
  ).toThrow('Expected a source field');
@@ -49,7 +49,7 @@ describe('source-table options', () => {
49
49
  orders:
50
50
  query: SELECT * FROM orders
51
51
  `;
52
- const parsed = SqlSyncRules.fromYaml(yaml, { defaultSchema: 'app', throwOnError: false });
52
+ const parsed = parseSyncConfig(yaml, { defaultSchema: 'app', throwOnError: false });
53
53
  expect(parsed.errors).toContainEqual(
54
54
  expect.objectContaining({ type: 'fatal', message: expect.stringContaining(message) })
55
55
  );
@@ -72,7 +72,7 @@ describe('source-table options', () => {
72
72
 
73
73
  test('keeps valid filters on a partial plan with an unrelated SQL error', () => {
74
74
  const yaml = CONFIGURED.replace('SELECT * FROM orders', 'SELECT FROM');
75
- const parsed = SqlSyncRules.fromYaml(yaml, { defaultSchema: 'app', throwOnError: false });
75
+ const parsed = parseSyncConfig(yaml, { defaultSchema: 'app', throwOnError: false });
76
76
  expect(parsed.errors.some((error) => error.type === 'fatal')).toBe(true);
77
77
  const update = updateSyncRulesFromConfig(parsed);
78
78
  expect(update.config.plan!.plan.sourceTableConfig).toEqual({
@@ -80,7 +80,7 @@ describe('source-table options', () => {
80
80
  });
81
81
  });
82
82
  test('requires replacement processing when filters change or are removed', () => {
83
- const deploy = (yaml: string) => updateSyncRulesFromConfig(SqlSyncRules.fromYaml(yaml, { defaultSchema: 'app' }));
83
+ const deploy = (yaml: string) => updateSyncRulesFromConfig(parseSyncConfig(yaml, { defaultSchema: 'app' }));
84
84
  const first = deploy(CONFIGURED);
85
85
  const same = deploy(CONFIGURED.replace('SELECT *', 'SELECT id'));
86
86
  const changed = deploy(CONFIGURED.replace('true', 'false'));
@@ -95,7 +95,7 @@ describe('source-table options', () => {
95
95
  expect(isCompatible([first.config.plan], removed.config, logger)).toBe(false);
96
96
  });
97
97
  test('requires replacement processing when saved source-table options no longer validate', () => {
98
- const update = updateSyncRulesFromConfig(SqlSyncRules.fromYaml(CONFIGURED, { defaultSchema: 'app' }));
98
+ const update = updateSyncRulesFromConfig(parseSyncConfig(CONFIGURED, { defaultSchema: 'app' }));
99
99
  const existing = structuredClone(update.config.plan!);
100
100
  // A saved expression can become unsupported after a module or version change.
101
101
  Object.assign(existing.plan.sourceTableConfig!.orders!, { mongodb_filter_expression: { $unsupported: [] } });