@powersync/service-core 1.27.2 → 1.28.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (134) hide show
  1. package/CHANGELOG.md +36 -0
  2. package/LICENSE +201 -67
  3. package/dist/api/RouteAPI.d.ts +6 -0
  4. package/dist/api/api-metrics.js +4 -0
  5. package/dist/api/api-metrics.js.map +1 -1
  6. package/dist/api/diagnostics.d.ts +5 -0
  7. package/dist/api/diagnostics.js +17 -2
  8. package/dist/api/diagnostics.js.map +1 -1
  9. package/dist/metrics/connection-metrics.d.ts +28 -0
  10. package/dist/metrics/connection-metrics.js +55 -0
  11. package/dist/metrics/connection-metrics.js.map +1 -0
  12. package/dist/metrics/metrics-index.d.ts +1 -0
  13. package/dist/metrics/metrics-index.js +1 -0
  14. package/dist/metrics/metrics-index.js.map +1 -1
  15. package/dist/metrics/metrics-interfaces.d.ts +7 -1
  16. package/dist/metrics/metrics-interfaces.js.map +1 -1
  17. package/dist/modules/loader.js +8 -2
  18. package/dist/modules/loader.js.map +1 -1
  19. package/dist/replication/AbstractReplicationJob.d.ts +2 -0
  20. package/dist/replication/AbstractReplicationJob.js +17 -1
  21. package/dist/replication/AbstractReplicationJob.js.map +1 -1
  22. package/dist/replication/AbstractReplicator.d.ts +8 -18
  23. package/dist/replication/AbstractReplicator.js +63 -64
  24. package/dist/replication/AbstractReplicator.js.map +1 -1
  25. package/dist/replication/ReplicationModule.js +0 -6
  26. package/dist/replication/ReplicationModule.js.map +1 -1
  27. package/dist/routes/configure-fastify.js +5 -1
  28. package/dist/routes/configure-fastify.js.map +1 -1
  29. package/dist/routes/endpoints/admin.js +24 -22
  30. package/dist/routes/endpoints/admin.js.map +1 -1
  31. package/dist/routes/endpoints/socket-route.js +19 -27
  32. package/dist/routes/endpoints/socket-route.js.map +1 -1
  33. package/dist/routes/endpoints/sync-rules.js +17 -40
  34. package/dist/routes/endpoints/sync-rules.js.map +1 -1
  35. package/dist/routes/endpoints/sync-stream.js +25 -23
  36. package/dist/routes/endpoints/sync-stream.js.map +1 -1
  37. package/dist/routes/hooks.d.ts +1 -1
  38. package/dist/routes/hooks.js +2 -1
  39. package/dist/routes/hooks.js.map +1 -1
  40. package/dist/routes/sync-connection.d.ts +7 -0
  41. package/dist/routes/sync-connection.js +34 -0
  42. package/dist/routes/sync-connection.js.map +1 -0
  43. package/dist/storage/BucketStorageFactory.d.ts +30 -22
  44. package/dist/storage/BucketStorageFactory.js +14 -14
  45. package/dist/storage/BucketStorageFactory.js.map +1 -1
  46. package/dist/storage/PersistedSyncConfigContent.js +2 -2
  47. package/dist/storage/PersistedSyncConfigContent.js.map +1 -1
  48. package/dist/storage/SyncRulesBucketStorage.d.ts +23 -2
  49. package/dist/storage/SyncRulesBucketStorage.js.map +1 -1
  50. package/dist/storage/storage-metrics.js +2 -1
  51. package/dist/storage/storage-metrics.js.map +1 -1
  52. package/dist/streams/BroadcastIterable.js +8 -7
  53. package/dist/streams/BroadcastIterable.js.map +1 -1
  54. package/dist/streams/LastValueSink.d.ts +2 -2
  55. package/dist/streams/LastValueSink.js +8 -13
  56. package/dist/streams/LastValueSink.js.map +1 -1
  57. package/dist/streams/merge.d.ts +15 -31
  58. package/dist/streams/merge.js +136 -152
  59. package/dist/streams/merge.js.map +1 -1
  60. package/dist/sync/BucketChecksumState.d.ts +3 -2
  61. package/dist/sync/BucketChecksumState.js +12 -5
  62. package/dist/sync/BucketChecksumState.js.map +1 -1
  63. package/dist/sync/sync.js +48 -38
  64. package/dist/sync/sync.js.map +1 -1
  65. package/dist/sync/util.d.ts +1 -2
  66. package/dist/sync/util.js +2 -5
  67. package/dist/sync/util.js.map +1 -1
  68. package/dist/system/ServiceContext.d.ts +7 -0
  69. package/dist/system/ServiceContext.js +3 -3
  70. package/dist/system/ServiceContext.js.map +1 -1
  71. package/dist/system/SyncConfigEngine.d.ts +135 -0
  72. package/dist/system/SyncConfigEngine.js +186 -0
  73. package/dist/system/SyncConfigEngine.js.map +1 -0
  74. package/dist/system/system-index.d.ts +1 -0
  75. package/dist/system/system-index.js +1 -0
  76. package/dist/system/system-index.js.map +1 -1
  77. package/dist/util/config/types.d.ts +4 -0
  78. package/package.json +8 -9
  79. package/src/api/RouteAPI.ts +6 -0
  80. package/src/api/api-metrics.ts +5 -0
  81. package/src/api/diagnostics.ts +22 -2
  82. package/src/metrics/connection-metrics.ts +75 -0
  83. package/src/metrics/metrics-index.ts +1 -0
  84. package/src/metrics/metrics-interfaces.ts +8 -1
  85. package/src/modules/loader.ts +16 -2
  86. package/src/replication/AbstractReplicationJob.ts +17 -1
  87. package/src/replication/AbstractReplicator.ts +83 -81
  88. package/src/replication/ReplicationModule.ts +0 -5
  89. package/src/routes/configure-fastify.ts +7 -1
  90. package/src/routes/endpoints/admin.ts +27 -20
  91. package/src/routes/endpoints/socket-route.ts +26 -38
  92. package/src/routes/endpoints/sync-rules.ts +22 -46
  93. package/src/routes/endpoints/sync-stream.ts +31 -27
  94. package/src/routes/hooks.ts +5 -1
  95. package/src/routes/sync-connection.ts +39 -0
  96. package/src/storage/BucketStorageFactory.ts +45 -29
  97. package/src/storage/PersistedSyncConfigContent.ts +2 -2
  98. package/src/storage/SyncRulesBucketStorage.ts +23 -2
  99. package/src/storage/storage-metrics.ts +2 -1
  100. package/src/streams/BroadcastIterable.ts +8 -7
  101. package/src/streams/LastValueSink.ts +10 -16
  102. package/src/streams/merge.ts +149 -169
  103. package/src/sync/BucketChecksumState.ts +17 -5
  104. package/src/sync/sync.ts +71 -57
  105. package/src/sync/util.ts +2 -7
  106. package/src/system/ServiceContext.ts +8 -3
  107. package/src/system/SyncConfigEngine.ts +247 -0
  108. package/src/system/system-index.ts +1 -0
  109. package/src/util/config/types.ts +4 -0
  110. package/test/src/AbstractReplicator.test.ts +375 -45
  111. package/test/src/PersistedSyncConfigContent.test.ts +5 -5
  112. package/test/src/SyncConfigEngine.test.ts +208 -0
  113. package/test/src/broadcast_iterable.test.ts +59 -31
  114. package/test/src/config.test.ts +65 -3
  115. package/test/src/connection-metrics-export.test.ts +76 -0
  116. package/test/src/connection-metrics.test.ts +184 -0
  117. package/test/src/diagnostics.test.ts +3 -3
  118. package/test/src/last_value_sink.test.ts +34 -0
  119. package/test/src/merge_iterable.test.ts +304 -55
  120. package/test/src/module-loader.test.ts +14 -1
  121. package/test/src/recording-metrics.ts +36 -0
  122. package/test/src/routes/admin.test.ts +2 -2
  123. package/test/src/routes/concurrency-limit.test.ts +108 -0
  124. package/test/src/routes/connection-metrics.test.ts +269 -0
  125. package/test/src/routes/invalid-config.test.ts +184 -0
  126. package/test/src/routes/mocks.ts +26 -13
  127. package/test/src/routes/stream-disconnect.integration.test.ts +129 -0
  128. package/test/src/routes/stream.test.ts +200 -107
  129. package/test/src/routes/sync-rules.test.ts +19 -1
  130. package/test/src/source_table_config.test.ts +7 -7
  131. package/test/src/sync/BucketChecksumState.test.ts +10 -10
  132. package/test/src/sync/stream_response.test.ts +389 -0
  133. package/test/src/sync/util.test.ts +14 -3
  134. package/tsconfig.tsbuildinfo +1 -1
@@ -1,200 +1,180 @@
1
- import { throwIfAborted } from 'ix/aborterror.js';
2
- import { AsyncIterableX } from 'ix/asynciterable/index.js';
3
- import { wrapWithAbort } from 'ix/asynciterable/operators/withabort.js';
4
1
  import { safeRace } from '../sync/safeRace.js';
2
+ import type { IterableSource } from './BroadcastIterable.js';
3
+
4
+ type ReadResult<T> = { index: number } & PromiseSettledResult<IteratorResult<T>>;
5
+ type Failure = { error: unknown };
5
6
 
6
7
  /**
7
8
  * Merge multiple source AsyncIterables into one output AsyncIterable.
9
+ * The results from all iterables are interleaved. When any iterable is done, the output stops.
8
10
  *
9
- * The results from all iterables are interleaved.
11
+ * Sources receive a signal owned by this merge. They must use it to unblock pending reads:
12
+ * async generators queue return() behind next(), so closing an unabortable read can block forever.
13
+ * The signal is aborted before closing all source iterators, and completion waits for their cleanup.
14
+ * Cleanup failures propagate to the caller; multiple failures are combined in an AggregateError.
10
15
  *
11
- * When any iterable is done, the output stops.
16
+ * Loosely based on IxJS merge, with these differences:
17
+ * 1. Fix an issue with uncaught errors, partially fixed in https://github.com/ReactiveX/IxJS/pull/354.
18
+ * Only call next() on inner iterators when next() is called on the outer one, and handle every
19
+ * pending read immediately, including reads whose results are not consumed before shutdown.
20
+ * 2. Call return() on all inner iterators when the outer one returns.
21
+ * 3. Return when the first iterator returns.
22
+ * Original implementation: https://github.com/ReactiveX/IxJS/blob/f07b7ef4095120f1ef21a4023030c75b36335cd1/src/asynciterable/merge.ts
12
23
  *
13
- * @param source The source AsyncIterables.
24
+ * @param sources Factories for source iterables that observe the supplied abort signal.
14
25
  * @param signal Optional signal to abort iteration.
15
- * @returns The merged AsyncIterable.
16
26
  */
17
- export function mergeAsyncIterables<T>(source: AsyncIterable<T>[], signal?: AbortSignal): AsyncIterable<T> {
18
- return mergeAsyncIterablesNew(source, signal);
19
- // return mergeAsyncIterablesOld(source, signal);
20
- }
21
-
22
- export function mergeAsyncIterablesNew<T>(source: AsyncIterable<T>[], signal?: AbortSignal): AsyncIterable<T> {
23
- return new MergedAsyncIterable(source, signal);
24
- }
25
-
26
- export function mergeAsyncIterablesOld<T>(source: AsyncIterable<T>[], signal?: AbortSignal): AsyncIterable<T> {
27
- return wrapWithAbort(new FixedMergeAsyncIterable(source, { race: true }), signal);
28
- }
29
-
30
- const NEVER_PROMISE = new Promise<never>(() => {});
31
-
32
- type MergeResult<T> = { value: T; index: number };
33
-
34
- function wrapPromiseWithIndex<T>(promise: Promise<T>, index: number) {
35
- return promise.then((value) => ({ value, index })) as Promise<MergeResult<T>>;
27
+ export function mergeAsyncIterables<T>(sources: IterableSource<T>[], signal?: AbortSignal): AsyncIterable<T> {
28
+ return {
29
+ [Symbol.asyncIterator]() {
30
+ return new MergedAsyncIterator(sources, signal);
31
+ }
32
+ };
36
33
  }
37
34
 
38
- /**
39
- * Re-implementation of FixedMergeAsyncIterable, without using async generators.
40
- *
41
- * The functionality should be the same in most cases.
42
- */
43
- class MergedAsyncIterable<T> implements AsyncIterable<T> {
44
- private _source: AsyncIterable<T>[];
45
- private _signal?: AbortSignal;
35
+ class MergedAsyncIterator<T> implements AsyncIterator<T> {
36
+ private readonly controller = new AbortController();
37
+ private readonly signal: AbortSignal;
38
+ private readonly iterators: AsyncIterator<T>[] = [];
39
+ private readonly pending: (Promise<ReadResult<T>> | null)[] = [];
40
+ private initialized = false;
41
+ private closing: Promise<unknown[]> | undefined;
42
+ private nextInFlight: Promise<void> = Promise.resolve();
43
+ private readonly onAbort = () => {
44
+ // Cleanup can start while the consumer is paused. Its settled errors remain available to next()/return().
45
+ void this.close({ error: this.parentSignal!.reason });
46
+ };
47
+
48
+ constructor(
49
+ private readonly sources: IterableSource<T>[],
50
+ private readonly parentSignal?: AbortSignal
51
+ ) {
52
+ this.signal = parentSignal ? AbortSignal.any([parentSignal, this.controller.signal]) : this.controller.signal;
53
+ if (parentSignal && !parentSignal.aborted) {
54
+ parentSignal.addEventListener('abort', this.onAbort, { once: true });
55
+ }
56
+ }
46
57
 
47
- constructor(source: AsyncIterable<T>[], signal?: AbortSignal) {
48
- this._source = source;
49
- this._signal = signal;
58
+ next(): Promise<IteratorResult<T>> {
59
+ // Serialize next() calls, while allowing return()/throw() to abort an in-flight read immediately.
60
+ const next = this.nextInFlight.then(() => this.read());
61
+ this.nextInFlight = next.then(
62
+ () => {},
63
+ () => {}
64
+ );
65
+ return next;
50
66
  }
51
67
 
52
- [Symbol.asyncIterator](): AsyncIterator<T> {
53
- const signal = this._signal;
54
- throwIfAborted(signal);
68
+ async return(): Promise<IteratorResult<T>> {
69
+ throwFailures(await this.close());
70
+ return { done: true, value: undefined };
71
+ }
55
72
 
56
- const length = this._source.length;
57
- const iterators = new Array<AsyncIterator<T>>(length);
58
- const nexts = new Array<Promise<MergeResult<IteratorResult<T>>> | null>(length);
73
+ async throw(error: unknown): Promise<IteratorResult<T>> {
74
+ throwFailures(await this.close({ error }, true));
75
+ return { done: true, value: undefined };
76
+ }
59
77
 
60
- for (let i = 0; i < length; i++) {
61
- const iterator = wrapWithAbort(this._source[i], signal)[Symbol.asyncIterator]();
62
- iterators[i] = iterator;
63
- nexts[i] = null;
78
+ private async read(): Promise<IteratorResult<T>> {
79
+ if (this.closing) {
80
+ throwFailures(await this.closing);
81
+ return { done: true, value: undefined };
64
82
  }
65
-
66
- const returnIterators = async (_value?: any): Promise<IteratorResult<T>> => {
67
- for (let iter of iterators) {
68
- // Do not wait for return() to complete, since it may block
69
- iter.return?.();
70
- }
71
- return { value: undefined, done: true };
72
- };
73
-
74
- return {
75
- next: async (): Promise<IteratorResult<T>> => {
76
- for (let i = 0; i < length; i++) {
77
- if (nexts[i] == null) {
78
- nexts[i] = wrapPromiseWithIndex(iterators[i].next(), i);
79
- }
80
- }
81
-
82
- try {
83
- // IMPORTANT! safeRace here acts like Promise.race, but avoids major memory leaks.
84
- const {
85
- value: { done, value },
86
- index
87
- } = (await safeRace(nexts as Array<Promise<MergeResult<IteratorResult<T>>>>))!;
88
- if (done) {
89
- // One of the source iterators is done - return them all
90
- await returnIterators();
91
-
92
- return {
93
- value: undefined,
94
- done: true
95
- };
96
- } else {
97
- // Consume the result, which will cause the next one to be requested on the
98
- // next iteration.
99
- nexts[index] = null;
100
-
101
- return {
102
- value: value,
103
- done: false
104
- };
105
- }
106
- } catch (e) {
107
- // One of the source iterators raised an error - return all others
108
- // and propagate the error
109
- await returnIterators();
110
- throw e;
83
+ try {
84
+ this.signal.throwIfAborted();
85
+ if (!this.initialized) {
86
+ this.initialized = true;
87
+ for (const source of this.sources) {
88
+ this.iterators.push(source(this.signal)[Symbol.asyncIterator]());
89
+ this.pending.push(null);
111
90
  }
112
- },
113
- return: () => {
114
- return returnIterators();
115
- },
116
- throw: async (e) => {
117
- for (let iter of iterators) {
118
- // Do not wait for throw() to complete, since it may block
119
- iter.throw?.(e);
91
+ }
92
+ for (const [index, iterator] of this.iterators.entries()) {
93
+ if (this.pending[index] == null) {
94
+ // Settle errors immediately, even if another source wins and the consumer stops reading.
95
+ this.pending[index] = Promise.resolve()
96
+ .then(() => iterator.next())
97
+ .then(
98
+ (value): ReadResult<T> => ({ index, status: 'fulfilled', value }),
99
+ (reason): ReadResult<T> => ({ index, status: 'rejected', reason })
100
+ );
120
101
  }
121
- throw e;
122
102
  }
123
- };
124
- }
125
- }
126
-
127
- /**
128
- * Loosely based on IxJS merge, but with some changes:
129
- * 1. Fix an issue with uncaught errors.
130
- * Partially fixed: https://github.com/ReactiveX/IxJS/pull/354
131
- * Essentially, we only want to call next() on inner iterators when next() is called on the outer one.
132
- * 2. Call return() on all inner iterators when the outer one returns.
133
- * 3. Returning when the first iterator returns.
134
- *
135
- * https://github.com/ReactiveX/IxJS/blob/f07b7ef4095120f1ef21a4023030c75b36335cd1/src/asynciterable/merge.ts
136
- */
137
- export class FixedMergeAsyncIterable<T> extends AsyncIterableX<T> {
138
- private _source: AsyncIterable<T>[];
139
- private _race: boolean;
140
-
141
- /**
142
- *
143
- * @param source
144
- * @param options Specify `race: true` to stop the iterator when any inner one returns, instead of waiting for all.
145
- */
146
- constructor(source: AsyncIterable<T>[], options?: { race?: boolean }) {
147
- super();
148
- this._source = source;
149
- this._race = options?.race ?? false;
103
+ if (this.pending.length === 0) {
104
+ return await this.return();
105
+ }
106
+ // Like Promise.race, but avoids retaining callbacks on long-lived pending reads.
107
+ const result = await safeRace(this.pending as Promise<ReadResult<T>>[]);
108
+ if (this.closing) {
109
+ throwFailures(await this.closing);
110
+ return { done: true, value: undefined };
111
+ }
112
+ if (result.status === 'rejected') {
113
+ throw result.reason;
114
+ }
115
+ if (result.value.done) {
116
+ return await this.return();
117
+ }
118
+ this.pending[result.index] = null;
119
+ return result.value;
120
+ } catch (error) {
121
+ throwFailures(await this.close({ error }));
122
+ throw error;
123
+ }
150
124
  }
151
125
 
152
- async *[Symbol.asyncIterator](signal?: AbortSignal): AsyncIterator<T> {
153
- throwIfAborted(signal);
154
- const length = this._source.length;
155
- const iterators = new Array<AsyncIterator<T>>(length);
156
- const nexts = new Array<Promise<MergeResult<IteratorResult<T>>>>(length);
157
- let active = length;
158
- for (let i = 0; i < length; i++) {
159
- const iterator = wrapWithAbort(this._source[i], signal)[Symbol.asyncIterator]();
160
- iterators[i] = iterator;
161
- nexts[i] = wrapPromiseWithIndex(iterator.next(), i);
126
+ private close(failure?: Failure, injectError = false): Promise<unknown[]> {
127
+ if (this.closing) {
128
+ return this.closing;
162
129
  }
163
-
164
- try {
165
- while (active > 0) {
166
- // IMPORTANT! safeRace here acts like Promise.race, but avoids major memory leaks.
167
- const {
168
- value: { done, value },
169
- index
170
- } = await safeRace(nexts);
171
- if (done) {
172
- nexts[index] = NEVER_PROMISE;
173
- active--;
174
- if (this._race) {
175
- break;
130
+ // Publish the closing state before invoking source code, including synchronous abort listeners.
131
+ this.closing = Promise.resolve().then(async () => {
132
+ this.parentSignal?.removeEventListener('abort', this.onAbort);
133
+ this.controller.abort(failure?.error);
134
+ const errors = failure ? [failure.error] : [];
135
+ const collect = (error: unknown) => {
136
+ // Aborting pending reads is expected. Do not hide the original failure or unrelated cleanup errors.
137
+ if (error !== this.signal.reason && !(error instanceof Error && error.name === 'AbortError')) {
138
+ if (!errors.includes(error)) {
139
+ errors.push(error);
176
140
  }
177
- } else {
178
- const iterator$ = iterators[index];
179
- nexts[index] = NEVER_PROMISE;
180
- try {
181
- yield value;
182
- } catch (e) {
183
- // iter.throw() was called on the merged iterator
184
- // Propagate the error to the source iterators
185
- for (let iter of iterators) {
186
- iter.throw?.(e);
141
+ }
142
+ };
143
+ const results = await Promise.allSettled(
144
+ this.iterators.map(async (iterator) => {
145
+ if (injectError && iterator.throw) {
146
+ try {
147
+ await iterator.throw(failure!.error);
148
+ } catch (error) {
149
+ collect(error);
187
150
  }
188
- throw e;
189
151
  }
190
- nexts[index] = wrapPromiseWithIndex(iterator$.next(), index);
152
+ // Even an iterator that catches throw() and yields again must be closed.
153
+ await iterator.return?.();
154
+ })
155
+ );
156
+ for (const result of results) {
157
+ if (result.status === 'rejected') {
158
+ collect(result.reason);
191
159
  }
192
160
  }
193
- } finally {
194
- for (let iter of iterators) {
195
- // This may be an early return - return all inner iterators
196
- iter.return?.();
161
+ // A source can fail while another source is finishing. Observe those pending reads as well.
162
+ for (const result of await Promise.all(this.pending)) {
163
+ if (result?.status === 'rejected') {
164
+ collect(result.reason);
165
+ }
197
166
  }
198
- }
167
+ return errors;
168
+ });
169
+ return this.closing;
170
+ }
171
+ }
172
+
173
+ function throwFailures(errors: unknown[]): void {
174
+ if (errors.length === 1) {
175
+ throw errors[0];
176
+ }
177
+ if (errors.length > 1) {
178
+ throw new AggregateError(errors, 'Multiple errors while closing merged iterators', { cause: errors[0] });
199
179
  }
200
180
  }
@@ -136,13 +136,15 @@ export class BucketChecksumState {
136
136
  *
137
137
  * When the checkpoint line is sent to the client, call `CheckpointLine.advance()` to update the internal state.
138
138
  * A line may be safely discarded (not sent to the client) if `advance()` is not called.
139
+ * Cancellation is checked between storage reads; in-flight reads are allowed to finish.
139
140
  *
140
141
  * @param next The updated checkpoint
141
142
  * @returns A {@link CheckpointLine} if any of the buckets watched by this connected was updated, or otherwise `null`.
142
143
  */
143
144
  async buildNextCheckpointLine(
144
145
  next: storage.StorageCheckpointUpdate,
145
- tracer?: PerformanceTracer<SyncCheckpointTraceCategory>
146
+ tracer?: PerformanceTracer<SyncCheckpointTraceCategory>,
147
+ signal?: AbortSignal
146
148
  ): Promise<CheckpointLine | null> {
147
149
  tracer ??= new PerformanceTracer<SyncCheckpointTraceCategory>(`sync checkpoint ${next.base.checkpoint}`);
148
150
 
@@ -151,7 +153,8 @@ export class BucketChecksumState {
151
153
 
152
154
  const storage = this.bucketStorage;
153
155
 
154
- const update = await this.parameterState.getCheckpointUpdate(next, tracer);
156
+ const update = await this.parameterState.getCheckpointUpdate(next, tracer, signal);
157
+ signal?.throwIfAborted();
155
158
  const { buckets: allBuckets, updatedBuckets, usedParameterResults } = update;
156
159
  const forceFullChecksum = this.forceFullChecksumForNextCheckpoint;
157
160
 
@@ -241,6 +244,8 @@ export class BucketChecksumState {
241
244
  checksumMap = await storage.getChecksums(base, bucketList, { requestHint });
242
245
  }
243
246
 
247
+ signal?.throwIfAborted();
248
+
244
249
  // Subset of buckets for which there may be new data in this batch.
245
250
  let bucketsToFetch: ResolvedBucket[];
246
251
  let bucketDataRequestHint: storage.BucketRequestHint;
@@ -584,13 +589,15 @@ export class BucketParameterState {
584
589
 
585
590
  async getCheckpointUpdate(
586
591
  checkpoint: storage.StorageCheckpointUpdate,
587
- tracer: PerformanceTracer<SyncCheckpointTraceCategory>
592
+ tracer: PerformanceTracer<SyncCheckpointTraceCategory>,
593
+ signal?: AbortSignal
588
594
  ): Promise<CheckpointUpdate> {
595
+ signal?.throwIfAborted();
589
596
  const querier = this.querier;
590
597
  let update: CheckpointUpdate;
591
598
  if (querier.hasDynamicBuckets) {
592
599
  try {
593
- update = await this.getCheckpointUpdateDynamic(checkpoint, tracer);
600
+ update = await this.getCheckpointUpdateDynamic(checkpoint, tracer, signal);
594
601
  } catch (e: unknown) {
595
602
  if (e instanceof ParameterSetLimitExceededError) {
596
603
  // Too many parameter results, create a breakdown of which streams are responsible for the most queries and
@@ -655,7 +662,8 @@ export class BucketParameterState {
655
662
  */
656
663
  private async getCheckpointUpdateDynamic(
657
664
  checkpoint: storage.StorageCheckpointUpdate,
658
- tracer: PerformanceTracer<SyncCheckpointTraceCategory>
665
+ tracer: PerformanceTracer<SyncCheckpointTraceCategory>,
666
+ signal?: AbortSignal
659
667
  ): Promise<CheckpointUpdate> {
660
668
  const querier = this.querier;
661
669
  const staticBuckets = this.staticBuckets.values();
@@ -695,6 +703,7 @@ export class BucketParameterState {
695
703
 
696
704
  dynamicBuckets = await querier.queryDynamicBucketDescriptions({
697
705
  getParameterSets: async (lookups, definition) => {
706
+ signal?.throwIfAborted();
698
707
  if (lookups.length > parameterLimit) {
699
708
  // Sync Streams can chain parameter lookups, so a large output from an earlier call may become the input
700
709
  // here. We reuse the output limit as a generous upper bound; legitimate queries are much smaller.
@@ -714,6 +723,7 @@ export class BucketParameterState {
714
723
  try {
715
724
  using _ = tracer.span('parameters', definition);
716
725
  const results = await checkpoint.base.getParameterSets(lookups, remainingBudget);
726
+ signal?.throwIfAborted();
717
727
  const numRows = results.reduce((a, b) => a + b.rows.length, 0);
718
728
 
719
729
  lookupLog.push({ definition, resultsOrLimit: numRows, didExceedLimit: false });
@@ -731,6 +741,8 @@ export class BucketParameterState {
731
741
  }
732
742
  });
733
743
 
744
+ signal?.throwIfAborted();
745
+
734
746
  this.cachedDynamicBuckets = dynamicBuckets;
735
747
  this.cachedDynamicBucketSet = new Set<string>(dynamicBuckets.map((b) => b.bucket));
736
748
  this.lookupsFromPreviousCheckpoint = recordedLookups;
package/src/sync/sync.ts CHANGED
@@ -21,6 +21,12 @@ interface ActiveCheckpointTrace {
21
21
  span: Span;
22
22
  }
23
23
 
24
+ interface CheckpointAndLine {
25
+ checkpoint: storage.ReplicationCheckpoint;
26
+ line: CheckpointLine | null;
27
+ trace: ActiveCheckpointTrace | null;
28
+ }
29
+
24
30
  export interface SyncStreamParameters {
25
31
  syncContext: SyncContext;
26
32
  bucketStorage: storage.SyncRulesBucketStorage;
@@ -54,47 +60,36 @@ export async function* streamResponse(
54
60
  } = options;
55
61
  const logger = options.logger ?? defaultLogger;
56
62
 
57
- // We also need to be able to abort, so we create our own controller.
58
- const controller = new AbortController();
59
- if (signal) {
60
- signal.addEventListener(
61
- 'abort',
62
- () => {
63
- controller.abort();
64
- },
65
- { once: true }
66
- );
67
- if (signal.aborted) {
68
- controller.abort();
69
- }
70
- }
71
- const ki = tokenStream(token, controller.signal, tokenStreamOptions);
72
- const stream = streamResponseInner(
73
- syncContext,
74
- bucketStorage,
75
- syncRules,
76
- params,
77
- token,
78
- tracker,
79
- controller.signal,
80
- logger,
81
- isEncodingAsBson
63
+ // Merge the two streams, and abort as soon as one of the streams ends or the consumer stops.
64
+ const merged = mergeAsyncIterables(
65
+ [
66
+ (signal) =>
67
+ streamResponseInner(
68
+ syncContext,
69
+ bucketStorage,
70
+ syncRules,
71
+ params,
72
+ token,
73
+ tracker,
74
+ signal,
75
+ logger,
76
+ isEncodingAsBson
77
+ ),
78
+ (signal) => tokenStream(token, signal, tokenStreamOptions)
79
+ ],
80
+ signal
82
81
  );
83
- // Merge the two streams, and abort as soon as one of the streams end.
84
- const merged = mergeAsyncIterables([stream, ki], controller.signal);
85
82
 
86
83
  try {
87
84
  yield* merged;
88
85
  } catch (e) {
89
- if (isAbortError(e)) {
86
+ // Parent cancellation ends this response normally, even when its reason is not an AbortError.
87
+ // Match the reason itself so unrelated iteration or cleanup failures still propagate.
88
+ if (isAbortError(e) || (signal?.aborted && Object.is(e, signal.reason))) {
90
89
  return;
91
90
  } else {
92
91
  throw e;
93
92
  }
94
- } finally {
95
- // This ensures all the underlying streams are aborted as soon as possible if the
96
- // parent loop stops.
97
- controller.abort();
98
93
  }
99
94
  }
100
95
 
@@ -119,23 +114,18 @@ async function* streamResponseInner(
119
114
  syncRequest: params,
120
115
  logger: logger
121
116
  });
122
- // The checkpoint iterator can have a pending next() while a data batch fails.
123
- // Aborting this on cleanup makes that pending next() settle, so the watcher subscription
124
- // is released instead of staying open until another checkpoint arrives. See the cleanup
125
- // in the finally block below.
117
+ // A checkpoint read or calculation can be pending while a data batch fails.
118
+ // Aborting this on cleanup stops that work once in-flight storage reads finish, so the
119
+ // watcher subscription is released instead of waiting for another checkpoint to arrive.
120
+ // See the cleanup in the finally block below.
126
121
  const checkpointWatchController = new AbortController();
122
+ const checkpointSignal = AbortSignal.any([signal, checkpointWatchController.signal]);
127
123
  const stream = bucketStorage.watchCheckpointChanges({
128
124
  user_id: checkpointUserId,
129
- signal: AbortSignal.any([signal, checkpointWatchController.signal])
125
+ signal: checkpointSignal
130
126
  });
131
127
  const newCheckpoints = stream[Symbol.asyncIterator]();
132
128
 
133
- type CheckpointAndLine = {
134
- checkpoint: storage.ReplicationCheckpoint;
135
- line: CheckpointLine | null;
136
- trace: ActiveCheckpointTrace | null;
137
- };
138
-
139
129
  async function waitForNewCheckpointLine(): Promise<IteratorResult<CheckpointAndLine>> {
140
130
  const next = await newCheckpoints.next();
141
131
  if (next.done) {
@@ -150,7 +140,7 @@ async function* streamResponseInner(
150
140
  };
151
141
 
152
142
  try {
153
- const line = await checksumState.buildNextCheckpointLine(next.value, trace.tracer);
143
+ const line = await checksumState.buildNextCheckpointLine(next.value, trace.tracer, checkpointSignal);
154
144
  return { done: false, value: { checkpoint: cp, line, trace: line == null ? null : trace } };
155
145
  } catch (e) {
156
146
  if (e instanceof storage.CheckpointChecksumInvalidatedError) {
@@ -184,10 +174,10 @@ async function* streamResponseInner(
184
174
  }
185
175
 
186
176
  let activeCheckpoint: { checkpoint: storage.ReplicationCheckpoint; trace: ActiveCheckpointTrace } | null = null;
177
+ let iterationFailure: { error: unknown } | undefined;
178
+ let nextCheckpointPromise: Promise<PromiseSettledResult<IteratorResult<CheckpointAndLine>>> | undefined;
187
179
 
188
180
  try {
189
- let nextCheckpointPromise: Promise<PromiseSettledResult<IteratorResult<CheckpointAndLine>>> | undefined;
190
-
191
181
  do {
192
182
  if (!nextCheckpointPromise) {
193
183
  // Wrap in a settledPromise, so that abort errors after the parent stopped iterating
@@ -344,6 +334,9 @@ async function* streamResponseInner(
344
334
  await new Promise((resolve) => setTimeout(resolve, 10));
345
335
  }
346
336
  } while (!signal.aborted);
337
+ } catch (error) {
338
+ iterationFailure = { error };
339
+ throw error;
347
340
  } finally {
348
341
  if (activeCheckpoint != null) {
349
342
  const { checkpoint, trace } = activeCheckpoint;
@@ -353,17 +346,38 @@ async function* streamResponseInner(
353
346
  });
354
347
  }
355
348
 
356
- // The abort makes a pending next() settle as soon as the storage watcher observes it,
357
- // releasing the subscription. return() is still needed for the case where there is no
358
- // pending next(): the watcher is then suspended at a yield and never observes the abort.
359
- // Neither is awaited: async generators queue return() behind a pending next(), so
360
- // awaiting it here would keep the connection open until the next checkpoint arrives.
361
- checkpointWatchController.abort();
362
- newCheckpoints.return?.().catch((e) => {
363
- if (!isAbortError(e)) {
364
- logger.warn('Error while closing the checkpoint watcher', e);
365
- }
366
- });
349
+ await closeCheckpointWatcher(checkpointWatchController, newCheckpoints, nextCheckpointPromise, iterationFailure);
350
+ }
351
+ }
352
+
353
+ async function closeCheckpointWatcher(
354
+ controller: AbortController,
355
+ watcher: AsyncIterator<unknown>,
356
+ pending: Promise<PromiseSettledResult<IteratorResult<CheckpointAndLine>>> | undefined,
357
+ iterationFailure?: { error: unknown }
358
+ ): Promise<void> {
359
+ // The abort makes a pending next() settle as soon as the storage watcher observes it,
360
+ // releasing the subscription. return() is still needed for the case where there is no
361
+ // pending next(): the watcher is then suspended at a yield and never observes the abort.
362
+ // Abort before awaiting return(): async generators queue it behind a pending next().
363
+ // This ordering prevents the connection stall fixed in #773, without detaching cleanup.
364
+ controller.abort(iterationFailure?.error);
365
+ // A pending read can fail in the watcher's finally block while its queued return() succeeds.
366
+ // Observe both results so that cleanup failures are not silently lost.
367
+ const [closed, read] = await Promise.all([settledPromise(Promise.resolve().then(() => watcher.return?.())), pending]);
368
+ const errors = iterationFailure ? [iterationFailure.error] : [];
369
+ for (const result of [closed, read]) {
370
+ if (result?.status === 'rejected' && !isAbortError(result.reason) && !errors.includes(result.reason)) {
371
+ errors.push(result.reason);
372
+ }
373
+ }
374
+ if (read?.status === 'fulfilled' && !read.value.done) {
375
+ read.value.value.trace?.span.end();
376
+ }
377
+ if (errors.length > 1) {
378
+ throw new AggregateError(errors, 'Checkpoint iteration and cleanup both failed', { cause: errors[0] });
379
+ } else if (errors.length === 1) {
380
+ throw errors[0];
367
381
  }
368
382
  }
369
383
 
package/src/sync/util.ts CHANGED
@@ -2,7 +2,6 @@ import * as timers from 'timers/promises';
2
2
 
3
3
  import { SemaphoreInterface } from 'async-mutex';
4
4
  import { serialize } from 'bson';
5
- import { AbortError } from 'ix/aborterror.js';
6
5
  import * as util from '../util/util-index.js';
7
6
  import { RequestTracker } from './RequestTracker.js';
8
7
 
@@ -25,14 +24,10 @@ const DEFAULT_TOKEN_STREAM_OPTIONS: TokenStreamOptions = {
25
24
  };
26
25
 
27
26
  /**
28
- * Recognize both ix's AbortError and native abort errors, which Node represents
29
- * as DOMExceptions with the name "AbortError".
27
+ * Recognize DOM AbortErrors and Node's timer/stream abort errors by name.
30
28
  */
31
29
  export function isAbortError(error: unknown): boolean {
32
- return (
33
- error instanceof AbortError ||
34
- (typeof error == 'object' && error != null && 'name' in error && error.name === 'AbortError')
35
- );
30
+ return typeof error == 'object' && error != null && 'name' in error && error.name === 'AbortError';
36
31
  }
37
32
 
38
33
  /**