@powersync/service-core 1.27.2 → 1.28.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +30 -0
- package/LICENSE +201 -67
- package/dist/api/RouteAPI.d.ts +6 -0
- package/dist/api/api-metrics.js +4 -0
- package/dist/api/api-metrics.js.map +1 -1
- package/dist/api/diagnostics.d.ts +5 -0
- package/dist/api/diagnostics.js +17 -2
- package/dist/api/diagnostics.js.map +1 -1
- package/dist/metrics/connection-metrics.d.ts +28 -0
- package/dist/metrics/connection-metrics.js +55 -0
- package/dist/metrics/connection-metrics.js.map +1 -0
- package/dist/metrics/metrics-index.d.ts +1 -0
- package/dist/metrics/metrics-index.js +1 -0
- package/dist/metrics/metrics-index.js.map +1 -1
- package/dist/metrics/metrics-interfaces.d.ts +7 -1
- package/dist/metrics/metrics-interfaces.js.map +1 -1
- package/dist/modules/loader.js +8 -2
- package/dist/modules/loader.js.map +1 -1
- package/dist/replication/AbstractReplicationJob.d.ts +2 -0
- package/dist/replication/AbstractReplicationJob.js +17 -1
- package/dist/replication/AbstractReplicationJob.js.map +1 -1
- package/dist/replication/AbstractReplicator.d.ts +8 -18
- package/dist/replication/AbstractReplicator.js +63 -64
- package/dist/replication/AbstractReplicator.js.map +1 -1
- package/dist/replication/ReplicationModule.js +0 -6
- package/dist/replication/ReplicationModule.js.map +1 -1
- package/dist/routes/configure-fastify.js +5 -1
- package/dist/routes/configure-fastify.js.map +1 -1
- package/dist/routes/endpoints/admin.js +24 -22
- package/dist/routes/endpoints/admin.js.map +1 -1
- package/dist/routes/endpoints/socket-route.js +19 -27
- package/dist/routes/endpoints/socket-route.js.map +1 -1
- package/dist/routes/endpoints/sync-rules.js +17 -40
- package/dist/routes/endpoints/sync-rules.js.map +1 -1
- package/dist/routes/endpoints/sync-stream.js +25 -23
- package/dist/routes/endpoints/sync-stream.js.map +1 -1
- package/dist/routes/hooks.d.ts +1 -1
- package/dist/routes/hooks.js +2 -1
- package/dist/routes/hooks.js.map +1 -1
- package/dist/routes/sync-connection.d.ts +7 -0
- package/dist/routes/sync-connection.js +34 -0
- package/dist/routes/sync-connection.js.map +1 -0
- package/dist/storage/BucketStorageFactory.d.ts +30 -22
- package/dist/storage/BucketStorageFactory.js +14 -14
- package/dist/storage/BucketStorageFactory.js.map +1 -1
- package/dist/storage/PersistedSyncConfigContent.js +2 -2
- package/dist/storage/PersistedSyncConfigContent.js.map +1 -1
- package/dist/storage/SyncRulesBucketStorage.d.ts +23 -2
- package/dist/storage/SyncRulesBucketStorage.js.map +1 -1
- package/dist/streams/BroadcastIterable.js +8 -7
- package/dist/streams/BroadcastIterable.js.map +1 -1
- package/dist/streams/LastValueSink.d.ts +2 -2
- package/dist/streams/LastValueSink.js +8 -13
- package/dist/streams/LastValueSink.js.map +1 -1
- package/dist/streams/merge.d.ts +15 -31
- package/dist/streams/merge.js +136 -152
- package/dist/streams/merge.js.map +1 -1
- package/dist/sync/BucketChecksumState.d.ts +3 -2
- package/dist/sync/BucketChecksumState.js +12 -5
- package/dist/sync/BucketChecksumState.js.map +1 -1
- package/dist/sync/sync.js +48 -38
- package/dist/sync/sync.js.map +1 -1
- package/dist/sync/util.d.ts +1 -2
- package/dist/sync/util.js +2 -5
- package/dist/sync/util.js.map +1 -1
- package/dist/system/ServiceContext.d.ts +7 -0
- package/dist/system/ServiceContext.js +3 -3
- package/dist/system/ServiceContext.js.map +1 -1
- package/dist/system/SyncConfigEngine.d.ts +135 -0
- package/dist/system/SyncConfigEngine.js +186 -0
- package/dist/system/SyncConfigEngine.js.map +1 -0
- package/dist/system/system-index.d.ts +1 -0
- package/dist/system/system-index.js +1 -0
- package/dist/system/system-index.js.map +1 -1
- package/dist/util/config/types.d.ts +4 -0
- package/package.json +8 -9
- package/src/api/RouteAPI.ts +6 -0
- package/src/api/api-metrics.ts +5 -0
- package/src/api/diagnostics.ts +22 -2
- package/src/metrics/connection-metrics.ts +75 -0
- package/src/metrics/metrics-index.ts +1 -0
- package/src/metrics/metrics-interfaces.ts +8 -1
- package/src/modules/loader.ts +16 -2
- package/src/replication/AbstractReplicationJob.ts +17 -1
- package/src/replication/AbstractReplicator.ts +83 -81
- package/src/replication/ReplicationModule.ts +0 -5
- package/src/routes/configure-fastify.ts +7 -1
- package/src/routes/endpoints/admin.ts +27 -20
- package/src/routes/endpoints/socket-route.ts +26 -38
- package/src/routes/endpoints/sync-rules.ts +22 -46
- package/src/routes/endpoints/sync-stream.ts +31 -27
- package/src/routes/hooks.ts +5 -1
- package/src/routes/sync-connection.ts +39 -0
- package/src/storage/BucketStorageFactory.ts +45 -29
- package/src/storage/PersistedSyncConfigContent.ts +2 -2
- package/src/storage/SyncRulesBucketStorage.ts +23 -2
- package/src/streams/BroadcastIterable.ts +8 -7
- package/src/streams/LastValueSink.ts +10 -16
- package/src/streams/merge.ts +149 -169
- package/src/sync/BucketChecksumState.ts +17 -5
- package/src/sync/sync.ts +71 -57
- package/src/sync/util.ts +2 -7
- package/src/system/ServiceContext.ts +8 -3
- package/src/system/SyncConfigEngine.ts +247 -0
- package/src/system/system-index.ts +1 -0
- package/src/util/config/types.ts +4 -0
- package/test/src/AbstractReplicator.test.ts +375 -45
- package/test/src/PersistedSyncConfigContent.test.ts +5 -5
- package/test/src/SyncConfigEngine.test.ts +208 -0
- package/test/src/broadcast_iterable.test.ts +59 -31
- package/test/src/config.test.ts +65 -3
- package/test/src/connection-metrics-export.test.ts +76 -0
- package/test/src/connection-metrics.test.ts +184 -0
- package/test/src/diagnostics.test.ts +3 -3
- package/test/src/last_value_sink.test.ts +34 -0
- package/test/src/merge_iterable.test.ts +304 -55
- package/test/src/module-loader.test.ts +14 -1
- package/test/src/recording-metrics.ts +36 -0
- package/test/src/routes/admin.test.ts +2 -2
- package/test/src/routes/concurrency-limit.test.ts +108 -0
- package/test/src/routes/connection-metrics.test.ts +269 -0
- package/test/src/routes/invalid-config.test.ts +184 -0
- package/test/src/routes/mocks.ts +26 -13
- package/test/src/routes/stream-disconnect.integration.test.ts +129 -0
- package/test/src/routes/stream.test.ts +200 -107
- package/test/src/routes/sync-rules.test.ts +19 -1
- package/test/src/source_table_config.test.ts +7 -7
- package/test/src/sync/BucketChecksumState.test.ts +10 -10
- package/test/src/sync/stream_response.test.ts +389 -0
- package/test/src/sync/util.test.ts +14 -3
- package/tsconfig.tsbuildinfo +1 -1
package/src/streams/merge.ts
CHANGED
|
@@ -1,200 +1,180 @@
|
|
|
1
|
-
import { throwIfAborted } from 'ix/aborterror.js';
|
|
2
|
-
import { AsyncIterableX } from 'ix/asynciterable/index.js';
|
|
3
|
-
import { wrapWithAbort } from 'ix/asynciterable/operators/withabort.js';
|
|
4
1
|
import { safeRace } from '../sync/safeRace.js';
|
|
2
|
+
import type { IterableSource } from './BroadcastIterable.js';
|
|
3
|
+
|
|
4
|
+
type ReadResult<T> = { index: number } & PromiseSettledResult<IteratorResult<T>>;
|
|
5
|
+
type Failure = { error: unknown };
|
|
5
6
|
|
|
6
7
|
/**
|
|
7
8
|
* Merge multiple source AsyncIterables into one output AsyncIterable.
|
|
9
|
+
* The results from all iterables are interleaved. When any iterable is done, the output stops.
|
|
8
10
|
*
|
|
9
|
-
*
|
|
11
|
+
* Sources receive a signal owned by this merge. They must use it to unblock pending reads:
|
|
12
|
+
* async generators queue return() behind next(), so closing an unabortable read can block forever.
|
|
13
|
+
* The signal is aborted before closing all source iterators, and completion waits for their cleanup.
|
|
14
|
+
* Cleanup failures propagate to the caller; multiple failures are combined in an AggregateError.
|
|
10
15
|
*
|
|
11
|
-
*
|
|
16
|
+
* Loosely based on IxJS merge, with these differences:
|
|
17
|
+
* 1. Fix an issue with uncaught errors, partially fixed in https://github.com/ReactiveX/IxJS/pull/354.
|
|
18
|
+
* Only call next() on inner iterators when next() is called on the outer one, and handle every
|
|
19
|
+
* pending read immediately, including reads whose results are not consumed before shutdown.
|
|
20
|
+
* 2. Call return() on all inner iterators when the outer one returns.
|
|
21
|
+
* 3. Return when the first iterator returns.
|
|
22
|
+
* Original implementation: https://github.com/ReactiveX/IxJS/blob/f07b7ef4095120f1ef21a4023030c75b36335cd1/src/asynciterable/merge.ts
|
|
12
23
|
*
|
|
13
|
-
* @param
|
|
24
|
+
* @param sources Factories for source iterables that observe the supplied abort signal.
|
|
14
25
|
* @param signal Optional signal to abort iteration.
|
|
15
|
-
* @returns The merged AsyncIterable.
|
|
16
26
|
*/
|
|
17
|
-
export function mergeAsyncIterables<T>(
|
|
18
|
-
return
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
return new MergedAsyncIterable(source, signal);
|
|
24
|
-
}
|
|
25
|
-
|
|
26
|
-
export function mergeAsyncIterablesOld<T>(source: AsyncIterable<T>[], signal?: AbortSignal): AsyncIterable<T> {
|
|
27
|
-
return wrapWithAbort(new FixedMergeAsyncIterable(source, { race: true }), signal);
|
|
28
|
-
}
|
|
29
|
-
|
|
30
|
-
const NEVER_PROMISE = new Promise<never>(() => {});
|
|
31
|
-
|
|
32
|
-
type MergeResult<T> = { value: T; index: number };
|
|
33
|
-
|
|
34
|
-
function wrapPromiseWithIndex<T>(promise: Promise<T>, index: number) {
|
|
35
|
-
return promise.then((value) => ({ value, index })) as Promise<MergeResult<T>>;
|
|
27
|
+
export function mergeAsyncIterables<T>(sources: IterableSource<T>[], signal?: AbortSignal): AsyncIterable<T> {
|
|
28
|
+
return {
|
|
29
|
+
[Symbol.asyncIterator]() {
|
|
30
|
+
return new MergedAsyncIterator(sources, signal);
|
|
31
|
+
}
|
|
32
|
+
};
|
|
36
33
|
}
|
|
37
34
|
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
private
|
|
45
|
-
private
|
|
35
|
+
class MergedAsyncIterator<T> implements AsyncIterator<T> {
|
|
36
|
+
private readonly controller = new AbortController();
|
|
37
|
+
private readonly signal: AbortSignal;
|
|
38
|
+
private readonly iterators: AsyncIterator<T>[] = [];
|
|
39
|
+
private readonly pending: (Promise<ReadResult<T>> | null)[] = [];
|
|
40
|
+
private initialized = false;
|
|
41
|
+
private closing: Promise<unknown[]> | undefined;
|
|
42
|
+
private nextInFlight: Promise<void> = Promise.resolve();
|
|
43
|
+
private readonly onAbort = () => {
|
|
44
|
+
// Cleanup can start while the consumer is paused. Its settled errors remain available to next()/return().
|
|
45
|
+
void this.close({ error: this.parentSignal!.reason });
|
|
46
|
+
};
|
|
47
|
+
|
|
48
|
+
constructor(
|
|
49
|
+
private readonly sources: IterableSource<T>[],
|
|
50
|
+
private readonly parentSignal?: AbortSignal
|
|
51
|
+
) {
|
|
52
|
+
this.signal = parentSignal ? AbortSignal.any([parentSignal, this.controller.signal]) : this.controller.signal;
|
|
53
|
+
if (parentSignal && !parentSignal.aborted) {
|
|
54
|
+
parentSignal.addEventListener('abort', this.onAbort, { once: true });
|
|
55
|
+
}
|
|
56
|
+
}
|
|
46
57
|
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
this.
|
|
58
|
+
next(): Promise<IteratorResult<T>> {
|
|
59
|
+
// Serialize next() calls, while allowing return()/throw() to abort an in-flight read immediately.
|
|
60
|
+
const next = this.nextInFlight.then(() => this.read());
|
|
61
|
+
this.nextInFlight = next.then(
|
|
62
|
+
() => {},
|
|
63
|
+
() => {}
|
|
64
|
+
);
|
|
65
|
+
return next;
|
|
50
66
|
}
|
|
51
67
|
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
68
|
+
async return(): Promise<IteratorResult<T>> {
|
|
69
|
+
throwFailures(await this.close());
|
|
70
|
+
return { done: true, value: undefined };
|
|
71
|
+
}
|
|
55
72
|
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
73
|
+
async throw(error: unknown): Promise<IteratorResult<T>> {
|
|
74
|
+
throwFailures(await this.close({ error }, true));
|
|
75
|
+
return { done: true, value: undefined };
|
|
76
|
+
}
|
|
59
77
|
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
78
|
+
private async read(): Promise<IteratorResult<T>> {
|
|
79
|
+
if (this.closing) {
|
|
80
|
+
throwFailures(await this.closing);
|
|
81
|
+
return { done: true, value: undefined };
|
|
64
82
|
}
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
};
|
|
73
|
-
|
|
74
|
-
return {
|
|
75
|
-
next: async (): Promise<IteratorResult<T>> => {
|
|
76
|
-
for (let i = 0; i < length; i++) {
|
|
77
|
-
if (nexts[i] == null) {
|
|
78
|
-
nexts[i] = wrapPromiseWithIndex(iterators[i].next(), i);
|
|
79
|
-
}
|
|
80
|
-
}
|
|
81
|
-
|
|
82
|
-
try {
|
|
83
|
-
// IMPORTANT! safeRace here acts like Promise.race, but avoids major memory leaks.
|
|
84
|
-
const {
|
|
85
|
-
value: { done, value },
|
|
86
|
-
index
|
|
87
|
-
} = (await safeRace(nexts as Array<Promise<MergeResult<IteratorResult<T>>>>))!;
|
|
88
|
-
if (done) {
|
|
89
|
-
// One of the source iterators is done - return them all
|
|
90
|
-
await returnIterators();
|
|
91
|
-
|
|
92
|
-
return {
|
|
93
|
-
value: undefined,
|
|
94
|
-
done: true
|
|
95
|
-
};
|
|
96
|
-
} else {
|
|
97
|
-
// Consume the result, which will cause the next one to be requested on the
|
|
98
|
-
// next iteration.
|
|
99
|
-
nexts[index] = null;
|
|
100
|
-
|
|
101
|
-
return {
|
|
102
|
-
value: value,
|
|
103
|
-
done: false
|
|
104
|
-
};
|
|
105
|
-
}
|
|
106
|
-
} catch (e) {
|
|
107
|
-
// One of the source iterators raised an error - return all others
|
|
108
|
-
// and propagate the error
|
|
109
|
-
await returnIterators();
|
|
110
|
-
throw e;
|
|
83
|
+
try {
|
|
84
|
+
this.signal.throwIfAborted();
|
|
85
|
+
if (!this.initialized) {
|
|
86
|
+
this.initialized = true;
|
|
87
|
+
for (const source of this.sources) {
|
|
88
|
+
this.iterators.push(source(this.signal)[Symbol.asyncIterator]());
|
|
89
|
+
this.pending.push(null);
|
|
111
90
|
}
|
|
112
|
-
}
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
91
|
+
}
|
|
92
|
+
for (const [index, iterator] of this.iterators.entries()) {
|
|
93
|
+
if (this.pending[index] == null) {
|
|
94
|
+
// Settle errors immediately, even if another source wins and the consumer stops reading.
|
|
95
|
+
this.pending[index] = Promise.resolve()
|
|
96
|
+
.then(() => iterator.next())
|
|
97
|
+
.then(
|
|
98
|
+
(value): ReadResult<T> => ({ index, status: 'fulfilled', value }),
|
|
99
|
+
(reason): ReadResult<T> => ({ index, status: 'rejected', reason })
|
|
100
|
+
);
|
|
120
101
|
}
|
|
121
|
-
throw e;
|
|
122
102
|
}
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
}
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
* @param options Specify `race: true` to stop the iterator when any inner one returns, instead of waiting for all.
|
|
145
|
-
*/
|
|
146
|
-
constructor(source: AsyncIterable<T>[], options?: { race?: boolean }) {
|
|
147
|
-
super();
|
|
148
|
-
this._source = source;
|
|
149
|
-
this._race = options?.race ?? false;
|
|
103
|
+
if (this.pending.length === 0) {
|
|
104
|
+
return await this.return();
|
|
105
|
+
}
|
|
106
|
+
// Like Promise.race, but avoids retaining callbacks on long-lived pending reads.
|
|
107
|
+
const result = await safeRace(this.pending as Promise<ReadResult<T>>[]);
|
|
108
|
+
if (this.closing) {
|
|
109
|
+
throwFailures(await this.closing);
|
|
110
|
+
return { done: true, value: undefined };
|
|
111
|
+
}
|
|
112
|
+
if (result.status === 'rejected') {
|
|
113
|
+
throw result.reason;
|
|
114
|
+
}
|
|
115
|
+
if (result.value.done) {
|
|
116
|
+
return await this.return();
|
|
117
|
+
}
|
|
118
|
+
this.pending[result.index] = null;
|
|
119
|
+
return result.value;
|
|
120
|
+
} catch (error) {
|
|
121
|
+
throwFailures(await this.close({ error }));
|
|
122
|
+
throw error;
|
|
123
|
+
}
|
|
150
124
|
}
|
|
151
125
|
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
const iterators = new Array<AsyncIterator<T>>(length);
|
|
156
|
-
const nexts = new Array<Promise<MergeResult<IteratorResult<T>>>>(length);
|
|
157
|
-
let active = length;
|
|
158
|
-
for (let i = 0; i < length; i++) {
|
|
159
|
-
const iterator = wrapWithAbort(this._source[i], signal)[Symbol.asyncIterator]();
|
|
160
|
-
iterators[i] = iterator;
|
|
161
|
-
nexts[i] = wrapPromiseWithIndex(iterator.next(), i);
|
|
126
|
+
private close(failure?: Failure, injectError = false): Promise<unknown[]> {
|
|
127
|
+
if (this.closing) {
|
|
128
|
+
return this.closing;
|
|
162
129
|
}
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
active--;
|
|
174
|
-
if (this._race) {
|
|
175
|
-
break;
|
|
130
|
+
// Publish the closing state before invoking source code, including synchronous abort listeners.
|
|
131
|
+
this.closing = Promise.resolve().then(async () => {
|
|
132
|
+
this.parentSignal?.removeEventListener('abort', this.onAbort);
|
|
133
|
+
this.controller.abort(failure?.error);
|
|
134
|
+
const errors = failure ? [failure.error] : [];
|
|
135
|
+
const collect = (error: unknown) => {
|
|
136
|
+
// Aborting pending reads is expected. Do not hide the original failure or unrelated cleanup errors.
|
|
137
|
+
if (error !== this.signal.reason && !(error instanceof Error && error.name === 'AbortError')) {
|
|
138
|
+
if (!errors.includes(error)) {
|
|
139
|
+
errors.push(error);
|
|
176
140
|
}
|
|
177
|
-
}
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
iter.throw?.(e);
|
|
141
|
+
}
|
|
142
|
+
};
|
|
143
|
+
const results = await Promise.allSettled(
|
|
144
|
+
this.iterators.map(async (iterator) => {
|
|
145
|
+
if (injectError && iterator.throw) {
|
|
146
|
+
try {
|
|
147
|
+
await iterator.throw(failure!.error);
|
|
148
|
+
} catch (error) {
|
|
149
|
+
collect(error);
|
|
187
150
|
}
|
|
188
|
-
throw e;
|
|
189
151
|
}
|
|
190
|
-
|
|
152
|
+
// Even an iterator that catches throw() and yields again must be closed.
|
|
153
|
+
await iterator.return?.();
|
|
154
|
+
})
|
|
155
|
+
);
|
|
156
|
+
for (const result of results) {
|
|
157
|
+
if (result.status === 'rejected') {
|
|
158
|
+
collect(result.reason);
|
|
191
159
|
}
|
|
192
160
|
}
|
|
193
|
-
|
|
194
|
-
for (
|
|
195
|
-
|
|
196
|
-
|
|
161
|
+
// A source can fail while another source is finishing. Observe those pending reads as well.
|
|
162
|
+
for (const result of await Promise.all(this.pending)) {
|
|
163
|
+
if (result?.status === 'rejected') {
|
|
164
|
+
collect(result.reason);
|
|
165
|
+
}
|
|
197
166
|
}
|
|
198
|
-
|
|
167
|
+
return errors;
|
|
168
|
+
});
|
|
169
|
+
return this.closing;
|
|
170
|
+
}
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
function throwFailures(errors: unknown[]): void {
|
|
174
|
+
if (errors.length === 1) {
|
|
175
|
+
throw errors[0];
|
|
176
|
+
}
|
|
177
|
+
if (errors.length > 1) {
|
|
178
|
+
throw new AggregateError(errors, 'Multiple errors while closing merged iterators', { cause: errors[0] });
|
|
199
179
|
}
|
|
200
180
|
}
|
|
@@ -136,13 +136,15 @@ export class BucketChecksumState {
|
|
|
136
136
|
*
|
|
137
137
|
* When the checkpoint line is sent to the client, call `CheckpointLine.advance()` to update the internal state.
|
|
138
138
|
* A line may be safely discarded (not sent to the client) if `advance()` is not called.
|
|
139
|
+
* Cancellation is checked between storage reads; in-flight reads are allowed to finish.
|
|
139
140
|
*
|
|
140
141
|
* @param next The updated checkpoint
|
|
141
142
|
* @returns A {@link CheckpointLine} if any of the buckets watched by this connected was updated, or otherwise `null`.
|
|
142
143
|
*/
|
|
143
144
|
async buildNextCheckpointLine(
|
|
144
145
|
next: storage.StorageCheckpointUpdate,
|
|
145
|
-
tracer?: PerformanceTracer<SyncCheckpointTraceCategory
|
|
146
|
+
tracer?: PerformanceTracer<SyncCheckpointTraceCategory>,
|
|
147
|
+
signal?: AbortSignal
|
|
146
148
|
): Promise<CheckpointLine | null> {
|
|
147
149
|
tracer ??= new PerformanceTracer<SyncCheckpointTraceCategory>(`sync checkpoint ${next.base.checkpoint}`);
|
|
148
150
|
|
|
@@ -151,7 +153,8 @@ export class BucketChecksumState {
|
|
|
151
153
|
|
|
152
154
|
const storage = this.bucketStorage;
|
|
153
155
|
|
|
154
|
-
const update = await this.parameterState.getCheckpointUpdate(next, tracer);
|
|
156
|
+
const update = await this.parameterState.getCheckpointUpdate(next, tracer, signal);
|
|
157
|
+
signal?.throwIfAborted();
|
|
155
158
|
const { buckets: allBuckets, updatedBuckets, usedParameterResults } = update;
|
|
156
159
|
const forceFullChecksum = this.forceFullChecksumForNextCheckpoint;
|
|
157
160
|
|
|
@@ -241,6 +244,8 @@ export class BucketChecksumState {
|
|
|
241
244
|
checksumMap = await storage.getChecksums(base, bucketList, { requestHint });
|
|
242
245
|
}
|
|
243
246
|
|
|
247
|
+
signal?.throwIfAborted();
|
|
248
|
+
|
|
244
249
|
// Subset of buckets for which there may be new data in this batch.
|
|
245
250
|
let bucketsToFetch: ResolvedBucket[];
|
|
246
251
|
let bucketDataRequestHint: storage.BucketRequestHint;
|
|
@@ -584,13 +589,15 @@ export class BucketParameterState {
|
|
|
584
589
|
|
|
585
590
|
async getCheckpointUpdate(
|
|
586
591
|
checkpoint: storage.StorageCheckpointUpdate,
|
|
587
|
-
tracer: PerformanceTracer<SyncCheckpointTraceCategory
|
|
592
|
+
tracer: PerformanceTracer<SyncCheckpointTraceCategory>,
|
|
593
|
+
signal?: AbortSignal
|
|
588
594
|
): Promise<CheckpointUpdate> {
|
|
595
|
+
signal?.throwIfAborted();
|
|
589
596
|
const querier = this.querier;
|
|
590
597
|
let update: CheckpointUpdate;
|
|
591
598
|
if (querier.hasDynamicBuckets) {
|
|
592
599
|
try {
|
|
593
|
-
update = await this.getCheckpointUpdateDynamic(checkpoint, tracer);
|
|
600
|
+
update = await this.getCheckpointUpdateDynamic(checkpoint, tracer, signal);
|
|
594
601
|
} catch (e: unknown) {
|
|
595
602
|
if (e instanceof ParameterSetLimitExceededError) {
|
|
596
603
|
// Too many parameter results, create a breakdown of which streams are responsible for the most queries and
|
|
@@ -655,7 +662,8 @@ export class BucketParameterState {
|
|
|
655
662
|
*/
|
|
656
663
|
private async getCheckpointUpdateDynamic(
|
|
657
664
|
checkpoint: storage.StorageCheckpointUpdate,
|
|
658
|
-
tracer: PerformanceTracer<SyncCheckpointTraceCategory
|
|
665
|
+
tracer: PerformanceTracer<SyncCheckpointTraceCategory>,
|
|
666
|
+
signal?: AbortSignal
|
|
659
667
|
): Promise<CheckpointUpdate> {
|
|
660
668
|
const querier = this.querier;
|
|
661
669
|
const staticBuckets = this.staticBuckets.values();
|
|
@@ -695,6 +703,7 @@ export class BucketParameterState {
|
|
|
695
703
|
|
|
696
704
|
dynamicBuckets = await querier.queryDynamicBucketDescriptions({
|
|
697
705
|
getParameterSets: async (lookups, definition) => {
|
|
706
|
+
signal?.throwIfAborted();
|
|
698
707
|
if (lookups.length > parameterLimit) {
|
|
699
708
|
// Sync Streams can chain parameter lookups, so a large output from an earlier call may become the input
|
|
700
709
|
// here. We reuse the output limit as a generous upper bound; legitimate queries are much smaller.
|
|
@@ -714,6 +723,7 @@ export class BucketParameterState {
|
|
|
714
723
|
try {
|
|
715
724
|
using _ = tracer.span('parameters', definition);
|
|
716
725
|
const results = await checkpoint.base.getParameterSets(lookups, remainingBudget);
|
|
726
|
+
signal?.throwIfAborted();
|
|
717
727
|
const numRows = results.reduce((a, b) => a + b.rows.length, 0);
|
|
718
728
|
|
|
719
729
|
lookupLog.push({ definition, resultsOrLimit: numRows, didExceedLimit: false });
|
|
@@ -731,6 +741,8 @@ export class BucketParameterState {
|
|
|
731
741
|
}
|
|
732
742
|
});
|
|
733
743
|
|
|
744
|
+
signal?.throwIfAborted();
|
|
745
|
+
|
|
734
746
|
this.cachedDynamicBuckets = dynamicBuckets;
|
|
735
747
|
this.cachedDynamicBucketSet = new Set<string>(dynamicBuckets.map((b) => b.bucket));
|
|
736
748
|
this.lookupsFromPreviousCheckpoint = recordedLookups;
|
package/src/sync/sync.ts
CHANGED
|
@@ -21,6 +21,12 @@ interface ActiveCheckpointTrace {
|
|
|
21
21
|
span: Span;
|
|
22
22
|
}
|
|
23
23
|
|
|
24
|
+
interface CheckpointAndLine {
|
|
25
|
+
checkpoint: storage.ReplicationCheckpoint;
|
|
26
|
+
line: CheckpointLine | null;
|
|
27
|
+
trace: ActiveCheckpointTrace | null;
|
|
28
|
+
}
|
|
29
|
+
|
|
24
30
|
export interface SyncStreamParameters {
|
|
25
31
|
syncContext: SyncContext;
|
|
26
32
|
bucketStorage: storage.SyncRulesBucketStorage;
|
|
@@ -54,47 +60,36 @@ export async function* streamResponse(
|
|
|
54
60
|
} = options;
|
|
55
61
|
const logger = options.logger ?? defaultLogger;
|
|
56
62
|
|
|
57
|
-
//
|
|
58
|
-
const
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
syncRules,
|
|
76
|
-
params,
|
|
77
|
-
token,
|
|
78
|
-
tracker,
|
|
79
|
-
controller.signal,
|
|
80
|
-
logger,
|
|
81
|
-
isEncodingAsBson
|
|
63
|
+
// Merge the two streams, and abort as soon as one of the streams ends or the consumer stops.
|
|
64
|
+
const merged = mergeAsyncIterables(
|
|
65
|
+
[
|
|
66
|
+
(signal) =>
|
|
67
|
+
streamResponseInner(
|
|
68
|
+
syncContext,
|
|
69
|
+
bucketStorage,
|
|
70
|
+
syncRules,
|
|
71
|
+
params,
|
|
72
|
+
token,
|
|
73
|
+
tracker,
|
|
74
|
+
signal,
|
|
75
|
+
logger,
|
|
76
|
+
isEncodingAsBson
|
|
77
|
+
),
|
|
78
|
+
(signal) => tokenStream(token, signal, tokenStreamOptions)
|
|
79
|
+
],
|
|
80
|
+
signal
|
|
82
81
|
);
|
|
83
|
-
// Merge the two streams, and abort as soon as one of the streams end.
|
|
84
|
-
const merged = mergeAsyncIterables([stream, ki], controller.signal);
|
|
85
82
|
|
|
86
83
|
try {
|
|
87
84
|
yield* merged;
|
|
88
85
|
} catch (e) {
|
|
89
|
-
|
|
86
|
+
// Parent cancellation ends this response normally, even when its reason is not an AbortError.
|
|
87
|
+
// Match the reason itself so unrelated iteration or cleanup failures still propagate.
|
|
88
|
+
if (isAbortError(e) || (signal?.aborted && Object.is(e, signal.reason))) {
|
|
90
89
|
return;
|
|
91
90
|
} else {
|
|
92
91
|
throw e;
|
|
93
92
|
}
|
|
94
|
-
} finally {
|
|
95
|
-
// This ensures all the underlying streams are aborted as soon as possible if the
|
|
96
|
-
// parent loop stops.
|
|
97
|
-
controller.abort();
|
|
98
93
|
}
|
|
99
94
|
}
|
|
100
95
|
|
|
@@ -119,23 +114,18 @@ async function* streamResponseInner(
|
|
|
119
114
|
syncRequest: params,
|
|
120
115
|
logger: logger
|
|
121
116
|
});
|
|
122
|
-
//
|
|
123
|
-
// Aborting this on cleanup
|
|
124
|
-
// is released instead of
|
|
125
|
-
// in the finally block below.
|
|
117
|
+
// A checkpoint read or calculation can be pending while a data batch fails.
|
|
118
|
+
// Aborting this on cleanup stops that work once in-flight storage reads finish, so the
|
|
119
|
+
// watcher subscription is released instead of waiting for another checkpoint to arrive.
|
|
120
|
+
// See the cleanup in the finally block below.
|
|
126
121
|
const checkpointWatchController = new AbortController();
|
|
122
|
+
const checkpointSignal = AbortSignal.any([signal, checkpointWatchController.signal]);
|
|
127
123
|
const stream = bucketStorage.watchCheckpointChanges({
|
|
128
124
|
user_id: checkpointUserId,
|
|
129
|
-
signal:
|
|
125
|
+
signal: checkpointSignal
|
|
130
126
|
});
|
|
131
127
|
const newCheckpoints = stream[Symbol.asyncIterator]();
|
|
132
128
|
|
|
133
|
-
type CheckpointAndLine = {
|
|
134
|
-
checkpoint: storage.ReplicationCheckpoint;
|
|
135
|
-
line: CheckpointLine | null;
|
|
136
|
-
trace: ActiveCheckpointTrace | null;
|
|
137
|
-
};
|
|
138
|
-
|
|
139
129
|
async function waitForNewCheckpointLine(): Promise<IteratorResult<CheckpointAndLine>> {
|
|
140
130
|
const next = await newCheckpoints.next();
|
|
141
131
|
if (next.done) {
|
|
@@ -150,7 +140,7 @@ async function* streamResponseInner(
|
|
|
150
140
|
};
|
|
151
141
|
|
|
152
142
|
try {
|
|
153
|
-
const line = await checksumState.buildNextCheckpointLine(next.value, trace.tracer);
|
|
143
|
+
const line = await checksumState.buildNextCheckpointLine(next.value, trace.tracer, checkpointSignal);
|
|
154
144
|
return { done: false, value: { checkpoint: cp, line, trace: line == null ? null : trace } };
|
|
155
145
|
} catch (e) {
|
|
156
146
|
if (e instanceof storage.CheckpointChecksumInvalidatedError) {
|
|
@@ -184,10 +174,10 @@ async function* streamResponseInner(
|
|
|
184
174
|
}
|
|
185
175
|
|
|
186
176
|
let activeCheckpoint: { checkpoint: storage.ReplicationCheckpoint; trace: ActiveCheckpointTrace } | null = null;
|
|
177
|
+
let iterationFailure: { error: unknown } | undefined;
|
|
178
|
+
let nextCheckpointPromise: Promise<PromiseSettledResult<IteratorResult<CheckpointAndLine>>> | undefined;
|
|
187
179
|
|
|
188
180
|
try {
|
|
189
|
-
let nextCheckpointPromise: Promise<PromiseSettledResult<IteratorResult<CheckpointAndLine>>> | undefined;
|
|
190
|
-
|
|
191
181
|
do {
|
|
192
182
|
if (!nextCheckpointPromise) {
|
|
193
183
|
// Wrap in a settledPromise, so that abort errors after the parent stopped iterating
|
|
@@ -344,6 +334,9 @@ async function* streamResponseInner(
|
|
|
344
334
|
await new Promise((resolve) => setTimeout(resolve, 10));
|
|
345
335
|
}
|
|
346
336
|
} while (!signal.aborted);
|
|
337
|
+
} catch (error) {
|
|
338
|
+
iterationFailure = { error };
|
|
339
|
+
throw error;
|
|
347
340
|
} finally {
|
|
348
341
|
if (activeCheckpoint != null) {
|
|
349
342
|
const { checkpoint, trace } = activeCheckpoint;
|
|
@@ -353,17 +346,38 @@ async function* streamResponseInner(
|
|
|
353
346
|
});
|
|
354
347
|
}
|
|
355
348
|
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
349
|
+
await closeCheckpointWatcher(checkpointWatchController, newCheckpoints, nextCheckpointPromise, iterationFailure);
|
|
350
|
+
}
|
|
351
|
+
}
|
|
352
|
+
|
|
353
|
+
async function closeCheckpointWatcher(
|
|
354
|
+
controller: AbortController,
|
|
355
|
+
watcher: AsyncIterator<unknown>,
|
|
356
|
+
pending: Promise<PromiseSettledResult<IteratorResult<CheckpointAndLine>>> | undefined,
|
|
357
|
+
iterationFailure?: { error: unknown }
|
|
358
|
+
): Promise<void> {
|
|
359
|
+
// The abort makes a pending next() settle as soon as the storage watcher observes it,
|
|
360
|
+
// releasing the subscription. return() is still needed for the case where there is no
|
|
361
|
+
// pending next(): the watcher is then suspended at a yield and never observes the abort.
|
|
362
|
+
// Abort before awaiting return(): async generators queue it behind a pending next().
|
|
363
|
+
// This ordering prevents the connection stall fixed in #773, without detaching cleanup.
|
|
364
|
+
controller.abort(iterationFailure?.error);
|
|
365
|
+
// A pending read can fail in the watcher's finally block while its queued return() succeeds.
|
|
366
|
+
// Observe both results so that cleanup failures are not silently lost.
|
|
367
|
+
const [closed, read] = await Promise.all([settledPromise(Promise.resolve().then(() => watcher.return?.())), pending]);
|
|
368
|
+
const errors = iterationFailure ? [iterationFailure.error] : [];
|
|
369
|
+
for (const result of [closed, read]) {
|
|
370
|
+
if (result?.status === 'rejected' && !isAbortError(result.reason) && !errors.includes(result.reason)) {
|
|
371
|
+
errors.push(result.reason);
|
|
372
|
+
}
|
|
373
|
+
}
|
|
374
|
+
if (read?.status === 'fulfilled' && !read.value.done) {
|
|
375
|
+
read.value.value.trace?.span.end();
|
|
376
|
+
}
|
|
377
|
+
if (errors.length > 1) {
|
|
378
|
+
throw new AggregateError(errors, 'Checkpoint iteration and cleanup both failed', { cause: errors[0] });
|
|
379
|
+
} else if (errors.length === 1) {
|
|
380
|
+
throw errors[0];
|
|
367
381
|
}
|
|
368
382
|
}
|
|
369
383
|
|
package/src/sync/util.ts
CHANGED
|
@@ -2,7 +2,6 @@ import * as timers from 'timers/promises';
|
|
|
2
2
|
|
|
3
3
|
import { SemaphoreInterface } from 'async-mutex';
|
|
4
4
|
import { serialize } from 'bson';
|
|
5
|
-
import { AbortError } from 'ix/aborterror.js';
|
|
6
5
|
import * as util from '../util/util-index.js';
|
|
7
6
|
import { RequestTracker } from './RequestTracker.js';
|
|
8
7
|
|
|
@@ -25,14 +24,10 @@ const DEFAULT_TOKEN_STREAM_OPTIONS: TokenStreamOptions = {
|
|
|
25
24
|
};
|
|
26
25
|
|
|
27
26
|
/**
|
|
28
|
-
* Recognize
|
|
29
|
-
* as DOMExceptions with the name "AbortError".
|
|
27
|
+
* Recognize DOM AbortErrors and Node's timer/stream abort errors by name.
|
|
30
28
|
*/
|
|
31
29
|
export function isAbortError(error: unknown): boolean {
|
|
32
|
-
return
|
|
33
|
-
error instanceof AbortError ||
|
|
34
|
-
(typeof error == 'object' && error != null && 'name' in error && error.name === 'AbortError')
|
|
35
|
-
);
|
|
30
|
+
return typeof error == 'object' && error != null && 'name' in error && error.name === 'AbortError';
|
|
36
31
|
}
|
|
37
32
|
|
|
38
33
|
/**
|