@powersync/common 1.50.0 → 1.52.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bundle.cjs +558 -481
- package/dist/bundle.cjs.map +1 -1
- package/dist/bundle.mjs +558 -480
- package/dist/bundle.mjs.map +1 -1
- package/dist/bundle.node.cjs +556 -481
- package/dist/bundle.node.cjs.map +1 -1
- package/dist/bundle.node.mjs +556 -480
- package/dist/bundle.node.mjs.map +1 -1
- package/dist/index.d.cts +73 -73
- package/lib/client/AbstractPowerSyncDatabase.js +3 -3
- package/lib/client/AbstractPowerSyncDatabase.js.map +1 -1
- package/lib/client/sync/stream/AbstractRemote.d.ts +29 -8
- package/lib/client/sync/stream/AbstractRemote.js +154 -177
- package/lib/client/sync/stream/AbstractRemote.js.map +1 -1
- package/lib/client/sync/stream/AbstractStreamingSyncImplementation.d.ts +1 -0
- package/lib/client/sync/stream/AbstractStreamingSyncImplementation.js +69 -88
- package/lib/client/sync/stream/AbstractStreamingSyncImplementation.js.map +1 -1
- package/lib/index.d.ts +1 -1
- package/lib/index.js +0 -1
- package/lib/index.js.map +1 -1
- package/lib/utils/async.d.ts +0 -9
- package/lib/utils/async.js +0 -9
- package/lib/utils/async.js.map +1 -1
- package/lib/utils/mutex.d.ts +32 -3
- package/lib/utils/mutex.js +85 -36
- package/lib/utils/mutex.js.map +1 -1
- package/lib/utils/queue.d.ts +16 -0
- package/lib/utils/queue.js +42 -0
- package/lib/utils/queue.js.map +1 -0
- package/lib/utils/stream_transform.d.ts +39 -0
- package/lib/utils/stream_transform.js +206 -0
- package/lib/utils/stream_transform.js.map +1 -0
- package/package.json +9 -7
- package/src/client/AbstractPowerSyncDatabase.ts +3 -3
- package/src/client/sync/stream/AbstractRemote.ts +182 -206
- package/src/client/sync/stream/AbstractStreamingSyncImplementation.ts +82 -83
- package/src/index.ts +1 -1
- package/src/utils/async.ts +0 -11
- package/src/utils/mutex.ts +111 -48
- package/src/utils/queue.ts +48 -0
- package/src/utils/stream_transform.ts +252 -0
- package/lib/utils/DataStream.d.ts +0 -62
- package/lib/utils/DataStream.js +0 -169
- package/lib/utils/DataStream.js.map +0 -1
- package/src/utils/DataStream.ts +0 -222
|
@@ -0,0 +1,252 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* An async iterator that can't be cancelled.
|
|
3
|
+
*
|
|
4
|
+
* To keep data flow simple, we always pass an explicit cancellation token when subscribing to async streams. Once the
|
|
5
|
+
* {@link AbortSignal} aborts, iterators are supposed to clean up and then emit a final `{done: true}` event. This means
|
|
6
|
+
* that there's no way to distinguish between streams that have completed normally and streams that have been cancelled,
|
|
7
|
+
* but that is acceptable for our uses of this.
|
|
8
|
+
*/
|
|
9
|
+
export type SimpleAsyncIterator<T> = Pick<AsyncIterator<T>, 'next'>;
|
|
10
|
+
|
|
11
|
+
export const doneResult: IteratorReturnResult<any> = { done: true, value: undefined };
|
|
12
|
+
|
|
13
|
+
export function valueResult<T>(value: T) {
|
|
14
|
+
return { done: false, value };
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
/**
|
|
18
|
+
* A variant of {@link Array.map} for async iterators.
|
|
19
|
+
*/
|
|
20
|
+
export function map<T1, T2>(source: SimpleAsyncIterator<T1>, map: (source: T1) => T2): SimpleAsyncIterator<T2> {
|
|
21
|
+
return {
|
|
22
|
+
next: async () => {
|
|
23
|
+
const value = await source.next();
|
|
24
|
+
if (value.done) {
|
|
25
|
+
return value;
|
|
26
|
+
} else {
|
|
27
|
+
return { value: map(value.value) };
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
};
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
export interface InjectableIterator<T> extends SimpleAsyncIterator<T> {
|
|
34
|
+
inject(event: T): void;
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
/**
|
|
38
|
+
* Expands a source async iterator by allowing to inject events asynchronously.
|
|
39
|
+
*
|
|
40
|
+
* The resulting iterator will emit all events from its source. Additionally though, events can be injected. These
|
|
41
|
+
* events are dropped once the main iterator completes, but are otherwise forwarded.
|
|
42
|
+
*
|
|
43
|
+
* The iterator completes when its source completes, and it supports backpressure by only calling `next()` on the source
|
|
44
|
+
* in response to a `next()` call from downstream if no pending injected events can be dispatched.
|
|
45
|
+
*/
|
|
46
|
+
export function injectable<T>(source: SimpleAsyncIterator<T>): InjectableIterator<T> {
|
|
47
|
+
type Waiter = { resolve: (t: IteratorResult<T>) => void; reject: (e: unknown) => void };
|
|
48
|
+
|
|
49
|
+
let sourceIsDone = false;
|
|
50
|
+
let waiter: Waiter | undefined = undefined; // An active, waiting next() call.
|
|
51
|
+
// A pending upstream event that couldn't be dispatched because inject() has been called before it was resolved.
|
|
52
|
+
let pendingSourceEvent: ((w: Waiter) => void) | null = null;
|
|
53
|
+
|
|
54
|
+
let pendingInjectedEvents: T[] = [];
|
|
55
|
+
|
|
56
|
+
const consumeWaiter = () => {
|
|
57
|
+
const pending = waiter;
|
|
58
|
+
waiter = undefined;
|
|
59
|
+
return pending;
|
|
60
|
+
};
|
|
61
|
+
|
|
62
|
+
const fetchFromSource = () => {
|
|
63
|
+
const resolveWaiter = (propagate: (w: Waiter) => void) => {
|
|
64
|
+
const active = consumeWaiter();
|
|
65
|
+
if (active) {
|
|
66
|
+
propagate(active);
|
|
67
|
+
} else {
|
|
68
|
+
pendingSourceEvent = propagate;
|
|
69
|
+
}
|
|
70
|
+
};
|
|
71
|
+
|
|
72
|
+
const nextFromSource = source.next();
|
|
73
|
+
nextFromSource.then(
|
|
74
|
+
(value) => {
|
|
75
|
+
sourceIsDone = value.done == true;
|
|
76
|
+
resolveWaiter((w) => w.resolve(value));
|
|
77
|
+
},
|
|
78
|
+
(error) => {
|
|
79
|
+
resolveWaiter((w) => w.reject(error));
|
|
80
|
+
}
|
|
81
|
+
);
|
|
82
|
+
};
|
|
83
|
+
|
|
84
|
+
return {
|
|
85
|
+
next: () => {
|
|
86
|
+
return new Promise((resolve, reject) => {
|
|
87
|
+
// First priority: Dispatch ready upstream events.
|
|
88
|
+
if (sourceIsDone) {
|
|
89
|
+
return resolve(doneResult);
|
|
90
|
+
}
|
|
91
|
+
if (pendingSourceEvent) {
|
|
92
|
+
pendingSourceEvent({ resolve, reject });
|
|
93
|
+
pendingSourceEvent = null;
|
|
94
|
+
return;
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
// Second priority: Dispatch injected events
|
|
98
|
+
if (pendingInjectedEvents.length) {
|
|
99
|
+
return resolve(valueResult(pendingInjectedEvents.shift()!));
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
// Nothing pending? Fetch from source
|
|
103
|
+
waiter = { resolve, reject };
|
|
104
|
+
return fetchFromSource();
|
|
105
|
+
});
|
|
106
|
+
},
|
|
107
|
+
inject: (event) => {
|
|
108
|
+
const pending = consumeWaiter();
|
|
109
|
+
if (pending != null) {
|
|
110
|
+
pending.resolve(valueResult(event));
|
|
111
|
+
} else {
|
|
112
|
+
pendingInjectedEvents.push(event);
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
};
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
/**
|
|
119
|
+
* Splits a byte stream at line endings, emitting each line as a string.
|
|
120
|
+
*/
|
|
121
|
+
export function extractJsonLines(
|
|
122
|
+
source: SimpleAsyncIterator<Uint8Array>,
|
|
123
|
+
decoder: TextDecoder
|
|
124
|
+
): SimpleAsyncIterator<string> {
|
|
125
|
+
let buffer = '';
|
|
126
|
+
const pendingLines: string[] = [];
|
|
127
|
+
let isFinalEvent = false;
|
|
128
|
+
|
|
129
|
+
return {
|
|
130
|
+
next: async () => {
|
|
131
|
+
while (true) {
|
|
132
|
+
if (isFinalEvent) {
|
|
133
|
+
return doneResult;
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
{
|
|
137
|
+
const first = pendingLines.shift();
|
|
138
|
+
if (first) {
|
|
139
|
+
return { done: false, value: first };
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
const { done, value } = await source.next();
|
|
144
|
+
if (done) {
|
|
145
|
+
const remaining = buffer.trim();
|
|
146
|
+
if (remaining.length != 0) {
|
|
147
|
+
isFinalEvent = true;
|
|
148
|
+
return { done: false, value: remaining };
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
return doneResult;
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
const data = decoder.decode(value, { stream: true });
|
|
155
|
+
buffer += data;
|
|
156
|
+
|
|
157
|
+
const lines = buffer.split('\n');
|
|
158
|
+
for (let i = 0; i < lines.length - 1; i++) {
|
|
159
|
+
const l = lines[i].trim();
|
|
160
|
+
if (l.length > 0) {
|
|
161
|
+
pendingLines.push(l);
|
|
162
|
+
}
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
buffer = lines[lines.length - 1];
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
};
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
/**
|
|
172
|
+
* Splits a concatenated stream of BSON objects by emitting individual objects.
|
|
173
|
+
*/
|
|
174
|
+
export function extractBsonObjects(source: SimpleAsyncIterator<Uint8Array>): SimpleAsyncIterator<Uint8Array> {
|
|
175
|
+
// Fully read but not emitted yet.
|
|
176
|
+
const completedObjects: Uint8Array[] = [];
|
|
177
|
+
|
|
178
|
+
// Whether source has returned { done: true }. We do the same once completed objects have been emitted.
|
|
179
|
+
let isDone = false;
|
|
180
|
+
|
|
181
|
+
const lengthBuffer = new DataView(new ArrayBuffer(4));
|
|
182
|
+
let objectBody: Uint8Array | null = null;
|
|
183
|
+
// If we're parsing the length field, a number between 1 and 4 (inclusive) describing remaining bytes in the header.
|
|
184
|
+
// If we're consuming a document, the bytes remaining.
|
|
185
|
+
let remainingLength = 4;
|
|
186
|
+
|
|
187
|
+
return {
|
|
188
|
+
async next(): Promise<IteratorResult<Uint8Array>> {
|
|
189
|
+
while (true) {
|
|
190
|
+
// Before fetching new data from upstream, return completed objects.
|
|
191
|
+
if (completedObjects.length) {
|
|
192
|
+
return valueResult(completedObjects.shift()!);
|
|
193
|
+
}
|
|
194
|
+
if (isDone) {
|
|
195
|
+
return doneResult;
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
const upstreamEvent = await source.next();
|
|
199
|
+
if (upstreamEvent.done) {
|
|
200
|
+
isDone = true;
|
|
201
|
+
if (objectBody || remainingLength != 4) {
|
|
202
|
+
throw new Error('illegal end of stream in BSON object');
|
|
203
|
+
}
|
|
204
|
+
return doneResult;
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
const chunk = upstreamEvent.value;
|
|
208
|
+
for (let i = 0; i < chunk.length; ) {
|
|
209
|
+
const availableInData = chunk.length - i;
|
|
210
|
+
|
|
211
|
+
if (objectBody) {
|
|
212
|
+
// We're in the middle of reading a BSON document.
|
|
213
|
+
const bytesToRead = Math.min(availableInData, remainingLength);
|
|
214
|
+
const copySource = new Uint8Array(chunk.buffer, chunk.byteOffset + i, bytesToRead);
|
|
215
|
+
objectBody.set(copySource, objectBody.length - remainingLength);
|
|
216
|
+
i += bytesToRead;
|
|
217
|
+
remainingLength -= bytesToRead;
|
|
218
|
+
|
|
219
|
+
if (remainingLength == 0) {
|
|
220
|
+
completedObjects.push(objectBody);
|
|
221
|
+
|
|
222
|
+
// Prepare to read another document, starting with its length
|
|
223
|
+
objectBody = null;
|
|
224
|
+
remainingLength = 4;
|
|
225
|
+
}
|
|
226
|
+
} else {
|
|
227
|
+
// Copy up to 4 bytes into lengthBuffer, depending on how many we still need.
|
|
228
|
+
const bytesToRead = Math.min(availableInData, remainingLength);
|
|
229
|
+
for (let j = 0; j < bytesToRead; j++) {
|
|
230
|
+
lengthBuffer.setUint8(4 - remainingLength + j, chunk[i + j]);
|
|
231
|
+
}
|
|
232
|
+
i += bytesToRead;
|
|
233
|
+
remainingLength -= bytesToRead;
|
|
234
|
+
|
|
235
|
+
if (remainingLength == 0) {
|
|
236
|
+
// Transition from reading length header to reading document. Subtracting 4 because the length of the
|
|
237
|
+
// header is included in length.
|
|
238
|
+
const length = lengthBuffer.getInt32(0, true /* little endian */);
|
|
239
|
+
remainingLength = length - 4;
|
|
240
|
+
if (remainingLength < 1) {
|
|
241
|
+
throw new Error(`invalid length for bson: ${length}`);
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
objectBody = new Uint8Array(length);
|
|
245
|
+
new DataView(objectBody.buffer).setInt32(0, length, true);
|
|
246
|
+
}
|
|
247
|
+
}
|
|
248
|
+
}
|
|
249
|
+
}
|
|
250
|
+
}
|
|
251
|
+
};
|
|
252
|
+
}
|
|
@@ -1,62 +0,0 @@
|
|
|
1
|
-
import { ILogger } from 'js-logger';
|
|
2
|
-
import { BaseListener, BaseObserver } from './BaseObserver.js';
|
|
3
|
-
export type DataStreamOptions<ParsedData, SourceData> = {
|
|
4
|
-
mapLine?: (line: SourceData) => ParsedData;
|
|
5
|
-
/**
|
|
6
|
-
* Close the stream if any consumer throws an error
|
|
7
|
-
*/
|
|
8
|
-
closeOnError?: boolean;
|
|
9
|
-
pressure?: {
|
|
10
|
-
highWaterMark?: number;
|
|
11
|
-
lowWaterMark?: number;
|
|
12
|
-
};
|
|
13
|
-
logger?: ILogger;
|
|
14
|
-
};
|
|
15
|
-
export type DataStreamCallback<Data extends any = any> = (data: Data) => Promise<void>;
|
|
16
|
-
export interface DataStreamListener<Data extends any = any> extends BaseListener {
|
|
17
|
-
data: (data: Data) => Promise<void>;
|
|
18
|
-
closed: () => void;
|
|
19
|
-
error: (error: Error) => void;
|
|
20
|
-
highWater: () => Promise<void>;
|
|
21
|
-
lowWater: () => Promise<void>;
|
|
22
|
-
}
|
|
23
|
-
export declare const DEFAULT_PRESSURE_LIMITS: {
|
|
24
|
-
highWater: number;
|
|
25
|
-
lowWater: number;
|
|
26
|
-
};
|
|
27
|
-
/**
|
|
28
|
-
* A very basic implementation of a data stream with backpressure support which does not use
|
|
29
|
-
* native JS streams or async iterators.
|
|
30
|
-
* This is handy for environments such as React Native which need polyfills for the above.
|
|
31
|
-
*/
|
|
32
|
-
export declare class DataStream<ParsedData, SourceData = any> extends BaseObserver<DataStreamListener<ParsedData>> {
|
|
33
|
-
protected options?: DataStreamOptions<ParsedData, SourceData> | undefined;
|
|
34
|
-
dataQueue: SourceData[];
|
|
35
|
-
protected isClosed: boolean;
|
|
36
|
-
protected processingPromise: Promise<void> | null;
|
|
37
|
-
protected notifyDataAdded: (() => void) | null;
|
|
38
|
-
protected logger: ILogger;
|
|
39
|
-
protected mapLine: (line: SourceData) => ParsedData;
|
|
40
|
-
constructor(options?: DataStreamOptions<ParsedData, SourceData> | undefined);
|
|
41
|
-
get highWatermark(): number;
|
|
42
|
-
get lowWatermark(): number;
|
|
43
|
-
get closed(): boolean;
|
|
44
|
-
close(): Promise<void>;
|
|
45
|
-
/**
|
|
46
|
-
* Enqueues data for the consumers to read
|
|
47
|
-
*/
|
|
48
|
-
enqueueData(data: SourceData): void;
|
|
49
|
-
/**
|
|
50
|
-
* Reads data once from the data stream
|
|
51
|
-
* @returns a Data payload or Null if the stream closed.
|
|
52
|
-
*/
|
|
53
|
-
read(): Promise<ParsedData | null>;
|
|
54
|
-
/**
|
|
55
|
-
* Executes a callback for each data item in the stream
|
|
56
|
-
*/
|
|
57
|
-
forEach(callback: DataStreamCallback<ParsedData>): () => void;
|
|
58
|
-
protected processQueue(): Promise<void> | undefined;
|
|
59
|
-
protected hasDataReader(): boolean;
|
|
60
|
-
protected _processQueue(): Promise<void>;
|
|
61
|
-
protected iterateAsyncErrored(cb: (l: Partial<DataStreamListener<ParsedData>>) => Promise<void>): Promise<void>;
|
|
62
|
-
}
|
package/lib/utils/DataStream.js
DELETED
|
@@ -1,169 +0,0 @@
|
|
|
1
|
-
import Logger from 'js-logger';
|
|
2
|
-
import { BaseObserver } from './BaseObserver.js';
|
|
3
|
-
export const DEFAULT_PRESSURE_LIMITS = {
|
|
4
|
-
highWater: 10,
|
|
5
|
-
lowWater: 0
|
|
6
|
-
};
|
|
7
|
-
/**
|
|
8
|
-
* A very basic implementation of a data stream with backpressure support which does not use
|
|
9
|
-
* native JS streams or async iterators.
|
|
10
|
-
* This is handy for environments such as React Native which need polyfills for the above.
|
|
11
|
-
*/
|
|
12
|
-
export class DataStream extends BaseObserver {
|
|
13
|
-
options;
|
|
14
|
-
dataQueue;
|
|
15
|
-
isClosed;
|
|
16
|
-
processingPromise;
|
|
17
|
-
notifyDataAdded;
|
|
18
|
-
logger;
|
|
19
|
-
mapLine;
|
|
20
|
-
constructor(options) {
|
|
21
|
-
super();
|
|
22
|
-
this.options = options;
|
|
23
|
-
this.processingPromise = null;
|
|
24
|
-
this.isClosed = false;
|
|
25
|
-
this.dataQueue = [];
|
|
26
|
-
this.mapLine = options?.mapLine ?? ((line) => line);
|
|
27
|
-
this.logger = options?.logger ?? Logger.get('DataStream');
|
|
28
|
-
if (options?.closeOnError) {
|
|
29
|
-
const l = this.registerListener({
|
|
30
|
-
error: (ex) => {
|
|
31
|
-
l?.();
|
|
32
|
-
this.close();
|
|
33
|
-
}
|
|
34
|
-
});
|
|
35
|
-
}
|
|
36
|
-
}
|
|
37
|
-
get highWatermark() {
|
|
38
|
-
return this.options?.pressure?.highWaterMark ?? DEFAULT_PRESSURE_LIMITS.highWater;
|
|
39
|
-
}
|
|
40
|
-
get lowWatermark() {
|
|
41
|
-
return this.options?.pressure?.lowWaterMark ?? DEFAULT_PRESSURE_LIMITS.lowWater;
|
|
42
|
-
}
|
|
43
|
-
get closed() {
|
|
44
|
-
return this.isClosed;
|
|
45
|
-
}
|
|
46
|
-
async close() {
|
|
47
|
-
this.isClosed = true;
|
|
48
|
-
await this.processingPromise;
|
|
49
|
-
this.iterateListeners((l) => l.closed?.());
|
|
50
|
-
// Discard any data in the queue
|
|
51
|
-
this.dataQueue = [];
|
|
52
|
-
this.listeners.clear();
|
|
53
|
-
}
|
|
54
|
-
/**
|
|
55
|
-
* Enqueues data for the consumers to read
|
|
56
|
-
*/
|
|
57
|
-
enqueueData(data) {
|
|
58
|
-
if (this.isClosed) {
|
|
59
|
-
throw new Error('Cannot enqueue data into closed stream.');
|
|
60
|
-
}
|
|
61
|
-
this.dataQueue.push(data);
|
|
62
|
-
this.notifyDataAdded?.();
|
|
63
|
-
this.processQueue();
|
|
64
|
-
}
|
|
65
|
-
/**
|
|
66
|
-
* Reads data once from the data stream
|
|
67
|
-
* @returns a Data payload or Null if the stream closed.
|
|
68
|
-
*/
|
|
69
|
-
async read() {
|
|
70
|
-
if (this.closed) {
|
|
71
|
-
return null;
|
|
72
|
-
}
|
|
73
|
-
// Wait for any pending processing to complete first.
|
|
74
|
-
// This ensures we register our listener before calling processQueue(),
|
|
75
|
-
// avoiding a race where processQueue() sees no reader and returns early.
|
|
76
|
-
if (this.processingPromise) {
|
|
77
|
-
await this.processingPromise;
|
|
78
|
-
}
|
|
79
|
-
// Re-check after await - stream may have closed while we were waiting
|
|
80
|
-
if (this.closed) {
|
|
81
|
-
return null;
|
|
82
|
-
}
|
|
83
|
-
return new Promise((resolve, reject) => {
|
|
84
|
-
const l = this.registerListener({
|
|
85
|
-
data: async (data) => {
|
|
86
|
-
resolve(data);
|
|
87
|
-
// Remove the listener
|
|
88
|
-
l?.();
|
|
89
|
-
},
|
|
90
|
-
closed: () => {
|
|
91
|
-
resolve(null);
|
|
92
|
-
l?.();
|
|
93
|
-
},
|
|
94
|
-
error: (ex) => {
|
|
95
|
-
reject(ex);
|
|
96
|
-
l?.();
|
|
97
|
-
}
|
|
98
|
-
});
|
|
99
|
-
this.processQueue();
|
|
100
|
-
});
|
|
101
|
-
}
|
|
102
|
-
/**
|
|
103
|
-
* Executes a callback for each data item in the stream
|
|
104
|
-
*/
|
|
105
|
-
forEach(callback) {
|
|
106
|
-
if (this.dataQueue.length <= this.lowWatermark) {
|
|
107
|
-
this.iterateAsyncErrored(async (l) => l.lowWater?.());
|
|
108
|
-
}
|
|
109
|
-
return this.registerListener({
|
|
110
|
-
data: callback
|
|
111
|
-
});
|
|
112
|
-
}
|
|
113
|
-
processQueue() {
|
|
114
|
-
if (this.processingPromise) {
|
|
115
|
-
return;
|
|
116
|
-
}
|
|
117
|
-
const promise = (this.processingPromise = this._processQueue());
|
|
118
|
-
promise.finally(() => {
|
|
119
|
-
this.processingPromise = null;
|
|
120
|
-
});
|
|
121
|
-
return promise;
|
|
122
|
-
}
|
|
123
|
-
hasDataReader() {
|
|
124
|
-
return Array.from(this.listeners.values()).some((l) => !!l.data);
|
|
125
|
-
}
|
|
126
|
-
async _processQueue() {
|
|
127
|
-
/**
|
|
128
|
-
* Allow listeners to mutate the queue before processing.
|
|
129
|
-
* This allows for operations such as dropping or compressing data
|
|
130
|
-
* on high water or requesting more data on low water.
|
|
131
|
-
*/
|
|
132
|
-
if (this.dataQueue.length >= this.highWatermark) {
|
|
133
|
-
await this.iterateAsyncErrored(async (l) => l.highWater?.());
|
|
134
|
-
}
|
|
135
|
-
if (this.isClosed || !this.hasDataReader()) {
|
|
136
|
-
return;
|
|
137
|
-
}
|
|
138
|
-
if (this.dataQueue.length) {
|
|
139
|
-
const data = this.dataQueue.shift();
|
|
140
|
-
const mapped = this.mapLine(data);
|
|
141
|
-
await this.iterateAsyncErrored(async (l) => l.data?.(mapped));
|
|
142
|
-
}
|
|
143
|
-
if (this.dataQueue.length <= this.lowWatermark) {
|
|
144
|
-
const dataAdded = new Promise((resolve) => {
|
|
145
|
-
this.notifyDataAdded = resolve;
|
|
146
|
-
});
|
|
147
|
-
await Promise.race([this.iterateAsyncErrored(async (l) => l.lowWater?.()), dataAdded]);
|
|
148
|
-
this.notifyDataAdded = null;
|
|
149
|
-
}
|
|
150
|
-
if (this.dataQueue.length > 0) {
|
|
151
|
-
setTimeout(() => this.processQueue());
|
|
152
|
-
}
|
|
153
|
-
}
|
|
154
|
-
async iterateAsyncErrored(cb) {
|
|
155
|
-
// Important: We need to copy the listeners, as calling a listener could result in adding another
|
|
156
|
-
// listener, resulting in infinite loops.
|
|
157
|
-
const listeners = Array.from(this.listeners.values());
|
|
158
|
-
for (let i of listeners) {
|
|
159
|
-
try {
|
|
160
|
-
await cb(i);
|
|
161
|
-
}
|
|
162
|
-
catch (ex) {
|
|
163
|
-
this.logger.error(ex);
|
|
164
|
-
this.iterateListeners((l) => l.error?.(ex));
|
|
165
|
-
}
|
|
166
|
-
}
|
|
167
|
-
}
|
|
168
|
-
}
|
|
169
|
-
//# sourceMappingURL=DataStream.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"DataStream.js","sourceRoot":"","sources":["../../src/utils/DataStream.ts"],"names":[],"mappings":"AAAA,OAAO,MAAmB,MAAM,WAAW,CAAC;AAC5C,OAAO,EAAgB,YAAY,EAAE,MAAM,mBAAmB,CAAC;AA0B/D,MAAM,CAAC,MAAM,uBAAuB,GAAG;IACrC,SAAS,EAAE,EAAE;IACb,QAAQ,EAAE,CAAC;CACZ,CAAC;AAEF;;;;GAIG;AACH,MAAM,OAAO,UAAyC,SAAQ,YAA4C;IAYlF;IAXtB,SAAS,CAAe;IAEd,QAAQ,CAAU;IAElB,iBAAiB,CAAuB;IACxC,eAAe,CAAsB;IAErC,MAAM,CAAU;IAEhB,OAAO,CAAmC;IAEpD,YAAsB,OAAmD;QACvE,KAAK,EAAE,CAAC;QADY,YAAO,GAAP,OAAO,CAA4C;QAEvE,IAAI,CAAC,iBAAiB,GAAG,IAAI,CAAC;QAC9B,IAAI,CAAC,QAAQ,GAAG,KAAK,CAAC;QACtB,IAAI,CAAC,SAAS,GAAG,EAAE,CAAC;QACpB,IAAI,CAAC,OAAO,GAAG,OAAO,EAAE,OAAO,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAW,CAAC,CAAC;QAE3D,IAAI,CAAC,MAAM,GAAG,OAAO,EAAE,MAAM,IAAI,MAAM,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;QAE1D,IAAI,OAAO,EAAE,YAAY,EAAE,CAAC;YAC1B,MAAM,CAAC,GAAG,IAAI,CAAC,gBAAgB,CAAC;gBAC9B,KAAK,EAAE,CAAC,EAAE,EAAE,EAAE;oBACZ,CAAC,EAAE,EAAE,CAAC;oBACN,IAAI,CAAC,KAAK,EAAE,CAAC;gBACf,CAAC;aACF,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,IAAI,aAAa;QACf,OAAO,IAAI,CAAC,OAAO,EAAE,QAAQ,EAAE,aAAa,IAAI,uBAAuB,CAAC,SAAS,CAAC;IACpF,CAAC;IAED,IAAI,YAAY;QACd,OAAO,IAAI,CAAC,OAAO,EAAE,QAAQ,EAAE,YAAY,IAAI,uBAAuB,CAAC,QAAQ,CAAC;IAClF,CAAC;IAED,IAAI,MAAM;QACR,OAAO,IAAI,CAAC,QAAQ,CAAC;IACvB,CAAC;IAED,KAAK,CAAC,KAAK;QACT,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;QACrB,MAAM,IAAI,CAAC,iBAAiB,CAAC;QAC7B,IAAI,CAAC,gBAAgB,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;QAC3C,gCAAgC;QAChC,IAAI,CAAC,SAAS,GAAG,EAAE,CAAC;QACpB,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,CAAC;IACzB,CAAC;IAED;;OAEG;IACH,WAAW,CAAC,IAAgB;QAC1B,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YAClB,MAAM,IAAI,KAAK,CAAC,yCAAyC,CAAC,CAAC;QAC7D,CAAC;QAED,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC1B,IAAI,CAAC,eAAe,EAAE,EAAE,CAAC;QAEzB,IAAI,CAAC,YAAY,EAAE,CAAC;IACtB,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,IAAI;QACR,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YAChB,OAAO,IAAI,CAAC;QACd,CAAC;QAED,qDAAqD;QACrD,uEAAuE;QACvE,yEAAyE;QACzE,IAAI,IAAI,CAAC,iBAAiB,EAAE,CAAC;YAC3B,MAAM,IAAI,CAAC,iBAAiB,CAAC;QAC/B,CAAC;QAED,sEAAsE;QACtE,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YAChB,OAAO,IAAI,CAAC;QACd,CAAC;QAED,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACrC,MAAM,CAAC,GAAG,IAAI,CAAC,gBAAgB,CAAC;gBAC9B,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,EAAE;oBACnB,OAAO,CAAC,IAAI,CAAC,CAAC;oBACd,sBAAsB;oBACtB,CAAC,EAAE,EAAE,CAAC;gBACR,CAAC;gBACD,MAAM,EAAE,GAAG,EAAE;oBACX,OAAO,CAAC,IAAI,CAAC,CAAC;oBACd,CAAC,EAAE,EAAE,CAAC;gBACR,CAAC;gBACD,KAAK,EAAE,CAAC,EAAE,EAAE,EAAE;oBACZ,MAAM,CAAC,EAAE,CAAC,CAAC;oBACX,CAAC,EAAE,EAAE,CAAC;gBACR,CAAC;aACF,CAAC,CAAC;YAEH,IAAI,CAAC,YAAY,EAAE,CAAC;QACtB,CAAC,CAAC,CAAC;IACL,CAAC;IAED;;OAEG;IACH,OAAO,CAAC,QAAwC;QAC9C,IAAI,IAAI,CAAC,SAAS,CAAC,MAAM,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;YAC/C,IAAI,CAAC,mBAAmB,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;QACxD,CAAC;QAED,OAAO,IAAI,CAAC,gBAAgB,CAAC;YAC3B,IAAI,EAAE,QAAQ;SACf,CAAC,CAAC;IACL,CAAC;IAES,YAAY;QACpB,IAAI,IAAI,CAAC,iBAAiB,EAAE,CAAC;YAC3B,OAAO;QACT,CAAC;QAED,MAAM,OAAO,GAAG,CAAC,IAAI,CAAC,iBAAiB,GAAG,IAAI,CAAC,aAAa,EAAE,CAAC,CAAC;QAChE,OAAO,CAAC,OAAO,CAAC,GAAG,EAAE;YACnB,IAAI,CAAC,iBAAiB,GAAG,IAAI,CAAC;QAChC,CAAC,CAAC,CAAC;QACH,OAAO,OAAO,CAAC;IACjB,CAAC;IAES,aAAa;QACrB,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IACnE,CAAC;IAES,KAAK,CAAC,aAAa;QAC3B;;;;WAIG;QACH,IAAI,IAAI,CAAC,SAAS,CAAC,MAAM,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;YAChD,MAAM,IAAI,CAAC,mBAAmB,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;QAC/D,CAAC;QAED,IAAI,IAAI,CAAC,QAAQ,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,EAAE,CAAC;YAC3C,OAAO;QACT,CAAC;QAED,IAAI,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,CAAC;YAC1B,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,EAAG,CAAC;YACrC,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;YAClC,MAAM,IAAI,CAAC,mBAAmB,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC;QAChE,CAAC;QAED,IAAI,IAAI,CAAC,SAAS,CAAC,MAAM,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;YAC/C,MAAM,SAAS,GAAG,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;gBAC9C,IAAI,CAAC,eAAe,GAAG,OAAO,CAAC;YACjC,CAAC,CAAC,CAAC;YAEH,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,mBAAmB,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,EAAE,SAAS,CAAC,CAAC,CAAC;YACvF,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC;QAC9B,CAAC;QAED,IAAI,IAAI,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC9B,UAAU,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,YAAY,EAAE,CAAC,CAAC;QACxC,CAAC;IACH,CAAC;IAES,KAAK,CAAC,mBAAmB,CAAC,EAAiE;QACnG,iGAAiG;QACjG,yCAAyC;QACzC,MAAM,SAAS,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,CAAC,CAAC;QACtD,KAAK,IAAI,CAAC,IAAI,SAAS,EAAE,CAAC;YACxB,IAAI,CAAC;gBACH,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC;YACd,CAAC;YAAC,OAAO,EAAE,EAAE,CAAC;gBACZ,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;gBACtB,IAAI,CAAC,gBAAgB,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;YAC9C,CAAC;QACH,CAAC;IACH,CAAC;CACF"}
|