@peerbit/stream 3.0.11 → 3.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/esm/index.d.ts +7 -7
- package/lib/esm/index.js +28 -26
- package/lib/esm/index.js.map +1 -1
- package/lib/esm/pushable-lanes.d.ts +70 -0
- package/lib/esm/pushable-lanes.js +262 -0
- package/lib/esm/pushable-lanes.js.map +1 -0
- package/package.json +5 -3
- package/src/index.ts +34 -33
- package/src/pushable-lanes.ts +385 -0
|
@@ -0,0 +1,262 @@
|
|
|
1
|
+
import defer from "p-defer";
|
|
2
|
+
import GenericFIFO from "fast-fifo";
|
|
3
|
+
export class AbortError extends Error {
|
|
4
|
+
type;
|
|
5
|
+
code;
|
|
6
|
+
constructor(message, code) {
|
|
7
|
+
super(message ?? "The operation was aborted");
|
|
8
|
+
this.type = "aborted";
|
|
9
|
+
this.code = code ?? "ABORT_ERR";
|
|
10
|
+
}
|
|
11
|
+
}
|
|
12
|
+
/**
|
|
13
|
+
* Fifo but with total readableLength counter
|
|
14
|
+
*/
|
|
15
|
+
class Uint8ArrayFifo extends GenericFIFO {
|
|
16
|
+
size = 0;
|
|
17
|
+
push(val) {
|
|
18
|
+
if (val.value) {
|
|
19
|
+
this.size += val.value.byteLength;
|
|
20
|
+
}
|
|
21
|
+
return super.push(val);
|
|
22
|
+
}
|
|
23
|
+
shift() {
|
|
24
|
+
const shifted = super.shift();
|
|
25
|
+
if (shifted?.value) {
|
|
26
|
+
this.size -= shifted.value.byteLength;
|
|
27
|
+
}
|
|
28
|
+
return shifted;
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
/**
|
|
32
|
+
* A queue consisting of multiple 'lanes' with different priority to be emptied.
|
|
33
|
+
* The lane with index 0 will empty before lane with index 1 etc..
|
|
34
|
+
* TODO add an additional proprty to control whether we we pick objects from slower lanes
|
|
35
|
+
* so no lane get really "stuck"
|
|
36
|
+
*/
|
|
37
|
+
class Uint8arrayPriorityQueue {
|
|
38
|
+
lanes;
|
|
39
|
+
constructor(options = { lanes: 1 }) {
|
|
40
|
+
this.lanes = new Array(options.lanes);
|
|
41
|
+
for (let i = 0; i < this.lanes.length; i++) {
|
|
42
|
+
this.lanes[i] = new Uint8ArrayFifo();
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
get size() {
|
|
46
|
+
let sum = 0;
|
|
47
|
+
for (const lane of this.lanes) {
|
|
48
|
+
sum += lane.size;
|
|
49
|
+
}
|
|
50
|
+
return sum;
|
|
51
|
+
}
|
|
52
|
+
push(val, lane) {
|
|
53
|
+
return this.lanes[lane].push(val);
|
|
54
|
+
}
|
|
55
|
+
shift() {
|
|
56
|
+
// fetch the first non undefined item.
|
|
57
|
+
// by iterating from index 0 up we define that lanes with lower index have higher prioirity
|
|
58
|
+
for (const lane of this.lanes) {
|
|
59
|
+
const element = lane.shift();
|
|
60
|
+
if (element) {
|
|
61
|
+
return element;
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
return undefined;
|
|
65
|
+
}
|
|
66
|
+
isEmpty() {
|
|
67
|
+
for (const lane of this.lanes) {
|
|
68
|
+
if (!lane.isEmpty()) {
|
|
69
|
+
return false;
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
return true;
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
export function pushableLanes(options = {}) {
|
|
76
|
+
return _pushable(options);
|
|
77
|
+
}
|
|
78
|
+
// Modified from https://github.com/alanshaw/it-pushable
|
|
79
|
+
function _pushable(options) {
|
|
80
|
+
options = options ?? {};
|
|
81
|
+
let onEnd = options.onEnd;
|
|
82
|
+
let buffer = new Uint8arrayPriorityQueue(options.lanes ? { lanes: options.lanes } : undefined);
|
|
83
|
+
let pushable;
|
|
84
|
+
let onNext;
|
|
85
|
+
let ended;
|
|
86
|
+
let drain = defer();
|
|
87
|
+
const getNext = () => {
|
|
88
|
+
const next = buffer.shift();
|
|
89
|
+
if (next == null) {
|
|
90
|
+
return { done: true };
|
|
91
|
+
}
|
|
92
|
+
if (next.error != null) {
|
|
93
|
+
throw next.error;
|
|
94
|
+
}
|
|
95
|
+
return {
|
|
96
|
+
done: next.done === true,
|
|
97
|
+
// @ts-expect-error if done is false, value will be present
|
|
98
|
+
value: next.value
|
|
99
|
+
};
|
|
100
|
+
};
|
|
101
|
+
const waitNext = async () => {
|
|
102
|
+
try {
|
|
103
|
+
if (!buffer.isEmpty()) {
|
|
104
|
+
return getNext();
|
|
105
|
+
}
|
|
106
|
+
if (ended) {
|
|
107
|
+
return { done: true };
|
|
108
|
+
}
|
|
109
|
+
return await new Promise((resolve, reject) => {
|
|
110
|
+
onNext = (next, lane) => {
|
|
111
|
+
onNext = null;
|
|
112
|
+
buffer.push(next, lane);
|
|
113
|
+
try {
|
|
114
|
+
resolve(getNext());
|
|
115
|
+
}
|
|
116
|
+
catch (err) {
|
|
117
|
+
reject(err);
|
|
118
|
+
}
|
|
119
|
+
return pushable;
|
|
120
|
+
};
|
|
121
|
+
});
|
|
122
|
+
}
|
|
123
|
+
finally {
|
|
124
|
+
if (buffer.isEmpty()) {
|
|
125
|
+
// settle promise in the microtask queue to give consumers a chance to
|
|
126
|
+
// await after calling .push
|
|
127
|
+
queueMicrotask(() => {
|
|
128
|
+
drain.resolve();
|
|
129
|
+
drain = defer();
|
|
130
|
+
});
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
};
|
|
134
|
+
const bufferNext = (next, lane) => {
|
|
135
|
+
if (onNext != null) {
|
|
136
|
+
return onNext(next, lane);
|
|
137
|
+
}
|
|
138
|
+
buffer.push(next, lane);
|
|
139
|
+
return pushable;
|
|
140
|
+
};
|
|
141
|
+
const bufferError = (err) => {
|
|
142
|
+
buffer = new Uint8ArrayFifo();
|
|
143
|
+
if (onNext != null) {
|
|
144
|
+
return onNext({ error: err }, 0);
|
|
145
|
+
}
|
|
146
|
+
buffer.push({ error: err });
|
|
147
|
+
return pushable;
|
|
148
|
+
};
|
|
149
|
+
const push = (value, lane = 0) => {
|
|
150
|
+
if (ended) {
|
|
151
|
+
return pushable;
|
|
152
|
+
}
|
|
153
|
+
return bufferNext({ done: false, value }, lane);
|
|
154
|
+
};
|
|
155
|
+
const end = (err) => {
|
|
156
|
+
if (ended)
|
|
157
|
+
return pushable;
|
|
158
|
+
ended = true;
|
|
159
|
+
return err != null ? bufferError(err) : bufferNext({ done: true }, 0);
|
|
160
|
+
};
|
|
161
|
+
const _return = () => {
|
|
162
|
+
buffer = new Uint8ArrayFifo();
|
|
163
|
+
end();
|
|
164
|
+
return { done: true };
|
|
165
|
+
};
|
|
166
|
+
const _throw = (err) => {
|
|
167
|
+
end(err);
|
|
168
|
+
return { done: true };
|
|
169
|
+
};
|
|
170
|
+
pushable = {
|
|
171
|
+
[Symbol.asyncIterator]() {
|
|
172
|
+
return this;
|
|
173
|
+
},
|
|
174
|
+
next: waitNext,
|
|
175
|
+
return: _return,
|
|
176
|
+
throw: _throw,
|
|
177
|
+
push,
|
|
178
|
+
end,
|
|
179
|
+
get readableLength() {
|
|
180
|
+
return buffer.size;
|
|
181
|
+
},
|
|
182
|
+
getReadableLength(lane) {
|
|
183
|
+
if (lane == null) {
|
|
184
|
+
return buffer.size;
|
|
185
|
+
}
|
|
186
|
+
if (buffer instanceof Uint8arrayPriorityQueue) {
|
|
187
|
+
return buffer.lanes[lane].size;
|
|
188
|
+
}
|
|
189
|
+
throw new Error("Missing lane info");
|
|
190
|
+
},
|
|
191
|
+
onEmpty: async (options) => {
|
|
192
|
+
const signal = options?.signal;
|
|
193
|
+
signal?.throwIfAborted();
|
|
194
|
+
if (buffer.isEmpty()) {
|
|
195
|
+
return;
|
|
196
|
+
}
|
|
197
|
+
let cancel;
|
|
198
|
+
let listener;
|
|
199
|
+
if (signal != null) {
|
|
200
|
+
cancel = new Promise((resolve, reject) => {
|
|
201
|
+
listener = () => {
|
|
202
|
+
reject(new AbortError());
|
|
203
|
+
};
|
|
204
|
+
signal.addEventListener("abort", listener);
|
|
205
|
+
});
|
|
206
|
+
}
|
|
207
|
+
try {
|
|
208
|
+
await Promise.race([drain.promise, cancel]);
|
|
209
|
+
}
|
|
210
|
+
finally {
|
|
211
|
+
if (listener != null && signal != null) {
|
|
212
|
+
signal?.removeEventListener("abort", listener);
|
|
213
|
+
}
|
|
214
|
+
}
|
|
215
|
+
}
|
|
216
|
+
};
|
|
217
|
+
if (onEnd == null) {
|
|
218
|
+
return pushable;
|
|
219
|
+
}
|
|
220
|
+
const _pushable = pushable;
|
|
221
|
+
pushable = {
|
|
222
|
+
[Symbol.asyncIterator]() {
|
|
223
|
+
return this;
|
|
224
|
+
},
|
|
225
|
+
next() {
|
|
226
|
+
return _pushable.next();
|
|
227
|
+
},
|
|
228
|
+
throw(err) {
|
|
229
|
+
_pushable.throw(err);
|
|
230
|
+
if (onEnd != null) {
|
|
231
|
+
onEnd(err);
|
|
232
|
+
onEnd = undefined;
|
|
233
|
+
}
|
|
234
|
+
return { done: true };
|
|
235
|
+
},
|
|
236
|
+
return() {
|
|
237
|
+
_pushable.return();
|
|
238
|
+
if (onEnd != null) {
|
|
239
|
+
onEnd();
|
|
240
|
+
onEnd = undefined;
|
|
241
|
+
}
|
|
242
|
+
return { done: true };
|
|
243
|
+
},
|
|
244
|
+
push,
|
|
245
|
+
end(err) {
|
|
246
|
+
_pushable.end(err);
|
|
247
|
+
if (onEnd != null) {
|
|
248
|
+
onEnd(err);
|
|
249
|
+
onEnd = undefined;
|
|
250
|
+
}
|
|
251
|
+
return pushable;
|
|
252
|
+
},
|
|
253
|
+
get readableLength() {
|
|
254
|
+
return _pushable.readableLength;
|
|
255
|
+
},
|
|
256
|
+
onEmpty: (opts) => {
|
|
257
|
+
return _pushable.onEmpty(opts);
|
|
258
|
+
}
|
|
259
|
+
};
|
|
260
|
+
return pushable;
|
|
261
|
+
}
|
|
262
|
+
//# sourceMappingURL=pushable-lanes.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"pushable-lanes.js","sourceRoot":"","sources":["../../src/pushable-lanes.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,SAAS,CAAC;AAC5B,OAAO,WAAW,MAAM,WAAW,CAAC;AAEpC,MAAM,OAAO,UAAW,SAAQ,KAAK;IACpC,IAAI,CAAS;IACb,IAAI,CAAS;IAEb,YAAY,OAAgB,EAAE,IAAa;QAC1C,KAAK,CAAC,OAAO,IAAI,2BAA2B,CAAC,CAAC;QAC9C,IAAI,CAAC,IAAI,GAAG,SAAS,CAAC;QACtB,IAAI,CAAC,IAAI,GAAG,IAAI,IAAI,WAAW,CAAC;IACjC,CAAC;CACD;AA4ED;;GAEG;AACH,MAAM,cAAiD,SAAQ,WAE9D;IACA,IAAI,GAAW,CAAC,CAAC;IACjB,IAAI,CAAC,GAAY;QAChB,IAAI,GAAG,CAAC,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,IAAI,IAAI,GAAG,CAAC,KAAK,CAAC,UAAU,CAAC;QACnC,CAAC;QACD,OAAO,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACxB,CAAC;IAED,KAAK;QACJ,MAAM,OAAO,GAAG,KAAK,CAAC,KAAK,EAAE,CAAC;QAC9B,IAAI,OAAO,EAAE,KAAK,EAAE,CAAC;YACpB,IAAI,CAAC,IAAI,IAAI,OAAO,CAAC,KAAK,CAAC,UAAU,CAAC;QACvC,CAAC;QACD,OAAO,OAAO,CAAC;IAChB,CAAC;CACD;AAED;;;;;GAKG;AACH,MAAM,uBAAuB;IAC5B,KAAK,CAAsB;IAC3B,YAAY,UAA6B,EAAE,KAAK,EAAE,CAAC,EAAE;QACpD,IAAI,CAAC,KAAK,GAAG,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;QACtC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAC5C,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,IAAI,cAAc,EAAE,CAAC;QACtC,CAAC;IACF,CAAC;IAED,IAAI,IAAI;QACP,IAAI,GAAG,GAAG,CAAC,CAAC;QACZ,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YAC/B,GAAG,IAAI,IAAI,CAAC,IAAI,CAAC;QAClB,CAAC;QACD,OAAO,GAAG,CAAC;IACZ,CAAC;IACD,IAAI,CAAC,GAAY,EAAE,IAAY;QAC9B,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACnC,CAAC;IACD,KAAK;QACJ,sCAAsC;QACtC,2FAA2F;QAC3F,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YAC/B,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,EAAE,CAAC;YAC7B,IAAI,OAAO,EAAE,CAAC;gBACb,OAAO,OAAO,CAAC;YAChB,CAAC;QACF,CAAC;QACD,OAAO,SAAS,CAAC;IAClB,CAAC;IACD,OAAO;QACN,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YAC/B,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,CAAC;gBACrB,OAAO,KAAK,CAAC;YACd,CAAC;QACF,CAAC;QACD,OAAO,IAAI,CAAC;IACb,CAAC;CACD;AAED,MAAM,UAAU,aAAa,CAC5B,UAAmB,EAAE;IAErB,OAAO,SAAS,CAAkC,OAAO,CAAC,CAAC;AAC5D,CAAC;AAED,wDAAwD;AACxD,SAAS,SAAS,CACjB,OAAiB;IAEjB,OAAO,GAAG,OAAO,IAAI,EAAE,CAAC;IACxB,IAAI,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC;IAC1B,IAAI,MAAM,GACT,IAAI,uBAAuB,CAC1B,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,SAAS,CACpD,CAAC;IACH,IAAI,QAAa,CAAC;IAClB,IAAI,MAAmE,CAAC;IACxE,IAAI,KAAc,CAAC;IACnB,IAAI,KAAK,GAAG,KAAK,EAAE,CAAC;IAEpB,MAAM,OAAO,GAAG,GAA0B,EAAE;QAC3C,MAAM,IAAI,GAA+B,MAAM,CAAC,KAAK,EAAE,CAAC;QAExD,IAAI,IAAI,IAAI,IAAI,EAAE,CAAC;YAClB,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;QACvB,CAAC;QAED,IAAI,IAAI,CAAC,KAAK,IAAI,IAAI,EAAE,CAAC;YACxB,MAAM,IAAI,CAAC,KAAK,CAAC;QAClB,CAAC;QAED,OAAO;YACN,IAAI,EAAE,IAAI,CAAC,IAAI,KAAK,IAAI;YACxB,2DAA2D;YAC3D,KAAK,EAAE,IAAI,CAAC,KAAK;SACjB,CAAC;IACH,CAAC,CAAC;IAEF,MAAM,QAAQ,GAAG,KAAK,IAAoC,EAAE;QAC3D,IAAI,CAAC;YACJ,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,EAAE,CAAC;gBACvB,OAAO,OAAO,EAAE,CAAC;YAClB,CAAC;YAED,IAAI,KAAK,EAAE,CAAC;gBACX,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;YACvB,CAAC;YAED,OAAO,MAAM,IAAI,OAAO,CAAwB,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;gBACnE,MAAM,GAAG,CAAC,IAAoB,EAAE,IAAY,EAAE,EAAE;oBAC/C,MAAM,GAAG,IAAI,CAAC;oBACd,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;oBAExB,IAAI,CAAC;wBACJ,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC;oBACpB,CAAC;oBAAC,OAAO,GAAG,EAAE,CAAC;wBACd,MAAM,CAAC,GAAG,CAAC,CAAC;oBACb,CAAC;oBAED,OAAO,QAAQ,CAAC;gBACjB,CAAC,CAAC;YACH,CAAC,CAAC,CAAC;QACJ,CAAC;gBAAS,CAAC;YACV,IAAI,MAAM,CAAC,OAAO,EAAE,EAAE,CAAC;gBACtB,sEAAsE;gBACtE,4BAA4B;gBAC5B,cAAc,CAAC,GAAG,EAAE;oBACnB,KAAK,CAAC,OAAO,EAAE,CAAC;oBAChB,KAAK,GAAG,KAAK,EAAE,CAAC;gBACjB,CAAC,CAAC,CAAC;YACJ,CAAC;QACF,CAAC;IACF,CAAC,CAAC;IAEF,MAAM,UAAU,GAAG,CAAC,IAAoB,EAAE,IAAY,EAAc,EAAE;QACrE,IAAI,MAAM,IAAI,IAAI,EAAE,CAAC;YACpB,OAAO,MAAM,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;QAC3B,CAAC;QAED,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;QACxB,OAAO,QAAQ,CAAC;IACjB,CAAC,CAAC;IAEF,MAAM,WAAW,GAAG,CAAC,GAAU,EAAc,EAAE;QAC9C,MAAM,GAAG,IAAI,cAAc,EAAE,CAAC;QAE9B,IAAI,MAAM,IAAI,IAAI,EAAE,CAAC;YACpB,OAAO,MAAM,CAAC,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC,CAAC;QAClC,CAAC;QAED,MAAM,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,CAAC;QAC5B,OAAO,QAAQ,CAAC;IACjB,CAAC,CAAC;IAEF,MAAM,IAAI,GAAG,CAAC,KAAe,EAAE,OAAe,CAAC,EAAc,EAAE;QAC9D,IAAI,KAAK,EAAE,CAAC;YACX,OAAO,QAAQ,CAAC;QACjB,CAAC;QAED,OAAO,UAAU,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,EAAE,IAAI,CAAC,CAAC;IACjD,CAAC,CAAC;IACF,MAAM,GAAG,GAAG,CAAC,GAAW,EAAc,EAAE;QACvC,IAAI,KAAK;YAAE,OAAO,QAAQ,CAAC;QAC3B,KAAK,GAAG,IAAI,CAAC;QAEb,OAAO,GAAG,IAAI,IAAI,CAAC,CAAC,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC;IACvE,CAAC,CAAC;IACF,MAAM,OAAO,GAAG,GAAe,EAAE;QAChC,MAAM,GAAG,IAAI,cAAc,EAAE,CAAC;QAC9B,GAAG,EAAE,CAAC;QAEN,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;IACvB,CAAC,CAAC;IACF,MAAM,MAAM,GAAG,CAAC,GAAU,EAAc,EAAE;QACzC,GAAG,CAAC,GAAG,CAAC,CAAC;QAET,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;IACvB,CAAC,CAAC;IAEF,QAAQ,GAAG;QACV,CAAC,MAAM,CAAC,aAAa,CAAC;YACrB,OAAO,IAAI,CAAC;QACb,CAAC;QACD,IAAI,EAAE,QAAQ;QACd,MAAM,EAAE,OAAO;QACf,KAAK,EAAE,MAAM;QACb,IAAI;QACJ,GAAG;QACH,IAAI,cAAc;YACjB,OAAO,MAAM,CAAC,IAAI,CAAC;QACpB,CAAC;QAED,iBAAiB,CAAC,IAAa;YAC9B,IAAI,IAAI,IAAI,IAAI,EAAE,CAAC;gBAClB,OAAO,MAAM,CAAC,IAAI,CAAC;YACpB,CAAC;YAED,IAAI,MAAM,YAAY,uBAAuB,EAAE,CAAC;gBAC/C,OAAO,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC;YAChC,CAAC;YACD,MAAM,IAAI,KAAK,CAAC,mBAAmB,CAAC,CAAC;QACtC,CAAC;QACD,OAAO,EAAE,KAAK,EAAE,OAAsB,EAAE,EAAE;YACzC,MAAM,MAAM,GAAG,OAAO,EAAE,MAAM,CAAC;YAC/B,MAAM,EAAE,cAAc,EAAE,CAAC;YAEzB,IAAI,MAAM,CAAC,OAAO,EAAE,EAAE,CAAC;gBACtB,OAAO;YACR,CAAC;YAED,IAAI,MAAiC,CAAC;YACtC,IAAI,QAAkC,CAAC;YAEvC,IAAI,MAAM,IAAI,IAAI,EAAE,CAAC;gBACpB,MAAM,GAAG,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;oBACxC,QAAQ,GAAG,GAAG,EAAE;wBACf,MAAM,CAAC,IAAI,UAAU,EAAE,CAAC,CAAC;oBAC1B,CAAC,CAAC;oBAEF,MAAM,CAAC,gBAAgB,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;gBAC5C,CAAC,CAAC,CAAC;YACJ,CAAC;YAED,IAAI,CAAC;gBACJ,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC;YAC7C,CAAC;oBAAS,CAAC;gBACV,IAAI,QAAQ,IAAI,IAAI,IAAI,MAAM,IAAI,IAAI,EAAE,CAAC;oBACxC,MAAM,EAAE,mBAAmB,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;gBAChD,CAAC;YACF,CAAC;QACF,CAAC;KACD,CAAC;IAEF,IAAI,KAAK,IAAI,IAAI,EAAE,CAAC;QACnB,OAAO,QAAQ,CAAC;IACjB,CAAC;IAED,MAAM,SAAS,GAAG,QAAQ,CAAC;IAE3B,QAAQ,GAAG;QACV,CAAC,MAAM,CAAC,aAAa,CAAC;YACrB,OAAO,IAAI,CAAC;QACb,CAAC;QACD,IAAI;YACH,OAAO,SAAS,CAAC,IAAI,EAAE,CAAC;QACzB,CAAC;QACD,KAAK,CAAC,GAAU;YACf,SAAS,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YAErB,IAAI,KAAK,IAAI,IAAI,EAAE,CAAC;gBACnB,KAAK,CAAC,GAAG,CAAC,CAAC;gBACX,KAAK,GAAG,SAAS,CAAC;YACnB,CAAC;YAED,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;QACvB,CAAC;QACD,MAAM;YACL,SAAS,CAAC,MAAM,EAAE,CAAC;YAEnB,IAAI,KAAK,IAAI,IAAI,EAAE,CAAC;gBACnB,KAAK,EAAE,CAAC;gBACR,KAAK,GAAG,SAAS,CAAC;YACnB,CAAC;YAED,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;QACvB,CAAC;QACD,IAAI;QACJ,GAAG,CAAC,GAAU;YACb,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YAEnB,IAAI,KAAK,IAAI,IAAI,EAAE,CAAC;gBACnB,KAAK,CAAC,GAAG,CAAC,CAAC;gBACX,KAAK,GAAG,SAAS,CAAC;YACnB,CAAC;YAED,OAAO,QAAQ,CAAC;QACjB,CAAC;QACD,IAAI,cAAc;YACjB,OAAO,SAAS,CAAC,cAAc,CAAC;QACjC,CAAC;QACD,OAAO,EAAE,CAAC,IAAmB,EAAE,EAAE;YAChC,OAAO,SAAS,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QAChC,CAAC;KACD,CAAC;IAEF,OAAO,QAAQ,CAAC;AACjB,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@peerbit/stream",
|
|
3
|
-
"version": "3.0
|
|
3
|
+
"version": "3.1.0",
|
|
4
4
|
"description": "A building block for direct streaming protocols",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"sideEffects": false,
|
|
@@ -50,7 +50,8 @@
|
|
|
50
50
|
"dao.xyz"
|
|
51
51
|
],
|
|
52
52
|
"devDependencies": {
|
|
53
|
-
"@peerbit/libp2p-test-utils": "2.1.
|
|
53
|
+
"@peerbit/libp2p-test-utils": "2.1.2",
|
|
54
|
+
"@types/fast-fifo": "^1.0.2",
|
|
54
55
|
"@types/yallist": "^4.0.1"
|
|
55
56
|
},
|
|
56
57
|
"dependencies": {
|
|
@@ -59,9 +60,10 @@
|
|
|
59
60
|
"@peerbit/crypto": "2.1.7",
|
|
60
61
|
"@peerbit/stream-interface": "^3.0.9",
|
|
61
62
|
"abortable-iterator": "^5.0.1",
|
|
63
|
+
"fast-fifo": "^1.3.2",
|
|
62
64
|
"libp2p": "^1.2.1",
|
|
63
65
|
"p-queue": "^8.0.1",
|
|
64
66
|
"yallist": "^4.0.0"
|
|
65
67
|
},
|
|
66
|
-
"gitHead": "
|
|
68
|
+
"gitHead": "0953a22f7a7ad21e89d0d4ee0673b0569da27f87"
|
|
67
69
|
}
|
package/src/index.ts
CHANGED
|
@@ -3,8 +3,7 @@ import { pipe } from "it-pipe";
|
|
|
3
3
|
import Queue from "p-queue";
|
|
4
4
|
import type { PeerId } from "@libp2p/interface";
|
|
5
5
|
import type { Connection } from "@libp2p/interface";
|
|
6
|
-
import
|
|
7
|
-
import { pushable } from "it-pushable";
|
|
6
|
+
import { PushableLanes, pushableLanes } from "./pushable-lanes.js";
|
|
8
7
|
import type { Stream } from "@libp2p/interface";
|
|
9
8
|
import { Uint8ArrayList } from "uint8arraylist";
|
|
10
9
|
import { abortableSource } from "abortable-iterator";
|
|
@@ -13,7 +12,6 @@ import { MAX_ROUTE_DISTANCE, Routes } from "./routes.js";
|
|
|
13
12
|
import type { IncomingStreamData, Registrar } from "@libp2p/interface-internal";
|
|
14
13
|
import type { AddressManager } from "@libp2p/interface-internal";
|
|
15
14
|
import type { ConnectionManager } from "@libp2p/interface-internal";
|
|
16
|
-
|
|
17
15
|
import { PeerStore } from "@libp2p/interface";
|
|
18
16
|
import pDefer from "p-defer";
|
|
19
17
|
|
|
@@ -116,7 +114,8 @@ export class PeerStreams extends TypedEventEmitter<PeerStreamEvents> {
|
|
|
116
114
|
/**
|
|
117
115
|
* Write stream - it's preferable to use the write method
|
|
118
116
|
*/
|
|
119
|
-
public outboundStream?:
|
|
117
|
+
public outboundStream?: PushableLanes<Uint8Array>;
|
|
118
|
+
|
|
120
119
|
/**
|
|
121
120
|
* Read stream
|
|
122
121
|
*/
|
|
@@ -124,11 +123,11 @@ export class PeerStreams extends TypedEventEmitter<PeerStreamEvents> {
|
|
|
124
123
|
/**
|
|
125
124
|
* The raw outbound stream, as retrieved from conn.newStream
|
|
126
125
|
*/
|
|
127
|
-
public
|
|
126
|
+
public rawOutboundStream?: Stream;
|
|
128
127
|
/**
|
|
129
128
|
* The raw inbound stream, as retrieved from the callback from libp2p.handle
|
|
130
129
|
*/
|
|
131
|
-
public
|
|
130
|
+
public rawInboundStream?: Stream;
|
|
132
131
|
/**
|
|
133
132
|
* An AbortController for controlled shutdown of the treams
|
|
134
133
|
*/
|
|
@@ -176,7 +175,7 @@ export class PeerStreams extends TypedEventEmitter<PeerStreamEvents> {
|
|
|
176
175
|
* Send a message to this peer.
|
|
177
176
|
* Throws if there is no `stream` to write to available.
|
|
178
177
|
*/
|
|
179
|
-
write(data: Uint8Array | Uint8ArrayList) {
|
|
178
|
+
write(data: Uint8Array | Uint8ArrayList, priority = false) {
|
|
180
179
|
if (data.length > MAX_DATA_LENGTH_OUT) {
|
|
181
180
|
throw new Error(
|
|
182
181
|
`Message too large (${data.length * 1e-6}) mb). Needs to be less than ${
|
|
@@ -190,13 +189,13 @@ export class PeerStreams extends TypedEventEmitter<PeerStreamEvents> {
|
|
|
190
189
|
}
|
|
191
190
|
|
|
192
191
|
this.usedBandWidthTracker.add(data.byteLength);
|
|
193
|
-
|
|
194
192
|
this.outboundStream.push(
|
|
195
|
-
data instanceof Uint8Array ? data : data.subarray()
|
|
193
|
+
data instanceof Uint8Array ? data : data.subarray(),
|
|
194
|
+
priority || this.outboundStream.getReadableLength(0) === 0 ? 0 : 1
|
|
196
195
|
);
|
|
197
196
|
}
|
|
198
197
|
|
|
199
|
-
async waitForWrite(bytes: Uint8Array | Uint8ArrayList) {
|
|
198
|
+
async waitForWrite(bytes: Uint8Array | Uint8ArrayList, priority = false) {
|
|
200
199
|
if (this.closed) {
|
|
201
200
|
logger.error("Failed to send to stream: " + this.peerId + ". Closed");
|
|
202
201
|
return;
|
|
@@ -235,13 +234,13 @@ export class PeerStreams extends TypedEventEmitter<PeerStreamEvents> {
|
|
|
235
234
|
|
|
236
235
|
await outboundPromise
|
|
237
236
|
.then(() => {
|
|
238
|
-
this.write(bytes);
|
|
237
|
+
this.write(bytes, priority);
|
|
239
238
|
})
|
|
240
239
|
.catch((error) => {
|
|
241
240
|
throw error;
|
|
242
241
|
});
|
|
243
242
|
} else {
|
|
244
|
-
this.write(bytes);
|
|
243
|
+
this.write(bytes, priority);
|
|
245
244
|
}
|
|
246
245
|
}
|
|
247
246
|
|
|
@@ -253,9 +252,9 @@ export class PeerStreams extends TypedEventEmitter<PeerStreamEvents> {
|
|
|
253
252
|
// The inbound stream is:
|
|
254
253
|
// - abortable, set to only return on abort, rather than throw
|
|
255
254
|
// - transformed with length-prefix transform
|
|
256
|
-
this.
|
|
255
|
+
this.rawInboundStream = stream;
|
|
257
256
|
this.inboundStream = abortableSource(
|
|
258
|
-
pipe(this.
|
|
257
|
+
pipe(this.rawInboundStream, (source) =>
|
|
259
258
|
lp.decode(source, { maxDataLength: MAX_DATA_LENGTH_IN })
|
|
260
259
|
),
|
|
261
260
|
this.inboundAbortController.signal,
|
|
@@ -284,21 +283,17 @@ export class PeerStreams extends TypedEventEmitter<PeerStreamEvents> {
|
|
|
284
283
|
);
|
|
285
284
|
return;
|
|
286
285
|
}
|
|
287
|
-
this.
|
|
288
|
-
this.outboundStream =
|
|
289
|
-
objectMode: false
|
|
290
|
-
});
|
|
286
|
+
this.rawOutboundStream = stream;
|
|
287
|
+
this.outboundStream = pushableLanes({ lanes: 2 });
|
|
291
288
|
|
|
292
289
|
pipe(
|
|
293
290
|
this.outboundStream,
|
|
294
291
|
(source) => lp.encode(source),
|
|
295
|
-
this.
|
|
292
|
+
this.rawOutboundStream
|
|
296
293
|
).catch(logError);
|
|
297
294
|
|
|
298
295
|
// Emit if the connection is new
|
|
299
296
|
this.dispatchEvent(new CustomEvent("stream:outbound"));
|
|
300
|
-
|
|
301
|
-
return this.outboundStream;
|
|
302
297
|
}
|
|
303
298
|
|
|
304
299
|
/**
|
|
@@ -314,21 +309,23 @@ export class PeerStreams extends TypedEventEmitter<PeerStreamEvents> {
|
|
|
314
309
|
// End the outbound stream
|
|
315
310
|
if (this.outboundStream != null) {
|
|
316
311
|
await this.outboundStream.return();
|
|
317
|
-
await this.
|
|
312
|
+
await this.rawOutboundStream?.abort(new AbortError("Closed"));
|
|
318
313
|
}
|
|
314
|
+
|
|
319
315
|
// End the inbound stream
|
|
320
316
|
if (this.inboundStream != null) {
|
|
321
317
|
this.inboundAbortController.abort();
|
|
322
|
-
await this.
|
|
318
|
+
await this.rawInboundStream?.close();
|
|
323
319
|
}
|
|
324
320
|
|
|
325
321
|
this.usedBandWidthTracker.stop();
|
|
326
322
|
|
|
327
323
|
this.dispatchEvent(new CustomEvent("close"));
|
|
328
324
|
|
|
329
|
-
this.
|
|
325
|
+
this.rawOutboundStream = undefined;
|
|
330
326
|
this.outboundStream = undefined;
|
|
331
|
-
|
|
327
|
+
|
|
328
|
+
this.rawInboundStream = undefined;
|
|
332
329
|
this.inboundStream = undefined;
|
|
333
330
|
}
|
|
334
331
|
}
|
|
@@ -1565,8 +1562,7 @@ export abstract class DirectStream<
|
|
|
1565
1562
|
clear();
|
|
1566
1563
|
deliveryDeferredPromise.reject(
|
|
1567
1564
|
new DeliveryError(
|
|
1568
|
-
|
|
1569
|
-
ev.detail.hashcode()
|
|
1565
|
+
`At least one recipent became unreachable while delivering messsage of type$ ${message.constructor.name}} to ${ev.detail.hashcode()}`
|
|
1570
1566
|
)
|
|
1571
1567
|
);
|
|
1572
1568
|
}
|
|
@@ -1603,9 +1599,7 @@ export abstract class DirectStream<
|
|
|
1603
1599
|
if (!hasAll && willGetAllAcknowledgements) {
|
|
1604
1600
|
deliveryDeferredPromise.reject(
|
|
1605
1601
|
new DeliveryError(
|
|
1606
|
-
|
|
1607
|
-
this.publicKeyHash
|
|
1608
|
-
} Failed to get message ${idString} ${filterMessageForSeenCounter} ${[
|
|
1602
|
+
`Failed to get message ${idString} ${filterMessageForSeenCounter} ${[
|
|
1609
1603
|
...messageToSet
|
|
1610
1604
|
]} delivery acknowledges from all nodes (${
|
|
1611
1605
|
fastestNodesReached.size
|
|
@@ -1699,6 +1693,11 @@ export abstract class DirectStream<
|
|
|
1699
1693
|
if (this.stopping || !this.started) {
|
|
1700
1694
|
throw new NotStartedError();
|
|
1701
1695
|
}
|
|
1696
|
+
const isPriorityMessage =
|
|
1697
|
+
message.header.mode instanceof SilentDelivery ||
|
|
1698
|
+
message.header.mode instanceof AnyWhere
|
|
1699
|
+
? false
|
|
1700
|
+
: true;
|
|
1702
1701
|
|
|
1703
1702
|
let delivereyPromise: Promise<void> | undefined = undefined as any;
|
|
1704
1703
|
|
|
@@ -1766,7 +1765,8 @@ export abstract class DirectStream<
|
|
|
1766
1765
|
const promises: Promise<any>[] = [];
|
|
1767
1766
|
for (const [neighbour, _distantPeers] of fanout) {
|
|
1768
1767
|
const stream = this.peers.get(neighbour);
|
|
1769
|
-
stream &&
|
|
1768
|
+
stream &&
|
|
1769
|
+
promises.push(stream.waitForWrite(bytes, isPriorityMessage));
|
|
1770
1770
|
}
|
|
1771
1771
|
await Promise.all(promises);
|
|
1772
1772
|
return delivereyPromise; // we are done sending the message in all direction with updates 'to' lists
|
|
@@ -1812,7 +1812,7 @@ export abstract class DirectStream<
|
|
|
1812
1812
|
|
|
1813
1813
|
sentOnce = true;
|
|
1814
1814
|
|
|
1815
|
-
promises.push(id.waitForWrite(bytes));
|
|
1815
|
+
promises.push(id.waitForWrite(bytes, isPriorityMessage));
|
|
1816
1816
|
}
|
|
1817
1817
|
await Promise.all(promises);
|
|
1818
1818
|
|
|
@@ -1959,7 +1959,8 @@ export abstract class DirectStream<
|
|
|
1959
1959
|
getQueuedBytes(): number {
|
|
1960
1960
|
let sum = 0;
|
|
1961
1961
|
for (const peer of this.peers) {
|
|
1962
|
-
|
|
1962
|
+
const out = peer[1].outboundStream;
|
|
1963
|
+
sum += out ? out.readableLength : 0;
|
|
1963
1964
|
}
|
|
1964
1965
|
return sum;
|
|
1965
1966
|
}
|