@nxtedition/lib 28.0.6 → 28.0.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/app.js +38 -29
- package/package.json +2 -2
- package/sequence.js +12 -12
- package/shared.js +139 -191
package/app.js
CHANGED
|
@@ -635,19 +635,7 @@ export function makeApp(appConfig, onTerminateOrMeta, metaOrNull) {
|
|
|
635
635
|
stats$ = rxjs.timer(0, 10e3).pipe(rx.map(() => ({})))
|
|
636
636
|
}
|
|
637
637
|
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
let memoryUsageMap
|
|
641
|
-
if (isMainThread) {
|
|
642
|
-
memoryUsageMap = new Map()
|
|
643
|
-
memoryUsageBC.onmessage = ({ data: { data, id } }) => {
|
|
644
|
-
memoryUsageMap.set(id, data)
|
|
645
|
-
}
|
|
646
|
-
}
|
|
647
|
-
|
|
648
|
-
setInterval(() => {
|
|
649
|
-
memoryUsageBC.postMessage({ data: process.memoryUsage(), id: serviceWorkerId })
|
|
650
|
-
}, 1e3).unref()
|
|
638
|
+
let statsMap
|
|
651
639
|
|
|
652
640
|
const startTime = Date.now()
|
|
653
641
|
stats$ = stats$.pipe(
|
|
@@ -679,12 +667,21 @@ export function makeApp(appConfig, onTerminateOrMeta, metaOrNull) {
|
|
|
679
667
|
totalArrayBuffers: 0,
|
|
680
668
|
}
|
|
681
669
|
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
|
|
670
|
+
const http = {
|
|
671
|
+
userAgent,
|
|
672
|
+
pending: globalThis._nxt_lib_http_pending?.size,
|
|
673
|
+
totalPending: 0,
|
|
674
|
+
}
|
|
675
|
+
|
|
676
|
+
const undici = {
|
|
677
|
+
sockets: globalThis.__undici_sockets?.size ?? 0,
|
|
678
|
+
totalSockets: 0,
|
|
679
|
+
}
|
|
680
|
+
|
|
681
|
+
if (statsMap) {
|
|
682
|
+
for (const stats of statsMap.values()) {
|
|
683
|
+
http.totalPending += stats.http?.pending ?? 0
|
|
684
|
+
undici.totalSockets += stats.undici?.sockets ?? 0
|
|
688
685
|
}
|
|
689
686
|
}
|
|
690
687
|
|
|
@@ -700,13 +697,8 @@ export function makeApp(appConfig, onTerminateOrMeta, metaOrNull) {
|
|
|
700
697
|
resourceLimits,
|
|
701
698
|
utilization: performance.eventLoopUtilization?.(elu2, elu1),
|
|
702
699
|
heap: v8.getHeapStatistics(),
|
|
703
|
-
http
|
|
704
|
-
|
|
705
|
-
pending: globalThis._nxt_lib_http_pending?.size,
|
|
706
|
-
},
|
|
707
|
-
undici: {
|
|
708
|
-
sockets: globalThis.__undici_sockets?.size ?? 0,
|
|
709
|
-
},
|
|
700
|
+
http,
|
|
701
|
+
undici,
|
|
710
702
|
}
|
|
711
703
|
}),
|
|
712
704
|
),
|
|
@@ -726,11 +718,20 @@ export function makeApp(appConfig, onTerminateOrMeta, metaOrNull) {
|
|
|
726
718
|
rx.refCount(),
|
|
727
719
|
)
|
|
728
720
|
|
|
721
|
+
const statsBC = new BroadcastChannel('nxt:app:stats').unref()
|
|
722
|
+
if (isMainThread) {
|
|
723
|
+
statsMap = new Map()
|
|
724
|
+
statsBC.onmessage = ({ data: { data, id } }) => {
|
|
725
|
+
statsMap.set(id, data)
|
|
726
|
+
}
|
|
727
|
+
}
|
|
728
|
+
|
|
729
729
|
monitorProviders.stats$ = stats$
|
|
730
730
|
|
|
731
731
|
if (process.env.NODE_ENV === 'production') {
|
|
732
732
|
appDestroyers.unshift(
|
|
733
733
|
stats$.pipe(rx.auditTime(10e3)).subscribe((stats) => {
|
|
734
|
+
statsBC.postMessage({ id: threadId, data: stats })
|
|
734
735
|
logger.debug(stats, 'STATS')
|
|
735
736
|
}),
|
|
736
737
|
)
|
|
@@ -779,7 +780,7 @@ export function makeApp(appConfig, onTerminateOrMeta, metaOrNull) {
|
|
|
779
780
|
rx.repeatWhen((complete$) => complete$.pipe(rx.delay(10e3))),
|
|
780
781
|
),
|
|
781
782
|
stats$.pipe(
|
|
782
|
-
rx.map(({ memory, heap, utilization, undici }) => {
|
|
783
|
+
rx.map(({ memory, heap, utilization, undici, http }) => {
|
|
783
784
|
const messages = []
|
|
784
785
|
|
|
785
786
|
if (memory?.containerLimit) {
|
|
@@ -812,8 +813,16 @@ export function makeApp(appConfig, onTerminateOrMeta, metaOrNull) {
|
|
|
812
813
|
if (undici) {
|
|
813
814
|
messages.push({
|
|
814
815
|
id: 'app:undici_upstream_sockets',
|
|
815
|
-
level: undici.
|
|
816
|
-
msg: `Undici: ${undici.
|
|
816
|
+
level: undici.totalSockets > 8192 ? 50 : undici.totalSockets > 4096 ? 40 : 30,
|
|
817
|
+
msg: `Undici: ${undici.totalSockets} upstream connected`,
|
|
818
|
+
})
|
|
819
|
+
}
|
|
820
|
+
|
|
821
|
+
if (http) {
|
|
822
|
+
messages.push({
|
|
823
|
+
id: 'app:http_pending_requests',
|
|
824
|
+
level: http.totalPending > 8192 ? 50 : http.totalPending > 4096 ? 40 : 30,
|
|
825
|
+
msg: `HTTP: ${http.totalPending} pending requests`,
|
|
817
826
|
})
|
|
818
827
|
}
|
|
819
828
|
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@nxtedition/lib",
|
|
3
|
-
"version": "28.0.
|
|
3
|
+
"version": "28.0.8",
|
|
4
4
|
"license": "UNLICENSED",
|
|
5
5
|
"author": "Robert Nagy <robert.nagy@boffins.se>",
|
|
6
6
|
"type": "module",
|
|
@@ -92,5 +92,5 @@
|
|
|
92
92
|
"pino": ">=7.0.0",
|
|
93
93
|
"rxjs": "^7.0.0"
|
|
94
94
|
},
|
|
95
|
-
"gitHead": "
|
|
95
|
+
"gitHead": "5d6fb19ef1936e749aadecc51007ddda71dcf0a3"
|
|
96
96
|
}
|
package/sequence.js
CHANGED
|
@@ -96,11 +96,10 @@ export class Sequence {
|
|
|
96
96
|
this.#value = '0'
|
|
97
97
|
} else if (Array.isArray(value)) {
|
|
98
98
|
this.#identity = identity
|
|
99
|
-
this.#count = 0
|
|
100
99
|
for (const part of value) {
|
|
101
100
|
if (typeof part === 'string') {
|
|
102
|
-
const [sequenceStr, id] = part.split(ID_SEP)
|
|
103
|
-
const sequence =
|
|
101
|
+
const [sequenceStr, id] = part.split(ID_SEP, 2)
|
|
102
|
+
const sequence = Number(sequenceStr)
|
|
104
103
|
this.#parts.push(id, sequence)
|
|
105
104
|
this.#identity = null
|
|
106
105
|
this.#count += part
|
|
@@ -117,16 +116,15 @@ export class Sequence {
|
|
|
117
116
|
}
|
|
118
117
|
assert(this.#identity === 0 || this.#parts.length > 0)
|
|
119
118
|
} else if (typeof value === 'string') {
|
|
120
|
-
const [countStr, token] = value.split('-')
|
|
121
|
-
const count = parseInt(countStr)
|
|
119
|
+
const [countStr, token] = value.split('-', 2)
|
|
122
120
|
if (token) {
|
|
123
121
|
for (const str of token.split('_')) {
|
|
124
122
|
const [sequenceStr, id] = str.split(ID_SEP)
|
|
125
|
-
const sequence =
|
|
123
|
+
const sequence = Number(sequenceStr)
|
|
126
124
|
this.#parts.push(id, sequence)
|
|
127
125
|
}
|
|
128
126
|
}
|
|
129
|
-
this.#count =
|
|
127
|
+
this.#count = Number(countStr)
|
|
130
128
|
this.#value = value
|
|
131
129
|
} else if (value instanceof Sequence) {
|
|
132
130
|
this.#count = value.#count
|
|
@@ -141,10 +139,6 @@ export class Sequence {
|
|
|
141
139
|
throw new Error('invalid sequence count')
|
|
142
140
|
}
|
|
143
141
|
|
|
144
|
-
if (!Array.isArray(this.#parts)) {
|
|
145
|
-
throw new Error('invalid sequence parts')
|
|
146
|
-
}
|
|
147
|
-
|
|
148
142
|
{
|
|
149
143
|
let count = 0
|
|
150
144
|
for (let n = 0; n < this.#parts.length; n += 2) {
|
|
@@ -289,7 +283,13 @@ export class Sequence {
|
|
|
289
283
|
throw new TypeError('strict must be a boolean')
|
|
290
284
|
}
|
|
291
285
|
|
|
292
|
-
if (
|
|
286
|
+
if (
|
|
287
|
+
strict &&
|
|
288
|
+
(other.identity || this.identity) &&
|
|
289
|
+
other.identity !== this.identity &&
|
|
290
|
+
this.#value !== '0' &&
|
|
291
|
+
other.#value !== '0'
|
|
292
|
+
) {
|
|
293
293
|
throw new Error('Cannot compare sequences with different identities')
|
|
294
294
|
}
|
|
295
295
|
|
package/shared.js
CHANGED
|
@@ -1,6 +1,3 @@
|
|
|
1
|
-
import stream from 'node:stream'
|
|
2
|
-
import assert from 'node:assert'
|
|
3
|
-
|
|
4
1
|
// By placing the read and write indices far apart (multiples of a common
|
|
5
2
|
// cache line size, 64 bytes), we prevent "false sharing". This is a
|
|
6
3
|
// low-level CPU optimization where two cores writing to different variables
|
|
@@ -21,11 +18,22 @@ const HWM_COUNT = 1024 // 1024 items
|
|
|
21
18
|
* @returns {{sharedState: SharedArrayBuffer, sharedBuffer: SharedArrayBuffer}}
|
|
22
19
|
*/
|
|
23
20
|
export function alloc(size) {
|
|
21
|
+
if (!Number.isInteger(size)) {
|
|
22
|
+
throw new TypeError('size must be a positive integer')
|
|
23
|
+
}
|
|
24
|
+
if (size <= 0) {
|
|
25
|
+
throw new RangeError('size must be a positive integer')
|
|
26
|
+
}
|
|
27
|
+
if (size >= 2 ** 31 - 8) {
|
|
28
|
+
throw new RangeError('size exceeds maximum of 2GB minus header size')
|
|
29
|
+
}
|
|
30
|
+
|
|
24
31
|
return {
|
|
25
32
|
// A small buffer for sharing state (read/write pointers).
|
|
26
33
|
sharedState: new SharedArrayBuffer(128),
|
|
27
34
|
// The main buffer for transferring data.
|
|
28
|
-
|
|
35
|
+
// We need another 8 bytes for entry headers.
|
|
36
|
+
sharedBuffer: new SharedArrayBuffer(size + 8),
|
|
29
37
|
}
|
|
30
38
|
}
|
|
31
39
|
|
|
@@ -35,6 +43,16 @@ export function alloc(size) {
|
|
|
35
43
|
* @returns {{ readSome: function }}
|
|
36
44
|
*/
|
|
37
45
|
export function reader({ sharedState, sharedBuffer }) {
|
|
46
|
+
if (!(sharedState instanceof SharedArrayBuffer)) {
|
|
47
|
+
throw new TypeError('sharedState must be a SharedArrayBuffer')
|
|
48
|
+
}
|
|
49
|
+
if (!(sharedBuffer instanceof SharedArrayBuffer)) {
|
|
50
|
+
throw new TypeError('sharedBuffer must be a SharedArrayBuffer')
|
|
51
|
+
}
|
|
52
|
+
if (sharedBuffer.byteLength >= 2 ** 31) {
|
|
53
|
+
throw new RangeError('Shared buffer size exceeds maximum of 2GB')
|
|
54
|
+
}
|
|
55
|
+
|
|
38
56
|
const state = new Int32Array(sharedState)
|
|
39
57
|
const size = sharedBuffer.byteLength
|
|
40
58
|
const buffer = Buffer.from(sharedBuffer)
|
|
@@ -52,7 +70,7 @@ export function reader({ sharedState, sharedBuffer }) {
|
|
|
52
70
|
|
|
53
71
|
/**
|
|
54
72
|
* Reads a batch of messages from the buffer.
|
|
55
|
-
* @param {(data: {buffer: Buffer, view: DataView, offset: number, length: number}) => void} next Callback to process a message.
|
|
73
|
+
* @param {(data: {buffer: Buffer, view: DataView, offset: number, length: number}) => void|boolean} next Callback to process a message.
|
|
56
74
|
* @returns {number} The number of messages read.
|
|
57
75
|
*/
|
|
58
76
|
function readSome(next) {
|
|
@@ -70,10 +88,11 @@ export function reader({ sharedState, sharedBuffer }) {
|
|
|
70
88
|
const dataPos = readPos + 4
|
|
71
89
|
const dataLen = view.getInt32(dataPos - 4, true) | 0
|
|
72
90
|
|
|
91
|
+
bytes += 4
|
|
92
|
+
|
|
73
93
|
// A length of -1 is a special marker indicating the writer has
|
|
74
94
|
// wrapped around to the beginning of the buffer.
|
|
75
95
|
if (dataLen === -1) {
|
|
76
|
-
bytes += 4
|
|
77
96
|
readPos = 0
|
|
78
97
|
// After wrapping, we must re-check against the writer's position.
|
|
79
98
|
// It's possible the writer is now at a position > 0.
|
|
@@ -86,22 +105,17 @@ export function reader({ sharedState, sharedBuffer }) {
|
|
|
86
105
|
throw new Error('Data exceeds buffer size')
|
|
87
106
|
}
|
|
88
107
|
|
|
89
|
-
bytes += dataLen
|
|
90
108
|
readPos += 4 + dataLen
|
|
109
|
+
|
|
110
|
+
bytes += dataLen
|
|
91
111
|
count += 1
|
|
92
112
|
|
|
93
113
|
// This is a "zero-copy" operation. We don't copy the data out.
|
|
94
114
|
// Instead, we pass a "view" into the shared buffer.
|
|
95
115
|
data.offset = dataPos
|
|
96
116
|
data.length = dataLen
|
|
97
|
-
const cont = next(data)
|
|
98
|
-
|
|
99
|
-
if (readPos === writePos) {
|
|
100
|
-
// If we reach the end of the buffer, we must re-check the writer's position.
|
|
101
|
-
writePos = Atomics.load(state, WRITE_INDEX) | 0
|
|
102
|
-
}
|
|
103
117
|
|
|
104
|
-
if (
|
|
118
|
+
if (next(data) === false) {
|
|
105
119
|
break
|
|
106
120
|
}
|
|
107
121
|
}
|
|
@@ -125,6 +139,16 @@ export function reader({ sharedState, sharedBuffer }) {
|
|
|
125
139
|
* @returns {{ write: function, cork: function(function) }}
|
|
126
140
|
*/
|
|
127
141
|
export function writer({ sharedState, sharedBuffer }, { yield: onYield, logger } = {}) {
|
|
142
|
+
if (!(sharedState instanceof SharedArrayBuffer)) {
|
|
143
|
+
throw new TypeError('sharedState must be a SharedArrayBuffer')
|
|
144
|
+
}
|
|
145
|
+
if (!(sharedBuffer instanceof SharedArrayBuffer)) {
|
|
146
|
+
throw new TypeError('sharedBuffer must be a SharedArrayBuffer')
|
|
147
|
+
}
|
|
148
|
+
if (sharedBuffer.byteLength >= 2 ** 31) {
|
|
149
|
+
throw new RangeError('Shared buffer size exceeds maximum of 2GB')
|
|
150
|
+
}
|
|
151
|
+
|
|
128
152
|
const state = new Int32Array(sharedState)
|
|
129
153
|
const size = sharedBuffer.byteLength
|
|
130
154
|
const buffer = Buffer.from(sharedBuffer)
|
|
@@ -140,7 +164,7 @@ export function writer({ sharedState, sharedBuffer }, { yield: onYield, logger }
|
|
|
140
164
|
let readPos = Atomics.load(state, READ_INDEX) | 0
|
|
141
165
|
let writePos = Atomics.load(state, WRITE_INDEX) | 0
|
|
142
166
|
|
|
143
|
-
let yielding =
|
|
167
|
+
let yielding = 0
|
|
144
168
|
let corked = 0
|
|
145
169
|
let pending = 0
|
|
146
170
|
|
|
@@ -153,22 +177,22 @@ export function writer({ sharedState, sharedBuffer }, { yield: onYield, logger }
|
|
|
153
177
|
* @param {number} delay The timeout for Atomics.wait.
|
|
154
178
|
*/
|
|
155
179
|
function _yield(delay) {
|
|
156
|
-
if (yielding) {
|
|
157
|
-
throw new Error('
|
|
180
|
+
if (yielding > 128) {
|
|
181
|
+
throw new Error('Detected possible deadlock: writer yielding too many times')
|
|
158
182
|
}
|
|
159
183
|
|
|
160
184
|
// First, ensure the very latest write position is visible to the reader.
|
|
161
185
|
_flush()
|
|
162
186
|
|
|
163
187
|
if (onYield) {
|
|
164
|
-
yielding
|
|
188
|
+
yielding += 1
|
|
165
189
|
try {
|
|
166
190
|
// Call the user-provided yield function, if any. This can be important
|
|
167
191
|
// if the writer is waiting for the reader to process data which would
|
|
168
192
|
// otherwise deadlock.
|
|
169
193
|
onYield()
|
|
170
194
|
} finally {
|
|
171
|
-
yielding
|
|
195
|
+
yielding -= 1
|
|
172
196
|
}
|
|
173
197
|
}
|
|
174
198
|
|
|
@@ -176,6 +200,8 @@ export function writer({ sharedState, sharedBuffer }, { yield: onYield, logger }
|
|
|
176
200
|
// to sleep, consuming no CPU, until the reader changes the READ_INDEX.
|
|
177
201
|
if (delay > 0) {
|
|
178
202
|
Atomics.wait(state, READ_INDEX, readPos, delay)
|
|
203
|
+
} else {
|
|
204
|
+
Atomics.pause()
|
|
179
205
|
}
|
|
180
206
|
|
|
181
207
|
// After waking up, refresh the local view of the reader's position.
|
|
@@ -192,24 +218,22 @@ export function writer({ sharedState, sharedBuffer }, { yield: onYield, logger }
|
|
|
192
218
|
// 4-byte header for the *next* message (for wrap-around check).
|
|
193
219
|
const required = len + 4 + 4
|
|
194
220
|
|
|
195
|
-
if (required < 0) {
|
|
196
|
-
throw new Error(`Required length ${required} is negative, expected at least 0`)
|
|
197
|
-
}
|
|
198
|
-
if (required > size) {
|
|
199
|
-
throw new Error(`Required length ${required} exceeds buffer size ${size}`)
|
|
200
|
-
}
|
|
201
|
-
|
|
202
221
|
if (writePos >= readPos) {
|
|
203
|
-
// Case 1: The writer is ahead of the reader. [ 0
|
|
204
|
-
// There is free space from W to the end (
|
|
222
|
+
// Case 1: The writer is ahead of the reader. [ 0 - R ... W - size ]
|
|
223
|
+
// There is free space from W to the end (s) and from 0 to R.
|
|
224
|
+
|
|
205
225
|
if (size - writePos >= required) {
|
|
206
226
|
// Enough space at the end of the buffer.
|
|
207
227
|
return true
|
|
208
228
|
}
|
|
209
229
|
|
|
230
|
+
if (readPos === 0) {
|
|
231
|
+
readPos = Atomics.load(state, READ_INDEX) | 0
|
|
232
|
+
}
|
|
233
|
+
|
|
210
234
|
// Not enough space at the end. Check if there's space at the beginning.
|
|
211
235
|
if (readPos === 0) {
|
|
212
|
-
// Reader is at the
|
|
236
|
+
// Reader is at the beginning, so no space to wrap around into.
|
|
213
237
|
return false
|
|
214
238
|
}
|
|
215
239
|
|
|
@@ -220,17 +244,24 @@ export function writer({ sharedState, sharedBuffer }, { yield: onYield, logger }
|
|
|
220
244
|
writePos = 0
|
|
221
245
|
|
|
222
246
|
if (writePos + 4 > size) {
|
|
247
|
+
// assertion
|
|
223
248
|
throw new Error(`Write position ${writePos} with next header exceeds buffer size ${size}`)
|
|
224
249
|
}
|
|
225
250
|
if (writePos === readPos) {
|
|
251
|
+
// assertion
|
|
226
252
|
throw new Error(`Write position ${writePos} cannot equal read position ${readPos}`)
|
|
227
253
|
}
|
|
228
254
|
|
|
229
255
|
Atomics.store(state, WRITE_INDEX, writePos)
|
|
230
256
|
}
|
|
231
257
|
|
|
232
|
-
// Case 2: The writer has wrapped around. [ 0 ... W
|
|
258
|
+
// Case 2: The writer has wrapped around. [ 0 ... W - R ... s ]
|
|
233
259
|
// The only free space is between W and R.
|
|
260
|
+
|
|
261
|
+
if (readPos - writePos < required) {
|
|
262
|
+
readPos = Atomics.load(state, READ_INDEX) | 0
|
|
263
|
+
}
|
|
264
|
+
|
|
234
265
|
return readPos - writePos >= required
|
|
235
266
|
}
|
|
236
267
|
|
|
@@ -245,84 +276,114 @@ export function writer({ sharedState, sharedBuffer }, { yield: onYield, logger }
|
|
|
245
276
|
}
|
|
246
277
|
}
|
|
247
278
|
|
|
279
|
+
function _flush() {
|
|
280
|
+
if (pending > 0) {
|
|
281
|
+
Atomics.store(state, WRITE_INDEX, writePos)
|
|
282
|
+
pending = 0
|
|
283
|
+
}
|
|
284
|
+
}
|
|
285
|
+
|
|
248
286
|
/**
|
|
249
287
|
* Performs the actual write into the buffer after space has been acquired.
|
|
250
|
-
* @param {number}
|
|
288
|
+
* @param {number} dataCap Max length of the payload.
|
|
251
289
|
* @param {({ buffer, view, offset, length }) => number} fn The callback that writes the data.
|
|
252
290
|
* @returns {void}
|
|
253
291
|
*/
|
|
254
|
-
function _write(
|
|
292
|
+
function _write(dataCap, fn) {
|
|
255
293
|
const dataPos = writePos + 4
|
|
256
294
|
|
|
257
295
|
data.offset = dataPos
|
|
258
|
-
data.length =
|
|
296
|
+
data.length = dataCap
|
|
259
297
|
|
|
260
298
|
// The user-provided function writes the data and returns the final position.
|
|
261
299
|
// We calculate the actual bytes written from that.
|
|
300
|
+
// NOTE: This is unsafe as the user function can write beyond the reserved length.
|
|
262
301
|
const dataLen = fn(data) - dataPos
|
|
263
302
|
|
|
303
|
+
if (typeof dataLen !== 'number') {
|
|
304
|
+
throw new TypeError('"fn" must return the number of bytes written')
|
|
305
|
+
}
|
|
264
306
|
if (dataLen < 0) {
|
|
265
|
-
throw new
|
|
307
|
+
throw new RangeError(`"fn" returned a negative number ${dataLen}`)
|
|
266
308
|
}
|
|
267
|
-
if (dataLen >
|
|
268
|
-
throw new
|
|
309
|
+
if (dataLen > dataCap) {
|
|
310
|
+
throw new RangeError(`"fn" returned a number ${dataLen} that exceeds capacity ${dataCap}`)
|
|
269
311
|
}
|
|
312
|
+
|
|
270
313
|
if (dataPos + dataLen > size) {
|
|
314
|
+
// assertion
|
|
271
315
|
throw new Error(`Data position ${dataPos} with length ${dataLen} exceeds buffer size ${size}`)
|
|
272
316
|
}
|
|
273
317
|
|
|
274
|
-
|
|
275
|
-
view.setInt32(writePos, dataLen, true)
|
|
276
|
-
writePos += 4 + dataLen
|
|
318
|
+
const nextPos = writePos + 4 + dataLen
|
|
277
319
|
|
|
278
|
-
if (
|
|
279
|
-
|
|
320
|
+
if (nextPos + 4 > size) {
|
|
321
|
+
// assertion
|
|
322
|
+
throw new Error(`Write position ${nextPos} with next header exceeds buffer size ${size}`)
|
|
280
323
|
}
|
|
281
|
-
if (
|
|
282
|
-
|
|
324
|
+
if (nextPos === readPos) {
|
|
325
|
+
// assertion
|
|
326
|
+
throw new Error(`Write position ${nextPos} cannot equal read position ${readPos}`)
|
|
283
327
|
}
|
|
284
328
|
|
|
329
|
+
// Write the actual length of the data into the 4-byte header.
|
|
330
|
+
view.setInt32(writePos, dataLen, true)
|
|
331
|
+
writePos += 4 + dataLen
|
|
332
|
+
pending += 4 + dataLen
|
|
333
|
+
|
|
285
334
|
// This is the "corking" optimization. Instead of calling Atomics.store
|
|
286
335
|
// on every write, we batch them. We either write when a certain
|
|
287
336
|
// amount of data is pending (HWM_BYTES) or at the end of the current
|
|
288
|
-
//
|
|
337
|
+
// event loop tick. This drastically reduces atomic operation overhead.
|
|
289
338
|
if (pending >= HWM_BYTES) {
|
|
290
339
|
Atomics.store(state, WRITE_INDEX, writePos)
|
|
291
340
|
pending = 0
|
|
292
|
-
} else {
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
corked += 1
|
|
296
|
-
setImmediate(_uncork)
|
|
297
|
-
}
|
|
341
|
+
} else if (corked === 0) {
|
|
342
|
+
corked += 1
|
|
343
|
+
setImmediate(_uncork)
|
|
298
344
|
}
|
|
299
345
|
}
|
|
300
346
|
|
|
301
347
|
/**
|
|
302
|
-
* Public write method. Acquires space and writes data.
|
|
348
|
+
* Public write method. Acquires space and synchronously writes data with a timeout. Will
|
|
349
|
+
* wait until space is available.
|
|
350
|
+
* Writing more than "len" bytes in the callback will cause undefined behavior.
|
|
303
351
|
* @param {number} len The maximum expected length of the payload.
|
|
304
352
|
* @param {({ buffer, view, offset, length }) => number} fn The callback that writes the data.
|
|
353
|
+
* @param {number} [timeout=60000] The maximum time to wait for space in milliseconds.
|
|
305
354
|
*/
|
|
306
|
-
function writeSync(len, fn) {
|
|
355
|
+
function writeSync(len, fn, timeout = 60e3) {
|
|
356
|
+
if (typeof len !== 'number') {
|
|
357
|
+
throw new TypeError('"len" must be a non-negative number')
|
|
358
|
+
}
|
|
307
359
|
if (len < 0) {
|
|
308
|
-
throw new
|
|
360
|
+
throw new RangeError(`"len" ${len} is negative`)
|
|
309
361
|
}
|
|
310
362
|
if (len >= 2 ** 31 || len > size - 8) {
|
|
311
|
-
throw new Error(`
|
|
363
|
+
throw new Error(`"len" ${len} exceeds maximum allowed size ${size - 8}`)
|
|
364
|
+
}
|
|
365
|
+
if (typeof fn !== 'function') {
|
|
366
|
+
throw new TypeError('"fn" must be a function')
|
|
367
|
+
}
|
|
368
|
+
if (typeof timeout !== 'number') {
|
|
369
|
+
throw new TypeError('"timeout" must be a non-negative number')
|
|
370
|
+
}
|
|
371
|
+
if (timeout < 0) {
|
|
372
|
+
throw new RangeError('"timeout" must be a non-negative number')
|
|
312
373
|
}
|
|
313
374
|
|
|
314
375
|
if (!_acquire(len)) {
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
_yield(3)
|
|
376
|
+
const startTime = performance.now()
|
|
377
|
+
logger?.warn({ readPos, writePos }, 'yield started')
|
|
378
|
+
_yield(0)
|
|
379
|
+
for (let n = 0; !_acquire(len); n++) {
|
|
380
|
+
if (performance.now() - startTime > timeout) {
|
|
381
|
+
throw new Error('Timeout while waiting for space in the buffer')
|
|
322
382
|
}
|
|
323
|
-
|
|
324
|
-
logger?.warn({ readPos, writePos, elapsedTime }, 'yield completed')
|
|
383
|
+
_yield(3)
|
|
325
384
|
}
|
|
385
|
+
const elapsedTime = performance.now() - startTime
|
|
386
|
+
logger?.warn({ readPos, writePos, elapsedTime }, 'yield completed')
|
|
326
387
|
}
|
|
327
388
|
|
|
328
389
|
_write(len, fn)
|
|
@@ -332,19 +393,28 @@ export function writer({ sharedState, sharedBuffer }, { yield: onYield, logger }
|
|
|
332
393
|
}
|
|
333
394
|
}
|
|
334
395
|
|
|
396
|
+
/**
|
|
397
|
+
* Public write method. Acquires space and tries to writes data.
|
|
398
|
+
* Writing more than "len" bytes in the callback will cause undefined behavior.
|
|
399
|
+
* @param {number} len The maximum expected length of the payload.
|
|
400
|
+
* @param {({ buffer, view, offset, length }) => number} fn The callback that writes the data.
|
|
401
|
+
*/
|
|
335
402
|
function tryWrite(len, fn) {
|
|
403
|
+
if (typeof len !== 'number') {
|
|
404
|
+
throw new TypeError('"len" must be a non-negative number')
|
|
405
|
+
}
|
|
336
406
|
if (len < 0) {
|
|
337
|
-
throw new
|
|
407
|
+
throw new RangeError(`"len" ${len} is negative`)
|
|
338
408
|
}
|
|
339
409
|
if (len >= 2 ** 31 || len > size - 8) {
|
|
340
|
-
throw new Error(`
|
|
410
|
+
throw new Error(`"len" ${len} exceeds maximum allowed size ${size - 8}`)
|
|
411
|
+
}
|
|
412
|
+
if (typeof fn !== 'function') {
|
|
413
|
+
throw new TypeError('"fn" must be a function')
|
|
341
414
|
}
|
|
342
415
|
|
|
343
416
|
if (!_acquire(len)) {
|
|
344
|
-
|
|
345
|
-
if (!_acquire(len)) {
|
|
346
|
-
return false
|
|
347
|
-
}
|
|
417
|
+
return false
|
|
348
418
|
}
|
|
349
419
|
|
|
350
420
|
_write(len, fn)
|
|
@@ -356,13 +426,6 @@ export function writer({ sharedState, sharedBuffer }, { yield: onYield, logger }
|
|
|
356
426
|
return true
|
|
357
427
|
}
|
|
358
428
|
|
|
359
|
-
function _flush() {
|
|
360
|
-
if (pending > 0) {
|
|
361
|
-
Atomics.store(state, WRITE_INDEX, writePos)
|
|
362
|
-
pending = 0
|
|
363
|
-
}
|
|
364
|
-
}
|
|
365
|
-
|
|
366
429
|
function cork(callback) {
|
|
367
430
|
corked += 1
|
|
368
431
|
try {
|
|
@@ -374,118 +437,3 @@ export function writer({ sharedState, sharedBuffer }, { yield: onYield, logger }
|
|
|
374
437
|
|
|
375
438
|
return { tryWrite, writeSync, cork }
|
|
376
439
|
}
|
|
377
|
-
|
|
378
|
-
export class Writable extends stream.Writable {
|
|
379
|
-
#writer
|
|
380
|
-
#retries = 0
|
|
381
|
-
#timeout = null
|
|
382
|
-
|
|
383
|
-
#chunk
|
|
384
|
-
#encoding
|
|
385
|
-
#callback
|
|
386
|
-
|
|
387
|
-
constructor({ state, ...options }) {
|
|
388
|
-
super({ ...options })
|
|
389
|
-
this.#writer = writer(state)
|
|
390
|
-
}
|
|
391
|
-
|
|
392
|
-
_write(chunk, encoding, callback) {
|
|
393
|
-
if (chunk.byteLength === 0) {
|
|
394
|
-
callback(null)
|
|
395
|
-
return
|
|
396
|
-
}
|
|
397
|
-
|
|
398
|
-
assert(!this.#timeout)
|
|
399
|
-
|
|
400
|
-
this.#chunk = chunk
|
|
401
|
-
this.#encoding = encoding
|
|
402
|
-
this.#callback = callback
|
|
403
|
-
this._writeSome()
|
|
404
|
-
}
|
|
405
|
-
|
|
406
|
-
_final(callback) {
|
|
407
|
-
this.#chunk = Buffer.allocUnsafe(0)
|
|
408
|
-
this.#encoding = null
|
|
409
|
-
this.#callback = callback
|
|
410
|
-
this._writeSome()
|
|
411
|
-
}
|
|
412
|
-
|
|
413
|
-
_writeSome = () => {
|
|
414
|
-
this.#timeout = null
|
|
415
|
-
|
|
416
|
-
if (this.#writer.tryWrite(this.#chunk.byteLength, this._doWrite)) {
|
|
417
|
-
const callback = this.#callback
|
|
418
|
-
this.#retries = 0
|
|
419
|
-
this.#chunk = null
|
|
420
|
-
this.#encoding = null
|
|
421
|
-
this.#callback = null
|
|
422
|
-
callback(null)
|
|
423
|
-
} else {
|
|
424
|
-
this.#retries += 1
|
|
425
|
-
this.#timeout = setTimeout(this._writeSome, Math.min(10, this.#retries))
|
|
426
|
-
}
|
|
427
|
-
}
|
|
428
|
-
|
|
429
|
-
_doWrite = (data) => {
|
|
430
|
-
const written =
|
|
431
|
-
typeof this.#chunk === 'string'
|
|
432
|
-
? data.buffer.write(this.#chunk, data.offset, data.length, this.#encoding)
|
|
433
|
-
: this.#chunk.copy(data.buffer, data.offset, 0, this.#chunk.length)
|
|
434
|
-
return data.offset + written
|
|
435
|
-
}
|
|
436
|
-
|
|
437
|
-
_destroy(err, callback) {
|
|
438
|
-
if (this.#timeout != null) {
|
|
439
|
-
clearTimeout(this.#timeout)
|
|
440
|
-
this.#timeout = null
|
|
441
|
-
}
|
|
442
|
-
callback(err)
|
|
443
|
-
}
|
|
444
|
-
}
|
|
445
|
-
|
|
446
|
-
export class Readable extends stream.Readable {
|
|
447
|
-
#reader
|
|
448
|
-
#retries = 0
|
|
449
|
-
#timeout
|
|
450
|
-
|
|
451
|
-
constructor({ state, ...options }) {
|
|
452
|
-
super(options)
|
|
453
|
-
this.#reader = reader(state)
|
|
454
|
-
}
|
|
455
|
-
|
|
456
|
-
_read() {
|
|
457
|
-
assert(!this.#timeout)
|
|
458
|
-
|
|
459
|
-
this._readSome()
|
|
460
|
-
}
|
|
461
|
-
|
|
462
|
-
_readSome = () => {
|
|
463
|
-
this.#timeout = null
|
|
464
|
-
|
|
465
|
-
const count = this.#reader.readSome((data) => {
|
|
466
|
-
if (data.length === 0) {
|
|
467
|
-
this.push(null)
|
|
468
|
-
return false
|
|
469
|
-
}
|
|
470
|
-
|
|
471
|
-
const chunk = Buffer.allocUnsafe(data.length)
|
|
472
|
-
data.buffer.copy(chunk, 0, data.offset, data.offset + data.length)
|
|
473
|
-
return this.push(chunk)
|
|
474
|
-
})
|
|
475
|
-
|
|
476
|
-
if (count > 0 || this.readableEnded) {
|
|
477
|
-
this.#retries = 0
|
|
478
|
-
} else {
|
|
479
|
-
this.#retries += 1
|
|
480
|
-
this.#timeout = setTimeout(this._readSome, Math.min(10, this.#retries))
|
|
481
|
-
}
|
|
482
|
-
}
|
|
483
|
-
|
|
484
|
-
_destroy(err, callback) {
|
|
485
|
-
if (this.#timeout != null) {
|
|
486
|
-
clearTimeout(this.#timeout)
|
|
487
|
-
this.#timeout = null
|
|
488
|
-
}
|
|
489
|
-
callback(err)
|
|
490
|
-
}
|
|
491
|
-
}
|