hyperjoin 0.0.0 → 1.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,160 @@
1
+ const Hypercore = require('hypercore')
2
+ const { getStreamError } = require('streamx')
3
+ const b4a = require('b4a')
4
+ const { toHex } = b4a
5
+
6
+ const CONNECTION_IDLE_TIMEOUT = 3 * 60 * 1000 // 3 Minutes
7
+
8
+ module.exports = class Connection {
9
+ constructor(replicator, stream, peerId, initiator) {
10
+ this.replicator = replicator
11
+ this.initiator = initiator
12
+ this.stream = stream
13
+ this.peerId = peerId
14
+ this._onclose = this._onclose.bind(this)
15
+ this._onchannelopen = this._onchannelopen.bind(this)
16
+ stream.on('close', this._onclose)
17
+
18
+ this.bytesRead = -1
19
+ this.bytesWritten = -1
20
+ this.lastActivity = -1
21
+ this.refresh()
22
+ this._survived = 0 // incremented on gc
23
+
24
+ this.info.markConnected(this.initiator)
25
+
26
+ this._setReplicating = null
27
+ this.replicating = new Promise((resolve) => {
28
+ this._setReplicating = resolve
29
+ })
30
+
31
+ this.destroyed = new Promise((resolve) => {
32
+ if (stream.destroyed) resolve()
33
+ else stream.once('close', resolve)
34
+ })
35
+ }
36
+
37
+ get protomux() {
38
+ return Hypercore.getProtocolMuxer(this.stream)
39
+ }
40
+
41
+ get info() {
42
+ return this.replicator.dialer.lookup(this.peerId)
43
+ }
44
+
45
+ get connectedAt() {
46
+ return this.info.connectedAt
47
+ }
48
+
49
+ isIdle(now = Date.now()) {
50
+ // Prevent GC before replicating upto 6s
51
+ if (this._setReplicating && now - this.connectedAt < 6000) return false
52
+
53
+ // TODO: Needs a fix see test/gc-connection.js
54
+ // if (this._protomuxSessions() === 0) return true
55
+
56
+ // no traffic observed
57
+ if (now - this.lastActivity > CONNECTION_IDLE_TIMEOUT) return true
58
+
59
+ // all-channels closed on local & remote
60
+ if (this.protomux.isIdle()) return true
61
+
62
+ return false
63
+ }
64
+
65
+ // @deprecated & unused - kept for reference
66
+ // see test/gc-connection.js
67
+ _protomuxSessions() {
68
+ const mux = this.protomux
69
+
70
+ let opened = 0
71
+ for (const ch of mux) {
72
+ if (ch.protocol !== 'hypercore/alpha') continue
73
+
74
+ // BUG! need to know if ch.opened on remote
75
+ // all local known channels are eagerly opened
76
+ // on corestore.replicate() and never closed
77
+ // or marked as in-use.
78
+
79
+ const dkeystr = toHex(ch.id)
80
+
81
+ for (const session of this.replicator.sessions.values()) {
82
+ if (!session.active) continue
83
+
84
+ if (session.hasCore(dkeystr)) {
85
+ opened++
86
+ break
87
+ }
88
+ }
89
+ }
90
+
91
+ return opened
92
+ }
93
+
94
+ _onclose() {
95
+ const rep = this.replicator
96
+
97
+ const err = getStreamError(this.stream)
98
+
99
+ const key = toHex(this.peerId)
100
+ if (this.initiator) {
101
+ rep.outbound.delete(key)
102
+ rep.stats.connects.client.closed++
103
+ } else {
104
+ rep.inbound.delete(key)
105
+ rep.stats.connects.server.closed++
106
+ }
107
+
108
+ if (this._setReplicating) {
109
+ this._setReplicating() // settle
110
+ this._setReplicating = null
111
+ }
112
+
113
+ this.info.markDisconnected(this.initiator, err)
114
+ rep.dialer.reschedule(this.info)
115
+ rep.dialer.refresh() // reschedule fast-reconnects
116
+
117
+ rep.emit('disconnected', this)
118
+ rep.emit('update')
119
+ }
120
+
121
+ _watchChannels() {
122
+ this.protomux.watch({ protocol: 'hypercore/alpha' }, this._onchannelopen)
123
+ }
124
+
125
+ _onchannelopen() {
126
+ try {
127
+ if (this._setReplicating) {
128
+ this._setReplicating()
129
+ this._setReplicating = null
130
+ }
131
+
132
+ this.protomux.unwatch({ protocol: 'hypercore/alpha' }, this._onchannelopen)
133
+ } catch (err) {
134
+ // noop
135
+ }
136
+ }
137
+
138
+ refresh(now = Date.now()) {
139
+ // secret-stream bytes written;
140
+ // should exclude heartbeat
141
+ if (
142
+ this.bytesRead === this.stream.rawBytesRead &&
143
+ this.bytesWritten === this.stream.rawBytesWritten
144
+ ) {
145
+ return
146
+ }
147
+
148
+ this.bytesRead = this.stream.rawBytesRead
149
+ this.bytesWritten = this.stream.rawBytesWritten
150
+ this.lastActivity = now
151
+ }
152
+
153
+ get destroying() {
154
+ return this.stream.destroying
155
+ }
156
+
157
+ destroy(err) {
158
+ this.stream.destroy(err)
159
+ }
160
+ }
package/lib/dialer.js ADDED
@@ -0,0 +1,370 @@
1
+ const { toHex } = require('b4a')
2
+ const Peer = require('./peer')
3
+ const OrderedSet = require('./ordered-set')
4
+
5
+ const MAX_PARALLEL = 3
6
+ const MIN_REFRESH_INTERVAL = 1000
7
+
8
+ class Dialer {
9
+ constructor(handlers, opts = {}) {
10
+ const {
11
+ ondial = async (peer) => {},
12
+ isConnected = (hex) => false,
13
+ want = () => Infinity,
14
+ canRelay = () => false,
15
+ onban = (peer) => {},
16
+ ondialerror = (err) => {
17
+ throw err
18
+ }
19
+ } = handlers
20
+
21
+ this._ondial = ondial
22
+ this._ondialerror = ondialerror
23
+ this._isConnected = isConnected
24
+ this._canRelay = canRelay
25
+ this._onban = onban
26
+ this._want = want
27
+
28
+ const { suspended = false, maxParallel = MAX_PARALLEL, jitter = 500 } = opts
29
+
30
+ this._activeTopics = new Set()
31
+
32
+ this.jitter = jitter
33
+
34
+ this.peers = new Map() // hex(peerId) -> Peer
35
+ this._orderedPeers = new OrderedSet((peer) => peer._nextAt)
36
+ this.suspended = suspended
37
+ this.destroyed = false
38
+
39
+ this._slots = new Semaphore(maxParallel)
40
+
41
+ // cursors
42
+ this._scheduled = -1
43
+ this._processed = -1
44
+ this._flushreqs = []
45
+ this._inflightWork = new Set()
46
+ this._externallyThrottled = false
47
+
48
+ if (!this.suspended) this._start()
49
+ }
50
+
51
+ get inflight() {
52
+ return this._inflightWork.size
53
+ }
54
+
55
+ reschedule(peer) {
56
+ if (!this._orderedPeers.has(peer)) return
57
+
58
+ this._orderedPeers.add(peer)
59
+
60
+ if (!peer.isBanned || peer._banReported) return
61
+
62
+ peer._banReported = true
63
+ this._onban(peer)
64
+ }
65
+
66
+ lookup(publicKey, relayAddresses) {
67
+ const key = toHex(publicKey)
68
+
69
+ let peer = this.peers.get(key)
70
+ if (!peer) {
71
+ peer = new Peer(publicKey)
72
+ this.peers.set(key, peer)
73
+ this._orderedPeers.add(peer)
74
+ }
75
+
76
+ // new is always better
77
+ if (relayAddresses && !peer.isBanned) peer.updateRelays(relayAddresses)
78
+
79
+ return peer
80
+ }
81
+
82
+ seen(dhtPeer, topic, now = Date.now()) {
83
+ const { publicKey, relayAddresses } = dhtPeer
84
+ const peer = this.lookup(publicKey, relayAddresses)
85
+
86
+ peer.observe(topic, now)
87
+ this.reschedule(peer)
88
+ this.refresh(now)
89
+
90
+ return peer
91
+ }
92
+
93
+ _start() {
94
+ this.refresh()
95
+ }
96
+
97
+ async flush(now = Date.now()) {
98
+ const inflight = Promise.all(this._inflightWork)
99
+
100
+ // dialer is externally held; resolve when inflights complete
101
+ if (this._externallyThrottled) {
102
+ return inflight
103
+ }
104
+
105
+ let target = -1
106
+
107
+ // TODO: O(1)
108
+ for (const peer of this.peers.values()) {
109
+ if (!this._isEligable(peer, now)) continue
110
+
111
+ if (peer.retryAt > target) target = peer.retryAt
112
+ }
113
+
114
+ if (target === -1) return inflight
115
+
116
+ if (this._flushreqs.length) {
117
+ const { target: prevTarget, promise } = this._flushreqs[this._flushreqs.length - 1]
118
+
119
+ // de-dupe prior work
120
+ if (target <= prevTarget) return promise
121
+ }
122
+
123
+ if (target <= this._processed) return inflight // no work, no-op
124
+
125
+ // create snapshot of known work
126
+ let done = null
127
+ const promise = new Promise((resolve) => {
128
+ done = resolve
129
+ }).then(() => inflight)
130
+
131
+ this._flushreqs.push({ target, promise, done })
132
+
133
+ return promise
134
+ }
135
+
136
+ _settleFlush(upto) {
137
+ this._processed = Math.max(this._processed, upto)
138
+
139
+ while (this._flushreqs.length) {
140
+ const { target } = this._flushreqs[0]
141
+ if (target > this._processed) break
142
+
143
+ // defer to last
144
+ if (target === this._processed) {
145
+ let n = 0
146
+
147
+ for (const peer of this.peers.values()) {
148
+ if (!this._isEligable(peer, this._processed)) continue
149
+
150
+ if (peer.retryAt === this._processed) n++
151
+ }
152
+
153
+ if (n) return
154
+ }
155
+
156
+ // settle pending flush
157
+ const { done } = this._flushreqs.shift()
158
+ done()
159
+ }
160
+ }
161
+
162
+ refresh(now = Date.now()) {
163
+ if (this._refreshTimer) clearTimeout(this._refreshTimer)
164
+ this._refreshTimer = null
165
+
166
+ if (this.suspended) return false
167
+ if (!this.peers.size) return false // queue empty
168
+
169
+ // future|cancellable work is not scheduled
170
+ const due = this._orderedPeers.upTo(now)
171
+
172
+ // collect garbage
173
+ for (const peer of due) {
174
+ if (!peer.isExpired(now)) continue
175
+
176
+ this.peers.delete(toHex(peer.publicKey))
177
+ this._orderedPeers.delete(peer)
178
+ }
179
+
180
+ const wanted = this._want()
181
+
182
+ for (const peer of due) {
183
+ // max inflight reached
184
+ if (this._slots.isFull) {
185
+ break
186
+ }
187
+
188
+ // max capacity achieved
189
+ if (wanted - this._inflightWork.size < 1) {
190
+ this._externallyThrottled = true
191
+
192
+ // TODO: re-evaulte Dialer.flush() use and name.
193
+ // Replicator.flush() seems to be only used by test-suites
194
+
195
+ // dump all flush requests
196
+ const inflight = Promise.all(this._inflightWork)
197
+ while (this._flushreqs.length) {
198
+ this._flushreqs.shift().done(inflight)
199
+ }
200
+
201
+ break
202
+ }
203
+ this._externallyThrottled = false
204
+
205
+ // not eligable
206
+ if (!this._isEligable(peer, now)) continue
207
+
208
+ // already queued
209
+ const previousVisit = peer._refreshedAt
210
+ peer._refreshedAt = now
211
+ if (previousVisit >= peer._nextAt) continue
212
+
213
+ // dial!
214
+ const startedAt = Date.now()
215
+ const attempt = this._dial(peer, now)
216
+ .catch(this._ondialerror)
217
+ .finally(() => {
218
+ this._inflightWork.delete(attempt)
219
+ this.refresh(now + (Date.now() - startedAt))
220
+ })
221
+
222
+ this._inflightWork.add(attempt)
223
+ }
224
+
225
+ this._scheduleRefresh() // auto-refresh
226
+ }
227
+
228
+ _isEligable(peer, now) {
229
+ if (this.suspended) return false
230
+
231
+ // already connected
232
+ if (this._isConnected(peer.publicKey)) return false
233
+
234
+ if (!this._hasActiveTopic(peer)) return false
235
+
236
+ return peer.shouldDial(now)
237
+ }
238
+
239
+ _hasActiveTopic(peer) {
240
+ for (const hex of peer._topics.keys()) {
241
+ if (this._activeTopics.has(hex)) return true
242
+ }
243
+
244
+ return false
245
+ }
246
+
247
+ async _dial(peer, now = Date.now()) {
248
+ const waitStart = Date.now()
249
+ await this._slots.acquire() // MAX_PARALLEL
250
+
251
+ if (this.jitter > 0) {
252
+ await new Promise((resolve) => setTimeout(resolve, Math.random() * this.jitter))
253
+ }
254
+
255
+ let postWait = now + (Date.now() - waitStart)
256
+
257
+ const { _nextAt: nextAt } = peer
258
+
259
+ const actionComplete = () => {
260
+ this._settleFlush(nextAt)
261
+ this._slots.release()
262
+ }
263
+
264
+ if (!this._isEligable(peer, postWait) || this._want() < 1) {
265
+ peer._refreshedAt = -1 // let refresh() re-process
266
+ actionComplete()
267
+ return
268
+ }
269
+
270
+ const attemptDone = peer.markAttempt(true, { canRelay: this._canRelay() })
271
+ this.reschedule(peer)
272
+
273
+ try {
274
+ const dialFailed = await this._ondial(peer, postWait)
275
+ postWait = now + (Date.now() - waitStart)
276
+ attemptDone(!!dialFailed, dialFailed, postWait)
277
+ this.reschedule(peer)
278
+ } catch (err) {
279
+ attemptDone(true)
280
+ this.reschedule(peer)
281
+ this._ondialerror(err)
282
+ }
283
+
284
+ postWait = now + (Date.now() - waitStart)
285
+
286
+ actionComplete()
287
+ }
288
+
289
+ _scheduleRefresh(now = Date.now()) {
290
+ if (this.suspended) return
291
+ if (this._refreshTimer) return // already scheduled
292
+
293
+ // earliest peer that is not yet due
294
+ const peer = this._orderedPeers.after(now)
295
+ if (!peer) return
296
+
297
+ const nextAt = peer._nextAt
298
+ const delay = Math.min(nextAt - now, MIN_REFRESH_INTERVAL)
299
+
300
+ this._refreshTimer = setTimeout(() => this.refresh(), delay)
301
+
302
+ this._scheduled = nextAt
303
+ }
304
+
305
+ suspend() {
306
+ this.suspended = true
307
+ const suspended = this.flush()
308
+
309
+ if (this._refreshTimer) clearTimeout(this._refreshTimer)
310
+ this._refreshTimer = null
311
+
312
+ return suspended
313
+ }
314
+
315
+ resume(now = Date.now()) {
316
+ this.suspended = false
317
+ this.refresh(now)
318
+ }
319
+
320
+ resetBackoff(since = 0) {
321
+ for (const peer of this.peers.values()) {
322
+ if (Math.max(peer.attemptedAt, peer.disconnectedAt) < since) continue
323
+
324
+ peer.resetAttempts()
325
+ this.reschedule(peer)
326
+ }
327
+ this.refresh()
328
+ }
329
+
330
+ topicEnable(topic) {
331
+ if (typeof topic !== 'string') topic = toHex(topic)
332
+ this._activeTopics.add(topic)
333
+ }
334
+
335
+ topicDisable(topic) {
336
+ if (typeof topic !== 'string') topic = toHex(topic)
337
+ this._activeTopics.delete(topic)
338
+ }
339
+
340
+ destroy() {
341
+ this.suspend()
342
+ }
343
+ }
344
+
345
+ class Semaphore {
346
+ constructor(permits = 3) {
347
+ this.n = permits
348
+ this.queue = []
349
+ }
350
+
351
+ get isFull() {
352
+ return this.queue.length >= this.n
353
+ }
354
+
355
+ acquire() {
356
+ if (this.n > 0) {
357
+ this.n--
358
+ return Promise.resolve()
359
+ }
360
+ return new Promise((resolve) => this.queue.push(resolve))
361
+ }
362
+
363
+ release() {
364
+ const next = this.queue.shift()
365
+ if (next) next()
366
+ else this.n++
367
+ }
368
+ }
369
+
370
+ module.exports = Dialer
@@ -0,0 +1,40 @@
1
+ // returns sorted array
2
+ // lowest index equals least priority
3
+ function sortConnections(connections, now = Date.now()) {
4
+ return connections.sort(tieredSort)
5
+
6
+ function score(connection, other) {
7
+ let w = 0
8
+
9
+ const peer = connection.info
10
+
11
+ // Keep explicit scans
12
+ // TODO: synchronize with scan lifecycle (V2)
13
+ if (now - peer.scannedAt < 60000) w += 1 << 4
14
+
15
+ // Keep established
16
+ if (connection._survived > 0) w += 1 << 3
17
+
18
+ // Keep recently active
19
+ if (connection.lastActivity > other.lastActivity) w += 1 << 2
20
+
21
+ // Keep oldest
22
+ if (connection.connectedAt < other.connectedAt) w += 1 << 1
23
+
24
+ // TODO: keep
25
+ // - from pinned topic
26
+ // - good throughput
27
+
28
+ return w
29
+ }
30
+
31
+ function tieredSort(a, b) {
32
+ const weightA = score(a, b)
33
+ const weightB = score(b, a)
34
+ return weightA - weightB
35
+ }
36
+ }
37
+
38
+ module.exports = {
39
+ sortConnections
40
+ }
@@ -0,0 +1,102 @@
1
+ // A set kept sorted by `orderOf(item)`, lowest first.
2
+ //
3
+ // Contract:
4
+ // - call add(item) after every write that can change orderOf(item)
5
+ // - call delete(item) to release an item, the set holds its items strongly
6
+ module.exports = class OrderedSet {
7
+ constructor(orderOf = (item) => -1) {
8
+ this._orderOf = orderOf
9
+ this._orders = new WeakMap() // item -> order it is sorted under
10
+ this._sorted = []
11
+ }
12
+
13
+ get size() {
14
+ return this._sorted.length
15
+ }
16
+
17
+ has(item) {
18
+ return this._orders.has(item)
19
+ }
20
+
21
+ // insert, or move to where orderOf(item) now puts it
22
+ add(item) {
23
+ const order = this._orderOf(item)
24
+
25
+ if (this._orders.has(item)) {
26
+ if (this._orders.get(item) === order) return this
27
+ this._sorted.splice(this._indexOf(item), 1)
28
+ }
29
+
30
+ this._orders.set(item, order)
31
+ this._sorted.splice(this._upperBound(order), 0, item)
32
+
33
+ return this
34
+ }
35
+
36
+ delete(item) {
37
+ if (!this._orders.has(item)) return false
38
+
39
+ this._sorted.splice(this._indexOf(item), 1)
40
+ this._orders.delete(item)
41
+
42
+ return true
43
+ }
44
+
45
+ clear() {
46
+ this._orders = new WeakMap()
47
+ this._sorted = []
48
+ }
49
+
50
+ // lowest ordered item
51
+ first() {
52
+ return this._sorted[0]
53
+ }
54
+
55
+ // copy of the items ordered at or below `order`
56
+ upTo(order) {
57
+ return this._sorted.slice(0, this._upperBound(order))
58
+ }
59
+
60
+ // first item ordered above `order`
61
+ after(order) {
62
+ return this._sorted[this._upperBound(order)]
63
+ }
64
+
65
+ [Symbol.iterator]() {
66
+ return this._sorted[Symbol.iterator]()
67
+ }
68
+
69
+ _indexOf(item) {
70
+ let i = this._lowerBound(this._orders.get(item))
71
+ while (this._sorted[i] !== item) i++
72
+ return i
73
+ }
74
+
75
+ // first index ordered at or above `order`
76
+ _lowerBound(order) {
77
+ let lo = 0
78
+ let hi = this._sorted.length
79
+
80
+ while (lo < hi) {
81
+ const mid = (lo + hi) >>> 1
82
+ if (this._orders.get(this._sorted[mid]) < order) lo = mid + 1
83
+ else hi = mid
84
+ }
85
+
86
+ return lo
87
+ }
88
+
89
+ // first index ordered above `order`
90
+ _upperBound(order) {
91
+ let lo = 0
92
+ let hi = this._sorted.length
93
+
94
+ while (lo < hi) {
95
+ const mid = (lo + hi) >>> 1
96
+ if (this._orders.get(this._sorted[mid]) <= order) lo = mid + 1
97
+ else hi = mid
98
+ }
99
+
100
+ return lo
101
+ }
102
+ }