@helia/bitswap 1.0.0 → 1.0.1-0a528bb
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.min.js +1 -1
- package/dist/src/constants.d.ts +2 -0
- package/dist/src/constants.d.ts.map +1 -1
- package/dist/src/constants.js +2 -0
- package/dist/src/constants.js.map +1 -1
- package/dist/src/index.d.ts +20 -7
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js.map +1 -1
- package/dist/src/network.d.ts +7 -4
- package/dist/src/network.d.ts.map +1 -1
- package/dist/src/network.js +55 -131
- package/dist/src/network.js.map +1 -1
- package/dist/src/pb/message.d.ts +6 -6
- package/dist/src/pb/message.d.ts.map +1 -1
- package/dist/src/pb/message.js +37 -20
- package/dist/src/pb/message.js.map +1 -1
- package/dist/src/peer-want-lists/index.d.ts +3 -1
- package/dist/src/peer-want-lists/index.d.ts.map +1 -1
- package/dist/src/peer-want-lists/index.js +6 -2
- package/dist/src/peer-want-lists/index.js.map +1 -1
- package/dist/src/peer-want-lists/ledger.d.ts +3 -1
- package/dist/src/peer-want-lists/ledger.d.ts.map +1 -1
- package/dist/src/peer-want-lists/ledger.js +8 -0
- package/dist/src/peer-want-lists/ledger.js.map +1 -1
- package/dist/src/stats.d.ts +2 -1
- package/dist/src/stats.d.ts.map +1 -1
- package/dist/src/stats.js +4 -4
- package/dist/src/stats.js.map +1 -1
- package/dist/src/utils/merge-messages.d.ts +3 -0
- package/dist/src/utils/merge-messages.d.ts.map +1 -0
- package/dist/src/utils/merge-messages.js +51 -0
- package/dist/src/utils/merge-messages.js.map +1 -0
- package/dist/src/utils/split-message.d.ts +18 -0
- package/dist/src/utils/split-message.d.ts.map +1 -0
- package/dist/src/utils/split-message.js +101 -0
- package/dist/src/utils/split-message.js.map +1 -0
- package/dist/src/want-list.d.ts +2 -1
- package/dist/src/want-list.d.ts.map +1 -1
- package/dist/src/want-list.js +8 -5
- package/dist/src/want-list.js.map +1 -1
- package/package.json +4 -4
- package/src/constants.ts +2 -0
- package/src/index.ts +22 -8
- package/src/network.ts +65 -154
- package/src/pb/message.ts +40 -20
- package/src/peer-want-lists/index.ts +8 -3
- package/src/peer-want-lists/ledger.ts +11 -1
- package/src/stats.ts +6 -5
- package/src/utils/merge-messages.ts +70 -0
- package/src/utils/split-message.ts +133 -0
- package/src/want-list.ts +12 -6
- package/dist/typedoc-urls.json +0 -20
package/src/index.ts
CHANGED
|
@@ -117,14 +117,6 @@ export interface BitswapOptions {
|
|
|
117
117
|
*/
|
|
118
118
|
protocol?: string
|
|
119
119
|
|
|
120
|
-
/**
|
|
121
|
-
* When a new peer connects, sending our WantList should complete within this
|
|
122
|
-
* many ms
|
|
123
|
-
*
|
|
124
|
-
* @default 5000
|
|
125
|
-
*/
|
|
126
|
-
messageSendTimeout?: number
|
|
127
|
-
|
|
128
120
|
/**
|
|
129
121
|
* When sending want list updates to peers, how many messages to send at once
|
|
130
122
|
*
|
|
@@ -167,6 +159,28 @@ export interface BitswapOptions {
|
|
|
167
159
|
* @default 1024
|
|
168
160
|
*/
|
|
169
161
|
maxSizeReplaceHasWithBlock?: number
|
|
162
|
+
|
|
163
|
+
/**
|
|
164
|
+
* The maximum size in bytes of a message that we will send. If a message is
|
|
165
|
+
* larger than this (due to lots of blocks or wantlist entries) it will be
|
|
166
|
+
* broken up into several smaller messages that are under this size.
|
|
167
|
+
*
|
|
168
|
+
* @see https://github.com/ipfs/boxo/blob/eeea414587350401b6b804f0574ed8436833331d/bitswap/client/internal/messagequeue/messagequeue.go#L33
|
|
169
|
+
*
|
|
170
|
+
* @default 2097152
|
|
171
|
+
*/
|
|
172
|
+
maxOutgoingMessageSize?: number
|
|
173
|
+
|
|
174
|
+
/**
|
|
175
|
+
* The maximum size in bytes of an incoming message that we will process.
|
|
176
|
+
*
|
|
177
|
+
* Messages larger than this will cause the incoming stream to be reset.
|
|
178
|
+
*
|
|
179
|
+
* Defaults to `maxOutgoingMessageSize`
|
|
180
|
+
*
|
|
181
|
+
* @default 2097152
|
|
182
|
+
*/
|
|
183
|
+
maxIncomingMessageSize?: number
|
|
170
184
|
}
|
|
171
185
|
|
|
172
186
|
export const createBitswap = (components: BitswapComponents, options: BitswapOptions = {}): Bitswap => {
|
package/src/network.ts
CHANGED
|
@@ -1,55 +1,25 @@
|
|
|
1
1
|
import { CodeError, TypedEventEmitter, setMaxListeners } from '@libp2p/interface'
|
|
2
2
|
import { PeerQueue, type PeerQueueJobOptions } from '@libp2p/utils/peer-queue'
|
|
3
|
-
import { anySignal } from 'any-signal'
|
|
4
|
-
import debug from 'debug'
|
|
5
3
|
import drain from 'it-drain'
|
|
6
4
|
import * as lp from 'it-length-prefixed'
|
|
7
|
-
import { lpStream } from 'it-length-prefixed-stream'
|
|
8
5
|
import map from 'it-map'
|
|
9
6
|
import { pipe } from 'it-pipe'
|
|
10
7
|
import take from 'it-take'
|
|
11
|
-
import { base64 } from 'multiformats/bases/base64'
|
|
12
|
-
import { CID } from 'multiformats/cid'
|
|
13
8
|
import { CustomProgressEvent } from 'progress-events'
|
|
14
9
|
import { raceEvent } from 'race-event'
|
|
15
|
-
import {
|
|
16
|
-
import { BITSWAP_120, DEFAULT_MAX_INBOUND_STREAMS, DEFAULT_MAX_OUTBOUND_STREAMS, DEFAULT_MAX_PROVIDERS_PER_REQUEST, DEFAULT_MESSAGE_RECEIVE_TIMEOUT, DEFAULT_MESSAGE_SEND_CONCURRENCY, DEFAULT_MESSAGE_SEND_TIMEOUT, DEFAULT_RUN_ON_TRANSIENT_CONNECTIONS } from './constants.js'
|
|
10
|
+
import { BITSWAP_120, DEFAULT_MAX_INBOUND_STREAMS, DEFAULT_MAX_INCOMING_MESSAGE_SIZE, DEFAULT_MAX_OUTBOUND_STREAMS, DEFAULT_MAX_OUTGOING_MESSAGE_SIZE, DEFAULT_MAX_PROVIDERS_PER_REQUEST, DEFAULT_MESSAGE_RECEIVE_TIMEOUT, DEFAULT_MESSAGE_SEND_CONCURRENCY, DEFAULT_RUN_ON_TRANSIENT_CONNECTIONS } from './constants.js'
|
|
17
11
|
import { BitswapMessage } from './pb/message.js'
|
|
12
|
+
import { mergeMessages } from './utils/merge-messages.js'
|
|
13
|
+
import { splitMessage } from './utils/split-message.js'
|
|
18
14
|
import type { WantOptions } from './bitswap.js'
|
|
19
15
|
import type { MultihashHasherLoader } from './index.js'
|
|
20
|
-
import type { Block
|
|
16
|
+
import type { Block } from './pb/message.js'
|
|
21
17
|
import type { Provider, Routing } from '@helia/interface/routing'
|
|
22
|
-
import type { Libp2p, AbortOptions, Connection, PeerId, IncomingStreamData, Topology, ComponentLogger, IdentifyResult, Counter } from '@libp2p/interface'
|
|
18
|
+
import type { Libp2p, AbortOptions, Connection, PeerId, IncomingStreamData, Topology, ComponentLogger, IdentifyResult, Counter, Metrics } from '@libp2p/interface'
|
|
23
19
|
import type { Logger } from '@libp2p/logger'
|
|
20
|
+
import type { CID } from 'multiformats/cid'
|
|
24
21
|
import type { ProgressEvent, ProgressOptions } from 'progress-events'
|
|
25
22
|
|
|
26
|
-
// Add a formatter for a bitswap message
|
|
27
|
-
debug.formatters.B = (b?: BitswapMessage): string => {
|
|
28
|
-
if (b == null) {
|
|
29
|
-
return 'undefined'
|
|
30
|
-
}
|
|
31
|
-
|
|
32
|
-
return JSON.stringify({
|
|
33
|
-
blocks: b.blocks?.map(b => ({
|
|
34
|
-
data: `${uint8ArrayToString(b.data, 'base64').substring(0, 10)}...`,
|
|
35
|
-
prefix: uint8ArrayToString(b.prefix, 'base64')
|
|
36
|
-
})),
|
|
37
|
-
blockPresences: b.blockPresences?.map(p => ({
|
|
38
|
-
...p,
|
|
39
|
-
cid: CID.decode(p.cid).toString()
|
|
40
|
-
})),
|
|
41
|
-
wantlist: b.wantlist == null
|
|
42
|
-
? undefined
|
|
43
|
-
: {
|
|
44
|
-
full: b.wantlist.full,
|
|
45
|
-
entries: b.wantlist.entries.map(e => ({
|
|
46
|
-
...e,
|
|
47
|
-
cid: CID.decode(e.cid).toString()
|
|
48
|
-
}))
|
|
49
|
-
}
|
|
50
|
-
}, null, 2)
|
|
51
|
-
}
|
|
52
|
-
|
|
53
23
|
export type BitswapNetworkProgressEvents =
|
|
54
24
|
ProgressEvent<'bitswap:network:dial', PeerId>
|
|
55
25
|
|
|
@@ -68,16 +38,18 @@ export interface NetworkInit {
|
|
|
68
38
|
maxInboundStreams?: number
|
|
69
39
|
maxOutboundStreams?: number
|
|
70
40
|
messageReceiveTimeout?: number
|
|
71
|
-
messageSendTimeout?: number
|
|
72
41
|
messageSendConcurrency?: number
|
|
73
42
|
protocols?: string[]
|
|
74
43
|
runOnTransientConnections?: boolean
|
|
44
|
+
maxOutgoingMessageSize?: number
|
|
45
|
+
maxIncomingMessageSize?: number
|
|
75
46
|
}
|
|
76
47
|
|
|
77
48
|
export interface NetworkComponents {
|
|
78
49
|
routing: Routing
|
|
79
50
|
logger: ComponentLogger
|
|
80
51
|
libp2p: Libp2p
|
|
52
|
+
metrics?: Metrics
|
|
81
53
|
}
|
|
82
54
|
|
|
83
55
|
export interface BitswapMessageEventDetail {
|
|
@@ -107,8 +79,9 @@ export class Network extends TypedEventEmitter<NetworkEvents> {
|
|
|
107
79
|
private registrarIds: string[]
|
|
108
80
|
private readonly metrics: { blocksSent?: Counter, dataSent?: Counter }
|
|
109
81
|
private readonly sendQueue: PeerQueue<void, SendMessageJobOptions>
|
|
110
|
-
private readonly messageSendTimeout: number
|
|
111
82
|
private readonly runOnTransientConnections: boolean
|
|
83
|
+
private readonly maxOutgoingMessageSize: number
|
|
84
|
+
private readonly maxIncomingMessageSize: number
|
|
112
85
|
|
|
113
86
|
constructor (components: NetworkComponents, init: NetworkInit = {}) {
|
|
114
87
|
super()
|
|
@@ -125,16 +98,17 @@ export class Network extends TypedEventEmitter<NetworkEvents> {
|
|
|
125
98
|
this.maxInboundStreams = init.maxInboundStreams ?? DEFAULT_MAX_INBOUND_STREAMS
|
|
126
99
|
this.maxOutboundStreams = init.maxOutboundStreams ?? DEFAULT_MAX_OUTBOUND_STREAMS
|
|
127
100
|
this.messageReceiveTimeout = init.messageReceiveTimeout ?? DEFAULT_MESSAGE_RECEIVE_TIMEOUT
|
|
128
|
-
this.messageSendTimeout = init.messageSendTimeout ?? DEFAULT_MESSAGE_SEND_TIMEOUT
|
|
129
101
|
this.runOnTransientConnections = init.runOnTransientConnections ?? DEFAULT_RUN_ON_TRANSIENT_CONNECTIONS
|
|
102
|
+
this.maxIncomingMessageSize = init.maxIncomingMessageSize ?? DEFAULT_MAX_OUTGOING_MESSAGE_SIZE
|
|
103
|
+
this.maxOutgoingMessageSize = init.maxOutgoingMessageSize ?? init.maxIncomingMessageSize ?? DEFAULT_MAX_INCOMING_MESSAGE_SIZE
|
|
130
104
|
this.metrics = {
|
|
131
|
-
blocksSent: components.
|
|
132
|
-
dataSent: components.
|
|
105
|
+
blocksSent: components.metrics?.registerCounter('helia_bitswap_sent_blocks_total'),
|
|
106
|
+
dataSent: components.metrics?.registerCounter('helia_bitswap_sent_data_bytes_total')
|
|
133
107
|
}
|
|
134
108
|
|
|
135
109
|
this.sendQueue = new PeerQueue({
|
|
136
110
|
concurrency: init.messageSendConcurrency ?? DEFAULT_MESSAGE_SEND_CONCURRENCY,
|
|
137
|
-
metrics: components.
|
|
111
|
+
metrics: components.metrics,
|
|
138
112
|
metricName: 'helia_bitswap_message_send_queue'
|
|
139
113
|
})
|
|
140
114
|
this.sendQueue.addEventListener('error', (evt) => {
|
|
@@ -212,21 +186,29 @@ export class Network extends TypedEventEmitter<NetworkEvents> {
|
|
|
212
186
|
Promise.resolve().then(async () => {
|
|
213
187
|
this.log('incoming new bitswap %s stream from %p', stream.protocol, connection.remotePeer)
|
|
214
188
|
const abortListener = (): void => {
|
|
215
|
-
|
|
189
|
+
if (stream.status === 'open') {
|
|
190
|
+
stream.abort(new CodeError('Incoming Bitswap stream timed out', 'ERR_TIMEOUT'))
|
|
191
|
+
} else {
|
|
192
|
+
this.log('stream aborted with status', stream.status)
|
|
193
|
+
}
|
|
216
194
|
}
|
|
217
195
|
|
|
218
196
|
let signal = AbortSignal.timeout(this.messageReceiveTimeout)
|
|
219
197
|
setMaxListeners(Infinity, signal)
|
|
220
198
|
signal.addEventListener('abort', abortListener)
|
|
221
199
|
|
|
200
|
+
await stream.closeWrite()
|
|
201
|
+
|
|
222
202
|
await pipe(
|
|
223
203
|
stream,
|
|
224
|
-
(source) => lp.decode(source
|
|
204
|
+
(source) => lp.decode(source, {
|
|
205
|
+
maxDataLength: this.maxIncomingMessageSize
|
|
206
|
+
}),
|
|
225
207
|
async (source) => {
|
|
226
208
|
for await (const data of source) {
|
|
227
209
|
try {
|
|
228
210
|
const message = BitswapMessage.decode(data)
|
|
229
|
-
this.log('incoming new bitswap %s message from %p
|
|
211
|
+
this.log('incoming new bitswap %s message from %p on stream', stream.protocol, connection.remotePeer, stream.id)
|
|
230
212
|
|
|
231
213
|
this.safeDispatchEvent('bitswap:message', {
|
|
232
214
|
detail: {
|
|
@@ -241,7 +223,7 @@ export class Network extends TypedEventEmitter<NetworkEvents> {
|
|
|
241
223
|
setMaxListeners(Infinity, signal)
|
|
242
224
|
signal.addEventListener('abort', abortListener)
|
|
243
225
|
} catch (err: any) {
|
|
244
|
-
this.log.error('error reading incoming bitswap message from %p', connection.remotePeer, err)
|
|
226
|
+
this.log.error('error reading incoming bitswap message from %p on stream', connection.remotePeer, stream.id, err)
|
|
245
227
|
stream.abort(err)
|
|
246
228
|
break
|
|
247
229
|
}
|
|
@@ -309,58 +291,55 @@ export class Network extends TypedEventEmitter<NetworkEvents> {
|
|
|
309
291
|
pendingBytes: msg.pendingBytes ?? 0
|
|
310
292
|
}
|
|
311
293
|
|
|
312
|
-
const
|
|
313
|
-
|
|
314
|
-
|
|
294
|
+
const existingJob = this.sendQueue.queue.find(job => {
|
|
295
|
+
return peerId.equals(job.options.peerId) && job.status === 'queued'
|
|
296
|
+
})
|
|
315
297
|
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
298
|
+
if (existingJob != null) {
|
|
299
|
+
// merge messages instead of adding new job
|
|
300
|
+
existingJob.options.message = mergeMessages(existingJob.options.message, message)
|
|
301
|
+
|
|
302
|
+
await existingJob.join({
|
|
303
|
+
signal: options?.signal
|
|
319
304
|
})
|
|
320
305
|
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
existingJob.options.message = mergeMessages(existingJob.options.message, message)
|
|
306
|
+
return
|
|
307
|
+
}
|
|
324
308
|
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
})
|
|
309
|
+
await this.sendQueue.add(async (options) => {
|
|
310
|
+
const message = options?.message
|
|
328
311
|
|
|
329
|
-
|
|
312
|
+
if (message == null) {
|
|
313
|
+
throw new CodeError('No message to send', 'ERR_NO_MESSAGE')
|
|
330
314
|
}
|
|
331
315
|
|
|
332
|
-
|
|
333
|
-
const message = options?.message
|
|
316
|
+
this.log('sendMessage to %p', peerId)
|
|
334
317
|
|
|
335
|
-
|
|
336
|
-
throw new CodeError('No message to send', 'ERR_NO_MESSAGE')
|
|
337
|
-
}
|
|
318
|
+
options?.onProgress?.(new CustomProgressEvent<PeerId>('bitswap:network:send-wantlist', peerId))
|
|
338
319
|
|
|
339
|
-
|
|
320
|
+
const stream = await this.libp2p.dialProtocol(peerId, BITSWAP_120, options)
|
|
321
|
+
await stream.closeRead()
|
|
340
322
|
|
|
341
|
-
|
|
323
|
+
try {
|
|
324
|
+
await pipe(
|
|
325
|
+
splitMessage(message, this.maxOutgoingMessageSize),
|
|
326
|
+
(source) => lp.encode(source),
|
|
327
|
+
stream
|
|
328
|
+
)
|
|
342
329
|
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
} catch (err: any) {
|
|
350
|
-
options?.onProgress?.(new CustomProgressEvent<{ peer: PeerId, error: Error }>('bitswap:network:send-wantlist:error', { peer: peerId, error: err }))
|
|
351
|
-
this.log.error('error sending message to %p', peerId, err)
|
|
352
|
-
stream.abort(err)
|
|
353
|
-
}
|
|
330
|
+
await stream.close(options)
|
|
331
|
+
} catch (err: any) {
|
|
332
|
+
options?.onProgress?.(new CustomProgressEvent<{ peer: PeerId, error: Error }>('bitswap:network:send-wantlist:error', { peer: peerId, error: err }))
|
|
333
|
+
this.log.error('error sending message to %p', peerId, err)
|
|
334
|
+
stream.abort(err)
|
|
335
|
+
}
|
|
354
336
|
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
} finally {
|
|
362
|
-
signal.clear()
|
|
363
|
-
}
|
|
337
|
+
this._updateSentStats(message.blocks)
|
|
338
|
+
}, {
|
|
339
|
+
peerId,
|
|
340
|
+
signal: options?.signal,
|
|
341
|
+
message
|
|
342
|
+
})
|
|
364
343
|
}
|
|
365
344
|
|
|
366
345
|
/**
|
|
@@ -409,71 +388,3 @@ export class Network extends TypedEventEmitter<NetworkEvents> {
|
|
|
409
388
|
this.metrics.blocksSent?.increment(blocks.length)
|
|
410
389
|
}
|
|
411
390
|
}
|
|
412
|
-
|
|
413
|
-
function mergeMessages (messageA: BitswapMessage, messageB: BitswapMessage): BitswapMessage {
|
|
414
|
-
const wantListEntries = new Map<string, WantlistEntry>(
|
|
415
|
-
(messageA.wantlist?.entries ?? []).map(entry => ([
|
|
416
|
-
base64.encode(entry.cid),
|
|
417
|
-
entry
|
|
418
|
-
]))
|
|
419
|
-
)
|
|
420
|
-
|
|
421
|
-
for (const entry of messageB.wantlist?.entries ?? []) {
|
|
422
|
-
const key = base64.encode(entry.cid)
|
|
423
|
-
const existingEntry = wantListEntries.get(key)
|
|
424
|
-
|
|
425
|
-
if (existingEntry != null) {
|
|
426
|
-
// take highest priority
|
|
427
|
-
if (existingEntry.priority > entry.priority) {
|
|
428
|
-
entry.priority = existingEntry.priority
|
|
429
|
-
}
|
|
430
|
-
|
|
431
|
-
// take later values if passed, otherwise use earlier ones
|
|
432
|
-
entry.cancel = entry.cancel ?? existingEntry.cancel
|
|
433
|
-
entry.wantType = entry.wantType ?? existingEntry.wantType
|
|
434
|
-
entry.sendDontHave = entry.sendDontHave ?? existingEntry.sendDontHave
|
|
435
|
-
}
|
|
436
|
-
|
|
437
|
-
wantListEntries.set(key, entry)
|
|
438
|
-
}
|
|
439
|
-
|
|
440
|
-
const blockPresences = new Map<string, BlockPresence>(
|
|
441
|
-
messageA.blockPresences.map(presence => ([
|
|
442
|
-
base64.encode(presence.cid),
|
|
443
|
-
presence
|
|
444
|
-
]))
|
|
445
|
-
)
|
|
446
|
-
|
|
447
|
-
for (const blockPresence of messageB.blockPresences) {
|
|
448
|
-
const key = base64.encode(blockPresence.cid)
|
|
449
|
-
|
|
450
|
-
// override earlier block presence with later one as if duplicated it is
|
|
451
|
-
// likely to be more accurate since it is more recent
|
|
452
|
-
blockPresences.set(key, blockPresence)
|
|
453
|
-
}
|
|
454
|
-
|
|
455
|
-
const blocks = new Map<string, Block>(
|
|
456
|
-
messageA.blocks.map(block => ([
|
|
457
|
-
base64.encode(block.data),
|
|
458
|
-
block
|
|
459
|
-
]))
|
|
460
|
-
)
|
|
461
|
-
|
|
462
|
-
for (const block of messageB.blocks) {
|
|
463
|
-
const key = base64.encode(block.data)
|
|
464
|
-
|
|
465
|
-
blocks.set(key, block)
|
|
466
|
-
}
|
|
467
|
-
|
|
468
|
-
const output: BitswapMessage = {
|
|
469
|
-
wantlist: {
|
|
470
|
-
full: messageA.wantlist?.full ?? messageB.wantlist?.full ?? false,
|
|
471
|
-
entries: [...wantListEntries.values()]
|
|
472
|
-
},
|
|
473
|
-
blockPresences: [...blockPresences.values()],
|
|
474
|
-
blocks: [...blocks.values()],
|
|
475
|
-
pendingBytes: messageA.pendingBytes + messageB.pendingBytes
|
|
476
|
-
}
|
|
477
|
-
|
|
478
|
-
return output
|
|
479
|
-
}
|
package/src/pb/message.ts
CHANGED
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
/* eslint-disable @typescript-eslint/no-unnecessary-boolean-literal-compare */
|
|
5
5
|
/* eslint-disable @typescript-eslint/no-empty-interface */
|
|
6
6
|
|
|
7
|
-
import { type Codec, decodeMessage, encodeMessage, enumeration, message } from 'protons-runtime'
|
|
7
|
+
import { type Codec, CodeError, decodeMessage, type DecodeOptions, encodeMessage, enumeration, message } from 'protons-runtime'
|
|
8
8
|
import { alloc as uint8ArrayAlloc } from 'uint8arrays/alloc'
|
|
9
9
|
import type { Uint8ArrayList } from 'uint8arraylist'
|
|
10
10
|
|
|
@@ -69,7 +69,7 @@ export namespace WantlistEntry {
|
|
|
69
69
|
if (opts.lengthDelimited !== false) {
|
|
70
70
|
w.ldelim()
|
|
71
71
|
}
|
|
72
|
-
}, (reader, length) => {
|
|
72
|
+
}, (reader, length, opts = {}) => {
|
|
73
73
|
const obj: any = {
|
|
74
74
|
cid: uint8ArrayAlloc(0),
|
|
75
75
|
priority: 0
|
|
@@ -119,8 +119,8 @@ export namespace WantlistEntry {
|
|
|
119
119
|
return encodeMessage(obj, WantlistEntry.codec())
|
|
120
120
|
}
|
|
121
121
|
|
|
122
|
-
export const decode = (buf: Uint8Array | Uint8ArrayList): WantlistEntry => {
|
|
123
|
-
return decodeMessage(buf, WantlistEntry.codec())
|
|
122
|
+
export const decode = (buf: Uint8Array | Uint8ArrayList, opts?: DecodeOptions<WantlistEntry>): WantlistEntry => {
|
|
123
|
+
return decodeMessage(buf, WantlistEntry.codec(), opts)
|
|
124
124
|
}
|
|
125
125
|
}
|
|
126
126
|
|
|
@@ -154,7 +154,7 @@ export namespace Wantlist {
|
|
|
154
154
|
if (opts.lengthDelimited !== false) {
|
|
155
155
|
w.ldelim()
|
|
156
156
|
}
|
|
157
|
-
}, (reader, length) => {
|
|
157
|
+
}, (reader, length, opts = {}) => {
|
|
158
158
|
const obj: any = {
|
|
159
159
|
entries: []
|
|
160
160
|
}
|
|
@@ -166,7 +166,13 @@ export namespace Wantlist {
|
|
|
166
166
|
|
|
167
167
|
switch (tag >>> 3) {
|
|
168
168
|
case 1: {
|
|
169
|
-
obj.entries.
|
|
169
|
+
if (opts.limits?.entries != null && obj.entries.length === opts.limits.entries) {
|
|
170
|
+
throw new CodeError('decode error - map field "entries" had too many elements', 'ERR_MAX_LENGTH')
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
obj.entries.push(WantlistEntry.codec().decode(reader, reader.uint32(), {
|
|
174
|
+
limits: opts.limits?.entries$
|
|
175
|
+
}))
|
|
170
176
|
break
|
|
171
177
|
}
|
|
172
178
|
case 2: {
|
|
@@ -191,8 +197,8 @@ export namespace Wantlist {
|
|
|
191
197
|
return encodeMessage(obj, Wantlist.codec())
|
|
192
198
|
}
|
|
193
199
|
|
|
194
|
-
export const decode = (buf: Uint8Array | Uint8ArrayList): Wantlist => {
|
|
195
|
-
return decodeMessage(buf, Wantlist.codec())
|
|
200
|
+
export const decode = (buf: Uint8Array | Uint8ArrayList, opts?: DecodeOptions<Wantlist>): Wantlist => {
|
|
201
|
+
return decodeMessage(buf, Wantlist.codec(), opts)
|
|
196
202
|
}
|
|
197
203
|
}
|
|
198
204
|
|
|
@@ -224,7 +230,7 @@ export namespace Block {
|
|
|
224
230
|
if (opts.lengthDelimited !== false) {
|
|
225
231
|
w.ldelim()
|
|
226
232
|
}
|
|
227
|
-
}, (reader, length) => {
|
|
233
|
+
}, (reader, length, opts = {}) => {
|
|
228
234
|
const obj: any = {
|
|
229
235
|
prefix: uint8ArrayAlloc(0),
|
|
230
236
|
data: uint8ArrayAlloc(0)
|
|
@@ -262,8 +268,8 @@ export namespace Block {
|
|
|
262
268
|
return encodeMessage(obj, Block.codec())
|
|
263
269
|
}
|
|
264
270
|
|
|
265
|
-
export const decode = (buf: Uint8Array | Uint8ArrayList): Block => {
|
|
266
|
-
return decodeMessage(buf, Block.codec())
|
|
271
|
+
export const decode = (buf: Uint8Array | Uint8ArrayList, opts?: DecodeOptions<Block>): Block => {
|
|
272
|
+
return decodeMessage(buf, Block.codec(), opts)
|
|
267
273
|
}
|
|
268
274
|
}
|
|
269
275
|
|
|
@@ -310,7 +316,7 @@ export namespace BlockPresence {
|
|
|
310
316
|
if (opts.lengthDelimited !== false) {
|
|
311
317
|
w.ldelim()
|
|
312
318
|
}
|
|
313
|
-
}, (reader, length) => {
|
|
319
|
+
}, (reader, length, opts = {}) => {
|
|
314
320
|
const obj: any = {
|
|
315
321
|
cid: uint8ArrayAlloc(0),
|
|
316
322
|
type: BlockPresenceType.HaveBlock
|
|
@@ -348,8 +354,8 @@ export namespace BlockPresence {
|
|
|
348
354
|
return encodeMessage(obj, BlockPresence.codec())
|
|
349
355
|
}
|
|
350
356
|
|
|
351
|
-
export const decode = (buf: Uint8Array | Uint8ArrayList): BlockPresence => {
|
|
352
|
-
return decodeMessage(buf, BlockPresence.codec())
|
|
357
|
+
export const decode = (buf: Uint8Array | Uint8ArrayList, opts?: DecodeOptions<BlockPresence>): BlockPresence => {
|
|
358
|
+
return decodeMessage(buf, BlockPresence.codec(), opts)
|
|
353
359
|
}
|
|
354
360
|
}
|
|
355
361
|
|
|
@@ -397,7 +403,7 @@ export namespace BitswapMessage {
|
|
|
397
403
|
if (opts.lengthDelimited !== false) {
|
|
398
404
|
w.ldelim()
|
|
399
405
|
}
|
|
400
|
-
}, (reader, length) => {
|
|
406
|
+
}, (reader, length, opts = {}) => {
|
|
401
407
|
const obj: any = {
|
|
402
408
|
blocks: [],
|
|
403
409
|
blockPresences: [],
|
|
@@ -411,15 +417,29 @@ export namespace BitswapMessage {
|
|
|
411
417
|
|
|
412
418
|
switch (tag >>> 3) {
|
|
413
419
|
case 1: {
|
|
414
|
-
obj.wantlist = Wantlist.codec().decode(reader, reader.uint32()
|
|
420
|
+
obj.wantlist = Wantlist.codec().decode(reader, reader.uint32(), {
|
|
421
|
+
limits: opts.limits?.wantlist
|
|
422
|
+
})
|
|
415
423
|
break
|
|
416
424
|
}
|
|
417
425
|
case 3: {
|
|
418
|
-
obj.blocks.
|
|
426
|
+
if (opts.limits?.blocks != null && obj.blocks.length === opts.limits.blocks) {
|
|
427
|
+
throw new CodeError('decode error - map field "blocks" had too many elements', 'ERR_MAX_LENGTH')
|
|
428
|
+
}
|
|
429
|
+
|
|
430
|
+
obj.blocks.push(Block.codec().decode(reader, reader.uint32(), {
|
|
431
|
+
limits: opts.limits?.blocks$
|
|
432
|
+
}))
|
|
419
433
|
break
|
|
420
434
|
}
|
|
421
435
|
case 4: {
|
|
422
|
-
obj.blockPresences.
|
|
436
|
+
if (opts.limits?.blockPresences != null && obj.blockPresences.length === opts.limits.blockPresences) {
|
|
437
|
+
throw new CodeError('decode error - map field "blockPresences" had too many elements', 'ERR_MAX_LENGTH')
|
|
438
|
+
}
|
|
439
|
+
|
|
440
|
+
obj.blockPresences.push(BlockPresence.codec().decode(reader, reader.uint32(), {
|
|
441
|
+
limits: opts.limits?.blockPresences$
|
|
442
|
+
}))
|
|
423
443
|
break
|
|
424
444
|
}
|
|
425
445
|
case 5: {
|
|
@@ -444,7 +464,7 @@ export namespace BitswapMessage {
|
|
|
444
464
|
return encodeMessage(obj, BitswapMessage.codec())
|
|
445
465
|
}
|
|
446
466
|
|
|
447
|
-
export const decode = (buf: Uint8Array | Uint8ArrayList): BitswapMessage => {
|
|
448
|
-
return decodeMessage(buf, BitswapMessage.codec())
|
|
467
|
+
export const decode = (buf: Uint8Array | Uint8ArrayList, opts?: DecodeOptions<BitswapMessage>): BitswapMessage => {
|
|
468
|
+
return decodeMessage(buf, BitswapMessage.codec(), opts)
|
|
449
469
|
}
|
|
450
470
|
}
|
|
@@ -6,7 +6,7 @@ import { Ledger } from './ledger.js'
|
|
|
6
6
|
import type { BitswapNotifyProgressEvents, WantListEntry } from '../index.js'
|
|
7
7
|
import type { Network } from '../network.js'
|
|
8
8
|
import type { BitswapMessage } from '../pb/message.js'
|
|
9
|
-
import type { ComponentLogger, Libp2p, Logger, PeerId } from '@libp2p/interface'
|
|
9
|
+
import type { ComponentLogger, Libp2p, Logger, Metrics, PeerId } from '@libp2p/interface'
|
|
10
10
|
import type { PeerMap } from '@libp2p/peer-collections'
|
|
11
11
|
import type { Blockstore } from 'interface-blockstore'
|
|
12
12
|
import type { AbortOptions } from 'it-length-prefixed-stream'
|
|
@@ -21,6 +21,7 @@ export interface PeerWantListsComponents {
|
|
|
21
21
|
network: Network
|
|
22
22
|
libp2p: Libp2p
|
|
23
23
|
logger: ComponentLogger
|
|
24
|
+
metrics?: Metrics
|
|
24
25
|
}
|
|
25
26
|
|
|
26
27
|
export interface PeerLedger {
|
|
@@ -37,16 +38,18 @@ export class PeerWantLists {
|
|
|
37
38
|
public readonly ledgerMap: PeerMap<Ledger>
|
|
38
39
|
private readonly maxSizeReplaceHasWithBlock?: number
|
|
39
40
|
private readonly log: Logger
|
|
41
|
+
private readonly logger: ComponentLogger
|
|
40
42
|
|
|
41
43
|
constructor (components: PeerWantListsComponents, init: PeerWantListsInit = {}) {
|
|
42
44
|
this.blockstore = components.blockstore
|
|
43
45
|
this.network = components.network
|
|
44
46
|
this.maxSizeReplaceHasWithBlock = init.maxSizeReplaceHasWithBlock
|
|
45
47
|
this.log = components.logger.forComponent('helia:bitswap:peer-want-lists')
|
|
48
|
+
this.logger = components.logger
|
|
46
49
|
|
|
47
50
|
this.ledgerMap = trackedPeerMap({
|
|
48
51
|
name: 'helia_bitswap_ledger_map',
|
|
49
|
-
metrics: components.
|
|
52
|
+
metrics: components.metrics
|
|
50
53
|
})
|
|
51
54
|
|
|
52
55
|
this.network.addEventListener('bitswap:message', (evt) => {
|
|
@@ -100,7 +103,8 @@ export class PeerWantLists {
|
|
|
100
103
|
ledger = new Ledger({
|
|
101
104
|
peerId,
|
|
102
105
|
blockstore: this.blockstore,
|
|
103
|
-
network: this.network
|
|
106
|
+
network: this.network,
|
|
107
|
+
logger: this.logger
|
|
104
108
|
}, {
|
|
105
109
|
maxSizeReplaceHasWithBlock: this.maxSizeReplaceHasWithBlock
|
|
106
110
|
})
|
|
@@ -141,6 +145,7 @@ export class PeerWantLists {
|
|
|
141
145
|
}
|
|
142
146
|
}
|
|
143
147
|
|
|
148
|
+
this.log('send blocks to peer')
|
|
144
149
|
await ledger.sendBlocksToPeer()
|
|
145
150
|
}
|
|
146
151
|
|
|
@@ -3,7 +3,7 @@ import { DEFAULT_MAX_SIZE_REPLACE_HAS_WITH_BLOCK } from '../constants.js'
|
|
|
3
3
|
import { BlockPresenceType, type BitswapMessage, WantType } from '../pb/message.js'
|
|
4
4
|
import { cidToPrefix } from '../utils/cid-prefix.js'
|
|
5
5
|
import type { Network } from '../network.js'
|
|
6
|
-
import type { PeerId } from '@libp2p/interface'
|
|
6
|
+
import type { ComponentLogger, Logger, PeerId } from '@libp2p/interface'
|
|
7
7
|
import type { Blockstore } from 'interface-blockstore'
|
|
8
8
|
import type { AbortOptions } from 'it-length-prefixed-stream'
|
|
9
9
|
import type { CID } from 'multiformats/cid'
|
|
@@ -12,6 +12,7 @@ export interface LedgerComponents {
|
|
|
12
12
|
peerId: PeerId
|
|
13
13
|
blockstore: Blockstore
|
|
14
14
|
network: Network
|
|
15
|
+
logger: ComponentLogger
|
|
15
16
|
}
|
|
16
17
|
|
|
17
18
|
export interface LedgerInit {
|
|
@@ -56,12 +57,14 @@ export class Ledger {
|
|
|
56
57
|
public bytesReceived: number
|
|
57
58
|
public lastExchange?: number
|
|
58
59
|
private readonly maxSizeReplaceHasWithBlock: number
|
|
60
|
+
private readonly log: Logger
|
|
59
61
|
|
|
60
62
|
constructor (components: LedgerComponents, init: LedgerInit) {
|
|
61
63
|
this.peerId = components.peerId
|
|
62
64
|
this.blockstore = components.blockstore
|
|
63
65
|
this.network = components.network
|
|
64
66
|
this.wants = new Map()
|
|
67
|
+
this.log = components.logger.forComponent(`helia:bitswap:ledger:${components.peerId}`)
|
|
65
68
|
|
|
66
69
|
this.exchangeCount = 0
|
|
67
70
|
this.bytesSent = 0
|
|
@@ -96,6 +99,8 @@ export class Ledger {
|
|
|
96
99
|
const has = await this.blockstore.has(entry.cid, options)
|
|
97
100
|
|
|
98
101
|
if (!has) {
|
|
102
|
+
this.log('do not have block for %c', entry.cid)
|
|
103
|
+
|
|
99
104
|
// we don't have the requested block and the remote is not interested
|
|
100
105
|
// in us telling them that
|
|
101
106
|
if (!entry.sendDontHave) {
|
|
@@ -121,6 +126,7 @@ export class Ledger {
|
|
|
121
126
|
// do they want the block or just us to tell them we have the block
|
|
122
127
|
if (entry.wantType === WantType.WantHave) {
|
|
123
128
|
if (block.byteLength < this.maxSizeReplaceHasWithBlock) {
|
|
129
|
+
this.log('sending have and block for %c', entry.cid)
|
|
124
130
|
// if the block is small we just send it to them
|
|
125
131
|
sentBlocks.add(key)
|
|
126
132
|
message.blocks.push({
|
|
@@ -128,6 +134,7 @@ export class Ledger {
|
|
|
128
134
|
prefix: cidToPrefix(entry.cid)
|
|
129
135
|
})
|
|
130
136
|
} else {
|
|
137
|
+
this.log('sending have for %c', entry.cid)
|
|
131
138
|
// otherwise tell them we have the block
|
|
132
139
|
message.blockPresences.push({
|
|
133
140
|
cid: entry.cid.bytes,
|
|
@@ -135,6 +142,7 @@ export class Ledger {
|
|
|
135
142
|
})
|
|
136
143
|
}
|
|
137
144
|
} else {
|
|
145
|
+
this.log('sending block for %c', entry.cid)
|
|
138
146
|
// they want the block, send it to them
|
|
139
147
|
sentBlocks.add(key)
|
|
140
148
|
message.blocks.push({
|
|
@@ -146,7 +154,9 @@ export class Ledger {
|
|
|
146
154
|
|
|
147
155
|
// only send the message if we actually have something to send
|
|
148
156
|
if (message.blocks.length > 0 || message.blockPresences.length > 0) {
|
|
157
|
+
this.log('sending message')
|
|
149
158
|
await this.network.sendMessage(this.peerId, message, options)
|
|
159
|
+
this.log('sent message')
|
|
150
160
|
|
|
151
161
|
// update accounting
|
|
152
162
|
this.sentBytes(message.blocks.reduce((acc, curr) => acc + curr.data.byteLength, 0))
|
package/src/stats.ts
CHANGED
|
@@ -1,7 +1,8 @@
|
|
|
1
|
-
import type { Libp2p, MetricGroup, PeerId } from '@libp2p/interface'
|
|
1
|
+
import type { Libp2p, MetricGroup, Metrics, PeerId } from '@libp2p/interface'
|
|
2
2
|
|
|
3
3
|
export interface StatsComponents {
|
|
4
4
|
libp2p: Libp2p
|
|
5
|
+
metrics?: Metrics
|
|
5
6
|
}
|
|
6
7
|
|
|
7
8
|
export class Stats {
|
|
@@ -11,10 +12,10 @@ export class Stats {
|
|
|
11
12
|
private readonly duplicateDataReceived?: MetricGroup
|
|
12
13
|
|
|
13
14
|
constructor (components: StatsComponents) {
|
|
14
|
-
this.blocksReceived = components.
|
|
15
|
-
this.duplicateBlocksReceived = components.
|
|
16
|
-
this.dataReceived = components.
|
|
17
|
-
this.duplicateDataReceived = components.
|
|
15
|
+
this.blocksReceived = components.metrics?.registerMetricGroup('helia_bitswap_received_blocks')
|
|
16
|
+
this.duplicateBlocksReceived = components.metrics?.registerMetricGroup('helia_bitswap_duplicate_received_blocks')
|
|
17
|
+
this.dataReceived = components.metrics?.registerMetricGroup('helia_bitswap_data_received_bytes')
|
|
18
|
+
this.duplicateDataReceived = components.metrics?.registerMetricGroup('helia_bitswap_duplicate_data_received_bytes')
|
|
18
19
|
}
|
|
19
20
|
|
|
20
21
|
updateBlocksReceived (count: number = 1, peerId?: PeerId): void {
|