@helia/bitswap 1.0.0 → 1.0.1-0a528bb

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (52) hide show
  1. package/dist/index.min.js +1 -1
  2. package/dist/src/constants.d.ts +2 -0
  3. package/dist/src/constants.d.ts.map +1 -1
  4. package/dist/src/constants.js +2 -0
  5. package/dist/src/constants.js.map +1 -1
  6. package/dist/src/index.d.ts +20 -7
  7. package/dist/src/index.d.ts.map +1 -1
  8. package/dist/src/index.js.map +1 -1
  9. package/dist/src/network.d.ts +7 -4
  10. package/dist/src/network.d.ts.map +1 -1
  11. package/dist/src/network.js +55 -131
  12. package/dist/src/network.js.map +1 -1
  13. package/dist/src/pb/message.d.ts +6 -6
  14. package/dist/src/pb/message.d.ts.map +1 -1
  15. package/dist/src/pb/message.js +37 -20
  16. package/dist/src/pb/message.js.map +1 -1
  17. package/dist/src/peer-want-lists/index.d.ts +3 -1
  18. package/dist/src/peer-want-lists/index.d.ts.map +1 -1
  19. package/dist/src/peer-want-lists/index.js +6 -2
  20. package/dist/src/peer-want-lists/index.js.map +1 -1
  21. package/dist/src/peer-want-lists/ledger.d.ts +3 -1
  22. package/dist/src/peer-want-lists/ledger.d.ts.map +1 -1
  23. package/dist/src/peer-want-lists/ledger.js +8 -0
  24. package/dist/src/peer-want-lists/ledger.js.map +1 -1
  25. package/dist/src/stats.d.ts +2 -1
  26. package/dist/src/stats.d.ts.map +1 -1
  27. package/dist/src/stats.js +4 -4
  28. package/dist/src/stats.js.map +1 -1
  29. package/dist/src/utils/merge-messages.d.ts +3 -0
  30. package/dist/src/utils/merge-messages.d.ts.map +1 -0
  31. package/dist/src/utils/merge-messages.js +51 -0
  32. package/dist/src/utils/merge-messages.js.map +1 -0
  33. package/dist/src/utils/split-message.d.ts +18 -0
  34. package/dist/src/utils/split-message.d.ts.map +1 -0
  35. package/dist/src/utils/split-message.js +101 -0
  36. package/dist/src/utils/split-message.js.map +1 -0
  37. package/dist/src/want-list.d.ts +2 -1
  38. package/dist/src/want-list.d.ts.map +1 -1
  39. package/dist/src/want-list.js +8 -5
  40. package/dist/src/want-list.js.map +1 -1
  41. package/package.json +4 -4
  42. package/src/constants.ts +2 -0
  43. package/src/index.ts +22 -8
  44. package/src/network.ts +65 -154
  45. package/src/pb/message.ts +40 -20
  46. package/src/peer-want-lists/index.ts +8 -3
  47. package/src/peer-want-lists/ledger.ts +11 -1
  48. package/src/stats.ts +6 -5
  49. package/src/utils/merge-messages.ts +70 -0
  50. package/src/utils/split-message.ts +133 -0
  51. package/src/want-list.ts +12 -6
  52. package/dist/typedoc-urls.json +0 -20
package/src/index.ts CHANGED
@@ -117,14 +117,6 @@ export interface BitswapOptions {
117
117
  */
118
118
  protocol?: string
119
119
 
120
- /**
121
- * When a new peer connects, sending our WantList should complete within this
122
- * many ms
123
- *
124
- * @default 5000
125
- */
126
- messageSendTimeout?: number
127
-
128
120
  /**
129
121
  * When sending want list updates to peers, how many messages to send at once
130
122
  *
@@ -167,6 +159,28 @@ export interface BitswapOptions {
167
159
  * @default 1024
168
160
  */
169
161
  maxSizeReplaceHasWithBlock?: number
162
+
163
+ /**
164
+ * The maximum size in bytes of a message that we will send. If a message is
165
+ * larger than this (due to lots of blocks or wantlist entries) it will be
166
+ * broken up into several smaller messages that are under this size.
167
+ *
168
+ * @see https://github.com/ipfs/boxo/blob/eeea414587350401b6b804f0574ed8436833331d/bitswap/client/internal/messagequeue/messagequeue.go#L33
169
+ *
170
+ * @default 2097152
171
+ */
172
+ maxOutgoingMessageSize?: number
173
+
174
+ /**
175
+ * The maximum size in bytes of an incoming message that we will process.
176
+ *
177
+ * Messages larger than this will cause the incoming stream to be reset.
178
+ *
179
+ * Defaults to `maxOutgoingMessageSize`
180
+ *
181
+ * @default 2097152
182
+ */
183
+ maxIncomingMessageSize?: number
170
184
  }
171
185
 
172
186
  export const createBitswap = (components: BitswapComponents, options: BitswapOptions = {}): Bitswap => {
package/src/network.ts CHANGED
@@ -1,55 +1,25 @@
1
1
  import { CodeError, TypedEventEmitter, setMaxListeners } from '@libp2p/interface'
2
2
  import { PeerQueue, type PeerQueueJobOptions } from '@libp2p/utils/peer-queue'
3
- import { anySignal } from 'any-signal'
4
- import debug from 'debug'
5
3
  import drain from 'it-drain'
6
4
  import * as lp from 'it-length-prefixed'
7
- import { lpStream } from 'it-length-prefixed-stream'
8
5
  import map from 'it-map'
9
6
  import { pipe } from 'it-pipe'
10
7
  import take from 'it-take'
11
- import { base64 } from 'multiformats/bases/base64'
12
- import { CID } from 'multiformats/cid'
13
8
  import { CustomProgressEvent } from 'progress-events'
14
9
  import { raceEvent } from 'race-event'
15
- import { toString as uint8ArrayToString } from 'uint8arrays/to-string'
16
- import { BITSWAP_120, DEFAULT_MAX_INBOUND_STREAMS, DEFAULT_MAX_OUTBOUND_STREAMS, DEFAULT_MAX_PROVIDERS_PER_REQUEST, DEFAULT_MESSAGE_RECEIVE_TIMEOUT, DEFAULT_MESSAGE_SEND_CONCURRENCY, DEFAULT_MESSAGE_SEND_TIMEOUT, DEFAULT_RUN_ON_TRANSIENT_CONNECTIONS } from './constants.js'
10
+ import { BITSWAP_120, DEFAULT_MAX_INBOUND_STREAMS, DEFAULT_MAX_INCOMING_MESSAGE_SIZE, DEFAULT_MAX_OUTBOUND_STREAMS, DEFAULT_MAX_OUTGOING_MESSAGE_SIZE, DEFAULT_MAX_PROVIDERS_PER_REQUEST, DEFAULT_MESSAGE_RECEIVE_TIMEOUT, DEFAULT_MESSAGE_SEND_CONCURRENCY, DEFAULT_RUN_ON_TRANSIENT_CONNECTIONS } from './constants.js'
17
11
  import { BitswapMessage } from './pb/message.js'
12
+ import { mergeMessages } from './utils/merge-messages.js'
13
+ import { splitMessage } from './utils/split-message.js'
18
14
  import type { WantOptions } from './bitswap.js'
19
15
  import type { MultihashHasherLoader } from './index.js'
20
- import type { Block, BlockPresence, WantlistEntry } from './pb/message.js'
16
+ import type { Block } from './pb/message.js'
21
17
  import type { Provider, Routing } from '@helia/interface/routing'
22
- import type { Libp2p, AbortOptions, Connection, PeerId, IncomingStreamData, Topology, ComponentLogger, IdentifyResult, Counter } from '@libp2p/interface'
18
+ import type { Libp2p, AbortOptions, Connection, PeerId, IncomingStreamData, Topology, ComponentLogger, IdentifyResult, Counter, Metrics } from '@libp2p/interface'
23
19
  import type { Logger } from '@libp2p/logger'
20
+ import type { CID } from 'multiformats/cid'
24
21
  import type { ProgressEvent, ProgressOptions } from 'progress-events'
25
22
 
26
- // Add a formatter for a bitswap message
27
- debug.formatters.B = (b?: BitswapMessage): string => {
28
- if (b == null) {
29
- return 'undefined'
30
- }
31
-
32
- return JSON.stringify({
33
- blocks: b.blocks?.map(b => ({
34
- data: `${uint8ArrayToString(b.data, 'base64').substring(0, 10)}...`,
35
- prefix: uint8ArrayToString(b.prefix, 'base64')
36
- })),
37
- blockPresences: b.blockPresences?.map(p => ({
38
- ...p,
39
- cid: CID.decode(p.cid).toString()
40
- })),
41
- wantlist: b.wantlist == null
42
- ? undefined
43
- : {
44
- full: b.wantlist.full,
45
- entries: b.wantlist.entries.map(e => ({
46
- ...e,
47
- cid: CID.decode(e.cid).toString()
48
- }))
49
- }
50
- }, null, 2)
51
- }
52
-
53
23
  export type BitswapNetworkProgressEvents =
54
24
  ProgressEvent<'bitswap:network:dial', PeerId>
55
25
 
@@ -68,16 +38,18 @@ export interface NetworkInit {
68
38
  maxInboundStreams?: number
69
39
  maxOutboundStreams?: number
70
40
  messageReceiveTimeout?: number
71
- messageSendTimeout?: number
72
41
  messageSendConcurrency?: number
73
42
  protocols?: string[]
74
43
  runOnTransientConnections?: boolean
44
+ maxOutgoingMessageSize?: number
45
+ maxIncomingMessageSize?: number
75
46
  }
76
47
 
77
48
  export interface NetworkComponents {
78
49
  routing: Routing
79
50
  logger: ComponentLogger
80
51
  libp2p: Libp2p
52
+ metrics?: Metrics
81
53
  }
82
54
 
83
55
  export interface BitswapMessageEventDetail {
@@ -107,8 +79,9 @@ export class Network extends TypedEventEmitter<NetworkEvents> {
107
79
  private registrarIds: string[]
108
80
  private readonly metrics: { blocksSent?: Counter, dataSent?: Counter }
109
81
  private readonly sendQueue: PeerQueue<void, SendMessageJobOptions>
110
- private readonly messageSendTimeout: number
111
82
  private readonly runOnTransientConnections: boolean
83
+ private readonly maxOutgoingMessageSize: number
84
+ private readonly maxIncomingMessageSize: number
112
85
 
113
86
  constructor (components: NetworkComponents, init: NetworkInit = {}) {
114
87
  super()
@@ -125,16 +98,17 @@ export class Network extends TypedEventEmitter<NetworkEvents> {
125
98
  this.maxInboundStreams = init.maxInboundStreams ?? DEFAULT_MAX_INBOUND_STREAMS
126
99
  this.maxOutboundStreams = init.maxOutboundStreams ?? DEFAULT_MAX_OUTBOUND_STREAMS
127
100
  this.messageReceiveTimeout = init.messageReceiveTimeout ?? DEFAULT_MESSAGE_RECEIVE_TIMEOUT
128
- this.messageSendTimeout = init.messageSendTimeout ?? DEFAULT_MESSAGE_SEND_TIMEOUT
129
101
  this.runOnTransientConnections = init.runOnTransientConnections ?? DEFAULT_RUN_ON_TRANSIENT_CONNECTIONS
102
+ this.maxIncomingMessageSize = init.maxIncomingMessageSize ?? DEFAULT_MAX_OUTGOING_MESSAGE_SIZE
103
+ this.maxOutgoingMessageSize = init.maxOutgoingMessageSize ?? init.maxIncomingMessageSize ?? DEFAULT_MAX_INCOMING_MESSAGE_SIZE
130
104
  this.metrics = {
131
- blocksSent: components.libp2p.metrics?.registerCounter('helia_bitswap_sent_blocks_total'),
132
- dataSent: components.libp2p.metrics?.registerCounter('helia_bitswap_sent_data_bytes_total')
105
+ blocksSent: components.metrics?.registerCounter('helia_bitswap_sent_blocks_total'),
106
+ dataSent: components.metrics?.registerCounter('helia_bitswap_sent_data_bytes_total')
133
107
  }
134
108
 
135
109
  this.sendQueue = new PeerQueue({
136
110
  concurrency: init.messageSendConcurrency ?? DEFAULT_MESSAGE_SEND_CONCURRENCY,
137
- metrics: components.libp2p.metrics,
111
+ metrics: components.metrics,
138
112
  metricName: 'helia_bitswap_message_send_queue'
139
113
  })
140
114
  this.sendQueue.addEventListener('error', (evt) => {
@@ -212,21 +186,29 @@ export class Network extends TypedEventEmitter<NetworkEvents> {
212
186
  Promise.resolve().then(async () => {
213
187
  this.log('incoming new bitswap %s stream from %p', stream.protocol, connection.remotePeer)
214
188
  const abortListener = (): void => {
215
- stream.abort(new CodeError('Incoming Bitswap stream timed out', 'ERR_TIMEOUT'))
189
+ if (stream.status === 'open') {
190
+ stream.abort(new CodeError('Incoming Bitswap stream timed out', 'ERR_TIMEOUT'))
191
+ } else {
192
+ this.log('stream aborted with status', stream.status)
193
+ }
216
194
  }
217
195
 
218
196
  let signal = AbortSignal.timeout(this.messageReceiveTimeout)
219
197
  setMaxListeners(Infinity, signal)
220
198
  signal.addEventListener('abort', abortListener)
221
199
 
200
+ await stream.closeWrite()
201
+
222
202
  await pipe(
223
203
  stream,
224
- (source) => lp.decode(source),
204
+ (source) => lp.decode(source, {
205
+ maxDataLength: this.maxIncomingMessageSize
206
+ }),
225
207
  async (source) => {
226
208
  for await (const data of source) {
227
209
  try {
228
210
  const message = BitswapMessage.decode(data)
229
- this.log('incoming new bitswap %s message from %p %B', stream.protocol, connection.remotePeer, message)
211
+ this.log('incoming new bitswap %s message from %p on stream', stream.protocol, connection.remotePeer, stream.id)
230
212
 
231
213
  this.safeDispatchEvent('bitswap:message', {
232
214
  detail: {
@@ -241,7 +223,7 @@ export class Network extends TypedEventEmitter<NetworkEvents> {
241
223
  setMaxListeners(Infinity, signal)
242
224
  signal.addEventListener('abort', abortListener)
243
225
  } catch (err: any) {
244
- this.log.error('error reading incoming bitswap message from %p', connection.remotePeer, err)
226
+ this.log.error('error reading incoming bitswap message from %p on stream', connection.remotePeer, stream.id, err)
245
227
  stream.abort(err)
246
228
  break
247
229
  }
@@ -309,58 +291,55 @@ export class Network extends TypedEventEmitter<NetworkEvents> {
309
291
  pendingBytes: msg.pendingBytes ?? 0
310
292
  }
311
293
 
312
- const timeoutSignal = AbortSignal.timeout(this.messageSendTimeout)
313
- const signal = anySignal([timeoutSignal, options?.signal])
314
- setMaxListeners(Infinity, timeoutSignal, signal)
294
+ const existingJob = this.sendQueue.queue.find(job => {
295
+ return peerId.equals(job.options.peerId) && job.status === 'queued'
296
+ })
315
297
 
316
- try {
317
- const existingJob = this.sendQueue.queue.find(job => {
318
- return peerId.equals(job.options.peerId) && job.status === 'queued'
298
+ if (existingJob != null) {
299
+ // merge messages instead of adding new job
300
+ existingJob.options.message = mergeMessages(existingJob.options.message, message)
301
+
302
+ await existingJob.join({
303
+ signal: options?.signal
319
304
  })
320
305
 
321
- if (existingJob != null) {
322
- // merge messages instead of adding new job
323
- existingJob.options.message = mergeMessages(existingJob.options.message, message)
306
+ return
307
+ }
324
308
 
325
- await existingJob.join({
326
- signal
327
- })
309
+ await this.sendQueue.add(async (options) => {
310
+ const message = options?.message
328
311
 
329
- return
312
+ if (message == null) {
313
+ throw new CodeError('No message to send', 'ERR_NO_MESSAGE')
330
314
  }
331
315
 
332
- await this.sendQueue.add(async (options) => {
333
- const message = options?.message
316
+ this.log('sendMessage to %p', peerId)
334
317
 
335
- if (message == null) {
336
- throw new CodeError('No message to send', 'ERR_NO_MESSAGE')
337
- }
318
+ options?.onProgress?.(new CustomProgressEvent<PeerId>('bitswap:network:send-wantlist', peerId))
338
319
 
339
- this.log('sendMessage to %p %B', peerId, message)
320
+ const stream = await this.libp2p.dialProtocol(peerId, BITSWAP_120, options)
321
+ await stream.closeRead()
340
322
 
341
- options?.onProgress?.(new CustomProgressEvent<PeerId>('bitswap:network:send-wantlist', peerId))
323
+ try {
324
+ await pipe(
325
+ splitMessage(message, this.maxOutgoingMessageSize),
326
+ (source) => lp.encode(source),
327
+ stream
328
+ )
342
329
 
343
- const stream = await this.libp2p.dialProtocol(peerId, BITSWAP_120, options)
344
-
345
- try {
346
- const lp = lpStream(stream)
347
- await lp.write(BitswapMessage.encode(message), options)
348
- await lp.unwrap().close(options)
349
- } catch (err: any) {
350
- options?.onProgress?.(new CustomProgressEvent<{ peer: PeerId, error: Error }>('bitswap:network:send-wantlist:error', { peer: peerId, error: err }))
351
- this.log.error('error sending message to %p', peerId, err)
352
- stream.abort(err)
353
- }
330
+ await stream.close(options)
331
+ } catch (err: any) {
332
+ options?.onProgress?.(new CustomProgressEvent<{ peer: PeerId, error: Error }>('bitswap:network:send-wantlist:error', { peer: peerId, error: err }))
333
+ this.log.error('error sending message to %p', peerId, err)
334
+ stream.abort(err)
335
+ }
354
336
 
355
- this._updateSentStats(message.blocks)
356
- }, {
357
- peerId,
358
- signal,
359
- message
360
- })
361
- } finally {
362
- signal.clear()
363
- }
337
+ this._updateSentStats(message.blocks)
338
+ }, {
339
+ peerId,
340
+ signal: options?.signal,
341
+ message
342
+ })
364
343
  }
365
344
 
366
345
  /**
@@ -409,71 +388,3 @@ export class Network extends TypedEventEmitter<NetworkEvents> {
409
388
  this.metrics.blocksSent?.increment(blocks.length)
410
389
  }
411
390
  }
412
-
413
- function mergeMessages (messageA: BitswapMessage, messageB: BitswapMessage): BitswapMessage {
414
- const wantListEntries = new Map<string, WantlistEntry>(
415
- (messageA.wantlist?.entries ?? []).map(entry => ([
416
- base64.encode(entry.cid),
417
- entry
418
- ]))
419
- )
420
-
421
- for (const entry of messageB.wantlist?.entries ?? []) {
422
- const key = base64.encode(entry.cid)
423
- const existingEntry = wantListEntries.get(key)
424
-
425
- if (existingEntry != null) {
426
- // take highest priority
427
- if (existingEntry.priority > entry.priority) {
428
- entry.priority = existingEntry.priority
429
- }
430
-
431
- // take later values if passed, otherwise use earlier ones
432
- entry.cancel = entry.cancel ?? existingEntry.cancel
433
- entry.wantType = entry.wantType ?? existingEntry.wantType
434
- entry.sendDontHave = entry.sendDontHave ?? existingEntry.sendDontHave
435
- }
436
-
437
- wantListEntries.set(key, entry)
438
- }
439
-
440
- const blockPresences = new Map<string, BlockPresence>(
441
- messageA.blockPresences.map(presence => ([
442
- base64.encode(presence.cid),
443
- presence
444
- ]))
445
- )
446
-
447
- for (const blockPresence of messageB.blockPresences) {
448
- const key = base64.encode(blockPresence.cid)
449
-
450
- // override earlier block presence with later one as if duplicated it is
451
- // likely to be more accurate since it is more recent
452
- blockPresences.set(key, blockPresence)
453
- }
454
-
455
- const blocks = new Map<string, Block>(
456
- messageA.blocks.map(block => ([
457
- base64.encode(block.data),
458
- block
459
- ]))
460
- )
461
-
462
- for (const block of messageB.blocks) {
463
- const key = base64.encode(block.data)
464
-
465
- blocks.set(key, block)
466
- }
467
-
468
- const output: BitswapMessage = {
469
- wantlist: {
470
- full: messageA.wantlist?.full ?? messageB.wantlist?.full ?? false,
471
- entries: [...wantListEntries.values()]
472
- },
473
- blockPresences: [...blockPresences.values()],
474
- blocks: [...blocks.values()],
475
- pendingBytes: messageA.pendingBytes + messageB.pendingBytes
476
- }
477
-
478
- return output
479
- }
package/src/pb/message.ts CHANGED
@@ -4,7 +4,7 @@
4
4
  /* eslint-disable @typescript-eslint/no-unnecessary-boolean-literal-compare */
5
5
  /* eslint-disable @typescript-eslint/no-empty-interface */
6
6
 
7
- import { type Codec, decodeMessage, encodeMessage, enumeration, message } from 'protons-runtime'
7
+ import { type Codec, CodeError, decodeMessage, type DecodeOptions, encodeMessage, enumeration, message } from 'protons-runtime'
8
8
  import { alloc as uint8ArrayAlloc } from 'uint8arrays/alloc'
9
9
  import type { Uint8ArrayList } from 'uint8arraylist'
10
10
 
@@ -69,7 +69,7 @@ export namespace WantlistEntry {
69
69
  if (opts.lengthDelimited !== false) {
70
70
  w.ldelim()
71
71
  }
72
- }, (reader, length) => {
72
+ }, (reader, length, opts = {}) => {
73
73
  const obj: any = {
74
74
  cid: uint8ArrayAlloc(0),
75
75
  priority: 0
@@ -119,8 +119,8 @@ export namespace WantlistEntry {
119
119
  return encodeMessage(obj, WantlistEntry.codec())
120
120
  }
121
121
 
122
- export const decode = (buf: Uint8Array | Uint8ArrayList): WantlistEntry => {
123
- return decodeMessage(buf, WantlistEntry.codec())
122
+ export const decode = (buf: Uint8Array | Uint8ArrayList, opts?: DecodeOptions<WantlistEntry>): WantlistEntry => {
123
+ return decodeMessage(buf, WantlistEntry.codec(), opts)
124
124
  }
125
125
  }
126
126
 
@@ -154,7 +154,7 @@ export namespace Wantlist {
154
154
  if (opts.lengthDelimited !== false) {
155
155
  w.ldelim()
156
156
  }
157
- }, (reader, length) => {
157
+ }, (reader, length, opts = {}) => {
158
158
  const obj: any = {
159
159
  entries: []
160
160
  }
@@ -166,7 +166,13 @@ export namespace Wantlist {
166
166
 
167
167
  switch (tag >>> 3) {
168
168
  case 1: {
169
- obj.entries.push(WantlistEntry.codec().decode(reader, reader.uint32()))
169
+ if (opts.limits?.entries != null && obj.entries.length === opts.limits.entries) {
170
+ throw new CodeError('decode error - map field "entries" had too many elements', 'ERR_MAX_LENGTH')
171
+ }
172
+
173
+ obj.entries.push(WantlistEntry.codec().decode(reader, reader.uint32(), {
174
+ limits: opts.limits?.entries$
175
+ }))
170
176
  break
171
177
  }
172
178
  case 2: {
@@ -191,8 +197,8 @@ export namespace Wantlist {
191
197
  return encodeMessage(obj, Wantlist.codec())
192
198
  }
193
199
 
194
- export const decode = (buf: Uint8Array | Uint8ArrayList): Wantlist => {
195
- return decodeMessage(buf, Wantlist.codec())
200
+ export const decode = (buf: Uint8Array | Uint8ArrayList, opts?: DecodeOptions<Wantlist>): Wantlist => {
201
+ return decodeMessage(buf, Wantlist.codec(), opts)
196
202
  }
197
203
  }
198
204
 
@@ -224,7 +230,7 @@ export namespace Block {
224
230
  if (opts.lengthDelimited !== false) {
225
231
  w.ldelim()
226
232
  }
227
- }, (reader, length) => {
233
+ }, (reader, length, opts = {}) => {
228
234
  const obj: any = {
229
235
  prefix: uint8ArrayAlloc(0),
230
236
  data: uint8ArrayAlloc(0)
@@ -262,8 +268,8 @@ export namespace Block {
262
268
  return encodeMessage(obj, Block.codec())
263
269
  }
264
270
 
265
- export const decode = (buf: Uint8Array | Uint8ArrayList): Block => {
266
- return decodeMessage(buf, Block.codec())
271
+ export const decode = (buf: Uint8Array | Uint8ArrayList, opts?: DecodeOptions<Block>): Block => {
272
+ return decodeMessage(buf, Block.codec(), opts)
267
273
  }
268
274
  }
269
275
 
@@ -310,7 +316,7 @@ export namespace BlockPresence {
310
316
  if (opts.lengthDelimited !== false) {
311
317
  w.ldelim()
312
318
  }
313
- }, (reader, length) => {
319
+ }, (reader, length, opts = {}) => {
314
320
  const obj: any = {
315
321
  cid: uint8ArrayAlloc(0),
316
322
  type: BlockPresenceType.HaveBlock
@@ -348,8 +354,8 @@ export namespace BlockPresence {
348
354
  return encodeMessage(obj, BlockPresence.codec())
349
355
  }
350
356
 
351
- export const decode = (buf: Uint8Array | Uint8ArrayList): BlockPresence => {
352
- return decodeMessage(buf, BlockPresence.codec())
357
+ export const decode = (buf: Uint8Array | Uint8ArrayList, opts?: DecodeOptions<BlockPresence>): BlockPresence => {
358
+ return decodeMessage(buf, BlockPresence.codec(), opts)
353
359
  }
354
360
  }
355
361
 
@@ -397,7 +403,7 @@ export namespace BitswapMessage {
397
403
  if (opts.lengthDelimited !== false) {
398
404
  w.ldelim()
399
405
  }
400
- }, (reader, length) => {
406
+ }, (reader, length, opts = {}) => {
401
407
  const obj: any = {
402
408
  blocks: [],
403
409
  blockPresences: [],
@@ -411,15 +417,29 @@ export namespace BitswapMessage {
411
417
 
412
418
  switch (tag >>> 3) {
413
419
  case 1: {
414
- obj.wantlist = Wantlist.codec().decode(reader, reader.uint32())
420
+ obj.wantlist = Wantlist.codec().decode(reader, reader.uint32(), {
421
+ limits: opts.limits?.wantlist
422
+ })
415
423
  break
416
424
  }
417
425
  case 3: {
418
- obj.blocks.push(Block.codec().decode(reader, reader.uint32()))
426
+ if (opts.limits?.blocks != null && obj.blocks.length === opts.limits.blocks) {
427
+ throw new CodeError('decode error - map field "blocks" had too many elements', 'ERR_MAX_LENGTH')
428
+ }
429
+
430
+ obj.blocks.push(Block.codec().decode(reader, reader.uint32(), {
431
+ limits: opts.limits?.blocks$
432
+ }))
419
433
  break
420
434
  }
421
435
  case 4: {
422
- obj.blockPresences.push(BlockPresence.codec().decode(reader, reader.uint32()))
436
+ if (opts.limits?.blockPresences != null && obj.blockPresences.length === opts.limits.blockPresences) {
437
+ throw new CodeError('decode error - map field "blockPresences" had too many elements', 'ERR_MAX_LENGTH')
438
+ }
439
+
440
+ obj.blockPresences.push(BlockPresence.codec().decode(reader, reader.uint32(), {
441
+ limits: opts.limits?.blockPresences$
442
+ }))
423
443
  break
424
444
  }
425
445
  case 5: {
@@ -444,7 +464,7 @@ export namespace BitswapMessage {
444
464
  return encodeMessage(obj, BitswapMessage.codec())
445
465
  }
446
466
 
447
- export const decode = (buf: Uint8Array | Uint8ArrayList): BitswapMessage => {
448
- return decodeMessage(buf, BitswapMessage.codec())
467
+ export const decode = (buf: Uint8Array | Uint8ArrayList, opts?: DecodeOptions<BitswapMessage>): BitswapMessage => {
468
+ return decodeMessage(buf, BitswapMessage.codec(), opts)
449
469
  }
450
470
  }
@@ -6,7 +6,7 @@ import { Ledger } from './ledger.js'
6
6
  import type { BitswapNotifyProgressEvents, WantListEntry } from '../index.js'
7
7
  import type { Network } from '../network.js'
8
8
  import type { BitswapMessage } from '../pb/message.js'
9
- import type { ComponentLogger, Libp2p, Logger, PeerId } from '@libp2p/interface'
9
+ import type { ComponentLogger, Libp2p, Logger, Metrics, PeerId } from '@libp2p/interface'
10
10
  import type { PeerMap } from '@libp2p/peer-collections'
11
11
  import type { Blockstore } from 'interface-blockstore'
12
12
  import type { AbortOptions } from 'it-length-prefixed-stream'
@@ -21,6 +21,7 @@ export interface PeerWantListsComponents {
21
21
  network: Network
22
22
  libp2p: Libp2p
23
23
  logger: ComponentLogger
24
+ metrics?: Metrics
24
25
  }
25
26
 
26
27
  export interface PeerLedger {
@@ -37,16 +38,18 @@ export class PeerWantLists {
37
38
  public readonly ledgerMap: PeerMap<Ledger>
38
39
  private readonly maxSizeReplaceHasWithBlock?: number
39
40
  private readonly log: Logger
41
+ private readonly logger: ComponentLogger
40
42
 
41
43
  constructor (components: PeerWantListsComponents, init: PeerWantListsInit = {}) {
42
44
  this.blockstore = components.blockstore
43
45
  this.network = components.network
44
46
  this.maxSizeReplaceHasWithBlock = init.maxSizeReplaceHasWithBlock
45
47
  this.log = components.logger.forComponent('helia:bitswap:peer-want-lists')
48
+ this.logger = components.logger
46
49
 
47
50
  this.ledgerMap = trackedPeerMap({
48
51
  name: 'helia_bitswap_ledger_map',
49
- metrics: components.libp2p.metrics
52
+ metrics: components.metrics
50
53
  })
51
54
 
52
55
  this.network.addEventListener('bitswap:message', (evt) => {
@@ -100,7 +103,8 @@ export class PeerWantLists {
100
103
  ledger = new Ledger({
101
104
  peerId,
102
105
  blockstore: this.blockstore,
103
- network: this.network
106
+ network: this.network,
107
+ logger: this.logger
104
108
  }, {
105
109
  maxSizeReplaceHasWithBlock: this.maxSizeReplaceHasWithBlock
106
110
  })
@@ -141,6 +145,7 @@ export class PeerWantLists {
141
145
  }
142
146
  }
143
147
 
148
+ this.log('send blocks to peer')
144
149
  await ledger.sendBlocksToPeer()
145
150
  }
146
151
 
@@ -3,7 +3,7 @@ import { DEFAULT_MAX_SIZE_REPLACE_HAS_WITH_BLOCK } from '../constants.js'
3
3
  import { BlockPresenceType, type BitswapMessage, WantType } from '../pb/message.js'
4
4
  import { cidToPrefix } from '../utils/cid-prefix.js'
5
5
  import type { Network } from '../network.js'
6
- import type { PeerId } from '@libp2p/interface'
6
+ import type { ComponentLogger, Logger, PeerId } from '@libp2p/interface'
7
7
  import type { Blockstore } from 'interface-blockstore'
8
8
  import type { AbortOptions } from 'it-length-prefixed-stream'
9
9
  import type { CID } from 'multiformats/cid'
@@ -12,6 +12,7 @@ export interface LedgerComponents {
12
12
  peerId: PeerId
13
13
  blockstore: Blockstore
14
14
  network: Network
15
+ logger: ComponentLogger
15
16
  }
16
17
 
17
18
  export interface LedgerInit {
@@ -56,12 +57,14 @@ export class Ledger {
56
57
  public bytesReceived: number
57
58
  public lastExchange?: number
58
59
  private readonly maxSizeReplaceHasWithBlock: number
60
+ private readonly log: Logger
59
61
 
60
62
  constructor (components: LedgerComponents, init: LedgerInit) {
61
63
  this.peerId = components.peerId
62
64
  this.blockstore = components.blockstore
63
65
  this.network = components.network
64
66
  this.wants = new Map()
67
+ this.log = components.logger.forComponent(`helia:bitswap:ledger:${components.peerId}`)
65
68
 
66
69
  this.exchangeCount = 0
67
70
  this.bytesSent = 0
@@ -96,6 +99,8 @@ export class Ledger {
96
99
  const has = await this.blockstore.has(entry.cid, options)
97
100
 
98
101
  if (!has) {
102
+ this.log('do not have block for %c', entry.cid)
103
+
99
104
  // we don't have the requested block and the remote is not interested
100
105
  // in us telling them that
101
106
  if (!entry.sendDontHave) {
@@ -121,6 +126,7 @@ export class Ledger {
121
126
  // do they want the block or just us to tell them we have the block
122
127
  if (entry.wantType === WantType.WantHave) {
123
128
  if (block.byteLength < this.maxSizeReplaceHasWithBlock) {
129
+ this.log('sending have and block for %c', entry.cid)
124
130
  // if the block is small we just send it to them
125
131
  sentBlocks.add(key)
126
132
  message.blocks.push({
@@ -128,6 +134,7 @@ export class Ledger {
128
134
  prefix: cidToPrefix(entry.cid)
129
135
  })
130
136
  } else {
137
+ this.log('sending have for %c', entry.cid)
131
138
  // otherwise tell them we have the block
132
139
  message.blockPresences.push({
133
140
  cid: entry.cid.bytes,
@@ -135,6 +142,7 @@ export class Ledger {
135
142
  })
136
143
  }
137
144
  } else {
145
+ this.log('sending block for %c', entry.cid)
138
146
  // they want the block, send it to them
139
147
  sentBlocks.add(key)
140
148
  message.blocks.push({
@@ -146,7 +154,9 @@ export class Ledger {
146
154
 
147
155
  // only send the message if we actually have something to send
148
156
  if (message.blocks.length > 0 || message.blockPresences.length > 0) {
157
+ this.log('sending message')
149
158
  await this.network.sendMessage(this.peerId, message, options)
159
+ this.log('sent message')
150
160
 
151
161
  // update accounting
152
162
  this.sentBytes(message.blocks.reduce((acc, curr) => acc + curr.data.byteLength, 0))
package/src/stats.ts CHANGED
@@ -1,7 +1,8 @@
1
- import type { Libp2p, MetricGroup, PeerId } from '@libp2p/interface'
1
+ import type { Libp2p, MetricGroup, Metrics, PeerId } from '@libp2p/interface'
2
2
 
3
3
  export interface StatsComponents {
4
4
  libp2p: Libp2p
5
+ metrics?: Metrics
5
6
  }
6
7
 
7
8
  export class Stats {
@@ -11,10 +12,10 @@ export class Stats {
11
12
  private readonly duplicateDataReceived?: MetricGroup
12
13
 
13
14
  constructor (components: StatsComponents) {
14
- this.blocksReceived = components.libp2p.metrics?.registerMetricGroup('helia_bitswap_received_blocks')
15
- this.duplicateBlocksReceived = components.libp2p.metrics?.registerMetricGroup('helia_bitswap_duplicate_received_blocks')
16
- this.dataReceived = components.libp2p.metrics?.registerMetricGroup('helia_bitswap_data_received_bytes')
17
- this.duplicateDataReceived = components.libp2p.metrics?.registerMetricGroup('helia_bitswap_duplicate_data_received_bytes')
15
+ this.blocksReceived = components.metrics?.registerMetricGroup('helia_bitswap_received_blocks')
16
+ this.duplicateBlocksReceived = components.metrics?.registerMetricGroup('helia_bitswap_duplicate_received_blocks')
17
+ this.dataReceived = components.metrics?.registerMetricGroup('helia_bitswap_data_received_bytes')
18
+ this.duplicateDataReceived = components.metrics?.registerMetricGroup('helia_bitswap_duplicate_data_received_bytes')
18
19
  }
19
20
 
20
21
  updateBlocksReceived (count: number = 1, peerId?: PeerId): void {