@helia/bitswap 4.0.8 → 4.0.9-3aeb3977

package/src/index.ts

@@ -14,7 +14,7 @@ import type { WantType } from './pb/message.ts'
 import type { BlockBrokerGetBlockProgressEvents, CreateSessionOptions, HasherLoader, HeliaMixin, ProviderOptions, SessionBlockBroker } from '@helia/interface'
 import type { Routing } from '@helia/interface'
 import type { HeliaWithLibp2p } from '@helia/libp2p'
-import type { Libp2p, AbortOptions, Startable, ComponentLogger, Metrics, PeerId } from '@libp2p/interface'
+import type { Libp2p, AbortOptions, Startable, ComponentLogger, Metrics, PeerId, ServiceMap } from '@libp2p/interface'
 import type { Blockstore } from 'interface-blockstore'
 import type { CID } from 'multiformats/cid'
 import type { ProgressEvent, ProgressOptions } from 'progress-events'
@@ -127,7 +127,8 @@ export interface BitswapOptions {
   protocol?: string
 
   /**
-   * When sending want list updates to peers, how many messages to send at once
+   * When sending messages to peers (our wantlist updates, blocks from their
+   * wantlist), how many messages to send at once
    *
    * @default 50
    */
@@ -149,12 +150,26 @@ export interface BitswapOptions {
   messageReceiveTimeout?: number
 
   /**
-   * When sending blocks to peers, how many messages to send at once
+   * When sending blocks to individual peers, how many messages to send at once.
+   * This should be left as the default.
    *
-   * @default 50
+   * @default 1
    */
   sendBlocksConcurrency?: number
 
+  /**
+   * Sending blocks is triggered by an incoming Bitswap message from the peer.
+   * To prevent runaway message sending, message send operations are stored in
+   * a queue. This setting defines the maximum queue length.
+   *
+   * Once the queue exceeds this length no further message send operations will
+   * be queued. Local wantlists are updated synchronously so the updated wants
+   * should be handled by send operations still in the queue.
+   *
+   * @default 10
+   */
+  sendBlocksMaxQueueLength?: number
+
   /**
    * When sending blocks to peers, timeout after this many milliseconds.
    * This is useful for preventing slow/large peer-connections from consuming
@@ -234,7 +249,7 @@ export interface BitswapOptions {
 /**
  * Return a Helia node augmented with a libp2p instance
  */
-export function withBitswap (helia: HeliaWithLibp2p, options: BitswapBlockBrokerInit = {}): HeliaWithLibp2p {
+export function withBitswap <M extends ServiceMap = ServiceMap> (helia: HeliaWithLibp2p<M>, options: BitswapBlockBrokerInit = {}): HeliaWithLibp2p<M> {
   const mixin: HeliaMixin<HeliaWithLibp2p> = {
     name: 'bitswap',
     start: async (helia) => {

package/src/peer-want-lists/index.ts

@@ -4,8 +4,9 @@ import { Ledger } from './ledger.ts'
 import type { BitswapNotifyProgressEvents, PeerWantListEntry } from '../index.ts'
 import type { Network } from '../network.ts'
 import type { BitswapMessage } from '../pb/message.ts'
-import type { AbortOptions, ComponentLogger, Libp2p, Logger, Metrics, PeerId } from '@libp2p/interface'
+import type { AbortOptions, ComponentLogger, Libp2p, Logger, Metrics, PeerId, Startable } from '@libp2p/interface'
 import type { PeerMap } from '@libp2p/peer-collections'
+import type { Queue } from '@libp2p/utils'
 import type { Blockstore } from 'interface-blockstore'
 import type { ProgressOptions } from 'progress-events'
 
@@ -13,6 +14,8 @@ export interface PeerWantListsInit {
   maxSizeReplaceHasWithBlock?: number
   doNotResendBlockWindow?: number
   maxWantListSize?: number
+  sendBlocksConcurrency?: number
+  sendBlocksMaxQueueLength?: number
 }
 
 export interface PeerWantListsComponents {
@@ -29,15 +32,18 @@ export interface PeerLedger {
   sent: number
   received: number
   exchanged: number
+  sendQueue: Queue
 }
 
-export class PeerWantLists {
+export class PeerWantLists implements Startable {
   public blockstore: Blockstore
   public network: Network
   public readonly ledgerMap: PeerMap<Ledger>
   private readonly maxSizeReplaceHasWithBlock?: number
   private readonly doNotResendBlockWindow?: number
   private readonly maxWantListSize?: number
+  private readonly sendBlocksConcurrency?: number
+  private readonly sendBlocksMaxQueueLength?: number
   private readonly log: Logger
   private readonly logger: ComponentLogger
 
@@ -47,6 +53,8 @@ export class PeerWantLists {
     this.maxSizeReplaceHasWithBlock = init.maxSizeReplaceHasWithBlock
     this.doNotResendBlockWindow = init.doNotResendBlockWindow
     this.maxWantListSize = init.maxWantListSize
+    this.sendBlocksConcurrency = init.sendBlocksConcurrency
+    this.sendBlocksMaxQueueLength = init.sendBlocksMaxQueueLength
     this.log = components.logger.forComponent('helia:bitswap:peer-want-lists')
     this.logger = components.logger
 
@@ -56,16 +64,27 @@ export class PeerWantLists {
     })
 
     this.network.addEventListener('bitswap:message', (evt) => {
-      this.receiveMessage(evt.detail.connection.remotePeer, evt.detail.message)
-        .catch(err => {
-          this.log.error('error receiving bitswap message from %p - %e', evt.detail.connection.remotePeer, err)
-        })
+      try {
+        this.receiveMessage(evt.detail.connection.remotePeer, evt.detail.message)
+      } catch (err) {
+        this.log.error('error receiving bitswap message from %p - %e', evt.detail.connection.remotePeer, err)
+      }
     })
     this.network.addEventListener('peer:disconnected', evt => {
       this.peerDisconnected(evt.detail)
     })
   }
 
+  start (): void {
+
+  }
+
+  stop (): void {
+    this.ledgerMap.forEach(ledger => {
+      ledger.stop()
+    })
+  }
+
   ledgerForPeer (peerId: PeerId): PeerLedger | undefined {
     const ledger = this.ledgerMap.get(peerId)
 
@@ -78,7 +97,8 @@ export class PeerWantLists {
       value: ledger.debtRatio(),
       sent: ledger.bytesSent,
       received: ledger.bytesReceived,
-      exchanged: ledger.exchangeCount
+      exchanged: ledger.exchangeCount,
+      sendQueue: ledger.sendQueue
     }
   }
 
@@ -102,7 +122,7 @@ export class PeerWantLists {
   /**
    * Handle incoming messages
    */
-  async receiveMessage (peerId: PeerId, message: BitswapMessage): Promise<void> {
+  receiveMessage (peerId: PeerId, message: BitswapMessage): void {
     let ledger = this.ledgerMap.get(peerId)
 
     if (ledger == null) {
@@ -114,22 +134,14 @@ export class PeerWantLists {
       }, {
         maxSizeReplaceHasWithBlock: this.maxSizeReplaceHasWithBlock,
         doNotResendBlockWindow: this.doNotResendBlockWindow,
-        maxWantListSize: this.maxWantListSize
+        maxWantListSize: this.maxWantListSize,
+        sendBlocksConcurrency: this.sendBlocksConcurrency,
+        sendBlocksMaxQueueLength: this.sendBlocksMaxQueueLength
       })
       this.ledgerMap.set(peerId, ledger)
     }
 
-    // record the amount of block data received
-    ledger.receivedBytes(message.blocks?.reduce((acc, curr) => acc + curr.data.byteLength, 0) ?? 0)
-
-    // remove any expired wants
-    ledger.removeExpiredWants()
-
-    // add new wants
-    ledger.addWants(message.wantlist)
-
-    this.log('send blocks to peer')
-    await ledger.sendBlocksToPeer()
+    ledger.queueSendOperation(message)
   }
 
   async receivedBlock (cid: CID, options: ProgressOptions<BitswapNotifyProgressEvents> & AbortOptions): Promise<void> {
@@ -142,7 +154,7 @@ export class PeerWantLists {
     }
 
     await Promise.all(
-      ledgers.map(async (ledger) => ledger.sendBlocksToPeer(options))
+      ledgers.map(async (ledger) => ledger.queueSendOperation(undefined, options))
     )
   }
 

package/src/peer-want-lists/ledger.ts

@@ -1,14 +1,18 @@
+import { Queue } from '@libp2p/utils'
+import { NotFoundError } from 'interface-store'
 import toBuffer from 'it-to-buffer'
 import { CID } from 'multiformats/cid'
 import { toString as uint8ArrayToString } from 'uint8arrays/to-string'
-import { DEFAULT_MAX_SIZE_REPLACE_HAS_WITH_BLOCK, DEFAULT_DO_NOT_RESEND_BLOCK_WINDOW, DEFAULT_MAX_WANTLIST_SIZE } from '../constants.ts'
+import { DEFAULT_MAX_SIZE_REPLACE_HAS_WITH_BLOCK, DEFAULT_DO_NOT_RESEND_BLOCK_WINDOW, DEFAULT_MAX_WANTLIST_SIZE, DEFAULT_PEER_MESSAGE_SEND_CONCURRENCY, DEFAULT_PEER_MAX_MESSAGE_SEND_QUEUE_LENGTH } from '../constants.ts'
 import { BlockPresenceType, WantType } from '../pb/message.ts'
 import { QueuedBitswapMessage } from '../utils/bitswap-message.ts'
 import { cidToPrefix } from '../utils/cid-prefix.ts'
+import type { BitswapNotifyProgressEvents } from '../index.ts'
 import type { Network } from '../network.ts'
-import type { Wantlist } from '../pb/message.ts'
+import type { BitswapMessage, Wantlist } from '../pb/message.ts'
 import type { AbortOptions, ComponentLogger, Logger, PeerId } from '@libp2p/interface'
 import type { Blockstore } from 'interface-blockstore'
+import type { ProgressOptions } from 'progress-events'
 
 export interface LedgerComponents {
   peerId: PeerId
@@ -21,6 +25,9 @@ export interface LedgerInit {
   maxSizeReplaceHasWithBlock?: number
   doNotResendBlockWindow?: number
   maxWantListSize?: number
+  sendBlocksConcurrency?: number
+  sendBlocksMaxQueueLength?: number
+  sendBlocksTimeout?: number
 }
 
 export interface PeerWantListEntry {
@@ -89,6 +96,7 @@ export class Ledger {
   public bytesSent: number
   public bytesReceived: number
   public lastExchange?: number
+  public readonly sendQueue: Queue
   private readonly maxSizeReplaceHasWithBlock: number
   private readonly log: Logger
   private readonly doNotResendBlockWindow: number
@@ -107,6 +115,15 @@ export class Ledger {
     this.maxSizeReplaceHasWithBlock = init.maxSizeReplaceHasWithBlock ?? DEFAULT_MAX_SIZE_REPLACE_HAS_WITH_BLOCK
     this.doNotResendBlockWindow = init.doNotResendBlockWindow ?? DEFAULT_DO_NOT_RESEND_BLOCK_WINDOW
     this.maxWantListSize = init.maxWantListSize ?? DEFAULT_MAX_WANTLIST_SIZE
+
+    this.sendQueue = new Queue({
+      concurrency: init.sendBlocksConcurrency ?? DEFAULT_PEER_MESSAGE_SEND_CONCURRENCY,
+      maxSize: init.sendBlocksMaxQueueLength ?? DEFAULT_PEER_MAX_MESSAGE_SEND_QUEUE_LENGTH
+    })
+  }
+
+  stop (): void {
+    this.sendQueue.abort()
   }
 
   sentBytes (n: number): void {
@@ -261,7 +278,37 @@ export class Ledger {
     return this.wants.has(cidStr)
   }
 
-  public async sendBlocksToPeer (options?: AbortOptions): Promise<void> {
+  public queueSendOperation (message?: BitswapMessage, options?: ProgressOptions<BitswapNotifyProgressEvents> & AbortOptions): void {
+    if (message != null) {
+      // record the amount of block data received
+      this.receivedBytes(message.blocks?.reduce((acc, curr) => acc + curr.data.byteLength, 0) ?? 0)
+
+      // add new wants
+      this.addWants(message.wantlist)
+    }
+
+    try {
+      this.sendQueue.add(async (options) => {
+        this.log('send blocks to peer')
+        await this.sendBlocksToPeer(options)
+      }, options)
+        .catch(err => {
+          if (err.name === 'QueueFullError') {
+            this.log.error('Dropping bitswap message as peer send queue is full')
+          } else {
+            this.log.error('Error when trying to add queue a send operation')
+          }
+        })
+    } catch (err: any) {
+      if (err.name === 'QueueFullError') {
+        this.log.error('Dropping bitswap message as peer send queue is full')
+      } else {
+        throw err
+      }
+    }
+  }
+
+  private async sendBlocksToPeer (options?: AbortOptions): Promise<void> {
     const message = new QueuedBitswapMessage()
     const sentBlocks = new Set<string>()
 
@@ -270,7 +317,15 @@ export class Ledger {
 
     // pick unsent wants
     const unsent = [...this.wants.entries()]
-      .filter(([key, value]) => value.status === 'want')
+      .filter(([key, value]) => {
+        // don't process the same want more than once per send iteration
+        return value.status === 'want'
+      })
+
+    // break out of the for loop early if we have no work to do
+    if (unsent.length === 0) {
+      return
+    }
 
     // update status, ensure we don't send the same blocks repeatedly
     unsent.forEach(([key, value]) => {
@@ -279,6 +334,15 @@ export class Ledger {
 
     for (const [key, entry] of unsent) {
       try {
+        // call `.has` and not `.get` because `.get` can cause network
+        // operations to occur
+        const has = await this.blockstore.has(entry.cid, options)
+
+        if (!has) {
+          // can't sent a block we don't have
+          throw new NotFoundError()
+        }
+
         const block = await toBuffer(this.blockstore.get(entry.cid, options))
 
         // ensure we still need to send the block/status, status may have

package/dist/typedoc-urls.json

@@ -1,29 +0,0 @@
-{
-  "codec": "https://ipfs.github.io/helia/functions/_helia_bitswap.WantType.codec.html",
-  "WantType": "https://ipfs.github.io/helia/enums/_helia_bitswap.WantType.html",
-  "Bitswap": "https://ipfs.github.io/helia/interfaces/_helia_bitswap.Bitswap.html",
-  ".:Bitswap": "https://ipfs.github.io/helia/interfaces/_helia_bitswap.Bitswap.html",
-  "BitswapBlockBrokerInit": "https://ipfs.github.io/helia/interfaces/_helia_bitswap.BitswapBlockBrokerInit.html",
-  "BitswapComponents": "https://ipfs.github.io/helia/interfaces/_helia_bitswap.BitswapComponents.html",
-  ".:BitswapComponents": "https://ipfs.github.io/helia/interfaces/_helia_bitswap.BitswapComponents.html",
-  "BitswapOptions": "https://ipfs.github.io/helia/interfaces/_helia_bitswap.BitswapOptions.html",
-  ".:BitswapOptions": "https://ipfs.github.io/helia/interfaces/_helia_bitswap.BitswapOptions.html",
-  "BitswapProvider": "https://ipfs.github.io/helia/interfaces/_helia_bitswap.BitswapProvider.html",
-  "PeerWantListEntry": "https://ipfs.github.io/helia/interfaces/_helia_bitswap.PeerWantListEntry.html",
-  ".:PeerWantListEntry": "https://ipfs.github.io/helia/interfaces/_helia_bitswap.PeerWantListEntry.html",
-  "WantListEntry": "https://ipfs.github.io/helia/interfaces/_helia_bitswap.WantListEntry.html",
-  ".:WantListEntry": "https://ipfs.github.io/helia/interfaces/_helia_bitswap.WantListEntry.html",
-  "BitswapNetworkNotifyProgressEvents": "https://ipfs.github.io/helia/types/_helia_bitswap.BitswapNetworkNotifyProgressEvents.html",
-  "BitswapNetworkProgressEvents": "https://ipfs.github.io/helia/types/_helia_bitswap.BitswapNetworkProgressEvents.html",
-  "BitswapNetworkWantProgressEvents": "https://ipfs.github.io/helia/types/_helia_bitswap.BitswapNetworkWantProgressEvents.html",
-  "BitswapNotifyProgressEvents": "https://ipfs.github.io/helia/types/_helia_bitswap.BitswapNotifyProgressEvents.html",
-  ".:BitswapNotifyProgressEvents": "https://ipfs.github.io/helia/types/_helia_bitswap.BitswapNotifyProgressEvents.html",
-  "BitswapWantBlockProgressEvents": "https://ipfs.github.io/helia/types/_helia_bitswap.BitswapWantBlockProgressEvents.html",
-  ".:BitswapWantBlockProgressEvents": "https://ipfs.github.io/helia/types/_helia_bitswap.BitswapWantBlockProgressEvents.html",
-  "BitswapWantProgressEvents": "https://ipfs.github.io/helia/types/_helia_bitswap.BitswapWantProgressEvents.html",
-  ".:BitswapWantProgressEvents": "https://ipfs.github.io/helia/types/_helia_bitswap.BitswapWantProgressEvents.html",
-  "WantStatus": "https://ipfs.github.io/helia/types/_helia_bitswap.WantStatus.html",
-  ".:WantStatus": "https://ipfs.github.io/helia/types/_helia_bitswap.WantStatus.html",
-  "withBitswap": "https://ipfs.github.io/helia/functions/_helia_bitswap.withBitswap.html",
-  ".:withBitswap": "https://ipfs.github.io/helia/functions/_helia_bitswap.withBitswap.html"
-}