@helia/bitswap 3.1.2-92480ee8 → 3.1.2-eaeb734d

package/src/peer-want-lists/index.ts

@@ -1,9 +1,5 @@
-/* eslint-disable max-depth */
-
 import { trackedPeerMap } from '@libp2p/peer-collections'
 import { CID } from 'multiformats/cid'
-import { toString as uint8ArrayToString } from 'uint8arrays/to-string'
-import { WantType } from '../pb/message.js'
 import { Ledger } from './ledger.js'
 import type { BitswapNotifyProgressEvents, PeerWantListEntry } from '../index.js'
 import type { Network } from '../network.js'
@@ -16,6 +12,7 @@ import type { ProgressOptions } from 'progress-events'
 export interface PeerWantListsInit {
   maxSizeReplaceHasWithBlock?: number
   doNotResendBlockWindow?: number
+  maxWantListSize?: number
 }
 
 export interface PeerWantListsComponents {
@@ -40,6 +37,7 @@ export class PeerWantLists {
   public readonly ledgerMap: PeerMap<Ledger>
   private readonly maxSizeReplaceHasWithBlock?: number
   private readonly doNotResendBlockWindow?: number
+  private readonly maxWantListSize?: number
   private readonly log: Logger
   private readonly logger: ComponentLogger
 
@@ -48,6 +46,7 @@ export class PeerWantLists {
     this.network = components.network
     this.maxSizeReplaceHasWithBlock = init.maxSizeReplaceHasWithBlock
     this.doNotResendBlockWindow = init.doNotResendBlockWindow
+    this.maxWantListSize = init.maxWantListSize
     this.log = components.logger.forComponent('helia:bitswap:peer-want-lists')
     this.logger = components.logger
 
@@ -93,7 +92,7 @@ export class PeerWantLists {
     // remove any expired wants
     ledger.removeExpiredWants()
 
-    return [...ledger.wants.values()]
+    return ledger.getWants()
   }
 
   peers (): PeerId[] {
@@ -114,7 +113,8 @@ export class PeerWantLists {
         logger: this.logger
       }, {
         maxSizeReplaceHasWithBlock: this.maxSizeReplaceHasWithBlock,
-        doNotResendBlockWindow: this.doNotResendBlockWindow
+        doNotResendBlockWindow: this.doNotResendBlockWindow,
+        maxWantListSize: this.maxWantListSize
       })
       this.ledgerMap.set(peerId, ledger)
     }
@@ -125,68 +125,18 @@ export class PeerWantLists {
     // remove any expired wants
     ledger.removeExpiredWants()
 
-    if (message.wantlist != null) {
-      // if the message has a full wantlist, clear the current wantlist
-      if (message.wantlist.full === true) {
-        ledger.wants.clear()
-      }
-
-      // clear cancelled wants and add new wants to the ledger
-      for (const entry of message.wantlist.entries) {
-        const cid = CID.decode(entry.cid)
-        const cidStr = uint8ArrayToString(cid.multihash.bytes, 'base64')
-
-        if (entry.cancel === true) {
-          this.log('peer %p cancelled want of block for %c', peerId, cid)
-          ledger.wants.delete(cidStr)
-        } else {
-          if (entry.wantType === WantType.WantHave) {
-            this.log('peer %p wanted block presence for %c', peerId, cid)
-          } else {
-            this.log('peer %p wanted block for %c', peerId, cid)
-          }
-
-          const existingWant = ledger.wants.get(cidStr)
-
-          // we are already tracking a want for this CID, just update the fields
-          if (existingWant != null) {
-            const sentOrSending = existingWant.status === 'sent' || existingWant.status === 'sending'
-            const wantTypeUpgrade = existingWant.wantType === WantType.WantHave && (entry.wantType == null || entry.wantType === WantType.WantBlock)
-
-            // allow upgrade from WantHave to WantBlock if we've previously
-            // sent or are sending a WantHave
-            if (sentOrSending && wantTypeUpgrade) {
-              existingWant.status = 'want'
-            }
-
-            existingWant.priority = entry.priority
-            existingWant.wantType = entry.wantType ?? WantType.WantBlock
-            existingWant.sendDontHave = entry.sendDontHave ?? false
-            continue
-          }
-
-          // add a new want
-          ledger.wants.set(cidStr, {
-            cid,
-            priority: entry.priority,
-            wantType: entry.wantType ?? WantType.WantBlock,
-            sendDontHave: entry.sendDontHave ?? false,
-            status: 'want'
-          })
-        }
-      }
-    }
+    // add new wants
+    ledger.addWants(message.wantlist)
 
     this.log('send blocks to peer')
     await ledger.sendBlocksToPeer()
   }
 
   async receivedBlock (cid: CID, options: ProgressOptions<BitswapNotifyProgressEvents> & AbortOptions): Promise<void> {
-    const cidStr = uint8ArrayToString(cid.multihash.bytes, 'base64')
     const ledgers: Ledger[] = []
 
     for (const ledger of this.ledgerMap.values()) {
-      if (ledger.wants.has(cidStr)) {
+      if (ledger.hasWant(cid)) {
         ledgers.push(ledger)
       }
     }

package/src/peer-want-lists/ledger.ts

@@ -1,12 +1,14 @@
 import toBuffer from 'it-to-buffer'
-import { DEFAULT_MAX_SIZE_REPLACE_HAS_WITH_BLOCK, DEFAULT_DO_NOT_RESEND_BLOCK_WINDOW } from '../constants.js'
+import { CID } from 'multiformats/cid'
+import { toString as uint8ArrayToString } from 'uint8arrays/to-string'
+import { DEFAULT_MAX_SIZE_REPLACE_HAS_WITH_BLOCK, DEFAULT_DO_NOT_RESEND_BLOCK_WINDOW, DEFAULT_MAX_WANTLIST_SIZE } from '../constants.js'
 import { BlockPresenceType, WantType } from '../pb/message.js'
 import { QueuedBitswapMessage } from '../utils/bitswap-message.js'
 import { cidToPrefix } from '../utils/cid-prefix.js'
 import type { Network } from '../network.js'
+import type { Wantlist } from '../pb/message.js'
 import type { AbortOptions, ComponentLogger, Logger, PeerId } from '@libp2p/interface'
 import type { Blockstore } from 'interface-blockstore'
-import type { CID } from 'multiformats/cid'
 
 export interface LedgerComponents {
   peerId: PeerId
@@ -18,6 +20,7 @@ export interface LedgerComponents {
 export interface LedgerInit {
   maxSizeReplaceHasWithBlock?: number
   doNotResendBlockWindow?: number
+  maxWantListSize?: number
 }
 
 export interface PeerWantListEntry {
@@ -59,13 +62,29 @@ export interface PeerWantListEntry {
    * peer. Once it has expired the peer can request the block a subsequent time.
    */
   expires?: number
+
+  /**
+   * A timestamp of when this entry was created
+   */
+  created: number
+
+  /**
+   * If this field is false, we have attempted to send this WantList entry but
+   * found there is no block for the CID in the blockstore and we are
+   * optimistically waiting to see if we come across it later.
+   *
+   * We only perform the check when we are about to send the block, by which
+   * point the entry status is 'sending' so this value with either be false or
+   * not set
+   */
+  haveBlock?: false
 }
 
 export class Ledger {
   public peerId: PeerId
   private readonly blockstore: Blockstore
   private readonly network: Network
-  public wants: Map<string, PeerWantListEntry>
+  private wants: Map<string, PeerWantListEntry>
   public exchangeCount: number
   public bytesSent: number
   public bytesReceived: number
@@ -73,6 +92,7 @@ export class Ledger {
   private readonly maxSizeReplaceHasWithBlock: number
   private readonly log: Logger
   private readonly doNotResendBlockWindow: number
+  private readonly maxWantListSize: number
 
   constructor (components: LedgerComponents, init: LedgerInit) {
     this.peerId = components.peerId
@@ -86,6 +106,7 @@ export class Ledger {
     this.bytesReceived = 0
     this.maxSizeReplaceHasWithBlock = init.maxSizeReplaceHasWithBlock ?? DEFAULT_MAX_SIZE_REPLACE_HAS_WITH_BLOCK
     this.doNotResendBlockWindow = init.doNotResendBlockWindow ?? DEFAULT_DO_NOT_RESEND_BLOCK_WINDOW
+    this.maxWantListSize = init.maxWantListSize ?? DEFAULT_MAX_WANTLIST_SIZE
   }
 
   sentBytes (n: number): void {
@@ -113,6 +134,133 @@ export class Ledger {
     })
   }
 
+  public addWants (wantlist?: Wantlist): void {
+    if (wantlist == null) {
+      return
+    }
+
+    // if the message has a full wantlist, remove all entries not currently
+    // being sent to the peer
+    if (wantlist.full === true) {
+      this.wants.forEach((value, key) => {
+        if (value.status === 'want') {
+          this.wants.delete(key)
+        }
+      })
+    }
+
+    // clear cancelled wants and add new wants to the ledger
+    for (const entry of wantlist.entries) {
+      const cid = CID.decode(entry.cid)
+      const cidStr = uint8ArrayToString(cid.multihash.bytes, 'base64')
+
+      if (entry.cancel === true) {
+        this.log('peer %p cancelled want of block for %c', this.peerId, cid)
+        this.wants.delete(cidStr)
+      } else {
+        if (entry.wantType === WantType.WantHave) {
+          this.log('peer %p wanted block presence for %c', this.peerId, cid)
+        } else {
+          this.log('peer %p wanted block for %c', this.peerId, cid)
+        }
+
+        const existingWant = this.wants.get(cidStr)
+
+        // we are already tracking a want for this CID, just update the fields
+        if (existingWant != null) {
+          const sentOrSending = existingWant.status === 'sent' || existingWant.status === 'sending'
+          const wantTypeUpgrade = existingWant.wantType === WantType.WantHave && (entry.wantType == null || entry.wantType === WantType.WantBlock)
+
+          // allow upgrade from WantHave to WantBlock if we've previously
+          // sent or are sending a WantHave
+          if (sentOrSending && wantTypeUpgrade) {
+            existingWant.status = 'want'
+          }
+
+          existingWant.priority = entry.priority
+          existingWant.wantType = entry.wantType ?? WantType.WantBlock
+          existingWant.sendDontHave = entry.sendDontHave ?? false
+          continue
+        }
+
+        // add a new want
+        this.wants.set(cidStr, {
+          cid,
+          priority: entry.priority,
+          wantType: entry.wantType ?? WantType.WantBlock,
+          sendDontHave: entry.sendDontHave ?? false,
+          status: 'want',
+          created: Date.now()
+        })
+      }
+    }
+
+    // if we have exceeded maxWantListSize, truncate the list - first select
+    // wants that are not currently being sent to the user
+    const wants = [...this.wants.entries()]
+      .filter(([key, entry]) => entry.status === 'want')
+
+    if (wants.length > this.maxWantListSize) {
+      this.truncateWants(wants)
+    }
+  }
+
+  private truncateWants (wants: Array<[string, PeerWantListEntry]>): void {
+    // sort wants by priority, lack of block presence, then age so the wants
+    // to be evicted are a older, low priority wants that we don't have the
+    // block for
+    wants = wants
+      .sort((a, b) => {
+        if (a[1].created < b[1].created) {
+          return -1
+        }
+
+        if (b[1].created < a[1].created) {
+          return 1
+        }
+
+        return 0
+      })
+      .sort((a, b) => {
+        if (a[1].haveBlock === false) {
+          return -1
+        }
+
+        if (b[1].haveBlock === false) {
+          return 1
+        }
+
+        return 0
+      })
+      .sort((a, b) => {
+        if (a[1].priority < b[1].priority) {
+          return -1
+        }
+
+        if (b[1].priority < a[1].priority) {
+          return 1
+        }
+
+        return 0
+      })
+
+    const toRemove = wants.length - this.maxWantListSize
+
+    for (let i = 0; i < toRemove; i++) {
+      this.wants.delete(wants[i][0])
+    }
+  }
+
+  public getWants (): PeerWantListEntry[] {
+    return [...this.wants.values()]
+  }
+
+  public hasWant (cid: CID): boolean {
+    const cidStr = uint8ArrayToString(cid.multihash.bytes, 'base64')
+
+    return this.wants.has(cidStr)
+  }
+
   public async sendBlocksToPeer (options?: AbortOptions): Promise<void> {
     const message = new QueuedBitswapMessage()
     const sentBlocks = new Set<string>()
@@ -179,6 +327,9 @@ export class Ledger {
         // reset status to try again later
         entry.status = 'want'
 
+        // used to maybe delete this want later if the want list grows too large
+        entry.haveBlock = false
+
         this.log('do not have block for %c', entry.cid)
 
         // we don't have the requested block and the remote is not interested