@helia/bitswap 3.1.2 → 3.1.3-83e1f40d

package/src/peer-want-lists/ledger.ts

@@ -1,12 +1,14 @@
 import toBuffer from 'it-to-buffer'
-import { DEFAULT_MAX_SIZE_REPLACE_HAS_WITH_BLOCK } from '../constants.js'
-import { BlockPresenceType, WantType } from '../pb/message.js'
-import { QueuedBitswapMessage } from '../utils/bitswap-message.js'
-import { cidToPrefix } from '../utils/cid-prefix.js'
-import type { Network } from '../network.js'
+import { CID } from 'multiformats/cid'
+import { toString as uint8ArrayToString } from 'uint8arrays/to-string'
+import { DEFAULT_MAX_SIZE_REPLACE_HAS_WITH_BLOCK, DEFAULT_DO_NOT_RESEND_BLOCK_WINDOW, DEFAULT_MAX_WANTLIST_SIZE } from '../constants.ts'
+import { BlockPresenceType, WantType } from '../pb/message.ts'
+import { QueuedBitswapMessage } from '../utils/bitswap-message.ts'
+import { cidToPrefix } from '../utils/cid-prefix.ts'
+import type { Network } from '../network.ts'
+import type { Wantlist } from '../pb/message.ts'
 import type { AbortOptions, ComponentLogger, Logger, PeerId } from '@libp2p/interface'
 import type { Blockstore } from 'interface-blockstore'
-import type { CID } from 'multiformats/cid'
 
 export interface LedgerComponents {
   peerId: PeerId
@@ -17,6 +19,8 @@ export interface LedgerComponents {
 
 export interface LedgerInit {
   maxSizeReplaceHasWithBlock?: number
+  doNotResendBlockWindow?: number
+  maxWantListSize?: number
 }
 
 export interface PeerWantListEntry {
@@ -45,19 +49,50 @@ export interface PeerWantListEntry {
    * If we don't have the block and we've told them we don't have the block
    */
   sentDoNotHave?: boolean
+
+  /**
+   * If the status is `sending` or `sent`, the block for this CID is or has been
+   * sent to the peer so we should not attempt to send it again
+   */
+  status: 'want' | 'sending' | 'sent'
+
+  /**
+   * A timestamp for when this want should be removed from the list, typically
+   * this is set with the `sent` status to prevent sending duplicate blocks to a
+   * peer. Once it has expired the peer can request the block a subsequent time.
+   */
+  expires?: number
+
+  /**
+   * A timestamp of when this entry was created
+   */
+  created: number
+
+  /**
+   * If this field is false, we have attempted to send this WantList entry but
+   * found there is no block for the CID in the blockstore and we are
+   * optimistically waiting to see if we come across it later.
+   *
+   * We only perform the check when we are about to send the block, by which
+   * point the entry status is 'sending' so this value with either be false or
+   * not set
+   */
+  haveBlock?: false
 }
 
 export class Ledger {
   public peerId: PeerId
   private readonly blockstore: Blockstore
   private readonly network: Network
-  public wants: Map<string, PeerWantListEntry>
+  private wants: Map<string, PeerWantListEntry>
   public exchangeCount: number
   public bytesSent: number
   public bytesReceived: number
   public lastExchange?: number
   private readonly maxSizeReplaceHasWithBlock: number
   private readonly log: Logger
+  private readonly doNotResendBlockWindow: number
+  private readonly maxWantListSize: number
 
   constructor (components: LedgerComponents, init: LedgerInit) {
     this.peerId = components.peerId
@@ -70,6 +105,8 @@ export class Ledger {
     this.bytesSent = 0
     this.bytesReceived = 0
     this.maxSizeReplaceHasWithBlock = init.maxSizeReplaceHasWithBlock ?? DEFAULT_MAX_SIZE_REPLACE_HAS_WITH_BLOCK
+    this.doNotResendBlockWindow = init.doNotResendBlockWindow ?? DEFAULT_DO_NOT_RESEND_BLOCK_WINDOW
+    this.maxWantListSize = init.maxWantListSize ?? DEFAULT_MAX_WANTLIST_SIZE
   }
 
   sentBytes (n: number): void {
@@ -88,18 +125,174 @@ export class Ledger {
     return (this.bytesSent / (this.bytesReceived + 1)) // +1 is to prevent division by zero
   }
 
+  removeExpiredWants (): void {
+    // remove any expired wants
+    this.wants.forEach((value, key) => {
+      if (value.expires != null && value.expires < Date.now()) {
+        this.wants.delete(key)
+      }
+    })
+  }
+
+  public addWants (wantlist?: Wantlist): void {
+    if (wantlist == null) {
+      return
+    }
+
+    // if the message has a full wantlist, remove all entries not currently
+    // being sent to the peer
+    if (wantlist.full === true) {
+      this.wants.forEach((value, key) => {
+        if (value.status === 'want') {
+          this.wants.delete(key)
+        }
+      })
+    }
+
+    // clear cancelled wants and add new wants to the ledger
+    for (const entry of wantlist.entries) {
+      const cid = CID.decode(entry.cid)
+      const cidStr = uint8ArrayToString(cid.multihash.bytes, 'base64')
+
+      if (entry.cancel === true) {
+        this.log('peer %p cancelled want of block for %c', this.peerId, cid)
+        this.wants.delete(cidStr)
+      } else {
+        if (entry.wantType === WantType.WantHave) {
+          this.log('peer %p wanted block presence for %c', this.peerId, cid)
+        } else {
+          this.log('peer %p wanted block for %c', this.peerId, cid)
+        }
+
+        const existingWant = this.wants.get(cidStr)
+
+        // we are already tracking a want for this CID, just update the fields
+        if (existingWant != null) {
+          const sentOrSending = existingWant.status === 'sent' || existingWant.status === 'sending'
+          const wantTypeUpgrade = existingWant.wantType === WantType.WantHave && (entry.wantType == null || entry.wantType === WantType.WantBlock)
+
+          // allow upgrade from WantHave to WantBlock if we've previously
+          // sent or are sending a WantHave
+          if (sentOrSending && wantTypeUpgrade) {
+            existingWant.status = 'want'
+          }
+
+          existingWant.priority = entry.priority
+          existingWant.wantType = entry.wantType ?? WantType.WantBlock
+          existingWant.sendDontHave = entry.sendDontHave ?? false
+          continue
+        }
+
+        // add a new want
+        this.wants.set(cidStr, {
+          cid,
+          priority: entry.priority,
+          wantType: entry.wantType ?? WantType.WantBlock,
+          sendDontHave: entry.sendDontHave ?? false,
+          status: 'want',
+          created: Date.now()
+        })
+      }
+    }
+
+    // if we have exceeded maxWantListSize, truncate the list - first select
+    // wants that are not currently being sent to the user
+    const wants = [...this.wants.entries()]
+      .filter(([key, entry]) => entry.status === 'want')
+
+    if (wants.length > this.maxWantListSize) {
+      this.truncateWants(wants)
+    }
+  }
+
+  private truncateWants (wants: Array<[string, PeerWantListEntry]>): void {
+    // sort wants by priority, lack of block presence, then age so the wants
+    // to be evicted are a older, low priority wants that we don't have the
+    // block for
+    wants = wants
+      .sort((a, b) => {
+        if (a[1].created < b[1].created) {
+          return -1
+        }
+
+        if (b[1].created < a[1].created) {
+          return 1
+        }
+
+        return 0
+      })
+      .sort((a, b) => {
+        if (a[1].haveBlock === false) {
+          return -1
+        }
+
+        if (b[1].haveBlock === false) {
+          return 1
+        }
+
+        return 0
+      })
+      .sort((a, b) => {
+        if (a[1].priority < b[1].priority) {
+          return -1
+        }
+
+        if (b[1].priority < a[1].priority) {
+          return 1
+        }
+
+        return 0
+      })
+
+    const toRemove = wants.length - this.maxWantListSize
+
+    for (let i = 0; i < toRemove; i++) {
+      this.wants.delete(wants[i][0])
+    }
+  }
+
+  public getWants (): PeerWantListEntry[] {
+    return [...this.wants.values()]
+  }
+
+  public hasWant (cid: CID): boolean {
+    const cidStr = uint8ArrayToString(cid.multihash.bytes, 'base64')
+
+    return this.wants.has(cidStr)
+  }
+
   public async sendBlocksToPeer (options?: AbortOptions): Promise<void> {
     const message = new QueuedBitswapMessage()
     const sentBlocks = new Set<string>()
 
-    for (const [key, entry] of this.wants.entries()) {
+    // remove any expired wants
+    this.removeExpiredWants()
+
+    // pick unsent wants
+    const unsent = [...this.wants.entries()]
+      .filter(([key, value]) => value.status === 'want')
+
+    // update status, ensure we don't send the same blocks repeatedly
+    unsent.forEach(([key, value]) => {
+      value.status = 'sending'
+    })
+
+    for (const [key, entry] of unsent) {
       try {
         const block = await toBuffer(this.blockstore.get(entry.cid, options))
 
+        // ensure we still need to send the block/status, status may have
+        // changed due to incoming message while we were waiting for async block
+        // load
+        if (entry.status !== 'sending') {
+          continue
+        }
+
         // do they want the block or just us to tell them we have the block
         if (entry.wantType === WantType.WantHave) {
           if (block.byteLength < this.maxSizeReplaceHasWithBlock) {
             this.log('sending have and block for %c', entry.cid)
+
             // if the block is small we just send it to them
             sentBlocks.add(key)
             message.addBlock(entry.cid, {
@@ -123,11 +316,20 @@ export class Ledger {
             prefix: cidToPrefix(entry.cid)
           })
         }
+
+        entry.status = 'sent'
+        entry.expires = Date.now() + this.doNotResendBlockWindow
       } catch (err: any) {
         if (err.name !== 'NotFoundError') {
           throw err
         }
 
+        // reset status to try again later
+        entry.status = 'want'
+
+        // used to maybe delete this want later if the want list grows too large
+        entry.haveBlock = false
+
         this.log('do not have block for %c', entry.cid)
 
         // we don't have the requested block and the remote is not interested
@@ -157,12 +359,6 @@ export class Ledger {
 
       // update accounting
       this.sentBytes([...message.blocks.values()].reduce((acc, curr) => acc + curr.data.byteLength, 0))
-
-      // remove sent blocks from local copy of their want list - they can still
-      // re-request if required
-      for (const key of sentBlocks) {
-        this.wants.delete(key)
-      }
     }
   }
 }

package/src/session.ts

@@ -1,9 +1,9 @@
 import { AbstractSession } from '@helia/utils'
 import { isPeerId } from '@libp2p/interface'
 import { CustomProgressEvent } from 'progress-events'
-import type { BitswapProvider, BitswapWantProgressEvents } from './index.js'
-import type { Network } from './network.js'
-import type { WantList } from './want-list.js'
+import type { BitswapProvider, BitswapWantProgressEvents } from './index.ts'
+import type { Network } from './network.ts'
+import type { WantList } from './want-list.ts'
 import type { BlockRetrievalOptions, CreateSessionOptions } from '@helia/interface'
 import type { ComponentLogger, Libp2p, PeerId } from '@libp2p/interface'
 import type { Multiaddr } from '@multiformats/multiaddr'

package/src/utils/bitswap-message.ts

@@ -1,5 +1,5 @@
 import { base64 } from 'multiformats/bases/base64'
-import type { Block, BlockPresence, WantlistEntry } from '../pb/message.js'
+import type { Block, BlockPresence, WantlistEntry } from '../pb/message.ts'
 import type { CID } from 'multiformats'
 
 /**

package/src/utils/cid-prefix.ts

@@ -1,4 +1,4 @@
-import ve from './varint-encoder.js'
+import ve from './varint-encoder.ts'
 import type { CID } from 'multiformats/cid'
 
 export function cidToPrefix (cid: CID): Uint8Array {

package/src/utils/merge-messages.ts

@@ -1,4 +1,4 @@
-import type { QueuedBitswapMessage } from './bitswap-message.js'
+import type { QueuedBitswapMessage } from './bitswap-message.ts'
 
 export function mergeMessages (existingMessage: QueuedBitswapMessage, newMessage: QueuedBitswapMessage): QueuedBitswapMessage {
   for (const [key, entry] of newMessage.wantlist.entries()) {

package/src/utils/split-message.ts

@@ -1,7 +1,7 @@
 import { encodingLength } from 'uint8-varint'
-import { BlockTooLargeError } from '../errors.js'
-import { BitswapMessage, Block, BlockPresence, WantlistEntry } from '../pb/message.js'
-import type { QueuedBitswapMessage } from './bitswap-message.js'
+import { BlockTooLargeError } from '../errors.ts'
+import { BitswapMessage, Block, BlockPresence, WantlistEntry } from '../pb/message.ts'
+import type { QueuedBitswapMessage } from './bitswap-message.ts'
 
 /**
  * https://github.com/ipfs/kubo/issues/4473#issuecomment-350390693

package/src/want-list.ts

@@ -7,13 +7,13 @@ import pDefer from 'p-defer'
 import { raceEvent } from 'race-event'
 import { equals as uint8ArrayEquals } from 'uint8arrays/equals'
 import { toString as uint8ArrayToString } from 'uint8arrays/to-string'
-import { DEFAULT_MESSAGE_SEND_DELAY } from './constants.js'
-import { BlockPresenceType, WantType } from './pb/message.js'
-import { QueuedBitswapMessage } from './utils/bitswap-message.js'
-import vd from './utils/varint-decoder.js'
-import type { BitswapNotifyProgressEvents, MultihashHasherLoader } from './index.js'
-import type { BitswapNetworkWantProgressEvents, Network } from './network.js'
-import type { BitswapMessage } from './pb/message.js'
+import { DEFAULT_MESSAGE_SEND_DELAY } from './constants.ts'
+import { BlockPresenceType, WantType } from './pb/message.ts'
+import { QueuedBitswapMessage } from './utils/bitswap-message.ts'
+import vd from './utils/varint-decoder.ts'
+import type { BitswapNotifyProgressEvents, MultihashHasherLoader } from './index.ts'
+import type { BitswapNetworkWantProgressEvents, Network } from './network.ts'
+import type { BitswapMessage } from './pb/message.ts'
 import type { ComponentLogger, PeerId, Startable, AbortOptions, Libp2p, TypedEventTarget, Metrics } from '@libp2p/interface'
 import type { Logger } from '@libp2p/logger'
 import type { PeerMap } from '@libp2p/peer-collections'

package/dist/typedoc-urls.json

@@ -1,26 +0,0 @@
-{
-  "codec": "https://ipfs.github.io/helia/functions/_helia_bitswap.WantType.codec.html",
-  "WantType": "https://ipfs.github.io/helia/enums/_helia_bitswap.WantType.html",
-  "Bitswap": "https://ipfs.github.io/helia/interfaces/_helia_bitswap.Bitswap.html",
-  ".:Bitswap": "https://ipfs.github.io/helia/interfaces/_helia_bitswap.Bitswap.html",
-  "BitswapComponents": "https://ipfs.github.io/helia/interfaces/_helia_bitswap.BitswapComponents.html",
-  ".:BitswapComponents": "https://ipfs.github.io/helia/interfaces/_helia_bitswap.BitswapComponents.html",
-  "BitswapOptions": "https://ipfs.github.io/helia/interfaces/_helia_bitswap.BitswapOptions.html",
-  ".:BitswapOptions": "https://ipfs.github.io/helia/interfaces/_helia_bitswap.BitswapOptions.html",
-  "BitswapProvider": "https://ipfs.github.io/helia/interfaces/_helia_bitswap.BitswapProvider.html",
-  "MultihashHasherLoader": "https://ipfs.github.io/helia/interfaces/_helia_bitswap.MultihashHasherLoader.html",
-  ".:MultihashHasherLoader": "https://ipfs.github.io/helia/interfaces/_helia_bitswap.MultihashHasherLoader.html",
-  "WantListEntry": "https://ipfs.github.io/helia/interfaces/_helia_bitswap.WantListEntry.html",
-  ".:WantListEntry": "https://ipfs.github.io/helia/interfaces/_helia_bitswap.WantListEntry.html",
-  "BitswapNetworkNotifyProgressEvents": "https://ipfs.github.io/helia/types/_helia_bitswap.BitswapNetworkNotifyProgressEvents.html",
-  "BitswapNetworkProgressEvents": "https://ipfs.github.io/helia/types/_helia_bitswap.BitswapNetworkProgressEvents.html",
-  "BitswapNetworkWantProgressEvents": "https://ipfs.github.io/helia/types/_helia_bitswap.BitswapNetworkWantProgressEvents.html",
-  "BitswapNotifyProgressEvents": "https://ipfs.github.io/helia/types/_helia_bitswap.BitswapNotifyProgressEvents.html",
-  ".:BitswapNotifyProgressEvents": "https://ipfs.github.io/helia/types/_helia_bitswap.BitswapNotifyProgressEvents.html",
-  "BitswapWantBlockProgressEvents": "https://ipfs.github.io/helia/types/_helia_bitswap.BitswapWantBlockProgressEvents.html",
-  ".:BitswapWantBlockProgressEvents": "https://ipfs.github.io/helia/types/_helia_bitswap.BitswapWantBlockProgressEvents.html",
-  "BitswapWantProgressEvents": "https://ipfs.github.io/helia/types/_helia_bitswap.BitswapWantProgressEvents.html",
-  ".:BitswapWantProgressEvents": "https://ipfs.github.io/helia/types/_helia_bitswap.BitswapWantProgressEvents.html",
-  "createBitswap": "https://ipfs.github.io/helia/functions/_helia_bitswap.createBitswap.html",
-  ".:createBitswap": "https://ipfs.github.io/helia/functions/_helia_bitswap.createBitswap.html"
-}