@helia/bitswap 0.0.0 → 1.0.0-59de059

package/src/want-list.ts

@@ -1,5 +1,5 @@
-import { AbortError } from '@libp2p/interface'
-import { trackedPeerMap, PeerSet } from '@libp2p/peer-collections'
+import { TypedEventEmitter, setMaxListeners } from '@libp2p/interface'
+import { trackedPeerMap } from '@libp2p/peer-collections'
 import { trackedMap } from '@libp2p/utils/tracked-map'
 import all from 'it-all'
 import filter from 'it-filter'
@@ -8,14 +8,16 @@ import { pipe } from 'it-pipe'
 import { CID } from 'multiformats/cid'
 import { sha256 } from 'multiformats/hashes/sha2'
 import pDefer from 'p-defer'
+import { raceEvent } from 'race-event'
+import { equals as uint8ArrayEquals } from 'uint8arrays/equals'
 import { toString as uint8ArrayToString } from 'uint8arrays/to-string'
 import { DEFAULT_MESSAGE_SEND_DELAY } from './constants.js'
 import { BlockPresenceType, WantType } from './pb/message.js'
 import vd from './utils/varint-decoder.js'
-import type { MultihashHasherLoader } from './index.js'
+import type { BitswapNotifyProgressEvents, MultihashHasherLoader } from './index.js'
 import type { BitswapNetworkWantProgressEvents, Network } from './network.js'
 import type { BitswapMessage } from './pb/message.js'
-import type { ComponentLogger, Metrics, PeerId, Startable, AbortOptions } from '@libp2p/interface'
+import type { ComponentLogger, PeerId, Startable, AbortOptions, Libp2p, TypedEventTarget } from '@libp2p/interface'
 import type { Logger } from '@libp2p/logger'
 import type { PeerMap } from '@libp2p/peer-collections'
 import type { DeferredPromise } from 'p-defer'
@@ -24,7 +26,7 @@ import type { ProgressOptions } from 'progress-events'
 export interface WantListComponents {
   network: Network
   logger: ComponentLogger
-  metrics?: Metrics
+  libp2p: Libp2p
 }
 
 export interface WantListInit {
@@ -58,29 +60,9 @@ export interface WantListEntry {
    * Whether the remote should tell us if they have the block or not
    */
   sendDontHave: boolean
-
-  /**
-   * If this set has members, the want will only be sent to these peers
-   */
-  session: PeerSet
-
-  /**
-   * Promises returned from `.wantBlock` for this block
-   */
-  blockWantListeners: Array<DeferredPromise<WantBlockResult>>
-
-  /**
-   * Promises returned from `.wantPresence` for this block
-   */
-  blockPresenceListeners: Array<DeferredPromise<WantPresenceResult>>
 }
 
 export interface WantOptions extends AbortOptions, ProgressOptions<BitswapNetworkWantProgressEvents> {
-  /**
-   * If set, this WantList entry will only be sent to this peer
-   */
-  peerId?: PeerId
-
   /**
    * Allow prioritising blocks
    */
@@ -108,7 +90,12 @@ export interface WantHaveResult {
 
 export type WantPresenceResult = WantDontHaveResult | WantHaveResult
 
-export class WantList implements Startable {
+export interface WantListEvents {
+  block: CustomEvent<WantBlockResult>
+  presence: CustomEvent<WantPresenceResult>
+}
+
+export class WantList extends TypedEventEmitter<WantListEvents> implements Startable, TypedEventTarget<WantListEvents> {
   /**
    * Tracks what CIDs we've previously sent to which peers
    */
@@ -119,15 +106,19 @@ export class WantList implements Startable {
   private readonly sendMessagesDelay: number
   private sendMessagesTimeout?: ReturnType<typeof setTimeout>
   private readonly hashLoader?: MultihashHasherLoader
+  private sendingMessages?: DeferredPromise<void>
 
   constructor (components: WantListComponents, init: WantListInit = {}) {
+    super()
+
+    setMaxListeners(Infinity, this)
     this.peers = trackedPeerMap({
-      name: 'ipfs_bitswap_peers',
-      metrics: components.metrics
+      name: 'helia_bitswap_peers',
+      metrics: components.libp2p.metrics
     })
     this.wants = trackedMap({
-      name: 'ipfs_bitswap_wantlist',
-      metrics: components.metrics
+      name: 'helia_bitswap_wantlist',
+      metrics: components.libp2p.metrics
     })
     this.network = components.network
     this.sendMessagesDelay = init.sendMessagesDelay ?? DEFAULT_MESSAGE_SEND_DELAY
@@ -160,17 +151,10 @@ export class WantList implements Startable {
     if (entry == null) {
       entry = {
         cid,
-        session: new PeerSet(),
         priority: options.priority ?? 1,
         wantType: options.wantType ?? WantType.WantBlock,
         cancel: false,
-        sendDontHave: true,
-        blockWantListeners: [],
-        blockPresenceListeners: []
-      }
-
-      if (options.peerId != null) {
-        entry.session.add(options.peerId)
+        sendDontHave: true
       }
 
       this.wants.set(cidStr, entry)
@@ -182,46 +166,41 @@ export class WantList implements Startable {
       entry.wantType = WantType.WantBlock
     }
 
-    // if this want was part of a session..
-    if (entry.session.size > 0) {
-      // if the new want is also part of a session, expand the want session to
-      // include both sets of peers
-      if (options.peerId != null) {
-        entry.session.add(options.peerId)
-      }
-
-      // if the new want is not part of a session, make this want a non-session
-      // want - nb. this will cause this WantList entry to be sent to every peer
-      // instead of just the ones in the session
-      if (options.peerId == null) {
-        entry.session.clear()
-      }
-    }
-
-    // add a promise that will be resolved or rejected when the response arrives
-    let deferred: DeferredPromise<WantBlockResult | WantPresenceResult>
-
-    if (options.wantType === WantType.WantBlock) {
-      const p = deferred = pDefer<WantBlockResult>()
+    // broadcast changes
+    await this.sendMessagesDebounced()
 
-      entry.blockWantListeners.push(p)
-    } else {
-      const p = deferred = pDefer<WantPresenceResult>()
+    try {
+      if (options.wantType === WantType.WantBlock) {
+        const event = await raceEvent<CustomEvent<WantBlockResult>>(this, 'block', options?.signal, {
+          filter: (event) => {
+            return uint8ArrayEquals(cid.multihash.digest, event.detail.cid.multihash.digest)
+          },
+          errorMessage: 'Want was aborted'
+        })
 
-      entry.blockPresenceListeners.push(p)
-    }
+        return event.detail
+      }
 
-    // reject the promise if the want is rejected
-    const abortListener = (): void => {
-      this.log('want for %c was aborted, cancelling want', cid)
+      const event = await raceEvent<CustomEvent<WantPresenceResult>>(this, 'presence', options?.signal, {
+        filter: (event) => {
+          return uint8ArrayEquals(cid.multihash.digest, event.detail.cid.multihash.digest)
+        },
+        errorMessage: 'Want was aborted'
+      })
 
-      if (entry != null) {
+      return event.detail
+    } finally {
+      if (options.signal?.aborted === true) {
+        this.log('want for %c was aborted, cancelling want', cid)
         entry.cancel = true
+        // broadcast changes
+        await this.sendMessagesDebounced()
       }
-
-      deferred.reject(new AbortError('Want was aborted'))
     }
-    options.signal?.addEventListener('abort', abortListener)
+  }
+
+  private async sendMessagesDebounced (): Promise<void> {
+    await this.sendingMessages?.promise
 
     // broadcast changes
     clearTimeout(this.sendMessagesTimeout)
@@ -231,77 +210,65 @@ export class WantList implements Startable {
           this.log('error sending messages to peers', err)
         })
     }, this.sendMessagesDelay)
-
-    try {
-      return await deferred.promise
-    } finally {
-      // remove listener
-      options.signal?.removeEventListener('abort', abortListener)
-      // remove deferred promise
-      if (options.wantType === WantType.WantBlock) {
-        entry.blockWantListeners = entry.blockWantListeners.filter(recipient => recipient !== deferred)
-      } else {
-        entry.blockPresenceListeners = entry.blockPresenceListeners.filter(recipient => recipient !== deferred)
-      }
-    }
   }
 
   private async sendMessages (): Promise<void> {
-    for (const [peerId, sentWants] of this.peers) {
-      const sent = new Set<string>()
-      const message: Partial<BitswapMessage> = {
-        wantlist: {
-          full: false,
-          entries: pipe(
-            this.wants.entries(),
-            (source) => filter(source, ([key, entry]) => {
-              // skip session-only wants
-              if (entry.session.size > 0 && !entry.session.has(peerId)) {
-                return false
-              }
-
-              const sentPreviously = sentWants.has(key)
-
-              // don't cancel if we've not sent it to them before
-              if (entry.cancel) {
-                return sentPreviously
-              }
-
-              // only send if we've not sent it to them before
-              return !sentPreviously
-            }),
-            (source) => map(source, ([key, entry]) => {
-              sent.add(key)
-
-              return {
-                cid: entry.cid.bytes,
-                priority: entry.priority,
-                wantType: entry.wantType,
-                cancel: entry.cancel,
-                sendDontHave: entry.sendDontHave
-              }
-            }),
-            (source) => all(source)
-          )
+    this.sendingMessages = pDefer()
+
+    await Promise.all(
+      [...this.peers.entries()].map(async ([peerId, sentWants]) => {
+        const sent = new Set<string>()
+        const message: Partial<BitswapMessage> = {
+          wantlist: {
+            full: false,
+            entries: pipe(
+              this.wants.entries(),
+              (source) => filter(source, ([key, entry]) => {
+                const sentPreviously = sentWants.has(key)
+
+                // don't cancel if we've not sent it to them before
+                if (entry.cancel) {
+                  return sentPreviously
+                }
+
+                // only send if we've not sent it to them before
+                return !sentPreviously
+              }),
+              (source) => map(source, ([key, entry]) => {
+                sent.add(key)
+
+                return {
+                  cid: entry.cid.bytes,
+                  priority: entry.priority,
+                  wantType: entry.wantType,
+                  cancel: entry.cancel,
+                  sendDontHave: entry.sendDontHave
+                }
+              }),
+              (source) => all(source)
+            )
+          }
         }
-      }
 
-      if (message.wantlist?.entries.length === 0) {
-        return
-      }
+        if (message.wantlist?.entries.length === 0) {
+          return
+        }
 
-      // add message to send queue
-      try {
-        await this.network.sendMessage(peerId, message)
+        // add message to send queue
+        try {
+          await this.network.sendMessage(peerId, message)
 
-        // update list of messages sent to remote
-        for (const key of sent) {
-          sentWants.add(key)
+          // update list of messages sent to remote
+          for (const key of sent) {
+            sentWants.add(key)
+          }
+        } catch (err: any) {
+          this.log.error('error sending full wantlist to new peer', err)
         }
-      } catch (err: any) {
-        this.log.error('error sending full wantlist to new peer', err)
-      }
-    }
+      })
+    ).catch(err => {
+      this.log.error('error sending messages', err)
+    })
 
     // queued all message sends, remove cancelled wants from wantlist and sent
     // wants
@@ -314,6 +281,8 @@ export class WantList implements Startable {
         }
       }
     }
+
+    this.sendingMessages.resolve()
   }
 
   has (cid: CID): boolean {
@@ -324,38 +293,28 @@ export class WantList implements Startable {
   /**
    * Add a CID to the wantlist
    */
-  async wantPresence (cid: CID, options: WantOptions = {}): Promise<WantPresenceResult> {
-    if (options.peerId != null && this.peers.get(options.peerId) == null) {
-      const cidStr = uint8ArrayToString(cid.multihash.bytes, 'base64')
-
-      try {
-        // if we don't have them as a peer, add them
-        this.peers.set(options.peerId, new Set([cidStr]))
-
-        // sending WantHave directly to peer
-        await this.network.sendMessage(options.peerId, {
-          wantlist: {
-            full: false,
-            entries: [{
-              cid: cid.bytes,
-              sendDontHave: true,
-              wantType: WantType.WantHave,
-              priority: 1
-            }]
-          }
-        })
-      } catch (err) {
-        // sending failed, remove them as a peer
-        this.peers.delete(options.peerId)
-
-        throw err
+  async wantSessionPresence (cid: CID, peerId: PeerId, options: WantOptions = {}): Promise<WantPresenceResult> {
+    // sending WantHave directly to peer
+    await this.network.sendMessage(peerId, {
+      wantlist: {
+        full: false,
+        entries: [{
+          cid: cid.bytes,
+          sendDontHave: true,
+          wantType: WantType.WantHave,
+          priority: 1
+        }]
       }
-    }
+    })
 
-    return this.addEntry(cid, {
-      ...options,
-      wantType: WantType.WantHave
+    // wait for peer response
+    const event = await raceEvent<CustomEvent<WantHaveResult | WantDontHaveResult>>(this, 'presence', options.signal, {
+      filter: (event) => {
+        return peerId.equals(event.detail.sender) && uint8ArrayEquals(cid.multihash.digest, event.detail.cid.multihash.digest)
+      }
     })
+
+    return event.detail
   }
 
   /**
@@ -368,15 +327,56 @@ export class WantList implements Startable {
     })
   }
 
+  /**
+   * Add a CID to the wantlist
+   */
+  async wantSessionBlock (cid: CID, peerId: PeerId, options: WantOptions = {}): Promise<WantPresenceResult> {
+    // sending WantBlockResult directly to peer
+    await this.network.sendMessage(peerId, {
+      wantlist: {
+        full: false,
+        entries: [{
+          cid: cid.bytes,
+          sendDontHave: true,
+          wantType: WantType.WantBlock,
+          priority: 1
+        }]
+      }
+    })
+
+    // wait for peer response
+    const event = await raceEvent<CustomEvent<WantPresenceResult>>(this, 'presence', options.signal, {
+      filter: (event) => {
+        return peerId.equals(event.detail.sender) && uint8ArrayEquals(cid.multihash.digest, event.detail.cid.multihash.digest)
+      }
+    })
+
+    return event.detail
+  }
+
+  /**
+   * Invoked when a block has been received from an external source
+   */
+  async receivedBlock (cid: CID, options: ProgressOptions<BitswapNotifyProgressEvents> & AbortOptions): Promise<void> {
+    const cidStr = uint8ArrayToString(cid.multihash.bytes, 'base64')
+
+    const entry = this.wants.get(cidStr)
+
+    if (entry == null) {
+      return
+    }
+
+    entry.cancel = true
+
+    await this.sendMessagesDebounced()
+  }
+
   /**
    * Invoked when a message is received from a bitswap peer
    */
   private async receiveMessage (sender: PeerId, message: BitswapMessage): Promise<void> {
-    this.log('received message from %p', sender)
-
-    // blocks received
-    const blockResults: WantBlockResult[] = []
-    const presenceResults: WantPresenceResult[] = []
+    this.log('received message %d from %p with %d blocks', sender, message.blocks.length)
+    let blocksCancelled = false
 
     // process blocks
     for (const block of message.blocks) {
@@ -384,7 +384,6 @@ export class WantList implements Startable {
         continue
       }
 
-      this.log('received block')
       const values = vd(block.prefix)
       const cidVersion = values[0]
       const multicodec = values[1]
@@ -398,71 +397,65 @@ export class WantList implements Startable {
         continue
       }
 
-      const hash = await hasher.digest(block.data)
+      let hash: any = hasher.digest(block.data)
+
+      if (hash.then != null) {
+        hash = await hash
+      }
+
       const cid = CID.create(cidVersion === 0 ? 0 : 1, multicodec, hash)
 
       this.log('received block from %p for %c', sender, cid)
 
-      blockResults.push({
-        sender,
-        cid,
-        block: block.data
-      })
-
-      presenceResults.push({
-        sender,
-        cid,
-        has: true
+      this.safeDispatchEvent<WantBlockResult>('block', {
+        detail: {
+          sender,
+          cid,
+          block: block.data
+        }
       })
-    }
-
-    // process block presences
-    for (const { cid: cidBytes, type } of message.blockPresences) {
-      const cid = CID.decode(cidBytes)
-
-      this.log('received %s from %p for %c', type, sender, cid)
 
-      presenceResults.push({
-        sender,
-        cid,
-        has: type === BlockPresenceType.HaveBlock
+      this.safeDispatchEvent<WantHaveResult | WantDontHaveResult>('presence', {
+        detail: {
+          sender,
+          cid,
+          has: true,
+          block: block.data
+        }
       })
-    }
 
-    for (const result of blockResults) {
-      const cidStr = uint8ArrayToString(result.cid.multihash.bytes, 'base64')
+      const cidStr = uint8ArrayToString(cid.multihash.bytes, 'base64')
       const entry = this.wants.get(cidStr)
 
       if (entry == null) {
-        return
+        continue
       }
 
-      const recipients = entry.blockWantListeners
-      entry.blockWantListeners = []
-      recipients.forEach((p) => {
-        p.resolve(result)
-      })
-
       // since we received the block, flip the cancel flag to send cancels to
       // any peers on the next message sending iteration, this will remove it
       // from the internal want list
       entry.cancel = true
+      blocksCancelled = true
     }
 
-    for (const result of presenceResults) {
-      const cidStr = uint8ArrayToString(result.cid.multihash.bytes, 'base64')
-      const entry = this.wants.get(cidStr)
+    // process block presences
+    for (const { cid: cidBytes, type } of message.blockPresences) {
+      const cid = CID.decode(cidBytes)
 
-      if (entry == null) {
-        return
-      }
+      this.log('received %s from %p for %c', type, sender, cid)
 
-      const recipients = entry.blockPresenceListeners
-      entry.blockPresenceListeners = []
-      recipients.forEach((p) => {
-        p.resolve(result)
+      this.safeDispatchEvent<WantHaveResult | WantDontHaveResult>('presence', {
+        detail: {
+          sender,
+          cid,
+          has: type === BlockPresenceType.HaveBlock
+        }
       })
     }
+
+    if (blocksCancelled) {
+      await this.sendMessagesDebounced()
+    }
   }
 
   /**
@@ -477,7 +470,6 @@ export class WantList implements Startable {
         full: true,
         entries: pipe(
           this.wants.entries(),
-          (source) => filter(source, ([key, entry]) => !entry.cancel && (entry.session.size > 0 && !entry.session.has(peerId))),
           (source) => filter(source, ([key, entry]) => !entry.cancel),
           (source) => map(source, ([key, entry]) => {
             sentWants.add(key)
@@ -525,5 +517,6 @@ export class WantList implements Startable {
 
   stop (): void {
     this.peers.clear()
+    clearTimeout(this.sendMessagesTimeout)
   }
 }

package/dist/typedoc-urls.json

@@ -1,24 +0,0 @@
-{
-  "Bitswap": "https://ipfs.github.io/helia/interfaces/_helia_bitswap.Bitswap.html",
-  ".:Bitswap": "https://ipfs.github.io/helia/interfaces/_helia_bitswap.Bitswap.html",
-  "BitswapComponents": "https://ipfs.github.io/helia/interfaces/_helia_bitswap.BitswapComponents.html",
-  ".:BitswapComponents": "https://ipfs.github.io/helia/interfaces/_helia_bitswap.BitswapComponents.html",
-  "BitswapOptions": "https://ipfs.github.io/helia/interfaces/_helia_bitswap.BitswapOptions.html",
-  ".:BitswapOptions": "https://ipfs.github.io/helia/interfaces/_helia_bitswap.BitswapOptions.html",
-  "BitswapSession": "https://ipfs.github.io/helia/interfaces/_helia_bitswap.BitswapSession.html",
-  ".:BitswapSession": "https://ipfs.github.io/helia/interfaces/_helia_bitswap.BitswapSession.html",
-  "CreateBitswapSessionOptions": "https://ipfs.github.io/helia/interfaces/_helia_bitswap.CreateBitswapSessionOptions.html",
-  ".:CreateBitswapSessionOptions": "https://ipfs.github.io/helia/interfaces/_helia_bitswap.CreateBitswapSessionOptions.html",
-  "MultihashHasherLoader": "https://ipfs.github.io/helia/interfaces/_helia_bitswap.MultihashHasherLoader.html",
-  ".:MultihashHasherLoader": "https://ipfs.github.io/helia/interfaces/_helia_bitswap.MultihashHasherLoader.html",
-  "WantListEntry": "https://ipfs.github.io/helia/interfaces/_helia_bitswap.WantListEntry.html",
-  ".:WantListEntry": "https://ipfs.github.io/helia/interfaces/_helia_bitswap.WantListEntry.html",
-  "BitswapNotifyProgressEvents": "https://ipfs.github.io/helia/types/_helia_bitswap.BitswapNotifyProgressEvents.html",
-  ".:BitswapNotifyProgressEvents": "https://ipfs.github.io/helia/types/_helia_bitswap.BitswapNotifyProgressEvents.html",
-  "BitswapWantBlockProgressEvents": "https://ipfs.github.io/helia/types/_helia_bitswap.BitswapWantBlockProgressEvents.html",
-  ".:BitswapWantBlockProgressEvents": "https://ipfs.github.io/helia/types/_helia_bitswap.BitswapWantBlockProgressEvents.html",
-  "BitswapWantProgressEvents": "https://ipfs.github.io/helia/types/_helia_bitswap.BitswapWantProgressEvents.html",
-  ".:BitswapWantProgressEvents": "https://ipfs.github.io/helia/types/_helia_bitswap.BitswapWantProgressEvents.html",
-  "createBitswap": "https://ipfs.github.io/helia/functions/_helia_bitswap.createBitswap.html",
-  ".:createBitswap": "https://ipfs.github.io/helia/functions/_helia_bitswap.createBitswap.html"
-}