@helia/bitswap 0.0.0 → 1.0.0-59de059

package/src/peer-want-lists/index.ts

@@ -6,7 +6,7 @@ import { Ledger } from './ledger.js'
 import type { BitswapNotifyProgressEvents, WantListEntry } from '../index.js'
 import type { Network } from '../network.js'
 import type { BitswapMessage } from '../pb/message.js'
-import type { ComponentLogger, Logger, Metrics, PeerId } from '@libp2p/interface'
+import type { ComponentLogger, Libp2p, Logger, PeerId } from '@libp2p/interface'
 import type { PeerMap } from '@libp2p/peer-collections'
 import type { Blockstore } from 'interface-blockstore'
 import type { AbortOptions } from 'it-length-prefixed-stream'
@@ -19,7 +19,7 @@ export interface PeerWantListsInit {
 export interface PeerWantListsComponents {
   blockstore: Blockstore
   network: Network
-  metrics?: Metrics
+  libp2p: Libp2p
   logger: ComponentLogger
 }
 
@@ -37,16 +37,18 @@ export class PeerWantLists {
   public readonly ledgerMap: PeerMap<Ledger>
   private readonly maxSizeReplaceHasWithBlock?: number
   private readonly log: Logger
+  private readonly logger: ComponentLogger
 
   constructor (components: PeerWantListsComponents, init: PeerWantListsInit = {}) {
     this.blockstore = components.blockstore
     this.network = components.network
     this.maxSizeReplaceHasWithBlock = init.maxSizeReplaceHasWithBlock
     this.log = components.logger.forComponent('helia:bitswap:peer-want-lists')
+    this.logger = components.logger
 
     this.ledgerMap = trackedPeerMap({
-      name: 'ipfs_bitswap_ledger_map',
-      metrics: components.metrics
+      name: 'helia_bitswap_ledger_map',
+      metrics: components.libp2p.metrics
     })
 
     this.network.addEventListener('bitswap:message', (evt) => {
@@ -100,7 +102,8 @@ export class PeerWantLists {
       ledger = new Ledger({
         peerId,
         blockstore: this.blockstore,
-        network: this.network
+        network: this.network,
+        logger: this.logger
       }, {
         maxSizeReplaceHasWithBlock: this.maxSizeReplaceHasWithBlock
       })
@@ -141,6 +144,7 @@ export class PeerWantLists {
       }
     }
 
+    this.log('send blocks to peer')
     await ledger.sendBlocksToPeer()
   }
 

package/src/peer-want-lists/ledger.ts

@@ -3,7 +3,7 @@ import { DEFAULT_MAX_SIZE_REPLACE_HAS_WITH_BLOCK } from '../constants.js'
 import { BlockPresenceType, type BitswapMessage, WantType } from '../pb/message.js'
 import { cidToPrefix } from '../utils/cid-prefix.js'
 import type { Network } from '../network.js'
-import type { PeerId } from '@libp2p/interface'
+import type { ComponentLogger, Logger, PeerId } from '@libp2p/interface'
 import type { Blockstore } from 'interface-blockstore'
 import type { AbortOptions } from 'it-length-prefixed-stream'
 import type { CID } from 'multiformats/cid'
@@ -12,6 +12,7 @@ export interface LedgerComponents {
   peerId: PeerId
   blockstore: Blockstore
   network: Network
+  logger: ComponentLogger
 }
 
 export interface LedgerInit {
@@ -56,12 +57,14 @@ export class Ledger {
   public bytesReceived: number
   public lastExchange?: number
   private readonly maxSizeReplaceHasWithBlock: number
+  private readonly log: Logger
 
   constructor (components: LedgerComponents, init: LedgerInit) {
     this.peerId = components.peerId
     this.blockstore = components.blockstore
     this.network = components.network
     this.wants = new Map()
+    this.log = components.logger.forComponent(`helia:bitswap:ledger:${components.peerId}`)
 
     this.exchangeCount = 0
     this.bytesSent = 0
@@ -96,6 +99,8 @@ export class Ledger {
       const has = await this.blockstore.has(entry.cid, options)
 
       if (!has) {
+        this.log('do not have block for %c', entry.cid)
+
         // we don't have the requested block and the remote is not interested
         // in us telling them that
         if (!entry.sendDontHave) {
@@ -121,6 +126,7 @@ export class Ledger {
       // do they want the block or just us to tell them we have the block
       if (entry.wantType === WantType.WantHave) {
         if (block.byteLength < this.maxSizeReplaceHasWithBlock) {
+          this.log('sending have and block for %c', entry.cid)
           // if the block is small we just send it to them
           sentBlocks.add(key)
           message.blocks.push({
@@ -128,6 +134,7 @@ export class Ledger {
             prefix: cidToPrefix(entry.cid)
           })
         } else {
+          this.log('sending have for %c', entry.cid)
           // otherwise tell them we have the block
           message.blockPresences.push({
             cid: entry.cid.bytes,
@@ -135,6 +142,7 @@ export class Ledger {
           })
         }
       } else {
+        this.log('sending block for %c', entry.cid)
         // they want the block, send it to them
         sentBlocks.add(key)
         message.blocks.push({
@@ -146,7 +154,9 @@ export class Ledger {
 
     // only send the message if we actually have something to send
     if (message.blocks.length > 0 || message.blockPresences.length > 0) {
+      this.log('sending message')
       await this.network.sendMessage(this.peerId, message, options)
+      this.log('sent message')
 
       // update accounting
       this.sentBytes(message.blocks.reduce((acc, curr) => acc + curr.data.byteLength, 0))

package/src/session.ts

@@ -1,16 +1,11 @@
-import { CodeError } from '@libp2p/interface'
-import { PeerSet } from '@libp2p/peer-collections'
-import { PeerQueue } from '@libp2p/utils/peer-queue'
-import map from 'it-map'
-import merge from 'it-merge'
-import pDefer, { type DeferredPromise } from 'p-defer'
-import type { BitswapWantProgressEvents, BitswapSession as BitswapSessionInterface } from './index.js'
+import { AbstractSession } from '@helia/utils'
+import type { BitswapWantProgressEvents } from './index.js'
 import type { Network } from './network.js'
 import type { WantList } from './want-list.js'
-import type { ComponentLogger, Logger, PeerId } from '@libp2p/interface'
+import type { CreateSessionOptions } from '@helia/interface'
+import type { ComponentLogger, PeerId } from '@libp2p/interface'
 import type { AbortOptions } from 'interface-store'
 import type { CID } from 'multiformats/cid'
-import type { ProgressOptions } from 'progress-events'
 
 export interface BitswapSessionComponents {
   network: Network
@@ -18,133 +13,49 @@ export interface BitswapSessionComponents {
   logger: ComponentLogger
 }
 
-export interface BitswapSessionInit extends AbortOptions {
-  root: CID
-  queryConcurrency: number
-  minProviders: number
-  maxProviders: number
-  connectedPeers: PeerId[]
-}
-
-class BitswapSession implements BitswapSessionInterface {
-  public readonly root: CID
-  public readonly peers: PeerSet
-  private readonly log: Logger
+class BitswapSession extends AbstractSession<PeerId, BitswapWantProgressEvents> {
   private readonly wantList: WantList
   private readonly network: Network
-  private readonly queue: PeerQueue
-  private readonly maxProviders: number
 
-  constructor (components: BitswapSessionComponents, init: BitswapSessionInit) {
-    this.peers = new PeerSet()
-    this.root = init.root
-    this.maxProviders = init.maxProviders
-    this.log = components.logger.forComponent(`helia:bitswap:session:${init.root}`)
+  constructor (components: BitswapSessionComponents, init: CreateSessionOptions) {
+    super(components, {
+      ...init,
+      name: 'helia:bitswap:session'
+    })
+
     this.wantList = components.wantList
     this.network = components.network
-
-    this.queue = new PeerQueue({
-      concurrency: init.queryConcurrency
-    })
-    this.queue.addEventListener('error', (evt) => {
-      this.log.error('error querying peer for %c', this.root, evt.detail)
-    })
   }
 
-  async want (cid: CID, options: AbortOptions & ProgressOptions<BitswapWantProgressEvents> = {}): Promise<Uint8Array> {
-    if (this.peers.size === 0) {
-      throw new CodeError('Bitswap session had no peers', 'ERR_NO_SESSION_PEERS')
-    }
-
-    this.log('sending WANT-BLOCK for %c to', cid, this.peers)
+  async queryProvider (cid: CID, provider: PeerId, options: AbortOptions): Promise<Uint8Array> {
+    this.log('sending WANT-BLOCK for %c to %p', cid, provider)
 
-    const result = await Promise.any(
-      [...this.peers].map(async peerId => {
-        return this.wantList.wantBlock(cid, {
-          peerId,
-          ...options
-        })
-      })
-    )
+    const result = await this.wantList.wantSessionBlock(cid, provider, options)
 
-    this.log('received block for %c from %p', cid, result.sender)
+    this.log('%p %s %c', provider, result.has ? 'has' : 'does not have', cid)
 
-    // TODO findNewProviders when promise.any throws aggregate error and signal
-    // is not aborted
+    if (result.has && result.block != null) {
+      return result.block
+    }
 
-    return result.block
+    throw new Error('Provider did not have block')
   }
 
-  async findNewProviders (cid: CID, count: number, options: AbortOptions = {}): Promise<void> {
-    const deferred: DeferredPromise<void> = pDefer()
-    let found = 0
-
-    this.log('find %d-%d new provider(s) for %c', count, this.maxProviders, cid)
-
-    const source = merge(
-      [...this.wantList.peers.keys()],
-      map(this.network.findProviders(cid, options), prov => prov.id)
-    )
-
-    void Promise.resolve()
-      .then(async () => {
-        for await (const peerId of source) {
-          // eslint-disable-next-line no-loop-func
-          await this.queue.add(async () => {
-            try {
-              this.log('asking potential session peer %p if they have %c', peerId, cid)
-              const result = await this.wantList.wantPresence(cid, {
-                peerId,
-                ...options
-              })
-
-              if (!result.has) {
-                this.log('potential session peer %p did not have %c', peerId, cid)
-                return
-              }
-
-              this.log('potential session peer %p had %c', peerId, cid)
-              found++
-
-              // add to list
-              this.peers.add(peerId)
-
-              if (found === count) {
-                this.log('found %d session peers', found)
-
-                deferred.resolve()
-              }
-
-              if (found === this.maxProviders) {
-                this.log('found max provider session peers', found)
-
-                this.queue.clear()
-              }
-            } catch (err: any) {
-              this.log.error('error querying potential session peer %p for %c', peerId, cid, err.errors ?? err)
-            }
-          }, {
-            peerId
-          })
-        }
-
-        this.log('found %d session peers total', found)
+  async * findNewProviders (cid: CID, options: AbortOptions = {}): AsyncGenerator<PeerId> {
+    for await (const provider of this.network.findProviders(cid, options)) {
+      yield provider.id
+    }
+  }
 
-        if (count > 0) {
-          deferred.reject(new CodeError(`Found ${found} of ${count} providers`, 'ERR_NO_PROVIDERS_FOUND'))
-        }
-      })
+  toEvictionKey (provider: PeerId): Uint8Array | string {
+    return provider.toBytes()
+  }
 
-    return deferred.promise
+  equals (providerA: PeerId, providerB: PeerId): boolean {
+    return providerA.equals(providerB)
   }
 }
 
-export async function createBitswapSession (components: BitswapSessionComponents, init: BitswapSessionInit): Promise<BitswapSessionInterface> {
-  const session = new BitswapSession(components, init)
-
-  await session.findNewProviders(init.root, init.minProviders, {
-    signal: init.signal
-  })
-
-  return session
+export function createBitswapSession (components: BitswapSessionComponents, init: CreateSessionOptions): BitswapSession {
+  return new BitswapSession(components, init)
 }

package/src/stats.ts

@@ -1,7 +1,7 @@
-import type { MetricGroup, Metrics, PeerId } from '@libp2p/interface'
+import type { Libp2p, MetricGroup, PeerId } from '@libp2p/interface'
 
 export interface StatsComponents {
-  metrics?: Metrics
+  libp2p: Libp2p
 }
 
 export class Stats {
@@ -11,10 +11,10 @@ export class Stats {
   private readonly duplicateDataReceived?: MetricGroup
 
   constructor (components: StatsComponents) {
-    this.blocksReceived = components.metrics?.registerMetricGroup('ipfs_bitswap_received_blocks')
-    this.duplicateBlocksReceived = components.metrics?.registerMetricGroup('ipfs_bitswap_duplicate_received_blocks')
-    this.dataReceived = components.metrics?.registerMetricGroup('ipfs_bitswap_data_received_bytes')
-    this.duplicateDataReceived = components.metrics?.registerMetricGroup('ipfs_bitswap_duplicate_data_received_bytes')
+    this.blocksReceived = components.libp2p.metrics?.registerMetricGroup('helia_bitswap_received_blocks')
+    this.duplicateBlocksReceived = components.libp2p.metrics?.registerMetricGroup('helia_bitswap_duplicate_received_blocks')
+    this.dataReceived = components.libp2p.metrics?.registerMetricGroup('helia_bitswap_data_received_bytes')
+    this.duplicateDataReceived = components.libp2p.metrics?.registerMetricGroup('helia_bitswap_duplicate_data_received_bytes')
   }
 
   updateBlocksReceived (count: number = 1, peerId?: PeerId): void {

package/src/utils/merge-messages.ts

@@ -0,0 +1,70 @@
+import { base64 } from 'multiformats/bases/base64'
+import type { BitswapMessage, Block, BlockPresence, WantlistEntry } from '../pb/message.js'
+
+export function mergeMessages (messageA: BitswapMessage, messageB: BitswapMessage): BitswapMessage {
+  const wantListEntries = new Map<string, WantlistEntry>(
+    (messageA.wantlist?.entries ?? []).map(entry => ([
+      base64.encode(entry.cid),
+      entry
+    ]))
+  )
+
+  for (const entry of messageB.wantlist?.entries ?? []) {
+    const key = base64.encode(entry.cid)
+    const existingEntry = wantListEntries.get(key)
+
+    if (existingEntry != null) {
+      // take highest priority
+      if (existingEntry.priority > entry.priority) {
+        entry.priority = existingEntry.priority
+      }
+
+      // take later values if passed, otherwise use earlier ones
+      entry.cancel = entry.cancel ?? existingEntry.cancel
+      entry.wantType = entry.wantType ?? existingEntry.wantType
+      entry.sendDontHave = entry.sendDontHave ?? existingEntry.sendDontHave
+    }
+
+    wantListEntries.set(key, entry)
+  }
+
+  const blockPresences = new Map<string, BlockPresence>(
+    messageA.blockPresences.map(presence => ([
+      base64.encode(presence.cid),
+      presence
+    ]))
+  )
+
+  for (const blockPresence of messageB.blockPresences) {
+    const key = base64.encode(blockPresence.cid)
+
+    // override earlier block presence with later one as if duplicated it is
+    // likely to be more accurate since it is more recent
+    blockPresences.set(key, blockPresence)
+  }
+
+  const blocks = new Map<string, Block>(
+    messageA.blocks.map(block => ([
+      base64.encode(block.data),
+      block
+    ]))
+  )
+
+  for (const block of messageB.blocks) {
+    const key = base64.encode(block.data)
+
+    blocks.set(key, block)
+  }
+
+  const output: BitswapMessage = {
+    wantlist: {
+      full: messageA.wantlist?.full ?? messageB.wantlist?.full ?? false,
+      entries: [...wantListEntries.values()]
+    },
+    blockPresences: [...blockPresences.values()],
+    blocks: [...blocks.values()],
+    pendingBytes: messageA.pendingBytes + messageB.pendingBytes
+  }
+
+  return output
+}

package/src/utils/split-message.ts

@@ -0,0 +1,131 @@
+/* eslint-disable complexity */
+import { CodeError } from '@libp2p/interface'
+import { encodingLength } from 'uint8-varint'
+import { BitswapMessage, Block, BlockPresence, WantlistEntry } from '../pb/message.js'
+
+/**
+ * Split the passed Bitswap message into multiple smaller messages that when
+ * serialized will be under the maximum message size.
+ *
+ * Since blocks are the largest thing to send, we first try to fit as many
+ * blocks as possible into the message, then add (smaller) block presences and
+ * wants until the max size is reached.
+ *
+ * If a block is encountered that is larger than the max message size an error
+ * will be thrown.
+ */
+export async function * splitMessage (message: BitswapMessage, maxSize: number): AsyncGenerator<Uint8Array> {
+  const wantListEntries = message.wantlist?.entries ?? []
+  const blockPresences = message.blockPresences
+  const blocks = message.blocks
+
+  let wantListIndex = 0
+  let blockPresencesIndex = 0
+  let blocksIndex = 0
+  let messagesSent = 0
+  let doneSending = false
+
+  while (true) {
+    const subMessage: Required<BitswapMessage> = {
+      wantlist: {
+        full: false,
+        entries: []
+      },
+      blockPresences: [],
+      blocks: [],
+      pendingBytes: 0
+    }
+
+    let size = 4
+
+    let { added, hasMore, newSize } = addToMessage(blocks, subMessage.blocks, blocksIndex, maxSize, size, calculateEncodedBlockSize)
+
+    blocksIndex += added
+    size = newSize
+    const haveMoreBlocks = hasMore
+
+    ;({ added, hasMore, newSize } = addToMessage(blockPresences, subMessage.blockPresences, blockPresencesIndex, maxSize, size, calculateEncodedBlockPresenceSize))
+
+    blockPresencesIndex += added
+    size = newSize
+    const haveMorePresences = hasMore
+
+    ;({ added, hasMore, newSize } = addToMessage(wantListEntries, subMessage.wantlist.entries, wantListIndex, maxSize, size, calculateEncodedWantlistEntrySize))
+
+    wantListIndex += added
+    size = newSize
+    const haveMoreWantlistEntries = hasMore
+
+    doneSending = !haveMoreBlocks && !haveMorePresences && !haveMoreWantlistEntries
+
+    // if we're only sending one message, and that message has the full wantlist
+    // make sure we let the remote know it's the full list
+    if (doneSending && messagesSent === 0 && message.wantlist?.full === true) {
+      subMessage.wantlist.full = true
+    }
+
+    yield BitswapMessage.encode(subMessage)
+
+    messagesSent++
+
+    if (doneSending) {
+      break
+    }
+  }
+}
+
+interface AddResult {
+  hasMore: boolean
+  added: number
+  newSize: number
+}
+
+function addToMessage <T> (input: T[], output: T[], start: number, maxSize: number, size: number, calculateSize: (arg: T) => number): AddResult {
+  let added = 0
+  let hasMore = false
+
+  // try to send as many blocks as possible
+  for (let i = start; i < input.length; i++) {
+    const item = input[i]
+    const itemSize = calculateSize(item)
+
+    if (itemSize > maxSize) {
+      throw new CodeError('Cannot send block as it is over the max message size', 'ERR_BLOCK_TOO_LARGE')
+    }
+
+    const newSize = size + itemSize
+
+    if (newSize >= maxSize) {
+      hasMore = true
+      break
+    }
+
+    output.push(item)
+    added++
+    size = newSize
+  }
+
+  return { hasMore, added, newSize: size }
+}
+
+function calculateEncodedBlockSize (block: Block): number {
+  // 3 is the "blocks" field number in message.proto
+  return calculateLength(3, Block.encode(block))
+}
+
+function calculateEncodedBlockPresenceSize (blockPresence: BlockPresence): number {
+  // 4 is the "blockPresences" field number in message.proto
+  return calculateLength(4, BlockPresence.encode(blockPresence))
+}
+
+function calculateEncodedWantlistEntrySize (entry: WantlistEntry): number {
+  // 1 is the "entries" field number in message.proto
+  return calculateLength(1, WantlistEntry.encode(entry))
+}
+
+function calculateLength (fieldNumber: number, data: Uint8Array): number {
+  const fieldNumberLength = encodingLength(fieldNumber)
+  const dataLengthLength = encodingLength(data.byteLength)
+
+  return fieldNumberLength + dataLengthLength + data.byteLength
+}