@helia/bitswap 0.0.0-329652a

package/src/peer-want-lists/ledger.ts

@@ -0,0 +1,161 @@
+/* eslint-disable max-depth */
+import { DEFAULT_MAX_SIZE_REPLACE_HAS_WITH_BLOCK } from '../constants.js'
+import { BlockPresenceType, type BitswapMessage, WantType } from '../pb/message.js'
+import { cidToPrefix } from '../utils/cid-prefix.js'
+import type { Network } from '../network.js'
+import type { PeerId } from '@libp2p/interface'
+import type { Blockstore } from 'interface-blockstore'
+import type { AbortOptions } from 'it-length-prefixed-stream'
+import type { CID } from 'multiformats/cid'
+
+export interface LedgerComponents {
+  peerId: PeerId
+  blockstore: Blockstore
+  network: Network
+}
+
+export interface LedgerInit {
+  maxSizeReplaceHasWithBlock?: number
+}
+
+export interface PeerWantListEntry {
+  /**
+   * The CID the peer has requested
+   */
+  cid: CID
+
+  /**
+   * The priority with which the remote should return the block
+   */
+  priority: number
+
+  /**
+   * If we want the block or if we want the remote to tell us if they have the
+   * block - note if the block is small they'll send it to us anyway.
+   */
+  wantType: WantType
+
+  /**
+   * Whether the remote should tell us if they have the block or not
+   */
+  sendDontHave: boolean
+
+  /**
+   * If we don't have the block and we've told them we don't have the block
+   */
+  sentDontHave?: boolean
+}
+
+export class Ledger {
+  public peerId: PeerId
+  private readonly blockstore: Blockstore
+  private readonly network: Network
+  public wants: Map<string, PeerWantListEntry>
+  public exchangeCount: number
+  public bytesSent: number
+  public bytesReceived: number
+  public lastExchange?: number
+  private readonly maxSizeReplaceHasWithBlock: number
+
+  constructor (components: LedgerComponents, init: LedgerInit) {
+    this.peerId = components.peerId
+    this.blockstore = components.blockstore
+    this.network = components.network
+    this.wants = new Map()
+
+    this.exchangeCount = 0
+    this.bytesSent = 0
+    this.bytesReceived = 0
+    this.maxSizeReplaceHasWithBlock = init.maxSizeReplaceHasWithBlock ?? DEFAULT_MAX_SIZE_REPLACE_HAS_WITH_BLOCK
+  }
+
+  sentBytes (n: number): void {
+    this.exchangeCount++
+    this.lastExchange = (new Date()).getTime()
+    this.bytesSent += n
+  }
+
+  receivedBytes (n: number): void {
+    this.exchangeCount++
+    this.lastExchange = (new Date()).getTime()
+    this.bytesReceived += n
+  }
+
+  debtRatio (): number {
+    return (this.bytesSent / (this.bytesReceived + 1)) // +1 is to prevent division by zero
+  }
+
+  public async sendBlocksToPeer (options?: AbortOptions): Promise<void> {
+    const message: Pick<BitswapMessage, 'blockPresences' | 'blocks'> = {
+      blockPresences: [],
+      blocks: []
+    }
+    const sentBlocks = new Set<string>()
+
+    for (const [key, entry] of this.wants.entries()) {
+      const has = await this.blockstore.has(entry.cid, options)
+
+      if (!has) {
+        // we don't have the requested block and the remote is not interested
+        // in us telling them that
+        if (!entry.sendDontHave) {
+          continue
+        }
+
+        // we have already told them we don't have the block
+        if (entry.sentDontHave === true) {
+          continue
+        }
+
+        entry.sentDontHave = true
+        message.blockPresences.push({
+          cid: entry.cid.bytes,
+          type: BlockPresenceType.DontHaveBlock
+        })
+
+        continue
+      }
+
+      const block = await this.blockstore.get(entry.cid, options)
+
+      // do they want the block or just us to tell them we have the block
+      if (entry.wantType === WantType.WantHave) {
+        if (block.byteLength < this.maxSizeReplaceHasWithBlock) {
+          // if the block is small we just send it to them
+          sentBlocks.add(key)
+          message.blocks.push({
+            data: block,
+            prefix: cidToPrefix(entry.cid)
+          })
+        } else {
+          // otherwise tell them we have the block
+          message.blockPresences.push({
+            cid: entry.cid.bytes,
+            type: BlockPresenceType.HaveBlock
+          })
+        }
+      } else {
+        // they want the block, send it to them
+        sentBlocks.add(key)
+        message.blocks.push({
+          data: block,
+          prefix: cidToPrefix(entry.cid)
+        })
+      }
+    }
+
+    // only send the message if we actually have something to send
+    if (message.blocks.length > 0 || message.blockPresences.length > 0) {
+      await this.network.sendMessage(this.peerId, message, options)
+
+      // update accounting
+      this.sentBytes(message.blocks.reduce((acc, curr) => acc + curr.data.byteLength, 0))
+
+      // remove sent blocks from local copy of their want list - they can still
+      // re-request if required
+      for (const key of sentBlocks) {
+        this.wants.delete(key)
+      }
+    }
+  }
+}

package/src/session.ts

@@ -0,0 +1,150 @@
+import { CodeError } from '@libp2p/interface'
+import { PeerSet } from '@libp2p/peer-collections'
+import { PeerQueue } from '@libp2p/utils/peer-queue'
+import map from 'it-map'
+import merge from 'it-merge'
+import pDefer, { type DeferredPromise } from 'p-defer'
+import type { BitswapWantProgressEvents, BitswapSession as BitswapSessionInterface } from './index.js'
+import type { Network } from './network.js'
+import type { WantList } from './want-list.js'
+import type { ComponentLogger, Logger, PeerId } from '@libp2p/interface'
+import type { AbortOptions } from 'interface-store'
+import type { CID } from 'multiformats/cid'
+import type { ProgressOptions } from 'progress-events'
+
+export interface BitswapSessionComponents {
+  network: Network
+  wantList: WantList
+  logger: ComponentLogger
+}
+
+export interface BitswapSessionInit extends AbortOptions {
+  root: CID
+  queryConcurrency: number
+  minProviders: number
+  maxProviders: number
+  connectedPeers: PeerId[]
+}
+
+class BitswapSession implements BitswapSessionInterface {
+  public readonly root: CID
+  public readonly peers: PeerSet
+  private readonly log: Logger
+  private readonly wantList: WantList
+  private readonly network: Network
+  private readonly queue: PeerQueue
+  private readonly maxProviders: number
+
+  constructor (components: BitswapSessionComponents, init: BitswapSessionInit) {
+    this.peers = new PeerSet()
+    this.root = init.root
+    this.maxProviders = init.maxProviders
+    this.log = components.logger.forComponent(`helia:bitswap:session:${init.root}`)
+    this.wantList = components.wantList
+    this.network = components.network
+
+    this.queue = new PeerQueue({
+      concurrency: init.queryConcurrency
+    })
+    this.queue.addEventListener('error', (evt) => {
+      this.log.error('error querying peer for %c', this.root, evt.detail)
+    })
+  }
+
+  async want (cid: CID, options: AbortOptions & ProgressOptions<BitswapWantProgressEvents> = {}): Promise<Uint8Array> {
+    if (this.peers.size === 0) {
+      throw new CodeError('Bitswap session had no peers', 'ERR_NO_SESSION_PEERS')
+    }
+
+    this.log('sending WANT-BLOCK for %c to', cid, this.peers)
+
+    const result = await Promise.any(
+      [...this.peers].map(async peerId => {
+        return this.wantList.wantBlock(cid, {
+          peerId,
+          ...options
+        })
+      })
+    )
+
+    this.log('received block for %c from %p', cid, result.sender)
+
+    // TODO findNewProviders when promise.any throws aggregate error and signal
+    // is not aborted
+
+    return result.block
+  }
+
+  async findNewProviders (cid: CID, count: number, options: AbortOptions = {}): Promise<void> {
+    const deferred: DeferredPromise<void> = pDefer()
+    let found = 0
+
+    this.log('find %d-%d new provider(s) for %c', count, this.maxProviders, cid)
+
+    const source = merge(
+      [...this.wantList.peers.keys()],
+      map(this.network.findProviders(cid, options), prov => prov.id)
+    )
+
+    void Promise.resolve()
+      .then(async () => {
+        for await (const peerId of source) {
+          // eslint-disable-next-line no-loop-func
+          await this.queue.add(async () => {
+            try {
+              this.log('asking potential session peer %p if they have %c', peerId, cid)
+              const result = await this.wantList.wantPresence(cid, {
+                peerId,
+                ...options
+              })
+
+              if (!result.has) {
+                this.log('potential session peer %p did not have %c', peerId, cid)
+                return
+              }
+
+              this.log('potential session peer %p had %c', peerId, cid)
+              found++
+
+              // add to list
+              this.peers.add(peerId)
+
+              if (found === count) {
+                this.log('found %d session peers', found)
+
+                deferred.resolve()
+              }
+
+              if (found === this.maxProviders) {
+                this.log('found max provider session peers', found)
+
+                this.queue.clear()
+              }
+            } catch (err: any) {
+              this.log.error('error querying potential session peer %p for %c', peerId, cid, err.errors ?? err)
+            }
+          }, {
+            peerId
+          })
+        }
+
+        this.log('found %d session peers total', found)
+
+        if (count > 0) {
+          deferred.reject(new CodeError(`Found ${found} of ${count} providers`, 'ERR_NO_PROVIDERS_FOUND'))
+        }
+      })
+
+    return deferred.promise
+  }
+}
+
+export async function createBitswapSession (components: BitswapSessionComponents, init: BitswapSessionInit): Promise<BitswapSessionInterface> {
+  const session = new BitswapSession(components, init)
+
+  await session.findNewProviders(init.root, init.minProviders, {
+    signal: init.signal
+  })
+
+  return session
+}

package/src/stats.ts

@@ -0,0 +1,67 @@
+import type { MetricGroup, Metrics, PeerId } from '@libp2p/interface'
+
+export interface StatsComponents {
+  metrics?: Metrics
+}
+
+export class Stats {
+  private readonly blocksReceived?: MetricGroup
+  private readonly duplicateBlocksReceived?: MetricGroup
+  private readonly dataReceived?: MetricGroup
+  private readonly duplicateDataReceived?: MetricGroup
+
+  constructor (components: StatsComponents) {
+    this.blocksReceived = components.metrics?.registerMetricGroup('ipfs_bitswap_received_blocks')
+    this.duplicateBlocksReceived = components.metrics?.registerMetricGroup('ipfs_bitswap_duplicate_received_blocks')
+    this.dataReceived = components.metrics?.registerMetricGroup('ipfs_bitswap_data_received_bytes')
+    this.duplicateDataReceived = components.metrics?.registerMetricGroup('ipfs_bitswap_duplicate_data_received_bytes')
+  }
+
+  updateBlocksReceived (count: number = 1, peerId?: PeerId): void {
+    const stats: Record<string, number | unknown> = {
+      global: count
+    }
+
+    if (peerId != null) {
+      stats[peerId.toString()] = count
+    }
+
+    this.blocksReceived?.increment(stats)
+  }
+
+  updateDuplicateBlocksReceived (count: number = 1, peerId?: PeerId): void {
+    const stats: Record<string, number | unknown> = {
+      global: count
+    }
+
+    if (peerId != null) {
+      stats[peerId.toString()] = count
+    }
+
+    this.duplicateBlocksReceived?.increment(stats)
+  }
+
+  updateDataReceived (bytes: number, peerId?: PeerId): void {
+    const stats: Record<string, number> = {
+      global: bytes
+    }
+
+    if (peerId != null) {
+      stats[peerId.toString()] = bytes
+    }
+
+    this.dataReceived?.increment(stats)
+  }
+
+  updateDuplicateDataReceived (bytes: number, peerId?: PeerId): void {
+    const stats: Record<string, number> = {
+      global: bytes
+    }
+
+    if (peerId != null) {
+      stats[peerId.toString()] = bytes
+    }
+
+    this.duplicateDataReceived?.increment(stats)
+  }
+}

package/src/utils/cid-prefix.ts

@@ -0,0 +1,8 @@
+import ve from './varint-encoder.js'
+import type { CID } from 'multiformats/cid'
+
+export function cidToPrefix (cid: CID): Uint8Array {
+  return ve([
+    cid.version, cid.code, cid.multihash.code, cid.multihash.digest.byteLength
+  ])
+}

package/src/utils/varint-decoder.ts

@@ -0,0 +1,19 @@
+import { decode, encodingLength } from 'uint8-varint'
+
+function varintDecoder (buf: Uint8Array): number[] {
+  if (!(buf instanceof Uint8Array)) {
+    throw new Error('arg needs to be a Uint8Array')
+  }
+
+  const result: number[] = []
+
+  while (buf.length > 0) {
+    const num = decode(buf)
+    result.push(num)
+    buf = buf.slice(encodingLength(num))
+  }
+
+  return result
+}
+
+export default varintDecoder

package/src/utils/varint-encoder.ts

@@ -0,0 +1,18 @@
+import { encode, encodingLength } from 'uint8-varint'
+
+function varintEncoder (buf: number[]): Uint8Array {
+  let out = new Uint8Array(buf.reduce((acc, curr) => {
+    return acc + encodingLength(curr)
+  }, 0))
+  let offset = 0
+
+  for (const num of buf) {
+    out = encode(num, out, offset)
+
+    offset += encodingLength(num)
+  }
+
+  return out
+}
+
+export default varintEncoder