@helia/bitswap 0.0.0-e582c63 → 1.0.0-338885f

package/src/bitswap.ts

@@ -1,5 +1,4 @@
 /* eslint-disable no-loop-func */
-import { DEFAULT_SESSION_MAX_PROVIDERS, DEFAULT_SESSION_MIN_PROVIDERS, DEFAULT_SESSION_PROVIDER_QUERY_CONCURRENCY } from '@helia/interface'
 import { setMaxListeners } from '@libp2p/interface'
 import { anySignal } from 'any-signal'
 import { Network } from './network.js'
@@ -7,7 +6,8 @@ import { PeerWantLists } from './peer-want-lists/index.js'
 import { createBitswapSession } from './session.js'
 import { Stats } from './stats.js'
 import { WantList } from './want-list.js'
-import type { BitswapOptions, Bitswap as BitswapInterface, BitswapWantProgressEvents, BitswapNotifyProgressEvents, BitswapSession, WantListEntry, BitswapComponents, CreateBitswapSessionOptions } from './index.js'
+import type { BitswapOptions, Bitswap as BitswapInterface, BitswapWantProgressEvents, BitswapNotifyProgressEvents, WantListEntry, BitswapComponents } from './index.js'
+import type { BlockBroker, CreateSessionOptions } from '@helia/interface'
 import type { ComponentLogger, PeerId } from '@libp2p/interface'
 import type { Logger } from '@libp2p/logger'
 import type { AbortOptions } from '@multiformats/multiaddr'
@@ -62,28 +62,18 @@ export class Bitswap implements BitswapInterface {
     }, init)
   }
 
-  async createSession (root: CID, options?: CreateBitswapSessionOptions): Promise<BitswapSession> {
-    const minProviders = options?.minProviders ?? DEFAULT_SESSION_MIN_PROVIDERS
-    const maxProviders = options?.maxProviders ?? DEFAULT_SESSION_MAX_PROVIDERS
-
+  createSession (options: CreateSessionOptions = {}): Required<Pick<BlockBroker<BitswapWantProgressEvents>, 'retrieve'>> {
     return createBitswapSession({
       wantList: this.wantList,
       network: this.network,
       logger: this.logger
-    }, {
-      root,
-      queryConcurrency: options?.providerQueryConcurrency ?? DEFAULT_SESSION_PROVIDER_QUERY_CONCURRENCY,
-      minProviders,
-      maxProviders,
-      connectedPeers: options?.queryConnectedPeers !== false ? [...this.wantList.peers.keys()] : [],
-      signal: options?.signal
-    })
+    }, options)
   }
 
   async want (cid: CID, options: WantOptions = {}): Promise<Uint8Array> {
     const controller = new AbortController()
-    setMaxListeners(Infinity, controller.signal)
     const signal = anySignal([controller.signal, options.signal])
+    setMaxListeners(Infinity, controller.signal, signal)
 
     // find providers and connect to them
     this.network.findAndConnect(cid, {
@@ -117,7 +107,10 @@ export class Bitswap implements BitswapInterface {
    * Sends notifications about the arrival of a block
    */
   async notify (cid: CID, block: Uint8Array, options: ProgressOptions<BitswapNotifyProgressEvents> & AbortOptions = {}): Promise<void> {
-    await this.peerWantLists.receivedBlock(cid, options)
+    await Promise.all([
+      this.peerWantLists.receivedBlock(cid, options),
+      this.wantList.receivedBlock(cid, options)
+    ])
   }
 
   getWantlist (): WantListEntry[] {

package/src/index.ts

@@ -9,10 +9,9 @@
 import { Bitswap as BitswapClass } from './bitswap.js'
 import type { BitswapNetworkNotifyProgressEvents, BitswapNetworkWantProgressEvents } from './network.js'
 import type { WantType } from './pb/message.js'
-import type { CreateSessionOptions } from '@helia/interface'
+import type { BlockBroker, CreateSessionOptions } from '@helia/interface'
 import type { Routing } from '@helia/interface/routing'
 import type { Libp2p, AbortOptions, Startable, ComponentLogger, Metrics, PeerId } from '@libp2p/interface'
-import type { PeerSet } from '@libp2p/peer-collections'
 import type { Blockstore } from 'interface-blockstore'
 import type { CID } from 'multiformats/cid'
 import type { MultihashHasher } from 'multiformats/hashes/interface'
@@ -29,52 +28,12 @@ export type BitswapWantBlockProgressEvents =
   ProgressEvent<'bitswap:want-block:block', CID> |
   BitswapNetworkWantProgressEvents
 
-/**
- * A bitswap session is a network overlay consisting of peers that all have the
- * first block in a file. Subsequent requests will only go to these peers.
- */
-export interface BitswapSession {
-  /**
-   * The peers in this session
-   */
-  peers: PeerSet
-
-  /**
-   * Fetch an additional CID from this DAG
-   */
-  want(cid: CID, options?: AbortOptions & ProgressOptions<BitswapWantProgressEvents>): Promise<Uint8Array>
-}
-
 export interface WantListEntry {
   cid: CID
   priority: number
   wantType: WantType
 }
 
-export interface CreateBitswapSessionOptions extends CreateSessionOptions<BitswapWantProgressEvents> {
-  /**
-   * If true, query connected peers before searching for providers via
-   * Helia routers
-   *
-   * @default true
-   */
-  queryConnectedPeers?: boolean
-
-  /**
-   * If true, search for providers via Helia routers to query for the root CID
-   *
-   * @default true
-   */
-  queryRoutingPeers?: boolean
-
-  /**
-   * The priority to use when querying availability of the root CID
-   *
-   * @default 1
-   */
-  priority?: number
-}
-
 export interface Bitswap extends Startable {
   /**
    * Returns the current state of the wantlist
@@ -100,7 +59,7 @@ export interface Bitswap extends Startable {
   /**
    * Start a session to retrieve a file from the network
    */
-  createSession(root: CID, options?: AbortOptions & ProgressOptions<BitswapWantProgressEvents>): Promise<BitswapSession>
+  createSession(options?: CreateSessionOptions<BitswapWantProgressEvents>): Required<Pick<BlockBroker<BitswapWantProgressEvents>, 'retrieve'>>
 }
 
 export interface MultihashHasherLoader {

package/src/network.ts

@@ -1,6 +1,5 @@
 import { CodeError, TypedEventEmitter, setMaxListeners } from '@libp2p/interface'
 import { PeerQueue, type PeerQueueJobOptions } from '@libp2p/utils/peer-queue'
-import { Circuit } from '@multiformats/multiaddr-matcher'
 import { anySignal } from 'any-signal'
 import debug from 'debug'
 import drain from 'it-drain'
@@ -14,13 +13,13 @@ import { CID } from 'multiformats/cid'
 import { CustomProgressEvent } from 'progress-events'
 import { raceEvent } from 'race-event'
 import { toString as uint8ArrayToString } from 'uint8arrays/to-string'
-import { BITSWAP_120, DEFAULT_MAX_INBOUND_STREAMS, DEFAULT_MAX_OUTBOUND_STREAMS, DEFAULT_MAX_PROVIDERS_PER_REQUEST, DEFAULT_MESSAGE_RECEIVE_TIMEOUT, DEFAULT_MESSAGE_SEND_TIMEOUT, DEFAULT_RUN_ON_TRANSIENT_CONNECTIONS } from './constants.js'
+import { BITSWAP_120, DEFAULT_MAX_INBOUND_STREAMS, DEFAULT_MAX_OUTBOUND_STREAMS, DEFAULT_MAX_PROVIDERS_PER_REQUEST, DEFAULT_MESSAGE_RECEIVE_TIMEOUT, DEFAULT_MESSAGE_SEND_CONCURRENCY, DEFAULT_MESSAGE_SEND_TIMEOUT, DEFAULT_RUN_ON_TRANSIENT_CONNECTIONS } from './constants.js'
 import { BitswapMessage } from './pb/message.js'
 import type { WantOptions } from './bitswap.js'
 import type { MultihashHasherLoader } from './index.js'
 import type { Block, BlockPresence, WantlistEntry } from './pb/message.js'
 import type { Provider, Routing } from '@helia/interface/routing'
-import type { Libp2p, AbortOptions, Connection, PeerId, IncomingStreamData, Topology, MetricGroup, ComponentLogger, Metrics, IdentifyResult } from '@libp2p/interface'
+import type { Libp2p, AbortOptions, Connection, PeerId, IncomingStreamData, Topology, ComponentLogger, IdentifyResult, Counter } from '@libp2p/interface'
 import type { Logger } from '@libp2p/logger'
 import type { ProgressEvent, ProgressOptions } from 'progress-events'
 
@@ -79,7 +78,6 @@ export interface NetworkComponents {
   routing: Routing
   logger: ComponentLogger
   libp2p: Libp2p
-  metrics?: Metrics
 }
 
 export interface BitswapMessageEventDetail {
@@ -107,7 +105,7 @@ export class Network extends TypedEventEmitter<NetworkEvents> {
   private readonly maxOutboundStreams: number
   private readonly messageReceiveTimeout: number
   private registrarIds: string[]
-  private readonly metrics?: { blocksSent: MetricGroup, dataSent: MetricGroup }
+  private readonly metrics: { blocksSent?: Counter, dataSent?: Counter }
   private readonly sendQueue: PeerQueue<void, SendMessageJobOptions>
   private readonly messageSendTimeout: number
   private readonly runOnTransientConnections: boolean
@@ -129,18 +127,15 @@ export class Network extends TypedEventEmitter<NetworkEvents> {
     this.messageReceiveTimeout = init.messageReceiveTimeout ?? DEFAULT_MESSAGE_RECEIVE_TIMEOUT
     this.messageSendTimeout = init.messageSendTimeout ?? DEFAULT_MESSAGE_SEND_TIMEOUT
     this.runOnTransientConnections = init.runOnTransientConnections ?? DEFAULT_RUN_ON_TRANSIENT_CONNECTIONS
-
-    if (components.metrics != null) {
-      this.metrics = {
-        blocksSent: components.metrics?.registerMetricGroup('ipfs_bitswap_sent_blocks'),
-        dataSent: components.metrics?.registerMetricGroup('ipfs_bitswap_sent_data_bytes')
-      }
+    this.metrics = {
+      blocksSent: components.libp2p.metrics?.registerCounter('helia_bitswap_sent_blocks_total'),
+      dataSent: components.libp2p.metrics?.registerCounter('helia_bitswap_sent_data_bytes_total')
     }
 
     this.sendQueue = new PeerQueue({
-      concurrency: init.messageSendConcurrency,
-      metrics: components.metrics,
-      metricName: 'ipfs_bitswap_message_send_queue'
+      concurrency: init.messageSendConcurrency ?? DEFAULT_MESSAGE_SEND_CONCURRENCY,
+      metrics: components.libp2p.metrics,
+      metricName: 'helia_bitswap_message_send_queue'
     })
     this.sendQueue.addEventListener('error', (evt) => {
       this.log.error('error sending wantlist to peer', evt.detail)
@@ -267,29 +262,12 @@ export class Network extends TypedEventEmitter<NetworkEvents> {
     options?.onProgress?.(new CustomProgressEvent<PeerId>('bitswap:network:find-providers', cid))
 
     for await (const provider of this.routing.findProviders(cid, options)) {
-      // unless we explicitly run on transient connections, skip peers that only
-      // have circuit relay addresses as bitswap won't run over them
-      if (!this.runOnTransientConnections) {
-        let hasDirectAddress = false
-
-        for (let ma of provider.multiaddrs) {
-          if (ma.getPeerId() == null) {
-            ma = ma.encapsulate(`/p2p/${provider.id}`)
-          }
-
-          if (!Circuit.exactMatch(ma)) {
-            hasDirectAddress = true
-            break
-          }
-        }
-
-        if (!hasDirectAddress) {
-          continue
-        }
-      }
+      // make sure we can dial the provider
+      const dialable = await this.libp2p.isDialable(provider.multiaddrs, {
+        runOnTransientConnection: this.runOnTransientConnections
+      })
 
-      // ignore non-bitswap providers
-      if (provider.protocols?.includes('transport-bitswap') === false) {
+      if (!dialable) {
         continue
       }
 
@@ -302,9 +280,9 @@ export class Network extends TypedEventEmitter<NetworkEvents> {
    */
   async findAndConnect (cid: CID, options?: WantOptions): Promise<void> {
     await drain(
-      take(
-        map(this.findProviders(cid, options), async provider => this.connectTo(provider.id, options)),
-        options?.maxProviders ?? DEFAULT_MAX_PROVIDERS_PER_REQUEST
+      map(
+        take(this.findProviders(cid, options), options?.maxProviders ?? DEFAULT_MAX_PROVIDERS_PER_REQUEST),
+        async provider => this.connectTo(provider.id, options)
       )
     )
       .catch(err => {
@@ -331,13 +309,16 @@ export class Network extends TypedEventEmitter<NetworkEvents> {
       pendingBytes: msg.pendingBytes ?? 0
     }
 
-    const signal = anySignal([AbortSignal.timeout(this.messageSendTimeout), options?.signal])
-    setMaxListeners(Infinity, signal)
+    const timeoutSignal = AbortSignal.timeout(this.messageSendTimeout)
+    const signal = anySignal([timeoutSignal, options?.signal])
+    setMaxListeners(Infinity, timeoutSignal, signal)
 
     try {
-      const existingJob = this.sendQueue.find(peerId)
+      const existingJob = this.sendQueue.queue.find(job => {
+        return peerId.equals(job.options.peerId) && job.status === 'queued'
+      })
 
-      if (existingJob?.status === 'queued') {
+      if (existingJob != null) {
         // merge messages instead of adding new job
         existingJob.options.message = mergeMessages(existingJob.options.message, message)
 
@@ -371,7 +352,7 @@ export class Network extends TypedEventEmitter<NetworkEvents> {
           stream.abort(err)
         }
 
-        this._updateSentStats(peerId, message.blocks)
+        this._updateSentStats(message.blocks)
       }, {
         peerId,
         signal,
@@ -417,23 +398,15 @@ export class Network extends TypedEventEmitter<NetworkEvents> {
     return connection
   }
 
-  _updateSentStats (peerId: PeerId, blocks: Block[] = []): void {
-    if (this.metrics != null) {
-      let bytes = 0
-
-      for (const block of blocks.values()) {
-        bytes += block.data.byteLength
-      }
+  _updateSentStats (blocks: Block[] = []): void {
+    let bytes = 0
 
-      this.metrics.dataSent.increment({
-        global: bytes,
-        [peerId.toString()]: bytes
-      })
-      this.metrics.blocksSent.increment({
-        global: blocks.length,
-        [peerId.toString()]: blocks.length
-      })
+    for (const block of blocks.values()) {
+      bytes += block.data.byteLength
     }
+
+    this.metrics.dataSent?.increment(bytes)
+    this.metrics.blocksSent?.increment(blocks.length)
   }
 }
 

package/src/peer-want-lists/index.ts

@@ -6,7 +6,7 @@ import { Ledger } from './ledger.js'
 import type { BitswapNotifyProgressEvents, WantListEntry } from '../index.js'
 import type { Network } from '../network.js'
 import type { BitswapMessage } from '../pb/message.js'
-import type { ComponentLogger, Logger, Metrics, PeerId } from '@libp2p/interface'
+import type { ComponentLogger, Libp2p, Logger, PeerId } from '@libp2p/interface'
 import type { PeerMap } from '@libp2p/peer-collections'
 import type { Blockstore } from 'interface-blockstore'
 import type { AbortOptions } from 'it-length-prefixed-stream'
@@ -19,7 +19,7 @@ export interface PeerWantListsInit {
 export interface PeerWantListsComponents {
   blockstore: Blockstore
   network: Network
-  metrics?: Metrics
+  libp2p: Libp2p
   logger: ComponentLogger
 }
 
@@ -45,8 +45,8 @@ export class PeerWantLists {
     this.log = components.logger.forComponent('helia:bitswap:peer-want-lists')
 
     this.ledgerMap = trackedPeerMap({
-      name: 'ipfs_bitswap_ledger_map',
-      metrics: components.metrics
+      name: 'helia_bitswap_ledger_map',
+      metrics: components.libp2p.metrics
     })
 
     this.network.addEventListener('bitswap:message', (evt) => {

package/src/session.ts

@@ -1,16 +1,11 @@
-import { CodeError } from '@libp2p/interface'
-import { PeerSet } from '@libp2p/peer-collections'
-import { PeerQueue } from '@libp2p/utils/peer-queue'
-import map from 'it-map'
-import merge from 'it-merge'
-import pDefer, { type DeferredPromise } from 'p-defer'
-import type { BitswapWantProgressEvents, BitswapSession as BitswapSessionInterface } from './index.js'
+import { AbstractSession } from '@helia/utils'
+import type { BitswapWantProgressEvents } from './index.js'
 import type { Network } from './network.js'
 import type { WantList } from './want-list.js'
-import type { ComponentLogger, Logger, PeerId } from '@libp2p/interface'
+import type { CreateSessionOptions } from '@helia/interface'
+import type { ComponentLogger, PeerId } from '@libp2p/interface'
 import type { AbortOptions } from 'interface-store'
 import type { CID } from 'multiformats/cid'
-import type { ProgressOptions } from 'progress-events'
 
 export interface BitswapSessionComponents {
   network: Network
@@ -18,133 +13,49 @@ export interface BitswapSessionComponents {
   logger: ComponentLogger
 }
 
-export interface BitswapSessionInit extends AbortOptions {
-  root: CID
-  queryConcurrency: number
-  minProviders: number
-  maxProviders: number
-  connectedPeers: PeerId[]
-}
-
-class BitswapSession implements BitswapSessionInterface {
-  public readonly root: CID
-  public readonly peers: PeerSet
-  private readonly log: Logger
+class BitswapSession extends AbstractSession<PeerId, BitswapWantProgressEvents> {
   private readonly wantList: WantList
   private readonly network: Network
-  private readonly queue: PeerQueue
-  private readonly maxProviders: number
 
-  constructor (components: BitswapSessionComponents, init: BitswapSessionInit) {
-    this.peers = new PeerSet()
-    this.root = init.root
-    this.maxProviders = init.maxProviders
-    this.log = components.logger.forComponent(`helia:bitswap:session:${init.root}`)
+  constructor (components: BitswapSessionComponents, init: CreateSessionOptions) {
+    super(components, {
+      ...init,
+      name: 'helia:bitswap:session'
+    })
+
     this.wantList = components.wantList
     this.network = components.network
-
-    this.queue = new PeerQueue({
-      concurrency: init.queryConcurrency
-    })
-    this.queue.addEventListener('error', (evt) => {
-      this.log.error('error querying peer for %c', this.root, evt.detail)
-    })
   }
 
-  async want (cid: CID, options: AbortOptions & ProgressOptions<BitswapWantProgressEvents> = {}): Promise<Uint8Array> {
-    if (this.peers.size === 0) {
-      throw new CodeError('Bitswap session had no peers', 'ERR_NO_SESSION_PEERS')
-    }
-
-    this.log('sending WANT-BLOCK for %c to', cid, this.peers)
+  async queryProvider (cid: CID, provider: PeerId, options: AbortOptions): Promise<Uint8Array> {
+    this.log('sending WANT-BLOCK for %c to %p', cid, provider)
 
-    const result = await Promise.any(
-      [...this.peers].map(async peerId => {
-        return this.wantList.wantBlock(cid, {
-          peerId,
-          ...options
-        })
-      })
-    )
+    const result = await this.wantList.wantSessionBlock(cid, provider, options)
 
-    this.log('received block for %c from %p', cid, result.sender)
+    this.log('%p %s %c', provider, result.has ? 'has' : 'does not have', cid)
 
-    // TODO findNewProviders when promise.any throws aggregate error and signal
-    // is not aborted
+    if (result.has && result.block != null) {
+      return result.block
+    }
 
-    return result.block
+    throw new Error('Provider did not have block')
   }
 
-  async findNewProviders (cid: CID, count: number, options: AbortOptions = {}): Promise<void> {
-    const deferred: DeferredPromise<void> = pDefer()
-    let found = 0
-
-    this.log('find %d-%d new provider(s) for %c', count, this.maxProviders, cid)
-
-    const source = merge(
-      [...this.wantList.peers.keys()],
-      map(this.network.findProviders(cid, options), prov => prov.id)
-    )
-
-    void Promise.resolve()
-      .then(async () => {
-        for await (const peerId of source) {
-          // eslint-disable-next-line no-loop-func
-          await this.queue.add(async () => {
-            try {
-              this.log('asking potential session peer %p if they have %c', peerId, cid)
-              const result = await this.wantList.wantPresence(cid, {
-                peerId,
-                ...options
-              })
-
-              if (!result.has) {
-                this.log('potential session peer %p did not have %c', peerId, cid)
-                return
-              }
-
-              this.log('potential session peer %p had %c', peerId, cid)
-              found++
-
-              // add to list
-              this.peers.add(peerId)
-
-              if (found === count) {
-                this.log('found %d session peers', found)
-
-                deferred.resolve()
-              }
-
-              if (found === this.maxProviders) {
-                this.log('found max provider session peers', found)
-
-                this.queue.clear()
-              }
-            } catch (err: any) {
-              this.log.error('error querying potential session peer %p for %c', peerId, cid, err.errors ?? err)
-            }
-          }, {
-            peerId
-          })
-        }
-
-        this.log('found %d session peers total', found)
+  async * findNewProviders (cid: CID, options: AbortOptions = {}): AsyncGenerator<PeerId> {
+    for await (const provider of this.network.findProviders(cid, options)) {
+      yield provider.id
+    }
+  }
 
-        if (count > 0) {
-          deferred.reject(new CodeError(`Found ${found} of ${count} providers`, 'ERR_NO_PROVIDERS_FOUND'))
-        }
-      })
+  toEvictionKey (provider: PeerId): Uint8Array | string {
+    return provider.toBytes()
+  }
 
-    return deferred.promise
+  equals (providerA: PeerId, providerB: PeerId): boolean {
+    return providerA.equals(providerB)
   }
 }
 
-export async function createBitswapSession (components: BitswapSessionComponents, init: BitswapSessionInit): Promise<BitswapSessionInterface> {
-  const session = new BitswapSession(components, init)
-
-  await session.findNewProviders(init.root, init.minProviders, {
-    signal: init.signal
-  })
-
-  return session
+export function createBitswapSession (components: BitswapSessionComponents, init: CreateSessionOptions): BitswapSession {
+  return new BitswapSession(components, init)
 }

package/src/stats.ts

@@ -1,7 +1,7 @@
-import type { MetricGroup, Metrics, PeerId } from '@libp2p/interface'
+import type { Libp2p, MetricGroup, PeerId } from '@libp2p/interface'
 
 export interface StatsComponents {
-  metrics?: Metrics
+  libp2p: Libp2p
 }
 
 export class Stats {
@@ -11,10 +11,10 @@ export class Stats {
   private readonly duplicateDataReceived?: MetricGroup
 
   constructor (components: StatsComponents) {
-    this.blocksReceived = components.metrics?.registerMetricGroup('ipfs_bitswap_received_blocks')
-    this.duplicateBlocksReceived = components.metrics?.registerMetricGroup('ipfs_bitswap_duplicate_received_blocks')
-    this.dataReceived = components.metrics?.registerMetricGroup('ipfs_bitswap_data_received_bytes')
-    this.duplicateDataReceived = components.metrics?.registerMetricGroup('ipfs_bitswap_duplicate_data_received_bytes')
+    this.blocksReceived = components.libp2p.metrics?.registerMetricGroup('helia_bitswap_received_blocks')
+    this.duplicateBlocksReceived = components.libp2p.metrics?.registerMetricGroup('helia_bitswap_duplicate_received_blocks')
+    this.dataReceived = components.libp2p.metrics?.registerMetricGroup('helia_bitswap_data_received_bytes')
+    this.duplicateDataReceived = components.libp2p.metrics?.registerMetricGroup('helia_bitswap_duplicate_data_received_bytes')
   }
 
   updateBlocksReceived (count: number = 1, peerId?: PeerId): void {