@helia/utils 0.1.0-ecf5394 → 0.2.0

package/src/abstract-session.ts

@@ -0,0 +1,287 @@
+import { DEFAULT_SESSION_MIN_PROVIDERS, DEFAULT_SESSION_MAX_PROVIDERS } from '@helia/interface'
+import { CodeError, TypedEventEmitter, setMaxListeners } from '@libp2p/interface'
+import { Queue } from '@libp2p/utils/queue'
+import { base64 } from 'multiformats/bases/base64'
+import pDefer from 'p-defer'
+import { BloomFilter } from './bloom-filter.js'
+import type { BlockBroker, BlockRetrievalOptions, CreateSessionOptions } from '@helia/interface'
+import type { AbortOptions, ComponentLogger, Logger } from '@libp2p/interface'
+import type { CID } from 'multiformats/cid'
+import type { DeferredPromise } from 'p-defer'
+import type { ProgressEvent } from 'progress-events'
+
+export interface AbstractSessionComponents {
+  logger: ComponentLogger
+}
+
+export interface AbstractCreateSessionOptions extends CreateSessionOptions {
+  name: string
+}
+
+export interface BlockstoreSessionEvents<Provider> {
+  provider: CustomEvent<Provider>
+}
+
+export abstract class AbstractSession<Provider, RetrieveBlockProgressEvents extends ProgressEvent> extends TypedEventEmitter<BlockstoreSessionEvents<Provider>> implements BlockBroker<RetrieveBlockProgressEvents> {
+  private intialPeerSearchComplete?: Promise<void>
+  private readonly requests: Map<string, Promise<Uint8Array>>
+  private readonly name: string
+  protected log: Logger
+  protected logger: ComponentLogger
+  private readonly minProviders: number
+  private readonly maxProviders: number
+  public readonly providers: Provider[]
+  private readonly evictionFilter: BloomFilter
+
+  constructor (components: AbstractSessionComponents, init: AbstractCreateSessionOptions) {
+    super()
+
+    setMaxListeners(Infinity, this)
+    this.name = init.name
+    this.logger = components.logger
+    this.log = components.logger.forComponent(this.name)
+    this.requests = new Map()
+    this.minProviders = init.minProviders ?? DEFAULT_SESSION_MIN_PROVIDERS
+    this.maxProviders = init.maxProviders ?? DEFAULT_SESSION_MAX_PROVIDERS
+    this.providers = []
+    this.evictionFilter = BloomFilter.create(this.maxProviders)
+  }
+
+  async retrieve (cid: CID, options: BlockRetrievalOptions<RetrieveBlockProgressEvents> = {}): Promise<Uint8Array> {
+    // see if we are already requesting this CID in this session
+    const cidStr = base64.encode(cid.multihash.bytes)
+    const existingJob = this.requests.get(cidStr)
+
+    if (existingJob != null) {
+      this.log('join existing request for %c', cid)
+      return existingJob
+    }
+
+    const deferred: DeferredPromise<Uint8Array> = pDefer()
+    this.requests.set(cidStr, deferred.promise)
+
+    if (this.providers.length === 0) {
+      let first = false
+
+      if (this.intialPeerSearchComplete == null) {
+        first = true
+        this.log = this.logger.forComponent(`${this.name}:${cid}`)
+        this.intialPeerSearchComplete = this.findProviders(cid, this.minProviders, options)
+      }
+
+      await this.intialPeerSearchComplete
+
+      if (first) {
+        this.log('found initial session peers for %c', cid)
+      }
+    }
+
+    let foundBlock = false
+
+    // this queue manages outgoing requests - as new peers are added to the
+    // session they will be added to the queue so we can request the current
+    // block from multiple peers as they are discovered
+    const queue = new Queue<Uint8Array, { provider: Provider, priority?: number }>({
+      concurrency: this.maxProviders
+    })
+    queue.addEventListener('error', () => {})
+    queue.addEventListener('failure', (evt) => {
+      this.log.error('error querying provider %o, evicting from session', evt.detail.job.options.provider, evt.detail.error)
+      this.evict(evt.detail.job.options.provider)
+    })
+    queue.addEventListener('success', (evt) => {
+      // peer has sent block, return it to the caller
+      foundBlock = true
+      deferred.resolve(evt.detail.result)
+    })
+    queue.addEventListener('idle', () => {
+      if (foundBlock || options.signal?.aborted === true) {
+        // we either found the block or the user gave up
+        return
+      }
+
+      // find more session peers and retry
+      Promise.resolve()
+        .then(async () => {
+          this.log('no session peers had block for for %c, finding new providers', cid)
+
+          // evict this.minProviders random providers to make room for more
+          for (let i = 0; i < this.minProviders; i++) {
+            if (this.providers.length === 0) {
+              break
+            }
+
+            const provider = this.providers[Math.floor(Math.random() * this.providers.length)]
+            this.evict(provider)
+          }
+
+          // find new providers for the CID
+          await this.findProviders(cid, this.minProviders, options)
+
+          // keep trying until the abort signal fires
+          this.log('found new providers re-retrieving %c', cid)
+          this.requests.delete(cidStr)
+          deferred.resolve(await this.retrieve(cid, options))
+        })
+        .catch(err => {
+          this.log.error('could not find new providers for %c', cid, err)
+          deferred.reject(err)
+        })
+    })
+
+    const peerAddedToSessionListener = (event: CustomEvent<Provider>): void => {
+      queue.add(async () => {
+        return this.queryProvider(cid, event.detail, options)
+      }, {
+        provider: event.detail
+      })
+        .catch(err => {
+          if (options.signal?.aborted === true) {
+            // skip logging error if signal was aborted because abort can happen
+            // on success (e.g. another session found the block)
+            return
+          }
+
+          this.log.error('error retrieving session block for %c', cid, err)
+        })
+    }
+
+    // add new session peers to query as they are discovered
+    this.addEventListener('provider', peerAddedToSessionListener)
+
+    // query each session peer directly
+    Promise.all([...this.providers].map(async (provider) => {
+      return queue.add(async () => {
+        return this.queryProvider(cid, provider, options)
+      }, {
+        provider
+      })
+    }))
+      .catch(err => {
+        if (options.signal?.aborted === true) {
+          // skip logging error if signal was aborted because abort can happen
+          // on success (e.g. another session found the block)
+          return
+        }
+
+        this.log.error('error retrieving session block for %c', cid, err)
+      })
+
+    try {
+      return await deferred.promise
+    } finally {
+      this.removeEventListener('provider', peerAddedToSessionListener)
+      queue.clear()
+      this.requests.delete(cidStr)
+    }
+  }
+
+  evict (provider: Provider): void {
+    this.evictionFilter.add(this.toEvictionKey(provider))
+    const index = this.providers.findIndex(prov => this.equals(prov, provider))
+
+    if (index === -1) {
+      return
+    }
+
+    this.providers.splice(index, 1)
+  }
+
+  isEvicted (provider: Provider): boolean {
+    return this.providers.some(prov => this.equals(prov, provider))
+  }
+
+  hasProvider (provider: Provider): boolean {
+    // dedupe existing gateways
+    if (this.providers.find(prov => this.equals(prov, provider)) != null) {
+      return true
+    }
+
+    // dedupe failed session peers
+    if (this.isEvicted(provider)) {
+      return true
+    }
+
+    return false
+  }
+
+  private async findProviders (cid: CID, count: number, options: AbortOptions): Promise<void> {
+    const deferred: DeferredPromise<void> = pDefer()
+    let found = 0
+
+    // run async to resolve the deferred promise when `count` providers are
+    // found but continue util this.providers reaches this.maxProviders
+    void Promise.resolve()
+      .then(async () => {
+        this.log('finding %d-%d new provider(s) for %c', count, this.maxProviders, cid)
+
+        for await (const provider of this.findNewProviders(cid, options)) {
+          if (found === this.maxProviders || options.signal?.aborted === true) {
+            break
+          }
+
+          if (this.hasProvider(provider)) {
+            continue
+          }
+
+          this.log('found %d/%d new providers', found, this.maxProviders)
+          this.providers.push(provider)
+
+          // let the new peer join current queries
+          this.safeDispatchEvent('provider', {
+            detail: provider
+          })
+
+          found++
+
+          if (found === count) {
+            this.log('session is ready')
+            deferred.resolve()
+            // continue finding peers until we reach this.maxProviders
+          }
+
+          if (this.providers.length === this.maxProviders) {
+            this.log('found max session peers', found)
+            break
+          }
+        }
+
+        this.log('found %d/%d new session peers', found, this.maxProviders)
+
+        if (found < count) {
+          throw new CodeError(`Found ${found} of ${count} ${this.name} providers for ${cid}`, 'ERR_INSUFFICIENT_PROVIDERS_FOUND')
+        }
+      })
+      .catch(err => {
+        this.log.error('error searching routing for potential session peers for %c', cid, err.errors ?? err)
+        deferred.reject(err)
+      })
+
+    return deferred.promise
+  }
+
+  /**
+   * This method should search for new providers and yield them.
+   */
+  abstract findNewProviders (cid: CID, options: AbortOptions): AsyncGenerator<Provider>
+
+  /**
+   * The subclass should contact the provider and request the block from it.
+   *
+   * If the provider cannot provide the block an error should be thrown.
+   *
+   * The provider will then be excluded from ongoing queries.
+   */
+  abstract queryProvider (cid: CID, provider: Provider, options: AbortOptions): Promise<Uint8Array>
+
+  /**
+   * Turn a provider into a concise Uint8Array representation for use in a Bloom
+   * filter
+   */
+  abstract toEvictionKey (provider: Provider): Uint8Array | string
+
+  /**
+   * Return `true` if we consider one provider to be the same as another
+   */
+  abstract equals (providerA: Provider, providerB: Provider): boolean
+}

package/src/bloom-filter.ts

@@ -0,0 +1,141 @@
+// ported from xxbloom - https://github.com/ceejbot/xxbloom/blob/master/LICENSE
+import { randomBytes } from '@libp2p/crypto'
+import mur from 'murmurhash3js-revisited'
+import { Uint8ArrayList } from 'uint8arraylist'
+import { alloc } from 'uint8arrays/alloc'
+import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string'
+
+const LN2_SQUARED = Math.LN2 * Math.LN2
+
+export interface BloomFilterOptions {
+  seeds?: number[]
+  hashes?: number
+  bits?: number
+}
+
+export class BloomFilter {
+  /**
+   * Create a `BloomFilter` with the smallest `bits` and `hashes` value for the
+   * specified item count and error rate.
+   */
+  static create (itemcount: number, errorRate: number = 0.005): BloomFilter {
+    const opts = optimize(itemcount, errorRate)
+    return new BloomFilter(opts)
+  }
+
+  public readonly seeds: number[]
+  public readonly bits: number
+  public buffer: Uint8Array
+
+  constructor (options: BloomFilterOptions = {}) {
+    if (options.seeds != null) {
+      this.seeds = options.seeds
+    } else {
+      this.seeds = generateSeeds(options.hashes ?? 8)
+    }
+
+    this.bits = options.bits ?? 1024
+    this.buffer = alloc(Math.ceil(this.bits / 8))
+  }
+
+  /**
+   * Add an item to the filter
+   */
+  add (item: Uint8Array | string): void {
+    if (typeof item === 'string') {
+      item = uint8ArrayFromString(item)
+    }
+
+    for (let i = 0; i < this.seeds.length; i++) {
+      const hash = mur.x86.hash32(item, this.seeds[i])
+      const bit = hash % this.bits
+
+      this.setbit(bit)
+    }
+  }
+
+  /**
+   * Test if the filter has an item. If it returns false it definitely does not
+   * have the item. If it returns true, it probably has the item but there's
+   * an `errorRate` chance it doesn't.
+   */
+  has (item: Uint8Array | string): boolean {
+    if (typeof item === 'string') {
+      item = uint8ArrayFromString(item)
+    }
+
+    for (let i = 0; i < this.seeds.length; i++) {
+      const hash = mur.x86.hash32(item, this.seeds[i])
+      const bit = hash % this.bits
+
+      const isSet = this.getbit(bit)
+
+      if (!isSet) {
+        return false
+      }
+    }
+
+    return true
+  }
+
+  /**
+   * Reset the filter
+   */
+  clear (): void {
+    this.buffer.fill(0)
+  }
+
+  setbit (bit: number): void {
+    let pos = 0
+    let shift = bit
+    while (shift > 7) {
+      pos++
+      shift -= 8
+    }
+
+    let bitfield = this.buffer[pos]
+    bitfield |= (0x1 << shift)
+    this.buffer[pos] = bitfield
+  }
+
+  getbit (bit: number): boolean {
+    let pos = 0
+    let shift = bit
+    while (shift > 7) {
+      pos++
+      shift -= 8
+    }
+
+    const bitfield = this.buffer[pos]
+    return (bitfield & (0x1 << shift)) !== 0
+  }
+}
+
+function optimize (itemcount: number, errorRate: number = 0.005): { bits: number, hashes: number } {
+  const bits = Math.round(-1 * itemcount * Math.log(errorRate) / LN2_SQUARED)
+  const hashes = Math.round((bits / itemcount) * Math.LN2)
+
+  return { bits, hashes }
+}
+
+function generateSeeds (count: number): number[] {
+  let buf: Uint8ArrayList
+  let j: number
+  const seeds = []
+
+  for (let i = 0; i < count; i++) {
+    buf = new Uint8ArrayList(randomBytes(4))
+    seeds[i] = buf.getUint32(0, true)
+
+    // Make sure we don't end up with two identical seeds,
+    // which is unlikely but possible.
+    for (j = 0; j < i; j++) {
+      if (seeds[i] === seeds[j]) {
+        i--
+        break
+      }
+    }
+  }
+
+  return seeds
+}

package/src/index.ts

@@ -39,6 +39,9 @@ import type { Datastore } from 'interface-datastore'
 import type { CID } from 'multiformats/cid'
 import type { MultihashHasher } from 'multiformats/hashes/interface'
 
+export { AbstractSession, type AbstractCreateSessionOptions } from './abstract-session.js'
+export { BloomFilter } from './bloom-filter.js'
+
 /**
  * Options used to create a Helia node.
  */
@@ -101,6 +104,24 @@ export interface HeliaInit {
    */
   routers?: Array<Partial<Routing>>
 
+  /**
+   * During provider lookups, peers can be returned from routing implementations
+   * with no multiaddrs.
+   *
+   * This can happen when they've been retrieved from network peers that only
+   * store multiaddrs for a limited amount of time.
+   *
+   * When this happens the peer's info has to be looked up with a further query.
+   *
+   * To not have this query block the yielding of other providers returned with
+   * multiaddrs, a separate queue is used to perform this lookup.
+   *
+   * This config value controls the concurrency of that queue.
+   *
+   * @default 5
+   */
+  providerLookupConcurrency?: number
+
   /**
    * Components used by subclasses
    */
@@ -119,6 +140,7 @@ interface Components {
   dagWalkers: Record<number, DAGWalker>
   logger: ComponentLogger
   blockBrokers: BlockBroker[]
+  routing: Routing
   dns: DNS
 }
 
@@ -140,6 +162,7 @@ export class Helia implements HeliaInterface {
     this.dagWalkers = defaultDagWalkers(init.dagWalkers)
     this.dns = init.dns ?? dns()
 
+    // @ts-expect-error routing is not set
     const components: Components = {
       blockstore: init.blockstore,
       datastore: init.datastore,
@@ -151,19 +174,7 @@ export class Helia implements HeliaInterface {
       ...(init.components ?? {})
     }
 
-    components.blockBrokers = init.blockBrokers.map((fn) => {
-      return fn(components)
-    })
-
-    const networkedStorage = new NetworkedStorage(components)
-
-    this.pins = new PinsImpl(init.datastore, networkedStorage, this.dagWalkers)
-
-    this.blockstore = new BlockStorage(networkedStorage, this.pins, {
-      holdGcLock: init.holdGcLock ?? true
-    })
-    this.datastore = init.datastore
-    this.routing = new RoutingClass(components, {
+    this.routing = components.routing = new RoutingClass(components, {
       routers: (init.routers ?? []).flatMap((router: any) => {
         // if the router itself is a router
         const routers = [
@@ -181,7 +192,19 @@ export class Helia implements HeliaInterface {
         }
 
         return routers
-      })
+      }),
+      providerLookupConcurrency: init.providerLookupConcurrency
+    })
+
+    const networkedStorage = new NetworkedStorage(components)
+    this.pins = new PinsImpl(init.datastore, networkedStorage, this.dagWalkers)
+    this.blockstore = new BlockStorage(networkedStorage, this.pins, {
+      holdGcLock: init.holdGcLock ?? true
+    })
+    this.datastore = init.datastore
+
+    components.blockBrokers = init.blockBrokers.map((fn) => {
+      return fn(components)
     })
   }
 

package/src/routing.ts

@@ -1,12 +1,15 @@
 import { CodeError, start, stop } from '@libp2p/interface'
-import { PeerSet } from '@libp2p/peer-collections'
+import { PeerQueue } from '@libp2p/utils/peer-queue'
 import merge from 'it-merge'
 import type { Routing as RoutingInterface, Provider, RoutingOptions } from '@helia/interface'
 import type { AbortOptions, ComponentLogger, Logger, PeerId, PeerInfo, Startable } from '@libp2p/interface'
 import type { CID } from 'multiformats/cid'
 
+const DEFAULT_PROVIDER_LOOKUP_CONCURRENCY = 5
+
 export interface RoutingInit {
   routers: Array<Partial<RoutingInterface>>
+  providerLookupConcurrency?: number
 }
 
 export interface RoutingComponents {
@@ -16,10 +19,12 @@ export interface RoutingComponents {
 export class Routing implements RoutingInterface, Startable {
   private readonly log: Logger
   private readonly routers: Array<Partial<RoutingInterface>>
+  private readonly providerLookupConcurrency: number
 
   constructor (components: RoutingComponents, init: RoutingInit) {
     this.log = components.logger.forComponent('helia:routing')
     this.routers = init.routers ?? []
+    this.providerLookupConcurrency = init.providerLookupConcurrency ?? DEFAULT_PROVIDER_LOOKUP_CONCURRENCY
   }
 
   async start (): Promise<void> {
@@ -31,16 +36,25 @@ export class Routing implements RoutingInterface, Startable {
   }
 
   /**
-   * Iterates over all content routers in parallel to find providers of the given key
+   * Iterates over all content routers in parallel to find providers of the
+   * given key
    */
   async * findProviders (key: CID, options: RoutingOptions = {}): AsyncIterable<Provider> {
     if (this.routers.length === 0) {
       throw new CodeError('No content routers available', 'ERR_NO_ROUTERS_AVAILABLE')
     }
 
-    const seen = new PeerSet()
+    // provider multiaddrs are only cached for a limited time, so they can come
+    // back as an empty array - when this happens we have to do a FIND_PEER
+    // query to get updated addresses, but we shouldn't block on this so use a
+    // separate bounded queue to perform this lookup
+    const queue = new PeerQueue<Provider | null>({
+      concurrency: this.providerLookupConcurrency
+    })
+    queue.addEventListener('error', () => {})
 
     for await (const peer of merge(
+      queue.toGenerator(),
       ...supports(this.routers, 'findProviders')
         .map(router => router.findProviders(key, options))
     )) {
@@ -50,12 +64,42 @@ export class Routing implements RoutingInterface, Startable {
         continue
       }
 
-      // deduplicate peers
-      if (seen.has(peer.id)) {
-        continue
-      }
+      peer.multiaddrs = peer.multiaddrs.map(ma => {
+        if (ma.getPeerId() != null) {
+          return ma
+        }
 
-      seen.add(peer.id)
+        return ma.encapsulate(`/p2p/${peer.id}`)
+      })
+
+      // have to refresh peer info for this peer to get updated multiaddrs
+      if (peer.multiaddrs.length === 0) {
+        // already looking this peer up
+        if (queue.find(peer.id) != null) {
+          continue
+        }
+
+        queue.add(async () => {
+          try {
+            const provider = await this.findPeer(peer.id, options)
+
+            if (provider.multiaddrs.length === 0) {
+              return null
+            }
+
+            return provider
+          } catch (err) {
+            this.log.error('could not load multiaddrs for peer', peer.id, err)
+            return null
+          }
+        }, {
+          peerId: peer.id,
+          signal: options.signal
+        })
+          .catch(err => {
+            this.log.error('could not load multiaddrs for peer', peer.id, err)
+          })
+      }
 
       yield peer
     }
@@ -142,8 +186,6 @@ export class Routing implements RoutingInterface, Startable {
       throw new CodeError('No peer routers available', 'ERR_NO_ROUTERS_AVAILABLE')
     }
 
-    const seen = new PeerSet()
-
     for await (const peer of merge(
       ...supports(this.routers, 'getClosestPeers')
         .map(router => router.getClosestPeers(key, options))
@@ -152,13 +194,6 @@ export class Routing implements RoutingInterface, Startable {
         continue
       }
 
-      // deduplicate peers
-      if (seen.has(peer.id)) {
-        continue
-      }
-
-      seen.add(peer.id)
-
       yield peer
     }
   }