@helia/utils 1.4.0-7d471a02 → 1.4.0-93aa4645

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@helia/utils",
-  "version": "1.4.0-7d471a02",
+  "version": "1.4.0-93aa4645",
   "description": "Shared code that implements the Helia API",
   "license": "Apache-2.0 OR MIT",
   "homepage": "https://github.com/ipfs/helia/tree/main/packages/utils#readme",
@@ -32,13 +32,6 @@
       "import": "./dist/src/index.js"
     }
   },
-  "eslintConfig": {
-    "extends": "ipfs",
-    "parserOptions": {
-      "project": true,
-      "sourceType": "module"
-    }
-  },
   "scripts": {
     "clean": "aegir clean",
     "lint": "aegir lint",
@@ -54,43 +47,45 @@
     "test:electron-main": "aegir test -t electron-main"
   },
   "dependencies": {
-    "@helia/interface": "5.4.0-7d471a02",
-    "@ipld/dag-cbor": "^9.2.2",
-    "@ipld/dag-json": "^10.2.3",
-    "@ipld/dag-pb": "^4.1.3",
-    "@libp2p/interface": "^2.5.0",
-    "@libp2p/keychain": "^5.2.8",
-    "@libp2p/logger": "^5.1.8",
-    "@libp2p/utils": "^6.5.1",
-    "@multiformats/dns": "^1.0.6",
-    "@multiformats/multiaddr": "^12.4.0",
+    "@helia/interface": "5.4.0-93aa4645",
+    "@ipld/dag-cbor": "^9.2.5",
+    "@ipld/dag-json": "^10.2.5",
+    "@ipld/dag-pb": "^4.1.5",
+    "@libp2p/interface": "^3.0.2",
+    "@libp2p/keychain": "^6.0.5",
+    "@libp2p/logger": "^6.0.5",
+    "@libp2p/utils": "^7.0.5",
+    "@multiformats/dns": "^1.0.9",
+    "@multiformats/multiaddr": "^13.0.1",
     "any-signal": "^4.1.1",
-    "blockstore-core": "^5.0.2",
-    "cborg": "^4.2.6",
-    "interface-blockstore": "^5.3.1",
-    "interface-datastore": "^8.3.1",
-    "interface-store": "^6.0.2",
-    "it-drain": "^3.0.7",
-    "it-filter": "^3.1.1",
-    "it-foreach": "^2.1.1",
-    "it-merge": "^3.0.5",
-    "libp2p": "^2.9.0",
-    "mortice": "^3.0.6",
-    "multiformats": "^13.3.1",
+    "blockstore-core": "^6.0.2",
+    "cborg": "^4.2.15",
+    "interface-blockstore": "^6.0.1",
+    "interface-datastore": "^9.0.2",
+    "interface-store": "^7.0.0",
+    "it-drain": "^3.0.10",
+    "it-filter": "^3.1.4",
+    "it-foreach": "^2.1.4",
+    "it-merge": "^3.0.12",
+    "it-to-buffer": "^4.0.10",
+    "libp2p": "^3.0.6",
+    "mortice": "^3.3.1",
+    "multiformats": "^13.4.1",
     "p-defer": "^4.0.1",
     "progress-events": "^1.0.1",
     "uint8arrays": "^5.1.0"
   },
   "devDependencies": {
-    "@libp2p/crypto": "^5.0.7",
-    "@libp2p/peer-id": "^5.0.8",
-    "@types/sinon": "^17.0.3",
-    "aegir": "^47.0.7",
-    "datastore-core": "^10.0.2",
+    "@libp2p/crypto": "^5.1.12",
+    "@libp2p/peer-id": "^6.0.3",
+    "@types/sinon": "^17.0.4",
+    "aegir": "^47.0.22",
+    "datastore-core": "^11.0.2",
     "delay": "^6.0.0",
-    "it-all": "^3.0.6",
-    "race-signal": "^1.1.0",
-    "sinon": "^20.0.0",
+    "it-all": "^3.0.9",
+    "it-map": "^3.1.4",
+    "race-signal": "^2.0.0",
+    "sinon": "^21.0.0",
     "sinon-ts": "^2.0.0"
   },
   "browser": {

package/src/abstract-session.ts

@@ -1,12 +1,11 @@
 import { DEFAULT_SESSION_MIN_PROVIDERS, DEFAULT_SESSION_MAX_PROVIDERS, InsufficientProvidersError } from '@helia/interface'
 import { AbortError, TypedEventEmitter, setMaxListeners } from '@libp2p/interface'
-import { createScalableCuckooFilter } from '@libp2p/utils/filters'
-import { Queue } from '@libp2p/utils/queue'
+import { createScalableCuckooFilter, Queue } from '@libp2p/utils'
 import { base64 } from 'multiformats/bases/base64'
 import pDefer from 'p-defer'
 import type { BlockBroker, BlockRetrievalOptions, CreateSessionOptions } from '@helia/interface'
 import type { AbortOptions, ComponentLogger, Logger, PeerId } from '@libp2p/interface'
-import type { Filter } from '@libp2p/utils/filters'
+import type { Filter } from '@libp2p/utils'
 import type { Multiaddr } from '@multiformats/multiaddr'
 import type { CID } from 'multiformats/cid'
 import type { DeferredPromise } from 'p-defer'
@@ -88,7 +87,6 @@ export abstract class AbstractSession<Provider, RetrieveBlockProgressEvents exte
     const queue = new Queue<Uint8Array, { provider: Provider, priority?: number } & AbortOptions>({
       concurrency: this.maxProviders
     })
-    queue.addEventListener('error', () => {})
     queue.addEventListener('failure', (evt) => {
       this.log.error('error querying provider %o, evicting from session', evt.detail.job.options.provider, evt.detail.error)
       this.evict(evt.detail.job.options.provider)

package/src/index.ts

@@ -5,7 +5,7 @@
  * modules such as `helia`, `@helia/http`, etc.
  */
 
-import { contentRoutingSymbol, peerRoutingSymbol, start, stop } from '@libp2p/interface'
+import { contentRoutingSymbol, peerRoutingSymbol, start, stop, TypedEventEmitter } from '@libp2p/interface'
 import { defaultLogger } from '@libp2p/logger'
 import { dns } from '@multiformats/dns'
 import drain from 'it-drain'
@@ -18,7 +18,7 @@ import { getCodec } from './utils/get-codec.js'
 import { getHasher } from './utils/get-hasher.js'
 import { NetworkedStorage } from './utils/networked-storage.js'
 import type { BlockStorageInit } from './storage.js'
-import type { Await, CodecLoader, GCOptions, HasherLoader, Helia as HeliaInterface, Routing } from '@helia/interface'
+import type { Await, CodecLoader, GCOptions, HasherLoader, Helia as HeliaInterface, HeliaEvents, Routing } from '@helia/interface'
 import type { BlockBroker } from '@helia/interface/blocks'
 import type { Pins } from '@helia/interface/pins'
 import type { ComponentLogger, Libp2p, Logger, Metrics } from '@libp2p/interface'
@@ -187,6 +187,7 @@ export class Helia<T extends Libp2p> implements HeliaInterface<T> {
   public libp2p: T
   public blockstore: BlockStorage
   public datastore: Datastore
+  public events: TypedEventEmitter<HeliaEvents<T>>
   public pins: Pins
   public logger: ComponentLogger
   public routing: Routing
@@ -204,6 +205,7 @@ export class Helia<T extends Libp2p> implements HeliaInterface<T> {
     this.dns = init.dns ?? dns()
     this.metrics = init.metrics
     this.libp2p = init.libp2p
+    this.events = new TypedEventEmitter<HeliaEvents<T>>()
 
     // @ts-expect-error routing is not set
     const components: Components = {
@@ -261,6 +263,7 @@ export class Helia<T extends Libp2p> implements HeliaInterface<T> {
       this.routing,
       this.libp2p
     )
+    this.events.dispatchEvent(new CustomEvent('start', { detail: this }))
   }
 
   async stop (): Promise<void> {
@@ -270,6 +273,7 @@ export class Helia<T extends Libp2p> implements HeliaInterface<T> {
       this.routing,
       this.libp2p
     )
+    this.events.dispatchEvent(new CustomEvent('stop', { detail: this }))
   }
 
   async gc (options: GCOptions = {}): Promise<void> {

package/src/pins.ts

@@ -1,6 +1,7 @@
-import { Queue } from '@libp2p/utils/queue'
+import { Queue } from '@libp2p/utils'
 import * as cborg from 'cborg'
 import { Key } from 'interface-datastore'
+import toBuffer from 'it-to-buffer'
 import { base36 } from 'multiformats/bases/base36'
 import { createUnsafe } from 'multiformats/block'
 import { CID } from 'multiformats/cid'
@@ -124,7 +125,7 @@ export class PinsImpl implements Pins {
     }
 
     const codec = await this.getCodec(cid.code)
-    const bytes = await this.blockstore.get(cid, options)
+    const bytes = await toBuffer(this.blockstore.get(cid, options))
     const block = createUnsafe({ bytes, cid, codec })
 
     yield cid

package/src/routing.ts

@@ -1,6 +1,6 @@
 import { NoRoutersAvailableError } from '@helia/interface'
 import { NotFoundError, start, stop } from '@libp2p/interface'
-import { PeerQueue } from '@libp2p/utils/peer-queue'
+import { PeerQueue } from '@libp2p/utils'
 import merge from 'it-merge'
 import type { Routing as RoutingInterface, Provider, RoutingOptions } from '@helia/interface'
 import type { AbortOptions, ComponentLogger, Logger, Metrics, PeerId, PeerInfo, Startable } from '@libp2p/interface'
@@ -75,7 +75,6 @@ export class Routing implements RoutingInterface, Startable {
     const queue = new PeerQueue<Provider | null>({
       concurrency: this.providerLookupConcurrency
     })
-    queue.addEventListener('error', () => {})
 
     for await (const peer of merge(
       queue.toGenerator(),

package/src/storage.ts

@@ -3,7 +3,7 @@ import createMortice from 'mortice'
 import type { Blocks, Pair, DeleteManyBlocksProgressEvents, DeleteBlockProgressEvents, GetBlockProgressEvents, GetManyBlocksProgressEvents, PutManyBlocksProgressEvents, PutBlockProgressEvents, GetAllBlocksProgressEvents, GetOfflineOptions, SessionBlockstore } from '@helia/interface/blocks'
 import type { Pins } from '@helia/interface/pins'
 import type { AbortOptions, Startable } from '@libp2p/interface'
-import type { Blockstore } from 'interface-blockstore'
+import type { Blockstore, InputPair } from 'interface-blockstore'
 import type { AwaitIterable } from 'interface-store'
 import type { Mortice } from 'mortice'
 import type { CID } from 'multiformats/cid'
@@ -75,7 +75,7 @@ export class BlockStorage implements Blocks, Startable {
   /**
    * Put a multiple blocks to the underlying datastore
    */
-  async * putMany (blocks: AwaitIterable<{ cid: CID, block: Uint8Array }>, options: AbortOptions & ProgressOptions<PutManyBlocksProgressEvents> = {}): AsyncIterable<CID> {
+  async * putMany (blocks: AwaitIterable<InputPair>, options: AbortOptions & ProgressOptions<PutManyBlocksProgressEvents> = {}): AsyncGenerator<CID> {
     options?.signal?.throwIfAborted()
     const releaseLock = await this.lock.readLock()
 
@@ -89,12 +89,12 @@ export class BlockStorage implements Blocks, Startable {
   /**
    * Get a block by cid
    */
-  async get (cid: CID, options: GetOfflineOptions & AbortOptions & ProgressOptions<GetBlockProgressEvents> = {}): Promise<Uint8Array> {
+  async * get (cid: CID, options: GetOfflineOptions & AbortOptions & ProgressOptions<GetBlockProgressEvents> = {}): AsyncGenerator<Uint8Array> {
     options?.signal?.throwIfAborted()
     const releaseLock = await this.lock.readLock()
 
     try {
-      return await this.child.get(cid, options)
+      yield * this.child.get(cid, options)
     } finally {
       releaseLock()
     }
@@ -103,7 +103,7 @@ export class BlockStorage implements Blocks, Startable {
   /**
    * Get multiple blocks back from an (async) iterable of cids
    */
-  async * getMany (cids: AwaitIterable<CID>, options: GetOfflineOptions & AbortOptions & ProgressOptions<GetManyBlocksProgressEvents> = {}): AsyncIterable<Pair> {
+  async * getMany (cids: AwaitIterable<CID>, options: GetOfflineOptions & AbortOptions & ProgressOptions<GetManyBlocksProgressEvents> = {}): AsyncGenerator<Pair> {
     options?.signal?.throwIfAborted()
     const releaseLock = await this.lock.readLock()
 
@@ -135,7 +135,7 @@ export class BlockStorage implements Blocks, Startable {
   /**
    * Delete multiple blocks from the blockstore
    */
-  async * deleteMany (cids: AwaitIterable<CID>, options: AbortOptions & ProgressOptions<DeleteManyBlocksProgressEvents> = {}): AsyncIterable<CID> {
+  async * deleteMany (cids: AwaitIterable<CID>, options: AbortOptions & ProgressOptions<DeleteManyBlocksProgressEvents> = {}): AsyncGenerator<CID> {
     options?.signal?.throwIfAborted()
     const releaseLock = await this.lock.writeLock()
 
@@ -167,7 +167,7 @@ export class BlockStorage implements Blocks, Startable {
     }
   }
 
-  async * getAll (options: AbortOptions & ProgressOptions<GetAllBlocksProgressEvents> = {}): AsyncIterable<Pair> {
+  async * getAll (options: AbortOptions & ProgressOptions<GetAllBlocksProgressEvents> = {}): AsyncGenerator<Pair> {
     options?.signal?.throwIfAborted()
     const releaseLock = await this.lock.readLock()
 

package/src/utils/networked-storage.ts

@@ -9,7 +9,7 @@ import { isPromise } from './is-promise.js'
 import type { HasherLoader } from '@helia/interface'
 import type { BlockBroker, Blocks, Pair, DeleteManyBlocksProgressEvents, DeleteBlockProgressEvents, GetBlockProgressEvents, GetManyBlocksProgressEvents, PutManyBlocksProgressEvents, PutBlockProgressEvents, GetAllBlocksProgressEvents, GetOfflineOptions, BlockRetrievalOptions, CreateSessionOptions, SessionBlockstore } from '@helia/interface/blocks'
 import type { AbortOptions, ComponentLogger, Logger, LoggerOptions, Startable } from '@libp2p/interface'
-import type { Blockstore } from 'interface-blockstore'
+import type { Blockstore, InputPair } from 'interface-blockstore'
 import type { AwaitIterable } from 'interface-store'
 import type { CID } from 'multiformats/cid'
 import type { MultihashDigest, MultihashHasher } from 'multiformats/hashes/interface'
@@ -56,7 +56,7 @@ class Storage implements Blockstore {
     options.onProgress?.(new CustomProgressEvent<CID>('blocks:put:providers:notify', cid))
 
     await Promise.all(
-      this.components.blockBrokers.map(async broker => broker.announce?.(cid, block, options))
+      this.components.blockBrokers.map(async broker => broker.announce?.(cid, options))
     )
 
     options.onProgress?.(new CustomProgressEvent<CID>('blocks:put:blockstore:put', cid))
@@ -67,7 +67,7 @@ class Storage implements Blockstore {
   /**
    * Put a multiple blocks to the underlying datastore
    */
-  async * putMany (blocks: AwaitIterable<{ cid: CID, block: Uint8Array }>, options: AbortOptions & ProgressOptions<PutManyBlocksProgressEvents> = {}): AsyncIterable<CID> {
+  async * putMany (blocks: AwaitIterable<InputPair>, options: AbortOptions & ProgressOptions<PutManyBlocksProgressEvents> = {}): AsyncGenerator<CID> {
     const missingBlocks = filter(blocks, async ({ cid }): Promise<boolean> => {
       const has = await this.child.has(cid, options)
 
@@ -78,10 +78,10 @@ class Storage implements Blockstore {
       return !has
     })
 
-    const notifyEach = forEach(missingBlocks, async ({ cid, block }): Promise<void> => {
+    const notifyEach = forEach(missingBlocks, async ({ cid }): Promise<void> => {
       options.onProgress?.(new CustomProgressEvent<CID>('blocks:put-many:providers:notify', cid))
       await Promise.all(
-        this.components.blockBrokers.map(async broker => broker.announce?.(cid, block, options))
+        this.components.blockBrokers.map(async broker => broker.announce?.(cid, options))
       )
     })
 
@@ -92,7 +92,7 @@ class Storage implements Blockstore {
   /**
    * Get a block by cid
    */
-  async get (cid: CID, options: GetOfflineOptions & AbortOptions & ProgressOptions<GetBlockProgressEvents> = {}): Promise<Uint8Array> {
+  async * get (cid: CID, options: GetOfflineOptions & AbortOptions & ProgressOptions<GetBlockProgressEvents> = {}): AsyncGenerator<Uint8Array> {
     if (options.offline !== true && !(await this.child.has(cid, options))) {
       const hasher = await this.getHasher(cid.multihash.code)
 
@@ -108,21 +108,22 @@ class Storage implements Blockstore {
       // notify other block providers of the new block
       options.onProgress?.(new CustomProgressEvent<CID>('blocks:get:providers:notify', cid))
       await Promise.all(
-        this.components.blockBrokers.map(async broker => broker.announce?.(cid, block, options))
+        this.components.blockBrokers.map(async broker => broker.announce?.(cid, options))
       )
 
-      return block
+      yield block
+      return
     }
 
     options.onProgress?.(new CustomProgressEvent<CID>('blocks:get:blockstore:get', cid))
 
-    return this.child.get(cid, options)
+    yield * this.child.get(cid, options)
   }
 
   /**
    * Get multiple blocks back from an (async) iterable of cids
    */
-  async * getMany (cids: AwaitIterable<CID>, options: GetOfflineOptions & AbortOptions & ProgressOptions<GetManyBlocksProgressEvents> = {}): AsyncIterable<Pair> {
+  async * getMany (cids: AwaitIterable<CID>, options: GetOfflineOptions & AbortOptions & ProgressOptions<GetManyBlocksProgressEvents> = {}): AsyncGenerator<Pair> {
     options.onProgress?.(new CustomProgressEvent('blocks:get-many:blockstore:get-many'))
 
     yield * this.child.getMany(forEach(cids, async (cid): Promise<void> => {
@@ -141,7 +142,7 @@ class Storage implements Blockstore {
         // notify other block providers of the new block
         options.onProgress?.(new CustomProgressEvent<CID>('blocks:get-many:providers:notify', cid))
         await Promise.all(
-          this.components.blockBrokers.map(async broker => broker.announce?.(cid, block, options))
+          this.components.blockBrokers.map(async broker => broker.announce?.(cid, options))
         )
       }
     }))
@@ -159,7 +160,7 @@ class Storage implements Blockstore {
   /**
    * Delete multiple blocks from the blockstore
    */
-  async * deleteMany (cids: AwaitIterable<CID>, options: AbortOptions & ProgressOptions<DeleteManyBlocksProgressEvents> = {}): AsyncIterable<CID> {
+  async * deleteMany (cids: AwaitIterable<CID>, options: AbortOptions & ProgressOptions<DeleteManyBlocksProgressEvents> = {}): AsyncGenerator<CID> {
     options.onProgress?.(new CustomProgressEvent('blocks:delete-many:blockstore:delete-many'))
     yield * this.child.deleteMany((async function * (): AsyncGenerator<CID> {
       for await (const cid of cids) {
@@ -172,7 +173,7 @@ class Storage implements Blockstore {
     return this.child.has(cid, options)
   }
 
-  async * getAll (options: AbortOptions & ProgressOptions<GetAllBlocksProgressEvents> = {}): AwaitIterable<Pair> {
+  async * getAll (options: AbortOptions & ProgressOptions<GetAllBlocksProgressEvents> = {}): AsyncGenerator<Pair> {
     options.onProgress?.(new CustomProgressEvent('blocks:get-all:blockstore:get-many'))
     yield * this.child.getAll(options)
   }
@@ -280,7 +281,7 @@ class SessionStorage extends Storage implements SessionBlockstore {
   /**
    * Put a multiple blocks to the underlying datastore
    */
-  async * putMany (blocks: AwaitIterable<{ cid: CID, block: Uint8Array }>, options: AbortOptions & ProgressOptions<PutManyBlocksProgressEvents> = {}): AsyncIterable<CID> {
+  async * putMany (blocks: AwaitIterable<InputPair>, options: AbortOptions & ProgressOptions<PutManyBlocksProgressEvents> = {}): AsyncGenerator<CID> {
     const signal = anySignal([this.closeController.signal, options.signal])
     setMaxListeners(Infinity, signal)
 
@@ -297,12 +298,12 @@ class SessionStorage extends Storage implements SessionBlockstore {
   /**
    * Get a block by cid
    */
-  async get (cid: CID, options: GetOfflineOptions & AbortOptions & ProgressOptions<GetBlockProgressEvents> = {}): Promise<Uint8Array> {
+  async * get (cid: CID, options: GetOfflineOptions & AbortOptions & ProgressOptions<GetBlockProgressEvents> = {}): AsyncGenerator<Uint8Array> {
     const signal = anySignal([this.closeController.signal, options.signal])
     setMaxListeners(Infinity, signal)
 
     try {
-      return await super.get(cid, {
+      yield * super.get(cid, {
         ...options,
         signal
       })
@@ -314,7 +315,7 @@ class SessionStorage extends Storage implements SessionBlockstore {
   /**
    * Get multiple blocks back from an (async) iterable of cids
    */
-  async * getMany (cids: AwaitIterable<CID>, options: GetOfflineOptions & AbortOptions & ProgressOptions<GetManyBlocksProgressEvents> = {}): AsyncIterable<Pair> {
+  async * getMany (cids: AwaitIterable<CID>, options: GetOfflineOptions & AbortOptions & ProgressOptions<GetManyBlocksProgressEvents> = {}): AsyncGenerator<Pair> {
     const signal = anySignal([this.closeController.signal, options.signal])
     setMaxListeners(Infinity, signal)
 
@@ -348,7 +349,7 @@ class SessionStorage extends Storage implements SessionBlockstore {
   /**
    * Delete multiple blocks from the blockstore
    */
-  async * deleteMany (cids: AwaitIterable<CID>, options: AbortOptions & ProgressOptions<DeleteManyBlocksProgressEvents> = {}): AsyncIterable<CID> {
+  async * deleteMany (cids: AwaitIterable<CID>, options: AbortOptions & ProgressOptions<DeleteManyBlocksProgressEvents> = {}): AsyncGenerator<CID> {
     const signal = anySignal([this.closeController.signal, options.signal])
     setMaxListeners(Infinity, signal)
 
@@ -376,7 +377,7 @@ class SessionStorage extends Storage implements SessionBlockstore {
     }
   }
 
-  async * getAll (options: AbortOptions & ProgressOptions<GetAllBlocksProgressEvents> = {}): AwaitIterable<Pair> {
+  async * getAll (options: AbortOptions & ProgressOptions<GetAllBlocksProgressEvents> = {}): AsyncGenerator<Pair> {
     const signal = anySignal([this.closeController.signal, options.signal])
     setMaxListeners(Infinity, signal)