@helia/utils 1.3.2-a0266903 → 1.4.0-34d3ecd7

package/src/pins.ts

@@ -1,6 +1,7 @@
-import { Queue } from '@libp2p/utils/queue'
+import { Queue } from '@libp2p/utils'
 import * as cborg from 'cborg'
 import { Key } from 'interface-datastore'
+import toBuffer from 'it-to-buffer'
 import { base36 } from 'multiformats/bases/base36'
 import { createUnsafe } from 'multiformats/block'
 import { CID } from 'multiformats/cid'
@@ -124,7 +125,7 @@ export class PinsImpl implements Pins {
     }
 
     const codec = await this.getCodec(cid.code)
-    const bytes = await this.blockstore.get(cid, options)
+    const bytes = await toBuffer(this.blockstore.get(cid, options))
     const block = createUnsafe({ bytes, cid, codec })
 
     yield cid

package/src/routing.ts

@@ -1,6 +1,6 @@
 import { NoRoutersAvailableError } from '@helia/interface'
 import { NotFoundError, start, stop } from '@libp2p/interface'
-import { PeerQueue } from '@libp2p/utils/peer-queue'
+import { PeerQueue } from '@libp2p/utils'
 import merge from 'it-merge'
 import type { Routing as RoutingInterface, Provider, RoutingOptions } from '@helia/interface'
 import type { AbortOptions, ComponentLogger, Logger, Metrics, PeerId, PeerInfo, Startable } from '@libp2p/interface'
@@ -75,7 +75,6 @@ export class Routing implements RoutingInterface, Startable {
     const queue = new PeerQueue<Provider | null>({
       concurrency: this.providerLookupConcurrency
     })
-    queue.addEventListener('error', () => {})
 
     for await (const peer of merge(
       queue.toGenerator(),

package/src/storage.ts

@@ -3,7 +3,7 @@ import createMortice from 'mortice'
 import type { Blocks, Pair, DeleteManyBlocksProgressEvents, DeleteBlockProgressEvents, GetBlockProgressEvents, GetManyBlocksProgressEvents, PutManyBlocksProgressEvents, PutBlockProgressEvents, GetAllBlocksProgressEvents, GetOfflineOptions, SessionBlockstore } from '@helia/interface/blocks'
 import type { Pins } from '@helia/interface/pins'
 import type { AbortOptions, Startable } from '@libp2p/interface'
-import type { Blockstore } from 'interface-blockstore'
+import type { Blockstore, InputPair } from 'interface-blockstore'
 import type { AwaitIterable } from 'interface-store'
 import type { Mortice } from 'mortice'
 import type { CID } from 'multiformats/cid'
@@ -75,7 +75,7 @@ export class BlockStorage implements Blocks, Startable {
   /**
    * Put a multiple blocks to the underlying datastore
    */
-  async * putMany (blocks: AwaitIterable<{ cid: CID, block: Uint8Array }>, options: AbortOptions & ProgressOptions<PutManyBlocksProgressEvents> = {}): AsyncIterable<CID> {
+  async * putMany (blocks: AwaitIterable<InputPair>, options: AbortOptions & ProgressOptions<PutManyBlocksProgressEvents> = {}): AsyncGenerator<CID> {
     options?.signal?.throwIfAborted()
     const releaseLock = await this.lock.readLock()
 
@@ -89,12 +89,12 @@ export class BlockStorage implements Blocks, Startable {
   /**
    * Get a block by cid
    */
-  async get (cid: CID, options: GetOfflineOptions & AbortOptions & ProgressOptions<GetBlockProgressEvents> = {}): Promise<Uint8Array> {
+  async * get (cid: CID, options: GetOfflineOptions & AbortOptions & ProgressOptions<GetBlockProgressEvents> = {}): AsyncGenerator<Uint8Array> {
     options?.signal?.throwIfAborted()
     const releaseLock = await this.lock.readLock()
 
     try {
-      return await this.child.get(cid, options)
+      yield * this.child.get(cid, options)
     } finally {
       releaseLock()
     }
@@ -103,7 +103,7 @@ export class BlockStorage implements Blocks, Startable {
   /**
    * Get multiple blocks back from an (async) iterable of cids
    */
-  async * getMany (cids: AwaitIterable<CID>, options: GetOfflineOptions & AbortOptions & ProgressOptions<GetManyBlocksProgressEvents> = {}): AsyncIterable<Pair> {
+  async * getMany (cids: AwaitIterable<CID>, options: GetOfflineOptions & AbortOptions & ProgressOptions<GetManyBlocksProgressEvents> = {}): AsyncGenerator<Pair> {
     options?.signal?.throwIfAborted()
     const releaseLock = await this.lock.readLock()
 
@@ -135,7 +135,7 @@ export class BlockStorage implements Blocks, Startable {
   /**
    * Delete multiple blocks from the blockstore
    */
-  async * deleteMany (cids: AwaitIterable<CID>, options: AbortOptions & ProgressOptions<DeleteManyBlocksProgressEvents> = {}): AsyncIterable<CID> {
+  async * deleteMany (cids: AwaitIterable<CID>, options: AbortOptions & ProgressOptions<DeleteManyBlocksProgressEvents> = {}): AsyncGenerator<CID> {
     options?.signal?.throwIfAborted()
     const releaseLock = await this.lock.writeLock()
 
@@ -167,7 +167,7 @@ export class BlockStorage implements Blocks, Startable {
     }
   }
 
-  async * getAll (options: AbortOptions & ProgressOptions<GetAllBlocksProgressEvents> = {}): AsyncIterable<Pair> {
+  async * getAll (options: AbortOptions & ProgressOptions<GetAllBlocksProgressEvents> = {}): AsyncGenerator<Pair> {
     options?.signal?.throwIfAborted()
     const releaseLock = await this.lock.readLock()
 

package/src/utils/networked-storage.ts

@@ -9,7 +9,7 @@ import { isPromise } from './is-promise.js'
 import type { HasherLoader } from '@helia/interface'
 import type { BlockBroker, Blocks, Pair, DeleteManyBlocksProgressEvents, DeleteBlockProgressEvents, GetBlockProgressEvents, GetManyBlocksProgressEvents, PutManyBlocksProgressEvents, PutBlockProgressEvents, GetAllBlocksProgressEvents, GetOfflineOptions, BlockRetrievalOptions, CreateSessionOptions, SessionBlockstore } from '@helia/interface/blocks'
 import type { AbortOptions, ComponentLogger, Logger, LoggerOptions, Startable } from '@libp2p/interface'
-import type { Blockstore } from 'interface-blockstore'
+import type { Blockstore, InputPair } from 'interface-blockstore'
 import type { AwaitIterable } from 'interface-store'
 import type { CID } from 'multiformats/cid'
 import type { MultihashDigest, MultihashHasher } from 'multiformats/hashes/interface'
@@ -56,7 +56,7 @@ class Storage implements Blockstore {
     options.onProgress?.(new CustomProgressEvent<CID>('blocks:put:providers:notify', cid))
 
     await Promise.all(
-      this.components.blockBrokers.map(async broker => broker.announce?.(cid, block, options))
+      this.components.blockBrokers.map(async broker => broker.announce?.(cid, options))
     )
 
     options.onProgress?.(new CustomProgressEvent<CID>('blocks:put:blockstore:put', cid))
@@ -67,7 +67,7 @@ class Storage implements Blockstore {
   /**
    * Put a multiple blocks to the underlying datastore
    */
-  async * putMany (blocks: AwaitIterable<{ cid: CID, block: Uint8Array }>, options: AbortOptions & ProgressOptions<PutManyBlocksProgressEvents> = {}): AsyncIterable<CID> {
+  async * putMany (blocks: AwaitIterable<InputPair>, options: AbortOptions & ProgressOptions<PutManyBlocksProgressEvents> = {}): AsyncGenerator<CID> {
     const missingBlocks = filter(blocks, async ({ cid }): Promise<boolean> => {
       const has = await this.child.has(cid, options)
 
@@ -78,10 +78,10 @@ class Storage implements Blockstore {
       return !has
     })
 
-    const notifyEach = forEach(missingBlocks, async ({ cid, block }): Promise<void> => {
+    const notifyEach = forEach(missingBlocks, async ({ cid }): Promise<void> => {
       options.onProgress?.(new CustomProgressEvent<CID>('blocks:put-many:providers:notify', cid))
       await Promise.all(
-        this.components.blockBrokers.map(async broker => broker.announce?.(cid, block, options))
+        this.components.blockBrokers.map(async broker => broker.announce?.(cid, options))
       )
     })
 
@@ -92,7 +92,7 @@ class Storage implements Blockstore {
   /**
    * Get a block by cid
    */
-  async get (cid: CID, options: GetOfflineOptions & AbortOptions & ProgressOptions<GetBlockProgressEvents> = {}): Promise<Uint8Array> {
+  async * get (cid: CID, options: GetOfflineOptions & AbortOptions & ProgressOptions<GetBlockProgressEvents> = {}): AsyncGenerator<Uint8Array> {
     if (options.offline !== true && !(await this.child.has(cid, options))) {
       const hasher = await this.getHasher(cid.multihash.code)
 
@@ -108,21 +108,22 @@ class Storage implements Blockstore {
       // notify other block providers of the new block
       options.onProgress?.(new CustomProgressEvent<CID>('blocks:get:providers:notify', cid))
       await Promise.all(
-        this.components.blockBrokers.map(async broker => broker.announce?.(cid, block, options))
+        this.components.blockBrokers.map(async broker => broker.announce?.(cid, options))
       )
 
-      return block
+      yield block
+      return
     }
 
     options.onProgress?.(new CustomProgressEvent<CID>('blocks:get:blockstore:get', cid))
 
-    return this.child.get(cid, options)
+    yield * this.child.get(cid, options)
   }
 
   /**
    * Get multiple blocks back from an (async) iterable of cids
    */
-  async * getMany (cids: AwaitIterable<CID>, options: GetOfflineOptions & AbortOptions & ProgressOptions<GetManyBlocksProgressEvents> = {}): AsyncIterable<Pair> {
+  async * getMany (cids: AwaitIterable<CID>, options: GetOfflineOptions & AbortOptions & ProgressOptions<GetManyBlocksProgressEvents> = {}): AsyncGenerator<Pair> {
     options.onProgress?.(new CustomProgressEvent('blocks:get-many:blockstore:get-many'))
 
     yield * this.child.getMany(forEach(cids, async (cid): Promise<void> => {
@@ -141,7 +142,7 @@ class Storage implements Blockstore {
         // notify other block providers of the new block
         options.onProgress?.(new CustomProgressEvent<CID>('blocks:get-many:providers:notify', cid))
         await Promise.all(
-          this.components.blockBrokers.map(async broker => broker.announce?.(cid, block, options))
+          this.components.blockBrokers.map(async broker => broker.announce?.(cid, options))
         )
       }
     }))
@@ -159,7 +160,7 @@ class Storage implements Blockstore {
   /**
    * Delete multiple blocks from the blockstore
    */
-  async * deleteMany (cids: AwaitIterable<CID>, options: AbortOptions & ProgressOptions<DeleteManyBlocksProgressEvents> = {}): AsyncIterable<CID> {
+  async * deleteMany (cids: AwaitIterable<CID>, options: AbortOptions & ProgressOptions<DeleteManyBlocksProgressEvents> = {}): AsyncGenerator<CID> {
     options.onProgress?.(new CustomProgressEvent('blocks:delete-many:blockstore:delete-many'))
     yield * this.child.deleteMany((async function * (): AsyncGenerator<CID> {
       for await (const cid of cids) {
@@ -172,7 +173,7 @@ class Storage implements Blockstore {
     return this.child.has(cid, options)
   }
 
-  async * getAll (options: AbortOptions & ProgressOptions<GetAllBlocksProgressEvents> = {}): AwaitIterable<Pair> {
+  async * getAll (options: AbortOptions & ProgressOptions<GetAllBlocksProgressEvents> = {}): AsyncGenerator<Pair> {
     options.onProgress?.(new CustomProgressEvent('blocks:get-all:blockstore:get-many'))
     yield * this.child.getAll(options)
   }
@@ -280,7 +281,7 @@ class SessionStorage extends Storage implements SessionBlockstore {
   /**
    * Put a multiple blocks to the underlying datastore
    */
-  async * putMany (blocks: AwaitIterable<{ cid: CID, block: Uint8Array }>, options: AbortOptions & ProgressOptions<PutManyBlocksProgressEvents> = {}): AsyncIterable<CID> {
+  async * putMany (blocks: AwaitIterable<InputPair>, options: AbortOptions & ProgressOptions<PutManyBlocksProgressEvents> = {}): AsyncGenerator<CID> {
     const signal = anySignal([this.closeController.signal, options.signal])
     setMaxListeners(Infinity, signal)
 
@@ -297,12 +298,12 @@ class SessionStorage extends Storage implements SessionBlockstore {
   /**
    * Get a block by cid
    */
-  async get (cid: CID, options: GetOfflineOptions & AbortOptions & ProgressOptions<GetBlockProgressEvents> = {}): Promise<Uint8Array> {
+  async * get (cid: CID, options: GetOfflineOptions & AbortOptions & ProgressOptions<GetBlockProgressEvents> = {}): AsyncGenerator<Uint8Array> {
     const signal = anySignal([this.closeController.signal, options.signal])
     setMaxListeners(Infinity, signal)
 
     try {
-      return await super.get(cid, {
+      yield * super.get(cid, {
         ...options,
         signal
       })
@@ -314,7 +315,7 @@ class SessionStorage extends Storage implements SessionBlockstore {
   /**
    * Get multiple blocks back from an (async) iterable of cids
    */
-  async * getMany (cids: AwaitIterable<CID>, options: GetOfflineOptions & AbortOptions & ProgressOptions<GetManyBlocksProgressEvents> = {}): AsyncIterable<Pair> {
+  async * getMany (cids: AwaitIterable<CID>, options: GetOfflineOptions & AbortOptions & ProgressOptions<GetManyBlocksProgressEvents> = {}): AsyncGenerator<Pair> {
     const signal = anySignal([this.closeController.signal, options.signal])
     setMaxListeners(Infinity, signal)
 
@@ -348,7 +349,7 @@ class SessionStorage extends Storage implements SessionBlockstore {
   /**
    * Delete multiple blocks from the blockstore
    */
-  async * deleteMany (cids: AwaitIterable<CID>, options: AbortOptions & ProgressOptions<DeleteManyBlocksProgressEvents> = {}): AsyncIterable<CID> {
+  async * deleteMany (cids: AwaitIterable<CID>, options: AbortOptions & ProgressOptions<DeleteManyBlocksProgressEvents> = {}): AsyncGenerator<CID> {
     const signal = anySignal([this.closeController.signal, options.signal])
     setMaxListeners(Infinity, signal)
 
@@ -376,7 +377,7 @@ class SessionStorage extends Storage implements SessionBlockstore {
     }
   }
 
-  async * getAll (options: AbortOptions & ProgressOptions<GetAllBlocksProgressEvents> = {}): AwaitIterable<Pair> {
+  async * getAll (options: AbortOptions & ProgressOptions<GetAllBlocksProgressEvents> = {}): AsyncGenerator<Pair> {
     const signal = anySignal([this.closeController.signal, options.signal])
     setMaxListeners(Infinity, signal)