@helia/utils 0.1.0-ecf5394 → 0.2.0

package/src/storage.ts

@@ -1,6 +1,6 @@
 import { start, stop } from '@libp2p/interface'
 import createMortice from 'mortice'
-import type { Blocks, Pair, DeleteManyBlocksProgressEvents, DeleteBlockProgressEvents, GetBlockProgressEvents, GetManyBlocksProgressEvents, PutManyBlocksProgressEvents, PutBlockProgressEvents, GetAllBlocksProgressEvents, GetOfflineOptions } from '@helia/interface/blocks'
+import type { Blocks, Pair, DeleteManyBlocksProgressEvents, DeleteBlockProgressEvents, GetBlockProgressEvents, GetManyBlocksProgressEvents, PutManyBlocksProgressEvents, PutBlockProgressEvents, GetAllBlocksProgressEvents, GetOfflineOptions, SessionBlockstore } from '@helia/interface/blocks'
 import type { Pins } from '@helia/interface/pins'
 import type { AbortOptions, Startable } from '@libp2p/interface'
 import type { Blockstore } from 'interface-blockstore'
@@ -24,14 +24,14 @@ export interface GetOptions extends AbortOptions {
  */
 export class BlockStorage implements Blocks, Startable {
   public lock: Mortice
-  private readonly child: Blockstore
+  private readonly child: Blocks
   private readonly pins: Pins
   private started: boolean
 
   /**
    * Create a new BlockStorage
    */
-  constructor (blockstore: Blockstore, pins: Pins, options: BlockStorageInit = {}) {
+  constructor (blockstore: Blocks, pins: Pins, options: BlockStorageInit = {}) {
     this.child = blockstore
     this.pins = pins
     this.lock = createMortice({
@@ -62,6 +62,7 @@ export class BlockStorage implements Blocks, Startable {
    * Put a block to the underlying datastore
    */
   async put (cid: CID, block: Uint8Array, options: AbortOptions & ProgressOptions<PutBlockProgressEvents> = {}): Promise<CID> {
+    options?.signal?.throwIfAborted()
     const releaseLock = await this.lock.readLock()
 
     try {
@@ -75,6 +76,7 @@ export class BlockStorage implements Blocks, Startable {
    * Put a multiple blocks to the underlying datastore
    */
   async * putMany (blocks: AwaitIterable<{ cid: CID, block: Uint8Array }>, options: AbortOptions & ProgressOptions<PutManyBlocksProgressEvents> = {}): AsyncIterable<CID> {
+    options?.signal?.throwIfAborted()
     const releaseLock = await this.lock.readLock()
 
     try {
@@ -88,6 +90,7 @@ export class BlockStorage implements Blocks, Startable {
    * Get a block by cid
    */
   async get (cid: CID, options: GetOfflineOptions & AbortOptions & ProgressOptions<GetBlockProgressEvents> = {}): Promise<Uint8Array> {
+    options?.signal?.throwIfAborted()
     const releaseLock = await this.lock.readLock()
 
     try {
@@ -101,6 +104,7 @@ export class BlockStorage implements Blocks, Startable {
    * Get multiple blocks back from an (async) iterable of cids
    */
   async * getMany (cids: AwaitIterable<CID>, options: GetOfflineOptions & AbortOptions & ProgressOptions<GetManyBlocksProgressEvents> = {}): AsyncIterable<Pair> {
+    options?.signal?.throwIfAborted()
     const releaseLock = await this.lock.readLock()
 
     try {
@@ -114,6 +118,7 @@ export class BlockStorage implements Blocks, Startable {
    * Delete a block from the blockstore
    */
   async delete (cid: CID, options: AbortOptions & ProgressOptions<DeleteBlockProgressEvents> = {}): Promise<void> {
+    options?.signal?.throwIfAborted()
     const releaseLock = await this.lock.writeLock()
 
     try {
@@ -131,6 +136,7 @@ export class BlockStorage implements Blocks, Startable {
    * Delete multiple blocks from the blockstore
    */
   async * deleteMany (cids: AwaitIterable<CID>, options: AbortOptions & ProgressOptions<DeleteManyBlocksProgressEvents> = {}): AsyncIterable<CID> {
+    options?.signal?.throwIfAborted()
     const releaseLock = await this.lock.writeLock()
 
     try {
@@ -151,6 +157,7 @@ export class BlockStorage implements Blocks, Startable {
   }
 
   async has (cid: CID, options: AbortOptions = {}): Promise<boolean> {
+    options?.signal?.throwIfAborted()
     const releaseLock = await this.lock.readLock()
 
     try {
@@ -161,6 +168,7 @@ export class BlockStorage implements Blocks, Startable {
   }
 
   async * getAll (options: AbortOptions & ProgressOptions<GetAllBlocksProgressEvents> = {}): AsyncIterable<Pair> {
+    options?.signal?.throwIfAborted()
     const releaseLock = await this.lock.readLock()
 
     try {
@@ -169,4 +177,9 @@ export class BlockStorage implements Blocks, Startable {
       releaseLock()
     }
   }
+
+  createSession (root: CID, options?: AbortOptions): SessionBlockstore {
+    options?.signal?.throwIfAborted()
+    return this.child.createSession(root, options)
+  }
 }

package/src/utils/networked-storage.ts

@@ -1,12 +1,11 @@
-import { CodeError, start, stop } from '@libp2p/interface'
+import { CodeError, setMaxListeners, start, stop } from '@libp2p/interface'
 import { anySignal } from 'any-signal'
 import { IdentityBlockstore } from 'blockstore-core/identity'
-import { TieredBlockstore } from 'blockstore-core/tiered'
 import filter from 'it-filter'
 import forEach from 'it-foreach'
 import { CustomProgressEvent, type ProgressOptions } from 'progress-events'
 import { equals as uint8ArrayEquals } from 'uint8arrays/equals'
-import type { BlockBroker, Blocks, Pair, DeleteManyBlocksProgressEvents, DeleteBlockProgressEvents, GetBlockProgressEvents, GetManyBlocksProgressEvents, PutManyBlocksProgressEvents, PutBlockProgressEvents, GetAllBlocksProgressEvents, GetOfflineOptions, BlockRetriever, BlockAnnouncer, BlockRetrievalOptions } from '@helia/interface/blocks'
+import type { BlockBroker, Blocks, Pair, DeleteManyBlocksProgressEvents, DeleteBlockProgressEvents, GetBlockProgressEvents, GetManyBlocksProgressEvents, PutManyBlocksProgressEvents, PutBlockProgressEvents, GetAllBlocksProgressEvents, GetOfflineOptions, BlockRetrievalOptions, CreateSessionOptions, SessionBlockstore } from '@helia/interface/blocks'
 import type { AbortOptions, ComponentLogger, Logger, LoggerOptions, Startable } from '@libp2p/interface'
 import type { Blockstore } from 'interface-blockstore'
 import type { AwaitIterable } from 'interface-store'
@@ -17,80 +16,45 @@ export interface GetOptions extends AbortOptions {
   progress?(evt: Event): void
 }
 
-function isBlockRetriever (b: any): b is BlockRetriever {
-  return typeof b.retrieve === 'function'
-}
-
-function isBlockAnnouncer (b: any): b is BlockAnnouncer {
-  return typeof b.announce === 'function'
-}
-
-export interface NetworkedStorageComponents {
+export interface StorageComponents {
   blockstore: Blockstore
   logger: ComponentLogger
-  blockBrokers?: BlockBroker[]
-  hashers?: Record<number, MultihashHasher>
+  blockBrokers: BlockBroker[]
+  hashers: Record<number, MultihashHasher>
 }
 
-/**
- * Networked storage wraps a regular blockstore - when getting blocks if the
- * blocks are not present Bitswap will be used to fetch them from network peers.
- */
-export class NetworkedStorage implements Blocks, Startable {
-  private readonly child: Blockstore
-  private readonly blockRetrievers: BlockRetriever[]
-  private readonly blockAnnouncers: BlockAnnouncer[]
-  private readonly hashers: Record<number, MultihashHasher>
-  private started: boolean
-  private readonly log: Logger
+class Storage implements Blockstore {
+  protected readonly child: Blockstore
+  protected readonly hashers: Record<number, MultihashHasher>
+  protected log: Logger
+  protected readonly logger: ComponentLogger
+  protected readonly components: StorageComponents
 
   /**
    * Create a new BlockStorage
    */
-  constructor (components: NetworkedStorageComponents) {
+  constructor (components: StorageComponents) {
     this.log = components.logger.forComponent('helia:networked-storage')
-    this.child = new TieredBlockstore([
-      new IdentityBlockstore(),
-      components.blockstore
-    ])
-    this.blockRetrievers = (components.blockBrokers ?? []).filter(isBlockRetriever)
-    this.blockAnnouncers = (components.blockBrokers ?? []).filter(isBlockAnnouncer)
+    this.logger = components.logger
+    this.components = components
+    this.child = new IdentityBlockstore(components.blockstore)
     this.hashers = components.hashers ?? {}
-    this.started = false
-  }
-
-  isStarted (): boolean {
-    return this.started
-  }
-
-  async start (): Promise<void> {
-    await start(this.child, ...new Set([...this.blockRetrievers, ...this.blockAnnouncers]))
-    this.started = true
-  }
-
-  async stop (): Promise<void> {
-    await stop(this.child, ...new Set([...this.blockRetrievers, ...this.blockAnnouncers]))
-    this.started = false
-  }
-
-  unwrap (): Blockstore {
-    return this.child
   }
 
   /**
    * Put a block to the underlying datastore
    */
   async put (cid: CID, block: Uint8Array, options: AbortOptions & ProgressOptions<PutBlockProgressEvents> = {}): Promise<CID> {
-    if (await this.child.has(cid)) {
+    if (await this.child.has(cid, options)) {
       options.onProgress?.(new CustomProgressEvent<CID>('blocks:put:duplicate', cid))
       return cid
     }
 
     options.onProgress?.(new CustomProgressEvent<CID>('blocks:put:providers:notify', cid))
 
-    this.blockAnnouncers.forEach(provider => {
-      provider.announce(cid, block, options)
-    })
+    await Promise.all(
+      this.components.blockBrokers.map(async broker => broker.announce?.(cid, block, options))
+    )
 
     options.onProgress?.(new CustomProgressEvent<CID>('blocks:put:blockstore:put', cid))
 
@@ -102,7 +66,7 @@ export class NetworkedStorage implements Blocks, Startable {
    */
   async * putMany (blocks: AwaitIterable<{ cid: CID, block: Uint8Array }>, options: AbortOptions & ProgressOptions<PutManyBlocksProgressEvents> = {}): AsyncIterable<CID> {
     const missingBlocks = filter(blocks, async ({ cid }): Promise<boolean> => {
-      const has = await this.child.has(cid)
+      const has = await this.child.has(cid, options)
 
       if (has) {
         options.onProgress?.(new CustomProgressEvent<CID>('blocks:put-many:duplicate', cid))
@@ -111,11 +75,11 @@ export class NetworkedStorage implements Blocks, Startable {
       return !has
     })
 
-    const notifyEach = forEach(missingBlocks, ({ cid, block }): void => {
+    const notifyEach = forEach(missingBlocks, async ({ cid, block }): Promise<void> => {
       options.onProgress?.(new CustomProgressEvent<CID>('blocks:put-many:providers:notify', cid))
-      this.blockAnnouncers.forEach(provider => {
-        provider.announce(cid, block, options)
-      })
+      await Promise.all(
+        this.components.blockBrokers.map(async broker => broker.announce?.(cid, block, options))
+      )
     })
 
     options.onProgress?.(new CustomProgressEvent('blocks:put-many:blockstore:put-many'))
@@ -126,10 +90,10 @@ export class NetworkedStorage implements Blocks, Startable {
    * Get a block by cid
    */
   async get (cid: CID, options: GetOfflineOptions & AbortOptions & ProgressOptions<GetBlockProgressEvents> = {}): Promise<Uint8Array> {
-    if (options.offline !== true && !(await this.child.has(cid))) {
+    if (options.offline !== true && !(await this.child.has(cid, options))) {
       // we do not have the block locally, get it from a block provider
       options.onProgress?.(new CustomProgressEvent<CID>('blocks:get:providers:get', cid))
-      const block = await raceBlockRetrievers(cid, this.blockRetrievers, this.hashers[cid.multihash.code], {
+      const block = await raceBlockRetrievers(cid, this.components.blockBrokers, this.hashers[cid.multihash.code], {
         ...options,
         log: this.log
       })
@@ -138,9 +102,9 @@ export class NetworkedStorage implements Blocks, Startable {
 
       // notify other block providers of the new block
       options.onProgress?.(new CustomProgressEvent<CID>('blocks:get:providers:notify', cid))
-      this.blockAnnouncers.forEach(provider => {
-        provider.announce(cid, block, options)
-      })
+      await Promise.all(
+        this.components.blockBrokers.map(async broker => broker.announce?.(cid, block, options))
+      )
 
       return block
     }
@@ -157,10 +121,10 @@ export class NetworkedStorage implements Blocks, Startable {
     options.onProgress?.(new CustomProgressEvent('blocks:get-many:blockstore:get-many'))
 
     yield * this.child.getMany(forEach(cids, async (cid): Promise<void> => {
-      if (options.offline !== true && !(await this.child.has(cid))) {
+      if (options.offline !== true && !(await this.child.has(cid, options))) {
         // we do not have the block locally, get it from a block provider
         options.onProgress?.(new CustomProgressEvent<CID>('blocks:get-many:providers:get', cid))
-        const block = await raceBlockRetrievers(cid, this.blockRetrievers, this.hashers[cid.multihash.code], {
+        const block = await raceBlockRetrievers(cid, this.components.blockBrokers, this.hashers[cid.multihash.code], {
           ...options,
           log: this.log
         })
@@ -169,9 +133,9 @@ export class NetworkedStorage implements Blocks, Startable {
 
         // notify other block providers of the new block
         options.onProgress?.(new CustomProgressEvent<CID>('blocks:get-many:providers:notify', cid))
-        this.blockAnnouncers.forEach(provider => {
-          provider.announce(cid, block, options)
-        })
+        await Promise.all(
+          this.components.blockBrokers.map(async broker => broker.announce?.(cid, block, options))
+        )
       }
     }))
   }
@@ -207,6 +171,223 @@ export class NetworkedStorage implements Blocks, Startable {
   }
 }
 
+export type NetworkedStorageComponents = StorageComponents
+
+/**
+ * Networked storage wraps a regular blockstore - when getting blocks if the
+ * blocks are not present, the configured BlockBrokers will be used to fetch them.
+ */
+export class NetworkedStorage extends Storage implements Blocks, Startable {
+  private started: boolean
+
+  /**
+   * Create a new BlockStorage
+   */
+  constructor (components: NetworkedStorageComponents) {
+    super(components)
+
+    this.started = false
+  }
+
+  isStarted (): boolean {
+    return this.started
+  }
+
+  async start (): Promise<void> {
+    await start(this.child, ...this.components.blockBrokers)
+    this.started = true
+  }
+
+  async stop (): Promise<void> {
+    await stop(this.child, ...this.components.blockBrokers)
+    this.started = false
+  }
+
+  unwrap (): Blockstore {
+    return this.child
+  }
+
+  createSession (root: CID, options?: CreateSessionOptions): SessionBlockstore {
+    const blockBrokers = this.components.blockBrokers.map(broker => {
+      if (broker.createSession == null) {
+        return broker
+      }
+
+      return broker.createSession(options)
+    })
+
+    return new SessionStorage({
+      blockstore: this.child,
+      blockBrokers,
+      hashers: this.hashers,
+      logger: this.logger
+    }, {
+      root
+    })
+  }
+}
+
+interface SessionStorageInit {
+  root: CID
+}
+
+/**
+ * Storage subclass that can cancel any ongoing operation at any point.
+ */
+class SessionStorage extends Storage implements SessionBlockstore {
+  private readonly closeController: AbortController
+
+  constructor (components: StorageComponents, init: SessionStorageInit) {
+    super(components)
+
+    // because brokers are allowed to continue searching for providers after the
+    // session has been created, we need a way to tell them that the user has
+    // finished using the session any in-flight requests should be cancelled
+    this.closeController = new AbortController()
+    setMaxListeners(Infinity, this.closeController.signal)
+
+    this.log = components.logger.forComponent(`helia:session-storage${init.root}`)
+  }
+
+  close (): void {
+    this.closeController.abort()
+  }
+
+  /**
+   * Put a block to the underlying datastore
+   */
+  async put (cid: CID, block: Uint8Array, options: AbortOptions & ProgressOptions<PutBlockProgressEvents> = {}): Promise<CID> {
+    const signal = anySignal([this.closeController.signal, options.signal])
+    setMaxListeners(Infinity, signal)
+
+    try {
+      return await super.put(cid, block, {
+        ...options,
+        signal
+      })
+    } finally {
+      signal.clear()
+    }
+  }
+
+  /**
+   * Put a multiple blocks to the underlying datastore
+   */
+  async * putMany (blocks: AwaitIterable<{ cid: CID, block: Uint8Array }>, options: AbortOptions & ProgressOptions<PutManyBlocksProgressEvents> = {}): AsyncIterable<CID> {
+    const signal = anySignal([this.closeController.signal, options.signal])
+    setMaxListeners(Infinity, signal)
+
+    try {
+      yield * super.putMany(blocks, {
+        ...options,
+        signal
+      })
+    } finally {
+      signal.clear()
+    }
+  }
+
+  /**
+   * Get a block by cid
+   */
+  async get (cid: CID, options: GetOfflineOptions & AbortOptions & ProgressOptions<GetBlockProgressEvents> = {}): Promise<Uint8Array> {
+    const signal = anySignal([this.closeController.signal, options.signal])
+    setMaxListeners(Infinity, signal)
+
+    try {
+      return await super.get(cid, {
+        ...options,
+        signal
+      })
+    } finally {
+      signal.clear()
+    }
+  }
+
+  /**
+   * Get multiple blocks back from an (async) iterable of cids
+   */
+  async * getMany (cids: AwaitIterable<CID>, options: GetOfflineOptions & AbortOptions & ProgressOptions<GetManyBlocksProgressEvents> = {}): AsyncIterable<Pair> {
+    const signal = anySignal([this.closeController.signal, options.signal])
+    setMaxListeners(Infinity, signal)
+
+    try {
+      yield * super.getMany(cids, {
+        ...options,
+        signal
+      })
+    } finally {
+      signal.clear()
+    }
+  }
+
+  /**
+   * Delete a block from the blockstore
+   */
+  async delete (cid: CID, options: AbortOptions & ProgressOptions<DeleteBlockProgressEvents> = {}): Promise<void> {
+    const signal = anySignal([this.closeController.signal, options.signal])
+    setMaxListeners(Infinity, signal)
+
+    try {
+      await super.delete(cid, {
+        ...options,
+        signal
+      })
+    } finally {
+      signal.clear()
+    }
+  }
+
+  /**
+   * Delete multiple blocks from the blockstore
+   */
+  async * deleteMany (cids: AwaitIterable<CID>, options: AbortOptions & ProgressOptions<DeleteManyBlocksProgressEvents> = {}): AsyncIterable<CID> {
+    const signal = anySignal([this.closeController.signal, options.signal])
+    setMaxListeners(Infinity, signal)
+
+    try {
+      yield * super.deleteMany(cids, {
+        ...options,
+        signal
+      })
+    } finally {
+      signal.clear()
+    }
+  }
+
+  async has (cid: CID, options: AbortOptions = {}): Promise<boolean> {
+    const signal = anySignal([this.closeController.signal, options.signal])
+    setMaxListeners(Infinity, signal)
+
+    try {
+      return await super.has(cid, {
+        ...options,
+        signal
+      })
+    } finally {
+      signal.clear()
+    }
+  }
+
+  async * getAll (options: AbortOptions & ProgressOptions<GetAllBlocksProgressEvents> = {}): AwaitIterable<Pair> {
+    const signal = anySignal([this.closeController.signal, options.signal])
+    setMaxListeners(Infinity, signal)
+
+    try {
+      yield * super.getAll({
+        ...options,
+        signal
+      })
+    } finally {
+      signal.clear()
+    }
+  }
+}
+
+function isRetrievingBlockBroker (broker: BlockBroker): broker is Required<Pick<BlockBroker, 'retrieve'>> {
+  return typeof broker.retrieve === 'function'
+}
+
 export const getCidBlockVerifierFunction = (cid: CID, hasher: MultihashHasher): Required<BlockRetrievalOptions>['validateFn'] => {
   if (hasher == null) {
     throw new CodeError(`No hasher configured for multihash code 0x${cid.multihash.code.toString(16)}, please configure one. You can look up which hash this is at https://github.com/multiformats/multicodec/blob/master/table.csv`, 'ERR_UNKNOWN_HASH_ALG')
@@ -227,40 +408,53 @@ export const getCidBlockVerifierFunction = (cid: CID, hasher: MultihashHasher):
  * Race block providers cancelling any pending requests once the block has been
  * found.
  */
-async function raceBlockRetrievers (cid: CID, providers: BlockRetriever[], hasher: MultihashHasher, options: AbortOptions & LoggerOptions): Promise<Uint8Array> {
+async function raceBlockRetrievers (cid: CID, blockBrokers: BlockBroker[], hasher: MultihashHasher, options: AbortOptions & LoggerOptions): Promise<Uint8Array> {
   const validateFn = getCidBlockVerifierFunction(cid, hasher)
 
   const controller = new AbortController()
   const signal = anySignal([controller.signal, options.signal])
+  setMaxListeners(Infinity, controller.signal, signal)
+
+  const retrievers: Array<Required<Pick<BlockBroker, 'retrieve'>>> = []
+
+  for (const broker of blockBrokers) {
+    if (isRetrievingBlockBroker(broker)) {
+      retrievers.push(broker)
+    }
+  }
 
   try {
     return await Promise.any(
-      providers.map(async provider => {
-        try {
-          let blocksWereValidated = false
-          const block = await provider.retrieve(cid, {
-            ...options,
-            signal,
-            validateFn: async (block: Uint8Array): Promise<void> => {
+      retrievers
+        .map(async retriever => {
+          try {
+            let blocksWereValidated = false
+            const block = await retriever.retrieve(cid, {
+              ...options,
+              signal,
+              validateFn: async (block: Uint8Array): Promise<void> => {
+                await validateFn(block)
+                blocksWereValidated = true
+              }
+            })
+
+            if (!blocksWereValidated) {
+              // the blockBroker either did not throw an error when attempting to validate the block
+              // or did not call the validateFn at all. We should validate the block ourselves
               await validateFn(block)
-              blocksWereValidated = true
             }
-          })
 
-          if (!blocksWereValidated) {
-            // the blockBroker either did not throw an error when attempting to validate the block
-            // or did not call the validateFn at all. We should validate the block ourselves
-            await validateFn(block)
+            return block
+          } catch (err) {
+            options.log.error('could not retrieve verified block for %c', cid, err)
+            throw err
           }
-
-          return block
-        } catch (err) {
-          options.log.error('could not retrieve verified block for %c', cid, err)
-          throw err
-        }
-      })
+        })
     )
   } finally {
+    // we have the block from the fastest block retriever, abort any still
+    // in-flight retrieve attempts
+    controller.abort()
     signal.clear()
   }
 }