@helia/utils 2.5.2-070b7528 → 2.5.2-73a28eda

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@helia/utils",
-  "version": "2.5.2-070b7528",
+  "version": "2.5.2-73a28eda",
   "description": "Shared code that implements the Helia API",
   "license": "Apache-2.0 OR MIT",
   "homepage": "https://github.com/ipfs/helia/tree/main/packages/utils#readme",
@@ -48,49 +48,47 @@
     "test:electron-main": "aegir test -t electron-main"
   },
   "dependencies": {
-    "@helia/interface": "6.2.1-070b7528",
-    "@ipld/dag-cbor": "^9.2.6",
-    "@ipld/dag-json": "^10.2.7",
+    "@helia/interface": "6.2.1-73a28eda",
+    "@ipld/dag-cbor": "^10.0.0",
+    "@ipld/dag-json": "^11.0.0",
     "@ipld/dag-pb": "^4.1.5",
+    "@ipshipyard/crypto": "^1.1.0",
+    "@ipshipyard/keychain": "^1.0.2",
     "@libp2p/interface": "^3.2.0",
-    "@libp2p/keychain": "^6.0.12",
     "@libp2p/utils": "^7.0.15",
     "@multiformats/dns": "^1.0.13",
     "@multiformats/multiaddr": "^13.0.1",
     "any-signal": "^4.2.0",
-    "blockstore-core": "^6.1.3",
+    "blockstore-core": "^7.0.1",
     "cborg": "^5.1.0",
-    "interface-blockstore": "^6.0.2",
-    "interface-datastore": "^9.0.3",
-    "interface-store": "^7.0.2",
+    "interface-blockstore": "^7.0.1",
+    "interface-datastore": "^10.0.1",
     "it-drain": "^3.0.12",
     "it-filter": "^3.1.5",
     "it-foreach": "^2.1.6",
     "it-merge": "^3.0.13",
-    "it-to-buffer": "^4.0.12",
+    "it-to-buffer": "^5.0.0",
     "libp2p": "^3.2.0",
     "mortice": "^3.3.1",
-    "multiformats": "^13.4.2",
+    "multiformats": "^14.0.0",
     "p-defer": "^4.0.1",
     "progress-events": "^1.1.0",
     "race-signal": "^2.0.0",
-    "uint8arrays": "^5.1.0"
+    "uint8arrays": "^6.1.1"
   },
   "devDependencies": {
     "@libp2p/crypto": "^5.1.15",
     "@libp2p/logger": "^6.2.4",
     "@libp2p/peer-id": "^6.0.6",
     "@types/sinon": "^21.0.1",
+    "abort-error": "^1.0.2",
     "aegir": "^48.0.4",
-    "datastore-core": "^11.0.3",
+    "datastore-core": "^12.0.1",
     "delay": "^7.0.0",
     "it-all": "^3.0.11",
     "it-map": "^3.1.5",
     "sinon": "^22.0.0",
     "sinon-ts": "^2.0.0"
   },
-  "browser": {
-    "./dist/src/utils/libp2p-defaults.js": "./dist/src/utils/libp2p-defaults.browser.js"
-  },
   "sideEffects": false
 }

package/src/errors.ts

@@ -32,3 +32,8 @@ export class BlockNotFoundWhileOfflineError extends Error {
   static name = 'BlockNotFoundWhileOfflineError'
   name = 'BlockNotFoundWhileOfflineError'
 }
+
+export class DecryptionFailedError extends Error {
+  static name = 'DecryptionFailedError'
+  name = 'DecryptionFailedError'
+}

package/src/index.ts

@@ -5,6 +5,7 @@
  * modules such as `helia`, `@helia/http`, etc.
  */
 
+import { keychain } from '@ipshipyard/keychain'
 import { contentRoutingSymbol, peerRoutingSymbol, start, stop, TypedEventEmitter } from '@libp2p/interface'
 import { dns } from '@multiformats/dns'
 import drain from 'it-drain'
@@ -14,14 +15,15 @@ import { Routing as RoutingClass } from './routing.ts'
 import { BlockStorage } from './storage.ts'
 import { assertDatastoreVersionIsCurrent } from './utils/datastore-version.ts'
 import { getCodec } from './utils/get-codec.ts'
+import { getCrypto } from './utils/get-crypto.ts'
 import { getHasher } from './utils/get-hasher.ts'
 import { NetworkedStorage } from './utils/networked-storage.ts'
 import type { BlockStorageInit } from './storage.ts'
-import type { Await, CodecLoader, GCOptions, HasherLoader, Helia as HeliaInterface, HeliaEvents, Routing } from '@helia/interface'
+import type { CodecLoader, GCOptions, HasherLoader, Helia as HeliaInterface, HeliaEvents, Routing, CryptoLoader, Crypto } from '@helia/interface'
 import type { BlockBroker } from '@helia/interface/blocks'
 import type { Pins } from '@helia/interface/pins'
+import type { Keychain, KeychainInit } from '@ipshipyard/keychain'
 import type { ComponentLogger, ContentRouting, Libp2p, Logger, Metrics, PeerRouting } from '@libp2p/interface'
-import type { KeychainInit } from '@libp2p/keychain'
 import type { DNS } from '@multiformats/dns'
 import type { Blockstore } from 'interface-blockstore'
 import type { Datastore } from 'interface-datastore'
@@ -88,7 +90,7 @@ export interface HeliaInit<T extends Libp2p = Libp2p> {
    * An optional function that can load a MultihashHasher on demand. May return
    * a promise.
    */
-  loadHasher?(code: number): Await<MultihashHasher>
+  loadHasher?(code: number): MultihashHasher | Promise<MultihashHasher>
 
   /**
    * In order to pin CIDs that correspond to a DAG, it's necessary to know
@@ -101,7 +103,7 @@ export interface HeliaInit<T extends Libp2p = Libp2p> {
    * An optional function that can load a BlockCodec on demand. May return a
    * promise.
    */
-  loadCodec?(code: number): Await<BlockCodec<any, any>>
+  loadCodec?(code: number): BlockCodec<any, any> | Promise<BlockCodec<any, any>>
 
   /**
    * A list of strategies used to fetch blocks when they are not present in
@@ -109,6 +111,16 @@ export interface HeliaInit<T extends Libp2p = Libp2p> {
    */
   blockBrokers: Array<(components: any) => BlockBroker>
 
+  /**
+   * A list of pre-supported public/private key implementations
+   */
+  cryptos?: Array<Crypto>
+
+  /**
+   * Dynamically load a cryptography implementation
+   */
+  loadCrypto?: CryptoLoader
+
   /**
    * Garbage collection requires preventing blockstore writes during searches
    * for unpinned blocks as DAGs are typically pinned after they've been
@@ -187,9 +199,11 @@ interface Components {
   blockBrokers: BlockBroker[]
   routing: Routing
   dns: DNS
+  keychain: Keychain
   metrics?: Metrics
   getCodec: CodecLoader
   getHasher: HasherLoader
+  getCrypto: CryptoLoader
 }
 
 export class Helia<T extends Libp2p> implements HeliaInterface<T> {
@@ -202,7 +216,9 @@ export class Helia<T extends Libp2p> implements HeliaInterface<T> {
   public routing: Routing
   public getCodec: CodecLoader
   public getHasher: HasherLoader
+  public getCrypto: CryptoLoader
   public dns: DNS
+  public keychain: Keychain
   public metrics?: Metrics
   private readonly log: Logger
 
@@ -211,12 +227,13 @@ export class Helia<T extends Libp2p> implements HeliaInterface<T> {
     this.log = this.logger.forComponent('helia')
     this.getHasher = getHasher(init.hashers, init.loadHasher)
     this.getCodec = getCodec(init.codecs, init.loadCodec)
+    this.getCrypto = getCrypto(init.cryptos, init.loadCrypto)
     this.dns = init.dns ?? dns()
     this.metrics = init.metrics
     this.libp2p = init.libp2p
     this.events = new TypedEventEmitter<HeliaEvents<T>>()
 
-    // @ts-expect-error routing is not set
+    // @ts-expect-error routing and keychain are not set
     const components: Components = {
       blockstore: init.blockstore,
       datastore: init.datastore,
@@ -225,11 +242,14 @@ export class Helia<T extends Libp2p> implements HeliaInterface<T> {
       blockBrokers: [],
       getHasher: this.getHasher,
       getCodec: this.getCodec,
+      getCrypto: this.getCrypto,
       dns: this.dns,
       metrics: this.metrics,
       ...(init.components ?? {})
     }
 
+    this.keychain = components.keychain = keychain()(components)
+
     this.routing = components.routing = new RoutingClass(components, {
       routers: (init.routers ?? []).flatMap((router: Partial<Routing> | ((components: any) => Partial<Routing>)) => {
         if (typeof router === 'function') {

package/src/storage.ts

@@ -6,7 +6,6 @@ import type { Blocks, Pair, DeleteManyBlocksProgressEvents, DeleteBlockProgressE
 import type { Pins } from '@helia/interface/pins'
 import type { AbortOptions, Startable } from '@libp2p/interface'
 import type { Blockstore, InputPair } from 'interface-blockstore'
-import type { AwaitIterable } from 'interface-store'
 import type { Mortice } from 'mortice'
 import type { CID } from 'multiformats/cid'
 import type { ProgressOptions } from 'progress-events'
@@ -79,7 +78,7 @@ export class BlockStorage implements Blocks, Startable {
   /**
    * Put a multiple blocks to the underlying datastore
    */
-  async * putMany (blocks: AwaitIterable<InputPair>, options: AbortOptions & ProgressOptions<PutManyBlocksProgressEvents> = {}): AsyncGenerator<CID> {
+  async * putMany (blocks: Iterable<InputPair> | AsyncIterable<InputPair>, options: AbortOptions & ProgressOptions<PutManyBlocksProgressEvents> = {}): AsyncGenerator<CID> {
     options?.signal?.throwIfAborted()
     const releaseLock = await this.lock.readLock()
 
@@ -107,7 +106,7 @@ export class BlockStorage implements Blocks, Startable {
   /**
    * Get multiple blocks back from an (async) iterable of cids
    */
-  async * getMany (cids: AwaitIterable<CID>, options: GetOfflineOptions & AbortOptions & ProgressOptions<GetManyBlocksProgressEvents> = {}): AsyncGenerator<Pair> {
+  async * getMany (cids: Iterable<CID> | AsyncIterable<CID>, options: GetOfflineOptions & AbortOptions & ProgressOptions<GetManyBlocksProgressEvents> = {}): AsyncGenerator<Pair> {
     options?.signal?.throwIfAborted()
     const releaseLock = await this.lock.readLock()
 
@@ -142,7 +141,7 @@ export class BlockStorage implements Blocks, Startable {
   /**
    * Delete multiple blocks from the blockstore
    */
-  async * deleteMany (cids: AwaitIterable<CID>, options: AbortOptions & ProgressOptions<DeleteManyBlocksProgressEvents> = {}): AsyncGenerator<CID> {
+  async * deleteMany (cids: Iterable<CID> | AsyncIterable<CID>, options: AbortOptions & ProgressOptions<DeleteManyBlocksProgressEvents> = {}): AsyncGenerator<CID> {
     options?.signal?.throwIfAborted()
     const releaseLock = await this.lock.writeLock()
 

package/src/utils/constants.ts

@@ -0,0 +1,3 @@
+export const SALT_LENGTH = 16
+export const KEY_SIZE = 32
+export const ITERATIONS = 10000

package/src/utils/get-codec.ts

@@ -1,5 +1,3 @@
-/* eslint max-depth: ["error", 7] */
-
 import { UnknownCodecError } from '@helia/interface'
 import * as dagCbor from '@ipld/dag-cbor'
 import * as dagJson from '@ipld/dag-json'
@@ -7,10 +5,10 @@ import * as dagPb from '@ipld/dag-pb'
 import * as json from 'multiformats/codecs/json'
 import * as raw from 'multiformats/codecs/raw'
 import { isPromise } from './is-promise.ts'
-import type { Await } from '@helia/interface'
+import type { CodecLoader } from '@helia/interface'
 import type { BlockCodec } from 'multiformats/codecs/interface'
 
-export function getCodec <T = any, Code extends number = any> (initialCodecs: Array<BlockCodec<any, any>> = [], loadCodec?: (code: number) => Await<BlockCodec<any, any>>): (code: Code) => Await<BlockCodec<Code, T>> {
+export function getCodec (initialCodecs: Array<BlockCodec<any, any>> = [], loadCodec?: CodecLoader): CodecLoader {
   const codecs: Record<number, BlockCodec<any, any>> = {
     [dagPb.code]: dagPb,
     [raw.code]: raw,

package/src/utils/get-crypto.ts

@@ -0,0 +1,44 @@
+import { UnknownCryptoError } from '@helia/interface'
+import { ecdsaCrypto, ed25519Crypto, rsaCrypto } from '@ipshipyard/crypto'
+import { isPromise } from './is-promise.ts'
+import type { CryptoLoader } from '@helia/interface'
+import type { Crypto } from '@ipshipyard/crypto'
+
+export function getCrypto (initialCryptos: Array<Crypto> = [], loadCrypto?: CryptoLoader): CryptoLoader {
+  const cryptos: Record<string | number, Crypto> = {}
+
+  initialCryptos = [
+    ecdsaCrypto(),
+    ed25519Crypto(),
+    rsaCrypto(),
+    ...initialCryptos
+  ]
+
+  initialCryptos.forEach(crypto => {
+    cryptos[crypto.type] = crypto
+    cryptos[crypto.code] = crypto
+  })
+
+  return async (nameOrCode) => {
+    let crypto = cryptos[nameOrCode]
+
+    if (crypto == null && loadCrypto != null) {
+      const res = loadCrypto(nameOrCode)
+
+      if (isPromise(res)) {
+        crypto = await res
+      } else {
+        crypto = res
+      }
+
+      cryptos[crypto.type] = crypto
+      cryptos[crypto.code] = crypto
+    }
+
+    if (crypto != null) {
+      return crypto
+    }
+
+    throw new UnknownCryptoError(`Could not load crypto for ${crypto}`)
+  }
+}

package/src/utils/get-hasher.ts

@@ -2,10 +2,10 @@ import { UnknownHashAlgorithmError } from '@helia/interface'
 import { identity } from 'multiformats/hashes/identity'
 import { sha256, sha512 } from 'multiformats/hashes/sha2'
 import { isPromise } from './is-promise.ts'
-import type { Await } from '@helia/interface'
+import type { HasherLoader } from '@helia/interface'
 import type { MultihashHasher } from 'multiformats/hashes/interface'
 
-export function getHasher (initialHashers: MultihashHasher[] = [], loadHasher?: (code: number) => Await<MultihashHasher>): (code: number) => Await<MultihashHasher> {
+export function getHasher (initialHashers: MultihashHasher[] = [], loadHasher?: HasherLoader): HasherLoader {
   const hashers: Record<number, MultihashHasher> = {
     [sha256.code]: sha256,
     [sha512.code]: sha512,

package/src/utils/session-storage.ts

@@ -6,7 +6,6 @@ import type { Pair, DeleteManyBlocksProgressEvents, DeleteBlockProgressEvents, G
 import type { AbortOptions, PeerId } from '@libp2p/interface'
 import type { Multiaddr } from '@multiformats/multiaddr'
 import type { InputPair } from 'interface-blockstore'
-import type { AwaitIterable } from 'interface-store'
 import type { CID } from 'multiformats/cid'
 import type { ProgressOptions } from 'progress-events'
 
@@ -63,7 +62,7 @@ export class SessionStorage extends Storage<SessionBlockBroker> implements Sessi
   /**
    * Put a multiple blocks to the underlying datastore
    */
-  async * putMany (blocks: AwaitIterable<InputPair>, options: AbortOptions & ProgressOptions<PutManyBlocksProgressEvents> = {}): AsyncGenerator<CID> {
+  async * putMany (blocks: Iterable<InputPair> | AsyncIterable<InputPair>, options: AbortOptions & ProgressOptions<PutManyBlocksProgressEvents> = {}): AsyncGenerator<CID> {
     const signal = anySignal([this.closeController.signal, options.signal])
     setMaxListeners(Infinity, signal)
 
@@ -97,7 +96,7 @@ export class SessionStorage extends Storage<SessionBlockBroker> implements Sessi
   /**
    * Get multiple blocks back from an (async) iterable of cids
    */
-  async * getMany (cids: AwaitIterable<CID>, options: GetOfflineOptions & AbortOptions & ProgressOptions<GetManyBlocksProgressEvents> = {}): AsyncGenerator<Pair> {
+  async * getMany (cids: Iterable<CID> | AsyncIterable<CID>, options: GetOfflineOptions & AbortOptions & ProgressOptions<GetManyBlocksProgressEvents> = {}): AsyncGenerator<Pair> {
     const signal = anySignal([this.closeController.signal, options.signal])
     setMaxListeners(Infinity, signal)
 
@@ -131,7 +130,7 @@ export class SessionStorage extends Storage<SessionBlockBroker> implements Sessi
   /**
    * Delete multiple blocks from the blockstore
    */
-  async * deleteMany (cids: AwaitIterable<CID>, options: AbortOptions & ProgressOptions<DeleteManyBlocksProgressEvents> = {}): AsyncGenerator<CID> {
+  async * deleteMany (cids: Iterable<CID> | AsyncIterable<CID>, options: AbortOptions & ProgressOptions<DeleteManyBlocksProgressEvents> = {}): AsyncGenerator<CID> {
     const signal = anySignal([this.closeController.signal, options.signal])
     setMaxListeners(Infinity, signal)
 

package/src/utils/storage.ts

@@ -11,7 +11,6 @@ import type { HasherLoader } from '@helia/interface'
 import type { BlockBroker, Pair, DeleteManyBlocksProgressEvents, DeleteBlockProgressEvents, GetBlockProgressEvents, GetManyBlocksProgressEvents, PutManyBlocksProgressEvents, PutBlockProgressEvents, GetAllBlocksProgressEvents, GetOfflineOptions, BlockRetrievalOptions } from '@helia/interface/blocks'
 import type { AbortOptions, ComponentLogger, Logger, LoggerOptions } from '@libp2p/interface'
 import type { Blockstore, InputPair } from 'interface-blockstore'
-import type { AwaitIterable } from 'interface-store'
 import type { CID } from 'multiformats/cid'
 import type { MultihashDigest, MultihashHasher } from 'multiformats/hashes/interface'
 import type { ProgressEvent, ProgressOptions } from 'progress-events'
@@ -72,7 +71,7 @@ export class Storage <Broker extends BlockBroker<ProgressEvent<any, any>, Progre
   /**
    * Put a multiple blocks to the underlying datastore
    */
-  async * putMany (blocks: AwaitIterable<InputPair>, options: AbortOptions & ProgressOptions<PutManyBlocksProgressEvents> = {}): AsyncGenerator<CID> {
+  async * putMany (blocks: Iterable<InputPair> | AsyncIterable<InputPair>, options: AbortOptions & ProgressOptions<PutManyBlocksProgressEvents> = {}): AsyncGenerator<CID> {
     const missingBlocks = filter(blocks, async ({ cid }): Promise<boolean> => {
       const has = await this.child.has(cid, options)
 
@@ -137,7 +136,7 @@ export class Storage <Broker extends BlockBroker<ProgressEvent<any, any>, Progre
   /**
    * Get multiple blocks back from an (async) iterable of cids
    */
-  async * getMany (cids: AwaitIterable<CID>, options: GetOfflineOptions & AbortOptions & ProgressOptions<GetManyBlocksProgressEvents> = {}): AsyncGenerator<Pair> {
+  async * getMany (cids: Iterable<CID> | AsyncIterable<CID>, options: GetOfflineOptions & AbortOptions & ProgressOptions<GetManyBlocksProgressEvents> = {}): AsyncGenerator<Pair> {
     options.onProgress?.(new CustomProgressEvent('blocks:get-many:blockstore:get-many'))
 
     yield * this.child.getMany(forEach(cids, async (cid): Promise<void> => {
@@ -182,7 +181,7 @@ export class Storage <Broker extends BlockBroker<ProgressEvent<any, any>, Progre
   /**
    * Delete multiple blocks from the blockstore
    */
-  async * deleteMany (cids: AwaitIterable<CID>, options: AbortOptions & ProgressOptions<DeleteManyBlocksProgressEvents> = {}): AsyncGenerator<CID> {
+  async * deleteMany (cids: Iterable<CID> | AsyncIterable<CID>, options: AbortOptions & ProgressOptions<DeleteManyBlocksProgressEvents> = {}): AsyncGenerator<CID> {
     options.onProgress?.(new CustomProgressEvent('blocks:delete-many:blockstore:delete-many'))
     yield * this.child.deleteMany((async function * (): AsyncGenerator<CID> {
       for await (const cid of cids) {