@helia/utils 2.5.2-6f8165b5 → 2.5.2-9114743f

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (63) hide show
  1. package/dist/index.min.js +22 -1
  2. package/dist/index.min.js.map +4 -4
  3. package/dist/src/abstract-session.d.ts +3 -3
  4. package/dist/src/abstract-session.d.ts.map +1 -1
  5. package/dist/src/abstract-session.js.map +1 -1
  6. package/dist/src/errors.d.ts +4 -0
  7. package/dist/src/errors.d.ts.map +1 -1
  8. package/dist/src/errors.js +4 -0
  9. package/dist/src/errors.js.map +1 -1
  10. package/dist/src/graph-walker.d.ts +3 -21
  11. package/dist/src/graph-walker.d.ts.map +1 -1
  12. package/dist/src/graph-walker.js +20 -17
  13. package/dist/src/graph-walker.js.map +1 -1
  14. package/dist/src/index.d.ts +39 -36
  15. package/dist/src/index.d.ts.map +1 -1
  16. package/dist/src/index.js +62 -30
  17. package/dist/src/index.js.map +1 -1
  18. package/dist/src/pins.d.ts.map +1 -1
  19. package/dist/src/pins.js +11 -39
  20. package/dist/src/pins.js.map +1 -1
  21. package/dist/src/routing.d.ts +7 -5
  22. package/dist/src/routing.d.ts.map +1 -1
  23. package/dist/src/routing.js +10 -4
  24. package/dist/src/routing.js.map +1 -1
  25. package/dist/src/utils/constants.d.ts +4 -0
  26. package/dist/src/utils/constants.d.ts.map +1 -0
  27. package/dist/src/utils/constants.js +4 -0
  28. package/dist/src/utils/constants.js.map +1 -0
  29. package/dist/src/utils/get-codec.d.ts +2 -1
  30. package/dist/src/utils/get-codec.d.ts.map +1 -1
  31. package/dist/src/utils/get-codec.js +0 -1
  32. package/dist/src/utils/get-codec.js.map +1 -1
  33. package/dist/src/utils/get-crypto.d.ts +4 -0
  34. package/dist/src/utils/get-crypto.d.ts.map +1 -0
  35. package/dist/src/utils/get-crypto.js +35 -0
  36. package/dist/src/utils/get-crypto.js.map +1 -0
  37. package/dist/src/utils/get-hasher.d.ts +2 -1
  38. package/dist/src/utils/get-hasher.d.ts.map +1 -1
  39. package/dist/src/utils/get-hasher.js.map +1 -1
  40. package/dist/src/utils/is-cid.d.ts +3 -0
  41. package/dist/src/utils/is-cid.d.ts.map +1 -0
  42. package/dist/src/utils/is-cid.js +8 -0
  43. package/dist/src/utils/is-cid.js.map +1 -0
  44. package/dist/src/utils/networked-storage.d.ts +2 -1
  45. package/dist/src/utils/networked-storage.d.ts.map +1 -1
  46. package/dist/src/utils/networked-storage.js.map +1 -1
  47. package/dist/src/utils/session-storage.d.ts +2 -2
  48. package/dist/src/utils/session-storage.d.ts.map +1 -1
  49. package/dist/src/utils/session-storage.js.map +1 -1
  50. package/package.json +21 -22
  51. package/src/abstract-session.ts +4 -4
  52. package/src/errors.ts +5 -0
  53. package/src/graph-walker.ts +30 -43
  54. package/src/index.ts +108 -73
  55. package/src/pins.ts +12 -50
  56. package/src/routing.ts +23 -10
  57. package/src/utils/constants.ts +3 -0
  58. package/src/utils/get-codec.ts +2 -3
  59. package/src/utils/get-crypto.ts +44 -0
  60. package/src/utils/get-hasher.ts +2 -1
  61. package/src/utils/is-cid.ts +9 -0
  62. package/src/utils/networked-storage.ts +2 -1
  63. package/src/utils/session-storage.ts +2 -2
package/src/errors.ts CHANGED
@@ -32,3 +32,8 @@ export class BlockNotFoundWhileOfflineError extends Error {
32
32
  static name = 'BlockNotFoundWhileOfflineError'
33
33
  name = 'BlockNotFoundWhileOfflineError'
34
34
  }
35
+
36
+ export class DecryptionFailedError extends Error {
37
+ static name = 'DecryptionFailedError'
38
+ name = 'DecryptionFailedError'
39
+ }
@@ -2,29 +2,17 @@ import { Queue } from '@libp2p/utils'
2
2
  import filter from 'it-filter'
3
3
  import toBuffer from 'it-to-buffer'
4
4
  import { createUnsafe } from 'multiformats/block'
5
- import type { CodecLoader } from '@helia/interface'
5
+ import type {
6
+ GraphWalker,
7
+ GraphWalkerComponents,
8
+ GraphWalkerInit,
9
+ GraphNode,
10
+ WalkOptions
11
+ } from '@helia/interface'
6
12
  import type { AbortOptions } from '@libp2p/interface'
7
- import type { Blockstore } from 'interface-blockstore'
8
- import type { BlockView, CID, Version } from 'multiformats'
13
+ import type { CID } from 'multiformats'
9
14
 
10
- export interface GraphWalkerComponents {
11
- blockstore: Blockstore
12
- getCodec: CodecLoader
13
- }
14
-
15
- export interface GraphWalkerInit {
16
-
17
- }
18
-
19
- export interface GraphNode <T = unknown, C extends number = number, A extends number = number, V extends Version = 0 | 1> {
20
- block: BlockView<T, C, A, V>
21
- depth: number
22
- path: CID[]
23
- }
24
-
25
- export interface GraphWalker {
26
- walk <T = any> (cid: CID, options?: WalkOptions<T>): AsyncGenerator<GraphNode<T>>
27
- }
15
+ export type { GraphWalker, GraphWalkerComponents, GraphWalkerInit, GraphNode, WalkOptions }
28
16
 
29
17
  /**
30
18
  * A depth-first walker descends into child blocks before processing successor
@@ -55,10 +43,6 @@ interface JobOptions extends AbortOptions {
55
43
  path: CID[]
56
44
  }
57
45
 
58
- export interface WalkOptions<T> extends AbortOptions {
59
- includeChild?(child: CID, parent: BlockView<T, number, number, 0 | 1>): boolean
60
- }
61
-
62
46
  abstract class AbstractGraphWalker {
63
47
  private readonly components: GraphWalkerComponents
64
48
 
@@ -70,6 +54,7 @@ abstract class AbstractGraphWalker {
70
54
  const queue = this.getQueue()
71
55
  const gen = filter(queue.toGenerator(options), (node) => node != null) as AsyncGenerator<GraphNode<T>>
72
56
  let finished = false
57
+ const maxDepth = options?.depth ?? Infinity
73
58
 
74
59
  const job = async (opts: JobOptions): Promise<GraphNode<T> | undefined> => {
75
60
  const cid = opts.cid
@@ -80,25 +65,27 @@ abstract class AbstractGraphWalker {
80
65
  codec: await this.components.getCodec(cid.code)
81
66
  })
82
67
 
83
- for (const [, linkedCid] of block.links()) {
84
- if (options?.includeChild?.(linkedCid, block) === false) {
85
- continue
86
- }
87
-
88
- queue.add(job, {
89
- ...opts,
90
- cid: linkedCid,
91
- depth: opts.depth + 1,
92
- path: [...opts.path, linkedCid]
93
- })
94
- // eslint-disable-next-line no-loop-func
95
- .catch(err => {
96
- // only throw if the generator is still yielding results, otherwise
97
- // it can cause unhandled promise rejections
98
- if (!finished) {
99
- gen.throw(err)
100
- }
68
+ if (opts.depth < maxDepth) {
69
+ for (const [, linkedCid] of block.links()) {
70
+ if (options?.includeChild?.(linkedCid, block) === false) {
71
+ continue
72
+ }
73
+
74
+ queue.add(job, {
75
+ ...opts,
76
+ cid: linkedCid,
77
+ depth: opts.depth + 1,
78
+ path: [...opts.path, linkedCid]
101
79
  })
80
+ // eslint-disable-next-line no-loop-func
81
+ .catch(err => {
82
+ // only throw if the generator is still yielding results, otherwise
83
+ // it can cause unhandled promise rejections
84
+ if (!finished) {
85
+ gen.throw(err)
86
+ }
87
+ })
88
+ }
102
89
  }
103
90
 
104
91
  return {
package/src/index.ts CHANGED
@@ -5,8 +5,12 @@
5
5
  * modules such as `helia`, `@helia/http`, etc.
6
6
  */
7
7
 
8
- import { contentRoutingSymbol, peerRoutingSymbol, start, stop, TypedEventEmitter } from '@libp2p/interface'
8
+ import { keychain } from '@ipshipyard/keychain'
9
+ import { start, stop, TypedEventEmitter } from '@libp2p/interface'
9
10
  import { dns } from '@multiformats/dns'
11
+ import { defaultLogger } from 'birnam'
12
+ import { MemoryBlockstore } from 'blockstore-core'
13
+ import { MemoryDatastore } from 'datastore-core'
10
14
  import drain from 'it-drain'
11
15
  import { CustomProgressEvent } from 'progress-events'
12
16
  import { PinsImpl } from './pins.ts'
@@ -14,18 +18,18 @@ import { Routing as RoutingClass } from './routing.ts'
14
18
  import { BlockStorage } from './storage.ts'
15
19
  import { assertDatastoreVersionIsCurrent } from './utils/datastore-version.ts'
16
20
  import { getCodec } from './utils/get-codec.ts'
21
+ import { getCrypto } from './utils/get-crypto.ts'
17
22
  import { getHasher } from './utils/get-hasher.ts'
18
23
  import { NetworkedStorage } from './utils/networked-storage.ts'
19
24
  import type { BlockStorageInit } from './storage.ts'
20
- import type { CodecLoader, GCOptions, HasherLoader, Helia as HeliaInterface, HeliaEvents, Routing } from '@helia/interface'
25
+ import type { CodecLoader, GCOptions, HasherLoader, Helia as HeliaInterface, HeliaEvents, Routing, CryptoLoader, Crypto, NodeInfo, Router, HeliaMixin } from '@helia/interface'
21
26
  import type { BlockBroker } from '@helia/interface/blocks'
22
27
  import type { Pins } from '@helia/interface/pins'
23
- import type { ComponentLogger, ContentRouting, Libp2p, Logger, Metrics, PeerRouting } from '@libp2p/interface'
24
- import type { KeychainInit } from '@libp2p/keychain'
28
+ import type { Keychain, KeychainInit } from '@ipshipyard/keychain'
29
+ import type { ComponentLogger, Logger, Metrics } from '@libp2p/interface'
25
30
  import type { DNS } from '@multiformats/dns'
26
31
  import type { Blockstore } from 'interface-blockstore'
27
32
  import type { Datastore } from 'interface-datastore'
28
- import type { Libp2pOptions } from 'libp2p'
29
33
  import type { BlockCodec } from 'multiformats'
30
34
  import type { CID } from 'multiformats/cid'
31
35
  import type { MultihashHasher } from 'multiformats/hashes/interface'
@@ -33,6 +37,8 @@ import type { MultihashHasher } from 'multiformats/hashes/interface'
33
37
  export { AbstractSession } from './abstract-session.ts'
34
38
  export type { AbstractCreateSessionOptions, BlockstoreSessionEvents, AbstractSessionComponents } from './abstract-session.ts'
35
39
 
40
+ export { isCID } from './utils/is-cid.ts'
41
+
36
42
  export type { BlockStorage, BlockStorageInit }
37
43
 
38
44
  export { breadthFirstWalker, depthFirstWalker, naturalOrderWalker } from './graph-walker.ts'
@@ -41,26 +47,7 @@ export type { GraphWalkerComponents, GraphWalkerInit, GraphNode, GraphWalker } f
41
47
  /**
42
48
  * Options used to create a Helia node.
43
49
  */
44
- export interface HeliaInit<T extends Libp2p = Libp2p> {
45
- /**
46
- * A libp2p node is required to perform network operations. Either a
47
- * pre-configured node or options to configure a node can be passed
48
- * here.
49
- *
50
- * If node options are passed, they will be merged with the default
51
- * config for the current platform. In this case all passed config
52
- * keys will replace those from the default config.
53
- *
54
- * The libp2p `start` option is not supported, instead please pass `start` in
55
- * the root of the HeliaInit object.
56
- */
57
- libp2p: T | Omit<Libp2pOptions<any>, 'start'>
58
-
59
- /**
60
- * Pass `false` to not start the Helia node
61
- */
62
- start?: boolean
63
-
50
+ export interface HeliaInit {
64
51
  /**
65
52
  * By default Helia stores the node's PeerId in an encrypted form in a
66
53
  * libp2p keystore. These options control how that keystore is configured.
@@ -70,12 +57,12 @@ export interface HeliaInit<T extends Libp2p = Libp2p> {
70
57
  /**
71
58
  * The blockstore is where blocks are stored
72
59
  */
73
- blockstore: Blockstore
60
+ blockstore?: Blockstore
74
61
 
75
62
  /**
76
63
  * The datastore is where data is stored
77
64
  */
78
- datastore: Datastore
65
+ datastore?: Datastore
79
66
 
80
67
  /**
81
68
  * By default sha256, sha512 and identity hashes are supported for
@@ -104,10 +91,14 @@ export interface HeliaInit<T extends Libp2p = Libp2p> {
104
91
  loadCodec?(code: number): BlockCodec<any, any> | Promise<BlockCodec<any, any>>
105
92
 
106
93
  /**
107
- * A list of strategies used to fetch blocks when they are not present in
108
- * the local blockstore
94
+ * A list of pre-supported public/private key implementations
109
95
  */
110
- blockBrokers: Array<(components: any) => BlockBroker>
96
+ cryptos?: Array<Crypto>
97
+
98
+ /**
99
+ * Dynamically load a cryptography implementation
100
+ */
101
+ loadCrypto?: CryptoLoader
111
102
 
112
103
  /**
113
104
  * Garbage collection requires preventing blockstore writes during searches
@@ -131,11 +122,17 @@ export interface HeliaInit<T extends Libp2p = Libp2p> {
131
122
  */
132
123
  logger?: ComponentLogger
133
124
 
125
+ /**
126
+ * A list of strategies used to fetch blocks when they are not present in
127
+ * the local blockstore
128
+ */
129
+ blockBrokers?: Array<BlockBroker | ((components: any) => BlockBroker)>
130
+
134
131
  /**
135
132
  * Routers perform operations such as looking up content providers,
136
133
  * information about network peers or getting/putting records.
137
134
  */
138
- routers?: Array<Partial<Routing> | ((components: any) => Partial<Routing>)>
135
+ routers?: Array<Router | ((components: any) => Router)>
139
136
 
140
137
  /**
141
138
  * During provider lookups, peers can be returned from routing implementations
@@ -180,58 +177,72 @@ export interface HeliaInit<T extends Libp2p = Libp2p> {
180
177
  }
181
178
 
182
179
  interface Components {
183
- libp2p: Libp2p
184
180
  blockstore: Blockstore
185
181
  datastore: Datastore
186
182
  logger: ComponentLogger
187
183
  blockBrokers: BlockBroker[]
188
184
  routing: Routing
189
185
  dns: DNS
186
+ keychain: Keychain
190
187
  metrics?: Metrics
191
188
  getCodec: CodecLoader
192
189
  getHasher: HasherLoader
190
+ getCrypto: CryptoLoader
193
191
  }
194
192
 
195
- export class Helia<T extends Libp2p> implements HeliaInterface<T> {
196
- public libp2p: T
193
+ export class Helia implements HeliaInterface {
194
+ public info: NodeInfo
197
195
  public blockstore: BlockStorage
198
196
  public datastore: Datastore
199
- public events: TypedEventEmitter<HeliaEvents<T>>
197
+ public events: TypedEventEmitter<HeliaEvents<this>>
200
198
  public pins: Pins
201
199
  public logger: ComponentLogger
202
- public routing: Routing
200
+ public routing: RoutingClass
203
201
  public getCodec: CodecLoader
204
202
  public getHasher: HasherLoader
203
+ public getCrypto: CryptoLoader
205
204
  public dns: DNS
205
+ public keychain: Keychain
206
206
  public metrics?: Metrics
207
+ public status: 'stopped' | 'stopping' | 'starting' | 'started'
207
208
  private readonly log: Logger
209
+ private readonly blockBrokers: BlockBroker[]
210
+ private readonly mixins: HeliaMixin[]
208
211
 
209
- constructor (init: Omit<HeliaInit, 'start' | 'libp2p'> & { libp2p: T }) {
210
- this.logger = init.logger ?? init.libp2p.logger
212
+ constructor (init: HeliaInit & { name: string, version: string }) {
213
+ this.info = {
214
+ name: init.name,
215
+ version: init.version
216
+ }
217
+ this.logger = init.logger ?? defaultLogger()
211
218
  this.log = this.logger.forComponent('helia')
212
219
  this.getHasher = getHasher(init.hashers, init.loadHasher)
213
220
  this.getCodec = getCodec(init.codecs, init.loadCodec)
221
+ this.getCrypto = getCrypto(init.cryptos, init.loadCrypto)
214
222
  this.dns = init.dns ?? dns()
215
223
  this.metrics = init.metrics
216
- this.libp2p = init.libp2p
217
- this.events = new TypedEventEmitter<HeliaEvents<T>>()
224
+ this.events = new TypedEventEmitter<HeliaEvents<typeof this>>()
225
+ this.status = 'stopped'
226
+ this.mixins = []
218
227
 
219
- // @ts-expect-error routing is not set
228
+ // @ts-expect-error routing and keychain are not set
220
229
  const components: Components = {
221
- blockstore: init.blockstore,
222
- datastore: init.datastore,
230
+ blockstore: init.blockstore ?? new MemoryBlockstore(),
231
+ datastore: init.datastore ?? new MemoryDatastore(),
223
232
  logger: this.logger,
224
- libp2p: this.libp2p,
225
233
  blockBrokers: [],
226
234
  getHasher: this.getHasher,
227
235
  getCodec: this.getCodec,
236
+ getCrypto: this.getCrypto,
228
237
  dns: this.dns,
229
238
  metrics: this.metrics,
230
239
  ...(init.components ?? {})
231
240
  }
232
241
 
242
+ this.keychain = components.keychain = keychain()(components)
243
+
233
244
  this.routing = components.routing = new RoutingClass(components, {
234
- routers: (init.routers ?? []).flatMap((router: Partial<Routing> | ((components: any) => Partial<Routing>)) => {
245
+ routers: (init.routers ?? []).flatMap((router: Router | ((components: any) => Router)) => {
235
246
  if (typeof router === 'function') {
236
247
  router = router(components)
237
248
  }
@@ -241,54 +252,86 @@ export class Helia<T extends Libp2p> implements HeliaInterface<T> {
241
252
  router
242
253
  ]
243
254
 
244
- // if the router provides a libp2p-style ContentRouter
245
- const contentRouting = asContentRouting(router)
246
- if (contentRouting != null) {
247
- routers.push(contentRouting)
248
- }
249
-
250
- // if the router provides a libp2p-style PeerRouter
251
- const peerRouting = asPeerRouting(router)
252
- if (peerRouting != null) {
253
- routers.push(peerRouting)
254
- }
255
-
256
255
  return routers
257
256
  }),
258
257
  providerLookupConcurrency: init.providerLookupConcurrency
259
258
  })
260
259
 
261
- components.blockBrokers = init.blockBrokers.map((fn) => {
262
- return fn(components)
260
+ this.blockBrokers = components.blockBrokers = (init.blockBrokers ?? []).map((broker) => {
261
+ if (typeof broker === 'function') {
262
+ broker = broker(components)
263
+ }
264
+
265
+ return broker
263
266
  })
264
267
 
265
268
  const networkedStorage = new NetworkedStorage(components, init)
266
- this.pins = new PinsImpl(init.datastore, networkedStorage, this.getCodec)
269
+ this.pins = new PinsImpl(components.datastore, networkedStorage, this.getCodec)
267
270
  this.blockstore = new BlockStorage(networkedStorage, this.pins, this.routing, {
268
271
  holdGcLock: init.holdGcLock ?? true
269
272
  })
270
- this.datastore = init.datastore
273
+ this.datastore = components.datastore
274
+ }
275
+
276
+ hasRouter (name: string): boolean {
277
+ return this.routing.hasRouter(name)
271
278
  }
272
279
 
273
- async start (): Promise<void> {
280
+ addRouter (router: Router): void {
281
+ this.routing.addRouter(router)
282
+ }
283
+
284
+ hasBlockBroker (name: string): boolean {
285
+ return this.blockBrokers.findIndex(b => b.name === name) !== -1
286
+ }
287
+
288
+ addBlockBroker (blockBroker: BlockBroker): void {
289
+ this.blockBrokers.push(blockBroker)
290
+ }
291
+
292
+ addMixin (mixin: HeliaMixin): void {
293
+ this.mixins.push(mixin)
294
+ }
295
+
296
+ async start (): Promise<this> {
297
+ this.status = 'starting'
298
+
274
299
  await assertDatastoreVersionIsCurrent(this.datastore)
275
300
  await start(
276
301
  this.blockstore,
277
302
  this.datastore,
278
303
  this.routing,
279
- this.libp2p
304
+ ...this.blockBrokers
280
305
  )
306
+
307
+ for (const mixin of this.mixins) {
308
+ await mixin.start?.(this)
309
+ }
310
+
311
+ this.status = 'started'
281
312
  this.events.dispatchEvent(new CustomEvent('start', { detail: this }))
313
+
314
+ return this
282
315
  }
283
316
 
284
- async stop (): Promise<void> {
317
+ async stop (): Promise<this> {
318
+ this.status = 'stopping'
319
+
320
+ for (const mixin of this.mixins) {
321
+ await mixin.stop?.(this)
322
+ }
323
+
285
324
  await stop(
286
325
  this.blockstore,
287
326
  this.datastore,
288
327
  this.routing,
289
- this.libp2p
328
+ ...this.blockBrokers
290
329
  )
330
+
331
+ this.status = 'stopped'
291
332
  this.events.dispatchEvent(new CustomEvent('stop', { detail: this }))
333
+
334
+ return this
292
335
  }
293
336
 
294
337
  async gc (options: GCOptions = {}): Promise<void> {
@@ -323,11 +366,3 @@ export class Helia<T extends Libp2p> implements HeliaInterface<T> {
323
366
  this.log('gc finished')
324
367
  }
325
368
  }
326
-
327
- function asContentRouting (obj?: any): ContentRouting | undefined {
328
- return obj?.[contentRoutingSymbol]
329
- }
330
-
331
- function asPeerRouting (obj?: any): PeerRouting | undefined {
332
- return obj?.[peerRoutingSymbol]
333
- }
package/src/pins.ts CHANGED
@@ -1,22 +1,18 @@
1
1
  import { InvalidParametersError } from '@libp2p/interface'
2
- import { Queue } from '@libp2p/utils'
3
2
  import * as cborg from 'cborg'
4
3
  import { Key } from 'interface-datastore'
5
- import toBuffer from 'it-to-buffer'
6
4
  import { base36 } from 'multiformats/bases/base36'
7
- import { createUnsafe } from 'multiformats/block'
8
5
  import { CID } from 'multiformats/cid'
9
6
  import { CustomProgressEvent } from 'progress-events'
10
7
  import { equals as uint8ArrayEquals } from 'uint8arrays/equals'
11
8
  import { AlreadyPinnedError } from './errors.ts'
9
+ import { depthFirstWalker } from './graph-walker.ts'
12
10
  import type { CodecLoader } from '@helia/interface'
13
- import type { GetBlockProgressEvents } from '@helia/interface/blocks'
14
- import type { AddOptions, AddPinEvents, IsPinnedOptions, LsOptions, Pin, Pins, RmOptions } from '@helia/interface/pins'
11
+ import type { AddOptions, IsPinnedOptions, LsOptions, Pin, Pins, RmOptions } from '@helia/interface/pins'
15
12
  import type { AbortOptions } from '@libp2p/interface'
16
13
  import type { Blockstore } from 'interface-blockstore'
17
14
  import type { Datastore } from 'interface-datastore'
18
15
  import type { Version } from 'multiformats/cid'
19
- import type { ProgressOptions } from 'progress-events'
20
16
 
21
17
  interface DatastorePin {
22
18
  /**
@@ -49,11 +45,6 @@ interface WithPinnedBlockCallback {
49
45
  const DATASTORE_PIN_PREFIX = '/pin/'
50
46
  const DATASTORE_BLOCK_PREFIX = '/pinned-block/'
51
47
  const DATASTORE_ENCODING = base36
52
- const DAG_WALK_QUEUE_CONCURRENCY = 1
53
-
54
- interface WalkDagOptions extends AbortOptions, ProgressOptions<GetBlockProgressEvents | AddPinEvents> {
55
- depth: number
56
- }
57
48
 
58
49
  function toDSKey (cid: CID): Key {
59
50
  if (cid.version === 0) {
@@ -87,15 +78,13 @@ export class PinsImpl implements Pins {
87
78
  throw new InvalidParametersError('Depth must be greater than or equal to 0')
88
79
  }
89
80
 
90
- // use a queue to walk the DAG instead of recursion so we can traverse very large DAGs
91
- const queue = new Queue<AsyncGenerator<CID>>({
92
- concurrency: DAG_WALK_QUEUE_CONCURRENCY
81
+ const walker = (options.walker ?? depthFirstWalker())({
82
+ blockstore: this.blockstore,
83
+ getCodec: this.getCodec
93
84
  })
94
85
 
95
- for await (const childCid of this.#walkDag(cid, queue, {
96
- ...options,
97
- depth
98
- })) {
86
+ for await (const node of walker.walk(cid, { ...options, depth })) {
87
+ const childCid = node.block.cid
99
88
  await this.#updatePinnedBlock(childCid, (pinnedBlock: DatastorePinnedBlock) => {
100
89
  // do not update pinned block if this block is already pinned by this CID
101
90
  if (pinnedBlock.pinnedBy.find(c => uint8ArrayEquals(c, cid.bytes)) != null) {
@@ -118,31 +107,6 @@ export class PinsImpl implements Pins {
118
107
  await this.datastore.put(pinKey, cborg.encode(pin), options)
119
108
  }
120
109
 
121
- /**
122
- * Walk a DAG in an iterable fashion
123
- */
124
- async * #walkDag (cid: CID, queue: Queue<AsyncGenerator<CID>>, options: WalkDagOptions): AsyncGenerator<CID> {
125
- if (options.depth === -1) {
126
- return
127
- }
128
-
129
- const codec = await this.getCodec(cid.code)
130
- const bytes = await toBuffer(this.blockstore.get(cid, options))
131
- const block = createUnsafe({ bytes, cid, codec })
132
-
133
- yield cid
134
-
135
- // walk dag, ensure all blocks are present
136
- for (const [,cid] of block.links()) {
137
- yield * await queue.add(async () => {
138
- return this.#walkDag(cid, queue, {
139
- ...options,
140
- depth: options.depth - 1
141
- })
142
- })
143
- }
144
- }
145
-
146
110
  /**
147
111
  * Update the pin count for the CID
148
112
  */
@@ -186,15 +150,13 @@ export class PinsImpl implements Pins {
186
150
 
187
151
  await this.datastore.delete(pinKey, options)
188
152
 
189
- // use a queue to walk the DAG instead of recursion so we can traverse very large DAGs
190
- const queue = new Queue<AsyncGenerator<CID>>({
191
- concurrency: DAG_WALK_QUEUE_CONCURRENCY
153
+ const walker = (options.walker ?? depthFirstWalker())({
154
+ blockstore: this.blockstore,
155
+ getCodec: this.getCodec
192
156
  })
193
157
 
194
- for await (const childCid of this.#walkDag(cid, queue, {
195
- ...options,
196
- depth: pin.depth
197
- })) {
158
+ for await (const node of walker.walk(cid, { ...options, depth: pin.depth })) {
159
+ const childCid = node.block.cid
198
160
  await this.#updatePinnedBlock(childCid, (pinnedBlock): boolean => {
199
161
  pinnedBlock.pinCount--
200
162
  pinnedBlock.pinnedBy = pinnedBlock.pinnedBy.filter(c => uint8ArrayEquals(c, cid.bytes))
package/src/routing.ts CHANGED
@@ -1,18 +1,19 @@
1
1
  import { NoRoutersAvailableError } from '@helia/interface'
2
2
  import { NotFoundError, start, stop } from '@libp2p/interface'
3
- import { PeerQueue } from '@libp2p/utils'
3
+ import { Queue } from '@libp2p/utils'
4
4
  import merge from 'it-merge'
5
5
  import { CustomProgressEvent } from 'progress-events'
6
6
  import { toString as uint8ArrayToString } from 'uint8arrays/to-string'
7
7
  import { GetFailedError } from './errors.ts'
8
- import type { Routing as RoutingInterface, Provider, RoutingOptions, RoutingFindProvidersProgressEvents, RoutingProvideProgressEvents, RoutingPutProgressEvents, RoutingGetProgressEvents, RoutingFindPeerProgressEvents, RoutingGetClosestPeersProgressEvents, RoutingCancelReprovideProgressEvents } from '@helia/interface'
9
- import type { ComponentLogger, Logger, Metrics, PeerId, PeerInfo, Startable } from '@libp2p/interface'
8
+ import type { Routing as RoutingInterface, Provider, RoutingOptions, RoutingFindProvidersProgressEvents, RoutingProvideProgressEvents, RoutingPutProgressEvents, RoutingGetProgressEvents, RoutingFindPeerProgressEvents, RoutingGetClosestPeersProgressEvents, RoutingCancelReprovideProgressEvents, Router, Peer } from '@helia/interface'
9
+ import type { ComponentLogger, Logger, Metrics, Startable } from '@libp2p/interface'
10
+ import type { AbortOptions } from 'abort-error'
10
11
  import type { CID } from 'multiformats/cid'
11
12
 
12
13
  const DEFAULT_PROVIDER_LOOKUP_CONCURRENCY = 5
13
14
 
14
15
  export interface RoutingInit {
15
- routers: Array<Partial<RoutingInterface>>
16
+ routers: Router[]
16
17
  providerLookupConcurrency?: number
17
18
  }
18
19
 
@@ -21,11 +22,15 @@ export interface RoutingComponents {
21
22
  metrics?: Metrics
22
23
  }
23
24
 
25
+ interface PeerQueueOptions extends AbortOptions {
26
+ peer: CID
27
+ }
28
+
24
29
  export class Routing implements RoutingInterface, Startable {
25
30
  public name: string
26
31
 
27
32
  private readonly log: Logger
28
- private readonly routers: Array<Partial<RoutingInterface>>
33
+ private readonly routers: Router[]
29
34
  private readonly providerLookupConcurrency: number
30
35
 
31
36
  constructor (components: RoutingComponents, init: RoutingInit) {
@@ -65,6 +70,14 @@ export class Routing implements RoutingInterface, Startable {
65
70
  await stop(...this.routers)
66
71
  }
67
72
 
73
+ hasRouter (name: string): boolean {
74
+ return this.routers.findIndex(r => r.name === name) !== -1
75
+ }
76
+
77
+ addRouter (router: Router): void {
78
+ this.routers.push(router)
79
+ }
80
+
68
81
  /**
69
82
  * Iterates over all content routers in parallel to find providers of the
70
83
  * given key
@@ -78,7 +91,7 @@ export class Routing implements RoutingInterface, Startable {
78
91
  // back as an empty array - when this happens we have to do a FIND_PEER
79
92
  // query to get updated addresses, but we shouldn't block on this so use a
80
93
  // separate bounded queue to perform this lookup
81
- const queue = new PeerQueue<Provider | null>({
94
+ const queue = new Queue<Provider | null, PeerQueueOptions>({
82
95
  concurrency: this.providerLookupConcurrency
83
96
  })
84
97
 
@@ -145,7 +158,7 @@ export class Routing implements RoutingInterface, Startable {
145
158
  // have to refresh peer info for this peer to get updated multiaddrs
146
159
  if (peer.multiaddrs.length === 0) {
147
160
  // already looking this peer up
148
- if (queue.find(peer.id) != null) {
161
+ if (queue.queue.find(job => job.options.peer.equals(peer.id)) != null) {
149
162
  continue
150
163
  }
151
164
 
@@ -167,7 +180,7 @@ export class Routing implements RoutingInterface, Startable {
167
180
  return null
168
181
  }
169
182
  }, {
170
- peerId: peer.id,
183
+ peer: peer.id,
171
184
  signal: options.signal
172
185
  })
173
186
  .catch(err => {
@@ -298,7 +311,7 @@ export class Routing implements RoutingInterface, Startable {
298
311
  /**
299
312
  * Iterates over all peer routers in parallel to find the given peer
300
313
  */
301
- async findPeer (id: PeerId, options?: RoutingOptions<RoutingFindPeerProgressEvents>): Promise<PeerInfo> {
314
+ async findPeer (id: CID, options?: RoutingOptions<RoutingFindPeerProgressEvents>): Promise<Peer> {
302
315
  if (this.routers.length === 0) {
303
316
  throw new NoRoutersAvailableError('No peer routers available')
304
317
  }
@@ -339,7 +352,7 @@ export class Routing implements RoutingInterface, Startable {
339
352
  /**
340
353
  * Attempt to find the closest peers on the network to the given key
341
354
  */
342
- async * getClosestPeers (key: Uint8Array, options: RoutingOptions<RoutingGetClosestPeersProgressEvents> = {}): AsyncIterable<PeerInfo> {
355
+ async * getClosestPeers (key: Uint8Array, options: RoutingOptions<RoutingGetClosestPeersProgressEvents> = {}): AsyncIterable<Peer> {
343
356
  if (this.routers.length === 0) {
344
357
  throw new NoRoutersAvailableError('No peer routers available')
345
358
  }
@@ -0,0 +1,3 @@
1
+ export const SALT_LENGTH = 16
2
+ export const KEY_SIZE = 32
3
+ export const ITERATIONS = 10000
@@ -1,5 +1,3 @@
1
- /* eslint max-depth: ["error", 7] */
2
-
3
1
  import { UnknownCodecError } from '@helia/interface'
4
2
  import * as dagCbor from '@ipld/dag-cbor'
5
3
  import * as dagJson from '@ipld/dag-json'
@@ -7,9 +5,10 @@ import * as dagPb from '@ipld/dag-pb'
7
5
  import * as json from 'multiformats/codecs/json'
8
6
  import * as raw from 'multiformats/codecs/raw'
9
7
  import { isPromise } from './is-promise.ts'
8
+ import type { CodecLoader } from '@helia/interface'
10
9
  import type { BlockCodec } from 'multiformats/codecs/interface'
11
10
 
12
- export function getCodec <T = any, Code extends number = any> (initialCodecs: Array<BlockCodec<any, any>> = [], loadCodec?: (code: number) => BlockCodec<any, any> | Promise<BlockCodec<any, any>>): (code: Code) => BlockCodec<Code, T> | Promise<BlockCodec<Code, T>> {
11
+ export function getCodec (initialCodecs: Array<BlockCodec<any, any>> = [], loadCodec?: CodecLoader): CodecLoader {
13
12
  const codecs: Record<number, BlockCodec<any, any>> = {
14
13
  [dagPb.code]: dagPb,
15
14
  [raw.code]: raw,