@helia/utils 2.5.2-6f8165b5 → 2.5.2-9114743f
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.min.js +22 -1
- package/dist/index.min.js.map +4 -4
- package/dist/src/abstract-session.d.ts +3 -3
- package/dist/src/abstract-session.d.ts.map +1 -1
- package/dist/src/abstract-session.js.map +1 -1
- package/dist/src/errors.d.ts +4 -0
- package/dist/src/errors.d.ts.map +1 -1
- package/dist/src/errors.js +4 -0
- package/dist/src/errors.js.map +1 -1
- package/dist/src/graph-walker.d.ts +3 -21
- package/dist/src/graph-walker.d.ts.map +1 -1
- package/dist/src/graph-walker.js +20 -17
- package/dist/src/graph-walker.js.map +1 -1
- package/dist/src/index.d.ts +39 -36
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +62 -30
- package/dist/src/index.js.map +1 -1
- package/dist/src/pins.d.ts.map +1 -1
- package/dist/src/pins.js +11 -39
- package/dist/src/pins.js.map +1 -1
- package/dist/src/routing.d.ts +7 -5
- package/dist/src/routing.d.ts.map +1 -1
- package/dist/src/routing.js +10 -4
- package/dist/src/routing.js.map +1 -1
- package/dist/src/utils/constants.d.ts +4 -0
- package/dist/src/utils/constants.d.ts.map +1 -0
- package/dist/src/utils/constants.js +4 -0
- package/dist/src/utils/constants.js.map +1 -0
- package/dist/src/utils/get-codec.d.ts +2 -1
- package/dist/src/utils/get-codec.d.ts.map +1 -1
- package/dist/src/utils/get-codec.js +0 -1
- package/dist/src/utils/get-codec.js.map +1 -1
- package/dist/src/utils/get-crypto.d.ts +4 -0
- package/dist/src/utils/get-crypto.d.ts.map +1 -0
- package/dist/src/utils/get-crypto.js +35 -0
- package/dist/src/utils/get-crypto.js.map +1 -0
- package/dist/src/utils/get-hasher.d.ts +2 -1
- package/dist/src/utils/get-hasher.d.ts.map +1 -1
- package/dist/src/utils/get-hasher.js.map +1 -1
- package/dist/src/utils/is-cid.d.ts +3 -0
- package/dist/src/utils/is-cid.d.ts.map +1 -0
- package/dist/src/utils/is-cid.js +8 -0
- package/dist/src/utils/is-cid.js.map +1 -0
- package/dist/src/utils/networked-storage.d.ts +2 -1
- package/dist/src/utils/networked-storage.d.ts.map +1 -1
- package/dist/src/utils/networked-storage.js.map +1 -1
- package/dist/src/utils/session-storage.d.ts +2 -2
- package/dist/src/utils/session-storage.d.ts.map +1 -1
- package/dist/src/utils/session-storage.js.map +1 -1
- package/package.json +21 -22
- package/src/abstract-session.ts +4 -4
- package/src/errors.ts +5 -0
- package/src/graph-walker.ts +30 -43
- package/src/index.ts +108 -73
- package/src/pins.ts +12 -50
- package/src/routing.ts +23 -10
- package/src/utils/constants.ts +3 -0
- package/src/utils/get-codec.ts +2 -3
- package/src/utils/get-crypto.ts +44 -0
- package/src/utils/get-hasher.ts +2 -1
- package/src/utils/is-cid.ts +9 -0
- package/src/utils/networked-storage.ts +2 -1
- package/src/utils/session-storage.ts +2 -2
package/src/errors.ts
CHANGED
|
@@ -32,3 +32,8 @@ export class BlockNotFoundWhileOfflineError extends Error {
|
|
|
32
32
|
static name = 'BlockNotFoundWhileOfflineError'
|
|
33
33
|
name = 'BlockNotFoundWhileOfflineError'
|
|
34
34
|
}
|
|
35
|
+
|
|
36
|
+
export class DecryptionFailedError extends Error {
|
|
37
|
+
static name = 'DecryptionFailedError'
|
|
38
|
+
name = 'DecryptionFailedError'
|
|
39
|
+
}
|
package/src/graph-walker.ts
CHANGED
|
@@ -2,29 +2,17 @@ import { Queue } from '@libp2p/utils'
|
|
|
2
2
|
import filter from 'it-filter'
|
|
3
3
|
import toBuffer from 'it-to-buffer'
|
|
4
4
|
import { createUnsafe } from 'multiformats/block'
|
|
5
|
-
import type {
|
|
5
|
+
import type {
|
|
6
|
+
GraphWalker,
|
|
7
|
+
GraphWalkerComponents,
|
|
8
|
+
GraphWalkerInit,
|
|
9
|
+
GraphNode,
|
|
10
|
+
WalkOptions
|
|
11
|
+
} from '@helia/interface'
|
|
6
12
|
import type { AbortOptions } from '@libp2p/interface'
|
|
7
|
-
import type {
|
|
8
|
-
import type { BlockView, CID, Version } from 'multiformats'
|
|
13
|
+
import type { CID } from 'multiformats'
|
|
9
14
|
|
|
10
|
-
export
|
|
11
|
-
blockstore: Blockstore
|
|
12
|
-
getCodec: CodecLoader
|
|
13
|
-
}
|
|
14
|
-
|
|
15
|
-
export interface GraphWalkerInit {
|
|
16
|
-
|
|
17
|
-
}
|
|
18
|
-
|
|
19
|
-
export interface GraphNode <T = unknown, C extends number = number, A extends number = number, V extends Version = 0 | 1> {
|
|
20
|
-
block: BlockView<T, C, A, V>
|
|
21
|
-
depth: number
|
|
22
|
-
path: CID[]
|
|
23
|
-
}
|
|
24
|
-
|
|
25
|
-
export interface GraphWalker {
|
|
26
|
-
walk <T = any> (cid: CID, options?: WalkOptions<T>): AsyncGenerator<GraphNode<T>>
|
|
27
|
-
}
|
|
15
|
+
export type { GraphWalker, GraphWalkerComponents, GraphWalkerInit, GraphNode, WalkOptions }
|
|
28
16
|
|
|
29
17
|
/**
|
|
30
18
|
* A depth-first walker descends into child blocks before processing successor
|
|
@@ -55,10 +43,6 @@ interface JobOptions extends AbortOptions {
|
|
|
55
43
|
path: CID[]
|
|
56
44
|
}
|
|
57
45
|
|
|
58
|
-
export interface WalkOptions<T> extends AbortOptions {
|
|
59
|
-
includeChild?(child: CID, parent: BlockView<T, number, number, 0 | 1>): boolean
|
|
60
|
-
}
|
|
61
|
-
|
|
62
46
|
abstract class AbstractGraphWalker {
|
|
63
47
|
private readonly components: GraphWalkerComponents
|
|
64
48
|
|
|
@@ -70,6 +54,7 @@ abstract class AbstractGraphWalker {
|
|
|
70
54
|
const queue = this.getQueue()
|
|
71
55
|
const gen = filter(queue.toGenerator(options), (node) => node != null) as AsyncGenerator<GraphNode<T>>
|
|
72
56
|
let finished = false
|
|
57
|
+
const maxDepth = options?.depth ?? Infinity
|
|
73
58
|
|
|
74
59
|
const job = async (opts: JobOptions): Promise<GraphNode<T> | undefined> => {
|
|
75
60
|
const cid = opts.cid
|
|
@@ -80,25 +65,27 @@ abstract class AbstractGraphWalker {
|
|
|
80
65
|
codec: await this.components.getCodec(cid.code)
|
|
81
66
|
})
|
|
82
67
|
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
// eslint-disable-next-line no-loop-func
|
|
95
|
-
.catch(err => {
|
|
96
|
-
// only throw if the generator is still yielding results, otherwise
|
|
97
|
-
// it can cause unhandled promise rejections
|
|
98
|
-
if (!finished) {
|
|
99
|
-
gen.throw(err)
|
|
100
|
-
}
|
|
68
|
+
if (opts.depth < maxDepth) {
|
|
69
|
+
for (const [, linkedCid] of block.links()) {
|
|
70
|
+
if (options?.includeChild?.(linkedCid, block) === false) {
|
|
71
|
+
continue
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
queue.add(job, {
|
|
75
|
+
...opts,
|
|
76
|
+
cid: linkedCid,
|
|
77
|
+
depth: opts.depth + 1,
|
|
78
|
+
path: [...opts.path, linkedCid]
|
|
101
79
|
})
|
|
80
|
+
// eslint-disable-next-line no-loop-func
|
|
81
|
+
.catch(err => {
|
|
82
|
+
// only throw if the generator is still yielding results, otherwise
|
|
83
|
+
// it can cause unhandled promise rejections
|
|
84
|
+
if (!finished) {
|
|
85
|
+
gen.throw(err)
|
|
86
|
+
}
|
|
87
|
+
})
|
|
88
|
+
}
|
|
102
89
|
}
|
|
103
90
|
|
|
104
91
|
return {
|
package/src/index.ts
CHANGED
|
@@ -5,8 +5,12 @@
|
|
|
5
5
|
* modules such as `helia`, `@helia/http`, etc.
|
|
6
6
|
*/
|
|
7
7
|
|
|
8
|
-
import {
|
|
8
|
+
import { keychain } from '@ipshipyard/keychain'
|
|
9
|
+
import { start, stop, TypedEventEmitter } from '@libp2p/interface'
|
|
9
10
|
import { dns } from '@multiformats/dns'
|
|
11
|
+
import { defaultLogger } from 'birnam'
|
|
12
|
+
import { MemoryBlockstore } from 'blockstore-core'
|
|
13
|
+
import { MemoryDatastore } from 'datastore-core'
|
|
10
14
|
import drain from 'it-drain'
|
|
11
15
|
import { CustomProgressEvent } from 'progress-events'
|
|
12
16
|
import { PinsImpl } from './pins.ts'
|
|
@@ -14,18 +18,18 @@ import { Routing as RoutingClass } from './routing.ts'
|
|
|
14
18
|
import { BlockStorage } from './storage.ts'
|
|
15
19
|
import { assertDatastoreVersionIsCurrent } from './utils/datastore-version.ts'
|
|
16
20
|
import { getCodec } from './utils/get-codec.ts'
|
|
21
|
+
import { getCrypto } from './utils/get-crypto.ts'
|
|
17
22
|
import { getHasher } from './utils/get-hasher.ts'
|
|
18
23
|
import { NetworkedStorage } from './utils/networked-storage.ts'
|
|
19
24
|
import type { BlockStorageInit } from './storage.ts'
|
|
20
|
-
import type { CodecLoader, GCOptions, HasherLoader, Helia as HeliaInterface, HeliaEvents, Routing } from '@helia/interface'
|
|
25
|
+
import type { CodecLoader, GCOptions, HasherLoader, Helia as HeliaInterface, HeliaEvents, Routing, CryptoLoader, Crypto, NodeInfo, Router, HeliaMixin } from '@helia/interface'
|
|
21
26
|
import type { BlockBroker } from '@helia/interface/blocks'
|
|
22
27
|
import type { Pins } from '@helia/interface/pins'
|
|
23
|
-
import type {
|
|
24
|
-
import type {
|
|
28
|
+
import type { Keychain, KeychainInit } from '@ipshipyard/keychain'
|
|
29
|
+
import type { ComponentLogger, Logger, Metrics } from '@libp2p/interface'
|
|
25
30
|
import type { DNS } from '@multiformats/dns'
|
|
26
31
|
import type { Blockstore } from 'interface-blockstore'
|
|
27
32
|
import type { Datastore } from 'interface-datastore'
|
|
28
|
-
import type { Libp2pOptions } from 'libp2p'
|
|
29
33
|
import type { BlockCodec } from 'multiformats'
|
|
30
34
|
import type { CID } from 'multiformats/cid'
|
|
31
35
|
import type { MultihashHasher } from 'multiformats/hashes/interface'
|
|
@@ -33,6 +37,8 @@ import type { MultihashHasher } from 'multiformats/hashes/interface'
|
|
|
33
37
|
export { AbstractSession } from './abstract-session.ts'
|
|
34
38
|
export type { AbstractCreateSessionOptions, BlockstoreSessionEvents, AbstractSessionComponents } from './abstract-session.ts'
|
|
35
39
|
|
|
40
|
+
export { isCID } from './utils/is-cid.ts'
|
|
41
|
+
|
|
36
42
|
export type { BlockStorage, BlockStorageInit }
|
|
37
43
|
|
|
38
44
|
export { breadthFirstWalker, depthFirstWalker, naturalOrderWalker } from './graph-walker.ts'
|
|
@@ -41,26 +47,7 @@ export type { GraphWalkerComponents, GraphWalkerInit, GraphNode, GraphWalker } f
|
|
|
41
47
|
/**
|
|
42
48
|
* Options used to create a Helia node.
|
|
43
49
|
*/
|
|
44
|
-
export interface HeliaInit
|
|
45
|
-
/**
|
|
46
|
-
* A libp2p node is required to perform network operations. Either a
|
|
47
|
-
* pre-configured node or options to configure a node can be passed
|
|
48
|
-
* here.
|
|
49
|
-
*
|
|
50
|
-
* If node options are passed, they will be merged with the default
|
|
51
|
-
* config for the current platform. In this case all passed config
|
|
52
|
-
* keys will replace those from the default config.
|
|
53
|
-
*
|
|
54
|
-
* The libp2p `start` option is not supported, instead please pass `start` in
|
|
55
|
-
* the root of the HeliaInit object.
|
|
56
|
-
*/
|
|
57
|
-
libp2p: T | Omit<Libp2pOptions<any>, 'start'>
|
|
58
|
-
|
|
59
|
-
/**
|
|
60
|
-
* Pass `false` to not start the Helia node
|
|
61
|
-
*/
|
|
62
|
-
start?: boolean
|
|
63
|
-
|
|
50
|
+
export interface HeliaInit {
|
|
64
51
|
/**
|
|
65
52
|
* By default Helia stores the node's PeerId in an encrypted form in a
|
|
66
53
|
* libp2p keystore. These options control how that keystore is configured.
|
|
@@ -70,12 +57,12 @@ export interface HeliaInit<T extends Libp2p = Libp2p> {
|
|
|
70
57
|
/**
|
|
71
58
|
* The blockstore is where blocks are stored
|
|
72
59
|
*/
|
|
73
|
-
blockstore
|
|
60
|
+
blockstore?: Blockstore
|
|
74
61
|
|
|
75
62
|
/**
|
|
76
63
|
* The datastore is where data is stored
|
|
77
64
|
*/
|
|
78
|
-
datastore
|
|
65
|
+
datastore?: Datastore
|
|
79
66
|
|
|
80
67
|
/**
|
|
81
68
|
* By default sha256, sha512 and identity hashes are supported for
|
|
@@ -104,10 +91,14 @@ export interface HeliaInit<T extends Libp2p = Libp2p> {
|
|
|
104
91
|
loadCodec?(code: number): BlockCodec<any, any> | Promise<BlockCodec<any, any>>
|
|
105
92
|
|
|
106
93
|
/**
|
|
107
|
-
* A list of
|
|
108
|
-
* the local blockstore
|
|
94
|
+
* A list of pre-supported public/private key implementations
|
|
109
95
|
*/
|
|
110
|
-
|
|
96
|
+
cryptos?: Array<Crypto>
|
|
97
|
+
|
|
98
|
+
/**
|
|
99
|
+
* Dynamically load a cryptography implementation
|
|
100
|
+
*/
|
|
101
|
+
loadCrypto?: CryptoLoader
|
|
111
102
|
|
|
112
103
|
/**
|
|
113
104
|
* Garbage collection requires preventing blockstore writes during searches
|
|
@@ -131,11 +122,17 @@ export interface HeliaInit<T extends Libp2p = Libp2p> {
|
|
|
131
122
|
*/
|
|
132
123
|
logger?: ComponentLogger
|
|
133
124
|
|
|
125
|
+
/**
|
|
126
|
+
* A list of strategies used to fetch blocks when they are not present in
|
|
127
|
+
* the local blockstore
|
|
128
|
+
*/
|
|
129
|
+
blockBrokers?: Array<BlockBroker | ((components: any) => BlockBroker)>
|
|
130
|
+
|
|
134
131
|
/**
|
|
135
132
|
* Routers perform operations such as looking up content providers,
|
|
136
133
|
* information about network peers or getting/putting records.
|
|
137
134
|
*/
|
|
138
|
-
routers?: Array<
|
|
135
|
+
routers?: Array<Router | ((components: any) => Router)>
|
|
139
136
|
|
|
140
137
|
/**
|
|
141
138
|
* During provider lookups, peers can be returned from routing implementations
|
|
@@ -180,58 +177,72 @@ export interface HeliaInit<T extends Libp2p = Libp2p> {
|
|
|
180
177
|
}
|
|
181
178
|
|
|
182
179
|
interface Components {
|
|
183
|
-
libp2p: Libp2p
|
|
184
180
|
blockstore: Blockstore
|
|
185
181
|
datastore: Datastore
|
|
186
182
|
logger: ComponentLogger
|
|
187
183
|
blockBrokers: BlockBroker[]
|
|
188
184
|
routing: Routing
|
|
189
185
|
dns: DNS
|
|
186
|
+
keychain: Keychain
|
|
190
187
|
metrics?: Metrics
|
|
191
188
|
getCodec: CodecLoader
|
|
192
189
|
getHasher: HasherLoader
|
|
190
|
+
getCrypto: CryptoLoader
|
|
193
191
|
}
|
|
194
192
|
|
|
195
|
-
export class Helia
|
|
196
|
-
public
|
|
193
|
+
export class Helia implements HeliaInterface {
|
|
194
|
+
public info: NodeInfo
|
|
197
195
|
public blockstore: BlockStorage
|
|
198
196
|
public datastore: Datastore
|
|
199
|
-
public events: TypedEventEmitter<HeliaEvents<
|
|
197
|
+
public events: TypedEventEmitter<HeliaEvents<this>>
|
|
200
198
|
public pins: Pins
|
|
201
199
|
public logger: ComponentLogger
|
|
202
|
-
public routing:
|
|
200
|
+
public routing: RoutingClass
|
|
203
201
|
public getCodec: CodecLoader
|
|
204
202
|
public getHasher: HasherLoader
|
|
203
|
+
public getCrypto: CryptoLoader
|
|
205
204
|
public dns: DNS
|
|
205
|
+
public keychain: Keychain
|
|
206
206
|
public metrics?: Metrics
|
|
207
|
+
public status: 'stopped' | 'stopping' | 'starting' | 'started'
|
|
207
208
|
private readonly log: Logger
|
|
209
|
+
private readonly blockBrokers: BlockBroker[]
|
|
210
|
+
private readonly mixins: HeliaMixin[]
|
|
208
211
|
|
|
209
|
-
constructor (init:
|
|
210
|
-
this.
|
|
212
|
+
constructor (init: HeliaInit & { name: string, version: string }) {
|
|
213
|
+
this.info = {
|
|
214
|
+
name: init.name,
|
|
215
|
+
version: init.version
|
|
216
|
+
}
|
|
217
|
+
this.logger = init.logger ?? defaultLogger()
|
|
211
218
|
this.log = this.logger.forComponent('helia')
|
|
212
219
|
this.getHasher = getHasher(init.hashers, init.loadHasher)
|
|
213
220
|
this.getCodec = getCodec(init.codecs, init.loadCodec)
|
|
221
|
+
this.getCrypto = getCrypto(init.cryptos, init.loadCrypto)
|
|
214
222
|
this.dns = init.dns ?? dns()
|
|
215
223
|
this.metrics = init.metrics
|
|
216
|
-
this.
|
|
217
|
-
this.
|
|
224
|
+
this.events = new TypedEventEmitter<HeliaEvents<typeof this>>()
|
|
225
|
+
this.status = 'stopped'
|
|
226
|
+
this.mixins = []
|
|
218
227
|
|
|
219
|
-
// @ts-expect-error routing
|
|
228
|
+
// @ts-expect-error routing and keychain are not set
|
|
220
229
|
const components: Components = {
|
|
221
|
-
blockstore: init.blockstore,
|
|
222
|
-
datastore: init.datastore,
|
|
230
|
+
blockstore: init.blockstore ?? new MemoryBlockstore(),
|
|
231
|
+
datastore: init.datastore ?? new MemoryDatastore(),
|
|
223
232
|
logger: this.logger,
|
|
224
|
-
libp2p: this.libp2p,
|
|
225
233
|
blockBrokers: [],
|
|
226
234
|
getHasher: this.getHasher,
|
|
227
235
|
getCodec: this.getCodec,
|
|
236
|
+
getCrypto: this.getCrypto,
|
|
228
237
|
dns: this.dns,
|
|
229
238
|
metrics: this.metrics,
|
|
230
239
|
...(init.components ?? {})
|
|
231
240
|
}
|
|
232
241
|
|
|
242
|
+
this.keychain = components.keychain = keychain()(components)
|
|
243
|
+
|
|
233
244
|
this.routing = components.routing = new RoutingClass(components, {
|
|
234
|
-
routers: (init.routers ?? []).flatMap((router:
|
|
245
|
+
routers: (init.routers ?? []).flatMap((router: Router | ((components: any) => Router)) => {
|
|
235
246
|
if (typeof router === 'function') {
|
|
236
247
|
router = router(components)
|
|
237
248
|
}
|
|
@@ -241,54 +252,86 @@ export class Helia<T extends Libp2p> implements HeliaInterface<T> {
|
|
|
241
252
|
router
|
|
242
253
|
]
|
|
243
254
|
|
|
244
|
-
// if the router provides a libp2p-style ContentRouter
|
|
245
|
-
const contentRouting = asContentRouting(router)
|
|
246
|
-
if (contentRouting != null) {
|
|
247
|
-
routers.push(contentRouting)
|
|
248
|
-
}
|
|
249
|
-
|
|
250
|
-
// if the router provides a libp2p-style PeerRouter
|
|
251
|
-
const peerRouting = asPeerRouting(router)
|
|
252
|
-
if (peerRouting != null) {
|
|
253
|
-
routers.push(peerRouting)
|
|
254
|
-
}
|
|
255
|
-
|
|
256
255
|
return routers
|
|
257
256
|
}),
|
|
258
257
|
providerLookupConcurrency: init.providerLookupConcurrency
|
|
259
258
|
})
|
|
260
259
|
|
|
261
|
-
components.blockBrokers = init.blockBrokers.map((
|
|
262
|
-
|
|
260
|
+
this.blockBrokers = components.blockBrokers = (init.blockBrokers ?? []).map((broker) => {
|
|
261
|
+
if (typeof broker === 'function') {
|
|
262
|
+
broker = broker(components)
|
|
263
|
+
}
|
|
264
|
+
|
|
265
|
+
return broker
|
|
263
266
|
})
|
|
264
267
|
|
|
265
268
|
const networkedStorage = new NetworkedStorage(components, init)
|
|
266
|
-
this.pins = new PinsImpl(
|
|
269
|
+
this.pins = new PinsImpl(components.datastore, networkedStorage, this.getCodec)
|
|
267
270
|
this.blockstore = new BlockStorage(networkedStorage, this.pins, this.routing, {
|
|
268
271
|
holdGcLock: init.holdGcLock ?? true
|
|
269
272
|
})
|
|
270
|
-
this.datastore =
|
|
273
|
+
this.datastore = components.datastore
|
|
274
|
+
}
|
|
275
|
+
|
|
276
|
+
hasRouter (name: string): boolean {
|
|
277
|
+
return this.routing.hasRouter(name)
|
|
271
278
|
}
|
|
272
279
|
|
|
273
|
-
|
|
280
|
+
addRouter (router: Router): void {
|
|
281
|
+
this.routing.addRouter(router)
|
|
282
|
+
}
|
|
283
|
+
|
|
284
|
+
hasBlockBroker (name: string): boolean {
|
|
285
|
+
return this.blockBrokers.findIndex(b => b.name === name) !== -1
|
|
286
|
+
}
|
|
287
|
+
|
|
288
|
+
addBlockBroker (blockBroker: BlockBroker): void {
|
|
289
|
+
this.blockBrokers.push(blockBroker)
|
|
290
|
+
}
|
|
291
|
+
|
|
292
|
+
addMixin (mixin: HeliaMixin): void {
|
|
293
|
+
this.mixins.push(mixin)
|
|
294
|
+
}
|
|
295
|
+
|
|
296
|
+
async start (): Promise<this> {
|
|
297
|
+
this.status = 'starting'
|
|
298
|
+
|
|
274
299
|
await assertDatastoreVersionIsCurrent(this.datastore)
|
|
275
300
|
await start(
|
|
276
301
|
this.blockstore,
|
|
277
302
|
this.datastore,
|
|
278
303
|
this.routing,
|
|
279
|
-
this.
|
|
304
|
+
...this.blockBrokers
|
|
280
305
|
)
|
|
306
|
+
|
|
307
|
+
for (const mixin of this.mixins) {
|
|
308
|
+
await mixin.start?.(this)
|
|
309
|
+
}
|
|
310
|
+
|
|
311
|
+
this.status = 'started'
|
|
281
312
|
this.events.dispatchEvent(new CustomEvent('start', { detail: this }))
|
|
313
|
+
|
|
314
|
+
return this
|
|
282
315
|
}
|
|
283
316
|
|
|
284
|
-
async stop (): Promise<
|
|
317
|
+
async stop (): Promise<this> {
|
|
318
|
+
this.status = 'stopping'
|
|
319
|
+
|
|
320
|
+
for (const mixin of this.mixins) {
|
|
321
|
+
await mixin.stop?.(this)
|
|
322
|
+
}
|
|
323
|
+
|
|
285
324
|
await stop(
|
|
286
325
|
this.blockstore,
|
|
287
326
|
this.datastore,
|
|
288
327
|
this.routing,
|
|
289
|
-
this.
|
|
328
|
+
...this.blockBrokers
|
|
290
329
|
)
|
|
330
|
+
|
|
331
|
+
this.status = 'stopped'
|
|
291
332
|
this.events.dispatchEvent(new CustomEvent('stop', { detail: this }))
|
|
333
|
+
|
|
334
|
+
return this
|
|
292
335
|
}
|
|
293
336
|
|
|
294
337
|
async gc (options: GCOptions = {}): Promise<void> {
|
|
@@ -323,11 +366,3 @@ export class Helia<T extends Libp2p> implements HeliaInterface<T> {
|
|
|
323
366
|
this.log('gc finished')
|
|
324
367
|
}
|
|
325
368
|
}
|
|
326
|
-
|
|
327
|
-
function asContentRouting (obj?: any): ContentRouting | undefined {
|
|
328
|
-
return obj?.[contentRoutingSymbol]
|
|
329
|
-
}
|
|
330
|
-
|
|
331
|
-
function asPeerRouting (obj?: any): PeerRouting | undefined {
|
|
332
|
-
return obj?.[peerRoutingSymbol]
|
|
333
|
-
}
|
package/src/pins.ts
CHANGED
|
@@ -1,22 +1,18 @@
|
|
|
1
1
|
import { InvalidParametersError } from '@libp2p/interface'
|
|
2
|
-
import { Queue } from '@libp2p/utils'
|
|
3
2
|
import * as cborg from 'cborg'
|
|
4
3
|
import { Key } from 'interface-datastore'
|
|
5
|
-
import toBuffer from 'it-to-buffer'
|
|
6
4
|
import { base36 } from 'multiformats/bases/base36'
|
|
7
|
-
import { createUnsafe } from 'multiformats/block'
|
|
8
5
|
import { CID } from 'multiformats/cid'
|
|
9
6
|
import { CustomProgressEvent } from 'progress-events'
|
|
10
7
|
import { equals as uint8ArrayEquals } from 'uint8arrays/equals'
|
|
11
8
|
import { AlreadyPinnedError } from './errors.ts'
|
|
9
|
+
import { depthFirstWalker } from './graph-walker.ts'
|
|
12
10
|
import type { CodecLoader } from '@helia/interface'
|
|
13
|
-
import type {
|
|
14
|
-
import type { AddOptions, AddPinEvents, IsPinnedOptions, LsOptions, Pin, Pins, RmOptions } from '@helia/interface/pins'
|
|
11
|
+
import type { AddOptions, IsPinnedOptions, LsOptions, Pin, Pins, RmOptions } from '@helia/interface/pins'
|
|
15
12
|
import type { AbortOptions } from '@libp2p/interface'
|
|
16
13
|
import type { Blockstore } from 'interface-blockstore'
|
|
17
14
|
import type { Datastore } from 'interface-datastore'
|
|
18
15
|
import type { Version } from 'multiformats/cid'
|
|
19
|
-
import type { ProgressOptions } from 'progress-events'
|
|
20
16
|
|
|
21
17
|
interface DatastorePin {
|
|
22
18
|
/**
|
|
@@ -49,11 +45,6 @@ interface WithPinnedBlockCallback {
|
|
|
49
45
|
const DATASTORE_PIN_PREFIX = '/pin/'
|
|
50
46
|
const DATASTORE_BLOCK_PREFIX = '/pinned-block/'
|
|
51
47
|
const DATASTORE_ENCODING = base36
|
|
52
|
-
const DAG_WALK_QUEUE_CONCURRENCY = 1
|
|
53
|
-
|
|
54
|
-
interface WalkDagOptions extends AbortOptions, ProgressOptions<GetBlockProgressEvents | AddPinEvents> {
|
|
55
|
-
depth: number
|
|
56
|
-
}
|
|
57
48
|
|
|
58
49
|
function toDSKey (cid: CID): Key {
|
|
59
50
|
if (cid.version === 0) {
|
|
@@ -87,15 +78,13 @@ export class PinsImpl implements Pins {
|
|
|
87
78
|
throw new InvalidParametersError('Depth must be greater than or equal to 0')
|
|
88
79
|
}
|
|
89
80
|
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
81
|
+
const walker = (options.walker ?? depthFirstWalker())({
|
|
82
|
+
blockstore: this.blockstore,
|
|
83
|
+
getCodec: this.getCodec
|
|
93
84
|
})
|
|
94
85
|
|
|
95
|
-
for await (const
|
|
96
|
-
|
|
97
|
-
depth
|
|
98
|
-
})) {
|
|
86
|
+
for await (const node of walker.walk(cid, { ...options, depth })) {
|
|
87
|
+
const childCid = node.block.cid
|
|
99
88
|
await this.#updatePinnedBlock(childCid, (pinnedBlock: DatastorePinnedBlock) => {
|
|
100
89
|
// do not update pinned block if this block is already pinned by this CID
|
|
101
90
|
if (pinnedBlock.pinnedBy.find(c => uint8ArrayEquals(c, cid.bytes)) != null) {
|
|
@@ -118,31 +107,6 @@ export class PinsImpl implements Pins {
|
|
|
118
107
|
await this.datastore.put(pinKey, cborg.encode(pin), options)
|
|
119
108
|
}
|
|
120
109
|
|
|
121
|
-
/**
|
|
122
|
-
* Walk a DAG in an iterable fashion
|
|
123
|
-
*/
|
|
124
|
-
async * #walkDag (cid: CID, queue: Queue<AsyncGenerator<CID>>, options: WalkDagOptions): AsyncGenerator<CID> {
|
|
125
|
-
if (options.depth === -1) {
|
|
126
|
-
return
|
|
127
|
-
}
|
|
128
|
-
|
|
129
|
-
const codec = await this.getCodec(cid.code)
|
|
130
|
-
const bytes = await toBuffer(this.blockstore.get(cid, options))
|
|
131
|
-
const block = createUnsafe({ bytes, cid, codec })
|
|
132
|
-
|
|
133
|
-
yield cid
|
|
134
|
-
|
|
135
|
-
// walk dag, ensure all blocks are present
|
|
136
|
-
for (const [,cid] of block.links()) {
|
|
137
|
-
yield * await queue.add(async () => {
|
|
138
|
-
return this.#walkDag(cid, queue, {
|
|
139
|
-
...options,
|
|
140
|
-
depth: options.depth - 1
|
|
141
|
-
})
|
|
142
|
-
})
|
|
143
|
-
}
|
|
144
|
-
}
|
|
145
|
-
|
|
146
110
|
/**
|
|
147
111
|
* Update the pin count for the CID
|
|
148
112
|
*/
|
|
@@ -186,15 +150,13 @@ export class PinsImpl implements Pins {
|
|
|
186
150
|
|
|
187
151
|
await this.datastore.delete(pinKey, options)
|
|
188
152
|
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
153
|
+
const walker = (options.walker ?? depthFirstWalker())({
|
|
154
|
+
blockstore: this.blockstore,
|
|
155
|
+
getCodec: this.getCodec
|
|
192
156
|
})
|
|
193
157
|
|
|
194
|
-
for await (const
|
|
195
|
-
|
|
196
|
-
depth: pin.depth
|
|
197
|
-
})) {
|
|
158
|
+
for await (const node of walker.walk(cid, { ...options, depth: pin.depth })) {
|
|
159
|
+
const childCid = node.block.cid
|
|
198
160
|
await this.#updatePinnedBlock(childCid, (pinnedBlock): boolean => {
|
|
199
161
|
pinnedBlock.pinCount--
|
|
200
162
|
pinnedBlock.pinnedBy = pinnedBlock.pinnedBy.filter(c => uint8ArrayEquals(c, cid.bytes))
|
package/src/routing.ts
CHANGED
|
@@ -1,18 +1,19 @@
|
|
|
1
1
|
import { NoRoutersAvailableError } from '@helia/interface'
|
|
2
2
|
import { NotFoundError, start, stop } from '@libp2p/interface'
|
|
3
|
-
import {
|
|
3
|
+
import { Queue } from '@libp2p/utils'
|
|
4
4
|
import merge from 'it-merge'
|
|
5
5
|
import { CustomProgressEvent } from 'progress-events'
|
|
6
6
|
import { toString as uint8ArrayToString } from 'uint8arrays/to-string'
|
|
7
7
|
import { GetFailedError } from './errors.ts'
|
|
8
|
-
import type { Routing as RoutingInterface, Provider, RoutingOptions, RoutingFindProvidersProgressEvents, RoutingProvideProgressEvents, RoutingPutProgressEvents, RoutingGetProgressEvents, RoutingFindPeerProgressEvents, RoutingGetClosestPeersProgressEvents, RoutingCancelReprovideProgressEvents } from '@helia/interface'
|
|
9
|
-
import type { ComponentLogger, Logger, Metrics,
|
|
8
|
+
import type { Routing as RoutingInterface, Provider, RoutingOptions, RoutingFindProvidersProgressEvents, RoutingProvideProgressEvents, RoutingPutProgressEvents, RoutingGetProgressEvents, RoutingFindPeerProgressEvents, RoutingGetClosestPeersProgressEvents, RoutingCancelReprovideProgressEvents, Router, Peer } from '@helia/interface'
|
|
9
|
+
import type { ComponentLogger, Logger, Metrics, Startable } from '@libp2p/interface'
|
|
10
|
+
import type { AbortOptions } from 'abort-error'
|
|
10
11
|
import type { CID } from 'multiformats/cid'
|
|
11
12
|
|
|
12
13
|
const DEFAULT_PROVIDER_LOOKUP_CONCURRENCY = 5
|
|
13
14
|
|
|
14
15
|
export interface RoutingInit {
|
|
15
|
-
routers:
|
|
16
|
+
routers: Router[]
|
|
16
17
|
providerLookupConcurrency?: number
|
|
17
18
|
}
|
|
18
19
|
|
|
@@ -21,11 +22,15 @@ export interface RoutingComponents {
|
|
|
21
22
|
metrics?: Metrics
|
|
22
23
|
}
|
|
23
24
|
|
|
25
|
+
interface PeerQueueOptions extends AbortOptions {
|
|
26
|
+
peer: CID
|
|
27
|
+
}
|
|
28
|
+
|
|
24
29
|
export class Routing implements RoutingInterface, Startable {
|
|
25
30
|
public name: string
|
|
26
31
|
|
|
27
32
|
private readonly log: Logger
|
|
28
|
-
private readonly routers:
|
|
33
|
+
private readonly routers: Router[]
|
|
29
34
|
private readonly providerLookupConcurrency: number
|
|
30
35
|
|
|
31
36
|
constructor (components: RoutingComponents, init: RoutingInit) {
|
|
@@ -65,6 +70,14 @@ export class Routing implements RoutingInterface, Startable {
|
|
|
65
70
|
await stop(...this.routers)
|
|
66
71
|
}
|
|
67
72
|
|
|
73
|
+
hasRouter (name: string): boolean {
|
|
74
|
+
return this.routers.findIndex(r => r.name === name) !== -1
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
addRouter (router: Router): void {
|
|
78
|
+
this.routers.push(router)
|
|
79
|
+
}
|
|
80
|
+
|
|
68
81
|
/**
|
|
69
82
|
* Iterates over all content routers in parallel to find providers of the
|
|
70
83
|
* given key
|
|
@@ -78,7 +91,7 @@ export class Routing implements RoutingInterface, Startable {
|
|
|
78
91
|
// back as an empty array - when this happens we have to do a FIND_PEER
|
|
79
92
|
// query to get updated addresses, but we shouldn't block on this so use a
|
|
80
93
|
// separate bounded queue to perform this lookup
|
|
81
|
-
const queue = new
|
|
94
|
+
const queue = new Queue<Provider | null, PeerQueueOptions>({
|
|
82
95
|
concurrency: this.providerLookupConcurrency
|
|
83
96
|
})
|
|
84
97
|
|
|
@@ -145,7 +158,7 @@ export class Routing implements RoutingInterface, Startable {
|
|
|
145
158
|
// have to refresh peer info for this peer to get updated multiaddrs
|
|
146
159
|
if (peer.multiaddrs.length === 0) {
|
|
147
160
|
// already looking this peer up
|
|
148
|
-
if (queue.find(peer.id) != null) {
|
|
161
|
+
if (queue.queue.find(job => job.options.peer.equals(peer.id)) != null) {
|
|
149
162
|
continue
|
|
150
163
|
}
|
|
151
164
|
|
|
@@ -167,7 +180,7 @@ export class Routing implements RoutingInterface, Startable {
|
|
|
167
180
|
return null
|
|
168
181
|
}
|
|
169
182
|
}, {
|
|
170
|
-
|
|
183
|
+
peer: peer.id,
|
|
171
184
|
signal: options.signal
|
|
172
185
|
})
|
|
173
186
|
.catch(err => {
|
|
@@ -298,7 +311,7 @@ export class Routing implements RoutingInterface, Startable {
|
|
|
298
311
|
/**
|
|
299
312
|
* Iterates over all peer routers in parallel to find the given peer
|
|
300
313
|
*/
|
|
301
|
-
async findPeer (id:
|
|
314
|
+
async findPeer (id: CID, options?: RoutingOptions<RoutingFindPeerProgressEvents>): Promise<Peer> {
|
|
302
315
|
if (this.routers.length === 0) {
|
|
303
316
|
throw new NoRoutersAvailableError('No peer routers available')
|
|
304
317
|
}
|
|
@@ -339,7 +352,7 @@ export class Routing implements RoutingInterface, Startable {
|
|
|
339
352
|
/**
|
|
340
353
|
* Attempt to find the closest peers on the network to the given key
|
|
341
354
|
*/
|
|
342
|
-
async * getClosestPeers (key: Uint8Array, options: RoutingOptions<RoutingGetClosestPeersProgressEvents> = {}): AsyncIterable<
|
|
355
|
+
async * getClosestPeers (key: Uint8Array, options: RoutingOptions<RoutingGetClosestPeersProgressEvents> = {}): AsyncIterable<Peer> {
|
|
343
356
|
if (this.routers.length === 0) {
|
|
344
357
|
throw new NoRoutersAvailableError('No peer routers available')
|
|
345
358
|
}
|
package/src/utils/get-codec.ts
CHANGED
|
@@ -1,5 +1,3 @@
|
|
|
1
|
-
/* eslint max-depth: ["error", 7] */
|
|
2
|
-
|
|
3
1
|
import { UnknownCodecError } from '@helia/interface'
|
|
4
2
|
import * as dagCbor from '@ipld/dag-cbor'
|
|
5
3
|
import * as dagJson from '@ipld/dag-json'
|
|
@@ -7,9 +5,10 @@ import * as dagPb from '@ipld/dag-pb'
|
|
|
7
5
|
import * as json from 'multiformats/codecs/json'
|
|
8
6
|
import * as raw from 'multiformats/codecs/raw'
|
|
9
7
|
import { isPromise } from './is-promise.ts'
|
|
8
|
+
import type { CodecLoader } from '@helia/interface'
|
|
10
9
|
import type { BlockCodec } from 'multiformats/codecs/interface'
|
|
11
10
|
|
|
12
|
-
export function getCodec
|
|
11
|
+
export function getCodec (initialCodecs: Array<BlockCodec<any, any>> = [], loadCodec?: CodecLoader): CodecLoader {
|
|
13
12
|
const codecs: Record<number, BlockCodec<any, any>> = {
|
|
14
13
|
[dagPb.code]: dagPb,
|
|
15
14
|
[raw.code]: raw,
|