@helia/utils 2.3.5 → 2.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.min.js +1 -11
- package/dist/index.min.js.map +4 -4
- package/dist/src/abstract-session.d.ts +3 -1
- package/dist/src/abstract-session.d.ts.map +1 -1
- package/dist/src/abstract-session.js +15 -4
- package/dist/src/abstract-session.js.map +1 -1
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +4 -5
- package/dist/src/index.js.map +1 -1
- package/dist/src/utils/networked-storage.d.ts +7 -57
- package/dist/src/utils/networked-storage.d.ts.map +1 -1
- package/dist/src/utils/networked-storage.js +15 -372
- package/dist/src/utils/networked-storage.js.map +1 -1
- package/dist/src/utils/session-storage.d.ts +48 -0
- package/dist/src/utils/session-storage.d.ts.map +1 -0
- package/dist/src/utils/session-storage.js +148 -0
- package/dist/src/utils/session-storage.js.map +1 -0
- package/dist/src/utils/storage.d.ts +56 -0
- package/dist/src/utils/storage.d.ts.map +1 -0
- package/dist/src/utils/storage.js +225 -0
- package/dist/src/utils/storage.js.map +1 -0
- package/package.json +3 -3
- package/src/abstract-session.ts +20 -4
- package/src/index.ts +5 -6
- package/src/utils/networked-storage.ts +22 -461
- package/src/utils/session-storage.ts +175 -0
- package/src/utils/storage.ts +295 -0
|
@@ -1,217 +1,24 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
4
|
-
import
|
|
5
|
-
import
|
|
6
|
-
import {
|
|
7
|
-
import {
|
|
8
|
-
import {
|
|
9
|
-
import { isPromise } from './is-promise.js'
|
|
10
|
-
import type { HasherLoader } from '@helia/interface'
|
|
11
|
-
import type { BlockBroker, Blocks, Pair, DeleteManyBlocksProgressEvents, DeleteBlockProgressEvents, GetBlockProgressEvents, GetManyBlocksProgressEvents, PutManyBlocksProgressEvents, PutBlockProgressEvents, GetAllBlocksProgressEvents, GetOfflineOptions, BlockRetrievalOptions, CreateSessionOptions, SessionBlockstore } from '@helia/interface/blocks'
|
|
12
|
-
import type { AbortOptions, ComponentLogger, Logger, LoggerOptions, Startable } from '@libp2p/interface'
|
|
13
|
-
import type { Blockstore, InputPair } from 'interface-blockstore'
|
|
14
|
-
import type { AwaitIterable } from 'interface-store'
|
|
1
|
+
import { start, stop } from '@libp2p/interface'
|
|
2
|
+
import { InvalidConfigurationError } from '../errors.ts'
|
|
3
|
+
import { SessionStorage } from './session-storage.ts'
|
|
4
|
+
import { Storage } from './storage.ts'
|
|
5
|
+
import type { StorageComponents, StorageInit } from './storage.ts'
|
|
6
|
+
import type { BlockBroker, Blocks, CreateSessionOptions, SessionBlockstore } from '@helia/interface/blocks'
|
|
7
|
+
import type { AbortOptions, Startable } from '@libp2p/interface'
|
|
8
|
+
import type { Blockstore } from 'interface-blockstore'
|
|
15
9
|
import type { CID } from 'multiformats/cid'
|
|
16
|
-
import type { MultihashDigest, MultihashHasher } from 'multiformats/hashes/interface'
|
|
17
|
-
import type { ProgressOptions } from 'progress-events'
|
|
18
10
|
|
|
19
11
|
export interface GetOptions extends AbortOptions {
|
|
20
12
|
progress?(evt: Event): void
|
|
21
13
|
}
|
|
22
14
|
|
|
23
|
-
export
|
|
24
|
-
blockstore: Blockstore
|
|
25
|
-
logger: ComponentLogger
|
|
26
|
-
blockBrokers: BlockBroker[]
|
|
27
|
-
getHasher: HasherLoader
|
|
28
|
-
}
|
|
29
|
-
|
|
30
|
-
export interface StorageInit {
|
|
31
|
-
maxIdentityHashDigestLength?: number
|
|
32
|
-
}
|
|
33
|
-
|
|
34
|
-
const DEFAULT_MAX_IDENTITY_HASH_DIGEST_LENGTH = 128
|
|
35
|
-
|
|
36
|
-
class Storage implements Blockstore {
|
|
37
|
-
protected readonly child: Blockstore
|
|
38
|
-
protected readonly getHasher: HasherLoader
|
|
39
|
-
protected log: Logger
|
|
40
|
-
protected readonly logger: ComponentLogger
|
|
41
|
-
protected readonly components: StorageComponents
|
|
42
|
-
|
|
43
|
-
/**
|
|
44
|
-
* Create a new BlockStorage
|
|
45
|
-
*/
|
|
46
|
-
constructor (components: StorageComponents, init: StorageInit = {}) {
|
|
47
|
-
this.log = components.logger.forComponent('helia:networked-storage')
|
|
48
|
-
this.logger = components.logger
|
|
49
|
-
this.components = components
|
|
50
|
-
this.child = new IdentityBlockstore(components.blockstore, {
|
|
51
|
-
maxDigestLength: init.maxIdentityHashDigestLength ?? DEFAULT_MAX_IDENTITY_HASH_DIGEST_LENGTH
|
|
52
|
-
})
|
|
53
|
-
this.getHasher = components.getHasher
|
|
54
|
-
}
|
|
55
|
-
|
|
56
|
-
/**
|
|
57
|
-
* Put a block to the underlying datastore
|
|
58
|
-
*/
|
|
59
|
-
async put (cid: CID, block: Uint8Array, options: AbortOptions & ProgressOptions<PutBlockProgressEvents> = {}): Promise<CID> {
|
|
60
|
-
if (await this.child.has(cid, options)) {
|
|
61
|
-
options.onProgress?.(new CustomProgressEvent<CID>('blocks:put:duplicate', cid))
|
|
62
|
-
return cid
|
|
63
|
-
}
|
|
64
|
-
|
|
65
|
-
options.onProgress?.(new CustomProgressEvent<CID>('blocks:put:providers:notify', cid))
|
|
66
|
-
|
|
67
|
-
await Promise.all(
|
|
68
|
-
this.components.blockBrokers.map(async broker => broker.announce?.(cid, options))
|
|
69
|
-
)
|
|
70
|
-
|
|
71
|
-
options.onProgress?.(new CustomProgressEvent<CID>('blocks:put:blockstore:put', cid))
|
|
72
|
-
|
|
73
|
-
return this.child.put(cid, block, options)
|
|
74
|
-
}
|
|
75
|
-
|
|
76
|
-
/**
|
|
77
|
-
* Put a multiple blocks to the underlying datastore
|
|
78
|
-
*/
|
|
79
|
-
async * putMany (blocks: AwaitIterable<InputPair>, options: AbortOptions & ProgressOptions<PutManyBlocksProgressEvents> = {}): AsyncGenerator<CID> {
|
|
80
|
-
const missingBlocks = filter(blocks, async ({ cid }): Promise<boolean> => {
|
|
81
|
-
const has = await this.child.has(cid, options)
|
|
82
|
-
|
|
83
|
-
if (has) {
|
|
84
|
-
options.onProgress?.(new CustomProgressEvent<CID>('blocks:put-many:duplicate', cid))
|
|
85
|
-
}
|
|
86
|
-
|
|
87
|
-
return !has
|
|
88
|
-
})
|
|
89
|
-
|
|
90
|
-
const notifyEach = forEach(missingBlocks, async ({ cid }): Promise<void> => {
|
|
91
|
-
options.onProgress?.(new CustomProgressEvent<CID>('blocks:put-many:providers:notify', cid))
|
|
92
|
-
await Promise.all(
|
|
93
|
-
this.components.blockBrokers.map(async broker => broker.announce?.(cid, options))
|
|
94
|
-
)
|
|
95
|
-
})
|
|
96
|
-
|
|
97
|
-
options.onProgress?.(new CustomProgressEvent('blocks:put-many:blockstore:put-many'))
|
|
98
|
-
yield * this.child.putMany(notifyEach, options)
|
|
99
|
-
}
|
|
100
|
-
|
|
101
|
-
/**
|
|
102
|
-
* Get a block by cid
|
|
103
|
-
*/
|
|
104
|
-
async * get (cid: CID, options: GetOfflineOptions & AbortOptions & ProgressOptions<GetBlockProgressEvents> = {}): AsyncGenerator<Uint8Array> {
|
|
105
|
-
const has = await this.child.has(cid, options)
|
|
106
|
-
const offline = options.offline === true
|
|
107
|
-
|
|
108
|
-
if (!has) {
|
|
109
|
-
if (offline) {
|
|
110
|
-
throw new BlockNotFoundWhileOfflineError('The block was present in the blockstore and the node is running offline so cannot fetch it')
|
|
111
|
-
}
|
|
112
|
-
|
|
113
|
-
const hasher = await this.getHasher(cid.multihash.code)
|
|
114
|
-
options?.signal?.throwIfAborted()
|
|
115
|
-
|
|
116
|
-
// we do not have the block locally, get it from a block provider
|
|
117
|
-
|
|
118
|
-
options.onProgress?.(new CustomProgressEvent<CID>('blocks:get:providers:get', cid))
|
|
119
|
-
const block = await raceBlockRetrievers(cid, this.components.blockBrokers, hasher, {
|
|
120
|
-
...options,
|
|
121
|
-
log: this.log
|
|
122
|
-
})
|
|
123
|
-
options.onProgress?.(new CustomProgressEvent<CID>('blocks:get:blockstore:put', cid))
|
|
124
|
-
await this.child.put(cid, block, options)
|
|
125
|
-
|
|
126
|
-
// notify other block providers of the new block
|
|
127
|
-
options.onProgress?.(new CustomProgressEvent<CID>('blocks:get:providers:notify', cid))
|
|
128
|
-
await Promise.all(
|
|
129
|
-
this.components.blockBrokers.map(async broker => broker.announce?.(cid, options))
|
|
130
|
-
)
|
|
131
|
-
|
|
132
|
-
yield block
|
|
133
|
-
return
|
|
134
|
-
}
|
|
135
|
-
|
|
136
|
-
options.onProgress?.(new CustomProgressEvent<CID>('blocks:get:blockstore:get', cid))
|
|
137
|
-
|
|
138
|
-
yield * this.child.get(cid, options)
|
|
139
|
-
}
|
|
140
|
-
|
|
141
|
-
/**
|
|
142
|
-
* Get multiple blocks back from an (async) iterable of cids
|
|
143
|
-
*/
|
|
144
|
-
async * getMany (cids: AwaitIterable<CID>, options: GetOfflineOptions & AbortOptions & ProgressOptions<GetManyBlocksProgressEvents> = {}): AsyncGenerator<Pair> {
|
|
145
|
-
options.onProgress?.(new CustomProgressEvent('blocks:get-many:blockstore:get-many'))
|
|
146
|
-
|
|
147
|
-
yield * this.child.getMany(forEach(cids, async (cid): Promise<void> => {
|
|
148
|
-
const has = await this.child.has(cid, options)
|
|
149
|
-
const offline = options.offline === true
|
|
150
|
-
|
|
151
|
-
if (!has) {
|
|
152
|
-
if (offline) {
|
|
153
|
-
throw new BlockNotFoundWhileOfflineError('The block was present in the blockstore and the node is running offline so cannot fetch it')
|
|
154
|
-
}
|
|
155
|
-
|
|
156
|
-
const hasher = await this.getHasher(cid.multihash.code)
|
|
157
|
-
options?.signal?.throwIfAborted()
|
|
158
|
-
|
|
159
|
-
// we do not have the block locally, get it from a block provider
|
|
160
|
-
options.onProgress?.(new CustomProgressEvent<CID>('blocks:get-many:providers:get', cid))
|
|
161
|
-
const block = await raceBlockRetrievers(cid, this.components.blockBrokers, hasher, {
|
|
162
|
-
...options,
|
|
163
|
-
log: this.log
|
|
164
|
-
})
|
|
165
|
-
options.onProgress?.(new CustomProgressEvent<CID>('blocks:get-many:blockstore:put', cid))
|
|
166
|
-
await this.child.put(cid, block, options)
|
|
167
|
-
|
|
168
|
-
// notify other block providers of the new block
|
|
169
|
-
options.onProgress?.(new CustomProgressEvent<CID>('blocks:get-many:providers:notify', cid))
|
|
170
|
-
await Promise.all(
|
|
171
|
-
this.components.blockBrokers.map(async broker => broker.announce?.(cid, options))
|
|
172
|
-
)
|
|
173
|
-
}
|
|
174
|
-
}))
|
|
175
|
-
}
|
|
176
|
-
|
|
177
|
-
/**
|
|
178
|
-
* Delete a block from the blockstore
|
|
179
|
-
*/
|
|
180
|
-
async delete (cid: CID, options: AbortOptions & ProgressOptions<DeleteBlockProgressEvents> = {}): Promise<void> {
|
|
181
|
-
options.onProgress?.(new CustomProgressEvent<CID>('blocks:delete:blockstore:delete', cid))
|
|
182
|
-
|
|
183
|
-
await this.child.delete(cid, options)
|
|
184
|
-
}
|
|
185
|
-
|
|
186
|
-
/**
|
|
187
|
-
* Delete multiple blocks from the blockstore
|
|
188
|
-
*/
|
|
189
|
-
async * deleteMany (cids: AwaitIterable<CID>, options: AbortOptions & ProgressOptions<DeleteManyBlocksProgressEvents> = {}): AsyncGenerator<CID> {
|
|
190
|
-
options.onProgress?.(new CustomProgressEvent('blocks:delete-many:blockstore:delete-many'))
|
|
191
|
-
yield * this.child.deleteMany((async function * (): AsyncGenerator<CID> {
|
|
192
|
-
for await (const cid of cids) {
|
|
193
|
-
yield cid
|
|
194
|
-
}
|
|
195
|
-
}()), options)
|
|
196
|
-
}
|
|
197
|
-
|
|
198
|
-
async has (cid: CID, options: AbortOptions = {}): Promise<boolean> {
|
|
199
|
-
return this.child.has(cid, options)
|
|
200
|
-
}
|
|
201
|
-
|
|
202
|
-
async * getAll (options: AbortOptions & ProgressOptions<GetAllBlocksProgressEvents> = {}): AsyncGenerator<Pair> {
|
|
203
|
-
options.onProgress?.(new CustomProgressEvent('blocks:get-all:blockstore:get-many'))
|
|
204
|
-
yield * this.child.getAll(options)
|
|
205
|
-
}
|
|
206
|
-
}
|
|
207
|
-
|
|
208
|
-
export type NetworkedStorageComponents = StorageComponents
|
|
15
|
+
export type NetworkedStorageComponents = StorageComponents<BlockBroker>
|
|
209
16
|
|
|
210
17
|
/**
|
|
211
18
|
* Networked storage wraps a regular blockstore - when getting blocks if the
|
|
212
19
|
* blocks are not present, the configured BlockBrokers will be used to fetch them.
|
|
213
20
|
*/
|
|
214
|
-
export class NetworkedStorage extends Storage implements Blocks, Startable {
|
|
21
|
+
export class NetworkedStorage extends Storage<BlockBroker> implements Blocks, Startable {
|
|
215
22
|
private started: boolean
|
|
216
23
|
|
|
217
24
|
/**
|
|
@@ -228,12 +35,12 @@ export class NetworkedStorage extends Storage implements Blocks, Startable {
|
|
|
228
35
|
}
|
|
229
36
|
|
|
230
37
|
async start (): Promise<void> {
|
|
231
|
-
await start(this.child, ...this.
|
|
38
|
+
await start(this.child, ...this.blockBrokers)
|
|
232
39
|
this.started = true
|
|
233
40
|
}
|
|
234
41
|
|
|
235
42
|
async stop (): Promise<void> {
|
|
236
|
-
await stop(this.child, ...this.
|
|
43
|
+
await stop(this.child, ...this.blockBrokers)
|
|
237
44
|
this.started = false
|
|
238
45
|
}
|
|
239
46
|
|
|
@@ -242,13 +49,17 @@ export class NetworkedStorage extends Storage implements Blocks, Startable {
|
|
|
242
49
|
}
|
|
243
50
|
|
|
244
51
|
createSession (root: CID, options?: CreateSessionOptions): SessionBlockstore {
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
}
|
|
52
|
+
if (this.blockBrokers.length === 0) {
|
|
53
|
+
throw new InvalidConfigurationError('No block brokers configured')
|
|
54
|
+
}
|
|
249
55
|
|
|
250
|
-
|
|
251
|
-
|
|
56
|
+
const blockBrokers = this.blockBrokers
|
|
57
|
+
.map(broker => broker.createSession?.(options))
|
|
58
|
+
.filter(broker => broker != null)
|
|
59
|
+
|
|
60
|
+
if (blockBrokers.length === 0) {
|
|
61
|
+
throw new InvalidConfigurationError(`No configured block brokers support sessions - tried ${this.blockBrokers.map(b => b.name).join(', ')}`)
|
|
62
|
+
}
|
|
252
63
|
|
|
253
64
|
return new SessionStorage({
|
|
254
65
|
blockstore: this.child,
|
|
@@ -260,253 +71,3 @@ export class NetworkedStorage extends Storage implements Blocks, Startable {
|
|
|
260
71
|
})
|
|
261
72
|
}
|
|
262
73
|
}
|
|
263
|
-
|
|
264
|
-
interface SessionStorageInit {
|
|
265
|
-
root: CID
|
|
266
|
-
}
|
|
267
|
-
|
|
268
|
-
/**
|
|
269
|
-
* Storage subclass that can cancel any ongoing operation at any point.
|
|
270
|
-
*/
|
|
271
|
-
class SessionStorage extends Storage implements SessionBlockstore {
|
|
272
|
-
private readonly closeController: AbortController
|
|
273
|
-
|
|
274
|
-
constructor (components: StorageComponents, init: SessionStorageInit) {
|
|
275
|
-
super(components)
|
|
276
|
-
|
|
277
|
-
// because brokers are allowed to continue searching for providers after the
|
|
278
|
-
// session has been created, we need a way to tell them that the user has
|
|
279
|
-
// finished using the session any in-flight requests should be cancelled
|
|
280
|
-
this.closeController = new AbortController()
|
|
281
|
-
setMaxListeners(Infinity, this.closeController.signal)
|
|
282
|
-
|
|
283
|
-
this.log = components.logger.forComponent(`helia:session-storage:${init.root}`)
|
|
284
|
-
}
|
|
285
|
-
|
|
286
|
-
close (): void {
|
|
287
|
-
this.closeController.abort()
|
|
288
|
-
}
|
|
289
|
-
|
|
290
|
-
/**
|
|
291
|
-
* Put a block to the underlying datastore
|
|
292
|
-
*/
|
|
293
|
-
async put (cid: CID, block: Uint8Array, options: AbortOptions & ProgressOptions<PutBlockProgressEvents> = {}): Promise<CID> {
|
|
294
|
-
const signal = anySignal([this.closeController.signal, options.signal])
|
|
295
|
-
setMaxListeners(Infinity, signal)
|
|
296
|
-
|
|
297
|
-
try {
|
|
298
|
-
return await super.put(cid, block, {
|
|
299
|
-
...options,
|
|
300
|
-
signal
|
|
301
|
-
})
|
|
302
|
-
} finally {
|
|
303
|
-
signal.clear()
|
|
304
|
-
}
|
|
305
|
-
}
|
|
306
|
-
|
|
307
|
-
/**
|
|
308
|
-
* Put a multiple blocks to the underlying datastore
|
|
309
|
-
*/
|
|
310
|
-
async * putMany (blocks: AwaitIterable<InputPair>, options: AbortOptions & ProgressOptions<PutManyBlocksProgressEvents> = {}): AsyncGenerator<CID> {
|
|
311
|
-
const signal = anySignal([this.closeController.signal, options.signal])
|
|
312
|
-
setMaxListeners(Infinity, signal)
|
|
313
|
-
|
|
314
|
-
try {
|
|
315
|
-
yield * super.putMany(blocks, {
|
|
316
|
-
...options,
|
|
317
|
-
signal
|
|
318
|
-
})
|
|
319
|
-
} finally {
|
|
320
|
-
signal.clear()
|
|
321
|
-
}
|
|
322
|
-
}
|
|
323
|
-
|
|
324
|
-
/**
|
|
325
|
-
* Get a block by cid
|
|
326
|
-
*/
|
|
327
|
-
async * get (cid: CID, options: GetOfflineOptions & AbortOptions & ProgressOptions<GetBlockProgressEvents> = {}): AsyncGenerator<Uint8Array> {
|
|
328
|
-
const signal = anySignal([this.closeController.signal, options.signal])
|
|
329
|
-
setMaxListeners(Infinity, signal)
|
|
330
|
-
|
|
331
|
-
try {
|
|
332
|
-
yield * super.get(cid, {
|
|
333
|
-
...options,
|
|
334
|
-
signal
|
|
335
|
-
})
|
|
336
|
-
} finally {
|
|
337
|
-
signal.clear()
|
|
338
|
-
}
|
|
339
|
-
}
|
|
340
|
-
|
|
341
|
-
/**
|
|
342
|
-
* Get multiple blocks back from an (async) iterable of cids
|
|
343
|
-
*/
|
|
344
|
-
async * getMany (cids: AwaitIterable<CID>, options: GetOfflineOptions & AbortOptions & ProgressOptions<GetManyBlocksProgressEvents> = {}): AsyncGenerator<Pair> {
|
|
345
|
-
const signal = anySignal([this.closeController.signal, options.signal])
|
|
346
|
-
setMaxListeners(Infinity, signal)
|
|
347
|
-
|
|
348
|
-
try {
|
|
349
|
-
yield * super.getMany(cids, {
|
|
350
|
-
...options,
|
|
351
|
-
signal
|
|
352
|
-
})
|
|
353
|
-
} finally {
|
|
354
|
-
signal.clear()
|
|
355
|
-
}
|
|
356
|
-
}
|
|
357
|
-
|
|
358
|
-
/**
|
|
359
|
-
* Delete a block from the blockstore
|
|
360
|
-
*/
|
|
361
|
-
async delete (cid: CID, options: AbortOptions & ProgressOptions<DeleteBlockProgressEvents> = {}): Promise<void> {
|
|
362
|
-
const signal = anySignal([this.closeController.signal, options.signal])
|
|
363
|
-
setMaxListeners(Infinity, signal)
|
|
364
|
-
|
|
365
|
-
try {
|
|
366
|
-
await super.delete(cid, {
|
|
367
|
-
...options,
|
|
368
|
-
signal
|
|
369
|
-
})
|
|
370
|
-
} finally {
|
|
371
|
-
signal.clear()
|
|
372
|
-
}
|
|
373
|
-
}
|
|
374
|
-
|
|
375
|
-
/**
|
|
376
|
-
* Delete multiple blocks from the blockstore
|
|
377
|
-
*/
|
|
378
|
-
async * deleteMany (cids: AwaitIterable<CID>, options: AbortOptions & ProgressOptions<DeleteManyBlocksProgressEvents> = {}): AsyncGenerator<CID> {
|
|
379
|
-
const signal = anySignal([this.closeController.signal, options.signal])
|
|
380
|
-
setMaxListeners(Infinity, signal)
|
|
381
|
-
|
|
382
|
-
try {
|
|
383
|
-
yield * super.deleteMany(cids, {
|
|
384
|
-
...options,
|
|
385
|
-
signal
|
|
386
|
-
})
|
|
387
|
-
} finally {
|
|
388
|
-
signal.clear()
|
|
389
|
-
}
|
|
390
|
-
}
|
|
391
|
-
|
|
392
|
-
async has (cid: CID, options: AbortOptions = {}): Promise<boolean> {
|
|
393
|
-
const signal = anySignal([this.closeController.signal, options.signal])
|
|
394
|
-
setMaxListeners(Infinity, signal)
|
|
395
|
-
|
|
396
|
-
try {
|
|
397
|
-
return await super.has(cid, {
|
|
398
|
-
...options,
|
|
399
|
-
signal
|
|
400
|
-
})
|
|
401
|
-
} finally {
|
|
402
|
-
signal.clear()
|
|
403
|
-
}
|
|
404
|
-
}
|
|
405
|
-
|
|
406
|
-
async * getAll (options: AbortOptions & ProgressOptions<GetAllBlocksProgressEvents> = {}): AsyncGenerator<Pair> {
|
|
407
|
-
const signal = anySignal([this.closeController.signal, options.signal])
|
|
408
|
-
setMaxListeners(Infinity, signal)
|
|
409
|
-
|
|
410
|
-
try {
|
|
411
|
-
yield * super.getAll({
|
|
412
|
-
...options,
|
|
413
|
-
signal
|
|
414
|
-
})
|
|
415
|
-
} finally {
|
|
416
|
-
signal.clear()
|
|
417
|
-
}
|
|
418
|
-
}
|
|
419
|
-
}
|
|
420
|
-
|
|
421
|
-
function isRetrievingBlockBroker (broker: BlockBroker): broker is Required<Pick<BlockBroker, 'retrieve'>> {
|
|
422
|
-
return typeof broker.retrieve === 'function'
|
|
423
|
-
}
|
|
424
|
-
|
|
425
|
-
export const getCidBlockVerifierFunction = (cid: CID, hasher: MultihashHasher): Required<BlockRetrievalOptions>['validateFn'] => {
|
|
426
|
-
if (hasher == null) {
|
|
427
|
-
throw new InvalidParametersError(`No hasher configured for multihash code 0x${cid.multihash.code.toString(16)}, please configure one. You can look up which hash this is at https://github.com/multiformats/multicodec/blob/master/table.csv`)
|
|
428
|
-
}
|
|
429
|
-
|
|
430
|
-
return async (block: Uint8Array): Promise<void> => {
|
|
431
|
-
// verify block
|
|
432
|
-
let hash: MultihashDigest<number>
|
|
433
|
-
const res = hasher.digest(block, {
|
|
434
|
-
// support truncated hashes where they are truncated
|
|
435
|
-
truncate: cid.multihash.digest.byteLength
|
|
436
|
-
})
|
|
437
|
-
|
|
438
|
-
if (isPromise(res)) {
|
|
439
|
-
hash = await res
|
|
440
|
-
} else {
|
|
441
|
-
hash = res
|
|
442
|
-
}
|
|
443
|
-
|
|
444
|
-
if (!uint8ArrayEquals(hash.digest, cid.multihash.digest)) {
|
|
445
|
-
// if a hash mismatch occurs for a TrustlessGatewayBlockBroker, we should try another gateway
|
|
446
|
-
throw new InvalidMultihashError('Hash of downloaded block did not match multihash from passed CID')
|
|
447
|
-
}
|
|
448
|
-
}
|
|
449
|
-
}
|
|
450
|
-
|
|
451
|
-
/**
|
|
452
|
-
* Race block providers cancelling any pending requests once the block has been
|
|
453
|
-
* found.
|
|
454
|
-
*/
|
|
455
|
-
async function raceBlockRetrievers (cid: CID, blockBrokers: BlockBroker[], hasher: MultihashHasher, options: AbortOptions & LoggerOptions): Promise<Uint8Array> {
|
|
456
|
-
const validateFn = getCidBlockVerifierFunction(cid, hasher)
|
|
457
|
-
|
|
458
|
-
const controller = new AbortController()
|
|
459
|
-
const signal = anySignal([controller.signal, options.signal])
|
|
460
|
-
setMaxListeners(Infinity, controller.signal, signal)
|
|
461
|
-
|
|
462
|
-
const retrievers: Array<Required<Pick<BlockBroker, 'retrieve'>>> = []
|
|
463
|
-
|
|
464
|
-
for (const broker of blockBrokers) {
|
|
465
|
-
if (isRetrievingBlockBroker(broker)) {
|
|
466
|
-
retrievers.push(broker)
|
|
467
|
-
}
|
|
468
|
-
}
|
|
469
|
-
|
|
470
|
-
if (retrievers.length === 0) {
|
|
471
|
-
throw new InvalidConfigurationError(`No block brokers capable of retrieving blocks are configured, the CID ${cid} cannot be fetched from the network`)
|
|
472
|
-
}
|
|
473
|
-
|
|
474
|
-
try {
|
|
475
|
-
return await Promise.any(
|
|
476
|
-
retrievers
|
|
477
|
-
.map(async retriever => {
|
|
478
|
-
try {
|
|
479
|
-
let blocksWereValidated = false
|
|
480
|
-
const block = await retriever.retrieve(cid, {
|
|
481
|
-
...options,
|
|
482
|
-
signal,
|
|
483
|
-
validateFn: async (block: Uint8Array): Promise<void> => {
|
|
484
|
-
await validateFn(block)
|
|
485
|
-
options.signal?.throwIfAborted()
|
|
486
|
-
blocksWereValidated = true
|
|
487
|
-
}
|
|
488
|
-
})
|
|
489
|
-
|
|
490
|
-
if (!blocksWereValidated) {
|
|
491
|
-
// the blockBroker either did not throw an error when attempting to validate the block
|
|
492
|
-
// or did not call the validateFn at all. We should validate the block ourselves
|
|
493
|
-
await validateFn(block)
|
|
494
|
-
options.signal?.throwIfAborted()
|
|
495
|
-
}
|
|
496
|
-
|
|
497
|
-
return block
|
|
498
|
-
} catch (err) {
|
|
499
|
-
options.log.error('could not retrieve verified block for %c - %e', cid, err)
|
|
500
|
-
throw err
|
|
501
|
-
}
|
|
502
|
-
})
|
|
503
|
-
)
|
|
504
|
-
} catch (err: any) {
|
|
505
|
-
throw new LoadBlockFailedError(err.errors, `Failed to load block for ${cid}`)
|
|
506
|
-
} finally {
|
|
507
|
-
// we have the block from the fastest block retriever, abort any still
|
|
508
|
-
// in-flight retrieve attempts
|
|
509
|
-
controller.abort()
|
|
510
|
-
signal.clear()
|
|
511
|
-
}
|
|
512
|
-
}
|
|
@@ -0,0 +1,175 @@
|
|
|
1
|
+
import { setMaxListeners } from '@libp2p/interface'
|
|
2
|
+
import { anySignal } from 'any-signal'
|
|
3
|
+
import { Storage } from './storage.ts'
|
|
4
|
+
import type { StorageComponents } from './storage.ts'
|
|
5
|
+
import type { Pair, DeleteManyBlocksProgressEvents, DeleteBlockProgressEvents, GetBlockProgressEvents, GetManyBlocksProgressEvents, PutManyBlocksProgressEvents, PutBlockProgressEvents, GetAllBlocksProgressEvents, GetOfflineOptions, SessionBlockstore, SessionBlockBroker } from '@helia/interface/blocks'
|
|
6
|
+
import type { AbortOptions, PeerId } from '@libp2p/interface'
|
|
7
|
+
import type { Multiaddr } from '@multiformats/multiaddr'
|
|
8
|
+
import type { InputPair } from 'interface-blockstore'
|
|
9
|
+
import type { AwaitIterable } from 'interface-store'
|
|
10
|
+
import type { CID } from 'multiformats/cid'
|
|
11
|
+
import type { ProgressOptions } from 'progress-events'
|
|
12
|
+
|
|
13
|
+
export interface SessionStorageInit {
|
|
14
|
+
root: CID
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
/**
|
|
18
|
+
* Storage subclass that can cancel any ongoing operation at any point.
|
|
19
|
+
*/
|
|
20
|
+
export class SessionStorage extends Storage<SessionBlockBroker> implements SessionBlockstore {
|
|
21
|
+
private readonly closeController: AbortController
|
|
22
|
+
|
|
23
|
+
constructor (components: StorageComponents<SessionBlockBroker>, init: SessionStorageInit) {
|
|
24
|
+
super(components)
|
|
25
|
+
|
|
26
|
+
// because brokers are allowed to continue searching for providers after the
|
|
27
|
+
// session has been created, we need a way to tell them that the user has
|
|
28
|
+
// finished using the session any in-flight requests should be cancelled
|
|
29
|
+
this.closeController = new AbortController()
|
|
30
|
+
setMaxListeners(Infinity, this.closeController.signal)
|
|
31
|
+
|
|
32
|
+
this.log = components.logger.forComponent(`helia:session-storage:${init.root}`)
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
close (): void {
|
|
36
|
+
this.closeController.abort()
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
async addPeer (peer: PeerId | Multiaddr | Multiaddr[], options?: AbortOptions): Promise<void> {
|
|
40
|
+
await Promise.all(
|
|
41
|
+
this.blockBrokers
|
|
42
|
+
.map(broker => broker.addPeer(peer, options))
|
|
43
|
+
)
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
/**
|
|
47
|
+
* Put a block to the underlying datastore
|
|
48
|
+
*/
|
|
49
|
+
async put (cid: CID, block: Uint8Array, options: AbortOptions & ProgressOptions<PutBlockProgressEvents> = {}): Promise<CID> {
|
|
50
|
+
const signal = anySignal([this.closeController.signal, options.signal])
|
|
51
|
+
setMaxListeners(Infinity, signal)
|
|
52
|
+
|
|
53
|
+
try {
|
|
54
|
+
return await super.put(cid, block, {
|
|
55
|
+
...options,
|
|
56
|
+
signal
|
|
57
|
+
})
|
|
58
|
+
} finally {
|
|
59
|
+
signal.clear()
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
/**
|
|
64
|
+
* Put a multiple blocks to the underlying datastore
|
|
65
|
+
*/
|
|
66
|
+
async * putMany (blocks: AwaitIterable<InputPair>, options: AbortOptions & ProgressOptions<PutManyBlocksProgressEvents> = {}): AsyncGenerator<CID> {
|
|
67
|
+
const signal = anySignal([this.closeController.signal, options.signal])
|
|
68
|
+
setMaxListeners(Infinity, signal)
|
|
69
|
+
|
|
70
|
+
try {
|
|
71
|
+
yield * super.putMany(blocks, {
|
|
72
|
+
...options,
|
|
73
|
+
signal
|
|
74
|
+
})
|
|
75
|
+
} finally {
|
|
76
|
+
signal.clear()
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
/**
|
|
81
|
+
* Get a block by cid
|
|
82
|
+
*/
|
|
83
|
+
async * get (cid: CID, options: GetOfflineOptions & AbortOptions & ProgressOptions<GetBlockProgressEvents> = {}): AsyncGenerator<Uint8Array> {
|
|
84
|
+
const signal = anySignal([this.closeController.signal, options.signal])
|
|
85
|
+
setMaxListeners(Infinity, signal)
|
|
86
|
+
|
|
87
|
+
try {
|
|
88
|
+
yield * super.get(cid, {
|
|
89
|
+
...options,
|
|
90
|
+
signal
|
|
91
|
+
})
|
|
92
|
+
} finally {
|
|
93
|
+
signal.clear()
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
/**
|
|
98
|
+
* Get multiple blocks back from an (async) iterable of cids
|
|
99
|
+
*/
|
|
100
|
+
async * getMany (cids: AwaitIterable<CID>, options: GetOfflineOptions & AbortOptions & ProgressOptions<GetManyBlocksProgressEvents> = {}): AsyncGenerator<Pair> {
|
|
101
|
+
const signal = anySignal([this.closeController.signal, options.signal])
|
|
102
|
+
setMaxListeners(Infinity, signal)
|
|
103
|
+
|
|
104
|
+
try {
|
|
105
|
+
yield * super.getMany(cids, {
|
|
106
|
+
...options,
|
|
107
|
+
signal
|
|
108
|
+
})
|
|
109
|
+
} finally {
|
|
110
|
+
signal.clear()
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
/**
|
|
115
|
+
* Delete a block from the blockstore
|
|
116
|
+
*/
|
|
117
|
+
async delete (cid: CID, options: AbortOptions & ProgressOptions<DeleteBlockProgressEvents> = {}): Promise<void> {
|
|
118
|
+
const signal = anySignal([this.closeController.signal, options.signal])
|
|
119
|
+
setMaxListeners(Infinity, signal)
|
|
120
|
+
|
|
121
|
+
try {
|
|
122
|
+
await super.delete(cid, {
|
|
123
|
+
...options,
|
|
124
|
+
signal
|
|
125
|
+
})
|
|
126
|
+
} finally {
|
|
127
|
+
signal.clear()
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
/**
|
|
132
|
+
* Delete multiple blocks from the blockstore
|
|
133
|
+
*/
|
|
134
|
+
async * deleteMany (cids: AwaitIterable<CID>, options: AbortOptions & ProgressOptions<DeleteManyBlocksProgressEvents> = {}): AsyncGenerator<CID> {
|
|
135
|
+
const signal = anySignal([this.closeController.signal, options.signal])
|
|
136
|
+
setMaxListeners(Infinity, signal)
|
|
137
|
+
|
|
138
|
+
try {
|
|
139
|
+
yield * super.deleteMany(cids, {
|
|
140
|
+
...options,
|
|
141
|
+
signal
|
|
142
|
+
})
|
|
143
|
+
} finally {
|
|
144
|
+
signal.clear()
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
async has (cid: CID, options: AbortOptions = {}): Promise<boolean> {
|
|
149
|
+
const signal = anySignal([this.closeController.signal, options.signal])
|
|
150
|
+
setMaxListeners(Infinity, signal)
|
|
151
|
+
|
|
152
|
+
try {
|
|
153
|
+
return await super.has(cid, {
|
|
154
|
+
...options,
|
|
155
|
+
signal
|
|
156
|
+
})
|
|
157
|
+
} finally {
|
|
158
|
+
signal.clear()
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
async * getAll (options: AbortOptions & ProgressOptions<GetAllBlocksProgressEvents> = {}): AsyncGenerator<Pair> {
|
|
163
|
+
const signal = anySignal([this.closeController.signal, options.signal])
|
|
164
|
+
setMaxListeners(Infinity, signal)
|
|
165
|
+
|
|
166
|
+
try {
|
|
167
|
+
yield * super.getAll({
|
|
168
|
+
...options,
|
|
169
|
+
signal
|
|
170
|
+
})
|
|
171
|
+
} finally {
|
|
172
|
+
signal.clear()
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
}
|