@helia/utils 0.3.3-ec8bf66 → 0.3.3-f5a03fc
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.min.js +1 -1
- package/dist/src/index.d.ts +16 -11
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +9 -17
- package/dist/src/index.js.map +1 -1
- package/dist/src/pins.d.ts +3 -3
- package/dist/src/pins.d.ts.map +1 -1
- package/dist/src/pins.js +8 -9
- package/dist/src/pins.js.map +1 -1
- package/dist/src/utils/get-codec.d.ts +4 -0
- package/dist/src/utils/get-codec.d.ts.map +1 -0
- package/dist/src/utils/get-codec.js +38 -0
- package/dist/src/utils/get-codec.js.map +1 -0
- package/dist/src/utils/get-hasher.d.ts +4 -0
- package/dist/src/utils/get-hasher.d.ts.map +1 -0
- package/dist/src/utils/get-hasher.js +32 -0
- package/dist/src/utils/get-hasher.js.map +1 -0
- package/dist/src/utils/is-promise.d.ts +2 -0
- package/dist/src/utils/is-promise.d.ts.map +1 -0
- package/dist/src/utils/is-promise.js +4 -0
- package/dist/src/utils/is-promise.js.map +1 -0
- package/dist/src/utils/networked-storage.d.ts +3 -2
- package/dist/src/utils/networked-storage.d.ts.map +1 -1
- package/dist/src/utils/networked-storage.js +16 -6
- package/dist/src/utils/networked-storage.js.map +1 -1
- package/package.json +3 -3
- package/src/index.ts +27 -23
- package/src/pins.ts +9 -12
- package/src/utils/get-codec.ts +47 -0
- package/src/utils/get-hasher.ts +40 -0
- package/src/utils/is-promise.ts +3 -0
- package/src/utils/networked-storage.ts +21 -8
- package/dist/src/utils/dag-walkers.d.ts +0 -28
- package/dist/src/utils/dag-walkers.d.ts.map +0 -1
- package/dist/src/utils/dag-walkers.js +0 -171
- package/dist/src/utils/dag-walkers.js.map +0 -1
- package/dist/src/utils/default-hashers.d.ts +0 -3
- package/dist/src/utils/default-hashers.d.ts.map +0 -1
- package/dist/src/utils/default-hashers.js +0 -15
- package/dist/src/utils/default-hashers.js.map +0 -1
- package/src/utils/dag-walkers.ts +0 -198
- package/src/utils/default-hashers.ts +0 -18
package/src/index.ts
CHANGED
|
@@ -25,19 +25,20 @@ import { CustomProgressEvent } from 'progress-events'
|
|
|
25
25
|
import { PinsImpl } from './pins.js'
|
|
26
26
|
import { Routing as RoutingClass } from './routing.js'
|
|
27
27
|
import { BlockStorage } from './storage.js'
|
|
28
|
-
import { defaultDagWalkers } from './utils/dag-walkers.js'
|
|
29
28
|
import { assertDatastoreVersionIsCurrent } from './utils/datastore-version.js'
|
|
30
|
-
import {
|
|
29
|
+
import { getCodec } from './utils/get-codec.js'
|
|
30
|
+
import { getHasher } from './utils/get-hasher.js'
|
|
31
31
|
import { NetworkedStorage } from './utils/networked-storage.js'
|
|
32
|
-
import type {
|
|
32
|
+
import type { Await, CodecLoader, GCOptions, HasherLoader, Helia as HeliaInterface, Routing } from '@helia/interface'
|
|
33
33
|
import type { BlockBroker } from '@helia/interface/blocks'
|
|
34
34
|
import type { Pins } from '@helia/interface/pins'
|
|
35
35
|
import type { ComponentLogger, Logger, Metrics } from '@libp2p/interface'
|
|
36
36
|
import type { DNS } from '@multiformats/dns'
|
|
37
37
|
import type { Blockstore } from 'interface-blockstore'
|
|
38
38
|
import type { Datastore } from 'interface-datastore'
|
|
39
|
+
import type { BlockCodec } from 'multiformats'
|
|
39
40
|
import type { CID } from 'multiformats/cid'
|
|
40
|
-
import type {
|
|
41
|
+
import type { MultihashHasher } from 'multiformats/hashes/interface'
|
|
41
42
|
|
|
42
43
|
export { AbstractSession, type AbstractCreateSessionOptions } from './abstract-session.js'
|
|
43
44
|
export { BloomFilter } from './bloom-filter.js'
|
|
@@ -63,12 +64,24 @@ export interface HeliaInit {
|
|
|
63
64
|
*/
|
|
64
65
|
hashers?: MultihashHasher[]
|
|
65
66
|
|
|
67
|
+
/**
|
|
68
|
+
* An optional function that can load a MultihashHasher on demand. May return
|
|
69
|
+
* a promise.
|
|
70
|
+
*/
|
|
71
|
+
loadHasher?(code: number): Await<MultihashHasher>
|
|
72
|
+
|
|
66
73
|
/**
|
|
67
74
|
* In order to pin CIDs that correspond to a DAG, it's necessary to know
|
|
68
75
|
* how to traverse that DAG. DAGWalkers take a block and yield any CIDs
|
|
69
76
|
* encoded within that block.
|
|
70
77
|
*/
|
|
71
|
-
|
|
78
|
+
codecs?: Array<BlockCodec<any, any>>
|
|
79
|
+
|
|
80
|
+
/**
|
|
81
|
+
* An optional function that can load a BlockCodec on demand. May return a
|
|
82
|
+
* promise.
|
|
83
|
+
*/
|
|
84
|
+
loadCodec?(code: number): Await<BlockCodec<any, any>>
|
|
72
85
|
|
|
73
86
|
/**
|
|
74
87
|
* A list of strategies used to fetch blocks when they are not present in
|
|
@@ -142,13 +155,13 @@ export interface HeliaInit {
|
|
|
142
155
|
interface Components {
|
|
143
156
|
blockstore: Blockstore
|
|
144
157
|
datastore: Datastore
|
|
145
|
-
hashers: Record<number, MultihashHasher>
|
|
146
|
-
dagWalkers: Record<number, DAGWalker>
|
|
147
158
|
logger: ComponentLogger
|
|
148
159
|
blockBrokers: BlockBroker[]
|
|
149
160
|
routing: Routing
|
|
150
161
|
dns: DNS
|
|
151
162
|
metrics?: Metrics
|
|
163
|
+
getCodec: CodecLoader
|
|
164
|
+
getHasher: HasherLoader
|
|
152
165
|
}
|
|
153
166
|
|
|
154
167
|
export class Helia implements HeliaInterface {
|
|
@@ -157,8 +170,8 @@ export class Helia implements HeliaInterface {
|
|
|
157
170
|
public pins: Pins
|
|
158
171
|
public logger: ComponentLogger
|
|
159
172
|
public routing: Routing
|
|
160
|
-
public
|
|
161
|
-
public
|
|
173
|
+
public getCodec: CodecLoader
|
|
174
|
+
public getHasher: HasherLoader
|
|
162
175
|
public dns: DNS
|
|
163
176
|
public metrics?: Metrics
|
|
164
177
|
private readonly log: Logger
|
|
@@ -166,8 +179,8 @@ export class Helia implements HeliaInterface {
|
|
|
166
179
|
constructor (init: HeliaInit) {
|
|
167
180
|
this.logger = init.logger ?? defaultLogger()
|
|
168
181
|
this.log = this.logger.forComponent('helia')
|
|
169
|
-
this.
|
|
170
|
-
this.
|
|
182
|
+
this.getHasher = getHasher(init.hashers, init.loadHasher)
|
|
183
|
+
this.getCodec = getCodec(init.codecs, init.loadCodec)
|
|
171
184
|
this.dns = init.dns ?? dns()
|
|
172
185
|
this.metrics = init.metrics
|
|
173
186
|
|
|
@@ -175,10 +188,10 @@ export class Helia implements HeliaInterface {
|
|
|
175
188
|
const components: Components = {
|
|
176
189
|
blockstore: init.blockstore,
|
|
177
190
|
datastore: init.datastore,
|
|
178
|
-
hashers: this.hashers,
|
|
179
|
-
dagWalkers: this.dagWalkers,
|
|
180
191
|
logger: this.logger,
|
|
181
192
|
blockBrokers: [],
|
|
193
|
+
getHasher: this.getHasher,
|
|
194
|
+
getCodec: this.getCodec,
|
|
182
195
|
dns: this.dns,
|
|
183
196
|
metrics: this.metrics,
|
|
184
197
|
...(init.components ?? {})
|
|
@@ -207,7 +220,7 @@ export class Helia implements HeliaInterface {
|
|
|
207
220
|
})
|
|
208
221
|
|
|
209
222
|
const networkedStorage = new NetworkedStorage(components)
|
|
210
|
-
this.pins = new PinsImpl(init.datastore, networkedStorage, this.
|
|
223
|
+
this.pins = new PinsImpl(init.datastore, networkedStorage, this.getCodec)
|
|
211
224
|
this.blockstore = new BlockStorage(networkedStorage, this.pins, {
|
|
212
225
|
holdGcLock: init.holdGcLock ?? true
|
|
213
226
|
})
|
|
@@ -267,12 +280,3 @@ export class Helia implements HeliaInterface {
|
|
|
267
280
|
this.log('gc finished')
|
|
268
281
|
}
|
|
269
282
|
}
|
|
270
|
-
|
|
271
|
-
/**
|
|
272
|
-
* Used to check that the passed multihash has the passed code
|
|
273
|
-
*
|
|
274
|
-
* Remove after https://github.com/multiformats/js-multiformats/pull/308
|
|
275
|
-
*/
|
|
276
|
-
export function hasCode <T extends number> (digest: MultihashDigest, code: T): digest is MultihashDigest<T> {
|
|
277
|
-
return digest.code === code
|
|
278
|
-
}
|
package/src/pins.ts
CHANGED
|
@@ -2,10 +2,11 @@ import { Queue } from '@libp2p/utils/queue'
|
|
|
2
2
|
import * as cborg from 'cborg'
|
|
3
3
|
import { type Datastore, Key } from 'interface-datastore'
|
|
4
4
|
import { base36 } from 'multiformats/bases/base36'
|
|
5
|
+
import { createUnsafe } from 'multiformats/block'
|
|
5
6
|
import { CID, type Version } from 'multiformats/cid'
|
|
6
7
|
import { CustomProgressEvent, type ProgressOptions } from 'progress-events'
|
|
7
8
|
import { equals as uint8ArrayEquals } from 'uint8arrays/equals'
|
|
8
|
-
import type {
|
|
9
|
+
import type { CodecLoader } from '@helia/interface'
|
|
9
10
|
import type { GetBlockProgressEvents } from '@helia/interface/blocks'
|
|
10
11
|
import type { AddOptions, AddPinEvents, IsPinnedOptions, LsOptions, Pin, Pins, RmOptions } from '@helia/interface/pins'
|
|
11
12
|
import type { AbortOptions } from '@libp2p/interface'
|
|
@@ -59,12 +60,12 @@ function toDSKey (cid: CID): Key {
|
|
|
59
60
|
export class PinsImpl implements Pins {
|
|
60
61
|
private readonly datastore: Datastore
|
|
61
62
|
private readonly blockstore: Blockstore
|
|
62
|
-
private readonly
|
|
63
|
+
private readonly getCodec: CodecLoader
|
|
63
64
|
|
|
64
|
-
constructor (datastore: Datastore, blockstore: Blockstore,
|
|
65
|
+
constructor (datastore: Datastore, blockstore: Blockstore, getCodec: CodecLoader) {
|
|
65
66
|
this.datastore = datastore
|
|
66
67
|
this.blockstore = blockstore
|
|
67
|
-
this.
|
|
68
|
+
this.getCodec = getCodec
|
|
68
69
|
}
|
|
69
70
|
|
|
70
71
|
async * add (cid: CID<unknown, number, number, Version>, options: AddOptions = {}): AsyncGenerator<CID, void, undefined> {
|
|
@@ -119,18 +120,14 @@ export class PinsImpl implements Pins {
|
|
|
119
120
|
return
|
|
120
121
|
}
|
|
121
122
|
|
|
122
|
-
const
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
throw new Error(`No dag walker found for cid codec ${cid.code}`)
|
|
126
|
-
}
|
|
127
|
-
|
|
128
|
-
const block = await this.blockstore.get(cid, options)
|
|
123
|
+
const codec = await this.getCodec(cid.code)
|
|
124
|
+
const bytes = await this.blockstore.get(cid, options)
|
|
125
|
+
const block = createUnsafe({ bytes, cid, codec })
|
|
129
126
|
|
|
130
127
|
yield cid
|
|
131
128
|
|
|
132
129
|
// walk dag, ensure all blocks are present
|
|
133
|
-
for await (const cid of
|
|
130
|
+
for await (const [,cid] of block.links()) {
|
|
134
131
|
yield * await queue.add(async () => {
|
|
135
132
|
return this.#walkDag(cid, queue, {
|
|
136
133
|
...options,
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
/* eslint max-depth: ["error", 7] */
|
|
2
|
+
|
|
3
|
+
import { UnknownCodecError } from '@helia/interface'
|
|
4
|
+
import * as dagCbor from '@ipld/dag-cbor'
|
|
5
|
+
import * as dagJson from '@ipld/dag-json'
|
|
6
|
+
import * as dagPb from '@ipld/dag-pb'
|
|
7
|
+
import * as json from 'multiformats/codecs/json'
|
|
8
|
+
import * as raw from 'multiformats/codecs/raw'
|
|
9
|
+
import { isPromise } from './is-promise.js'
|
|
10
|
+
import type { Await } from '@helia/interface'
|
|
11
|
+
import type { BlockCodec } from 'multiformats/codecs/interface'
|
|
12
|
+
|
|
13
|
+
export function getCodec <T = any, Code extends number = any> (initialCodecs: Array<BlockCodec<any, any>> = [], loadCodec?: (code: number) => Await<BlockCodec<any, any>>): (code: Code) => Await<BlockCodec<Code, T>> {
|
|
14
|
+
const codecs: Record<number, BlockCodec<any, any>> = {
|
|
15
|
+
[dagPb.code]: dagPb,
|
|
16
|
+
[raw.code]: raw,
|
|
17
|
+
[dagCbor.code]: dagCbor,
|
|
18
|
+
[dagJson.code]: dagJson,
|
|
19
|
+
[json.code]: json
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
initialCodecs.forEach(codec => {
|
|
23
|
+
codecs[codec.code] = codec
|
|
24
|
+
})
|
|
25
|
+
|
|
26
|
+
return async (code) => {
|
|
27
|
+
let codec = codecs[code]
|
|
28
|
+
|
|
29
|
+
if (codec == null && loadCodec != null) {
|
|
30
|
+
const res = loadCodec(code)
|
|
31
|
+
|
|
32
|
+
if (isPromise(res)) {
|
|
33
|
+
codec = await res
|
|
34
|
+
} else {
|
|
35
|
+
codec = res
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
codecs[codec.code] = codec
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
if (codec != null) {
|
|
42
|
+
return codec
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
throw new UnknownCodecError(`Could not load codec for ${code}`)
|
|
46
|
+
}
|
|
47
|
+
}
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
import { UnknownHashAlgorithmError } from '@helia/interface'
|
|
2
|
+
import { identity } from 'multiformats/hashes/identity'
|
|
3
|
+
import { sha256, sha512 } from 'multiformats/hashes/sha2'
|
|
4
|
+
import { isPromise } from './is-promise.js'
|
|
5
|
+
import type { Await } from '@helia/interface'
|
|
6
|
+
import type { MultihashHasher } from 'multiformats/hashes/interface'
|
|
7
|
+
|
|
8
|
+
export function getHasher (initialHashers: MultihashHasher[] = [], loadHasher?: (code: number) => Await<MultihashHasher>): (code: number) => Await<MultihashHasher> {
|
|
9
|
+
const hashers: Record<number, MultihashHasher> = {
|
|
10
|
+
[sha256.code]: sha256,
|
|
11
|
+
[sha512.code]: sha512,
|
|
12
|
+
[identity.code]: identity
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
initialHashers.forEach(hasher => {
|
|
16
|
+
hashers[hasher.code] = hasher
|
|
17
|
+
})
|
|
18
|
+
|
|
19
|
+
return async (code) => {
|
|
20
|
+
let hasher = hashers[code]
|
|
21
|
+
|
|
22
|
+
if (hasher == null && loadHasher != null) {
|
|
23
|
+
const res = loadHasher(code)
|
|
24
|
+
|
|
25
|
+
if (isPromise(res)) {
|
|
26
|
+
hasher = await res
|
|
27
|
+
} else {
|
|
28
|
+
hasher = res
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
hashers[hasher.code] = hasher
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
if (hasher != null) {
|
|
35
|
+
return hasher
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
throw new UnknownHashAlgorithmError(`No hasher configured for multihash code 0x${code.toString(16)}, please configure one. You can look up which hash this is at https://github.com/multiformats/multicodec/blob/master/table.csv`)
|
|
39
|
+
}
|
|
40
|
+
}
|
|
@@ -5,12 +5,14 @@ import filter from 'it-filter'
|
|
|
5
5
|
import forEach from 'it-foreach'
|
|
6
6
|
import { CustomProgressEvent, type ProgressOptions } from 'progress-events'
|
|
7
7
|
import { equals as uint8ArrayEquals } from 'uint8arrays/equals'
|
|
8
|
+
import { isPromise } from './is-promise.js'
|
|
9
|
+
import type { HasherLoader } from '@helia/interface'
|
|
8
10
|
import type { BlockBroker, Blocks, Pair, DeleteManyBlocksProgressEvents, DeleteBlockProgressEvents, GetBlockProgressEvents, GetManyBlocksProgressEvents, PutManyBlocksProgressEvents, PutBlockProgressEvents, GetAllBlocksProgressEvents, GetOfflineOptions, BlockRetrievalOptions, CreateSessionOptions, SessionBlockstore } from '@helia/interface/blocks'
|
|
9
11
|
import type { AbortOptions, ComponentLogger, Logger, LoggerOptions, Startable } from '@libp2p/interface'
|
|
10
12
|
import type { Blockstore } from 'interface-blockstore'
|
|
11
13
|
import type { AwaitIterable } from 'interface-store'
|
|
12
14
|
import type { CID } from 'multiformats/cid'
|
|
13
|
-
import type { MultihashHasher } from 'multiformats/hashes/interface'
|
|
15
|
+
import type { MultihashDigest, MultihashHasher } from 'multiformats/hashes/interface'
|
|
14
16
|
|
|
15
17
|
export interface GetOptions extends AbortOptions {
|
|
16
18
|
progress?(evt: Event): void
|
|
@@ -20,12 +22,12 @@ export interface StorageComponents {
|
|
|
20
22
|
blockstore: Blockstore
|
|
21
23
|
logger: ComponentLogger
|
|
22
24
|
blockBrokers: BlockBroker[]
|
|
23
|
-
|
|
25
|
+
getHasher: HasherLoader
|
|
24
26
|
}
|
|
25
27
|
|
|
26
28
|
class Storage implements Blockstore {
|
|
27
29
|
protected readonly child: Blockstore
|
|
28
|
-
protected readonly
|
|
30
|
+
protected readonly getHasher: HasherLoader
|
|
29
31
|
protected log: Logger
|
|
30
32
|
protected readonly logger: ComponentLogger
|
|
31
33
|
protected readonly components: StorageComponents
|
|
@@ -38,7 +40,7 @@ class Storage implements Blockstore {
|
|
|
38
40
|
this.logger = components.logger
|
|
39
41
|
this.components = components
|
|
40
42
|
this.child = new IdentityBlockstore(components.blockstore)
|
|
41
|
-
this.
|
|
43
|
+
this.getHasher = components.getHasher
|
|
42
44
|
}
|
|
43
45
|
|
|
44
46
|
/**
|
|
@@ -91,9 +93,11 @@ class Storage implements Blockstore {
|
|
|
91
93
|
*/
|
|
92
94
|
async get (cid: CID, options: GetOfflineOptions & AbortOptions & ProgressOptions<GetBlockProgressEvents> = {}): Promise<Uint8Array> {
|
|
93
95
|
if (options.offline !== true && !(await this.child.has(cid, options))) {
|
|
96
|
+
const hasher = await this.getHasher(cid.multihash.code)
|
|
97
|
+
|
|
94
98
|
// we do not have the block locally, get it from a block provider
|
|
95
99
|
options.onProgress?.(new CustomProgressEvent<CID>('blocks:get:providers:get', cid))
|
|
96
|
-
const block = await raceBlockRetrievers(cid, this.components.blockBrokers,
|
|
100
|
+
const block = await raceBlockRetrievers(cid, this.components.blockBrokers, hasher, {
|
|
97
101
|
...options,
|
|
98
102
|
log: this.log
|
|
99
103
|
})
|
|
@@ -122,9 +126,11 @@ class Storage implements Blockstore {
|
|
|
122
126
|
|
|
123
127
|
yield * this.child.getMany(forEach(cids, async (cid): Promise<void> => {
|
|
124
128
|
if (options.offline !== true && !(await this.child.has(cid, options))) {
|
|
129
|
+
const hasher = await this.getHasher(cid.multihash.code)
|
|
130
|
+
|
|
125
131
|
// we do not have the block locally, get it from a block provider
|
|
126
132
|
options.onProgress?.(new CustomProgressEvent<CID>('blocks:get-many:providers:get', cid))
|
|
127
|
-
const block = await raceBlockRetrievers(cid, this.components.blockBrokers,
|
|
133
|
+
const block = await raceBlockRetrievers(cid, this.components.blockBrokers, hasher, {
|
|
128
134
|
...options,
|
|
129
135
|
log: this.log
|
|
130
136
|
})
|
|
@@ -219,7 +225,7 @@ export class NetworkedStorage extends Storage implements Blocks, Startable {
|
|
|
219
225
|
return new SessionStorage({
|
|
220
226
|
blockstore: this.child,
|
|
221
227
|
blockBrokers,
|
|
222
|
-
|
|
228
|
+
getHasher: this.getHasher,
|
|
223
229
|
logger: this.logger
|
|
224
230
|
}, {
|
|
225
231
|
root
|
|
@@ -395,7 +401,14 @@ export const getCidBlockVerifierFunction = (cid: CID, hasher: MultihashHasher):
|
|
|
395
401
|
|
|
396
402
|
return async (block: Uint8Array): Promise<void> => {
|
|
397
403
|
// verify block
|
|
398
|
-
|
|
404
|
+
let hash: MultihashDigest<number>
|
|
405
|
+
const res = hasher.digest(block)
|
|
406
|
+
|
|
407
|
+
if (isPromise(res)) {
|
|
408
|
+
hash = await res
|
|
409
|
+
} else {
|
|
410
|
+
hash = res
|
|
411
|
+
}
|
|
399
412
|
|
|
400
413
|
if (!uint8ArrayEquals(hash.digest, cid.multihash.digest)) {
|
|
401
414
|
// if a hash mismatch occurs for a TrustlessGatewayBlockBroker, we should try another gateway
|
|
@@ -1,28 +0,0 @@
|
|
|
1
|
-
import type { DAGWalker } from '@helia/interface';
|
|
2
|
-
/**
|
|
3
|
-
* Dag walker for dag-pb CIDs
|
|
4
|
-
*/
|
|
5
|
-
export declare const dagPbWalker: DAGWalker;
|
|
6
|
-
/**
|
|
7
|
-
* Dag walker for raw CIDs
|
|
8
|
-
*/
|
|
9
|
-
export declare const rawWalker: DAGWalker;
|
|
10
|
-
/**
|
|
11
|
-
* Dag walker for dag-cbor CIDs. Does not actually use dag-cbor since
|
|
12
|
-
* all we are interested in is extracting the the CIDs from the block
|
|
13
|
-
* so we can just use cborg for that.
|
|
14
|
-
*/
|
|
15
|
-
export declare const dagCborWalker: DAGWalker;
|
|
16
|
-
/**
|
|
17
|
-
* Dag walker for dag-json CIDs. Does not actually use dag-json since
|
|
18
|
-
* all we are interested in is extracting the the CIDs from the block
|
|
19
|
-
* so we can just use cborg/json for that.
|
|
20
|
-
*/
|
|
21
|
-
export declare const dagJsonWalker: DAGWalker;
|
|
22
|
-
/**
|
|
23
|
-
* Dag walker for json CIDs. JSON has no facility for linking to
|
|
24
|
-
* external blocks so the walker is a no-op.
|
|
25
|
-
*/
|
|
26
|
-
export declare const jsonWalker: DAGWalker;
|
|
27
|
-
export declare function defaultDagWalkers(walkers?: DAGWalker[]): Record<number, DAGWalker>;
|
|
28
|
-
//# sourceMappingURL=dag-walkers.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"dag-walkers.d.ts","sourceRoot":"","sources":["../../../src/utils/dag-walkers.ts"],"names":[],"mappings":"AAYA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAA;AAEjD;;GAEG;AACH,eAAO,MAAM,WAAW,EAAE,SAOzB,CAAA;AAED;;GAEG;AACH,eAAO,MAAM,SAAS,EAAE,SAKvB,CAAA;AAKD;;;;GAIG;AACH,eAAO,MAAM,aAAa,EAAE,SAuB3B,CAAA;AAsED;;;;GAIG;AACH,eAAO,MAAM,aAAa,EAAE,SA6B3B,CAAA;AAED;;;GAGG;AACH,eAAO,MAAM,UAAU,EAAE,SAGxB,CAAA;AAED,wBAAgB,iBAAiB,CAAE,OAAO,GAAE,SAAS,EAAO,GAAG,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,CAevF"}
|
|
@@ -1,171 +0,0 @@
|
|
|
1
|
-
/* eslint max-depth: ["error", 7] */
|
|
2
|
-
import * as dagCbor from '@ipld/dag-cbor';
|
|
3
|
-
import * as dagJson from '@ipld/dag-json';
|
|
4
|
-
import * as dagPb from '@ipld/dag-pb';
|
|
5
|
-
import * as cborg from 'cborg';
|
|
6
|
-
import { Type, Token } from 'cborg';
|
|
7
|
-
import * as cborgJson from 'cborg/json';
|
|
8
|
-
import { CID } from 'multiformats';
|
|
9
|
-
import { base64 } from 'multiformats/bases/base64';
|
|
10
|
-
import * as json from 'multiformats/codecs/json';
|
|
11
|
-
import * as raw from 'multiformats/codecs/raw';
|
|
12
|
-
/**
|
|
13
|
-
* Dag walker for dag-pb CIDs
|
|
14
|
-
*/
|
|
15
|
-
export const dagPbWalker = {
|
|
16
|
-
codec: dagPb.code,
|
|
17
|
-
*walk(block) {
|
|
18
|
-
const node = dagPb.decode(block);
|
|
19
|
-
yield* node.Links.map(l => l.Hash);
|
|
20
|
-
}
|
|
21
|
-
};
|
|
22
|
-
/**
|
|
23
|
-
* Dag walker for raw CIDs
|
|
24
|
-
*/
|
|
25
|
-
export const rawWalker = {
|
|
26
|
-
codec: raw.code,
|
|
27
|
-
*walk() {
|
|
28
|
-
// no embedded CIDs in a raw block
|
|
29
|
-
}
|
|
30
|
-
};
|
|
31
|
-
// https://github.com/ipfs/go-ipfs/issues/3570#issuecomment-273931692
|
|
32
|
-
const CID_TAG = 42;
|
|
33
|
-
/**
|
|
34
|
-
* Dag walker for dag-cbor CIDs. Does not actually use dag-cbor since
|
|
35
|
-
* all we are interested in is extracting the the CIDs from the block
|
|
36
|
-
* so we can just use cborg for that.
|
|
37
|
-
*/
|
|
38
|
-
export const dagCborWalker = {
|
|
39
|
-
codec: dagCbor.code,
|
|
40
|
-
*walk(block) {
|
|
41
|
-
const cids = [];
|
|
42
|
-
const tags = [];
|
|
43
|
-
tags[CID_TAG] = (bytes) => {
|
|
44
|
-
if (bytes[0] !== 0) {
|
|
45
|
-
throw new Error('Invalid CID for CBOR tag 42; expected leading 0x00');
|
|
46
|
-
}
|
|
47
|
-
const cid = CID.decode(bytes.subarray(1)); // ignore leading 0x00
|
|
48
|
-
cids.push(cid);
|
|
49
|
-
return cid;
|
|
50
|
-
};
|
|
51
|
-
cborg.decode(block, {
|
|
52
|
-
tags
|
|
53
|
-
});
|
|
54
|
-
yield* cids;
|
|
55
|
-
}
|
|
56
|
-
};
|
|
57
|
-
/**
|
|
58
|
-
* Borrowed from @ipld/dag-json
|
|
59
|
-
*/
|
|
60
|
-
class DagJsonTokenizer extends cborgJson.Tokenizer {
|
|
61
|
-
tokenBuffer;
|
|
62
|
-
constructor(data, options) {
|
|
63
|
-
super(data, options);
|
|
64
|
-
this.tokenBuffer = [];
|
|
65
|
-
}
|
|
66
|
-
done() {
|
|
67
|
-
return this.tokenBuffer.length === 0 && super.done();
|
|
68
|
-
}
|
|
69
|
-
_next() {
|
|
70
|
-
if (this.tokenBuffer.length > 0) {
|
|
71
|
-
// @ts-expect-error https://github.com/Microsoft/TypeScript/issues/30406
|
|
72
|
-
return this.tokenBuffer.pop();
|
|
73
|
-
}
|
|
74
|
-
return super.next();
|
|
75
|
-
}
|
|
76
|
-
/**
|
|
77
|
-
* Implements rules outlined in https://github.com/ipld/specs/pull/356
|
|
78
|
-
*/
|
|
79
|
-
next() {
|
|
80
|
-
const token = this._next();
|
|
81
|
-
if (token.type === Type.map) {
|
|
82
|
-
const keyToken = this._next();
|
|
83
|
-
if (keyToken.type === Type.string && keyToken.value === '/') {
|
|
84
|
-
const valueToken = this._next();
|
|
85
|
-
if (valueToken.type === Type.string) { // *must* be a CID
|
|
86
|
-
const breakToken = this._next(); // swallow the end-of-map token
|
|
87
|
-
if (breakToken.type !== Type.break) {
|
|
88
|
-
throw new Error('Invalid encoded CID form');
|
|
89
|
-
}
|
|
90
|
-
this.tokenBuffer.push(valueToken); // CID.parse will pick this up after our tag token
|
|
91
|
-
return new Token(Type.tag, 42, 0);
|
|
92
|
-
}
|
|
93
|
-
if (valueToken.type === Type.map) {
|
|
94
|
-
const innerKeyToken = this._next();
|
|
95
|
-
if (innerKeyToken.type === Type.string && innerKeyToken.value === 'bytes') {
|
|
96
|
-
const innerValueToken = this._next();
|
|
97
|
-
if (innerValueToken.type === Type.string) { // *must* be Bytes
|
|
98
|
-
for (let i = 0; i < 2; i++) {
|
|
99
|
-
const breakToken = this._next(); // swallow two end-of-map tokens
|
|
100
|
-
if (breakToken.type !== Type.break) {
|
|
101
|
-
throw new Error('Invalid encoded Bytes form');
|
|
102
|
-
}
|
|
103
|
-
}
|
|
104
|
-
const bytes = base64.decode(`m${innerValueToken.value}`);
|
|
105
|
-
return new Token(Type.bytes, bytes, innerValueToken.value.length);
|
|
106
|
-
}
|
|
107
|
-
this.tokenBuffer.push(innerValueToken); // bail
|
|
108
|
-
}
|
|
109
|
-
this.tokenBuffer.push(innerKeyToken); // bail
|
|
110
|
-
}
|
|
111
|
-
this.tokenBuffer.push(valueToken); // bail
|
|
112
|
-
}
|
|
113
|
-
this.tokenBuffer.push(keyToken); // bail
|
|
114
|
-
}
|
|
115
|
-
return token;
|
|
116
|
-
}
|
|
117
|
-
}
|
|
118
|
-
/**
|
|
119
|
-
* Dag walker for dag-json CIDs. Does not actually use dag-json since
|
|
120
|
-
* all we are interested in is extracting the the CIDs from the block
|
|
121
|
-
* so we can just use cborg/json for that.
|
|
122
|
-
*/
|
|
123
|
-
export const dagJsonWalker = {
|
|
124
|
-
codec: dagJson.code,
|
|
125
|
-
*walk(block) {
|
|
126
|
-
const cids = [];
|
|
127
|
-
const tags = [];
|
|
128
|
-
tags[CID_TAG] = (string) => {
|
|
129
|
-
const cid = CID.parse(string);
|
|
130
|
-
cids.push(cid);
|
|
131
|
-
return cid;
|
|
132
|
-
};
|
|
133
|
-
cborgJson.decode(block, {
|
|
134
|
-
tags,
|
|
135
|
-
tokenizer: new DagJsonTokenizer(block, {
|
|
136
|
-
tags,
|
|
137
|
-
allowIndefinite: true,
|
|
138
|
-
allowUndefined: true,
|
|
139
|
-
allowNaN: true,
|
|
140
|
-
allowInfinity: true,
|
|
141
|
-
allowBigInt: true,
|
|
142
|
-
strict: false,
|
|
143
|
-
rejectDuplicateMapKeys: false
|
|
144
|
-
})
|
|
145
|
-
});
|
|
146
|
-
yield* cids;
|
|
147
|
-
}
|
|
148
|
-
};
|
|
149
|
-
/**
|
|
150
|
-
* Dag walker for json CIDs. JSON has no facility for linking to
|
|
151
|
-
* external blocks so the walker is a no-op.
|
|
152
|
-
*/
|
|
153
|
-
export const jsonWalker = {
|
|
154
|
-
codec: json.code,
|
|
155
|
-
*walk() { }
|
|
156
|
-
};
|
|
157
|
-
export function defaultDagWalkers(walkers = []) {
|
|
158
|
-
const output = {};
|
|
159
|
-
[
|
|
160
|
-
dagPbWalker,
|
|
161
|
-
rawWalker,
|
|
162
|
-
dagCborWalker,
|
|
163
|
-
dagJsonWalker,
|
|
164
|
-
jsonWalker,
|
|
165
|
-
...walkers
|
|
166
|
-
].forEach(dagWalker => {
|
|
167
|
-
output[dagWalker.codec] = dagWalker;
|
|
168
|
-
});
|
|
169
|
-
return output;
|
|
170
|
-
}
|
|
171
|
-
//# sourceMappingURL=dag-walkers.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"dag-walkers.js","sourceRoot":"","sources":["../../../src/utils/dag-walkers.ts"],"names":[],"mappings":"AAAA,oCAAoC;AAEpC,OAAO,KAAK,OAAO,MAAM,gBAAgB,CAAA;AACzC,OAAO,KAAK,OAAO,MAAM,gBAAgB,CAAA;AACzC,OAAO,KAAK,KAAK,MAAM,cAAc,CAAA;AACrC,OAAO,KAAK,KAAK,MAAM,OAAO,CAAA;AAC9B,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,OAAO,CAAA;AACnC,OAAO,KAAK,SAAS,MAAM,YAAY,CAAA;AACvC,OAAO,EAAE,GAAG,EAAE,MAAM,cAAc,CAAA;AAClC,OAAO,EAAE,MAAM,EAAE,MAAM,2BAA2B,CAAA;AAClD,OAAO,KAAK,IAAI,MAAM,0BAA0B,CAAA;AAChD,OAAO,KAAK,GAAG,MAAM,yBAAyB,CAAA;AAG9C;;GAEG;AACH,MAAM,CAAC,MAAM,WAAW,GAAc;IACpC,KAAK,EAAE,KAAK,CAAC,IAAI;IACjB,CAAE,IAAI,CAAE,KAAK;QACX,MAAM,IAAI,GAAG,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;QAEhC,KAAM,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAA;IACrC,CAAC;CACF,CAAA;AAED;;GAEG;AACH,MAAM,CAAC,MAAM,SAAS,GAAc;IAClC,KAAK,EAAE,GAAG,CAAC,IAAI;IACf,CAAE,IAAI;QACJ,kCAAkC;IACpC,CAAC;CACF,CAAA;AAED,qEAAqE;AACrE,MAAM,OAAO,GAAG,EAAE,CAAA;AAElB;;;;GAIG;AACH,MAAM,CAAC,MAAM,aAAa,GAAc;IACtC,KAAK,EAAE,OAAO,CAAC,IAAI;IACnB,CAAE,IAAI,CAAE,KAAK;QACX,MAAM,IAAI,GAAU,EAAE,CAAA;QACtB,MAAM,IAAI,GAAuB,EAAE,CAAA;QACnC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE,EAAE;YACxB,IAAI,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,EAAE,CAAC;gBACnB,MAAM,IAAI,KAAK,CAAC,oDAAoD,CAAC,CAAA;YACvE,CAAC;YAED,MAAM,GAAG,GAAG,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAA,CAAC,sBAAsB;YAEhE,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;YAEd,OAAO,GAAG,CAAA;QACZ,CAAC,CAAA;QAED,KAAK,CAAC,MAAM,CAAC,KAAK,EAAE;YAClB,IAAI;SACL,CAAC,CAAA;QAEF,KAAM,CAAC,CAAC,IAAI,CAAA;IACd,CAAC;CACF,CAAA;AAED;;GAEG;AACH,MAAM,gBAAiB,SAAQ,SAAS,CAAC,SAAS;IAC/B,WAAW,CAAe;IAE3C,YAAa,IAAgB,EAAE,OAA6B;QAC1D,KAAK,CAAC,IAAI,EAAE,OAAO,CAAC,CAAA;QAEpB,IAAI,CAAC,WAAW,GAAG,EAAE,CAAA;IACvB,CAAC;IAED,IAAI;QACF,OAAO,IAAI,CAAC,WAAW,CAAC,MAAM,KAAK,CAAC,IAAI,KAAK,CAAC,IAAI,EAAE,CAAA;IACtD,CAAC;IAED,KAAK;QACH,IAAI,IAAI,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAChC,wEAAwE;YACxE,OAAO,IAAI,CAAC,WAAW,CAAC,GAAG,EAAE,CAAA;QAC/B,CAAC;QACD,OAAO,KAAK,CAAC,IAAI,EAAE,CAAA;IACrB,CAAC;IAED;;OAEG;IACH,IAAI;QACF,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,EAAE,CAAA;QAE1B,IAAI,KAAK,CAAC,IAAI,KAAK,IAAI,CAAC,GAAG,EAAE,CAAC;YAC5B,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,EAAE,CAAA;YAC7B,IAAI,QAAQ,CAAC,IAAI,KAAK,IAAI,CAAC,MAAM,IAAI,QAAQ,CAAC,KAAK,KAAK,GAAG,EAAE,CAAC;gBAC5D,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,EAAE,CAAA;gBAC/B,IAAI,UAAU,CAAC,IAAI,KAAK,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,kBAAkB;oBACvD,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,EAAE,CAAA,CAAC,+BAA+B;oBAC/D,IAAI,UAAU,CAAC,IAAI,KAAK,IAAI,CAAC,KAAK,EAAE,CAAC;wBACnC,MAAM,IAAI,KAAK,CAAC,0BAA0B,CAAC,CAAA;oBAC7C,CAAC;oBACD,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,UAAU,CAAC,CAAA,CAAC,kDAAkD;oBACpF,OAAO,IAAI,KAAK,CAAC,IAAI,CAAC,GAAG,EAAE,EAAE,EAAE,CAAC,CAAC,CAAA;gBACnC,CAAC;gBACD,IAAI,UAAU,CAAC,IAAI,KAAK,IAAI,CAAC,GAAG,EAAE,CAAC;oBACjC,MAAM,aAAa,GAAG,IAAI,CAAC,KAAK,EAAE,CAAA;oBAClC,IAAI,aAAa,CAAC,IAAI,KAAK,IAAI,CAAC,MAAM,IAAI,aAAa,CAAC,KAAK,KAAK,OAAO,EAAE,CAAC;wBAC1E,MAAM,eAAe,GAAG,IAAI,CAAC,KAAK,EAAE,CAAA;wBACpC,IAAI,eAAe,CAAC,IAAI,KAAK,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,kBAAkB;4BAC5D,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;gCAC3B,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,EAAE,CAAA,CAAC,gCAAgC;gCAChE,IAAI,UAAU,CAAC,IAAI,KAAK,IAAI,CAAC,KAAK,EAAE,CAAC;oCACnC,MAAM,IAAI,KAAK,CAAC,4BAA4B,CAAC,CAAA;gCAC/C,CAAC;4BACH,CAAC;4BACD,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,eAAe,CAAC,KAAK,EAAE,CAAC,CAAA;4BACxD,OAAO,IAAI,KAAK,CAAC,IAAI,CAAC,KAAK,EAAE,KAAK,EAAE,eAAe,CAAC,KAAK,CAAC,MAAM,CAAC,CAAA;wBACnE,CAAC;wBACD,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,eAAe,CAAC,CAAA,CAAC,OAAO;oBAChD,CAAC;oBACD,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,aAAa,CAAC,CAAA,CAAC,OAAO;gBAC9C,CAAC;gBACD,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,UAAU,CAAC,CAAA,CAAC,OAAO;YAC3C,CAAC;YACD,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA,CAAC,OAAO;QACzC,CAAC;QACD,OAAO,KAAK,CAAA;IACd,CAAC;CACF;AAED;;;;GAIG;AACH,MAAM,CAAC,MAAM,aAAa,GAAc;IACtC,KAAK,EAAE,OAAO,CAAC,IAAI;IACnB,CAAE,IAAI,CAAE,KAAK;QACX,MAAM,IAAI,GAAU,EAAE,CAAA;QACtB,MAAM,IAAI,GAAuB,EAAE,CAAA;QACnC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,EAAE;YACzB,MAAM,GAAG,GAAG,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,CAAA;YAE7B,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;YAEd,OAAO,GAAG,CAAA;QACZ,CAAC,CAAA;QAED,SAAS,CAAC,MAAM,CAAC,KAAK,EAAE;YACtB,IAAI;YACJ,SAAS,EAAE,IAAI,gBAAgB,CAAC,KAAK,EAAE;gBACrC,IAAI;gBACJ,eAAe,EAAE,IAAI;gBACrB,cAAc,EAAE,IAAI;gBACpB,QAAQ,EAAE,IAAI;gBACd,aAAa,EAAE,IAAI;gBACnB,WAAW,EAAE,IAAI;gBACjB,MAAM,EAAE,KAAK;gBACb,sBAAsB,EAAE,KAAK;aAC9B,CAAC;SACH,CAAC,CAAA;QAEF,KAAM,CAAC,CAAC,IAAI,CAAA;IACd,CAAC;CACF,CAAA;AAED;;;GAGG;AACH,MAAM,CAAC,MAAM,UAAU,GAAc;IACnC,KAAK,EAAE,IAAI,CAAC,IAAI;IAChB,CAAE,IAAI,KAAK,CAAC;CACb,CAAA;AAED,MAAM,UAAU,iBAAiB,CAAE,UAAuB,EAAE;IAC1D,MAAM,MAAM,GAA8B,EAAE,CAE3C;IAAA;QACC,WAAW;QACX,SAAS;QACT,aAAa;QACb,aAAa;QACb,UAAU;QACV,GAAG,OAAO;KACX,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE;QACpB,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,GAAG,SAAS,CAAA;IACrC,CAAC,CAAC,CAAA;IAEF,OAAO,MAAM,CAAA;AACf,CAAC"}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"default-hashers.d.ts","sourceRoot":"","sources":["../../../src/utils/default-hashers.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,+BAA+B,CAAA;AAEpE,wBAAgB,cAAc,CAAE,OAAO,GAAE,eAAe,EAAO,GAAG,MAAM,CAAC,MAAM,EAAE,eAAe,CAAC,CAahG"}
|
|
@@ -1,15 +0,0 @@
|
|
|
1
|
-
import { identity } from 'multiformats/hashes/identity';
|
|
2
|
-
import { sha256, sha512 } from 'multiformats/hashes/sha2';
|
|
3
|
-
export function defaultHashers(hashers = []) {
|
|
4
|
-
const output = {};
|
|
5
|
-
[
|
|
6
|
-
sha256,
|
|
7
|
-
sha512,
|
|
8
|
-
identity,
|
|
9
|
-
...hashers
|
|
10
|
-
].forEach(hasher => {
|
|
11
|
-
output[hasher.code] = hasher;
|
|
12
|
-
});
|
|
13
|
-
return output;
|
|
14
|
-
}
|
|
15
|
-
//# sourceMappingURL=default-hashers.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"default-hashers.js","sourceRoot":"","sources":["../../../src/utils/default-hashers.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,8BAA8B,CAAA;AACvD,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,0BAA0B,CAAA;AAGzD,MAAM,UAAU,cAAc,CAAE,UAA6B,EAAE;IAC7D,MAAM,MAAM,GAAoC,EAAE,CAEjD;IAAA;QACC,MAAM;QACN,MAAM;QACN,QAAQ;QACR,GAAG,OAAO;KACX,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE;QACjB,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,MAAM,CAAA;IAC9B,CAAC,CAAC,CAAA;IAEF,OAAO,MAAM,CAAA;AACf,CAAC"}
|