@autonomys/auto-dag-data 1.0.7 → 1.0.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +26 -16
- package/dist/compression/index.d.ts.map +1 -1
- package/dist/compression/index.js +1 -1
- package/dist/encryption/index.d.ts +3 -2
- package/dist/encryption/index.d.ts.map +1 -1
- package/dist/ipld/chunker.d.ts +4 -2
- package/dist/ipld/chunker.d.ts.map +1 -1
- package/dist/ipld/chunker.js +23 -21
- package/dist/ipld/nodes.d.ts.map +1 -1
- package/dist/ipld/nodes.js +12 -11
- package/jest.config.ts +1 -0
- package/package.json +2 -2
- package/src/compression/index.ts +1 -1
- package/src/encryption/index.ts +3 -2
- package/src/ipld/chunker.ts +17 -15
- package/src/ipld/nodes.ts +12 -11
- package/tests/chunker.spec.ts +133 -4
- package/tests/compression.spec.ts +81 -0
- package/tests/encryption.spec.ts +21 -0
- package/tests/fileRetrievability.spec.ts +181 -0
- package/tests/nodes.spec.ts +25 -2
- package/tests/offchainMetadata.spec.ts +149 -0
package/src/ipld/nodes.ts
CHANGED
|
@@ -2,13 +2,13 @@ import { CID } from 'multiformats/cid'
|
|
|
2
2
|
import { FileUploadOptions, OffchainMetadata } from '../metadata/index.js'
|
|
3
3
|
import { encodeIPLDNodeData, MetadataType } from '../metadata/onchain/index.js'
|
|
4
4
|
import { stringifyMetadata } from '../utils/metadata.js'
|
|
5
|
-
import {
|
|
5
|
+
import { DEFAULT_NODE_MAX_SIZE, ensureNodeMaxSize } from './chunker.js'
|
|
6
6
|
import { createNode, PBNode } from './index.js'
|
|
7
7
|
|
|
8
8
|
/// Creates a file chunk ipld node
|
|
9
9
|
export const createFileChunkIpldNode = (
|
|
10
10
|
data: Buffer,
|
|
11
|
-
maxNodeSize: number =
|
|
11
|
+
maxNodeSize: number = DEFAULT_NODE_MAX_SIZE,
|
|
12
12
|
): PBNode =>
|
|
13
13
|
ensureNodeMaxSize(
|
|
14
14
|
createNode(
|
|
@@ -31,7 +31,7 @@ export const createChunkedFileIpldNode = (
|
|
|
31
31
|
size: bigint,
|
|
32
32
|
linkDepth: number,
|
|
33
33
|
name?: string,
|
|
34
|
-
maxNodeSize: number =
|
|
34
|
+
maxNodeSize: number = DEFAULT_NODE_MAX_SIZE,
|
|
35
35
|
uploadOptions?: FileUploadOptions,
|
|
36
36
|
): PBNode =>
|
|
37
37
|
ensureNodeMaxSize(
|
|
@@ -53,7 +53,7 @@ export const createFileInlinkIpldNode = (
|
|
|
53
53
|
links: CID[],
|
|
54
54
|
size: number,
|
|
55
55
|
linkDepth: number,
|
|
56
|
-
maxNodeSize: number =
|
|
56
|
+
maxNodeSize: number = DEFAULT_NODE_MAX_SIZE,
|
|
57
57
|
): PBNode =>
|
|
58
58
|
ensureNodeMaxSize(
|
|
59
59
|
createNode(
|
|
@@ -74,7 +74,7 @@ export const createSingleFileIpldNode = (
|
|
|
74
74
|
data: Buffer,
|
|
75
75
|
name?: string,
|
|
76
76
|
uploadOptions?: FileUploadOptions,
|
|
77
|
-
maxNodeSize: number =
|
|
77
|
+
maxNodeSize: number = DEFAULT_NODE_MAX_SIZE,
|
|
78
78
|
): PBNode =>
|
|
79
79
|
ensureNodeMaxSize(
|
|
80
80
|
createNode(
|
|
@@ -98,7 +98,7 @@ export const createMetadataInlinkIpldNode = (
|
|
|
98
98
|
links: CID[],
|
|
99
99
|
size: number,
|
|
100
100
|
linkDepth: number,
|
|
101
|
-
maxNodeSize: number =
|
|
101
|
+
maxNodeSize: number = DEFAULT_NODE_MAX_SIZE,
|
|
102
102
|
): PBNode =>
|
|
103
103
|
ensureNodeMaxSize(
|
|
104
104
|
createNode(
|
|
@@ -129,7 +129,7 @@ export const createSingleMetadataIpldNode = (data: Buffer, name?: string): PBNod
|
|
|
129
129
|
|
|
130
130
|
export const createMetadataChunkIpldNode = (
|
|
131
131
|
data: Buffer,
|
|
132
|
-
maxNodeSize: number =
|
|
132
|
+
maxNodeSize: number = DEFAULT_NODE_MAX_SIZE,
|
|
133
133
|
): PBNode =>
|
|
134
134
|
ensureNodeMaxSize(
|
|
135
135
|
createNode(
|
|
@@ -148,7 +148,7 @@ export const createChunkedMetadataIpldNode = (
|
|
|
148
148
|
size: bigint,
|
|
149
149
|
linkDepth: number,
|
|
150
150
|
name?: string,
|
|
151
|
-
maxNodeSize: number =
|
|
151
|
+
maxNodeSize: number = DEFAULT_NODE_MAX_SIZE,
|
|
152
152
|
): PBNode =>
|
|
153
153
|
ensureNodeMaxSize(
|
|
154
154
|
createNode(
|
|
@@ -171,7 +171,7 @@ export const createFolderIpldNode = (
|
|
|
171
171
|
name: string,
|
|
172
172
|
linkDepth: number,
|
|
173
173
|
size: bigint,
|
|
174
|
-
maxNodeSize: number =
|
|
174
|
+
maxNodeSize: number = DEFAULT_NODE_MAX_SIZE,
|
|
175
175
|
uploadOptions?: FileUploadOptions,
|
|
176
176
|
): PBNode =>
|
|
177
177
|
ensureNodeMaxSize(
|
|
@@ -191,7 +191,7 @@ export const createFolderIpldNode = (
|
|
|
191
191
|
export const createFolderInlinkIpldNode = (
|
|
192
192
|
links: CID[],
|
|
193
193
|
linkDepth: number,
|
|
194
|
-
maxNodeSize: number =
|
|
194
|
+
maxNodeSize: number = DEFAULT_NODE_MAX_SIZE,
|
|
195
195
|
): PBNode =>
|
|
196
196
|
ensureNodeMaxSize(
|
|
197
197
|
createNode(
|
|
@@ -207,7 +207,7 @@ export const createFolderInlinkIpldNode = (
|
|
|
207
207
|
/// Creates a metadata ipld node
|
|
208
208
|
export const createMetadataNode = (
|
|
209
209
|
metadata: OffchainMetadata,
|
|
210
|
-
maxNodeSize: number =
|
|
210
|
+
maxNodeSize: number = DEFAULT_NODE_MAX_SIZE,
|
|
211
211
|
): PBNode => {
|
|
212
212
|
const data = Buffer.from(stringifyMetadata(metadata))
|
|
213
213
|
|
|
@@ -218,6 +218,7 @@ export const createMetadataNode = (
|
|
|
218
218
|
name: metadata.name,
|
|
219
219
|
linkDepth: 0,
|
|
220
220
|
data,
|
|
221
|
+
size: BigInt(data.length).valueOf(),
|
|
221
222
|
}),
|
|
222
223
|
),
|
|
223
224
|
maxNodeSize,
|
package/tests/chunker.spec.ts
CHANGED
|
@@ -1,14 +1,24 @@
|
|
|
1
1
|
import { BaseBlockstore, MemoryBlockstore } from 'blockstore-core'
|
|
2
|
-
import { cidOfNode, cidToString, createSingleFileIpldNode } from '../src'
|
|
3
2
|
import {
|
|
3
|
+
cidOfNode,
|
|
4
|
+
cidToString,
|
|
5
|
+
createFileChunkIpldNode,
|
|
6
|
+
createSingleFileIpldNode,
|
|
7
|
+
fileBuilders,
|
|
8
|
+
} from '../src'
|
|
9
|
+
import {
|
|
10
|
+
DEFAULT_MAX_CHUNK_SIZE,
|
|
4
11
|
LINK_SIZE_IN_BYTES,
|
|
12
|
+
MAX_NAME_SIZE,
|
|
5
13
|
NODE_METADATA_SIZE,
|
|
14
|
+
processBufferToIPLDFormatFromChunks,
|
|
15
|
+
processChunksToIPLDFormat,
|
|
6
16
|
processFileToIPLDFormat,
|
|
7
17
|
processFolderToIPLDFormat,
|
|
8
18
|
processMetadataToIPLDFormat,
|
|
9
19
|
} from '../src/ipld/chunker'
|
|
10
|
-
import { createNode, decodeNode, PBNode } from '../src/ipld/utils'
|
|
11
|
-
import {
|
|
20
|
+
import { createNode, decodeNode, encodeNode, PBNode } from '../src/ipld/utils'
|
|
21
|
+
import { fileMetadata, IPLDNodeData, MetadataType, OffchainMetadata } from '../src/metadata'
|
|
12
22
|
|
|
13
23
|
describe('chunker', () => {
|
|
14
24
|
describe('file creation', () => {
|
|
@@ -88,6 +98,22 @@ describe('chunker', () => {
|
|
|
88
98
|
})
|
|
89
99
|
})
|
|
90
100
|
|
|
101
|
+
it('create a file with long name should throw an error', async () => {
|
|
102
|
+
const name = 'a'.repeat(MAX_NAME_SIZE + 1)
|
|
103
|
+
const blockstore = new MemoryBlockstore()
|
|
104
|
+
expect(() =>
|
|
105
|
+
processFileToIPLDFormat(blockstore, [Buffer.from('hello')], BigInt(5), name),
|
|
106
|
+
).toThrow(`Filename is too long: ${name.length} > ${MAX_NAME_SIZE}`)
|
|
107
|
+
})
|
|
108
|
+
|
|
109
|
+
it('create a file with long name from buffer should throw an error', async () => {
|
|
110
|
+
const name = 'a'.repeat(MAX_NAME_SIZE + 1)
|
|
111
|
+
const blockstore = new MemoryBlockstore()
|
|
112
|
+
await expect(
|
|
113
|
+
processBufferToIPLDFormatFromChunks(blockstore, [], name, BigInt(5), fileBuilders),
|
|
114
|
+
).rejects.toThrow(`Filename is too long: ${name.length} > ${MAX_NAME_SIZE}`)
|
|
115
|
+
})
|
|
116
|
+
|
|
91
117
|
it('create a file dag with inlinks', async () => {
|
|
92
118
|
const chunkLength = 1000
|
|
93
119
|
const maxNodeSize = chunkLength + NODE_METADATA_SIZE
|
|
@@ -194,6 +220,89 @@ describe('chunker', () => {
|
|
|
194
220
|
expect(rootCount).toBe(1)
|
|
195
221
|
expect(inlinkCount).toBe(3)
|
|
196
222
|
})
|
|
223
|
+
|
|
224
|
+
it('create a folder with long name should throw an error', async () => {
|
|
225
|
+
const name = 'a'.repeat(MAX_NAME_SIZE + 1)
|
|
226
|
+
const blockstore = new MemoryBlockstore()
|
|
227
|
+
await expect(processFolderToIPLDFormat(blockstore, [], name, BigInt(1000))).rejects.toThrow(
|
|
228
|
+
`Filename is too long: ${name.length} > ${MAX_NAME_SIZE}`,
|
|
229
|
+
)
|
|
230
|
+
})
|
|
231
|
+
})
|
|
232
|
+
|
|
233
|
+
describe('asyncronous file creation', () => {
|
|
234
|
+
it('process chunks to IPLD format should return the leftover buffer', async () => {
|
|
235
|
+
const filename = 'test.txt'
|
|
236
|
+
const chunkSize = DEFAULT_MAX_CHUNK_SIZE
|
|
237
|
+
const chunksCount = 1.5
|
|
238
|
+
const buffer = Buffer.from(
|
|
239
|
+
Array.from({ length: chunkSize * chunksCount })
|
|
240
|
+
.map(() => Math.floor(Math.random() * 16).toString(16))
|
|
241
|
+
.join(''),
|
|
242
|
+
)
|
|
243
|
+
|
|
244
|
+
const leftoverSize = buffer.length % chunkSize
|
|
245
|
+
const blockstore = new MemoryBlockstore()
|
|
246
|
+
const leftover = await processChunksToIPLDFormat(blockstore, [buffer], fileBuilders)
|
|
247
|
+
expect(leftover.length).toBe(leftoverSize)
|
|
248
|
+
})
|
|
249
|
+
|
|
250
|
+
it('process chunks with exact chunk size len(leftover)=0', async () => {
|
|
251
|
+
const filename = 'test.txt'
|
|
252
|
+
const chunkSize = DEFAULT_MAX_CHUNK_SIZE
|
|
253
|
+
const chunksCount = 4
|
|
254
|
+
const buffer = Buffer.from(
|
|
255
|
+
Array.from({ length: chunkSize * chunksCount })
|
|
256
|
+
.map(() => Math.floor(Math.random() * 16).toString(16))
|
|
257
|
+
.join(''),
|
|
258
|
+
)
|
|
259
|
+
|
|
260
|
+
const blockstore = new MemoryBlockstore()
|
|
261
|
+
const leftover = await processChunksToIPLDFormat(blockstore, [buffer], fileBuilders)
|
|
262
|
+
|
|
263
|
+
expect(leftover.length).toBe(0)
|
|
264
|
+
})
|
|
265
|
+
|
|
266
|
+
it('process file by chunks', async () => {
|
|
267
|
+
const filename = 'test.txt'
|
|
268
|
+
const chunkSize = DEFAULT_MAX_CHUNK_SIZE
|
|
269
|
+
const chunksCount = 4.5
|
|
270
|
+
const buffer = Buffer.from(
|
|
271
|
+
Array.from({ length: chunkSize * chunksCount })
|
|
272
|
+
.map(() => Math.floor(Math.random() * 16).toString(16))
|
|
273
|
+
.join(''),
|
|
274
|
+
)
|
|
275
|
+
|
|
276
|
+
const blockstore = new MemoryBlockstore()
|
|
277
|
+
const leftover = await processChunksToIPLDFormat(blockstore, [buffer], fileBuilders)
|
|
278
|
+
const leftoverCid = createFileChunkIpldNode(leftover)
|
|
279
|
+
await blockstore.put(cidOfNode(leftoverCid), encodeNode(leftoverCid))
|
|
280
|
+
|
|
281
|
+
const mapCIDs = (async function* () {
|
|
282
|
+
for await (const { cid } of blockstore.getAll()) {
|
|
283
|
+
yield cid
|
|
284
|
+
}
|
|
285
|
+
})()
|
|
286
|
+
|
|
287
|
+
const headCID = await processBufferToIPLDFormatFromChunks(
|
|
288
|
+
blockstore,
|
|
289
|
+
mapCIDs,
|
|
290
|
+
filename,
|
|
291
|
+
BigInt(buffer.length),
|
|
292
|
+
fileBuilders,
|
|
293
|
+
)
|
|
294
|
+
|
|
295
|
+
const headNode = decodeNode(await blockstore.get(headCID))
|
|
296
|
+
expect(headNode?.Links.length).toBe(Math.ceil(chunksCount))
|
|
297
|
+
expect(cidToString(headNode?.Links[headNode.Links.length - 1].Hash)).toEqual(
|
|
298
|
+
cidToString(cidOfNode(leftoverCid)),
|
|
299
|
+
)
|
|
300
|
+
const ipldMetadata = IPLDNodeData.decode(headNode?.Data ?? new Uint8Array())
|
|
301
|
+
expect(ipldMetadata.name).toBe(filename)
|
|
302
|
+
expect(ipldMetadata.type).toBe(MetadataType.File)
|
|
303
|
+
expect(ipldMetadata.linkDepth).toBe(1)
|
|
304
|
+
expect(ipldMetadata.size!.toString()).toBe(buffer.length.toString())
|
|
305
|
+
})
|
|
197
306
|
})
|
|
198
307
|
|
|
199
308
|
describe('metadata creation', () => {
|
|
@@ -209,11 +318,31 @@ describe('chunker', () => {
|
|
|
209
318
|
}
|
|
210
319
|
|
|
211
320
|
const blockstore = new MemoryBlockstore()
|
|
212
|
-
|
|
321
|
+
await processMetadataToIPLDFormat(blockstore, metadata)
|
|
213
322
|
const nodes = await nodesFromBlockstore(blockstore)
|
|
214
323
|
expect(nodes.length).toBe(1)
|
|
215
324
|
})
|
|
216
325
|
|
|
326
|
+
it('create a metadata dag with long name should throw an error', async () => {
|
|
327
|
+
const name = 'a'.repeat(MAX_NAME_SIZE + 1)
|
|
328
|
+
const metadata = fileMetadata(
|
|
329
|
+
cidOfNode(createNode(Buffer.from(Math.random().toString()))),
|
|
330
|
+
[
|
|
331
|
+
{
|
|
332
|
+
cid: cidToString(cidOfNode(createNode(Buffer.from(Math.random().toString())))),
|
|
333
|
+
size: BigInt(1000),
|
|
334
|
+
},
|
|
335
|
+
],
|
|
336
|
+
BigInt(1000),
|
|
337
|
+
name,
|
|
338
|
+
)
|
|
339
|
+
|
|
340
|
+
const blockstore = new MemoryBlockstore()
|
|
341
|
+
await expect(processMetadataToIPLDFormat(blockstore, metadata)).rejects.toThrow(
|
|
342
|
+
`Filename is too long: ${name.length} > ${MAX_NAME_SIZE}`,
|
|
343
|
+
)
|
|
344
|
+
})
|
|
345
|
+
|
|
217
346
|
it('large metadata dag represented into multiple nodes', async () => {
|
|
218
347
|
const metadata: OffchainMetadata = {
|
|
219
348
|
type: 'file',
|
|
@@ -1,5 +1,10 @@
|
|
|
1
|
+
import { AwaitIterable } from 'interface-store'
|
|
1
2
|
import { compressFile, COMPRESSION_CHUNK_SIZE, CompressionAlgorithm, decompressFile } from '../src'
|
|
2
3
|
|
|
4
|
+
const awaitIterable = async (it: AwaitIterable<Buffer>) => {
|
|
5
|
+
for await (const _ of it);
|
|
6
|
+
}
|
|
7
|
+
|
|
3
8
|
describe('compression', () => {
|
|
4
9
|
it('compresses and decompresses a file with default options', async () => {
|
|
5
10
|
const file = Buffer.from('hello'.repeat(1000))
|
|
@@ -55,4 +60,80 @@ describe('compression', () => {
|
|
|
55
60
|
|
|
56
61
|
expect(decompressedBuffer.toString()).toBe(file.toString())
|
|
57
62
|
})
|
|
63
|
+
|
|
64
|
+
it('asynchronously iterates over the compressed file for chunked compression', async () => {
|
|
65
|
+
const chunkSize = COMPRESSION_CHUNK_SIZE
|
|
66
|
+
const chunks = 5
|
|
67
|
+
const chunk = Buffer.from('hello'.repeat(chunkSize))
|
|
68
|
+
const compressed = compressFile(
|
|
69
|
+
(async function* () {
|
|
70
|
+
for (let i = 0; i < chunks; i++) {
|
|
71
|
+
yield chunk
|
|
72
|
+
await new Promise((resolve) => setTimeout(resolve, 50))
|
|
73
|
+
}
|
|
74
|
+
})(),
|
|
75
|
+
{
|
|
76
|
+
level: 9,
|
|
77
|
+
algorithm: CompressionAlgorithm.ZLIB,
|
|
78
|
+
},
|
|
79
|
+
)
|
|
80
|
+
|
|
81
|
+
await awaitIterable(compressed)
|
|
82
|
+
}, 10_000)
|
|
83
|
+
|
|
84
|
+
it('throws an error if the compression algorithm is not supported', async () => {
|
|
85
|
+
await expect(
|
|
86
|
+
awaitIterable(compressFile([Buffer.from('hello')], { algorithm: 'efwhhgfew' as any })),
|
|
87
|
+
).rejects.toThrow('Unsupported compression algorithm')
|
|
88
|
+
})
|
|
89
|
+
|
|
90
|
+
it('throws an error if the compression level is invalid', async () => {
|
|
91
|
+
await expect(
|
|
92
|
+
awaitIterable(
|
|
93
|
+
compressFile([Buffer.from('hello')], {
|
|
94
|
+
algorithm: CompressionAlgorithm.ZLIB,
|
|
95
|
+
level: -1 as any,
|
|
96
|
+
}),
|
|
97
|
+
),
|
|
98
|
+
).rejects.toThrow('Invalid compression level')
|
|
99
|
+
})
|
|
100
|
+
|
|
101
|
+
it('throws an error if the chunk size is invalid', async () => {
|
|
102
|
+
await expect(
|
|
103
|
+
awaitIterable(
|
|
104
|
+
compressFile([Buffer.from('hello')], {
|
|
105
|
+
algorithm: CompressionAlgorithm.ZLIB,
|
|
106
|
+
chunkSize: 0,
|
|
107
|
+
}),
|
|
108
|
+
),
|
|
109
|
+
).rejects.toThrow('Invalid chunk size')
|
|
110
|
+
})
|
|
111
|
+
|
|
112
|
+
it('throws an error if the decompression algorithm is not supported', async () => {
|
|
113
|
+
await expect(
|
|
114
|
+
awaitIterable(decompressFile([Buffer.from('hello')], { algorithm: 'efwhhgfew' as any })),
|
|
115
|
+
).rejects.toThrow('Unsupported compression algorithm')
|
|
116
|
+
})
|
|
117
|
+
|
|
118
|
+
it('throws an error if the decompression chunk size is invalid', async () => {
|
|
119
|
+
await expect(
|
|
120
|
+
awaitIterable(
|
|
121
|
+
decompressFile([Buffer.from('hello')], {
|
|
122
|
+
chunkSize: 0,
|
|
123
|
+
algorithm: CompressionAlgorithm.ZLIB,
|
|
124
|
+
}),
|
|
125
|
+
),
|
|
126
|
+
).rejects.toThrow('Invalid chunk size')
|
|
127
|
+
})
|
|
128
|
+
|
|
129
|
+
it('throws an error if the compression level is invalid', async () => {
|
|
130
|
+
await expect(
|
|
131
|
+
awaitIterable(
|
|
132
|
+
decompressFile([Buffer.from('hello')], {
|
|
133
|
+
level: -1 as any,
|
|
134
|
+
algorithm: CompressionAlgorithm.ZLIB,
|
|
135
|
+
}),
|
|
136
|
+
),
|
|
137
|
+
).rejects.toThrow('Invalid compression level')
|
|
138
|
+
})
|
|
58
139
|
})
|
package/tests/encryption.spec.ts
CHANGED
|
@@ -1,5 +1,10 @@
|
|
|
1
|
+
import { AwaitIterable } from 'interface-store'
|
|
1
2
|
import { decryptFile, encryptFile, EncryptionAlgorithm } from '../src'
|
|
2
3
|
|
|
4
|
+
const awaitIterable = async (it: AwaitIterable<Buffer>) => {
|
|
5
|
+
for await (const _ of it);
|
|
6
|
+
}
|
|
7
|
+
|
|
3
8
|
describe('encryption', () => {
|
|
4
9
|
it('encrypts and decrypts a file with default chunk size', async () => {
|
|
5
10
|
const chunk = 'hello'
|
|
@@ -101,4 +106,20 @@ describe('encryption', () => {
|
|
|
101
106
|
decryptedBuffer = Buffer.concat([decryptedBuffer, chunk])
|
|
102
107
|
}
|
|
103
108
|
})
|
|
109
|
+
|
|
110
|
+
it('throws an error if the encryption algorithm is not supported', async () => {
|
|
111
|
+
await expect(
|
|
112
|
+
awaitIterable(
|
|
113
|
+
encryptFile([Buffer.from('hello')], 'password', { algorithm: 'efwhhgfew' as any }),
|
|
114
|
+
),
|
|
115
|
+
).rejects.toThrow('Unsupported encryption algorithm')
|
|
116
|
+
})
|
|
117
|
+
|
|
118
|
+
it('throws an error if the decryption algorithm is not supported', async () => {
|
|
119
|
+
await expect(
|
|
120
|
+
awaitIterable(
|
|
121
|
+
decryptFile([Buffer.from('hello')], 'password', { algorithm: 'efwhhgfew' as any }),
|
|
122
|
+
),
|
|
123
|
+
).rejects.toThrow('Unsupported encryption algorithm')
|
|
124
|
+
})
|
|
104
125
|
})
|
|
@@ -0,0 +1,181 @@
|
|
|
1
|
+
import { MemoryBlockstore } from 'blockstore-core'
|
|
2
|
+
import {
|
|
3
|
+
createChunkedFileIpldNode,
|
|
4
|
+
createSingleFileIpldNode,
|
|
5
|
+
decodeIPLDNodeData,
|
|
6
|
+
decodeNode,
|
|
7
|
+
DEFAULT_MAX_CHUNK_SIZE,
|
|
8
|
+
encodeNode,
|
|
9
|
+
fileBuilders,
|
|
10
|
+
processBufferToIPLDFormatFromChunks,
|
|
11
|
+
processChunksToIPLDFormat,
|
|
12
|
+
processFileToIPLDFormat,
|
|
13
|
+
} from '../src'
|
|
14
|
+
|
|
15
|
+
describe('file retrievability', () => {
|
|
16
|
+
it('should be able to retrieve a file', () => {
|
|
17
|
+
const filename = 'test.txt'
|
|
18
|
+
const buffer = Buffer.from('hello world')
|
|
19
|
+
const encodedNode = encodeNode(createSingleFileIpldNode(buffer, filename))
|
|
20
|
+
|
|
21
|
+
const decodedNode = decodeIPLDNodeData(encodedNode)
|
|
22
|
+
|
|
23
|
+
expect(decodedNode.name).toBe(filename)
|
|
24
|
+
expect(decodedNode.size!.toString()).toBe(buffer.length.toString())
|
|
25
|
+
expect(Buffer.from(decodedNode.data ?? '').toString()).toBe(buffer.toString())
|
|
26
|
+
})
|
|
27
|
+
|
|
28
|
+
it('should be able to retrieve a file with chunked file', async () => {
|
|
29
|
+
const filename = 'test.txt'
|
|
30
|
+
const expectedChunks = 4
|
|
31
|
+
const fileSize = expectedChunks * DEFAULT_MAX_CHUNK_SIZE
|
|
32
|
+
const buffer = Buffer.from(
|
|
33
|
+
Array.from({ length: fileSize })
|
|
34
|
+
.map(() => Math.floor(Math.random() * 16).toString(16))
|
|
35
|
+
.join(''),
|
|
36
|
+
)
|
|
37
|
+
const blockstore = new MemoryBlockstore()
|
|
38
|
+
const headCID = await processFileToIPLDFormat(
|
|
39
|
+
blockstore,
|
|
40
|
+
[buffer],
|
|
41
|
+
BigInt(buffer.length),
|
|
42
|
+
filename,
|
|
43
|
+
)
|
|
44
|
+
|
|
45
|
+
const node = await blockstore.get(headCID)
|
|
46
|
+
const decodedNode = decodeNode(node)
|
|
47
|
+
|
|
48
|
+
expect(decodedNode.Links.length).toBe(expectedChunks)
|
|
49
|
+
const chunks = await Promise.all(
|
|
50
|
+
decodedNode.Links.map(async (e) => {
|
|
51
|
+
const chunk = await blockstore.get(e.Hash)
|
|
52
|
+
const decodedChunk = decodeIPLDNodeData(chunk)
|
|
53
|
+
expect(decodedChunk.data).toBeDefined()
|
|
54
|
+
return Buffer.from(decodedChunk.data!)
|
|
55
|
+
}),
|
|
56
|
+
)
|
|
57
|
+
|
|
58
|
+
const allChunks = await Promise.all(chunks)
|
|
59
|
+
const finalBuffer = Buffer.concat(allChunks)
|
|
60
|
+
expect(finalBuffer.toString()).toBe(buffer.toString())
|
|
61
|
+
})
|
|
62
|
+
|
|
63
|
+
it('should be able to retrieve a file with chunked file with uneven chunk size', async () => {
|
|
64
|
+
const filename = 'test.txt'
|
|
65
|
+
const expectedChunks = 1.5
|
|
66
|
+
const fileSize = Math.floor(expectedChunks * DEFAULT_MAX_CHUNK_SIZE)
|
|
67
|
+
const buffer = Buffer.from(
|
|
68
|
+
Array.from({ length: fileSize })
|
|
69
|
+
.map(() => Math.floor(Math.random() * 16).toString(16))
|
|
70
|
+
.join(''),
|
|
71
|
+
)
|
|
72
|
+
|
|
73
|
+
const blockstore = new MemoryBlockstore()
|
|
74
|
+
const headCID = await processFileToIPLDFormat(
|
|
75
|
+
blockstore,
|
|
76
|
+
[buffer],
|
|
77
|
+
BigInt(buffer.length),
|
|
78
|
+
filename,
|
|
79
|
+
)
|
|
80
|
+
|
|
81
|
+
const node = await blockstore.get(headCID)
|
|
82
|
+
const decodedNode = decodeNode(node)
|
|
83
|
+
|
|
84
|
+
expect(decodedNode.Links.length).toBe(Math.ceil(expectedChunks))
|
|
85
|
+
const chunks = await Promise.all(
|
|
86
|
+
decodedNode.Links.map(async (e) => {
|
|
87
|
+
const chunk = await blockstore.get(e.Hash)
|
|
88
|
+
const decodedChunk = decodeIPLDNodeData(chunk)
|
|
89
|
+
expect(decodedChunk.data).toBeDefined()
|
|
90
|
+
return Buffer.from(decodedChunk.data!)
|
|
91
|
+
}),
|
|
92
|
+
)
|
|
93
|
+
|
|
94
|
+
const allChunks = await Promise.all(chunks)
|
|
95
|
+
const finalBuffer = Buffer.concat(allChunks)
|
|
96
|
+
expect(finalBuffer.toString()).toBe(buffer.toString())
|
|
97
|
+
})
|
|
98
|
+
|
|
99
|
+
it('should be able to retrieve a file with chunked file with different chunk size and uneven chunk size ', async () => {
|
|
100
|
+
const filename = 'test.txt'
|
|
101
|
+
|
|
102
|
+
const expectedChunks = 2
|
|
103
|
+
const chunkSize = Math.floor((DEFAULT_MAX_CHUNK_SIZE * 100) / 121)
|
|
104
|
+
const fileSize = Math.floor(expectedChunks * chunkSize)
|
|
105
|
+
const buffer = Buffer.from(
|
|
106
|
+
Array.from({ length: fileSize })
|
|
107
|
+
.map(() => Math.floor(Math.random() * 16).toString(16))
|
|
108
|
+
.join(''),
|
|
109
|
+
)
|
|
110
|
+
|
|
111
|
+
const blockstore = new MemoryBlockstore()
|
|
112
|
+
const headCID = await processFileToIPLDFormat(
|
|
113
|
+
blockstore,
|
|
114
|
+
[buffer],
|
|
115
|
+
BigInt(buffer.length),
|
|
116
|
+
filename,
|
|
117
|
+
)
|
|
118
|
+
|
|
119
|
+
const node = await blockstore.get(headCID)
|
|
120
|
+
const decodedNode = decodeNode(node)
|
|
121
|
+
|
|
122
|
+
expect(decodedNode.Links.length).toBe(Math.ceil(expectedChunks))
|
|
123
|
+
const chunks = await Promise.all(
|
|
124
|
+
decodedNode.Links.map(async (e) => {
|
|
125
|
+
const chunk = await blockstore.get(e.Hash)
|
|
126
|
+
const decodedChunk = decodeIPLDNodeData(chunk)
|
|
127
|
+
expect(decodedChunk.data).toBeDefined()
|
|
128
|
+
return Buffer.from(decodedChunk.data!)
|
|
129
|
+
}),
|
|
130
|
+
)
|
|
131
|
+
|
|
132
|
+
const allChunks = await Promise.all(chunks)
|
|
133
|
+
const finalBuffer = Buffer.concat(allChunks)
|
|
134
|
+
expect(finalBuffer.toString()).toBe(buffer.toString())
|
|
135
|
+
})
|
|
136
|
+
|
|
137
|
+
it('should retrieve a file generated asynchronously', async () => {
|
|
138
|
+
const filename = 'test.txt'
|
|
139
|
+
const chunkSize = DEFAULT_MAX_CHUNK_SIZE
|
|
140
|
+
const chunksCount = 50
|
|
141
|
+
const buffer = Buffer.from(
|
|
142
|
+
Array.from({ length: chunkSize * chunksCount })
|
|
143
|
+
.map(() => Math.floor(Math.random() * 16).toString(16))
|
|
144
|
+
.join(''),
|
|
145
|
+
)
|
|
146
|
+
|
|
147
|
+
const blockstore = new MemoryBlockstore()
|
|
148
|
+
await processChunksToIPLDFormat(blockstore, [buffer], fileBuilders)
|
|
149
|
+
|
|
150
|
+
const mapCIDs = (async function* () {
|
|
151
|
+
for await (const { cid } of blockstore.getAll()) {
|
|
152
|
+
yield cid
|
|
153
|
+
}
|
|
154
|
+
})()
|
|
155
|
+
|
|
156
|
+
const headCID = await processBufferToIPLDFormatFromChunks(
|
|
157
|
+
blockstore,
|
|
158
|
+
mapCIDs,
|
|
159
|
+
filename,
|
|
160
|
+
BigInt(buffer.length),
|
|
161
|
+
fileBuilders,
|
|
162
|
+
)
|
|
163
|
+
|
|
164
|
+
const node = await blockstore.get(headCID)
|
|
165
|
+
const decodedNode = decodeNode(node)
|
|
166
|
+
|
|
167
|
+
expect(decodedNode.Links.length).toBe(chunksCount)
|
|
168
|
+
const chunks = await Promise.all(
|
|
169
|
+
decodedNode.Links.map(async (e) => {
|
|
170
|
+
const chunk = await blockstore.get(e.Hash)
|
|
171
|
+
const decodedChunk = decodeIPLDNodeData(chunk)
|
|
172
|
+
expect(decodedChunk.data).toBeDefined()
|
|
173
|
+
return Buffer.from(decodedChunk.data!)
|
|
174
|
+
}),
|
|
175
|
+
)
|
|
176
|
+
|
|
177
|
+
const allChunks = await Promise.all(chunks)
|
|
178
|
+
const finalBuffer = Buffer.concat(allChunks)
|
|
179
|
+
expect(finalBuffer.toString()).toBe(buffer.toString())
|
|
180
|
+
})
|
|
181
|
+
})
|
package/tests/nodes.spec.ts
CHANGED
|
@@ -1,11 +1,14 @@
|
|
|
1
1
|
import {
|
|
2
2
|
cidOfNode,
|
|
3
|
+
cidToString,
|
|
3
4
|
createChunkedFileIpldNode,
|
|
4
5
|
createFileChunkIpldNode,
|
|
5
6
|
createSingleFileIpldNode,
|
|
7
|
+
fileMetadata,
|
|
6
8
|
} from '../src/index.js'
|
|
7
|
-
import { createNode,
|
|
9
|
+
import { createMetadataNode, createNode, DEFAULT_NODE_MAX_SIZE } from '../src/ipld/index.js'
|
|
8
10
|
import { IPLDNodeData, MetadataType } from '../src/metadata/onchain/protobuf/OnchainMetadata.js'
|
|
11
|
+
import { stringifyMetadata } from '../src/utils/metadata.js'
|
|
9
12
|
|
|
10
13
|
describe('node creation', () => {
|
|
11
14
|
describe('files nodes', () => {
|
|
@@ -30,7 +33,7 @@ describe('node creation', () => {
|
|
|
30
33
|
})
|
|
31
34
|
|
|
32
35
|
it('single file root node | buffer too large', () => {
|
|
33
|
-
const maxNodeSize =
|
|
36
|
+
const maxNodeSize = DEFAULT_NODE_MAX_SIZE
|
|
34
37
|
const buffer = Buffer.from('h'.repeat(maxNodeSize))
|
|
35
38
|
expect(() => createSingleFileIpldNode(buffer, 'test.txt')).toThrow()
|
|
36
39
|
})
|
|
@@ -77,4 +80,24 @@ describe('node creation', () => {
|
|
|
77
80
|
expect(decoded.linkDepth).toBe(0)
|
|
78
81
|
})
|
|
79
82
|
})
|
|
83
|
+
|
|
84
|
+
describe('metadata nodes', () => {
|
|
85
|
+
it('metadata node | correctly params setup', () => {
|
|
86
|
+
const randomCID = cidOfNode(createNode(Buffer.from(Math.random().toString())))
|
|
87
|
+
const metadata = fileMetadata(
|
|
88
|
+
randomCID,
|
|
89
|
+
[{ cid: cidToString(randomCID), size: BigInt(1000) }],
|
|
90
|
+
BigInt(1000),
|
|
91
|
+
'test.txt',
|
|
92
|
+
)
|
|
93
|
+
const metadataSize = Buffer.from(stringifyMetadata(metadata)).length
|
|
94
|
+
|
|
95
|
+
const metadataNode = createMetadataNode(metadata)
|
|
96
|
+
|
|
97
|
+
const decoded = IPLDNodeData.decode(metadataNode.Data ?? new Uint8Array())
|
|
98
|
+
expect(decoded.type).toBe(MetadataType.Metadata)
|
|
99
|
+
expect(decoded.name).toBe('test.txt')
|
|
100
|
+
expect(decoded.size!.toString()).toBe(BigInt(metadataSize).toString())
|
|
101
|
+
})
|
|
102
|
+
})
|
|
80
103
|
})
|