@autonomys/auto-dag-data 0.8.2

package/tests/chunker.spec.ts

@@ -0,0 +1,294 @@
+import { BaseBlockstore, MemoryBlockstore } from 'blockstore-core'
+import { cidOfNode, cidToString } from '../src'
+import {
+  processFileToIPLDFormat,
+  processFolderToIPLDFormat,
+  processMetadataToIPLDFormat,
+} from '../src/ipld/chunker'
+import { createNode, decodeNode, PBNode } from '../src/ipld/utils'
+import { IPLDNodeData, MetadataType, OffchainMetadata } from '../src/metadata'
+
+describe('chunker', () => {
+  describe('file creation', () => {
+    it('create a file dag from a small buffer', async () => {
+      const text = 'hello world'
+      const size = text.length
+      const name = 'test.txt'
+      const blockstore = new MemoryBlockstore()
+
+      await processFileToIPLDFormat(blockstore, bufferToIterable(Buffer.from(text)), size, name)
+      const nodes = await nodesFromBlockstore(blockstore)
+      expect(nodes.length).toBe(1)
+
+      const node = nodes[0]
+      expect(node).toBeDefined()
+      expect(node?.Data).toBeDefined()
+      const decoded = IPLDNodeData.decode(node?.Data ?? new Uint8Array())
+      expect(decoded.data).toBeDefined()
+
+      /// Check the metadata
+      expect(decoded.name).toBe(name)
+      expect(decoded.type).toBe(MetadataType.File)
+      expect(Buffer.from(decoded.data ?? '').toString()).toBe(text)
+      expect(decoded.linkDepth).toBe(0)
+      expect(decoded.size).toBe(text.length)
+
+      /// Check no links
+      expect(node?.Links.length).toBe(0)
+    })
+
+    it('create a file dag from a large buffer', async () => {
+      const maxChunkSize = 1000
+      const chunkNum = 10
+      const chunk = 'h'.repeat(maxChunkSize)
+      const text = chunk.repeat(chunkNum)
+      const size = text.length
+
+      const name = 'test.txt'
+      /// 1 chunk + root
+      const EXPECTED_NODE_COUNT = 2
+
+      const blockstore = new MemoryBlockstore()
+      const headCID = await processFileToIPLDFormat(
+        blockstore,
+        bufferToIterable(Buffer.from(text)),
+        size,
+        name,
+        {
+          maxChunkSize,
+          maxLinkPerNode: maxChunkSize / 64,
+        },
+      )
+
+      const nodes = await nodesFromBlockstore(blockstore)
+      expect(nodes.length).toBe(EXPECTED_NODE_COUNT)
+
+      const head = decodeNode(await blockstore.get(headCID))
+      expect(head?.Data).toBeDefined()
+      expect(head).toBeDefined()
+      expect(head?.Links.length).toBe(chunkNum)
+
+      const decoded = IPLDNodeData.decode(head?.Data ?? new Uint8Array())
+      expect(decoded.name).toBe(name)
+      expect(decoded.type).toBe(MetadataType.File)
+      expect(decoded.linkDepth).toBe(1)
+      expect(decoded.size).toBe(text.length)
+
+      nodes.forEach((node) => {
+        if (cidToString(cidOfNode(node)) !== cidToString(headCID)) {
+          expect(node?.Links.length).toBe(0)
+        }
+      })
+    })
+
+    it('create a file dag with inlinks', async () => {
+      const maxChunkSize = 1000
+      const chunkNum = 10
+      const chunk = 'h'.repeat(maxChunkSize)
+      const name = 'test.txt'
+      const text = chunk.repeat(chunkNum)
+      const size = text.length
+
+      /// 1 chunks + 2 inlinks + root
+      const EXPECTED_NODE_COUNT = 4
+
+      const blockstore = new MemoryBlockstore()
+      const headCID = await processFileToIPLDFormat(
+        blockstore,
+        bufferToIterable(Buffer.from(text)),
+        size,
+        name,
+        {
+          maxChunkSize,
+          maxLinkPerNode: 4,
+        },
+      )
+
+      const nodes = await nodesFromBlockstore(blockstore)
+      expect(nodes.length).toBe(EXPECTED_NODE_COUNT)
+
+      let [rootCount, inlinkCount, chunkCount] = [0, 0, 0]
+
+      nodes.forEach((node) => {
+        const decoded = IPLDNodeData.decode(node?.Data ?? new Uint8Array())
+        if (decoded.type === MetadataType.File) {
+          rootCount++
+        } else if (decoded.type === MetadataType.FileInlink) {
+          inlinkCount++
+        } else if (decoded.type === MetadataType.FileChunk) {
+          chunkCount++
+        } else {
+          throw new Error('Unexpected node type')
+        }
+      })
+
+      expect(rootCount).toBe(1)
+      expect(inlinkCount).toBe(2)
+      expect(chunkCount).toBe(1)
+    })
+  })
+
+  describe('folder creation', () => {
+    it('create a folder dag from a small buffer', async () => {
+      const links = Array.from({ length: 1 }, () =>
+        cidOfNode(createNode(Buffer.from(Math.random().toString()))),
+      )
+      const name = 'folder'
+      const size = 1000
+      const blockstore = new MemoryBlockstore()
+      const headCID = processFolderToIPLDFormat(blockstore, links, name, size, {
+        maxLinkPerNode: 4,
+      })
+
+      const nodes = await nodesFromBlockstore(blockstore)
+      expect(nodes.length).toBe(1)
+      const node = nodes[0]
+      expect(node).toBeDefined()
+      expect(node?.Data).toBeDefined()
+      const decoded = IPLDNodeData.decode(node?.Data ?? new Uint8Array())
+      expect(decoded.name).toBe(name)
+      expect(decoded.type).toBe(MetadataType.Folder)
+      expect(decoded.linkDepth).toBe(0)
+      expect(decoded.size).toBe(size)
+    })
+
+    it('create a folder dag with inlinks', async () => {
+      const links = Array.from({ length: 10 }, () =>
+        cidOfNode(createNode(Buffer.from(Math.random().toString()))),
+      )
+      const name = 'folder'
+      const size = 1000
+
+      /// 3 inlinks + root
+      const EXPECTED_NODE_COUNT = 4
+
+      const blockstore = new MemoryBlockstore()
+      const headCID = processFolderToIPLDFormat(blockstore, links, name, size, {
+        maxLinkPerNode: 4,
+      })
+
+      const nodes = await nodesFromBlockstore(blockstore)
+      expect(nodes.length).toBe(EXPECTED_NODE_COUNT)
+
+      let [rootCount, inlinkCount] = [0, 0, 0]
+
+      nodes.forEach((node) => {
+        const decoded = IPLDNodeData.decode(node?.Data ?? new Uint8Array())
+        if (decoded.type === MetadataType.Folder) {
+          rootCount++
+        } else if (decoded.type === MetadataType.FolderInlink) {
+          inlinkCount++
+        } else {
+          throw new Error('Unexpected node type')
+        }
+      })
+
+      expect(rootCount).toBe(1)
+      expect(inlinkCount).toBe(3)
+    })
+  })
+
+  describe('metadata creation', () => {
+    it('create a metadata dag from a small buffer', async () => {
+      const metadata: OffchainMetadata = {
+        type: 'file',
+        dataCid: 'test',
+        name: 'test',
+        mimeType: 'text/plain',
+        totalSize: 1000,
+        totalChunks: 10,
+        chunks: [],
+      }
+
+      const blockstore = new MemoryBlockstore()
+      const headCID = await processMetadataToIPLDFormat(blockstore, metadata)
+      const nodes = await nodesFromBlockstore(blockstore)
+      expect(nodes.length).toBe(1)
+    })
+
+    it('large metadata dag represented into multiple nodes', async () => {
+      const metadata: OffchainMetadata = {
+        type: 'file',
+        dataCid: 'test',
+        name: 'test',
+        mimeType: 'text/plain'.repeat(100),
+        totalSize: 1000,
+        totalChunks: 10,
+        chunks: [],
+      }
+
+      const blockstore = new MemoryBlockstore()
+      const headCID = await processMetadataToIPLDFormat(blockstore, metadata, {
+        maxChunkSize: 200,
+        maxLinkPerNode: 2,
+      })
+      const nodes = await nodesFromBlockstore(blockstore)
+      expect(nodes.length).toBeGreaterThan(1)
+    })
+  })
+
+  describe('asyncronous chunking equivalence', () => {
+    it('chunk a small file buffer', async () => {
+      const buffer = Buffer.from('hello world')
+      const blockstore = new MemoryBlockstore()
+      const chunkedBlockstore = new MemoryBlockstore()
+      const singleBufferCID = await processFileToIPLDFormat(
+        blockstore,
+        bufferToIterable(buffer),
+        buffer.length,
+        'test.txt',
+      )
+      const chunkedBufferCID = await processFileToIPLDFormat(
+        chunkedBlockstore,
+        separateBufferToIterable(buffer, 5),
+        buffer.length,
+        'test.txt',
+      )
+
+      expect(singleBufferCID).toEqual(chunkedBufferCID)
+    })
+
+    it('chunk a large file buffer', async () => {
+      const buffer = Buffer.from('hello world')
+      const blockstore = new MemoryBlockstore()
+      const chunkedBlockstore = new MemoryBlockstore()
+      const singleBufferCID = await processFileToIPLDFormat(
+        blockstore,
+        bufferToIterable(buffer),
+        buffer.length,
+        'test.txt',
+      )
+      const chunkedBufferCID = await processFileToIPLDFormat(
+        chunkedBlockstore,
+        separateBufferToIterable(buffer, 5),
+        buffer.length,
+        'test.txt',
+      )
+
+      expect(singleBufferCID).toEqual(chunkedBufferCID)
+    })
+  })
+})
+
+const bufferToIterable = (buffer: Buffer): AsyncIterable<Buffer> => {
+  return (async function* () {
+    yield buffer
+  })()
+}
+
+const separateBufferToIterable = (buffer: Buffer, chunkSize: number): AsyncIterable<Buffer> => {
+  return (async function* () {
+    while (buffer.length > 0) {
+      yield buffer.subarray(0, chunkSize)
+      buffer = buffer.subarray(chunkSize)
+    }
+  })()
+}
+
+const nodesFromBlockstore = async (blockstore: BaseBlockstore): Promise<PBNode[]> => {
+  const nodes: PBNode[] = []
+  for await (const pair of blockstore.getAll()) {
+    nodes.push(decodeNode(pair.block))
+  }
+  return nodes
+}

package/tests/cid.spec.ts

@@ -0,0 +1,20 @@
+import { CID } from 'multiformats'
+import { cidOfNode, cidToString, stringToCid } from '../src/cid/index.js'
+import { createNode } from '../src/ipld/index.js'
+
+const randomCIDString = cidOfNode(createNode(new Uint8Array([]), [])).toString()
+
+describe('CID', () => {
+  it('should generate a valid CID from a node', () => {
+    const node = createNode(new Uint8Array([1, 2, 3]), [])
+    const cid = cidOfNode(node).toString()
+    expect(cid).toBeDefined()
+    expect(cidToString(stringToCid(cid))).toEqual(cid)
+  })
+
+  it('should convert CID string back to CID object', () => {
+    const cidObject = stringToCid(randomCIDString)
+    expect(cidObject).toBeInstanceOf(CID)
+    expect(cidToString(cidObject)).toEqual(randomCIDString)
+  })
+})

package/tests/compression.spec.ts

@@ -0,0 +1,58 @@
+import { compressFile, COMPRESSION_CHUNK_SIZE, CompressionAlgorithm, decompressFile } from '../src'
+
+describe('compression', () => {
+  it('compresses and decompresses a file with default options', async () => {
+    const file = Buffer.from('hello'.repeat(1000))
+
+    const compressed = compressFile(
+      (async function* () {
+        yield file
+      })(),
+      {
+        algorithm: CompressionAlgorithm.ZLIB,
+        level: 9,
+      },
+    )
+
+    const decompressed = decompressFile(compressed, {
+      algorithm: CompressionAlgorithm.ZLIB,
+      level: 9,
+    })
+
+    let decompressedBuffer = Buffer.alloc(0)
+    for await (const chunk of decompressed) {
+      decompressedBuffer = Buffer.concat([decompressedBuffer, chunk])
+    }
+
+    expect(decompressedBuffer.toString()).toBe(file.toString())
+  })
+
+  it('compresses and decompresses a file with custom chunk size', async () => {
+    const file = Buffer.from('hello'.repeat(1000))
+    const chunkSize = 100
+
+    const compressed = compressFile(
+      (async function* () {
+        yield file
+      })(),
+      {
+        algorithm: CompressionAlgorithm.ZLIB,
+        level: 9,
+        chunkSize,
+      },
+    )
+
+    const decompressed = decompressFile(compressed, {
+      algorithm: CompressionAlgorithm.ZLIB,
+      level: 9,
+      chunkSize,
+    })
+
+    let decompressedBuffer = Buffer.alloc(0)
+    for await (const chunk of decompressed) {
+      decompressedBuffer = Buffer.concat([decompressedBuffer, chunk])
+    }
+
+    expect(decompressedBuffer.toString()).toBe(file.toString())
+  })
+})

package/tests/encryption.spec.ts

@@ -0,0 +1,67 @@
+import { decryptFile, encryptFile, EncryptionAlgorithm } from '../src'
+
+describe('encryption', () => {
+  it('encrypts and decrypts a file with default chunk size', async () => {
+    const chunk = 'hello'
+    const file = Buffer.from(chunk.repeat(1000))
+    const password = 'password'
+    const salt = 'salt'
+
+    const encrypted = encryptFile(
+      (async function* () {
+        yield file
+      })(),
+      password,
+      {
+        algorithm: EncryptionAlgorithm.AES_256_GCM,
+      },
+    )
+
+    const decrypted = decryptFile(encrypted, password, {
+      algorithm: EncryptionAlgorithm.AES_256_GCM,
+    })
+
+    let decryptedBuffer = Buffer.alloc(0)
+    for await (const chunk of decrypted) {
+      decryptedBuffer = Buffer.concat([decryptedBuffer, chunk])
+    }
+
+    expect(decryptedBuffer.toString()).toBe(file.toString())
+  })
+
+  it('encrypts and decrypts a file with custom chunk size', async () => {
+    const chunk = 'hello'
+    const file = Buffer.from(chunk.repeat(1000))
+    const password = 'password'
+    const salt = 'salt'
+    const IV_SIZE = 16
+    const TAG_SIZE = 16
+
+    const encryptingSize = 100
+
+    const encrypted = encryptFile(
+      (async function* () {
+        yield file
+      })(),
+      password,
+      {
+        algorithm: EncryptionAlgorithm.AES_256_GCM,
+        chunkSize: encryptingSize,
+      },
+    )
+
+    const decryptingSize = encryptingSize + IV_SIZE + TAG_SIZE
+
+    const decrypted = decryptFile(encrypted, password, {
+      algorithm: EncryptionAlgorithm.AES_256_GCM,
+      chunkSize: decryptingSize,
+    })
+
+    let decryptedBuffer = Buffer.alloc(0)
+    for await (const chunk of decrypted) {
+      decryptedBuffer = Buffer.concat([decryptedBuffer, chunk])
+    }
+
+    expect(decryptedBuffer.toString()).toBe(file.toString())
+  })
+})

package/tests/nodes.spec.ts

@@ -0,0 +1,74 @@
+import {
+  cidOfNode,
+  createChunkedFileIpldNode,
+  createFileChunkIpldNode,
+  createSingleFileIpldNode,
+} from '../src/index.js'
+import { createNode } from '../src/ipld/index.js'
+import { IPLDNodeData, MetadataType } from '../src/metadata/onchain/protobuf/OnchainMetadata.js'
+
+describe('node creation', () => {
+  describe('files nodes', () => {
+    it('single file node | correctly params setup', () => {
+      const filename = 'test.txt'
+      const buffer = Buffer.from('hello world')
+      const node = createSingleFileIpldNode(buffer, filename)
+      const decoded = IPLDNodeData.decode(node.Data ?? new Uint8Array())
+      expect(decoded.name).toBe(filename)
+      expect(decoded.size).toBe(buffer.length)
+      expect(Buffer.from(decoded.data ?? '').toString()).toBe(buffer.toString())
+    })
+
+    it('single file root node | no name', () => {
+      const buffer = Buffer.from('hello world')
+      const node = createSingleFileIpldNode(buffer)
+      const decoded = IPLDNodeData.decode(node.Data ?? new Uint8Array())
+      expect(decoded.type).toBe(MetadataType.File)
+      expect(decoded.name).toBeUndefined()
+      expect(decoded.size).toBe(buffer.length)
+      expect(Buffer.from(decoded.data ?? '').toString()).toBe(buffer.toString())
+    })
+
+    it('chunked file root node | correctly params setup', () => {
+      const links = Array.from({ length: 10 }, () =>
+        cidOfNode(createNode(Buffer.from(Math.random().toString()))),
+      )
+      const size = 1000
+      const linkDepth = 1
+      const filename = 'test.txt'
+      const node = createChunkedFileIpldNode(links, size, linkDepth, filename)
+
+      const decoded = IPLDNodeData.decode(node.Data ?? new Uint8Array())
+      expect(decoded.type).toBe(MetadataType.File)
+      expect(decoded.name).toBe(filename)
+      expect(decoded.size).toBe(size)
+      expect(decoded.linkDepth).toBe(linkDepth)
+    })
+
+    it('chunked file root node | no name', () => {
+      const links = Array.from({ length: 10 }, () =>
+        cidOfNode(createNode(Buffer.from(Math.random().toString()))),
+      )
+      const size = 1000
+      const linkDepth = 1
+      const node = createChunkedFileIpldNode(links, size, linkDepth)
+
+      const decoded = IPLDNodeData.decode(node.Data ?? new Uint8Array())
+      expect(decoded.type).toBe(MetadataType.File)
+      expect(decoded.name).toBeUndefined()
+      expect(decoded.size).toBe(size)
+      expect(decoded.linkDepth).toBe(linkDepth)
+    })
+
+    it('file chunk node | correctly params setup', () => {
+      const buffer = Buffer.from('hello world')
+      const node = createFileChunkIpldNode(buffer)
+
+      const decoded = IPLDNodeData.decode(node.Data ?? new Uint8Array())
+      expect(decoded.type).toBe(MetadataType.FileChunk)
+      expect(decoded.name).toBeUndefined()
+      expect(decoded.size).toBe(buffer.length)
+      expect(decoded.linkDepth).toBe(0)
+    })
+  })
+})

package/tsconfig.json

@@ -0,0 +1,14 @@
+{
+  "extends": "../../tsconfig.json",
+  "compilerOptions": {
+    "outDir": "./dist",
+    "rootDir": "./src/",
+    "moduleResolution": "NodeNext",
+    "module": "NodeNext",
+    "forceConsistentCasingInFileNames": true,
+    "skipLibCheck": true,
+    "stripInternal": true,
+    "resolveJsonModule": true
+  },
+  "include": ["src"]
+}