@autonomys/auto-drive 0.7.0 → 0.7.2

package/tests/chunker.spec.ts

@@ -1,17 +1,26 @@
-import { createNode } from '@ipld/dag-pb'
-import { cidOfNode } from '../src'
-import { createFileIPLDDag, createFolderIPLDDag, createMetadataIPLDDag } from '../src/ipld/chunker'
+import { BaseBlockstore, MemoryBlockstore } from 'blockstore-core'
+import { cidOfNode, cidToString } from '../src'
+import {
+  processFileToIPLDFormat,
+  processFolderToIPLDFormat,
+  processMetadataToIPLDFormat,
+} from '../src/ipld/chunker'
+import { createNode, decodeNode, PBNode } from '../src/ipld/utils'
 import { IPLDNodeData, MetadataType, OffchainMetadata } from '../src/metadata'
 
 describe('chunker', () => {
   describe('file creation', () => {
-    it('create a file dag from a small buffer', () => {
+    it('create a file dag from a small buffer', async () => {
       const text = 'hello world'
+      const size = text.length
       const name = 'test.txt'
-      const dag = createFileIPLDDag(Buffer.from(text), name)
-      expect(dag.nodes.size).toBe(1)
+      const blockstore = new MemoryBlockstore()
 
-      const node = dag.nodes.get(dag.headCID)
+      await processFileToIPLDFormat(blockstore, bufferToIterable(Buffer.from(text)), size, name)
+      const nodes = await nodesFromBlockstore(blockstore)
+      expect(nodes.length).toBe(1)
+
+      const node = nodes[0]
       expect(node).toBeDefined()
       expect(node?.Data).toBeDefined()
       const decoded = IPLDNodeData.decode(node?.Data ?? new Uint8Array())
@@ -28,22 +37,33 @@ describe('chunker', () => {
       expect(node?.Links.length).toBe(0)
     })
 
-    it('create a file dag from a large buffer', () => {
-      const chunkSize = 1000
+    it('create a file dag from a large buffer', async () => {
+      const maxChunkSize = 1000
       const chunkNum = 10
-      const chunk = 'h'.repeat(chunkSize)
+      const chunk = 'h'.repeat(maxChunkSize)
       const text = chunk.repeat(chunkNum)
+      const size = text.length
 
       const name = 'test.txt'
+      /// 1 chunk + root
+      const EXPECTED_NODE_COUNT = 2
 
-      const dag = createFileIPLDDag(Buffer.from(text), name, {
-        chunkSize,
-        maxLinkPerNode: chunkSize / 64,
-      })
+      const blockstore = new MemoryBlockstore()
+      const headCID = await processFileToIPLDFormat(
+        blockstore,
+        bufferToIterable(Buffer.from(text)),
+        size,
+        name,
+        {
+          maxChunkSize,
+          maxLinkPerNode: maxChunkSize / 64,
+        },
+      )
 
-      expect(dag.nodes.size).toBe(chunkNum + 1)
+      const nodes = await nodesFromBlockstore(blockstore)
+      expect(nodes.length).toBe(EXPECTED_NODE_COUNT)
 
-      const head = dag.nodes.get(dag.headCID)
+      const head = decodeNode(await blockstore.get(headCID))
       expect(head?.Data).toBeDefined()
       expect(head).toBeDefined()
       expect(head?.Links.length).toBe(chunkNum)
@@ -54,33 +74,42 @@ describe('chunker', () => {
       expect(decoded.linkDepth).toBe(1)
       expect(decoded.size).toBe(text.length)
 
-      Array.from(dag.nodes.entries()).forEach(([cid, node]) => {
-        if (cid !== dag.headCID) {
+      nodes.forEach((node) => {
+        if (cidToString(cidOfNode(node)) !== cidToString(headCID)) {
           expect(node?.Links.length).toBe(0)
         }
       })
     })
 
-    it('create a file dag with inlinks', () => {
-      const chunkSize = 1000
+    it('create a file dag with inlinks', async () => {
+      const maxChunkSize = 1000
       const chunkNum = 10
-      const chunk = 'h'.repeat(chunkSize)
+      const chunk = 'h'.repeat(maxChunkSize)
       const name = 'test.txt'
       const text = chunk.repeat(chunkNum)
+      const size = text.length
 
-      /// 10 chunks + 3 inlinks + root
-      const EXPECTED_NODE_COUNT = 14
+      /// 1 chunks + 2 inlinks + root
+      const EXPECTED_NODE_COUNT = 4
 
-      const dag = createFileIPLDDag(Buffer.from(text), name, {
-        chunkSize,
-        maxLinkPerNode: 4,
-      })
+      const blockstore = new MemoryBlockstore()
+      const headCID = await processFileToIPLDFormat(
+        blockstore,
+        bufferToIterable(Buffer.from(text)),
+        size,
+        name,
+        {
+          maxChunkSize,
+          maxLinkPerNode: 4,
+        },
+      )
 
-      expect(dag.nodes.size).toBe(EXPECTED_NODE_COUNT)
+      const nodes = await nodesFromBlockstore(blockstore)
+      expect(nodes.length).toBe(EXPECTED_NODE_COUNT)
 
       let [rootCount, inlinkCount, chunkCount] = [0, 0, 0]
 
-      Array.from(dag.nodes.values()).forEach((node) => {
+      nodes.forEach((node) => {
         const decoded = IPLDNodeData.decode(node?.Data ?? new Uint8Array())
         if (decoded.type === MetadataType.File) {
           rootCount++
@@ -94,24 +123,26 @@ describe('chunker', () => {
       })
 
       expect(rootCount).toBe(1)
-      expect(inlinkCount).toBe(3)
-      expect(chunkCount).toBe(10)
+      expect(inlinkCount).toBe(2)
+      expect(chunkCount).toBe(1)
     })
   })
 
   describe('folder creation', () => {
-    it('create a folder dag from a small buffer', () => {
+    it('create a folder dag from a small buffer', async () => {
       const links = Array.from({ length: 1 }, () =>
         cidOfNode(createNode(Buffer.from(Math.random().toString()))),
       )
       const name = 'folder'
       const size = 1000
-      const dag = createFolderIPLDDag(links, name, size, {
+      const blockstore = new MemoryBlockstore()
+      const headCID = processFolderToIPLDFormat(blockstore, links, name, size, {
         maxLinkPerNode: 4,
       })
 
-      expect(dag.nodes.size).toBe(1)
-      const node = dag.nodes.get(dag.headCID)
+      const nodes = await nodesFromBlockstore(blockstore)
+      expect(nodes.length).toBe(1)
+      const node = nodes[0]
       expect(node).toBeDefined()
       expect(node?.Data).toBeDefined()
       const decoded = IPLDNodeData.decode(node?.Data ?? new Uint8Array())
@@ -121,7 +152,7 @@ describe('chunker', () => {
       expect(decoded.size).toBe(size)
     })
 
-    it('create a folder dag with inlinks', () => {
+    it('create a folder dag with inlinks', async () => {
       const links = Array.from({ length: 10 }, () =>
         cidOfNode(createNode(Buffer.from(Math.random().toString()))),
       )
@@ -131,15 +162,17 @@ describe('chunker', () => {
       /// 3 inlinks + root
       const EXPECTED_NODE_COUNT = 4
 
-      const dag = createFolderIPLDDag(links, name, size, {
+      const blockstore = new MemoryBlockstore()
+      const headCID = processFolderToIPLDFormat(blockstore, links, name, size, {
         maxLinkPerNode: 4,
       })
 
-      expect(dag.nodes.size).toBe(EXPECTED_NODE_COUNT)
+      const nodes = await nodesFromBlockstore(blockstore)
+      expect(nodes.length).toBe(EXPECTED_NODE_COUNT)
 
       let [rootCount, inlinkCount] = [0, 0, 0]
 
-      Array.from(dag.nodes.values()).forEach((node) => {
+      nodes.forEach((node) => {
         const decoded = IPLDNodeData.decode(node?.Data ?? new Uint8Array())
         if (decoded.type === MetadataType.Folder) {
           rootCount++
@@ -156,7 +189,7 @@ describe('chunker', () => {
   })
 
   describe('metadata creation', () => {
-    it('create a metadata dag from a small buffer', () => {
+    it('create a metadata dag from a small buffer', async () => {
       const metadata: OffchainMetadata = {
         type: 'file',
         dataCid: 'test',
@@ -167,11 +200,13 @@ describe('chunker', () => {
         chunks: [],
       }
 
-      const dag = createMetadataIPLDDag(metadata)
-      expect(dag.nodes.size).toBe(1)
+      const blockstore = new MemoryBlockstore()
+      const headCID = await processMetadataToIPLDFormat(blockstore, metadata)
+      const nodes = await nodesFromBlockstore(blockstore)
+      expect(nodes.length).toBe(1)
     })
 
-    it('large metadata dag represented into multiple nodes', () => {
+    it('large metadata dag represented into multiple nodes', async () => {
       const metadata: OffchainMetadata = {
         type: 'file',
         dataCid: 'test',
@@ -182,11 +217,78 @@ describe('chunker', () => {
         chunks: [],
       }
 
-      const dag = createMetadataIPLDDag(metadata, {
-        chunkSize: 200,
+      const blockstore = new MemoryBlockstore()
+      const headCID = await processMetadataToIPLDFormat(blockstore, metadata, {
+        maxChunkSize: 200,
         maxLinkPerNode: 2,
       })
-      expect(dag.nodes.size).toBeGreaterThan(1)
+      const nodes = await nodesFromBlockstore(blockstore)
+      expect(nodes.length).toBeGreaterThan(1)
+    })
+  })
+
+  describe('asyncronous chunking equivalence', () => {
+    it('chunk a small file buffer', async () => {
+      const buffer = Buffer.from('hello world')
+      const blockstore = new MemoryBlockstore()
+      const chunkedBlockstore = new MemoryBlockstore()
+      const singleBufferCID = await processFileToIPLDFormat(
+        blockstore,
+        bufferToIterable(buffer),
+        buffer.length,
+        'test.txt',
+      )
+      const chunkedBufferCID = await processFileToIPLDFormat(
+        chunkedBlockstore,
+        separateBufferToIterable(buffer, 5),
+        buffer.length,
+        'test.txt',
+      )
+
+      expect(singleBufferCID).toEqual(chunkedBufferCID)
+    })
+
+    it('chunk a large file buffer', async () => {
+      const buffer = Buffer.from('hello world')
+      const blockstore = new MemoryBlockstore()
+      const chunkedBlockstore = new MemoryBlockstore()
+      const singleBufferCID = await processFileToIPLDFormat(
+        blockstore,
+        bufferToIterable(buffer),
+        buffer.length,
+        'test.txt',
+      )
+      const chunkedBufferCID = await processFileToIPLDFormat(
+        chunkedBlockstore,
+        separateBufferToIterable(buffer, 5),
+        buffer.length,
+        'test.txt',
+      )
+
+      expect(singleBufferCID).toEqual(chunkedBufferCID)
     })
   })
 })
+
+const bufferToIterable = (buffer: Buffer): AsyncIterable<Buffer> => {
+  return (async function* () {
+    yield buffer
+  })()
+}
+
+const separateBufferToIterable = (buffer: Buffer, chunkSize: number): AsyncIterable<Buffer> => {
+  return (async function* () {
+    while (buffer.length > 0) {
+      yield buffer.subarray(0, chunkSize)
+      buffer = buffer.subarray(chunkSize)
+    }
+  })()
+}
+
+const nodesFromBlockstore = async (blockstore: BaseBlockstore): Promise<PBNode[]> => {
+  const nodes: PBNode[] = []
+  for await (const pair of blockstore.getAll()) {
+    nodes.push(decodeNode(pair.block))
+  }
+  return nodes
+}

package/tests/cid.spec.ts

@@ -1,6 +1,6 @@
-import { createNode } from '@ipld/dag-pb'
 import { CID } from 'multiformats'
 import { cidOfNode, cidToString, stringToCid } from '../src/cid/index.js'
+import { createNode } from '../src/ipld/index.js'
 
 const randomCIDString = cidOfNode(createNode(new Uint8Array([]), [])).toString()
 

package/tests/nodes.spec.ts

@@ -1,10 +1,10 @@
-import { createNode } from '@ipld/dag-pb'
 import {
   cidOfNode,
   createChunkedFileIpldNode,
   createFileChunkIpldNode,
   createSingleFileIpldNode,
 } from '../src/index.js'
+import { createNode } from '../src/ipld/index.js'
 import { IPLDNodeData, MetadataType } from '../src/metadata/onchain/protobuf/OnchainMetadata.js'
 
 describe('node creation', () => {