@autonomys/auto-drive 0.5.0

package/src/metadata/onchain/protobuf/OnchainMetadata.ts

@@ -0,0 +1,133 @@
+/* eslint-disable import/export */
+/* eslint-disable complexity */
+/* eslint-disable @typescript-eslint/no-namespace */
+/* eslint-disable @typescript-eslint/no-unnecessary-boolean-literal-compare */
+/* eslint-disable @typescript-eslint/no-empty-interface */
+
+import { type Codec, decodeMessage, type DecodeOptions, encodeMessage, enumeration, message } from 'protons-runtime'
+import type { Uint8ArrayList } from 'uint8arraylist'
+
+export interface IPLDNodeData {
+  type: MetadataType
+  linkDepth: number
+  size?: number
+  name?: string
+  data?: Uint8Array
+}
+
+export namespace IPLDNodeData {
+  let _codec: Codec<IPLDNodeData>
+
+  export const codec = (): Codec<IPLDNodeData> => {
+    if (_codec == null) {
+      _codec = message<IPLDNodeData>((obj, w, opts = {}) => {
+        if (opts.lengthDelimited !== false) {
+          w.fork()
+        }
+
+        if (obj.type != null && __MetadataTypeValues[obj.type] !== 0) {
+          w.uint32(8)
+          MetadataType.codec().encode(obj.type, w)
+        }
+
+        if ((obj.linkDepth != null && obj.linkDepth !== 0)) {
+          w.uint32(16)
+          w.int32(obj.linkDepth)
+        }
+
+        if (obj.size != null) {
+          w.uint32(24)
+          w.int32(obj.size)
+        }
+
+        if (obj.name != null) {
+          w.uint32(34)
+          w.string(obj.name)
+        }
+
+        if (obj.data != null) {
+          w.uint32(42)
+          w.bytes(obj.data)
+        }
+
+        if (opts.lengthDelimited !== false) {
+          w.ldelim()
+        }
+      }, (reader, length, opts = {}) => {
+        const obj: any = {
+          type: MetadataType.File,
+          linkDepth: 0
+        }
+
+        const end = length == null ? reader.len : reader.pos + length
+
+        while (reader.pos < end) {
+          const tag = reader.uint32()
+
+          switch (tag >>> 3) {
+            case 1: {
+              obj.type = MetadataType.codec().decode(reader)
+              break
+            }
+            case 2: {
+              obj.linkDepth = reader.int32()
+              break
+            }
+            case 3: {
+              obj.size = reader.int32()
+              break
+            }
+            case 4: {
+              obj.name = reader.string()
+              break
+            }
+            case 5: {
+              obj.data = reader.bytes()
+              break
+            }
+            default: {
+              reader.skipType(tag & 7)
+              break
+            }
+          }
+        }
+
+        return obj
+      })
+    }
+
+    return _codec
+  }
+
+  export const encode = (obj: Partial<IPLDNodeData>): Uint8Array => {
+    return encodeMessage(obj, IPLDNodeData.codec())
+  }
+
+  export const decode = (buf: Uint8Array | Uint8ArrayList, opts?: DecodeOptions<IPLDNodeData>): IPLDNodeData => {
+    return decodeMessage(buf, IPLDNodeData.codec(), opts)
+  }
+}
+
+export enum MetadataType {
+  File = 'File',
+  FileInlink = 'FileInlink',
+  FileChunk = 'FileChunk',
+  Folder = 'Folder',
+  FolderInlink = 'FolderInlink',
+  Metadata = 'Metadata'
+}
+
+enum __MetadataTypeValues {
+  File = 0,
+  FileInlink = 1,
+  FileChunk = 2,
+  Folder = 3,
+  FolderInlink = 4,
+  Metadata = 5
+}
+
+export namespace MetadataType {
+  export const codec = (): Codec<MetadataType> => {
+    return enumeration<MetadataType>(__MetadataTypeValues)
+  }
+}

package/src/metadata/onchain/utils.ts

@@ -0,0 +1,15 @@
+import { decode } from '@ipld/dag-pb'
+import { IPLDNodeData } from '../onchain/index.js'
+
+export const encodeIPLDNodeData = (metadata: IPLDNodeData): Uint8Array => {
+  return IPLDNodeData.encode(metadata)
+}
+
+export const decodeIPLDNodeData = (data: Uint8Array): IPLDNodeData => {
+  const decoded = decode(data)
+  if (!decoded.Data) {
+    throw new Error('Invalid data')
+  }
+
+  return IPLDNodeData.decode(decoded.Data)
+}

package/tests/chunker.spec.ts

@@ -0,0 +1,157 @@
+import { createNode } from '@ipld/dag-pb'
+import { cidOfNode } from '../src'
+import { createFileIPLDDag, createFolderIPLDDag } from '../src/ipld/chunker'
+import { IPLDNodeData, MetadataType } from '../src/metadata'
+
+describe('chunker', () => {
+  describe('file creation', () => {
+    it('create a file dag from a small buffer', () => {
+      const text = 'hello world'
+      const name = 'test.txt'
+      const dag = createFileIPLDDag(Buffer.from(text), name)
+      expect(dag.nodes.size).toBe(1)
+
+      const node = dag.nodes.get(dag.headCID)
+      expect(node).toBeDefined()
+      expect(node?.Data).toBeDefined()
+      const decoded = IPLDNodeData.decode(node?.Data ?? new Uint8Array())
+      expect(decoded.data).toBeDefined()
+
+      /// Check the metadata
+      expect(decoded.name).toBe(name)
+      expect(decoded.type).toBe(MetadataType.File)
+      expect(Buffer.from(decoded.data ?? '').toString()).toBe(text)
+      expect(decoded.linkDepth).toBe(0)
+      expect(decoded.size).toBe(text.length)
+
+      /// Check no links
+      expect(node?.Links.length).toBe(0)
+    })
+
+    it('create a file dag from a large buffer', () => {
+      const chunkSize = 1000
+      const chunkNum = 10
+      const chunk = 'h'.repeat(chunkSize)
+      const text = chunk.repeat(chunkNum)
+
+      const name = 'test.txt'
+
+      const dag = createFileIPLDDag(Buffer.from(text), name, {
+        chunkSize,
+        maxLinkPerNode: chunkSize / 64,
+      })
+
+      expect(dag.nodes.size).toBe(chunkNum + 1)
+
+      const head = dag.nodes.get(dag.headCID)
+      expect(head?.Data).toBeDefined()
+      expect(head).toBeDefined()
+      expect(head?.Links.length).toBe(chunkNum)
+
+      const decoded = IPLDNodeData.decode(head?.Data ?? new Uint8Array())
+      expect(decoded.name).toBe(name)
+      expect(decoded.type).toBe(MetadataType.File)
+      expect(decoded.linkDepth).toBe(1)
+      expect(decoded.size).toBe(text.length)
+
+      Array.from(dag.nodes.entries()).forEach(([cid, node]) => {
+        if (cid !== dag.headCID) {
+          expect(node?.Links.length).toBe(0)
+        }
+      })
+    })
+
+    it('create a file dag with inlinks', () => {
+      const chunkSize = 1000
+      const chunkNum = 10
+      const chunk = 'h'.repeat(chunkSize)
+      const name = 'test.txt'
+      const text = chunk.repeat(chunkNum)
+
+      /// 10 chunks + 3 inlinks + root
+      const EXPECTED_NODE_COUNT = 14
+
+      const dag = createFileIPLDDag(Buffer.from(text), name, {
+        chunkSize,
+        maxLinkPerNode: 4,
+      })
+
+      expect(dag.nodes.size).toBe(EXPECTED_NODE_COUNT)
+
+      let [rootCount, inlinkCount, chunkCount] = [0, 0, 0]
+
+      Array.from(dag.nodes.values()).forEach((node) => {
+        const decoded = IPLDNodeData.decode(node?.Data ?? new Uint8Array())
+        if (decoded.type === MetadataType.File) {
+          rootCount++
+        } else if (decoded.type === MetadataType.FileInlink) {
+          inlinkCount++
+        } else if (decoded.type === MetadataType.FileChunk) {
+          chunkCount++
+        } else {
+          throw new Error('Unexpected node type')
+        }
+      })
+
+      expect(rootCount).toBe(1)
+      expect(inlinkCount).toBe(3)
+      expect(chunkCount).toBe(10)
+    })
+  })
+
+  describe('folder creation', () => {
+    it('create a folder dag from a small buffer', () => {
+      const links = Array.from({ length: 1 }, () =>
+        cidOfNode(createNode(Buffer.from(Math.random().toString()))),
+      )
+      const name = 'folder'
+      const size = 1000
+      const dag = createFolderIPLDDag(links, name, size, {
+        maxLinkPerNode: 4,
+      })
+
+      expect(dag.nodes.size).toBe(1)
+      const node = dag.nodes.get(dag.headCID)
+      expect(node).toBeDefined()
+      expect(node?.Data).toBeDefined()
+      const decoded = IPLDNodeData.decode(node?.Data ?? new Uint8Array())
+      expect(decoded.name).toBe(name)
+      expect(decoded.type).toBe(MetadataType.Folder)
+      expect(decoded.linkDepth).toBe(0)
+      expect(decoded.size).toBe(size)
+    })
+
+    it('create a folder dag with inlinks', () => {
+      const links = Array.from({ length: 10 }, () =>
+        cidOfNode(createNode(Buffer.from(Math.random().toString()))),
+      )
+      const name = 'folder'
+      const size = 1000
+
+      /// 3 inlinks + root
+      const EXPECTED_NODE_COUNT = 4
+
+      const dag = createFolderIPLDDag(links, name, size, {
+        maxLinkPerNode: 4,
+      })
+
+      expect(dag.nodes.size).toBe(EXPECTED_NODE_COUNT)
+
+      let [rootCount, inlinkCount] = [0, 0, 0]
+
+      Array.from(dag.nodes.values()).forEach((node) => {
+        const decoded = IPLDNodeData.decode(node?.Data ?? new Uint8Array())
+        if (decoded.type === MetadataType.Folder) {
+          rootCount++
+        } else if (decoded.type === MetadataType.FolderInlink) {
+          inlinkCount++
+        } else {
+          throw new Error('Unexpected node type')
+        }
+      })
+
+      expect(rootCount).toBe(1)
+      expect(inlinkCount).toBe(3)
+    })
+  })
+})

package/tests/cid.spec.ts

@@ -0,0 +1,20 @@
+import { createNode } from '@ipld/dag-pb'
+import { CID } from 'multiformats'
+import { cidOfNode, cidToString, stringToCid } from '../src/cid/index.js'
+
+const randomCIDString = cidOfNode(createNode(new Uint8Array([]), [])).toString()
+
+describe('CID', () => {
+  it('should generate a valid CID from a node', () => {
+    const node = createNode(new Uint8Array([1, 2, 3]), [])
+    const cid = cidOfNode(node).toString()
+    expect(cid).toBeDefined()
+    expect(cidToString(stringToCid(cid))).toEqual(cid)
+  })
+
+  it('should convert CID string back to CID object', () => {
+    const cidObject = stringToCid(randomCIDString)
+    expect(cidObject).toBeInstanceOf(CID)
+    expect(cidToString(cidObject)).toEqual(randomCIDString)
+  })
+})

package/tests/nodes.spec.ts

@@ -0,0 +1,70 @@
+import { createNode } from '@ipld/dag-pb'
+import { cidOfNode, createChunkedFileIpldNode, createSingleFileIpldNode } from '../src/index.js'
+import { createChunkIpldNode } from '../src/ipld/nodes.js'
+import { IPLDNodeData, MetadataType } from '../src/metadata/onchain/protobuf/OnchainMetadata.js'
+
+describe('node creation', () => {
+  describe('files nodes', () => {
+    it('single file node | correctly params setup', () => {
+      const filename = 'test.txt'
+      const buffer = Buffer.from('hello world')
+      const node = createSingleFileIpldNode(buffer, filename)
+      const decoded = IPLDNodeData.decode(node.Data ?? new Uint8Array())
+      expect(decoded.name).toBe(filename)
+      expect(decoded.size).toBe(buffer.length)
+      expect(Buffer.from(decoded.data ?? '').toString()).toBe(buffer.toString())
+    })
+
+    it('single file root node | no name', () => {
+      const buffer = Buffer.from('hello world')
+      const node = createSingleFileIpldNode(buffer)
+      const decoded = IPLDNodeData.decode(node.Data ?? new Uint8Array())
+      expect(decoded.type).toBe(MetadataType.File)
+      expect(decoded.name).toBeUndefined()
+      expect(decoded.size).toBe(buffer.length)
+      expect(Buffer.from(decoded.data ?? '').toString()).toBe(buffer.toString())
+    })
+
+    it('chunked file root node | correctly params setup', () => {
+      const links = Array.from({ length: 10 }, () =>
+        cidOfNode(createNode(Buffer.from(Math.random().toString()))),
+      )
+      const size = 1000
+      const linkDepth = 1
+      const filename = 'test.txt'
+      const node = createChunkedFileIpldNode(links, size, linkDepth, filename)
+
+      const decoded = IPLDNodeData.decode(node.Data ?? new Uint8Array())
+      expect(decoded.type).toBe(MetadataType.File)
+      expect(decoded.name).toBe(filename)
+      expect(decoded.size).toBe(size)
+      expect(decoded.linkDepth).toBe(linkDepth)
+    })
+
+    it('chunked file root node | no name', () => {
+      const links = Array.from({ length: 10 }, () =>
+        cidOfNode(createNode(Buffer.from(Math.random().toString()))),
+      )
+      const size = 1000
+      const linkDepth = 1
+      const node = createChunkedFileIpldNode(links, size, linkDepth)
+
+      const decoded = IPLDNodeData.decode(node.Data ?? new Uint8Array())
+      expect(decoded.type).toBe(MetadataType.File)
+      expect(decoded.name).toBeUndefined()
+      expect(decoded.size).toBe(size)
+      expect(decoded.linkDepth).toBe(linkDepth)
+    })
+
+    it('file chunk node | correctly params setup', () => {
+      const buffer = Buffer.from('hello world')
+      const node = createChunkIpldNode(buffer)
+
+      const decoded = IPLDNodeData.decode(node.Data ?? new Uint8Array())
+      expect(decoded.type).toBe(MetadataType.FileChunk)
+      expect(decoded.name).toBeUndefined()
+      expect(decoded.size).toBe(buffer.length)
+      expect(decoded.linkDepth).toBe(0)
+    })
+  })
+})

package/tsconfig.json

@@ -0,0 +1,14 @@
+{
+  "extends": "../../tsconfig.json",
+  "compilerOptions": {
+    "outDir": "./dist",
+    "rootDir": "./src/",
+    "moduleResolution": "NodeNext",
+    "module": "NodeNext",
+    "forceConsistentCasingInFileNames": true,
+    "skipLibCheck": true,
+    "stripInternal": true,
+    "resolveJsonModule": true
+  },
+  "include": ["src"]
+}