@autonomys/auto-drive 0.6.3 → 0.6.4

package/README.md

@@ -128,6 +128,27 @@ const cidString = cidToString(cid)
 console.log(`CID of the file DAG: ${cidString}`)
 ```
 
+### Example: Converting Metadata To DAG
+
+```typescript
+import {
+  createMetadataIPLDDag,
+  cidOfNode,
+  cidToString,
+  type OffchainMetadata,
+} from '@autonomys/auto-drive'
+import fs from 'fs'
+
+const metadata: OffchainMetadata = fs.readFileSync('path/to/your/metadata.json')
+
+const dag = createMetadataIPLDDag(metadata)
+
+const cid = cidOfNode(dag.headCID)
+const cidString = cidToString(cid)
+
+console.log(`CID of the metadata DAG: ${cidString}`)
+```
+
 ## License
 
 This project is licensed under the MIT License. See the [LICENSE](LICENSE) file for details.

package/dist/metadata/offchain/file.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"file.d.ts","sourceRoot":"","sources":["../../../src/metadata/offchain/file.ts"],"names":[],"mappings":"AAAA,OAAO,EAA0B,OAAO,EAAE,MAAM,gBAAgB,CAAA;AAEhE,MAAM,MAAM,oBAAoB,GAAG;IACjC,IAAI,EAAE,MAAM,CAAA;IACZ,OAAO,EAAE,MAAM,CAAA;IACf,IAAI,CAAC,EAAE,MAAM,CAAA;IACb,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB,SAAS,EAAE,MAAM,CAAA;IACjB,WAAW,EAAE,MAAM,CAAA;IACnB,MAAM,EAAE,SAAS,EAAE,CAAA;CACpB,CAAA;AAED,MAAM,WAAW,SAAS;IACxB,IAAI,EAAE,MAAM,CAAA;IACZ,GAAG,EAAE,MAAM,CAAA;CACZ;AAED,eAAO,MAAM,YAAY,QAClB,OAAO,aACD,MAAM,SACV,MAAM,aACF,MAAM,KAChB,oBAaF,CAAA"}
+{"version":3,"file":"file.d.ts","sourceRoot":"","sources":["../../../src/metadata/offchain/file.ts"],"names":[],"mappings":"AAAA,OAAO,EAA0B,OAAO,EAA8B,MAAM,gBAAgB,CAAA;AAE5F,MAAM,MAAM,oBAAoB,GAAG;IACjC,IAAI,EAAE,MAAM,CAAA;IACZ,OAAO,EAAE,MAAM,CAAA;IACf,IAAI,CAAC,EAAE,MAAM,CAAA;IACb,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB,SAAS,EAAE,MAAM,CAAA;IACjB,WAAW,EAAE,MAAM,CAAA;IACnB,MAAM,EAAE,SAAS,EAAE,CAAA;CACpB,CAAA;AAED,MAAM,WAAW,SAAS;IACxB,IAAI,EAAE,MAAM,CAAA;IACZ,GAAG,EAAE,MAAM,CAAA;CACZ;AAED,eAAO,MAAM,YAAY,QAClB,OAAO,aACD,MAAM,SACV,MAAM,aACF,MAAM,KAChB,oBAiBF,CAAA"}

package/dist/metadata/offchain/file.js

@@ -1,13 +1,14 @@
-import { cidOfNode, cidToString } from '../../index.js';
+import { cidOfNode, cidToString, IPLDNodeData } from '../../index.js';
 export const fileMetadata = (dag, totalSize, name, mimeType) => {
+    const chunks = Array.from(dag.nodes.values()).filter((n) => n.Data && IPLDNodeData.decode(n.Data).data);
     return {
         type: 'file',
         dataCid: cidToString(dag.headCID),
         name,
         mimeType,
         totalSize,
-        totalChunks: dag.nodes.size,
-        chunks: Array.from(dag.nodes.values()).map((chunk) => {
+        totalChunks: chunks.length,
+        chunks: chunks.map((chunk) => {
             var _a, _b;
             return ({
                 cid: cidToString(cidOfNode(chunk)),

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@autonomys/auto-drive",
   "packageManager": "yarn@4.1.1",
-  "version": "0.6.3",
+  "version": "0.6.4",
   "license": "MIT",
   "main": "dist/index.js",
   "repository": {
@@ -39,5 +39,5 @@
     "protons": "^7.6.0",
     "protons-runtime": "^5.5.0"
   },
-  "gitHead": "68a54c6f18e51cb1f32a5997c04230a6107ab631"
+  "gitHead": "cbeaa151644f75d43e7e36b5a817d580bff09ebd"
 }

package/src/ipld/builders.ts

@@ -0,0 +1,38 @@
+import { PBNode } from '@ipld/dag-pb'
+import { CID } from 'multiformats/cid'
+import {
+  createChunkedFileIpldNode,
+  createChunkedMetadataIpldNode,
+  createFileChunkIpldNode,
+  createFileInlinkIpldNode,
+  createMetadataChunkIpldNode,
+  createMetadataInlinkIpldNode,
+  createSingleFileIpldNode,
+  createSingleMetadataIpldNode,
+} from './nodes.js'
+
+export interface Builders {
+  inlink: (links: CID[], size: number, linkDepth: number, chunkSize: number) => PBNode
+  chunk: (data: Buffer) => PBNode
+  root: (
+    links: CID[],
+    size: number,
+    linkDepth: number,
+    name?: string,
+    maxNodeSize?: number,
+  ) => PBNode
+  single: (data: Buffer, filename?: string) => PBNode
+}
+export const metadataBuilders: Builders = {
+  inlink: createMetadataInlinkIpldNode,
+  chunk: createMetadataChunkIpldNode,
+  root: createChunkedMetadataIpldNode,
+  single: createSingleMetadataIpldNode,
+}
+
+export const fileBuilders: Builders = {
+  inlink: createFileInlinkIpldNode,
+  chunk: createFileChunkIpldNode,
+  root: createChunkedFileIpldNode,
+  single: createSingleFileIpldNode,
+}

package/src/ipld/chunker.ts

@@ -1,14 +1,9 @@
 import { PBNode } from '@ipld/dag-pb'
 import { CID } from 'multiformats'
 import { cidOfNode } from '../cid/index.js'
-import {
-  createChunkedFileIpldNode,
-  createChunkIpldNode,
-  createFileInlinkIpldNode,
-  createFolderInlinkIpldNode,
-  createFolderIpldNode,
-  createSingleFileIpldNode,
-} from './nodes.js'
+import { OffchainMetadata } from '../metadata/index.js'
+import { Builders, fileBuilders, metadataBuilders } from './builders.js'
+import { createFolderInlinkIpldNode, createFolderIpldNode } from './nodes.js'
 import { chunkBuffer, encodeNode } from './utils.js'
 
 export const DEFAULT_MAX_CHUNK_SIZE = 1024 * 64
@@ -27,8 +22,32 @@ export const createFileIPLDDag = (
     maxLinkPerNode: DEFAULT_MAX_LINK_PER_NODE,
   },
 ): IPLDDag => {
-  if (file.length <= chunkSize) {
-    const head = createSingleFileIpldNode(file, filename)
+  return createBufferIPLDDag(file, filename, fileBuilders, { chunkSize, maxLinkPerNode })
+}
+
+export const createMetadataIPLDDag = (
+  metadata: OffchainMetadata,
+  limits: { chunkSize: number; maxLinkPerNode: number } = {
+    chunkSize: DEFAULT_MAX_CHUNK_SIZE,
+    maxLinkPerNode: DEFAULT_MAX_LINK_PER_NODE,
+  },
+): IPLDDag => {
+  const buffer = Buffer.from(JSON.stringify(metadata))
+  const name = `${metadata.name}.metadata.json`
+  return createBufferIPLDDag(buffer, name, metadataBuilders, limits)
+}
+
+const createBufferIPLDDag = (
+  buffer: Buffer,
+  filename: string | undefined,
+  builders: Builders,
+  { chunkSize, maxLinkPerNode }: { chunkSize: number; maxLinkPerNode: number } = {
+    chunkSize: DEFAULT_MAX_CHUNK_SIZE,
+    maxLinkPerNode: DEFAULT_MAX_LINK_PER_NODE,
+  },
+): IPLDDag => {
+  if (buffer.length <= chunkSize) {
+    const head = builders.single(buffer, filename)
     const headCID = cidOfNode(head)
     return {
       headCID,
@@ -36,12 +55,12 @@ export const createFileIPLDDag = (
     }
   }
 
-  const bufferChunks = chunkBuffer(file, chunkSize)
+  const bufferChunks = chunkBuffer(buffer, chunkSize)
 
   const nodes = new Map<CID, PBNode>()
 
   let CIDs: CID[] = bufferChunks.map((chunk) => {
-    const node = createChunkIpldNode(chunk)
+    const node = builders.chunk(chunk)
     const cid = cidOfNode(node)
     nodes.set(cid, node)
 
@@ -54,7 +73,7 @@ export const createFileIPLDDag = (
     for (let i = 0; i < CIDs.length; i += maxLinkPerNode) {
       const chunk = CIDs.slice(i, i + maxLinkPerNode)
 
-      const node = createFileInlinkIpldNode(chunk, chunk.length, depth, chunkSize)
+      const node = builders.inlink(chunk, chunk.length, depth, chunkSize)
       const cid = cidOfNode(node)
       nodes.set(cid, node)
       newCIDs.push(cid)
@@ -62,7 +81,7 @@ export const createFileIPLDDag = (
     depth++
     CIDs = newCIDs
   }
-  const head = createChunkedFileIpldNode(CIDs, file.length, depth, filename, chunkSize)
+  const head = builders.root(CIDs, buffer.length, depth, filename, chunkSize)
   const headCID = cidOfNode(head)
   nodes.set(headCID, head)
 

package/src/ipld/nodes.ts

@@ -4,8 +4,8 @@ import { OffchainMetadata } from '../metadata/index.js'
 import { encodeIPLDNodeData, MetadataType } from '../metadata/onchain/index.js'
 import { DEFAULT_MAX_CHUNK_SIZE, ensureNodeMaxSize } from './chunker.js'
 
-/// Creates a chunk ipld node
-export const createChunkIpldNode = (data: Buffer): PBNode =>
+/// Creates a file chunk ipld node
+export const createFileChunkIpldNode = (data: Buffer): PBNode =>
   createNode(
     encodeIPLDNodeData({
       type: MetadataType.FileChunk,
@@ -40,7 +40,6 @@ export const createChunkedFileIpldNode = (
   )
 // Creates a file ipld node
 // links: the CIDs of the file's contents
-// @todo: add the file's metadata
 export const createFileInlinkIpldNode = (
   links: CID[],
   size: number,
@@ -74,6 +73,72 @@ export const createSingleFileIpldNode = (data: Buffer, name?: string): PBNode =>
     [],
   )
 
+// Creates a file ipld node
+// links: the CIDs of the file's contents
+// @todo: add the file's metadata
+export const createMetadataInlinkIpldNode = (
+  links: CID[],
+  size: number,
+  linkDepth: number,
+  maxNodeSize: number = DEFAULT_MAX_CHUNK_SIZE,
+): PBNode =>
+  ensureNodeMaxSize(
+    createNode(
+      encodeIPLDNodeData({
+        type: MetadataType.FileInlink,
+        size,
+        linkDepth,
+      }),
+      links.map((cid) => ({ Hash: cid })),
+    ),
+    maxNodeSize,
+  )
+
+// Creates a file ipld node
+// links: the CIDs of the file's contents
+// @todo: add the file's metadata
+export const createSingleMetadataIpldNode = (data: Buffer, name?: string): PBNode =>
+  createNode(
+    encodeIPLDNodeData({
+      type: MetadataType.Metadata,
+      name,
+      size: data.length,
+      linkDepth: 0,
+      data,
+    }),
+    [],
+  )
+
+export const createMetadataChunkIpldNode = (data: Buffer): PBNode =>
+  createNode(
+    encodeIPLDNodeData({
+      type: MetadataType.MetadataChunk,
+      size: data.length,
+      linkDepth: 0,
+      data,
+    }),
+  )
+
+export const createChunkedMetadataIpldNode = (
+  links: CID[],
+  size: number,
+  linkDepth: number,
+  name?: string,
+  maxNodeSize: number = DEFAULT_MAX_CHUNK_SIZE,
+): PBNode =>
+  ensureNodeMaxSize(
+    createNode(
+      encodeIPLDNodeData({
+        type: MetadataType.Metadata,
+        name,
+        size,
+        linkDepth,
+      }),
+      links.map((cid) => ({ Hash: cid })),
+    ),
+    maxNodeSize,
+  )
+
 // Creates a folder ipld node
 // links: the CIDs of the folder's contents
 // @todo: add the folder's metadata

package/src/metadata/onchain/protobuf/OnchainMetadata.proto

@@ -16,4 +16,6 @@ enum MetadataType {
   Folder = 3;
   FolderInlink = 4;
   Metadata = 5;
+  MetadataInlink = 6;
+  MetadataChunk = 7;
 }

package/src/metadata/onchain/protobuf/OnchainMetadata.ts

@@ -114,7 +114,9 @@ export enum MetadataType {
   FileChunk = 'FileChunk',
   Folder = 'Folder',
   FolderInlink = 'FolderInlink',
-  Metadata = 'Metadata'
+  Metadata = 'Metadata',
+  MetadataInlink = 'MetadataInlink',
+  MetadataChunk = 'MetadataChunk'
 }
 
 enum __MetadataTypeValues {
@@ -123,7 +125,9 @@ enum __MetadataTypeValues {
   FileChunk = 2,
   Folder = 3,
   FolderInlink = 4,
-  Metadata = 5
+  Metadata = 5,
+  MetadataInlink = 6,
+  MetadataChunk = 7
 }
 
 export namespace MetadataType {

package/tests/chunker.spec.ts

@@ -1,7 +1,7 @@
 import { createNode } from '@ipld/dag-pb'
 import { cidOfNode } from '../src'
-import { createFileIPLDDag, createFolderIPLDDag } from '../src/ipld/chunker'
-import { IPLDNodeData, MetadataType } from '../src/metadata'
+import { createFileIPLDDag, createFolderIPLDDag, createMetadataIPLDDag } from '../src/ipld/chunker'
+import { IPLDNodeData, MetadataType, OffchainMetadata } from '../src/metadata'
 
 describe('chunker', () => {
   describe('file creation', () => {
@@ -154,4 +154,39 @@ describe('chunker', () => {
       expect(inlinkCount).toBe(3)
     })
   })
+
+  describe('metadata creation', () => {
+    it('create a metadata dag from a small buffer', () => {
+      const metadata: OffchainMetadata = {
+        type: 'file',
+        dataCid: 'test',
+        name: 'test',
+        mimeType: 'text/plain',
+        totalSize: 1000,
+        totalChunks: 10,
+        chunks: [],
+      }
+
+      const dag = createMetadataIPLDDag(metadata)
+      expect(dag.nodes.size).toBe(1)
+    })
+
+    it('large metadata dag represented into multiple nodes', () => {
+      const metadata: OffchainMetadata = {
+        type: 'file',
+        dataCid: 'test',
+        name: 'test',
+        mimeType: 'text/plain'.repeat(100),
+        totalSize: 1000,
+        totalChunks: 10,
+        chunks: [],
+      }
+
+      const dag = createMetadataIPLDDag(metadata, {
+        chunkSize: 200,
+        maxLinkPerNode: 2,
+      })
+      expect(dag.nodes.size).toBeGreaterThan(1)
+    })
+  })
 })

package/tests/nodes.spec.ts

@@ -1,6 +1,10 @@
 import { createNode } from '@ipld/dag-pb'
-import { cidOfNode, createChunkedFileIpldNode, createSingleFileIpldNode } from '../src/index.js'
-import { createChunkIpldNode } from '../src/ipld/nodes.js'
+import {
+  cidOfNode,
+  createChunkedFileIpldNode,
+  createFileChunkIpldNode,
+  createSingleFileIpldNode,
+} from '../src/index.js'
 import { IPLDNodeData, MetadataType } from '../src/metadata/onchain/protobuf/OnchainMetadata.js'
 
 describe('node creation', () => {
@@ -58,7 +62,7 @@ describe('node creation', () => {
 
     it('file chunk node | correctly params setup', () => {
       const buffer = Buffer.from('hello world')
-      const node = createChunkIpldNode(buffer)
+      const node = createFileChunkIpldNode(buffer)
 
       const decoded = IPLDNodeData.decode(node.Data ?? new Uint8Array())
       expect(decoded.type).toBe(MetadataType.FileChunk)