@autonomys/auto-drive 0.6.2 → 0.6.4

package/LICENSE

@@ -0,0 +1,18 @@
+MIT License
+
+Copyright (c) 2024 Autonomys Network (autonomys.xyz)
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+1. **Attribution**: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+2. **No Warranty**: THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+
+3. **Limitation of Liability**: IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+---

package/README.md

@@ -1 +1,167 @@
-# auto-drive
+# Auto-Drive
+
+![Autonomys Banner](https://github.com/autonomys/auto-sdk/blob/main/.github/images/autonomys-banner.webp)
+
+[![Latest Github release](https://img.shields.io/github/v/tag/autonomys/auto-sdk.svg)](https://github.com/autonomys/auto-sdk/tags)
+[![Build status of the main branch on Linux/OSX](https://img.shields.io/github/actions/workflow/status/autonomys/auto-sdk/build.yaml?branch=main&label=Linux%2FOSX%20build)](https://github.com/autonomys/auto-sdk/actions/workflows/build.yaml)
+[![npm version](https://badge.fury.io/js/@autonomys%2Fauto-drive.svg)](https://badge.fury.io/js/@autonomys/auto-drive)
+
+## Overview
+
+The **Autonomys Auto Drive SDK** (`@autonomys/auto-drive`) provides utilities for creating and managing IPLD DAGs (InterPlanetary Linked Data Directed Acyclic Graphs) for files and folders. It facilitates chunking large files, handling metadata, and creating folder structures suitable for distributed storage systems like IPFS.
+
+## Features
+
+- **File Chunking and DAG Creation**: Efficiently split large files into smaller chunks and create IPLD DAGs.
+- **Folder Structure Creation**: Generate IPLD DAGs for directory structures.
+- **Metadata Handling**: Add and manage metadata for files and folders.
+- **CID Management**: Utilities for working with Content Identifiers (CIDs).
+- **TypeScript Support**: Fully typed for enhanced developer experience.
+
+## Installation
+
+You can install Auto-Drive using npm or yarn:
+
+```bash
+npm install @autonomys/auto-drive
+```
+
+or
+
+```bash
+yarn add @autonomys/auto-drive
+```
+
+## Usage
+
+### Creating an IPLD DAG from a File
+
+To create an IPLD DAG from a file, you can use the `createFileIPLDDag` function:
+
+```typescript
+import { createFileIPLDDag } from '@autonomys/auto-drive'
+import fs from 'fs'
+
+const fileBuffer = fs.readFileSync('path/to/your/file.txt')
+
+const dag = createFileIPLDDag(fileBuffer, 'file.txt')
+```
+
+### Creating an IPLD DAG from a Folder
+
+To create an IPLD DAG from a folder, you can use the `createFolderIPLDDag` function:
+
+```typescript
+import { createFolderIPLDDag } from '@autonomys/auto-drive'
+import { CID } from 'multiformats'
+
+// Example child CIDs and folder information
+const childCIDs: CID[] = [
+  /* array of CIDs */
+]
+const folderName = 'my-folder'
+const folderSize = 1024 // size in bytes
+
+const folderDag = createFolderIPLDDag(childCIDs, folderName, folderSize)
+```
+
+### Working with CIDs
+
+You can use functions from the `cid` module to work with CIDs:
+
+```typescript
+import { cidOfNode, cidToString, stringToCid } from '@autonomys/auto-drive'
+
+// Create a CID from a node
+const cid = cidOfNode(dag.head)
+
+// Convert the CID to a string
+const cidString = cidToString(cid)
+
+// Parse a string back into a CID
+const parsedCID = stringToCid(cidString)
+```
+
+### Encoding and Decoding Nodes
+
+You can encode and decode IPLD nodes:
+
+```typescript
+import { encodeNode, decodeNode } from '@autonomys/auto-drive'
+
+// Encode a node
+const encodedNode = encodeNode(dag.head)
+
+// Decode a node
+const decodedNode = decodeNode(encodedNode)
+```
+
+### Handling Metadata
+
+To add metadata to a node, you can create a metadata node:
+
+```typescript
+import { createMetadataNode } from '@autonomys/auto-drive'
+
+const metadata = {
+  name: 'My File',
+  description: 'This is a sample file',
+  // ... other metadata fields
+}
+
+const metadataNode = createMetadataNode(metadata)
+```
+
+### Example: Creating a File DAG and Converting to CID
+
+```typescript
+import { createFileIPLDDag, cidOfNode, cidToString } from '@autonomys/auto-drive'
+import fs from 'fs'
+
+const fileBuffer = fs.readFileSync('path/to/your/file.txt')
+
+const dag = createFileIPLDDag(fileBuffer, 'file.txt')
+
+const cid = cidOfNode(dag.headCID)
+const cidString = cidToString(cid)
+
+console.log(`CID of the file DAG: ${cidString}`)
+```
+
+### Example: Converting Metadata To DAG
+
+```typescript
+import {
+  createMetadataIPLDDag,
+  cidOfNode,
+  cidToString,
+  type OffchainMetadata,
+} from '@autonomys/auto-drive'
+import fs from 'fs'
+
+const metadata: OffchainMetadata = fs.readFileSync('path/to/your/metadata.json')
+
+const dag = createMetadataIPLDDag(metadata)
+
+const cid = cidOfNode(dag.headCID)
+const cidString = cidToString(cid)
+
+console.log(`CID of the metadata DAG: ${cidString}`)
+```
+
+## License
+
+This project is licensed under the MIT License. See the [LICENSE](LICENSE) file for details.
+
+## Additional Resources
+
+- **Autonomys Academy**: Learn more at [Autonomys Academy](https://academy.autonomys.xyz).
+- **Auto-Utils Package**: Utility functions used alongside `auto-drive` can be found in [`@autonomys/auto-utils`](../Auto-Utils/README.md).
+
+## Contact
+
+If you have any questions or need support, feel free to reach out:
+
+- **GitHub Issues**: [GitHub Issues Page](https://github.com/autonomys/auto-sdk/issues)
+
+We appreciate your feedback and contributions!

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@autonomys/auto-drive",
   "packageManager": "yarn@4.1.1",
-  "version": "0.6.2",
+  "version": "0.6.4",
   "license": "MIT",
   "main": "dist/index.js",
   "repository": {
@@ -39,5 +39,5 @@
     "protons": "^7.6.0",
     "protons-runtime": "^5.5.0"
   },
-  "gitHead": "c49eb0479d010f277abe0fde54665a7a06dd00b8"
+  "gitHead": "cbeaa151644f75d43e7e36b5a817d580bff09ebd"
 }

package/src/ipld/builders.ts

@@ -0,0 +1,38 @@
+import { PBNode } from '@ipld/dag-pb'
+import { CID } from 'multiformats/cid'
+import {
+  createChunkedFileIpldNode,
+  createChunkedMetadataIpldNode,
+  createFileChunkIpldNode,
+  createFileInlinkIpldNode,
+  createMetadataChunkIpldNode,
+  createMetadataInlinkIpldNode,
+  createSingleFileIpldNode,
+  createSingleMetadataIpldNode,
+} from './nodes.js'
+
+export interface Builders {
+  inlink: (links: CID[], size: number, linkDepth: number, chunkSize: number) => PBNode
+  chunk: (data: Buffer) => PBNode
+  root: (
+    links: CID[],
+    size: number,
+    linkDepth: number,
+    name?: string,
+    maxNodeSize?: number,
+  ) => PBNode
+  single: (data: Buffer, filename?: string) => PBNode
+}
+export const metadataBuilders: Builders = {
+  inlink: createMetadataInlinkIpldNode,
+  chunk: createMetadataChunkIpldNode,
+  root: createChunkedMetadataIpldNode,
+  single: createSingleMetadataIpldNode,
+}
+
+export const fileBuilders: Builders = {
+  inlink: createFileInlinkIpldNode,
+  chunk: createFileChunkIpldNode,
+  root: createChunkedFileIpldNode,
+  single: createSingleFileIpldNode,
+}

package/src/ipld/chunker.ts

@@ -1,14 +1,9 @@
 import { PBNode } from '@ipld/dag-pb'
 import { CID } from 'multiformats'
 import { cidOfNode } from '../cid/index.js'
-import {
-  createChunkedFileIpldNode,
-  createChunkIpldNode,
-  createFileInlinkIpldNode,
-  createFolderInlinkIpldNode,
-  createFolderIpldNode,
-  createSingleFileIpldNode,
-} from './nodes.js'
+import { OffchainMetadata } from '../metadata/index.js'
+import { Builders, fileBuilders, metadataBuilders } from './builders.js'
+import { createFolderInlinkIpldNode, createFolderIpldNode } from './nodes.js'
 import { chunkBuffer, encodeNode } from './utils.js'
 
 export const DEFAULT_MAX_CHUNK_SIZE = 1024 * 64
@@ -27,8 +22,32 @@ export const createFileIPLDDag = (
     maxLinkPerNode: DEFAULT_MAX_LINK_PER_NODE,
   },
 ): IPLDDag => {
-  if (file.length <= chunkSize) {
-    const head = createSingleFileIpldNode(file, filename)
+  return createBufferIPLDDag(file, filename, fileBuilders, { chunkSize, maxLinkPerNode })
+}
+
+export const createMetadataIPLDDag = (
+  metadata: OffchainMetadata,
+  limits: { chunkSize: number; maxLinkPerNode: number } = {
+    chunkSize: DEFAULT_MAX_CHUNK_SIZE,
+    maxLinkPerNode: DEFAULT_MAX_LINK_PER_NODE,
+  },
+): IPLDDag => {
+  const buffer = Buffer.from(JSON.stringify(metadata))
+  const name = `${metadata.name}.metadata.json`
+  return createBufferIPLDDag(buffer, name, metadataBuilders, limits)
+}
+
+const createBufferIPLDDag = (
+  buffer: Buffer,
+  filename: string | undefined,
+  builders: Builders,
+  { chunkSize, maxLinkPerNode }: { chunkSize: number; maxLinkPerNode: number } = {
+    chunkSize: DEFAULT_MAX_CHUNK_SIZE,
+    maxLinkPerNode: DEFAULT_MAX_LINK_PER_NODE,
+  },
+): IPLDDag => {
+  if (buffer.length <= chunkSize) {
+    const head = builders.single(buffer, filename)
     const headCID = cidOfNode(head)
     return {
       headCID,
@@ -36,12 +55,12 @@ export const createFileIPLDDag = (
     }
   }
 
-  const bufferChunks = chunkBuffer(file, chunkSize)
+  const bufferChunks = chunkBuffer(buffer, chunkSize)
 
   const nodes = new Map<CID, PBNode>()
 
   let CIDs: CID[] = bufferChunks.map((chunk) => {
-    const node = createChunkIpldNode(chunk)
+    const node = builders.chunk(chunk)
     const cid = cidOfNode(node)
     nodes.set(cid, node)
 
@@ -54,7 +73,7 @@ export const createFileIPLDDag = (
     for (let i = 0; i < CIDs.length; i += maxLinkPerNode) {
       const chunk = CIDs.slice(i, i + maxLinkPerNode)
 
-      const node = createFileInlinkIpldNode(chunk, chunk.length, depth, chunkSize)
+      const node = builders.inlink(chunk, chunk.length, depth, chunkSize)
       const cid = cidOfNode(node)
       nodes.set(cid, node)
       newCIDs.push(cid)
@@ -62,7 +81,7 @@ export const createFileIPLDDag = (
     depth++
     CIDs = newCIDs
   }
-  const head = createChunkedFileIpldNode(CIDs, file.length, depth, filename, chunkSize)
+  const head = builders.root(CIDs, buffer.length, depth, filename, chunkSize)
   const headCID = cidOfNode(head)
   nodes.set(headCID, head)
 

package/src/ipld/nodes.ts

@@ -4,8 +4,8 @@ import { OffchainMetadata } from '../metadata/index.js'
 import { encodeIPLDNodeData, MetadataType } from '../metadata/onchain/index.js'
 import { DEFAULT_MAX_CHUNK_SIZE, ensureNodeMaxSize } from './chunker.js'
 
-/// Creates a chunk ipld node
-export const createChunkIpldNode = (data: Buffer): PBNode =>
+/// Creates a file chunk ipld node
+export const createFileChunkIpldNode = (data: Buffer): PBNode =>
   createNode(
     encodeIPLDNodeData({
       type: MetadataType.FileChunk,
@@ -40,7 +40,6 @@ export const createChunkedFileIpldNode = (
   )
 // Creates a file ipld node
 // links: the CIDs of the file's contents
-// @todo: add the file's metadata
 export const createFileInlinkIpldNode = (
   links: CID[],
   size: number,
@@ -74,6 +73,72 @@ export const createSingleFileIpldNode = (data: Buffer, name?: string): PBNode =>
     [],
   )
 
+// Creates a file ipld node
+// links: the CIDs of the file's contents
+// @todo: add the file's metadata
+export const createMetadataInlinkIpldNode = (
+  links: CID[],
+  size: number,
+  linkDepth: number,
+  maxNodeSize: number = DEFAULT_MAX_CHUNK_SIZE,
+): PBNode =>
+  ensureNodeMaxSize(
+    createNode(
+      encodeIPLDNodeData({
+        type: MetadataType.FileInlink,
+        size,
+        linkDepth,
+      }),
+      links.map((cid) => ({ Hash: cid })),
+    ),
+    maxNodeSize,
+  )
+
+// Creates a file ipld node
+// links: the CIDs of the file's contents
+// @todo: add the file's metadata
+export const createSingleMetadataIpldNode = (data: Buffer, name?: string): PBNode =>
+  createNode(
+    encodeIPLDNodeData({
+      type: MetadataType.Metadata,
+      name,
+      size: data.length,
+      linkDepth: 0,
+      data,
+    }),
+    [],
+  )
+
+export const createMetadataChunkIpldNode = (data: Buffer): PBNode =>
+  createNode(
+    encodeIPLDNodeData({
+      type: MetadataType.MetadataChunk,
+      size: data.length,
+      linkDepth: 0,
+      data,
+    }),
+  )
+
+export const createChunkedMetadataIpldNode = (
+  links: CID[],
+  size: number,
+  linkDepth: number,
+  name?: string,
+  maxNodeSize: number = DEFAULT_MAX_CHUNK_SIZE,
+): PBNode =>
+  ensureNodeMaxSize(
+    createNode(
+      encodeIPLDNodeData({
+        type: MetadataType.Metadata,
+        name,
+        size,
+        linkDepth,
+      }),
+      links.map((cid) => ({ Hash: cid })),
+    ),
+    maxNodeSize,
+  )
+
 // Creates a folder ipld node
 // links: the CIDs of the folder's contents
 // @todo: add the folder's metadata

package/src/metadata/onchain/protobuf/OnchainMetadata.proto

@@ -16,4 +16,6 @@ enum MetadataType {
   Folder = 3;
   FolderInlink = 4;
   Metadata = 5;
+  MetadataInlink = 6;
+  MetadataChunk = 7;
 }

package/src/metadata/onchain/protobuf/OnchainMetadata.ts

@@ -114,7 +114,9 @@ export enum MetadataType {
   FileChunk = 'FileChunk',
   Folder = 'Folder',
   FolderInlink = 'FolderInlink',
-  Metadata = 'Metadata'
+  Metadata = 'Metadata',
+  MetadataInlink = 'MetadataInlink',
+  MetadataChunk = 'MetadataChunk'
 }
 
 enum __MetadataTypeValues {
@@ -123,7 +125,9 @@ enum __MetadataTypeValues {
   FileChunk = 2,
   Folder = 3,
   FolderInlink = 4,
-  Metadata = 5
+  Metadata = 5,
+  MetadataInlink = 6,
+  MetadataChunk = 7
 }
 
 export namespace MetadataType {

package/tests/chunker.spec.ts

@@ -1,7 +1,7 @@
 import { createNode } from '@ipld/dag-pb'
 import { cidOfNode } from '../src'
-import { createFileIPLDDag, createFolderIPLDDag } from '../src/ipld/chunker'
-import { IPLDNodeData, MetadataType } from '../src/metadata'
+import { createFileIPLDDag, createFolderIPLDDag, createMetadataIPLDDag } from '../src/ipld/chunker'
+import { IPLDNodeData, MetadataType, OffchainMetadata } from '../src/metadata'
 
 describe('chunker', () => {
   describe('file creation', () => {
@@ -154,4 +154,39 @@ describe('chunker', () => {
       expect(inlinkCount).toBe(3)
     })
   })
+
+  describe('metadata creation', () => {
+    it('create a metadata dag from a small buffer', () => {
+      const metadata: OffchainMetadata = {
+        type: 'file',
+        dataCid: 'test',
+        name: 'test',
+        mimeType: 'text/plain',
+        totalSize: 1000,
+        totalChunks: 10,
+        chunks: [],
+      }
+
+      const dag = createMetadataIPLDDag(metadata)
+      expect(dag.nodes.size).toBe(1)
+    })
+
+    it('large metadata dag represented into multiple nodes', () => {
+      const metadata: OffchainMetadata = {
+        type: 'file',
+        dataCid: 'test',
+        name: 'test',
+        mimeType: 'text/plain'.repeat(100),
+        totalSize: 1000,
+        totalChunks: 10,
+        chunks: [],
+      }
+
+      const dag = createMetadataIPLDDag(metadata, {
+        chunkSize: 200,
+        maxLinkPerNode: 2,
+      })
+      expect(dag.nodes.size).toBeGreaterThan(1)
+    })
+  })
 })

package/tests/nodes.spec.ts

@@ -1,6 +1,10 @@
 import { createNode } from '@ipld/dag-pb'
-import { cidOfNode, createChunkedFileIpldNode, createSingleFileIpldNode } from '../src/index.js'
-import { createChunkIpldNode } from '../src/ipld/nodes.js'
+import {
+  cidOfNode,
+  createChunkedFileIpldNode,
+  createFileChunkIpldNode,
+  createSingleFileIpldNode,
+} from '../src/index.js'
 import { IPLDNodeData, MetadataType } from '../src/metadata/onchain/protobuf/OnchainMetadata.js'
 
 describe('node creation', () => {
@@ -58,7 +62,7 @@ describe('node creation', () => {
 
     it('file chunk node | correctly params setup', () => {
       const buffer = Buffer.from('hello world')
-      const node = createChunkIpldNode(buffer)
+      const node = createFileChunkIpldNode(buffer)
 
       const decoded = IPLDNodeData.decode(node.Data ?? new Uint8Array())
       expect(decoded.type).toBe(MetadataType.FileChunk)