@autonomys/auto-drive 0.5.0

package/dist/metadata/onchain/protobuf/onchainMetadata.js

@@ -0,0 +1,108 @@
+/* eslint-disable import/export */
+/* eslint-disable complexity */
+/* eslint-disable @typescript-eslint/no-namespace */
+/* eslint-disable @typescript-eslint/no-unnecessary-boolean-literal-compare */
+/* eslint-disable @typescript-eslint/no-empty-interface */
+import { decodeMessage, encodeMessage, enumeration, message } from 'protons-runtime';
+export var IPLDNodeData;
+(function (IPLDNodeData) {
+    let _codec;
+    IPLDNodeData.codec = () => {
+        if (_codec == null) {
+            _codec = message((obj, w, opts = {}) => {
+                if (opts.lengthDelimited !== false) {
+                    w.fork();
+                }
+                if (obj.type != null && __MetadataTypeValues[obj.type] !== 0) {
+                    w.uint32(8);
+                    MetadataType.codec().encode(obj.type, w);
+                }
+                if ((obj.linkDepth != null && obj.linkDepth !== 0)) {
+                    w.uint32(16);
+                    w.int32(obj.linkDepth);
+                }
+                if (obj.size != null) {
+                    w.uint32(24);
+                    w.int32(obj.size);
+                }
+                if (obj.name != null) {
+                    w.uint32(34);
+                    w.string(obj.name);
+                }
+                if (obj.data != null) {
+                    w.uint32(42);
+                    w.bytes(obj.data);
+                }
+                if (opts.lengthDelimited !== false) {
+                    w.ldelim();
+                }
+            }, (reader, length, opts = {}) => {
+                const obj = {
+                    type: MetadataType.File,
+                    linkDepth: 0
+                };
+                const end = length == null ? reader.len : reader.pos + length;
+                while (reader.pos < end) {
+                    const tag = reader.uint32();
+                    switch (tag >>> 3) {
+                        case 1: {
+                            obj.type = MetadataType.codec().decode(reader);
+                            break;
+                        }
+                        case 2: {
+                            obj.linkDepth = reader.int32();
+                            break;
+                        }
+                        case 3: {
+                            obj.size = reader.int32();
+                            break;
+                        }
+                        case 4: {
+                            obj.name = reader.string();
+                            break;
+                        }
+                        case 5: {
+                            obj.data = reader.bytes();
+                            break;
+                        }
+                        default: {
+                            reader.skipType(tag & 7);
+                            break;
+                        }
+                    }
+                }
+                return obj;
+            });
+        }
+        return _codec;
+    };
+    IPLDNodeData.encode = (obj) => {
+        return encodeMessage(obj, IPLDNodeData.codec());
+    };
+    IPLDNodeData.decode = (buf, opts) => {
+        return decodeMessage(buf, IPLDNodeData.codec(), opts);
+    };
+})(IPLDNodeData || (IPLDNodeData = {}));
+export var MetadataType;
+(function (MetadataType) {
+    MetadataType["File"] = "File";
+    MetadataType["FileInlink"] = "FileInlink";
+    MetadataType["FileChunk"] = "FileChunk";
+    MetadataType["Folder"] = "Folder";
+    MetadataType["FolderInlink"] = "FolderInlink";
+    MetadataType["Metadata"] = "Metadata";
+})(MetadataType || (MetadataType = {}));
+var __MetadataTypeValues;
+(function (__MetadataTypeValues) {
+    __MetadataTypeValues[__MetadataTypeValues["File"] = 0] = "File";
+    __MetadataTypeValues[__MetadataTypeValues["FileInlink"] = 1] = "FileInlink";
+    __MetadataTypeValues[__MetadataTypeValues["FileChunk"] = 2] = "FileChunk";
+    __MetadataTypeValues[__MetadataTypeValues["Folder"] = 3] = "Folder";
+    __MetadataTypeValues[__MetadataTypeValues["FolderInlink"] = 4] = "FolderInlink";
+    __MetadataTypeValues[__MetadataTypeValues["Metadata"] = 5] = "Metadata";
+})(__MetadataTypeValues || (__MetadataTypeValues = {}));
+(function (MetadataType) {
+    MetadataType.codec = () => {
+        return enumeration(__MetadataTypeValues);
+    };
+})(MetadataType || (MetadataType = {}));

package/dist/metadata/onchain/utils.d.ts

@@ -0,0 +1,4 @@
+import { IPLDNodeData } from '../onchain/index.js';
+export declare const encodeIPLDNodeData: (metadata: IPLDNodeData) => Uint8Array;
+export declare const decodeIPLDNodeData: (data: Uint8Array) => IPLDNodeData;
+//# sourceMappingURL=utils.d.ts.map

package/dist/metadata/onchain/utils.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../../../src/metadata/onchain/utils.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAA;AAElD,eAAO,MAAM,kBAAkB,aAAc,YAAY,KAAG,UAE3D,CAAA;AAED,eAAO,MAAM,kBAAkB,SAAU,UAAU,KAAG,YAOrD,CAAA"}

package/dist/metadata/onchain/utils.js

@@ -0,0 +1,12 @@
+import { decode } from '@ipld/dag-pb';
+import { IPLDNodeData } from '../onchain/index.js';
+export const encodeIPLDNodeData = (metadata) => {
+    return IPLDNodeData.encode(metadata);
+};
+export const decodeIPLDNodeData = (data) => {
+    const decoded = decode(data);
+    if (!decoded.Data) {
+        throw new Error('Invalid data');
+    }
+    return IPLDNodeData.decode(decoded.Data);
+};

package/jest.config.ts

@@ -0,0 +1,17 @@
+const { createDefaultEsmPreset } = require('ts-jest')
+
+module.exports = {
+  ...createDefaultEsmPreset(),
+  extensionsToTreatAsEsm: ['.ts'],
+  moduleNameMapper: {
+    '^(\\.{1,2}/.*)\\.js$': '$1',
+  },
+  transform: {
+    '^.+\\.tsx?$': [
+      'ts-jest',
+      {
+        useESM: true,
+      },
+    ],
+  },
+}

package/package.json

@@ -0,0 +1,42 @@
+{
+  "name": "@autonomys/auto-drive",
+  "packageManager": "yarn@4.1.1",
+  "version": "0.5.0",
+  "main": "dist/index.js",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/autonomys/auto-sdk"
+  },
+  "author": {
+    "name": "Autonomys",
+    "url": "https://www.autonomys.net"
+  },
+  "type": "module",
+  "scripts": {
+    "build": "tsc",
+    "pb": "yarn protons src/metadata/onchain/protobuf/OnchainMetadata.proto -o src/metadata/onchain/protobuf",
+    "clean": "rm -rf dist",
+    "format": "prettier --write \"src/**/*.ts\"",
+    "test": "yarn node --experimental-vm-modules $(yarn bin jest)"
+  },
+  "exports": {
+    ".": "./dist/index.js",
+    "./protobuf": "./dist/metadata/onchain/protobuf/onchainMetadata.js"
+  },
+  "devDependencies": {
+    "@types/jest": "^29.5.13",
+    "jest": "^29.7.0",
+    "protobufjs": "^7.4.0",
+    "ts-jest": "^29.2.5",
+    "typescript": "^5.6.2"
+  },
+  "dependencies": {
+    "@ipld/dag-pb": "^4.1.2",
+    "blake3": "1.1.0",
+    "multiformats": "^13.2.2",
+    "protobufjs": "^7.4.0",
+    "protons": "^7.6.0",
+    "protons-runtime": "^5.5.0"
+  },
+  "gitHead": "b7b40dfa6b9cb8bf189fd9d5d3a0ad190b21197a"
+}

package/src/cid/index.ts

@@ -0,0 +1,26 @@
+import { encode, PBNode } from '@ipld/dag-pb'
+import { hash } from 'blake3'
+import { CID } from 'multiformats/cid'
+import * as base32 from 'multiformats/bases/base32'
+import * as raw from 'multiformats/codecs/raw'
+import { create } from 'multiformats/hashes/digest'
+
+export const BLAKE3_CODE = 0x1f
+
+export const cidOfNode = (node: PBNode) => {
+  return cidFromBlakeHash(hash(encode(node)))
+}
+
+export const cidToString = (cid: CID) => {
+  return cid.toString(base32.base32)
+}
+
+export const stringToCid = (str: string) => {
+  return CID.parse(str, base32.base32)
+}
+
+export const cidFromBlakeHash = (hash: Buffer) => {
+  return CID.create(1, raw.code, create(BLAKE3_CODE, hash))
+}
+
+export const blake3HashFromCid = (cid: CID) => cid.multihash.digest

package/src/index.ts

@@ -0,0 +1,3 @@
+export * from './cid/index.js'
+export * from './ipld/index.js'
+export * from './metadata/index.js'

package/src/ipld/chunker.ts

@@ -0,0 +1,117 @@
+import { PBNode } from '@ipld/dag-pb'
+import { CID } from 'multiformats'
+import { cidOfNode } from '../cid/index.js'
+import {
+  createChunkedFileIpldNode,
+  createChunkIpldNode,
+  createFileInlinkIpldNode,
+  createFolderInlinkIpldNode,
+  createFolderIpldNode,
+  createSingleFileIpldNode,
+} from './nodes.js'
+import { chunkBuffer, encodeNode } from './utils.js'
+
+export const DEFAULT_MAX_CHUNK_SIZE = 1024 * 64
+export const DEFAULT_MAX_LINK_PER_NODE = DEFAULT_MAX_CHUNK_SIZE / 64
+
+export interface IPLDDag {
+  headCID: CID
+  nodes: Map<CID, PBNode>
+}
+
+export const createFileIPLDDag = (
+  file: Buffer,
+  filename?: string,
+  { chunkSize, maxLinkPerNode }: { chunkSize: number; maxLinkPerNode: number } = {
+    chunkSize: DEFAULT_MAX_CHUNK_SIZE,
+    maxLinkPerNode: DEFAULT_MAX_LINK_PER_NODE,
+  },
+): IPLDDag => {
+  if (file.length <= chunkSize) {
+    const head = createSingleFileIpldNode(file, filename)
+    const headCID = cidOfNode(head)
+    return {
+      headCID,
+      nodes: new Map([[headCID, head]]),
+    }
+  }
+
+  const bufferChunks = chunkBuffer(file, chunkSize)
+
+  const nodes = new Map<CID, PBNode>()
+
+  let CIDs: CID[] = bufferChunks.map((chunk) => {
+    const node = createChunkIpldNode(chunk)
+    const cid = cidOfNode(node)
+    nodes.set(cid, node)
+
+    return cid
+  })
+
+  let depth = 1
+  while (CIDs.length > maxLinkPerNode) {
+    const newCIDs: CID[] = []
+    for (let i = 0; i < CIDs.length; i += maxLinkPerNode) {
+      const chunk = CIDs.slice(i, i + maxLinkPerNode)
+
+      const node = createFileInlinkIpldNode(chunk, chunk.length, depth, chunkSize)
+      const cid = cidOfNode(node)
+      nodes.set(cid, node)
+      newCIDs.push(cid)
+    }
+    depth++
+    CIDs = newCIDs
+  }
+  const head = createChunkedFileIpldNode(CIDs, file.length, depth, filename, chunkSize)
+  const headCID = cidOfNode(head)
+  nodes.set(headCID, head)
+
+  return {
+    headCID,
+    nodes,
+  }
+}
+
+export const createFolderIPLDDag = (
+  children: CID[],
+  name: string,
+  size: number,
+  { maxLinkPerNode }: { maxLinkPerNode: number } = { maxLinkPerNode: DEFAULT_MAX_LINK_PER_NODE },
+): IPLDDag => {
+  const nodes = new Map<CID, PBNode>()
+  let cids = children
+  let depth = 0
+  while (cids.length > maxLinkPerNode) {
+    const newCIDs: CID[] = []
+    for (let i = 0; i < cids.length; i += maxLinkPerNode) {
+      const chunk = cids.slice(i, i + maxLinkPerNode)
+      const node = createFolderInlinkIpldNode(chunk, depth)
+      const cid = cidOfNode(node)
+      nodes.set(cid, node)
+      newCIDs.push(cid)
+    }
+    cids = newCIDs
+    depth++
+  }
+
+  const node = createFolderIpldNode(cids, name, depth, size)
+  const cid = cidOfNode(node)
+  nodes.set(cid, node)
+
+  return {
+    headCID: cid,
+    nodes,
+  }
+}
+
+export const ensureNodeMaxSize = (
+  node: PBNode,
+  maxSize: number = DEFAULT_MAX_CHUNK_SIZE,
+): PBNode => {
+  const nodeSize = encodeNode(node).byteLength
+  if (nodeSize > maxSize) {
+    throw new Error(`Node is too large to fit in a single chunk: ${nodeSize} > ${maxSize}`)
+  }
+
+  return node
+}

package/src/ipld/index.ts

@@ -0,0 +1,3 @@
+export * from './chunker.js'
+export * from './nodes.js'
+export { encodeNode } from './utils.js'

package/src/ipld/nodes.ts

@@ -0,0 +1,134 @@
+import { createNode, PBNode } from '@ipld/dag-pb'
+import { CID } from 'multiformats/cid'
+import { OffchainMetadata } from '../metadata/index.js'
+import { encodeIPLDNodeData, MetadataType } from '../metadata/onchain/index.js'
+import { DEFAULT_MAX_CHUNK_SIZE, ensureNodeMaxSize } from './chunker.js'
+
+/// Creates a chunk ipld node
+export const createChunkIpldNode = (data: Buffer): PBNode =>
+  createNode(
+    encodeIPLDNodeData({
+      type: MetadataType.FileChunk,
+      size: data.length,
+      linkDepth: 0,
+      data,
+    }),
+    [],
+  )
+
+// Creates a file ipld node
+// links: the CIDs of the file's contents
+// @todo: add the file's metadata
+export const createChunkedFileIpldNode = (
+  links: CID[],
+  size: number,
+  linkDepth: number,
+  name?: string,
+  maxNodeSize: number = DEFAULT_MAX_CHUNK_SIZE,
+): PBNode =>
+  ensureNodeMaxSize(
+    createNode(
+      encodeIPLDNodeData({
+        type: MetadataType.File,
+        name,
+        size,
+        linkDepth,
+      }),
+      links.map((cid) => ({ Hash: cid })),
+    ),
+    maxNodeSize,
+  )
+// Creates a file ipld node
+// links: the CIDs of the file's contents
+// @todo: add the file's metadata
+export const createFileInlinkIpldNode = (
+  links: CID[],
+  size: number,
+  linkDepth: number,
+  maxNodeSize: number = DEFAULT_MAX_CHUNK_SIZE,
+): PBNode =>
+  ensureNodeMaxSize(
+    createNode(
+      encodeIPLDNodeData({
+        type: MetadataType.FileInlink,
+        size,
+        linkDepth,
+      }),
+      links.map((cid) => ({ Hash: cid })),
+    ),
+    maxNodeSize,
+  )
+
+// Creates a file ipld node
+// links: the CIDs of the file's contents
+// @todo: add the file's metadata
+export const createSingleFileIpldNode = (data: Buffer, name?: string): PBNode =>
+  createNode(
+    encodeIPLDNodeData({
+      type: MetadataType.File,
+      name,
+      size: data.length,
+      linkDepth: 0,
+      data,
+    }),
+    [],
+  )
+
+// Creates a folder ipld node
+// links: the CIDs of the folder's contents
+// @todo: add the folder's metadata
+export const createFolderIpldNode = (
+  links: CID[],
+  name: string,
+  linkDepth: number,
+  size: number,
+  maxNodeSize: number = DEFAULT_MAX_CHUNK_SIZE,
+): PBNode =>
+  ensureNodeMaxSize(
+    createNode(
+      encodeIPLDNodeData({
+        type: MetadataType.Folder,
+        name,
+        size,
+        linkDepth,
+      }),
+      links.map((cid) => ({ Hash: cid })),
+    ),
+    maxNodeSize,
+  )
+
+export const createFolderInlinkIpldNode = (
+  links: CID[],
+  linkDepth: number,
+  maxNodeSize: number = DEFAULT_MAX_CHUNK_SIZE,
+): PBNode =>
+  ensureNodeMaxSize(
+    createNode(
+      encodeIPLDNodeData({
+        type: MetadataType.FolderInlink,
+        linkDepth,
+      }),
+      links.map((cid) => ({ Hash: cid })),
+    ),
+    maxNodeSize,
+  )
+
+/// Creates a metadata ipld node
+export const createMetadataNode = (
+  metadata: OffchainMetadata,
+  maxNodeSize: number = DEFAULT_MAX_CHUNK_SIZE,
+): PBNode => {
+  const data = Buffer.from(JSON.stringify(metadata))
+
+  return ensureNodeMaxSize(
+    createNode(
+      encodeIPLDNodeData({
+        type: MetadataType.Metadata,
+        name: metadata.name,
+        linkDepth: 0,
+        data,
+      }),
+    ),
+    maxNodeSize,
+  )
+}

package/src/ipld/utils.ts

@@ -0,0 +1,15 @@
+import { decode, encode, PBNode } from '@ipld/dag-pb'
+
+export const chunkBuffer = (buffer: Buffer, chunkSize: number) => {
+  const chunks: Buffer[] = []
+
+  for (let i = 0; i < buffer.length; i += chunkSize) {
+    chunks.push(Buffer.from(buffer.buffer.slice(i, i + chunkSize)))
+  }
+
+  return chunks
+}
+
+export const encodeNode = (node: PBNode): Buffer => Buffer.from(encode(node))
+
+export const decodeNode = (data: Uint8Array): PBNode => decode(data)

package/src/metadata/index.ts

@@ -0,0 +1,2 @@
+export * from './offchain/index.js'
+export * from './onchain/index.js'

package/src/metadata/offchain/base.ts

@@ -0,0 +1,4 @@
+import { OffchainFileMetadata } from './file.js'
+import { OffchainFolderMetadata } from './folder.js'
+
+export type OffchainMetadata = OffchainFolderMetadata | OffchainFileMetadata

package/src/metadata/offchain/file.ts

@@ -0,0 +1,36 @@
+import { cidOfNode, cidToString, IPLDDag } from '../../index.js'
+
+export type OffchainFileMetadata = {
+  type: 'file'
+  dataCid: string
+  name?: string
+  mimeType?: string
+  totalSize: number
+  totalChunks: number
+  chunks: ChunkInfo[]
+}
+
+export interface ChunkInfo {
+  size: number
+  cid: string
+}
+
+export const fileMetadata = (
+  dag: IPLDDag,
+  totalSize: number,
+  name?: string,
+  mimeType?: string,
+): OffchainFileMetadata => {
+  return {
+    type: 'file',
+    dataCid: cidToString(dag.headCID),
+    name,
+    mimeType,
+    totalSize,
+    totalChunks: dag.nodes.size,
+    chunks: Array.from(dag.nodes.values()).map((chunk) => ({
+      cid: cidToString(cidOfNode(chunk)),
+      size: chunk.Data?.length ?? 0,
+    })),
+  }
+}

package/src/metadata/offchain/folder.ts

@@ -0,0 +1,28 @@
+interface ChildrenMetadata {
+  type: 'folder' | 'file'
+  name?: string
+  cid: string
+  totalSize: number
+}
+
+export type OffchainFolderMetadata = {
+  type: 'folder'
+  dataCid: string
+  name?: string
+  totalSize: number
+  totalFiles: number
+  children: ChildrenMetadata[]
+}
+
+export const folderMetadata = (
+  cid: string,
+  children: ChildrenMetadata[],
+): OffchainFolderMetadata => {
+  return {
+    dataCid: cid,
+    totalSize: children.reduce((acc, child) => acc + child.totalSize, 0),
+    totalFiles: children.length,
+    children,
+    type: 'folder',
+  }
+}

package/src/metadata/offchain/index.ts

@@ -0,0 +1,3 @@
+export * from './base.js'
+export * from './file.js'
+export * from './folder.js'

package/src/metadata/onchain/index.ts

@@ -0,0 +1,2 @@
+export * from './protobuf/OnchainMetadata.js'
+export * from './utils.js'

package/src/metadata/onchain/protobuf/OnchainMetadata.proto

@@ -0,0 +1,19 @@
+syntax = "proto3";
+
+message IPLDNodeData {
+  MetadataType type = 1;
+  int32 linkDepth = 2;
+  optional int32 size = 3;
+  optional string name = 4;
+  optional bytes data = 5;
+}
+
+// MetadataType defines the possible types of metadata.
+enum MetadataType {
+  File = 0;
+  FileInlink = 1;
+  FileChunk = 2;
+  Folder = 3;
+  FolderInlink = 4;
+  Metadata = 5;
+}