@autonomys/auto-dag-data 0.8.2

package/src/ipld/nodes.ts

@@ -0,0 +1,208 @@
+import { CID } from 'multiformats/cid'
+import { FileUploadOptions, OffchainMetadata } from '../metadata/index.js'
+import { encodeIPLDNodeData, MetadataType } from '../metadata/onchain/index.js'
+import { DEFAULT_MAX_CHUNK_SIZE, ensureNodeMaxSize } from './chunker.js'
+import { createNode, PBNode } from './index.js'
+
+/// Creates a file chunk ipld node
+export const createFileChunkIpldNode = (data: Buffer): PBNode =>
+  createNode(
+    encodeIPLDNodeData({
+      type: MetadataType.FileChunk,
+      size: data.length,
+      linkDepth: 0,
+      data,
+    }),
+    [],
+  )
+
+// Creates a file ipld node
+// links: the CIDs of the file's contents
+// @todo: add the file's metadata
+export const createChunkedFileIpldNode = (
+  links: CID[],
+  size: number,
+  linkDepth: number,
+  name?: string,
+  maxNodeSize: number = DEFAULT_MAX_CHUNK_SIZE,
+  uploadOptions?: FileUploadOptions,
+): PBNode =>
+  ensureNodeMaxSize(
+    createNode(
+      encodeIPLDNodeData({
+        type: MetadataType.File,
+        name,
+        size,
+        linkDepth,
+        uploadOptions,
+      }),
+      links.map((cid) => ({ Hash: cid })),
+    ),
+    maxNodeSize,
+  )
+// Creates a file ipld node
+// links: the CIDs of the file's contents
+export const createFileInlinkIpldNode = (
+  links: CID[],
+  size: number,
+  linkDepth: number,
+  maxNodeSize: number = DEFAULT_MAX_CHUNK_SIZE,
+): PBNode =>
+  ensureNodeMaxSize(
+    createNode(
+      encodeIPLDNodeData({
+        type: MetadataType.FileInlink,
+        size,
+        linkDepth,
+      }),
+      links.map((cid) => ({ Hash: cid })),
+    ),
+    maxNodeSize,
+  )
+
+// Creates a file ipld node
+// links: the CIDs of the file's contents
+// @todo: add the file's metadata
+export const createSingleFileIpldNode = (
+  data: Buffer,
+  name?: string,
+  uploadOptions?: FileUploadOptions,
+): PBNode =>
+  createNode(
+    encodeIPLDNodeData({
+      type: MetadataType.File,
+      name,
+      size: data.length,
+      linkDepth: 0,
+      data,
+      uploadOptions,
+    }),
+    [],
+  )
+
+// Creates a file ipld node
+// links: the CIDs of the file's contents
+// @todo: add the file's metadata
+export const createMetadataInlinkIpldNode = (
+  links: CID[],
+  size: number,
+  linkDepth: number,
+  maxNodeSize: number = DEFAULT_MAX_CHUNK_SIZE,
+): PBNode =>
+  ensureNodeMaxSize(
+    createNode(
+      encodeIPLDNodeData({
+        type: MetadataType.FileInlink,
+        size,
+        linkDepth,
+      }),
+      links.map((cid) => ({ Hash: cid })),
+    ),
+    maxNodeSize,
+  )
+
+// Creates a file ipld node
+// links: the CIDs of the file's contents
+// @todo: add the file's metadata
+export const createSingleMetadataIpldNode = (data: Buffer, name?: string): PBNode =>
+  createNode(
+    encodeIPLDNodeData({
+      type: MetadataType.Metadata,
+      name,
+      size: data.length,
+      linkDepth: 0,
+      data,
+    }),
+    [],
+  )
+
+export const createMetadataChunkIpldNode = (data: Buffer): PBNode =>
+  createNode(
+    encodeIPLDNodeData({
+      type: MetadataType.MetadataChunk,
+      size: data.length,
+      linkDepth: 0,
+      data,
+    }),
+  )
+
+export const createChunkedMetadataIpldNode = (
+  links: CID[],
+  size: number,
+  linkDepth: number,
+  name?: string,
+  maxNodeSize: number = DEFAULT_MAX_CHUNK_SIZE,
+): PBNode =>
+  ensureNodeMaxSize(
+    createNode(
+      encodeIPLDNodeData({
+        type: MetadataType.Metadata,
+        name,
+        size,
+        linkDepth,
+      }),
+      links.map((cid) => ({ Hash: cid })),
+    ),
+    maxNodeSize,
+  )
+
+// Creates a folder ipld node
+// links: the CIDs of the folder's contents
+// @todo: add the folder's metadata
+export const createFolderIpldNode = (
+  links: CID[],
+  name: string,
+  linkDepth: number,
+  size: number,
+  maxNodeSize: number = DEFAULT_MAX_CHUNK_SIZE,
+  uploadOptions?: FileUploadOptions,
+): PBNode =>
+  ensureNodeMaxSize(
+    createNode(
+      encodeIPLDNodeData({
+        type: MetadataType.Folder,
+        name,
+        size,
+        linkDepth,
+        uploadOptions,
+      }),
+      links.map((cid) => ({ Hash: cid })),
+    ),
+    maxNodeSize,
+  )
+
+export const createFolderInlinkIpldNode = (
+  links: CID[],
+  linkDepth: number,
+  maxNodeSize: number = DEFAULT_MAX_CHUNK_SIZE,
+): PBNode =>
+  ensureNodeMaxSize(
+    createNode(
+      encodeIPLDNodeData({
+        type: MetadataType.FolderInlink,
+        linkDepth,
+      }),
+      links.map((cid) => ({ Hash: cid })),
+    ),
+    maxNodeSize,
+  )
+
+/// Creates a metadata ipld node
+export const createMetadataNode = (
+  metadata: OffchainMetadata,
+  maxNodeSize: number = DEFAULT_MAX_CHUNK_SIZE,
+): PBNode => {
+  const data = Buffer.from(JSON.stringify(metadata))
+
+  return ensureNodeMaxSize(
+    createNode(
+      encodeIPLDNodeData({
+        type: MetadataType.Metadata,
+        name: metadata.name,
+        linkDepth: 0,
+        data,
+      }),
+    ),
+    maxNodeSize,
+  )
+}

package/src/ipld/utils.ts

@@ -0,0 +1,21 @@
+import { createNode, decode, encode, PBNode } from '@ipld/dag-pb'
+import { AwaitIterable } from 'interface-store'
+
+export const chunkBuffer = async function* (
+  buffer: AwaitIterable<Buffer>,
+  { maxChunkSize, ignoreLastChunk = false }: { maxChunkSize: number; ignoreLastChunk?: boolean },
+): AsyncIterable<Buffer> {
+  let target = Buffer.alloc(0)
+  for await (let chunk of buffer) {
+    target = Buffer.concat([target, chunk])
+    while (target.length >= maxChunkSize) {
+      yield target.subarray(0, maxChunkSize)
+      target = target.subarray(maxChunkSize)
+    }
+  }
+  if (target.length > 0 && !ignoreLastChunk) {
+    yield target
+  }
+}
+
+export { createNode, decode as decodeNode, encode as encodeNode, PBNode }

package/src/metadata/index.ts

@@ -0,0 +1,2 @@
+export * from './offchain/index.js'
+export * from './onchain/index.js'

package/src/metadata/offchain/base.ts

@@ -0,0 +1,4 @@
+import { OffchainFileMetadata } from './file.js'
+import { OffchainFolderMetadata } from './folder.js'
+
+export type OffchainMetadata = OffchainFolderMetadata | OffchainFileMetadata

package/src/metadata/offchain/file.ts

@@ -0,0 +1,41 @@
+import { CID } from 'multiformats'
+import { cidToString, FileUploadOptions } from '../../index.js'
+
+export type OffchainFileMetadata = {
+  type: 'file'
+  dataCid: string
+  name?: string
+  mimeType?: string
+  totalSize: number
+  totalChunks: number
+  chunks: ChunkInfo[]
+  uploadOptions?: FileUploadOptions
+}
+
+export interface ChunkInfo {
+  size: number
+  cid: string
+}
+
+export const fileMetadata = (
+  headCID: CID,
+  chunks: ChunkInfo[],
+  totalSize: number,
+  name?: string | null,
+  mimeType?: string | null,
+  uploadOptions: FileUploadOptions = {
+    compression: undefined,
+    encryption: undefined,
+  },
+): OffchainFileMetadata => {
+  return {
+    type: 'file',
+    dataCid: cidToString(headCID),
+    name: name ?? undefined,
+    mimeType: mimeType ?? undefined,
+    totalSize,
+    totalChunks: chunks.length,
+    chunks,
+    uploadOptions,
+  }
+}

package/src/metadata/offchain/folder.ts

@@ -0,0 +1,54 @@
+import { CID } from 'multiformats'
+import { cidOfNode, cidToString } from '../../cid/index.js'
+import { PBNode } from '../../ipld/index.js'
+import { FileUploadOptions, IPLDNodeData, MetadataType } from '../onchain/index.js'
+
+interface ChildrenMetadata {
+  type: 'folder' | 'file'
+  name?: string
+  cid: string
+  totalSize: number
+}
+
+export type OffchainFolderMetadata = {
+  type: 'folder'
+  dataCid: string
+  name?: string
+  totalSize: number
+  totalFiles: number
+  children: ChildrenMetadata[]
+  uploadOptions: FileUploadOptions
+}
+
+export const childrenMetadataFromNode = (node: PBNode): ChildrenMetadata => {
+  const ipldData = IPLDNodeData.decode(node.Data!)
+  if (ipldData.type !== MetadataType.File && ipldData.type !== MetadataType.Folder) {
+    throw new Error('Invalid metadata type')
+  }
+
+  return {
+    type: ipldData.type === MetadataType.File ? 'file' : 'folder',
+    cid: cidToString(cidOfNode(node)),
+    totalSize: ipldData.size ?? 0,
+    name: ipldData.name,
+  }
+}
+
+export const folderMetadata = (
+  cid: CID | string,
+  children: ChildrenMetadata[],
+  name?: string | null,
+  uploadOptions: FileUploadOptions = {},
+): OffchainFolderMetadata => {
+  cid = typeof cid === 'string' ? cid : cidToString(cid)
+
+  return {
+    dataCid: cid,
+    totalSize: children.reduce((acc, child) => acc + child.totalSize, 0),
+    totalFiles: children.length,
+    children,
+    type: 'folder',
+    name: name ?? undefined,
+    uploadOptions,
+  }
+}

package/src/metadata/offchain/index.ts

@@ -0,0 +1,3 @@
+export * from './base.js'
+export * from './file.js'
+export * from './folder.js'

package/src/metadata/onchain/index.ts

@@ -0,0 +1,2 @@
+export * from './protobuf/OnchainMetadata.js'
+export * from './utils.js'

package/src/metadata/onchain/protobuf/OnchainMetadata.proto

@@ -0,0 +1,46 @@
+syntax = "proto3";
+
+message IPLDNodeData {
+  MetadataType type = 1;
+  int32 linkDepth = 2;
+  optional int32 size = 3;
+  optional string name = 4;
+  optional bytes data = 5;
+  optional FileUploadOptions uploadOptions = 6;
+}
+
+// MetadataType defines the possible types of metadata.
+enum MetadataType {
+  File = 0;
+  FileInlink = 1;
+  FileChunk = 2;
+  Folder = 3;
+  FolderInlink = 4;
+  Metadata = 5;
+  MetadataInlink = 6;
+  MetadataChunk = 7;
+}
+
+message FileUploadOptions {
+  optional CompressionOptions compression = 1;
+  optional EncryptionOptions encryption = 2;
+}
+
+message CompressionOptions {
+  CompressionAlgorithm algorithm = 1;
+  optional int32 level = 2;
+  optional int32 chunkSize = 3;
+}
+
+message EncryptionOptions {
+  EncryptionAlgorithm algorithm = 1;
+  optional int32 chunkSize = 2;
+}
+
+enum CompressionAlgorithm {
+  ZLIB = 0;
+}
+
+enum EncryptionAlgorithm {
+  AES_256_GCM = 0;
+}