@autonomys/auto-drive 0.7.3 → 0.8.2

package/src/ipld/chunker.ts

@@ -1,194 +0,0 @@
-import type { BaseBlockstore } from 'blockstore-core'
-import type { AwaitIterable } from 'interface-store'
-import { CID } from 'multiformats'
-import { cidOfNode } from '../cid/index.js'
-import { decodeIPLDNodeData, OffchainMetadata } from '../metadata/index.js'
-import { Builders, fileBuilders, metadataBuilders } from './builders.js'
-import { createFolderInlinkIpldNode, createFolderIpldNode } from './nodes.js'
-import { chunkBuffer, encodeNode, PBNode } from './utils.js'
-
-export const DEFAULT_MAX_CHUNK_SIZE = 64 * 1024
-
-const ESTIMATED_LINK_SIZE_IN_BYTES = 64
-export const DEFAULT_MAX_LINK_PER_NODE = DEFAULT_MAX_CHUNK_SIZE / ESTIMATED_LINK_SIZE_IN_BYTES
-
-export const processFileToIPLDFormat = (
-  blockstore: BaseBlockstore,
-  file: AwaitIterable<Buffer>,
-  totalSize: number,
-  filename?: string,
-  { maxChunkSize, maxLinkPerNode }: { maxChunkSize: number; maxLinkPerNode: number } = {
-    maxChunkSize: DEFAULT_MAX_CHUNK_SIZE,
-    maxLinkPerNode: DEFAULT_MAX_LINK_PER_NODE,
-  },
-): Promise<CID> => {
-  return processBufferToIPLDFormat(blockstore, file, filename, totalSize, fileBuilders, {
-    maxChunkSize,
-    maxLinkPerNode,
-  })
-}
-
-export const processMetadataToIPLDFormat = async (
-  blockstore: BaseBlockstore,
-  metadata: OffchainMetadata,
-  limits: { maxChunkSize: number; maxLinkPerNode: number } = {
-    maxChunkSize: DEFAULT_MAX_CHUNK_SIZE,
-    maxLinkPerNode: DEFAULT_MAX_LINK_PER_NODE,
-  },
-): Promise<CID> => {
-  const buffer = Buffer.from(JSON.stringify(metadata))
-  const name = `${metadata.name}.metadata.json`
-  return processBufferToIPLDFormat(
-    blockstore,
-    (async function* () {
-      yield buffer
-    })(),
-    name,
-    buffer.byteLength,
-    metadataBuilders,
-    limits,
-  )
-}
-
-const processBufferToIPLDFormat = async (
-  blockstore: BaseBlockstore,
-  buffer: AwaitIterable<Buffer>,
-  filename: string | undefined,
-  totalSize: number,
-  builders: Builders,
-  { maxChunkSize, maxLinkPerNode }: { maxChunkSize: number; maxLinkPerNode: number } = {
-    maxChunkSize: DEFAULT_MAX_CHUNK_SIZE,
-    maxLinkPerNode: DEFAULT_MAX_LINK_PER_NODE,
-  },
-): Promise<CID> => {
-  const bufferChunks = chunkBuffer(buffer, { maxChunkSize })
-
-  let CIDs: CID[] = []
-  for await (const chunk of bufferChunks) {
-    const node = builders.chunk(chunk)
-    const cid = cidOfNode(node)
-    await blockstore.put(cid, encodeNode(node))
-    CIDs.push(cid)
-  }
-
-  return processBufferToIPLDFormatFromChunks(blockstore, CIDs, filename, totalSize, builders, {
-    maxLinkPerNode,
-    maxChunkSize,
-  })
-}
-
-export const processBufferToIPLDFormatFromChunks = async (
-  blockstore: BaseBlockstore,
-  chunks: AwaitIterable<CID>,
-  filename: string | undefined,
-  totalSize: number,
-  builders: Builders,
-  { maxLinkPerNode, maxChunkSize }: { maxLinkPerNode: number; maxChunkSize: number } = {
-    maxLinkPerNode: DEFAULT_MAX_LINK_PER_NODE,
-    maxChunkSize: DEFAULT_MAX_CHUNK_SIZE,
-  },
-): Promise<CID> => {
-  let chunkCount = 0
-  let CIDs: CID[] = []
-  for await (const chunk of chunks) {
-    CIDs.push(chunk)
-    chunkCount++
-  }
-
-  if (CIDs.length === 1) {
-    const nodeBytes = await blockstore.get(CIDs[0])
-    await blockstore.delete(CIDs[0])
-    const data = decodeIPLDNodeData(nodeBytes)
-    const singleNode = builders.single(Buffer.from(data.data!), filename)
-    await blockstore.put(cidOfNode(singleNode), encodeNode(singleNode))
-    const headCID = cidOfNode(singleNode)
-
-    return headCID
-  }
-
-  let depth = 1
-  while (CIDs.length > maxLinkPerNode) {
-    const newCIDs: CID[] = []
-    for (let i = 0; i < CIDs.length; i += maxLinkPerNode) {
-      const chunk = CIDs.slice(i, i + maxLinkPerNode)
-
-      const node = builders.inlink(chunk, chunk.length, depth, maxChunkSize)
-      const cid = cidOfNode(node)
-      await blockstore.put(cid, encodeNode(node))
-      newCIDs.push(cid)
-    }
-    depth++
-    CIDs = newCIDs
-  }
-  const head = builders.root(CIDs, totalSize, depth, filename, maxChunkSize)
-  const headCID = cidOfNode(head)
-  await blockstore.put(headCID, encodeNode(head))
-
-  return headCID
-}
-
-export const processFolderToIPLDFormat = async (
-  blockstore: BaseBlockstore,
-  children: CID[],
-  name: string,
-  size: number,
-  { maxLinkPerNode }: { maxLinkPerNode: number } = { maxLinkPerNode: DEFAULT_MAX_LINK_PER_NODE },
-): Promise<CID> => {
-  let cids = children
-  let depth = 0
-  while (cids.length > maxLinkPerNode) {
-    const newCIDs: CID[] = []
-    for (let i = 0; i < cids.length; i += maxLinkPerNode) {
-      const chunk = cids.slice(i, i + maxLinkPerNode)
-      const node = createFolderInlinkIpldNode(chunk, depth)
-      const cid = cidOfNode(node)
-      await blockstore.put(cid, encodeNode(node))
-      newCIDs.push(cid)
-    }
-    cids = newCIDs
-    depth++
-  }
-
-  const node = createFolderIpldNode(cids, name, depth, size)
-  const cid = cidOfNode(node)
-  await blockstore.put(cid, encodeNode(node))
-
-  return cid
-}
-
-/**
- * Process chunks to IPLD format, return the last chunk if it's not full
- * @returns the last chunk if it's not full, otherwise an empty buffer
- */
-export const processChunksToIPLDFormat = async (
-  blockstore: BaseBlockstore,
-  chunks: AwaitIterable<Buffer>,
-  builders: Builders,
-  { maxChunkSize = DEFAULT_MAX_CHUNK_SIZE }: { maxChunkSize?: number },
-): Promise<Buffer> => {
-  const bufferChunks = chunkBuffer(chunks, { maxChunkSize, ignoreLastChunk: false })
-
-  for await (const chunk of bufferChunks) {
-    if (chunk.byteLength < maxChunkSize) {
-      return chunk
-    }
-
-    const node = builders.chunk(chunk)
-    const cid = cidOfNode(node)
-    await blockstore.put(cid, encodeNode(node))
-  }
-
-  return Buffer.alloc(0)
-}
-
-export const ensureNodeMaxSize = (
-  node: PBNode,
-  maxSize: number = DEFAULT_MAX_CHUNK_SIZE,
-): PBNode => {
-  const nodeSize = encodeNode(node).byteLength
-  if (nodeSize > maxSize) {
-    throw new Error(`Node is too large to fit in a single chunk: ${nodeSize} > ${maxSize}`)
-  }
-
-  return node
-}

package/src/ipld/index.ts

@@ -1,4 +0,0 @@
-export * from './builders.js'
-export * from './chunker.js'
-export * from './nodes.js'
-export * from './utils.js'

package/src/ipld/nodes.ts

@@ -1,199 +0,0 @@
-import { CID } from 'multiformats/cid'
-import { createNode, PBNode } from '../ipld/index.js'
-import { OffchainMetadata } from '../metadata/index.js'
-import { encodeIPLDNodeData, MetadataType } from '../metadata/onchain/index.js'
-import { DEFAULT_MAX_CHUNK_SIZE, ensureNodeMaxSize } from './chunker.js'
-
-/// Creates a file chunk ipld node
-export const createFileChunkIpldNode = (data: Buffer): PBNode =>
-  createNode(
-    encodeIPLDNodeData({
-      type: MetadataType.FileChunk,
-      size: data.length,
-      linkDepth: 0,
-      data,
-    }),
-    [],
-  )
-
-// Creates a file ipld node
-// links: the CIDs of the file's contents
-// @todo: add the file's metadata
-export const createChunkedFileIpldNode = (
-  links: CID[],
-  size: number,
-  linkDepth: number,
-  name?: string,
-  maxNodeSize: number = DEFAULT_MAX_CHUNK_SIZE,
-): PBNode =>
-  ensureNodeMaxSize(
-    createNode(
-      encodeIPLDNodeData({
-        type: MetadataType.File,
-        name,
-        size,
-        linkDepth,
-      }),
-      links.map((cid) => ({ Hash: cid })),
-    ),
-    maxNodeSize,
-  )
-// Creates a file ipld node
-// links: the CIDs of the file's contents
-export const createFileInlinkIpldNode = (
-  links: CID[],
-  size: number,
-  linkDepth: number,
-  maxNodeSize: number = DEFAULT_MAX_CHUNK_SIZE,
-): PBNode =>
-  ensureNodeMaxSize(
-    createNode(
-      encodeIPLDNodeData({
-        type: MetadataType.FileInlink,
-        size,
-        linkDepth,
-      }),
-      links.map((cid) => ({ Hash: cid })),
-    ),
-    maxNodeSize,
-  )
-
-// Creates a file ipld node
-// links: the CIDs of the file's contents
-// @todo: add the file's metadata
-export const createSingleFileIpldNode = (data: Buffer, name?: string): PBNode =>
-  createNode(
-    encodeIPLDNodeData({
-      type: MetadataType.File,
-      name,
-      size: data.length,
-      linkDepth: 0,
-      data,
-    }),
-    [],
-  )
-
-// Creates a file ipld node
-// links: the CIDs of the file's contents
-// @todo: add the file's metadata
-export const createMetadataInlinkIpldNode = (
-  links: CID[],
-  size: number,
-  linkDepth: number,
-  maxNodeSize: number = DEFAULT_MAX_CHUNK_SIZE,
-): PBNode =>
-  ensureNodeMaxSize(
-    createNode(
-      encodeIPLDNodeData({
-        type: MetadataType.FileInlink,
-        size,
-        linkDepth,
-      }),
-      links.map((cid) => ({ Hash: cid })),
-    ),
-    maxNodeSize,
-  )
-
-// Creates a file ipld node
-// links: the CIDs of the file's contents
-// @todo: add the file's metadata
-export const createSingleMetadataIpldNode = (data: Buffer, name?: string): PBNode =>
-  createNode(
-    encodeIPLDNodeData({
-      type: MetadataType.Metadata,
-      name,
-      size: data.length,
-      linkDepth: 0,
-      data,
-    }),
-    [],
-  )
-
-export const createMetadataChunkIpldNode = (data: Buffer): PBNode =>
-  createNode(
-    encodeIPLDNodeData({
-      type: MetadataType.MetadataChunk,
-      size: data.length,
-      linkDepth: 0,
-      data,
-    }),
-  )
-
-export const createChunkedMetadataIpldNode = (
-  links: CID[],
-  size: number,
-  linkDepth: number,
-  name?: string,
-  maxNodeSize: number = DEFAULT_MAX_CHUNK_SIZE,
-): PBNode =>
-  ensureNodeMaxSize(
-    createNode(
-      encodeIPLDNodeData({
-        type: MetadataType.Metadata,
-        name,
-        size,
-        linkDepth,
-      }),
-      links.map((cid) => ({ Hash: cid })),
-    ),
-    maxNodeSize,
-  )
-
-// Creates a folder ipld node
-// links: the CIDs of the folder's contents
-// @todo: add the folder's metadata
-export const createFolderIpldNode = (
-  links: CID[],
-  name: string,
-  linkDepth: number,
-  size: number,
-  maxNodeSize: number = DEFAULT_MAX_CHUNK_SIZE,
-): PBNode =>
-  ensureNodeMaxSize(
-    createNode(
-      encodeIPLDNodeData({
-        type: MetadataType.Folder,
-        name,
-        size,
-        linkDepth,
-      }),
-      links.map((cid) => ({ Hash: cid })),
-    ),
-    maxNodeSize,
-  )
-
-export const createFolderInlinkIpldNode = (
-  links: CID[],
-  linkDepth: number,
-  maxNodeSize: number = DEFAULT_MAX_CHUNK_SIZE,
-): PBNode =>
-  ensureNodeMaxSize(
-    createNode(
-      encodeIPLDNodeData({
-        type: MetadataType.FolderInlink,
-        linkDepth,
-      }),
-      links.map((cid) => ({ Hash: cid })),
-    ),
-    maxNodeSize,
-  )
-
-/// Creates a metadata ipld node
-export const createMetadataNode = (
-  metadata: OffchainMetadata,
-  maxNodeSize: number = DEFAULT_MAX_CHUNK_SIZE,
-): PBNode => {
-  const data = Buffer.from(JSON.stringify(metadata))
-
-  return ensureNodeMaxSize(
-    createNode(
-      encodeIPLDNodeData({
-        type: MetadataType.Metadata,
-        name: metadata.name,
-        linkDepth: 0,
-        data,
-      }),
-    ),
-    maxNodeSize,
-  )
-}

package/src/ipld/utils.ts

@@ -1,21 +0,0 @@
-import { createNode, decode, encode, PBNode } from '@ipld/dag-pb'
-import { AwaitIterable } from 'interface-store'
-
-export const chunkBuffer = async function* (
-  buffer: AwaitIterable<Buffer>,
-  { maxChunkSize, ignoreLastChunk = false }: { maxChunkSize: number; ignoreLastChunk?: boolean },
-): AsyncIterable<Buffer> {
-  let target = Buffer.alloc(0)
-  for await (let chunk of buffer) {
-    target = Buffer.concat([target, chunk])
-    while (target.length >= maxChunkSize) {
-      yield target.subarray(0, maxChunkSize)
-      target = target.subarray(maxChunkSize)
-    }
-  }
-  if (target.length > 0 && !ignoreLastChunk) {
-    yield target
-  }
-}
-
-export { createNode, decode as decodeNode, encode as encodeNode, PBNode }

package/src/metadata/index.ts

@@ -1,2 +0,0 @@
-export * from './offchain/index.js'
-export * from './onchain/index.js'

package/src/metadata/offchain/base.ts

@@ -1,4 +0,0 @@
-import { OffchainFileMetadata } from './file.js'
-import { OffchainFolderMetadata } from './folder.js'
-
-export type OffchainMetadata = OffchainFolderMetadata | OffchainFileMetadata

package/src/metadata/offchain/file.ts

@@ -1,35 +0,0 @@
-import { CID } from 'multiformats'
-import { cidToString } from '../../index.js'
-
-export type OffchainFileMetadata = {
-  type: 'file'
-  dataCid: string
-  name?: string
-  mimeType?: string
-  totalSize: number
-  totalChunks: number
-  chunks: ChunkInfo[]
-}
-
-export interface ChunkInfo {
-  size: number
-  cid: string
-}
-
-export const fileMetadata = (
-  headCID: CID,
-  chunks: ChunkInfo[],
-  totalSize: number,
-  name?: string | null,
-  mimeType?: string | null,
-): OffchainFileMetadata => {
-  return {
-    type: 'file',
-    dataCid: cidToString(headCID),
-    name: name ?? undefined,
-    mimeType: mimeType ?? undefined,
-    totalSize,
-    totalChunks: chunks.length,
-    chunks,
-  }
-}

package/src/metadata/offchain/folder.ts

@@ -1,51 +0,0 @@
-import { CID } from 'multiformats'
-import { cidOfNode, cidToString } from '../../cid/index.js'
-import { PBNode } from '../../ipld/index.js'
-import { IPLDNodeData, MetadataType } from '../onchain/index.js'
-
-interface ChildrenMetadata {
-  type: 'folder' | 'file'
-  name?: string
-  cid: string
-  totalSize: number
-}
-
-export type OffchainFolderMetadata = {
-  type: 'folder'
-  dataCid: string
-  name?: string
-  totalSize: number
-  totalFiles: number
-  children: ChildrenMetadata[]
-}
-
-export const childrenMetadataFromNode = (node: PBNode): ChildrenMetadata => {
-  const ipldData = IPLDNodeData.decode(node.Data!)
-  if (ipldData.type !== MetadataType.File && ipldData.type !== MetadataType.Folder) {
-    throw new Error('Invalid metadata type')
-  }
-
-  return {
-    type: ipldData.type === MetadataType.File ? 'file' : 'folder',
-    cid: cidToString(cidOfNode(node)),
-    totalSize: ipldData.size ?? 0,
-    name: ipldData.name,
-  }
-}
-
-export const folderMetadata = (
-  cid: CID | string,
-  children: ChildrenMetadata[],
-  name?: string | null,
-): OffchainFolderMetadata => {
-  cid = typeof cid === 'string' ? cid : cidToString(cid)
-
-  return {
-    dataCid: cid,
-    totalSize: children.reduce((acc, child) => acc + child.totalSize, 0),
-    totalFiles: children.length,
-    children,
-    type: 'folder',
-    name: name ?? undefined,
-  }
-}

package/src/metadata/offchain/index.ts

@@ -1,3 +0,0 @@
-export * from './base.js'
-export * from './file.js'
-export * from './folder.js'

package/src/metadata/onchain/index.ts

@@ -1,2 +0,0 @@
-export * from './protobuf/OnchainMetadata.js'
-export * from './utils.js'

package/src/metadata/onchain/protobuf/OnchainMetadata.proto

@@ -1,21 +0,0 @@
-syntax = "proto3";
-
-message IPLDNodeData {
-  MetadataType type = 1;
-  int32 linkDepth = 2;
-  optional int32 size = 3;
-  optional string name = 4;
-  optional bytes data = 5;
-}
-
-// MetadataType defines the possible types of metadata.
-enum MetadataType {
-  File = 0;
-  FileInlink = 1;
-  FileChunk = 2;
-  Folder = 3;
-  FolderInlink = 4;
-  Metadata = 5;
-  MetadataInlink = 6;
-  MetadataChunk = 7;
-}

package/src/metadata/onchain/protobuf/OnchainMetadata.ts

@@ -1,137 +0,0 @@
-/* eslint-disable import/export */
-/* eslint-disable complexity */
-/* eslint-disable @typescript-eslint/no-namespace */
-/* eslint-disable @typescript-eslint/no-unnecessary-boolean-literal-compare */
-/* eslint-disable @typescript-eslint/no-empty-interface */
-
-import { type Codec, decodeMessage, type DecodeOptions, encodeMessage, enumeration, message } from 'protons-runtime'
-import type { Uint8ArrayList } from 'uint8arraylist'
-
-export interface IPLDNodeData {
-  type: MetadataType
-  linkDepth: number
-  size?: number
-  name?: string
-  data?: Uint8Array
-}
-
-export namespace IPLDNodeData {
-  let _codec: Codec<IPLDNodeData>
-
-  export const codec = (): Codec<IPLDNodeData> => {
-    if (_codec == null) {
-      _codec = message<IPLDNodeData>((obj, w, opts = {}) => {
-        if (opts.lengthDelimited !== false) {
-          w.fork()
-        }
-
-        if (obj.type != null && __MetadataTypeValues[obj.type] !== 0) {
-          w.uint32(8)
-          MetadataType.codec().encode(obj.type, w)
-        }
-
-        if ((obj.linkDepth != null && obj.linkDepth !== 0)) {
-          w.uint32(16)
-          w.int32(obj.linkDepth)
-        }
-
-        if (obj.size != null) {
-          w.uint32(24)
-          w.int32(obj.size)
-        }
-
-        if (obj.name != null) {
-          w.uint32(34)
-          w.string(obj.name)
-        }
-
-        if (obj.data != null) {
-          w.uint32(42)
-          w.bytes(obj.data)
-        }
-
-        if (opts.lengthDelimited !== false) {
-          w.ldelim()
-        }
-      }, (reader, length, opts = {}) => {
-        const obj: any = {
-          type: MetadataType.File,
-          linkDepth: 0
-        }
-
-        const end = length == null ? reader.len : reader.pos + length
-
-        while (reader.pos < end) {
-          const tag = reader.uint32()
-
-          switch (tag >>> 3) {
-            case 1: {
-              obj.type = MetadataType.codec().decode(reader)
-              break
-            }
-            case 2: {
-              obj.linkDepth = reader.int32()
-              break
-            }
-            case 3: {
-              obj.size = reader.int32()
-              break
-            }
-            case 4: {
-              obj.name = reader.string()
-              break
-            }
-            case 5: {
-              obj.data = reader.bytes()
-              break
-            }
-            default: {
-              reader.skipType(tag & 7)
-              break
-            }
-          }
-        }
-
-        return obj
-      })
-    }
-
-    return _codec
-  }
-
-  export const encode = (obj: Partial<IPLDNodeData>): Uint8Array => {
-    return encodeMessage(obj, IPLDNodeData.codec())
-  }
-
-  export const decode = (buf: Uint8Array | Uint8ArrayList, opts?: DecodeOptions<IPLDNodeData>): IPLDNodeData => {
-    return decodeMessage(buf, IPLDNodeData.codec(), opts)
-  }
-}
-
-export enum MetadataType {
-  File = 'File',
-  FileInlink = 'FileInlink',
-  FileChunk = 'FileChunk',
-  Folder = 'Folder',
-  FolderInlink = 'FolderInlink',
-  Metadata = 'Metadata',
-  MetadataInlink = 'MetadataInlink',
-  MetadataChunk = 'MetadataChunk'
-}
-
-enum __MetadataTypeValues {
-  File = 0,
-  FileInlink = 1,
-  FileChunk = 2,
-  Folder = 3,
-  FolderInlink = 4,
-  Metadata = 5,
-  MetadataInlink = 6,
-  MetadataChunk = 7
-}
-
-export namespace MetadataType {
-  export const codec = (): Codec<MetadataType> => {
-    return enumeration<MetadataType>(__MetadataTypeValues)
-  }
-}

package/src/metadata/onchain/utils.ts

@@ -1,15 +0,0 @@
-import { decodeNode } from '../../ipld/index.js'
-import { IPLDNodeData } from '../onchain/index.js'
-
-export const encodeIPLDNodeData = (metadata: IPLDNodeData): Uint8Array => {
-  return IPLDNodeData.encode(metadata)
-}
-
-export const decodeIPLDNodeData = (data: Uint8Array): IPLDNodeData => {
-  const decoded = decodeNode(data)
-  if (!decoded.Data) {
-    throw new Error('Invalid data')
-  }
-
-  return IPLDNodeData.decode(decoded.Data)
-}