@autonomys/auto-drive 0.7.1 → 0.7.3

package/src/ipld/builders.ts

@@ -1,5 +1,5 @@
-import { PBNode } from '@ipld/dag-pb'
 import { CID } from 'multiformats/cid'
+import { PBNode } from '../ipld/index.js'
 import {
   createChunkedFileIpldNode,
   createChunkedMetadataIpldNode,

package/src/ipld/chunker.ts

@@ -1,71 +1,110 @@
-import { PBNode } from '@ipld/dag-pb'
+import type { BaseBlockstore } from 'blockstore-core'
+import type { AwaitIterable } from 'interface-store'
 import { CID } from 'multiformats'
 import { cidOfNode } from '../cid/index.js'
-import { OffchainMetadata } from '../metadata/index.js'
+import { decodeIPLDNodeData, OffchainMetadata } from '../metadata/index.js'
 import { Builders, fileBuilders, metadataBuilders } from './builders.js'
 import { createFolderInlinkIpldNode, createFolderIpldNode } from './nodes.js'
-import { chunkBuffer, encodeNode } from './utils.js'
+import { chunkBuffer, encodeNode, PBNode } from './utils.js'
 
-export const DEFAULT_MAX_CHUNK_SIZE = 1024 * 64
-export const DEFAULT_MAX_LINK_PER_NODE = DEFAULT_MAX_CHUNK_SIZE / 64
+export const DEFAULT_MAX_CHUNK_SIZE = 64 * 1024
 
-export interface IPLDDag {
-  headCID: CID
-  nodes: Map<CID, PBNode>
-}
+const ESTIMATED_LINK_SIZE_IN_BYTES = 64
+export const DEFAULT_MAX_LINK_PER_NODE = DEFAULT_MAX_CHUNK_SIZE / ESTIMATED_LINK_SIZE_IN_BYTES
 
-export const createFileIPLDDag = (
-  file: Buffer,
+export const processFileToIPLDFormat = (
+  blockstore: BaseBlockstore,
+  file: AwaitIterable<Buffer>,
+  totalSize: number,
   filename?: string,
-  { chunkSize, maxLinkPerNode }: { chunkSize: number; maxLinkPerNode: number } = {
-    chunkSize: DEFAULT_MAX_CHUNK_SIZE,
+  { maxChunkSize, maxLinkPerNode }: { maxChunkSize: number; maxLinkPerNode: number } = {
+    maxChunkSize: DEFAULT_MAX_CHUNK_SIZE,
     maxLinkPerNode: DEFAULT_MAX_LINK_PER_NODE,
   },
-): IPLDDag => {
-  return createBufferIPLDDag(file, filename, fileBuilders, { chunkSize, maxLinkPerNode })
+): Promise<CID> => {
+  return processBufferToIPLDFormat(blockstore, file, filename, totalSize, fileBuilders, {
+    maxChunkSize,
+    maxLinkPerNode,
+  })
 }
 
-export const createMetadataIPLDDag = (
+export const processMetadataToIPLDFormat = async (
+  blockstore: BaseBlockstore,
   metadata: OffchainMetadata,
-  limits: { chunkSize: number; maxLinkPerNode: number } = {
-    chunkSize: DEFAULT_MAX_CHUNK_SIZE,
+  limits: { maxChunkSize: number; maxLinkPerNode: number } = {
+    maxChunkSize: DEFAULT_MAX_CHUNK_SIZE,
     maxLinkPerNode: DEFAULT_MAX_LINK_PER_NODE,
   },
-): IPLDDag => {
+): Promise<CID> => {
   const buffer = Buffer.from(JSON.stringify(metadata))
   const name = `${metadata.name}.metadata.json`
-  return createBufferIPLDDag(buffer, name, metadataBuilders, limits)
+  return processBufferToIPLDFormat(
+    blockstore,
+    (async function* () {
+      yield buffer
+    })(),
+    name,
+    buffer.byteLength,
+    metadataBuilders,
+    limits,
+  )
 }
 
-const createBufferIPLDDag = (
-  buffer: Buffer,
+const processBufferToIPLDFormat = async (
+  blockstore: BaseBlockstore,
+  buffer: AwaitIterable<Buffer>,
   filename: string | undefined,
+  totalSize: number,
   builders: Builders,
-  { chunkSize, maxLinkPerNode }: { chunkSize: number; maxLinkPerNode: number } = {
-    chunkSize: DEFAULT_MAX_CHUNK_SIZE,
+  { maxChunkSize, maxLinkPerNode }: { maxChunkSize: number; maxLinkPerNode: number } = {
+    maxChunkSize: DEFAULT_MAX_CHUNK_SIZE,
     maxLinkPerNode: DEFAULT_MAX_LINK_PER_NODE,
   },
-): IPLDDag => {
-  if (buffer.length <= chunkSize) {
-    const head = builders.single(buffer, filename)
-    const headCID = cidOfNode(head)
-    return {
-      headCID,
-      nodes: new Map([[headCID, head]]),
-    }
-  }
+): Promise<CID> => {
+  const bufferChunks = chunkBuffer(buffer, { maxChunkSize })
 
-  const bufferChunks = chunkBuffer(buffer, chunkSize)
-
-  const nodes = new Map<CID, PBNode>()
-
-  let CIDs: CID[] = bufferChunks.map((chunk) => {
+  let CIDs: CID[] = []
+  for await (const chunk of bufferChunks) {
     const node = builders.chunk(chunk)
     const cid = cidOfNode(node)
-    nodes.set(cid, node)
+    await blockstore.put(cid, encodeNode(node))
+    CIDs.push(cid)
+  }
 
-    return cid
+  return processBufferToIPLDFormatFromChunks(blockstore, CIDs, filename, totalSize, builders, {
+    maxLinkPerNode,
+    maxChunkSize,
   })
+}
+
+export const processBufferToIPLDFormatFromChunks = async (
+  blockstore: BaseBlockstore,
+  chunks: AwaitIterable<CID>,
+  filename: string | undefined,
+  totalSize: number,
+  builders: Builders,
+  { maxLinkPerNode, maxChunkSize }: { maxLinkPerNode: number; maxChunkSize: number } = {
+    maxLinkPerNode: DEFAULT_MAX_LINK_PER_NODE,
+    maxChunkSize: DEFAULT_MAX_CHUNK_SIZE,
+  },
+): Promise<CID> => {
+  let chunkCount = 0
+  let CIDs: CID[] = []
+  for await (const chunk of chunks) {
+    CIDs.push(chunk)
+    chunkCount++
+  }
+
+  if (CIDs.length === 1) {
+    const nodeBytes = await blockstore.get(CIDs[0])
+    await blockstore.delete(CIDs[0])
+    const data = decodeIPLDNodeData(nodeBytes)
+    const singleNode = builders.single(Buffer.from(data.data!), filename)
+    await blockstore.put(cidOfNode(singleNode), encodeNode(singleNode))
+    const headCID = cidOfNode(singleNode)
+
+    return headCID
+  }
 
   let depth = 1
   while (CIDs.length > maxLinkPerNode) {
@@ -73,31 +112,28 @@ const createBufferIPLDDag = (
     for (let i = 0; i < CIDs.length; i += maxLinkPerNode) {
       const chunk = CIDs.slice(i, i + maxLinkPerNode)
 
-      const node = builders.inlink(chunk, chunk.length, depth, chunkSize)
+      const node = builders.inlink(chunk, chunk.length, depth, maxChunkSize)
       const cid = cidOfNode(node)
-      nodes.set(cid, node)
+      await blockstore.put(cid, encodeNode(node))
       newCIDs.push(cid)
     }
     depth++
     CIDs = newCIDs
   }
-  const head = builders.root(CIDs, buffer.length, depth, filename, chunkSize)
+  const head = builders.root(CIDs, totalSize, depth, filename, maxChunkSize)
   const headCID = cidOfNode(head)
-  nodes.set(headCID, head)
+  await blockstore.put(headCID, encodeNode(head))
 
-  return {
-    headCID,
-    nodes,
-  }
+  return headCID
 }
 
-export const createFolderIPLDDag = (
+export const processFolderToIPLDFormat = async (
+  blockstore: BaseBlockstore,
   children: CID[],
   name: string,
   size: number,
   { maxLinkPerNode }: { maxLinkPerNode: number } = { maxLinkPerNode: DEFAULT_MAX_LINK_PER_NODE },
-): IPLDDag => {
-  const nodes = new Map<CID, PBNode>()
+): Promise<CID> => {
   let cids = children
   let depth = 0
   while (cids.length > maxLinkPerNode) {
@@ -106,7 +142,7 @@ export const createFolderIPLDDag = (
       const chunk = cids.slice(i, i + maxLinkPerNode)
       const node = createFolderInlinkIpldNode(chunk, depth)
       const cid = cidOfNode(node)
-      nodes.set(cid, node)
+      await blockstore.put(cid, encodeNode(node))
       newCIDs.push(cid)
     }
     cids = newCIDs
@@ -115,12 +151,34 @@ export const createFolderIPLDDag = (
 
   const node = createFolderIpldNode(cids, name, depth, size)
   const cid = cidOfNode(node)
-  nodes.set(cid, node)
+  await blockstore.put(cid, encodeNode(node))
 
-  return {
-    headCID: cid,
-    nodes,
+  return cid
+}
+
+/**
+ * Process chunks to IPLD format, return the last chunk if it's not full
+ * @returns the last chunk if it's not full, otherwise an empty buffer
+ */
+export const processChunksToIPLDFormat = async (
+  blockstore: BaseBlockstore,
+  chunks: AwaitIterable<Buffer>,
+  builders: Builders,
+  { maxChunkSize = DEFAULT_MAX_CHUNK_SIZE }: { maxChunkSize?: number },
+): Promise<Buffer> => {
+  const bufferChunks = chunkBuffer(chunks, { maxChunkSize, ignoreLastChunk: false })
+
+  for await (const chunk of bufferChunks) {
+    if (chunk.byteLength < maxChunkSize) {
+      return chunk
+    }
+
+    const node = builders.chunk(chunk)
+    const cid = cidOfNode(node)
+    await blockstore.put(cid, encodeNode(node))
   }
+
+  return Buffer.alloc(0)
 }
 
 export const ensureNodeMaxSize = (

package/src/ipld/index.ts

@@ -1,3 +1,4 @@
+export * from './builders.js'
 export * from './chunker.js'
 export * from './nodes.js'
-export { encodeNode } from './utils.js'
+export * from './utils.js'

package/src/ipld/nodes.ts

@@ -1,5 +1,5 @@
-import { createNode, PBNode } from '@ipld/dag-pb'
 import { CID } from 'multiformats/cid'
+import { createNode, PBNode } from '../ipld/index.js'
 import { OffchainMetadata } from '../metadata/index.js'
 import { encodeIPLDNodeData, MetadataType } from '../metadata/onchain/index.js'
 import { DEFAULT_MAX_CHUNK_SIZE, ensureNodeMaxSize } from './chunker.js'

package/src/ipld/utils.ts

@@ -1,15 +1,21 @@
-import { decode, encode, PBNode } from '@ipld/dag-pb'
+import { createNode, decode, encode, PBNode } from '@ipld/dag-pb'
+import { AwaitIterable } from 'interface-store'
 
-export const chunkBuffer = (buffer: Buffer, chunkSize: number) => {
-  const chunks: Buffer[] = []
-
-  for (let i = 0; i < buffer.length; i += chunkSize) {
-    chunks.push(Buffer.from(buffer.buffer.slice(i, i + chunkSize)))
+export const chunkBuffer = async function* (
+  buffer: AwaitIterable<Buffer>,
+  { maxChunkSize, ignoreLastChunk = false }: { maxChunkSize: number; ignoreLastChunk?: boolean },
+): AsyncIterable<Buffer> {
+  let target = Buffer.alloc(0)
+  for await (let chunk of buffer) {
+    target = Buffer.concat([target, chunk])
+    while (target.length >= maxChunkSize) {
+      yield target.subarray(0, maxChunkSize)
+      target = target.subarray(maxChunkSize)
+    }
+  }
+  if (target.length > 0 && !ignoreLastChunk) {
+    yield target
   }
-
-  return chunks
 }
 
-export const encodeNode = (node: PBNode): Buffer => Buffer.from(encode(node))
-
-export const decodeNode = (data: Uint8Array): PBNode => decode(data)
+export { createNode, decode as decodeNode, encode as encodeNode, PBNode }

package/src/metadata/offchain/file.ts

@@ -1,4 +1,5 @@
-import { cidOfNode, cidToString, IPLDDag, IPLDNodeData, MetadataType } from '../../index.js'
+import { CID } from 'multiformats'
+import { cidToString } from '../../index.js'
 
 export type OffchainFileMetadata = {
   type: 'file'
@@ -16,25 +17,19 @@ export interface ChunkInfo {
 }
 
 export const fileMetadata = (
-  dag: IPLDDag,
+  headCID: CID,
+  chunks: ChunkInfo[],
   totalSize: number,
-  name?: string,
-  mimeType?: string,
+  name?: string | null,
+  mimeType?: string | null,
 ): OffchainFileMetadata => {
-  const chunks = Array.from(dag.nodes.values()).filter(
-    (n) => n.Data && IPLDNodeData.decode(n.Data).data,
-  )
-
   return {
     type: 'file',
-    dataCid: cidToString(dag.headCID),
-    name,
-    mimeType,
+    dataCid: cidToString(headCID),
+    name: name ?? undefined,
+    mimeType: mimeType ?? undefined,
     totalSize,
     totalChunks: chunks.length,
-    chunks: chunks.map((chunk) => ({
-      cid: cidToString(cidOfNode(chunk)),
-      size: chunk.Data?.length ?? 0,
-    })),
+    chunks,
   }
 }

package/src/metadata/offchain/folder.ts

@@ -1,3 +1,8 @@
+import { CID } from 'multiformats'
+import { cidOfNode, cidToString } from '../../cid/index.js'
+import { PBNode } from '../../ipld/index.js'
+import { IPLDNodeData, MetadataType } from '../onchain/index.js'
+
 interface ChildrenMetadata {
   type: 'folder' | 'file'
   name?: string
@@ -14,17 +19,33 @@ export type OffchainFolderMetadata = {
   children: ChildrenMetadata[]
 }
 
+export const childrenMetadataFromNode = (node: PBNode): ChildrenMetadata => {
+  const ipldData = IPLDNodeData.decode(node.Data!)
+  if (ipldData.type !== MetadataType.File && ipldData.type !== MetadataType.Folder) {
+    throw new Error('Invalid metadata type')
+  }
+
+  return {
+    type: ipldData.type === MetadataType.File ? 'file' : 'folder',
+    cid: cidToString(cidOfNode(node)),
+    totalSize: ipldData.size ?? 0,
+    name: ipldData.name,
+  }
+}
+
 export const folderMetadata = (
-  cid: string,
+  cid: CID | string,
   children: ChildrenMetadata[],
-  name?: string,
+  name?: string | null,
 ): OffchainFolderMetadata => {
+  cid = typeof cid === 'string' ? cid : cidToString(cid)
+
   return {
     dataCid: cid,
     totalSize: children.reduce((acc, child) => acc + child.totalSize, 0),
     totalFiles: children.length,
     children,
     type: 'folder',
-    name,
+    name: name ?? undefined,
   }
 }

package/src/metadata/onchain/utils.ts

@@ -1,4 +1,4 @@
-import { decode } from '@ipld/dag-pb'
+import { decodeNode } from '../../ipld/index.js'
 import { IPLDNodeData } from '../onchain/index.js'
 
 export const encodeIPLDNodeData = (metadata: IPLDNodeData): Uint8Array => {
@@ -6,7 +6,7 @@ export const encodeIPLDNodeData = (metadata: IPLDNodeData): Uint8Array => {
 }
 
 export const decodeIPLDNodeData = (data: Uint8Array): IPLDNodeData => {
-  const decoded = decode(data)
+  const decoded = decodeNode(data)
   if (!decoded.Data) {
     throw new Error('Invalid data')
   }