@autonomys/auto-dag-data 0.8.2

package/dist/utils/async.js

@@ -0,0 +1,48 @@
+var __asyncValues = (this && this.__asyncValues) || function (o) {
+    if (!Symbol.asyncIterator) throw new TypeError("Symbol.asyncIterator is not defined.");
+    var m = o[Symbol.asyncIterator], i;
+    return m ? m.call(o) : (o = typeof __values === "function" ? __values(o) : o[Symbol.iterator](), i = {}, verb("next"), verb("throw"), verb("return"), i[Symbol.asyncIterator] = function () { return this; }, i);
+    function verb(n) { i[n] = o[n] && function (v) { return new Promise(function (resolve, reject) { v = o[n](v), settle(resolve, reject, v.done, v.value); }); }; }
+    function settle(resolve, reject, d, v) { Promise.resolve(v).then(function(v) { resolve({ value: v, done: d }); }, reject); }
+};
+var __await = (this && this.__await) || function (v) { return this instanceof __await ? (this.v = v, this) : new __await(v); }
+var __asyncGenerator = (this && this.__asyncGenerator) || function (thisArg, _arguments, generator) {
+    if (!Symbol.asyncIterator) throw new TypeError("Symbol.asyncIterator is not defined.");
+    var g = generator.apply(thisArg, _arguments || []), i, q = [];
+    return i = Object.create((typeof AsyncIterator === "function" ? AsyncIterator : Object).prototype), verb("next"), verb("throw"), verb("return", awaitReturn), i[Symbol.asyncIterator] = function () { return this; }, i;
+    function awaitReturn(f) { return function (v) { return Promise.resolve(v).then(f, reject); }; }
+    function verb(n, f) { if (g[n]) { i[n] = function (v) { return new Promise(function (a, b) { q.push([n, v, a, b]) > 1 || resume(n, v); }); }; if (f) i[n] = f(i[n]); } }
+    function resume(n, v) { try { step(g[n](v)); } catch (e) { settle(q[0][3], e); } }
+    function step(r) { r.value instanceof __await ? Promise.resolve(r.value.v).then(fulfill, reject) : settle(q[0][2], r); }
+    function fulfill(value) { resume("next", value); }
+    function reject(value) { resume("throw", value); }
+    function settle(f, v) { if (f(v), q.shift(), q.length) resume(q[0][0], q[0][1]); }
+};
+export const asyncByChunk = function (iterable_1, chunkSize_1) {
+    return __asyncGenerator(this, arguments, function* (iterable, chunkSize, ignoreLastChunk = false) {
+        var _a, e_1, _b, _c;
+        let accumulated = Buffer.alloc(0);
+        try {
+            for (var _d = true, iterable_2 = __asyncValues(iterable), iterable_2_1; iterable_2_1 = yield __await(iterable_2.next()), _a = iterable_2_1.done, !_a; _d = true) {
+                _c = iterable_2_1.value;
+                _d = false;
+                const chunk = _c;
+                accumulated = Buffer.concat([accumulated, chunk]);
+                while (accumulated.length >= chunkSize) {
+                    yield yield __await(accumulated.subarray(0, chunkSize));
+                    accumulated = accumulated.subarray(chunkSize);
+                }
+            }
+        }
+        catch (e_1_1) { e_1 = { error: e_1_1 }; }
+        finally {
+            try {
+                if (!_d && !_a && (_b = iterable_2.return)) yield __await(_b.call(iterable_2));
+            }
+            finally { if (e_1) throw e_1.error; }
+        }
+        if (accumulated.length > 0 && !ignoreLastChunk) {
+            yield yield __await(accumulated);
+        }
+    });
+};

package/dist/utils/index.d.ts

@@ -0,0 +1,2 @@
+export * from './async.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/utils/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/utils/index.ts"],"names":[],"mappings":"AAAA,cAAc,YAAY,CAAA"}

package/dist/utils/index.js

@@ -0,0 +1 @@
+export * from './async.js';

package/dist/utils/types.d.ts

@@ -0,0 +1,2 @@
+export type PickPartial<T, K extends keyof T> = Pick<T, K> & Partial<T>;
+//# sourceMappingURL=types.d.ts.map

package/dist/utils/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/utils/types.ts"],"names":[],"mappings":"AAAA,MAAM,MAAM,WAAW,CAAC,CAAC,EAAE,CAAC,SAAS,MAAM,CAAC,IAAI,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,CAAA"}

package/dist/utils/types.js

@@ -0,0 +1 @@
+export {};

package/jest.config.ts

@@ -0,0 +1,17 @@
+const { createDefaultEsmPreset } = require('ts-jest')
+
+module.exports = {
+  ...createDefaultEsmPreset(),
+  extensionsToTreatAsEsm: ['.ts'],
+  moduleNameMapper: {
+    '^(\\.{1,2}/.*)\\.js$': '$1',
+  },
+  transform: {
+    '^.+\\.tsx?$': [
+      'ts-jest',
+      {
+        useESM: true,
+      },
+    ],
+  },
+}

package/package.json

@@ -0,0 +1,46 @@
+{
+  "name": "@autonomys/auto-dag-data",
+  "packageManager": "yarn@4.1.1",
+  "version": "0.8.2",
+  "license": "MIT",
+  "main": "dist/index.js",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/autonomys/auto-sdk"
+  },
+  "author": {
+    "name": "Autonomys",
+    "url": "https://www.autonomys.net"
+  },
+  "type": "module",
+  "scripts": {
+    "build": "tsc",
+    "pb": "yarn protons src/metadata/onchain/protobuf/OnchainMetadata.proto -o src/metadata/onchain/protobuf",
+    "clean": "rm -rf dist",
+    "format": "prettier --write \"src/**/*.ts\"",
+    "test": "yarn node --experimental-vm-modules $(yarn bin jest)"
+  },
+  "exports": {
+    ".": "./dist/index.js",
+    "./protobuf": "./dist/metadata/onchain/protobuf/onchainMetadata.js"
+  },
+  "devDependencies": {
+    "@types/jest": "^29.5.13",
+    "blockstore-core": "^5.0.2",
+    "interface-store": "^6.0.2",
+    "jest": "^29.7.0",
+    "protobufjs": "^7.4.0",
+    "ts-jest": "^29.2.5",
+    "typescript": "^5.6.2"
+  },
+  "dependencies": {
+    "@ipld/dag-pb": "^4.1.2",
+    "blake3": "1.1.0",
+    "fflate": "^0.8.2",
+    "multiformats": "^13.2.2",
+    "protobufjs": "^7.4.0",
+    "protons": "^7.6.0",
+    "protons-runtime": "^5.5.0"
+  },
+  "gitHead": "027bc98a4e2191c6ce374b3db807a7ae67b7dd10"
+}

package/src/cid/index.ts

@@ -0,0 +1,26 @@
+import { hash } from 'blake3'
+import * as base32 from 'multiformats/bases/base32'
+import { CID } from 'multiformats/cid'
+import * as raw from 'multiformats/codecs/raw'
+import { create } from 'multiformats/hashes/digest'
+import { encodeNode, PBNode } from '../ipld/utils.js'
+
+export const BLAKE3_CODE = 0x1f
+
+export const cidOfNode = (node: PBNode) => {
+  return cidFromBlakeHash(hash(encodeNode(node)))
+}
+
+export const cidToString = (cid: CID) => {
+  return cid.toString(base32.base32)
+}
+
+export const stringToCid = (str: string) => {
+  return CID.parse(str, base32.base32)
+}
+
+export const cidFromBlakeHash = (hash: Buffer) => {
+  return CID.create(1, raw.code, create(BLAKE3_CODE, hash))
+}
+
+export const blake3HashFromCid = (cid: CID) => cid.multihash.digest

package/src/compression/index.ts

@@ -0,0 +1,84 @@
+import { Unzlib, Zlib } from 'fflate'
+import type { AwaitIterable } from 'interface-store'
+import { CompressionAlgorithm } from '../metadata/index.js'
+import { asyncByChunk } from '../utils/async.js'
+import type { PickPartial } from '../utils/types.js'
+import { CompressionOptions } from './types.js'
+
+export const COMPRESSION_CHUNK_SIZE = 1024 * 1024
+
+export async function* compressFile(
+  file: AwaitIterable<Buffer>,
+  {
+    level = 9,
+    chunkSize = COMPRESSION_CHUNK_SIZE,
+    algorithm,
+  }: PickPartial<CompressionOptions, 'algorithm'>,
+): AsyncIterable<Buffer> {
+  if (algorithm !== CompressionAlgorithm.ZLIB) {
+    throw new Error('Unsupported compression algorithm')
+  }
+  if (level < 0 || level > 9) {
+    throw new Error('Invalid compression level')
+  }
+  if (chunkSize <= 0) {
+    throw new Error('Invalid chunk size')
+  }
+
+  const zlib = new Zlib({ level })
+  const compressedChunks: Buffer[] = []
+
+  zlib.ondata = (chunk) => {
+    compressedChunks.push(Buffer.from(chunk))
+  }
+
+  for await (const chunk of asyncByChunk(file, chunkSize)) {
+    zlib.push(chunk, false)
+    while (compressedChunks.length > 0) {
+      yield compressedChunks.shift()!
+    }
+  }
+
+  zlib.push(new Uint8Array(), true)
+  while (compressedChunks.length > 0) {
+    yield compressedChunks.shift()!
+  }
+}
+
+export async function* decompressFile(
+  compressedFile: AwaitIterable<Buffer>,
+  {
+    chunkSize = COMPRESSION_CHUNK_SIZE,
+    algorithm = CompressionAlgorithm.ZLIB,
+    level = 9,
+  }: PickPartial<CompressionOptions, 'algorithm'>,
+): AsyncIterable<Buffer> {
+  if (algorithm !== CompressionAlgorithm.ZLIB) {
+    throw new Error('Unsupported compression algorithm')
+  }
+  if (chunkSize <= 0) {
+    throw new Error('Invalid chunk size')
+  }
+  if (level < 0 || level > 9) {
+    throw new Error('Invalid compression level')
+  }
+
+  const unzlib = new Unzlib()
+  const decompressedChunks: Buffer[] = []
+
+  unzlib.ondata = (chunk) => {
+    decompressedChunks.push(Buffer.from(chunk))
+  }
+
+  for await (const chunk of asyncByChunk(compressedFile, chunkSize)) {
+    unzlib.push(chunk, false)
+    while (decompressedChunks.length > 0) {
+      yield decompressedChunks.shift()!
+    }
+  }
+
+  unzlib.push(new Uint8Array(), true)
+  while (decompressedChunks.length > 0) {
+    yield decompressedChunks.shift()!
+  }
+}

package/src/compression/types.ts

@@ -0,0 +1,11 @@
+import { CompressionAlgorithm } from '../metadata/index.js'
+
+export type CompressionLevel = 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9
+
+export type ZLibOptions = {
+  algorithm: CompressionAlgorithm.ZLIB
+  level: CompressionLevel
+  chunkSize: number
+}
+
+export type CompressionOptions = ZLibOptions

package/src/encryption/index.ts

@@ -0,0 +1,99 @@
+import { Crypto } from '@peculiar/webcrypto'
+import { randomBytes } from 'crypto'
+import { EncryptionAlgorithm, EncryptionOptions } from '../metadata/index.js'
+import { asyncByChunk } from '../utils/async.js'
+import type { PickPartial } from '../utils/types.js'
+import { PasswordGenerationOptions } from './types.js'
+
+const crypto = new Crypto()
+
+export const ENCRYPTING_CHUNK_SIZE = 1024 * 1024
+const IV_SIZE = 16
+const TAG_SIZE = 16
+const ENCRYPTED_CHUNK_SIZE = ENCRYPTING_CHUNK_SIZE + IV_SIZE + TAG_SIZE
+const SALT_SIZE = 32
+
+export const getKeyFromPassword = async ({ password, salt }: PasswordGenerationOptions) => {
+  const encoder = new TextEncoder()
+  const saltHash =
+    typeof salt === 'string' ? await crypto.subtle.digest('SHA-256', encoder.encode(salt)) : salt
+
+  const keyMaterial = await crypto.subtle.importKey(
+    'raw',
+    encoder.encode(password),
+    'PBKDF2',
+    false,
+    ['deriveBits', 'deriveKey'],
+  )
+
+  return crypto.subtle.deriveKey(
+    {
+      name: 'PBKDF2',
+      salt: saltHash,
+      iterations: 100000,
+      hash: 'SHA-256',
+    },
+    keyMaterial,
+    { name: 'AES-GCM', length: 256 },
+    false,
+    ['encrypt', 'decrypt'],
+  )
+}
+
+export const encryptFile = async function* (
+  file: AsyncIterable<Buffer>,
+  password: string,
+  { chunkSize = ENCRYPTING_CHUNK_SIZE, algorithm }: PickPartial<EncryptionOptions, 'algorithm'>,
+): AsyncIterable<Buffer> {
+  if (algorithm !== EncryptionAlgorithm.AES_256_GCM) {
+    throw new Error('Unsupported encryption algorithm')
+  }
+
+  const salt = randomBytes(SALT_SIZE)
+  const key = await getKeyFromPassword({ password, salt })
+
+  yield salt
+
+  for await (const chunk of asyncByChunk(file, chunkSize)) {
+    const iv = crypto.getRandomValues(new Uint8Array(IV_SIZE))
+    const encrypted = await crypto.subtle.encrypt({ name: 'AES-GCM', iv }, key, chunk)
+    yield Buffer.concat([Buffer.from(iv), Buffer.from(encrypted)])
+  }
+}
+
+export const decryptFile = async function* (
+  file: AsyncIterable<Buffer>,
+  password: string,
+  { chunkSize = ENCRYPTED_CHUNK_SIZE, algorithm }: PickPartial<EncryptionOptions, 'algorithm'>,
+): AsyncIterable<Buffer> {
+  if (algorithm !== EncryptionAlgorithm.AES_256_GCM) {
+    throw new Error('Unsupported encryption algorithm')
+  }
+
+  let key: CryptoKey | undefined = undefined
+  let chunks = Buffer.alloc(0)
+  for await (const chunk of file) {
+    chunks = Buffer.concat([chunks, chunk])
+
+    if (chunks.length >= SALT_SIZE && !key) {
+      const salt = chunks.subarray(0, 32)
+      key = await getKeyFromPassword({ password, salt })
+      chunks = chunks.subarray(SALT_SIZE)
+    }
+
+    while (key && chunks.length >= chunkSize) {
+      const iv = chunks.subarray(0, IV_SIZE)
+      const encryptedChunk = chunk.subarray(IV_SIZE, chunkSize)
+      const decrypted = await crypto.subtle.decrypt({ name: 'AES-GCM', iv }, key, encryptedChunk)
+      chunks = chunks.subarray(chunkSize)
+      yield Buffer.from(decrypted)
+    }
+  }
+
+  if (key && chunks.length > 0) {
+    const iv = chunks.subarray(0, IV_SIZE)
+    const encryptedChunk = chunks.subarray(IV_SIZE, chunkSize)
+    const decrypted = await crypto.subtle.decrypt({ name: 'AES-GCM', iv }, key, encryptedChunk)
+    yield Buffer.from(decrypted)
+  }
+}

package/src/encryption/types.ts

@@ -0,0 +1,4 @@
+export type PasswordGenerationOptions = {
+  password: string
+  salt: string | Uint8Array
+}

package/src/index.ts

@@ -0,0 +1,5 @@
+export * from './cid/index.js'
+export * from './compression/index.js'
+export * from './encryption/index.js'
+export * from './ipld/index.js'
+export * from './metadata/index.js'

package/src/ipld/builders.ts

@@ -0,0 +1,40 @@
+import { CID } from 'multiformats/cid'
+import { FileUploadOptions } from '../metadata/index.js'
+import { PBNode } from './index.js'
+import {
+  createChunkedFileIpldNode,
+  createChunkedMetadataIpldNode,
+  createFileChunkIpldNode,
+  createFileInlinkIpldNode,
+  createMetadataChunkIpldNode,
+  createMetadataInlinkIpldNode,
+  createSingleFileIpldNode,
+  createSingleMetadataIpldNode,
+} from './nodes.js'
+
+export interface Builders {
+  inlink: (links: CID[], size: number, linkDepth: number, chunkSize: number) => PBNode
+  chunk: (data: Buffer) => PBNode
+  root: (
+    links: CID[],
+    size: number,
+    linkDepth: number,
+    name?: string,
+    maxNodeSize?: number,
+    fileUploadOptions?: FileUploadOptions,
+  ) => PBNode
+  single: (data: Buffer, filename?: string, fileUploadOptions?: FileUploadOptions) => PBNode
+}
+export const metadataBuilders: Builders = {
+  inlink: createMetadataInlinkIpldNode,
+  chunk: createMetadataChunkIpldNode,
+  root: createChunkedMetadataIpldNode,
+  single: createSingleMetadataIpldNode,
+}
+
+export const fileBuilders: Builders = {
+  inlink: createFileInlinkIpldNode,
+  chunk: createFileChunkIpldNode,
+  root: createChunkedFileIpldNode,
+  single: createSingleFileIpldNode,
+}

package/src/ipld/chunker.ts

@@ -0,0 +1,245 @@
+import type { BaseBlockstore } from 'blockstore-core'
+import type { AwaitIterable } from 'interface-store'
+import { CID } from 'multiformats'
+import { cidOfNode } from '../cid/index.js'
+import { decodeIPLDNodeData, FileUploadOptions, OffchainMetadata } from '../metadata/index.js'
+import { Builders, fileBuilders, metadataBuilders } from './builders.js'
+import { createFolderInlinkIpldNode, createFolderIpldNode } from './nodes.js'
+import { chunkBuffer, encodeNode, PBNode } from './utils.js'
+
+type ChunkerLimits = {
+  maxChunkSize: number
+  maxLinkPerNode: number
+}
+
+type ChunkerOptions = ChunkerLimits & FileUploadOptions
+
+export const DEFAULT_MAX_CHUNK_SIZE = 64 * 1024
+
+const ESTIMATED_LINK_SIZE_IN_BYTES = 64
+export const DEFAULT_MAX_LINK_PER_NODE = DEFAULT_MAX_CHUNK_SIZE / ESTIMATED_LINK_SIZE_IN_BYTES
+
+export const processFileToIPLDFormat = (
+  blockstore: BaseBlockstore,
+  file: AwaitIterable<Buffer>,
+  totalSize: number,
+  filename?: string,
+  {
+    maxChunkSize = DEFAULT_MAX_CHUNK_SIZE,
+    maxLinkPerNode = DEFAULT_MAX_LINK_PER_NODE,
+    encryption = undefined,
+    compression = undefined,
+  }: Partial<ChunkerOptions> = {
+    maxChunkSize: DEFAULT_MAX_CHUNK_SIZE,
+    maxLinkPerNode: DEFAULT_MAX_LINK_PER_NODE,
+    encryption: undefined,
+    compression: undefined,
+  },
+): Promise<CID> => {
+  return processBufferToIPLDFormat(blockstore, file, filename, totalSize, fileBuilders, {
+    maxChunkSize,
+    maxLinkPerNode,
+    encryption,
+    compression,
+  })
+}
+
+export const processMetadataToIPLDFormat = async (
+  blockstore: BaseBlockstore,
+  metadata: OffchainMetadata,
+  limits: { maxChunkSize: number; maxLinkPerNode: number } = {
+    maxChunkSize: DEFAULT_MAX_CHUNK_SIZE,
+    maxLinkPerNode: DEFAULT_MAX_LINK_PER_NODE,
+  },
+): Promise<CID> => {
+  const buffer = Buffer.from(JSON.stringify(metadata))
+  const name = `${metadata.name}.metadata.json`
+  return processBufferToIPLDFormat(
+    blockstore,
+    (async function* () {
+      yield buffer
+    })(),
+    name,
+    buffer.byteLength,
+    metadataBuilders,
+    limits,
+  )
+}
+
+const processBufferToIPLDFormat = async (
+  blockstore: BaseBlockstore,
+  buffer: AwaitIterable<Buffer>,
+  filename: string | undefined,
+  totalSize: number,
+  builders: Builders,
+  {
+    maxChunkSize = DEFAULT_MAX_CHUNK_SIZE,
+    maxLinkPerNode = DEFAULT_MAX_LINK_PER_NODE,
+    encryption = undefined,
+    compression = undefined,
+  }: ChunkerOptions = {
+    maxChunkSize: DEFAULT_MAX_CHUNK_SIZE,
+    maxLinkPerNode: DEFAULT_MAX_LINK_PER_NODE,
+    encryption: undefined,
+    compression: undefined,
+  },
+): Promise<CID> => {
+  const bufferChunks = chunkBuffer(buffer, { maxChunkSize: maxChunkSize })
+
+  let CIDs: CID[] = []
+  for await (const chunk of bufferChunks) {
+    const node = builders.chunk(chunk)
+    const cid = cidOfNode(node)
+    await blockstore.put(cid, encodeNode(node))
+    CIDs.push(cid)
+  }
+
+  return processBufferToIPLDFormatFromChunks(blockstore, CIDs, filename, totalSize, builders, {
+    maxLinkPerNode,
+    maxChunkSize,
+    encryption,
+    compression,
+  })
+}
+
+export const processBufferToIPLDFormatFromChunks = async (
+  blockstore: BaseBlockstore,
+  chunks: AwaitIterable<CID>,
+  filename: string | undefined,
+  totalSize: number,
+  builders: Builders,
+  {
+    maxChunkSize = DEFAULT_MAX_CHUNK_SIZE,
+    maxLinkPerNode = DEFAULT_MAX_LINK_PER_NODE,
+    encryption = undefined,
+    compression = undefined,
+  }: Partial<ChunkerOptions> = {
+    maxChunkSize: DEFAULT_MAX_CHUNK_SIZE,
+    maxLinkPerNode: DEFAULT_MAX_LINK_PER_NODE,
+    encryption: undefined,
+    compression: undefined,
+  },
+): Promise<CID> => {
+  let chunkCount = 0
+  let CIDs: CID[] = []
+  for await (const chunk of chunks) {
+    CIDs.push(chunk)
+    chunkCount++
+  }
+
+  if (CIDs.length === 1) {
+    const nodeBytes = await blockstore.get(CIDs[0])
+    await blockstore.delete(CIDs[0])
+    const data = decodeIPLDNodeData(nodeBytes)
+    const singleNode = builders.single(Buffer.from(data.data!), filename, {
+      compression,
+      encryption,
+    })
+    await blockstore.put(cidOfNode(singleNode), encodeNode(singleNode))
+    const headCID = cidOfNode(singleNode)
+
+    return headCID
+  }
+
+  let depth = 1
+  while (CIDs.length > maxLinkPerNode) {
+    const newCIDs: CID[] = []
+    for (let i = 0; i < CIDs.length; i += maxLinkPerNode) {
+      const chunk = CIDs.slice(i, i + maxLinkPerNode)
+
+      const node = builders.inlink(chunk, chunk.length, depth, maxChunkSize)
+      const cid = cidOfNode(node)
+      await blockstore.put(cid, encodeNode(node))
+      newCIDs.push(cid)
+    }
+    depth++
+    CIDs = newCIDs
+  }
+  const head = builders.root(CIDs, totalSize, depth, filename, maxChunkSize, {
+    compression,
+    encryption,
+  })
+  const headCID = cidOfNode(head)
+  await blockstore.put(headCID, encodeNode(head))
+
+  return headCID
+}
+
+export const processFolderToIPLDFormat = async (
+  blockstore: BaseBlockstore,
+  children: CID[],
+  name: string,
+  size: number,
+  {
+    maxLinkPerNode = DEFAULT_MAX_LINK_PER_NODE,
+    maxChunkSize = DEFAULT_MAX_CHUNK_SIZE,
+    compression = undefined,
+    encryption = undefined,
+  }: Partial<ChunkerOptions> = {
+    maxLinkPerNode: DEFAULT_MAX_LINK_PER_NODE,
+    maxChunkSize: DEFAULT_MAX_CHUNK_SIZE,
+    compression: undefined,
+    encryption: undefined,
+  },
+): Promise<CID> => {
+  let cids = children
+  let depth = 0
+  while (cids.length > maxLinkPerNode) {
+    const newCIDs: CID[] = []
+    for (let i = 0; i < cids.length; i += maxLinkPerNode) {
+      const chunk = cids.slice(i, i + maxLinkPerNode)
+      const node = createFolderInlinkIpldNode(chunk, depth)
+      const cid = cidOfNode(node)
+      await blockstore.put(cid, encodeNode(node))
+      newCIDs.push(cid)
+    }
+    cids = newCIDs
+    depth++
+  }
+
+  const node = createFolderIpldNode(cids, name, depth, size, maxChunkSize, {
+    compression,
+    encryption,
+  })
+  const cid = cidOfNode(node)
+  await blockstore.put(cid, encodeNode(node))
+
+  return cid
+}
+
+/**
+ * Process chunks to IPLD format, return the last chunk if it's not full
+ * @returns the last chunk if it's not full, otherwise an empty buffer
+ */
+export const processChunksToIPLDFormat = async (
+  blockstore: BaseBlockstore,
+  chunks: AwaitIterable<Buffer>,
+  builders: Builders,
+  { maxChunkSize = DEFAULT_MAX_CHUNK_SIZE }: { maxChunkSize?: number },
+): Promise<Buffer> => {
+  const bufferChunks = chunkBuffer(chunks, { maxChunkSize, ignoreLastChunk: false })
+
+  for await (const chunk of bufferChunks) {
+    if (chunk.byteLength < maxChunkSize) {
+      return chunk
+    }
+
+    const node = builders.chunk(chunk)
+    const cid = cidOfNode(node)
+    await blockstore.put(cid, encodeNode(node))
+  }
+
+  return Buffer.alloc(0)
+}
+
+export const ensureNodeMaxSize = (
+  node: PBNode,
+  maxSize: number = DEFAULT_MAX_CHUNK_SIZE,
+): PBNode => {
+  const nodeSize = encodeNode(node).byteLength
+  if (nodeSize > maxSize) {
+    throw new Error(`Node is too large to fit in a single chunk: ${nodeSize} > ${maxSize}`)
+  }
+
+  return node
+}

package/src/ipld/index.ts

@@ -0,0 +1,4 @@
+export * from './builders.js'
+export * from './chunker.js'
+export * from './nodes.js'
+export * from './utils.js'