@autonomys/auto-dag-data 1.0.7 → 1.0.8

package/README.md

@@ -8,7 +8,7 @@
 
 ## Overview
 
-The **Autonomys Auto DAG Data SDK** (`@autonomys/auto-dag-data`) provides utilities for creating and managing IPLD DAGs (InterPlanetary Linked Data Directed Acyclic Graphs) for files and folders. It facilitates chunking large files, handling metadata, and creating folder structures suitable for distributed storage systems like IPFS.
+The **Autonomys Auto Dag Data SDK** (`@autonomys/auto-dag-data`) provides utilities for creating and managing IPLD DAGs (InterPlanetary Linked Data Directed Acyclic Graphs) for files and folders. It facilitates chunking large files, handling metadata, and creating folder structures suitable for distributed storage systems like IPFS.
 
 ## Features
 
@@ -20,7 +20,7 @@ The **Autonomys Auto DAG Data SDK** (`@autonomys/auto-dag-data`) provides utilit
 
 ## Installation
 
-You can install Auto-DAG-Data using npm or yarn:
+You can install Auto-Dag-Data using npm or yarn:
 
 ```bash
 npm install @autonomys/auto-dag-data
@@ -36,23 +36,27 @@ yarn add @autonomys/auto-dag-data
 
 ### Creating an IPLD DAG from a File
 
-To create an IPLD DAG from a file, you can use the `createFileIPLDDag` function:
+To create an IPLD DAG from a file, you can use the `processFileToIPLDFormat` function:
 
 ```typescript
-import { createFileIPLDDag } from '@autonomys/auto-dag-data'
+import { processFileToIPLDFormat } from '@autonomys/auto-dag-data'
+import { MemoryBlockstore } from 'blockstore-core/memory'
 import fs from 'fs'
 
-const fileBuffer = fs.readFileSync('path/to/your/file.txt')
+const fileStream = fs.createReadStream('path/to/your/file.txt')
+const fileSize = fs.statSync('path/to/your/file.txt').size
 
-const dag = createFileIPLDDag(fileBuffer, 'file.txt')
+const blockstore = new MemoryBlockstore()
+const fileCID = processFileToIPLDFormat(blockstore, fileStream, totalSize, 'file.txt')
 ```
 
 ### Creating an IPLD DAG from a Folder
 
-To create an IPLD DAG from a folder, you can use the `createFolderIPLDDag` function:
+To generate an IPLD DAG from a folder, you can use the `processFolderToIPLDFormat` function:
 
 ```typescript
-import { createFolderIPLDDag } from '@autonomys/auto-dag-data'
+import { processFolderToIPLDFormat, decodeNode } from '@autonomys/auto-dag-data'
+import { MemoryBlockstore } from 'blockstore-core/memory'
 import { CID } from 'multiformats'
 
 // Example child CIDs and folder information
@@ -60,9 +64,12 @@ const childCIDs: CID[] = [
   /* array of CIDs */
 ]
 const folderName = 'my-folder'
-const folderSize = 1024 // size in bytes
+const folderSize = 1024 // size in bytes (the sum of their children size)
 
-const folderDag = createFolderIPLDDag(childCIDs, folderName, folderSize)
+const blockstore = new MemoryBlockstore()
+const folderCID = processFolderToIPLDFormat(blockstore, childCIDs, folderName, folderSize)
+
+const node = decodeNode(blockstore.get(folderCID))
 ```
 
 ### Working with CIDs
@@ -115,14 +122,16 @@ const metadataNode = createMetadataNode(metadata)
 ### Example: Creating a File DAG and Converting to CID
 
 ```typescript
-import { createFileIPLDDag, cidOfNode, cidToString } from '@autonomys/auto-dag-data'
+import { processFileToIPLDFormat } from '@autonomys/auto-dag-data'
+import { MemoryBlockstore } from 'blockstore-core/memory'
 import fs from 'fs'
 
-const fileBuffer = fs.readFileSync('path/to/your/file.txt')
+const fileStream = fs.createReadStream('path/to/your/file.txt')
+const fileSize = fs.statSync('path/to/your/file.txt').size
 
-const dag = createFileIPLDDag(fileBuffer, 'file.txt')
+const blockstore = new MemoryBlockstore()
+const cid = processFileToIPLDFormat(blockstore, fileStream, totalSize, 'file.txt')
 
-const cid = cidOfNode(dag.headCID)
 const cidString = cidToString(cid)
 
 console.log(`CID of the file DAG: ${cidString}`)
@@ -137,13 +146,14 @@ import {
   cidToString,
   type OffchainMetadata,
 } from '@autonomys/auto-dag-data'
+import { MemoryBlockstore } from 'blockstore-core/memory'
 import fs from 'fs'
 
 const metadata: OffchainMetadata = fs.readFileSync('path/to/your/metadata.json')
 
-const dag = createMetadataIPLDDag(metadata)
+const blockstore = new MemoryBlockstore()
+const cid = processMetadataToIPLDFormat(blockstore, metadata)
 
-const cid = cidOfNode(dag.headCID)
 const cidString = cidToString(cid)
 
 console.log(`CID of the metadata DAG: ${cidString}`)

package/dist/ipld/chunker.d.ts

@@ -9,6 +9,7 @@ type ChunkerLimits = {
     maxLinkPerNode: number;
 };
 type ChunkerOptions = ChunkerLimits & FileUploadOptions;
+export declare const DEFAULT_NODE_MAX_SIZE = 65535;
 export declare const NODE_METADATA_SIZE: number;
 export declare const DEFAULT_MAX_CHUNK_SIZE: number;
 export declare const LINK_SIZE_IN_BYTES = 40;
@@ -24,8 +25,8 @@ export declare const processFolderToIPLDFormat: (blockstore: BaseBlockstore, chi
  * Process chunks to IPLD format, return the last chunk if it's not full
  * @returns the last chunk if it's not full, otherwise an empty buffer
  */
-export declare const processChunksToIPLDFormat: (blockstore: BaseBlockstore, chunks: AwaitIterable<Buffer>, builders: Builders, { maxNodeSize }: {
-    maxNodeSize?: number;
+export declare const processChunksToIPLDFormat: (blockstore: BaseBlockstore, chunks: AwaitIterable<Buffer>, builders: Builders, { maxChunkSize }?: {
+    maxChunkSize?: number;
 }) => Promise<Buffer>;
 export declare const ensureNodeMaxSize: (node: PBNode, maxSize?: number) => PBNode;
 export {};

package/dist/ipld/chunker.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"chunker.d.ts","sourceRoot":"","sources":["../../src/ipld/chunker.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAA;AACrD,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAA;AACpD,OAAO,EAAE,GAAG,EAAE,MAAM,cAAc,CAAA;AAElC,OAAO,EAAsB,iBAAiB,EAAE,gBAAgB,EAAE,MAAM,sBAAsB,CAAA;AAE9F,OAAO,EAAE,QAAQ,EAAkC,MAAM,eAAe,CAAA;AAExE,OAAO,EAA2B,MAAM,EAAE,MAAM,YAAY,CAAA;AAE5D,KAAK,aAAa,GAAG;IACnB,WAAW,EAAE,MAAM,CAAA;IACnB,cAAc,EAAE,MAAM,CAAA;CACvB,CAAA;AAED,KAAK,cAAc,GAAG,aAAa,GAAG,iBAAiB,CAAA;AAmBvD,eAAO,MAAM,kBAAkB,QAML,CAAA;AAE1B,eAAO,MAAM,sBAAsB,QAA6C,CAAA;AAEhF,eAAO,MAAM,kBAAkB,KAAK,CAAA;AACpC,eAAO,MAAM,yBAAyB,QAA0D,CAAA;AAEhG,eAAO,MAAM,uBAAuB,eACtB,cAAc,QACpB,aAAa,CAAC,MAAM,CAAC,aAChB,MAAM,aACN,MAAM,8DAMd,OAAO,CAAC,cAAc,CAAC,KAMzB,OAAO,CAAC,GAAG,CAWb,CAAA;AAED,eAAO,MAAM,2BAA2B,eAC1B,cAAc,YAChB,gBAAgB,WAClB;IAAE,WAAW,EAAE,MAAM,CAAC;IAAC,cAAc,EAAE,MAAM,CAAA;CAAE,KAItD,OAAO,CAAC,GAAG,CAiBb,CAAA;AA0CD,eAAO,MAAM,mCAAmC,eAClC,cAAc,UAClB,aAAa,CAAC,GAAG,CAAC,YAChB,MAAM,GAAG,SAAS,aACjB,MAAM,YACP,QAAQ,2EAMf,OAAO,CAAC,cAAc,CAAC,KAMzB,OAAO,CAAC,GAAG,CAgDb,CAAA;AAED,eAAO,MAAM,yBAAyB,eACxB,cAAc,YAChB,GAAG,EAAE,QACT,MAAM,QACN,MAAM,2EAMT,OAAO,CAAC,cAAc,CAAC,KAMzB,OAAO,CAAC,GAAG,CA4Bb,CAAA;AAED;;;GAGG;AACH,eAAO,MAAM,yBAAyB,eACxB,cAAc,UAClB,aAAa,CAAC,MAAM,CAAC,YACnB,QAAQ,mBACwB;IAAE,WAAW,CAAC,EAAE,MAAM,CAAA;CAAE,KACjE,OAAO,CAAC,MAAM,CAiBhB,CAAA;AAED,eAAO,MAAM,iBAAiB,SACtB,MAAM,YACH,MAAM,KACd,MAOF,CAAA"}
+{"version":3,"file":"chunker.d.ts","sourceRoot":"","sources":["../../src/ipld/chunker.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAA;AACrD,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAA;AACpD,OAAO,EAAE,GAAG,EAAE,MAAM,cAAc,CAAA;AAElC,OAAO,EAAsB,iBAAiB,EAAE,gBAAgB,EAAE,MAAM,sBAAsB,CAAA;AAE9F,OAAO,EAAE,QAAQ,EAAkC,MAAM,eAAe,CAAA;AAExE,OAAO,EAA2B,MAAM,EAAE,MAAM,YAAY,CAAA;AAE5D,KAAK,aAAa,GAAG;IACnB,WAAW,EAAE,MAAM,CAAA;IACnB,cAAc,EAAE,MAAM,CAAA;CACvB,CAAA;AAED,KAAK,cAAc,GAAG,aAAa,GAAG,iBAAiB,CAAA;AAEvD,eAAO,MAAM,qBAAqB,QAAQ,CAAA;AAiB1C,eAAO,MAAM,kBAAkB,QAML,CAAA;AAE1B,eAAO,MAAM,sBAAsB,QAA6C,CAAA;AAEhF,eAAO,MAAM,kBAAkB,KAAK,CAAA;AACpC,eAAO,MAAM,yBAAyB,QAA0D,CAAA;AAEhG,eAAO,MAAM,uBAAuB,eACtB,cAAc,QACpB,aAAa,CAAC,MAAM,CAAC,aAChB,MAAM,aACN,MAAM,8DAMd,OAAO,CAAC,cAAc,CAAC,KAMzB,OAAO,CAAC,GAAG,CAWb,CAAA;AAED,eAAO,MAAM,2BAA2B,eAC1B,cAAc,YAChB,gBAAgB,WAClB;IAAE,WAAW,EAAE,MAAM,CAAC;IAAC,cAAc,EAAE,MAAM,CAAA;CAAE,KAItD,OAAO,CAAC,GAAG,CAiBb,CAAA;AA0CD,eAAO,MAAM,mCAAmC,eAClC,cAAc,UAClB,aAAa,CAAC,GAAG,CAAC,YAChB,MAAM,GAAG,SAAS,aACjB,MAAM,YACP,QAAQ,2EAMf,OAAO,CAAC,cAAc,CAAC,KAMzB,OAAO,CAAC,GAAG,CAgDb,CAAA;AAED,eAAO,MAAM,yBAAyB,eACxB,cAAc,YAChB,GAAG,EAAE,QACT,MAAM,QACN,MAAM,2EAMT,OAAO,CAAC,cAAc,CAAC,KAMzB,OAAO,CAAC,GAAG,CA4Bb,CAAA;AAED;;;GAGG;AACH,eAAO,MAAM,yBAAyB,eACxB,cAAc,UAClB,aAAa,CAAC,MAAM,CAAC,YACnB,QAAQ,qBACyB;IAAE,YAAY,CAAC,EAAE,MAAM,CAAA;CAAE,KAGnE,OAAO,CAAC,MAAM,CAiBhB,CAAA;AAED,eAAO,MAAM,iBAAiB,SACtB,MAAM,YACH,MAAM,KACd,MAOF,CAAA"}

package/dist/ipld/chunker.js

@@ -33,7 +33,7 @@ import { stringifyMetadata } from '../utils/metadata.js';
 import { fileBuilders, metadataBuilders } from './builders.js';
 import { createFolderInlinkIpldNode, createFolderIpldNode } from './nodes.js';
 import { chunkBuffer, encodeNode } from './utils.js';
-const DEFAULT_NODE_MAX_SIZE = 65535;
+export const DEFAULT_NODE_MAX_SIZE = 65535;
 // u8 -> 1 byte (may grow in the future but unlikely further than 255)
 const NODE_TYPE_SIZE = 1;
 // u32 -> 4 bytes
@@ -57,8 +57,8 @@ export const NODE_METADATA_SIZE = NODE_TYPE_SIZE +
 export const DEFAULT_MAX_CHUNK_SIZE = DEFAULT_NODE_MAX_SIZE - NODE_METADATA_SIZE;
 export const LINK_SIZE_IN_BYTES = 40;
 export const DEFAULT_MAX_LINK_PER_NODE = Math.floor(DEFAULT_MAX_CHUNK_SIZE / LINK_SIZE_IN_BYTES);
-export const processFileToIPLDFormat = (blockstore, file, totalSize, filename, { maxNodeSize = DEFAULT_MAX_CHUNK_SIZE, maxLinkPerNode = DEFAULT_MAX_LINK_PER_NODE, encryption = undefined, compression = undefined, } = {
-    maxNodeSize: DEFAULT_MAX_CHUNK_SIZE,
+export const processFileToIPLDFormat = (blockstore, file, totalSize, filename, { maxNodeSize = DEFAULT_NODE_MAX_SIZE, maxLinkPerNode = DEFAULT_MAX_LINK_PER_NODE, encryption = undefined, compression = undefined, } = {
+    maxNodeSize: DEFAULT_NODE_MAX_SIZE,
     maxLinkPerNode: DEFAULT_MAX_LINK_PER_NODE,
     encryption: undefined,
     compression: undefined,
@@ -74,7 +74,7 @@ export const processFileToIPLDFormat = (blockstore, file, totalSize, filename, {
     });
 };
 export const processMetadataToIPLDFormat = (blockstore_1, metadata_1, ...args_1) => __awaiter(void 0, [blockstore_1, metadata_1, ...args_1], void 0, function* (blockstore, metadata, limits = {
-    maxNodeSize: DEFAULT_MAX_CHUNK_SIZE,
+    maxNodeSize: DEFAULT_NODE_MAX_SIZE,
     maxLinkPerNode: DEFAULT_MAX_LINK_PER_NODE,
 }) {
     if (metadata.name && metadata.name.length > MAX_NAME_SIZE) {
@@ -87,8 +87,8 @@ export const processMetadataToIPLDFormat = (blockstore_1, metadata_1, ...args_1)
         });
     })(), metadata.name, BigInt(buffer.byteLength), metadataBuilders, limits);
 });
-const processBufferToIPLDFormat = (blockstore_1, buffer_1, filename_1, totalSize_1, builders_1, ...args_1) => __awaiter(void 0, [blockstore_1, buffer_1, filename_1, totalSize_1, builders_1, ...args_1], void 0, function* (blockstore, buffer, filename, totalSize, builders, { maxNodeSize: maxNodeSize = DEFAULT_MAX_CHUNK_SIZE, maxLinkPerNode = DEFAULT_MAX_LINK_PER_NODE, encryption = undefined, compression = undefined, } = {
-    maxNodeSize: DEFAULT_MAX_CHUNK_SIZE,
+const processBufferToIPLDFormat = (blockstore_1, buffer_1, filename_1, totalSize_1, builders_1, ...args_1) => __awaiter(void 0, [blockstore_1, buffer_1, filename_1, totalSize_1, builders_1, ...args_1], void 0, function* (blockstore, buffer, filename, totalSize, builders, { maxNodeSize: maxNodeSize = DEFAULT_NODE_MAX_SIZE, maxLinkPerNode = DEFAULT_MAX_LINK_PER_NODE, encryption = undefined, compression = undefined, } = {
+    maxNodeSize: DEFAULT_NODE_MAX_SIZE,
     maxLinkPerNode: DEFAULT_MAX_LINK_PER_NODE,
     encryption: undefined,
     compression: undefined,
@@ -124,8 +124,8 @@ const processBufferToIPLDFormat = (blockstore_1, buffer_1, filename_1, totalSize
         compression,
     });
 });
-export const processBufferToIPLDFormatFromChunks = (blockstore_1, chunks_1, filename_1, totalSize_1, builders_1, ...args_1) => __awaiter(void 0, [blockstore_1, chunks_1, filename_1, totalSize_1, builders_1, ...args_1], void 0, function* (blockstore, chunks, filename, totalSize, builders, { maxNodeSize: maxNodeSize = DEFAULT_MAX_CHUNK_SIZE, maxLinkPerNode = DEFAULT_MAX_LINK_PER_NODE, encryption = undefined, compression = undefined, } = {
-    maxNodeSize: DEFAULT_MAX_CHUNK_SIZE,
+export const processBufferToIPLDFormatFromChunks = (blockstore_1, chunks_1, filename_1, totalSize_1, builders_1, ...args_1) => __awaiter(void 0, [blockstore_1, chunks_1, filename_1, totalSize_1, builders_1, ...args_1], void 0, function* (blockstore, chunks, filename, totalSize, builders, { maxNodeSize: maxNodeSize = DEFAULT_NODE_MAX_SIZE, maxLinkPerNode = DEFAULT_MAX_LINK_PER_NODE, encryption = undefined, compression = undefined, } = {
+    maxNodeSize: DEFAULT_NODE_MAX_SIZE,
     maxLinkPerNode: DEFAULT_MAX_LINK_PER_NODE,
     encryption: undefined,
     compression: undefined,
@@ -186,9 +186,9 @@ export const processBufferToIPLDFormatFromChunks = (blockstore_1, chunks_1, file
     yield blockstore.put(headCID, encodeNode(head));
     return headCID;
 });
-export const processFolderToIPLDFormat = (blockstore_1, children_1, name_1, size_1, ...args_1) => __awaiter(void 0, [blockstore_1, children_1, name_1, size_1, ...args_1], void 0, function* (blockstore, children, name, size, { maxLinkPerNode = DEFAULT_MAX_LINK_PER_NODE, maxNodeSize: maxNodeSize = DEFAULT_MAX_CHUNK_SIZE, compression = undefined, encryption = undefined, } = {
+export const processFolderToIPLDFormat = (blockstore_1, children_1, name_1, size_1, ...args_1) => __awaiter(void 0, [blockstore_1, children_1, name_1, size_1, ...args_1], void 0, function* (blockstore, children, name, size, { maxLinkPerNode = DEFAULT_MAX_LINK_PER_NODE, maxNodeSize: maxNodeSize = DEFAULT_NODE_MAX_SIZE, compression = undefined, encryption = undefined, } = {
     maxLinkPerNode: DEFAULT_MAX_LINK_PER_NODE,
-    maxNodeSize: DEFAULT_MAX_CHUNK_SIZE,
+    maxNodeSize: DEFAULT_NODE_MAX_SIZE,
     compression: undefined,
     encryption: undefined,
 }) {
@@ -221,18 +221,20 @@ export const processFolderToIPLDFormat = (blockstore_1, children_1, name_1, size
  * Process chunks to IPLD format, return the last chunk if it's not full
  * @returns the last chunk if it's not full, otherwise an empty buffer
  */
-export const processChunksToIPLDFormat = (blockstore_1, chunks_1, builders_1, _a) => __awaiter(void 0, [blockstore_1, chunks_1, builders_1, _a], void 0, function* (blockstore, chunks, builders, { maxNodeSize = DEFAULT_MAX_CHUNK_SIZE }) {
-    var _b, e_3, _c, _d;
+export const processChunksToIPLDFormat = (blockstore_1, chunks_1, builders_1, ...args_1) => __awaiter(void 0, [blockstore_1, chunks_1, builders_1, ...args_1], void 0, function* (blockstore, chunks, builders, { maxChunkSize = DEFAULT_MAX_CHUNK_SIZE } = {
+    maxChunkSize: DEFAULT_MAX_CHUNK_SIZE,
+}) {
+    var _a, e_3, _b, _c;
     const bufferChunks = chunkBuffer(chunks, {
-        maxChunkSize: maxNodeSize - NODE_METADATA_SIZE,
+        maxChunkSize,
         ignoreLastChunk: false,
     });
     try {
-        for (var _e = true, bufferChunks_2 = __asyncValues(bufferChunks), bufferChunks_2_1; bufferChunks_2_1 = yield bufferChunks_2.next(), _b = bufferChunks_2_1.done, !_b; _e = true) {
-            _d = bufferChunks_2_1.value;
-            _e = false;
-            const chunk = _d;
-            if (chunk.byteLength < maxNodeSize) {
+        for (var _d = true, bufferChunks_2 = __asyncValues(bufferChunks), bufferChunks_2_1; bufferChunks_2_1 = yield bufferChunks_2.next(), _a = bufferChunks_2_1.done, !_a; _d = true) {
+            _c = bufferChunks_2_1.value;
+            _d = false;
+            const chunk = _c;
+            if (chunk.byteLength < maxChunkSize) {
                 return chunk;
             }
             const node = builders.chunk(chunk);
@@ -243,13 +245,13 @@ export const processChunksToIPLDFormat = (blockstore_1, chunks_1, builders_1, _a
     catch (e_3_1) { e_3 = { error: e_3_1 }; }
     finally {
         try {
-            if (!_e && !_b && (_c = bufferChunks_2.return)) yield _c.call(bufferChunks_2);
+            if (!_d && !_a && (_b = bufferChunks_2.return)) yield _b.call(bufferChunks_2);
         }
         finally { if (e_3) throw e_3.error; }
     }
     return Buffer.alloc(0);
 });
-export const ensureNodeMaxSize = (node, maxSize = DEFAULT_MAX_CHUNK_SIZE) => {
+export const ensureNodeMaxSize = (node, maxSize = DEFAULT_NODE_MAX_SIZE) => {
     const nodeSize = encodeNode(node).byteLength;
     if (nodeSize > maxSize) {
         throw new Error(`Node is too large to fit in a single chunk: ${nodeSize} > ${maxSize}`);

package/dist/ipld/nodes.js

@@ -1,9 +1,9 @@
 import { encodeIPLDNodeData, MetadataType } from '../metadata/onchain/index.js';
 import { stringifyMetadata } from '../utils/metadata.js';
-import { DEFAULT_MAX_CHUNK_SIZE, ensureNodeMaxSize } from './chunker.js';
+import { DEFAULT_NODE_MAX_SIZE, ensureNodeMaxSize } from './chunker.js';
 import { createNode } from './index.js';
 /// Creates a file chunk ipld node
-export const createFileChunkIpldNode = (data, maxNodeSize = DEFAULT_MAX_CHUNK_SIZE) => ensureNodeMaxSize(createNode(encodeIPLDNodeData({
+export const createFileChunkIpldNode = (data, maxNodeSize = DEFAULT_NODE_MAX_SIZE) => ensureNodeMaxSize(createNode(encodeIPLDNodeData({
     type: MetadataType.FileChunk,
     size: BigInt(data.length).valueOf(),
     linkDepth: 0,
@@ -12,7 +12,7 @@ export const createFileChunkIpldNode = (data, maxNodeSize = DEFAULT_MAX_CHUNK_SI
 // Creates a file ipld node
 // links: the CIDs of the file's contents
 // @todo: add the file's metadata
-export const createChunkedFileIpldNode = (links, size, linkDepth, name, maxNodeSize = DEFAULT_MAX_CHUNK_SIZE, uploadOptions) => ensureNodeMaxSize(createNode(encodeIPLDNodeData({
+export const createChunkedFileIpldNode = (links, size, linkDepth, name, maxNodeSize = DEFAULT_NODE_MAX_SIZE, uploadOptions) => ensureNodeMaxSize(createNode(encodeIPLDNodeData({
     type: MetadataType.File,
     name,
     size,
@@ -21,7 +21,7 @@ export const createChunkedFileIpldNode = (links, size, linkDepth, name, maxNodeS
 }), links.map((cid) => ({ Hash: cid }))), maxNodeSize);
 // Creates a file ipld node
 // links: the CIDs of the file's contents
-export const createFileInlinkIpldNode = (links, size, linkDepth, maxNodeSize = DEFAULT_MAX_CHUNK_SIZE) => ensureNodeMaxSize(createNode(encodeIPLDNodeData({
+export const createFileInlinkIpldNode = (links, size, linkDepth, maxNodeSize = DEFAULT_NODE_MAX_SIZE) => ensureNodeMaxSize(createNode(encodeIPLDNodeData({
     type: MetadataType.FileInlink,
     size: BigInt(size).valueOf(),
     linkDepth,
@@ -29,7 +29,7 @@ export const createFileInlinkIpldNode = (links, size, linkDepth, maxNodeSize = D
 // Creates a file ipld node
 // links: the CIDs of the file's contents
 // @todo: add the file's metadata
-export const createSingleFileIpldNode = (data, name, uploadOptions, maxNodeSize = DEFAULT_MAX_CHUNK_SIZE) => ensureNodeMaxSize(createNode(encodeIPLDNodeData({
+export const createSingleFileIpldNode = (data, name, uploadOptions, maxNodeSize = DEFAULT_NODE_MAX_SIZE) => ensureNodeMaxSize(createNode(encodeIPLDNodeData({
     type: MetadataType.File,
     name,
     size: BigInt(data.length).valueOf(),
@@ -40,7 +40,7 @@ export const createSingleFileIpldNode = (data, name, uploadOptions, maxNodeSize
 // Creates a file ipld node
 // links: the CIDs of the file's contents
 // @todo: add the file's metadata
-export const createMetadataInlinkIpldNode = (links, size, linkDepth, maxNodeSize = DEFAULT_MAX_CHUNK_SIZE) => ensureNodeMaxSize(createNode(encodeIPLDNodeData({
+export const createMetadataInlinkIpldNode = (links, size, linkDepth, maxNodeSize = DEFAULT_NODE_MAX_SIZE) => ensureNodeMaxSize(createNode(encodeIPLDNodeData({
     type: MetadataType.FileInlink,
     size: BigInt(size).valueOf(),
     linkDepth,
@@ -55,13 +55,13 @@ export const createSingleMetadataIpldNode = (data, name) => createNode(encodeIPL
     linkDepth: 0,
     data,
 }), []);
-export const createMetadataChunkIpldNode = (data, maxNodeSize = DEFAULT_MAX_CHUNK_SIZE) => ensureNodeMaxSize(createNode(encodeIPLDNodeData({
+export const createMetadataChunkIpldNode = (data, maxNodeSize = DEFAULT_NODE_MAX_SIZE) => ensureNodeMaxSize(createNode(encodeIPLDNodeData({
     type: MetadataType.MetadataChunk,
     size: BigInt(data.length).valueOf(),
     linkDepth: 0,
     data,
 })), maxNodeSize);
-export const createChunkedMetadataIpldNode = (links, size, linkDepth, name, maxNodeSize = DEFAULT_MAX_CHUNK_SIZE) => ensureNodeMaxSize(createNode(encodeIPLDNodeData({
+export const createChunkedMetadataIpldNode = (links, size, linkDepth, name, maxNodeSize = DEFAULT_NODE_MAX_SIZE) => ensureNodeMaxSize(createNode(encodeIPLDNodeData({
     type: MetadataType.Metadata,
     name,
     size,
@@ -70,19 +70,19 @@ export const createChunkedMetadataIpldNode = (links, size, linkDepth, name, maxN
 // Creates a folder ipld node
 // links: the CIDs of the folder's contents
 // @todo: add the folder's metadata
-export const createFolderIpldNode = (links, name, linkDepth, size, maxNodeSize = DEFAULT_MAX_CHUNK_SIZE, uploadOptions) => ensureNodeMaxSize(createNode(encodeIPLDNodeData({
+export const createFolderIpldNode = (links, name, linkDepth, size, maxNodeSize = DEFAULT_NODE_MAX_SIZE, uploadOptions) => ensureNodeMaxSize(createNode(encodeIPLDNodeData({
     type: MetadataType.Folder,
     name,
     size,
     linkDepth,
     uploadOptions,
 }), links.map((cid) => ({ Hash: cid }))), maxNodeSize);
-export const createFolderInlinkIpldNode = (links, linkDepth, maxNodeSize = DEFAULT_MAX_CHUNK_SIZE) => ensureNodeMaxSize(createNode(encodeIPLDNodeData({
+export const createFolderInlinkIpldNode = (links, linkDepth, maxNodeSize = DEFAULT_NODE_MAX_SIZE) => ensureNodeMaxSize(createNode(encodeIPLDNodeData({
     type: MetadataType.FolderInlink,
     linkDepth,
 }), links.map((cid) => ({ Hash: cid }))), maxNodeSize);
 /// Creates a metadata ipld node
-export const createMetadataNode = (metadata, maxNodeSize = DEFAULT_MAX_CHUNK_SIZE) => {
+export const createMetadataNode = (metadata, maxNodeSize = DEFAULT_NODE_MAX_SIZE) => {
     const data = Buffer.from(stringifyMetadata(metadata));
     return ensureNodeMaxSize(createNode(encodeIPLDNodeData({
         type: MetadataType.Metadata,

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@autonomys/auto-dag-data",
   "packageManager": "yarn@4.1.1",
-  "version": "1.0.7",
+  "version": "1.0.8",
   "license": "MIT",
   "main": "dist/index.js",
   "repository": {
@@ -48,5 +48,5 @@
     "protons": "^7.6.0",
     "protons-runtime": "^5.5.0"
   },
-  "gitHead": "2ee93dd0867c16e0a42f2f47b2fa66bc41ba29cf"
+  "gitHead": "88b1b90467db7dd1e301387451ed727189f9808c"
 }

package/src/ipld/chunker.ts

@@ -15,7 +15,7 @@ type ChunkerLimits = {
 
 type ChunkerOptions = ChunkerLimits & FileUploadOptions
 
-const DEFAULT_NODE_MAX_SIZE = 65535
+export const DEFAULT_NODE_MAX_SIZE = 65535
 
 // u8 -> 1 byte (may grow in the future but unlikely further than 255)
 const NODE_TYPE_SIZE = 1
@@ -51,12 +51,12 @@ export const processFileToIPLDFormat = (
   totalSize: bigint,
   filename?: string,
   {
-    maxNodeSize = DEFAULT_MAX_CHUNK_SIZE,
+    maxNodeSize = DEFAULT_NODE_MAX_SIZE,
     maxLinkPerNode = DEFAULT_MAX_LINK_PER_NODE,
     encryption = undefined,
     compression = undefined,
   }: Partial<ChunkerOptions> = {
-    maxNodeSize: DEFAULT_MAX_CHUNK_SIZE,
+    maxNodeSize: DEFAULT_NODE_MAX_SIZE,
     maxLinkPerNode: DEFAULT_MAX_LINK_PER_NODE,
     encryption: undefined,
     compression: undefined,
@@ -78,7 +78,7 @@ export const processMetadataToIPLDFormat = async (
   blockstore: BaseBlockstore,
   metadata: OffchainMetadata,
   limits: { maxNodeSize: number; maxLinkPerNode: number } = {
-    maxNodeSize: DEFAULT_MAX_CHUNK_SIZE,
+    maxNodeSize: DEFAULT_NODE_MAX_SIZE,
     maxLinkPerNode: DEFAULT_MAX_LINK_PER_NODE,
   },
 ): Promise<CID> => {
@@ -107,12 +107,12 @@ const processBufferToIPLDFormat = async (
   totalSize: bigint,
   builders: Builders,
   {
-    maxNodeSize: maxNodeSize = DEFAULT_MAX_CHUNK_SIZE,
+    maxNodeSize: maxNodeSize = DEFAULT_NODE_MAX_SIZE,
     maxLinkPerNode = DEFAULT_MAX_LINK_PER_NODE,
     encryption = undefined,
     compression = undefined,
   }: ChunkerOptions = {
-    maxNodeSize: DEFAULT_MAX_CHUNK_SIZE,
+    maxNodeSize: DEFAULT_NODE_MAX_SIZE,
     maxLinkPerNode: DEFAULT_MAX_LINK_PER_NODE,
     encryption: undefined,
     compression: undefined,
@@ -147,12 +147,12 @@ export const processBufferToIPLDFormatFromChunks = async (
   totalSize: bigint,
   builders: Builders,
   {
-    maxNodeSize: maxNodeSize = DEFAULT_MAX_CHUNK_SIZE,
+    maxNodeSize: maxNodeSize = DEFAULT_NODE_MAX_SIZE,
     maxLinkPerNode = DEFAULT_MAX_LINK_PER_NODE,
     encryption = undefined,
     compression = undefined,
   }: Partial<ChunkerOptions> = {
-    maxNodeSize: DEFAULT_MAX_CHUNK_SIZE,
+    maxNodeSize: DEFAULT_NODE_MAX_SIZE,
     maxLinkPerNode: DEFAULT_MAX_LINK_PER_NODE,
     encryption: undefined,
     compression: undefined,
@@ -214,12 +214,12 @@ export const processFolderToIPLDFormat = async (
   size: bigint,
   {
     maxLinkPerNode = DEFAULT_MAX_LINK_PER_NODE,
-    maxNodeSize: maxNodeSize = DEFAULT_MAX_CHUNK_SIZE,
+    maxNodeSize: maxNodeSize = DEFAULT_NODE_MAX_SIZE,
     compression = undefined,
     encryption = undefined,
   }: Partial<ChunkerOptions> = {
     maxLinkPerNode: DEFAULT_MAX_LINK_PER_NODE,
-    maxNodeSize: DEFAULT_MAX_CHUNK_SIZE,
+    maxNodeSize: DEFAULT_NODE_MAX_SIZE,
     compression: undefined,
     encryption: undefined,
   },
@@ -261,15 +261,17 @@ export const processChunksToIPLDFormat = async (
   blockstore: BaseBlockstore,
   chunks: AwaitIterable<Buffer>,
   builders: Builders,
-  { maxNodeSize = DEFAULT_MAX_CHUNK_SIZE }: { maxNodeSize?: number },
+  { maxChunkSize = DEFAULT_MAX_CHUNK_SIZE }: { maxChunkSize?: number } = {
+    maxChunkSize: DEFAULT_MAX_CHUNK_SIZE,
+  },
 ): Promise<Buffer> => {
   const bufferChunks = chunkBuffer(chunks, {
-    maxChunkSize: maxNodeSize - NODE_METADATA_SIZE,
+    maxChunkSize,
     ignoreLastChunk: false,
   })
 
   for await (const chunk of bufferChunks) {
-    if (chunk.byteLength < maxNodeSize) {
+    if (chunk.byteLength < maxChunkSize) {
       return chunk
     }
 
@@ -283,7 +285,7 @@ export const processChunksToIPLDFormat = async (
 
 export const ensureNodeMaxSize = (
   node: PBNode,
-  maxSize: number = DEFAULT_MAX_CHUNK_SIZE,
+  maxSize: number = DEFAULT_NODE_MAX_SIZE,
 ): PBNode => {
   const nodeSize = encodeNode(node).byteLength
   if (nodeSize > maxSize) {

package/src/ipld/nodes.ts

@@ -2,13 +2,13 @@ import { CID } from 'multiformats/cid'
 import { FileUploadOptions, OffchainMetadata } from '../metadata/index.js'
 import { encodeIPLDNodeData, MetadataType } from '../metadata/onchain/index.js'
 import { stringifyMetadata } from '../utils/metadata.js'
-import { DEFAULT_MAX_CHUNK_SIZE, ensureNodeMaxSize } from './chunker.js'
+import { DEFAULT_NODE_MAX_SIZE, ensureNodeMaxSize } from './chunker.js'
 import { createNode, PBNode } from './index.js'
 
 /// Creates a file chunk ipld node
 export const createFileChunkIpldNode = (
   data: Buffer,
-  maxNodeSize: number = DEFAULT_MAX_CHUNK_SIZE,
+  maxNodeSize: number = DEFAULT_NODE_MAX_SIZE,
 ): PBNode =>
   ensureNodeMaxSize(
     createNode(
@@ -31,7 +31,7 @@ export const createChunkedFileIpldNode = (
   size: bigint,
   linkDepth: number,
   name?: string,
-  maxNodeSize: number = DEFAULT_MAX_CHUNK_SIZE,
+  maxNodeSize: number = DEFAULT_NODE_MAX_SIZE,
   uploadOptions?: FileUploadOptions,
 ): PBNode =>
   ensureNodeMaxSize(
@@ -53,7 +53,7 @@ export const createFileInlinkIpldNode = (
   links: CID[],
   size: number,
   linkDepth: number,
-  maxNodeSize: number = DEFAULT_MAX_CHUNK_SIZE,
+  maxNodeSize: number = DEFAULT_NODE_MAX_SIZE,
 ): PBNode =>
   ensureNodeMaxSize(
     createNode(
@@ -74,7 +74,7 @@ export const createSingleFileIpldNode = (
   data: Buffer,
   name?: string,
   uploadOptions?: FileUploadOptions,
-  maxNodeSize: number = DEFAULT_MAX_CHUNK_SIZE,
+  maxNodeSize: number = DEFAULT_NODE_MAX_SIZE,
 ): PBNode =>
   ensureNodeMaxSize(
     createNode(
@@ -98,7 +98,7 @@ export const createMetadataInlinkIpldNode = (
   links: CID[],
   size: number,
   linkDepth: number,
-  maxNodeSize: number = DEFAULT_MAX_CHUNK_SIZE,
+  maxNodeSize: number = DEFAULT_NODE_MAX_SIZE,
 ): PBNode =>
   ensureNodeMaxSize(
     createNode(
@@ -129,7 +129,7 @@ export const createSingleMetadataIpldNode = (data: Buffer, name?: string): PBNod
 
 export const createMetadataChunkIpldNode = (
   data: Buffer,
-  maxNodeSize: number = DEFAULT_MAX_CHUNK_SIZE,
+  maxNodeSize: number = DEFAULT_NODE_MAX_SIZE,
 ): PBNode =>
   ensureNodeMaxSize(
     createNode(
@@ -148,7 +148,7 @@ export const createChunkedMetadataIpldNode = (
   size: bigint,
   linkDepth: number,
   name?: string,
-  maxNodeSize: number = DEFAULT_MAX_CHUNK_SIZE,
+  maxNodeSize: number = DEFAULT_NODE_MAX_SIZE,
 ): PBNode =>
   ensureNodeMaxSize(
     createNode(
@@ -171,7 +171,7 @@ export const createFolderIpldNode = (
   name: string,
   linkDepth: number,
   size: bigint,
-  maxNodeSize: number = DEFAULT_MAX_CHUNK_SIZE,
+  maxNodeSize: number = DEFAULT_NODE_MAX_SIZE,
   uploadOptions?: FileUploadOptions,
 ): PBNode =>
   ensureNodeMaxSize(
@@ -191,7 +191,7 @@ export const createFolderIpldNode = (
 export const createFolderInlinkIpldNode = (
   links: CID[],
   linkDepth: number,
-  maxNodeSize: number = DEFAULT_MAX_CHUNK_SIZE,
+  maxNodeSize: number = DEFAULT_NODE_MAX_SIZE,
 ): PBNode =>
   ensureNodeMaxSize(
     createNode(
@@ -207,7 +207,7 @@ export const createFolderInlinkIpldNode = (
 /// Creates a metadata ipld node
 export const createMetadataNode = (
   metadata: OffchainMetadata,
-  maxNodeSize: number = DEFAULT_MAX_CHUNK_SIZE,
+  maxNodeSize: number = DEFAULT_NODE_MAX_SIZE,
 ): PBNode => {
   const data = Buffer.from(stringifyMetadata(metadata))
 

package/tests/nodes.spec.ts

@@ -4,7 +4,7 @@ import {
   createFileChunkIpldNode,
   createSingleFileIpldNode,
 } from '../src/index.js'
-import { createNode, DEFAULT_MAX_CHUNK_SIZE } from '../src/ipld/index.js'
+import { createNode, DEFAULT_MAX_CHUNK_SIZE, DEFAULT_NODE_MAX_SIZE } from '../src/ipld/index.js'
 import { IPLDNodeData, MetadataType } from '../src/metadata/onchain/protobuf/OnchainMetadata.js'
 
 describe('node creation', () => {
@@ -30,7 +30,7 @@ describe('node creation', () => {
     })
 
     it('single file root node | buffer too large', () => {
-      const maxNodeSize = DEFAULT_MAX_CHUNK_SIZE
+      const maxNodeSize = DEFAULT_NODE_MAX_SIZE
       const buffer = Buffer.from('h'.repeat(maxNodeSize))
       expect(() => createSingleFileIpldNode(buffer, 'test.txt')).toThrow()
     })