@autonomys/auto-dag-data 1.0.6 → 1.0.7

package/README.md

@@ -1,4 +1,4 @@
-# Auto DAG Data
+# Autonomys Auto DAG Data SDK
 
 ![Autonomys Banner](https://github.com/autonomys/auto-sdk/blob/main/.github/images/autonomys-banner.webp)
 

package/dist/ipld/builders.d.ts

@@ -3,8 +3,8 @@ import { FileUploadOptions } from '../metadata/index.js';
 import { PBNode } from './index.js';
 export interface Builders {
     inlink: (links: CID[], size: number, linkDepth: number, chunkSize: number) => PBNode;
-    chunk: (data: Buffer) => PBNode;
-    root: (links: CID[], size: number, linkDepth: number, name?: string, maxNodeSize?: number, fileUploadOptions?: FileUploadOptions) => PBNode;
+    chunk: (data: Buffer, maxNodeSize?: number) => PBNode;
+    root: (links: CID[], size: bigint, linkDepth: number, name?: string, maxNodeSize?: number, fileUploadOptions?: FileUploadOptions) => PBNode;
     single: (data: Buffer, filename?: string, fileUploadOptions?: FileUploadOptions) => PBNode;
 }
 export declare const metadataBuilders: Builders;

package/dist/ipld/builders.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"builders.d.ts","sourceRoot":"","sources":["../../src/ipld/builders.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,MAAM,kBAAkB,CAAA;AACtC,OAAO,EAAE,iBAAiB,EAAE,MAAM,sBAAsB,CAAA;AACxD,OAAO,EAAE,MAAM,EAAE,MAAM,YAAY,CAAA;AAYnC,MAAM,WAAW,QAAQ;IACvB,MAAM,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,KAAK,MAAM,CAAA;IACpF,KAAK,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,MAAM,CAAA;IAC/B,IAAI,EAAE,CACJ,KAAK,EAAE,GAAG,EAAE,EACZ,IAAI,EAAE,MAAM,EACZ,SAAS,EAAE,MAAM,EACjB,IAAI,CAAC,EAAE,MAAM,EACb,WAAW,CAAC,EAAE,MAAM,EACpB,iBAAiB,CAAC,EAAE,iBAAiB,KAClC,MAAM,CAAA;IACX,MAAM,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,EAAE,iBAAiB,CAAC,EAAE,iBAAiB,KAAK,MAAM,CAAA;CAC3F;AACD,eAAO,MAAM,gBAAgB,EAAE,QAK9B,CAAA;AAED,eAAO,MAAM,YAAY,EAAE,QAK1B,CAAA"}
+{"version":3,"file":"builders.d.ts","sourceRoot":"","sources":["../../src/ipld/builders.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,MAAM,kBAAkB,CAAA;AACtC,OAAO,EAAE,iBAAiB,EAAE,MAAM,sBAAsB,CAAA;AACxD,OAAO,EAAE,MAAM,EAAE,MAAM,YAAY,CAAA;AAYnC,MAAM,WAAW,QAAQ;IACvB,MAAM,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,KAAK,MAAM,CAAA;IACpF,KAAK,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,WAAW,CAAC,EAAE,MAAM,KAAK,MAAM,CAAA;IACrD,IAAI,EAAE,CACJ,KAAK,EAAE,GAAG,EAAE,EACZ,IAAI,EAAE,MAAM,EACZ,SAAS,EAAE,MAAM,EACjB,IAAI,CAAC,EAAE,MAAM,EACb,WAAW,CAAC,EAAE,MAAM,EACpB,iBAAiB,CAAC,EAAE,iBAAiB,KAClC,MAAM,CAAA;IACX,MAAM,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,EAAE,iBAAiB,CAAC,EAAE,iBAAiB,KAAK,MAAM,CAAA;CAC3F;AACD,eAAO,MAAM,gBAAgB,EAAE,QAK9B,CAAA;AAED,eAAO,MAAM,YAAY,EAAE,QAK1B,CAAA"}

package/dist/ipld/chunker.d.ts

@@ -5,25 +5,27 @@ import { FileUploadOptions, OffchainMetadata } from '../metadata/index.js';
 import { Builders } from './builders.js';
 import { PBNode } from './utils.js';
 type ChunkerLimits = {
-    maxChunkSize: number;
+    maxNodeSize: number;
     maxLinkPerNode: number;
 };
 type ChunkerOptions = ChunkerLimits & FileUploadOptions;
+export declare const NODE_METADATA_SIZE: number;
 export declare const DEFAULT_MAX_CHUNK_SIZE: number;
+export declare const LINK_SIZE_IN_BYTES = 40;
 export declare const DEFAULT_MAX_LINK_PER_NODE: number;
-export declare const processFileToIPLDFormat: (blockstore: BaseBlockstore, file: AwaitIterable<Buffer>, totalSize: number, filename?: string, { maxChunkSize, maxLinkPerNode, encryption, compression, }?: Partial<ChunkerOptions>) => Promise<CID>;
+export declare const processFileToIPLDFormat: (blockstore: BaseBlockstore, file: AwaitIterable<Buffer>, totalSize: bigint, filename?: string, { maxNodeSize, maxLinkPerNode, encryption, compression, }?: Partial<ChunkerOptions>) => Promise<CID>;
 export declare const processMetadataToIPLDFormat: (blockstore: BaseBlockstore, metadata: OffchainMetadata, limits?: {
-    maxChunkSize: number;
+    maxNodeSize: number;
     maxLinkPerNode: number;
 }) => Promise<CID>;
-export declare const processBufferToIPLDFormatFromChunks: (blockstore: BaseBlockstore, chunks: AwaitIterable<CID>, filename: string | undefined, totalSize: number, builders: Builders, { maxChunkSize, maxLinkPerNode, encryption, compression, }?: Partial<ChunkerOptions>) => Promise<CID>;
-export declare const processFolderToIPLDFormat: (blockstore: BaseBlockstore, children: CID[], name: string, size: number, { maxLinkPerNode, maxChunkSize, compression, encryption, }?: Partial<ChunkerOptions>) => Promise<CID>;
+export declare const processBufferToIPLDFormatFromChunks: (blockstore: BaseBlockstore, chunks: AwaitIterable<CID>, filename: string | undefined, totalSize: bigint, builders: Builders, { maxNodeSize: maxNodeSize, maxLinkPerNode, encryption, compression, }?: Partial<ChunkerOptions>) => Promise<CID>;
+export declare const processFolderToIPLDFormat: (blockstore: BaseBlockstore, children: CID[], name: string, size: bigint, { maxLinkPerNode, maxNodeSize: maxNodeSize, compression, encryption, }?: Partial<ChunkerOptions>) => Promise<CID>;
 /**
  * Process chunks to IPLD format, return the last chunk if it's not full
  * @returns the last chunk if it's not full, otherwise an empty buffer
  */
-export declare const processChunksToIPLDFormat: (blockstore: BaseBlockstore, chunks: AwaitIterable<Buffer>, builders: Builders, { maxChunkSize }: {
-    maxChunkSize?: number;
+export declare const processChunksToIPLDFormat: (blockstore: BaseBlockstore, chunks: AwaitIterable<Buffer>, builders: Builders, { maxNodeSize }: {
+    maxNodeSize?: number;
 }) => Promise<Buffer>;
 export declare const ensureNodeMaxSize: (node: PBNode, maxSize?: number) => PBNode;
 export {};

package/dist/ipld/chunker.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"chunker.d.ts","sourceRoot":"","sources":["../../src/ipld/chunker.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAA;AACrD,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAA;AACpD,OAAO,EAAE,GAAG,EAAE,MAAM,cAAc,CAAA;AAElC,OAAO,EAAsB,iBAAiB,EAAE,gBAAgB,EAAE,MAAM,sBAAsB,CAAA;AAC9F,OAAO,EAAE,QAAQ,EAAkC,MAAM,eAAe,CAAA;AAExE,OAAO,EAA2B,MAAM,EAAE,MAAM,YAAY,CAAA;AAE5D,KAAK,aAAa,GAAG;IACnB,YAAY,EAAE,MAAM,CAAA;IACpB,cAAc,EAAE,MAAM,CAAA;CACvB,CAAA;AAED,KAAK,cAAc,GAAG,aAAa,GAAG,iBAAiB,CAAA;AAEvD,eAAO,MAAM,sBAAsB,QAAY,CAAA;AAG/C,eAAO,MAAM,yBAAyB,QAAwD,CAAA;AAE9F,eAAO,MAAM,uBAAuB,eACtB,cAAc,QACpB,aAAa,CAAC,MAAM,CAAC,aAChB,MAAM,aACN,MAAM,+DAMd,OAAO,CAAC,cAAc,CAAC,KAMzB,OAAO,CAAC,GAAG,CAOb,CAAA;AAED,eAAO,MAAM,2BAA2B,eAC1B,cAAc,YAChB,gBAAgB,WAClB;IAAE,YAAY,EAAE,MAAM,CAAC;IAAC,cAAc,EAAE,MAAM,CAAA;CAAE,KAIvD,OAAO,CAAC,GAAG,CAab,CAAA;AAsCD,eAAO,MAAM,mCAAmC,eAClC,cAAc,UAClB,aAAa,CAAC,GAAG,CAAC,YAChB,MAAM,GAAG,SAAS,aACjB,MAAM,YACP,QAAQ,+DAMf,OAAO,CAAC,cAAc,CAAC,KAMzB,OAAO,CAAC,GAAG,CA4Cb,CAAA;AAED,eAAO,MAAM,yBAAyB,eACxB,cAAc,YAChB,GAAG,EAAE,QACT,MAAM,QACN,MAAM,+DAMT,OAAO,CAAC,cAAc,CAAC,KAMzB,OAAO,CAAC,GAAG,CAwBb,CAAA;AAED;;;GAGG;AACH,eAAO,MAAM,yBAAyB,eACxB,cAAc,UAClB,aAAa,CAAC,MAAM,CAAC,YACnB,QAAQ,oBACyB;IAAE,YAAY,CAAC,EAAE,MAAM,CAAA;CAAE,KACnE,OAAO,CAAC,MAAM,CAchB,CAAA;AAED,eAAO,MAAM,iBAAiB,SACtB,MAAM,YACH,MAAM,KACd,MAOF,CAAA"}
+{"version":3,"file":"chunker.d.ts","sourceRoot":"","sources":["../../src/ipld/chunker.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAA;AACrD,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAA;AACpD,OAAO,EAAE,GAAG,EAAE,MAAM,cAAc,CAAA;AAElC,OAAO,EAAsB,iBAAiB,EAAE,gBAAgB,EAAE,MAAM,sBAAsB,CAAA;AAE9F,OAAO,EAAE,QAAQ,EAAkC,MAAM,eAAe,CAAA;AAExE,OAAO,EAA2B,MAAM,EAAE,MAAM,YAAY,CAAA;AAE5D,KAAK,aAAa,GAAG;IACnB,WAAW,EAAE,MAAM,CAAA;IACnB,cAAc,EAAE,MAAM,CAAA;CACvB,CAAA;AAED,KAAK,cAAc,GAAG,aAAa,GAAG,iBAAiB,CAAA;AAmBvD,eAAO,MAAM,kBAAkB,QAML,CAAA;AAE1B,eAAO,MAAM,sBAAsB,QAA6C,CAAA;AAEhF,eAAO,MAAM,kBAAkB,KAAK,CAAA;AACpC,eAAO,MAAM,yBAAyB,QAA0D,CAAA;AAEhG,eAAO,MAAM,uBAAuB,eACtB,cAAc,QACpB,aAAa,CAAC,MAAM,CAAC,aAChB,MAAM,aACN,MAAM,8DAMd,OAAO,CAAC,cAAc,CAAC,KAMzB,OAAO,CAAC,GAAG,CAWb,CAAA;AAED,eAAO,MAAM,2BAA2B,eAC1B,cAAc,YAChB,gBAAgB,WAClB;IAAE,WAAW,EAAE,MAAM,CAAC;IAAC,cAAc,EAAE,MAAM,CAAA;CAAE,KAItD,OAAO,CAAC,GAAG,CAiBb,CAAA;AA0CD,eAAO,MAAM,mCAAmC,eAClC,cAAc,UAClB,aAAa,CAAC,GAAG,CAAC,YAChB,MAAM,GAAG,SAAS,aACjB,MAAM,YACP,QAAQ,2EAMf,OAAO,CAAC,cAAc,CAAC,KAMzB,OAAO,CAAC,GAAG,CAgDb,CAAA;AAED,eAAO,MAAM,yBAAyB,eACxB,cAAc,YAChB,GAAG,EAAE,QACT,MAAM,QACN,MAAM,2EAMT,OAAO,CAAC,cAAc,CAAC,KAMzB,OAAO,CAAC,GAAG,CA4Bb,CAAA;AAED;;;GAGG;AACH,eAAO,MAAM,yBAAyB,eACxB,cAAc,UAClB,aAAa,CAAC,MAAM,CAAC,YACnB,QAAQ,mBACwB;IAAE,WAAW,CAAC,EAAE,MAAM,CAAA;CAAE,KACjE,OAAO,CAAC,MAAM,CAiBhB,CAAA;AAED,eAAO,MAAM,iBAAiB,SACtB,MAAM,YACH,MAAM,KACd,MAOF,CAAA"}

package/dist/ipld/chunker.js

@@ -29,45 +29,75 @@ var __asyncValues = (this && this.__asyncValues) || function (o) {
 };
 import { cidOfNode } from '../cid/index.js';
 import { decodeIPLDNodeData } from '../metadata/index.js';
+import { stringifyMetadata } from '../utils/metadata.js';
 import { fileBuilders, metadataBuilders } from './builders.js';
 import { createFolderInlinkIpldNode, createFolderIpldNode } from './nodes.js';
 import { chunkBuffer, encodeNode } from './utils.js';
-export const DEFAULT_MAX_CHUNK_SIZE = 64 * 1024;
-const ESTIMATED_LINK_SIZE_IN_BYTES = 64;
-export const DEFAULT_MAX_LINK_PER_NODE = DEFAULT_MAX_CHUNK_SIZE / ESTIMATED_LINK_SIZE_IN_BYTES;
-export const processFileToIPLDFormat = (blockstore, file, totalSize, filename, { maxChunkSize = DEFAULT_MAX_CHUNK_SIZE, maxLinkPerNode = DEFAULT_MAX_LINK_PER_NODE, encryption = undefined, compression = undefined, } = {
-    maxChunkSize: DEFAULT_MAX_CHUNK_SIZE,
+const DEFAULT_NODE_MAX_SIZE = 65535;
+// u8 -> 1 byte (may grow in the future but unlikely further than 255)
+const NODE_TYPE_SIZE = 1;
+// u32 -> 4 bytes
+const NODE_LINK_DEPTH_SIZE = 4;
+// u64 -> 8 bytes
+const NODE_SIZE_SIZE = 8;
+// Limit at 255 string length (Mac Limit)
+const MAX_NAME_SIZE = 255;
+const END_OF_STRING_BYTE = 1;
+const NODE_NAME_SIZE = MAX_NAME_SIZE + END_OF_STRING_BYTE;
+// Upload options may be amplified in the future
+const NODE_UPLOAD_OPTIONS_SIZE = 100;
+// Reserve 100 bytes for future use
+const NODE_RESERVED_SIZE = 100;
+export const NODE_METADATA_SIZE = NODE_TYPE_SIZE +
+    NODE_LINK_DEPTH_SIZE +
+    NODE_SIZE_SIZE +
+    NODE_NAME_SIZE +
+    NODE_RESERVED_SIZE +
+    NODE_UPLOAD_OPTIONS_SIZE;
+export const DEFAULT_MAX_CHUNK_SIZE = DEFAULT_NODE_MAX_SIZE - NODE_METADATA_SIZE;
+export const LINK_SIZE_IN_BYTES = 40;
+export const DEFAULT_MAX_LINK_PER_NODE = Math.floor(DEFAULT_MAX_CHUNK_SIZE / LINK_SIZE_IN_BYTES);
+export const processFileToIPLDFormat = (blockstore, file, totalSize, filename, { maxNodeSize = DEFAULT_MAX_CHUNK_SIZE, maxLinkPerNode = DEFAULT_MAX_LINK_PER_NODE, encryption = undefined, compression = undefined, } = {
+    maxNodeSize: DEFAULT_MAX_CHUNK_SIZE,
     maxLinkPerNode: DEFAULT_MAX_LINK_PER_NODE,
     encryption: undefined,
     compression: undefined,
 }) => {
+    if (filename && filename.length > MAX_NAME_SIZE) {
+        throw new Error(`Filename is too long: ${filename.length} > ${MAX_NAME_SIZE}`);
+    }
     return processBufferToIPLDFormat(blockstore, file, filename, totalSize, fileBuilders, {
-        maxChunkSize,
+        maxNodeSize,
         maxLinkPerNode,
         encryption,
         compression,
     });
 };
 export const processMetadataToIPLDFormat = (blockstore_1, metadata_1, ...args_1) => __awaiter(void 0, [blockstore_1, metadata_1, ...args_1], void 0, function* (blockstore, metadata, limits = {
-    maxChunkSize: DEFAULT_MAX_CHUNK_SIZE,
+    maxNodeSize: DEFAULT_MAX_CHUNK_SIZE,
     maxLinkPerNode: DEFAULT_MAX_LINK_PER_NODE,
 }) {
-    const buffer = Buffer.from(JSON.stringify(metadata));
-    const name = `${metadata.name}.metadata.json`;
+    if (metadata.name && metadata.name.length > MAX_NAME_SIZE) {
+        throw new Error(`Filename is too long: ${metadata.name.length} > ${MAX_NAME_SIZE}`);
+    }
+    const buffer = Buffer.from(stringifyMetadata(metadata));
     return processBufferToIPLDFormat(blockstore, (function () {
         return __asyncGenerator(this, arguments, function* () {
             yield yield __await(buffer);
         });
-    })(), name, buffer.byteLength, metadataBuilders, limits);
+    })(), metadata.name, BigInt(buffer.byteLength), metadataBuilders, limits);
 });
-const processBufferToIPLDFormat = (blockstore_1, buffer_1, filename_1, totalSize_1, builders_1, ...args_1) => __awaiter(void 0, [blockstore_1, buffer_1, filename_1, totalSize_1, builders_1, ...args_1], void 0, function* (blockstore, buffer, filename, totalSize, builders, { maxChunkSize = DEFAULT_MAX_CHUNK_SIZE, maxLinkPerNode = DEFAULT_MAX_LINK_PER_NODE, encryption = undefined, compression = undefined, } = {
-    maxChunkSize: DEFAULT_MAX_CHUNK_SIZE,
+const processBufferToIPLDFormat = (blockstore_1, buffer_1, filename_1, totalSize_1, builders_1, ...args_1) => __awaiter(void 0, [blockstore_1, buffer_1, filename_1, totalSize_1, builders_1, ...args_1], void 0, function* (blockstore, buffer, filename, totalSize, builders, { maxNodeSize: maxNodeSize = DEFAULT_MAX_CHUNK_SIZE, maxLinkPerNode = DEFAULT_MAX_LINK_PER_NODE, encryption = undefined, compression = undefined, } = {
+    maxNodeSize: DEFAULT_MAX_CHUNK_SIZE,
     maxLinkPerNode: DEFAULT_MAX_LINK_PER_NODE,
     encryption: undefined,
     compression: undefined,
 }) {
     var _a, e_1, _b, _c;
-    const bufferChunks = chunkBuffer(buffer, { maxChunkSize: maxChunkSize });
+    if (filename && filename.length > MAX_NAME_SIZE) {
+        throw new Error(`Filename is too long: ${filename.length} > ${MAX_NAME_SIZE}`);
+    }
+    const bufferChunks = chunkBuffer(buffer, { maxChunkSize: maxNodeSize - NODE_METADATA_SIZE });
     let CIDs = [];
     try {
         for (var _d = true, bufferChunks_1 = __asyncValues(bufferChunks), bufferChunks_1_1; bufferChunks_1_1 = yield bufferChunks_1.next(), _a = bufferChunks_1_1.done, !_a; _d = true) {
@@ -89,19 +119,22 @@ const processBufferToIPLDFormat = (blockstore_1, buffer_1, filename_1, totalSize
     }
     return processBufferToIPLDFormatFromChunks(blockstore, CIDs, filename, totalSize, builders, {
         maxLinkPerNode,
-        maxChunkSize,
+        maxNodeSize,
         encryption,
         compression,
     });
 });
-export const processBufferToIPLDFormatFromChunks = (blockstore_1, chunks_1, filename_1, totalSize_1, builders_1, ...args_1) => __awaiter(void 0, [blockstore_1, chunks_1, filename_1, totalSize_1, builders_1, ...args_1], void 0, function* (blockstore, chunks, filename, totalSize, builders, { maxChunkSize = DEFAULT_MAX_CHUNK_SIZE, maxLinkPerNode = DEFAULT_MAX_LINK_PER_NODE, encryption = undefined, compression = undefined, } = {
-    maxChunkSize: DEFAULT_MAX_CHUNK_SIZE,
+export const processBufferToIPLDFormatFromChunks = (blockstore_1, chunks_1, filename_1, totalSize_1, builders_1, ...args_1) => __awaiter(void 0, [blockstore_1, chunks_1, filename_1, totalSize_1, builders_1, ...args_1], void 0, function* (blockstore, chunks, filename, totalSize, builders, { maxNodeSize: maxNodeSize = DEFAULT_MAX_CHUNK_SIZE, maxLinkPerNode = DEFAULT_MAX_LINK_PER_NODE, encryption = undefined, compression = undefined, } = {
+    maxNodeSize: DEFAULT_MAX_CHUNK_SIZE,
     maxLinkPerNode: DEFAULT_MAX_LINK_PER_NODE,
     encryption: undefined,
     compression: undefined,
 }) {
     var _a, chunks_2, chunks_2_1;
     var _b, e_2, _c, _d;
+    if (filename && filename.length > MAX_NAME_SIZE) {
+        throw new Error(`Filename is too long: ${filename.length} > ${MAX_NAME_SIZE}`);
+    }
     let chunkCount = 0;
     let CIDs = [];
     try {
@@ -137,7 +170,7 @@ export const processBufferToIPLDFormatFromChunks = (blockstore_1, chunks_1, file
         const newCIDs = [];
         for (let i = 0; i < CIDs.length; i += maxLinkPerNode) {
             const chunk = CIDs.slice(i, i + maxLinkPerNode);
-            const node = builders.inlink(chunk, chunk.length, depth, maxChunkSize);
+            const node = builders.inlink(chunk, chunk.length, depth, maxNodeSize);
             const cid = cidOfNode(node);
             yield blockstore.put(cid, encodeNode(node));
             newCIDs.push(cid);
@@ -145,7 +178,7 @@ export const processBufferToIPLDFormatFromChunks = (blockstore_1, chunks_1, file
         depth++;
         CIDs = newCIDs;
     }
-    const head = builders.root(CIDs, totalSize, depth, filename, maxChunkSize, {
+    const head = builders.root(CIDs, totalSize, depth, filename, maxNodeSize, {
         compression,
         encryption,
     });
@@ -153,12 +186,15 @@ export const processBufferToIPLDFormatFromChunks = (blockstore_1, chunks_1, file
     yield blockstore.put(headCID, encodeNode(head));
     return headCID;
 });
-export const processFolderToIPLDFormat = (blockstore_1, children_1, name_1, size_1, ...args_1) => __awaiter(void 0, [blockstore_1, children_1, name_1, size_1, ...args_1], void 0, function* (blockstore, children, name, size, { maxLinkPerNode = DEFAULT_MAX_LINK_PER_NODE, maxChunkSize = DEFAULT_MAX_CHUNK_SIZE, compression = undefined, encryption = undefined, } = {
+export const processFolderToIPLDFormat = (blockstore_1, children_1, name_1, size_1, ...args_1) => __awaiter(void 0, [blockstore_1, children_1, name_1, size_1, ...args_1], void 0, function* (blockstore, children, name, size, { maxLinkPerNode = DEFAULT_MAX_LINK_PER_NODE, maxNodeSize: maxNodeSize = DEFAULT_MAX_CHUNK_SIZE, compression = undefined, encryption = undefined, } = {
     maxLinkPerNode: DEFAULT_MAX_LINK_PER_NODE,
-    maxChunkSize: DEFAULT_MAX_CHUNK_SIZE,
+    maxNodeSize: DEFAULT_MAX_CHUNK_SIZE,
     compression: undefined,
     encryption: undefined,
 }) {
+    if (name.length > MAX_NAME_SIZE) {
+        throw new Error(`Filename is too long: ${name.length} > ${MAX_NAME_SIZE}`);
+    }
     let cids = children;
     let depth = 0;
     while (cids.length > maxLinkPerNode) {
@@ -173,7 +209,7 @@ export const processFolderToIPLDFormat = (blockstore_1, children_1, name_1, size
         cids = newCIDs;
         depth++;
     }
-    const node = createFolderIpldNode(cids, name, depth, size, maxChunkSize, {
+    const node = createFolderIpldNode(cids, name, depth, size, maxNodeSize, {
         compression,
         encryption,
     });
@@ -185,15 +221,18 @@ export const processFolderToIPLDFormat = (blockstore_1, children_1, name_1, size
  * Process chunks to IPLD format, return the last chunk if it's not full
  * @returns the last chunk if it's not full, otherwise an empty buffer
  */
-export const processChunksToIPLDFormat = (blockstore_1, chunks_1, builders_1, _a) => __awaiter(void 0, [blockstore_1, chunks_1, builders_1, _a], void 0, function* (blockstore, chunks, builders, { maxChunkSize = DEFAULT_MAX_CHUNK_SIZE }) {
+export const processChunksToIPLDFormat = (blockstore_1, chunks_1, builders_1, _a) => __awaiter(void 0, [blockstore_1, chunks_1, builders_1, _a], void 0, function* (blockstore, chunks, builders, { maxNodeSize = DEFAULT_MAX_CHUNK_SIZE }) {
     var _b, e_3, _c, _d;
-    const bufferChunks = chunkBuffer(chunks, { maxChunkSize, ignoreLastChunk: false });
+    const bufferChunks = chunkBuffer(chunks, {
+        maxChunkSize: maxNodeSize - NODE_METADATA_SIZE,
+        ignoreLastChunk: false,
+    });
     try {
         for (var _e = true, bufferChunks_2 = __asyncValues(bufferChunks), bufferChunks_2_1; bufferChunks_2_1 = yield bufferChunks_2.next(), _b = bufferChunks_2_1.done, !_b; _e = true) {
             _d = bufferChunks_2_1.value;
             _e = false;
             const chunk = _d;
-            if (chunk.byteLength < maxChunkSize) {
+            if (chunk.byteLength < maxNodeSize) {
                 return chunk;
             }
             const node = builders.chunk(chunk);

package/dist/ipld/nodes.d.ts

@@ -1,15 +1,15 @@
 import { CID } from 'multiformats/cid';
 import { FileUploadOptions, OffchainMetadata } from '../metadata/index.js';
 import { PBNode } from './index.js';
-export declare const createFileChunkIpldNode: (data: Buffer) => PBNode;
-export declare const createChunkedFileIpldNode: (links: CID[], size: number, linkDepth: number, name?: string, maxNodeSize?: number, uploadOptions?: FileUploadOptions) => PBNode;
+export declare const createFileChunkIpldNode: (data: Buffer, maxNodeSize?: number) => PBNode;
+export declare const createChunkedFileIpldNode: (links: CID[], size: bigint, linkDepth: number, name?: string, maxNodeSize?: number, uploadOptions?: FileUploadOptions) => PBNode;
 export declare const createFileInlinkIpldNode: (links: CID[], size: number, linkDepth: number, maxNodeSize?: number) => PBNode;
-export declare const createSingleFileIpldNode: (data: Buffer, name?: string, uploadOptions?: FileUploadOptions) => PBNode;
+export declare const createSingleFileIpldNode: (data: Buffer, name?: string, uploadOptions?: FileUploadOptions, maxNodeSize?: number) => PBNode;
 export declare const createMetadataInlinkIpldNode: (links: CID[], size: number, linkDepth: number, maxNodeSize?: number) => PBNode;
 export declare const createSingleMetadataIpldNode: (data: Buffer, name?: string) => PBNode;
-export declare const createMetadataChunkIpldNode: (data: Buffer) => PBNode;
-export declare const createChunkedMetadataIpldNode: (links: CID[], size: number, linkDepth: number, name?: string, maxNodeSize?: number) => PBNode;
-export declare const createFolderIpldNode: (links: CID[], name: string, linkDepth: number, size: number, maxNodeSize?: number, uploadOptions?: FileUploadOptions) => PBNode;
+export declare const createMetadataChunkIpldNode: (data: Buffer, maxNodeSize?: number) => PBNode;
+export declare const createChunkedMetadataIpldNode: (links: CID[], size: bigint, linkDepth: number, name?: string, maxNodeSize?: number) => PBNode;
+export declare const createFolderIpldNode: (links: CID[], name: string, linkDepth: number, size: bigint, maxNodeSize?: number, uploadOptions?: FileUploadOptions) => PBNode;
 export declare const createFolderInlinkIpldNode: (links: CID[], linkDepth: number, maxNodeSize?: number) => PBNode;
 export declare const createMetadataNode: (metadata: OffchainMetadata, maxNodeSize?: number) => PBNode;
 //# sourceMappingURL=nodes.d.ts.map

package/dist/ipld/nodes.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"nodes.d.ts","sourceRoot":"","sources":["../../src/ipld/nodes.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,MAAM,kBAAkB,CAAA;AACtC,OAAO,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,MAAM,sBAAsB,CAAA;AAG1E,OAAO,EAAc,MAAM,EAAE,MAAM,YAAY,CAAA;AAG/C,eAAO,MAAM,uBAAuB,SAAU,MAAM,KAAG,MASpD,CAAA;AAKH,eAAO,MAAM,yBAAyB,UAC7B,GAAG,EAAE,QACN,MAAM,aACD,MAAM,SACV,MAAM,gBACA,MAAM,kBACH,iBAAiB,KAChC,MAaA,CAAA;AAGH,eAAO,MAAM,wBAAwB,UAC5B,GAAG,EAAE,QACN,MAAM,aACD,MAAM,gBACJ,MAAM,KAClB,MAWA,CAAA;AAKH,eAAO,MAAM,wBAAwB,SAC7B,MAAM,SACL,MAAM,kBACG,iBAAiB,KAChC,MAWA,CAAA;AAKH,eAAO,MAAM,4BAA4B,UAChC,GAAG,EAAE,QACN,MAAM,aACD,MAAM,gBACJ,MAAM,KAClB,MAWA,CAAA;AAKH,eAAO,MAAM,4BAA4B,SAAU,MAAM,SAAS,MAAM,KAAG,MAUxE,CAAA;AAEH,eAAO,MAAM,2BAA2B,SAAU,MAAM,KAAG,MAQxD,CAAA;AAEH,eAAO,MAAM,6BAA6B,UACjC,GAAG,EAAE,QACN,MAAM,aACD,MAAM,SACV,MAAM,gBACA,MAAM,KAClB,MAYA,CAAA;AAKH,eAAO,MAAM,oBAAoB,UACxB,GAAG,EAAE,QACN,MAAM,aACD,MAAM,QACX,MAAM,gBACC,MAAM,kBACH,iBAAiB,KAChC,MAaA,CAAA;AAEH,eAAO,MAAM,0BAA0B,UAC9B,GAAG,EAAE,aACD,MAAM,gBACJ,MAAM,KAClB,MAUA,CAAA;AAGH,eAAO,MAAM,kBAAkB,aACnB,gBAAgB,gBACb,MAAM,KAClB,MAcF,CAAA"}
+{"version":3,"file":"nodes.d.ts","sourceRoot":"","sources":["../../src/ipld/nodes.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,MAAM,kBAAkB,CAAA;AACtC,OAAO,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,MAAM,sBAAsB,CAAA;AAI1E,OAAO,EAAc,MAAM,EAAE,MAAM,YAAY,CAAA;AAG/C,eAAO,MAAM,uBAAuB,SAC5B,MAAM,gBACC,MAAM,KAClB,MAYA,CAAA;AAKH,eAAO,MAAM,yBAAyB,UAC7B,GAAG,EAAE,QACN,MAAM,aACD,MAAM,SACV,MAAM,gBACA,MAAM,kBACH,iBAAiB,KAChC,MAaA,CAAA;AAGH,eAAO,MAAM,wBAAwB,UAC5B,GAAG,EAAE,QACN,MAAM,aACD,MAAM,gBACJ,MAAM,KAClB,MAWA,CAAA;AAKH,eAAO,MAAM,wBAAwB,SAC7B,MAAM,SACL,MAAM,kBACG,iBAAiB,gBACpB,MAAM,KAClB,MAcA,CAAA;AAKH,eAAO,MAAM,4BAA4B,UAChC,GAAG,EAAE,QACN,MAAM,aACD,MAAM,gBACJ,MAAM,KAClB,MAWA,CAAA;AAKH,eAAO,MAAM,4BAA4B,SAAU,MAAM,SAAS,MAAM,KAAG,MAUxE,CAAA;AAEH,eAAO,MAAM,2BAA2B,SAChC,MAAM,gBACC,MAAM,KAClB,MAWA,CAAA;AAEH,eAAO,MAAM,6BAA6B,UACjC,GAAG,EAAE,QACN,MAAM,aACD,MAAM,SACV,MAAM,gBACA,MAAM,KAClB,MAYA,CAAA;AAKH,eAAO,MAAM,oBAAoB,UACxB,GAAG,EAAE,QACN,MAAM,aACD,MAAM,QACX,MAAM,gBACC,MAAM,kBACH,iBAAiB,KAChC,MAaA,CAAA;AAEH,eAAO,MAAM,0BAA0B,UAC9B,GAAG,EAAE,aACD,MAAM,gBACJ,MAAM,KAClB,MAUA,CAAA;AAGH,eAAO,MAAM,kBAAkB,aACnB,gBAAgB,gBACb,MAAM,KAClB,MAcF,CAAA"}

package/dist/ipld/nodes.js

@@ -1,13 +1,14 @@
 import { encodeIPLDNodeData, MetadataType } from '../metadata/onchain/index.js';
+import { stringifyMetadata } from '../utils/metadata.js';
 import { DEFAULT_MAX_CHUNK_SIZE, ensureNodeMaxSize } from './chunker.js';
 import { createNode } from './index.js';
 /// Creates a file chunk ipld node
-export const createFileChunkIpldNode = (data) => createNode(encodeIPLDNodeData({
+export const createFileChunkIpldNode = (data, maxNodeSize = DEFAULT_MAX_CHUNK_SIZE) => ensureNodeMaxSize(createNode(encodeIPLDNodeData({
     type: MetadataType.FileChunk,
-    size: data.length,
+    size: BigInt(data.length).valueOf(),
     linkDepth: 0,
     data,
-}), []);
+}), []), maxNodeSize);
 // Creates a file ipld node
 // links: the CIDs of the file's contents
 // @todo: add the file's metadata
@@ -22,26 +23,26 @@ export const createChunkedFileIpldNode = (links, size, linkDepth, name, maxNodeS
 // links: the CIDs of the file's contents
 export const createFileInlinkIpldNode = (links, size, linkDepth, maxNodeSize = DEFAULT_MAX_CHUNK_SIZE) => ensureNodeMaxSize(createNode(encodeIPLDNodeData({
     type: MetadataType.FileInlink,
-    size,
+    size: BigInt(size).valueOf(),
     linkDepth,
 }), links.map((cid) => ({ Hash: cid }))), maxNodeSize);
 // Creates a file ipld node
 // links: the CIDs of the file's contents
 // @todo: add the file's metadata
-export const createSingleFileIpldNode = (data, name, uploadOptions) => createNode(encodeIPLDNodeData({
+export const createSingleFileIpldNode = (data, name, uploadOptions, maxNodeSize = DEFAULT_MAX_CHUNK_SIZE) => ensureNodeMaxSize(createNode(encodeIPLDNodeData({
     type: MetadataType.File,
     name,
-    size: data.length,
+    size: BigInt(data.length).valueOf(),
     linkDepth: 0,
     data,
     uploadOptions,
-}), []);
+}), []), maxNodeSize);
 // Creates a file ipld node
 // links: the CIDs of the file's contents
 // @todo: add the file's metadata
 export const createMetadataInlinkIpldNode = (links, size, linkDepth, maxNodeSize = DEFAULT_MAX_CHUNK_SIZE) => ensureNodeMaxSize(createNode(encodeIPLDNodeData({
     type: MetadataType.FileInlink,
-    size,
+    size: BigInt(size).valueOf(),
     linkDepth,
 }), links.map((cid) => ({ Hash: cid }))), maxNodeSize);
 // Creates a file ipld node
@@ -50,16 +51,16 @@ export const createMetadataInlinkIpldNode = (links, size, linkDepth, maxNodeSize
 export const createSingleMetadataIpldNode = (data, name) => createNode(encodeIPLDNodeData({
     type: MetadataType.Metadata,
     name,
-    size: data.length,
+    size: BigInt(data.length).valueOf(),
     linkDepth: 0,
     data,
 }), []);
-export const createMetadataChunkIpldNode = (data) => createNode(encodeIPLDNodeData({
+export const createMetadataChunkIpldNode = (data, maxNodeSize = DEFAULT_MAX_CHUNK_SIZE) => ensureNodeMaxSize(createNode(encodeIPLDNodeData({
     type: MetadataType.MetadataChunk,
-    size: data.length,
+    size: BigInt(data.length).valueOf(),
     linkDepth: 0,
     data,
-}));
+})), maxNodeSize);
 export const createChunkedMetadataIpldNode = (links, size, linkDepth, name, maxNodeSize = DEFAULT_MAX_CHUNK_SIZE) => ensureNodeMaxSize(createNode(encodeIPLDNodeData({
     type: MetadataType.Metadata,
     name,
@@ -82,7 +83,7 @@ export const createFolderInlinkIpldNode = (links, linkDepth, maxNodeSize = DEFAU
 }), links.map((cid) => ({ Hash: cid }))), maxNodeSize);
 /// Creates a metadata ipld node
 export const createMetadataNode = (metadata, maxNodeSize = DEFAULT_MAX_CHUNK_SIZE) => {
-    const data = Buffer.from(JSON.stringify(metadata));
+    const data = Buffer.from(stringifyMetadata(metadata));
     return ensureNodeMaxSize(createNode(encodeIPLDNodeData({
         type: MetadataType.Metadata,
         name: metadata.name,

package/dist/metadata/offchain/file.d.ts

@@ -5,14 +5,14 @@ export type OffchainFileMetadata = {
     dataCid: string;
     name?: string;
     mimeType?: string;
-    totalSize: number;
+    totalSize: bigint;
     totalChunks: number;
     chunks: ChunkInfo[];
     uploadOptions?: FileUploadOptions;
 };
 export interface ChunkInfo {
-    size: number;
+    size: bigint;
     cid: string;
 }
-export declare const fileMetadata: (headCID: CID, chunks: ChunkInfo[], totalSize: number, name?: string | null, mimeType?: string | null, uploadOptions?: FileUploadOptions) => OffchainFileMetadata;
+export declare const fileMetadata: (headCID: CID, chunks: ChunkInfo[], totalSize: bigint, name?: string | null, mimeType?: string | null, uploadOptions?: FileUploadOptions) => OffchainFileMetadata;
 //# sourceMappingURL=file.d.ts.map

package/dist/metadata/offchain/folder.d.ts

@@ -5,13 +5,13 @@ interface ChildrenMetadata {
     type: 'folder' | 'file';
     name?: string;
     cid: string;
-    totalSize: number;
+    totalSize: bigint;
 }
 export type OffchainFolderMetadata = {
     type: 'folder';
     dataCid: string;
     name?: string;
-    totalSize: number;
+    totalSize: bigint;
     totalFiles: number;
     children: ChildrenMetadata[];
     uploadOptions: FileUploadOptions;

package/dist/metadata/offchain/folder.js

@@ -9,7 +9,7 @@ export const childrenMetadataFromNode = (node) => {
     return {
         type: ipldData.type === MetadataType.File ? 'file' : 'folder',
         cid: cidToString(cidOfNode(node)),
-        totalSize: (_a = ipldData.size) !== null && _a !== void 0 ? _a : 0,
+        totalSize: (_a = ipldData.size) !== null && _a !== void 0 ? _a : BigInt(0).valueOf(),
         name: ipldData.name,
     };
 };
@@ -17,7 +17,7 @@ export const folderMetadata = (cid, children, name, uploadOptions = {}) => {
     cid = typeof cid === 'string' ? cid : cidToString(cid);
     return {
         dataCid: cid,
-        totalSize: children.reduce((acc, child) => acc + child.totalSize, 0),
+        totalSize: children.reduce((acc, child) => acc + child.totalSize, BigInt(0).valueOf()),
         totalFiles: children.length,
         children,
         type: 'folder',

package/dist/metadata/onchain/protobuf/OnchainMetadata.d.ts

@@ -3,7 +3,7 @@ import type { Uint8ArrayList } from 'uint8arraylist';
 export interface IPLDNodeData {
     type: MetadataType;
     linkDepth: number;
-    size?: number;
+    size?: bigint;
     name?: string;
     data?: Uint8Array;
     uploadOptions?: FileUploadOptions;

package/dist/metadata/onchain/protobuf/OnchainMetadata.js

@@ -23,7 +23,7 @@ export var IPLDNodeData;
                 }
                 if (obj.size != null) {
                     w.uint32(24);
-                    w.int32(obj.size);
+                    w.int64(obj.size);
                 }
                 if (obj.name != null) {
                     w.uint32(34);
@@ -59,7 +59,7 @@ export var IPLDNodeData;
                             break;
                         }
                         case 3: {
-                            obj.size = reader.int32();
+                            obj.size = reader.int64();
                             break;
                         }
                         case 4: {

package/dist/utils/metadata.d.ts

@@ -0,0 +1,3 @@
+import { OffchainMetadata } from '../metadata/index.js';
+export declare const stringifyMetadata: (metadata: OffchainMetadata) => string;
+//# sourceMappingURL=metadata.d.ts.map

package/dist/utils/metadata.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"metadata.d.ts","sourceRoot":"","sources":["../../src/utils/metadata.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,EAAE,MAAM,sBAAsB,CAAA;AAEvD,eAAO,MAAM,iBAAiB,aAAc,gBAAgB,KAAG,MAG5D,CAAA"}

package/dist/utils/metadata.js

@@ -0,0 +1 @@
+export const stringifyMetadata = (metadata) => JSON.stringify(metadata, (_, v) => typeof v === 'bigint' || v instanceof BigInt ? v.toString() : v);

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@autonomys/auto-dag-data",
   "packageManager": "yarn@4.1.1",
-  "version": "1.0.6",
+  "version": "1.0.7",
   "license": "MIT",
   "main": "dist/index.js",
   "repository": {
@@ -38,6 +38,7 @@
   },
   "dependencies": {
     "@ipld/dag-pb": "^4.1.2",
+    "@peculiar/webcrypto": "^1.5.0",
     "@webbuf/blake3": "^3.0.26",
     "@webbuf/fixedbuf": "^3.0.26",
     "@webbuf/webbuf": "^3.0.26",

package/src/ipld/builders.ts

@@ -14,10 +14,10 @@ import {
 
 export interface Builders {
   inlink: (links: CID[], size: number, linkDepth: number, chunkSize: number) => PBNode
-  chunk: (data: Buffer) => PBNode
+  chunk: (data: Buffer, maxNodeSize?: number) => PBNode
   root: (
     links: CID[],
-    size: number,
+    size: bigint,
     linkDepth: number,
     name?: string,
     maxNodeSize?: number,