@net-protocol/storage 0.1.8 → 0.1.9

package/dist/index.d.mts

@@ -535,6 +535,99 @@ declare function detectFileTypeFromBase64(base64Data: string): string | undefine
  */
 declare function base64ToDataUri(base64Data: string): string;
 
+/**
+ * Streaming utilities for processing large files without loading them entirely into memory.
+ * Uses file.slice() to read in chunks, keeping memory usage low.
+ */
+/**
+ * Result from processing a single file chunk
+ */
+interface StreamingChunkResult {
+    /** Index of this chunk (0-based) */
+    chunkIndex: number;
+    /** Keccak256 hash of the chunk content (66 chars) */
+    hash: string;
+    /** Compressed chunks ready for ChunkedStorage (array of hex strings) */
+    compressedChunks: string[];
+}
+/**
+ * Result from processing an entire file via streaming
+ */
+interface StreamingProcessResult {
+    /** All chunk hashes in order */
+    hashes: string[];
+    /** All compressed chunk arrays in order */
+    allCompressedChunks: string[][];
+    /** Total number of chunks */
+    totalChunks: number;
+    /** Whether the file was treated as binary */
+    isBinary: boolean;
+}
+/**
+ * Detects if a file should be treated as binary based on its MIME type.
+ * Binary files will be base64 encoded; text files will be read as text.
+ *
+ * @param file - The file to check
+ * @returns true if the file should be treated as binary
+ */
+declare function isBinaryFile(file: File): boolean;
+/**
+ * Reads a slice of a file as a string.
+ * For binary files, returns base64-encoded data.
+ * For text files, returns the raw text.
+ *
+ * @param file - The file to read from
+ * @param offset - Start byte offset
+ * @param size - Number of bytes to read
+ * @param isBinary - Whether to read as binary (base64) or text
+ * @param isFirstChunk - Whether this is the first chunk (for data URI prefix)
+ * @returns The chunk as a string
+ */
+declare function readFileSlice(file: File, offset: number, size: number, isBinary: boolean, isFirstChunk: boolean): Promise<string>;
+/**
+ * Async generator that processes a file in streaming chunks.
+ * Reads the file in 80KB slices, hashes and compresses each,
+ * yielding results one at a time to keep memory usage low.
+ *
+ * @param file - The file to process
+ * @param chunkSize - Size of each chunk in bytes (default: 80KB)
+ * @yields StreamingChunkResult for each chunk processed
+ *
+ * @example
+ * ```typescript
+ * const hashes: string[] = [];
+ * const transactions: TransactionConfig[] = [];
+ *
+ * for await (const result of processFileStreaming(file)) {
+ *   hashes.push(result.hash);
+ *   transactions.push(createTransaction(result.compressedChunks));
+ * }
+ *
+ * const metadata = generateXmlMetadata(hashes, 0, operatorAddress);
+ * ```
+ */
+declare function processFileStreaming(file: File, chunkSize?: number): AsyncGenerator<StreamingChunkResult>;
+/**
+ * Processes an entire file via streaming and returns all results.
+ * This is a convenience function that collects all generator results.
+ *
+ * For very large files, prefer using processFileStreaming directly
+ * to process chunks as they're generated.
+ *
+ * @param file - The file to process
+ * @param onProgress - Optional callback for progress updates
+ * @returns All hashes and compressed chunks
+ */
+declare function processFileStreamingComplete(file: File, onProgress?: (current: number, total: number) => void): Promise<StreamingProcessResult>;
+/**
+ * Estimates the number of chunks for a file without reading it.
+ *
+ * @param fileSize - Size of the file in bytes
+ * @param isBinary - Whether the file is binary (uses smaller chunk size for base64 alignment)
+ * @returns Estimated number of chunks
+ */
+declare function estimateChunkCount(fileSize: number, isBinary?: boolean): number;
+
 declare const STORAGE_CONTRACT: {
     abi: Abi;
     address: `0x${string}`;
@@ -570,4 +663,4 @@ declare const CONCURRENT_XML_FETCHES = 3;
  */
 declare function resolveXmlRecursive(content: string, defaultOperator: string, client: PublicClient, maxDepth: number, visited?: Set<string>, inheritedOperator?: string): Promise<string>;
 
-export { type BulkStorageKey, CHUNKED_STORAGE_CONTRACT, CHUNKED_STORAGE_READER_CONTRACT, CONCURRENT_XML_FETCHES, type ChunkedMetadata, MAX_XML_DEPTH, SAFE_STORAGE_READER_CONTRACT, STORAGE_CONTRACT, STORAGE_ROUTER_CONTRACT, StorageClient, type StorageClientOptions, type StorageData, type UseStorageOptions, type UseXmlStorageOptions, type XmlReference$1 as XmlReference, assembleChunks, base64ToDataUri, chunkData, chunkDataForStorage, computeTopLevelHash, containsXmlReferences, detectFileTypeFromBase64, detectStorageType, encodeStorageKeyForUrl, fileToDataUri, formatStorageKeyForDisplay, generateStorageEmbedTag, generateXmlMetadata, generateXmlMetadataWithSource, getChunkCount, getReferenceKey, getStorageKeyBytes, parseNetReferences, processDataForStorage, resolveOperator, resolveXmlRecursive, shouldSuggestXmlStorage, useBulkStorage, useStorage, useStorageForOperator, useStorageForOperatorAndKey, useStorageFromRouter, useStorageTotalWrites, useXmlStorage, validateDataSize };
+export { type BulkStorageKey, CHUNKED_STORAGE_CONTRACT, CHUNKED_STORAGE_READER_CONTRACT, CONCURRENT_XML_FETCHES, type ChunkedMetadata, MAX_XML_DEPTH, SAFE_STORAGE_READER_CONTRACT, STORAGE_CONTRACT, STORAGE_ROUTER_CONTRACT, StorageClient, type StorageClientOptions, type StorageData, type StreamingChunkResult, type StreamingProcessResult, type UseStorageOptions, type UseXmlStorageOptions, type XmlReference$1 as XmlReference, assembleChunks, base64ToDataUri, chunkData, chunkDataForStorage, computeTopLevelHash, containsXmlReferences, detectFileTypeFromBase64, detectStorageType, encodeStorageKeyForUrl, estimateChunkCount, fileToDataUri, formatStorageKeyForDisplay, generateStorageEmbedTag, generateXmlMetadata, generateXmlMetadataWithSource, getChunkCount, getReferenceKey, getStorageKeyBytes, isBinaryFile, parseNetReferences, processDataForStorage, processFileStreaming, processFileStreamingComplete, readFileSlice, resolveOperator, resolveXmlRecursive, shouldSuggestXmlStorage, useBulkStorage, useStorage, useStorageForOperator, useStorageForOperatorAndKey, useStorageFromRouter, useStorageTotalWrites, useXmlStorage, validateDataSize };

package/dist/index.d.ts

@@ -535,6 +535,99 @@ declare function detectFileTypeFromBase64(base64Data: string): string | undefine
  */
 declare function base64ToDataUri(base64Data: string): string;
 
+/**
+ * Streaming utilities for processing large files without loading them entirely into memory.
+ * Uses file.slice() to read in chunks, keeping memory usage low.
+ */
+/**
+ * Result from processing a single file chunk
+ */
+interface StreamingChunkResult {
+    /** Index of this chunk (0-based) */
+    chunkIndex: number;
+    /** Keccak256 hash of the chunk content (66 chars) */
+    hash: string;
+    /** Compressed chunks ready for ChunkedStorage (array of hex strings) */
+    compressedChunks: string[];
+}
+/**
+ * Result from processing an entire file via streaming
+ */
+interface StreamingProcessResult {
+    /** All chunk hashes in order */
+    hashes: string[];
+    /** All compressed chunk arrays in order */
+    allCompressedChunks: string[][];
+    /** Total number of chunks */
+    totalChunks: number;
+    /** Whether the file was treated as binary */
+    isBinary: boolean;
+}
+/**
+ * Detects if a file should be treated as binary based on its MIME type.
+ * Binary files will be base64 encoded; text files will be read as text.
+ *
+ * @param file - The file to check
+ * @returns true if the file should be treated as binary
+ */
+declare function isBinaryFile(file: File): boolean;
+/**
+ * Reads a slice of a file as a string.
+ * For binary files, returns base64-encoded data.
+ * For text files, returns the raw text.
+ *
+ * @param file - The file to read from
+ * @param offset - Start byte offset
+ * @param size - Number of bytes to read
+ * @param isBinary - Whether to read as binary (base64) or text
+ * @param isFirstChunk - Whether this is the first chunk (for data URI prefix)
+ * @returns The chunk as a string
+ */
+declare function readFileSlice(file: File, offset: number, size: number, isBinary: boolean, isFirstChunk: boolean): Promise<string>;
+/**
+ * Async generator that processes a file in streaming chunks.
+ * Reads the file in 80KB slices, hashes and compresses each,
+ * yielding results one at a time to keep memory usage low.
+ *
+ * @param file - The file to process
+ * @param chunkSize - Size of each chunk in bytes (default: 80KB)
+ * @yields StreamingChunkResult for each chunk processed
+ *
+ * @example
+ * ```typescript
+ * const hashes: string[] = [];
+ * const transactions: TransactionConfig[] = [];
+ *
+ * for await (const result of processFileStreaming(file)) {
+ *   hashes.push(result.hash);
+ *   transactions.push(createTransaction(result.compressedChunks));
+ * }
+ *
+ * const metadata = generateXmlMetadata(hashes, 0, operatorAddress);
+ * ```
+ */
+declare function processFileStreaming(file: File, chunkSize?: number): AsyncGenerator<StreamingChunkResult>;
+/**
+ * Processes an entire file via streaming and returns all results.
+ * This is a convenience function that collects all generator results.
+ *
+ * For very large files, prefer using processFileStreaming directly
+ * to process chunks as they're generated.
+ *
+ * @param file - The file to process
+ * @param onProgress - Optional callback for progress updates
+ * @returns All hashes and compressed chunks
+ */
+declare function processFileStreamingComplete(file: File, onProgress?: (current: number, total: number) => void): Promise<StreamingProcessResult>;
+/**
+ * Estimates the number of chunks for a file without reading it.
+ *
+ * @param fileSize - Size of the file in bytes
+ * @param isBinary - Whether the file is binary (uses smaller chunk size for base64 alignment)
+ * @returns Estimated number of chunks
+ */
+declare function estimateChunkCount(fileSize: number, isBinary?: boolean): number;
+
 declare const STORAGE_CONTRACT: {
     abi: Abi;
     address: `0x${string}`;
@@ -570,4 +663,4 @@ declare const CONCURRENT_XML_FETCHES = 3;
  */
 declare function resolveXmlRecursive(content: string, defaultOperator: string, client: PublicClient, maxDepth: number, visited?: Set<string>, inheritedOperator?: string): Promise<string>;
 
-export { type BulkStorageKey, CHUNKED_STORAGE_CONTRACT, CHUNKED_STORAGE_READER_CONTRACT, CONCURRENT_XML_FETCHES, type ChunkedMetadata, MAX_XML_DEPTH, SAFE_STORAGE_READER_CONTRACT, STORAGE_CONTRACT, STORAGE_ROUTER_CONTRACT, StorageClient, type StorageClientOptions, type StorageData, type UseStorageOptions, type UseXmlStorageOptions, type XmlReference$1 as XmlReference, assembleChunks, base64ToDataUri, chunkData, chunkDataForStorage, computeTopLevelHash, containsXmlReferences, detectFileTypeFromBase64, detectStorageType, encodeStorageKeyForUrl, fileToDataUri, formatStorageKeyForDisplay, generateStorageEmbedTag, generateXmlMetadata, generateXmlMetadataWithSource, getChunkCount, getReferenceKey, getStorageKeyBytes, parseNetReferences, processDataForStorage, resolveOperator, resolveXmlRecursive, shouldSuggestXmlStorage, useBulkStorage, useStorage, useStorageForOperator, useStorageForOperatorAndKey, useStorageFromRouter, useStorageTotalWrites, useXmlStorage, validateDataSize };
+export { type BulkStorageKey, CHUNKED_STORAGE_CONTRACT, CHUNKED_STORAGE_READER_CONTRACT, CONCURRENT_XML_FETCHES, type ChunkedMetadata, MAX_XML_DEPTH, SAFE_STORAGE_READER_CONTRACT, STORAGE_CONTRACT, STORAGE_ROUTER_CONTRACT, StorageClient, type StorageClientOptions, type StorageData, type StreamingChunkResult, type StreamingProcessResult, type UseStorageOptions, type UseXmlStorageOptions, type XmlReference$1 as XmlReference, assembleChunks, base64ToDataUri, chunkData, chunkDataForStorage, computeTopLevelHash, containsXmlReferences, detectFileTypeFromBase64, detectStorageType, encodeStorageKeyForUrl, estimateChunkCount, fileToDataUri, formatStorageKeyForDisplay, generateStorageEmbedTag, generateXmlMetadata, generateXmlMetadataWithSource, getChunkCount, getReferenceKey, getStorageKeyBytes, isBinaryFile, parseNetReferences, processDataForStorage, processFileStreaming, processFileStreamingComplete, readFileSlice, resolveOperator, resolveXmlRecursive, shouldSuggestXmlStorage, useBulkStorage, useStorage, useStorageForOperator, useStorageForOperatorAndKey, useStorageFromRouter, useStorageTotalWrites, useXmlStorage, validateDataSize };

package/dist/index.js

@@ -2209,6 +2209,129 @@ function base64ToDataUri(base64Data) {
   const mimeType = detectFileTypeFromBase64(base64Data) || "application/octet-stream";
   return `data:${mimeType};base64,${base64Data}`;
 }
+var STREAMING_CHUNK_SIZE = 80 * 1e3;
+var BINARY_CHUNK_SIZE = 79998;
+function isBinaryFile(file) {
+  const mimeType = file.type.toLowerCase();
+  const textTypes = [
+    "text/",
+    "application/json",
+    "application/xml",
+    "application/javascript",
+    "application/typescript",
+    "application/x-javascript",
+    "application/ecmascript"
+  ];
+  for (const textType of textTypes) {
+    if (mimeType.startsWith(textType)) {
+      return false;
+    }
+  }
+  if (!mimeType || mimeType === "application/octet-stream") {
+    const extension = file.name.split(".").pop()?.toLowerCase() || "";
+    const textExtensions = [
+      "txt",
+      "md",
+      "json",
+      "xml",
+      "html",
+      "htm",
+      "css",
+      "js",
+      "ts",
+      "jsx",
+      "tsx",
+      "yaml",
+      "yml",
+      "toml",
+      "ini",
+      "cfg",
+      "conf",
+      "log",
+      "csv",
+      "svg"
+    ];
+    return !textExtensions.includes(extension);
+  }
+  return true;
+}
+async function readFileSlice(file, offset, size, isBinary, isFirstChunk) {
+  const blob = file.slice(offset, offset + size);
+  if (isBinary) {
+    const arrayBuffer = await blob.arrayBuffer();
+    const bytes = new Uint8Array(arrayBuffer);
+    let base64 = "";
+    const chunkSize = 32766;
+    for (let i = 0; i < bytes.length; i += chunkSize) {
+      const chunk = bytes.slice(i, i + chunkSize);
+      base64 += btoa(String.fromCharCode(...chunk));
+    }
+    if (isFirstChunk) {
+      const mimeType = detectFileTypeFromBase64(base64) || file.type || "application/octet-stream";
+      return `data:${mimeType};base64,${base64}`;
+    }
+    return base64;
+  } else {
+    return await blob.text();
+  }
+}
+async function* processFileStreaming(file, chunkSize = STREAMING_CHUNK_SIZE) {
+  const binary = isBinaryFile(file);
+  const effectiveChunkSize = binary ? BINARY_CHUNK_SIZE : chunkSize;
+  let offset = 0;
+  let chunkIndex = 0;
+  while (offset < file.size) {
+    const chunkString = await readFileSlice(
+      file,
+      offset,
+      effectiveChunkSize,
+      binary,
+      chunkIndex === 0
+    );
+    const hash = core.keccak256HashString(chunkString);
+    const compressedChunks = chunkDataForStorage(chunkString);
+    yield {
+      chunkIndex,
+      hash,
+      compressedChunks
+    };
+    offset += effectiveChunkSize;
+    chunkIndex++;
+  }
+  if (chunkIndex === 0) {
+    const emptyString = binary ? `data:${file.type || "application/octet-stream"};base64,` : "";
+    const hash = core.keccak256HashString(emptyString);
+    const compressedChunks = chunkDataForStorage(emptyString);
+    yield {
+      chunkIndex: 0,
+      hash,
+      compressedChunks
+    };
+  }
+}
+async function processFileStreamingComplete(file, onProgress) {
+  const totalChunks = Math.max(1, Math.ceil(file.size / STREAMING_CHUNK_SIZE));
+  const hashes = [];
+  const allCompressedChunks = [];
+  const binary = isBinaryFile(file);
+  let processed = 0;
+  for await (const result of processFileStreaming(file)) {
+    hashes.push(result.hash);
+    allCompressedChunks.push(result.compressedChunks);
+    processed++;
+    onProgress?.(processed, totalChunks);
+  }
+  return {
+    hashes,
+    allCompressedChunks,
+    totalChunks: hashes.length,
+    isBinary: binary
+  };
+}
+function estimateChunkCount(fileSize, isBinary = true) {
+  const chunkSize = isBinary ? BINARY_CHUNK_SIZE : STREAMING_CHUNK_SIZE;
+  return Math.max(1, Math.ceil(fileSize / chunkSize));
+}
 
 exports.CHUNKED_STORAGE_CONTRACT = CHUNKED_STORAGE_CONTRACT;
 exports.CHUNKED_STORAGE_READER_CONTRACT = CHUNKED_STORAGE_READER_CONTRACT;
@@ -2227,6 +2350,7 @@ exports.containsXmlReferences = containsXmlReferences;
 exports.detectFileTypeFromBase64 = detectFileTypeFromBase64;
 exports.detectStorageType = detectStorageType;
 exports.encodeStorageKeyForUrl = encodeStorageKeyForUrl;
+exports.estimateChunkCount = estimateChunkCount;
 exports.fileToDataUri = fileToDataUri;
 exports.formatStorageKeyForDisplay = formatStorageKeyForDisplay;
 exports.generateStorageEmbedTag = generateStorageEmbedTag;
@@ -2235,8 +2359,12 @@ exports.generateXmlMetadataWithSource = generateXmlMetadataWithSource;
 exports.getChunkCount = getChunkCount;
 exports.getReferenceKey = getReferenceKey;
 exports.getStorageKeyBytes = getStorageKeyBytes;
+exports.isBinaryFile = isBinaryFile;
 exports.parseNetReferences = parseNetReferences;
 exports.processDataForStorage = processDataForStorage;
+exports.processFileStreaming = processFileStreaming;
+exports.processFileStreamingComplete = processFileStreamingComplete;
+exports.readFileSlice = readFileSlice;
 exports.resolveOperator = resolveOperator;
 exports.resolveXmlRecursive = resolveXmlRecursive;
 exports.shouldSuggestXmlStorage = shouldSuggestXmlStorage;