@net-protocol/storage 0.1.8 → 0.1.9

package/dist/index.mjs

@@ -2202,7 +2202,130 @@ function base64ToDataUri(base64Data) {
   const mimeType = detectFileTypeFromBase64(base64Data) || "application/octet-stream";
   return `data:${mimeType};base64,${base64Data}`;
 }
+var STREAMING_CHUNK_SIZE = 80 * 1e3;
+var BINARY_CHUNK_SIZE = 79998;
+function isBinaryFile(file) {
+  const mimeType = file.type.toLowerCase();
+  const textTypes = [
+    "text/",
+    "application/json",
+    "application/xml",
+    "application/javascript",
+    "application/typescript",
+    "application/x-javascript",
+    "application/ecmascript"
+  ];
+  for (const textType of textTypes) {
+    if (mimeType.startsWith(textType)) {
+      return false;
+    }
+  }
+  if (!mimeType || mimeType === "application/octet-stream") {
+    const extension = file.name.split(".").pop()?.toLowerCase() || "";
+    const textExtensions = [
+      "txt",
+      "md",
+      "json",
+      "xml",
+      "html",
+      "htm",
+      "css",
+      "js",
+      "ts",
+      "jsx",
+      "tsx",
+      "yaml",
+      "yml",
+      "toml",
+      "ini",
+      "cfg",
+      "conf",
+      "log",
+      "csv",
+      "svg"
+    ];
+    return !textExtensions.includes(extension);
+  }
+  return true;
+}
+async function readFileSlice(file, offset, size, isBinary, isFirstChunk) {
+  const blob = file.slice(offset, offset + size);
+  if (isBinary) {
+    const arrayBuffer = await blob.arrayBuffer();
+    const bytes = new Uint8Array(arrayBuffer);
+    let base64 = "";
+    const chunkSize = 32766;
+    for (let i = 0; i < bytes.length; i += chunkSize) {
+      const chunk = bytes.slice(i, i + chunkSize);
+      base64 += btoa(String.fromCharCode(...chunk));
+    }
+    if (isFirstChunk) {
+      const mimeType = detectFileTypeFromBase64(base64) || file.type || "application/octet-stream";
+      return `data:${mimeType};base64,${base64}`;
+    }
+    return base64;
+  } else {
+    return await blob.text();
+  }
+}
+async function* processFileStreaming(file, chunkSize = STREAMING_CHUNK_SIZE) {
+  const binary = isBinaryFile(file);
+  const effectiveChunkSize = binary ? BINARY_CHUNK_SIZE : chunkSize;
+  let offset = 0;
+  let chunkIndex = 0;
+  while (offset < file.size) {
+    const chunkString = await readFileSlice(
+      file,
+      offset,
+      effectiveChunkSize,
+      binary,
+      chunkIndex === 0
+    );
+    const hash = keccak256HashString(chunkString);
+    const compressedChunks = chunkDataForStorage(chunkString);
+    yield {
+      chunkIndex,
+      hash,
+      compressedChunks
+    };
+    offset += effectiveChunkSize;
+    chunkIndex++;
+  }
+  if (chunkIndex === 0) {
+    const emptyString = binary ? `data:${file.type || "application/octet-stream"};base64,` : "";
+    const hash = keccak256HashString(emptyString);
+    const compressedChunks = chunkDataForStorage(emptyString);
+    yield {
+      chunkIndex: 0,
+      hash,
+      compressedChunks
+    };
+  }
+}
+async function processFileStreamingComplete(file, onProgress) {
+  const totalChunks = Math.max(1, Math.ceil(file.size / STREAMING_CHUNK_SIZE));
+  const hashes = [];
+  const allCompressedChunks = [];
+  const binary = isBinaryFile(file);
+  let processed = 0;
+  for await (const result of processFileStreaming(file)) {
+    hashes.push(result.hash);
+    allCompressedChunks.push(result.compressedChunks);
+    processed++;
+    onProgress?.(processed, totalChunks);
+  }
+  return {
+    hashes,
+    allCompressedChunks,
+    totalChunks: hashes.length,
+    isBinary: binary
+  };
+}
+function estimateChunkCount(fileSize, isBinary = true) {
+  const chunkSize = isBinary ? BINARY_CHUNK_SIZE : STREAMING_CHUNK_SIZE;
+  return Math.max(1, Math.ceil(fileSize / chunkSize));
+}
 
-export { CHUNKED_STORAGE_CONTRACT, CHUNKED_STORAGE_READER_CONTRACT, CONCURRENT_XML_FETCHES, MAX_XML_DEPTH, SAFE_STORAGE_READER_CONTRACT, STORAGE_CONTRACT, STORAGE_ROUTER_CONTRACT, StorageClient, assembleChunks, base64ToDataUri, chunkData, chunkDataForStorage, computeTopLevelHash, containsXmlReferences, detectFileTypeFromBase64, detectStorageType, encodeStorageKeyForUrl, fileToDataUri, formatStorageKeyForDisplay, generateStorageEmbedTag, generateXmlMetadata, generateXmlMetadataWithSource, getChunkCount, getReferenceKey, getStorageKeyBytes, parseNetReferences, processDataForStorage, resolveOperator, resolveXmlRecursive, shouldSuggestXmlStorage, useBulkStorage, useStorage, useStorageForOperator, useStorageForOperatorAndKey, useStorageFromRouter, useStorageTotalWrites, useXmlStorage, validateDataSize };
+export { CHUNKED_STORAGE_CONTRACT, CHUNKED_STORAGE_READER_CONTRACT, CONCURRENT_XML_FETCHES, MAX_XML_DEPTH, SAFE_STORAGE_READER_CONTRACT, STORAGE_CONTRACT, STORAGE_ROUTER_CONTRACT, StorageClient, assembleChunks, base64ToDataUri, chunkData, chunkDataForStorage, computeTopLevelHash, containsXmlReferences, detectFileTypeFromBase64, detectStorageType, encodeStorageKeyForUrl, estimateChunkCount, fileToDataUri, formatStorageKeyForDisplay, generateStorageEmbedTag, generateXmlMetadata, generateXmlMetadataWithSource, getChunkCount, getReferenceKey, getStorageKeyBytes, isBinaryFile, parseNetReferences, processDataForStorage, processFileStreaming, processFileStreamingComplete, readFileSlice, resolveOperator, resolveXmlRecursive, shouldSuggestXmlStorage, useBulkStorage, useStorage, useStorageForOperator, useStorageForOperatorAndKey, useStorageFromRouter, useStorageTotalWrites, useXmlStorage, validateDataSize };
 //# sourceMappingURL=index.mjs.map
 //# sourceMappingURL=index.mjs.map