@muhkoo/connect 0.10.9-alpha.0 → 0.10.10-alpha.0

package/dist/connect.d.ts

@@ -1706,6 +1706,14 @@ declare class FileStorage {
         data: Uint8Array;
         stat: FileStat;
     }>;
+    /**
+     * Streaming inverse of {@link writeFileToShards}: yields each chunk's
+     * decrypted plaintext in order WITHOUT concatenating. Peak memory stays at
+     * ~one chunk, so multi-GB files (which overflow a single `Uint8Array`) can be
+     * piped straight to a file/socket. Same shard sourcing (cache → peers →
+     * origin) and per-chunk auth as {@link readFileFromShards}.
+     */
+    readChunksFromShards(manifest: FileManifest): AsyncGenerator<Uint8Array>;
     /** As {@link writeFileToShards}, plus POSTs the manifest to `input.spaceId`. */
     writeFile(input: WriteFileInput): Promise<FileStat>;
     /** As {@link readFileFromShards}, but fetches the manifest from a space first. */
@@ -4146,6 +4154,11 @@ declare class Space {
         data: Uint8Array;
         stat: FileStat;
     }>;
+    /** Streaming read: async-iterate a stored file's decrypted chunks in order,
+     *  never holding the whole file in memory. Use for multi-GB files (a single
+     *  `Uint8Array` can't hold them) — e.g. a transcoder piping the raw to a temp
+     *  file. Same peer/origin shard sourcing as {@link getFile}. */
+    getFileStream(manifest: FileManifest): AsyncGenerator<Uint8Array>;
     /** Declare a file's shards (ref++ + meter) for a manifest already stored —
      *  used to back-fill accounting for files written before commit existed, or to
      *  register a manifest received via sharing. `putFile` already commits new

package/dist/server/index.js

@@ -2976,7 +2976,7 @@ function base64UrlDecode(s) {
  * expect (and isn't serving a stale cached bundle). Keep this in sync with
  * `package.json` `version`.
  */
-const VERSION = "0.10.9-alpha.0";
+const VERSION = "0.10.10-alpha.0";
 
 /**
  * `HttpClient` — the one place the unified {@link Client} talks to the
@@ -5832,31 +5832,37 @@ class FileStorage {
      */
     async writeFileToShards(input) {
         await this.codec.ready(this.rsWasmModule);
-        const data = await toUint8Array(input.data);
         const fileId = input.fileId ?? generateId();
         const createdAt = Date.now();
         const chunks = [];
-        const totalChunks = Math.max(1, Math.ceil(data.length / this.chunkSize));
+        // Read the source per-chunk so the peak memory footprint stays at
+        // ~chunkSize even for multi-GB files. A Blob/File is sliced lazily (only
+        // the current chunk's bytes are materialized) — reading the whole thing
+        // up front would blow past the browser's ~2GB ArrayBuffer cap and throw
+        // "the file could not be read". A caller that already has the bytes in a
+        // Uint8Array just gets zero-copy subarray views.
+        const asBytes = input.data instanceof Uint8Array ? input.data : null;
+        const size = asBytes ? asBytes.length : input.data.size;
+        const readChunk = async (start, end) => asBytes ? asBytes.subarray(start, end)
+            : new Uint8Array(await input.data.slice(start, end).arrayBuffer());
+        const totalChunks = Math.max(1, Math.ceil(size / this.chunkSize));
         input.onProgress?.(0, totalChunks);
-        // Slicing + encrypting + encoding + uploading is done per-chunk so
-        // the peak memory footprint stays at ~chunkSize even for huge files.
-        // Within a chunk, the shard uploads are parallelized up to
-        // `this.concurrency`.
-        for (let chunkIndex = 0; chunkIndex * this.chunkSize < Math.max(1, data.length); chunkIndex++) {
+        // Within a chunk, the shard uploads are parallelized up to `this.concurrency`.
+        for (let chunkIndex = 0; chunkIndex * this.chunkSize < Math.max(1, size); chunkIndex++) {
             const start = chunkIndex * this.chunkSize;
-            const end = Math.min(start + this.chunkSize, data.length);
-            const plaintext = data.subarray(start, end);
+            const end = Math.min(start + this.chunkSize, size);
+            const plaintext = await readChunk(start, end);
             chunks.push(await this.writeChunk(plaintext, chunkIndex));
             input.onProgress?.(chunks.length, totalChunks);
             // Stop after one iteration if the file is empty — we still emit a
             // single zero-length chunk so the manifest has a stable shape.
-            if (data.length === 0)
+            if (size === 0)
                 break;
         }
         const manifest = {
             id: fileId,
             name: input.metadata.name,
-            size: data.length,
+            size,
             type: input.metadata.type,
             path: input.metadata.path,
             lastModified: input.metadata.lastModified ?? createdAt,
@@ -5892,6 +5898,19 @@ class FileStorage {
         }
         return { data: out, stat: manifestToStat(manifest) };
     }
+    /**
+     * Streaming inverse of {@link writeFileToShards}: yields each chunk's
+     * decrypted plaintext in order WITHOUT concatenating. Peak memory stays at
+     * ~one chunk, so multi-GB files (which overflow a single `Uint8Array`) can be
+     * piped straight to a file/socket. Same shard sourcing (cache → peers →
+     * origin) and per-chunk auth as {@link readFileFromShards}.
+     */
+    async *readChunksFromShards(manifest) {
+        await this.codec.ready(this.rsWasmModule);
+        for (const chunkManifest of [...manifest.chunks].sort((a, b) => a.chunkIndex - b.chunkIndex)) {
+            yield await this.readChunk(chunkManifest);
+        }
+    }
     // -------------------------------------------------------------------------
     // Space-backed convenience wrappers — encrypt + upload + write manifest.
     // Require a `SharedSpaceClient` to have been provided at construction.
@@ -6025,14 +6044,6 @@ class FileStorage {
         await Promise.all(workers);
     }
 }
-/** Normalize `data` to a flat `Uint8Array`. */
-async function toUint8Array(data) {
-    if (data instanceof Uint8Array)
-        return data;
-    // Blob covers File too (File extends Blob in every runtime we target).
-    const ab = await data.arrayBuffer();
-    return new Uint8Array(ab);
-}
 
 /**
  * HTTP client for the gated file-manifest store on a SharedSpace
@@ -7348,6 +7359,13 @@ class Space {
     async getFile(manifest) {
         return this.fileStorage().readFileFromShards(manifest);
     }
+    /** Streaming read: async-iterate a stored file's decrypted chunks in order,
+     *  never holding the whole file in memory. Use for multi-GB files (a single
+     *  `Uint8Array` can't hold them) — e.g. a transcoder piping the raw to a temp
+     *  file. Same peer/origin shard sourcing as {@link getFile}. */
+    getFileStream(manifest) {
+        return this.fileStorage().readChunksFromShards(manifest);
+    }
     /** Declare a file's shards (ref++ + meter) for a manifest already stored —
      *  used to back-fill accounting for files written before commit existed, or to
      *  register a manifest received via sharing. `putFile` already commits new