@peerbit/please-lib 2.0.1 → 2.0.3

package/lib/esm/index.js

@@ -32,17 +32,51 @@ var __runInitializers = (this && this.__runInitializers) || function (thisArg, i
     }
     return useValue ? value : void 0;
 };
-import { field, variant, vec, option } from "@dao-xyz/borsh";
+import { field, variant, option } from "@dao-xyz/borsh";
 import { Program } from "@peerbit/program";
-import { Documents, SearchRequest, StringMatch, StringMatchMethod, Or, IsNull, } from "@peerbit/document";
-import { sha256Base64Sync, randomBytes } from "@peerbit/crypto";
+import { Documents, SearchRequest, StringMatch, StringMatchMethod, IsNull, } from "@peerbit/document";
+import { sha256Base64Sync, randomBytes, toBase64, toBase64URL, } from "@peerbit/crypto";
 import { concat } from "uint8arrays";
 import { sha256Sync } from "@peerbit/crypto";
 import { TrustedNetwork } from "@peerbit/trusted-network";
-import PQueue from "p-queue";
+import { SHA256 } from "@stablelib/sha256";
+const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
+const isRetryableChunkLookupError = (error) => error instanceof Error &&
+    (error.name === "AbortError" ||
+        error.message.includes("fanout channel closed"));
 export class AbstractFile {
+    async getFile(files, properties) {
+        const chunks = [];
+        for await (const chunk of this.streamFile(files, properties)) {
+            chunks.push(chunk);
+        }
+        return (properties?.as === "chunks" ? chunks : concat(chunks));
+    }
+    async writeFile(files, writable, properties) {
+        try {
+            for await (const chunk of this.streamFile(files, properties)) {
+                await writable.write(chunk);
+            }
+            await writable.close?.();
+        }
+        catch (error) {
+            if (writable.abort) {
+                try {
+                    await writable.abort(error);
+                }
+                catch {
+                    // Ignore writable cleanup failures.
+                }
+            }
+            throw error;
+        }
+    }
 }
 let IndexableFile = (() => {
+    let _classDecorators = [variant("files_indexable_file")];
+    let _classDescriptor;
+    let _classExtraInitializers = [];
+    let _classThis;
     let _id_decorators;
     let _id_initializers = [];
     let _id_extraInitializers = [];
@@ -55,18 +89,22 @@ let IndexableFile = (() => {
     let _parentId_decorators;
     let _parentId_initializers = [];
     let _parentId_extraInitializers = [];
-    return class IndexableFile {
+    var IndexableFile = class {
+        static { _classThis = this; }
         static {
             const _metadata = typeof Symbol === "function" && Symbol.metadata ? Object.create(null) : void 0;
             _id_decorators = [field({ type: "string" })];
             _name_decorators = [field({ type: "string" })];
-            _size_decorators = [field({ type: "u32" })];
+            _size_decorators = [field({ type: "u64" })];
             _parentId_decorators = [field({ type: option("string") })];
             __esDecorate(null, null, _id_decorators, { kind: "field", name: "id", static: false, private: false, access: { has: obj => "id" in obj, get: obj => obj.id, set: (obj, value) => { obj.id = value; } }, metadata: _metadata }, _id_initializers, _id_extraInitializers);
             __esDecorate(null, null, _name_decorators, { kind: "field", name: "name", static: false, private: false, access: { has: obj => "name" in obj, get: obj => obj.name, set: (obj, value) => { obj.name = value; } }, metadata: _metadata }, _name_initializers, _name_extraInitializers);
             __esDecorate(null, null, _size_decorators, { kind: "field", name: "size", static: false, private: false, access: { has: obj => "size" in obj, get: obj => obj.size, set: (obj, value) => { obj.size = value; } }, metadata: _metadata }, _size_initializers, _size_extraInitializers);
             __esDecorate(null, null, _parentId_decorators, { kind: "field", name: "parentId", static: false, private: false, access: { has: obj => "parentId" in obj, get: obj => obj.parentId, set: (obj, value) => { obj.parentId = value; } }, metadata: _metadata }, _parentId_initializers, _parentId_extraInitializers);
-            if (_metadata) Object.defineProperty(this, Symbol.metadata, { enumerable: true, configurable: true, writable: true, value: _metadata });
+            __esDecorate(null, _classDescriptor = { value: _classThis }, _classDecorators, { kind: "class", name: _classThis.name, metadata: _metadata }, null, _classExtraInitializers);
+            IndexableFile = _classThis = _classDescriptor.value;
+            if (_metadata) Object.defineProperty(_classThis, Symbol.metadata, { enumerable: true, configurable: true, writable: true, value: _metadata });
+            __runInitializers(_classThis, _classExtraInitializers);
         }
         id = __runInitializers(this, _id_initializers, void 0);
         name = (__runInitializers(this, _id_extraInitializers), __runInitializers(this, _name_initializers, void 0));
@@ -80,9 +118,32 @@ let IndexableFile = (() => {
             this.parentId = file.parentId;
         }
     };
+    return IndexableFile = _classThis;
 })();
 export { IndexableFile };
 const TINY_FILE_SIZE_LIMIT = 5 * 1e6; // 6mb
+const LARGE_FILE_SEGMENT_SIZE = TINY_FILE_SIZE_LIMIT / 10;
+const LARGE_FILE_TARGET_CHUNK_COUNT = 1024;
+const CHUNK_SIZE_GRANULARITY = 64 * 1024;
+const MAX_LARGE_FILE_SEGMENT_SIZE = TINY_FILE_SIZE_LIMIT - 256 * 1024;
+const LARGE_FILE_CHUNK_LOOKUP_TIMEOUT_MS = 5 * 60 * 1000;
+const TINY_FILE_SIZE_LIMIT_BIGINT = BigInt(TINY_FILE_SIZE_LIMIT);
+const roundUpTo = (value, multiple) => Math.ceil(value / multiple) * multiple;
+const getLargeFileSegmentSize = (size) => Math.min(MAX_LARGE_FILE_SEGMENT_SIZE, roundUpTo(Math.max(LARGE_FILE_SEGMENT_SIZE, Math.ceil(Number(size) / LARGE_FILE_TARGET_CHUNK_COUNT)), CHUNK_SIZE_GRANULARITY));
+const getChunkStart = (index, chunkSize = LARGE_FILE_SEGMENT_SIZE) => index * chunkSize;
+const getChunkEnd = (index, size, chunkSize = LARGE_FILE_SEGMENT_SIZE) => Math.min((index + 1) * chunkSize, Number(size));
+const getChunkCount = (size, chunkSize = LARGE_FILE_SEGMENT_SIZE) => Math.ceil(Number(size) / chunkSize);
+const getChunkId = (parentId, index) => `${parentId}:${index}`;
+const createUploadId = () => toBase64URL(randomBytes(16));
+const isBlobLike = (value) => typeof Blob !== "undefined" && value instanceof Blob;
+const readBlobChunk = async (blob, index, chunkSize) => new Uint8Array(await blob
+    .slice(getChunkStart(index, chunkSize), getChunkEnd(index, blob.size, chunkSize))
+    .arrayBuffer());
+const ensureSourceSize = (actual, expected) => {
+    if (actual !== expected) {
+        throw new Error(`Source size changed during upload. Expected ${expected} bytes, got ${actual}`);
+    }
+};
 let TinyFile = (() => {
     let _classDecorators = [variant(0)];
     let _classDescriptor;
@@ -98,9 +159,15 @@ let TinyFile = (() => {
     let _file_decorators;
     let _file_initializers = [];
     let _file_extraInitializers = [];
+    let _hash_decorators;
+    let _hash_initializers = [];
+    let _hash_extraInitializers = [];
     let _parentId_decorators;
     let _parentId_initializers = [];
     let _parentId_extraInitializers = [];
+    let _index_decorators;
+    let _index_initializers = [];
+    let _index_extraInitializers = [];
     var TinyFile = class extends _classSuper {
         static { _classThis = this; }
         static {
@@ -108,11 +175,15 @@ let TinyFile = (() => {
             _id_decorators = [field({ type: "string" })];
             _name_decorators = [field({ type: "string" })];
             _file_decorators = [field({ type: Uint8Array })];
+            _hash_decorators = [field({ type: "string" })];
             _parentId_decorators = [field({ type: option("string") })];
+            _index_decorators = [field({ type: option("u32") })];
             __esDecorate(null, null, _id_decorators, { kind: "field", name: "id", static: false, private: false, access: { has: obj => "id" in obj, get: obj => obj.id, set: (obj, value) => { obj.id = value; } }, metadata: _metadata }, _id_initializers, _id_extraInitializers);
             __esDecorate(null, null, _name_decorators, { kind: "field", name: "name", static: false, private: false, access: { has: obj => "name" in obj, get: obj => obj.name, set: (obj, value) => { obj.name = value; } }, metadata: _metadata }, _name_initializers, _name_extraInitializers);
             __esDecorate(null, null, _file_decorators, { kind: "field", name: "file", static: false, private: false, access: { has: obj => "file" in obj, get: obj => obj.file, set: (obj, value) => { obj.file = value; } }, metadata: _metadata }, _file_initializers, _file_extraInitializers);
+            __esDecorate(null, null, _hash_decorators, { kind: "field", name: "hash", static: false, private: false, access: { has: obj => "hash" in obj, get: obj => obj.hash, set: (obj, value) => { obj.hash = value; } }, metadata: _metadata }, _hash_initializers, _hash_extraInitializers);
             __esDecorate(null, null, _parentId_decorators, { kind: "field", name: "parentId", static: false, private: false, access: { has: obj => "parentId" in obj, get: obj => obj.parentId, set: (obj, value) => { obj.parentId = value; } }, metadata: _metadata }, _parentId_initializers, _parentId_extraInitializers);
+            __esDecorate(null, null, _index_decorators, { kind: "field", name: "index", static: false, private: false, access: { has: obj => "index" in obj, get: obj => obj.index, set: (obj, value) => { obj.index = value; } }, metadata: _metadata }, _index_initializers, _index_extraInitializers);
             __esDecorate(null, _classDescriptor = { value: _classThis }, _classDecorators, { kind: "class", name: _classThis.name, metadata: _metadata }, null, _classExtraInitializers);
             TinyFile = _classThis = _classDescriptor.value;
             if (_metadata) Object.defineProperty(_classThis, Symbol.metadata, { enumerable: true, configurable: true, writable: true, value: _metadata });
@@ -121,24 +192,32 @@ let TinyFile = (() => {
         id = __runInitializers(this, _id_initializers, void 0);
         name = (__runInitializers(this, _id_extraInitializers), __runInitializers(this, _name_initializers, void 0));
         file = (__runInitializers(this, _name_extraInitializers), __runInitializers(this, _file_initializers, void 0)); // 10 mb imit
-        parentId = (__runInitializers(this, _file_extraInitializers), __runInitializers(this, _parentId_initializers, void 0));
+        hash = (__runInitializers(this, _file_extraInitializers), __runInitializers(this, _hash_initializers, void 0));
+        parentId = (__runInitializers(this, _hash_extraInitializers), __runInitializers(this, _parentId_initializers, void 0));
+        index = (__runInitializers(this, _parentId_extraInitializers), __runInitializers(this, _index_initializers, void 0));
         get size() {
-            return this.file.byteLength;
+            return BigInt(this.file.byteLength);
         }
         constructor(properties) {
             super();
-            __runInitializers(this, _parentId_extraInitializers);
-            this.id = properties.id || sha256Base64Sync(properties.file);
+            __runInitializers(this, _index_extraInitializers);
+            this.parentId = properties.parentId;
+            this.index = properties.index;
+            this.id =
+                properties.id ||
+                    (properties.parentId != null && properties.index != null
+                        ? `${properties.parentId}:${properties.index}`
+                        : sha256Base64Sync(properties.file));
             this.name = properties.name;
             this.file = properties.file;
-            this.parentId = properties.parentId;
+            this.hash = properties.hash || sha256Base64Sync(properties.file);
         }
-        async getFile(_files, properties) {
-            if (sha256Base64Sync(this.file) !== this.id) {
+        async *streamFile(_files, properties) {
+            if (sha256Base64Sync(this.file) !== this.hash) {
                 throw new Error("Hash does not match the file content");
             }
             properties?.progress?.(1);
-            return Promise.resolve(properties?.as == "chunks" ? [this.file] : this.file);
+            yield this.file;
         }
         async delete() {
             // Do nothing, since no releated files where created
@@ -159,135 +238,165 @@ let LargeFile = (() => {
     let _name_decorators;
     let _name_initializers = [];
     let _name_extraInitializers = [];
-    let _fileIds_decorators;
-    let _fileIds_initializers = [];
-    let _fileIds_extraInitializers = [];
     let _size_decorators;
     let _size_initializers = [];
     let _size_extraInitializers = [];
+    let _chunkCount_decorators;
+    let _chunkCount_initializers = [];
+    let _chunkCount_extraInitializers = [];
+    let _ready_decorators;
+    let _ready_initializers = [];
+    let _ready_extraInitializers = [];
+    let _finalHash_decorators;
+    let _finalHash_initializers = [];
+    let _finalHash_extraInitializers = [];
     var LargeFile = class extends _classSuper {
         static { _classThis = this; }
         static {
             const _metadata = typeof Symbol === "function" && Symbol.metadata ? Object.create(_classSuper[Symbol.metadata] ?? null) : void 0;
             _id_decorators = [field({ type: "string" })];
             _name_decorators = [field({ type: "string" })];
-            _fileIds_decorators = [field({ type: vec("string") })];
-            _size_decorators = [field({ type: "u32" })];
+            _size_decorators = [field({ type: "u64" })];
+            _chunkCount_decorators = [field({ type: "u32" })];
+            _ready_decorators = [field({ type: "bool" })];
+            _finalHash_decorators = [field({ type: option("string") })];
             __esDecorate(null, null, _id_decorators, { kind: "field", name: "id", static: false, private: false, access: { has: obj => "id" in obj, get: obj => obj.id, set: (obj, value) => { obj.id = value; } }, metadata: _metadata }, _id_initializers, _id_extraInitializers);
             __esDecorate(null, null, _name_decorators, { kind: "field", name: "name", static: false, private: false, access: { has: obj => "name" in obj, get: obj => obj.name, set: (obj, value) => { obj.name = value; } }, metadata: _metadata }, _name_initializers, _name_extraInitializers);
-            __esDecorate(null, null, _fileIds_decorators, { kind: "field", name: "fileIds", static: false, private: false, access: { has: obj => "fileIds" in obj, get: obj => obj.fileIds, set: (obj, value) => { obj.fileIds = value; } }, metadata: _metadata }, _fileIds_initializers, _fileIds_extraInitializers);
             __esDecorate(null, null, _size_decorators, { kind: "field", name: "size", static: false, private: false, access: { has: obj => "size" in obj, get: obj => obj.size, set: (obj, value) => { obj.size = value; } }, metadata: _metadata }, _size_initializers, _size_extraInitializers);
+            __esDecorate(null, null, _chunkCount_decorators, { kind: "field", name: "chunkCount", static: false, private: false, access: { has: obj => "chunkCount" in obj, get: obj => obj.chunkCount, set: (obj, value) => { obj.chunkCount = value; } }, metadata: _metadata }, _chunkCount_initializers, _chunkCount_extraInitializers);
+            __esDecorate(null, null, _ready_decorators, { kind: "field", name: "ready", static: false, private: false, access: { has: obj => "ready" in obj, get: obj => obj.ready, set: (obj, value) => { obj.ready = value; } }, metadata: _metadata }, _ready_initializers, _ready_extraInitializers);
+            __esDecorate(null, null, _finalHash_decorators, { kind: "field", name: "finalHash", static: false, private: false, access: { has: obj => "finalHash" in obj, get: obj => obj.finalHash, set: (obj, value) => { obj.finalHash = value; } }, metadata: _metadata }, _finalHash_initializers, _finalHash_extraInitializers);
             __esDecorate(null, _classDescriptor = { value: _classThis }, _classDecorators, { kind: "class", name: _classThis.name, metadata: _metadata }, null, _classExtraInitializers);
             LargeFile = _classThis = _classDescriptor.value;
             if (_metadata) Object.defineProperty(_classThis, Symbol.metadata, { enumerable: true, configurable: true, writable: true, value: _metadata });
             __runInitializers(_classThis, _classExtraInitializers);
         }
-        id = __runInitializers(this, _id_initializers, void 0); // hash
+        id = __runInitializers(this, _id_initializers, void 0);
         name = (__runInitializers(this, _id_extraInitializers), __runInitializers(this, _name_initializers, void 0));
-        fileIds = (__runInitializers(this, _name_extraInitializers), __runInitializers(this, _fileIds_initializers, void 0));
-        size = (__runInitializers(this, _fileIds_extraInitializers), __runInitializers(this, _size_initializers, void 0));
+        size = (__runInitializers(this, _name_extraInitializers), __runInitializers(this, _size_initializers, void 0));
+        chunkCount = (__runInitializers(this, _size_extraInitializers), __runInitializers(this, _chunkCount_initializers, void 0));
+        ready = (__runInitializers(this, _chunkCount_extraInitializers), __runInitializers(this, _ready_initializers, void 0));
+        finalHash = (__runInitializers(this, _ready_extraInitializers), __runInitializers(this, _finalHash_initializers, void 0));
         constructor(properties) {
             super();
-            __runInitializers(this, _size_extraInitializers);
-            this.id = properties.id;
+            __runInitializers(this, _finalHash_extraInitializers);
+            this.id = properties.id || createUploadId();
             this.name = properties.name;
-            this.fileIds = properties.fileIds;
             this.size = properties.size;
-        }
-        static async create(name, file, files, progress) {
-            const segmetSize = TINY_FILE_SIZE_LIMIT / 10; // 10% of the small size limit
-            const fileIds = [];
-            const id = sha256Base64Sync(file);
-            const fileSize = file.byteLength;
-            progress?.(0);
-            const end = Math.ceil(file.byteLength / segmetSize);
-            for (let i = 0; i < end; i++) {
-                progress?.((i + 1) / end);
-                fileIds.push(await files.add(name + "/" + i, file.subarray(i * segmetSize, Math.min((i + 1) * segmetSize, file.byteLength)), id));
-            }
-            progress?.(1);
-            return new LargeFile({ id, name, fileIds: fileIds, size: fileSize });
+            this.chunkCount = properties.chunkCount;
+            this.ready = properties.ready ?? false;
+            this.finalHash = properties.finalHash;
         }
         get parentId() {
             // Large file can never have a parent
             return undefined;
         }
-        async fetchChunks(files) {
-            const expectedIds = new Set(this.fileIds);
-            const allFiles = await files.files.index.search(new SearchRequest({
-                query: [
-                    new Or([...expectedIds].map((x) => new StringMatch({ key: "id", value: x }))),
-                ],
-                fetch: 0xffffffff,
-            }));
-            return allFiles;
+        async fetchChunks(files, properties) {
+            const chunks = new Map();
+            const totalTimeout = properties?.timeout ?? 30_000;
+            const deadline = Date.now() + totalTimeout;
+            const queryTimeout = Math.min(totalTimeout, 5_000);
+            const searchOptions = {
+                local: true,
+                remote: {
+                    timeout: queryTimeout,
+                    throwOnMissing: false,
+                    // Chunk queries return the full TinyFile document including its
+                    // bytes. Observer reads can stream that result directly, while
+                    // actual replicators should still persist downloaded chunks.
+                    replicate: files.persistChunkReads,
+                },
+            };
+            const recordChunks = (results) => {
+                for (const chunk of results) {
+                    if (chunk instanceof TinyFile &&
+                        chunk.parentId === this.id &&
+                        chunk.index != null &&
+                        !chunks.has(chunk.index)) {
+                        chunks.set(chunk.index, chunk);
+                    }
+                }
+            };
+            while (chunks.size < this.chunkCount && Date.now() < deadline) {
+                const before = chunks.size;
+                recordChunks(await files.files.index.search(new SearchRequest({
+                    query: new StringMatch({ key: "parentId", value: this.id }),
+                    fetch: 0xffffffff,
+                }), searchOptions));
+                if (chunks.size === before && chunks.size < this.chunkCount) {
+                    await sleep(250);
+                }
+            }
+            return [...chunks.values()].sort((a, b) => (a.index || 0) - (b.index || 0));
         }
         async delete(files) {
-            await Promise.all((await this.fetchChunks(files)).map((x) => x.delete(files)));
+            await Promise.all((await this.fetchChunks(files)).map((x) => files.files.del(x.id)));
         }
-        async getFile(files, properties) {
-            // Get all sub files (SmallFiles) and concatinate them in the right order (the order of this.fileIds)
-            properties?.progress?.(0);
-            const allChunks = await this.fetchChunks(files);
-            const fetchQueue = new PQueue({ concurrency: 10 });
-            let fetchError = undefined;
-            fetchQueue.on("error", (err) => {
-                fetchError = err;
-            });
-            const chunks = new Map();
-            const expectedIds = new Set(this.fileIds);
-            if (allChunks.length > 0) {
-                let c = 0;
-                for (const r of allChunks) {
-                    if (chunks.has(r.id)) {
-                        // chunk already added;
+        async resolveChunk(files, index, knownChunks, properties) {
+            const totalTimeout = properties?.timeout ?? LARGE_FILE_CHUNK_LOOKUP_TIMEOUT_MS;
+            const deadline = Date.now() + totalTimeout;
+            const attemptTimeout = Math.min(totalTimeout, 5_000);
+            const chunkId = getChunkId(this.id, index);
+            while (Date.now() < deadline) {
+                const cached = knownChunks.get(index);
+                if (cached) {
+                    return cached;
+                }
+                try {
+                    const chunk = await files.files.index.get(chunkId, {
+                        local: true,
+                        waitFor: attemptTimeout,
+                        remote: {
+                            timeout: attemptTimeout,
+                            wait: {
+                                timeout: attemptTimeout,
+                                behavior: "keep-open",
+                            },
+                            throwOnMissing: false,
+                            retryMissingResponses: true,
+                            replicate: files.persistChunkReads,
+                        },
+                    });
+                    if (chunk instanceof TinyFile &&
+                        chunk.parentId === this.id &&
+                        chunk.index === index) {
+                        knownChunks.set(index, chunk);
+                        return chunk;
                     }
-                    if (!expectedIds.has(r.id)) {
-                        // chunk is not part of this file
+                }
+                catch (error) {
+                    if (!isRetryableChunkLookupError(error)) {
+                        throw error;
                     }
-                    fetchQueue
-                        .add(async () => {
-                        let lastError = undefined;
-                        for (let i = 0; i < 3; i++) {
-                            try {
-                                const chunk = await r.getFile(files, {
-                                    as: "joined",
-                                    timeout: properties?.timeout,
-                                });
-                                if (!chunk) {
-                                    throw new Error("Failed to fetch chunk");
-                                }
-                                chunks.set(r.id, chunk);
-                                c++;
-                                properties?.progress?.(c / allChunks.length);
-                                return;
-                            }
-                            catch (error) {
-                                // try 3 times
-                                lastError = error;
-                            }
-                        }
-                        throw lastError;
-                    })
-                        .catch(() => {
-                        fetchQueue.clear(); // Dont do anything more since we failed to fetch one block
-                    });
                 }
+                await sleep(250);
             }
-            await fetchQueue.onIdle();
-            if (fetchError || chunks.size !== expectedIds.size) {
-                throw new Error(`Failed to resolve file. Recieved ${chunks.size}/${expectedIds.size} chunks`);
+            throw new Error(`Failed to resolve chunk ${index + 1}/${this.chunkCount} for file ${this.id}`);
+        }
+        async *streamFile(files, properties) {
+            if (!this.ready) {
+                throw new Error("File is still uploading");
+            }
+            properties?.progress?.(0);
+            let processed = 0;
+            const hasher = this.finalHash ? new SHA256() : undefined;
+            const knownChunks = new Map();
+            for (let index = 0; index < this.chunkCount; index++) {
+                const chunkFile = await this.resolveChunk(files, index, knownChunks, {
+                    timeout: properties?.timeout,
+                });
+                const chunk = await chunkFile.getFile(files, {
+                    as: "joined",
+                    timeout: properties?.timeout,
+                });
+                hasher?.update(chunk);
+                processed += chunk.byteLength;
+                properties?.progress?.(processed / Math.max(Number(this.size), 1));
+                yield chunk;
+            }
+            if (hasher && toBase64(hasher.digest()) !== this.finalHash) {
+                throw new Error("File hash does not match the expected content");
             }
-            const chunkContentResolved = await Promise.all(this.fileIds.map(async (x) => {
-                const chunkValue = await chunks.get(x);
-                if (!chunkValue) {
-                    throw new Error("Failed to retrieve chunk with id: " + x);
-                }
-                return chunkValue;
-            }));
-            return (properties?.as == "chunks"
-                ? chunkContentResolved
-                : concat(chunkContentResolved));
         }
     };
     return LargeFile = _classThis;
@@ -332,9 +441,9 @@ let Files = (() => {
         name = (__runInitializers(this, _id_extraInitializers), __runInitializers(this, _name_initializers, void 0));
         trustGraph = (__runInitializers(this, _name_extraInitializers), __runInitializers(this, _trustGraph_initializers, void 0));
         files = (__runInitializers(this, _trustGraph_extraInitializers), __runInitializers(this, _files_initializers, void 0));
+        persistChunkReads = __runInitializers(this, _files_extraInitializers);
         constructor(properties = {}) {
             super();
-            __runInitializers(this, _files_extraInitializers);
             this.id = properties.id || randomBytes(32);
             this.name = properties.name || "";
             this.trustGraph = properties.rootKey
@@ -347,22 +456,159 @@ let Files = (() => {
                     properties.rootKey?.bytes || new Uint8Array(0),
                 ])),
             });
+            this.persistChunkReads = true;
         }
         async add(name, file, parentId, progress) {
-            let toPut;
+            if (isBlobLike(file)) {
+                return this.addBlob(name, file, parentId, progress);
+            }
+            progress?.(0);
+            if (BigInt(file.byteLength) <= TINY_FILE_SIZE_LIMIT_BIGINT) {
+                const tinyFile = new TinyFile({ name, file, parentId });
+                await this.files.put(tinyFile);
+                progress?.(1);
+                return tinyFile.id;
+            }
+            const chunkSize = getLargeFileSegmentSize(file.byteLength);
+            return this.addChunkedFile(name, BigInt(file.byteLength), (index) => Promise.resolve(file.subarray(getChunkStart(index, chunkSize), getChunkEnd(index, file.byteLength, chunkSize))), chunkSize, parentId, progress);
+        }
+        async addBlob(name, file, parentId, progress) {
             progress?.(0);
-            if (file.byteLength <= TINY_FILE_SIZE_LIMIT) {
-                toPut = new TinyFile({ name, file, parentId });
+            if (BigInt(file.size) <= TINY_FILE_SIZE_LIMIT_BIGINT) {
+                const tinyFile = new TinyFile({
+                    name,
+                    file: new Uint8Array(await file.arrayBuffer()),
+                    parentId,
+                });
+                await this.files.put(tinyFile);
+                progress?.(1);
+                return tinyFile.id;
             }
-            else {
-                if (parentId) {
-                    throw new Error("Unexpected that a LargeFile to have a parent");
+            const chunkSize = getLargeFileSegmentSize(file.size);
+            return this.addChunkedFile(name, BigInt(file.size), (index) => readBlobChunk(file, index, chunkSize), chunkSize, parentId, progress);
+        }
+        async addSource(name, source, parentId, progress) {
+            progress?.(0);
+            if (source.size <= TINY_FILE_SIZE_LIMIT_BIGINT) {
+                const chunks = [];
+                let processed = 0n;
+                for await (const chunk of source.readChunks(TINY_FILE_SIZE_LIMIT)) {
+                    chunks.push(chunk);
+                    processed += BigInt(chunk.byteLength);
+                }
+                ensureSourceSize(processed, source.size);
+                const tinyFile = new TinyFile({
+                    name,
+                    file: chunks.length === 0 ? new Uint8Array(0) : concat(chunks),
+                    parentId,
+                });
+                await this.files.put(tinyFile);
+                progress?.(1);
+                return tinyFile.id;
+            }
+            if (parentId) {
+                throw new Error("Unexpected that a LargeFile to have a parent");
+            }
+            const size = source.size;
+            const chunkSize = getLargeFileSegmentSize(size);
+            const uploadId = createUploadId();
+            const manifest = new LargeFile({
+                id: uploadId,
+                name,
+                size,
+                chunkCount: getChunkCount(size, chunkSize),
+                ready: false,
+            });
+            await this.files.put(manifest);
+            const hasher = new SHA256();
+            try {
+                let uploadedBytes = 0n;
+                let chunkCount = 0;
+                for await (const chunkBytes of source.readChunks(chunkSize)) {
+                    hasher.update(chunkBytes);
+                    await this.files.put(new TinyFile({
+                        name: name + "/" + chunkCount,
+                        file: chunkBytes,
+                        parentId: uploadId,
+                        index: chunkCount,
+                    }));
+                    uploadedBytes += BigInt(chunkBytes.byteLength);
+                    chunkCount++;
+                    progress?.(Number(uploadedBytes) / Math.max(Number(size), 1));
                 }
-                toPut = await LargeFile.create(name, file, this, progress);
+                ensureSourceSize(uploadedBytes, source.size);
+                await this.files.put(new LargeFile({
+                    id: uploadId,
+                    name,
+                    size,
+                    chunkCount,
+                    ready: true,
+                    finalHash: toBase64(hasher.digest()),
+                }));
+            }
+            catch (error) {
+                await this.cleanupChunkedUpload(uploadId).catch(() => { });
+                await this.files.del(uploadId).catch(() => { });
+                throw error;
             }
-            await this.files.put(toPut);
             progress?.(1);
-            return toPut.id;
+            return uploadId;
+        }
+        async cleanupChunkedUpload(uploadId) {
+            const chunks = await this.files.index.search(new SearchRequest({
+                query: new StringMatch({
+                    key: "parentId",
+                    value: uploadId,
+                }),
+                fetch: 0xffffffff,
+            }), { local: true });
+            await Promise.all(chunks.map((chunk) => this.files.del(chunk.id)));
+        }
+        async addChunkedFile(name, size, getChunk, chunkSize, parentId, progress) {
+            if (parentId) {
+                throw new Error("Unexpected that a LargeFile to have a parent");
+            }
+            const uploadId = createUploadId();
+            const chunkCount = getChunkCount(size, chunkSize);
+            const manifest = new LargeFile({
+                id: uploadId,
+                name,
+                size,
+                chunkCount,
+                ready: false,
+            });
+            await this.files.put(manifest);
+            const hasher = new SHA256();
+            try {
+                let uploadedBytes = 0;
+                for (let i = 0; i < chunkCount; i++) {
+                    const chunkBytes = await getChunk(i);
+                    hasher.update(chunkBytes);
+                    await this.files.put(new TinyFile({
+                        name: name + "/" + i,
+                        file: chunkBytes,
+                        parentId: uploadId,
+                        index: i,
+                    }));
+                    uploadedBytes += chunkBytes.byteLength;
+                    progress?.(uploadedBytes / Math.max(Number(size), 1));
+                }
+                await this.files.put(new LargeFile({
+                    id: uploadId,
+                    name,
+                    size,
+                    chunkCount,
+                    ready: true,
+                    finalHash: toBase64(hasher.digest()),
+                }));
+            }
+            catch (error) {
+                await this.cleanupChunkedUpload(uploadId).catch(() => { });
+                await this.files.del(uploadId).catch(() => { });
+                throw error;
+            }
+            progress?.(1);
+            return uploadId;
         }
         async removeById(id) {
             const file = await this.files.index.get(id);
@@ -394,7 +640,10 @@ let Files = (() => {
             }), {
                 local: true,
                 remote: {
-                    throwOnMissing: true,
+                    // Allow partial results while the network is still forming. If we
+                    // throw on missing shards here, the UI can appear "empty" until
+                    // all shard roots respond, which feels broken during joins/churn.
+                    throwOnMissing: false,
                     replicate: true, // sync here because this, because we might want to access it offline, even though we are not replicators
                 },
             });
@@ -407,6 +656,45 @@ let Files = (() => {
             }));
             return count;
         }
+        async resolveById(id, properties) {
+            return this.files.index.get(id, {
+                local: true,
+                waitFor: properties?.timeout,
+                remote: {
+                    timeout: properties?.timeout ?? 10 * 1000,
+                    wait: properties?.timeout
+                        ? {
+                            timeout: properties.timeout,
+                            behavior: "keep-open",
+                        }
+                        : undefined,
+                    throwOnMissing: false,
+                    retryMissingResponses: true,
+                    replicate: properties?.replicate,
+                },
+            });
+        }
+        async resolveByName(name, properties) {
+            const results = await this.files.index.search(new SearchRequest({
+                query: [
+                    new StringMatch({
+                        key: "name",
+                        value: name,
+                        caseInsensitive: false,
+                        method: StringMatchMethod.exact,
+                    }),
+                ],
+                fetch: 1,
+            }), {
+                local: true,
+                remote: {
+                    timeout: properties?.timeout ?? 10 * 1000,
+                    throwOnMissing: false,
+                    replicate: properties?.replicate,
+                },
+            });
+            return results[0];
+        }
         /**
          * Get by name
          * @param id
@@ -461,6 +749,7 @@ let Files = (() => {
         }
         // Setup lifecycle, will be invoked on 'open'
         async open(args) {
+            this.persistChunkReads = args?.replicate !== false;
             await this.trustGraph?.open({
                 replicate: args?.replicate,
             });