@coderook/cli 0.22.1 → 0.23.0

package/dist/desktop-app/src/main/compress.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.looksCompressed = looksCompressed;
+exports.looksCompressed = void 0;
 exports.encodeForUpload = encodeForUpload;
 /**
  * Compressing an object before it is sent.
@@ -29,6 +29,9 @@ exports.encodeForUpload = encodeForUpload;
 const node_crypto_1 = require("node:crypto");
 const node_zlib_1 = require("node:zlib");
 const node_util_1 = require("node:util");
+const compression_policy_1 = require("../shared/compression_policy");
+var compression_policy_2 = require("../shared/compression_policy");
+Object.defineProperty(exports, "looksCompressed", { enumerable: true, get: function () { return compression_policy_2.looksCompressed; } });
 const deflate = (0, node_util_1.promisify)(node_zlib_1.gzip);
 /*
   Extensions whose contents are already compressed. Matching on the name is a
@@ -36,30 +39,6 @@ const deflate = (0, node_util_1.promisify)(node_zlib_1.gzip);
   is measured rather than guessed, so a wrong entry here costs a little time
   and never a wrong result.
 */
-const ALREADY_COMPRESSED = new Set([
-    ".7z", ".aac", ".avi", ".br", ".bz2", ".cbx", ".docx", ".flac", ".gif",
-    ".gz", ".heic", ".jpeg", ".jpg", ".jxl", ".m4a", ".m4v", ".mkv", ".mov",
-    ".mp3", ".mp4", ".odp", ".ods", ".odt", ".ogg", ".opus", ".pdf", ".png",
-    ".pptx", ".rar", ".safetensors", ".webm", ".webp", ".xlsx", ".xz", ".zip",
-    ".zst",
-]);
-/**
- * Below this, compression is not worth the round trip through zlib — the
- * saving is measured in bytes and every file pays the CPU.
- */
-const SMALLEST_WORTH_TRYING = 4 * 1024;
-/**
- * Keep the compressed form only if it saved something worth having. A file
- * that shrinks by two percent is not worth the service decompressing it on
- * every read for the rest of its life.
- */
-const WORTHWHILE = 0.95;
-function looksCompressed(relativePath) {
-    const dot = relativePath.lastIndexOf(".");
-    if (dot < 0)
-        return false;
-    return ALREADY_COMPRESSED.has(relativePath.slice(dot).toLowerCase());
-}
 /**
  * Decide how one object should travel.
  *
@@ -73,15 +52,11 @@ async function encodeForUpload(original, relativePath, allowGzip) {
         encoding: "identity",
         storedSha256: "",
     };
-    if (!allowGzip)
-        return plain;
-    if (original.byteLength < SMALLEST_WORTH_TRYING)
-        return plain;
-    if (looksCompressed(relativePath))
+    if (!(0, compression_policy_1.shouldTryGzip)(relativePath, original.byteLength, allowGzip))
         return plain;
     try {
         const packed = await deflate(original, { level: 6 });
-        if (packed.byteLength >= original.byteLength * WORTHWHILE)
+        if (!(0, compression_policy_1.gzipIsWorthKeeping)(original.byteLength, packed.byteLength))
             return plain;
         return {
             body: new Uint8Array(packed),

package/dist/desktop-app/src/main/download.js

@@ -174,8 +174,39 @@ class Downloader {
         }
         return response;
     }
+    /**
+     * Read a complete JSON response under the same retry rule as the request.
+     *
+     * `fetch()` resolves as soon as the headers arrive. A connection can still
+     * disappear while `.json()` is consuming the body; treating the response
+     * object as success meant a large file list failed outside the retry loop.
+     * These routes are reads, so repeating the whole request is safe and is the
+     * only way to replace a truncated body.
+     */
+    async json(route) {
+        let wait = retry_js_1.RETRY_FIRST_WAIT_MS;
+        for (let attempt = 1;; attempt += 1) {
+            this.check();
+            try {
+                return await (await this.attempt(route)).json();
+            }
+            catch (error) {
+                if (error instanceof DownloadCancelled)
+                    throw error;
+                if (attempt >= retry_js_1.RETRY_ATTEMPTS || !(0, retry_js_1.worthRetrying)(error))
+                    throw error;
+                try {
+                    await (0, retry_js_1.pauseFor)(this.controller.signal, wait);
+                }
+                catch {
+                    throw new DownloadCancelled();
+                }
+                wait *= 2;
+            }
+        }
+    }
     async versions(repositoryId) {
-        const body = (await (await this.request(`/v1/repositories/${repositoryId}/versions`)).json());
+        const body = await this.json(`/v1/repositories/${repositoryId}/versions`);
         return (body.versions ?? []).map((row) => ({
             id: String(row.id ?? ""),
             sequence: Number(row.sequence ?? 0),
@@ -187,13 +218,26 @@ class Downloader {
         }));
     }
     async files(repositoryId, versionId) {
-        const body = (await (await this.request(`/v1/repositories/${repositoryId}/versions/${versionId}/files`)).json());
+        const body = await this.json(`/v1/repositories/${repositoryId}/versions/${versionId}/files`);
         return (body.files ?? []).map((row) => {
             const pieces = Array.isArray(row.chunks) ? row.chunks : null;
+            const packed = row.pack && typeof row.pack === "object"
+                ? row.pack
+                : null;
             return {
                 path: String(row.path ?? ""),
+                ...(row.objectId ? { objectId: String(row.objectId) } : {}),
                 sha256: String(row.sha256 ?? ""),
                 sourceSize: Number(row.sourceSize ?? 0),
+                ...(packed?.objectId
+                    ? {
+                        pack: {
+                            objectId: String(packed.objectId),
+                            offset: Number(packed.offset ?? 0),
+                            length: Number(packed.length ?? 0),
+                        },
+                    }
+                    : {}),
                 ...(pieces && pieces.length
                     ? {
                         chunks: pieces.map((piece) => {
@@ -240,6 +284,92 @@ class Downloader {
         await (0, promises_1.mkdir)(node_path_1.default.dirname(target), { recursive: true });
         return this.fetchInto(repositoryId, versionId, file, target);
     }
+    /**
+     * Reconstruct and verify every file in a Version without materialising the
+     * whole project at once. Solid packs are fetched once and large files use a
+     * single reusable scratch path, so the disk requirement is bounded by the
+     * largest individual file rather than the complete snapshot.
+     */
+    async verify(repositoryId, versionId, scratch, report) {
+        const files = await this.files(repositoryId, versionId);
+        if (!files.length)
+            throw new Error("That version has no files");
+        const totalBytes = files.reduce((total, file) => total + file.sourceSize, 0);
+        const packed = new Map();
+        for (const file of files) {
+            const id = file.pack?.objectId;
+            if (!id)
+                continue;
+            const group = packed.get(id) ?? [];
+            group.push(file);
+            packed.set(id, group);
+        }
+        await (0, promises_1.rm)(scratch, { recursive: true, force: true });
+        await (0, promises_1.mkdir)(scratch, { recursive: true });
+        const temporary = node_path_1.default.join(scratch, "current-file.partial");
+        const restoredPacks = new Set();
+        const manifest = {};
+        let written = 0;
+        let bytes = 0;
+        const progress = (filePath) => report({
+            files: written,
+            totalFiles: files.length,
+            bytes,
+            totalBytes,
+            path: filePath,
+            percent: underway(bytes, totalBytes),
+        });
+        try {
+            for (const file of files) {
+                this.check();
+                progress(file.path);
+                const packId = file.pack?.objectId;
+                if (packId && restoredPacks.has(packId))
+                    continue;
+                if (packId) {
+                    const reply = await this.request(`/v1/repositories/${repositoryId}/objects/${packId}`);
+                    const body = Buffer.from(await reply.arrayBuffer());
+                    const packDigest = (0, node_crypto_1.createHash)("sha256").update(body).digest("hex");
+                    const expectedPackDigest = reply.headers.get("x-coderook-sha256");
+                    if (expectedPackDigest && packDigest !== expectedPackDigest) {
+                        throw new Error("A solid pack did not arrive intact");
+                    }
+                    for (const member of packed.get(packId) ?? []) {
+                        const offset = member.pack.offset;
+                        const length = member.pack.length;
+                        if (offset < 0 || length < 0 || offset + length > body.byteLength) {
+                            throw new Error(`${member.path} points outside its solid pack`);
+                        }
+                        const digest = (0, node_crypto_1.createHash)("sha256")
+                            .update(body.subarray(offset, offset + length))
+                            .digest("hex");
+                        if (member.sha256 && digest !== member.sha256) {
+                            throw new Error(`${member.path} did not arrive intact`);
+                        }
+                        manifest[member.path] = digest;
+                        written += 1;
+                        bytes += member.sourceSize;
+                        progress(member.path);
+                    }
+                    restoredPacks.add(packId);
+                    continue;
+                }
+                const digest = await this.fetchInto(repositoryId, versionId, file, temporary);
+                if (file.sha256 && digest !== file.sha256) {
+                    throw new Error(`${file.path} did not arrive intact`);
+                }
+                manifest[file.path] = digest;
+                written += 1;
+                bytes += file.sourceSize;
+                await (0, promises_1.rm)(temporary, { force: true });
+            }
+            progress("Done");
+            return { files: written, bytes, manifest };
+        }
+        finally {
+            await (0, promises_1.rm)(scratch, { recursive: true, force: true });
+        }
+    }
     async fetchInto(repositoryId, versionId, file, target) {
         const whole = (0, node_crypto_1.createHash)("sha256");
         const pieces = file.chunks ?? [];
@@ -269,8 +399,10 @@ class Downloader {
                 }
             })()
             : (async function* () {
-                const reply = await request(`/v1/repositories/${repositoryId}/versions/${versionId}/file` +
-                    `?path=${encodeURIComponent(file.path)}`);
+                const reply = await request(file.objectId
+                    ? `/v1/repositories/${repositoryId}/objects/${file.objectId}`
+                    : `/v1/repositories/${repositoryId}/versions/${versionId}/file` +
+                        `?path=${encodeURIComponent(file.path)}`);
                 if (!reply.body)
                     return;
                 for await (const block of node_stream_1.Readable.fromWeb(reply.body)) {
@@ -322,6 +454,16 @@ class Downloader {
         try {
             let written = 0;
             let bytes = 0;
+            const packed = new Map();
+            for (const file of files) {
+                const id = file.pack?.objectId;
+                if (!id)
+                    continue;
+                const group = packed.get(id) ?? [];
+                group.push(file);
+                packed.set(id, group);
+            }
+            const restoredPacks = new Set();
             for (const file of files) {
                 this.check();
                 report({
@@ -337,6 +479,60 @@ class Downloader {
                 if (parts.some((part) => part === ".." || part.includes("\0"))) {
                     throw new Error(`That version contains an unsafe path: ${file.path}`);
                 }
+                const packId = file.pack?.objectId;
+                if (packId && restoredPacks.has(packId))
+                    continue;
+                if (packId) {
+                    /*
+                      A solid pack is one compressed object containing many small files.
+          
+                      Asking the single-file route for every member made a large restore
+                      fetch and decompress the same pack thousands of times. The files
+                      listing already gives us the pack id and each member's exact
+                      window, so fetch the decoded pack once, verify it, and materialise
+                      every member before releasing the bounded section-sized buffer.
+                    */
+                    const reply = await this.request(`/v1/repositories/${repositoryId}/objects/${packId}`);
+                    const body = Buffer.from(await reply.arrayBuffer());
+                    const packDigest = (0, node_crypto_1.createHash)("sha256").update(body).digest("hex");
+                    const expectedPackDigest = reply.headers.get("x-coderook-sha256");
+                    if (expectedPackDigest && packDigest !== expectedPackDigest) {
+                        throw new Error("A solid pack did not arrive intact");
+                    }
+                    for (const member of packed.get(packId) ?? []) {
+                        this.check();
+                        const memberParts = member.path.split("/").filter(Boolean);
+                        if (memberParts.some((part) => part === ".." || part.includes("\0"))) {
+                            throw new Error(`That version contains an unsafe path: ${member.path}`);
+                        }
+                        const offset = member.pack.offset;
+                        const length = member.pack.length;
+                        if (offset < 0 || length < 0 || offset + length > body.byteLength) {
+                            throw new Error(`${member.path} points outside its solid pack`);
+                        }
+                        const contents = body.subarray(offset, offset + length);
+                        const digest = (0, node_crypto_1.createHash)("sha256").update(contents).digest("hex");
+                        if (member.sha256 && digest !== member.sha256) {
+                            throw new Error(`${member.path} did not arrive intact`);
+                        }
+                        const full = node_path_1.default.join(staging, ...memberParts);
+                        await (0, promises_1.mkdir)(node_path_1.default.dirname(full), { recursive: true });
+                        await (0, promises_1.writeFile)(full, contents);
+                        manifest[member.path] = digest;
+                        written += 1;
+                        bytes += member.sourceSize;
+                        report({
+                            files: written,
+                            totalFiles: files.length,
+                            bytes,
+                            totalBytes,
+                            path: member.path,
+                            percent: underway(bytes, totalBytes),
+                        });
+                    }
+                    restoredPacks.add(packId);
+                    continue;
+                }
                 const full = node_path_1.default.join(staging, ...parts);
                 await (0, promises_1.mkdir)(node_path_1.default.dirname(full), { recursive: true });
                 const digest = await this.fetchInto(repositoryId, versionId, file, full);