@coderook/cli 0.30.2 → 0.31.1

package/dist/desktop-app/src/main/detect.js

@@ -25,6 +25,7 @@ exports.rulesFromSuggestions = rulesFromSuggestions;
  * exactly as long as nothing is applied without somebody saying so.
  */
 const node_path_1 = __importDefault(require("node:path"));
+const format_js_1 = require("../shared/format.js");
 /*
   Everything here is recoverable from something else in the project, or is
   another program's working state. Nothing that could be somebody's only copy
@@ -302,6 +303,46 @@ const BULK_FILES = 2_000;
 const BULK_SHARE = 0.7;
 /** Above this share of hand-written files it is source, not output. */
 const BULK_SOURCE_SHARE = 0.1;
+/**
+ * And the same question asked in bytes, for a folder that is huge but not many.
+ *
+ * Counting files misses the shape that actually stops a save finishing. A real
+ * project — 253 files — kept thirteen `.gguf` model weights in `output/`: 73
+ * files, 10.67 GB, 99% of everything it would have sent and 29% of its file
+ * count. Every threshold above is about counts, so nothing was suggested, and
+ * the app set out to upload 10.7 GB of rebuildable model output without a word.
+ * It never finished, and the project still has no version.
+ *
+ * A gigabyte is the floor because below it the upload is not the problem and
+ * the question is not worth asking. The share is the same 0.7: a folder that is
+ * most of the bytes, whatever its file count.
+ */
+const BULK_BYTES = 1024 * 1024 * 1024;
+/*
+  Media somebody made, as opposed to bytes a machine emitted.
+
+  Needed only by the size test, and needed badly there. `SOURCE_EXTENSIONS`
+  above asks "did a person type this", which is the right question for a folder
+  judged on its file count — a cache holds no `.ts` files. It is the wrong
+  question for a folder judged on its size, because a game's art holds no `.ts`
+  files either, and by bytes the art is almost always most of the project. The
+  count rule has a calibration case for exactly that folder; a size rule without
+  this list would walk straight back into it and offer to leave the game out.
+*/
+const AUTHORED_EXTENSIONS = new Set([
+    ".aseprite", ".blend", ".bmp", ".dds", ".exr", ".fbx", ".flac", ".gif",
+    ".glb", ".gltf", ".hdr", ".jpeg", ".jpg", ".m4a", ".mov", ".mp3", ".mp4",
+    ".obj", ".ogg", ".otf", ".pdf", ".png", ".psd", ".svg", ".tga", ".tif",
+    ".tiff", ".ttf", ".wav", ".webm", ".webp", ".woff", ".woff2",
+]);
+/**
+ * Above this share of a folder's *bytes* it is a library of made things.
+ *
+ * By bytes rather than by file count, because that is the currency the test it
+ * guards is arguing in: one 4 GB video beside a thousand generated logs is a
+ * folder of video, and counting files would call it a folder of logs.
+ */
+const AUTHORED_BYTE_SHARE = 0.5;
 /**
  * One enormous folder that no list of names could have known about.
  *
@@ -309,42 +350,70 @@ const BULK_SOURCE_SHARE = 0.1;
  * things every project has and not at all for the one a person invented. This
  * repository's own `work/` is 63,210 of the 64,554 files it would otherwise
  * send — scratch output from a test harness, named nothing in particular, and
- * invisible to every rule in this file.
+ * invisible to every rule in this file. Another project's `output/` is 10.67 GB
+ * of model weights in 73 files, which no count-based threshold can see at all.
  *
- * So this asks about shape instead of name: one folder that is most of the
- * upload and holds no source in it. Never recommended, because the same shape
- * is also what a genuine dataset looks like, and a project whose work really
- * is fifty thousand data files should be asked once and then believed.
+ * So this asks about shape instead of name, in both currencies: one folder that
+ * is most of what would be sent — by file count or by size — and holds no
+ * source in it. Never recommended, because the same shape is also what a
+ * genuine dataset looks like, and a project whose work really is fifty thousand
+ * data files should be asked once and then believed.
  */
 function suggestBulkDirectories(files) {
-    if (files.length < BULK_FILES)
-        return [];
     const tally = new Map();
+    let totalBytes = 0;
     for (const entry of files) {
         const normalised = entry.path.split("\\").join("/");
+        totalBytes += entry.size;
         const top = normalised.includes("/") ? normalised.slice(0, normalised.indexOf("/")) : "";
         /* A file at the root belongs to no folder, so it cannot be one of these. */
         if (!top)
             continue;
-        const held = tally.get(top) ?? { bytes: 0, files: 0, source: 0 };
+        const held = tally.get(top) ?? { bytes: 0, files: 0, source: 0, authored: 0 };
+        const extension = node_path_1.default.extname(normalised).toLowerCase();
         held.bytes += entry.size;
         held.files += 1;
-        if (SOURCE_EXTENSIONS.has(node_path_1.default.extname(normalised).toLowerCase()))
+        if (SOURCE_EXTENSIONS.has(extension))
             held.source += 1;
+        if (AUTHORED_EXTENSIONS.has(extension))
+            held.authored += entry.size;
         tally.set(top, held);
     }
     const found = [];
     for (const [top, held] of tally) {
-        if (held.files < BULK_FILES)
-            continue;
-        if (held.files / files.length < BULK_SHARE)
-            continue;
+        /*
+          Whatever else is true of it, a folder with source in it is somebody's
+          work. This is the one test both shapes below have to pass.
+        */
         if (held.source / held.files >= BULK_SOURCE_SHARE)
             continue;
+        const byCount = files.length >= BULK_FILES &&
+            held.files >= BULK_FILES &&
+            held.files / files.length >= BULK_SHARE;
+        const byBytes = held.bytes >= BULK_BYTES &&
+            held.bytes / Math.max(totalBytes, 1) >= BULK_SHARE &&
+            /*
+              And it is not a library of made things. A game's art is most of a game
+              by size and is the last thing anybody wants left out of the save.
+            */
+            held.authored / Math.max(held.bytes, 1) < AUTHORED_BYTE_SHARE;
+        if (!byCount && !byBytes)
+            continue;
+        /*
+          Described in whichever currency makes the case. "73 of 253 files" is not
+          an argument for leaving anything out; "10.7 GB of 10.8 GB" is the entire
+          argument, and the person reading it is deciding whether to spend an
+          evening uploading. Where both are true, size is still the thing that
+          hurts, so size is what gets said.
+        */
+        const reason = byBytes
+            ? `${(0, format_js_1.formatBytes)(held.bytes)} of ${(0, format_js_1.formatBytes)(totalBytes)}, ` +
+                `and no source in it — generated?`
+            : `${held.files.toLocaleString()} of ${files.length.toLocaleString()} files, ` +
+                `and no source in it — generated?`;
         found.push({
             pattern: `${top}/`,
-            reason: `${held.files.toLocaleString()} of ${files.length.toLocaleString()} files, ` +
-                `and no source in it — generated?`,
+            reason,
             bytes: held.bytes,
             files: held.files,
             recommended: false,

package/dist/desktop-app/src/main/download.js

@@ -61,6 +61,23 @@ async function pruneEmpty(root, removed) {
         directory = node_path_1.default.dirname(directory);
     }
 }
+/**
+ * How long one request may go unanswered before it is treated as lost.
+ *
+ * The same three minutes the uploader uses, and for the same reason: what
+ * follows a deadline here is a retry, so the cost of it firing early on a
+ * genuinely slow call is one repeated request.
+ */
+const REQUEST_DEADLINE_MS = 180_000;
+/**
+ * And how long a whole-version archive may take to arrive.
+ *
+ * The service reads every file in the version out of object storage to build
+ * it, so the wait is proportional to the project rather than to the network.
+ * Fifteen minutes is far beyond any archive measured and still an answer
+ * rather than an indefinite wait.
+ */
+const ARCHIVE_DEADLINE_MS = 900_000;
 /**
  * A percentage for work that is still going.
  *
@@ -127,7 +144,18 @@ class Downloader {
             }
         }
     }
-    async attempt(route) {
+    async attempt(route, 
+    /*
+      Longer where the service has to build the answer before it can send it.
+
+      A deadline sized for an ordinary request aborts a large archive halfway
+      through making it, and the git import quietly falls back to fetching
+      every file on its own — so the fix for a slow clone became the cause of
+      one. The archive for a 21,071-file version takes about ninety seconds
+      to assemble and longer under load; three minutes is a request that has
+      failed, not one that is still working.
+    */
+    deadline = REQUEST_DEADLINE_MS) {
         const token = await this.credentials.token();
         if (!token)
             throw new Error("Sign in again before downloading");
@@ -137,7 +165,19 @@ class Downloader {
                 "user-agent": "CodeRook/0.1",
                 ...(0, identify_js_1.clientHeaders)(),
             },
-            signal: this.controller.signal,
+            /*
+              The download's own cancellation, and a deadline.
+      
+              This carried only the cancel signal, so a request the service never
+              answered waited for ever — the same defect that was fixed in the
+              uploader and in the command line's own client, and missed here. A
+              restore that stops with no bytes moving and no message is the worse
+              half of it: the person is left without the thing they were restoring.
+            */
+            signal: AbortSignal.any([
+                this.controller.signal,
+                AbortSignal.timeout(deadline),
+            ]),
         });
         if (!response.ok) {
             const text = await response.text().catch(() => "");
@@ -217,6 +257,22 @@ class Downloader {
             authorName: String(row.authorName ?? ""),
         }));
     }
+    /**
+     * A whole version as one archive, for a caller that wants all of it.
+     *
+     * The git import asked for each file in its own request, which on a
+     * 21,071-file project is 21,071 round trips — four hours in series, and
+     * about half an hour even eight at a time. The service already builds the
+     * whole version as a single stored archive for the download button; this
+     * is that, for a caller that would otherwise ask file by file.
+     *
+     * Returned whole rather than streamed because the caller is handing the
+     * files to `git fast-import`, which wants them one after another anyway.
+     */
+    async archive(repositoryId, versionId) {
+        const response = await this.attempt(`/v1/repositories/${repositoryId}/versions/${versionId}/archive`, ARCHIVE_DEADLINE_MS);
+        return new Uint8Array(await response.arrayBuffer());
+    }
     async files(repositoryId, versionId) {
         const body = await this.json(`/v1/repositories/${repositoryId}/versions/${versionId}/files`);
         return (body.files ?? []).map((row) => {

package/dist/desktop-app/src/main/upload.js

@@ -21,6 +21,7 @@ const identify_js_1 = require("./identify.js");
 const worktree_js_1 = require("./worktree.js");
 const publish_name_js_1 = require("./publish_name.js");
 const compress_js_1 = require("./compress.js");
+const compression_policy_js_1 = require("../shared/compression_policy.js");
 const chunking_js_1 = require("../shared/chunking.js");
 const publication_js_1 = require("../shared/publication.js");
 const profile_js_1 = require("./profile.js");
@@ -125,9 +126,21 @@ class Uploader {
     /** One shared preflight, so parallel upload lanes cannot race /health. */
     capabilityRequest = null;
     telemetry = new telemetry_js_1.RepositoryTelemetry();
-    /** Planning runs the real partition/compression choices without body I/O. */
+    /*
+      Planning makes the real partition and compression choices, and for a
+      chunked file that means reading it: a chunk's identity is the digest of its
+      bytes, and the quote has to name every one. This comment used to say the
+      plan did no body I/O, which was never true, and the belief that it was is
+      how the send came to do all of that reading a second time.
+    */
     planning = false;
     plannedObjects = new Map();
+    /**
+     * Where each chunked file was cut, as the plan found it, so the send does not
+     * cut it again. Keyed by path and whole-file digest: a file that changed in
+     * between has a different digest, finds nothing here, and is walked afresh.
+     */
+    plannedLayouts = new Map();
     plannedQuote = null;
     /*
       Digests worked out while planning, so sending does not work them out again.
@@ -145,6 +158,20 @@ class Uploader {
     /** Whether this save has already said that packing was refused. */
     packRefusalSaid = false;
     plannedDigests = new Map();
+    /**
+     * Every digest this save actually took, for a caller that keeps a record.
+     *
+     * The desktop keeps one and hands it back through `knownDigests`, which is
+     * what stops an unchanged file being read twice. The command line had no
+     * way to keep one because this was private, so it re-read and re-hashed
+     * whole projects on every save.
+     *
+     * Only files that were read are in here — an entry served from the caller's
+     * own record was never taken and does not need writing down again.
+     */
+    digestsTaken() {
+        return new Map(this.plannedDigests);
+    }
     planningCreatedRepositoryId = null;
     /** Attached to every object write after the server admits the exact plan. */
     activeQuoteId = null;
@@ -306,6 +333,7 @@ class Uploader {
         this.plannedQuote = null;
         this.planningCreatedRepositoryId = null;
         this.plannedObjects.clear();
+        this.plannedLayouts.clear();
         try {
             await this.runPass(request, report);
             const quote = this.plannedQuote;
@@ -2061,18 +2089,36 @@ class Uploader {
      */
     async putChunked(repositoryId, declaration, profile, onOffset) {
         /*
-          Read in windows, not all at once.
+          Three steps, and each file is read for as long as it has to be.
     
-          This used to load the whole file to chunk it, which for the files that
-          reach this path — everything past sixteen megabytes — meant an allocation
-          the size of the file. A three gigabyte video was a three gigabyte buffer
-          and an out-of-memory crash partway through an upload, which is the one
-          failure worse than a slow one: it happens after the work.
+          This used to walk the file once, asking the service about every four
+          chunks as it went and waiting for each answer before reading on — then
+          the send walked it again and asked all over again. A 10.6 GB project is
+          8,395 chunks: 2,099 lookups per pass, one after another, at about a
+          third of a second each. Twelve minutes of waiting per pass, on a stage
+          the window labels "Working out what to send", before a byte had moved.
     
-          The window holds at most one read plus one maximum chunk, whatever the
-          file is. Exactly how cbx.ts packs a bundle, and for the same reason.
+          Now: find where the file is cut and what each piece's digest is (the plan
+          does this; the send is handed the plan's answer), ask about every digest
+          at once — one request per five thousand — and then read only the pieces
+          that are not already held, by position, several at a time.
         */
-        const chunks = [];
+        const key = `${declaration.path}\n${declaration.sha256}`;
+        let layout = this.planning ? undefined : this.plannedLayouts.get(key);
+        /*
+          Walking the file is half this file's progress when it happens here and
+          none of it when the plan already did it, so the bar neither jumps back
+          between the two halves nor sits still through the first.
+        */
+        const walking = !layout;
+        if (!layout) {
+            layout = await this.layoutOf(declaration, profile, (offset) => onOffset(Math.floor(offset / 2)));
+            if (this.planning)
+                this.plannedLayouts.set(key, layout);
+        }
+        const already = walking ? declaration.size / 2 : 0;
+        const share = walking ? 0.5 : 1;
+        const chunks = new Array(layout.length);
         let reused = 0;
         let sent = 0;
         let done = 0;
@@ -2082,6 +2128,12 @@ class Uploader {
             else
                 sent += bytes;
         };
+        /*
+          Asked all at once, because nothing about the question needs the bytes.
+          The old batch was capped at four because each entry held a piece in
+          memory; the digests alone are sixty-four characters each.
+        */
+        const held = await this.storedInBulk(repositoryId, layout.map((piece) => piece.digest));
         /*
           A file's chunks go up several at a time.
     
@@ -2091,68 +2143,108 @@ class Uploader {
           out and only the big one was left — five lanes idle while one crawled.
           For a folder that is *one* large file, that was the whole upload.
     
-          Bounded tighter than the file lanes because each one holds a piece in
-          memory: four in flight of at most a maximum chunk each, rather than six.
+          Bounded because each lane holds one piece in memory: four in flight of
+          at most a maximum chunk each.
         */
         const CHUNK_LANES = 4;
-        const active = new Set();
-        const dispatch = async (piece, at, digest) => {
-            const task = this.sendChunk(repositoryId, declaration, piece, digest, chunks, at, note).finally(() => {
-                active.delete(task);
-                done += piece.length;
-                onOffset(done);
-            });
-            active.add(task);
-            if (active.size >= CHUNK_LANES)
-                await Promise.race(active);
-        };
         /*
-          Ask about a bounded group of chunks before sending any of them.
-    
-          The old path asked once per chunk, with four GETs in flight. Besides
-          spending one network round trip per piece, an unavailable lookup was
-          deliberately treated as a miss, so a transient failure retransmitted
-          every otherwise reusable chunk. The website already uses the bulk
-          endpoint. Reusing it here gives Desktop and CLI the same wire behaviour
-          while retaining the safe "send it if unsure" fallback.
-    
-          A batch is no larger than the existing chunk lane count, so this does
-          not increase the memory ceiling: at most four bounded pieces are held.
+          One verdict for this file, shared by its lanes, and consulted only where
+          the plan has not already decided. The send takes each chunk's encoding
+          from the plan instead, because the service holds every object to exactly
+          what was quoted and rejects anything else — and this verdict depends on
+          which of four lanes finishes compressing first, so two passes asking it
+          separately could disagree about a chunk right where it gives up.
         */
-        const batch = [];
-        const flush = async () => {
-            if (!batch.length)
-                return;
-            const ready = batch.splice(0, batch.length);
-            const held = await this.storedInBulk(repositoryId, ready.map((item) => item.digest));
-            for (const item of ready) {
-                const stored = held.get(item.digest);
-                if (stored) {
-                    note("reused", 0);
-                    chunks[item.at] = {
-                        objectId: stored.objectId,
-                        sourceSize: item.piece.length,
-                        storedSize: stored.size,
-                    };
-                    done += item.piece.length;
-                    onOffset(done);
-                    continue;
+        const gzip = (0, compression_policy_js_1.freshGzipVerdict)();
+        const pieces = layout;
+        const file = await (0, promises_1.open)(declaration.full, "r");
+        try {
+            let cursor = 0;
+            const lane = async () => {
+                for (;;) {
+                    const at = cursor;
+                    cursor += 1;
+                    if (at >= pieces.length)
+                        return;
+                    this.check();
+                    const piece = pieces[at];
+                    const stored = held.get(piece.digest);
+                    if (stored) {
+                        note("reused", 0);
+                        chunks[at] = {
+                            objectId: stored.objectId,
+                            sourceSize: piece.length,
+                            storedSize: stored.size,
+                        };
+                    }
+                    else {
+                        const bytes = Buffer.allocUnsafe(piece.length);
+                        let got = 0;
+                        while (got < piece.length) {
+                            const { bytesRead } = await file.read(bytes, got, piece.length - got, piece.offset + got);
+                            if (!bytesRead)
+                                break;
+                            got += bytesRead;
+                        }
+                        /*
+                          Checked, because the layout may be the plan's and the file may
+                          have been written to since without its size or time moving. The
+                          service would refuse the piece anyway; this says why, in words.
+                        */
+                        if (got !== piece.length ||
+                            (0, node_crypto_1.createHash)("sha256").update(bytes).digest("hex") !== piece.digest) {
+                            throw new Error(`${declaration.path} changed while it was being saved. ` +
+                                "Save again to send what it holds now.");
+                        }
+                        await this.sendChunk(repositoryId, declaration, bytes, piece.digest, chunks, at, note, gzip);
+                    }
+                    done += piece.length;
+                    onOffset(Math.floor(already + done * share));
                 }
-                await dispatch(item.piece, item.at, item.digest);
+            };
+            await Promise.all(Array.from({ length: Math.min(CHUNK_LANES, pieces.length) }, () => lane()));
+        }
+        finally {
+            await file.close();
+        }
+        /*
+          Order is the file. Every position must be filled: a hole would mean a
+          chunk that never landed, and publishing around it would produce a version
+          whose bytes are individually verified and collectively wrong.
+    
+          Counted by index rather than with `some`, which skips the holes of a
+          sparse array — the one thing this check exists to find — and so could
+          never have fired.
+        */
+        for (let at = 0; at < pieces.length; at += 1) {
+            if (!chunks[at]) {
+                throw new Error(`${declaration.path}: a chunk did not finish uploading`);
             }
-            await Promise.all(active);
-        };
-        const queue = async (piece) => {
-            batch.push({
-                piece,
-                at: ordinal++,
-                digest: (0, node_crypto_1.createHash)("sha256").update(piece).digest("hex"),
-            });
-            if (batch.length >= CHUNK_LANES)
-                await flush();
+        }
+        return {
+            chunks: chunks,
+            reused,
+            sentBytes: sent,
         };
-        let ordinal = 0;
+    }
+    /**
+     * Where a file is cut into chunks, and the digest of each piece.
+     *
+     * Read in windows, not all at once. This used to load the whole file to
+     * chunk it, which for the files that reach this path meant an allocation the
+     * size of the file: a three gigabyte video was a three gigabyte buffer and an
+     * out-of-memory crash partway through an upload. The window holds at most
+     * one read plus one maximum chunk, whatever the file is — exactly how cbx.ts
+     * packs a bundle, and for the same reason.
+     *
+     * Nothing is kept but offsets and digests, so a ten gigabyte file costs a few
+     * hundred kilobytes to remember between the plan and the send.
+     */
+    async layoutOf(declaration, profile, onScanned) {
+        const layout = [];
+        const digestOf = (bytes) => (0, node_crypto_1.createHash)("sha256").update(bytes).digest("hex");
         let pending = Buffer.alloc(0);
+        let pendingAt = 0;
         for await (const block of (0, node_fs_1.createReadStream)(declaration.full, {
             highWaterMark: 8 * 1024 * 1024,
         })) {
@@ -2161,10 +2253,16 @@ class Uploader {
             pending = pending.length ? Buffer.concat([pending, incoming]) : incoming;
             let from = 0;
             for (const cut of (0, chunking_js_1.cutPoints)(pending, profile)) {
-                await queue(Buffer.from(pending.subarray(from, cut)));
+                layout.push({
+                    offset: pendingAt + from,
+                    length: cut - from,
+                    digest: digestOf(pending.subarray(from, cut)),
+                });
                 from = cut;
             }
+            pendingAt += from;
             pending = Buffer.from(pending.subarray(from));
+            onScanned(pendingAt);
         }
         /*
           cutPoints returns the boundaries *inside* what it was given and never the
@@ -2172,23 +2270,10 @@ class Uploader {
           mention — the packer in cbx.ts emits that tail itself, after its loop.
           Dropping it truncates every chunked file by exactly its last piece.
         */
-        if (pending.length)
-            await queue(pending);
-        await flush();
-        /*
-          Order is the file. Every position must be filled: a hole would mean a
-          chunk that never landed, and publishing around it would produce a version
-          whose bytes are individually verified and collectively wrong.
-        */
-        const ordered = chunks.slice(0, ordinal);
-        if (ordered.some((one) => !one)) {
-            throw new Error(`${declaration.path}: a chunk did not finish uploading`);
+        if (pending.length) {
+            layout.push({ offset: pendingAt, length: pending.length, digest: digestOf(pending) });
         }
-        return {
-            chunks: ordered,
-            reused,
-            sentBytes: sent,
-        };
+        return layout;
     }
     /** Send one chunk, reusing it if the account already holds those bytes. */
     async sendChunk(repositoryId, declaration, piece, digest, 
@@ -2199,10 +2284,29 @@ class Uploader {
       large file out of sequence, which produces bytes that are all present,
       all verified individually, and wrong.
     */
-    chunks, at, note) {
+    chunks, at, note, 
+    /** This file's running verdict on compression; see {@link noteChunkOutcome}. */
+    gzip) {
         {
             this.check();
-            const encoded = await (0, compress_js_1.encodeForUpload)(new Uint8Array(piece), declaration.path, await this.gzipAllowed());
+            /*
+              What the plan decided, where it decided. The quote names each object's
+              encoding and the service rejects a piece that arrives any other way,
+              so the send repeats the plan's answer rather than working one out. The
+              same bytes with the same answer compress to the same bytes, which is
+              what makes repeating it exact.
+            */
+            const planned = this.planning ? undefined : this.plannedObjects.get(digest);
+            const encoded = await (0, compress_js_1.encodeForUpload)(new Uint8Array(piece), declaration.path, planned
+                ? planned.encoding === "gzip"
+                : gzip.allowed && (await this.gzipAllowed()));
+            /*
+              Read the outcome back rather than the guess. A chunk that kept gzip
+              proves the file is worth compressing; a run of chunks that did not
+              proves it is not, and there is no reason to ask the rest of a gigabyte.
+            */
+            if (!planned)
+                (0, compression_policy_js_1.noteChunkOutcome)(gzip, encoded.encoding === "gzip");
             const query = encoded.encoding === "gzip"
                 ? `?kind=chunk&role=chunk&encoding=gzip` +
                     `&logicalSize=${piece.length}` +