@coderook/cli 0.30.2 → 0.31.0

package/.claude-plugin/plugin.json

@@ -2,7 +2,7 @@
   "name": "coderook",
   "displayName": "CodeRook",
   "description": "Save, browse and restore whole-snapshot versions of a project on CodeRook, from Claude Code.",
-  "version": "0.30.2",
+  "version": "0.31.0",
   "author": {
     "name": "ACCA Gaming Productions",
     "url": "https://coderook.com"

package/dist/cli/src/cli.js

@@ -623,7 +623,21 @@ async function commandSubmit(parsed) {
       which is the wrong way round — so this is that capability, asked for
       plainly.
     */
-    const files = await (0, worktree_js_1.changedFiles)(folder, rules, baseline, hasFlag(parsed, "sync") ? "synchronize" : "add-and-update");
+    /*
+      Digests this machine has taken before, read before the scan rather than
+      before the upload.
+  
+      The uploader has always accepted these and the desktop has always passed
+      them, but on the command line the expensive part happens earlier: the
+      scan below reads and hashes the whole folder to decide what changed, and
+      that is where a 21,071-file project spent 169 of its 299 seconds. The
+      same map serves both — `changedFiles` fills it in place, so what the scan
+      learns is what the upload is handed.
+    */
+    const knownDigests = link?.repositoryId
+        ? await (0, config_js_1.readDigests)(link.repositoryId)
+        : new Map();
+    const files = await (0, worktree_js_1.changedFiles)(folder, rules, baseline, hasFlag(parsed, "sync") ? "synchronize" : "add-and-update", knownDigests);
     /*
       An upgraded folder with no materialisation record and something that
       differs from its recorded version is ambiguous in the one way that
@@ -788,6 +802,7 @@ Pass ${accent("--allow-secrets")} if these are not real keys.`);
     */
     const uploadRequest = (0, publish_js_1.uploadRequestFor)({
         localPath: folder,
+        ...(knownDigests.size ? { knownDigests } : {}),
         changed: files.filter((file) => !file.deleted).map((file) => file.path),
         deleted: files.filter((file) => file.deleted).map((file) => file.path),
         /*
@@ -994,6 +1009,15 @@ The connection failed: ${text}`));
         local: result.local,
         manifest: result.manifest,
     });
+    /*
+      What this save hashed, kept for the next one.
+  
+      Merged rather than replaced: `digestsTaken` holds only the files actually
+      read this time, and everything served from the existing record has to
+      survive or the next save reads it all again — which is precisely the cost
+      this exists to avoid.
+    */
+    await (0, config_js_1.writeDigests)(result.repositoryId, new Map([...knownDigests, ...uploader.digestsTaken()]));
     if (result.repeated) {
         console.log(`Already saved as ${accent(`v${result.sequence}`)} by an earlier attempt;` +
             ` nothing was sent again.`);

package/dist/cli/src/config.js

@@ -13,6 +13,8 @@ exports.keyFor = keyFor;
 exports.readLink = readLink;
 exports.forgetLink = forgetLink;
 exports.writeLink = writeLink;
+exports.readDigests = readDigests;
+exports.writeDigests = writeDigests;
 /**
  * Where the command-line tool keeps its credential and its project links.
  *
@@ -182,3 +184,24 @@ async function writeLink(localPath, link) {
     };
     await writePrivate(linksFile(), JSON.stringify(links, null, 2));
 }
+const digestsFile = (projectId) => node_path_1.default.join(configDirectory(), "digests", `${projectId}.json`);
+async function readDigests(projectId) {
+    try {
+        const raw = await (0, promises_1.readFile)(digestsFile(projectId), "utf8");
+        return new Map(Object.entries(JSON.parse(raw)));
+    }
+    catch {
+        /* No record yet, or one this build cannot read: hash everything. */
+        return new Map();
+    }
+}
+async function writeDigests(projectId, digests) {
+    if (!digests.size)
+        return;
+    try {
+        await writePrivate(digestsFile(projectId), JSON.stringify(Object.fromEntries(digests)));
+    }
+    catch {
+        /* The next save hashes again, which is where this started. */
+    }
+}

package/dist/cli/src/git_remote.js

@@ -114,6 +114,7 @@ const node_os_1 = __importDefault(require("node:os"));
 const node_path_1 = __importDefault(require("node:path"));
 const node_process_1 = __importDefault(require("node:process"));
 const upload_js_1 = require("../../desktop-app/src/main/upload.js");
+const stored_zip_js_1 = require("./stored_zip.js");
 const tracks_js_1 = require("../../desktop-app/src/main/tracks.js");
 const identify_js_1 = require("../../desktop-app/src/main/identify.js");
 const config_js_1 = require("./config.js");
@@ -533,6 +534,17 @@ async function recordPushedCommits(marks) {
  * Contents are deduplicated by SHA-256 across the whole history, so a file
  * unchanged across a thousand versions is downloaded and sent to git once.
  */
+/*
+  How many blobs a clone fetches at once, and how much it will hold doing it.
+
+  Eight lanes for the same reason the folder walks use eight: these are
+  latency, not work, and the measured curve flattens there. The byte ceiling
+  is what stops a window of large files becoming a window of memory — one
+  oversized file still goes through on its own, because the count check comes
+  first and an empty window always accepts one more.
+*/
+const BLOB_LANES = 8;
+const BLOB_WINDOW_BYTES = 64 * 1024 * 1024;
 async function doImport(refs, url) {
     const { owner, slug } = (0, git_history_js_1.parseRemoteUrl)(url);
     const project = await (0, api_js_1.findProject)(owner ? `${owner}/${slug}` : slug);
@@ -639,22 +651,99 @@ async function doImport(refs, url) {
         let done = 0;
         for (const version of fresh) {
             const files = await downloader.files(repositoryId, version.id);
-            // Blobs first: fast-import requires a mark to exist before a commit
-            // names it.
-            for (const file of files) {
-                if (blobMark.has(file.sha256))
-                    continue;
-                const target = node_path_1.default.join(scratch, "blob.bin");
-                await (0, promises_1.rm)(target, { force: true });
-                await downloader.fileTo(repositoryId, version.id, file, target);
-                const bytes = await (0, promises_1.readFile)(target);
-                const mark = nextMark++;
-                blobMark.set(file.sha256, mark);
-                send("blob");
-                send(`mark :${mark}`);
-                send(`data ${bytes.length}`);
-                node_process_1.default.stdout.write(bytes);
-                node_process_1.default.stdout.write("\n");
+            /*
+              Blobs first: fast-import requires a mark to exist before a commit
+              names it. Fetched a window at a time and written out in order.
+      
+              This asked for one file, waited for it, wrote it to a scratch path,
+              read it back, and only then asked for the next — so a clone cost one
+              network round trip per file, in series. Against a 21,071-file project
+              that is 21,071 round trips at about seven tenths of a second each,
+              and a clone that never finished: measured over twenty seconds it was
+              using 0.17s of processor and moving two kilobytes.
+      
+              The stream fast-import reads has to stay ordered, so the fetching is
+              what overlaps, not the writing. The window is bounded by count and by
+              bytes together, because sixteen source files and sixteen video files
+              are not the same amount to be holding at once.
+            */
+            /*
+              The whole version in one request, where that works.
+      
+              Fetching per file is one round trip each: 21,071 of them on a real
+              Unity project, which is four hours in series and about half an hour
+              even eight at a time. The archive is the same bytes in one request.
+      
+              Anything unexpected about it — an entry this cannot read, a short
+              download, a service that does not offer the route — falls through to
+              the per-file path below, which is slow and known to work. A clone
+              that is slow is a nuisance; a clone that writes the wrong bytes is
+              not recoverable by trying again.
+            */
+            const fromArchive = new Map();
+            try {
+                const packed = await downloader.archive(repositoryId, version.id);
+                for (const entry of (0, stored_zip_js_1.readStoredZip)(packed)) {
+                    fromArchive.set(entry.path, entry.bytes);
+                }
+            }
+            catch (error) {
+                /*
+                  Said out loud, not swallowed.
+        
+                  This fell back silently, and the silence cost an hour: a deadline
+                  meant for ordinary requests was aborting the archive, the import
+                  dropped to fetching every file on its own, and from the outside it
+                  looked exactly like the slow clone the archive was added to fix.
+                  A fallback nobody can see is a fallback nobody can diagnose.
+                */
+                say(`  the archive for v${version.sequence} could not be used ` +
+                    `(${error instanceof Error ? error.message : String(error)}); ` +
+                    `fetching its files one by one`);
+                fromArchive.clear();
+            }
+            const seenBlob = new Set();
+            const wantedBlobs = files.filter((file) => {
+                if (blobMark.has(file.sha256) || seenBlob.has(file.sha256))
+                    return false;
+                seenBlob.add(file.sha256);
+                return true;
+            });
+            for (let at = 0; at < wantedBlobs.length;) {
+                const window = [];
+                let held = 0;
+                while (at < wantedBlobs.length && window.length < BLOB_LANES) {
+                    const size = Number(wantedBlobs[at].sourceSize ?? 0);
+                    if (window.length > 0 && held + size > BLOB_WINDOW_BYTES)
+                        break;
+                    held += size;
+                    window.push(wantedBlobs[at]);
+                    at += 1;
+                }
+                const fetched = await Promise.all(window.map(async (file, lane) => {
+                    /* Already in hand, if the archive carried it. */
+                    const packed = fromArchive.get(file.path);
+                    if (packed)
+                        return { file, bytes: packed };
+                    /*
+                      A scratch path per lane. They shared one name, which is fine
+                      while only one download is ever in flight and silently wrong
+                      the moment two are.
+                    */
+                    const target = node_path_1.default.join(scratch, `blob-${lane}.bin`);
+                    await (0, promises_1.rm)(target, { force: true });
+                    await downloader.fileTo(repositoryId, version.id, file, target);
+                    return { file, bytes: await (0, promises_1.readFile)(target) };
+                }));
+                for (const { file, bytes } of fetched) {
+                    const mark = nextMark++;
+                    blobMark.set(file.sha256, mark);
+                    send("blob");
+                    send(`mark :${mark}`);
+                    send(`data ${bytes.length}`);
+                    node_process_1.default.stdout.write(bytes);
+                    node_process_1.default.stdout.write("\n");
+                }
             }
             const mark = nextMark++;
             commitMark.set(version.id, mark);

package/dist/cli/src/publish.js

@@ -23,6 +23,11 @@ function uploadRequestFor(input) {
             ? { expectedHeadVersionId: input.baseVersionId }
             : {}),
         ...(input.known ? { known: input.known } : {}),
+        /*
+          Digests this machine has taken before, so an unchanged file is not read
+          again. The desktop has always passed these; the command line never did.
+        */
+        ...(input.knownDigests ? { knownDigests: input.knownDigests } : {}),
     };
 }
 function classifyPublishFailure(error) {

package/dist/cli/src/stored_zip.js

@@ -0,0 +1,85 @@
+"use strict";
+/**
+ * Reading the archive CodeRook builds, without a compression library.
+ *
+ * A clone fetched every file individually: one HTTP round trip per blob, in
+ * series. Against a 21,071-file project that is 21,071 round trips, which at
+ * seven tenths of a second each is four hours — and the clone that exposed
+ * this simply never finished. Fetching eight at a time cut it to about half
+ * an hour; asking for the version as one archive cuts it to one request.
+ *
+ * Written by hand for the same reason `backend/src/archive.ts` writes it by
+ * hand: the entries are *stored*, never deflated, so the whole job is a
+ * header, a body, and skipping the directory at the end. Pulling in a zip
+ * library to not decompress anything would be a poor trade.
+ *
+ * Only the format that writer produces is accepted. Anything else — a
+ * deflated entry, a size deferred to a data descriptor, a Zip64 field — is
+ * refused rather than guessed at, and the caller falls back to fetching the
+ * files one by one. A reader that quietly mangles a project is worse than a
+ * slow one.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.readStoredZip = readStoredZip;
+const LOCAL_HEADER = 0x04034b50;
+const CENTRAL_HEADER = 0x02014b50;
+/* An archive of nothing has no entries and no directory records — it opens
+   with this, and a reader that only knew the other two would refuse it. */
+const END_OF_DIRECTORY = 0x06054b50;
+/** Set in the general-purpose flags when sizes come after the body. */
+const SIZES_DEFERRED = 0x0008;
+const STORED = 0;
+/**
+ * Every entry in a stored archive, in the order it was written.
+ *
+ * Deliberately whole-buffer rather than streaming: the caller is holding one
+ * version's files to hand to fast-import, which needs them in order anyway,
+ * and a version large enough for this to matter is one the archive path
+ * should not be taking.
+ */
+function readStoredZip(archive) {
+    const view = new DataView(archive.buffer, archive.byteOffset, archive.byteLength);
+    const names = new TextDecoder("utf-8", { fatal: false });
+    const entries = [];
+    let at = 0;
+    while (at + 4 <= archive.length) {
+        const signature = view.getUint32(at, true);
+        /* Either record marks the end of the entries. */
+        if (signature === CENTRAL_HEADER || signature === END_OF_DIRECTORY)
+            break;
+        if (signature !== LOCAL_HEADER) {
+            throw new Error(`Not a stored archive: unexpected record at byte ${at}`);
+        }
+        if (at + 30 > archive.length)
+            throw new Error("Archive ends inside a header");
+        const flags = view.getUint16(at + 6, true);
+        const method = view.getUint16(at + 8, true);
+        const compressed = view.getUint32(at + 18, true);
+        const nameLength = view.getUint16(at + 26, true);
+        const extraLength = view.getUint16(at + 28, true);
+        if (method !== STORED) {
+            throw new Error(`Archive entry is compressed (method ${method})`);
+        }
+        if (flags & SIZES_DEFERRED) {
+            throw new Error("Archive entry defers its size to a data descriptor");
+        }
+        /*
+          0xffffffff is Zip64's marker for "the real size is in an extra field".
+          This writer never emits one, so seeing it means the archive did not come
+          from where it is assumed to have come from.
+        */
+        if (compressed === 0xffffffff)
+            throw new Error("Archive entry needs Zip64");
+        const nameAt = at + 30;
+        const bodyAt = nameAt + nameLength + extraLength;
+        const end = bodyAt + compressed;
+        if (end > archive.length)
+            throw new Error("Archive ends inside an entry");
+        entries.push({
+            path: names.decode(archive.subarray(nameAt, nameAt + nameLength)),
+            bytes: archive.subarray(bodyAt, end),
+        });
+        at = end;
+    }
+    return entries;
+}

package/dist/desktop-app/src/main/download.js

@@ -61,6 +61,23 @@ async function pruneEmpty(root, removed) {
         directory = node_path_1.default.dirname(directory);
     }
 }
+/**
+ * How long one request may go unanswered before it is treated as lost.
+ *
+ * The same three minutes the uploader uses, and for the same reason: what
+ * follows a deadline here is a retry, so the cost of it firing early on a
+ * genuinely slow call is one repeated request.
+ */
+const REQUEST_DEADLINE_MS = 180_000;
+/**
+ * And how long a whole-version archive may take to arrive.
+ *
+ * The service reads every file in the version out of object storage to build
+ * it, so the wait is proportional to the project rather than to the network.
+ * Fifteen minutes is far beyond any archive measured and still an answer
+ * rather than an indefinite wait.
+ */
+const ARCHIVE_DEADLINE_MS = 900_000;
 /**
  * A percentage for work that is still going.
  *
@@ -127,7 +144,18 @@ class Downloader {
             }
         }
     }
-    async attempt(route) {
+    async attempt(route, 
+    /*
+      Longer where the service has to build the answer before it can send it.
+
+      A deadline sized for an ordinary request aborts a large archive halfway
+      through making it, and the git import quietly falls back to fetching
+      every file on its own — so the fix for a slow clone became the cause of
+      one. The archive for a 21,071-file version takes about ninety seconds
+      to assemble and longer under load; three minutes is a request that has
+      failed, not one that is still working.
+    */
+    deadline = REQUEST_DEADLINE_MS) {
         const token = await this.credentials.token();
         if (!token)
             throw new Error("Sign in again before downloading");
@@ -137,7 +165,19 @@ class Downloader {
                 "user-agent": "CodeRook/0.1",
                 ...(0, identify_js_1.clientHeaders)(),
             },
-            signal: this.controller.signal,
+            /*
+              The download's own cancellation, and a deadline.
+      
+              This carried only the cancel signal, so a request the service never
+              answered waited for ever — the same defect that was fixed in the
+              uploader and in the command line's own client, and missed here. A
+              restore that stops with no bytes moving and no message is the worse
+              half of it: the person is left without the thing they were restoring.
+            */
+            signal: AbortSignal.any([
+                this.controller.signal,
+                AbortSignal.timeout(deadline),
+            ]),
         });
         if (!response.ok) {
             const text = await response.text().catch(() => "");
@@ -217,6 +257,22 @@ class Downloader {
             authorName: String(row.authorName ?? ""),
         }));
     }
+    /**
+     * A whole version as one archive, for a caller that wants all of it.
+     *
+     * The git import asked for each file in its own request, which on a
+     * 21,071-file project is 21,071 round trips — four hours in series, and
+     * about half an hour even eight at a time. The service already builds the
+     * whole version as a single stored archive for the download button; this
+     * is that, for a caller that would otherwise ask file by file.
+     *
+     * Returned whole rather than streamed because the caller is handing the
+     * files to `git fast-import`, which wants them one after another anyway.
+     */
+    async archive(repositoryId, versionId) {
+        const response = await this.attempt(`/v1/repositories/${repositoryId}/versions/${versionId}/archive`, ARCHIVE_DEADLINE_MS);
+        return new Uint8Array(await response.arrayBuffer());
+    }
     async files(repositoryId, versionId) {
         const body = await this.json(`/v1/repositories/${repositoryId}/versions/${versionId}/files`);
         return (body.files ?? []).map((row) => {

package/dist/desktop-app/src/main/upload.js

@@ -145,6 +145,20 @@ class Uploader {
     /** Whether this save has already said that packing was refused. */
     packRefusalSaid = false;
     plannedDigests = new Map();
+    /**
+     * Every digest this save actually took, for a caller that keeps a record.
+     *
+     * The desktop keeps one and hands it back through `knownDigests`, which is
+     * what stops an unchanged file being read twice. The command line had no
+     * way to keep one because this was private, so it re-read and re-hashed
+     * whole projects on every save.
+     *
+     * Only files that were read are in here — an entry served from the caller's
+     * own record was never taken and does not need writing down again.
+     */
+    digestsTaken() {
+        return new Map(this.plannedDigests);
+    }
     planningCreatedRepositoryId = null;
     /** Attached to every object write after the server admits the exact plan. */
     activeQuoteId = null;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@coderook/cli",
-  "version": "0.30.2",
+  "version": "0.31.0",
   "description": "CodeRook from the command line, on any operating system",
   "license": "SEE LICENSE IN LICENSE.txt",
   "homepage": "https://coderook.com",