@coderook/cli 0.31.1 → 0.31.3

package/.claude-plugin/plugin.json

@@ -2,7 +2,7 @@
   "name": "coderook",
   "displayName": "CodeRook",
   "description": "Save, browse and restore whole-snapshot versions of a project on CodeRook, from Claude Code.",
-  "version": "0.31.1",
+  "version": "0.31.3",
   "author": {
     "name": "ACCA Gaming Productions",
     "url": "https://coderook.com"

package/dist/cli/src/cli.js

@@ -18,6 +18,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
 const promises_1 = require("node:readline/promises");
 const node_os_1 = __importDefault(require("node:os"));
 const promises_2 = require("node:fs/promises");
+const node_fs_1 = require("node:fs");
 const node_path_1 = __importDefault(require("node:path"));
 const node_process_1 = __importDefault(require("node:process"));
 const api_js_1 = require("./api.js");
@@ -385,6 +386,22 @@ async function commandStatus(parsed) {
             `— run ${accent("cbx get")} to bring ${stale.length === 1 ? "it" : "them"} down.`));
         for (const [path] of stale.slice(0, 10))
             console.log(dim(`  behind  ${path}`));
+        /*
+          Behind, and not here at all. Usually somebody else's file that has not
+          been fetched — but it is also what a file deleted here looked like after
+          a save that did not send the deletion, because until 0.31.2 that save
+          dropped it from this folder's record. Told plainly, since the one advice
+          above would bring back a file the person meant to be rid of.
+        */
+        const absent = stale.filter(([file]) => !(0, node_fs_1.existsSync)(node_path_1.default.join(folder, ...file.split("/"))));
+        if (absent.length) {
+            const one = absent.length === 1;
+            const which = absent.length === stale.length
+                ? one ? "It is not" : "They are not"
+                : `${absent.length} of them ${one ? "is" : "are"} not`;
+            console.log(dim(`${which} in this folder at all. If you deleted ${one ? "it" : "them"} yourself, ` +
+                `run ${accent("cbx get")}, delete ${one ? "it" : "them"} again, and save with ${accent("--sync")}.`));
+        }
     }
     /** The deletions, and what a plain save will do about them. */
     /*

package/dist/cli/src/git_history.js

@@ -22,6 +22,8 @@ exports.stamp = stamp;
 exports.importRef = importRef;
 exports.quotePath = quotePath;
 exports.refPlacements = refPlacements;
+exports.withinDepth = withinDepth;
+exports.historyLeftOut = historyLeftOut;
 /**
  * Git's constant hash for the empty tree.
  *
@@ -262,3 +264,59 @@ function refPlacements(wanted, markOfVersion) {
     }
     return { placed, unplaceable };
 }
+/**
+ * The versions within `depth` generations of these heads: 1 is the heads
+ * alone, 2 adds their parents, and so on.
+ *
+ * What `git clone --depth` means here. The oldest version taken is imported
+ * with no parent, so it becomes a root commit — which also means its commit id
+ * differs from the same version's id in a full clone, because a commit's id
+ * covers its parents. That is the same trade git's own shallow clones make.
+ */
+function withinDepth(heads, all, depth) {
+    const byId = new Map(all.map((version) => [version.id, version]));
+    const kept = new Set();
+    let level = heads.filter((id) => byId.has(id));
+    for (let generation = 0; generation < depth && level.length; generation += 1) {
+        const next = [];
+        for (const id of level) {
+            if (kept.has(id))
+                continue;
+            kept.add(id);
+            for (const parent of byId.get(id).parentVersionIds) {
+                if (byId.has(parent) && !kept.has(parent))
+                    next.push(parent);
+            }
+        }
+        level = next;
+    }
+    return kept;
+}
+/**
+ * History an earlier shallow clone chose not to take.
+ *
+ * Every ancestor of something already imported that was not itself imported.
+ * A full clone imported every ancestor, so for it this is always empty; for a
+ * shallow one it is everything below the cut. Without it the first ordinary
+ * `git fetch` after a `--depth 1` clone would import the whole history anyway,
+ * as a second, disconnected line of commits nobody asked for.
+ */
+function historyLeftOut(all, imported) {
+    const byId = new Map(all.map((version) => [version.id, version]));
+    const have = new Set(imported);
+    const left = new Set();
+    const pending = [];
+    for (const id of have) {
+        for (const parent of byId.get(id)?.parentVersionIds ?? [])
+            pending.push(parent);
+    }
+    while (pending.length) {
+        const id = pending.pop();
+        if (have.has(id) || left.has(id) || !byId.has(id))
+            continue;
+        left.add(id);
+        for (const parent of byId.get(id).parentVersionIds)
+            pending.push(parent);
+    }
+    return left;
+}

package/dist/cli/src/git_remote.js

@@ -544,8 +544,14 @@ async function recordPushedCommits(marks) {
   first and an empty window always accepts one more.
 */
 const BLOB_LANES = 8;
+/** Fewer new files than this, and one request for them each is quicker. */
+const ARCHIVE_MIN_FILES = 64;
+/** The share of a version's bytes that must be new before it is fetched whole. */
+const ARCHIVE_MIN_SHARE = 0.5;
 const BLOB_WINDOW_BYTES = 64 * 1024 * 1024;
-async function doImport(refs, url) {
+async function doImport(refs, url, 
+/** From `git clone --depth N`; only honoured on a first import. */
+depth = null) {
     const { owner, slug } = (0, git_history_js_1.parseRemoteUrl)(url);
     const project = await (0, api_js_1.findProject)(owner ? `${owner}/${slug}` : slug);
     if (!project) {
@@ -593,6 +599,26 @@ async function doImport(refs, url) {
     let nextMark = 1;
     for (const mark of commitMark.values())
         nextMark = Math.max(nextMark, mark + 1);
+    /*
+      What this import takes, out of everything the heads can reach.
+  
+      Never history an earlier shallow clone left out: its first ordinary fetch
+      would otherwise bring the lot in as a second line of commits. And on a
+      first import with `--depth`, only that many generations of each line.
+    */
+    const leftOut = (0, git_history_js_1.historyLeftOut)(all, commitMark.keys());
+    let chosen = order.filter((version) => !leftOut.has(version.id));
+    if (depth && depth > 0) {
+        if (commitMark.size) {
+            say(`  --depth applies to a new clone; fetching everything newer than what is here.`);
+        }
+        else {
+            const keep = (0, git_history_js_1.withinDepth)([...wanted.values()], all, depth);
+            chosen = chosen.filter((version) => keep.has(version.id));
+            say(`Taking the newest ${depth} version${depth === 1 ? "" : "s"} of each line ` +
+                `(--depth ${depth}); ${order.length - chosen.length} older left out.`);
+        }
+    }
     /*
       Point every ref git asked for at the commit its line's head became.
   
@@ -610,7 +636,7 @@ async function doImport(refs, url) {
             say(`  could not place ${ref}: no commit for its head version`);
         }
     };
-    const fresh = order.filter((version) => !commitMark.has(version.id));
+    const fresh = chosen.filter((version) => !commitMark.has(version.id));
     if (!fresh.length) {
         say(`Already up to date with ${slug}.`);
         /*
@@ -631,8 +657,8 @@ async function doImport(refs, url) {
         return;
     }
     say(`Importing ${fresh.length} version${fresh.length === 1 ? "" : "s"} from ${slug}` +
-        (order.length > fresh.length
-            ? ` (${order.length - fresh.length} already here)`
+        (chosen.length > fresh.length
+            ? ` (${chosen.length - fresh.length} already here)`
             : "") +
         ".");
     const { Downloader } = await Promise.resolve().then(() => __importStar(require("../../desktop-app/src/main/download.js")));
@@ -667,41 +693,7 @@ async function doImport(refs, url) {
               bytes together, because sixteen source files and sixteen video files
               are not the same amount to be holding at once.
             */
-            /*
-              The whole version in one request, where that works.
-      
-              Fetching per file is one round trip each: 21,071 of them on a real
-              Unity project, which is four hours in series and about half an hour
-              even eight at a time. The archive is the same bytes in one request.
-      
-              Anything unexpected about it — an entry this cannot read, a short
-              download, a service that does not offer the route — falls through to
-              the per-file path below, which is slow and known to work. A clone
-              that is slow is a nuisance; a clone that writes the wrong bytes is
-              not recoverable by trying again.
-            */
-            const fromArchive = new Map();
-            try {
-                const packed = await downloader.archive(repositoryId, version.id);
-                for (const entry of (0, stored_zip_js_1.readStoredZip)(packed)) {
-                    fromArchive.set(entry.path, entry.bytes);
-                }
-            }
-            catch (error) {
-                /*
-                  Said out loud, not swallowed.
-        
-                  This fell back silently, and the silence cost an hour: a deadline
-                  meant for ordinary requests was aborting the archive, the import
-                  dropped to fetching every file on its own, and from the outside it
-                  looked exactly like the slow clone the archive was added to fix.
-                  A fallback nobody can see is a fallback nobody can diagnose.
-                */
-                say(`  the archive for v${version.sequence} could not be used ` +
-                    `(${error instanceof Error ? error.message : String(error)}); ` +
-                    `fetching its files one by one`);
-                fromArchive.clear();
-            }
+            /* Only content not already imported: a blob is written once, by digest. */
             const seenBlob = new Set();
             const wantedBlobs = files.filter((file) => {
                 if (blobMark.has(file.sha256) || seenBlob.has(file.sha256))
@@ -709,6 +701,57 @@ async function doImport(refs, url) {
                 seenBlob.add(file.sha256);
                 return true;
             });
+            /*
+              The whole version in one request — but only when most of it is new.
+      
+              The archive is every file the version holds, and this used to fetch it
+              for every version and then keep only the files not already imported.
+              After the first, that is nearly none of them: a five-version clone of
+              a large project downloaded its whole snapshot five times over to use
+              the changed files out of each, which is why an archive added to speed
+              clones up left one taking eighty-two minutes.
+      
+              So it is weighed per version: where the files still wanted are at
+              least half of the version's bytes, one request for the lot wins; where
+              they are a handful, fetching just those does. Below a few dozen files
+              the round trips are not worth avoiding at all.
+      
+              Where it is worth it, fetching per file would be one round trip each:
+              21,071 of them on a real Unity project, four hours in series and about
+              half an hour even eight at a time.
+      
+              Anything unexpected about the archive — an entry this cannot read, a
+              short download, a service that does not offer the route — falls
+              through to the per-file path below, which is slow and known to work.
+              A clone that is slow is a nuisance; a clone that writes the wrong bytes
+              is not recoverable by trying again.
+            */
+            const bytesOf = (list) => list.reduce((sum, file) => sum + Number(file.sourceSize ?? 0), 0);
+            const archiveWorthIt = wantedBlobs.length >= ARCHIVE_MIN_FILES &&
+                bytesOf(wantedBlobs) >= bytesOf(files) * ARCHIVE_MIN_SHARE;
+            const fromArchive = new Map();
+            if (archiveWorthIt)
+                try {
+                    const packed = await downloader.archive(repositoryId, version.id);
+                    for (const entry of (0, stored_zip_js_1.readStoredZip)(packed)) {
+                        fromArchive.set(entry.path, entry.bytes);
+                    }
+                }
+                catch (error) {
+                    /*
+                      Said out loud, not swallowed.
+            
+                      This fell back silently, and the silence cost an hour: a deadline
+                      meant for ordinary requests was aborting the archive, the import
+                      dropped to fetching every file on its own, and from the outside it
+                      looked exactly like the slow clone the archive was added to fix.
+                      A fallback nobody can see is a fallback nobody can diagnose.
+                    */
+                    say(`  the archive for v${version.sequence} could not be used ` +
+                        `(${error instanceof Error ? error.message : String(error)}); ` +
+                        `fetching its files one by one`);
+                    fromArchive.clear();
+                }
             for (let at = 0; at < wantedBlobs.length;) {
                 const window = [];
                 let held = 0;
@@ -1393,6 +1436,8 @@ async function main(argv) {
       clone of a project you cannot see left an empty folder and exited zero.
     */
     let unservable = false;
+    /** `git clone --depth N`, when git passed one. */
+    let depth = null;
     for await (const line of lines()) {
         const command = line.trim();
         if (command === "capabilities") {
@@ -1406,9 +1451,19 @@ async function main(argv) {
             continue;
         }
         if (command.startsWith("option ")) {
-            // Nothing here changes behaviour yet, and claiming otherwise would make
-            // git believe a setting took effect.
-            send("unsupported");
+            /*
+              Only what is actually acted on is accepted: answering "ok" to anything
+              else would make git believe a setting took effect. Depth is honoured
+              by importing fewer versions; git passes it for `clone --depth`.
+            */
+            const [, name, value] = command.split(" ");
+            if (name === "depth" && /^[1-9][0-9]*$/.test(value ?? "")) {
+                depth = Number(value);
+                send("ok");
+            }
+            else {
+                send("unsupported");
+            }
             continue;
         }
         if (command === "list" || command === "list for-push") {
@@ -1521,7 +1576,7 @@ async function main(argv) {
             if (importing.length) {
                 const batch = importing.splice(0, importing.length);
                 try {
-                    await doImport(batch, url);
+                    await doImport(batch, url, depth);
                 }
                 catch (error) {
                     /*

package/dist/cli/src/stored_zip.js

@@ -42,11 +42,14 @@ function readStoredZip(archive) {
     const names = new TextDecoder("utf-8", { fatal: false });
     const entries = [];
     let at = 0;
+    let ended = false;
     while (at + 4 <= archive.length) {
         const signature = view.getUint32(at, true);
         /* Either record marks the end of the entries. */
-        if (signature === CENTRAL_HEADER || signature === END_OF_DIRECTORY)
+        if (signature === CENTRAL_HEADER || signature === END_OF_DIRECTORY) {
+            ended = true;
             break;
+        }
         if (signature !== LOCAL_HEADER) {
             throw new Error(`Not a stored archive: unexpected record at byte ${at}`);
         }
@@ -81,5 +84,42 @@ function readStoredZip(archive) {
         });
         at = end;
     }
+    /*
+      A zip is only whole once its directory arrives.
+  
+      Running out of bytes exactly between two entries used to read as the end
+      of the archive. It is also exactly what a download cut off at a file
+      boundary looks like — and the service's archive of a 9,010-file version
+      was being cut off, by a Worker limit, after 4,978 of them. Every one of
+      those read back perfectly, so the clone took them and fetched the other
+      4,032 one at a time without a word, which is why it took half an hour.
+      Now the directory has to be there, and has to count what was read.
+    */
+    if (!ended) {
+        throw new Error(`Archive was cut off after ${entries.length} entries: it has no directory`);
+    }
+    const counted = directoryCount(view);
+    if (counted === null) {
+        throw new Error("Archive has no end-of-directory record");
+    }
+    if (counted !== entries.length) {
+        throw new Error(`Archive says it holds ${counted} entries and ${entries.length} were read`);
+    }
     return entries;
 }
+/**
+ * How many entries the archive's end-of-directory record says it holds.
+ *
+ * Searched for from the end, as the format intends: the record is the last
+ * thing in the file, followed only by a comment of up to 64 KB.
+ */
+function directoryCount(view) {
+    const last = view.byteLength - 22;
+    const first = Math.max(0, last - 0xffff);
+    for (let at = last; at >= first; at -= 1) {
+        if (view.getUint32(at, true) === END_OF_DIRECTORY) {
+            return view.getUint16(at + 10, true);
+        }
+    }
+    return null;
+}

package/dist/desktop-app/src/main/upload.js

@@ -915,6 +915,53 @@ class Uploader {
         if (!completed.mergeTrack && !Object.keys(acceptedManifest).length) {
             acceptedManifest = Object.fromEntries([...(await this.versionFiles(repositoryId, completed.version.id))].map(([file, descriptor]) => [file, descriptor.sha256]));
         }
+        const manifest = Object.keys(acceptedManifest).length
+            ? acceptedManifest
+            : {
+                ...Object.fromEntries(reused.map((file) => [file.path, prior.get(file.path).sha256])),
+                ...Object.fromEntries(declarations.map((declaration) => [declaration.path, declaration.sha256])),
+            };
+        /*
+          What this folder holds after the save. A file that was sent holds
+          the bytes that were sent. Anything else keeps whatever digest was
+          recorded before: an unticked edit therefore stays pending rather
+          than looking saved, and a copy left stale by somebody else's merge
+          stays recognisably stale rather than looking like a new edit.
+    
+          Looked up in a map rather than by searching the declarations for each
+          file, which on a 47,000-file project was over a billion comparisons at
+          the very end of a save.
+        */
+        const sentDigest = new Map(declarations.map((declaration) => [declaration.path, declaration.sha256]));
+        const onDisk = new Set(everything.map((file) => file.path));
+        const local = Object.fromEntries(everything.flatMap((file) => {
+            const sent = sentDigest.get(file.path);
+            if (sent)
+                return [[file.path, sent]];
+            const before = request.known?.[file.path] ?? prior.get(file.path)?.sha256;
+            return before ? [[file.path, before]] : [];
+        }));
+        /*
+          And a file deleted here that this save carried forward keeps its place.
+    
+          A plain save adds and updates, so a file somebody deleted stays in the
+          version. It used to vanish from this record all the same, because the
+          record was built only from what is on disk — and the next status then
+          found the version holding a file this folder did not, called it
+          "behind", and offered to fetch it back. The person had deleted it; the
+          only thing that brought it back was being told to run `cbx get`.
+    
+          Only where the record already had it. A file that was never here — one
+          somebody else's save merged into the version — has no entry to keep, so
+          it stays "behind", which is exactly what it is.
+        */
+        for (const path of Object.keys(manifest)) {
+            if (onDisk.has(path))
+                continue;
+            const held = request.known?.[path];
+            if (held)
+                local[path] = held;
+        }
         report({
             stage: "done",
             files: declarations.length,
@@ -940,29 +987,8 @@ class Uploader {
             /** Selected, but the service already held the content. */
             alreadyStoredFiles: progress.alreadyOnAccount,
             telemetry: this.telemetry.snapshot(),
-            // A file that kept its old object records the digest of *that* copy,
-            // not of the file on disk, so an unticked edit is still pending next
-            // time rather than looking as though it had been saved.
-            /*
-              What this folder holds after the save. A file that was sent holds
-              the bytes that were sent. Anything else keeps whatever digest was
-              recorded before: an unticked edit therefore stays pending rather
-              than looking saved, and a copy left stale by somebody else's merge
-              stays recognisably stale rather than looking like a new edit.
-            */
-            local: Object.fromEntries(everything.flatMap((file) => {
-                const sent = declarations.find((one) => one.path === file.path);
-                if (sent)
-                    return [[file.path, sent.sha256]];
-                const before = request.known?.[file.path] ?? prior.get(file.path)?.sha256;
-                return before ? [[file.path, before]] : [];
-            })),
-            manifest: Object.keys(acceptedManifest).length
-                ? acceptedManifest
-                : {
-                    ...Object.fromEntries(reused.map((file) => [file.path, prior.get(file.path).sha256])),
-                    ...Object.fromEntries(declarations.map((declaration) => [declaration.path, declaration.sha256])),
-                },
+            local,
+            manifest,
         };
     }
     /**
@@ -1452,7 +1478,50 @@ class Uploader {
           Read once, here. Everything below works from the declarations rather
           than from the disk.
         */
-        const { declarations, totalBytes } = await this.hashSelection(request, sending, deletions, readFailures, vanished, report);
+        const hashed = await this.hashSelection(request, sending, deletions, readFailures, vanished, report);
+        /*
+          A file that is exactly what the last version held is kept, not sent.
+    
+          Both callers select everything — the command line says so with an empty
+          exclusion list, because its list of changes stops at twenty thousand —
+          and a selected file used to be a file to send. Most of a project is
+          small files, and small files are stored inside packs, so the service
+          cannot recognise one as already held; the whole lot was packed again and
+          sent again on every save. CodeBox's own saves sent 3.1 MB across 1,025
+          files to record a change to eight.
+    
+          The digest has just been taken, so the comparison is exact: same path,
+          same bytes, and the previous version's object for them — a pack slice,
+          its chunks, or a plain object — is named again instead.
+        */
+        const keptAsItWas = (declaration) => {
+            const was = prior.get(declaration.path);
+            return (!!was &&
+                was.sha256 === declaration.sha256 &&
+                !deletions.has(declaration.path) &&
+                !!(was.pack?.objectId || was.chunks?.length || was.objectId));
+        };
+        const declarations = hashed.declarations.filter((one) => !keptAsItWas(one));
+        let totalBytes = hashed.totalBytes;
+        for (const one of hashed.declarations) {
+            if (!keptAsItWas(one))
+                continue;
+            totalBytes -= one.size;
+            const held = prior.get(one.path);
+            reused.push({
+                path: one.path,
+                sourceSize: held.sourceSize,
+                storedSize: held.storedSize,
+                mediaType: held.mediaType,
+                added: 0,
+                removed: 0,
+                included: true,
+                deleted: false,
+                binary: false,
+                lines: 0,
+            });
+        }
+        (0, profile_js_1.counted)("files kept as they were", hashed.declarations.length - declarations.length);
         // 2. The project needs somewhere on the account to live.
         const repositoryId = await this.ensureRepository(request);
         // Learn the wire contract once, before the two section pipelines begin.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@coderook/cli",
-  "version": "0.31.1",
+  "version": "0.31.3",
   "description": "CodeRook from the command line, on any operating system",
   "license": "SEE LICENSE IN LICENSE.txt",
   "homepage": "https://coderook.com",