@coderook/cli 0.34.4 → 0.34.6

package/.claude-plugin/plugin.json

@@ -2,7 +2,7 @@
   "name": "coderook",
   "displayName": "CodeRook",
   "description": "Save, browse and restore whole-snapshot versions of a project on CodeRook, with cbx or plain git, from Claude Code.",
-  "version": "0.34.4",
+  "version": "0.34.6",
   "author": {
     "name": "ACCA Gaming Productions",
     "url": "https://coderook.com"

package/dist/cbx/src/core/licences.js

@@ -21,6 +21,7 @@ exports.LICENCE_FILE = exports.DEFAULT_LICENCE = exports.LICENCES = void 0;
 exports.licenceById = licenceById;
 exports.licenceText = licenceText;
 exports.detectLicence = detectLicence;
+exports.existingLicence = existingLicence;
 exports.licenceNewProject = licenceNewProject;
 const promises_1 = require("node:fs/promises");
 const node_path_1 = __importDefault(require("node:path"));
@@ -281,6 +282,23 @@ function detectLicence(text) {
  *
  * Returns the licence written, or null when nothing was.
  */
+/*
+  The names a licence goes by, as a file or as a folder of them.
+
+  Only the exact name LICENSE was looked for, which missed nearly every way a
+  project actually carries its licence — LICENSE.md, LICENCE.txt, COPYING,
+  the LICENSE-MIT and LICENSE-APACHE pair, a licenses folder. Found by saving
+  a copy of Blender: its licences live in a folder called `license`, which on
+  Windows answers to "LICENSE" as well, so the check opened a directory and
+  the save failed. Had the folder been named anything else, the save would
+  have put an MIT licence beside Blender's GPL one.
+*/
+const LICENCE_NAME = /^(licen[cs]es?|copying|copyright|unlicen[cs]e)([-._ ].*)?$/i;
+/** The licence file or folder this folder already has, if any. */
+async function existingLicence(folder) {
+    const names = await (0, promises_1.readdir)(folder).catch(() => []);
+    return names.find((name) => LICENCE_NAME.test(name)) ?? null;
+}
 async function licenceNewProject(folder, holder, skip = false) {
     /*
       No name, no licence. A copyright line naming nobody is worse than the
@@ -289,10 +307,19 @@ async function licenceNewProject(folder, holder, skip = false) {
     */
     if (skip || !holder.trim())
         return null;
+    /* Any licence at all is somebody's decision, whatever it is called. */
+    if (await existingLicence(folder))
+        return null;
     const target = node_path_1.default.join(folder, exports.LICENCE_FILE);
-    const already = await (0, promises_1.readFile)(target, "utf8").catch(() => null);
-    if (already !== null)
+    /*
+      A nudge, never a reason to fail. A folder that cannot be written to is
+      saved without one, exactly as it would have been before this existed.
+    */
+    try {
+        await (0, promises_1.writeFile)(target, licenceText(exports.DEFAULT_LICENCE, holder.trim()), { encoding: "utf8", flag: "wx" });
+    }
+    catch {
         return null;
-    await (0, promises_1.writeFile)(target, licenceText(exports.DEFAULT_LICENCE, holder.trim()), "utf8");
+    }
     return exports.DEFAULT_LICENCE;
 }

package/dist/cbx/src/core/upload.js

@@ -88,10 +88,14 @@ function mediaTypeOf(file) {
     return MEDIA_TYPES[node_path_1.default.extname(file).toLowerCase()] ?? "application/octet-stream";
 }
 /** Hash without holding the file in memory; some of these are large. */
-async function digestOf(full) {
+async function digestOf(full, onBytes) {
     const hash = (0, node_crypto_1.createHash)("sha256");
-    for await (const chunk of (0, node_fs_1.createReadStream)(full))
+    let read = 0;
+    for await (const chunk of (0, node_fs_1.createReadStream)(full)) {
         hash.update(chunk);
+        read += chunk.byteLength;
+        onBytes?.(read);
+    }
     return hash.digest("hex");
 }
 /**
@@ -243,6 +247,38 @@ class Uploader {
     /** A single attempt. Everything about repeating it lives in `call`. */
     async attempt(route, init) {
         const started = performance.now();
+        /*
+          A body that says how much of itself has gone.
+    
+          A batch of small files is one request of a few megabytes, and on an
+          ordinary home connection three of them in flight take ten seconds each.
+          Counted only when the answer came back, the bar stood still for those
+          ten seconds and then jumped — and the last few batches of a save looked
+          exactly like a save that had stopped. Handed over in slices, each slice
+          taken is counted as it is taken, which is what the network has accepted.
+          A fresh stream per attempt, so a retry counts from nothing again.
+        */
+        const streamed = init.onSent && init.body instanceof Uint8Array && init.body.byteLength > 0
+            ? (() => {
+                const whole = init.body;
+                const report = init.onSent;
+                const SLICE = 256 * 1024;
+                let at = 0;
+                report(0);
+                return new ReadableStream({
+                    pull(controller) {
+                        if (at >= whole.byteLength) {
+                            controller.close();
+                            return;
+                        }
+                        const next = whole.subarray(at, Math.min(at + SLICE, whole.byteLength));
+                        at += next.byteLength;
+                        controller.enqueue(next);
+                        report(at / whole.byteLength);
+                    },
+                });
+            })()
+            : null;
         const token = await this.credentials.token();
         if (!token)
             throw new Error("Sign in again before uploading");
@@ -258,7 +294,8 @@ class Uploader {
                     : {}),
                 ...(init.contentType ? { "content-type": init.contentType } : {}),
             },
-            body: init.body,
+            body: streamed ?? init.body,
+            ...(streamed ? { duplex: "half" } : {}),
             /*
               The upload's own cancellation, and a deadline.
       
@@ -575,6 +612,42 @@ class Uploader {
         let cursor = 0;
         /** Throttled: twenty thousand sends is its own cost, and nobody reads them. */
         let told = 0;
+        /*
+          Bytes, not just files.
+    
+          This said how many files were done and nothing else, so a project whose
+          weight is four 1.6 GB builds sat on "1 of 11" for three minutes on a
+          hard disk and then leapt: every report came after a whole file. The
+          sizes are known from the survey, so how much has been read is a share of
+          a known total, and the lanes say how far into a large file they are.
+        */
+        const readTotal = sending.reduce((sum, file) => sum + file.size, 0);
+        let readDone = 0;
+        const readingNow = new Map();
+        const tell = (path, force = false) => {
+            const now = Date.now();
+            if (!force && now - told <= 250)
+                return;
+            told = now;
+            let bytes = readDone;
+            for (const partial of readingNow.values())
+                bytes += partial;
+            report({
+                stage: "hash",
+                files: hashed,
+                totalFiles: sending.length,
+                bytes: Math.min(bytes, readTotal),
+                totalBytes: readTotal,
+                path,
+                /*
+                  Reading is the stretch from just past the walk to a fifth of
+                  the way through. Starting it at nought put the bar behind where
+                  it had already been and it went backwards between stages.
+                */
+                percent: 8 + Math.round((hashed / sending.length) * 12),
+                bytesPerSecond: 0,
+            });
+        };
         const lane = async () => {
             for (;;) {
                 const index = cursor;
@@ -597,6 +670,7 @@ class Uploader {
                     vanished.push(file.path);
                     readFailures.set(file.path, error instanceof Error ? error.message : String(error));
                     hashed += 1;
+                    readDone += file.size;
                     continue;
                 }
                 /*
@@ -630,13 +704,19 @@ class Uploader {
                         mediaType: mediaTypeOf(file.path),
                     };
                     hashed += 1;
+                    readDone += file.size;
                     continue;
                 }
                 let digest;
                 try {
-                    digest = await (0, profile_js_1.timed)("hash files", () => digestOf(full));
+                    digest = await (0, profile_js_1.timed)("hash files", () => digestOf(full, (read) => {
+                        readingNow.set(index, read);
+                        tell(file.path);
+                    }));
                 }
                 catch (error) {
+                    readingNow.delete(index);
+                    readDone += file.size;
                     vanished.push(file.path);
                     readFailures.set(file.path, error instanceof Error ? error.message : String(error));
                     hashed += 1;
@@ -651,25 +731,9 @@ class Uploader {
                     mediaType: mediaTypeOf(file.path),
                 };
                 hashed += 1;
-                const now = Date.now();
-                if (now - told > 80 || hashed === sending.length) {
-                    told = now;
-                    report({
-                        stage: "hash",
-                        files: hashed,
-                        totalFiles: sending.length,
-                        bytes: 0,
-                        totalBytes: 0,
-                        path: file.path,
-                        /*
-                          Reading is the stretch from just past the walk to a fifth of
-                          the way through. Starting it at nought put the bar behind where
-                          it had already been and it went backwards between stages.
-                        */
-                        percent: 8 + Math.round((hashed / sending.length) * 12),
-                        bytesPerSecond: 0,
-                    });
-                }
+                readingNow.delete(index);
+                readDone += file.size;
+                tell(file.path, hashed === sending.length);
             }
         };
         await Promise.all(Array.from({ length: Math.min(HASH_LANES, sending.length) }, () => lane()));
@@ -868,6 +932,11 @@ class Uploader {
           objects are immutable and content-addressed, so a retry reuses them.
         */
         (0, faults_js_1.maybeFail)("submit:after-objects");
+        const counted = request.countLines
+            ? await (0, profile_js_1.timed)("count the changed lines", () => request.countLines()).catch(() => null)
+            : null;
+        const linesAdded = counted ? counted.added : request.linesAdded;
+        const linesRemoved = counted ? counted.removed : request.linesRemoved;
         const completed = await (0, profile_js_1.timed)("publish the version", () => this.call(`/v1/repositories/${repositoryId}/versions`, {
             method: "POST",
             contentType: "application/json",
@@ -886,10 +955,8 @@ class Uploader {
                 ...(request.allowIgnored ? { allowIgnored: true } : {}),
                 ...(this.trackedByGit.length ? { trackedByGit: this.trackedByGit } : {}),
                 ...(request.allowSecrets ? { allowSecrets: true } : {}),
-                ...(request.linesAdded === undefined ? {} : { linesAdded: request.linesAdded }),
-                ...(request.linesRemoved === undefined
-                    ? {}
-                    : { linesRemoved: request.linesRemoved }),
+                ...(linesAdded === undefined ? {} : { linesAdded }),
+                ...(linesRemoved === undefined ? {} : { linesRemoved }),
                 /*
                   Names this attempt so a retry after a lost connection is answered
                   with the version already made, rather than making a second one.
@@ -1090,7 +1157,23 @@ class Uploader {
           after which the service already holds the bytes and the copy costs a
           question rather than a transfer.
         */
-        const sending = new Map();
+        const inFlightByDigest = new Map();
+        /*
+          What the account already holds, asked once for the whole save.
+    
+          Asked per section, the answer changed under the save's feet. The plan
+          stores nothing, so every section of the plan heard the same answer; the
+          send stores as it goes and runs sections side by side, so a later
+          section holding a copy of a file an earlier one had just sent heard
+          "already held" and left it out of its pack. The pack's bytes, and so its
+          digest, were then not the pack the plan had priced, and the service
+          refused it as "a file that was not part of what it asked to send". A
+          copy of Blender 4.5 — 326 contents present more than once — failed this
+          way every time. One answer, taken before anything is sent, is the answer
+          the plan was given.
+        */
+        let heldOnce = null;
+        const heldAtStart = () => (heldOnce ??= (0, profile_js_1.timed)("ask what is already held", () => this.storedInBulk(repositoryId, everyDeclaration.map((one) => one.sha256))));
         const runSection = async (section) => {
             const declarations = section.files.map((file) => declarationByPath.get(file.path));
             /*
@@ -1150,7 +1233,7 @@ class Uploader {
             asking took longer than the transfer. The answer per digest is a boolean
             and an identifier, so the whole question fits in one request.
           */
-            const alreadyHeld = await (0, profile_js_1.timed)("ask what is already held", () => this.storedInBulk(repositoryId, declarations.map((one) => one.sha256)));
+            const alreadyHeld = await heldAtStart();
             /*
               The small files go first, packed together.
         
@@ -1202,6 +1285,19 @@ class Uploader {
             if (smallForPacking.length > 1) {
                 let pack = [];
                 let packBytes = 0;
+                /* How many batches have had a lane of their own on the bar. */
+                let batchLanes = 0;
+                /*
+                  How long a source byte takes to land in a batch on this connection,
+                  learned from the batches that have landed.
+          
+                  Counting the body as it is handed over is not enough on its own: the
+                  operating system takes a few megabytes into its send buffer at once,
+                  so a whole batch reads as sent within a moment and then waits seconds
+                  for the wire. A batch in flight is shown no further along than its
+                  time so far at the pace earlier batches actually managed.
+                */
+                let secondsPerByte = 0;
                 /*
                   Batches overlap rather than queue.
           
@@ -1216,6 +1312,15 @@ class Uploader {
                   that is reasonable to have outstanding.
                 */
                 const BATCH_LANES = 3;
+                /*
+                  Time against the expected time, as a share of a batch: in step until
+                  nine tenths, then closing on the end without reaching it, so a batch
+                  that runs late keeps moving and never reads as landed before its
+                  answer says so.
+                */
+                const creep = (expectedShare) => expectedShare <= 0.9
+                    ? Math.max(0, expectedShare)
+                    : 0.9 + 0.09 * (1 - Math.exp(-(expectedShare - 0.9) / 0.3));
                 const flying = new Set();
                 const flush = async () => {
                     if (!pack.length)
@@ -1223,6 +1328,52 @@ class Uploader {
                     const taking = pack;
                     pack = [];
                     packBytes = 0;
+                    /*
+                      This batch's own lane on the bar, keyed below zero so it can never
+                      collide with a file's lane. Its source bytes are shared out as its
+                      body goes, and taken back off when the batch lands and is counted
+                      for real.
+                    */
+                    batchLanes += 1;
+                    const lane = -batchLanes;
+                    const batchSource = taking.reduce((sum, item) => sum + item.declaration.size, 0);
+                    let toldAt = 0;
+                    let handed = 0;
+                    const sentAt = Date.now();
+                    const show = () => {
+                        /*
+                          Until a batch has landed there is no pace of this connection to go
+                          by, and the first batches read as sent within a moment and then
+                          stood still for seconds. A cautious guess stands in until the first
+                          answer: a megabyte and a half a second, shared by the batches in
+                          flight. Too slow a guess costs only a jump forward when the batch
+                          lands; too fast a one would be the stall again.
+                        */
+                        const pace = secondsPerByte > 0 ? secondsPerByte : BATCH_LANES / 1_500_000;
+                        const elapsed = (Date.now() - sentAt) / 1000;
+                        const share = batchSource > 0 ? Math.min(handed, creep(elapsed / (pace * batchSource))) : handed;
+                        progress.lane(lane, Math.round(batchSource * share));
+                        const now = Date.now();
+                        if (now - toldAt < 250)
+                            return;
+                        toldAt = now;
+                        report({
+                            stage: "upload",
+                            files: advance(0),
+                            totalFiles: everyDeclaration.length,
+                            bytes: progress.bytes(),
+                            totalBytes,
+                            path: taking[taking.length - 1].declaration.path,
+                            percent: 20 + Math.round((progress.bytes() / Math.max(totalBytes, 1)) * 72),
+                            bytesPerSecond: progress.rate(),
+                        });
+                    };
+                    const onSent = (fraction) => {
+                        handed = fraction;
+                        show();
+                    };
+                    /* Keeps the estimate moving while the batch waits on the wire. */
+                    const pacing = setInterval(show, 300);
                     const task = (async () => {
                         let exactWireBytes = null;
                         try {
@@ -1254,7 +1405,7 @@ class Uploader {
                                     batched.set(digest, one);
                             }
                             const placed = !delta && built
-                                ? await (0, profile_js_1.timed)("send packs", () => this.putPack(repositoryId, packItems, built))
+                                ? await (0, profile_js_1.timed)("send packs", () => this.putPack(repositoryId, packItems, built, onSent))
                                 : null;
                             if (placed) {
                                 (0, profile_js_1.counted)("files sent in packs", taking.length);
@@ -1267,7 +1418,7 @@ class Uploader {
                                   Not worth packing — already-compressed content, or too few files.
                                   They still travel together, just as separate objects.
                                 */
-                                const answer = await (0, profile_js_1.timed)("send batches", () => this.putBatch(repositoryId, taking));
+                                const answer = await (0, profile_js_1.timed)("send batches", () => this.putBatch(repositoryId, taking, onSent));
                                 (0, profile_js_1.counted)("files sent in batches", taking.length);
                                 for (const [digest, one] of answer)
                                     batched.set(digest, one);
@@ -1297,6 +1448,13 @@ class Uploader {
                           below and counted a second time — which is how the progress bar came
                           to report a hundred and twenty percent of a forty megabyte upload.
                         */
+                        /* Landed or refused, its estimate comes off: what landed is counted below. */
+                        clearInterval(pacing);
+                        progress.laneDone(lane);
+                        if (batchSource > 0 && taking.some((item) => batched.has(item.declaration.sha256) || packedInto.has(item.declaration.sha256))) {
+                            const took = (Date.now() - sentAt) / 1000 / batchSource;
+                            secondsPerByte = secondsPerByte > 0 ? secondsPerByte * 0.6 + took * 0.4 : took;
+                        }
                         for (const item of taking) {
                             const digest = item.declaration.sha256;
                             if (!batched.has(digest) && !packedInto.has(digest))
@@ -1317,7 +1475,10 @@ class Uploader {
                             percent: 20 + Math.round((progress.bytes() / Math.max(totalBytes, 1)) * 72),
                             bytesPerSecond: progress.rate(),
                         });
-                    })().finally(() => flying.delete(task));
+                    })().finally(() => {
+                        clearInterval(pacing);
+                        flying.delete(task);
+                    });
                     flying.add(task);
                     if (flying.size >= BATCH_LANES)
                         await Promise.race(flying);
@@ -1359,12 +1520,12 @@ class Uploader {
             const queue = declarations.entries();
             const runWorker = async () => {
                 for (const [index, declaration] of queue) {
-                    const earlier = sending.get(declaration.sha256);
+                    const earlier = inFlightByDigest.get(declaration.sha256);
                     if (earlier)
                         await earlier;
                     let release = () => { };
                     if (!earlier) {
-                        sending.set(declaration.sha256, new Promise((done) => {
+                        inFlightByDigest.set(declaration.sha256, new Promise((done) => {
                             release = done;
                         }));
                     }
@@ -1831,7 +1992,7 @@ class Uploader {
      * service would not take it — in which case the caller falls back to sending
      * them separately, which is slower and larger but always works.
      */
-    async putPack(repositoryId, items, alreadyBuilt) {
+    async putPack(repositoryId, items, alreadyBuilt, onSent) {
         const built = alreadyBuilt ?? await (0, solid_js_1.buildSolidPack)(items.map((item) => ({
             sha256: item.declaration.sha256,
             body: item.body,
@@ -1867,6 +2028,7 @@ class Uploader {
             method: "POST",
             contentType: "application/octet-stream",
             body: packed,
+            ...(onSent ? { onSent } : {}),
         });
         const placed = new Map();
         /*
@@ -2128,7 +2290,7 @@ class Uploader {
      * content-addressed, so a retry is free and a partial batch costs nothing
      * but the objects it missed.
      */
-    async putBatch(repositoryId, items) {
+    async putBatch(repositoryId, items, onSent) {
         const landed = new Map();
         if (!items.length)
             return landed;
@@ -2170,6 +2332,7 @@ class Uploader {
             method: "POST",
             contentType: "application/octet-stream",
             body: packed,
+            ...(onSent ? { onSent } : {}),
         });
         for (const one of answer.objects ?? []) {
             landed.set(one.sha256, {

package/dist/cbx/src/core/worktree.js

@@ -33,6 +33,7 @@ exports.detectPrivateDirectories = detectPrivateDirectories;
 exports.detectPastedCredentials = detectPastedCredentials;
 exports.uploadConcerns = uploadConcerns;
 exports.isCredentialByName = isCredentialByName;
+exports.holdsOnlyCertificates = holdsOnlyCertificates;
 exports.detectSecrets = detectSecrets;
 /** Reading a project folder: changed files, diffs, and rule measurement. */
 const node_child_process_1 = require("node:child_process");
@@ -1737,6 +1738,13 @@ onScanned) {
     const wanted = include.filter((relative) => (0, secret_patterns_js_1.worthReading)(relative.slice(relative.lastIndexOf("/") + 1)));
     let cursor = 0;
     let stop = false;
+    /*
+      Every file looked at, read or passed over for its size. Reporting only
+      those read left the count short of its own total: a refused save of
+      Blender stopped on "3700 of 3747 files", which reads as a scan that gave
+      up rather than one that finished.
+    */
+    let looked = 0;
     const lane = async () => {
         for (;;) {
             if (stop)
@@ -1745,6 +1753,9 @@ onScanned) {
             cursor += 1;
             if (at >= wanted.length)
                 return;
+            looked += 1;
+            if (onScanned && looked % 50 === 0)
+                onScanned(looked, wanted.length);
             if (found.length >= MOST_FINDINGS) {
                 stop = true;
                 return;
@@ -1768,8 +1779,6 @@ onScanned) {
             }
             read += 1;
             bytes += contents.byteLength;
-            if (onScanned && read % 50 === 0)
-                onScanned(read, wanted.length);
             if (!(0, secret_patterns_js_1.looksLikeText)(contents))
                 continue;
             const credentials = (0, secret_patterns_js_1.findCredentials)(contents.toString("utf8"));
@@ -1778,6 +1787,8 @@ onScanned) {
         }
     };
     await Promise.all(Array.from({ length: Math.min(LANES, wanted.length) }, () => lane()));
+    if (onScanned && wanted.length)
+        onScanned(looked, wanted.length);
     /*
       `stop` is also set by the findings ceiling, which is a different statement
       from running out of budget — forty findings is plenty to act on, and the
@@ -1847,6 +1858,38 @@ function isCredentialByName(relativePath) {
         SECRET_SUFFIXES.some((suffix) => name.endsWith(suffix)) ||
         parts.slice(0, -1).includes("secrets"));
 }
+/*
+  A `.pem` is a container, not a kind of file.
+
+  It holds a private key as readily as a certificate, and the name cannot
+  tell the two apart — so every one was refused. That refused Blender: its
+  bundled Python carries `certifi/cacert.pem` twice, Mozilla's list of the
+  certificate authorities a program should trust, which ships inside every
+  pip there is and is public by design. A certificate is the half of a key
+  pair that exists to be handed out.
+
+  So a `.pem` is opened and judged by its blocks. Only certificates (and
+  revocation lists) clear it; a private key anywhere in it, a block of any
+  other kind, no blocks at all, or a file too large or unreadable to look
+  at all leave it a credential, as before.
+*/
+const CERTIFICATE_BUNDLE_LIMIT = 4 * 1024 * 1024;
+async function holdsOnlyCertificates(full) {
+    try {
+        const info = await (0, promises_1.stat)(full);
+        if (info.size > CERTIFICATE_BUNDLE_LIMIT)
+            return false;
+        const text = await (0, promises_1.readFile)(full, "latin1");
+        if (/PRIVATE KEY/i.test(text))
+            return false;
+        const blocks = text.match(/-----BEGIN [A-Z0-9 ]+-----/g) ?? [];
+        return (blocks.length > 0 &&
+            blocks.every((block) => /^-----BEGIN (TRUSTED |X509 )?(CERTIFICATE|CRL)-----$/.test(block)));
+    }
+    catch {
+        return false;
+    }
+}
 async function detectSecrets(root) {
     const found = [];
     let seen = 0;
@@ -1896,8 +1939,12 @@ async function detectSecrets(root) {
                       that templates are not credentials.
                     */
                     const relative = node_path_1.default.relative(root, full).split(node_path_1.default.sep).join("/");
-                    if (isCredentialByName(relative))
-                        found.push(relative);
+                    if (!isCredentialByName(relative))
+                        continue;
+                    if (entry.name.toLowerCase().endsWith(".pem") && (await holdsOnlyCertificates(full))) {
+                        continue;
+                    }
+                    found.push(relative);
                 }
             }
         };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@coderook/cli",
-  "version": "0.34.4",
+  "version": "0.34.6",
   "description": "The command line for CodeRook (coderook.com): version control and project hosting for projects of any size. Every save is a compressed snapshot, private or public, never used to train AI.",
   "license": "Apache-2.0",
   "homepage": "https://coderook.com",