@coderook/cli 0.34.4 → 0.34.5

package/.claude-plugin/plugin.json

@@ -2,7 +2,7 @@
   "name": "coderook",
   "displayName": "CodeRook",
   "description": "Save, browse and restore whole-snapshot versions of a project on CodeRook, with cbx or plain git, from Claude Code.",
-  "version": "0.34.4",
+  "version": "0.34.5",
   "author": {
     "name": "ACCA Gaming Productions",
     "url": "https://coderook.com"

package/dist/cbx/src/core/upload.js

@@ -88,10 +88,14 @@ function mediaTypeOf(file) {
     return MEDIA_TYPES[node_path_1.default.extname(file).toLowerCase()] ?? "application/octet-stream";
 }
 /** Hash without holding the file in memory; some of these are large. */
-async function digestOf(full) {
+async function digestOf(full, onBytes) {
     const hash = (0, node_crypto_1.createHash)("sha256");
-    for await (const chunk of (0, node_fs_1.createReadStream)(full))
+    let read = 0;
+    for await (const chunk of (0, node_fs_1.createReadStream)(full)) {
         hash.update(chunk);
+        read += chunk.byteLength;
+        onBytes?.(read);
+    }
     return hash.digest("hex");
 }
 /**
@@ -243,6 +247,38 @@ class Uploader {
     /** A single attempt. Everything about repeating it lives in `call`. */
     async attempt(route, init) {
         const started = performance.now();
+        /*
+          A body that says how much of itself has gone.
+    
+          A batch of small files is one request of a few megabytes, and on an
+          ordinary home connection three of them in flight take ten seconds each.
+          Counted only when the answer came back, the bar stood still for those
+          ten seconds and then jumped — and the last few batches of a save looked
+          exactly like a save that had stopped. Handed over in slices, each slice
+          taken is counted as it is taken, which is what the network has accepted.
+          A fresh stream per attempt, so a retry counts from nothing again.
+        */
+        const streamed = init.onSent && init.body instanceof Uint8Array && init.body.byteLength > 0
+            ? (() => {
+                const whole = init.body;
+                const report = init.onSent;
+                const SLICE = 256 * 1024;
+                let at = 0;
+                report(0);
+                return new ReadableStream({
+                    pull(controller) {
+                        if (at >= whole.byteLength) {
+                            controller.close();
+                            return;
+                        }
+                        const next = whole.subarray(at, Math.min(at + SLICE, whole.byteLength));
+                        at += next.byteLength;
+                        controller.enqueue(next);
+                        report(at / whole.byteLength);
+                    },
+                });
+            })()
+            : null;
         const token = await this.credentials.token();
         if (!token)
             throw new Error("Sign in again before uploading");
@@ -258,7 +294,8 @@ class Uploader {
                     : {}),
                 ...(init.contentType ? { "content-type": init.contentType } : {}),
             },
-            body: init.body,
+            body: streamed ?? init.body,
+            ...(streamed ? { duplex: "half" } : {}),
             /*
               The upload's own cancellation, and a deadline.
       
@@ -575,6 +612,42 @@ class Uploader {
         let cursor = 0;
         /** Throttled: twenty thousand sends is its own cost, and nobody reads them. */
         let told = 0;
+        /*
+          Bytes, not just files.
+    
+          This said how many files were done and nothing else, so a project whose
+          weight is four 1.6 GB builds sat on "1 of 11" for three minutes on a
+          hard disk and then leapt: every report came after a whole file. The
+          sizes are known from the survey, so how much has been read is a share of
+          a known total, and the lanes say how far into a large file they are.
+        */
+        const readTotal = sending.reduce((sum, file) => sum + file.size, 0);
+        let readDone = 0;
+        const readingNow = new Map();
+        const tell = (path, force = false) => {
+            const now = Date.now();
+            if (!force && now - told <= 250)
+                return;
+            told = now;
+            let bytes = readDone;
+            for (const partial of readingNow.values())
+                bytes += partial;
+            report({
+                stage: "hash",
+                files: hashed,
+                totalFiles: sending.length,
+                bytes: Math.min(bytes, readTotal),
+                totalBytes: readTotal,
+                path,
+                /*
+                  Reading is the stretch from just past the walk to a fifth of
+                  the way through. Starting it at nought put the bar behind where
+                  it had already been and it went backwards between stages.
+                */
+                percent: 8 + Math.round((hashed / sending.length) * 12),
+                bytesPerSecond: 0,
+            });
+        };
         const lane = async () => {
             for (;;) {
                 const index = cursor;
@@ -597,6 +670,7 @@ class Uploader {
                     vanished.push(file.path);
                     readFailures.set(file.path, error instanceof Error ? error.message : String(error));
                     hashed += 1;
+                    readDone += file.size;
                     continue;
                 }
                 /*
@@ -630,13 +704,19 @@ class Uploader {
                         mediaType: mediaTypeOf(file.path),
                     };
                     hashed += 1;
+                    readDone += file.size;
                     continue;
                 }
                 let digest;
                 try {
-                    digest = await (0, profile_js_1.timed)("hash files", () => digestOf(full));
+                    digest = await (0, profile_js_1.timed)("hash files", () => digestOf(full, (read) => {
+                        readingNow.set(index, read);
+                        tell(file.path);
+                    }));
                 }
                 catch (error) {
+                    readingNow.delete(index);
+                    readDone += file.size;
                     vanished.push(file.path);
                     readFailures.set(file.path, error instanceof Error ? error.message : String(error));
                     hashed += 1;
@@ -651,25 +731,9 @@ class Uploader {
                     mediaType: mediaTypeOf(file.path),
                 };
                 hashed += 1;
-                const now = Date.now();
-                if (now - told > 80 || hashed === sending.length) {
-                    told = now;
-                    report({
-                        stage: "hash",
-                        files: hashed,
-                        totalFiles: sending.length,
-                        bytes: 0,
-                        totalBytes: 0,
-                        path: file.path,
-                        /*
-                          Reading is the stretch from just past the walk to a fifth of
-                          the way through. Starting it at nought put the bar behind where
-                          it had already been and it went backwards between stages.
-                        */
-                        percent: 8 + Math.round((hashed / sending.length) * 12),
-                        bytesPerSecond: 0,
-                    });
-                }
+                readingNow.delete(index);
+                readDone += file.size;
+                tell(file.path, hashed === sending.length);
             }
         };
         await Promise.all(Array.from({ length: Math.min(HASH_LANES, sending.length) }, () => lane()));
@@ -1090,7 +1154,7 @@ class Uploader {
           after which the service already holds the bytes and the copy costs a
           question rather than a transfer.
         */
-        const sending = new Map();
+        const inFlightByDigest = new Map();
         const runSection = async (section) => {
             const declarations = section.files.map((file) => declarationByPath.get(file.path));
             /*
@@ -1202,6 +1266,19 @@ class Uploader {
             if (smallForPacking.length > 1) {
                 let pack = [];
                 let packBytes = 0;
+                /* How many batches have had a lane of their own on the bar. */
+                let batchLanes = 0;
+                /*
+                  How long a source byte takes to land in a batch on this connection,
+                  learned from the batches that have landed.
+          
+                  Counting the body as it is handed over is not enough on its own: the
+                  operating system takes a few megabytes into its send buffer at once,
+                  so a whole batch reads as sent within a moment and then waits seconds
+                  for the wire. A batch in flight is shown no further along than its
+                  time so far at the pace earlier batches actually managed.
+                */
+                let secondsPerByte = 0;
                 /*
                   Batches overlap rather than queue.
           
@@ -1216,6 +1293,15 @@ class Uploader {
                   that is reasonable to have outstanding.
                 */
                 const BATCH_LANES = 3;
+                /*
+                  Time against the expected time, as a share of a batch: in step until
+                  nine tenths, then closing on the end without reaching it, so a batch
+                  that runs late keeps moving and never reads as landed before its
+                  answer says so.
+                */
+                const creep = (expectedShare) => expectedShare <= 0.9
+                    ? Math.max(0, expectedShare)
+                    : 0.9 + 0.09 * (1 - Math.exp(-(expectedShare - 0.9) / 0.3));
                 const flying = new Set();
                 const flush = async () => {
                     if (!pack.length)
@@ -1223,6 +1309,52 @@ class Uploader {
                     const taking = pack;
                     pack = [];
                     packBytes = 0;
+                    /*
+                      This batch's own lane on the bar, keyed below zero so it can never
+                      collide with a file's lane. Its source bytes are shared out as its
+                      body goes, and taken back off when the batch lands and is counted
+                      for real.
+                    */
+                    batchLanes += 1;
+                    const lane = -batchLanes;
+                    const batchSource = taking.reduce((sum, item) => sum + item.declaration.size, 0);
+                    let toldAt = 0;
+                    let handed = 0;
+                    const sentAt = Date.now();
+                    const show = () => {
+                        /*
+                          Until a batch has landed there is no pace of this connection to go
+                          by, and the first batches read as sent within a moment and then
+                          stood still for seconds. A cautious guess stands in until the first
+                          answer: a megabyte and a half a second, shared by the batches in
+                          flight. Too slow a guess costs only a jump forward when the batch
+                          lands; too fast a one would be the stall again.
+                        */
+                        const pace = secondsPerByte > 0 ? secondsPerByte : BATCH_LANES / 1_500_000;
+                        const elapsed = (Date.now() - sentAt) / 1000;
+                        const share = batchSource > 0 ? Math.min(handed, creep(elapsed / (pace * batchSource))) : handed;
+                        progress.lane(lane, Math.round(batchSource * share));
+                        const now = Date.now();
+                        if (now - toldAt < 250)
+                            return;
+                        toldAt = now;
+                        report({
+                            stage: "upload",
+                            files: advance(0),
+                            totalFiles: everyDeclaration.length,
+                            bytes: progress.bytes(),
+                            totalBytes,
+                            path: taking[taking.length - 1].declaration.path,
+                            percent: 20 + Math.round((progress.bytes() / Math.max(totalBytes, 1)) * 72),
+                            bytesPerSecond: progress.rate(),
+                        });
+                    };
+                    const onSent = (fraction) => {
+                        handed = fraction;
+                        show();
+                    };
+                    /* Keeps the estimate moving while the batch waits on the wire. */
+                    const pacing = setInterval(show, 300);
                     const task = (async () => {
                         let exactWireBytes = null;
                         try {
@@ -1254,7 +1386,7 @@ class Uploader {
                                     batched.set(digest, one);
                             }
                             const placed = !delta && built
-                                ? await (0, profile_js_1.timed)("send packs", () => this.putPack(repositoryId, packItems, built))
+                                ? await (0, profile_js_1.timed)("send packs", () => this.putPack(repositoryId, packItems, built, onSent))
                                 : null;
                             if (placed) {
                                 (0, profile_js_1.counted)("files sent in packs", taking.length);
@@ -1267,7 +1399,7 @@ class Uploader {
                                   Not worth packing — already-compressed content, or too few files.
                                   They still travel together, just as separate objects.
                                 */
-                                const answer = await (0, profile_js_1.timed)("send batches", () => this.putBatch(repositoryId, taking));
+                                const answer = await (0, profile_js_1.timed)("send batches", () => this.putBatch(repositoryId, taking, onSent));
                                 (0, profile_js_1.counted)("files sent in batches", taking.length);
                                 for (const [digest, one] of answer)
                                     batched.set(digest, one);
@@ -1297,6 +1429,13 @@ class Uploader {
                           below and counted a second time — which is how the progress bar came
                           to report a hundred and twenty percent of a forty megabyte upload.
                         */
+                        /* Landed or refused, its estimate comes off: what landed is counted below. */
+                        clearInterval(pacing);
+                        progress.laneDone(lane);
+                        if (batchSource > 0 && taking.some((item) => batched.has(item.declaration.sha256) || packedInto.has(item.declaration.sha256))) {
+                            const took = (Date.now() - sentAt) / 1000 / batchSource;
+                            secondsPerByte = secondsPerByte > 0 ? secondsPerByte * 0.6 + took * 0.4 : took;
+                        }
                         for (const item of taking) {
                             const digest = item.declaration.sha256;
                             if (!batched.has(digest) && !packedInto.has(digest))
@@ -1317,7 +1456,10 @@ class Uploader {
                             percent: 20 + Math.round((progress.bytes() / Math.max(totalBytes, 1)) * 72),
                             bytesPerSecond: progress.rate(),
                         });
-                    })().finally(() => flying.delete(task));
+                    })().finally(() => {
+                        clearInterval(pacing);
+                        flying.delete(task);
+                    });
                     flying.add(task);
                     if (flying.size >= BATCH_LANES)
                         await Promise.race(flying);
@@ -1359,12 +1501,12 @@ class Uploader {
             const queue = declarations.entries();
             const runWorker = async () => {
                 for (const [index, declaration] of queue) {
-                    const earlier = sending.get(declaration.sha256);
+                    const earlier = inFlightByDigest.get(declaration.sha256);
                     if (earlier)
                         await earlier;
                     let release = () => { };
                     if (!earlier) {
-                        sending.set(declaration.sha256, new Promise((done) => {
+                        inFlightByDigest.set(declaration.sha256, new Promise((done) => {
                             release = done;
                         }));
                     }
@@ -1831,7 +1973,7 @@ class Uploader {
      * service would not take it — in which case the caller falls back to sending
      * them separately, which is slower and larger but always works.
      */
-    async putPack(repositoryId, items, alreadyBuilt) {
+    async putPack(repositoryId, items, alreadyBuilt, onSent) {
         const built = alreadyBuilt ?? await (0, solid_js_1.buildSolidPack)(items.map((item) => ({
             sha256: item.declaration.sha256,
             body: item.body,
@@ -1867,6 +2009,7 @@ class Uploader {
             method: "POST",
             contentType: "application/octet-stream",
             body: packed,
+            ...(onSent ? { onSent } : {}),
         });
         const placed = new Map();
         /*
@@ -2128,7 +2271,7 @@ class Uploader {
      * content-addressed, so a retry is free and a partial batch costs nothing
      * but the objects it missed.
      */
-    async putBatch(repositoryId, items) {
+    async putBatch(repositoryId, items, onSent) {
         const landed = new Map();
         if (!items.length)
             return landed;
@@ -2170,6 +2313,7 @@ class Uploader {
             method: "POST",
             contentType: "application/octet-stream",
             body: packed,
+            ...(onSent ? { onSent } : {}),
         });
         for (const one of answer.objects ?? []) {
             landed.set(one.sha256, {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@coderook/cli",
-  "version": "0.34.4",
+  "version": "0.34.5",
   "description": "The command line for CodeRook (coderook.com): version control and project hosting for projects of any size. Every save is a compressed snapshot, private or public, never used to train AI.",
   "license": "Apache-2.0",
   "homepage": "https://coderook.com",