nukebase 2.5.1 → 2.5.2

package/lib/project-commands.js

@@ -177,34 +177,83 @@ function splitBySize(files, sizeOf, n) {
 }
 
 // One progress line shared by every stream of a transfer. counter() gives each
-// stream its own pass-through that adds to the shared total.
+// stream its own pass-through that adds to the shared total; add(n) counts
+// bytes moved without a stream (storage uploads; negative takes back a failed
+// attempt).
+//
+// The speed is the rate over the last few seconds, not the average since the
+// start, and it is redrawn on a timer rather than when bytes arrive:
+//  - An upload's bytes are counted as they enter the request. The first
+//    several MB land in Node's and the OS's send buffers almost at once
+//    (measured ~10 MB on Windows), so a whole-run average opened far above
+//    the line speed and took a long time to fall. The window starts only once
+//    the buffers have filled (SETTLE_MS after the first byte); from then on
+//    the counted bytes move at the real pace.
+//  - Redrawing only on data froze the line during a stall, then jumped.
+// Live redraws go to a terminal only (in a log or an AI agent they pile up);
+// the final line, with the whole-run average, is always printed.
 function progressMeter(totalBytes) {
-  let bytes = 0;
-  let last = 0;
+  const TICK_MS = 250;      // redraw interval
+  const SETTLE_MS = 1000;   // after the first byte: buffers filling, not measured
+  const WINDOW_MS = 5000;   // speed = bytes over (at most) this long
+  // Shortest span worth showing a speed for. Bytes leave in steps of a few MB
+  // (the OS frees send buffer in bursts), so a shorter span can catch a gap
+  // between steps and read 0.
+  const MIN_SPAN_MS = 2000;
+  const SMOOTH = 0.3;       // per-tick weight of the newest reading
+  const tty = !!process.stdout.isTTY;
   const start = Date.now();
+  let bytes = 0;
+  let firstAt = 0;
+  let speed = null;         // bytes per second, smoothed; null until measured
+  const samples = [];       // [time, bytes], from SETTLE_MS after the first byte
   const mb = (n) => (n / 1024 / 1024).toFixed(2);
-  const draw = (force) => {
+
+  const measure = () => {
     const now = Date.now();
-    if (!force && now - last < 100) return;
-    last = now;
-    const secs = (now - start) / 1000;
-    const speed = secs > 0 ? (bytes / 1024 / 1024) / secs : 0;
-    const remaining = Math.max(0, totalBytes - bytes);
-    const eta = speed > 0 ? formatEta((remaining / 1024 / 1024) / speed) : '--:--';
+    if (bytes > 0 && !firstAt) firstAt = now;
+    if (!firstAt || now - firstAt < SETTLE_MS) return;
+    // All of an upload counted: the tail is still in the send buffers, going
+    // out at the same pace. Hold the speed rather than let it decay to 0.
+    if (totalBytes > 0 && bytes >= totalBytes) return;
+    samples.push([now, bytes]);
+    while (samples.length > 2 && now - samples[1][0] >= WINDOW_MS) samples.shift();
+    const [t0, b0] = samples[0];
+    if (now - t0 < MIN_SPAN_MS) return;
+    const rate = Math.max(0, (bytes - b0) / ((now - t0) / 1000));
+    speed = speed === null ? rate : speed + SMOOTH * (rate - speed);
+  };
+  const line = () => {
+    const shown = totalBytes > 0 ? Math.min(bytes, totalBytes) : bytes;
     const of = totalBytes > 0 ? ` of ${mb(totalBytes).padStart(7)}` : '';
-    process.stdout.write(`\r   ${mb(bytes).padStart(7)}${of} MB || ${speed.toFixed(2).padStart(6)} MB/s || ETA ${String(eta).padStart(5)}    `);
+    const rate = speed === null ? '    -- MB/s' : `${(speed / 1024 / 1024).toFixed(2).padStart(6)} MB/s (${Math.round(speed * 8 / 1e6)} Mbps)`;
+    const remaining = Math.max(0, totalBytes - bytes);
+    const eta = totalBytes > 0 && remaining === 0 ? 'finishing'
+      : speed > 0 && totalBytes > 0 ? formatEta(remaining / speed) : '--:--';
+    return `\r   ${mb(shown).padStart(7)}${of} MB || ${rate} || ETA ${String(eta).padStart(5)}    `;
   };
+  const tick = () => { measure(); if (tty) process.stdout.write(line()); };
+  const timer = setInterval(tick, TICK_MS);
+  if (timer.unref) timer.unref();
+
   return {
     counter() {
-      return new Transform({ transform(chunk, enc, cb) { bytes += chunk.length; draw(false); cb(null, chunk); } });
+      return new Transform({ transform(chunk, enc, cb) { bytes += chunk.length; cb(null, chunk); } });
     },
-    // Bytes moved without a stream to count them (storage uploads).
-    add(n) { bytes += n; draw(false); },
+    add(n) { bytes = Math.max(0, bytes + n); },
     bytes: () => bytes,
+    // For a transfer that failed: stop redrawing and end the line, so the
+    // error printed next is not drawn over. done() is for one that finished.
+    stop() {
+      clearInterval(timer);
+      if (tty && firstAt) process.stdout.write('\n');
+    },
     done() {
-      draw(true);
+      clearInterval(timer);
+      if (tty) process.stdout.write(line());
       const secs = Math.max((Date.now() - start) / 1000, 0.001);
-      console.log(`\n   ✓ ${mb(bytes)} MB in ${secs.toFixed(1)}s (avg: ${(bytes / 1024 / 1024 / secs).toFixed(2)} MB/s)`);
+      const avg = bytes / secs;
+      console.log(`${tty ? '\n' : ''}   ✓ ${mb(bytes)} MB in ${secs.toFixed(1)}s (avg: ${(avg / 1024 / 1024).toFixed(2)} MB/s, ${Math.round(avg * 8 / 1e6)} Mbps)`);
     }
   };
 }
@@ -283,6 +332,9 @@ async function downloadStreams(dest, files, sizeOf, totalBytes) {
         if (!stopRequested) throw err;
       }
     }));
+  } catch (err) {
+    meter.stop();
+    throw err;
   } finally {
     process.off('SIGINT', onSigint);
   }
@@ -1670,7 +1722,9 @@ async function push() {
   if (storageUp.length) {
     console.log(`\n  File storage (${storageUp.length} file(s)):`);
     const meter = progressMeter(storageBytes);
-    storageFailed = await storage.upload(API, authHeaders, root, storageLocal, storageUp, (n) => meter.add(n));
+    try {
+      storageFailed = await storage.upload(API, authHeaders, root, storageLocal, storageUp, (n) => meter.add(n));
+    } catch (e) { meter.stop(); throw e; }
     meter.done();
     if (storageFailed.length) {
       console.error(`\n❌ ${storageFailed.length} file(s) did not reach the storage; push again to send them:`);
@@ -2436,7 +2490,9 @@ async function backup() {
   if (storageChanged.length) {
     console.log(`\n   File storage (${storageChanged.length} file(s)):`);
     const meter = progressMeter(storageBytes);
-    storageFailed = await storage.download(API, authHeaders, dest, store.files, storageChanged, () => meter.counter());
+    try {
+      storageFailed = await storage.download(API, authHeaders, dest, store.files, storageChanged, () => meter.counter());
+    } catch (e) { meter.stop(); throw e; }
     meter.done();
   }
 

package/lib/storage.js

@@ -50,8 +50,12 @@ async function manifest(API, authHeaders) {
 }
 
 // One request to the bucket, IPv4, failing on IDLE_MS without progress.
-// Resolves { status, headers, stream } once headers arrive.
-function request(method, url, body) {
+// Resolves { status, headers, stream } once headers arrive. With onSent, the
+// body goes out in SLICE pieces and onSent(n) is called as each one is handed
+// to the network, so the progress line moves while a part uploads instead of
+// jumping by a whole part (up to 16 MB) when it finishes.
+const SLICE = 256 * 1024;
+function request(method, url, body, onSent) {
   return new Promise((resolve, reject) => {
     const u = new URL(url);
     const mod = u.protocol === 'http:' ? http : https;
@@ -64,10 +68,37 @@ function request(method, url, body) {
     });
     req.setTimeout(IDLE_MS, () => req.destroy(new Error('the storage stopped answering')));
     req.on('error', reject);
-    req.end(body || undefined);
+    if (!body || !onSent) { req.end(body || undefined); return; }
+    let off = 0;
+    const writeMore = () => {
+      while (off < body.length) {
+        const end = Math.min(off + SLICE, body.length);
+        const n = end - off;
+        const more = req.write(body.subarray(off, end), (err) => { if (!err) onSent(n); });
+        off = end;
+        if (!more) { req.once('drain', writeMore); return; }
+      }
+      req.end();
+    };
+    writeMore();
   });
 }
 
+// One upload attempt whose bytes are reported as they go out. If the attempt
+// fails, what it reported is taken back, so a retried part is not counted
+// twice and the line cannot pass the total.
+async function counted(onBytes, attempt) {
+  let n = 0;
+  let over = false;   // a write callback landing after the failure adds nothing
+  try {
+    return await attempt((k) => { if (over) return; n += k; onBytes(k); });
+  } catch (e) {
+    over = true;
+    if (n) onBytes(-n);
+    throw e;
+  }
+}
+
 async function retry(what, fn) {
   let last;
   for (let i = 0; i < TRIES; i++) {
@@ -214,12 +245,11 @@ async function upload(API, authHeaders, root, local, keys, onBytes) {
       const plan = await postJson(API, authHeaders, '/storage-upload', { key, size });
       if (plan.url) {
         const body = fs.readFileSync(file);
-        await retry(key, async () => {
-          const r = await request('PUT', plan.url, body);
+        await retry(key, () => counted(onBytes, async (onSent) => {
+          const r = await request('PUT', plan.url, body, onSent);
           r.stream.resume();
           if (r.status !== 200) throw new Error(`HTTP ${r.status}`);
-        });
-        onBytes(size);
+        }));
         sent.push(key);
         return;
       }
@@ -230,13 +260,12 @@ async function upload(API, authHeaders, root, local, keys, onBytes) {
           const len = Math.min(plan.partSize, size - i * plan.partSize);
           const buf = Buffer.alloc(len);
           fs.readSync(fd, buf, 0, len, i * plan.partSize);
-          await retry(`${key} part ${i + 1}`, async () => {
-            const r = await request('PUT', url, buf);
+          await retry(`${key} part ${i + 1}`, () => counted(onBytes, async (onSent) => {
+            const r = await request('PUT', url, buf, onSent);
             r.stream.resume();
             if (r.status !== 200 || !r.headers.etag) throw new Error(`HTTP ${r.status}`);
             etags[i] = r.headers.etag;
-          });
-          onBytes(len);
+          }));
         });
         await postJson(API, authHeaders, '/storage-upload-complete', { key, uploadId: plan.uploadId, etags });
         sent.push(key);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "nukebase",
-  "version": "2.5.1",
+  "version": "2.5.2",
   "description": "NukeBase CLI: create, deploy and manage NukeBase projects",
   "bin": {
     "nukebase": "bin/nukebase.js"