@coderook/cli 0.27.0 → 0.29.0

package/dist/desktop-app/src/main/worktree.js

@@ -14,7 +14,6 @@ exports.branchName = branchName;
 exports.changedFiles = changedFiles;
 exports.surveyFiles = surveyFiles;
 exports.totalSize = totalSize;
-exports.fileSizes = fileSizes;
 exports.fileDiff = fileDiff;
 exports.projectTree = projectTree;
 exports.evaluateRules = evaluateRules;
@@ -55,49 +54,49 @@ async function git(root, ...args) {
  * file would produce an incomplete project that looked backed up, so likely
  * secrets are warned about instead (docs/UPLOAD_POLICY.md).
  */
-exports.STARTER_IGNORE = `# Dependencies and generated output
-node_modules/
-dist/
-build/
-out/
-.next/
-target/
-__pycache__/
-.venv/
-venv/
-*.log
-
-# Large model weights
-models/**
-*.safetensors
-*.ckpt
-*.pt
-*.pth
-
-# Caches and local state
-.venv/
-.pytest_cache/
-.mypy_cache/
-.ruff_cache/
-.cache/
-*.pyc
-
-# A browser or Electron profile that has been left in the project folder.
-# These hold files another program keeps open — a LOCK that cannot be read
-# while it runs will stop a save outright — and nothing in them is the work.
-IndexedDB/
-Local Storage/
-Session Storage/
-Service Worker/
-Network/
-GPUCache/
-Code Cache/
-blob_storage/
-Local State
-Preferences
-
-# Archives of the project, inside the project
-*.cbx
+exports.STARTER_IGNORE = `# Dependencies and generated output
+node_modules/
+dist/
+build/
+out/
+.next/
+target/
+__pycache__/
+.venv/
+venv/
+*.log
+
+# Large model weights
+models/**
+*.safetensors
+*.ckpt
+*.pt
+*.pth
+
+# Caches and local state
+.venv/
+.pytest_cache/
+.mypy_cache/
+.ruff_cache/
+.cache/
+*.pyc
+
+# A browser or Electron profile that has been left in the project folder.
+# These hold files another program keeps open — a LOCK that cannot be read
+# while it runs will stop a save outright — and nothing in them is the work.
+IndexedDB/
+Local Storage/
+Session Storage/
+Service Worker/
+Network/
+GPUCache/
+Code Cache/
+blob_storage/
+Local State
+Preferences
+
+# Archives of the project, inside the project
+*.cbx
 `;
 /** The shared rules file, committed with the project. */
 exports.IGNORE_FILE = ".gitignore";
@@ -318,19 +317,69 @@ async function measure(full, size, known) {
  * upload when no version has ever been saved. The filter rules decide what
  * is a candidate; the baseline decides what is new.
  */
+/**
+ * The same files, ordered so that a cap spends itself across the folder.
+ *
+ * Taken in walk order, a cap is handed to whichever directory the walk
+ * reached first — and the directory a person cares about is rarely that one.
+ * Round-robin by directory means the first row of every directory is chosen
+ * before the second row of any, so a folder of forty-five thousand cache
+ * files cannot bury the eight hundred beside it that are the work.
+ */
+function spreadAcrossDirectories(found) {
+    const byDirectory = new Map();
+    for (const one of found) {
+        const at = one.relative.slice(0, one.relative.lastIndexOf("/") + 1);
+        const held = byDirectory.get(at);
+        if (held)
+            held.push(one);
+        else
+            byDirectory.set(at, [one]);
+    }
+    const queues = [...byDirectory.values()];
+    const deepest = queues.reduce((most, queue) => Math.max(most, queue.length), 0);
+    const spread = [];
+    for (let round = 0; round < deepest; round++) {
+        for (const queue of queues) {
+            const one = queue[round];
+            if (one)
+                spread.push(one);
+        }
+    }
+    return spread;
+}
 async function changedFiles(root, rules, baseline = null, mode = "add-and-update", 
 /*
   Digests already taken, and what the files looked like when they were.
   Read and written in place, so the caller keeps whatever this learns and
   the next scan does not read the same unchanged gigabyte again.
 */
-stats) {
+stats, 
+/** Filled with how much of the folder the cap allowed through. */
+outcome) {
     const layers = await collectLayers(root, rules);
     const files = [];
     const present = new Set();
     const pending = [root];
-    while (pending.length && files.length < LISTED_FILE_LIMIT) {
-        const directory = pending.pop();
+    /** Everything the rules would send, including what the cap keeps out. */
+    const found = [];
+    const limit = outcome?.limit ?? LISTED_FILE_LIMIT;
+    /*
+      Walk the whole folder first, and decide what to read afterwards.
+  
+      These used to be one pass: walk until twenty thousand rows had been
+      collected, then stop where it stood. That made the cap decide the shape of
+      the answer — the walk was depth first, so the first directory it descended
+      into took the entire allowance. In a Unity project that is `Library/`,
+      forty-five thousand files of import cache, and `Assets/` — the actual game
+      — appeared in the list nowhere at all.
+  
+      Walking is cheap: a readdir and a stat, no file is opened. Reading is what
+      costs, so reading is what the cap should govern, and it can only be spent
+      well once the whole folder is known.
+    */
+    for (let next = 0; next < pending.length; next++) {
+        const directory = pending[next];
         let entries;
         try {
             entries = await (0, promises_1.readdir)(directory, { withFileTypes: true });
@@ -338,6 +387,7 @@ stats) {
         catch {
             continue;
         }
+        const wanted = [];
         for (const entry of entries) {
             const full = node_path_1.default.join(directory, entry.name);
             if (entry.isSymbolicLink())
@@ -359,43 +409,75 @@ stats) {
                 continue;
             if ((0, rules_js_1.excludes)(relative, false, layers))
                 continue;
-            let size;
-            let mtimeMs;
+            wanted.push({ full, relative });
+        }
+        /*
+          Sized a directory at a time rather than a file at a time. One `stat` is
+          a syscall's round trip, and forty-seven thousand of them taken strictly
+          one after another is most of the scan; asked for together they overlap.
+        */
+        const sized = await Promise.all(wanted.map(async (one) => {
             try {
-                const info = await (0, promises_1.stat)(full);
-                size = info.size;
-                mtimeMs = info.mtimeMs;
+                const info = await (0, promises_1.stat)(one.full);
+                return { ...one, size: info.size, mtimeMs: info.mtimeMs };
             }
             catch {
-                continue;
+                /* Gone or unreadable since the walk began; the upload reports it. */
+                return null;
             }
-            present.add(relative);
-            const measured = await measure(full, size, {
-                mtimeMs,
-                cached: stats?.get(relative),
-                recorded: baseline?.has(relative) ?? false,
-            });
+        }));
+        for (const one of sized) {
+            if (!one)
+                continue;
             /*
-              Remembered whether or not it changed: the next scan wants to skip
-              reading this file again, and that is just as true of one that was
-              edited a moment ago as of one that never moves.
+              Recorded as present for every file the walk saw, capped or not.
+      
+              Synchronize mode calls anything in the last version that is missing
+              from this set a deletion. Filling it only for the rows that fitted
+              meant a folder past the cap proposed deleting every file the cap had
+              kept out — files sitting right there on disk.
             */
-            if (measured.hash)
-                stats?.set(relative, { size, mtimeMs, sha256: measured.hash });
-            const saved = baseline?.get(relative);
-            if (saved && measured.hash && saved === measured.hash)
-                continue;
-            files.push({
-                path: relative,
-                // Against a saved version the true line delta needs the old copy;
-                // until a version exists to fetch, a changed file counts as rewritten.
-                added: measured.lines,
-                removed: 0,
-                included: true,
-                binary: measured.binary,
-            });
+            present.add(one.relative);
+            outcome?.sizes.push({ path: one.relative, size: one.size });
+            found.push(one);
         }
     }
+    /*
+      Read in an order that gives every directory a share, and stop at the cap.
+  
+      An unchanged file costs a digest and produces no row, so this runs until
+      the list is full rather than over a fixed slice — a folder where nothing
+      has changed still has to be walked through to find that out.
+    */
+    for (const one of spreadAcrossDirectories(found)) {
+        if (files.length >= limit)
+            break;
+        const { full, relative, size, mtimeMs } = one;
+        const measured = await measure(full, size, {
+            mtimeMs,
+            cached: stats?.get(relative),
+            recorded: baseline?.has(relative) ?? false,
+        });
+        /*
+          Remembered whether or not it changed: the next scan wants to skip
+          reading this file again, and that is just as true of one that was
+          edited a moment ago as of one that never moves.
+        */
+        if (measured.hash)
+            stats?.set(relative, { size, mtimeMs, sha256: measured.hash });
+        const saved = baseline?.get(relative);
+        if (saved && measured.hash && saved === measured.hash)
+            continue;
+        files.push({
+            path: relative,
+            // Against a saved version the true line delta needs the old copy;
+            // until a version exists to fetch, a changed file counts as rewritten.
+            added: measured.lines,
+            removed: 0,
+            included: true,
+            binary: measured.binary,
+        });
+    }
     // A file that was in the last version and is gone now is only a change in
     // synchronize mode. The safe default adds and updates, and leaves the
     // saved copy of a missing file alone (docs/UPLOAD_POLICY.md).
@@ -413,6 +495,15 @@ stats) {
             });
         }
     }
+    if (outcome) {
+        outcome.listed = files.length;
+        /*
+          Deletions are rows the walk never saw, so they are added to both sides
+          rather than left out of the total — otherwise a synchronize scan could
+          report listing more files than the folder was found to hold.
+        */
+        outcome.total = Math.max(found.length, files.length);
+    }
     return files.sort((left, right) => left.path.localeCompare(right.path));
 }
 /** The size on disk of everything the rules would upload. */
@@ -483,29 +574,6 @@ async function totalSize(root, files) {
     }
     return total;
 }
-/**
- * The same walk, keeping each file's size rather than only the sum.
- *
- * Used where something has to reason about which files are large — the
- * exclusion suggestions, whose whole value is saying which folder is costing
- * the upload its time. Separate from totalSize so that the common path still
- * carries nothing it does not need.
- */
-async function fileSizes(root, files) {
-    const sized = [];
-    for (const file of files) {
-        try {
-            sized.push({
-                path: file.path,
-                size: (await (0, promises_1.stat)(node_path_1.default.join(root, file.path))).size,
-            });
-        }
-        catch {
-            /* deleted since the scan; it cannot be measured and does not count */
-        }
-    }
-    return sized;
-}
 /** The unified diff for one file, including files git does not track yet. */
 async function fileDiff(root, file, ignoreWhitespace = false) {
     const output = await git(root, "diff", "--no-ext-diff", "--unified=3", ...(ignoreWhitespace ? ["--ignore-all-space"] : []), "--", file);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@coderook/cli",
-  "version": "0.27.0",
+  "version": "0.29.0",
   "description": "CodeRook from the command line, on any operating system",
   "license": "SEE LICENSE IN LICENSE.txt",
   "homepage": "https://coderook.com",