@coderook/cli 0.12.0 → 0.14.0

package/dist/desktop-app/src/main/worktree.js

@@ -12,7 +12,9 @@ exports.collectLayers = collectLayers;
 exports.assertReadable = assertReadable;
 exports.branchName = branchName;
 exports.changedFiles = changedFiles;
+exports.surveyFiles = surveyFiles;
 exports.totalSize = totalSize;
+exports.fileSizes = fileSizes;
 exports.fileDiff = fileDiff;
 exports.projectTree = projectTree;
 exports.evaluateRules = evaluateRules;
@@ -20,6 +22,7 @@ exports.detectSecrets = detectSecrets;
 /** Reading a project folder: changed files, diffs, and rule measurement. */
 const node_child_process_1 = require("node:child_process");
 const node_crypto_1 = require("node:crypto");
+const node_fs_1 = require("node:fs");
 const promises_1 = require("node:fs/promises");
 const node_path_1 = __importDefault(require("node:path"));
 const node_util_1 = require("node:util");
@@ -47,24 +50,49 @@ async function git(root, ...args) {
  * file would produce an incomplete project that looked backed up, so likely
  * secrets are warned about instead (docs/UPLOAD_POLICY.md).
  */
-exports.STARTER_IGNORE = `# Dependencies and generated output
-node_modules/
-dist/
-build/
-out/
-.next/
-target/
-__pycache__/
-.venv/
-venv/
-*.log
-
-# Large model weights
-models/**
-*.safetensors
-*.ckpt
-*.pt
-*.pth
+exports.STARTER_IGNORE = `# Dependencies and generated output
+node_modules/
+dist/
+build/
+out/
+.next/
+target/
+__pycache__/
+.venv/
+venv/
+*.log
+
+# Large model weights
+models/**
+*.safetensors
+*.ckpt
+*.pt
+*.pth
+
+# Caches and local state
+.venv/
+.pytest_cache/
+.mypy_cache/
+.ruff_cache/
+.cache/
+*.pyc
+
+# A browser or Electron profile that has been left in the project folder.
+# These hold files another program keeps open — a LOCK that cannot be read
+# while it runs will stop a save outright — and nothing in them is the work.
+IndexedDB/
+Local Storage/
+Session Storage/
+Service Worker/
+Network/
+GPUCache/
+Code Cache/
+blob_storage/
+Local State
+Preferences
+
+# Archives of the project, inside the project
+*.cbx
 `;
 /** The shared rules file, committed with the project. */
 exports.IGNORE_FILE = ".gitignore";
@@ -196,10 +224,73 @@ async function branchName(root) {
 const LINE_COUNT_LIMIT = 4 * 1024 * 1024;
 /** Enough rows for any real project; past it the rail would be unusable. */
 const LISTED_FILE_LIMIT = 20_000;
-/** How a file reads: line count, or binary, or too big to bother. */
-async function measure(full, size) {
-    if (size > LINE_COUNT_LIMIT)
-        return { lines: 0, binary: true, hash: "" };
+/**
+ * The digest of a file, without holding it.
+ *
+ * Streamed because the files this exists for are the large ones, and reading
+ * a gigabyte into memory in order to hash it is the reason it was not being
+ * done at all.
+ */
+async function digestOf(full) {
+    try {
+        const hash = (0, node_crypto_1.createHash)("sha256");
+        for await (const block of (0, node_fs_1.createReadStream)(full, {
+            highWaterMark: 4 * 1024 * 1024,
+        })) {
+            hash.update(block);
+        }
+        return hash.digest("hex");
+    }
+    catch {
+        return "";
+    }
+}
+/**
+ * How a file reads: its line count, whether it is binary, and its digest.
+ *
+ * The size decides only whether the lines are counted. It used to decide
+ * whether the digest was taken as well, and those are not the same question:
+ * counting lines needs the whole file in memory, which is a real reason to
+ * stop at a few megabytes, while the digest is what decides whether the file
+ * changed at all.
+ *
+ * Returning no digest for a large file therefore did not mean "not measured",
+ * it meant "cannot match" — the comparison below skips a file only when its
+ * digest equals the recorded one, and an empty string never does. So every
+ * file over four megabytes was reported as changed on every scan, forever,
+ * however untouched it was. A project holding two large files offered to
+ * upload them again after each save, said "2 not uploaded yet" beside a panel
+ * confirming the account already held them, and could not be talked out of it
+ * by refreshing, because refreshing recomputed the same empty answer.
+ */
+async function measure(full, size, known) {
+    /*
+      Unchanged on disk, so unchanged in content. Checked before the size test
+      because the whole point is to answer without reading, and the files worth
+      not reading are the large ones.
+    */
+    const cached = known?.cached;
+    if (cached && cached.size === size && cached.mtimeMs === known.mtimeMs) {
+        return { lines: 0, binary: true, hash: cached.sha256 };
+    }
+    if (size > LINE_COUNT_LIMIT) {
+        /*
+          Read only when the answer could be "unchanged".
+    
+          The digest exists to be compared against a recorded one. A file the
+          record has never heard of is new whatever it hashes to, so taking the
+          digest decides nothing and costs a full read of the file.
+    
+          That is most of what a first scan is. Hashing large files was necessary
+          to stop them reading as changed forever, but doing it for files nothing
+          could match turned opening a six hundred megabyte project for the first
+          time into two minutes of reading to learn what the absence of a record
+          already said.
+        */
+        if (!known?.recorded)
+            return { lines: 0, binary: true, hash: "" };
+        return { lines: 0, binary: true, hash: await digestOf(full) };
+    }
     try {
         const contents = await (0, promises_1.readFile)(full);
         const binary = contents.subarray(0, 8192).includes(0);
@@ -222,7 +313,13 @@ async function measure(full, size) {
  * upload when no version has ever been saved. The filter rules decide what
  * is a candidate; the baseline decides what is new.
  */
-async function changedFiles(root, rules, baseline = null, mode = "add-and-update") {
+async function changedFiles(root, rules, baseline = null, mode = "add-and-update", 
+/*
+  Digests already taken, and what the files looked like when they were.
+  Read and written in place, so the caller keeps whatever this learns and
+  the next scan does not read the same unchanged gigabyte again.
+*/
+stats) {
     const layers = await collectLayers(root, rules);
     const files = [];
     const present = new Set();
@@ -258,14 +355,28 @@ async function changedFiles(root, rules, baseline = null, mode = "add-and-update
             if ((0, rules_js_1.excludes)(relative, false, layers))
                 continue;
             let size;
+            let mtimeMs;
             try {
-                size = (await (0, promises_1.stat)(full)).size;
+                const info = await (0, promises_1.stat)(full);
+                size = info.size;
+                mtimeMs = info.mtimeMs;
             }
             catch {
                 continue;
             }
             present.add(relative);
-            const measured = await measure(full, size);
+            const measured = await measure(full, size, {
+                mtimeMs,
+                cached: stats?.get(relative),
+                recorded: baseline?.has(relative) ?? false,
+            });
+            /*
+              Remembered whether or not it changed: the next scan wants to skip
+              reading this file again, and that is just as true of one that was
+              edited a moment ago as of one that never moves.
+            */
+            if (measured.hash)
+                stats?.set(relative, { size, mtimeMs, sha256: measured.hash });
             const saved = baseline?.get(relative);
             if (saved && measured.hash && saved === measured.hash)
                 continue;
@@ -300,6 +411,61 @@ async function changedFiles(root, rules, baseline = null, mode = "add-and-update
     return files.sort((left, right) => left.path.localeCompare(right.path));
 }
 /** The size on disk of everything the rules would upload. */
+/**
+ * Every file the rules would upload, with its size and nothing else.
+ *
+ * Deliberately not `changedFiles`. That function stops at twenty thousand rows
+ * because past it a list is unusable — a correct limit for a scrollbar and a
+ * catastrophic one for deciding what gets backed up, which is what it was also
+ * being used for. A project past that many files was silently uploaded in
+ * part.
+ *
+ * This walk has no cap and reads nothing: no line counting, no hashing, no
+ * content at all. That is what makes it affordable to run over everything
+ * before deciding anything.
+ */
+async function surveyFiles(root, rules) {
+    const layers = await collectLayers(root, rules);
+    const found = [];
+    const pending = [root];
+    while (pending.length) {
+        const directory = pending.pop();
+        let entries;
+        try {
+            entries = await (0, promises_1.readdir)(directory, { withFileTypes: true });
+        }
+        catch {
+            continue;
+        }
+        for (const entry of entries) {
+            const full = node_path_1.default.join(directory, entry.name);
+            if (entry.isSymbolicLink())
+                continue;
+            if ((0, rules_js_1.isUncounted)(entry.name))
+                continue;
+            const relative = node_path_1.default.relative(root, full).split(node_path_1.default.sep).join("/");
+            if (entry.isDirectory()) {
+                if ((0, rules_js_1.excludes)(relative, true, layers) &&
+                    !(0, rules_js_1.negationReachesInto)(relative, layers)) {
+                    continue;
+                }
+                pending.push(full);
+                continue;
+            }
+            if (!entry.isFile())
+                continue;
+            if ((0, rules_js_1.excludes)(relative, false, layers))
+                continue;
+            try {
+                found.push({ path: relative, size: (await (0, promises_1.stat)(full)).size });
+            }
+            catch {
+                /* Gone or unreadable since the walk began; the upload reports it. */
+            }
+        }
+    }
+    return found.sort((left, right) => left.path.localeCompare(right.path));
+}
 async function totalSize(root, files) {
     let total = 0;
     for (const file of files) {
@@ -312,6 +478,29 @@ async function totalSize(root, files) {
     }
     return total;
 }
+/**
+ * The same walk, keeping each file's size rather than only the sum.
+ *
+ * Used where something has to reason about which files are large — the
+ * exclusion suggestions, whose whole value is saying which folder is costing
+ * the upload its time. Separate from totalSize so that the common path still
+ * carries nothing it does not need.
+ */
+async function fileSizes(root, files) {
+    const sized = [];
+    for (const file of files) {
+        try {
+            sized.push({
+                path: file.path,
+                size: (await (0, promises_1.stat)(node_path_1.default.join(root, file.path))).size,
+            });
+        }
+        catch {
+            /* deleted since the scan; it cannot be measured and does not count */
+        }
+    }
+    return sized;
+}
 /** The unified diff for one file, including files git does not track yet. */
 async function fileDiff(root, file, ignoreWhitespace = false) {
     const output = await git(root, "diff", "--no-ext-diff", "--unified=3", ...(ignoreWhitespace ? ["--ignore-all-space"] : []), "--", file);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@coderook/cli",
-  "version": "0.12.0",
+  "version": "0.14.0",
   "description": "CodeRook from the command line, on any operating system",
   "license": "SEE LICENSE IN LICENSE.txt",
   "homepage": "https://coderook.com",