@coderook/cli 0.30.1 → 0.31.0

package/dist/desktop-app/src/main/upload.js

@@ -92,6 +92,15 @@ async function digestOf(full) {
         hash.update(chunk);
     return hash.digest("hex");
 }
+/**
+ * How long one request may go unanswered before it is treated as lost.
+ *
+ * Three minutes, which is far longer than any request this makes has taken
+ * and short enough that a save cannot sit silent all afternoon. What follows
+ * a deadline is a retry, not a failure, so the cost of it firing early on a
+ * genuinely slow call is one repeated request.
+ */
+const REQUEST_DEADLINE_MS = 180_000;
 class UploadCancelled extends Error {
     constructor() {
         super("Upload cancelled");
@@ -136,6 +145,20 @@ class Uploader {
     /** Whether this save has already said that packing was refused. */
     packRefusalSaid = false;
     plannedDigests = new Map();
+    /**
+     * Every digest this save actually took, for a caller that keeps a record.
+     *
+     * The desktop keeps one and hands it back through `knownDigests`, which is
+     * what stops an unchanged file being read twice. The command line had no
+     * way to keep one because this was private, so it re-read and re-hashed
+     * whole projects on every save.
+     *
+     * Only files that were read are in here — an entry served from the caller's
+     * own record was never taken and does not need writing down again.
+     */
+    digestsTaken() {
+        return new Map(this.plannedDigests);
+    }
     planningCreatedRepositoryId = null;
     /** Attached to every object write after the server admits the exact plan. */
     activeQuoteId = null;
@@ -220,7 +243,24 @@ class Uploader {
                 ...(init.contentType ? { "content-type": init.contentType } : {}),
             },
             body: init.body,
-            signal: this.controller.signal,
+            /*
+              The upload's own cancellation, and a deadline.
+      
+              This carried only the cancel signal, so a request the service never
+              answered waited for ever: a real save of a 38,913-file project stopped
+              dead with one open connection, no bytes moving either way and no CPU,
+              and nothing anywhere said so. `worthRetrying` has always named "a
+              request that timed out" as the case worth repeating — there was simply
+              no way for one to time out, so the branch could not be reached.
+      
+              Generous on purpose. The service legitimately spends tens of seconds
+              on a large batch, and a deadline tight enough to cut that short would
+              turn slow saves into failing ones.
+            */
+            signal: AbortSignal.any([
+                this.controller.signal,
+                AbortSignal.timeout(REQUEST_DEADLINE_MS),
+            ]),
         });
         const text = await response.text();
         this.telemetry.request(route.split("?")[0] ?? route, init.body, new TextEncoder().encode(text).byteLength, performance.now() - started);

package/dist/desktop-app/src/main/worktree.js

@@ -21,6 +21,8 @@ exports.surveyFiles = surveyFiles;
 exports.totalSize = totalSize;
 exports.fileDiff = fileDiff;
 exports.projectTree = projectTree;
+exports.treeFrom = treeFrom;
+exports.walkProject = walkProject;
 exports.evaluateRules = evaluateRules;
 exports.detectPrivateDirectories = detectPrivateDirectories;
 exports.detectPastedCredentials = detectPastedCredentials;
@@ -871,56 +873,85 @@ async function surveyFiles(root, rules,
 onFound) {
     const layers = await collectLayers(root, rules);
     const found = [];
-    const pending = [root];
-    while (pending.length) {
-        const directory = pending.pop();
-        let entries;
-        try {
-            entries = await (0, promises_1.readdir)(directory, { withFileTypes: true });
-        }
-        catch {
-            continue;
-        }
-        for (const entry of entries) {
-            const full = node_path_1.default.join(directory, entry.name);
-            if (entry.isSymbolicLink())
-                continue;
-            if ((0, rules_js_1.isUncounted)(entry.name))
-                continue;
-            const relative = node_path_1.default.relative(root, full).split(node_path_1.default.sep).join("/");
-            if (entry.isDirectory()) {
-                if ((0, rules_js_1.excludes)(relative, true, layers) &&
-                    !(0, rules_js_1.negationReachesInto)(relative, layers)) {
+    /*
+      A level at a time, eight in flight.
+  
+      Asking the disk for one thing and waiting for the answer before asking
+      for the next is most of what this cost: the same walk with eight lanes
+      open is about four times quicker, and a ninth buys nothing. The order the
+      answers arrive in does not matter because the list is sorted at the end.
+    */
+    let level = [root];
+    while (level.length) {
+        const next = [];
+        let at = 0;
+        const lane = async () => {
+            while (at < level.length) {
+                const directory = level[at];
+                at += 1;
+                let entries;
+                try {
+                    entries = await (0, promises_1.readdir)(directory, { withFileTypes: true });
+                }
+                catch {
                     continue;
                 }
-                pending.push(full);
-                continue;
-            }
-            if (!entry.isFile())
-                continue;
-            if ((0, rules_js_1.excludes)(relative, false, layers))
-                continue;
-            try {
-                found.push({ path: relative, size: (await (0, promises_1.stat)(full)).size });
-                if (onFound && found.length % 500 === 0)
-                    onFound(found.length);
-            }
-            catch {
-                /* Gone or unreadable since the walk began; the upload reports it. */
+                const here = [];
+                for (const entry of entries) {
+                    const full = node_path_1.default.join(directory, entry.name);
+                    if (entry.isSymbolicLink())
+                        continue;
+                    if ((0, rules_js_1.isUncounted)(entry.name))
+                        continue;
+                    const relative = node_path_1.default.relative(root, full).split(node_path_1.default.sep).join("/");
+                    if (entry.isDirectory()) {
+                        if ((0, rules_js_1.excludes)(relative, true, layers) &&
+                            !(0, rules_js_1.negationReachesInto)(relative, layers)) {
+                            continue;
+                        }
+                        next.push(full);
+                        continue;
+                    }
+                    if (!entry.isFile())
+                        continue;
+                    if ((0, rules_js_1.excludes)(relative, false, layers))
+                        continue;
+                    here.push({ full, relative });
+                }
+                for (let from = 0; from < here.length; from += WALK_LANES) {
+                    const sized = await Promise.all(here.slice(from, from + WALK_LANES).map(async (one) => {
+                        try {
+                            return { path: one.relative, size: (await (0, promises_1.stat)(one.full)).size };
+                        }
+                        catch {
+                            /* Gone or unreadable since the walk began; the upload says so. */
+                            return null;
+                        }
+                    }));
+                    for (const one of sized) {
+                        if (!one)
+                            continue;
+                        found.push(one);
+                        if (onFound && found.length % 500 === 0)
+                            onFound(found.length);
+                    }
+                }
             }
-        }
+        };
+        await Promise.all(Array.from({ length: Math.min(WALK_LANES, level.length) }, () => lane()));
+        level = next;
     }
     return found.sort((left, right) => left.path.localeCompare(right.path));
 }
 async function totalSize(root, files) {
     let total = 0;
-    for (const file of files) {
-        try {
-            total += (await (0, promises_1.stat)(node_path_1.default.join(root, file.path))).size;
-        }
-        catch {
-            /* deleted since the scan; it contributes nothing */
-        }
+    /* Same reason as the walk: these are all waiting, so let them wait together. */
+    for (let from = 0; from < files.length; from += WALK_LANES) {
+        const sizes = await Promise.all(files.slice(from, from + WALK_LANES).map((file) => (0, promises_1.stat)(node_path_1.default.join(root, file.path)).then((info) => info.size, 
+        /* deleted since the scan; it contributes nothing */
+        () => 0)));
+        for (const size of sizes)
+            total += size;
     }
     return total;
 }
@@ -1018,71 +1049,168 @@ async function fileDiff(root, file, ignoreWhitespace = false) {
  * than by writing glob patterns. Sizes are rolled up, because the only way
  * to decide whether to keep a folder is to know what it costs.
  */
-async function projectTree(root, limit = exports.EVALUATION_FILE_LIMIT) {
-    let seen = 0;
-    const walk = async (directory, relative) => {
+/**
+ * The folder as a tree, built from one walk rather than another descent.
+ *
+ * It used to recurse the disk itself, in step with an evaluation doing the
+ * same descent alongside it — two serial passes over a folder that on a real
+ * Unity project holds 47,387 files. Now both are worked out from a single
+ * concurrent walk, so the filter screen reads the disk once.
+ */
+async function projectTree(root, limitOrWalked = exports.EVALUATION_FILE_LIMIT) {
+    const walked = typeof limitOrWalked === "number"
+        ? await walkProject(root, { limit: limitOrWalked })
+        : limitOrWalked;
+    return treeFrom(walked);
+}
+/** Fold the walk into nodes, carrying sizes and counts up as it goes. */
+function treeFrom(walked) {
+    const root = {
+        path: "",
+        name: ".",
+        directory: true,
+        size: 0,
+        files: 0,
+        children: [],
+        ...(walked.truncated ? { truncated: true } : {}),
+    };
+    const folders = new Map([["", root]]);
+    const folderAt = (relative) => {
+        const held = folders.get(relative);
+        if (held)
+            return held;
+        const cut = relative.lastIndexOf("/");
+        const parent = folderAt(cut < 0 ? "" : relative.slice(0, cut));
         const node = {
             path: relative,
-            name: relative ? relative.slice(relative.lastIndexOf("/") + 1) : ".",
+            name: relative.slice(cut + 1),
             directory: true,
             size: 0,
             files: 0,
             children: [],
         };
-        let entries;
-        try {
-            entries = await (0, promises_1.readdir)(directory, { withFileTypes: true });
-        }
-        catch {
-            return node;
-        }
-        for (const entry of entries) {
-            if (entry.isSymbolicLink())
-                continue;
-            if ((0, rules_js_1.isUncounted)(entry.name))
-                continue;
-            const full = node_path_1.default.join(directory, entry.name);
-            const childPath = relative ? `${relative}/${entry.name}` : entry.name;
-            if (entry.isDirectory()) {
-                const child = await walk(full, childPath);
-                node.children.push(child);
-                node.size += child.size;
-                node.files += child.files;
-                continue;
-            }
-            if (!entry.isFile())
-                continue;
-            seen += 1;
-            if (seen > limit) {
-                node.truncated = true;
-                break;
-            }
-            let size = 0;
-            try {
-                size = (await (0, promises_1.stat)(full)).size;
-            }
-            catch {
-                continue;
-            }
-            node.children.push({
-                path: childPath,
-                name: entry.name,
-                directory: false,
-                size,
-                files: 1,
-            });
-            node.size += size;
-            node.files += 1;
+        parent.children.push(node);
+        folders.set(relative, node);
+        return node;
+    };
+    /* Empty folders are folders, and a tree that hides them is a different
+       folder from the one on disk. */
+    for (const directory of walked.directories)
+        folderAt(directory);
+    for (const file of walked.files) {
+        const cut = file.path.lastIndexOf("/");
+        const parent = folderAt(cut < 0 ? "" : file.path.slice(0, cut));
+        parent.children.push({
+            path: file.path,
+            name: file.path.slice(cut + 1),
+            directory: false,
+            size: file.size,
+            files: 1,
+        });
+        /* Up the chain, so a folder can be judged without opening it. */
+        for (let at = parent; at;) {
+            at.size += file.size;
+            at.files += 1;
+            const cutAt = at.path.lastIndexOf("/");
+            at = at.path === "" ? undefined : folders.get(cutAt < 0 ? "" : at.path.slice(0, cutAt));
         }
-        // Folders first, then files, each alphabetically — the familiar order.
+    }
+    // Folders first, then files, each alphabetically — the familiar order.
+    for (const node of folders.values()) {
         node.children.sort((left, right) => left.directory === right.directory
             ? left.name.localeCompare(right.name)
             : left.directory
                 ? -1
                 : 1);
-        return node;
-    };
-    return walk(root, "");
+    }
+    return root;
+}
+/**
+ * Every file under a folder, with its size, walked eight at a time.
+ *
+ * The filter screen used to walk the same folder twice over, once to build
+ * the tree and once to apply the rules, and both walks asked the disk for one
+ * thing at a time. On a 47,387-file Unity project that is 47,387 serial stat
+ * calls, and a stat is almost entirely waiting: the same walk with eight in
+ * flight is 2,483ms down to 638ms, and more lanes than eight buy nothing.
+ *
+ * So there is one walk, it is concurrent, and the tree and the evaluation are
+ * both worked out from what it returns rather than from the disk.
+ */
+const WALK_LANES = 8;
+async function walkProject(root, options = {}) {
+    const limit = options.limit ?? exports.EVALUATION_FILE_LIMIT;
+    const files = [];
+    const directories = [];
+    let truncated = false;
+    let stopped = false;
+    /* Breadth-first, a level at a time, so the lanes always have work. */
+    let level = [root];
+    while (level.length && !stopped) {
+        const next = [];
+        let at = 0;
+        const lane = async () => {
+            while (at < level.length) {
+                if (stopped)
+                    return;
+                const directory = level[at];
+                at += 1;
+                if (options.shouldStop?.()) {
+                    stopped = true;
+                    return;
+                }
+                let entries;
+                try {
+                    entries = await (0, promises_1.readdir)(directory, { withFileTypes: true });
+                }
+                catch {
+                    continue;
+                }
+                const here = [];
+                for (const entry of entries) {
+                    if (entry.isSymbolicLink())
+                        continue;
+                    if ((0, rules_js_1.isUncounted)(entry.name))
+                        continue;
+                    const full = node_path_1.default.join(directory, entry.name);
+                    if (entry.isDirectory()) {
+                        next.push(full);
+                        directories.push(relativeTo(root, full));
+                        continue;
+                    }
+                    if (entry.isFile())
+                        here.push(full);
+                }
+                for (let from = 0; from < here.length; from += WALK_LANES) {
+                    const sized = await Promise.all(here.slice(from, from + WALK_LANES).map(async (full) => {
+                        try {
+                            return { full, size: (await (0, promises_1.stat)(full)).size };
+                        }
+                        catch {
+                            return null;
+                        }
+                    }));
+                    for (const one of sized) {
+                        if (!one)
+                            continue;
+                        if (files.length >= limit) {
+                            truncated = true;
+                            stopped = true;
+                            return;
+                        }
+                        files.push({ path: relativeTo(root, one.full), size: one.size });
+                    }
+                }
+            }
+        };
+        await Promise.all(Array.from({ length: Math.min(WALK_LANES, level.length) }, () => lane()));
+        level = next;
+    }
+    return { files, directories, truncated };
+}
+/** A path under the root, written the way the rules are. */
+function relativeTo(root, full) {
+    return node_path_1.default.relative(root, full).split(node_path_1.default.sep).join("/");
 }
 /**
  * Measure what the supplied rules would upload, skip and force-keep.
@@ -1092,7 +1220,6 @@ async function projectTree(root, limit = exports.EVALUATION_FILE_LIMIT) {
  */
 async function evaluateRules(root, rules, options = {}) {
     const layers = await collectLayers(root, rules);
-    const limit = options.limit ?? exports.EVALUATION_FILE_LIMIT;
     const impactBytes = new Map();
     const impactFiles = new Map();
     const excluded = [];
@@ -1102,89 +1229,65 @@ async function evaluateRules(root, rules, options = {}) {
     let skippedBytes = 0;
     let keptFiles = 0;
     let keptBytes = 0;
-    let seen = 0;
-    let truncated = false;
-    const pending = [root];
-    while (pending.length) {
-        if (options.shouldStop?.())
-            return emptyEvaluation(true);
-        const directory = pending.pop();
-        let entries;
-        try {
-            entries = await (0, promises_1.readdir)(directory, { withFileTypes: true });
-        }
-        catch {
-            continue;
-        }
-        for (const entry of entries) {
-            if (options.shouldStop?.())
-                return emptyEvaluation(true);
-            const full = node_path_1.default.join(directory, entry.name);
-            if (entry.isSymbolicLink())
-                continue;
-            if ((0, rules_js_1.isUncounted)(entry.name))
-                continue;
-            if (entry.isDirectory()) {
-                pending.push(full);
-                continue;
-            }
-            if (!entry.isFile())
-                continue;
-            seen += 1;
-            if (seen > limit) {
-                truncated = true;
-                pending.length = 0;
-                break;
-            }
-            let size;
-            try {
-                size = (await (0, promises_1.stat)(full)).size;
-            }
-            catch {
-                continue;
-            }
-            const relative = node_path_1.default.relative(root, full).split(node_path_1.default.sep).join("/");
-            // The last rule to speak wins, across every .gitignore that governs
-            // this path. A `!` rule brings the file back and is worth reporting,
-            // because it is how "always include" is expressed.
-            /*
-              The directories above it count. Asking `decide` about the file alone
-              is what made this screen report a Unity project's whole Library as
-              going up: `/[Ll]ibrary/` speaks about the directory, so no rule
-              matched the file and every one of them looked included — while the
-              save, which has always walked the ancestors, left them all behind.
-            */
-            const rule = (0, rules_js_1.decidingRule)(relative, false, layers);
-            if (rule?.negated) {
-                keptFiles += 1;
-                keptBytes += size;
-                uploadFiles += 1;
-                uploadBytes += size;
-                excluded.push({
-                    path: relative,
-                    bytes: size,
-                    rule: `!${rule.pattern}`,
-                    forceKept: true,
-                });
-                continue;
-            }
-            if (rule) {
-                skippedFiles += 1;
-                skippedBytes += size;
-                impactBytes.set(rule.pattern, (impactBytes.get(rule.pattern) ?? 0) + size);
-                impactFiles.set(rule.pattern, (impactFiles.get(rule.pattern) ?? 0) + 1);
-                excluded.push({
-                    path: relative,
-                    bytes: size,
-                    rule: rule.pattern,
-                    forceKept: false,
-                });
-                continue;
-            }
+    const walked = options.walked ?? (await walkProject(root, options));
+    if (options.shouldStop?.())
+        return emptyEvaluation(true);
+    const truncated = walked.truncated;
+    /* Kept for this evaluation only, so no edit to the rules can outlive it. */
+    const folderAnswers = new Map();
+    for (const file of walked.files) {
+        const { path: relative, size } = file;
+        // The last rule to speak wins, across every .gitignore that governs
+        // this path. A `!` rule brings the file back and is worth reporting,
+        // because it is how "always include" is expressed.
+        /*
+          The directories above it count. Asking `decide` about the file alone
+          is what made this screen report a Unity project's whole Library as
+          going up: `/[Ll]ibrary/` speaks about the directory, so no rule
+          matched the file and every one of them looked included — while the
+          save, which has always walked the ancestors, left them all behind.
+        */
+        const rule = (0, rules_js_1.decidingRule)(relative, false, layers, folderAnswers);
+        if (rule?.negated) {
+            keptFiles += 1;
+            keptBytes += size;
             uploadFiles += 1;
             uploadBytes += size;
+            excluded.push({
+                path: relative,
+                bytes: size,
+                rule: `!${rule.pattern}`,
+                forceKept: true,
+            });
+            continue;
+        }
+        if (rule) {
+            skippedFiles += 1;
+            skippedBytes += size;
+            impactBytes.set(rule.pattern, (impactBytes.get(rule.pattern) ?? 0) + size);
+            impactFiles.set(rule.pattern, (impactFiles.get(rule.pattern) ?? 0) + 1);
+            excluded.push({
+                path: relative,
+                bytes: size,
+                rule: rule.pattern,
+                forceKept: false,
+            });
+            continue;
         }
+        uploadFiles += 1;
+        uploadBytes += size;
     }
+    /*
+      Sorted, and all of it.
+  
+      This used to hand back the four hundred largest, which was right for a
+      flat list somebody scrolls and wrong for what reads it now: the screen
+      folds these into folders, so a Unity project's Library — 45,739 files —
+      was drawn from four hundred of them and labelled "395 files" beside a
+      header that said 45,739. The grouping happens here, in this process, and
+      only the folded tree crosses to the window, so keeping all of them costs
+      an array nobody sends.
+    */
     excluded.sort((left, right) => right.bytes - left.bytes);
     const impacts = [...impactBytes.entries()]
         .map(([pattern, bytes]) => ({
@@ -1201,7 +1304,7 @@ async function evaluateRules(root, rules, options = {}) {
         keptFiles,
         keptBytes,
         impacts,
-        excluded: excluded.slice(0, 400),
+        excluded,
         truncated,
     };
 }
@@ -1496,39 +1599,60 @@ function isCredentialByName(relativePath) {
 }
 async function detectSecrets(root) {
     const found = [];
-    const pending = [root];
     let seen = 0;
-    while (pending.length && found.length < 50) {
-        const directory = pending.pop();
-        let entries;
-        try {
-            entries = await (0, promises_1.readdir)(directory, { withFileTypes: true });
-        }
-        catch {
-            continue;
-        }
-        for (const entry of entries) {
-            if (seen > 40_000)
-                return found;
-            const full = node_path_1.default.join(directory, entry.name);
-            if ((0, rules_js_1.isUncounted)(entry.name))
-                continue;
-            if (entry.isDirectory()) {
-                if (entry.name !== "node_modules")
-                    pending.push(full);
-                continue;
+    let enough = false;
+    /*
+      Every directory, eight at a time. This reads names and nothing else, so
+      it is pure waiting, and it was waiting one directory at a time over all
+      two and a half thousand of them on a real project.
+  
+      It still looks everywhere the rules do not. A credential inside a folder
+      that is currently ignored is one rule edit away from being sent, and this
+      is the check that is supposed to notice before that happens.
+    */
+    let level = [root];
+    while (level.length && !enough) {
+        const next = [];
+        let at = 0;
+        const lane = async () => {
+            while (at < level.length && !enough) {
+                const directory = level[at];
+                at += 1;
+                let entries;
+                try {
+                    entries = await (0, promises_1.readdir)(directory, { withFileTypes: true });
+                }
+                catch {
+                    continue;
+                }
+                for (const entry of entries) {
+                    if (found.length >= 50 || seen > 40_000) {
+                        enough = true;
+                        return;
+                    }
+                    const full = node_path_1.default.join(directory, entry.name);
+                    if ((0, rules_js_1.isUncounted)(entry.name))
+                        continue;
+                    if (entry.isDirectory()) {
+                        if (entry.name !== "node_modules")
+                            next.push(full);
+                        continue;
+                    }
+                    seen += 1;
+                    /*
+                      Asked of the one function rather than repeated here. These were
+                      two copies of the same rule, which is how `.env.example` came to
+                      be refused by the command line long after the service had learned
+                      that templates are not credentials.
+                    */
+                    const relative = node_path_1.default.relative(root, full).split(node_path_1.default.sep).join("/");
+                    if (isCredentialByName(relative))
+                        found.push(relative);
+                }
             }
-            seen += 1;
-            /*
-              Asked of the one function rather than repeated here. These were two
-              copies of the same rule, which is how `.env.example` came to be
-              refused by the command line long after the service had learned that
-              templates are not credentials.
-            */
-            const relative = node_path_1.default.relative(root, full).split(node_path_1.default.sep).join("/");
-            if (isCredentialByName(relative))
-                found.push(relative);
-        }
+        };
+        await Promise.all(Array.from({ length: Math.min(WALK_LANES, level.length) }, () => lane()));
+        level = next;
     }
     return found;
 }