@coderook/cli 0.45.0 → 0.46.0

package/.claude-plugin/plugin.json

@@ -2,7 +2,7 @@
   "name": "coderook",
   "displayName": "CodeRook",
   "description": "Save, browse and restore whole-snapshot versions of a project on CodeRook, with cbx or plain git, from Claude Code.",
-  "version": "0.45.0",
+  "version": "0.46.0",
   "author": {
     "name": "ACCA Gaming Productions",
     "url": "https://coderook.com"

package/dist/cbx/src/core/upload.js

@@ -20,6 +20,7 @@ const faults_js_1 = require("./faults.js");
 const identify_js_1 = require("./identify.js");
 const worktree_js_1 = require("./worktree.js");
 const rules_js_1 = require("./rules.js");
+const native_engine_js_1 = require("../shared/native_engine.js");
 const publish_name_js_1 = require("./publish_name.js");
 const compress_js_1 = require("./compress.js");
 const compression_policy_js_1 = require("../shared/compression_policy.js");
@@ -696,6 +697,33 @@ class Uploader {
                 bytesPerSecond: 0,
             });
         };
+        /*
+          Hashed ahead by the native engine, on every core, when there is enough
+          to read to be worth a process. Each answer carries the size and time the
+          file had while it was read, and is used below only when a fresh stat
+          still says the same — so it never vouches for bytes that have moved.
+          Anything it does not answer is hashed here as before.
+        */
+        const ahead = new Map();
+        {
+            const unknown = sending.filter((file) => {
+                const remembered = this.plannedDigests.get(file.path) ?? request.knownDigests?.get(file.path);
+                return !(remembered && remembered.size === file.size);
+            });
+            const bytes = unknown.reduce((sum, file) => sum + file.size, 0);
+            if ((0, native_engine_js_1.worthTheEngine)(unknown.length, bytes)) {
+                const answered = await (0, profile_js_1.timed)("hash files", () => (0, native_engine_js_1.engineMeasure)(request.localPath, unknown.map((file) => ({ path: file.path, count: false })), (done, _total, file) => {
+                    hashed = done;
+                    tell(file);
+                }, this.controller.signal));
+                this.check();
+                hashed = 0;
+                for (const [file, one] of answered ?? []) {
+                    if (one.sha256 && !one.error && typeof one.mtimeMs === "number")
+                        ahead.set(file, one);
+                }
+            }
+        }
         const lane = async () => {
             for (;;) {
                 const index = cursor;
@@ -757,6 +785,18 @@ class Uploader {
                     continue;
                 }
                 let digest;
+                const read = ahead.get(file.path);
+                if (read && read.size === size && read.mtimeMs === mtimeMs) {
+                    digest = read.sha256;
+                    if (fromPlan && fromPlan.sha256 !== digest)
+                        changedSincePlan.push(file.path);
+                    this.plannedDigests.set(file.path, { size, mtimeMs, sha256: digest });
+                    measured[index] = { path: file.path, full, size, sha256: digest, mediaType: mediaTypeOf(file.path) };
+                    hashed += 1;
+                    readDone += file.size;
+                    tell(file.path, hashed === sending.length);
+                    continue;
+                }
                 try {
                     digest = await (0, profile_js_1.timed)("hash files", () => digestOf(full, (read) => {
                         readingNow.set(index, read);

package/dist/cbx/src/core/worktree.js

@@ -46,6 +46,7 @@ const detect_js_1 = require("./detect.js");
 const ignore_templates_js_1 = require("./ignore_templates.js");
 const rules_js_1 = require("./rules.js");
 const secret_patterns_js_1 = require("./secret_patterns.js");
+const native_engine_js_1 = require("../shared/native_engine.js");
 const run = (0, node_util_1.promisify)(node_child_process_1.execFile);
 /* A code point, so nothing between here and disk can eat the escape. */
 const NEWLINE = String.fromCharCode(10);
@@ -799,6 +800,8 @@ async function branchName(root) {
 const LINE_COUNT_LIMIT = 4 * 1024 * 1024;
 /** Enough rows for any real project; past it the rail would be unusable. */
 const LISTED_FILE_LIMIT = 20_000;
+/** How many files the native engine is handed at once while a scan reads ahead. */
+const READ_AHEAD_FILES = 4_096;
 /**
  * The digest of a file, without holding it.
  *
@@ -927,10 +930,18 @@ async function changedFiles(root, rules, baseline = null, mode = "add-and-update
 */
 stats, 
 /** Filled with how much of the folder the cap allowed through. */
-outcome) {
-    const layers = await collectLayers(root, rules);
+outcome, 
+/** False for a read-only report: the listing's sizes and times may stand in for a stat (see engineScan). */
+exact = true) {
     /* What git already tracks goes up whatever a rule says; see gitTracked. */
     const tracked = await gitTracked(root);
+    /*
+      The native engine walks the folder itself when there is one, deciding every
+      rule as this file does and in the same order (engine/compat/rules-cross.mjs
+      compares the two), with one stat per file. Otherwise both passes run here.
+    */
+    const native = await (0, native_engine_js_1.engineScan)(root, rules, tracked, baseline?.size ?? stats?.size, exact);
+    const layers = native ? [] : await collectLayers(root, rules);
     const files = [];
     const present = new Set();
     const pending = [root];
@@ -951,7 +962,12 @@ outcome) {
       costs, so reading is what the cap should govern, and it can only be spent
       well once the whole folder is known.
     */
-    for (let next = 0; next < pending.length; next++) {
+    for (const one of native ?? []) {
+        present.add(one.path);
+        outcome?.sizes.push({ path: one.path, size: one.size });
+        found.push({ full: node_path_1.default.join(root, ...one.path.split("/")), relative: one.path, size: one.size, mtimeMs: one.mtimeMs });
+    }
+    for (let next = 0; !native && next < pending.length; next++) {
         const directory = pending[next];
         let entries;
         try {
@@ -1024,15 +1040,58 @@ outcome) {
       the list is full rather than over a fixed slice — a folder where nothing
       has changed still has to be walked through to find that out.
     */
-    for (const one of spreadAcrossDirectories(found)) {
+    const order = spreadAcrossDirectories(found);
+    /*
+      Read ahead by the native engine, a slice at a time in the same order, when
+      there is enough to read to be worth a process. A slice rather than all of
+      it, so the cap still stops the reading where it stops the list. Whatever
+      the engine does not answer is measured here as before, with the same rules:
+      an unchanged file is not read, nor a large one nothing could match.
+    */
+    const readAhead = new Map();
+    const needsReading = (one) => {
+        const cached = stats?.get(one.relative);
+        if (cached && cached.size === one.size && cached.mtimeMs === one.mtimeMs)
+            return false;
+        return one.size <= LINE_COUNT_LIMIT || (baseline?.has(one.relative) ?? false);
+    };
+    let readAheadTo = 0;
+    for (let at = 0; at < order.length; at += 1) {
         if (files.length >= limit)
             break;
+        if (at >= readAheadTo) {
+            const slice = [];
+            let bytes = 0;
+            let next = at;
+            while (next < order.length && slice.length < READ_AHEAD_FILES) {
+                const candidate = order[next];
+                if (needsReading(candidate)) {
+                    slice.push(candidate);
+                    bytes += candidate.size;
+                }
+                next += 1;
+            }
+            readAheadTo = next;
+            if ((0, native_engine_js_1.worthTheEngine)(slice.length, bytes)) {
+                const answered = await (0, native_engine_js_1.engineMeasure)(root, slice.map((one) => ({ path: one.relative, count: one.size <= LINE_COUNT_LIMIT })));
+                for (const one of slice) {
+                    const got = answered?.get(one.relative);
+                    /* Not as listed means it is being written; measured again below. */
+                    if (!got?.sha256 || got.error || got.size !== one.size || got.mtimeMs !== one.mtimeMs)
+                        continue;
+                    readAhead.set(one.relative, { lines: got.lines ?? 0, binary: got.binary ?? true, hash: got.sha256 });
+                }
+            }
+        }
+        const one = order[at];
         const { full, relative, size, mtimeMs } = one;
-        const measured = await measure(full, size, {
-            mtimeMs,
-            cached: stats?.get(relative),
-            recorded: baseline?.has(relative) ?? false,
-        });
+        const measured = readAhead.get(relative) ??
+            (await measure(full, size, {
+                mtimeMs,
+                cached: stats?.get(relative),
+                recorded: baseline?.has(relative) ?? false,
+            }));
+        readAhead.delete(relative);
         /*
           Remembered whether or not it changed: the next scan wants to skip
           reading this file again, and that is just as true of one that was
@@ -1107,8 +1166,16 @@ async function surveyFiles(root, rules,
   difference between waiting and wondering whether it has hung.
 */
 onFound) {
-    const layers = await collectLayers(root, rules);
     const tracked = await gitTracked(root);
+    /* The native engine's walk, when there is one: the same files, decided the same way. */
+    const native = await (0, native_engine_js_1.engineScan)(root, rules, tracked);
+    if (native) {
+        onFound?.(native.length);
+        return native
+            .map((one) => ({ path: one.path, size: one.size }))
+            .sort((left, right) => left.path.localeCompare(right.path));
+    }
+    const layers = await collectLayers(root, rules);
     const found = [];
     /*
       Every entry asks about each folder above it; those answers do not change

package/dist/cbx/src/local/history.js

@@ -182,13 +182,16 @@ async function log(repository, options = {}) {
 }
 async function status(repository) {
     const { save: head, tree } = await headTree(repository);
-    const reading = await (0, snapshot_js_1.readFolder)(repository, { store: false, previous: tree });
+    /* A report, not a save: the listing's sizes and times may stand in for a stat each. */
+    const reading = await (0, snapshot_js_1.readFolder)(repository, { store: false, previous: tree, exact: false });
     /*
       Kept, so the next status does not hash the same files again. The index
       is only a cache: an entry this writes without objects makes a save read
-      that file once more, never trust something it has not stored.
+      that file once more, never trust something it has not stored. Written
+      only when it changed; rewriting it whole was a good part of a quiet status.
     */
-    await repository.writeIndex(reading.index);
+    if (reading.indexChanged)
+        await repository.writeIndex(reading.index);
     const merging = await (0, merge_js_1.readMergeState)(repository);
     return {
         line: await repository.currentLine(),

package/dist/cbx/src/local/snapshot.js

@@ -24,6 +24,7 @@ const promises_1 = require("node:fs/promises");
 const node_path_1 = __importDefault(require("node:path"));
 const worktree_js_1 = require("../core/worktree.js");
 const chunking_js_1 = require("../shared/chunking.js");
+const native_engine_js_1 = require("../shared/native_engine.js");
 const safe_path_js_1 = require("../shared/safe_path.js");
 const READ_SIZE = 8 * 1024 * 1024;
 /*
@@ -58,8 +59,15 @@ async function inLanes(items, count, work) {
     await Promise.all(Array.from({ length: Math.min(count, items.length) }, () => lane()));
 }
 /** The files the rules would save, with what `stat` says about each. */
-async function listFolder(root) {
+async function listFolder(root, expectedFiles, exact = true) {
     const rules = await (0, worktree_js_1.readRules)(root);
+    /* The native engine walks and stats in one pass when there is one; otherwise the walk below and a stat each. */
+    const native = await (0, native_engine_js_1.engineScan)(root, rules, await (0, worktree_js_1.gitTracked)(root), expectedFiles, exact);
+    if (native) {
+        return native
+            .map((one) => ({ path: one.path, size: one.size, mtimeMs: one.mtimeMs, mode: one.mode }))
+            .sort((left, right) => (left.path < right.path ? -1 : left.path > right.path ? 1 : 0));
+    }
     const found = await (0, worktree_js_1.surveyFiles)(root, rules);
     const files = [];
     await inLanes(found, SMALL_LANES, async (one) => {
@@ -138,30 +146,61 @@ async function storeFile(repository, full, size) {
 async function readFolder(repository, options) {
     const index = await repository.readIndex();
     const previous = new Map(options.previous?.files.map((file) => [file.path, file]) ?? []);
-    const listed = await listFolder(repository.root);
+    const listed = await listFolder(repository.root, index.size || undefined, options.store || options.exact !== false);
     const files = new Map();
     const next = new Map();
     const racyBefore = Date.now() - RACY_WINDOW_MS;
-    let done = 0;
-    const one = async (file) => {
-        const full = node_path_1.default.join(repository.root, file.path);
+    const unchangedSince = (file) => {
         const known = index.get(file.path);
-        let sha256;
-        let chunks = [];
-        const unchanged = known &&
+        return known &&
             known.size === file.size &&
             known.mtimeMs === file.mtimeMs &&
-            (!options.store || known.chunks.length > 0 || file.size === 0);
-        if (unchanged) {
-            sha256 = known.sha256;
-            chunks = known.chunks;
-        }
-        else if (options.store) {
-            ({ sha256, chunks } = await storeFile(repository, full, file.size));
-        }
-        else {
-            sha256 = await digestFile(full, file.size);
+            (!options.store || known.chunks.length > 0 || file.size === 0)
+            ? known
+            : null;
+    };
+    /*
+      The files that have to be read, handed to the native engine in one go when
+      there are enough of them to be worth a process. It reads, hashes, cuts and
+      compresses them on every core and gives the same names and chunks storeFile
+      would; anything it does not answer is read here as before.
+    */
+    const toRead = listed.filter((file) => !unchangedSince(file));
+    let native = null;
+    let done = listed.length - toRead.length;
+    const readBytes = toRead.reduce((sum, file) => sum + file.size, 0);
+    if ((0, native_engine_js_1.worthTheEngine)(toRead.length, readBytes)) {
+        const report = options.progress
+            ? (count, _total, file) => options.progress(done + count, listed.length, file)
+            : undefined;
+        const wanted = toRead.map((file) => ({ path: file.path, size: file.size }));
+        native = options.store
+            ? await (0, native_engine_js_1.engineStore)(repository.objects.directory, repository.root, wanted, report)
+            : await (0, native_engine_js_1.engineMeasure)(repository.root, wanted.map((file) => ({ path: file.path, count: false })), report);
+        /* The engine wrote packs this store has not read yet. */
+        if (native && options.store)
+            repository.objects.rescan();
+        if (native)
+            done += toRead.filter((file) => native.get(file.path)?.sha256).length;
+    }
+    /* What is already known about a file without reading it: from the index, or from the engine. */
+    const answerFor = (file) => {
+        const known = unchangedSince(file);
+        if (known)
+            return { sha256: known.sha256, chunks: known.chunks, settled: true, fromEngine: false };
+        const answered = native?.get(file.path);
+        if (answered && !answered.error && answered.sha256 && (!options.store || answered.chunks)) {
+            return {
+                sha256: answered.sha256,
+                chunks: options.store ? answered.chunks : [],
+                /* Whether what was read is what was listed; a file still being written is not trusted next time. */
+                settled: answered.size === file.size && (options.store || answered.mtimeMs === file.mtimeMs),
+                fromEngine: true,
+            };
         }
+        return null;
+    };
+    const record = (file, sha256, chunks, settled) => {
         files.set(file.path, {
             path: file.path,
             size: file.size,
@@ -169,22 +208,60 @@ async function readFolder(repository, options) {
             executable: executableFrom(file.mode, previous.get(file.path)),
             chunks,
         });
-        if (file.mtimeMs < racyBefore) {
+        if (file.mtimeMs < racyBefore && settled) {
             next.set(file.path, { size: file.size, mtimeMs: file.mtimeMs, sha256, chunks });
         }
+    };
+    /*
+      Everything already answered is recorded straight away; only files that
+      still have to be read go through the lanes. A quiet status on a folder of
+      eight thousand files spent fifty milliseconds queueing files that needed
+      nothing done to them.
+    */
+    const toReadHere = [];
+    for (const file of listed) {
+        const answer = answerFor(file);
+        if (!answer) {
+            toReadHere.push(file);
+            continue;
+        }
+        record(file, answer.sha256, answer.chunks, answer.settled);
+        if (!answer.fromEngine) {
+            done += 1;
+            options.progress?.(done, listed.length, file.path);
+        }
+    }
+    const one = async (file) => {
+        const full = node_path_1.default.join(repository.root, file.path);
+        let sha256;
+        let chunks = [];
+        if (options.store) {
+            ({ sha256, chunks } = await storeFile(repository, full, file.size));
+        }
+        else {
+            sha256 = await digestFile(full, file.size);
+        }
+        record(file, sha256, chunks, true);
         done += 1;
         options.progress?.(done, listed.length, file.path);
     };
     const large = (file) => (0, chunking_js_1.profileForFileSize)(file.size) !== null;
     await Promise.all([
-        inLanes(listed.filter((file) => !large(file)), SMALL_LANES, one),
-        inLanes(listed.filter(large), LARGE_LANES, one),
+        inLanes(toReadHere.filter((file) => !large(file)), SMALL_LANES, one),
+        inLanes(toReadHere.filter(large), LARGE_LANES, one),
     ]);
     /* Nothing may name an object that is not on disk yet. */
     if (options.store)
         await repository.objects.flush();
     options.progress?.(listed.length, listed.length, "");
-    return { files, index: next };
+    const sameEntry = (left, right) => Boolean(right) &&
+        left.size === right.size &&
+        left.mtimeMs === right.mtimeMs &&
+        left.sha256 === right.sha256 &&
+        left.chunks.length === right.chunks.length &&
+        left.chunks.every((chunk, at) => chunk === right.chunks[at]);
+    const indexChanged = next.size !== index.size || [...next].some(([file, entry]) => !sameEntry(entry, index.get(file)));
+    return { files, index: next, indexChanged };
 }
 /** A tree entry's bytes, put back together from the store and checked. */
 async function contentsOf(repository, entry) {

package/dist/cbx/src/local/store.js

@@ -213,6 +213,15 @@ class ObjectStore {
         }
         return (0, cbx_js_1.openChunkFrame)(frame, digest);
     }
+    /**
+     * Read the pack indexes again on next use.
+     *
+     * For after something else has written packs into this store — the native
+     * engine does, during a save — since the list is read once and then kept.
+     */
+    rescan() {
+        this.packed = null;
+    }
     /** Close any packs held open for reading. Safe to call at any time. */
     async release() {
         const open = [...this.handles.values()];
@@ -231,6 +240,23 @@ class ObjectStore {
         return this.put(Buffer.from(JSON.stringify(value), "utf8"), { loose: true });
     }
     async getJson(digest) {
+        /*
+          Saves and trees are written on their own (putJson), so they are looked
+          for there first. Going through get() read every pack's index just to
+          find them, which grew with the history: the same object either way,
+          checked the same way.
+        */
+        if (!this.pending.has(digest) && !this.writing.has(digest)) {
+            let frame = null;
+            try {
+                frame = await (0, promises_1.readFile)(this.where(digest));
+            }
+            catch {
+                /* Not on its own; packed, perhaps. */
+            }
+            if (frame)
+                return JSON.parse((0, cbx_js_1.openChunkFrame)(frame, digest).toString("utf8"));
+        }
         return JSON.parse((await this.get(digest)).toString("utf8"));
     }
     /** Every object whose name starts with `prefix` (at least four characters). */