@coderook/cli 0.46.1 → 0.46.3

package/dist/cbx/src/local/history.js

@@ -126,7 +126,7 @@ async function save(repository, options) {
             previous: before,
             progress: options.progress,
         });
-        const tree = await repository.writeTree([...reading.files.values()]);
+        const tree = await repository.writeTree([...reading.files.values()], parent?.tree);
         await repository.writeIndex(reading.index);
         if (!merging && parent && parent.tree === tree) {
             return { saved: false, reason: "unchanged" };
@@ -148,6 +148,14 @@ async function save(repository, options) {
         await repository.setLineTip(line, made.id);
         if (merging)
             await (0, merge_js_1.clearMergeState)(repository);
+        /* The folder is what was just saved; said so, for the next quiet status. */
+        if (repository.objects.v2) {
+            await repository.writeIndex(reading.index, {
+                head: made.id,
+                clean: true,
+                complete: reading.index.size === reading.files.size,
+            });
+        }
         let bytes = 0;
         for (const file of reading.files.values())
             bytes += file.size;
@@ -190,13 +198,28 @@ async function status(repository) {
       that file once more, never trust something it has not stored. Written
       only when it changed; rewriting it whole was a good part of a quiet status.
     */
-    if (reading.indexChanged)
-        await repository.writeIndex(reading.index);
     const merging = await (0, merge_js_1.readMergeState)(repository);
+    const changes = compareTrees(byPath(tree), reading.files);
+    /*
+      And whether the folder is exactly the head, so the next quiet status can
+      be answered without Node (engine/src/quick.rs). Rewritten when what it
+      says changes, as well as when the index does.
+    */
+    const stamp = {
+        head: head?.id ?? null,
+        clean: !changeCount(changes) && !merging,
+        complete: reading.index.size === reading.files.size,
+    };
+    const told = repository.indexStamp;
+    if (reading.indexChanged ||
+        (repository.objects.v2 &&
+            (told?.head !== stamp.head || told?.clean !== stamp.clean || (told?.complete === true) !== stamp.complete))) {
+        await repository.writeIndex(reading.index, stamp);
+    }
     return {
         line: await repository.currentLine(),
         head,
-        changes: compareTrees(byPath(tree), reading.files),
+        changes,
         ...(merging
             ? {
                 merging: {

package/dist/cbx/src/local/remote.js

@@ -210,7 +210,22 @@ async function push(repository, options) {
                 }
                 await (0, promises_1.rm)(scratch, { recursive: true, force: true });
                 await (0, snapshot_js_1.writeEntriesInto)(repository, scratch, changed.map((file) => after.get(file)));
+                /*
+                  This history holds the base of every file it changes, as Git's sender
+                  does, so changed files go up as exact patches with no signatures
+                  fetched first.
+                */
+                const baseTree = before;
                 const request = {
+                    /* The listing of each version pushed is kept in the history, for the next push. */
+                    listingRoot: repository.directory,
+                    localBase: async (file, sha256) => {
+                        const entry = baseTree.get(file);
+                        if (!entry || entry.sha256 !== sha256)
+                            return null;
+                        const parts = await Promise.all(entry.chunks.map((name) => repository.objects.get(name)));
+                        return new Uint8Array(Buffer.concat(parts));
+                    },
                     localPath: scratch,
                     include: [...changed, ...changes.deleted],
                     deletions: changes.deleted,

package/dist/cbx/src/local/repository.js

@@ -3,12 +3,12 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.Repository = exports.DEFAULT_LINE = exports.FORMAT_LINE = exports.STORE_DIRECTORY = void 0;
+exports.Repository = exports.DEFAULT_LINE = exports.FORMAT_LINE_V2 = exports.FORMAT_LINE = exports.STORE_DIRECTORY = void 0;
 exports.lineNameProblem = lineNameProblem;
 /**
  * A local history: the `.cbx/` folder at the top of a project.
  *
- *   .cbx/format          "cbx-local 1", so a later layout can tell an older one
+ *   .cbx/format          "cbx-local 1" or "cbx-local 2", so a later layout can tell an older one
  *   .cbx/objects/        the object store (see store.ts)
  *   .cbx/lines/<name>    the id of the newest save on each line of work
  *   .cbx/state.json      which line the folder is on
@@ -28,6 +28,15 @@ const node_path_1 = __importDefault(require("node:path"));
 const store_js_1 = require("./store.js");
 exports.STORE_DIRECTORY = ".cbx";
 exports.FORMAT_LINE = "cbx-local 1";
+/*
+  The same objects, saves and trees, kept as solid blocks and deltas
+  (store_v2.ts, engine/V2-STORE.md). Read by every cbx that knows it; an
+  older one refuses the history and asks to be updated rather than misread it.
+  New histories are made in it only when CBX_LOCAL_FORMAT=2, until it has
+  proved itself on the benchmark suite.
+*/
+exports.FORMAT_LINE_V2 = "cbx-local 2";
+const FORMATS = new Set([exports.FORMAT_LINE, exports.FORMAT_LINE_V2]);
 exports.DEFAULT_LINE = "main";
 /**
  * Line names are file names under `.cbx/lines`, so they are held to what is
@@ -69,10 +78,10 @@ class Repository {
     root;
     directory;
     objects;
-    constructor(root) {
+    constructor(root, format = exports.FORMAT_LINE) {
         this.root = root;
         this.directory = node_path_1.default.join(root, exports.STORE_DIRECTORY);
-        this.objects = new store_js_1.ObjectStore(node_path_1.default.join(this.directory, "objects"));
+        this.objects = new store_js_1.ObjectStore(node_path_1.default.join(this.directory, "objects"), { v2: format === exports.FORMAT_LINE_V2 });
     }
     /** Start a history in `root`. Refuses a folder that already has one. */
     static async init(root, line = exports.DEFAULT_LINE) {
@@ -86,10 +95,11 @@ class Repository {
         }
         await (0, promises_1.mkdir)(node_path_1.default.join(directory, "objects"), { recursive: true });
         await (0, promises_1.mkdir)(node_path_1.default.join(directory, "lines"), { recursive: true });
-        await (0, promises_1.writeFile)(node_path_1.default.join(directory, "format"), `${exports.FORMAT_LINE}\n`);
+        const format = process.env.CBX_LOCAL_FORMAT === "2" ? exports.FORMAT_LINE_V2 : exports.FORMAT_LINE;
+        await (0, promises_1.writeFile)(node_path_1.default.join(directory, "format"), `${format}\n`);
         await (0, promises_1.writeFile)(node_path_1.default.join(directory, "state.json"), JSON.stringify({ line }));
         await (0, promises_1.writeFile)(node_path_1.default.join(directory, "config.json"), "{}");
-        return new Repository(full);
+        return new Repository(full, format);
     }
     /** The history `from` is inside, looking upward; null when there is none. */
     static async find(from) {
@@ -98,10 +108,10 @@ class Repository {
             const directory = node_path_1.default.join(at, exports.STORE_DIRECTORY);
             if (await exists(node_path_1.default.join(directory, "format"))) {
                 const format = (await (0, promises_1.readFile)(node_path_1.default.join(directory, "format"), "utf8")).trim();
-                if (format !== exports.FORMAT_LINE) {
+                if (!FORMATS.has(format)) {
                     throw new Error(`${directory} is in a format this cbx does not read (${format}). Update cbx.`);
                 }
-                return new Repository(at);
+                return new Repository(at, format);
             }
             const parent = node_path_1.default.dirname(at);
             if (parent === at)
@@ -218,7 +228,13 @@ class Repository {
         const id = await this.objects.putJson(canonical);
         return { ...canonical, id };
     }
-    async writeTree(files) {
+    /**
+     * Keep a tree. In a v2 history it goes into a pack as a delta against
+     * `base`, the tree it was made from: a tree names every file, so twenty
+     * saves of an 8,500-file project kept twenty megabyte-sized lists, where
+     * what changed between them was a few lines.
+     */
+    async writeTree(files, base) {
         const sorted = [...files].sort((left, right) => left.path < right.path ? -1 : left.path > right.path ? 1 : 0);
         const tree = {
             format: "cbx-tree",
@@ -231,6 +247,20 @@ class Repository {
                 chunks: file.chunks,
             })),
         };
+        if (this.objects.v2) {
+            const id = await this.objects.put(Buffer.from(JSON.stringify(tree), "utf8"), {
+                ...(base ? { base } : {}),
+                path: ".cbx-tree",
+                /*
+                  Every status rebuilds the head tree, so its chain is kept short: four
+                  deltas of a megabyte-sized list is a few milliseconds, twelve was not.
+                */
+                maxChain: 4,
+            });
+            /* On disk before any save can name it: a save is written on its own, at once. */
+            await this.objects.flush();
+            return id;
+        }
         return this.objects.putJson(tree);
     }
     /**
@@ -281,17 +311,66 @@ class Repository {
         return save;
     }
     /* ---- the index ------------------------------------------------------ */
+    /*
+      What a compact index last said about the folder as a whole: the save it
+      matched and whether it matched it exactly. Written by status, read by the
+      engine's quiet status (engine/src/quick.rs), which answers "nothing has
+      changed" from it after a walk, without Node. Any other write leaves it out.
+    */
+    stamp = null;
+    get indexStamp() {
+        return this.stamp;
+    }
     async readIndex() {
+        this.stamp = null;
         try {
             const raw = JSON.parse(await (0, promises_1.readFile)(node_path_1.default.join(this.directory, "index.json"), "utf8"));
+            if (raw.compact === 1) {
+                const told = raw.stamp;
+                if (told && (typeof told.head === "string" || told.head === null) && typeof told.clean === "boolean") {
+                    this.stamp = { head: told.head, clean: told.clean, ...(told.complete === true ? { complete: true } : {}) };
+                }
+                const index = new Map();
+                for (const [file, size, mtimeMs, sha256, chunks, runs] of raw.files) {
+                    index.set(file, {
+                        size,
+                        mtimeMs,
+                        sha256,
+                        chunks: chunks === 0 ? [sha256] : chunks,
+                        ...(runs === undefined ? {} : { executable: runs === 1 }),
+                    });
+                }
+                return index;
+            }
             return new Map(Object.entries(raw));
         }
         catch {
             return new Map();
         }
     }
-    async writeIndex(index) {
-        await writeAtomically(node_path_1.default.join(this.directory, "index.json"), JSON.stringify(Object.fromEntries(index)));
+    /*
+      A v2 history keeps the index compact: one array per file, and no list of
+      objects for a file that is one object named by its own digest, which is
+      nearly every file. It was 2.1 MB for 8,500 files, read and parsed on
+      every status. A v1 history keeps the old shape, which an older cbx reads.
+    */
+    async writeIndex(index, stamp) {
+        this.stamp = stamp && this.objects.v2 ? stamp : null;
+        const body = this.objects.v2
+            ? JSON.stringify({
+                compact: 1,
+                ...(stamp ? { stamp } : {}),
+                files: [...index].map(([file, entry]) => [
+                    file,
+                    entry.size,
+                    entry.mtimeMs,
+                    entry.sha256,
+                    entry.chunks.length === 1 && entry.chunks[0] === entry.sha256 ? 0 : entry.chunks,
+                    ...(entry.executable === undefined ? [] : [entry.executable ? 1 : 0]),
+                ]),
+            })
+            : JSON.stringify(Object.fromEntries(index));
+        await writeAtomically(node_path_1.default.join(this.directory, "index.json"), body);
     }
     /* ---- the lock ------------------------------------------------------- */
     /**

package/dist/cbx/src/local/snapshot.js

@@ -110,13 +110,65 @@ async function digestFile(full, size) {
  * that size, so an edit in the middle of a large asset stores the pieces
  * around the edit and reuses the rest.
  */
-async function storeFile(repository, full, size) {
+async function storeFile(repository, full, size, 
+/*
+  For a v2 history: where the file sits, and what it was last time (or,
+  for a new file, a file like it), so its objects can be kept as deltas.
+*/
+hints) {
     const profile = (0, chunking_js_1.profileForFileSize)(size);
     if (!profile) {
         const bytes = await (0, promises_1.readFile)(full);
-        const digest = await repository.objects.put(bytes);
+        const was = hints?.previous?.chunks.length === 1 ? hints.previous : hints?.similar;
+        const base = was?.chunks.length === 1 ? was.chunks[0] : undefined;
+        const digest = await repository.objects.put(bytes, {
+            ...(base ? { base } : {}),
+            ...(hints ? { path: hints.path } : {}),
+        });
         return { sha256: digest, chunks: bytes.length ? [digest] : [] };
     }
+    /*
+      The old pieces by where they started, so each new piece is kept as a
+      delta against the old piece at the same place: an edit inside a big file
+      changes the pieces around it, and those resemble what they replaced.
+    */
+    const before = hints?.previous?.chunks ?? [];
+    const starts = [];
+    if (before.length && repository.objects.v2) {
+        let at = 0;
+        for (const digest of before) {
+            starts.push(at);
+            const length = await repository.objects.rawSize(digest);
+            if (length === null) {
+                starts.length = 0;
+                break;
+            }
+            at += length;
+        }
+    }
+    const baseAt = (offset) => {
+        if (!starts.length)
+            return undefined;
+        let low = 0;
+        let high = starts.length - 1;
+        while (low < high) {
+            const middle = (low + high + 1) >> 1;
+            if (starts[middle] <= offset)
+                low = middle;
+            else
+                high = middle - 1;
+        }
+        return before[low];
+    };
+    let written = 0;
+    const keep = async (piece) => {
+        const base = baseAt(written);
+        written += piece.length;
+        return repository.objects.put(piece, {
+            ...(base ? { base } : {}),
+            ...(hints ? { path: hints.path } : {}),
+        });
+    };
     const whole = (0, node_crypto_1.createHash)("sha256");
     const chunks = [];
     let pending = Buffer.alloc(0);
@@ -126,13 +178,13 @@ async function storeFile(repository, full, size) {
         pending = pending.length ? Buffer.concat([pending, block]) : Buffer.from(block);
         let from = 0;
         for (const cut of (0, chunking_js_1.cutPoints)(pending, profile)) {
-            chunks.push(await repository.objects.put(Buffer.from(pending.subarray(from, cut))));
+            chunks.push(await keep(Buffer.from(pending.subarray(from, cut))));
             from = cut;
         }
         pending = pending.subarray(from);
     }
     if (pending.length)
-        chunks.push(await repository.objects.put(Buffer.from(pending)));
+        chunks.push(await keep(Buffer.from(pending)));
     return { sha256: whole.digest("hex"), chunks };
 }
 /**
@@ -169,7 +221,11 @@ async function readFolder(repository, options) {
     let native = null;
     let done = listed.length - toRead.length;
     const readBytes = toRead.reduce((sum, file) => sum + file.size, 0);
-    if ((0, native_engine_js_1.worthTheEngine)(toRead.length, readBytes)) {
+    /*
+      Not to store into a v2 history: the engine writes v1 packs, so what it
+      stored would miss the solid blocks and deltas. It still measures there.
+    */
+    if ((0, native_engine_js_1.worthTheEngine)(toRead.length, readBytes) && !(options.store && repository.objects.v2)) {
         const report = options.progress
             ? (count, _total, file) => options.progress(done + count, listed.length, file)
             : undefined;
@@ -201,15 +257,16 @@ async function readFolder(repository, options) {
         return null;
     };
     const record = (file, sha256, chunks, settled) => {
+        const executable = executableFrom(file.mode, previous.get(file.path));
         files.set(file.path, {
             path: file.path,
             size: file.size,
             sha256,
-            executable: executableFrom(file.mode, previous.get(file.path)),
+            executable,
             chunks,
         });
         if (file.mtimeMs < racyBefore && settled) {
-            next.set(file.path, { size: file.size, mtimeMs: file.mtimeMs, sha256, chunks });
+            next.set(file.path, { size: file.size, mtimeMs: file.mtimeMs, sha256, chunks, executable });
         }
     };
     /*
@@ -231,12 +288,50 @@ async function readFolder(repository, options) {
             options.progress?.(done, listed.length, file.path);
         }
     }
+    /*
+      A new file is kept against the file most like it the last save held:
+      one with the same extension in the same folder, nearest in size, and
+      within a factor of two. A copy made and edited is then a few bytes.
+    */
+    let alike = null;
+    const kindOf = (filePath) => {
+        const slash = filePath.lastIndexOf("/");
+        const name = filePath.slice(slash + 1);
+        const dot = name.lastIndexOf(".");
+        return `${filePath.slice(0, slash + 1)}\u0000${dot > 0 ? name.slice(dot).toLowerCase() : ""}`;
+    };
+    const hintsFor = (file) => {
+        const was = previous.get(file.path);
+        if (was)
+            return { previous: was };
+        if (!alike) {
+            alike = new Map();
+            for (const entry of previous.values()) {
+                if (entry.chunks.length !== 1)
+                    continue;
+                const key = kindOf(entry.path);
+                const list = alike.get(key);
+                if (list)
+                    list.push(entry);
+                else
+                    alike.set(key, [entry]);
+            }
+        }
+        let best;
+        for (const entry of alike.get(kindOf(file.path)) ?? []) {
+            if (entry.size * 2 < file.size || entry.size > file.size * 2)
+                continue;
+            if (!best || Math.abs(entry.size - file.size) < Math.abs(best.size - file.size))
+                best = entry;
+        }
+        return best ? { similar: best } : {};
+    };
     const one = async (file) => {
         const full = node_path_1.default.join(repository.root, file.path);
         let sha256;
         let chunks = [];
         if (options.store) {
-            ({ sha256, chunks } = await storeFile(repository, full, file.size));
+            ({ sha256, chunks } = await storeFile(repository, full, file.size, repository.objects.v2 ? { path: file.path, ...hintsFor(file) } : undefined));
         }
         else {
             sha256 = await digestFile(full, file.size);
@@ -251,13 +346,19 @@ async function readFolder(repository, options) {
         inLanes(toReadHere.filter(large), LARGE_LANES, one),
     ]);
     /* Nothing may name an object that is not on disk yet. */
-    if (options.store)
+    /*
+      A v2 history writes the files with the tree that names them, in one pack:
+      the save flushes after its tree (writeTree), so a pack is not spent on
+      the tree alone.
+    */
+    if (options.store && !repository.objects.v2)
         await repository.objects.flush();
     options.progress?.(listed.length, listed.length, "");
     const sameEntry = (left, right) => Boolean(right) &&
         left.size === right.size &&
         left.mtimeMs === right.mtimeMs &&
         left.sha256 === right.sha256 &&
+        left.executable === right.executable &&
         left.chunks.length === right.chunks.length &&
         left.chunks.every((chunk, at) => chunk === right.chunks[at]);
     const indexChanged = next.size !== index.size || [...next].some(([file, entry]) => !sameEntry(entry, index.get(file)));