@coderook/cli 0.46.1 → 0.46.3

package/dist/cbx/src/local/store.js

@@ -30,6 +30,8 @@ const node_crypto_1 = require("node:crypto");
 const promises_1 = require("node:fs/promises");
 const node_path_1 = __importDefault(require("node:path"));
 const cbx_js_1 = require("../core/cbx.js");
+const delta_js_1 = require("../shared/delta.js");
+const store_v2_js_1 = require("./store_v2.js");
 exports.DIGEST = /^[0-9a-f]{64}$/;
 /** Objects at most this big are packed rather than written on their own. */
 exports.PACKED_OBJECT_LIMIT = 256 * 1024;
@@ -38,6 +40,58 @@ const PACK_BYTES = 32 * 1024 * 1024;
 function digestOf(bytes) {
     return (0, node_crypto_1.createHash)("sha256").update(bytes).digest("hex");
 }
+/*
+  Every v2 index, searched in place. A lookup is a binary search per pack,
+  newest first, which is nothing for the few a status makes; a save or a
+  restore makes thousands, and once the searches would cost more than
+  decoding everything once, everything is decoded into one map.
+*/
+class V2Packs {
+    files = [];
+    map = null;
+    searches = 0;
+    add(file) {
+        this.files.push(file);
+        if (this.map)
+            for (const name of file.names())
+                this.map.set(name, { ...file.find(name), pack: file.pack });
+    }
+    get(digest) {
+        if (this.map)
+            return this.map.get(digest);
+        this.searches += this.files.length;
+        if (this.searches > 200_000) {
+            this.map = new Map();
+            for (const file of this.files) {
+                for (const name of file.names())
+                    this.map.set(name, { ...file.find(name), pack: file.pack });
+            }
+            return this.map.get(digest);
+        }
+        for (let index = this.files.length - 1; index >= 0; index -= 1) {
+            const found = this.files[index].find(digest);
+            if (found)
+                return { ...found, pack: this.files[index].pack };
+        }
+        return undefined;
+    }
+    has(digest) {
+        return this.get(digest) !== undefined;
+    }
+    *keys() {
+        for (const file of this.files)
+            yield* file.names();
+    }
+}
+/*
+  The largest object kept as a delta: the biggest piece any chunk profile
+  cuts (32 MiB). The transport stops at 8 MiB, and using its limit here kept
+  every edited piece of a large file whole again: JDK's 130 MB module image
+  is cut into pieces of up to 16 MiB.
+*/
+const LOCAL_DELTA_MAX_BYTES = 32 * 1024 * 1024;
+/* Solid blocks kept decompressed, so restoring the files of one block reads it once. */
+const BLOCKS_KEPT_BYTES = 160 * 1024 * 1024;
 class ObjectStore {
     directory;
     /* Fan-out folders already known to exist, so each is made once, not per object. */
@@ -57,8 +111,20 @@ class ObjectStore {
       because Windows will not delete a folder holding an open file.
     */
     handles = new Map();
-    constructor(directory) {
+    /*
+      A cbx-local 2 history (store_v2.ts): the same objects, kept as solid
+      blocks and deltas in v2 packs. A v1 history never writes them, but every
+      store reads both, so upgrading changes nothing already written.
+    */
+    v2;
+    v2Located = null;
+    pendingV2 = new Map();
+    writingV2 = new Map();
+    pendingV2Bytes = 0;
+    blocks = new Map();
+    constructor(directory, options = {}) {
         this.directory = directory;
+        this.v2 = options.v2 === true;
     }
     get packDirectory() {
         return node_path_1.default.join(this.directory, "packs");
@@ -68,9 +134,21 @@ class ObjectStore {
             throw new Error(`Not an object name: ${digest}`);
         return node_path_1.default.join(this.directory, digest.slice(0, 2), digest.slice(2));
     }
+    /*
+      Read once, by one caller: the lanes of a save ask at the same moment, and
+      a second caller that found the v1 list set while the v2 list was still
+      loading went on with no v2 list at all.
+    */
+    loadingPacks = null;
     async packs() {
-        if (this.packed)
+        if (this.packed && this.v2Located)
             return this.packed;
+        this.loadingPacks ??= this.loadPacks().finally(() => {
+            this.loadingPacks = null;
+        });
+        return this.loadingPacks;
+    }
+    async loadPacks() {
         const found = new Map();
         let names = [];
         try {
@@ -86,9 +164,33 @@ class ObjectStore {
                 found.set(digest, { pack, offset, length });
             }
         }
+        const v2 = await this.readV2Indexes();
         this.packed = found;
+        this.v2Located = v2;
+        return found;
+    }
+    get v2Directory() {
+        return node_path_1.default.join(this.directory, "v2");
+    }
+    async readV2Indexes() {
+        const found = new V2Packs();
+        let names = [];
+        try {
+            names = await (0, promises_1.readdir)(this.v2Directory);
+        }
+        catch {
+            /* No v2 packs. */
+        }
+        for (const name of names.filter((one) => one.endsWith(".idx")).sort()) {
+            const pack = node_path_1.default.join(this.v2Directory, name.replace(/\.idx$/, ".pack"));
+            found.add(new store_v2_js_1.V2IndexFile(pack, await (0, promises_1.readFile)(node_path_1.default.join(this.v2Directory, name))));
+        }
         return found;
     }
+    async v2Index() {
+        await this.packs();
+        return this.v2Located;
+    }
     async loose(digest) {
         try {
             await (0, promises_1.stat)(this.where(digest));
@@ -101,10 +203,59 @@ class ObjectStore {
     async has(digest) {
         if (this.pending.has(digest) || this.writing.has(digest))
             return true;
+        if (this.pendingV2.has(digest) || this.writingV2.has(digest))
+            return true;
         if ((await this.packs()).has(digest))
             return true;
+        if (this.v2Located.has(digest))
+            return true;
         return this.loose(digest);
     }
+    /**
+     * How many deltas stand under an object, or null when it is not held.
+     * Anything kept whole, in a v1 store or loose, stands on nothing.
+     */
+    async chainOf(digest) {
+        const waiting = this.pendingV2.get(digest) ?? this.writingV2.get(digest);
+        if (waiting)
+            return waiting.solid ? 0 : waiting.chain;
+        const entry = (await this.v2Index()).get(digest);
+        if (entry)
+            return entry.chain;
+        return (await this.has(digest)) ? 0 : null;
+    }
+    /** An object's own length without rebuilding it, or null when it is not held. */
+    async rawSize(digest) {
+        const waiting = this.pendingV2.get(digest) ?? this.writingV2.get(digest);
+        if (waiting)
+            return waiting.raw.length;
+        const entry = (await this.v2Index()).get(digest);
+        if (entry)
+            return entry.raw;
+        const frame = this.pending.get(digest) ?? this.writing.get(digest);
+        if (frame)
+            return Number(frame.readBigUInt64LE(8));
+        const located = this.packed.get(digest);
+        try {
+            const head = Buffer.alloc(16);
+            if (located) {
+                await (await this.handleFor(located.pack)).read(head, 0, 16, located.offset);
+            }
+            else {
+                const handle = await (0, promises_1.open)(this.where(digest), "r");
+                try {
+                    await handle.read(head, 0, 16, 0);
+                }
+                finally {
+                    await handle.close();
+                }
+            }
+            return Number(head.readBigUInt64LE(8));
+        }
+        catch {
+            return null;
+        }
+    }
     /**
      * Keep `raw`, returning its name.
      *
@@ -118,6 +269,10 @@ class ObjectStore {
         const digest = digestOf(raw);
         if (await this.has(digest))
             return digest;
+        if (this.v2 && !options.loose) {
+            await this.putV2(raw, digest, options.base, options.path, options.maxChain ?? store_v2_js_1.MAX_CHAIN);
+            return digest;
+        }
         const { frame } = (0, cbx_js_1.chunkFrame)(raw);
         if (!options.loose && raw.length <= exports.PACKED_OBJECT_LIMIT) {
             this.pending.set(digest, frame);
@@ -150,6 +305,17 @@ class ObjectStore {
         /* One pack at a time; a second caller waits for the first and goes again. */
         while (this.flushing)
             await this.flushing;
+        if (this.pendingV2.size) {
+            const taken = this.pendingV2;
+            this.pendingV2 = new Map();
+            this.pendingV2Bytes = 0;
+            this.writingV2 = taken;
+            this.flushing = this.writeV2Pack(taken).finally(() => {
+                this.writingV2 = new Map();
+                this.flushing = null;
+            });
+            await this.flushing;
+        }
         if (!this.pending.size)
             return;
         const taken = this.pending;
@@ -162,6 +328,190 @@ class ObjectStore {
         });
         await this.flushing;
     }
+    /**
+     * Keep one object in a v2 store: as a delta against `base` when that is
+     * clearly smaller than the object compressed on its own and the base does
+     * not already stand on too many others; otherwise joined into a solid
+     * block when small, or whole.
+     */
+    async putV2(raw, digest, base, filePath, maxChain) {
+        if (base && base !== digest && raw.length >= 64 && raw.length <= LOCAL_DELTA_MAX_BYTES) {
+            const depth = await this.chainOf(base);
+            if (depth !== null && depth + 1 < Math.min(maxChain, store_v2_js_1.MAX_CHAIN) + 1) {
+                let from = null;
+                try {
+                    from = await this.get(base);
+                }
+                catch {
+                    from = null;
+                }
+                if (from && from.length <= LOCAL_DELTA_MAX_BYTES) {
+                    const patch = Buffer.from((0, delta_js_1.createExactDeltaPatch)(from, raw, LOCAL_DELTA_MAX_BYTES));
+                    const squeezed = patch.length > 64 ? (0, store_v2_js_1.zstdAt)(patch, 3) : patch;
+                    const packed = squeezed.length < patch.length;
+                    const record = packed ? squeezed : patch;
+                    /* Against what the object costs compressed on its own, not its raw size. */
+                    const alone = Math.min(raw.length, (0, store_v2_js_1.zstdAt)(raw, 1).length);
+                    if (record.length < alone / 2 && alone - record.length >= 96) {
+                        await this.holdV2(digest, {
+                            solid: false,
+                            raw,
+                            record,
+                            kind: packed ? store_v2_js_1.KIND_DELTA_ZSTD : store_v2_js_1.KIND_DELTA,
+                            chain: depth + 1,
+                            base,
+                        });
+                        return;
+                    }
+                }
+            }
+        }
+        if (raw.length <= store_v2_js_1.SOLID_OBJECT_LIMIT) {
+            await this.holdV2(digest, { solid: true, raw, key: (0, store_v2_js_1.solidKey)(filePath), large: false });
+            return;
+        }
+        /* Larger and compressible: joined with others of its kind. Otherwise kept as it is. */
+        if ((0, store_v2_js_1.compresses)(raw)) {
+            await this.holdV2(digest, { solid: true, raw, key: (0, store_v2_js_1.solidKey)(filePath), large: true });
+            return;
+        }
+        await this.holdV2(digest, { solid: false, raw, record: raw, kind: store_v2_js_1.KIND_WHOLE, chain: 0 });
+    }
+    async holdV2(digest, held) {
+        this.pendingV2.set(digest, held);
+        this.pendingV2Bytes += held.raw.length;
+        /* Written when there is a pack's worth; callers flush at the end of a save either way. */
+        if (this.pendingV2Bytes >= store_v2_js_1.V2_PACK_BYTES)
+            await this.flush();
+    }
+    async writeV2Pack(objects) {
+        await (0, promises_1.mkdir)(this.v2Directory, { recursive: true });
+        const entries = new Map();
+        const records = [];
+        let offset = 0;
+        const solid = [...objects]
+            .filter((pair) => pair[1].solid)
+            .sort(([leftName, left], [rightName, right]) => left.key < right.key ? -1 : left.key > right.key ? 1 : leftName < rightName ? -1 : 1);
+        let block = [];
+        let blockBytes = 0;
+        let blockLarge = false;
+        const closeBlock = () => {
+            if (!block.length)
+                return;
+            const joined = Buffer.concat(block.map(([, raw]) => raw));
+            const stored = (0, store_v2_js_1.zstdAt)(joined, 9, true, blockLarge ? store_v2_js_1.LARGE_WINDOW_LOG : undefined, true);
+            let inner = 0;
+            for (const [digest, raw] of block) {
+                entries.set(digest, { kind: store_v2_js_1.KIND_SOLID, chain: 0, offset, length: stored.length, inner, raw: raw.length });
+                inner += raw.length;
+            }
+            records.push(stored);
+            offset += stored.length;
+            block = [];
+            blockBytes = 0;
+        };
+        /* Small objects first, then large: the two tiers never share a block. */
+        for (const large of [false, true]) {
+            const limit = large ? store_v2_js_1.LARGE_BLOCK_BYTES : store_v2_js_1.SOLID_BLOCK_BYTES;
+            blockLarge = large;
+            for (const [digest, held] of solid) {
+                if (held.large !== large)
+                    continue;
+                if (blockBytes && blockBytes + held.raw.length > limit)
+                    closeBlock();
+                block.push([digest, held.raw]);
+                blockBytes += held.raw.length;
+            }
+            closeBlock();
+        }
+        for (const [digest, held] of objects) {
+            if (held.solid)
+                continue;
+            entries.set(digest, {
+                kind: held.kind,
+                chain: held.chain,
+                offset,
+                length: held.record.length,
+                inner: 0,
+                raw: held.raw.length,
+                ...(held.base ? { base: held.base } : {}),
+            });
+            records.push(held.record);
+            offset += held.record.length;
+        }
+        const body = Buffer.concat(records);
+        const id = digestOf(body);
+        const pack = node_path_1.default.join(this.v2Directory, `${id}.pack`);
+        const idx = node_path_1.default.join(this.v2Directory, `${id}.idx`);
+        await (0, promises_1.writeFile)(`${pack}.partial`, body);
+        await (0, promises_1.rename)(`${pack}.partial`, pack);
+        const encoded = (0, store_v2_js_1.encodeIndex)(entries);
+        await (0, promises_1.writeFile)(`${idx}.partial`, encoded);
+        await (0, promises_1.rename)(`${idx}.partial`, idx);
+        (await this.v2Index()).add(new store_v2_js_1.V2IndexFile(pack, encoded));
+    }
+    handleFor(pack) {
+        let opening = this.handles.get(pack);
+        if (!opening) {
+            opening = (0, promises_1.open)(pack, "r");
+            this.handles.set(pack, opening);
+        }
+        return opening;
+    }
+    async readRecord(entry) {
+        const record = Buffer.alloc(entry.length);
+        const { bytesRead } = await (await this.handleFor(entry.pack)).read(record, 0, entry.length, entry.offset);
+        if (bytesRead !== entry.length)
+            throw new Error("A v2 pack ends before one of its records does");
+        return record;
+    }
+    /** Rebuild one object from a v2 pack and check it against its name. */
+    async getV2(digest, entry) {
+        let raw;
+        try {
+            if (entry.kind === store_v2_js_1.KIND_SOLID) {
+                const key = `${entry.pack}:${entry.offset}`;
+                let block = this.blocks.get(key);
+                if (block) {
+                    this.blocks.delete(key);
+                }
+                else {
+                    block = (0, store_v2_js_1.unzstd)(await this.readRecord(entry));
+                }
+                this.blocks.set(key, block);
+                let kept = 0;
+                for (const one of this.blocks.values())
+                    kept += one.length;
+                while (kept > BLOCKS_KEPT_BYTES && this.blocks.size > 1) {
+                    const oldest = this.blocks.keys().next().value;
+                    kept -= this.blocks.get(oldest).length;
+                    this.blocks.delete(oldest);
+                }
+                if (entry.inner + entry.raw > block.length)
+                    throw new Error("it reaches outside its block");
+                raw = Buffer.from(block.subarray(entry.inner, entry.inner + entry.raw));
+            }
+            else if (entry.kind === store_v2_js_1.KIND_WHOLE || entry.kind === store_v2_js_1.KIND_WHOLE_ZSTD) {
+                const record = await this.readRecord(entry);
+                raw = entry.kind === store_v2_js_1.KIND_WHOLE ? record : (0, store_v2_js_1.unzstd)(record);
+            }
+            else if (entry.kind === store_v2_js_1.KIND_DELTA || entry.kind === store_v2_js_1.KIND_DELTA_ZSTD) {
+                const record = await this.readRecord(entry);
+                const patch = entry.kind === store_v2_js_1.KIND_DELTA ? record : (0, store_v2_js_1.unzstd)(record);
+                raw = Buffer.from((0, delta_js_1.applyDeltaPatch)(await this.get(entry.base), patch, LOCAL_DELTA_MAX_BYTES));
+            }
+            else {
+                throw new Error("it is of a kind this cbx does not read");
+            }
+        }
+        catch (error) {
+            throw new Error(`Object ${digest.slice(0, 12)} could not be rebuilt: ${error instanceof Error ? error.message : String(error)}`);
+        }
+        if (digestOf(raw) !== digest) {
+            throw new Error(`Object ${digest.slice(0, 12)} is damaged: its bytes do not match its name`);
+        }
+        return raw;
+    }
     async writePack(objects) {
         await (0, promises_1.mkdir)(this.packDirectory, { recursive: true });
         const index = {};
@@ -189,14 +539,15 @@ class ObjectStore {
         const waiting = this.pending.get(digest) ?? this.writing.get(digest);
         if (waiting)
             return (0, cbx_js_1.openChunkFrame)(waiting, digest);
+        const held = this.pendingV2.get(digest) ?? this.writingV2.get(digest);
+        if (held)
+            return Buffer.from(held.raw);
         const located = (await this.packs()).get(digest);
+        const inV2 = !located ? this.v2Located.get(digest) : undefined;
+        if (inV2)
+            return this.getV2(digest, inV2);
         if (located) {
-            let opening = this.handles.get(located.pack);
-            if (!opening) {
-                opening = (0, promises_1.open)(located.pack, "r");
-                this.handles.set(located.pack, opening);
-            }
-            const handle = await opening;
+            const handle = await this.handleFor(located.pack);
             const frame = Buffer.alloc(located.length);
             const { bytesRead } = await handle.read(frame, 0, located.length, located.offset);
             if (bytesRead !== located.length) {
@@ -221,6 +572,7 @@ class ObjectStore {
      */
     rescan() {
         this.packed = null;
+        this.v2Located = null;
     }
     /** Close any packs held open for reading. Safe to call at any time. */
     async release() {
@@ -246,7 +598,7 @@ class ObjectStore {
           find them, which grew with the history: the same object either way,
           checked the same way.
         */
-        if (!this.pending.has(digest) && !this.writing.has(digest)) {
+        if (!this.pending.has(digest) && !this.writing.has(digest) && !this.pendingV2.has(digest) && !this.writingV2.has(digest)) {
             let frame = null;
             try {
                 frame = await (0, promises_1.readFile)(this.where(digest));
@@ -278,6 +630,10 @@ class ObjectStore {
             if (digest.startsWith(prefix))
                 found.add(digest);
         }
+        for (const digest of this.v2Located.keys()) {
+            if (digest.startsWith(prefix))
+                found.add(digest);
+        }
         return [...found];
     }
 }