jev-agent-tools 0.1.4 → 0.3.0

package/dist/adapters/git.js

@@ -0,0 +1,483 @@
+import { lstat, readlink } from "node:fs/promises";
+import { resolve } from "node:path";
+import { GIT_BLOB_BATCH_SIZE, STATE_MAX_CHARS, TIMEOUT_MS, } from "../constants.js";
+import { parseDiff } from "../core/diff.js";
+import { isSecretPath } from "../core/secret-path.js";
+import { buildUnits, } from "../core/units.js";
+import { admitGitPath, admitRepoMetadata, openRepoFile } from "./files.js";
+import { shareGitInventory } from "./git-inventory.js";
+import { loadSyntaxParser } from "./syntax.js";
+import { decodeUtf8, verifyGitUtf8 } from "./utf8.js";
+const config = [
+    "-c",
+    "color.ui=false",
+    "-c",
+    "core.quotePath=true",
+    "-c",
+    "diff.suppressBlankEmpty=false",
+    "-c",
+    "diff.relative=false",
+    "-c",
+    "diff.noprefix=false",
+    "-c",
+    "diff.mnemonicPrefix=false",
+    "-c",
+    "diff.algorithm=myers",
+    "-c",
+    "diff.indentHeuristic=false",
+];
+async function git(exec, options, args) {
+    try {
+        options.signal?.throwIfAborted();
+        const result = await exec("git", [...config, ...args], {
+            cwd: options.cwd,
+            timeout: TIMEOUT_MS,
+            signal: options.signal,
+        });
+        if (result.killed)
+            return {
+                ok: false,
+                cause: options.signal?.aborted ? "cancelled" : "git_failure",
+                error: "Git interrupted (cancelled or timed out).",
+            };
+        if (result.code !== 0)
+            return {
+                ok: false,
+                cause: "git_failure",
+                error: `Git failed: ${result.stderr}`,
+            };
+        return { ok: true, text: result.stdout };
+    }
+    catch (error) {
+        return {
+            ok: false,
+            cause: options.signal?.aborted ? "cancelled" : "git_failure",
+            error: `Unable to execute git: ${String(error)}`,
+        };
+    }
+}
+async function readWorktree(options, exec, path, inventory) {
+    let handle;
+    try {
+        options.signal?.throwIfAborted();
+        const location = await admitRepoMetadata(options.cwd, path, exec, options.signal, inventory);
+        if (!location.ok)
+            return { text: "", binary: false, limitation: "unreadable" };
+        const stat = await lstat(location.abs);
+        if (stat.isSymbolicLink())
+            return { text: await readlink(location.abs), binary: false };
+        if (stat.size > STATE_MAX_CHARS)
+            return { text: "", binary: false, limitation: "too_large" };
+        const opened = await openRepoFile(options.cwd, path, {
+            exec,
+            signal: options.signal,
+            inventory,
+        });
+        if (!opened.ok)
+            return { text: "", binary: false, limitation: "unreadable" };
+        handle = opened.handle;
+        const buffer = Buffer.alloc(STATE_MAX_CHARS + 1);
+        let offset = 0;
+        while (offset < buffer.length) {
+            const { bytesRead } = await handle.read(buffer, offset, buffer.length - offset, null);
+            if (!bytesRead)
+                break;
+            offset += bytesRead;
+        }
+        if (offset > STATE_MAX_CHARS)
+            return { text: "", binary: false, limitation: "too_large" };
+        const bytes = buffer.subarray(0, offset);
+        const decoded = decodeUtf8(bytes);
+        return decoded.ok
+            ? { text: decoded.text, binary: bytes.includes(0) }
+            : { text: "", binary: false, limitation: "not UTF-8 text" };
+    }
+    catch {
+        return { text: "", binary: false, limitation: "unreadable" };
+    }
+    finally {
+        await handle?.close();
+    }
+}
+async function readGitBlobs(exec, options, objects) {
+    const blobs = new Map();
+    const invalid = new Set();
+    for (let offset = 0; offset < objects.length; offset += GIT_BLOB_BATCH_SIZE) {
+        const batch = objects.slice(offset, offset + GIT_BLOB_BATCH_SIZE);
+        const result = await git(exec, options, [
+            "show",
+            "--no-ext-diff",
+            "--no-textconv",
+            "--end-of-options",
+            ...batch.map((object) => object.id),
+        ]);
+        if (!result.ok)
+            return result;
+        const bytes = Buffer.from(result.text, "utf8");
+        if (bytes.length !== batch.reduce((sum, object) => sum + object.size, 0)) {
+            // Host decoding can replace invalid UTF-8: never split subsequent blobs at shifted offsets.
+            for (const object of batch) {
+                const single = await git(exec, options, [
+                    "show",
+                    "--no-ext-diff",
+                    "--no-textconv",
+                    "--end-of-options",
+                    object.id,
+                ]);
+                if (!single.ok)
+                    return single;
+                const decoded = verifyGitUtf8(single.text, object.id, object.size);
+                if (decoded.ok)
+                    blobs.set(object.id, decoded.text);
+                else
+                    invalid.add(object.id);
+            }
+            continue;
+        }
+        let position = 0;
+        for (const object of batch) {
+            const decoded = verifyGitUtf8(bytes.subarray(position, position + object.size).toString("utf8"), object.id, object.size);
+            if (decoded.ok)
+                blobs.set(object.id, decoded.text);
+            else
+                invalid.add(object.id);
+            position += object.size;
+        }
+    }
+    return { ok: true, blobs, invalid };
+}
+export async function collectDiff(exec, options) {
+    exec = shareGitInventory(exec);
+    const root = await git(exec, options, ["rev-parse", "--show-toplevel"]);
+    if (!root.ok)
+        return root;
+    const cwd = root.text.replace(/\n$/, "");
+    const scoped = { ...options, cwd };
+    const location = await git(exec, options, ["rev-parse", "--show-prefix"]);
+    if (!location.ok)
+        return location;
+    const repositoryPrefix = location.text.replace(/\n$/, "");
+    const paths = [
+        "--",
+        ...(options.paths ?? []).map((path) => `:(top,literal)${repositoryPrefix}${path}`),
+    ];
+    const rawResult = await git(exec, scoped, [
+        "diff",
+        "--raw",
+        "-z",
+        "--no-ext-diff",
+        "--no-textconv",
+        "--ignore-submodules=none",
+        "--end-of-options",
+        options.base,
+        ...paths,
+    ]);
+    if (!rawResult.ok)
+        return rawResult;
+    const untrackedResult = await git(exec, scoped, [
+        "ls-files",
+        "--full-name",
+        "--others",
+        "--exclude-standard",
+        "-z",
+        ...paths,
+    ]);
+    if (!untrackedResult.ok)
+        return untrackedResult;
+    let candidates;
+    try {
+        candidates = parseDiff({
+            raw: rawResult.text,
+            numstat: "",
+            patch: "",
+            untracked: untrackedResult.text.split("\0").filter(Boolean),
+        });
+    }
+    catch {
+        return {
+            ok: false,
+            cause: "git_failure",
+            error: "Inconsistent Git snapshots; collect again.",
+            kind: "inconsistent_diff",
+        };
+    }
+    const unsupported = [];
+    // Secret-named paths (either side of a rename) stay named exclusions;
+    // excluding them from every later pathspec keeps Git from emitting their
+    // patch text or pairing them into a rename.
+    const secret = candidates
+        .filter((file) => isSecretPath(file.path) || isSecretPath(file.oldPath))
+        .map((file) => file.path);
+    await Promise.all(candidates.map(async (file) => {
+        if (secret.includes(file.path))
+            return;
+        try {
+            const stat = await lstat(resolve(cwd, file.path));
+            if (!stat.isFile() &&
+                !stat.isSymbolicLink() &&
+                file.oldMode !== "160000" &&
+                file.newMode !== "160000")
+                unsupported.push(file.path);
+        }
+        catch {
+            /* Missing tracked paths remain deletions. */
+        }
+    }));
+    for (const path of [...unsupported, ...secret])
+        paths.push(`:(top,exclude,literal)${path}`);
+    for (const file of candidates)
+        if (secret.includes(file.path) && file.oldPath !== file.path)
+            paths.push(`:(top,exclude,literal)${file.oldPath}`);
+    const prefix = [
+        "diff",
+        "--no-color",
+        "--no-ext-diff",
+        "--no-textconv",
+        "--find-renames",
+        "--src-prefix=a/",
+        "--dst-prefix=b/",
+        "--no-relative",
+        "--line-prefix=",
+        "--submodule=short",
+        "--ignore-submodules=none",
+    ];
+    const results = await Promise.all([
+        git(exec, scoped, [
+            ...prefix,
+            "--raw",
+            "-z",
+            "--end-of-options",
+            options.base,
+            ...paths,
+        ]),
+        git(exec, scoped, [
+            ...prefix,
+            "--numstat",
+            "-z",
+            "--end-of-options",
+            options.base,
+            ...paths,
+        ]),
+        git(exec, scoped, [
+            ...prefix,
+            "--patch",
+            "--unified=8",
+            "--end-of-options",
+            options.base,
+            ...paths,
+        ]),
+        git(exec, scoped, [
+            "ls-files",
+            "--full-name",
+            "--others",
+            "--exclude-standard",
+            "-z",
+            ...paths,
+        ]),
+        git(exec, scoped, [
+            "ls-files",
+            "--full-name",
+            "--cached",
+            "--others",
+            "--exclude-standard",
+            "-z",
+        ]),
+        git(exec, scoped, [
+            "ls-tree",
+            "-r",
+            "--long",
+            "-z",
+            "--full-tree",
+            "--end-of-options",
+            options.base,
+        ]),
+    ]);
+    for (const result of results)
+        if (!result.ok)
+            return result;
+    const [raw, numstat, patch, untracked, current, base] = results;
+    if (!raw.ok ||
+        !numstat.ok ||
+        !patch.ok ||
+        !untracked.ok ||
+        !current.ok ||
+        !base.ok)
+        return {
+            ok: false,
+            cause: "git_failure",
+            error: "Incomplete Git collection.",
+        };
+    const currentInventory = new Set(current.text.split("\0").filter(Boolean));
+    const baseInventory = new Map();
+    for (const entry of base.text.split("\0")) {
+        const separator = entry.indexOf("\t");
+        if (separator !== -1)
+            baseInventory.set(entry.slice(separator + 1), entry);
+    }
+    let files;
+    try {
+        files = parseDiff({
+            raw: raw.text,
+            numstat: numstat.text,
+            patch: patch.text,
+            verifyPaths: true,
+            untracked: untracked.text.split("\0").filter(Boolean),
+        });
+    }
+    catch {
+        return {
+            ok: false,
+            cause: "git_failure",
+            error: "Inconsistent Git snapshots; collect again.",
+            kind: "inconsistent_diff",
+        };
+    }
+    files.push(...candidates.filter((file) => unsupported.includes(file.path) || secret.includes(file.path)));
+    const objects = new Map();
+    for (const file of files) {
+        if (/^[A?]/.test(file.status) ||
+            file.binary ||
+            unsupported.includes(file.path) ||
+            secret.includes(file.path))
+            continue;
+        const admission = await admitGitPath(cwd, file.oldPath, options.base, exec, options.signal, baseInventory);
+        if (!admission.ok)
+            continue;
+        const entry = baseInventory.get(file.oldPath);
+        const match = entry &&
+            /^(?:100644|100755|120000) blob ([0-9a-f]+)\s+(\d+)\t/.exec(entry);
+        if (match)
+            objects.set(file.oldPath, { id: match[1] ?? "", size: Number(match[2]) });
+    }
+    const uniqueObjects = new Map([...objects.values()]
+        .filter((object) => object.size <= STATE_MAX_CHARS)
+        .map((object) => [object.id, object]));
+    const loaded = await readGitBlobs(exec, scoped, [...uniqueObjects.values()]);
+    if (!loaded.ok)
+        return loaded;
+    const collectFile = async (file) => {
+        if (secret.includes(file.path))
+            return {
+                ok: true,
+                file: {
+                    ...file,
+                    hunks: [],
+                    binary: false,
+                    before: null,
+                    after: null,
+                    limitation: "secret_pattern",
+                },
+            };
+        const historical = !/^[A?]/.test(file.status);
+        const admission = historical
+            ? await admitGitPath(cwd, file.oldPath, options.base, exec, options.signal, baseInventory)
+            : await admitRepoMetadata(cwd, file.path, exec, options.signal, currentInventory);
+        if (!admission.ok)
+            return {
+                ok: true,
+                file: {
+                    ...file,
+                    hunks: [],
+                    before: null,
+                    after: null,
+                    limitation: "unreadable",
+                },
+            };
+        if (unsupported.includes(file.path)) {
+            return {
+                ok: true,
+                file: { ...file, before: null, after: null, limitation: "unreadable" },
+            };
+        }
+        if (file.binary) {
+            return { ok: true, file: { ...file, before: null, after: null } };
+        }
+        if (file.oldMode === "160000" || file.newMode === "160000") {
+            return {
+                ok: true,
+                file: { ...file, before: null, after: null, limitation: "unreadable" },
+            };
+        }
+        if (file.status === "D") {
+            try {
+                const replacement = await lstat(resolve(cwd, file.path));
+                if (!replacement.isFile() && !replacement.isSymbolicLink()) {
+                    return {
+                        ok: true,
+                        file: {
+                            ...file,
+                            before: null,
+                            after: null,
+                            limitation: "unreadable",
+                        },
+                    };
+                }
+            }
+            catch {
+                /* A genuinely deleted path has no worktree version. */
+            }
+        }
+        let before = "";
+        let limitation;
+        if (!/^[A?]/.test(file.status)) {
+            const object = objects.get(file.oldPath);
+            if (!object)
+                return {
+                    ok: false,
+                    cause: "git_failure",
+                    error: "Incomplete Git base tree; collect again.",
+                };
+            if (object.size > STATE_MAX_CHARS)
+                limitation = "too_large";
+            else if (loaded.invalid.has(object.id))
+                limitation = "not UTF-8 text";
+            else
+                before = loaded.blobs.get(object.id) ?? "";
+        }
+        const after = file.status === "D"
+            ? { text: "", binary: false }
+            : await readWorktree(scoped, exec, file.path, currentInventory);
+        limitation = after.limitation ?? limitation;
+        const unverifiableHunk = limitation === "too_large" &&
+            file.hunks.some((hunk) => (hunk.beforeText ?? "").includes("\uFFFD") ||
+                (hunk.afterText ?? "").includes("\uFFFD"));
+        if (unverifiableHunk)
+            limitation = "not UTF-8 text";
+        return {
+            ok: true,
+            file: {
+                ...file,
+                hunks: after.limitation === "unreadable" || limitation === "not UTF-8 text"
+                    ? file.hunks.map(({ beforeText: _before, afterText: _after, ...hunk }) => hunk)
+                    : file.hunks,
+                binary: after.binary,
+                limitation,
+                before: /^[A?]/.test(file.status) ||
+                    limitation === "too_large" ||
+                    limitation === "not UTF-8 text"
+                    ? null
+                    : before,
+                after: file.status === "D" || limitation ? null : after.text,
+            },
+        };
+    };
+    const sources = [];
+    for (let offset = 0; offset < files.length; offset += 4) {
+        const batch = await Promise.all(files.slice(offset, offset + 4).map(collectFile));
+        for (const result of batch) {
+            if (!result.ok)
+                return result;
+            sources.push(result.file);
+        }
+    }
+    return { ok: true, files: sources };
+}
+export async function collectUnits(exec, options, sourceParser) {
+    const [diff, parser] = await Promise.all([
+        collectDiff(exec, options),
+        sourceParser ?? loadSyntaxParser(),
+    ]);
+    if (!diff.ok)
+        return diff;
+    return { ok: true, files: diff.files, ...buildUnits(diff.files, parser) };
+}

package/dist/adapters/locate-file.js

@@ -0,0 +1,197 @@
+import { LOCATE_LABEL_MAX_CHARS, LOCATE_READ_BUFFER_BYTES, LOCATE_WHOLE_MAX_BYTES, LOCATE_WHOLE_MAX_CHARS, LOCATE_WINDOW_LINES, LOCATE_WINDOW_MAX_SERIALIZED_CHARS, STATE_MAX_CHARS, } from "../constants.js";
+import { truncate } from "../core/truncate.js";
+import { openRepoFile } from "./files.js";
+import { createUtf8Decoder, decodeUtf8 } from "./utf8.js";
+/** The initial read is bounded; large-file outlines retain labels rather than source. */
+export async function readLocateFile(cwd, path, signal, exec) {
+    if (/^[a-z][a-z0-9+.-]*:\/\//i.test(path))
+        return {
+            ok: false,
+            cause: "forbidden_path",
+            error: `Internal URLs are not files; use read for ${path}.`,
+        };
+    const opened = await openRepoFile(cwd, path, { exec, signal });
+    if (!opened.ok)
+        return opened;
+    const handle = opened.handle;
+    try {
+        const stat = await handle.stat();
+        if (!stat.isFile())
+            return {
+                ok: false,
+                cause: "file_unavailable",
+                error: `Not a file: ${path}.`,
+            };
+        const buffer = Buffer.alloc(Math.min(stat.size + 1, LOCATE_WHOLE_MAX_BYTES + 1));
+        let count = 0;
+        while (count < buffer.length) {
+            signal?.throwIfAborted();
+            const read = await handle.read(buffer, count, buffer.length - count, null);
+            if (!read.bytesRead)
+                break;
+            count += read.bytesRead;
+        }
+        if (buffer.subarray(0, count).includes(0))
+            return {
+                ok: false,
+                cause: "binary_or_non_utf8",
+                error: `Not a text file: ${path}.`,
+            };
+        if (count < buffer.length) {
+            const decoded = decodeUtf8(buffer.subarray(0, count));
+            if (!decoded.ok)
+                return {
+                    ok: false,
+                    cause: "binary_or_non_utf8",
+                    error: `${path}: ${decoded.error}`,
+                };
+            const text = decoded.text;
+            if (text.length <= LOCATE_WHOLE_MAX_CHARS)
+                return {
+                    ok: true,
+                    kind: "whole",
+                    bytes: count,
+                    lines: text
+                        ? text.split("\n").length - Number(text.endsWith("\n"))
+                        : 0,
+                    text,
+                };
+        }
+        const scan = await scanRange(cwd, path, undefined, signal, exec);
+        return scan.ok
+            ? {
+                ok: true,
+                kind: "outline",
+                bytes: stat.size,
+                lines: scan.lines,
+                windows: scan.windows,
+            }
+            : scan;
+    }
+    catch (error) {
+        return {
+            ok: false,
+            cause: signal?.aborted ? "cancelled" : "file_unavailable",
+            error: `Cannot read ${path}: ${String(error)}`,
+        };
+    }
+    finally {
+        await handle.close();
+    }
+}
+export async function scanRange(cwd, path, range, signal, exec) {
+    const opened = await openRepoFile(cwd, path, { exec, signal });
+    if (!opened.ok)
+        return opened;
+    const handle = opened.handle;
+    const decode = createUtf8Decoder();
+    const buffer = Buffer.alloc(LOCATE_READ_BUFFER_BYTES);
+    let reachedEof = false;
+    let line = 1, label = "", pending = false, text = "", bytes = 0;
+    const windows = [];
+    let windowStart = 1, head = "", tail = "", serializedTextChars = 2;
+    const finishWindow = (end) => {
+        windows.push({
+            start: windowStart,
+            end,
+            label: label || `lines ${windowStart}-${end}`,
+            text: `${head}\n…\n${tail}`,
+            serializedTextChars,
+        });
+        windowStart = end + 1;
+        label = "";
+        head = "";
+        tail = "";
+        serializedTextChars = 2;
+    };
+    const consume = (chunk) => {
+        for (const piece of chunk.split(/(?<=\n)/)) {
+            if (!piece)
+                continue;
+            pending = true;
+            const excerptLimit = LOCATE_WINDOW_MAX_SERIALIZED_CHARS;
+            serializedTextChars += JSON.stringify(piece).length - 2;
+            head = truncate(head + piece, excerptLimit);
+            const combined = tail + piece;
+            const tailStart = Math.max(0, combined.length - excerptLimit);
+            tail = combined.slice(tailStart +
+                (tailStart > 0 && /[\uDC00-\uDFFF]/.test(combined[tailStart] ?? "")
+                    ? 1
+                    : 0));
+            if ((line - 1) % LOCATE_WINDOW_LINES === 0 &&
+                label.length < LOCATE_LABEL_MAX_CHARS)
+                label = truncate(label + piece.trim(), LOCATE_LABEL_MAX_CHARS);
+            if (range && line >= range.start && line <= range.end)
+                text += piece;
+            if (text.length > STATE_MAX_CHARS)
+                return false;
+            if (piece.endsWith("\n")) {
+                if (line - windowStart + 1 >= LOCATE_WINDOW_LINES ||
+                    serializedTextChars >= LOCATE_WINDOW_MAX_SERIALIZED_CHARS)
+                    finishWindow(line);
+                line++;
+                pending = false;
+            }
+        }
+        return true;
+    };
+    try {
+        while (true) {
+            signal?.throwIfAborted();
+            const read = await handle.read(buffer, 0, buffer.length, null);
+            if (!read.bytesRead) {
+                reachedEof = true;
+                break;
+            }
+            if (buffer.subarray(0, read.bytesRead).includes(0))
+                return {
+                    ok: false,
+                    cause: "binary_or_non_utf8",
+                    error: `Not a text file: ${path}.`,
+                };
+            bytes += read.bytesRead;
+            const decoded = decode(buffer.subarray(0, read.bytesRead));
+            if (!decoded.ok)
+                return {
+                    ok: false,
+                    cause: "binary_or_non_utf8",
+                    error: `${path}: ${decoded.error}`,
+                };
+            if (!consume(decoded.text))
+                return {
+                    ok: false,
+                    cause: "evidence_too_large",
+                    error: `Selected range exceeds ${STATE_MAX_CHARS} chars; read a narrower range directly.`,
+                };
+            if (range && line > range.end)
+                break;
+        }
+        const end = reachedEof ? decode() : { ok: true, text: "" };
+        if (!end.ok)
+            return {
+                ok: false,
+                cause: "binary_or_non_utf8",
+                error: `${path}: ${end.error}`,
+            };
+        if (!consume(end.text))
+            return {
+                ok: false,
+                cause: "evidence_too_large",
+                error: "Selected range too large.",
+            };
+        const lines = line - Number(!pending);
+        if (lines > 0 && (windows.at(-1)?.end ?? 0) < lines)
+            finishWindow(lines);
+        return { ok: true, bytes, lines, windows, text };
+    }
+    catch (error) {
+        return {
+            ok: false,
+            cause: signal?.aborted ? "cancelled" : "file_unavailable",
+            error: `Cannot read ${path}: ${String(error)}`,
+        };
+    }
+    finally {
+        await handle.close();
+    }
+}