jev-agent-tools 0.1.4 → 0.2.0

package/dist/adapters/files.js

@@ -0,0 +1,293 @@
+import { createHash } from "node:crypto";
+import { constants } from "node:fs";
+import { lstat, open, readlink, realpath, } from "node:fs/promises";
+import { isAbsolute, relative, resolve, sep } from "node:path";
+import { ASK_MAX_FILES, FIND_EXCERPT_CHARS, FIND_POINTER_EXCERPT_CHARS, FIND_READ_CHUNK_BYTES, STATE_MAX_CHARS, TIMEOUT_MS, } from "../constants.js";
+import { createExcerptCollector } from "../core/find.js";
+import { createUtf8Decoder, decodeUtf8 } from "./utf8.js";
+export async function checkFileAdmission(cwd, path, exec, signal, inventory) {
+    if (path.split(/[\\/]/).includes(".git"))
+        return { ok: false, error: `${path}: .git metadata: not sent to Jev` };
+    if (inventory)
+        return inventory.has(path)
+            ? { ok: true }
+            : { ok: false, error: `${path}: gitignored: not sent to Jev` };
+    if (!exec)
+        return { ok: true };
+    const listed = await exec("git", [
+        "ls-files",
+        "--cached",
+        "--others",
+        "--exclude-standard",
+        "-z",
+        "--",
+        path,
+    ], { cwd, timeout: TIMEOUT_MS, signal });
+    if (listed.killed)
+        return { ok: false, error: `${path}: file admission interrupted` };
+    if (listed.code === 0 && !listed.stdout.split("\0").includes(path))
+        return { ok: false, error: `${path}: gitignored: not sent to Jev` };
+    return { ok: true };
+}
+export async function resolveInsideRepo(root, path, options = {}) {
+    if (isAbsolute(path) || path.split(/[\\/]/).includes(".."))
+        return {
+            ok: false,
+            error: `Path must remain inside the repository: ${path}.`,
+        };
+    try {
+        const base = await realpath(root);
+        const requested = resolve(base, path);
+        const rel = relative(base, requested);
+        if (rel === ".." || rel.startsWith(`..${sep}`) || isAbsolute(rel))
+            return { ok: false, error: `Path escapes the repository: ${path}.` };
+        let current = base;
+        for (const part of rel.split(sep).filter(Boolean)) {
+            current = resolve(current, part);
+            if ((await lstat(current)).isSymbolicLink()) {
+                if (options.allowLeafSymlink && current === requested)
+                    return { ok: true, abs: requested, rel: rel.replaceAll(sep, "/") };
+                return { ok: false, error: `Symbolic links are not read: ${path}.` };
+            }
+        }
+        const abs = await realpath(requested);
+        const actual = relative(base, abs);
+        if (actual === ".." || actual.startsWith(`..${sep}`) || isAbsolute(actual))
+            return { ok: false, error: `Path escapes the repository: ${path}.` };
+        return { ok: true, abs, rel: actual.replaceAll(sep, "/") };
+    }
+    catch (error) {
+        return { ok: false, error: `Cannot resolve ${path}: ${String(error)}` };
+    }
+}
+/** Leaf-link metadata is admitted without following the target. */
+export async function admitRepoMetadata(cwd, path, exec, signal, inventory) {
+    const location = await resolveInsideRepo(cwd, path, {
+        allowLeafSymlink: true,
+    });
+    if (!location.ok)
+        return location;
+    const admission = await checkFileAdmission(cwd, location.rel, exec, signal, inventory);
+    return admission.ok ? location : admission;
+}
+/** History is admitted by its tracked base tree, independent of worktree ignores or links. */
+export async function admitGitPath(cwd, path, base, exec, signal, inventory) {
+    if (isAbsolute(path) || path.split(/[\\/]/).includes(".."))
+        return {
+            ok: false,
+            error: `Path must remain inside the repository: ${path}.`,
+        };
+    const admission = await checkFileAdmission(cwd, path, undefined, signal);
+    if (!admission.ok)
+        return admission;
+    // ls-tree cannot descend through a symlink or gitlink ancestor in the base tree.
+    let entry = inventory?.get(path);
+    if (!inventory) {
+        const tree = await exec("git", ["ls-tree", "-z", "--full-tree", "--end-of-options", base, "--", path], { cwd, timeout: TIMEOUT_MS, signal });
+        if (tree.code !== 0 || tree.killed)
+            return { ok: false, error: `${path}: not a file at comparison base` };
+        entry = tree.stdout
+            .split("\0")
+            .find((item) => item.slice(item.indexOf("\t") + 1) === path);
+    }
+    if (!entry || !/^(?:100644|100755|120000) blob /.test(entry))
+        return { ok: false, error: `${path}: not a file at comparison base` };
+    return { ok: true };
+}
+/** The only content opener for repository files. Callers own and close the handle. */
+export async function openRepoFile(cwd, path, options = {}) {
+    try {
+        options.signal?.throwIfAborted();
+        const location = await resolveInsideRepo(cwd, path);
+        if (!location.ok)
+            return location;
+        const admission = await checkFileAdmission(cwd, location.rel, options.exec, options.signal, options.inventory);
+        if (!admission.ok)
+            return admission;
+        const handle = await open(location.abs, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK);
+        try {
+            const stat = await handle.stat();
+            if (!stat.isFile()) {
+                await handle.close();
+                return { ok: false, error: `Not a file: ${path}.` };
+            }
+            return { ok: true, handle, size: stat.size };
+        }
+        catch (error) {
+            await handle.close();
+            throw error;
+        }
+    }
+    catch (error) {
+        return { ok: false, error: `Cannot read ${path}: ${String(error)}` };
+    }
+}
+export async function collectFiles(cwd, paths, signal, options = {}) {
+    if (paths.length > ASK_MAX_FILES)
+        return {
+            ok: false,
+            error: `paths exceeds ${ASK_MAX_FILES} files; split paths into several calls.`,
+        };
+    const results = await Promise.all(paths.map(async (path) => {
+        if (/^[a-z][a-z0-9+.-]*:\/\//i.test(path))
+            return {
+                ok: false,
+                error: `Internal URLs are not files; use read for ${path}.`,
+            };
+        let handle;
+        try {
+            signal?.throwIfAborted();
+            const opened = await openRepoFile(cwd, path, {
+                exec: options.exec,
+                signal,
+                inventory: options.inventory,
+            });
+            if (!opened.ok)
+                return opened;
+            handle = opened.handle;
+            const bytes = Buffer.alloc(Math.min(opened.size + 1, STATE_MAX_CHARS * 4 + 1));
+            let count = 0;
+            while (count < bytes.length) {
+                signal?.throwIfAborted();
+                const read = await handle.read(bytes, count, bytes.length - count, null);
+                if (!read.bytesRead)
+                    break;
+                count += read.bytesRead;
+            }
+            if (count === bytes.length)
+                return {
+                    ok: false,
+                    error: `File exceeds ${STATE_MAX_CHARS} chars: ${path}; files are never truncated. Split paths into several calls.`,
+                };
+            const decoded = decodeUtf8(bytes.subarray(0, count));
+            if (!decoded.ok)
+                return {
+                    ok: false,
+                    invalidUtf8: true,
+                    error: `${path} is not UTF-8 text; provide a UTF-8 text file instead.`,
+                };
+            const text = decoded.text;
+            if (count === bytes.length || text.length > STATE_MAX_CHARS)
+                return {
+                    ok: false,
+                    error: `File exceeds ${STATE_MAX_CHARS} chars: ${path}; files are never truncated. Split paths into several calls.`,
+                };
+            if (!text.length && !options.allowEmpty)
+                return { ok: false, error: `${path} is empty.` };
+            return {
+                ok: true,
+                path,
+                text,
+                identity: {
+                    requestedPath: path,
+                    resolvedPath: resolve(cwd, path),
+                    insertedPath: resolve(cwd, path),
+                    content: text,
+                    readSha256: createHash("sha256")
+                        .update(bytes.subarray(0, count))
+                        .digest("hex"),
+                    insertedSha256: createHash("sha256").update(text).digest("hex"),
+                },
+            };
+        }
+        catch (error) {
+            return {
+                ok: false,
+                error: `Cannot read ${path}: ${String(error)}`,
+            };
+        }
+        finally {
+            await handle?.close();
+        }
+    }));
+    const failure = results.find((result) => !result.ok && !(options.skipInvalidUtf8 && "invalidUtf8" in result));
+    if (failure && !failure.ok)
+        return failure;
+    return {
+        ok: true,
+        skipped: results.flatMap((result) => !result.ok && "invalidUtf8" in result ? [result.error] : []),
+        identities: results.flatMap((result) => result.ok ? [result.identity] : []),
+        files: Object.fromEntries(results.flatMap((result) => result.ok ? [[result.path, result.text]] : [])),
+    };
+}
+export async function collectSearchExcerpts(cwd, paths, keywords, signal, exec) {
+    const files = {}, pointers = {};
+    const ignored = [];
+    const limits = [];
+    const results = await Promise.all(paths.map(async (path) => {
+        if (/^[a-z][a-z0-9+.-]*:\/\//i.test(path))
+            return {
+                ok: false,
+                error: `Internal URLs are not files; use read for ${path}.`,
+            };
+        try {
+            const location = await admitRepoMetadata(cwd, path, exec, signal);
+            if (!location.ok) {
+                ignored.push(path);
+                return { ok: true };
+            }
+            if ((await lstat(location.abs)).isSymbolicLink()) {
+                const target = await readlink(location.abs);
+                files[path] = target;
+                pointers[path] = target;
+                return { ok: true };
+            }
+            const opened = await openRepoFile(cwd, path, { exec, signal });
+            if (!opened.ok) {
+                ignored.push(path);
+                return { ok: true };
+            }
+            const handle = opened.handle;
+            const content = createExcerptCollector(keywords, FIND_EXCERPT_CHARS);
+            const pointer = createExcerptCollector(keywords, FIND_POINTER_EXCERPT_CHARS);
+            const decode = createUtf8Decoder();
+            const stream = handle.createReadStream({
+                highWaterMark: FIND_READ_CHUNK_BYTES,
+                signal,
+            });
+            for await (const chunk of stream) {
+                const decoded = decode(chunk);
+                if (!decoded.ok) {
+                    limits.push({ path, reason: decoded.error });
+                    ignored.push(path);
+                    stream.destroy();
+                    return { ok: true };
+                }
+                const text = decoded.text;
+                if (text.includes("\0")) {
+                    ignored.push(path);
+                    stream.destroy();
+                    return { ok: true };
+                }
+                content.push(text);
+                pointer.push(text);
+            }
+            const end = decode();
+            if (!end.ok) {
+                limits.push({ path, reason: end.error });
+                ignored.push(path);
+                return { ok: true };
+            }
+            content.push(end.text);
+            pointer.push(end.text);
+            const excerpt = content.finish();
+            if (!excerpt.length) {
+                ignored.push(path);
+                return { ok: true };
+            }
+            files[path] = excerpt;
+            pointers[path] = pointer.finish();
+            return { ok: true };
+        }
+        catch (error) {
+            if (signal?.aborted)
+                return { ok: false, error: String(error) };
+            ignored.push(path);
+            return { ok: true };
+        }
+    }));
+    const failure = results.find((result) => !result.ok);
+    return failure && !failure.ok
+        ? failure
+        : { ok: true, files, pointers, ignored, limits };
+}

package/dist/adapters/find.js

@@ -0,0 +1,122 @@
+import { FIND_GREP_PAGE_SIZE, FIND_PATH_WEIGHT, TIMEOUT_MS, } from "../constants.js";
+import { pathAllowed } from "../core/find.js";
+export async function prefilter(exec, cwd, keywords, limit, scope, exclude, signal, native = true) {
+    try {
+        const listed = await exec("git", ["ls-files", "-z", "--cached", "--others", "--exclude-standard"], { cwd, timeout: TIMEOUT_MS, signal });
+        if (listed.killed || listed.code !== 0)
+            return {
+                ok: false,
+                error: `Cannot list repository files: ${listed.stderr || "git interrupted"}`,
+            };
+        const paths = [
+            ...new Set(listed.stdout
+                .split("\0")
+                .filter((path) => path && pathAllowed(path, scope, exclude))),
+        ];
+        if (!paths.length)
+            return { ok: true, paths: [], scopeFiles: 0 };
+        const allowed = new Set(paths);
+        let finder;
+        if (native) {
+            try {
+                // Static loading would break hosts where this optional native package is absent.
+                const { FileFinder } = await import("@ff-labs/fff-node");
+                const created = FileFinder.create({
+                    basePath: cwd,
+                    disableWatch: true,
+                });
+                if (created.ok) {
+                    finder = created.value;
+                    const ready = await finder.waitForScan(TIMEOUT_MS);
+                    if (!ready.ok || !ready.value) {
+                        finder.destroy();
+                        finder = undefined;
+                    }
+                }
+            }
+            catch {
+                /* Optional native package is unavailable in this host. */
+            }
+        }
+        if (!finder) {
+            const available = await exec("rg", ["--version"], {
+                cwd,
+                timeout: TIMEOUT_MS,
+                signal,
+            });
+            if (available.killed || available.code !== 0)
+                return { ok: false, error: "rg is required when fff is unavailable" };
+        }
+        try {
+            const matches = await Promise.all(keywords.map(async (word) => {
+                signal?.throwIfAborted();
+                if (finder) {
+                    const found = new Set();
+                    let cursor = null;
+                    do {
+                        const result = finder.multiGrep({
+                            patterns: [word],
+                            smartCase: true,
+                            maxMatchesPerFile: 1,
+                            pageSize: FIND_GREP_PAGE_SIZE,
+                            cursor,
+                        });
+                        if (!result.ok)
+                            throw new Error(result.error);
+                        for (const item of result.value.items)
+                            if (allowed.has(item.relativePath))
+                                found.add(item.relativePath);
+                        cursor = result.value.nextCursor;
+                    } while (cursor);
+                    return found;
+                }
+                const result = await exec("rg", [
+                    "--files-with-matches",
+                    "--hidden",
+                    "--null",
+                    "--fixed-strings",
+                    "--ignore-case",
+                    "--",
+                    word,
+                    ".",
+                ], { cwd, timeout: TIMEOUT_MS, signal });
+                if (result.killed ||
+                    (result.code !== 0 && result.code !== 1) ||
+                    (result.code === 1 && result.stderr.trim()))
+                    throw new Error(result.stderr || "rg interrupted");
+                return new Set(result.stdout
+                    .split("\0")
+                    .map((path) => path.replace(/^\.\//u, ""))
+                    .filter((path) => allowed.has(path)));
+            }));
+            const scores = paths.map((path) => ({
+                path,
+                score: keywords.reduce((sum, word, index) => {
+                    const matchesForWord = matches[index];
+                    if (!matchesForWord)
+                        return sum;
+                    const weight = Math.log(1 + paths.length / (1 + matchesForWord.size));
+                    return (sum +
+                        weight *
+                            (Number(matchesForWord.has(path)) +
+                                FIND_PATH_WEIGHT * Number(path.toLowerCase().includes(word))));
+                }, 0),
+            }));
+            scores.sort((a, b) => b.score - a.score || a.path.localeCompare(b.path));
+            return {
+                ok: true,
+                paths: scores
+                    .filter((item) => item.score > 0)
+                    .slice(0, limit)
+                    .map((item) => item.path),
+                scopeFiles: paths.length,
+            };
+        }
+        finally {
+            finder?.destroy();
+        }
+    }
+    catch (error) {
+        return { ok: false, error: `Cannot prefilter files: ${String(error)}` };
+    }
+}

package/dist/adapters/git-base.js

@@ -0,0 +1,26 @@
+import { TIMEOUT_MS } from "../constants.js";
+export async function resolveBase(exec, cwd, base, signal) {
+    if (!base)
+        return { ok: true, base: "HEAD" };
+    try {
+        const result = await exec("git", ["merge-base", "--", "HEAD", base], {
+            cwd,
+            timeout: TIMEOUT_MS,
+            signal,
+        });
+        if (result.killed)
+            return { ok: false, error: "Git interrupted (cancelled or timed out)." };
+        if (result.code === 0 && result.stdout.trim())
+            return { ok: true, base: result.stdout.trim() };
+        return {
+            ok: false,
+            error: `Cannot resolve base=${base}: history may be truncated or no shared ancestor is available. Run git fetch --unshallow (or git fetch --deepen=<count>), or choose a base that is an ancestor of HEAD.${result.stderr.trim() ? ` Git: ${result.stderr.trim()}` : ""}`,
+        };
+    }
+    catch (error) {
+        return {
+            ok: false,
+            error: `Unable to resolve base=${base}: ${String(error)}`,
+        };
+    }
+}

package/dist/adapters/git-inventory.js

@@ -0,0 +1,71 @@
+/** Fresh per tool call: HEAD/index timestamps cannot detect untracked or ignore changes. */
+export function shareGitInventory(exec) {
+    const snapshots = new Map();
+    return async (command, args, options) => {
+        const offset = args.indexOf("ls-files");
+        if (command !== "git" && command !== "rg")
+            snapshots.clear();
+        if (command !== "git" || offset === -1)
+            return exec(command, args, options);
+        const flags = args.slice(offset + 1);
+        if (!flags.includes("-z") ||
+            flags.includes("--ignored") ||
+            flags.some((flag) => flag.startsWith("--with-tree")))
+            return exec(command, args, options);
+        const separator = flags.indexOf("--");
+        const filters = separator === -1 ? [] : flags.slice(separator + 1);
+        if (filters.some((path) => !path.startsWith(":(top,literal)") &&
+            !path.startsWith(":(top,exclude,literal)") &&
+            /[*?[\]:]/.test(path)))
+            return exec(command, args, options);
+        const allowed = new Set([
+            "-z",
+            "--cached",
+            "--others",
+            "--exclude-standard",
+            "--full-name",
+        ]);
+        if ((separator === -1 ? flags : flags.slice(0, separator)).some((flag) => !allowed.has(flag)))
+            return exec(command, args, options);
+        const key = `${options.cwd}\0${flags.includes("--full-name")}`;
+        let snapshot = snapshots.get(key);
+        if (!snapshot) {
+            snapshot = exec("git", [
+                "ls-files",
+                "--cached",
+                "--others",
+                "--exclude-standard",
+                ...(flags.includes("--full-name") ? ["--full-name"] : []),
+                "-t",
+                "-z",
+            ], options);
+            snapshots.set(key, snapshot);
+        }
+        const result = await snapshot;
+        if (result.code !== 0 || result.killed)
+            return result;
+        const entries = result.stdout.split("\0").filter(Boolean);
+        const othersOnly = flags.includes("--others") && !flags.includes("--cached");
+        const trackedOnly = !flags.includes("--others");
+        const include = filters.filter((path) => !path.startsWith(":(top,exclude,literal)"));
+        const exclude = filters
+            .filter((path) => path.startsWith(":(top,exclude,literal)"))
+            .map((path) => path.slice(":(top,exclude,literal)".length));
+        const matches = (path, filter) => {
+            const literal = filter.replace(/^:\(top,literal\)/, "");
+            return path === literal || path.startsWith(`${literal}/`);
+        };
+        return {
+            ...result,
+            stdout: entries
+                .filter((entry) => !(othersOnly && entry[0] !== "?") &&
+                !(trackedOnly && entry[0] === "?"))
+                .map((entry) => entry.slice(2))
+                .filter((path) => (!include.length ||
+                include.some((filter) => matches(path, filter))) &&
+                !exclude.some((filter) => matches(path, filter)))
+                .map((path) => `${path}\0`)
+                .join(""),
+        };
+    };
+}