jev-agent-tools 0.1.3

package/src/adapters/files.ts

@@ -0,0 +1,411 @@
+import { createHash } from "node:crypto";
+import { constants } from "node:fs";
+import {
+  type FileHandle,
+  lstat,
+  open,
+  readlink,
+  realpath,
+} from "node:fs/promises";
+import { isAbsolute, relative, resolve, sep } from "node:path";
+import {
+  ASK_MAX_FILES,
+  FIND_EXCERPT_CHARS,
+  FIND_POINTER_EXCERPT_CHARS,
+  FIND_READ_CHUNK_BYTES,
+  STATE_MAX_CHARS,
+  TIMEOUT_MS,
+} from "../constants.ts";
+import { createExcerptCollector } from "../core/find.ts";
+import type { GitExec } from "../core/git.ts";
+import type { FileIdentity } from "../core/integrity.ts";
+import type { Result } from "../result.ts";
+import { createUtf8Decoder, decodeUtf8 } from "./utf8.ts";
+
+export async function checkFileAdmission(
+  cwd: string,
+  path: string,
+  exec?: GitExec,
+  signal?: AbortSignal,
+  inventory?: ReadonlySet<string>,
+): Promise<Result<object>> {
+  if (path.split(/[\\/]/).includes(".git"))
+    return { ok: false, error: `${path}: .git metadata: not sent to Jev` };
+  if (inventory)
+    return inventory.has(path)
+      ? { ok: true }
+      : { ok: false, error: `${path}: gitignored: not sent to Jev` };
+  if (!exec) return { ok: true };
+  const listed = await exec(
+    "git",
+    [
+      "ls-files",
+      "--cached",
+      "--others",
+      "--exclude-standard",
+      "-z",
+      "--",
+      path,
+    ],
+    { cwd, timeout: TIMEOUT_MS, signal },
+  );
+  if (listed.killed)
+    return { ok: false, error: `${path}: file admission interrupted` };
+  if (listed.code === 0 && !listed.stdout.split("\0").includes(path))
+    return { ok: false, error: `${path}: gitignored: not sent to Jev` };
+  return { ok: true };
+}
+
+export async function resolveInsideRepo(
+  root: string,
+  path: string,
+  options: { allowLeafSymlink?: boolean } = {},
+): Promise<Result<{ abs: string; rel: string }>> {
+  if (isAbsolute(path) || path.split(/[\\/]/).includes(".."))
+    return {
+      ok: false,
+      error: `Path must remain inside the repository: ${path}.`,
+    };
+  try {
+    const base = await realpath(root);
+    const requested = resolve(base, path);
+    const rel = relative(base, requested);
+    if (rel === ".." || rel.startsWith(`..${sep}`) || isAbsolute(rel))
+      return { ok: false, error: `Path escapes the repository: ${path}.` };
+    let current = base;
+    for (const part of rel.split(sep).filter(Boolean)) {
+      current = resolve(current, part);
+      if ((await lstat(current)).isSymbolicLink()) {
+        if (options.allowLeafSymlink && current === requested)
+          return { ok: true, abs: requested, rel: rel.replaceAll(sep, "/") };
+        return { ok: false, error: `Symbolic links are not read: ${path}.` };
+      }
+    }
+    const abs = await realpath(requested);
+    const actual = relative(base, abs);
+    if (actual === ".." || actual.startsWith(`..${sep}`) || isAbsolute(actual))
+      return { ok: false, error: `Path escapes the repository: ${path}.` };
+    return { ok: true, abs, rel: actual.replaceAll(sep, "/") };
+  } catch (error) {
+    return { ok: false, error: `Cannot resolve ${path}: ${String(error)}` };
+  }
+}
+
+/** Leaf-link metadata is admitted without following the target. */
+export async function admitRepoMetadata(
+  cwd: string,
+  path: string,
+  exec?: GitExec,
+  signal?: AbortSignal,
+  inventory?: ReadonlySet<string>,
+): Promise<Result<{ abs: string; rel: string }>> {
+  const location = await resolveInsideRepo(cwd, path, {
+    allowLeafSymlink: true,
+  });
+  if (!location.ok) return location;
+  const admission = await checkFileAdmission(
+    cwd,
+    location.rel,
+    exec,
+    signal,
+    inventory,
+  );
+  return admission.ok ? location : admission;
+}
+
+/** History is admitted by its tracked base tree, independent of worktree ignores or links. */
+export async function admitGitPath(
+  cwd: string,
+  path: string,
+  base: string,
+  exec: GitExec,
+  signal?: AbortSignal,
+  inventory?: ReadonlyMap<string, string>,
+): Promise<Result<object>> {
+  if (isAbsolute(path) || path.split(/[\\/]/).includes(".."))
+    return {
+      ok: false,
+      error: `Path must remain inside the repository: ${path}.`,
+    };
+  const admission = await checkFileAdmission(cwd, path, undefined, signal);
+  if (!admission.ok) return admission;
+  // ls-tree cannot descend through a symlink or gitlink ancestor in the base tree.
+  let entry = inventory?.get(path);
+  if (!inventory) {
+    const tree = await exec(
+      "git",
+      ["ls-tree", "-z", "--full-tree", "--end-of-options", base, "--", path],
+      { cwd, timeout: TIMEOUT_MS, signal },
+    );
+    if (tree.code !== 0 || tree.killed)
+      return { ok: false, error: `${path}: not a file at comparison base` };
+    entry = tree.stdout
+      .split("\0")
+      .find((item) => item.slice(item.indexOf("\t") + 1) === path);
+  }
+  if (!entry || !/^(?:100644|100755|120000) blob /.test(entry))
+    return { ok: false, error: `${path}: not a file at comparison base` };
+  return { ok: true };
+}
+
+/** The only content opener for repository files. Callers own and close the handle. */
+export async function openRepoFile(
+  cwd: string,
+  path: string,
+  options: {
+    exec?: GitExec;
+    signal?: AbortSignal;
+    inventory?: ReadonlySet<string>;
+  } = {},
+): Promise<Result<{ handle: FileHandle; size: number }>> {
+  try {
+    options.signal?.throwIfAborted();
+    const location = await resolveInsideRepo(cwd, path);
+    if (!location.ok) return location;
+    const admission = await checkFileAdmission(
+      cwd,
+      location.rel,
+      options.exec,
+      options.signal,
+      options.inventory,
+    );
+    if (!admission.ok) return admission;
+    const handle = await open(
+      location.abs,
+      constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK,
+    );
+    try {
+      const stat = await handle.stat();
+      if (!stat.isFile()) {
+        await handle.close();
+        return { ok: false, error: `Not a file: ${path}.` };
+      }
+      return { ok: true, handle, size: stat.size };
+    } catch (error) {
+      await handle.close();
+      throw error;
+    }
+  } catch (error) {
+    return { ok: false, error: `Cannot read ${path}: ${String(error)}` };
+  }
+}
+
+export async function collectFiles(
+  cwd: string,
+  paths: string[],
+  signal?: AbortSignal,
+  options: {
+    exec?: GitExec;
+    allowEmpty?: boolean;
+    skipInvalidUtf8?: boolean;
+    inventory?: ReadonlySet<string>;
+  } = {},
+): Promise<
+  Result<{
+    files: Record<string, string>;
+    identities: FileIdentity[];
+    skipped: string[];
+  }>
+> {
+  if (paths.length > ASK_MAX_FILES)
+    return {
+      ok: false,
+      error: `paths exceeds ${ASK_MAX_FILES} files; split paths into several calls.`,
+    };
+  const results = await Promise.all(
+    paths.map(async (path) => {
+      if (/^[a-z][a-z0-9+.-]*:\/\//i.test(path))
+        return {
+          ok: false as const,
+          error: `Internal URLs are not files; use read for ${path}.`,
+        };
+      let handle: FileHandle | undefined;
+      try {
+        signal?.throwIfAborted();
+        const opened = await openRepoFile(cwd, path, {
+          exec: options.exec,
+          signal,
+          inventory: options.inventory,
+        });
+        if (!opened.ok) return opened;
+        handle = opened.handle;
+        const bytes = Buffer.alloc(
+          Math.min(opened.size + 1, STATE_MAX_CHARS * 4 + 1),
+        );
+        let count = 0;
+        while (count < bytes.length) {
+          signal?.throwIfAborted();
+          const read = await handle.read(
+            bytes,
+            count,
+            bytes.length - count,
+            null,
+          );
+          if (!read.bytesRead) break;
+          count += read.bytesRead;
+        }
+        if (count === bytes.length)
+          return {
+            ok: false as const,
+            error: `File exceeds ${STATE_MAX_CHARS} chars: ${path}; files are never truncated. Split paths into several calls.`,
+          };
+        const decoded = decodeUtf8(bytes.subarray(0, count));
+        if (!decoded.ok)
+          return {
+            ok: false as const,
+            invalidUtf8: true,
+            error: `${path} is not UTF-8 text; provide a UTF-8 text file instead.`,
+          };
+        const text = decoded.text;
+        if (count === bytes.length || text.length > STATE_MAX_CHARS)
+          return {
+            ok: false as const,
+            error: `File exceeds ${STATE_MAX_CHARS} chars: ${path}; files are never truncated. Split paths into several calls.`,
+          };
+        if (!text.length && !options.allowEmpty)
+          return { ok: false as const, error: `${path} is empty.` };
+        return {
+          ok: true as const,
+          path,
+          text,
+          identity: {
+            requestedPath: path,
+            resolvedPath: resolve(cwd, path),
+            insertedPath: resolve(cwd, path),
+            content: text,
+            readSha256: createHash("sha256")
+              .update(bytes.subarray(0, count))
+              .digest("hex"),
+            insertedSha256: createHash("sha256").update(text).digest("hex"),
+          },
+        };
+      } catch (error) {
+        return {
+          ok: false as const,
+          error: `Cannot read ${path}: ${String(error)}`,
+        };
+      } finally {
+        await handle?.close();
+      }
+    }),
+  );
+  const failure = results.find(
+    (result) =>
+      !result.ok && !(options.skipInvalidUtf8 && "invalidUtf8" in result),
+  );
+  if (failure && !failure.ok) return failure;
+  return {
+    ok: true,
+    skipped: results.flatMap((result) =>
+      !result.ok && "invalidUtf8" in result ? [result.error] : [],
+    ),
+    identities: results.flatMap((result) =>
+      result.ok ? [result.identity] : [],
+    ),
+    files: Object.fromEntries(
+      results.flatMap((result) =>
+        result.ok ? [[result.path, result.text]] : [],
+      ),
+    ),
+  };
+}
+
+export async function collectSearchExcerpts(
+  cwd: string,
+  paths: readonly string[],
+  keywords: readonly string[],
+  signal?: AbortSignal,
+  exec?: GitExec,
+): Promise<
+  Result<{
+    files: Record<string, string>;
+    pointers: Record<string, string>;
+    ignored: string[];
+    limits: { path: string; reason: string }[];
+  }>
+> {
+  const files: Record<string, string> = {},
+    pointers: Record<string, string> = {};
+  const ignored: string[] = [];
+  const limits: { path: string; reason: string }[] = [];
+  const results = await Promise.all(
+    paths.map(async (path) => {
+      if (/^[a-z][a-z0-9+.-]*:\/\//i.test(path))
+        return {
+          ok: false as const,
+          error: `Internal URLs are not files; use read for ${path}.`,
+        };
+      try {
+        const location = await admitRepoMetadata(cwd, path, exec, signal);
+        if (!location.ok) {
+          ignored.push(path);
+          return { ok: true as const };
+        }
+        if ((await lstat(location.abs)).isSymbolicLink()) {
+          const target = await readlink(location.abs);
+          files[path] = target;
+          pointers[path] = target;
+          return { ok: true as const };
+        }
+        const opened = await openRepoFile(cwd, path, { exec, signal });
+        if (!opened.ok) {
+          ignored.push(path);
+          return { ok: true as const };
+        }
+        const handle = opened.handle;
+        const content = createExcerptCollector(keywords, FIND_EXCERPT_CHARS);
+        const pointer = createExcerptCollector(
+          keywords,
+          FIND_POINTER_EXCERPT_CHARS,
+        );
+        const decode = createUtf8Decoder();
+        const stream = handle.createReadStream({
+          highWaterMark: FIND_READ_CHUNK_BYTES,
+          signal,
+        });
+        for await (const chunk of stream) {
+          const decoded = decode(chunk as Buffer);
+          if (!decoded.ok) {
+            limits.push({ path, reason: decoded.error });
+            ignored.push(path);
+            stream.destroy();
+            return { ok: true as const };
+          }
+          const text = decoded.text;
+          if (text.includes("\0")) {
+            ignored.push(path);
+            stream.destroy();
+            return { ok: true as const };
+          }
+          content.push(text);
+          pointer.push(text);
+        }
+        const end = decode();
+        if (!end.ok) {
+          limits.push({ path, reason: end.error });
+          ignored.push(path);
+          return { ok: true as const };
+        }
+        content.push(end.text);
+        pointer.push(end.text);
+        const excerpt = content.finish();
+        if (!excerpt.length) {
+          ignored.push(path);
+          return { ok: true as const };
+        }
+        files[path] = excerpt;
+        pointers[path] = pointer.finish();
+        return { ok: true as const };
+      } catch (error) {
+        if (signal?.aborted)
+          return { ok: false as const, error: String(error) };
+        ignored.push(path);
+        return { ok: true as const };
+      }
+    }),
+  );
+  const failure = results.find((result) => !result.ok);
+  return failure && !failure.ok
+    ? failure
+    : { ok: true, files, pointers, ignored, limits };
+}

package/src/adapters/find.ts

@@ -0,0 +1,151 @@
+import type { FileFinder, GrepCursor } from "@ff-labs/fff-node";
+import {
+  FIND_GREP_PAGE_SIZE,
+  FIND_PATH_WEIGHT,
+  TIMEOUT_MS,
+} from "../constants.ts";
+import { pathAllowed } from "../core/find.ts";
+import type { GitExec } from "../core/git.ts";
+import type { Result } from "../result.ts";
+
+export async function prefilter(
+  exec: GitExec,
+  cwd: string,
+  keywords: readonly string[],
+  limit: number,
+  scope?: string | readonly string[],
+  exclude?: readonly string[],
+  signal?: AbortSignal,
+  native = true,
+): Promise<Result<{ paths: string[]; scopeFiles: number }>> {
+  try {
+    const listed = await exec(
+      "git",
+      ["ls-files", "-z", "--cached", "--others", "--exclude-standard"],
+      { cwd, timeout: TIMEOUT_MS, signal },
+    );
+    if (listed.killed || listed.code !== 0)
+      return {
+        ok: false,
+        error: `Cannot list repository files: ${listed.stderr || "git interrupted"}`,
+      };
+    const paths = [
+      ...new Set(
+        listed.stdout
+          .split("\0")
+          .filter((path) => path && pathAllowed(path, scope, exclude)),
+      ),
+    ];
+    if (!paths.length) return { ok: true, paths: [], scopeFiles: 0 };
+    const allowed = new Set(paths);
+    let finder: FileFinder | undefined;
+    if (native) {
+      try {
+        // Static loading would break hosts where this optional native package is absent.
+        const { FileFinder } = await import("@ff-labs/fff-node");
+        const created = FileFinder.create({
+          basePath: cwd,
+          disableWatch: true,
+        });
+        if (created.ok) {
+          finder = created.value;
+          const ready = await finder.waitForScan(TIMEOUT_MS);
+          if (!ready.ok || !ready.value) {
+            finder.destroy();
+            finder = undefined;
+          }
+        }
+      } catch {
+        /* Optional native package is unavailable in this host. */
+      }
+    }
+    if (!finder) {
+      const available = await exec("rg", ["--version"], {
+        cwd,
+        timeout: TIMEOUT_MS,
+        signal,
+      });
+      if (available.killed || available.code !== 0)
+        return { ok: false, error: "rg is required when fff is unavailable" };
+    }
+    try {
+      const matches = await Promise.all(
+        keywords.map(async (word) => {
+          signal?.throwIfAborted();
+          if (finder) {
+            const found = new Set<string>();
+            let cursor: GrepCursor | null = null;
+            do {
+              const result = finder.multiGrep({
+                patterns: [word],
+                smartCase: true,
+                maxMatchesPerFile: 1,
+                pageSize: FIND_GREP_PAGE_SIZE,
+                cursor,
+              });
+              if (!result.ok) throw new Error(result.error);
+              for (const item of result.value.items)
+                if (allowed.has(item.relativePath))
+                  found.add(item.relativePath);
+              cursor = result.value.nextCursor;
+            } while (cursor);
+            return found;
+          }
+          const result = await exec(
+            "rg",
+            [
+              "--files-with-matches",
+              "--hidden",
+              "--null",
+              "--fixed-strings",
+              "--ignore-case",
+              "--",
+              word,
+              ".",
+            ],
+            { cwd, timeout: TIMEOUT_MS, signal },
+          );
+          if (
+            result.killed ||
+            (result.code !== 0 && result.code !== 1) ||
+            (result.code === 1 && result.stderr.trim())
+          )
+            throw new Error(result.stderr || "rg interrupted");
+          return new Set(
+            result.stdout
+              .split("\0")
+              .map((path) => path.replace(/^\.\//u, ""))
+              .filter((path) => allowed.has(path)),
+          );
+        }),
+      );
+      const scores = paths.map((path) => ({
+        path,
+        score: keywords.reduce((sum, word, index) => {
+          const matchesForWord = matches[index];
+          if (!matchesForWord) return sum;
+          const weight = Math.log(1 + paths.length / (1 + matchesForWord.size));
+          return (
+            sum +
+            weight *
+              (Number(matchesForWord.has(path)) +
+                FIND_PATH_WEIGHT * Number(path.toLowerCase().includes(word)))
+          );
+        }, 0),
+      }));
+      scores.sort((a, b) => b.score - a.score || a.path.localeCompare(b.path));
+      return {
+        ok: true,
+        paths: scores
+          .filter((item) => item.score > 0)
+          .slice(0, limit)
+          .map((item) => item.path),
+        scopeFiles: paths.length,
+      };
+    } finally {
+      finder?.destroy();
+    }
+  } catch (error) {
+    return { ok: false, error: `Cannot prefilter files: ${String(error)}` };
+  }
+}

package/src/adapters/git-base.ts

@@ -0,0 +1,32 @@
+import { TIMEOUT_MS } from "../constants.ts";
+import type { GitExec } from "../core/git.ts";
+import type { Result } from "../result.ts";
+
+export async function resolveBase(
+  exec: GitExec,
+  cwd: string,
+  base?: string,
+  signal?: AbortSignal,
+): Promise<Result<{ base: string }>> {
+  if (!base) return { ok: true, base: "HEAD" };
+  try {
+    const result = await exec("git", ["merge-base", "--", "HEAD", base], {
+      cwd,
+      timeout: TIMEOUT_MS,
+      signal,
+    });
+    if (result.killed)
+      return { ok: false, error: "Git interrupted (cancelled or timed out)." };
+    if (result.code === 0 && result.stdout.trim())
+      return { ok: true, base: result.stdout.trim() };
+    return {
+      ok: false,
+      error: `Cannot resolve base=${base}: history may be truncated or no shared ancestor is available. Run git fetch --unshallow (or git fetch --deepen=<count>), or choose a base that is an ancestor of HEAD.${result.stderr.trim() ? ` Git: ${result.stderr.trim()}` : ""}`,
+    };
+  } catch (error) {
+    return {
+      ok: false,
+      error: `Unable to resolve base=${base}: ${String(error)}`,
+    };
+  }
+}

package/src/adapters/git-inventory.ts

@@ -0,0 +1,94 @@
+import type { GitExec } from "../core/git.ts";
+
+/** Fresh per tool call: HEAD/index timestamps cannot detect untracked or ignore changes. */
+export function shareGitInventory(exec: GitExec): GitExec {
+  const snapshots = new Map<string, ReturnType<GitExec>>();
+  return async (command, args, options) => {
+    const offset = args.indexOf("ls-files");
+    if (command !== "git" && command !== "rg") snapshots.clear();
+    if (command !== "git" || offset === -1) return exec(command, args, options);
+    const flags = args.slice(offset + 1);
+    if (
+      !flags.includes("-z") ||
+      flags.includes("--ignored") ||
+      flags.some((flag) => flag.startsWith("--with-tree"))
+    )
+      return exec(command, args, options);
+    const separator = flags.indexOf("--");
+    const filters = separator === -1 ? [] : flags.slice(separator + 1);
+    if (
+      filters.some(
+        (path) =>
+          !path.startsWith(":(top,literal)") &&
+          !path.startsWith(":(top,exclude,literal)") &&
+          /[*?[\]:]/.test(path),
+      )
+    )
+      return exec(command, args, options);
+    const allowed = new Set([
+      "-z",
+      "--cached",
+      "--others",
+      "--exclude-standard",
+      "--full-name",
+    ]);
+    if (
+      (separator === -1 ? flags : flags.slice(0, separator)).some(
+        (flag) => !allowed.has(flag),
+      )
+    )
+      return exec(command, args, options);
+    const key = `${options.cwd}\0${flags.includes("--full-name")}`;
+    let snapshot = snapshots.get(key);
+    if (!snapshot) {
+      snapshot = exec(
+        "git",
+        [
+          "ls-files",
+          "--cached",
+          "--others",
+          "--exclude-standard",
+          ...(flags.includes("--full-name") ? ["--full-name"] : []),
+          "-t",
+          "-z",
+        ],
+        options,
+      );
+      snapshots.set(key, snapshot);
+    }
+    const result = await snapshot;
+    if (result.code !== 0 || result.killed) return result;
+    const entries = result.stdout.split("\0").filter(Boolean);
+    const othersOnly =
+      flags.includes("--others") && !flags.includes("--cached");
+    const trackedOnly = !flags.includes("--others");
+    const include = filters.filter(
+      (path) => !path.startsWith(":(top,exclude,literal)"),
+    );
+    const exclude = filters
+      .filter((path) => path.startsWith(":(top,exclude,literal)"))
+      .map((path) => path.slice(":(top,exclude,literal)".length));
+    const matches = (path: string, filter: string) => {
+      const literal = filter.replace(/^:\(top,literal\)/, "");
+      return path === literal || path.startsWith(`${literal}/`);
+    };
+    return {
+      ...result,
+      stdout: entries
+        .filter(
+          (entry) =>
+            !(othersOnly && entry[0] !== "?") &&
+            !(trackedOnly && entry[0] === "?"),
+        )
+        .map((entry) => entry.slice(2))
+        .filter(
+          (path) =>
+            (!include.length ||
+              include.some((filter) => matches(path, filter))) &&
+            !exclude.some((filter) => matches(path, filter)),
+        )
+        .map((path) => `${path}\0`)
+        .join(""),
+    };
+  };
+}