jev-agent-tools 0.1.3

package/src/adapters/syntax.ts

@@ -0,0 +1,229 @@
+import type { SgNode } from "@ast-grep/napi";
+import type { SyntaxRootParser } from "../core/syntax.ts";
+import type {
+  Declaration,
+  DeclarationResult,
+  SyntaxParser,
+} from "../core/units.ts";
+
+let parserPromise: Promise<SyntaxRootParser | undefined> | undefined;
+/** Optional grammars resolve from this extension, not the repository under review. */
+export function loadSyntaxParser(): Promise<SyntaxParser | undefined> {
+  parserPromise ??= createParser();
+  return parserPromise;
+}
+/** Raw AST access shares grammar availability and registration with declaration extraction. */
+export function loadSyntaxRootParser(): Promise<SyntaxRootParser | undefined> {
+  parserPromise ??= createParser();
+  return parserPromise;
+}
+async function createParser(): Promise<SyntaxRootParser | undefined> {
+  try {
+    const napi = await import("@ast-grep/napi");
+    let python = false;
+    try {
+      const language = await import("@ast-grep/lang-python");
+      napi.registerDynamicLanguage({ python: language.default });
+      python = true;
+    } catch {
+      /* buildUnits reports the missing grammar for each affected file. */
+    }
+    const parser: SyntaxRootParser = {
+      supports: (path) =>
+        /\.[cm]?[jt]sx?$/.test(path) || (python && path.endsWith(".py")),
+      parseRaw(path, text) {
+        try {
+          const language = path.endsWith(".py")
+            ? "python"
+            : /\.[cm]?tsx?$/.test(path)
+              ? path.endsWith("tsx")
+                ? napi.Lang.Tsx
+                : napi.Lang.TypeScript
+              : path.endsWith(".jsx")
+                ? "jsx"
+                : napi.Lang.JavaScript;
+          const root = napi.parse(language, text).root();
+          return { ok: true, root };
+        } catch {
+          return { ok: false, error: "Source parser unavailable" };
+        }
+      },
+      parse(path, text) {
+        const parsed = this.parseRaw(path, text);
+        if (!parsed.ok) return parsed;
+        return parsed.root.find({ rule: { kind: "ERROR" } })
+          ? { ok: false, error: "Source syntax incomplete" }
+          : parsed;
+      },
+      declarations(path, text, options) {
+        const isPython = path.endsWith(".py");
+        const parsed = this.parseRaw(path, text);
+        if (!parsed.ok)
+          return { declarations: [], limits: [{ kind: "parse_failed" }] };
+        const root = parsed.root;
+        const result: Declaration[] = [];
+        const limits: DeclarationResult["limits"] = [];
+        const errors = root.findAll({ rule: { kind: "ERROR" } });
+        const declarationKind =
+          /^(?:function_declaration|generator_function_declaration|method_definition|function_definition|lexical_declaration|variable_declaration|type_alias_declaration|interface_declaration|enum_declaration|class_declaration|class_definition)$/;
+        if (
+          errors.some(
+            (error) =>
+              !error
+                .ancestors()
+                .some((node) => declarationKind.test(String(node.kind()))) &&
+              (error
+                .findAll({ rule: { regex: ".*" } })
+                .some((node) => declarationKind.test(String(node.kind()))) ||
+                /^(?:export\s+)?(?:function|const|let|var|class)\b/.test(
+                  error.text().trim(),
+                )),
+          )
+        )
+          return { declarations: [], limits: [{ kind: "parse_failed" }] };
+        const attributedErrors = new Set<number>();
+        const source = text.split("\n");
+        const visit = (node: SgNode) => {
+          const kind = String(node.kind());
+          // Most AST nodes cannot yield a declaration; avoid constructing their ancestor chains.
+          if (
+            !/^(?:function_declaration|generator_function_declaration|method_definition|function_definition|lexical_declaration|variable_declaration|type_alias_declaration|interface_declaration|enum_declaration|expression_statement|class_declaration|class_definition)$/.test(
+              kind,
+            )
+          ) {
+            for (const child of node.children()) visit(child);
+            return;
+          }
+          const ancestors = node.ancestors();
+          const functionNode =
+            /^(?:function_declaration|generator_function_declaration|method_definition|function_definition)$/.test(
+              kind,
+            );
+          const variable =
+            /^(?:lexical_declaration|variable_declaration|type_alias_declaration|interface_declaration|enum_declaration)$/.test(
+              kind,
+            ) &&
+            ancestors.every(
+              (a) => !/function|method|statement_block/.test(String(a.kind())),
+            );
+          const callableVariable =
+            variable &&
+            node.find({
+              rule: {
+                any: [
+                  { kind: "arrow_function" },
+                  { kind: "function_expression" },
+                ],
+              },
+            }) !== null;
+          const assignment =
+            isPython &&
+            kind === "expression_statement" &&
+            node.parent()?.kind() === "module" &&
+            node.children().some((child) => child.kind() === "assignment");
+          const classNode =
+            options?.descendAbove !== undefined &&
+            /^(?:class_declaration|class_definition)$/.test(kind);
+          if (functionNode || variable || assignment || classNode) {
+            let envelope = node;
+            if (
+              node.parent()?.kind() === "decorated_definition" ||
+              node.parent()?.kind() === "export_statement"
+            )
+              envelope = node.parent() ?? node;
+            const named =
+              node.field("name") ?? node.find({ rule: { kind: "identifier" } });
+            const owner = ancestors
+              .filter((a) =>
+                /^(?:class_definition|class_declaration|internal_module|module)$/.test(
+                  String(a.kind()),
+                ),
+              )
+              .reverse()
+              .map((a) => a.field("name")?.text())
+              .filter(Boolean);
+            let name = [...owner, named?.text() ?? "declaration"].join(".");
+            if (assignment)
+              name = node.text().split(/\s*[:=]/, 1)[0] ?? "declaration";
+            let start = envelope.range().start.line;
+            const body =
+              node.field("body") ??
+              (callableVariable
+                ? node
+                    .find({
+                      rule: {
+                        any: [
+                          { kind: "arrow_function" },
+                          { kind: "function_expression" },
+                        ],
+                      },
+                    })
+                    ?.field("body")
+                : null);
+            const tokens = (part: SgNode): string[] => {
+              const children = part.children();
+              if (
+                !children.length ||
+                /string|comment/.test(String(part.kind()))
+              )
+                return part.kind() === "comment" ? [] : [part.text()];
+              return children.flatMap(tokens);
+            };
+            const accessor =
+              kind === "method_definition"
+                ? node
+                    .children()
+                    .find((child) => /^(get|set)$/.test(String(child.kind())))
+                    ?.kind()
+                : undefined;
+            const end = envelope.range().end;
+            // Include the immediately preceding comment block without swallowing another declaration.
+            while (
+              start > 0 &&
+              /^\s*(?:\/\/|\/\*|\*|\*\/|#)/.test(source[start - 1] ?? "")
+            )
+              start--;
+            result.push({
+              kind:
+                functionNode || callableVariable ? "function" : "declaration",
+              name,
+              type: accessor ? `${kind}:${accessor}` : kind,
+              signatureEnd: body
+                ? body.range().start.line + (isPython ? 0 : 1)
+                : undefined,
+              body: body ? JSON.stringify(tokens(body)) : undefined,
+              exported: isPython
+                ? !name.split(".").some((part) => part.startsWith("_"))
+                : ancestors.some((a) => a.kind() === "export_statement") ||
+                  envelope.kind() === "export_statement",
+              start: start + 1,
+              end: end.line + (end.column === 0 ? 0 : 1),
+            });
+            const declarationErrors = envelope.findAll({
+              rule: { kind: "ERROR" },
+            });
+            if (declarationErrors.length) {
+              limits.push({ kind: "parse_partial", declaration: name });
+              for (const error of declarationErrors)
+                attributedErrors.add(error.id());
+            }
+            if (
+              options?.descendAbove === undefined ||
+              end.line + (end.column === 0 ? 0 : 1) - start <=
+                options.descendAbove
+            )
+              return;
+          }
+          for (const child of node.children()) visit(child);
+        };
+        visit(root);
+        if (errors.some((error) => !attributedErrors.has(error.id())))
+          limits.push({ kind: "parse_partial" });
+        return { declarations: result, limits };
+      },
+    };
+    return parser;
+  } catch {
+    return undefined;
+  }
+}

package/src/adapters/test-inventory.ts

@@ -0,0 +1,168 @@
+import { type FileHandle, lstat } from "node:fs/promises";
+import { TEST_SOURCE_MAX_BYTES, TIMEOUT_MS } from "../constants.ts";
+import type { GitExec } from "../core/git.ts";
+import type { ImportSource } from "../core/imports.ts";
+import type { Result } from "../result.ts";
+import {
+  checkFileAdmission,
+  openRepoFile,
+  resolveInsideRepo,
+} from "./files.ts";
+import { decodeUtf8 } from "./utf8.ts";
+
+export async function collectTestInventory(
+  exec: GitExec,
+  cwd: string,
+  signal?: AbortSignal,
+): Promise<
+  Result<{
+    cwd: string;
+    paths: readonly string[];
+    read(path: string): Promise<ImportSource | undefined>;
+    limits: readonly {
+      path: string;
+      kind: "unreadable" | "too_large" | "not UTF-8 text";
+    }[];
+  }>
+> {
+  const root = await exec("git", ["rev-parse", "--show-toplevel"], {
+    cwd,
+    timeout: TIMEOUT_MS,
+    signal,
+  });
+  if (root.code !== 0 || root.killed)
+    return {
+      ok: false,
+      error: root.stderr || "Cannot identify repository root.",
+    };
+  cwd = root.stdout.replace(/\n$/, "");
+  const listed = await exec(
+    "git",
+    [
+      "ls-files",
+      "--cached",
+      "--others",
+      "--exclude-standard",
+      "--full-name",
+      "-z",
+    ],
+    { cwd, timeout: TIMEOUT_MS, signal },
+  );
+  if (listed.code !== 0 || listed.killed)
+    return {
+      ok: false,
+      error: listed.stderr || "Cannot inventory repository files.",
+    };
+  const ignored = await exec(
+    "git",
+    ["ls-files", "--cached", "--ignored", "--exclude-standard", "-z"],
+    { cwd, timeout: TIMEOUT_MS, signal },
+  );
+  if (ignored.code !== 0 || ignored.killed)
+    return {
+      ok: false,
+      error: ignored.stderr || "Cannot identify ignored tracked files.",
+    };
+  const excluded = new Set(ignored.stdout.split("\0").filter(Boolean));
+  const paths = listed.stdout
+    .split("\0")
+    .filter(
+      (path) =>
+        path &&
+        (!excluded.has(path) ||
+          /(?:^|\/)(?:package|tsconfig[^/]*)\.json$/.test(path)),
+    );
+  const admitted = new Set(paths);
+  const limits: {
+    path: string;
+    kind: "unreadable" | "too_large" | "not UTF-8 text";
+  }[] = [];
+  const cache = new Map<string, Promise<ImportSource | undefined>>();
+  for (let offset = 0; offset < paths.length; offset += 16) {
+    const inspected = await Promise.all(
+      paths.slice(offset, offset + 16).map(async (path) => {
+        try {
+          signal?.throwIfAborted();
+          const admission = await checkFileAdmission(
+            cwd,
+            path,
+            exec,
+            signal,
+            admitted,
+          );
+          if (!admission.ok) return "unreadable" as const;
+          const resolved = await resolveInsideRepo(cwd, path);
+          if (!resolved.ok) return "unreadable" as const;
+          const stat = await lstat(resolved.abs);
+          if (!stat.isFile()) return "unreadable" as const;
+          return stat.size > TEST_SOURCE_MAX_BYTES
+            ? ("too_large" as const)
+            : undefined;
+        } catch {
+          return "unreadable" as const;
+        }
+      }),
+    );
+    for (const [index, kind] of inspected.entries()) {
+      if (!kind) continue;
+      const path = paths[offset + index];
+      if (path === undefined || cache.has(path)) continue;
+      limits.push({ path, kind });
+      cache.set(path, Promise.resolve(undefined));
+    }
+  }
+  const read = (path: string): Promise<ImportSource | undefined> => {
+    const cached = cache.get(path);
+    if (cached) return cached;
+    const pending = (async () => {
+      let handle: FileHandle | undefined;
+      try {
+        signal?.throwIfAborted();
+        const opened = await openRepoFile(cwd, path, {
+          exec,
+          signal,
+          inventory: admitted,
+        });
+        if (!opened.ok) return { path, kind: "unreadable" as const };
+        handle = opened.handle;
+        const stat = { size: opened.size };
+        if (stat.size > TEST_SOURCE_MAX_BYTES)
+          return { path, kind: "too_large" as const };
+        const bytes = Buffer.alloc(
+          Math.min(stat.size + 1, TEST_SOURCE_MAX_BYTES + 1),
+        );
+        let count = 0;
+        while (count < bytes.length) {
+          signal?.throwIfAborted();
+          const part = await handle.read(
+            bytes,
+            count,
+            bytes.length - count,
+            null,
+          );
+          if (!part.bytesRead) break;
+          count += part.bytesRead;
+        }
+        if (count === bytes.length) return { path, kind: "too_large" as const };
+        if (bytes.subarray(0, count).includes(0)) return { path, text: "" };
+        const decoded = decodeUtf8(bytes.subarray(0, count));
+        if (!decoded.ok) return { path, kind: "not UTF-8 text" as const };
+        const text = decoded.text;
+        return { path, text };
+      } catch {
+        return { path, kind: "unreadable" as const };
+      } finally {
+        await handle?.close();
+      }
+    })().then((file) => {
+      if (file.kind) {
+        limits.push({ path: file.path, kind: file.kind });
+        return undefined;
+      }
+      return { path: file.path, text: file.text };
+    });
+    cache.set(path, pending);
+    return pending;
+  };
+  return { ok: true, cwd, paths, read, limits };
+}

package/src/adapters/usage.ts

@@ -0,0 +1,21 @@
+import type { JudgmentMetadata } from "../jev/types.ts";
+
+export function hostUsage(isOmp: boolean, usage: JudgmentMetadata["usage"]) {
+  if (isOmp || !usage) return {};
+  return {
+    usage: {
+      input: usage.inputTokens,
+      output: 0,
+      cacheRead: 0,
+      cacheWrite: 0,
+      totalTokens: usage.inputTokens,
+      cost: {
+        input: usage.costUsd,
+        output: 0,
+        cacheRead: 0,
+        cacheWrite: 0,
+        total: usage.costUsd,
+      },
+    },
+  };
+}

package/src/adapters/utf8.ts

@@ -0,0 +1,57 @@
+import { isUtf8 } from "node:buffer";
+import { createHash } from "node:crypto";
+import type { Result } from "../result.ts";
+
+/** Callers enforce byte budgets before decoding. BOM and legitimate controls are preserved. */
+export function decodeUtf8(bytes: Uint8Array): Result<{ text: string }> {
+  return isUtf8(bytes)
+    ? {
+        ok: true,
+        text: Buffer.from(
+          bytes.buffer,
+          bytes.byteOffset,
+          bytes.byteLength,
+        ).toString("utf8"),
+      }
+    : { ok: false, error: "not UTF-8 text" };
+}
+
+/** Keep incomplete code points between chunks, but reject them at EOF. */
+export function createUtf8Decoder() {
+  const decoder = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true });
+  return (bytes?: Uint8Array): Result<{ text: string }> => {
+    try {
+      return {
+        ok: true,
+        text: decoder.decode(bytes, { stream: bytes !== undefined }),
+      };
+    } catch {
+      return { ok: false, error: "not UTF-8 text" };
+    }
+  };
+}
+
+/** Verify host-decoded text against the raw Git object; replacement cannot silently become evidence. */
+export function verifyGitUtf8(
+  text: string,
+  id?: string,
+  size?: number,
+): Result<{ text: string }> {
+  const bytes = Buffer.from(text, "utf8");
+  if (size !== undefined && bytes.length !== size)
+    return { ok: false, error: "not UTF-8 text" };
+  if (id && /^(?:[a-f0-9]{40}|[a-f0-9]{64})$/.test(id)) {
+    const actual = createHash(id.length === 64 ? "sha256" : "sha1")
+      .update(`blob ${bytes.length}\0`)
+      .update(bytes)
+      .digest("hex");
+    if (actual !== id) return { ok: false, error: "not UTF-8 text" };
+  } else if (text.includes("\uFFFD")) {
+    return {
+      ok: false,
+      error:
+        "not verifiable as UTF-8 text (contains U+FFFD after host decoding)",
+    };
+  }
+  return decodeUtf8(bytes);
+}

package/src/constants.ts

@@ -0,0 +1,109 @@
+export const STATE_MAX_CHARS = 80_000;
+export const TEST_SOURCE_MAX_BYTES = STATE_MAX_CHARS * 16;
+export const SELECT_FILE_SHARE = 0.8;
+export const ASK_NOTE_MAX_CHARS = 8_000;
+export const ASK_MAX_FILES = 20;
+export const ASK_TIMEOUT_S = 60;
+export const ASK_TIMEOUT_MAX_S = 300;
+export const OUTPUT_REPEAT_MIN = 5;
+export const OUTPUT_CHUNK_CHARS = 2_500;
+export const OUTPUT_FIND_MAX_CALLS = 40;
+export const OUTPUT_HEAD_SHARE = 0.05;
+export const OUTPUT_TAIL_SHARE = 0.15;
+// Unmeasured admission bound for each command output file; never truncate it silently.
+export const OUTPUT_FILE_MAX_BYTES = 64 * 1024 * 1024;
+export const OUTPUT_LINE_MAX_CHARS = 8_192;
+export const OUTPUT_SHAPE_MAX_COUNT = 2_048;
+export const OUTPUT_FAILURE_WINDOW_LINES = 20;
+export const CHOICE_MAX_OPTIONS = 255;
+export const MAX_FILES = 255;
+export const FILE_MAX_KB = STATE_MAX_CHARS / 1_000;
+export const SCORE_MIN_LEVELS = 2;
+export const SCORE_MAX_LEVELS = 10;
+export const TIMEOUT_MS = 10_000;
+export const UNIT_MAX_CHARS = 6_000;
+export const UNIT_CONTEXT_LINES = 8;
+export const SELECT_MIN = 0.5;
+export const BAND_BOOL_GRAY_A = 0.2;
+export const BAND_BOOL_YES_MIN = 1 - BAND_BOOL_GRAY_A;
+export const ORDER_REVERSE_BELOW = 0.85;
+export const ORDER_DISAGREE_MIN = 0.1;
+export const BAND_CHOICE_VERDICT_MIN = ORDER_REVERSE_BELOW;
+export const CANNOT_TELL_MIN = 0.3;
+export const REQUEST_MAX_TOKENS = 60_000;
+export const RATE_PER_SECOND = 8;
+export const CONCURRENCY = 8;
+export const REQUEST_ATTEMPTS = 3;
+export const RETRY_BASE_MS = 500;
+export const RETRY_MAX_MS = 5_000;
+export const HTTP_ERROR_MAX_CHARS = 300;
+export const QUESTION_TOKENS = 1 / 3.3;
+export const STATE_TOKENS = 0.283;
+export const REQUEST_BASE_TOKENS = 456;
+export const FLAG_MIN = 0.7;
+export const DOCS_CHECK_MIN = 0.2;
+export const DOCS_MAX_SECTIONS = 40;
+export const HOOK_BUDGET_MS = 15_000;
+/** Progressive collection time bound, not a coverage or wall-time guarantee. */
+export const DOCS_COLLECT_BUDGET_MS = 750;
+export const DOCS_NAME_MAX_FILES = 32;
+/** Initial per-anchor traversal bound; shared edge cache remains unbounded. */
+export const DOCS_ANCHOR_MAX_FILES = 32;
+export const WITNESS_LURE_MAX = 0.15;
+/** Small coverage decoys only; risk and large-witness policies remain unchanged. */
+export const COVERAGE_WITNESS_LURE_MAX = 0.2;
+export const WITNESS_LARGE_LURE_MAX = 0.25;
+export const WITNESS_LARGE_CHARS = 13_000;
+export const WITNESS_ETALON_MIN = 0.7;
+export const WITNESS_AUTO_MIN_CELLS = 8;
+export const CLOSURE_MAX_CHARS = 20_000;
+export const CALLER_DISPLAY_MAX_SPANS = 8;
+export const LIMIT_DISPLAY_MAX_PATHS = 4;
+export const DOCS_DISPLAY_MAX_SECTIONS = 5;
+export const FIND_NAME_CANDIDATES = 128;
+export const FIND_NAME_BATCH = 64;
+export const FIND_FILES_READ = 20;
+export const FIND_READ_PER_CALL = 5;
+export const FIND_EXCERPT_CHARS = 3_000;
+export const FIND_POINTER_EXCERPT_CHARS = 1_500;
+export const FIND_POINTER_MAX = 8;
+export const FIND_NAME_GUARD_MIN = 0.9;
+export const FIND_CONFIRM_MIN = 0.8;
+export const FIND_GOAL_MIN_WORDS = 5;
+export const FIND_QUICK_CANDIDATES = 32;
+export const FIND_QUICK_READ = 8;
+export const FIND_THOROUGH_CANDIDATES = 256;
+export const FIND_THOROUGH_READ = 40;
+export const FIND_EXCERPT_HEAD_CHARS = 1_000;
+export const FIND_CONTENT_MIN = 0.5;
+export const FIND_NAME_GUARD_TOP = 3;
+export const FIND_PATH_WEIGHT = 3;
+export const FIND_GREP_PAGE_SIZE = 1024;
+export const FIND_READ_CHUNK_BYTES = 16_384;
+export const LOCATE_MIN_KB = 19;
+export const LOCATE_VERDICT_MIN = 0.7;
+export const LOCATE_GRAY_MIN = 0.4;
+export const LOCATE_SHRINK_TOP = 3;
+export const LOCATE_SECTION_MAX_LINES = 150;
+export const LOCATE_SECTION_MIN_LINES = 8;
+export const LOCATE_WINDOW_LINES = 80;
+export const LOCATE_LABEL_MAX_CHARS = 120;
+export const LOCATE_WHOLE_MAX_CHARS = STATE_MAX_CHARS * 4;
+export const LOCATE_WHOLE_MAX_BYTES = LOCATE_WHOLE_MAX_CHARS * 4;
+export const LOCATE_READ_BUFFER_BYTES = 64 * 1024;
+// Unmeasured engineering bound: leaves room for multi-window refinement and JSON metadata.
+export const LOCATE_WINDOW_MAX_SERIALIZED_CHARS = STATE_MAX_CHARS / 4;
+// Keep SHA-256 IDs plus options below Windows' command-line limit.
+export const GIT_BLOB_BATCH_SIZE = 400;
+export const RUNNER_PACKAGE_MAX_BYTES = 64 * 1024;
+
+// ADR 0001: allowed local dependencies, including erased type imports.
+export const IMPORT_LAYERS = {
+  core: ["core", "constants.ts", "result.ts", "jev/types.ts"],
+  presets: ["presets", "core", "constants.ts", "result.ts", "jev/types.ts"],
+  adapters: ["adapters", "core", "constants.ts", "result.ts", "jev/types.ts"],
+  jev: ["jev", "core", "constants.ts", "result.ts"],
+  texts: ["texts", "constants.ts", "presets"],
+  "constants.ts": [],
+  "result.ts": [],
+} as const;