jev-agent-tools 0.1.3

package/src/core/test-state.ts

@@ -0,0 +1,99 @@
+import { STATE_MAX_CHARS } from "../constants.ts";
+import type { State } from "../jev/types.ts";
+import type { TestScenario } from "./test-discovery.ts";
+
+export interface TestStateBatch {
+  state: State;
+  scenarios: readonly TestScenario[];
+}
+export interface PreparedTestStates {
+  batches: readonly TestStateBatch[];
+  unjudged: readonly { scenario: TestScenario; reason: string }[];
+}
+
+/** Partition complete bodies with a single setup mask and additive serialized sizes. */
+export function prepareTestStates(
+  state: State,
+  scenarios: readonly TestScenario[],
+  maxChars = STATE_MAX_CHARS,
+): PreparedTestStates {
+  if (JSON.stringify(state).length <= maxChars)
+    return { batches: [{ state, scenarios }], unjudged: [] };
+  const file = state.testFile;
+  if (
+    !file ||
+    typeof file !== "object" ||
+    Array.isArray(file) ||
+    typeof file.text !== "string"
+  )
+    return {
+      batches: [],
+      unjudged: scenarios.map((scenario) => ({
+        scenario,
+        reason: "required test file absent from state",
+      })),
+    };
+  const source = file.text.split("\n");
+  const mask = new Int32Array(source.length + 1);
+  for (const scenario of scenarios)
+    if (scenario.range) {
+      const start = Math.max(0, scenario.range.start - 1);
+      const end = Math.min(source.length, scenario.range.end);
+      mask[start] = (mask[start] ?? 0) + 1;
+      mask[end] = (mask[end] ?? 0) - 1;
+    }
+  let active = 0;
+  const setup = `${source
+    .map((line, index) => {
+      active += mask[index] ?? 0;
+      return active ? "" : line;
+    })
+    .join("\n")}\n`;
+  const base: State = { ...state, testFile: { ...file, text: setup } };
+  const baseSize = JSON.stringify(base).length;
+  const batches: TestStateBatch[] = [];
+  const unjudged: { scenario: TestScenario; reason: string }[] = [];
+  let group: TestScenario[] = [];
+  let bodies: string[] = [];
+  let size = baseSize;
+  const flush = () => {
+    if (group.length)
+      batches.push({
+        state: {
+          ...base,
+          testFile: { ...file, text: setup + bodies.join("\n") },
+        },
+        scenarios: group,
+      });
+    group = [];
+    bodies = [];
+    size = baseSize;
+  };
+  for (const scenario of scenarios) {
+    if (!scenario.range) {
+      unjudged.push({
+        scenario,
+        reason:
+          "test body range unresolved; complete file exceeds state budget",
+      });
+      continue;
+    }
+    const body = source
+      .slice(scenario.range.start - 1, scenario.range.end)
+      .join("\n");
+    const bodySize = JSON.stringify(body).length - 2;
+    if (baseSize + bodySize > maxChars) {
+      unjudged.push({
+        scenario,
+        reason: `complete test body and required evidence exceed state budget ${maxChars}`,
+      });
+      continue;
+    }
+    if (group.length && size + 2 + bodySize > maxChars) flush();
+    size += bodySize + (group.length ? 2 : 0);
+    group.push(scenario);
+    bodies.push(body);
+  }
+  flush();
+  return { batches, unjudged };
+}

package/src/core/truncate.ts

@@ -0,0 +1,14 @@
+/** Keep a UTF-16 prefix without splitting a valid surrogate pair. */
+export function truncate(text: string, maxChars: number): string {
+  let end = Math.max(0, Math.min(text.length, Math.floor(maxChars)));
+  if (
+    end > 0 &&
+    end < text.length &&
+    text.charCodeAt(end - 1) >= 0xd800 &&
+    text.charCodeAt(end - 1) <= 0xdbff &&
+    text.charCodeAt(end) >= 0xdc00 &&
+    text.charCodeAt(end) <= 0xdfff
+  )
+    end--;
+  return text.slice(0, end);
+}

package/src/core/units.ts

@@ -0,0 +1,531 @@
+import { UNIT_CONTEXT_LINES, UNIT_MAX_CHARS } from "../constants.ts";
+import { type DiffFile, isTestFile } from "./diff.ts";
+import { truncate } from "./truncate.ts";
+
+export interface SourceFile extends DiffFile {
+  before: string | null;
+  after: string | null;
+  limitation?: "too_large" | "unreadable" | "not UTF-8 text";
+}
+export interface Declaration {
+  kind: "function" | "declaration";
+  name: string;
+  exported: boolean;
+  start: number;
+  end: number;
+  /** Syntax type disambiguates declarations sharing a qualified name. */
+  type?: string;
+  signatureEnd?: number;
+  /** Token-normalized body; literal contents are preserved. */
+  body?: string;
+}
+export interface DeclarationResult {
+  declarations: Declaration[];
+  limits: (
+    | { kind: "parse_failed" }
+    | { kind: "parse_partial"; declaration?: string }
+  )[];
+}
+export interface SyntaxParser {
+  declarations(
+    path: string,
+    text: string,
+    options?: { descendAbove?: number },
+  ): DeclarationResult;
+  supports(path: string): boolean;
+}
+export type UnitLimit =
+  | { kind: "grammar_absent"; file: string; language: string }
+  | {
+      kind:
+        | "parse_failed"
+        | "binary_ignored"
+        | "too_large"
+        | "unreadable"
+        | "not UTF-8 text"
+        | "rename_unpaired";
+      file: string;
+    }
+  | { kind: "truncated"; file: string; unit: string }
+  | { kind: "parse_partial"; file: string; declaration?: string };
+export interface EvidenceUnit {
+  id: string;
+  kind: "function" | "slice" | "declaration" | "file" | "hunk";
+  file: string;
+  name: string;
+  exported: boolean;
+  before: string | null;
+  after: string | null;
+  beforeRange: { start: number; end: number } | null;
+  afterRange: { start: number; end: number } | null;
+}
+const code = /\.(?:[cm]?[jt]sx?|py)$/;
+function lines(text: string | null): string[] {
+  return text === null ? [] : text.split("\n");
+}
+function overlaps(
+  start: number,
+  end: number,
+  point: number,
+  count: number,
+): boolean {
+  return count
+    ? point <= end && point + count - 1 >= start
+    : point > start && point <= end;
+}
+/** Conservative fallback: unknown regions remain hunks, never disappear. */
+export function heuristicDeclarations(
+  path: string,
+  text: string,
+): Declaration[] {
+  const source = lines(text);
+  const result: Declaration[] = [];
+  const python = path.endsWith(".py");
+  for (let i = 0; i < source.length; i++) {
+    const line = source[i] ?? "";
+    const match = python
+      ? /^(\s*)(?:async\s+)?def\s+(\w+)/.exec(line)
+      : /^\s*(?:export\s+(?:default\s+)?)?(?:async\s+)?function\s+(\w+)/.exec(
+          line,
+        );
+    const declaration = python
+      ? /^(\w+)\s*(?::[^=]+)?=/.exec(line)
+      : /^\s*(?:export\s+)?(?:const|let|var|type|interface|enum)\s+(\w+)/.exec(
+          line,
+        );
+    if (!match && !declaration) continue;
+    let start = i;
+    let end = i;
+    if (python && match) {
+      const indent = (match[1] ?? "").length;
+      while (start > 0 && (source[start - 1] ?? "").trim().startsWith("@"))
+        start--;
+      while (
+        end + 1 < source.length &&
+        (!(source[end + 1] ?? "").trim() ||
+          (source[end + 1] ?? "").search(/\S/) > indent)
+      )
+        end++;
+    } else if (!python) {
+      let depth = 0;
+      let opened = false;
+      for (let j = i; j < source.length; j++) {
+        for (const char of source[j] ?? "") {
+          if (char === "{") {
+            depth++;
+            opened = true;
+          } else if (char === "}") depth--;
+        }
+        end = j;
+        if (
+          (opened && depth <= 0) ||
+          (!opened && /;\s*$/.test(source[j] ?? ""))
+        )
+          break;
+        if (!opened && j === i && !match) break;
+      }
+    }
+    result.push({
+      kind: match ? "function" : "declaration",
+      name: match
+        ? (match[python ? 2 : 1] ?? "function")
+        : (declaration?.[1] ?? "declaration"),
+      exported: python
+        ? !line.trim().startsWith("def _")
+        : /\bexport\b/.test(line),
+      start: start + 1,
+      end: end + 1,
+    });
+    i = end;
+  }
+  return result;
+}
+
+export function buildUnits(
+  files: readonly SourceFile[],
+  parser?: SyntaxParser,
+): { units: EvidenceUnit[]; limits: UnitLimit[] } {
+  const units: EvidenceUnit[] = [];
+  const limits: UnitLimit[] = [];
+  for (const file of files) {
+    if (isTestFile(file.path)) continue;
+    if (file.binary) {
+      limits.push({ kind: "binary_ignored", file: file.path });
+      continue;
+    }
+    if (file.limitation) {
+      limits.push({ kind: file.limitation, file: file.path });
+      if (
+        file.limitation === "too_large" ||
+        file.limitation === "not UTF-8 text"
+      ) {
+        for (const hunk of file.hunks) {
+          const id = `u${String(units.length + 1).padStart(3, "0")}`;
+          const before = /^[A?]/.test(file.status)
+            ? null
+            : (hunk.beforeText ?? null);
+          const after = file.status === "D" ? null : (hunk.afterText ?? null);
+          const budget =
+            before === null
+              ? 0
+              : after === null
+                ? UNIT_MAX_CHARS
+                : Math.floor(UNIT_MAX_CHARS / 2);
+          units.push({
+            id,
+            kind: "hunk",
+            file: file.path,
+            name: `${file.path}:${hunk.afterStart}`,
+            exported: false,
+            before: before === null ? null : truncate(before, budget),
+            after:
+              after === null
+                ? null
+                : truncate(
+                    after,
+                    UNIT_MAX_CHARS - Math.min(before?.length ?? 0, budget),
+                  ),
+            beforeRange:
+              before === null
+                ? null
+                : {
+                    start: hunk.beforeStart,
+                    end: hunk.beforeStart + hunk.beforeCount - 1,
+                  },
+            afterRange:
+              after === null
+                ? null
+                : {
+                    start: hunk.afterStart,
+                    end: hunk.afterStart + hunk.afterCount - 1,
+                  },
+          });
+          if (
+            hunk.truncated ||
+            (before?.length ?? 0) + (after?.length ?? 0) > UNIT_MAX_CHARS
+          )
+            limits.push({ kind: "truncated", file: file.path, unit: id });
+        }
+      }
+      continue;
+    }
+    const oldLines = lines(file.before);
+    const newLines = lines(file.after);
+    const changes = file.hunks.flatMap((hunk) => hunk.changes);
+    if (
+      !changes.length &&
+      (file.before !== file.after || /^[RC]/.test(file.status))
+    )
+      changes.push({
+        beforeStart: 1,
+        beforeCount: oldLines.length,
+        afterStart: 1,
+        afterCount: newLines.length,
+      });
+    if (!changes.length) continue;
+    const add = (
+      kind: EvidenceUnit["kind"],
+      name: string,
+      exported: boolean,
+      oldRange: EvidenceUnit["beforeRange"],
+      newRange: EvidenceUnit["afterRange"],
+    ) => {
+      const excerpt = (
+        source: string[],
+        range: EvidenceUnit["beforeRange"],
+      ): string | null => {
+        if (!range) return null;
+        return source.slice(range.start - 1, range.end).join("\n");
+      };
+      let before = excerpt(oldLines, oldRange);
+      let after = excerpt(newLines, newRange);
+      const id = `u${String(units.length + 1).padStart(3, "0")}`;
+      const total = (before?.length ?? 0) + (after?.length ?? 0);
+      if (total > UNIT_MAX_CHARS) {
+        const beforeBudget =
+          before === null
+            ? 0
+            : after === null
+              ? UNIT_MAX_CHARS
+              : Math.floor(UNIT_MAX_CHARS / 2);
+        if (before !== null) before = truncate(before, beforeBudget);
+        if (after !== null)
+          after = truncate(after, UNIT_MAX_CHARS - (before?.length ?? 0));
+        limits.push({ kind: "truncated", file: file.path, unit: id });
+      }
+      units.push({
+        id,
+        kind,
+        file: file.path,
+        name,
+        exported,
+        before,
+        after,
+        beforeRange: oldRange,
+        afterRange: newRange,
+      });
+    };
+    if (
+      !code.test(file.path) &&
+      (file.before?.length ?? 0) + (file.after?.length ?? 0) <= UNIT_MAX_CHARS
+    ) {
+      add(
+        "file",
+        file.path,
+        false,
+        file.before === null ? null : { start: 1, end: oldLines.length },
+        file.after === null ? null : { start: 1, end: newLines.length },
+      );
+      continue;
+    }
+    const declarations = (text: string | null, path: string) => {
+      if (text === null || !code.test(path)) return [];
+      if (parser?.supports(path)) {
+        const result = parser.declarations(path, text);
+        for (const limit of result.limits) {
+          if (
+            !limits.some(
+              (existing) =>
+                existing.kind === limit.kind &&
+                existing.file === path &&
+                (limit.kind !== "parse_partial" ||
+                  (existing.kind === "parse_partial" &&
+                    existing.declaration === limit.declaration)),
+            )
+          )
+            limits.push({ ...limit, file: path });
+        }
+        if (!result.limits.some((limit) => limit.kind === "parse_failed"))
+          return result.declarations;
+      } else if (
+        !limits.some(
+          (limit) =>
+            limit.kind === "grammar_absent" && limit.file === file.path,
+        )
+      )
+        limits.push({
+          kind: "grammar_absent",
+          file: file.path,
+          language: path.endsWith(".py") ? "Python" : "JavaScript/TypeScript",
+        });
+      return heuristicDeclarations(path, text);
+    };
+    const beforeDeclarations = declarations(file.before, file.oldPath);
+    const afterDeclarations = declarations(file.after, file.path);
+    const key = (d: Declaration) => `${d.type ?? d.kind}:${d.name}`;
+    const pairedAfter = new Set<Declaration>();
+    const selected: { before?: Declaration; after?: Declaration }[] = [];
+    for (const before of beforeDeclarations) {
+      const after = afterDeclarations.find(
+        (d) => !pairedAfter.has(d) && key(d) === key(before),
+      );
+      if (after) pairedAfter.add(after);
+      selected.push({ before, after });
+    }
+    for (const pair of selected) {
+      const before = pair.before;
+      if (
+        pair.after ||
+        before?.kind !== "function" ||
+        before.body === undefined
+      )
+        continue;
+      const after = afterDeclarations.find(
+        (d) =>
+          !pairedAfter.has(d) &&
+          d.kind === "function" &&
+          d.type === before.type &&
+          d.body === before.body,
+      );
+      if (after) {
+        pair.after = after;
+        pairedAfter.add(after);
+      }
+    }
+    for (const after of afterDeclarations)
+      if (!pairedAfter.has(after)) selected.push({ after });
+    const touched = (d: Declaration | undefined, side: "before" | "after") =>
+      d !== undefined &&
+      changes.some((c) =>
+        overlaps(d.start, d.end, c[`${side}Start`], c[`${side}Count`]),
+      );
+    const changed = selected.filter(
+      (pair) => touched(pair.before, "before") || touched(pair.after, "after"),
+    );
+    if (
+      changed.some((p) => p.before?.kind === "function" && !p.after) &&
+      changed.some((p) => p.after?.kind === "function" && !p.before)
+    )
+      limits.push({ kind: "rename_unpaired", file: file.path });
+    for (const pair of changed) {
+      const declaration = pair.after ?? pair.before;
+      if (!declaration) continue;
+      const name = declaration.name;
+      const size =
+        (pair.before
+          ? oldLines.slice(pair.before.start - 1, pair.before.end).join("\n")
+              .length
+          : 0) +
+        (pair.after
+          ? newLines.slice(pair.after.start - 1, pair.after.end).join("\n")
+              .length
+          : 0);
+      const large = size > UNIT_MAX_CHARS;
+      const relevant = changes.filter(
+        (c) =>
+          (pair.before &&
+            overlaps(
+              pair.before.start,
+              pair.before.end,
+              c.beforeStart,
+              c.beforeCount,
+            )) ||
+          (pair.after &&
+            overlaps(
+              pair.after.start,
+              pair.after.end,
+              c.afterStart,
+              c.afterCount,
+            )),
+      );
+      const zones: (typeof changes)[] = [];
+      for (const change of relevant) {
+        const previous = zones[zones.length - 1];
+        const last = previous?.[previous.length - 1];
+        if (
+          last &&
+          change.beforeStart <=
+            last.beforeStart + last.beforeCount + 2 * UNIT_CONTEXT_LINES &&
+          change.afterStart <=
+            last.afterStart + last.afterCount + 2 * UNIT_CONTEXT_LINES
+        )
+          previous?.push(change);
+        else zones.push([change]);
+      }
+      for (const zone of large ? zones : [relevant]) {
+        const range = (
+          d: Declaration | undefined,
+          side: "before" | "after",
+        ) => {
+          if (!d) return null;
+          if (!large) return { start: d.start, end: d.end };
+          return {
+            start: Math.max(
+              d.start,
+              Math.min(...zone.map((c) => c[`${side}Start`])) -
+                UNIT_CONTEXT_LINES,
+            ),
+            end: Math.min(
+              d.end,
+              Math.max(
+                ...zone.map(
+                  (c) => c[`${side}Start`] + Math.max(1, c[`${side}Count`]) - 1,
+                ),
+              ) + UNIT_CONTEXT_LINES,
+            ),
+          };
+        };
+        add(
+          large ? "slice" : declaration.kind,
+          name,
+          declaration.exported,
+          range(pair.before, "before"),
+          range(pair.after, "after"),
+        );
+        if (!large) continue;
+        const unit = units[units.length - 1];
+        if (!unit) continue;
+        const omission = file.path.endsWith(".py")
+          ? "# … omitted …"
+          : "// … omitted …";
+        for (const side of ["before", "after"] as const) {
+          const d = pair[side];
+          const r = unit[`${side}Range`];
+          const source = side === "before" ? oldLines : newLines;
+          if (!d || !r || unit[side] === null) continue;
+          const signatureEnd = d.signatureEnd ?? d.start;
+          if (r.start > d.start) {
+            const signature = source
+              .slice(d.start - 1, Math.min(signatureEnd, r.start - 1))
+              .join("\n");
+            unit[side] =
+              `${signature}\n${r.start > signatureEnd + 1 ? `${omission}\n` : ""}${unit[side]}`;
+          }
+          if (r.end < d.end) unit[side] += `\n${omission}`;
+        }
+        if (
+          (unit.before?.length ?? 0) + (unit.after?.length ?? 0) >
+          UNIT_MAX_CHARS
+        ) {
+          const budget =
+            unit.after === null
+              ? UNIT_MAX_CHARS
+              : unit.before === null
+                ? 0
+                : Math.floor(UNIT_MAX_CHARS / 2);
+          if (unit.before !== null) unit.before = truncate(unit.before, budget);
+          if (unit.after !== null)
+            unit.after = truncate(
+              unit.after,
+              UNIT_MAX_CHARS - (unit.before?.length ?? 0),
+            );
+          if (!limits.some((l) => l.kind === "truncated" && l.unit === unit.id))
+            limits.push({ kind: "truncated", file: file.path, unit: unit.id });
+        }
+      }
+    }
+    for (const change of changes) {
+      const contains = (
+        declarations: Declaration[],
+        source: string[],
+        start: number,
+        count: number,
+      ) =>
+        count === 0 ||
+        source
+          .slice(Math.max(0, start - 1), start + count - 1)
+          .every(
+            (line, offset) =>
+              !line.trim() ||
+              declarations.some(
+                (d) => d.start <= start + offset && d.end >= start + offset,
+              ),
+          );
+      const covered =
+        (file.before === null ||
+          contains(
+            beforeDeclarations,
+            oldLines,
+            change.beforeStart,
+            change.beforeCount,
+          )) &&
+        (file.after === null ||
+          contains(
+            afterDeclarations,
+            newLines,
+            change.afterStart,
+            change.afterCount,
+          ));
+      if (covered) continue;
+      const range = (start: number, count: number, length: number) => ({
+        start: Math.max(1, start - UNIT_CONTEXT_LINES),
+        end: Math.min(
+          length,
+          start + Math.max(1, count) - 1 + UNIT_CONTEXT_LINES,
+        ),
+      });
+      add(
+        "hunk",
+        `${file.path}:${change.afterStart}`,
+        false,
+        file.before === null
+          ? null
+          : range(change.beforeStart, change.beforeCount, oldLines.length),
+        file.after === null
+          ? null
+          : range(change.afterStart, change.afterCount, newLines.length),
+      );
+    }
+  }
+  return { units, limits };
+}

package/src/describe.ts

@@ -0,0 +1,26 @@
+import * as constants from "./constants.ts";
+import type { Names } from "./host.ts";
+import { ASK_DESCRIPTION } from "./texts/ask.ts";
+export function interpolate(template: string, names: Names): string {
+  const values: Record<string, unknown> = {
+    ...constants,
+    BAND_BOOL_GRAY_A: constants.BAND_BOOL_GRAY_A.toFixed(2),
+    BAND_BOOL_YES_MIN: constants.BAND_BOOL_YES_MIN.toFixed(2),
+    BAND_CHOICE_VERDICT_MIN: constants.BAND_CHOICE_VERDICT_MIN.toFixed(2),
+    ...Object.fromEntries(
+      Object.entries(names).map(([key, value]) => [`names.${key}`, value]),
+    ),
+  };
+  const rendered = template.replace(/{{([^}]+)}}/g, (_match, key: string) => {
+    const value = values[key];
+    if (typeof value !== "string" && typeof value !== "number")
+      throw new Error(`Unknown description interpolation: ${key}`);
+    return String(value);
+  });
+  if (rendered.includes("{{"))
+    throw new Error("Unresolved description interpolation.");
+  return rendered;
+}
+export function describeAsk(names: Names): string {
+  return interpolate(ASK_DESCRIPTION, names);
+}