jev-agent-tools 0.1.3

package/src/core/integrity.ts

@@ -0,0 +1,64 @@
+import type { State } from "../jev/types.ts";
+import type { Ask } from "./asks.ts";
+import type { Limitation } from "./output.ts";
+export interface FileIdentity {
+  requestedPath: string;
+  resolvedPath: string;
+  insertedPath: string;
+  readSha256: string;
+  insertedSha256: string;
+  content: string;
+}
+export function checkIntegrity(
+  state: State,
+  asks: readonly Ask[],
+  files: readonly FileIdentity[] = [],
+  output?: { text: string; truncated: boolean; omittedChars: number },
+): { controls: Limitation[]; notes: Limitation[] } {
+  const controls: Limitation[] = [];
+  const notes: Limitation[] = [];
+  for (const file of files) {
+    if (file.resolvedPath !== file.insertedPath || !file.content.length)
+      controls.push({
+        fact: `integrity: wrong identity or empty file ${file.requestedPath}`,
+        next: `add the complete file to paths: ${file.requestedPath}`,
+      });
+    if (file.readSha256 !== file.insertedSha256)
+      controls.push({
+        fact: `integrity: fingerprint mismatch ${file.requestedPath}`,
+        next: `read the complete file again: ${file.requestedPath}`,
+      });
+  }
+  const out =
+    output?.text ??
+    (typeof state.output === "string" ? state.output : undefined);
+  if (out !== undefined)
+    for (const ask of asks) {
+      if (ask.about !== undefined && ask.about !== "output") continue;
+      const statements =
+        ask.intent === "verify"
+          ? Object.values(ask.claims)
+          : ask.intent === "decide"
+            ? Object.values(ask.hypotheses)
+            : ask.intent === "classify"
+              ? Object.values(ask.categories)
+              : [];
+      for (const statement of statements)
+        for (const match of statement.matchAll(
+          /`([^`]{4,})`|"([^"\n]{4,})"/g,
+        )) {
+          const literal = match[1] ?? match[2] ?? "";
+          if (!out.includes(literal))
+            controls.push({
+              fact: `integrity: cited line not in state: ${literal}`,
+              next: "add the cited output through command or state",
+            });
+        }
+    }
+  if (output?.truncated)
+    notes.push({
+      fact: `output truncated, ${output.omittedChars} chars omitted`,
+      next: "narrow command to retain decisive lines",
+    });
+  return { controls, notes };
+}

package/src/core/lexical.ts

@@ -0,0 +1,130 @@
+export interface LexicalToken {
+  value: string;
+  line: number;
+  offset: number;
+  string: boolean;
+}
+export interface LexicalSource {
+  tokenize(path: string, text: string, python?: boolean): LexicalToken[];
+  resolved?(path: string, text: string): void;
+}
+/** A lexical pass excludes comments and keeps literals separate from identifiers. */
+export function tokenize(text: string, python = false): LexicalToken[] {
+  const tokens: LexicalToken[] = [];
+  let i = 0;
+  let line = 1;
+  while (i < text.length) {
+    const c = text[i] ?? "";
+    if (/\s/.test(c)) {
+      if (c === "\n") line++;
+      i++;
+      continue;
+    }
+    if (python ? c === "#" : text.slice(i, i + 2) === "//") {
+      while (i < text.length && text[i] !== "\n") i++;
+      continue;
+    }
+    if (!python && text.slice(i, i + 2) === "/*") {
+      i += 2;
+      while (i < text.length && text.slice(i, i + 2) !== "*/") {
+        if (text[i] === "\n") line++;
+        i++;
+      }
+      i += 2;
+      continue;
+    }
+    const start = i;
+    const firstLine = line;
+    const previous = tokens[tokens.length - 1];
+    // A slash can begin a regex only where an expression can begin, not after
+    // an operand (where it is division). Keep the entire literal opaque.
+    if (
+      !python &&
+      c === "/" &&
+      (!previous ||
+        (!previous.string &&
+          (/^[([{,:;=!?&|+*%~^<>-]$/.test(previous.value) ||
+            [
+              "return",
+              "throw",
+              "case",
+              "delete",
+              "void",
+              "typeof",
+              "yield",
+              "await",
+              "in",
+              "of",
+            ].includes(previous.value))))
+    ) {
+      let end = i + 1;
+      let characterClass = false;
+      while (end < text.length && text[end] !== "\n" && text[end] !== "\r") {
+        const next = text[end];
+        if (next === "\\") {
+          end += 2;
+          continue;
+        }
+        if (next === "[") characterClass = true;
+        else if (next === "]") characterClass = false;
+        else if (next === "/" && !characterClass) break;
+        end++;
+      }
+      if (text[end] === "/") {
+        end++;
+        while (/[a-z]/i.test(text[end] ?? "")) end++;
+        tokens.push({
+          value: text.slice(i, end),
+          line,
+          offset: start,
+          string: false,
+        });
+        i = end;
+        continue;
+      }
+    }
+    if (c === "'" || c === '"' || c === "`") {
+      const quote =
+        python && text.slice(i, i + 3) === c.repeat(3) ? c.repeat(3) : c;
+      i += quote.length;
+      let value = "";
+      let dynamic = false;
+      while (i < text.length && text.slice(i, i + quote.length) !== quote) {
+        if (text[i] === "\n") line++;
+        if (c === "`" && text.slice(i, i + 2) === "${") dynamic = true;
+        if (text[i] === "\\") {
+          i++;
+          const escaped = text[i++] ?? "";
+          value +=
+            escaped === "n"
+              ? "\n"
+              : escaped === "t"
+                ? "\t"
+                : escaped === "r"
+                  ? "\r"
+                  : escaped;
+        } else value += text[i++];
+      }
+      const closed = text.slice(i, i + quote.length) === quote;
+      if (closed) i += quote.length;
+      tokens.push({
+        value,
+        line: firstLine,
+        offset: start,
+        string: closed && !dynamic,
+      });
+    } else if (/[\w$]/.test(c)) {
+      while (i < text.length && /[\w$]/.test(text[i] ?? "")) i++;
+      tokens.push({
+        value: text.slice(start, i),
+        line,
+        offset: start,
+        string: false,
+      });
+    } else {
+      i++;
+      tokens.push({ value: c, line, offset: start, string: false });
+    }
+  }
+  return tokens;
+}

package/src/core/locate.ts

@@ -0,0 +1,213 @@
+import {
+  CHOICE_MAX_OPTIONS,
+  LOCATE_VERDICT_MIN,
+  STATE_MAX_CHARS,
+} from "../constants.ts";
+import type { State } from "../jev/types.ts";
+import type { Candidate } from "./output.ts";
+import type { Section } from "./sections.ts";
+import { truncate } from "./truncate.ts";
+
+export function sectionState(
+  path: string,
+  goal: string,
+  sections: readonly Section[],
+  outline = false,
+): State {
+  return {
+    path,
+    goal,
+    sections: sections.map((section) => ({
+      id: section.id,
+      lines: `${section.start}-${section.end}`,
+      text: outline ? section.label : section.text,
+    })),
+  };
+}
+export function sectionStateFits(
+  path: string,
+  goal: string,
+  sections: readonly Section[],
+  outline = false,
+): boolean {
+  return (
+    sections.length < CHOICE_MAX_OPTIONS &&
+    JSON.stringify(sectionState(path, goal, sections, outline)).length <=
+      STATE_MAX_CHARS
+  );
+}
+/** Every planned block has at least two sections and fits its full-text refinement budget. */
+export function sectionOutline(
+  path: string,
+  goal: string,
+  sections: readonly Section[],
+): Section[] {
+  const base = JSON.stringify(sectionState(path, goal, [])).length;
+  const costs = new Map(
+    sections.map((section) => [
+      section,
+      JSON.stringify({
+        id: section.id,
+        lines: `${section.start}-${section.end}`,
+        text: "",
+      }).length +
+        (section.serializedTextChars ?? JSON.stringify(section.text).length) -
+        2,
+    ]),
+  );
+  const fits = (items: readonly Section[]) =>
+    items.length < CHOICE_MAX_OPTIONS &&
+    base + items.reduce((sum, item) => sum + (costs.get(item) ?? 0) + 1, 0) <=
+      STATE_MAX_CHARS;
+  const groups: Section[][] = [];
+  let current: Section[] = [],
+    currentSize = base;
+  for (const section of sections) {
+    const length = (costs.get(section) ?? 0) + 1;
+    if (base + length > STATE_MAX_CHARS) return [];
+    const owner = section.label.split(".")[0],
+      previousOwner = current.at(-1)?.label.split(".")[0];
+    if (
+      current.length &&
+      (currentSize + length > STATE_MAX_CHARS ||
+        current.length + 1 >= CHOICE_MAX_OPTIONS ||
+        (current.length >= 2 &&
+          section.label.includes(".") &&
+          owner !== previousOwner))
+    ) {
+      groups.push(current);
+      current = [];
+      currentSize = base;
+    }
+    current.push(section);
+    currentSize += length;
+  }
+  if (current.length) groups.push(current);
+  for (let index = 0; index < groups.length; index++) {
+    const group = groups[index];
+    if (!group || group.length >= 2) continue;
+    const previous = groups[index - 1],
+      next = groups[index + 1];
+    if (previous && fits([...previous, ...group])) {
+      previous.push(...group);
+      groups.splice(index, 1);
+      index--;
+      continue;
+    }
+    if (next && fits([...group, ...next])) {
+      next.unshift(...group);
+      groups.splice(index, 1);
+      index--;
+      continue;
+    }
+    if (previous && previous.length > 2) {
+      const moved = previous.at(-1);
+      if (moved && fits([moved, ...group])) {
+        previous.pop();
+        group.unshift(moved);
+        continue;
+      }
+    }
+    if (next && next.length > 2) {
+      const moved = next[0];
+      if (moved && fits([...group, moved])) {
+        next.shift();
+        group.push(moved);
+        continue;
+      }
+    }
+    return [];
+  }
+  return groups.map((group, index) => ({
+    id: `B${index + 1}`,
+    start: group[0]?.start ?? 0,
+    end: group.at(-1)?.end ?? 0,
+    label: group.map((s) => s.label).join(" / "),
+    text: group.map((s) => s.text).join("\n"),
+  }));
+}
+/** JSON-escaped budget, allocated equally then redistributed to larger blocks. */
+export function outlineEvidence(
+  path: string,
+  goal: string,
+  blocks: readonly Section[],
+): Section[] {
+  const overhead = JSON.stringify(
+    sectionState(
+      path,
+      goal,
+      blocks.map((block) => ({ ...block, text: "" })),
+    ),
+  ).length;
+  let remaining = Math.max(0, STATE_MAX_CHARS - overhead);
+  const allocations = blocks.map(() => 0);
+  let active = blocks.map((_, index) => index);
+  while (remaining > 0 && active.length) {
+    const share = Math.max(1, Math.floor(remaining / active.length));
+    const next: number[] = [];
+    for (const index of active) {
+      const block = blocks[index];
+      if (!block) continue;
+      const need =
+        JSON.stringify(block.text).length - 2 - (allocations[index] ?? 0);
+      const granted = Math.min(need, share, remaining);
+      allocations[index] = (allocations[index] ?? 0) + granted;
+      remaining -= granted;
+      if (need > granted) next.push(index);
+    }
+    active = next;
+  }
+  return blocks.map((block, index) => {
+    const budget = allocations[index] ?? 0;
+    if (JSON.stringify(block.text).length - 2 <= budget) return { ...block };
+    const excerpt = (chars: number) => {
+      const part = Math.floor(chars / 3);
+      const slice = (start: number) => {
+        if (start > 0 && /[\uDC00-\uDFFF]/.test(block.text[start] ?? ""))
+          start++;
+        return truncate(block.text.slice(start), part);
+      };
+      return `${slice(0)}\n…\n${slice(Math.max(0, Math.floor(block.text.length / 2) - Math.floor(part / 2)))}\n…\n${slice(Math.max(0, block.text.length - part))}`;
+    };
+    let low = 0,
+      high = Math.min(block.text.length, budget),
+      text = "";
+    while (low <= high) {
+      const middle = Math.floor((low + high) / 2),
+        candidate = excerpt(middle);
+      if (JSON.stringify(candidate).length - 2 <= budget) {
+        text = candidate;
+        low = middle + 1;
+      } else high = middle - 1;
+    }
+    return { ...block, text };
+  });
+}
+export function locateDisplay(
+  path: string,
+  sections: readonly Section[],
+  ranked: readonly { id: string; p: number }[],
+  forceUnsure: boolean,
+) {
+  const head = ranked[0];
+  if (!head) return undefined;
+  const display = (id: string, p: number): Candidate => {
+    const section = sections.find((s) => s.id === id);
+    return {
+      label: section ? `${path}:${section.start}-${section.end}` : "none",
+      value: { head: section?.label ?? "no section fits", p },
+    };
+  };
+  const band =
+    forceUnsure || head.p < LOCATE_VERDICT_MIN
+      ? ("unsure" as const)
+      : ("verdict" as const);
+  return {
+    primary: display(head.id, head.p),
+    band,
+    candidates:
+      band === "unsure"
+        ? ranked.slice(1, 2).map((s) => display(s.id, s.p))
+        : undefined,
+  };
+}

package/src/core/output.ts

@@ -0,0 +1,264 @@
+import {
+  BAND_BOOL_GRAY_A,
+  BAND_BOOL_YES_MIN,
+  BAND_CHOICE_VERDICT_MIN,
+  CANNOT_TELL_MIN,
+  LIMIT_DISPLAY_MAX_PATHS,
+} from "../constants.ts";
+import type { Answer } from "../jev/types.ts";
+
+export interface Limitation {
+  fact: string;
+  next: string;
+  cause?: string;
+  path?: string;
+  dependency?: string;
+  count?: number;
+}
+export interface Yield {
+  calls: number;
+  questions: number;
+  costUsd?: number;
+  cacheHits: number;
+  cacheRequests: number;
+  elapsedMs: number;
+}
+export type Band = "verdict" | "unsure" | "abstain";
+export interface DisplayValue {
+  head: string | number;
+  p: number;
+}
+export interface Candidate {
+  label: string;
+  value: DisplayValue;
+}
+interface CandidateFields {
+  orderDependent?: boolean;
+  candidates?: readonly Candidate[];
+}
+export type BandPolicy = (value: DisplayValue, input: AnswerInput) => Band;
+export type BudgetRefusal = { kind: "max_calls" | "session"; message: string };
+export interface AnswerInput extends CandidateFields {
+  label: string;
+  answer?: Exclude<Answer, { type: "unjudged" }>;
+  value?: DisplayValue;
+  band?: Band;
+  uncalibrated?: boolean;
+  missing?: string;
+  reason?: string;
+  verify?: boolean;
+}
+export type OutputLine =
+  | (CandidateFields & {
+      type: "answer";
+      label: string;
+      value: DisplayValue;
+      band: Band;
+      uncalibrated: boolean;
+      reason?: string;
+    })
+  | { type: "unjudged"; label: string; reason: string; next: string }
+  | { type: "fact"; fact: string; next: string }
+  | {
+      type: "limit-group";
+      cause: string;
+      count: number;
+      priority: boolean;
+      examples: readonly string[];
+      remaining: number;
+      next: string;
+    }
+  | {
+      type: "collection";
+      title: string;
+      count: number;
+      items: readonly string[];
+      next: string;
+    }
+  | { type: "unchecked"; items: readonly string[] }
+  | { type: "refusal"; message: string }
+  | { type: "list"; title: string; items: readonly string[] }
+  | (CandidateFields & {
+      type: "entry";
+      candidate: Candidate;
+      band: Band;
+      relevant?: readonly Candidate[];
+      nextRead?: readonly string[];
+      orderProbabilities?: readonly [number, number];
+    })
+  | {
+      type: "state";
+      files: number;
+      integrity: "ok" | "failed";
+      warnings: readonly string[];
+    }
+  | {
+      type: "command";
+      cwd: string;
+      framework: string;
+      config?: string;
+      project?: string;
+      executable: string;
+      args: readonly string[];
+    }
+  | { type: "yield"; value: Yield };
+export interface Envelope {
+  lines: readonly OutputLine[];
+}
+export interface EnvelopeInput {
+  state?: Omit<Extract<OutputLine, { type: "state" }>, "type">;
+  answers?: readonly AnswerInput[];
+  unjudged?: readonly { label: string; reason: string; next: string }[];
+  lines?: readonly Extract<
+    OutputLine,
+    { type: "list" | "entry" | "command" | "collection" }
+  >[];
+  policy?: BandPolicy;
+  controls?: readonly Limitation[];
+  limitations?: readonly Limitation[];
+  limitationPriorityPaths?: readonly string[];
+  unchecked?: readonly string[];
+  refusal?: string;
+  budget?: BudgetRefusal;
+  yield: Yield;
+}
+export function askBand(value: DisplayValue, input: AnswerInput): Band {
+  const answer = input.answer;
+  if (answer?.type === "bool")
+    return value.p <= BAND_BOOL_GRAY_A || value.p >= BAND_BOOL_YES_MIN
+      ? "verdict"
+      : "unsure";
+  if (answer && (answer.probabilities.cannot_tell ?? 0) >= CANNOT_TELL_MIN)
+    return "abstain";
+  return value.p >= BAND_CHOICE_VERDICT_MIN ? "verdict" : "unsure";
+}
+function displayValue(input: AnswerInput): DisplayValue {
+  if (input.value) return input.value;
+  const answer = input.answer;
+  if (!answer)
+    throw new Error("An answer requires a typed value or a Jev answer");
+  if (answer.type === "bool")
+    return { head: answer.p >= 0.5 ? "yes" : "no (not shown)", p: answer.p };
+  const p = input.verify
+    ? Math.max(
+        answer.probabilities.holds ?? 0,
+        answer.probabilities.contradicted ?? 0,
+        (answer.probabilities.not_addressed ?? 0) +
+          (answer.probabilities.cannot_tell ?? 0),
+      )
+    : Math.max(...Object.values(answer.probabilities));
+  return { head: answer.type === "choice" ? answer.choice : answer.score, p };
+}
+export function buildEnvelope(input: EnvelopeInput): Envelope {
+  const lines: OutputLine[] = [];
+  if (input.budget?.kind === "session" && input.yield.calls === 0)
+    return {
+      lines: [
+        { type: "refusal", message: input.budget.message },
+        { type: "yield", value: input.yield },
+      ],
+    };
+  if (input.state) lines.push({ type: "state", ...input.state });
+  const failed = (input.controls ?? [])
+    .map((control) => control.fact)
+    .join("; ");
+  for (const item of input.answers ?? []) {
+    const value = displayValue(item);
+    const initial = item.band ?? (input.policy ?? askBand)(value, item);
+    const band = initial === "verdict" && failed ? "unsure" : initial;
+    const reason =
+      band === "abstain"
+        ? (item.reason ??
+          item.missing ??
+          "proof missing; add decisive evidence to paths or command")
+        : failed && initial === "verdict"
+          ? failed
+          : item.reason;
+    lines.push({
+      type: "answer",
+      label: item.label,
+      value,
+      band,
+      uncalibrated: item.uncalibrated ?? false,
+      ...(reason ? { reason } : {}),
+      ...(item.orderDependent !== undefined
+        ? { orderDependent: item.orderDependent }
+        : {}),
+      ...(item.candidates ? { candidates: item.candidates } : {}),
+    });
+  }
+  for (const item of input.unjudged ?? [])
+    lines.push({ type: "unjudged", ...item });
+  for (const line of input.lines ?? [])
+    lines.push(
+      line.type === "entry" && line.band === "verdict" && failed
+        ? { ...line, band: "unsure" }
+        : line,
+    );
+  if (input.refusal) lines.push({ type: "refusal", message: input.refusal });
+  if (input.budget?.kind === "session")
+    lines.push({ type: "refusal", message: input.budget.message });
+  for (const fact of input.controls ?? [])
+    lines.push({ type: "fact", ...fact });
+  const groups = new Map<
+    string,
+    Omit<Extract<OutputLine, { type: "limit-group" }>, "examples"> & {
+      examples: string[];
+    }
+  >();
+  for (const limit of input.limitations ?? []) {
+    if (
+      !limit.cause ||
+      !limit.path ||
+      input.limitationPriorityPaths === undefined
+    ) {
+      lines.push({ type: "fact", fact: limit.fact, next: limit.next });
+      continue;
+    }
+    const priority =
+      input.limitationPriorityPaths?.includes(limit.path) ?? false;
+    const key = JSON.stringify([priority, limit.cause, limit.next]);
+    const group = groups.get(key) ?? {
+      type: "limit-group",
+      cause: limit.cause,
+      count: 0,
+      priority,
+      examples: [],
+      remaining: 0,
+      next: limit.next,
+    };
+    group.count += limit.count ?? 1;
+    if (priority || group.examples.length < LIMIT_DISPLAY_MAX_PATHS)
+      group.examples.push(
+        `${limit.path}${limit.dependency ? ` (${limit.dependency})` : ""}`,
+      );
+    else group.remaining++;
+    groups.set(key, group);
+  }
+  lines.push(
+    ...[...groups.values()].sort(
+      (a, b) => Number(b.priority) - Number(a.priority),
+    ),
+  );
+  if (input.budget?.kind === "max_calls")
+    lines.push({
+      type: "fact",
+      fact: input.budget.message,
+      next: "raise max_calls",
+    });
+  if (input.unchecked?.length)
+    lines.push({ type: "unchecked", items: input.unchecked });
+  const gray = lines.reduce(
+    (count, line) =>
+      count + Number(line.type === "answer" && line.band === "unsure"),
+    0,
+  );
+  if (gray > 0)
+    lines.push({
+      type: "fact",
+      fact: `${gray}/${input.answers?.length ?? 0} answers unsure`,
+      next: "read the decisive passage or add its file to paths",
+    });
+  lines.push({ type: "yield", value: input.yield });
+  return { lines };
+}