@agent-inspect/eval 2.0.0

package/dist/index.cjs

@@ -0,0 +1,733 @@
+'use strict';
+
+var readers = require('agent-inspect/readers');
+
+// packages/eval/src/index.ts
+function isTraceReadResult(value) {
+  return value !== null && typeof value === "object" && "runs" in value && "events" in value && "format" in value;
+}
+function traceInputFrom(value) {
+  return { type: "file", path: value instanceof URL ? value.pathname : value };
+}
+async function resolveRead(input, options) {
+  try {
+    if (typeof input === "string") {
+      const read2 = await readers.openTrace(traceInputFrom(input), {
+        ...options.format !== void 0 && options.format !== "auto" ? { format: options.format } : {}
+      });
+      return { read: read2, runId: options.runId, diagnostics: [] };
+    }
+    if (isTraceReadResult(input)) {
+      return { read: input, runId: options.runId, diagnostics: [] };
+    }
+    if (isTraceReadResult(input.trace)) {
+      return { read: input.trace, runId: options.runId ?? input.runId, diagnostics: [] };
+    }
+    const format = options.format ?? input.format;
+    const read = await readers.openTrace(traceInputFrom(input.trace), {
+      ...format !== void 0 && format !== "auto" ? { format } : {}
+    });
+    return { read, runId: options.runId ?? input.runId, diagnostics: [] };
+  } catch (error) {
+    return { diagnostics: [diagnosticFromError(error)] };
+  }
+}
+function diagnosticFromError(error) {
+  if (error instanceof readers.TraceReadError) {
+    const code = error.code === "unsupported_format" ? "AI_EVAL_UNSUPPORTED_FORMAT" : error.code === "ambiguous_format" ? "AI_EVAL_AMBIGUOUS_FORMAT" : "AI_EVAL_TRACE_UNREADABLE";
+    return { code, severity: "error", message: error.message };
+  }
+  return {
+    code: "AI_EVAL_TRACE_UNREADABLE",
+    severity: "error",
+    message: error instanceof Error ? error.message : String(error)
+  };
+}
+function flatten(nodes) {
+  return nodes.flatMap((node) => [node, ...flatten(node.children)]);
+}
+function selectRun(read, runId) {
+  if (runId !== void 0) {
+    const run = read.runs.find((candidate) => candidate.runId === runId);
+    return run === void 0 ? {
+      diagnostics: [
+        {
+          code: "AI_EVAL_RUN_SELECTION_REQUIRED",
+          severity: "error",
+          message: `Run not found: ${runId}.`
+        }
+      ]
+    } : { run, diagnostics: [] };
+  }
+  if (read.runs.length === 1) {
+    return { run: read.runs[0], diagnostics: [] };
+  }
+  if (read.runs.length === 0) {
+    return {
+      diagnostics: [
+        {
+          code: "AI_EVAL_RUN_SELECTION_REQUIRED",
+          severity: "error",
+          message: "No runs are available for eval."
+        }
+      ]
+    };
+  }
+  return {
+    diagnostics: [
+      {
+        code: "AI_EVAL_RUN_SELECTION_REQUIRED",
+        severity: "error",
+        message: "Multiple runs are available; select a run before eval."
+      }
+    ]
+  };
+}
+function errorResult(format, diagnostics, runId) {
+  const errors = diagnostics.filter((item) => item.severity === "error").length;
+  return {
+    ok: false,
+    status: "error",
+    format,
+    ...runId !== void 0 ? { runId } : {},
+    summary: { passed: 0, failed: 0, warnings: 0, errors },
+    findings: [],
+    diagnostics: [...diagnostics]
+  };
+}
+function normalizeFinding(rule, finding) {
+  return {
+    ...finding,
+    ruleId: finding.ruleId || rule.id,
+    severity: finding.severity ?? rule.severity ?? "error",
+    evidence: [...finding.evidence]
+  };
+}
+function compareFindings(a, b) {
+  const aEvidence = a.evidence[0];
+  const bEvidence = b.evidence[0];
+  return a.ruleId.localeCompare(b.ruleId) || (aEvidence?.runId ?? "").localeCompare(bEvidence?.runId ?? "") || (aEvidence?.eventId ?? "").localeCompare(bEvidence?.eventId ?? "") || (aEvidence?.path ?? "").localeCompare(bEvidence?.path ?? "") || a.message.localeCompare(b.message);
+}
+function summarize(findings, ruleCount) {
+  const failed = findings.filter((item) => item.status === "fail").length;
+  const warnings = findings.filter((item) => item.status === "warning").length;
+  const errors = findings.filter((item) => item.severity === "error").length;
+  return {
+    passed: Math.max(0, ruleCount - failed - warnings),
+    failed,
+    warnings,
+    errors
+  };
+}
+function defaultRules() {
+  return [checks.requireSuccess()];
+}
+async function evalRun(input, options = {}) {
+  const resolved = await resolveRead(input, options);
+  if (resolved.read === void 0) {
+    return errorResult("unknown", resolved.diagnostics);
+  }
+  const selected = selectRun(resolved.read, resolved.runId);
+  if (selected.run === void 0) {
+    return errorResult(resolved.read.format, selected.diagnostics, resolved.runId);
+  }
+  const rules = [...options.checks ?? defaultRules()].sort(
+    (a, b) => a.id.localeCompare(b.id)
+  );
+  const context = {
+    format: resolved.read.format,
+    run: selected.run,
+    nodes: flatten(selected.run.children),
+    events: resolved.read.events.filter((event) => event.runId === selected.run?.runId)
+  };
+  const diagnostics = [];
+  const findings = [];
+  for (const rule of rules) {
+    try {
+      findings.push(...rule.evaluate(context).map((finding) => normalizeFinding(rule, finding)));
+    } catch (error) {
+      diagnostics.push({
+        code: "AI_EVAL_RULE_FAILED",
+        severity: "error",
+        ruleId: rule.id,
+        message: `Eval rule ${rule.id} failed: ${error instanceof Error ? error.message : String(error)}`
+      });
+    }
+  }
+  if (diagnostics.length > 0) {
+    return errorResult(resolved.read.format, diagnostics, selected.run.runId);
+  }
+  const sortedFindings = findings.sort(compareFindings);
+  const summary = summarize(sortedFindings, rules.length);
+  const status = summary.failed > 0 ? "fail" : "pass";
+  return {
+    ok: status === "pass",
+    status,
+    format: resolved.read.format,
+    runId: selected.run.runId,
+    summary,
+    findings: sortedFindings,
+    diagnostics: []
+  };
+}
+function evidenceForRun(run, path) {
+  return [{ runId: run.runId, ...path !== void 0 ? { path } : {} }];
+}
+function evidenceForEvent(event, path) {
+  return [
+    {
+      runId: event.runId,
+      eventId: event.eventId,
+      ...event.parentId !== void 0 ? { parentId: event.parentId } : {},
+      kind: event.kind,
+      name: event.name,
+      ...path !== void 0 ? { path } : {}
+    }
+  ];
+}
+function fail(ruleId, message, evidence, expected, actual) {
+  return {
+    ruleId,
+    status: "fail",
+    severity: "error",
+    message,
+    ...expected !== void 0 ? { expected } : {},
+    ...actual !== void 0 ? { actual } : {},
+    evidence
+  };
+}
+function nodeNames(nodes, kind) {
+  return nodes.filter((node) => node.event.kind === kind).map((node) => node.event.name).sort((a, b) => a.localeCompare(b));
+}
+function hasAttribute(node, key) {
+  return node.event.attributes !== void 0 && Object.prototype.hasOwnProperty.call(node.event.attributes, key);
+}
+function numericAttribute(node, keys) {
+  const attrs = node.event.attributes;
+  if (attrs === void 0) return void 0;
+  for (const key of keys) {
+    const value = attrs[key];
+    if (typeof value === "number" && Number.isFinite(value)) return value;
+  }
+  return void 0;
+}
+function totalTokenCount(events) {
+  return events.reduce((total, event) => {
+    const usage = event.tokenUsage;
+    if (usage?.total !== void 0) return total + usage.total;
+    return total + (usage?.input ?? 0) + (usage?.output ?? 0);
+  }, 0);
+}
+function createRule(id, category, evaluate) {
+  return { id, category, severity: "error", evaluate };
+}
+var DEFAULT_ANSWER_KEYS = [
+  "answer",
+  "finalAnswer",
+  "final",
+  "response",
+  "result",
+  "output",
+  "outputPreview",
+  "completion",
+  "text"
+];
+var DEFAULT_CONTEXT_KEYS = [
+  "context",
+  "contexts",
+  "document",
+  "documents",
+  "retrieved",
+  "retrieval",
+  "chunks",
+  "chunk",
+  "source",
+  "sources",
+  "sourceText"
+];
+var DEFAULT_CITATION_KEYS = [
+  "citation",
+  "citations",
+  "reference",
+  "references",
+  "sourceId",
+  "sourceIds",
+  "source_id",
+  "source_ids"
+];
+var DEFAULT_SOURCE_ID_KEYS = [
+  ...DEFAULT_CITATION_KEYS,
+  "id",
+  "ids",
+  "documentId",
+  "documentIds",
+  "docId",
+  "docIds"
+];
+var DEFAULT_BANNED_UNSUPPORTED_PHRASES = [
+  "i don't have enough information",
+  "i do not have enough information",
+  "not enough context",
+  "cannot determine from the context",
+  "unable to determine from the provided context"
+];
+var STOP_WORDS = /* @__PURE__ */ new Set([
+  "a",
+  "an",
+  "and",
+  "are",
+  "but",
+  "for",
+  "from",
+  "have",
+  "into",
+  "not",
+  "that",
+  "the",
+  "their",
+  "this",
+  "was",
+  "were",
+  "with",
+  "you",
+  "your"
+]);
+function normalizeKey(key) {
+  return key.toLowerCase().replace(/[^a-z0-9]/g, "");
+}
+function keySet(keys) {
+  return new Set(keys.map(normalizeKey));
+}
+function valuesAsStrings(value, depth = 0) {
+  if (depth > 5) return [];
+  if (typeof value === "string") {
+    const trimmed = value.trim();
+    return trimmed.length > 0 ? [trimmed] : [];
+  }
+  if (typeof value === "number" || typeof value === "boolean") return [String(value)];
+  if (Array.isArray(value)) {
+    return value.flatMap((item) => valuesAsStrings(item, depth + 1));
+  }
+  if (value !== null && typeof value === "object") {
+    return Object.values(value).flatMap(
+      (item) => valuesAsStrings(item, depth + 1)
+    );
+  }
+  return [];
+}
+function collectTextFields(nodes, keys, preferredKinds = []) {
+  const wanted = keySet(keys);
+  const preferred = new Set(preferredKinds);
+  const orderedNodes = [...nodes].sort((a, b) => {
+    const aPreferred = preferred.has(a.event.kind) ? 0 : 1;
+    const bPreferred = preferred.has(b.event.kind) ? 0 : 1;
+    return aPreferred - bPreferred || a.event.eventId.localeCompare(b.event.eventId);
+  });
+  const fields = [];
+  for (const node of orderedNodes) {
+    const attrs = node.event.attributes;
+    if (attrs === void 0) continue;
+    for (const [key, value] of Object.entries(attrs)) {
+      if (!wanted.has(normalizeKey(key))) continue;
+      for (const text of valuesAsStrings(value)) {
+        fields.push({ text, node, path: `attributes.${key}` });
+      }
+    }
+  }
+  return fields;
+}
+function tokenize(text) {
+  return [...text.toLowerCase().matchAll(/[a-z0-9][a-z0-9'-]{2,}/g)].map((match) => match[0].replace(/^['-]+|['-]+$/g, "")).filter((token) => token.length > 2 && !STOP_WORDS.has(token));
+}
+function firstEvidence(fields, run, path) {
+  const first = fields[0];
+  return first === void 0 ? evidenceForRun(run, path) : evidenceForEvent(first.node.event, first.path);
+}
+function collectSourceIds(nodes, keys) {
+  const wanted = keySet(keys);
+  const ids = /* @__PURE__ */ new Set();
+  for (const node of nodes) {
+    const attrs = node.event.attributes;
+    if (attrs === void 0) continue;
+    for (const [key, value] of Object.entries(attrs)) {
+      if (!wanted.has(normalizeKey(key))) continue;
+      for (const candidate of valuesAsStrings(value)) {
+        const trimmed = candidate.trim();
+        if (trimmed.length > 0 && trimmed.length <= 128) ids.add(trimmed);
+      }
+    }
+  }
+  return [...ids].sort((a, b) => a.localeCompare(b));
+}
+function citationCount(answer, citationFields) {
+  const inline = answer.match(/\[[^\]\n]{1,40}\]|\([A-Za-z][A-Za-z0-9_-]{0,39}\)/g)?.length ?? 0;
+  return inline + citationFields.length;
+}
+function quotedSnippets(text, minLength) {
+  const snippets = /* @__PURE__ */ new Set();
+  const pattern = /"([^"\n]+)"|'([^'\n]+)'|“([^”\n]+)”/g;
+  for (const match of text.matchAll(pattern)) {
+    const snippet = (match[1] ?? match[2] ?? match[3] ?? "").trim();
+    if (snippet.length >= minLength) snippets.add(snippet);
+  }
+  return [...snippets].sort((a, b) => a.localeCompare(b));
+}
+function wordCount(text) {
+  return tokenize(text).length;
+}
+var checks = {
+  requireSuccess() {
+    return createRule(
+      "eval.requireSuccess",
+      "run",
+      (context) => context.run.status === "ok" ? [] : [
+        fail(
+          "eval.requireSuccess",
+          "Run did not complete successfully.",
+          evidenceForRun(context.run, "status"),
+          "ok",
+          context.run.status ?? "unknown"
+        )
+      ]
+    );
+  },
+  requiredTools(required) {
+    const expected = [...required].sort((a, b) => a.localeCompare(b));
+    return createRule("eval.requiredTools", "tool", (context) => {
+      const tools = new Set(nodeNames(context.nodes, "TOOL"));
+      return expected.filter((name) => !tools.has(name)).map(
+        (name) => fail(
+          "eval.requiredTools",
+          `Required tool ${name} did not appear.`,
+          evidenceForRun(context.run, "children"),
+          name,
+          [...tools].sort((a, b) => a.localeCompare(b))
+        )
+      );
+    });
+  },
+  forbiddenTools(forbidden) {
+    const blocked = [...forbidden].sort((a, b) => a.localeCompare(b));
+    return createRule(
+      "eval.forbiddenTools",
+      "tool",
+      (context) => context.nodes.filter((node) => node.event.kind === "TOOL" && blocked.includes(node.event.name)).map(
+        (node) => fail(
+          "eval.forbiddenTools",
+          `Forbidden tool ${node.event.name} appeared.`,
+          evidenceForEvent(node.event, "name"),
+          "tool absent",
+          node.event.name
+        )
+      )
+    );
+  },
+  maxDurationMs(maxDurationMs) {
+    return createRule(
+      "eval.maxDurationMs",
+      "run",
+      (context) => context.run.durationMs !== void 0 && context.run.durationMs > maxDurationMs ? [
+        fail(
+          "eval.maxDurationMs",
+          `Run duration exceeded ${maxDurationMs}ms.`,
+          evidenceForRun(context.run, "durationMs"),
+          { maxDurationMs },
+          context.run.durationMs
+        )
+      ] : []
+    );
+  },
+  maxDepth(maxDepth) {
+    return createRule("eval.maxDepth", "structure", (context) => {
+      const deepest = context.nodes.reduce(
+        (current, node) => current === void 0 || node.depth > current.depth ? node : current,
+        void 0
+      );
+      return deepest !== void 0 && deepest.depth > maxDepth ? [
+        fail(
+          "eval.maxDepth",
+          `Run tree depth exceeded ${maxDepth}.`,
+          evidenceForEvent(deepest.event, "depth"),
+          { maxDepth },
+          deepest.depth
+        )
+      ] : [];
+    });
+  },
+  maxRetries(maxRetries) {
+    return createRule(
+      "eval.maxRetries",
+      "structure",
+      (context) => context.nodes.flatMap((node) => {
+        const retries = numericAttribute(node, ["retryCount", "retries", "attempt"]);
+        return retries !== void 0 && retries > maxRetries ? [
+          fail(
+            "eval.maxRetries",
+            `Retry count exceeded ${maxRetries}.`,
+            evidenceForEvent(node.event, "attributes.retryCount"),
+            { maxRetries },
+            retries
+          )
+        ] : [];
+      })
+    );
+  },
+  maxTotalTokens(maxTotalTokens) {
+    return createRule("eval.maxTotalTokens", "llm", (context) => {
+      const total = totalTokenCount(context.events);
+      return total > maxTotalTokens ? [
+        fail(
+          "eval.maxTotalTokens",
+          `Total token usage exceeded ${maxTotalTokens}.`,
+          evidenceForRun(context.run, "tokenUsage.total"),
+          { maxTotalTokens },
+          total
+        )
+      ] : [];
+    });
+  },
+  noFailedSteps() {
+    return createRule(
+      "eval.noFailedSteps",
+      "run",
+      (context) => context.nodes.filter((node) => node.event.status === "error" || node.event.kind === "ERROR").map(
+        (node) => fail(
+          "eval.noFailedSteps",
+          "Run contains a failed step or error node.",
+          evidenceForEvent(node.event, "status"),
+          "no failed nodes",
+          node.event.status ?? node.event.kind
+        )
+      )
+    );
+  },
+  requiredRetrievalBeforeGeneration() {
+    return createRule("eval.requiredRetrievalBeforeGeneration", "retrieval", (context) => {
+      const firstLlmIndex = context.nodes.findIndex((node) => node.event.kind === "LLM");
+      if (firstLlmIndex === -1) return [];
+      const retrievalIndex = context.nodes.findIndex(
+        (node, index) => index < firstLlmIndex && node.event.kind === "RETRIEVER"
+      );
+      return retrievalIndex === -1 ? [
+        fail(
+          "eval.requiredRetrievalBeforeGeneration",
+          "No retrieval step appeared before the first LLM generation.",
+          evidenceForEvent(context.nodes[firstLlmIndex].event, "kind"),
+          "RETRIEVER before LLM",
+          "LLM before RETRIEVER"
+        )
+      ] : [];
+    });
+  },
+  requiredDecisionMetadata(keys) {
+    const required = [...keys].sort((a, b) => a.localeCompare(b));
+    return createRule("eval.requiredDecisionMetadata", "structure", (context) => {
+      const decisions = context.nodes.filter((node) => node.event.kind === "DECISION");
+      if (decisions.length === 0) {
+        return [
+          fail(
+            "eval.requiredDecisionMetadata",
+            "No decision node is available for required metadata.",
+            evidenceForRun(context.run, "children"),
+            { decisionMetadata: required },
+            "no decision nodes"
+          )
+        ];
+      }
+      return decisions.flatMap(
+        (node) => required.filter((key) => !hasAttribute(node, key)).map(
+          (key) => fail(
+            "eval.requiredDecisionMetadata",
+            `Decision metadata ${key} is missing.`,
+            evidenceForEvent(node.event, `attributes.${key}`),
+            key,
+            "missing"
+          )
+        )
+      );
+    });
+  },
+  contextOverlap(options = {}) {
+    const minOverlap = options.minOverlap ?? 0.1;
+    const minSharedTerms = options.minSharedTerms ?? 1;
+    const answerKeys = options.answerKeys ?? DEFAULT_ANSWER_KEYS;
+    const contextKeys = options.contextKeys ?? DEFAULT_CONTEXT_KEYS;
+    return createRule("eval.contextOverlap", "retrieval", (context) => {
+      const answers = collectTextFields(context.nodes, answerKeys, ["RESULT", "LLM", "AGENT"]);
+      const contexts = collectTextFields(context.nodes, contextKeys, ["RETRIEVER", "TOOL"]);
+      if (answers.length === 0 || contexts.length === 0) {
+        return [
+          fail(
+            "eval.contextOverlap",
+            "Answer and context text are required for overlap evaluation.",
+            firstEvidence(answers.length > 0 ? answers : contexts, context.run, "children"),
+            { answer: "present", context: "present" },
+            { answerFields: answers.length, contextFields: contexts.length }
+          )
+        ];
+      }
+      const answerTerms = new Set(tokenize(answers.map((field) => field.text).join(" ")));
+      const contextTerms = new Set(tokenize(contexts.map((field) => field.text).join(" ")));
+      const sharedTerms = [...answerTerms].filter((term) => contextTerms.has(term)).length;
+      const overlap = answerTerms.size === 0 ? 0 : sharedTerms / answerTerms.size;
+      return sharedTerms < minSharedTerms || overlap < minOverlap ? [
+        fail(
+          "eval.contextOverlap",
+          "Answer text did not sufficiently overlap retrieved context.",
+          firstEvidence(answers, context.run, "attributes.answer"),
+          { minOverlap, minSharedTerms },
+          {
+            answerTerms: answerTerms.size,
+            contextTerms: contextTerms.size,
+            sharedTerms,
+            overlap: Number(overlap.toFixed(4))
+          }
+        )
+      ] : [];
+    });
+  },
+  quoteOverlap(options = {}) {
+    const answerKeys = options.answerKeys ?? DEFAULT_ANSWER_KEYS;
+    const contextKeys = options.contextKeys ?? DEFAULT_CONTEXT_KEYS;
+    const minQuoteLength = options.minQuoteLength ?? 6;
+    const requireQuote = options.requireQuote ?? true;
+    return createRule("eval.quoteOverlap", "retrieval", (context) => {
+      const answers = collectTextFields(context.nodes, answerKeys, ["RESULT", "LLM", "AGENT"]);
+      const contexts = collectTextFields(context.nodes, contextKeys, ["RETRIEVER", "TOOL"]);
+      const answerText = answers.map((field) => field.text).join(" ");
+      const contextText = contexts.map((field) => field.text).join(" ").toLowerCase();
+      const quotes = quotedSnippets(answerText, minQuoteLength);
+      if (quotes.length === 0) {
+        return requireQuote ? [
+          fail(
+            "eval.quoteOverlap",
+            "Answer did not contain a quote for overlap evaluation.",
+            firstEvidence(answers, context.run, "attributes.answer"),
+            { quotedText: "present" },
+            { quoteCount: 0 }
+          )
+        ] : [];
+      }
+      const missing = quotes.filter((quote) => !contextText.includes(quote.toLowerCase()));
+      return missing.length > 0 ? [
+        fail(
+          "eval.quoteOverlap",
+          "Quoted answer text did not appear in retrieved context.",
+          firstEvidence(answers, context.run, "attributes.answer"),
+          { allQuotesInContext: true },
+          { quoteCount: quotes.length, missingQuotes: missing.length }
+        )
+      ] : [];
+    });
+  },
+  citationPresence(options = {}) {
+    const answerKeys = options.answerKeys ?? DEFAULT_ANSWER_KEYS;
+    const citationKeys = options.citationKeys ?? DEFAULT_CITATION_KEYS;
+    return createRule("eval.citationPresence", "retrieval", (context) => {
+      const answers = collectTextFields(context.nodes, answerKeys, ["RESULT", "LLM", "AGENT"]);
+      const citations = collectTextFields(context.nodes, citationKeys);
+      const count = citationCount(answers.map((field) => field.text).join(" "), citations);
+      return count === 0 ? [
+        fail(
+          "eval.citationPresence",
+          "Answer did not include citations or source references.",
+          firstEvidence(answers, context.run, "attributes.answer"),
+          { citationCount: ">= 1" },
+          { citationCount: 0 }
+        )
+      ] : [];
+    });
+  },
+  requiredSourceIds(requiredIds, options = {}) {
+    const expected = [...requiredIds].sort((a, b) => a.localeCompare(b));
+    const sourceIdKeys = options.sourceIdKeys ?? DEFAULT_SOURCE_ID_KEYS;
+    return createRule("eval.requiredSourceIds", "retrieval", (context) => {
+      const available = collectSourceIds(context.nodes, sourceIdKeys);
+      const availableSet = new Set(available);
+      const missing = expected.filter((id) => !availableSet.has(id));
+      return missing.length > 0 ? [
+        fail(
+          "eval.requiredSourceIds",
+          "Required source IDs were not present in trace context or citations.",
+          evidenceForRun(context.run, "children"),
+          { sourceIds: expected },
+          { missingSourceIds: missing, availableSourceIds: available.slice(0, 20) }
+        )
+      ] : [];
+    });
+  },
+  answerLengthBounds(options) {
+    const answerKeys = options.answerKeys ?? DEFAULT_ANSWER_KEYS;
+    return createRule("eval.answerLengthBounds", "llm", (context) => {
+      const answers = collectTextFields(context.nodes, answerKeys, ["RESULT", "LLM", "AGENT"]);
+      const answer = answers.map((field) => field.text).join(" ").trim();
+      const characters = answer.length;
+      const words = wordCount(answer);
+      const tooShort = options.minCharacters !== void 0 && characters < options.minCharacters || options.minWords !== void 0 && words < options.minWords;
+      const tooLong = options.maxCharacters !== void 0 && characters > options.maxCharacters || options.maxWords !== void 0 && words > options.maxWords;
+      return answer.length === 0 || tooShort || tooLong ? [
+        fail(
+          "eval.answerLengthBounds",
+          "Answer length fell outside required bounds.",
+          firstEvidence(answers, context.run, "attributes.answer"),
+          {
+            minCharacters: options.minCharacters,
+            maxCharacters: options.maxCharacters,
+            minWords: options.minWords,
+            maxWords: options.maxWords
+          },
+          { characters, words }
+        )
+      ] : [];
+    });
+  },
+  bannedUnsupportedPhrases(phrases = DEFAULT_BANNED_UNSUPPORTED_PHRASES, options = {}) {
+    const answerKeys = options.answerKeys ?? DEFAULT_ANSWER_KEYS;
+    const banned = [...phrases].map((phrase) => phrase.toLowerCase()).sort();
+    return createRule("eval.bannedUnsupportedPhrases", "safety", (context) => {
+      const answers = collectTextFields(context.nodes, answerKeys, ["RESULT", "LLM", "AGENT"]);
+      const answer = answers.map((field) => field.text).join(" ").toLowerCase();
+      const matches = banned.filter((phrase) => answer.includes(phrase));
+      return matches.length > 0 ? [
+        fail(
+          "eval.bannedUnsupportedPhrases",
+          "Answer contained banned unsupported-answer phrasing.",
+          firstEvidence(answers, context.run, "attributes.answer"),
+          { bannedPhraseCount: banned.length },
+          { matchedPhraseCount: matches.length }
+        )
+      ] : [];
+    });
+  }
+};
+function renderEvalMarkdown(result) {
+  const lines = [
+    `# AgentInspect Eval`,
+    "",
+    `Status: ${result.status}`,
+    `Format: ${result.format}`,
+    ...result.runId !== void 0 ? [`Run: ${result.runId}`] : [],
+    `Summary: ${result.summary.passed} passed, ${result.summary.failed} failed, ${result.summary.warnings} warnings, ${result.summary.errors} errors`
+  ];
+  if (result.diagnostics.length > 0) {
+    lines.push("", "## Diagnostics");
+    for (const diagnostic of result.diagnostics) {
+      lines.push(`- ${diagnostic.code}: ${diagnostic.message}`);
+    }
+  }
+  if (result.findings.length > 0) {
+    lines.push("", "## Findings");
+    for (const finding of result.findings) {
+      const path = finding.evidence[0]?.path;
+      lines.push(`- ${finding.ruleId}: ${finding.message}${path ? ` (${path})` : ""}`);
+    }
+  }
+  return `${lines.join("\n")}
+`;
+}
+
+exports.checks = checks;
+exports.evalRun = evalRun;
+exports.renderEvalMarkdown = renderEvalMarkdown;
+//# sourceMappingURL=index.cjs.map
+//# sourceMappingURL=index.cjs.map