@agent-inspect/eval 2.3.0 → 2.5.0

package/dist/index.mjs

@@ -1,4 +1,914 @@
 import { openTrace, TraceReadError } from 'agent-inspect/readers';
+import crypto, { createHash } from 'crypto';
+
+// packages/eval/src/index.ts
+var ALL_RULES = [
+  "circuit.same-tool-repetition",
+  "circuit.same-args-repetition",
+  "circuit.max-loop-iterations",
+  "circuit.max-retries",
+  "circuit.tool-timeout",
+  "circuit.runaway-llm-loop",
+  "circuit.excessive-branch-width"
+];
+function closed(ruleId, message) {
+  return { ruleId, status: "closed", severity: "info", message, evidence: [] };
+}
+function open(ruleId, message, evidence, severity = "error") {
+  return {
+    ruleId,
+    status: severity === "warning" ? "warn" : "open",
+    severity,
+    message,
+    evidence
+  };
+}
+function isToolEvent(event) {
+  const name = event.name.toLowerCase();
+  return event.kind === "tool" || name.startsWith("tool:") || name.startsWith("function:") || name.includes(".tool.") || name.startsWith("mcp:");
+}
+function isLlmEvent(event) {
+  const name = event.name.toLowerCase();
+  return event.kind === "llm" || name.startsWith("llm:") || name.includes(".llm.") || name.includes("generation");
+}
+function toolLabel(event) {
+  const attrs = event.attributes ?? {};
+  const fromAttr = attrs.toolName ?? attrs.tool ?? attrs.function;
+  if (typeof fromAttr === "string" && fromAttr.length > 0) return fromAttr;
+  return event.name.replace(/^(tool:|function:|mcp:)/i, "");
+}
+function argsHash(toolName, args) {
+  return createHash("sha256").update(`${toolName}:${JSON.stringify(args ?? null)}`).digest("hex").slice(0, 16);
+}
+function toolArgs(event) {
+  const attrs = event.attributes ?? {};
+  return attrs.arguments ?? attrs.args ?? attrs.input ?? attrs.parameters;
+}
+function durationMs(event) {
+  if (typeof event.durationMs === "number") return event.durationMs;
+  const attrs = event.attributes ?? {};
+  const fromAttr = attrs.durationMs ?? attrs.duration;
+  return typeof fromAttr === "number" ? fromAttr : void 0;
+}
+function attemptNumber(event) {
+  const attrs = event.attributes ?? {};
+  const value = attrs.attempt ?? attrs.retryAttempt ?? attrs.retryCount;
+  return typeof value === "number" ? value : void 0;
+}
+function evaluateSameToolRepetition(events, maxRepeats) {
+  const ruleId = "circuit.same-tool-repetition";
+  const counts = /* @__PURE__ */ new Map();
+  for (const event of events.filter(isToolEvent)) {
+    const label = toolLabel(event);
+    counts.set(label, (counts.get(label) ?? 0) + 1);
+  }
+  const evidence = [];
+  for (const [toolName, count] of counts) {
+    if (count > maxRepeats) {
+      evidence.push({ ruleId, toolName, count, threshold: maxRepeats });
+    }
+  }
+  if (evidence.length === 0) {
+    return closed(ruleId, "Tool repetition within threshold.");
+  }
+  return open(ruleId, "Same tool repeated beyond threshold.", evidence);
+}
+function evaluateSameArgsRepetition(events, maxRepeats) {
+  const ruleId = "circuit.same-args-repetition";
+  const counts = /* @__PURE__ */ new Map();
+  for (const event of events.filter(isToolEvent)) {
+    const label = toolLabel(event);
+    const hash = argsHash(label, toolArgs(event));
+    const key = `${label}:${hash}`;
+    const current = counts.get(key) ?? { toolName: label, count: 0 };
+    current.count += 1;
+    counts.set(key, current);
+  }
+  const evidence = [];
+  for (const entry of counts.values()) {
+    if (entry.count > maxRepeats) {
+      evidence.push({ ruleId, toolName: entry.toolName, count: entry.count, threshold: maxRepeats });
+    }
+  }
+  if (evidence.length === 0) {
+    return closed(ruleId, "Tool argument repetition within threshold.");
+  }
+  return open(ruleId, "Same tool arguments repeated beyond threshold.", evidence);
+}
+function evaluateMaxLoopIterations(events, maxIterations) {
+  const ruleId = "circuit.max-loop-iterations";
+  const iterationEvents = events.filter((event) => {
+    const attrs = event.attributes ?? {};
+    return typeof attrs.iteration === "number" || event.name.toLowerCase().includes("loop");
+  });
+  const maxSeen = iterationEvents.reduce((max, event) => {
+    const attrs = event.attributes ?? {};
+    const iteration = typeof attrs.iteration === "number" ? attrs.iteration : max;
+    return Math.max(max, iteration);
+  }, iterationEvents.length);
+  if (maxSeen <= maxIterations) {
+    return closed(ruleId, "Loop iterations within threshold.");
+  }
+  return open(ruleId, "Loop iterations exceeded threshold.", [
+    { ruleId, count: maxSeen, threshold: maxIterations }
+  ]);
+}
+function evaluateMaxRetries(events, maxRetries) {
+  const ruleId = "circuit.max-retries";
+  const attempts = events.map(attemptNumber).filter((value) => value !== void 0);
+  const maxAttempt = attempts.length > 0 ? Math.max(...attempts) : 0;
+  if (maxAttempt <= maxRetries) {
+    return closed(ruleId, "Retry count within threshold.");
+  }
+  return open(ruleId, "Retry count exceeded threshold.", [
+    { ruleId, count: maxAttempt, threshold: maxRetries }
+  ]);
+}
+function evaluateToolTimeout(events, maxDurationMs) {
+  const ruleId = "circuit.tool-timeout";
+  const evidence = [];
+  for (const event of events.filter(isToolEvent)) {
+    const duration = durationMs(event);
+    if (duration !== void 0 && duration > maxDurationMs) {
+      evidence.push({
+        ruleId,
+        toolName: toolLabel(event),
+        count: duration,
+        threshold: maxDurationMs,
+        eventId: event.eventId
+      });
+    }
+  }
+  if (evidence.length === 0) {
+    return closed(ruleId, "Tool durations within timeout.");
+  }
+  return open(ruleId, "Tool call exceeded configured timeout.", evidence, "warning");
+}
+function evaluateRunawayLlmLoop(events, maxLlmCalls) {
+  const ruleId = "circuit.runaway-llm-loop";
+  const llmCount = events.filter(isLlmEvent).length;
+  const hasTerminal = events.some((event) => {
+    const status = (event.status ?? event.attributes?.status ?? "").toString().toLowerCase();
+    return status === "ok" || status === "success" || status === "completed";
+  });
+  if (llmCount <= maxLlmCalls || hasTerminal) {
+    return closed(ruleId, "LLM call count within threshold or run completed.");
+  }
+  return open(ruleId, "Runaway LLM loop detected.", [
+    { ruleId, count: llmCount, threshold: maxLlmCalls }
+  ]);
+}
+function evaluateExcessiveBranchWidth(events, maxWidth) {
+  const ruleId = "circuit.excessive-branch-width";
+  const children = /* @__PURE__ */ new Map();
+  for (const event of events) {
+    const parentId = event.parentId;
+    if (!parentId) continue;
+    children.set(parentId, (children.get(parentId) ?? 0) + 1);
+  }
+  const evidence = [];
+  for (const [parentId, count] of children) {
+    if (count > maxWidth) {
+      evidence.push({ ruleId, path: parentId, count, threshold: maxWidth });
+    }
+  }
+  if (evidence.length === 0) {
+    return closed(ruleId, "Branch width within threshold.");
+  }
+  return open(ruleId, "Excessive parallel branch width detected.", evidence, "warning");
+}
+function runRule(ruleId, events, options) {
+  switch (ruleId) {
+    case "circuit.same-tool-repetition":
+      if (options.sameToolRepetition === void 0) return void 0;
+      return evaluateSameToolRepetition(events, options.sameToolRepetition.maxRepeats);
+    case "circuit.same-args-repetition":
+      if (options.sameArgsRepetition === void 0) return void 0;
+      return evaluateSameArgsRepetition(events, options.sameArgsRepetition.maxRepeats);
+    case "circuit.max-loop-iterations":
+      if (options.maxLoopIterations === void 0) return void 0;
+      return evaluateMaxLoopIterations(events, options.maxLoopIterations.maxIterations);
+    case "circuit.max-retries":
+      if (options.maxRetries === void 0) return void 0;
+      return evaluateMaxRetries(events, options.maxRetries.maxRetries);
+    case "circuit.tool-timeout":
+      if (options.toolTimeout === void 0) return void 0;
+      return evaluateToolTimeout(events, options.toolTimeout.maxDurationMs);
+    case "circuit.runaway-llm-loop":
+      if (options.runawayLlmLoop === void 0) return void 0;
+      return evaluateRunawayLlmLoop(events, options.runawayLlmLoop.maxLlmCalls);
+    case "circuit.excessive-branch-width":
+      if (options.excessiveBranchWidth === void 0) return void 0;
+      return evaluateExcessiveBranchWidth(events, options.excessiveBranchWidth.maxWidth);
+    default:
+      return void 0;
+  }
+}
+function runCircuits(events, options = {}) {
+  const selected = options.rules ?? ALL_RULES;
+  const results = [];
+  for (const ruleId of selected) {
+    const result = runRule(ruleId, events, options);
+    if (result) results.push(result);
+  }
+  const ok = !results.some((result) => result.status === "open" && result.severity === "error");
+  return { ok, results };
+}
+var DEFAULT_REDACT_KEYS = [
+  "authorization",
+  "cookie",
+  "token",
+  "apiKey",
+  "password",
+  "secret",
+  "email"
+];
+var SHARE_PROFILE_EXTRA_KEYS = [
+  "userEmail",
+  "customerEmail",
+  "phone",
+  "phoneNumber",
+  "address",
+  "ip",
+  "ipAddress",
+  "sessionId",
+  "requestId",
+  "correlationId",
+  "decisionId",
+  "groupId",
+  "customerId",
+  "userId",
+  "accountId",
+  "tenantId",
+  "orgId",
+  "organizationId",
+  "traceId",
+  "spanId",
+  "parentSpanId"
+];
+var STRICT_PROFILE_EXTRA_KEYS = [
+  "prompt",
+  "completion",
+  "input",
+  "output",
+  "inputPreview",
+  "outputPreview",
+  "message",
+  "messages",
+  "transcript",
+  "context",
+  "document",
+  "documents",
+  "chunk",
+  "chunks",
+  "retrieval",
+  "query"
+];
+function isRecord(value) {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+function toKey(key) {
+  return key.toLowerCase();
+}
+function stableHash(value) {
+  const hash = crypto.createHash("sha256").update(value, "utf8").digest("hex");
+  return hash.slice(0, 8);
+}
+function stringifyScalar(value) {
+  if (typeof value === "string") return value;
+  if (typeof value === "number" || typeof value === "boolean" || typeof value === "bigint") {
+    return String(value);
+  }
+  return void 0;
+}
+function patternDetector(options) {
+  return {
+    id: options.id,
+    severity: options.severity ?? "warning",
+    matchKind: "value",
+    detect(input) {
+      if (typeof input.value !== "string") return [];
+      options.pattern.lastIndex = 0;
+      return options.pattern.test(input.value) ? [{ action: "replace", severity: options.severity ?? "warning", matchKind: "value" }] : [];
+    }
+  };
+}
+function digitsOnly(value) {
+  return value.replace(/\D/g, "");
+}
+function passesLuhn(value) {
+  const digits = digitsOnly(value);
+  if (digits.length < 13 || digits.length > 19) return false;
+  let sum = 0;
+  let double = false;
+  for (let i = digits.length - 1; i >= 0; i -= 1) {
+    let digit = Number(digits[i]);
+    if (double) {
+      digit *= 2;
+      if (digit > 9) digit -= 9;
+    }
+    sum += digit;
+    double = !double;
+  }
+  return sum % 10 === 0;
+}
+var credentialDetectors = [
+  patternDetector({
+    id: "value.authorizationHeader",
+    pattern: /^(?:basic|bearer|digest|apikey)\s+[a-z0-9._~+/=-]+$/i,
+    severity: "error"
+  }),
+  patternDetector({
+    id: "value.bearerToken",
+    pattern: /\bbearer\s+[a-z0-9._~+/=-]{12,}\b/i,
+    severity: "error"
+  }),
+  patternDetector({
+    id: "value.cookie",
+    pattern: /\b[a-z0-9_.-]+=[^;\s]+(?:;\s*[a-z0-9_.-]+=[^;\s]+)+/i,
+    severity: "error"
+  }),
+  patternDetector({
+    id: "value.jwt",
+    pattern: /\beyJ[a-zA-Z0-9_-]{8,}\.[a-zA-Z0-9_-]{8,}\.[a-zA-Z0-9_-]{8,}\b/,
+    severity: "error"
+  }),
+  patternDetector({
+    id: "value.providerApiKey",
+    pattern: /\b(?:sk-(?:proj-)?[a-zA-Z0-9_-]{16,}|sk-ant-[a-zA-Z0-9_-]{16,}|AIza[0-9A-Za-z_-]{20,})\b/,
+    severity: "error"
+  }),
+  patternDetector({
+    id: "value.githubToken",
+    pattern: /\b(?:ghp|gho|ghu|ghs|ghr)_[A-Za-z0-9_]{20,}\b/,
+    severity: "error"
+  }),
+  patternDetector({
+    id: "value.awsAccessKey",
+    pattern: /\b(?:AKIA|ASIA)[A-Z0-9]{16}\b/,
+    severity: "error"
+  }),
+  patternDetector({
+    id: "value.privateKey",
+    pattern: /-----BEGIN [A-Z ]*PRIVATE KEY-----[\s\S]*-----END [A-Z ]*PRIVATE KEY-----/,
+    severity: "error"
+  }),
+  {
+    id: "value.creditCard",
+    severity: "error",
+    matchKind: "value",
+    detect(input) {
+      if (typeof input.value !== "string") return [];
+      const candidatePattern = /(?:\d[ -]?){13,19}/g;
+      for (const match of input.value.matchAll(candidatePattern)) {
+        const candidate = match[0] ?? "";
+        if (passesLuhn(candidate)) {
+          return [{ action: "replace", severity: "error", matchKind: "value" }];
+        }
+      }
+      return [];
+    }
+  }
+];
+var identifierDetectors = [
+  patternDetector({
+    id: "value.email",
+    pattern: /\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}\b/i
+  }),
+  patternDetector({
+    id: "value.phone",
+    pattern: /\b(?:\+\d{1,3}[\s.-]?)?(?:\(?\d{3}\)?[\s.-])\d{3}[\s.-]\d{4}\b/
+  }),
+  patternDetector({
+    id: "value.ipv4",
+    pattern: /\b(?:(?:25[0-5]|2[0-4]\d|1?\d?\d)\.){3}(?:25[0-5]|2[0-4]\d|1?\d?\d)\b/
+  }),
+  patternDetector({
+    id: "value.ipv6",
+    pattern: /\b(?:[0-9a-f]{1,4}:){2,7}[0-9a-f]{1,4}\b/i
+  })
+];
+function builtInDetectorsForProfile(profile) {
+  if (profile === "local") return credentialDetectors;
+  return [...credentialDetectors, ...identifierDetectors];
+}
+function compileRules(rules, extraKeys) {
+  const out = /* @__PURE__ */ new Map();
+  const set = (rule) => {
+    const key = toKey(rule.key);
+    out.set(key, { ...rule, key });
+  };
+  for (const key of DEFAULT_REDACT_KEYS) {
+    set({ key, strategy: "full" });
+  }
+  for (const key of extraKeys ?? []) {
+    if (typeof key === "string" && key.length > 0) {
+      set({ key, strategy: "full" });
+    }
+  }
+  for (const rule of rules ?? []) {
+    if (typeof rule === "string") {
+      set({ key: rule, strategy: "full" });
+      continue;
+    }
+    if (rule.strategy === "full") set({ key: rule.key, strategy: "full" });
+    if (rule.strategy === "hash") set({ key: rule.key, strategy: "hash" });
+    if (rule.strategy === "prefix") {
+      set({
+        key: rule.key,
+        strategy: "prefix",
+        keep: typeof rule.keep === "number" ? rule.keep : 8
+      });
+    }
+  }
+  return [...out.values()];
+}
+function actionForRule(rule) {
+  if (rule.strategy === "full") return "replace";
+  return rule.strategy;
+}
+function applyRule(rule, value, replacement) {
+  if (rule.strategy === "full") return replacement;
+  const asString = stringifyScalar(value);
+  if (rule.strategy === "prefix") {
+    if (asString === void 0) return replacement;
+    const keep = Math.max(0, Math.floor(rule.keep));
+    return asString.length <= keep ? `${asString}\u2026` : `${asString.slice(0, keep)}\u2026`;
+  }
+  if (rule.strategy === "hash") {
+    if (asString === void 0) return "[HASH:unknown]";
+    return `[HASH:${stableHash(asString)}]`;
+  }
+  return value;
+}
+function childPath(path, key) {
+  if (/^[A-Za-z_$][A-Za-z0-9_$]*$/.test(key)) {
+    return path ? `${path}.${key}` : key;
+  }
+  return `${path || "$"}[${JSON.stringify(key)}]`;
+}
+function indexPath(path, index) {
+  return `${path || "$"}[${index}]`;
+}
+function makeFinding(path, detector, action, matchKind, severity = "warning", preview) {
+  return preview === void 0 ? { path, detector, action, severity, matchKind } : { path, detector, action, severity, matchKind, preview };
+}
+function createRedactionProfile(profile = "local") {
+  switch (profile) {
+    case "local":
+      return { profile: "local", extraKeys: [] };
+    case "share":
+      return {
+        profile: "share",
+        extraKeys: SHARE_PROFILE_EXTRA_KEYS,
+        maxMetadataValueLengthCap: 500,
+        maxPreviewLengthCap: 200
+      };
+    case "strict":
+      return {
+        profile: "strict",
+        extraKeys: [...SHARE_PROFILE_EXTRA_KEYS, ...STRICT_PROFILE_EXTRA_KEYS],
+        maxMetadataValueLengthCap: 200,
+        maxPreviewLengthCap: 80
+      };
+  }
+}
+var Redactor = class {
+  #rules;
+  #detectors;
+  #profile;
+  #replacement;
+  #maxDepth;
+  #collectFindings;
+  constructor(options) {
+    const resolved = createRedactionProfile(options?.profile ?? "local");
+    this.#profile = resolved.profile;
+    this.#rules = compileRules(options?.rules, [
+      ...resolved.extraKeys,
+      ...options?.extraKeys ?? []
+    ]);
+    this.#detectors = [
+      ...builtInDetectorsForProfile(this.#profile),
+      ...options?.detectors ?? []
+    ];
+    this.#replacement = options?.replacement ?? "[REDACTED]";
+    this.#maxDepth = options?.maxDepth ?? 32;
+    this.#collectFindings = options?.collectFindings ?? true;
+  }
+  redactValue(key, value) {
+    return this.#redactValue(value, key, key, 0, {
+      findings: [],
+      seen: /* @__PURE__ */ new WeakMap()
+    });
+  }
+  redactRecord(record) {
+    return this.redact(record).value;
+  }
+  redact(value) {
+    const state = {
+      findings: [],
+      seen: /* @__PURE__ */ new WeakMap()
+    };
+    const redacted = this.#redactValue(value, void 0, "$", 0, state);
+    return {
+      value: redacted,
+      findings: state.findings,
+      redacted: state.findings.some((finding) => finding.action !== "keep"),
+      profile: this.#profile
+    };
+  }
+  #recordFinding(state, finding) {
+    if (this.#collectFindings) state.findings.push(finding);
+  }
+  #redactValue(value, key, path, depth, state) {
+    if (depth > this.#maxDepth) {
+      this.#recordFinding(
+        state,
+        makeFinding(path, "structure.maxDepth", "truncate", "value", "warning")
+      );
+      return "[Truncated]";
+    }
+    if (key !== void 0) {
+      const rule = this.#rules.find((candidate) => candidate.key === toKey(key));
+      if (rule) {
+        this.#recordFinding(
+          state,
+          makeFinding(path, `key.${rule.key}`, actionForRule(rule), "key", "warning")
+        );
+        return applyRule(rule, value, this.#replacement);
+      }
+    }
+    for (const detector of this.#detectors) {
+      const detections = detector.detect({ path, key, value });
+      for (const detection of detections) {
+        const action = detection.action ?? "replace";
+        this.#recordFinding(
+          state,
+          makeFinding(
+            path,
+            detector.id,
+            action,
+            detection.matchKind ?? detector.matchKind ?? "custom",
+            detection.severity ?? detector.severity ?? "warning",
+            detection.preview
+          )
+        );
+        if (action !== "keep") {
+          return detection.replacement ?? this.#replacement;
+        }
+      }
+    }
+    if (Array.isArray(value)) {
+      if (state.seen.has(value)) return state.seen.get(value);
+      const out = [];
+      state.seen.set(value, out);
+      value.forEach((item, index) => {
+        out[index] = this.#redactValue(item, void 0, indexPath(path, index), depth + 1, state);
+      });
+      return out;
+    }
+    if (isRecord(value)) {
+      if (state.seen.has(value)) return state.seen.get(value);
+      const out = {};
+      state.seen.set(value, out);
+      for (const [entryKey, entryValue] of Object.entries(value)) {
+        out[entryKey] = this.#redactValue(
+          entryValue,
+          entryKey,
+          childPath(path === "$" ? "" : path, entryKey),
+          depth + 1,
+          state
+        );
+      }
+      return out;
+    }
+    return value;
+  }
+};
+function createRedactor(options) {
+  return new Redactor(options);
+}
+function redact(value, options) {
+  return createRedactor(options).redact(value);
+}
+
+// packages/guardrails/src/rules.ts
+var DEFAULT_INJECTION_PATTERNS = [
+  "ignore previous instructions",
+  "ignore all prior",
+  "disregard your instructions",
+  "system prompt",
+  "you are now",
+  "jailbreak"
+];
+function pass(ruleId, message) {
+  return { ruleId, status: "pass", severity: "info", message, evidence: [] };
+}
+function fail(ruleId, message, evidence, severity = "error") {
+  return { ruleId, status: severity === "warning" ? "warn" : "fail", severity, message, evidence };
+}
+function boundedPreview(value, max = 80) {
+  if (value.length <= max) return value;
+  return `${value.slice(0, max - 3)}...`;
+}
+function evaluateBannedPhrase(text, options) {
+  const ruleId = "guardrail.banned-phrase";
+  const haystack = options.caseInsensitive !== false ? text.toLowerCase() : text;
+  const evidence = [];
+  for (const phrase of options.phrases) {
+    const needle = options.caseInsensitive !== false ? phrase.toLowerCase() : phrase;
+    if (needle.length > 0 && haystack.includes(needle)) {
+      evidence.push({ ruleId, match: phrase, preview: boundedPreview(text) });
+    }
+  }
+  if (evidence.length === 0) {
+    return pass(ruleId, "No banned phrases matched.");
+  }
+  return fail(ruleId, `Matched ${evidence.length} banned phrase(s).`, evidence);
+}
+var SEVERITY_RANK = { info: 0, warning: 1, error: 2 };
+function evaluatePiiLeak(value, options = {}) {
+  const ruleId = "guardrail.pii-leak";
+  const minSeverity = options.minSeverity ?? "warning";
+  const result = redact(value, { profile: options.profile ?? "share", collectFindings: true });
+  const findings = result.findings.filter(
+    (finding) => SEVERITY_RANK[finding.severity] >= SEVERITY_RANK[minSeverity]
+  );
+  if (findings.length === 0) {
+    return pass(ruleId, "No PII-style redaction findings.");
+  }
+  const evidence = findings.map((finding) => ({
+    ruleId,
+    path: finding.path,
+    detector: finding.detector,
+    preview: finding.preview
+  }));
+  return fail(ruleId, `Detected ${findings.length} PII-style finding(s).`, evidence);
+}
+function measureDepth(value) {
+  if (value === null || typeof value !== "object") return 0;
+  if (Array.isArray(value)) {
+    return 1 + Math.max(0, ...value.map((item) => measureDepth(item)));
+  }
+  const depths = Object.values(value).map((item) => measureDepth(item));
+  return 1 + (depths.length === 0 ? 0 : Math.max(...depths));
+}
+function maxStringLength(value) {
+  if (typeof value === "string") return value.length;
+  if (value === null || typeof value !== "object") return 0;
+  if (Array.isArray(value)) {
+    return Math.max(0, ...value.map((item) => maxStringLength(item)));
+  }
+  return Math.max(0, ...Object.values(value).map((item) => maxStringLength(item)));
+}
+function evaluateUnsafeToolArgs(toolName, toolArgs2, options = {}) {
+  const ruleId = "guardrail.unsafe-tool-args";
+  const blocked = new Set((options.blockedTools ?? []).map((name) => name.toLowerCase()));
+  const evidence = [];
+  if (blocked.has(toolName.toLowerCase())) {
+    evidence.push({ ruleId, preview: toolName, match: toolName });
+  }
+  const maxDepth = options.maxDepth ?? 12;
+  const depth = measureDepth(toolArgs2);
+  if (depth > maxDepth) {
+    evidence.push({ ruleId, path: "args", preview: `depth=${depth}` });
+  }
+  const maxLen = options.maxStringLength ?? 16384;
+  const longest = maxStringLength(toolArgs2);
+  if (longest > maxLen) {
+    evidence.push({ ruleId, path: "args", preview: `maxStringLength=${longest}` });
+  }
+  if (evidence.length === 0) {
+    return pass(ruleId, "Tool arguments within configured bounds.");
+  }
+  return fail(ruleId, "Unsafe or oversized tool arguments detected.", evidence);
+}
+function evaluatePromptInjection(text, options = {}) {
+  const ruleId = "guardrail.prompt-injection";
+  const patterns = options.patterns ?? DEFAULT_INJECTION_PATTERNS;
+  const haystack = text.toLowerCase();
+  const evidence = [];
+  for (const pattern of patterns) {
+    const needle = pattern.toLowerCase();
+    if (needle.length > 0 && haystack.includes(needle)) {
+      evidence.push({ ruleId, match: pattern, preview: boundedPreview(text) });
+    }
+  }
+  if (evidence.length === 0) {
+    return pass(ruleId, "No prompt-injection patterns matched.");
+  }
+  return fail(ruleId, `Matched ${evidence.length} injection pattern(s).`, evidence, "warning");
+}
+function validateSchemaField(value, field, path, evidence) {
+  const ruleId = "guardrail.structured-output";
+  if (field.type) {
+    const actual = value === null ? "null" : Array.isArray(value) ? "array" : typeof value;
+    if (actual !== field.type) {
+      evidence.push({ ruleId, path, preview: `expected ${field.type}, got ${actual}` });
+      return;
+    }
+  }
+  if (field.enum && !field.enum.some((item) => Object.is(item, value))) {
+    evidence.push({ ruleId, path, preview: "value not in enum" });
+  }
+  if (field.type === "object" && field.required && value && typeof value === "object" && !Array.isArray(value)) {
+    const record = value;
+    for (const key of field.required) {
+      if (!(key in record)) {
+        evidence.push({ ruleId, path: `${path}.${key}`, preview: "missing required key" });
+      }
+    }
+  }
+}
+function evaluateStructuredOutput(value, options) {
+  const ruleId = "guardrail.structured-output";
+  if (!value || typeof value !== "object" || Array.isArray(value)) {
+    return fail(ruleId, "Structured output must be an object.", [
+      { ruleId, preview: typeof value }
+    ]);
+  }
+  const record = value;
+  const evidence = [];
+  for (const [key, field] of Object.entries(options.schema)) {
+    validateSchemaField(record[key], field, key, evidence);
+  }
+  if (evidence.length === 0) {
+    return pass(ruleId, "Structured output matches schema subset.");
+  }
+  return fail(ruleId, "Structured output schema violation.", evidence);
+}
+function evaluateOversizeOutput(value, options = {}) {
+  const ruleId = "guardrail.oversize-output";
+  const text = typeof value === "string" ? value : JSON.stringify(value);
+  const maxLength = options.maxLength ?? options.maxSerializedLength ?? 32768;
+  if (text.length <= maxLength) {
+    return pass(ruleId, "Output within size limits.");
+  }
+  return fail(ruleId, `Output exceeds max length (${text.length} > ${maxLength}).`, [
+    { ruleId, preview: `length=${text.length}` }
+  ]);
+}
+function evaluateRequiredJsonShape(value, options) {
+  const ruleId = "guardrail.required-json-shape";
+  let parsed = value;
+  if (typeof value === "string") {
+    try {
+      parsed = JSON.parse(value);
+    } catch {
+      return fail(ruleId, "Value is not valid JSON.", [{ ruleId, preview: boundedPreview(value) }]);
+    }
+  }
+  if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) {
+    return fail(ruleId, "JSON value must be an object.", [{ ruleId, preview: typeof parsed }]);
+  }
+  const record = parsed;
+  const evidence = [];
+  for (const key of options.requiredKeys) {
+    if (!(key in record)) {
+      evidence.push({ ruleId, path: key, preview: "missing required key" });
+    }
+  }
+  if (evidence.length === 0) {
+    return pass(ruleId, "Required JSON keys present.");
+  }
+  return fail(ruleId, "Missing required JSON keys.", evidence);
+}
+
+// packages/guardrails/src/run.ts
+var ALL_RULES2 = [
+  "guardrail.banned-phrase",
+  "guardrail.pii-leak",
+  "guardrail.unsafe-tool-args",
+  "guardrail.prompt-injection",
+  "guardrail.structured-output",
+  "guardrail.oversize-output",
+  "guardrail.required-json-shape"
+];
+function isErrorFailure(result) {
+  return result.status === "fail" && result.severity === "error";
+}
+function runRule2(ruleId, input, options) {
+  switch (ruleId) {
+    case "guardrail.banned-phrase": {
+      if (!options.bannedPhrase || input.text === void 0) return void 0;
+      return evaluateBannedPhrase(input.text, options.bannedPhrase);
+    }
+    case "guardrail.pii-leak": {
+      if (input.value === void 0 && input.text === void 0) return void 0;
+      return evaluatePiiLeak(input.value ?? input.text, options.piiLeak);
+    }
+    case "guardrail.unsafe-tool-args": {
+      if (!input.toolName) return void 0;
+      return evaluateUnsafeToolArgs(input.toolName, input.toolArgs ?? {}, options.unsafeToolArgs);
+    }
+    case "guardrail.prompt-injection": {
+      if (input.text === void 0) return void 0;
+      return evaluatePromptInjection(input.text, options.promptInjection);
+    }
+    case "guardrail.structured-output": {
+      if (!options.structuredOutput || input.value === void 0) return void 0;
+      return evaluateStructuredOutput(input.value, options.structuredOutput);
+    }
+    case "guardrail.oversize-output": {
+      if (input.value === void 0 && input.text === void 0) return void 0;
+      return evaluateOversizeOutput(input.value ?? input.text, options.oversizeOutput);
+    }
+    case "guardrail.required-json-shape": {
+      if (!options.requiredJsonShape || input.value === void 0 && input.text === void 0) return void 0;
+      return evaluateRequiredJsonShape(input.value ?? input.text, options.requiredJsonShape);
+    }
+    default:
+      return void 0;
+  }
+}
+function runGuardrails(input, options = {}) {
+  const selected = options.rules ?? ALL_RULES2;
+  const results = [];
+  for (const ruleId of selected) {
+    const result = runRule2(ruleId, input, options);
+    if (result) results.push(result);
+  }
+  const ok = !results.some(isErrorFailure);
+  return { ok, results };
+}
+
+// packages/eval/src/safety-rules.ts
+function pass2(ruleId, message) {
+  return { ruleId, status: "pass", severity: "info", message, evidence: [] };
+}
+function failFinding(ruleId, message, expected, actual) {
+  return {
+    ruleId,
+    status: "fail",
+    severity: "error",
+    message,
+    expected,
+    actual,
+    evidence: []
+  };
+}
+function warnFinding(ruleId, message, actual) {
+  return {
+    ruleId,
+    status: "warning",
+    severity: "warning",
+    message,
+    actual,
+    evidence: []
+  };
+}
+function collectAnswerText(context) {
+  const parts = [];
+  for (const node of context.nodes) {
+    const attrs = node.event.attributes ?? {};
+    for (const key of ["answer", "output", "text", "content"]) {
+      const value = attrs[key];
+      if (typeof value === "string") parts.push(value);
+    }
+  }
+  return parts.join("\n");
+}
+function createEvalGuardrailRule(options) {
+  return {
+    id: "eval.guardrails",
+    category: "safety",
+    evaluate(context) {
+      const text = collectAnswerText(context);
+      const run = runGuardrails({ text, value: context.events }, options);
+      return run.results.flatMap((result) => {
+        if (result.status === "pass") return [pass2(result.ruleId, result.message)];
+        if (result.status === "warn") return [warnFinding(result.ruleId, result.message)];
+        return [failFinding(result.ruleId, result.message)];
+      });
+    }
+  };
+}
+function createEvalCircuitRule(options) {
+  return {
+    id: "eval.circuits",
+    category: "structure",
+    evaluate(context) {
+      const events = context.events.map((event) => ({
+        eventId: event.eventId,
+        runId: event.runId,
+        name: event.name,
+        kind: event.kind,
+        parentId: event.parentId,
+        startedAt: event.startedAt,
+        endedAt: event.endedAt,
+        durationMs: event.durationMs,
+        attributes: event.attributes,
+        status: event.status
+      }));
+      const run = runCircuits(events, options);
+      return run.results.flatMap((result) => {
+        if (result.status === "closed") return [pass2(result.ruleId, result.message)];
+        if (result.status === "warn") return [warnFinding(result.ruleId, result.message, result.evidence[0]?.count)];
+        return [failFinding(result.ruleId, result.message, result.evidence[0]?.threshold, result.evidence[0]?.count)];
+      });
+    }
+  };
+}
 
 // packages/eval/src/index.ts
 function isTraceReadResult(value) {
@@ -183,7 +1093,7 @@ function evidenceForEvent(event, path) {
     }
   ];
 }
-function fail(ruleId, message, evidence, expected, actual) {
+function fail2(ruleId, message, evidence, expected, actual) {
   return {
     ruleId,
     status: "fail",
@@ -379,7 +1289,7 @@ var checks = {
       "eval.requireSuccess",
       "run",
       (context) => context.run.status === "ok" ? [] : [
-        fail(
+        fail2(
           "eval.requireSuccess",
           "Run did not complete successfully.",
           evidenceForRun(context.run, "status"),
@@ -394,7 +1304,7 @@ var checks = {
     return createRule("eval.requiredTools", "tool", (context) => {
       const tools = new Set(nodeNames(context.nodes, "TOOL"));
       return expected.filter((name) => !tools.has(name)).map(
-        (name) => fail(
+        (name) => fail2(
           "eval.requiredTools",
           `Required tool ${name} did not appear.`,
           evidenceForRun(context.run, "children"),
@@ -410,7 +1320,7 @@ var checks = {
       "eval.forbiddenTools",
       "tool",
       (context) => context.nodes.filter((node) => node.event.kind === "TOOL" && blocked.includes(node.event.name)).map(
-        (node) => fail(
+        (node) => fail2(
           "eval.forbiddenTools",
           `Forbidden tool ${node.event.name} appeared.`,
           evidenceForEvent(node.event, "name"),
@@ -425,7 +1335,7 @@ var checks = {
       "eval.maxDurationMs",
       "run",
       (context) => context.run.durationMs !== void 0 && context.run.durationMs > maxDurationMs ? [
-        fail(
+        fail2(
           "eval.maxDurationMs",
           `Run duration exceeded ${maxDurationMs}ms.`,
           evidenceForRun(context.run, "durationMs"),
@@ -442,7 +1352,7 @@ var checks = {
         void 0
       );
       return deepest !== void 0 && deepest.depth > maxDepth ? [
-        fail(
+        fail2(
           "eval.maxDepth",
           `Run tree depth exceeded ${maxDepth}.`,
           evidenceForEvent(deepest.event, "depth"),
@@ -459,7 +1369,7 @@ var checks = {
       (context) => context.nodes.flatMap((node) => {
         const retries = numericAttribute(node, ["retryCount", "retries", "attempt"]);
         return retries !== void 0 && retries > maxRetries ? [
-          fail(
+          fail2(
             "eval.maxRetries",
             `Retry count exceeded ${maxRetries}.`,
             evidenceForEvent(node.event, "attributes.retryCount"),
@@ -474,7 +1384,7 @@ var checks = {
     return createRule("eval.maxTotalTokens", "llm", (context) => {
       const total = totalTokenCount(context.events);
       return total > maxTotalTokens ? [
-        fail(
+        fail2(
           "eval.maxTotalTokens",
           `Total token usage exceeded ${maxTotalTokens}.`,
           evidenceForRun(context.run, "tokenUsage.total"),
@@ -489,7 +1399,7 @@ var checks = {
       "eval.noFailedSteps",
       "run",
       (context) => context.nodes.filter((node) => node.event.status === "error" || node.event.kind === "ERROR").map(
-        (node) => fail(
+        (node) => fail2(
           "eval.noFailedSteps",
           "Run contains a failed step or error node.",
           evidenceForEvent(node.event, "status"),
@@ -507,7 +1417,7 @@ var checks = {
         (node, index) => index < firstLlmIndex && node.event.kind === "RETRIEVER"
       );
       return retrievalIndex === -1 ? [
-        fail(
+        fail2(
           "eval.requiredRetrievalBeforeGeneration",
           "No retrieval step appeared before the first LLM generation.",
           evidenceForEvent(context.nodes[firstLlmIndex].event, "kind"),
@@ -523,7 +1433,7 @@ var checks = {
       const decisions = context.nodes.filter((node) => node.event.kind === "DECISION");
       if (decisions.length === 0) {
         return [
-          fail(
+          fail2(
             "eval.requiredDecisionMetadata",
             "No decision node is available for required metadata.",
             evidenceForRun(context.run, "children"),
@@ -534,7 +1444,7 @@ var checks = {
       }
       return decisions.flatMap(
         (node) => required.filter((key) => !hasAttribute(node, key)).map(
-          (key) => fail(
+          (key) => fail2(
             "eval.requiredDecisionMetadata",
             `Decision metadata ${key} is missing.`,
             evidenceForEvent(node.event, `attributes.${key}`),
@@ -555,7 +1465,7 @@ var checks = {
       const contexts = collectTextFields(context.nodes, contextKeys, ["RETRIEVER", "TOOL"]);
       if (answers.length === 0 || contexts.length === 0) {
         return [
-          fail(
+          fail2(
             "eval.contextOverlap",
             "Answer and context text are required for overlap evaluation.",
             firstEvidence(answers.length > 0 ? answers : contexts, context.run, "children"),
@@ -569,7 +1479,7 @@ var checks = {
       const sharedTerms = [...answerTerms].filter((term) => contextTerms.has(term)).length;
       const overlap = answerTerms.size === 0 ? 0 : sharedTerms / answerTerms.size;
       return sharedTerms < minSharedTerms || overlap < minOverlap ? [
-        fail(
+        fail2(
           "eval.contextOverlap",
           "Answer text did not sufficiently overlap retrieved context.",
           firstEvidence(answers, context.run, "attributes.answer"),
@@ -597,7 +1507,7 @@ var checks = {
       const quotes = quotedSnippets(answerText, minQuoteLength);
       if (quotes.length === 0) {
         return requireQuote ? [
-          fail(
+          fail2(
             "eval.quoteOverlap",
             "Answer did not contain a quote for overlap evaluation.",
             firstEvidence(answers, context.run, "attributes.answer"),
@@ -608,7 +1518,7 @@ var checks = {
       }
       const missing = quotes.filter((quote) => !contextText.includes(quote.toLowerCase()));
       return missing.length > 0 ? [
-        fail(
+        fail2(
           "eval.quoteOverlap",
           "Quoted answer text did not appear in retrieved context.",
           firstEvidence(answers, context.run, "attributes.answer"),
@@ -626,7 +1536,7 @@ var checks = {
       const citations = collectTextFields(context.nodes, citationKeys);
       const count = citationCount(answers.map((field) => field.text).join(" "), citations);
       return count === 0 ? [
-        fail(
+        fail2(
           "eval.citationPresence",
           "Answer did not include citations or source references.",
           firstEvidence(answers, context.run, "attributes.answer"),
@@ -644,7 +1554,7 @@ var checks = {
       const availableSet = new Set(available);
       const missing = expected.filter((id) => !availableSet.has(id));
       return missing.length > 0 ? [
-        fail(
+        fail2(
           "eval.requiredSourceIds",
           "Required source IDs were not present in trace context or citations.",
           evidenceForRun(context.run, "children"),
@@ -664,7 +1574,7 @@ var checks = {
       const tooShort = options.minCharacters !== void 0 && characters < options.minCharacters || options.minWords !== void 0 && words < options.minWords;
       const tooLong = options.maxCharacters !== void 0 && characters > options.maxCharacters || options.maxWords !== void 0 && words > options.maxWords;
       return answer.length === 0 || tooShort || tooLong ? [
-        fail(
+        fail2(
           "eval.answerLengthBounds",
           "Answer length fell outside required bounds.",
           firstEvidence(answers, context.run, "attributes.answer"),
@@ -687,7 +1597,7 @@ var checks = {
       const answer = answers.map((field) => field.text).join(" ").toLowerCase();
       const matches = banned.filter((phrase) => answer.includes(phrase));
       return matches.length > 0 ? [
-        fail(
+        fail2(
           "eval.bannedUnsupportedPhrases",
           "Answer contained banned unsupported-answer phrasing.",
           firstEvidence(answers, context.run, "attributes.answer"),
@@ -724,6 +1634,6 @@ function renderEvalMarkdown(result) {
 `;
 }
 
-export { checks, evalRun, renderEvalMarkdown };
+export { checks, createEvalCircuitRule, createEvalGuardrailRule, evalRun, renderEvalMarkdown };
 //# sourceMappingURL=index.mjs.map
 //# sourceMappingURL=index.mjs.map