agent-inspect 2.4.0 → 2.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -4,7 +4,7 @@ import path13 from 'path';
4
4
  import { fileURLToPath, pathToFileURL } from 'url';
5
5
  import { Command, Option } from 'commander';
6
6
  import { AsyncLocalStorage } from 'async_hooks';
7
- import crypto, { webcrypto } from 'crypto';
7
+ import crypto, { webcrypto, createHash } from 'crypto';
8
8
  import { unlink, stat, mkdir, writeFile, appendFile, readdir, readFile, access, open } from 'fs/promises';
9
9
  import os from 'os';
10
10
  import process2, { stdin, stdout } from 'process';
@@ -12,7 +12,7 @@ import tty from 'tty';
12
12
  import { createInterface } from 'readline';
13
13
 
14
14
  // package.json
15
- var version = "2.4.0";
15
+ var version = "2.5.0";
16
16
 
17
17
  // packages/core/src/correlation-metadata.ts
18
18
  var TRACE_CORRELATION_KEYS = [
@@ -1429,18 +1429,18 @@ var proto = Object.defineProperties(() => {
1429
1429
  }
1430
1430
  }
1431
1431
  });
1432
- var createStyler = (open2, close, parent) => {
1432
+ var createStyler = (open3, close, parent) => {
1433
1433
  let openAll;
1434
1434
  let closeAll;
1435
1435
  if (parent === void 0) {
1436
- openAll = open2;
1436
+ openAll = open3;
1437
1437
  closeAll = close;
1438
1438
  } else {
1439
- openAll = parent.openAll + open2;
1439
+ openAll = parent.openAll + open3;
1440
1440
  closeAll = close + parent.closeAll;
1441
1441
  }
1442
1442
  return {
1443
- open: open2,
1443
+ open: open3,
1444
1444
  close,
1445
1445
  openAll,
1446
1446
  closeAll,
@@ -1505,15 +1505,15 @@ function getStatusIcon(status) {
1505
1505
  if (status === "error") return source_default.red("\u2716");
1506
1506
  return source_default.yellow("\u23F3");
1507
1507
  }
1508
- function renderStepLine(name, durationMs, status, depth) {
1508
+ function renderStepLine(name, durationMs2, status, depth) {
1509
1509
  try {
1510
1510
  const nm = formatTerminalName(name);
1511
1511
  const ind = getIndent(depth ?? 0);
1512
- if (status === "running" && durationMs === void 0) {
1512
+ if (status === "running" && durationMs2 === void 0) {
1513
1513
  return `${ind}${source_default.yellow("\u23F3")} ${nm}`;
1514
1514
  }
1515
- const hasDur = durationMs !== void 0 && Number.isFinite(durationMs);
1516
- const dur = hasDur ? formatDuration2(durationMs) : void 0;
1515
+ const hasDur = durationMs2 !== void 0 && Number.isFinite(durationMs2);
1516
+ const dur = hasDur ? formatDuration2(durationMs2) : void 0;
1517
1517
  if (status === "running") {
1518
1518
  return dur !== void 0 ? `${ind}${source_default.yellow("\u23F3")} ${nm} (${dur})` : `${ind}${source_default.yellow("\u23F3")} ${nm}`;
1519
1519
  }
@@ -1666,14 +1666,14 @@ async function extractMetadata(filePath, _quickScan) {
1666
1666
  } else {
1667
1667
  status = "unknown";
1668
1668
  }
1669
- const durationMs = explicitDurationMs ?? (startedAt !== void 0 && endedAt !== void 0 && Number.isFinite(startedAt) && Number.isFinite(endedAt) && endedAt >= startedAt ? endedAt - startedAt : void 0);
1669
+ const durationMs2 = explicitDurationMs ?? (startedAt !== void 0 && endedAt !== void 0 && Number.isFinite(startedAt) && Number.isFinite(endedAt) && endedAt >= startedAt ? endedAt - startedAt : void 0);
1670
1670
  return {
1671
1671
  runId: resolvedRunId,
1672
1672
  name,
1673
1673
  status,
1674
1674
  startedAt,
1675
1675
  endedAt,
1676
- durationMs,
1676
+ durationMs: durationMs2,
1677
1677
  eventCount: parsedTrace.sourceEventCount,
1678
1678
  filePath,
1679
1679
  fileSize: stats.size,
@@ -1694,7 +1694,7 @@ function buildRunSummary(events) {
1694
1694
  const runId = started?.runId ?? events.find((e) => typeof e.runId === "string")?.runId ?? "unknown-run";
1695
1695
  const name = typeof started?.name === "string" && started.name.trim() !== "" ? started.name : void 0;
1696
1696
  const status = lastCompleted ? lastCompleted.status : started ? "running" : "unknown";
1697
- const durationMs = lastCompleted && isFiniteNumber(lastCompleted.durationMs) ? lastCompleted.durationMs : void 0;
1697
+ const durationMs2 = lastCompleted && isFiniteNumber(lastCompleted.durationMs) ? lastCompleted.durationMs : void 0;
1698
1698
  started && isFiniteNumber(started.startTime) ? started.startTime : void 0;
1699
1699
  const steps = /* @__PURE__ */ new Map();
1700
1700
  for (const e of events) {
@@ -1784,7 +1784,7 @@ function buildRunSummary(events) {
1784
1784
  runId,
1785
1785
  name,
1786
1786
  status,
1787
- durationMs,
1787
+ durationMs: durationMs2,
1788
1788
  totalSteps,
1789
1789
  llmSteps,
1790
1790
  toolSteps,
@@ -3215,10 +3215,10 @@ function statusIcon(status) {
3215
3215
  if (status === "running") return "\u23F3";
3216
3216
  return "?";
3217
3217
  }
3218
- function durationCell(status, durationMs) {
3218
+ function durationCell(status, durationMs2) {
3219
3219
  if (status === "running" || status === "unknown") return "-";
3220
- if (durationMs !== void 0 && Number.isFinite(durationMs)) {
3221
- return formatDuration2(durationMs);
3220
+ if (durationMs2 !== void 0 && Number.isFinite(durationMs2)) {
3221
+ return formatDuration2(durationMs2);
3222
3222
  }
3223
3223
  return "-";
3224
3224
  }
@@ -3875,7 +3875,7 @@ var TreeBuilder = class {
3875
3875
  const startedAt = sorted.length > 0 ? sorted[0].timestamp : void 0;
3876
3876
  const endedAt = sorted.length > 0 ? sorted[sorted.length - 1].timestamp : void 0;
3877
3877
  const status = computeRunStatus(sorted);
3878
- const durationMs = startedAt !== void 0 && endedAt !== void 0 && Number.isFinite(startedAt) && Number.isFinite(endedAt) && endedAt >= startedAt && status !== "running" ? endedAt - startedAt : void 0;
3878
+ const durationMs2 = startedAt !== void 0 && endedAt !== void 0 && Number.isFinite(startedAt) && Number.isFinite(endedAt) && endedAt >= startedAt && status !== "running" ? endedAt - startedAt : void 0;
3879
3879
  const name = sorted.find((e) => e.kind === "RUN")?.name;
3880
3880
  out.push({
3881
3881
  runId,
@@ -3883,7 +3883,7 @@ var TreeBuilder = class {
3883
3883
  status,
3884
3884
  startedAt,
3885
3885
  endedAt: status === "running" ? void 0 : endedAt,
3886
- durationMs,
3886
+ durationMs: durationMs2,
3887
3887
  children: roots,
3888
3888
  metadata: {
3889
3889
  totalEvents: sorted.length,
@@ -4578,9 +4578,9 @@ function mapOpenInferenceSpan(span, index, version2) {
4578
4578
  if (endedAt !== void 0) {
4579
4579
  event.endedAt = endedAt;
4580
4580
  }
4581
- const durationMs = durationBetweenIso(startedAt, endedAt);
4582
- if (durationMs !== void 0) {
4583
- event.durationMs = durationMs;
4581
+ const durationMs2 = durationBetweenIso(startedAt, endedAt);
4582
+ if (durationMs2 !== void 0) {
4583
+ event.durationMs = durationMs2;
4584
4584
  }
4585
4585
  if (tokenUsage !== void 0) {
4586
4586
  event.tokenUsage = tokenUsage;
@@ -5165,9 +5165,9 @@ function mapOtlpSpan(context) {
5165
5165
  if (endedAt !== void 0) {
5166
5166
  event.endedAt = endedAt;
5167
5167
  }
5168
- const durationMs = durationBetweenIso(startedAt, endedAt);
5169
- if (durationMs !== void 0) {
5170
- event.durationMs = durationMs;
5168
+ const durationMs2 = durationBetweenIso(startedAt, endedAt);
5169
+ if (durationMs2 !== void 0) {
5170
+ event.durationMs = durationMs2;
5171
5171
  }
5172
5172
  if (tokenUsage !== void 0) {
5173
5173
  event.tokenUsage = tokenUsage;
@@ -6172,13 +6172,13 @@ var EventNormalizer = class {
6172
6172
  const timestampMissing = parsedTs === void 0;
6173
6173
  const parentIdKey = cfg.parentIdKey;
6174
6174
  const parentId = parentIdKey ? safeString(raw[parentIdKey]) : void 0;
6175
- let durationMs;
6175
+ let durationMs2;
6176
6176
  const durationKey = cfg.durationKey;
6177
6177
  if (durationKey) {
6178
6178
  const v = raw[durationKey];
6179
- if (isFiniteNumber2(v)) durationMs = v;
6179
+ if (isFiniteNumber2(v)) durationMs2 = v;
6180
6180
  } else if (isFiniteNumber2(raw.durationMs)) {
6181
- durationMs = raw.durationMs;
6181
+ durationMs2 = raw.durationMs;
6182
6182
  }
6183
6183
  let status;
6184
6184
  const statusKey = cfg.statusKey;
@@ -6225,7 +6225,7 @@ var EventNormalizer = class {
6225
6225
  kind,
6226
6226
  timestamp,
6227
6227
  ...status ? { status } : {},
6228
- ...durationMs !== void 0 ? { durationMs } : {},
6228
+ ...durationMs2 !== void 0 ? { durationMs: durationMs2 } : {},
6229
6229
  ...Object.keys(attributes).length > 0 ? { attributes } : {},
6230
6230
  confidence,
6231
6231
  source: {
@@ -7813,7 +7813,7 @@ function manualTraceEventsToRunTree(events) {
7813
7813
  }
7814
7814
  const startedAt = started.startTime;
7815
7815
  const endedAt = lastCompleted !== void 0 && runStatus !== "running" ? lastCompleted.endTime : void 0;
7816
- const durationMs = lastCompleted !== void 0 && Number.isFinite(lastCompleted.durationMs) ? lastCompleted.durationMs : void 0;
7816
+ const durationMs2 = lastCompleted !== void 0 && Number.isFinite(lastCompleted.durationMs) ? lastCompleted.durationMs : void 0;
7817
7817
  const steps = /* @__PURE__ */ new Map();
7818
7818
  for (const e of events) {
7819
7819
  if (e.event !== "step_started") continue;
@@ -7902,7 +7902,7 @@ function manualTraceEventsToRunTree(events) {
7902
7902
  status: runStatus,
7903
7903
  startedAt,
7904
7904
  endedAt,
7905
- durationMs,
7905
+ durationMs: durationMs2,
7906
7906
  children: roots,
7907
7907
  metadata: {
7908
7908
  totalEvents: inspectNodes.size,
@@ -8215,7 +8215,7 @@ function manualTraceEventsToComparableRun(events) {
8215
8215
  let runStatus;
8216
8216
  if (lastCompleted === void 0) runStatus = "running";
8217
8217
  else runStatus = lastCompleted.status;
8218
- const durationMs = lastCompleted !== void 0 && Number.isFinite(lastCompleted.durationMs) ? lastCompleted.durationMs : void 0;
8218
+ const durationMs2 = lastCompleted !== void 0 && Number.isFinite(lastCompleted.durationMs) ? lastCompleted.durationMs : void 0;
8219
8219
  const steps = /* @__PURE__ */ new Map();
8220
8220
  let order = 0;
8221
8221
  for (const e of events) {
@@ -8286,7 +8286,7 @@ function manualTraceEventsToComparableRun(events) {
8286
8286
  runId,
8287
8287
  name: rs.name,
8288
8288
  status: runStatus,
8289
- durationMs,
8289
+ durationMs: durationMs2,
8290
8290
  steps: roots
8291
8291
  };
8292
8292
  }
@@ -11180,7 +11180,7 @@ function createSafetyRawContentRule(options = {}) {
11180
11180
  }
11181
11181
  function createSafetySecretPatternRule(options = {}) {
11182
11182
  const patterns = options.patterns ?? DEFAULT_SECRET_PATTERNS;
11183
- const maxStringLength = options.maxStringLength ?? 4096;
11183
+ const maxStringLength2 = options.maxStringLength ?? 4096;
11184
11184
  return {
11185
11185
  id: "safety.secretPattern",
11186
11186
  category: "safety",
@@ -11190,7 +11190,7 @@ function createSafetySecretPatternRule(options = {}) {
11190
11190
  for (const event of context.events) {
11191
11191
  for (const entry of eventValueEntries(event, { includeSummaries: true, includeError: true })) {
11192
11192
  if (typeof entry.value !== "string") continue;
11193
- const sample = entry.value.slice(0, maxStringLength);
11193
+ const sample = entry.value.slice(0, maxStringLength2);
11194
11194
  for (const pattern of patterns) {
11195
11195
  pattern.pattern.lastIndex = 0;
11196
11196
  if (!pattern.pattern.test(sample)) continue;
@@ -11446,6 +11446,621 @@ function runTraceChecks(input3, options = {}) {
11446
11446
  diagnostics
11447
11447
  };
11448
11448
  }
11449
+ var ALL_RULES = [
11450
+ "circuit.same-tool-repetition",
11451
+ "circuit.same-args-repetition",
11452
+ "circuit.max-loop-iterations",
11453
+ "circuit.max-retries",
11454
+ "circuit.tool-timeout",
11455
+ "circuit.runaway-llm-loop",
11456
+ "circuit.excessive-branch-width"
11457
+ ];
11458
+ function closed(ruleId, message) {
11459
+ return { ruleId, status: "closed", severity: "info", message, evidence: [] };
11460
+ }
11461
+ function open2(ruleId, message, evidence, severity = "error") {
11462
+ return {
11463
+ ruleId,
11464
+ status: severity === "warning" ? "warn" : "open",
11465
+ severity,
11466
+ message,
11467
+ evidence
11468
+ };
11469
+ }
11470
+ function isToolEvent(event) {
11471
+ const name = event.name.toLowerCase();
11472
+ return event.kind === "tool" || name.startsWith("tool:") || name.startsWith("function:") || name.includes(".tool.") || name.startsWith("mcp:");
11473
+ }
11474
+ function isLlmEvent(event) {
11475
+ const name = event.name.toLowerCase();
11476
+ return event.kind === "llm" || name.startsWith("llm:") || name.includes(".llm.") || name.includes("generation");
11477
+ }
11478
+ function toolLabel(event) {
11479
+ const attrs = event.attributes ?? {};
11480
+ const fromAttr = attrs.toolName ?? attrs.tool ?? attrs.function;
11481
+ if (typeof fromAttr === "string" && fromAttr.length > 0) return fromAttr;
11482
+ return event.name.replace(/^(tool:|function:|mcp:)/i, "");
11483
+ }
11484
+ function argsHash(toolName2, args) {
11485
+ return createHash("sha256").update(`${toolName2}:${JSON.stringify(args ?? null)}`).digest("hex").slice(0, 16);
11486
+ }
11487
+ function toolArgs(event) {
11488
+ const attrs = event.attributes ?? {};
11489
+ return attrs.arguments ?? attrs.args ?? attrs.input ?? attrs.parameters;
11490
+ }
11491
+ function durationMs(event) {
11492
+ if (typeof event.durationMs === "number") return event.durationMs;
11493
+ const attrs = event.attributes ?? {};
11494
+ const fromAttr = attrs.durationMs ?? attrs.duration;
11495
+ return typeof fromAttr === "number" ? fromAttr : void 0;
11496
+ }
11497
+ function attemptNumber(event) {
11498
+ const attrs = event.attributes ?? {};
11499
+ const value = attrs.attempt ?? attrs.retryAttempt ?? attrs.retryCount;
11500
+ return typeof value === "number" ? value : void 0;
11501
+ }
11502
+ function evaluateSameToolRepetition(events, maxRepeats) {
11503
+ const ruleId = "circuit.same-tool-repetition";
11504
+ const counts = /* @__PURE__ */ new Map();
11505
+ for (const event of events.filter(isToolEvent)) {
11506
+ const label = toolLabel(event);
11507
+ counts.set(label, (counts.get(label) ?? 0) + 1);
11508
+ }
11509
+ const evidence = [];
11510
+ for (const [toolName2, count] of counts) {
11511
+ if (count > maxRepeats) {
11512
+ evidence.push({ ruleId, toolName: toolName2, count, threshold: maxRepeats });
11513
+ }
11514
+ }
11515
+ if (evidence.length === 0) {
11516
+ return closed(ruleId, "Tool repetition within threshold.");
11517
+ }
11518
+ return open2(ruleId, "Same tool repeated beyond threshold.", evidence);
11519
+ }
11520
+ function evaluateSameArgsRepetition(events, maxRepeats) {
11521
+ const ruleId = "circuit.same-args-repetition";
11522
+ const counts = /* @__PURE__ */ new Map();
11523
+ for (const event of events.filter(isToolEvent)) {
11524
+ const label = toolLabel(event);
11525
+ const hash = argsHash(label, toolArgs(event));
11526
+ const key = `${label}:${hash}`;
11527
+ const current = counts.get(key) ?? { toolName: label, count: 0 };
11528
+ current.count += 1;
11529
+ counts.set(key, current);
11530
+ }
11531
+ const evidence = [];
11532
+ for (const entry of counts.values()) {
11533
+ if (entry.count > maxRepeats) {
11534
+ evidence.push({ ruleId, toolName: entry.toolName, count: entry.count, threshold: maxRepeats });
11535
+ }
11536
+ }
11537
+ if (evidence.length === 0) {
11538
+ return closed(ruleId, "Tool argument repetition within threshold.");
11539
+ }
11540
+ return open2(ruleId, "Same tool arguments repeated beyond threshold.", evidence);
11541
+ }
11542
+ function evaluateMaxLoopIterations(events, maxIterations) {
11543
+ const ruleId = "circuit.max-loop-iterations";
11544
+ const iterationEvents = events.filter((event) => {
11545
+ const attrs = event.attributes ?? {};
11546
+ return typeof attrs.iteration === "number" || event.name.toLowerCase().includes("loop");
11547
+ });
11548
+ const maxSeen = iterationEvents.reduce((max, event) => {
11549
+ const attrs = event.attributes ?? {};
11550
+ const iteration = typeof attrs.iteration === "number" ? attrs.iteration : max;
11551
+ return Math.max(max, iteration);
11552
+ }, iterationEvents.length);
11553
+ if (maxSeen <= maxIterations) {
11554
+ return closed(ruleId, "Loop iterations within threshold.");
11555
+ }
11556
+ return open2(ruleId, "Loop iterations exceeded threshold.", [
11557
+ { ruleId, count: maxSeen, threshold: maxIterations }
11558
+ ]);
11559
+ }
11560
+ function evaluateMaxRetries(events, maxRetries) {
11561
+ const ruleId = "circuit.max-retries";
11562
+ const attempts = events.map(attemptNumber).filter((value) => value !== void 0);
11563
+ const maxAttempt = attempts.length > 0 ? Math.max(...attempts) : 0;
11564
+ if (maxAttempt <= maxRetries) {
11565
+ return closed(ruleId, "Retry count within threshold.");
11566
+ }
11567
+ return open2(ruleId, "Retry count exceeded threshold.", [
11568
+ { ruleId, count: maxAttempt, threshold: maxRetries }
11569
+ ]);
11570
+ }
11571
+ function evaluateToolTimeout(events, maxDurationMs) {
11572
+ const ruleId = "circuit.tool-timeout";
11573
+ const evidence = [];
11574
+ for (const event of events.filter(isToolEvent)) {
11575
+ const duration = durationMs(event);
11576
+ if (duration !== void 0 && duration > maxDurationMs) {
11577
+ evidence.push({
11578
+ ruleId,
11579
+ toolName: toolLabel(event),
11580
+ count: duration,
11581
+ threshold: maxDurationMs,
11582
+ eventId: event.eventId
11583
+ });
11584
+ }
11585
+ }
11586
+ if (evidence.length === 0) {
11587
+ return closed(ruleId, "Tool durations within timeout.");
11588
+ }
11589
+ return open2(ruleId, "Tool call exceeded configured timeout.", evidence, "warning");
11590
+ }
11591
+ function evaluateRunawayLlmLoop(events, maxLlmCalls) {
11592
+ const ruleId = "circuit.runaway-llm-loop";
11593
+ const llmCount = events.filter(isLlmEvent).length;
11594
+ const hasTerminal = events.some((event) => {
11595
+ const status = (event.status ?? event.attributes?.status ?? "").toString().toLowerCase();
11596
+ return status === "ok" || status === "success" || status === "completed";
11597
+ });
11598
+ if (llmCount <= maxLlmCalls || hasTerminal) {
11599
+ return closed(ruleId, "LLM call count within threshold or run completed.");
11600
+ }
11601
+ return open2(ruleId, "Runaway LLM loop detected.", [
11602
+ { ruleId, count: llmCount, threshold: maxLlmCalls }
11603
+ ]);
11604
+ }
11605
+ function evaluateExcessiveBranchWidth(events, maxWidth) {
11606
+ const ruleId = "circuit.excessive-branch-width";
11607
+ const children = /* @__PURE__ */ new Map();
11608
+ for (const event of events) {
11609
+ const parentId = event.parentId;
11610
+ if (!parentId) continue;
11611
+ children.set(parentId, (children.get(parentId) ?? 0) + 1);
11612
+ }
11613
+ const evidence = [];
11614
+ for (const [parentId, count] of children) {
11615
+ if (count > maxWidth) {
11616
+ evidence.push({ ruleId, path: parentId, count, threshold: maxWidth });
11617
+ }
11618
+ }
11619
+ if (evidence.length === 0) {
11620
+ return closed(ruleId, "Branch width within threshold.");
11621
+ }
11622
+ return open2(ruleId, "Excessive parallel branch width detected.", evidence, "warning");
11623
+ }
11624
+ function runRule(ruleId, events, options) {
11625
+ switch (ruleId) {
11626
+ case "circuit.same-tool-repetition":
11627
+ if (options.sameToolRepetition === void 0) return void 0;
11628
+ return evaluateSameToolRepetition(events, options.sameToolRepetition.maxRepeats);
11629
+ case "circuit.same-args-repetition":
11630
+ if (options.sameArgsRepetition === void 0) return void 0;
11631
+ return evaluateSameArgsRepetition(events, options.sameArgsRepetition.maxRepeats);
11632
+ case "circuit.max-loop-iterations":
11633
+ if (options.maxLoopIterations === void 0) return void 0;
11634
+ return evaluateMaxLoopIterations(events, options.maxLoopIterations.maxIterations);
11635
+ case "circuit.max-retries":
11636
+ if (options.maxRetries === void 0) return void 0;
11637
+ return evaluateMaxRetries(events, options.maxRetries.maxRetries);
11638
+ case "circuit.tool-timeout":
11639
+ if (options.toolTimeout === void 0) return void 0;
11640
+ return evaluateToolTimeout(events, options.toolTimeout.maxDurationMs);
11641
+ case "circuit.runaway-llm-loop":
11642
+ if (options.runawayLlmLoop === void 0) return void 0;
11643
+ return evaluateRunawayLlmLoop(events, options.runawayLlmLoop.maxLlmCalls);
11644
+ case "circuit.excessive-branch-width":
11645
+ if (options.excessiveBranchWidth === void 0) return void 0;
11646
+ return evaluateExcessiveBranchWidth(events, options.excessiveBranchWidth.maxWidth);
11647
+ default:
11648
+ return void 0;
11649
+ }
11650
+ }
11651
+ function runCircuits(events, options = {}) {
11652
+ const selected = options.rules ?? ALL_RULES;
11653
+ const results = [];
11654
+ for (const ruleId of selected) {
11655
+ const result = runRule(ruleId, events, options);
11656
+ if (result) results.push(result);
11657
+ }
11658
+ const ok = !results.some((result) => result.status === "open" && result.severity === "error");
11659
+ return { ok, results };
11660
+ }
11661
+
11662
+ // packages/guardrails/src/rules.ts
11663
+ var DEFAULT_INJECTION_PATTERNS = [
11664
+ "ignore previous instructions",
11665
+ "ignore all prior",
11666
+ "disregard your instructions",
11667
+ "system prompt",
11668
+ "you are now",
11669
+ "jailbreak"
11670
+ ];
11671
+ function pass(ruleId, message) {
11672
+ return { ruleId, status: "pass", severity: "info", message, evidence: [] };
11673
+ }
11674
+ function fail(ruleId, message, evidence, severity = "error") {
11675
+ return { ruleId, status: severity === "warning" ? "warn" : "fail", severity, message, evidence };
11676
+ }
11677
+ function boundedPreview(value, max = 80) {
11678
+ if (value.length <= max) return value;
11679
+ return `${value.slice(0, max - 3)}...`;
11680
+ }
11681
+ function evaluateBannedPhrase(text, options) {
11682
+ const ruleId = "guardrail.banned-phrase";
11683
+ const haystack = options.caseInsensitive !== false ? text.toLowerCase() : text;
11684
+ const evidence = [];
11685
+ for (const phrase of options.phrases) {
11686
+ const needle = options.caseInsensitive !== false ? phrase.toLowerCase() : phrase;
11687
+ if (needle.length > 0 && haystack.includes(needle)) {
11688
+ evidence.push({ ruleId, match: phrase, preview: boundedPreview(text) });
11689
+ }
11690
+ }
11691
+ if (evidence.length === 0) {
11692
+ return pass(ruleId, "No banned phrases matched.");
11693
+ }
11694
+ return fail(ruleId, `Matched ${evidence.length} banned phrase(s).`, evidence);
11695
+ }
11696
+ var SEVERITY_RANK2 = { info: 0, warning: 1, error: 2 };
11697
+ function evaluatePiiLeak(value, options = {}) {
11698
+ const ruleId = "guardrail.pii-leak";
11699
+ const minSeverity = options.minSeverity ?? "warning";
11700
+ const result = redact(value, { profile: options.profile ?? "share", collectFindings: true });
11701
+ const findings = result.findings.filter(
11702
+ (finding) => SEVERITY_RANK2[finding.severity] >= SEVERITY_RANK2[minSeverity]
11703
+ );
11704
+ if (findings.length === 0) {
11705
+ return pass(ruleId, "No PII-style redaction findings.");
11706
+ }
11707
+ const evidence = findings.map((finding) => ({
11708
+ ruleId,
11709
+ path: finding.path,
11710
+ detector: finding.detector,
11711
+ preview: finding.preview
11712
+ }));
11713
+ return fail(ruleId, `Detected ${findings.length} PII-style finding(s).`, evidence);
11714
+ }
11715
+ function measureDepth(value) {
11716
+ if (value === null || typeof value !== "object") return 0;
11717
+ if (Array.isArray(value)) {
11718
+ return 1 + Math.max(0, ...value.map((item) => measureDepth(item)));
11719
+ }
11720
+ const depths = Object.values(value).map((item) => measureDepth(item));
11721
+ return 1 + (depths.length === 0 ? 0 : Math.max(...depths));
11722
+ }
11723
+ function maxStringLength(value) {
11724
+ if (typeof value === "string") return value.length;
11725
+ if (value === null || typeof value !== "object") return 0;
11726
+ if (Array.isArray(value)) {
11727
+ return Math.max(0, ...value.map((item) => maxStringLength(item)));
11728
+ }
11729
+ return Math.max(0, ...Object.values(value).map((item) => maxStringLength(item)));
11730
+ }
11731
+ function evaluateUnsafeToolArgs(toolName2, toolArgs2, options = {}) {
11732
+ const ruleId = "guardrail.unsafe-tool-args";
11733
+ const blocked = new Set((options.blockedTools ?? []).map((name) => name.toLowerCase()));
11734
+ const evidence = [];
11735
+ if (blocked.has(toolName2.toLowerCase())) {
11736
+ evidence.push({ ruleId, preview: toolName2, match: toolName2 });
11737
+ }
11738
+ const maxDepth = options.maxDepth ?? 12;
11739
+ const depth = measureDepth(toolArgs2);
11740
+ if (depth > maxDepth) {
11741
+ evidence.push({ ruleId, path: "args", preview: `depth=${depth}` });
11742
+ }
11743
+ const maxLen = options.maxStringLength ?? 16384;
11744
+ const longest = maxStringLength(toolArgs2);
11745
+ if (longest > maxLen) {
11746
+ evidence.push({ ruleId, path: "args", preview: `maxStringLength=${longest}` });
11747
+ }
11748
+ if (evidence.length === 0) {
11749
+ return pass(ruleId, "Tool arguments within configured bounds.");
11750
+ }
11751
+ return fail(ruleId, "Unsafe or oversized tool arguments detected.", evidence);
11752
+ }
11753
+ function evaluatePromptInjection(text, options = {}) {
11754
+ const ruleId = "guardrail.prompt-injection";
11755
+ const patterns = options.patterns ?? DEFAULT_INJECTION_PATTERNS;
11756
+ const haystack = text.toLowerCase();
11757
+ const evidence = [];
11758
+ for (const pattern of patterns) {
11759
+ const needle = pattern.toLowerCase();
11760
+ if (needle.length > 0 && haystack.includes(needle)) {
11761
+ evidence.push({ ruleId, match: pattern, preview: boundedPreview(text) });
11762
+ }
11763
+ }
11764
+ if (evidence.length === 0) {
11765
+ return pass(ruleId, "No prompt-injection patterns matched.");
11766
+ }
11767
+ return fail(ruleId, `Matched ${evidence.length} injection pattern(s).`, evidence, "warning");
11768
+ }
11769
+ function validateSchemaField(value, field, path15, evidence) {
11770
+ const ruleId = "guardrail.structured-output";
11771
+ if (field.type) {
11772
+ const actual = value === null ? "null" : Array.isArray(value) ? "array" : typeof value;
11773
+ if (actual !== field.type) {
11774
+ evidence.push({ ruleId, path: path15, preview: `expected ${field.type}, got ${actual}` });
11775
+ return;
11776
+ }
11777
+ }
11778
+ if (field.enum && !field.enum.some((item) => Object.is(item, value))) {
11779
+ evidence.push({ ruleId, path: path15, preview: "value not in enum" });
11780
+ }
11781
+ if (field.type === "object" && field.required && value && typeof value === "object" && !Array.isArray(value)) {
11782
+ const record = value;
11783
+ for (const key of field.required) {
11784
+ if (!(key in record)) {
11785
+ evidence.push({ ruleId, path: `${path15}.${key}`, preview: "missing required key" });
11786
+ }
11787
+ }
11788
+ }
11789
+ }
11790
+ function evaluateStructuredOutput(value, options) {
11791
+ const ruleId = "guardrail.structured-output";
11792
+ if (!value || typeof value !== "object" || Array.isArray(value)) {
11793
+ return fail(ruleId, "Structured output must be an object.", [
11794
+ { ruleId, preview: typeof value }
11795
+ ]);
11796
+ }
11797
+ const record = value;
11798
+ const evidence = [];
11799
+ for (const [key, field] of Object.entries(options.schema)) {
11800
+ validateSchemaField(record[key], field, key, evidence);
11801
+ }
11802
+ if (evidence.length === 0) {
11803
+ return pass(ruleId, "Structured output matches schema subset.");
11804
+ }
11805
+ return fail(ruleId, "Structured output schema violation.", evidence);
11806
+ }
11807
+ function evaluateOversizeOutput(value, options = {}) {
11808
+ const ruleId = "guardrail.oversize-output";
11809
+ const text = typeof value === "string" ? value : JSON.stringify(value);
11810
+ const maxLength = options.maxLength ?? options.maxSerializedLength ?? 32768;
11811
+ if (text.length <= maxLength) {
11812
+ return pass(ruleId, "Output within size limits.");
11813
+ }
11814
+ return fail(ruleId, `Output exceeds max length (${text.length} > ${maxLength}).`, [
11815
+ { ruleId, preview: `length=${text.length}` }
11816
+ ]);
11817
+ }
11818
+ function evaluateRequiredJsonShape(value, options) {
11819
+ const ruleId = "guardrail.required-json-shape";
11820
+ let parsed = value;
11821
+ if (typeof value === "string") {
11822
+ try {
11823
+ parsed = JSON.parse(value);
11824
+ } catch {
11825
+ return fail(ruleId, "Value is not valid JSON.", [{ ruleId, preview: boundedPreview(value) }]);
11826
+ }
11827
+ }
11828
+ if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) {
11829
+ return fail(ruleId, "JSON value must be an object.", [{ ruleId, preview: typeof parsed }]);
11830
+ }
11831
+ const record = parsed;
11832
+ const evidence = [];
11833
+ for (const key of options.requiredKeys) {
11834
+ if (!(key in record)) {
11835
+ evidence.push({ ruleId, path: key, preview: "missing required key" });
11836
+ }
11837
+ }
11838
+ if (evidence.length === 0) {
11839
+ return pass(ruleId, "Required JSON keys present.");
11840
+ }
11841
+ return fail(ruleId, "Missing required JSON keys.", evidence);
11842
+ }
11843
+
11844
+ // packages/guardrails/src/run.ts
11845
+ var ALL_RULES2 = [
11846
+ "guardrail.banned-phrase",
11847
+ "guardrail.pii-leak",
11848
+ "guardrail.unsafe-tool-args",
11849
+ "guardrail.prompt-injection",
11850
+ "guardrail.structured-output",
11851
+ "guardrail.oversize-output",
11852
+ "guardrail.required-json-shape"
11853
+ ];
11854
+ function isErrorFailure(result) {
11855
+ return result.status === "fail" && result.severity === "error";
11856
+ }
11857
+ function runRule2(ruleId, input3, options) {
11858
+ switch (ruleId) {
11859
+ case "guardrail.banned-phrase": {
11860
+ if (!options.bannedPhrase || input3.text === void 0) return void 0;
11861
+ return evaluateBannedPhrase(input3.text, options.bannedPhrase);
11862
+ }
11863
+ case "guardrail.pii-leak": {
11864
+ if (input3.value === void 0 && input3.text === void 0) return void 0;
11865
+ return evaluatePiiLeak(input3.value ?? input3.text, options.piiLeak);
11866
+ }
11867
+ case "guardrail.unsafe-tool-args": {
11868
+ if (!input3.toolName) return void 0;
11869
+ return evaluateUnsafeToolArgs(input3.toolName, input3.toolArgs ?? {}, options.unsafeToolArgs);
11870
+ }
11871
+ case "guardrail.prompt-injection": {
11872
+ if (input3.text === void 0) return void 0;
11873
+ return evaluatePromptInjection(input3.text, options.promptInjection);
11874
+ }
11875
+ case "guardrail.structured-output": {
11876
+ if (!options.structuredOutput || input3.value === void 0) return void 0;
11877
+ return evaluateStructuredOutput(input3.value, options.structuredOutput);
11878
+ }
11879
+ case "guardrail.oversize-output": {
11880
+ if (input3.value === void 0 && input3.text === void 0) return void 0;
11881
+ return evaluateOversizeOutput(input3.value ?? input3.text, options.oversizeOutput);
11882
+ }
11883
+ case "guardrail.required-json-shape": {
11884
+ if (!options.requiredJsonShape || input3.value === void 0 && input3.text === void 0) return void 0;
11885
+ return evaluateRequiredJsonShape(input3.value ?? input3.text, options.requiredJsonShape);
11886
+ }
11887
+ default:
11888
+ return void 0;
11889
+ }
11890
+ }
11891
+ function runGuardrails(input3, options = {}) {
11892
+ const selected = options.rules ?? ALL_RULES2;
11893
+ const results = [];
11894
+ for (const ruleId of selected) {
11895
+ const result = runRule2(ruleId, input3, options);
11896
+ if (result) results.push(result);
11897
+ }
11898
+ const ok = !results.some(isErrorFailure);
11899
+ return { ok, results };
11900
+ }
11901
+
11902
+ // packages/cli/src/safety-extensions.ts
11903
+ var GUARDRAIL_ALIASES = {
11904
+ "banned-phrase": "guardrail.banned-phrase",
11905
+ "pii-leak": "guardrail.pii-leak",
11906
+ "unsafe-tool-args": "guardrail.unsafe-tool-args",
11907
+ "prompt-injection": "guardrail.prompt-injection",
11908
+ "structured-output": "guardrail.structured-output",
11909
+ "oversize-output": "guardrail.oversize-output",
11910
+ "required-json-shape": "guardrail.required-json-shape"
11911
+ };
11912
+ var CIRCUIT_ALIASES = {
11913
+ "same-tool-repetition": "circuit.same-tool-repetition",
11914
+ "same-args-repetition": "circuit.same-args-repetition",
11915
+ "max-loop-iterations": "circuit.max-loop-iterations",
11916
+ "max-retries": "circuit.max-retries",
11917
+ "tool-timeout": "circuit.tool-timeout",
11918
+ "runaway-llm-loop": "circuit.runaway-llm-loop",
11919
+ "excessive-branch-width": "circuit.excessive-branch-width"
11920
+ };
11921
+ function parseGuardrailRules(values) {
11922
+ if (!values?.length) return void 0;
11923
+ return values.map((value) => {
11924
+ const rule = GUARDRAIL_ALIASES[value] ?? value;
11925
+ return rule;
11926
+ });
11927
+ }
11928
+ function parseCircuitRules(values) {
11929
+ if (!values?.length) return void 0;
11930
+ return values.map((value) => {
11931
+ const rule = CIRCUIT_ALIASES[value] ?? value;
11932
+ return rule;
11933
+ });
11934
+ }
11935
+ function toSeverity(severity) {
11936
+ return severity;
11937
+ }
11938
+ function guardrailFinding(result) {
11939
+ return {
11940
+ ruleId: result.ruleId,
11941
+ severity: toSeverity(result.severity),
11942
+ status: result.status === "pass" ? "pass" : result.status === "warn" ? "warning" : "fail",
11943
+ message: result.message,
11944
+ evidence: result.evidence.map(
11945
+ (item) => ({
11946
+ path: item.preview ? `${item.path ?? "value"} (${item.preview})` : item.path
11947
+ })
11948
+ )
11949
+ };
11950
+ }
11951
+ function circuitFinding(result) {
11952
+ return {
11953
+ ruleId: result.ruleId,
11954
+ severity: toSeverity(result.severity),
11955
+ status: result.status === "closed" ? "pass" : result.status === "warn" ? "warning" : "fail",
11956
+ message: result.message,
11957
+ evidence: result.evidence.map(
11958
+ (item) => ({
11959
+ runId: item.runId,
11960
+ eventId: item.eventId,
11961
+ path: item.path,
11962
+ name: item.toolName
11963
+ })
11964
+ ),
11965
+ actual: result.evidence[0]?.count,
11966
+ expected: result.evidence[0]?.threshold
11967
+ };
11968
+ }
11969
+ function eventToCircuit(event) {
11970
+ return {
11971
+ eventId: event.eventId,
11972
+ runId: event.runId,
11973
+ name: event.name,
11974
+ kind: event.kind,
11975
+ parentId: event.parentId,
11976
+ startedAt: event.startedAt,
11977
+ endedAt: event.endedAt,
11978
+ durationMs: event.durationMs,
11979
+ attributes: event.attributes,
11980
+ status: event.status
11981
+ };
11982
+ }
11983
+ function collectGuardrailInputs(read) {
11984
+ const inputs = [];
11985
+ for (const event of read.events) {
11986
+ const attrs = event.attributes ?? {};
11987
+ for (const key of ["output", "answer", "text", "content", "message"]) {
11988
+ const value = attrs[key];
11989
+ if (typeof value === "string") inputs.push({ text: value });
11990
+ else if (value !== void 0) inputs.push({ value });
11991
+ }
11992
+ if (event.kind === "TOOL" || event.name.startsWith("tool:")) {
11993
+ inputs.push({
11994
+ toolName: String(attrs.toolName ?? attrs.tool ?? event.name),
11995
+ toolArgs: attrs.arguments ?? attrs.args ?? attrs.input
11996
+ });
11997
+ }
11998
+ }
11999
+ return inputs;
12000
+ }
12001
+ var DEFAULT_GUARDRAIL_OPTIONS = {
12002
+ bannedPhrase: { phrases: ["delete all data", "ignore all instructions"] },
12003
+ promptInjection: {},
12004
+ piiLeak: { profile: "share" }
12005
+ };
12006
+ var DEFAULT_CIRCUIT_OPTIONS = {
12007
+ sameToolRepetition: { maxRepeats: 3 },
12008
+ sameArgsRepetition: { maxRepeats: 2 },
12009
+ maxLoopIterations: { maxIterations: 20 },
12010
+ maxRetries: { maxRetries: 3 },
12011
+ toolTimeout: { maxDurationMs: 6e4 },
12012
+ runawayLlmLoop: { maxLlmCalls: 12 },
12013
+ excessiveBranchWidth: { maxWidth: 8 }
12014
+ };
12015
+ function mergeSafetyExtensions(result, read, options) {
12016
+ const findings = [...result.findings];
12017
+ let failed = result.summary.failed;
12018
+ let warnings = result.summary.warnings;
12019
+ let passed = result.summary.passed;
12020
+ const guardrailRules = parseGuardrailRules(options.guardrails);
12021
+ if (guardrailRules) {
12022
+ for (const input3 of collectGuardrailInputs(read)) {
12023
+ const run = runGuardrails(input3, {
12024
+ ...DEFAULT_GUARDRAIL_OPTIONS,
12025
+ rules: guardrailRules
12026
+ });
12027
+ for (const item of run.results) {
12028
+ const finding = guardrailFinding(item);
12029
+ findings.push(finding);
12030
+ if (finding.status === "fail") failed += 1;
12031
+ else if (finding.status === "warning") warnings += 1;
12032
+ else passed += 1;
12033
+ }
12034
+ }
12035
+ }
12036
+ const circuitRules = parseCircuitRules(options.circuits);
12037
+ if (circuitRules) {
12038
+ const circuitRun = runCircuits(
12039
+ read.events.map(eventToCircuit),
12040
+ { ...DEFAULT_CIRCUIT_OPTIONS, rules: circuitRules }
12041
+ );
12042
+ for (const item of circuitRun.results) {
12043
+ const finding = circuitFinding(item);
12044
+ findings.push(finding);
12045
+ if (finding.status === "fail") failed += 1;
12046
+ else if (finding.status === "warning") warnings += 1;
12047
+ else passed += 1;
12048
+ }
12049
+ }
12050
+ const ok = failed === 0 && result.summary.errors === 0;
12051
+ return {
12052
+ ...result,
12053
+ ok,
12054
+ status: failed > 0 ? "fail" : result.status,
12055
+ summary: {
12056
+ passed,
12057
+ failed,
12058
+ warnings,
12059
+ errors: result.summary.errors
12060
+ },
12061
+ findings
12062
+ };
12063
+ }
11449
12064
 
11450
12065
  // packages/cli/src/check.ts
11451
12066
  var DEFAULT_SELECT = ["run.status"];
@@ -11685,12 +12300,19 @@ async function checkCommand(target, options = {}, stdin = process.stdin) {
11685
12300
  }
11686
12301
  );
11687
12302
  perRun.push(
11688
- runTraceChecks(
11689
- { read },
12303
+ mergeSafetyExtensions(
12304
+ runTraceChecks(
12305
+ { read },
12306
+ {
12307
+ rules: built.rules,
12308
+ select: built.select,
12309
+ runId: meta.runId
12310
+ }
12311
+ ),
12312
+ read,
11690
12313
  {
11691
- rules: built.rules,
11692
- select: built.select,
11693
- runId: meta.runId
12314
+ ...options.guardrails ? { guardrails: options.guardrails } : {},
12315
+ ...options.circuit ? { circuits: options.circuit } : {}
11694
12316
  }
11695
12317
  )
11696
12318
  );
@@ -11709,12 +12331,19 @@ async function checkCommand(target, options = {}, stdin = process.stdin) {
11709
12331
  const read = await openTrace(input3, {
11710
12332
  ...options.format !== void 0 ? { format: options.format } : {}
11711
12333
  });
11712
- result = runTraceChecks(
11713
- { read },
12334
+ result = mergeSafetyExtensions(
12335
+ runTraceChecks(
12336
+ { read },
12337
+ {
12338
+ rules: built.rules,
12339
+ select: built.select,
12340
+ ...options.run !== void 0 ? { runId: options.run } : {}
12341
+ }
12342
+ ),
12343
+ read,
11714
12344
  {
11715
- rules: built.rules,
11716
- select: built.select,
11717
- ...options.run !== void 0 ? { runId: options.run } : {}
12345
+ ...options.guardrails ? { guardrails: options.guardrails } : {},
12346
+ ...options.circuit ? { circuits: options.circuit } : {}
11718
12347
  }
11719
12348
  );
11720
12349
  }
@@ -11918,7 +12547,7 @@ function evidenceForEvent(event, path15) {
11918
12547
  }
11919
12548
  ];
11920
12549
  }
11921
- function fail(ruleId, message, evidence, expected, actual) {
12550
+ function fail2(ruleId, message, evidence, expected, actual) {
11922
12551
  return {
11923
12552
  ruleId,
11924
12553
  status: "fail",
@@ -12114,7 +12743,7 @@ var checks = {
12114
12743
  "eval.requireSuccess",
12115
12744
  "run",
12116
12745
  (context) => context.run.status === "ok" ? [] : [
12117
- fail(
12746
+ fail2(
12118
12747
  "eval.requireSuccess",
12119
12748
  "Run did not complete successfully.",
12120
12749
  evidenceForRun(context.run, "status"),
@@ -12129,7 +12758,7 @@ var checks = {
12129
12758
  return createRule("eval.requiredTools", "tool", (context) => {
12130
12759
  const tools = new Set(nodeNames(context.nodes, "TOOL"));
12131
12760
  return expected.filter((name) => !tools.has(name)).map(
12132
- (name) => fail(
12761
+ (name) => fail2(
12133
12762
  "eval.requiredTools",
12134
12763
  `Required tool ${name} did not appear.`,
12135
12764
  evidenceForRun(context.run, "children"),
@@ -12145,7 +12774,7 @@ var checks = {
12145
12774
  "eval.forbiddenTools",
12146
12775
  "tool",
12147
12776
  (context) => context.nodes.filter((node) => node.event.kind === "TOOL" && blocked.includes(node.event.name)).map(
12148
- (node) => fail(
12777
+ (node) => fail2(
12149
12778
  "eval.forbiddenTools",
12150
12779
  `Forbidden tool ${node.event.name} appeared.`,
12151
12780
  evidenceForEvent(node.event, "name"),
@@ -12160,7 +12789,7 @@ var checks = {
12160
12789
  "eval.maxDurationMs",
12161
12790
  "run",
12162
12791
  (context) => context.run.durationMs !== void 0 && context.run.durationMs > maxDurationMs ? [
12163
- fail(
12792
+ fail2(
12164
12793
  "eval.maxDurationMs",
12165
12794
  `Run duration exceeded ${maxDurationMs}ms.`,
12166
12795
  evidenceForRun(context.run, "durationMs"),
@@ -12177,7 +12806,7 @@ var checks = {
12177
12806
  void 0
12178
12807
  );
12179
12808
  return deepest !== void 0 && deepest.depth > maxDepth ? [
12180
- fail(
12809
+ fail2(
12181
12810
  "eval.maxDepth",
12182
12811
  `Run tree depth exceeded ${maxDepth}.`,
12183
12812
  evidenceForEvent(deepest.event, "depth"),
@@ -12194,7 +12823,7 @@ var checks = {
12194
12823
  (context) => context.nodes.flatMap((node) => {
12195
12824
  const retries = numericAttribute(node, ["retryCount", "retries", "attempt"]);
12196
12825
  return retries !== void 0 && retries > maxRetries ? [
12197
- fail(
12826
+ fail2(
12198
12827
  "eval.maxRetries",
12199
12828
  `Retry count exceeded ${maxRetries}.`,
12200
12829
  evidenceForEvent(node.event, "attributes.retryCount"),
@@ -12209,7 +12838,7 @@ var checks = {
12209
12838
  return createRule("eval.maxTotalTokens", "llm", (context) => {
12210
12839
  const total = totalTokenCount(context.events);
12211
12840
  return total > maxTotalTokens ? [
12212
- fail(
12841
+ fail2(
12213
12842
  "eval.maxTotalTokens",
12214
12843
  `Total token usage exceeded ${maxTotalTokens}.`,
12215
12844
  evidenceForRun(context.run, "tokenUsage.total"),
@@ -12224,7 +12853,7 @@ var checks = {
12224
12853
  "eval.noFailedSteps",
12225
12854
  "run",
12226
12855
  (context) => context.nodes.filter((node) => node.event.status === "error" || node.event.kind === "ERROR").map(
12227
- (node) => fail(
12856
+ (node) => fail2(
12228
12857
  "eval.noFailedSteps",
12229
12858
  "Run contains a failed step or error node.",
12230
12859
  evidenceForEvent(node.event, "status"),
@@ -12242,7 +12871,7 @@ var checks = {
12242
12871
  (node, index) => index < firstLlmIndex && node.event.kind === "RETRIEVER"
12243
12872
  );
12244
12873
  return retrievalIndex === -1 ? [
12245
- fail(
12874
+ fail2(
12246
12875
  "eval.requiredRetrievalBeforeGeneration",
12247
12876
  "No retrieval step appeared before the first LLM generation.",
12248
12877
  evidenceForEvent(context.nodes[firstLlmIndex].event, "kind"),
@@ -12258,7 +12887,7 @@ var checks = {
12258
12887
  const decisions = context.nodes.filter((node) => node.event.kind === "DECISION");
12259
12888
  if (decisions.length === 0) {
12260
12889
  return [
12261
- fail(
12890
+ fail2(
12262
12891
  "eval.requiredDecisionMetadata",
12263
12892
  "No decision node is available for required metadata.",
12264
12893
  evidenceForRun(context.run, "children"),
@@ -12269,7 +12898,7 @@ var checks = {
12269
12898
  }
12270
12899
  return decisions.flatMap(
12271
12900
  (node) => required.filter((key) => !hasAttribute(node, key)).map(
12272
- (key) => fail(
12901
+ (key) => fail2(
12273
12902
  "eval.requiredDecisionMetadata",
12274
12903
  `Decision metadata ${key} is missing.`,
12275
12904
  evidenceForEvent(node.event, `attributes.${key}`),
@@ -12290,7 +12919,7 @@ var checks = {
12290
12919
  const contexts = collectTextFields(context.nodes, contextKeys, ["RETRIEVER", "TOOL"]);
12291
12920
  if (answers.length === 0 || contexts.length === 0) {
12292
12921
  return [
12293
- fail(
12922
+ fail2(
12294
12923
  "eval.contextOverlap",
12295
12924
  "Answer and context text are required for overlap evaluation.",
12296
12925
  firstEvidence(answers.length > 0 ? answers : contexts, context.run, "children"),
@@ -12304,7 +12933,7 @@ var checks = {
12304
12933
  const sharedTerms = [...answerTerms].filter((term) => contextTerms.has(term)).length;
12305
12934
  const overlap = answerTerms.size === 0 ? 0 : sharedTerms / answerTerms.size;
12306
12935
  return sharedTerms < minSharedTerms || overlap < minOverlap ? [
12307
- fail(
12936
+ fail2(
12308
12937
  "eval.contextOverlap",
12309
12938
  "Answer text did not sufficiently overlap retrieved context.",
12310
12939
  firstEvidence(answers, context.run, "attributes.answer"),
@@ -12332,7 +12961,7 @@ var checks = {
12332
12961
  const quotes = quotedSnippets(answerText, minQuoteLength);
12333
12962
  if (quotes.length === 0) {
12334
12963
  return requireQuote ? [
12335
- fail(
12964
+ fail2(
12336
12965
  "eval.quoteOverlap",
12337
12966
  "Answer did not contain a quote for overlap evaluation.",
12338
12967
  firstEvidence(answers, context.run, "attributes.answer"),
@@ -12343,7 +12972,7 @@ var checks = {
12343
12972
  }
12344
12973
  const missing = quotes.filter((quote) => !contextText.includes(quote.toLowerCase()));
12345
12974
  return missing.length > 0 ? [
12346
- fail(
12975
+ fail2(
12347
12976
  "eval.quoteOverlap",
12348
12977
  "Quoted answer text did not appear in retrieved context.",
12349
12978
  firstEvidence(answers, context.run, "attributes.answer"),
@@ -12361,7 +12990,7 @@ var checks = {
12361
12990
  const citations = collectTextFields(context.nodes, citationKeys);
12362
12991
  const count = citationCount(answers.map((field) => field.text).join(" "), citations);
12363
12992
  return count === 0 ? [
12364
- fail(
12993
+ fail2(
12365
12994
  "eval.citationPresence",
12366
12995
  "Answer did not include citations or source references.",
12367
12996
  firstEvidence(answers, context.run, "attributes.answer"),
@@ -12379,7 +13008,7 @@ var checks = {
12379
13008
  const availableSet = new Set(available);
12380
13009
  const missing = expected.filter((id) => !availableSet.has(id));
12381
13010
  return missing.length > 0 ? [
12382
- fail(
13011
+ fail2(
12383
13012
  "eval.requiredSourceIds",
12384
13013
  "Required source IDs were not present in trace context or citations.",
12385
13014
  evidenceForRun(context.run, "children"),
@@ -12399,7 +13028,7 @@ var checks = {
12399
13028
  const tooShort = options.minCharacters !== void 0 && characters < options.minCharacters || options.minWords !== void 0 && words < options.minWords;
12400
13029
  const tooLong = options.maxCharacters !== void 0 && characters > options.maxCharacters || options.maxWords !== void 0 && words > options.maxWords;
12401
13030
  return answer.length === 0 || tooShort || tooLong ? [
12402
- fail(
13031
+ fail2(
12403
13032
  "eval.answerLengthBounds",
12404
13033
  "Answer length fell outside required bounds.",
12405
13034
  firstEvidence(answers, context.run, "attributes.answer"),
@@ -12422,7 +13051,7 @@ var checks = {
12422
13051
  const answer = answers.map((field) => field.text).join(" ").toLowerCase();
12423
13052
  const matches = banned.filter((phrase) => answer.includes(phrase));
12424
13053
  return matches.length > 0 ? [
12425
- fail(
13054
+ fail2(
12426
13055
  "eval.bannedUnsupportedPhrases",
12427
13056
  "Answer contained banned unsupported-answer phrasing.",
12428
13057
  firstEvidence(answers, context.run, "attributes.answer"),
@@ -12800,7 +13429,7 @@ function invalidArgumentResult(command, error) {
12800
13429
  });
12801
13430
  }
12802
13431
  function buildSafetyRules(options) {
12803
- const maxStringLength = parseLimit3(options.maxStringLength, "--max-string-length") ?? DEFAULT_MAX_STRING_LENGTH;
13432
+ const maxStringLength2 = parseLimit3(options.maxStringLength, "--max-string-length") ?? DEFAULT_MAX_STRING_LENGTH;
12804
13433
  const maxArrayLength = parseLimit3(options.maxArrayLength, "--max-array-length") ?? DEFAULT_MAX_ARRAY_LENGTH;
12805
13434
  const maxObjectKeys = parseLimit3(options.maxObjectKeys, "--max-object-keys") ?? DEFAULT_MAX_OBJECT_KEYS;
12806
13435
  const maxSerializedBytes = parseLimit3(options.maxSerializedBytes, "--max-serialized-bytes") ?? DEFAULT_MAX_SERIALIZED_BYTES;
@@ -12809,7 +13438,7 @@ function buildSafetyRules(options) {
12809
13438
  createSafetyRedactionRule(),
12810
13439
  createSafetySecretPatternRule(),
12811
13440
  createSafetyOversizedAttributeRule({
12812
- maxStringLength,
13441
+ maxStringLength: maxStringLength2,
12813
13442
  maxArrayLength,
12814
13443
  maxObjectKeys,
12815
13444
  maxSerializedBytes
@@ -13693,6 +14322,14 @@ function createCliProgram() {
13693
14322
  ]).option("--max-total-tokens <number>", "add llm.usage with a max total-token budget").option("--session <id>", "check all runs in a workflow session (requires --dir)").option("--group <id>", "check all runs sharing a groupId (requires --dir)").option(
13694
14323
  "--correlate-group",
13695
14324
  "when using --session, also match synthetic group: session keys"
14325
+ ).option(
14326
+ "--guardrails <rule>",
14327
+ "run optional guardrail rules (repeatable): banned-phrase, pii-leak, prompt-injection, ...",
14328
+ (value, previous = []) => [...previous, value]
14329
+ ).option(
14330
+ "--circuit <rule>",
14331
+ "run optional circuit rules (repeatable): same-tool-repetition, max-retries, ...",
14332
+ (value, previous = []) => [...previous, value]
13696
14333
  ).action((target, opts) => {
13697
14334
  runCommand(() => checkCommand(target, opts));
13698
14335
  });