agent-inspect 6.29.4 → 6.29.6

package/packages/cli/dist/index.cjs

@@ -330,6 +330,28 @@ var init_types = __esm({
   }
 });
 
+// packages/core/src/types/inspect-event.ts
+function isAttributionConfidence(value) {
+  return typeof value === "string" && ATTRIBUTION_CONFIDENCES.includes(value);
+}
+var ATTRIBUTION_CONFIDENCES, ATTRIBUTION_CONFIDENCE_RANK;
+var init_inspect_event = __esm({
+  "packages/core/src/types/inspect-event.ts"() {
+    ATTRIBUTION_CONFIDENCES = [
+      "unknown",
+      "heuristic",
+      "correlated",
+      "explicit"
+    ];
+    ATTRIBUTION_CONFIDENCE_RANK = {
+      unknown: 0,
+      heuristic: 1,
+      correlated: 2,
+      explicit: 3
+    };
+  }
+});
+
 // packages/core/src/types/persisted-inspect-event.ts
 function isRecord3(value) {
   return typeof value === "object" && value !== null && !Array.isArray(value);
@@ -352,9 +374,6 @@ function isOptionalNonNegativeNumber(value) {
 function isInspectKind(value) {
   return typeof value === "string" && INSPECT_KINDS.includes(value);
 }
-function isAttributionConfidence(value) {
-  return typeof value === "string" && ATTRIBUTION_CONFIDENCES.includes(value);
-}
 function isPersistedEventSourceType(value) {
   return typeof value === "string" && PERSISTED_EVENT_SOURCE_TYPES.includes(value);
 }
@@ -429,9 +448,10 @@ function isPersistedInspectEvent(value) {
   }
   return true;
 }
-var INSPECT_KINDS, ATTRIBUTION_CONFIDENCES, PERSISTED_EVENT_SOURCE_TYPES, PERSISTED_EVENT_STATUSES;
+var INSPECT_KINDS, PERSISTED_EVENT_SOURCE_TYPES, PERSISTED_EVENT_STATUSES;
 var init_persisted_inspect_event = __esm({
   "packages/core/src/types/persisted-inspect-event.ts"() {
+    init_inspect_event();
     INSPECT_KINDS = [
       "RUN",
       "AGENT",
@@ -446,12 +466,6 @@ var init_persisted_inspect_event = __esm({
       "LOG",
       "OUTCOME"
     ];
-    ATTRIBUTION_CONFIDENCES = [
-      "explicit",
-      "correlated",
-      "heuristic",
-      "unknown"
-    ];
     PERSISTED_EVENT_SOURCE_TYPES = [
       "manual",
       "json-log",
@@ -5442,7 +5456,7 @@ function flattenNodes(nodes) {
 function buildFacts2(input3, selectedRun) {
   const scopedRuns = selectedRun ? [selectedRun] : input3.read.runs;
   const scopedRunIds = new Set(scopedRuns.map((run) => run.runId));
-  const scopedEvents = selectedRun === void 0 ? input3.read.events : input3.read.events.filter((event) => scopedRunIds.has(event.runId));
+  const scopedEvents2 = selectedRun === void 0 ? input3.read.events : input3.read.events.filter((event) => scopedRunIds.has(event.runId));
   const nodes = flattenNodes(scopedRuns.flatMap((run) => run.children));
   const nodesByEventId = /* @__PURE__ */ new Map();
   const childrenByParentId = /* @__PURE__ */ new Map();
@@ -5455,11 +5469,11 @@ function buildFacts2(input3, selectedRun) {
       childrenByParentId.set(parentId, children);
     }
   }
-  const projection = projectLogicalEvents(scopedEvents);
+  const projection = projectLogicalEvents(scopedEvents2);
   return {
     format: input3.read.format,
     runs: Object.freeze([...input3.read.runs]),
-    events: Object.freeze([...scopedEvents]),
+    events: Object.freeze([...scopedEvents2]),
     logicalEvents: projection.logicalEvents,
     logicalProjectionDiagnostics: projection.diagnostics,
     readerWarnings: Object.freeze([...input3.read.warnings]),
@@ -5800,6 +5814,18 @@ function limitFindings(findings, maxFindings) {
 function hasRedactionMarker(value, markers) {
   return markers.some((marker) => value.includes(marker)) || /^\[HASH:[A-Za-z0-9_-]+\]$/.test(value);
 }
+function isFullyRedactedValue(value, markers = ["[REDACTED]", "[REDACTED:"]) {
+  if (typeof value !== "string") return false;
+  const trimmed = value.trim();
+  if (trimmed.length === 0) return false;
+  if (/^\[HASH:[A-Za-z0-9_-]+\]$/.test(trimmed)) return true;
+  if (/^\[REDACTED:[^\]]*\]$/.test(trimmed)) return true;
+  for (const marker of markers) {
+    if (marker.endsWith(":")) continue;
+    if (trimmed === marker) return true;
+  }
+  return false;
+}
 function isSensitiveKey(key, sensitiveKeys) {
   return isCredentialSensitiveKey(key, sensitiveKeys);
 }
@@ -6118,8 +6144,26 @@ function createToolUsageRule(options) {
       const findings = [];
       for (const required of options.required ?? []) {
         if (!nameSet.has(required)) {
+          const otherKinds = [
+            ...new Set(
+              context.events.filter((event) => {
+                const kind = typeof event.kind === "string" ? event.kind.toUpperCase() : "";
+                if (kind === "TOOL") return false;
+                return resolveCanonicalToolName(event) === required || event.name === required || event.name === `tool:${required}` || event.name.endsWith(`:${required}`);
+              }).map(
+                (event) => typeof event.kind === "string" ? event.kind.toUpperCase() : "UNKNOWN"
+              )
+            )
+          ].sort();
+          const hint = otherKinds.length > 0 ? ` Required name appears under non-TOOL kind(s): ${otherKinds.join(", ")}. tools.required / requiredOrder match TOOL events only.` : "";
           findings.push(
-            failFinding("tool.usage", `Required tool ${required} did not appear.`, runEvidence(context.selectedRun), required, names)
+            failFinding(
+              "tool.usage",
+              `Required tool ${required} did not appear.${hint}`,
+              runEvidence(context.selectedRun),
+              required,
+              names
+            )
           );
         }
       }
@@ -6293,6 +6337,12 @@ function createStructureCycleRule() {
   };
 }
 function createStructureRelationshipRule(options = {}) {
+  if (options.minConfidence !== void 0 && !isAttributionConfidence(options.minConfidence)) {
+    throw new TypeError(
+      `createStructureRelationshipRule: minConfidence must be one of: ${ATTRIBUTION_CONFIDENCES.join(", ")}.`
+    );
+  }
+  const minConfidence = options.minConfidence;
   return {
     id: "structure.relationship",
     category: "structure",
@@ -6300,18 +6350,21 @@ function createStructureRelationshipRule(options = {}) {
     evaluate(context) {
       const byId = eventMap(semanticEvents(context));
       const findings = [];
-      const minConfidence = options.minConfidence;
       for (const event of semanticEvents(context)) {
-        if (minConfidence && CONFIDENCE_RANK[event.confidence] < CONFIDENCE_RANK[minConfidence]) {
-          findings.push(
-            failFinding(
-              "structure.relationship",
-              `Event confidence ${event.confidence} is below ${minConfidence}.`,
-              [eventEvidence(event, "confidence")],
-              { minConfidence },
-              event.confidence
-            )
-          );
+        if (minConfidence) {
+          const eventRank = CONFIDENCE_RANK[event.confidence] ?? CONFIDENCE_RANK.unknown;
+          const minRank = CONFIDENCE_RANK[minConfidence];
+          if (eventRank < minRank) {
+            findings.push(
+              failFinding(
+                "structure.relationship",
+                `Event confidence ${event.confidence} is below ${minConfidence}.`,
+                [eventEvidence(event, "confidence")],
+                { minConfidence },
+                event.confidence
+              )
+            );
+          }
         }
         if (!event.parentId) continue;
         if (event.parentId === event.eventId) {
@@ -6475,6 +6528,7 @@ function createSafetyRedactionRule(options = {}) {
 function createSafetyRawContentRule(options = {}) {
   const forbiddenKeys = options.forbiddenKeys ?? DEFAULT_RAW_CONTENT_KEYS;
   const safePathPrefixes = options.safePathPrefixes ?? DEFAULT_SAFE_RAW_CONTENT_PATH_PREFIXES;
+  const markers = options.redactedMarkers ?? ["[REDACTED]", "[REDACTED:"];
   return {
     id: "safety.rawPrompt",
     category: "safety",
@@ -6485,6 +6539,7 @@ function createSafetyRawContentRule(options = {}) {
         for (const entry of eventValueEntries(event, { includeSummaries: options.includeSummaries })) {
           const key = entry.key ?? lastPathSegment(entry.path);
           if (!isRawContentPath(entry.path, key, forbiddenKeys, safePathPrefixes)) continue;
+          if (isFullyRedactedValue(entry.value, markers)) continue;
           findings.push(
             failFinding(
               "safety.rawPrompt",
@@ -6882,6 +6937,7 @@ function runTraceChecks(input3, options = {}) {
 var SEVERITY_RANK, STATUS_RANK, CONFIDENCE_RANK, DEFAULT_SENSITIVE_KEYS, DEFAULT_RAW_CONTENT_KEYS, DEFAULT_SAFE_RAW_CONTENT_PATH_PREFIXES, SAFE_USAGE_LEAF_KEYS, DEFAULT_SECRET_PATTERNS;
 var init_checks2 = __esm({
   "packages/core/src/checks/index.ts"() {
+    init_inspect_event();
     init_programmatic();
     init_outcomes();
     init_sensitive_key();
@@ -6901,12 +6957,7 @@ var init_checks2 = __esm({
       warning: 1,
       pass: 2
     };
-    CONFIDENCE_RANK = {
-      unknown: 0,
-      heuristic: 1,
-      correlated: 2,
-      explicit: 3
-    };
+    CONFIDENCE_RANK = ATTRIBUTION_CONFIDENCE_RANK;
     DEFAULT_SENSITIVE_KEYS = DEFAULT_CREDENTIAL_SENSITIVE_KEYS;
     DEFAULT_RAW_CONTENT_KEYS = [
       "body",
@@ -13273,6 +13324,7 @@ var init_advanced = __esm({
     init_terminal();
     init_utils();
     init_types();
+    init_inspect_event();
     init_storage();
     init_read_trace();
     init_trace_directory();
@@ -13759,7 +13811,7 @@ var init_src = __esm({
 });
 
 // package.json
-var version = "6.29.4";
+var version = "6.29.6";
 
 // packages/cli/src/list.ts
 init_advanced();
@@ -18439,6 +18491,7 @@ async function loadRedactionPolicy(policyPath) {
 
 // packages/core/src/entries/checks.ts
 init_checks2();
+init_inspect_event();
 
 // packages/cli/src/trace-input.ts
 init_advanced();
@@ -19807,12 +19860,22 @@ function open2(ruleId, message, evidence, severity = "error") {
   };
 }
 function isToolEvent(event) {
+  const kind = typeof event.kind === "string" ? event.kind.toLowerCase() : "";
+  if (kind === "tool") return true;
+  if (kind === "llm" || kind === "run" || kind === "agent" || kind === "logic" || kind === "log" || kind === "outcome" || kind === "chain" || kind === "retriever" || kind === "decision" || kind === "result" || kind === "error") {
+    return false;
+  }
   const name = event.name.toLowerCase();
-  return event.kind === "tool" || name.startsWith("tool:") || name.startsWith("function:") || name.includes(".tool.") || name.startsWith("mcp:");
+  return name.startsWith("tool:") || name.startsWith("function:") || name.includes(".tool.") || name.startsWith("mcp:");
 }
 function isLlmEvent(event) {
+  const kind = typeof event.kind === "string" ? event.kind.toLowerCase() : "";
+  if (kind === "llm") return true;
+  if (kind === "tool" || kind === "run" || kind === "agent" || kind === "logic" || kind === "log" || kind === "outcome" || kind === "chain" || kind === "retriever" || kind === "decision" || kind === "result" || kind === "error") {
+    return false;
+  }
   const name = event.name.toLowerCase();
-  return event.kind === "llm" || name.startsWith("llm:") || name.includes(".llm.") || name.includes("generation");
+  return name.startsWith("llm:") || name.includes(".llm.") || name.includes("generation");
 }
 function toolLabel(event) {
   const attrs = event.attributes ?? {};
@@ -20257,6 +20320,14 @@ var CIRCUIT_ALIASES = {
   "runaway-llm-loop": "circuit.runaway-llm-loop",
   "excessive-branch-width": "circuit.excessive-branch-width"
 };
+var KNOWN_CIRCUIT_RULES = /* @__PURE__ */ new Set([
+  ...Object.values(CIRCUIT_ALIASES),
+  ...ALL_RULES
+]);
+var KNOWN_GUARDRAIL_RULES = /* @__PURE__ */ new Set([
+  ...Object.values(GUARDRAIL_ALIASES),
+  ...ALL_RULES2
+]);
 function parseGuardrailRules(values) {
   if (!values?.length) return void 0;
   return values.map((value) => {
@@ -20319,16 +20390,21 @@ function eventToCircuit(event) {
     status: event.status
   };
 }
-function collectGuardrailInputs(read) {
+function scopedEvents(read, runId) {
+  if (runId === void 0) return [...read.events];
+  return read.events.filter((event) => event.runId === runId);
+}
+function collectGuardrailInputs(events) {
   const inputs = [];
-  for (const event of read.events) {
+  for (const event of events) {
     const attrs = event.attributes ?? {};
     for (const key of ["output", "answer", "text", "content", "message"]) {
       const value = attrs[key];
       if (typeof value === "string") inputs.push({ text: value });
       else if (value !== void 0) inputs.push({ value });
     }
-    if (event.kind === "TOOL" || event.name.startsWith("tool:")) {
+    const kind = typeof event.kind === "string" ? event.kind.toLowerCase() : "";
+    if (kind === "tool" || event.name.startsWith("tool:")) {
       inputs.push({
         toolName: String(attrs.toolName ?? attrs.tool ?? event.name),
         toolArgs: attrs.arguments ?? attrs.args ?? attrs.input
@@ -20337,6 +20413,21 @@ function collectGuardrailInputs(read) {
   }
   return inputs;
 }
+function classifyExtensionExecution(findings, errored) {
+  if (errored) return "error";
+  if (findings.some((finding) => finding.status === "fail")) return "fail";
+  if (findings.some((finding) => finding.status === "warning")) return "warning";
+  return "pass";
+}
+function extensionExecution(ruleId, findings, options) {
+  return {
+    ruleId,
+    category: "safety",
+    status: classifyExtensionExecution(findings, options.errored === true),
+    findingCount: findings.length,
+    ...options.runId !== void 0 ? { runId: options.runId } : {}
+  };
+}
 var DEFAULT_GUARDRAIL_OPTIONS = {
   bannedPhrase: { phrases: ["delete all data", "ignore all instructions"] },
   promptInjection: {},
@@ -20353,53 +20444,116 @@ var DEFAULT_CIRCUIT_OPTIONS = {
 };
 function mergeSafetyExtensions(result, read, options) {
   const findings = [...result.findings];
+  const ruleExecutions = [...result.ruleExecutions ?? []];
   let failed = result.summary.failed;
   let warnings = result.summary.warnings;
   let passed = result.summary.passed;
+  let errors = result.summary.errors;
+  const events = scopedEvents(read, options.runId);
+  const runId = options.runId;
   const guardrailRules = parseGuardrailRules(options.guardrails);
   if (guardrailRules) {
-    for (const input3 of collectGuardrailInputs(read)) {
-      const run = runGuardrails(input3, {
-        ...DEFAULT_GUARDRAIL_OPTIONS,
-        rules: guardrailRules
-      });
-      for (const item of run.results) {
-        const finding = guardrailFinding(item);
+    const inputs = collectGuardrailInputs(events);
+    for (const ruleId of guardrailRules) {
+      if (!KNOWN_GUARDRAIL_RULES.has(ruleId)) {
+        const finding = {
+          ruleId,
+          severity: "error",
+          status: "fail",
+          message: `Unknown guardrail rule: ${ruleId}.`,
+          evidence: []
+        };
         findings.push(finding);
-        if (finding.status === "fail") failed += 1;
-        else if (finding.status === "warning") warnings += 1;
-        else passed += 1;
+        failed += 1;
+        ruleExecutions.push(extensionExecution(ruleId, [finding], { runId, errored: true }));
+        errors += 1;
+        continue;
+      }
+      const ruleFindings = [];
+      if (inputs.length === 0) {
+        ruleExecutions.push(extensionExecution(ruleId, [], { runId }));
+        continue;
+      }
+      for (const input3 of inputs) {
+        const run = runGuardrails(input3, {
+          ...DEFAULT_GUARDRAIL_OPTIONS,
+          rules: [ruleId]
+        });
+        for (const item of run.results) {
+          const finding = guardrailFinding(item);
+          ruleFindings.push(finding);
+          findings.push(finding);
+          if (finding.status === "fail") failed += 1;
+          else if (finding.status === "warning") warnings += 1;
+          else passed += 1;
+        }
       }
+      ruleExecutions.push(extensionExecution(ruleId, ruleFindings, { runId }));
     }
   }
   const circuitRules = parseCircuitRules(options.circuits);
   if (circuitRules) {
-    const circuitRun = runCircuits(
-      read.events.map(eventToCircuit),
-      { ...DEFAULT_CIRCUIT_OPTIONS, rules: circuitRules }
-    );
-    for (const item of circuitRun.results) {
-      const finding = circuitFinding(item);
-      findings.push(finding);
-      if (finding.status === "fail") failed += 1;
-      else if (finding.status === "warning") warnings += 1;
-      else passed += 1;
+    const projected = projectLogicalEvents(events);
+    const circuitEvents = projected.logicalEvents.map(eventToCircuit);
+    for (const ruleId of circuitRules) {
+      if (!KNOWN_CIRCUIT_RULES.has(ruleId)) {
+        const finding = {
+          ruleId,
+          severity: "error",
+          status: "fail",
+          message: `Unknown circuit rule: ${ruleId}.`,
+          evidence: []
+        };
+        findings.push(finding);
+        failed += 1;
+        errors += 1;
+        ruleExecutions.push(extensionExecution(ruleId, [finding], { runId, errored: true }));
+        continue;
+      }
+      const circuitRun = runCircuits(circuitEvents, {
+        ...DEFAULT_CIRCUIT_OPTIONS,
+        rules: [ruleId]
+      });
+      if (circuitRun.results.length === 0) {
+        const finding = {
+          ruleId,
+          severity: "error",
+          status: "fail",
+          message: `Circuit rule ${ruleId} produced no evaluation result.`,
+          evidence: []
+        };
+        findings.push(finding);
+        failed += 1;
+        errors += 1;
+        ruleExecutions.push(extensionExecution(ruleId, [finding], { runId, errored: true }));
+        continue;
+      }
+      const ruleFindings = [];
+      for (const item of circuitRun.results) {
+        const finding = circuitFinding(item);
+        ruleFindings.push(finding);
+        findings.push(finding);
+        if (finding.status === "fail") failed += 1;
+        else if (finding.status === "warning") warnings += 1;
+        else passed += 1;
+      }
+      ruleExecutions.push(extensionExecution(ruleId, ruleFindings, { runId }));
     }
   }
-  const ok = failed === 0 && result.summary.errors === 0;
+  const status = result.status === "error" || errors > 0 ? "error" : failed > 0 ? "fail" : result.status;
   return {
     ...result,
-    ok,
-    status: failed > 0 ? "fail" : result.status,
+    ok: status === "pass",
+    status,
     summary: {
       passed,
       failed,
       warnings,
-      errors: result.summary.errors,
-      rulesEvaluated: result.summary.rulesEvaluated ?? 0
+      errors,
+      rulesEvaluated: ruleExecutions.length
     },
     findings,
-    ruleExecutions: result.ruleExecutions ?? []
+    ruleExecutions
   };
 }
 
@@ -20603,7 +20757,7 @@ var SAFETY_KEYS = /* @__PURE__ */ new Set([
   "maxFindings"
 ]);
 var RUN_EXPECTED = /* @__PURE__ */ new Set(["ok", "error", "running"]);
-var CONFIDENCE_VALUES = /* @__PURE__ */ new Set(["exact", "high", "medium", "low", "inferred"]);
+var CONFIDENCE_LIST = ATTRIBUTION_CONFIDENCES.join(", ");
 var CheckConfigError = class extends Error {
   code;
   constructor(code, message) {
@@ -20745,10 +20899,10 @@ function parseStructureSection(value) {
   rejectUnknownKeys(record, STRUCTURE_KEYS, "checks.structure");
   const out = {};
   if (record.minConfidence !== void 0) {
-    if (typeof record.minConfidence !== "string" || !CONFIDENCE_VALUES.has(record.minConfidence)) {
+    if (!isAttributionConfidence(record.minConfidence)) {
       throw new CheckConfigError(
         "AI_CHECK_CONFIG_INVALID_VALUE",
-        `checks.structure.minConfidence must be one of: exact, high, medium, low, inferred.`
+        `checks.structure.minConfidence must be one of: ${CONFIDENCE_LIST}.`
       );
     }
     out.minConfidence = record.minConfidence;
@@ -21258,7 +21412,8 @@ async function checkCommand(target, options = {}, stdin = process.stdin) {
             read,
             {
               ...options.guardrails ? { guardrails: options.guardrails } : {},
-              ...options.circuit ? { circuits: options.circuit } : {}
+              ...options.circuit ? { circuits: options.circuit } : {},
+              runId: meta2.runId
             }
           )
         );
@@ -21311,7 +21466,8 @@ async function checkCommand(target, options = {}, stdin = process.stdin) {
         read,
         {
           ...options.guardrails ? { guardrails: options.guardrails } : {},
-          ...options.circuit ? { circuits: options.circuit } : {}
+          ...options.circuit ? { circuits: options.circuit } : {},
+          ...options.run !== void 0 ? { runId: options.run } : {}
         }
       );
     }
@@ -23490,10 +23646,10 @@ function selectRun4(read, runId) {
 }
 function summarizeTrace(read, run) {
   const runId = run?.runId;
-  const scopedEvents = runId === void 0 ? read.events : read.events.filter((event) => event.runId === runId);
+  const scopedEvents2 = runId === void 0 ? read.events : read.events.filter((event) => event.runId === runId);
   const eventsByKind = {};
   const eventsByStatus = {};
-  for (const event of scopedEvents) {
+  for (const event of scopedEvents2) {
     increment(eventsByKind, event.kind);
     increment(eventsByStatus, event.status);
   }
@@ -23503,7 +23659,7 @@ function summarizeTrace(read, run) {
     ...run?.status !== void 0 ? { runStatus: run.status } : {},
     ...run?.durationMs !== void 0 ? { runDurationMs: run.durationMs } : {},
     runCount: read.runs.length,
-    eventCount: scopedEvents.length,
+    eventCount: scopedEvents2.length,
     eventsByKind: Object.fromEntries(Object.entries(eventsByKind).sort()),
     eventsByStatus: Object.fromEntries(Object.entries(eventsByStatus).sort()),
     readerWarnings: read.warnings.length,
@@ -26901,7 +27057,7 @@ function createCliProgram() {
     "add run.maxStepDuration (e.g. 30s, 5m)"
   ).option("--require-completed", "add run.requireCompleted").option("--detect-stalls", "add run.stall for running or incomplete events").option(
     "--fail-on-observation <status>",
-    "fail when observed outcomes match status (passed, failed, unknown, skipped; comma-separated)"
+    "outcome gate: require \u22651 OUTCOME and fail when status matches (passed,failed,unknown,skipped; comma-separated). Does not invent outcomes from tool/LLM/run success; use --preset trajectory for structural checks"
   ).option("--session <id>", "check all runs in a workflow session (requires --dir)").option("--group <id>", "check all runs sharing a groupId (requires --dir)").option(
     "--correlate-group",
     "when using --session, also match synthetic group: session keys"
@@ -26914,7 +27070,7 @@ function createCliProgram() {
     "run optional circuit rules (repeatable): same-tool-repetition, max-retries, ...",
     (value, previous = []) => [...previous, value]
   ).addOption(
-    new commander.Option("--preset <name>", "additive check preset").choices(["trajectory", "safety", "comprehensive", "behavioral-session"])
+    new commander.Option("--preset <name>", "additive check preset (trajectory=structure only; safety=share checks; not interchangeable)").choices(["trajectory", "safety", "comprehensive", "behavioral-session"])
   ).addOption(
     new commander.Option(
       "--evidence-on <mode>",