agent-inspect 6.31.11 → 6.31.13

package/packages/core/dist/advanced.mjs

@@ -11,12 +11,12 @@ import { escapeHtml, flattenTree } from './chunk-CAWPF22J.mjs';
 import { resolveRedactionProfile, readTraceEventsFromFile, stringContainsHighConfidenceCredential } from './chunk-DK3OW6UI.mjs';
 export { DEFAULT_MAX_EVENT_BYTES, DEFAULT_MAX_METADATA_VALUE_LENGTH, DEFAULT_MAX_PREVIEW_LENGTH, getRunIdFromTraceFileName, initializeTraceFile, listTraceFiles, prepareMetadataForDisk, prepareTraceEventForDisk, readTraceEvents, readTraceFile, resolveRedactionProfile, resolveTraceSafetyOptions, serializeEvent, validateEvent, writeTraceEvent } from './chunk-DK3OW6UI.mjs';
 import { Redactor } from './chunk-VLVCLAPB.mjs';
-import { openTrace } from './chunk-QOC3PJO3.mjs';
-import './chunk-OAUNS5CS.mjs';
+import { openTrace } from './chunk-NJUSSJX3.mjs';
+import './chunk-JE3TUFAQ.mjs';
 import { projectLogicalEvents } from './chunk-AHUZNHFI.mjs';
 import './chunk-S4YWKV4G.mjs';
 import './chunk-S6LQBLX5.mjs';
-import './chunk-AK5EN4U2.mjs';
+import './chunk-YU6XT5A2.mjs';
 import { parseDuration, isTraceEvent, getTraceFilePath } from './chunk-7DHSWDKB.mjs';
 export { DEFAULT_TRACE_DIR_NAME, FALLBACK_TRACE_DIR, MAX_NAME_LENGTH, RUNS_DIR_NAME, createRunId, createStepId, ensureTraceDir, formatDuration, formatError, formatTimestamp, getDefaultTraceDir, getTraceFilePath, isStepStatus, isStepType, isTraceEvent, parseDuration, parseTraceJsonl, truncateName, unknownTraceFormatMessage, warn } from './chunk-7DHSWDKB.mjs';
 import { persistedInspectEventsToTraceEvents } from './chunk-EE5Z2GQ3.mjs';
@@ -381,12 +381,14 @@ function topKinds(run) {
 function countErrorNodes(nodes) {
   return nodes.filter((entry) => entry.node.event.status === "error").length;
 }
+function isExecutionStepKind(kind) {
+  return kind !== "RUN" && kind !== "OUTCOME";
+}
 function slowestNode(nodes) {
   return nodes.filter(
     (entry) => entry.node.event.durationMs !== void 0 && // The RUN boundary spans the whole run, so its duration always dominates
-    // and shadows every real step. Rank actual work nodes only, matching the
-    // stats slowest-step ranking, which never counts the run envelope.
-    entry.node.event.kind !== "RUN"
+    // and shadows every real step. OUTCOME is a linked result, not execution.
+    isExecutionStepKind(entry.node.event.kind)
   ).sort((a, b) => {
     const delta = (b.node.event.durationMs ?? 0) - (a.node.event.durationMs ?? 0);
     return delta !== 0 ? delta : a.index - b.index;
@@ -439,6 +441,12 @@ function buildFacts(run, redactor, events) {
     const scoped = events.filter((event) => event.runId === run.runId);
     const { logicalEvents: logical } = projectLogicalEvents(scoped);
     const logicalSteps = logical.filter((event) => event.kind !== "RUN");
+    const executionSteps = logical.filter((event) => isExecutionStepKind(event.kind));
+    const executionByKind = {
+      LLM: executionSteps.filter((event) => event.kind === "LLM").length,
+      TOOL: executionSteps.filter((event) => event.kind === "TOOL").length,
+      LOGIC: executionSteps.filter((event) => event.kind === "LOGIC").length
+    };
     facts.push(
       fact("run.rawEventCount", "Raw persisted event count", scoped.length, redactor),
       fact(
@@ -452,9 +460,21 @@ function buildFacts(run, redactor, events) {
         "Logical lifecycle event count (including RUN)",
         logical.length,
         redactor
+      ),
+      fact(
+        "run.executionStepCount",
+        "Execution step count (excludes RUN and OUTCOME)",
+        executionSteps.length,
+        redactor
+      ),
+      fact(
+        "run.executionStepKinds",
+        "Execution step kind counts",
+        executionByKind,
+        redactor
       )
     );
-    const slowestLogical = [...logicalSteps].filter((event) => typeof event.durationMs === "number").sort((a, b) => {
+    const slowestLogical = [...executionSteps].filter((event) => typeof event.durationMs === "number").sort((a, b) => {
       const delta = (b.durationMs ?? 0) - (a.durationMs ?? 0);
       return delta !== 0 ? delta : a.eventId.localeCompare(b.eventId);
     })[0];
@@ -3394,25 +3414,86 @@ function asStringArray(value, label) {
 function asPositiveNumber(value, label) {
   if (value === void 0) return void 0;
   if (typeof value !== "number" || !Number.isFinite(value) || value < 0) {
-    throw new Error(`${label} must be a non-negative number.`);
+    throw new Error(`${label} must be a non-negative finite number.`);
   }
   return value;
 }
-function asExpectConfig(value, label) {
+function asBoolean(value, label) {
   if (value === void 0) return void 0;
+  if (typeof value !== "boolean") {
+    throw new Error(`${label} must be a boolean (not a string or other type).`);
+  }
+  return value;
+}
+function assertObject(value, label) {
   if (typeof value !== "object" || value === null || Array.isArray(value)) {
     throw new Error(`${label} must be an object.`);
   }
-  const raw = value;
+  return value;
+}
+function assertKnownKeys(raw, allowed, label) {
+  const allowedSet = new Set(allowed);
+  for (const key of Object.keys(raw)) {
+    if (!allowedSet.has(key)) {
+      throw new Error(`${label} has unknown key "${key}".`);
+    }
+  }
+}
+var CASE_KEYS = [
+  "id",
+  "trace",
+  "runId",
+  "input",
+  "requireTools",
+  "forbidTools",
+  "maxDurationMs",
+  "expectedObservations",
+  "expect"
+];
+var EXPECT_KEYS = [
+  "checkStatus",
+  "findingRuleIds",
+  "rejectUnexpectedFindings",
+  "minAssertions"
+];
+var CHECKS_KEYS = ["select", "run", "tool", "llm"];
+var CHECKS_RUN_KEYS = ["maxDurationMs", "maxDepth"];
+var CHECKS_TOOL_KEYS = ["required", "forbidden"];
+var CHECKS_LLM_KEYS = ["allowedModels", "maxTotalTokens"];
+var EVAL_KEYS = [
+  "requireSuccess",
+  "requiredTools",
+  "forbiddenTools",
+  "maxDurationMs",
+  "maxDepth",
+  "maxRetries",
+  "maxTotalTokens"
+];
+var ROOT_KEYS = [
+  "name",
+  "traces",
+  "cases",
+  "checks",
+  "eval",
+  "redactionProfile",
+  "artifacts",
+  "baseline",
+  "candidate"
+];
+function asExpectConfig(value, label) {
+  if (value === void 0) return void 0;
+  const raw = assertObject(value, label);
+  assertKnownKeys(raw, EXPECT_KEYS, label);
   const checkStatus = raw.checkStatus;
   if (checkStatus !== "pass" && checkStatus !== "fail" && checkStatus !== "error") {
     throw new Error(`${label}.checkStatus must be "pass", "fail", or "error".`);
   }
   const findingRuleIds = asStringArray(raw.findingRuleIds, `${label}.findingRuleIds`);
   const minAssertions = asPositiveNumber(raw.minAssertions, `${label}.minAssertions`);
-  const rejectUnexpectedFindings = raw.rejectUnexpectedFindings === void 0 ? void 0 : typeof raw.rejectUnexpectedFindings === "boolean" ? raw.rejectUnexpectedFindings : (() => {
-    throw new Error(`${label}.rejectUnexpectedFindings must be a boolean.`);
-  })();
+  const rejectUnexpectedFindings = asBoolean(
+    raw.rejectUnexpectedFindings,
+    `${label}.rejectUnexpectedFindings`
+  );
   return {
     checkStatus,
     ...findingRuleIds !== void 0 ? { findingRuleIds } : {},
@@ -3420,6 +3501,56 @@ function asExpectConfig(value, label) {
     ...minAssertions !== void 0 ? { minAssertions } : {}
   };
 }
+function asChecksConfig(value, label) {
+  const raw = assertObject(value, label);
+  assertKnownKeys(raw, CHECKS_KEYS, label);
+  const checks = {};
+  const select = asStringArray(raw.select, `${label}.select`);
+  if (select !== void 0) checks.select = select;
+  if (raw.run !== void 0) {
+    const run = assertObject(raw.run, `${label}.run`);
+    assertKnownKeys(run, CHECKS_RUN_KEYS, `${label}.run`);
+    checks.run = {
+      ...asPositiveNumber(run.maxDurationMs, `${label}.run.maxDurationMs`) !== void 0 ? { maxDurationMs: asPositiveNumber(run.maxDurationMs, `${label}.run.maxDurationMs`) } : {},
+      ...asPositiveNumber(run.maxDepth, `${label}.run.maxDepth`) !== void 0 ? { maxDepth: asPositiveNumber(run.maxDepth, `${label}.run.maxDepth`) } : {}
+    };
+  }
+  if (raw.tool !== void 0) {
+    const tool = assertObject(raw.tool, `${label}.tool`);
+    assertKnownKeys(tool, CHECKS_TOOL_KEYS, `${label}.tool`);
+    checks.tool = {
+      ...asStringArray(tool.required, `${label}.tool.required`) !== void 0 ? { required: asStringArray(tool.required, `${label}.tool.required`) } : {},
+      ...asStringArray(tool.forbidden, `${label}.tool.forbidden`) !== void 0 ? { forbidden: asStringArray(tool.forbidden, `${label}.tool.forbidden`) } : {}
+    };
+  }
+  if (raw.llm !== void 0) {
+    const llm = assertObject(raw.llm, `${label}.llm`);
+    assertKnownKeys(llm, CHECKS_LLM_KEYS, `${label}.llm`);
+    checks.llm = {
+      ...asStringArray(llm.allowedModels, `${label}.llm.allowedModels`) !== void 0 ? { allowedModels: asStringArray(llm.allowedModels, `${label}.llm.allowedModels`) } : {},
+      ...asPositiveNumber(llm.maxTotalTokens, `${label}.llm.maxTotalTokens`) !== void 0 ? {
+        maxTotalTokens: asPositiveNumber(
+          llm.maxTotalTokens,
+          `${label}.llm.maxTotalTokens`
+        )
+      } : {}
+    };
+  }
+  return checks;
+}
+function asEvalConfig(value, label) {
+  const raw = assertObject(value, label);
+  assertKnownKeys(raw, EVAL_KEYS, label);
+  return {
+    ...asBoolean(raw.requireSuccess, `${label}.requireSuccess`) !== void 0 ? { requireSuccess: asBoolean(raw.requireSuccess, `${label}.requireSuccess`) } : {},
+    ...asStringArray(raw.requiredTools, `${label}.requiredTools`) !== void 0 ? { requiredTools: asStringArray(raw.requiredTools, `${label}.requiredTools`) } : {},
+    ...asStringArray(raw.forbiddenTools, `${label}.forbiddenTools`) !== void 0 ? { forbiddenTools: asStringArray(raw.forbiddenTools, `${label}.forbiddenTools`) } : {},
+    ...asPositiveNumber(raw.maxDurationMs, `${label}.maxDurationMs`) !== void 0 ? { maxDurationMs: asPositiveNumber(raw.maxDurationMs, `${label}.maxDurationMs`) } : {},
+    ...asPositiveNumber(raw.maxDepth, `${label}.maxDepth`) !== void 0 ? { maxDepth: asPositiveNumber(raw.maxDepth, `${label}.maxDepth`) } : {},
+    ...asPositiveNumber(raw.maxRetries, `${label}.maxRetries`) !== void 0 ? { maxRetries: asPositiveNumber(raw.maxRetries, `${label}.maxRetries`) } : {},
+    ...asPositiveNumber(raw.maxTotalTokens, `${label}.maxTotalTokens`) !== void 0 ? { maxTotalTokens: asPositiveNumber(raw.maxTotalTokens, `${label}.maxTotalTokens`) } : {}
+  };
+}
 function validateCaseConfig(value, index) {
   const diagnostics = [];
   if (typeof value !== "object" || value === null || Array.isArray(value)) {
@@ -3430,6 +3561,7 @@ function validateCaseConfig(value, index) {
   }
   const raw = value;
   try {
+    assertKnownKeys(raw, CASE_KEYS, `cases[${index}]`);
     const id = asString(raw.id, `cases[${index}].id`);
     if (id === void 0) {
       diagnostics.push(
@@ -3478,6 +3610,7 @@ function normalizeSuiteConfig(value) {
     throw new Error("Suite config must export an object.");
   }
   const raw = value;
+  assertKnownKeys(raw, ROOT_KEYS, "suite config");
   const name = asString(raw.name, "name");
   const traces = asString(raw.traces, "traces");
   if (name === void 0) throw new Error("name is required.");
@@ -3506,25 +3639,15 @@ function normalizeSuiteConfig(value) {
   }
   const config = { name, traces, cases };
   if (raw.checks !== void 0) {
-    if (typeof raw.checks !== "object" || Array.isArray(raw.checks)) {
-      throw new Error("checks must be an object.");
-    }
-    config.checks = raw.checks;
+    config.checks = asChecksConfig(raw.checks, "checks");
   }
   if (raw.eval !== void 0) {
-    if (typeof raw.eval !== "object" || Array.isArray(raw.eval)) {
-      throw new Error("eval must be an object.");
-    }
-    config.eval = raw.eval;
+    config.eval = asEvalConfig(raw.eval, "eval");
   }
   if (raw.artifacts !== void 0) {
-    if (typeof raw.artifacts !== "object" || Array.isArray(raw.artifacts)) {
-      throw new Error("artifacts must be an object.");
-    }
-    const outputDir = asString(
-      raw.artifacts.outputDir,
-      "artifacts.outputDir"
-    );
+    const artifacts = assertObject(raw.artifacts, "artifacts");
+    assertKnownKeys(artifacts, ["outputDir"], "artifacts");
+    const outputDir = asString(artifacts.outputDir, "artifacts.outputDir");
     config.artifacts = outputDir !== void 0 ? { outputDir } : {};
   }
   if (raw.baseline !== void 0) {