agent-inspect 6.14.1 → 6.15.0

package/packages/core/dist/checks.cjs

@@ -10,6 +10,59 @@ function _interopDefault (e) { return e && e.__esModule ? e : { default: e }; }
 var path__default = /*#__PURE__*/_interopDefault(path);
 var os__default = /*#__PURE__*/_interopDefault(os);
 
+// packages/core/src/diagnostics/programmatic.ts
+var PROGRAMMATIC_DIAGNOSTIC_SPECS = Object.freeze({
+  AI_TRACE_INPUT_INVALID: {
+    code: "AI_TRACE_INPUT_INVALID",
+    summary: 'Expected { type: "file", path }, { type: "directory", path }, { type: "string", content }, { type: "buffer", content }, or { type: "stdin" }.',
+    remediation: "For a file path, use openTraceFile(path).",
+    relatedCodes: ["invalid_input"]
+  },
+  AI_TRACE_FORMAT_UNSUPPORTED: {
+    code: "AI_TRACE_FORMAT_UNSUPPORTED",
+    summary: "No trace reader could detect the input format.",
+    remediation: "Pass an AgentInspect JSONL file via openTraceFile, or set options.format to a registered reader.",
+    relatedCodes: ["unsupported_format"]
+  },
+  AI_TRACE_FORMAT_AMBIGUOUS: {
+    code: "AI_TRACE_FORMAT_AMBIGUOUS",
+    summary: "Multiple trace readers matched the input with equal confidence.",
+    remediation: "Set options.format explicitly to disambiguate the reader.",
+    relatedCodes: ["ambiguous_format"]
+  },
+  AI_TRACE_FACTS_INPUT_NOT_NORMALIZED: {
+    code: "AI_TRACE_FACTS_INPUT_NOT_NORMALIZED",
+    summary: "TraceFacts requires TraceReadResult or PersistedInspectEvent[].",
+    remediation: "Use openTraceFile() to normalize a JSONL trace first."
+  },
+  AI_TRACE_CONTRACT_RUN_SELECTION_REQUIRED: {
+    code: "AI_TRACE_CONTRACT_RUN_SELECTION_REQUIRED",
+    summary: "Multiple runs are available; select a run before executing checks.",
+    remediation: "Pass options.runId or TraceCheckInput.selectedRun.",
+    relatedCodes: ["AI_CHECK_RUN_SELECTION_REQUIRED"]
+  },
+  AI_TRACE_RELATIONSHIP_SELF_PARENT: {
+    code: "AI_TRACE_RELATIONSHIP_SELF_PARENT",
+    summary: "A parentId equals its own event/step id.",
+    remediation: "Reject at capture (AI_LANGGRAPH_SELF_PARENT_REJECTED) or drop via logical projection; do not invent replacement parents.",
+    relatedCodes: [
+      "AI_LANGGRAPH_SELF_PARENT_REJECTED",
+      "AI_LOGICAL_SELF_PARENT_REMOVED"
+    ]
+  },
+  AI_TRACE_RELATIONSHIP_CYCLE: {
+    code: "AI_TRACE_RELATIONSHIP_CYCLE",
+    summary: "Trace contains a parentId cycle.",
+    remediation: "Use visibility-first tree linking for legacy fixtures; prefer acyclic capture for new adapter output.",
+    relatedCodes: ["structure.cycle"]
+  }
+});
+function formatProgrammaticDiagnostic(code, detail) {
+  const spec = PROGRAMMATIC_DIAGNOSTIC_SPECS[code];
+  const summary = detail?.trim() ? detail.trim() : spec.summary;
+  return `${code}: ${summary} Remediation: ${spec.remediation}`;
+}
+
 // packages/core/src/outcomes/types.ts
 var OUTCOME_ATTRIBUTE_STATUS_KEY = "outcomeStatus";
 var OUTCOME_ATTRIBUTE_EXPECTATION_KEY = "expectation";
@@ -64,6 +117,73 @@ function outcomesMatchingStatus(outcomes, statuses) {
   return outcomes.filter((outcome) => set.has(outcome.status));
 }
 
+// packages/core/src/safety/sensitive-key.ts
+function normalizeSensitiveKey(value) {
+  return value.toLowerCase().replace(/[^a-z0-9_]/g, "");
+}
+var NON_CREDENTIAL_TOKEN_CONFIG_KEYS = new Set(
+  [
+    "tokens",
+    "max_tokens",
+    "min_tokens",
+    "ls_max_tokens",
+    "token_count",
+    "token_limit",
+    "token_budget",
+    "input_tokens",
+    "output_tokens",
+    "total_tokens",
+    "cached_tokens",
+    "prompt_tokens",
+    "completion_tokens"
+  ].map(normalizeSensitiveKey)
+);
+var DEFAULT_CREDENTIAL_SENSITIVE_KEYS = [
+  "authorization",
+  "cookie",
+  "token",
+  "access_token",
+  "accesstoken",
+  "auth_token",
+  "authtoken",
+  "refresh_token",
+  "refreshtoken",
+  "id_token",
+  "idtoken",
+  "bearer_token",
+  "bearertoken",
+  "api_token",
+  "apitoken",
+  "apikey",
+  "api_key",
+  "password",
+  "secret",
+  "email"
+];
+function isTokenCredentialKey(normalized) {
+  if (NON_CREDENTIAL_TOKEN_CONFIG_KEYS.has(normalized)) return false;
+  if (normalized === "token") return true;
+  if (normalized.endsWith("tokens")) return false;
+  return normalized.endsWith("token");
+}
+function isCredentialSensitiveKey(key, sensitiveKeys = DEFAULT_CREDENTIAL_SENSITIVE_KEYS) {
+  if (!key) return false;
+  const normalized = normalizeSensitiveKey(key);
+  if (!normalized) return false;
+  if (NON_CREDENTIAL_TOKEN_CONFIG_KEYS.has(normalized)) return false;
+  for (const sensitive of sensitiveKeys) {
+    const s = normalizeSensitiveKey(sensitive);
+    if (!s) continue;
+    if (s === "token") {
+      if (isTokenCredentialKey(normalized)) return true;
+      continue;
+    }
+    if (normalized === s) return true;
+    if (normalized.endsWith(`_${s}`) || normalized.startsWith(`${s}_`)) return true;
+  }
+  return false;
+}
+
 // packages/core/src/checks/logical-events.ts
 function isRecord(value) {
   return typeof value === "object" && value !== null && !Array.isArray(value);
@@ -259,6 +379,26 @@ function projectLogicalEvents(events) {
       }
     }
     if (!remapped) {
+      if (originalParentId === event.eventId) {
+        diagnostics.push({
+          code: "AI_LOGICAL_SELF_PARENT_REMOVED",
+          message: formatProgrammaticDiagnostic(
+            "AI_TRACE_RELATIONSHIP_SELF_PARENT",
+            `Removed self-parent edge on ${event.eventId}.`
+          ),
+          eventIds: [event.eventId]
+        });
+        const { parentId: _drop, ...rest } = event;
+        normalized.push({
+          ...rest,
+          projection: {
+            ...event.projection,
+            parentNormalized: true,
+            originalParentId
+          }
+        });
+        continue;
+      }
       if (!logicalById.has(originalParentId) && !logicalByRawId.has(originalParentId)) {
         const mapping = event.attributes?.parentMapping;
         const unresolved = mapping === "unresolved" || event.attributes?.parentUnresolved === true || event.attributes?.unresolvedParent === true || // LangGraph/framework scaffolding sentinel labels are not event ids.
@@ -274,6 +414,26 @@ function projectLogicalEvents(events) {
       normalized.push(event);
       continue;
     }
+    if (nextParent === event.eventId) {
+      diagnostics.push({
+        code: "AI_LOGICAL_SELF_PARENT_REMOVED",
+        message: formatProgrammaticDiagnostic(
+          "AI_TRACE_RELATIONSHIP_SELF_PARENT",
+          `Removed self-parent edge on ${event.eventId} (was ${originalParentId}).`
+        ),
+        eventIds: [event.eventId]
+      });
+      const { parentId: _drop, ...rest } = event;
+      normalized.push({
+        ...rest,
+        projection: {
+          ...event.projection,
+          parentNormalized: true,
+          originalParentId
+        }
+      });
+      continue;
+    }
     diagnostics.push({
       code: "AI_LOGICAL_PARENT_REMAPPED",
       message: `Remapped parent ${originalParentId} \u2192 ${nextParent} for ${event.eventId}.`,
@@ -346,7 +506,45 @@ function summarizeSemanticParity(events) {
     diagnostics: projection.diagnostics
   };
 }
-function buildTraceFacts(events) {
+var TRACE_FACTS_INPUT_NOT_NORMALIZED = formatProgrammaticDiagnostic(
+  "AI_TRACE_FACTS_INPUT_NOT_NORMALIZED"
+);
+function isTraceReadResult(input) {
+  if (typeof input !== "object" || input === null || Array.isArray(input)) {
+    return false;
+  }
+  const record = input;
+  return Array.isArray(record.events) && Array.isArray(record.runs) && typeof record.format === "string" && Array.isArray(record.warnings);
+}
+function looksLikeRawV01TraceEvents(input) {
+  if (!Array.isArray(input) || input.length === 0) return false;
+  const first = input[0];
+  if (typeof first !== "object" || first === null) return false;
+  const row = first;
+  return typeof row.event === "string" && (row.schemaVersion === "0.1" || row.eventId === void 0);
+}
+function isPersistedInspectEventArray(input) {
+  if (!Array.isArray(input)) return false;
+  if (input.length === 0) return true;
+  const first = input[0];
+  if (typeof first !== "object" || first === null) return false;
+  const row = first;
+  return typeof row.eventId === "string" && (row.schemaVersion === "0.2" || row.schemaVersion === "1.0" || row.schemaVersion === "0.1") && typeof row.event !== "string";
+}
+function resolveTraceFactsEvents(input) {
+  if (isTraceReadResult(input)) {
+    return input.events;
+  }
+  if (looksLikeRawV01TraceEvents(input)) {
+    throw new TypeError(TRACE_FACTS_INPUT_NOT_NORMALIZED);
+  }
+  if (isPersistedInspectEventArray(input)) {
+    return input;
+  }
+  throw new TypeError(TRACE_FACTS_INPUT_NOT_NORMALIZED);
+}
+function buildTraceFacts(input) {
+  const events = resolveTraceFactsEvents(input);
   const projection = projectLogicalEvents(events);
   const toolsByName = /* @__PURE__ */ new Map();
   const llmEvents = [];
@@ -551,6 +749,9 @@ function evaluateTraceContract(input, contract, options = {}) {
     ...options.runId !== void 0 ? { runId: options.runId } : {}
   });
 }
+function evaluateTraceContractRead(read, contract, options = {}) {
+  return evaluateTraceContract({ read }, contract, options);
+}
 
 // packages/core/src/checks/index.ts
 var SEVERITY_RANK = {
@@ -569,16 +770,7 @@ var CONFIDENCE_RANK = {
   correlated: 2,
   explicit: 3
 };
-var DEFAULT_SENSITIVE_KEYS = [
-  "authorization",
-  "cookie",
-  "token",
-  "apikey",
-  "api_key",
-  "password",
-  "secret",
-  "email"
-];
+var DEFAULT_SENSITIVE_KEYS = DEFAULT_CREDENTIAL_SENSITIVE_KEYS;
 var DEFAULT_RAW_CONTENT_KEYS = [
   "body",
   "headers",
@@ -736,7 +928,13 @@ function resolveSelectedRun(input, runId) {
   if (input.read.runs.length === 0) {
     return {
       diagnostics: [
-        diagnostic("AI_CHECK_RUN_SELECTION_REQUIRED", "No runs are available for checks.")
+        diagnostic(
+          "AI_CHECK_RUN_SELECTION_REQUIRED",
+          formatProgrammaticDiagnostic(
+            "AI_TRACE_CONTRACT_RUN_SELECTION_REQUIRED",
+            "No runs are available for checks."
+          )
+        )
       ]
     };
   }
@@ -744,7 +942,7 @@ function resolveSelectedRun(input, runId) {
     diagnostics: [
       diagnostic(
         "AI_CHECK_RUN_SELECTION_REQUIRED",
-        "Multiple runs are available; select a run before executing checks."
+        formatProgrammaticDiagnostic("AI_TRACE_CONTRACT_RUN_SELECTION_REQUIRED")
       )
     ]
   };
@@ -1011,9 +1209,7 @@ function hasRedactionMarker(value, markers) {
   return markers.some((marker) => value.includes(marker)) || /^\[HASH:[A-Za-z0-9_-]+\]$/.test(value);
 }
 function isSensitiveKey(key, sensitiveKeys) {
-  if (!key) return false;
-  const normalized = normalizedKey(key);
-  return sensitiveKeys.some((sensitive) => normalized.includes(normalizedKey(sensitive)));
+  return isCredentialSensitiveKey(key, sensitiveKeys);
 }
 function isRawContentKey(key, forbiddenKeys) {
   if (!key) return false;
@@ -1372,7 +1568,7 @@ function createToolUsageRule(options) {
         }
       }
       const forbidden = new Set(options.forbidden ?? []);
-      const allowed = options.allowed ? new Set(options.allowed) : void 0;
+      const allowed = options.allowed?.length ? new Set(options.allowed) : void 0;
       for (const event of tools) {
         const name = toolName(event);
         if (forbidden.has(name)) {
@@ -1472,9 +1668,9 @@ function createLlmUsageRule(options) {
     evaluate(context) {
       const llms = finishedEvents(context, "LLM");
       const findings = [];
-      const allowedModels = options.allowedModels ? new Set(options.allowedModels) : void 0;
-      const allowedProviders = options.allowedProviders ? new Set(options.allowedProviders) : void 0;
-      const finishReasons = options.finishReasons ? new Set(options.finishReasons) : void 0;
+      const allowedModels = options.allowedModels?.length ? new Set(options.allowedModels) : void 0;
+      const allowedProviders = options.allowedProviders?.length ? new Set(options.allowedProviders) : void 0;
+      const finishReasons = options.finishReasons?.length ? new Set(options.finishReasons) : void 0;
       if (options.maxCalls !== void 0 && llms.length > options.maxCalls) {
         findings.push(
           failFinding(
@@ -1627,7 +1823,7 @@ function createStructureCycleRule() {
               findings.push(
                 failFinding(
                   "structure.cycle",
-                  "Trace contains a parentId cycle.",
+                  formatProgrammaticDiagnostic("AI_TRACE_RELATIONSHIP_CYCLE"),
                   cycle.map((item) => eventEvidence(item, "parentId")),
                   "acyclic parentId graph",
                   cycle.map((item) => item.eventId).sort()
@@ -1815,7 +2011,7 @@ function createSignalRule(ruleId, label, options, selectEvents, nameForEvent) {
         }
       }
       const forbidden = new Set(options.forbidden ?? []);
-      const allowed = options.allowed ? new Set(options.allowed) : void 0;
+      const allowed = options.allowed?.length ? new Set(options.allowed) : void 0;
       for (const event of events) {
         const name = nameForEvent(event);
         if (forbidden.has(name)) {
@@ -2295,6 +2491,7 @@ function runTraceChecks(input, options = {}) {
   };
 }
 
+exports.PROGRAMMATIC_DIAGNOSTIC_SPECS = PROGRAMMATIC_DIAGNOSTIC_SPECS;
 exports.buildTraceFacts = buildTraceFacts;
 exports.createBaselineRegressionRule = createBaselineRegressionRule;
 exports.createDecisionRule = createDecisionRule;
@@ -2323,6 +2520,8 @@ exports.createToolOrderingRule = createToolOrderingRule;
 exports.createToolUsageRule = createToolUsageRule;
 exports.defineTraceContract = defineTraceContract;
 exports.evaluateTraceContract = evaluateTraceContract;
+exports.evaluateTraceContractRead = evaluateTraceContractRead;
+exports.formatProgrammaticDiagnostic = formatProgrammaticDiagnostic;
 exports.projectLogicalEvents = projectLogicalEvents;
 exports.resolveCanonicalToolName = resolveCanonicalToolName;
 exports.runTraceChecks = runTraceChecks;