agent-inspect 6.3.0 → 6.4.1

package/packages/core/dist/checks.cjs

@@ -64,6 +64,117 @@ function outcomesMatchingStatus(outcomes, statuses) {
   return outcomes.filter((outcome) => set.has(outcome.status));
 }
 
+// packages/core/src/checks/contract.ts
+function contractFailFinding(ruleId, message, evidence, expected, actual) {
+  return {
+    ruleId,
+    severity: "error",
+    status: "fail",
+    message,
+    ...expected !== void 0 ? { expected } : {},
+    ...actual !== void 0 ? { actual } : {},
+    evidence: [...evidence]
+  };
+}
+function normalizeStatus(status) {
+  if (status === "success") return "ok";
+  if (status === "failed") return "error";
+  if (status === "running") return "running";
+  return "error";
+}
+function contractToRules(contract) {
+  const rules = [];
+  if (contract.run?.allowedStatuses?.length === 1) {
+    rules.push(
+      createRunStatusRule({
+        expected: normalizeStatus(contract.run.allowedStatuses[0]),
+        allowIncomplete: contract.run.requireCompleted === false
+      })
+    );
+  } else if (contract.run?.requireCompleted !== false) {
+    rules.push(createRunStatusRule({ allowIncomplete: false }));
+  }
+  if (contract.run?.maxDurationMs !== void 0) {
+    rules.push(createRunDurationRule({ maxDurationMs: contract.run.maxDurationMs }));
+  }
+  if (contract.run?.requireCompleted === true) {
+    rules.push(createStructureIncompleteRule({ requireEndedAtForStarted: true }));
+  }
+  if (contract.tools) {
+    rules.push(
+      createToolUsageRule({
+        ...contract.tools.required ? { required: contract.tools.required } : {},
+        ...contract.tools.forbidden ? { forbidden: contract.tools.forbidden } : {},
+        ...contract.tools.allowed ? { allowed: contract.tools.allowed } : {},
+        ...contract.tools.maxCalls !== void 0 ? { maxCount: contract.tools.maxCalls } : {}
+      })
+    );
+    const order = contract.tools.requiredOrder ?? [];
+    for (let i = 0; i < order.length - 1; i += 1) {
+      rules.push(createToolOrderingRule({ before: order[i], after: order[i + 1] }));
+    }
+  }
+  if (contract.llm) {
+    rules.push(
+      createLlmUsageRule({
+        ...contract.llm.maxCalls !== void 0 ? { maxCalls: contract.llm.maxCalls } : {},
+        ...contract.llm.maxTotalTokens !== void 0 ? { maxTotalTokens: contract.llm.maxTotalTokens } : {},
+        ...contract.llm.allowedModels ? { allowedModels: contract.llm.allowedModels } : {}
+      })
+    );
+  }
+  if (contract.observations) {
+    const required = contract.observations.required ?? [];
+    if (required.length > 0) {
+      rules.push({
+        id: "contract.observation.required",
+        category: "run",
+        defaultSeverity: "error",
+        evaluate(context) {
+          const outcomes = extractOutcomesFromPersistedEvents(context.events);
+          const names = new Set(outcomes.map((item) => item.name));
+          const missing = required.filter((name) => !names.has(name));
+          if (missing.length === 0) return [];
+          return [
+            contractFailFinding(
+              "contract.observation.required",
+              `Required observations missing: ${missing.join(", ")}`,
+              context.selectedRun ? [{ runId: context.selectedRun.runId, kind: "RUN", name: context.selectedRun.name }] : [],
+              required,
+              [...names]
+            )
+          ];
+        }
+      });
+    }
+    if (contract.observations.failOn?.length) {
+      rules.push(
+        createObservedOutcomeRule({
+          failOn: contract.observations.failOn.filter(
+            (status) => status === "failed" || status === "unknown" || status === "skipped"
+          )
+        })
+      );
+    }
+  }
+  return rules;
+}
+function defineTraceContract(input) {
+  return {
+    ...input.run ? { run: { ...input.run } } : {},
+    ...input.tools ? { tools: { ...input.tools } } : {},
+    ...input.llm ? { llm: { ...input.llm } } : {},
+    ...input.observations ? { observations: { ...input.observations } } : {}
+  };
+}
+function evaluateTraceContract(input, contract, options = {}) {
+  const rules = contractToRules(contract);
+  return runTraceChecks(input, {
+    rules,
+    ...options.runId !== void 0 ? { runId: options.runId } : {}
+  });
+}
+
 // packages/core/src/checks/index.ts
 var SEVERITY_RANK = {
   error: 0,
@@ -1734,6 +1845,8 @@ exports.createStructureRelationshipRule = createStructureRelationshipRule;
 exports.createToolFailureRule = createToolFailureRule;
 exports.createToolOrderingRule = createToolOrderingRule;
 exports.createToolUsageRule = createToolUsageRule;
+exports.defineTraceContract = defineTraceContract;
+exports.evaluateTraceContract = evaluateTraceContract;
 exports.runTraceChecks = runTraceChecks;
 //# sourceMappingURL=checks.cjs.map
 //# sourceMappingURL=checks.cjs.map