agent-inspect 4.2.0 → 4.4.0

package/packages/core/dist/checks.d.cts

@@ -1,6 +1,7 @@
 import { TraceReadResult, TraceReadWarning } from './readers.cjs';
-import { b as InspectRunTree, c as InspectNode, A as AttributionConfidence } from './inspect-event-CYAV7Hxm.cjs';
-import { P as PersistedInspectEvent } from './persisted-inspect-event-D5SjBz9b.cjs';
+import { b as InspectRunTree, c as InspectNode, A as AttributionConfidence } from './inspect-event-IqwOHbGo.cjs';
+import { P as PersistedInspectEvent } from './persisted-inspect-event-Gd-W8vB7.cjs';
+import { a as ObservedOutcomeStatus } from './types-CvsKGweN.cjs';
 
 /**
  * Experimental trace-check finding severity.
@@ -596,6 +597,13 @@ declare function createSafetyOversizedAttributeRule(options: SafetyOversizedAttr
  * evolve during the v1.x experimental period.
  */
 declare function createBaselineRegressionRule(options: BaselineRegressionRuleOptions): TraceCheckRule;
+/**
+ * Experimental rule for observed real-world outcomes (v4.4+).
+ */
+interface ObservedOutcomeRuleOptions {
+    failOn?: readonly ObservedOutcomeStatus[];
+}
+declare function createObservedOutcomeRule(options?: ObservedOutcomeRuleOptions): TraceCheckRule;
 /**
  * Run experimental deterministic trace checks against normalized reader output.
  *
@@ -607,4 +615,4 @@ declare function createBaselineRegressionRule(options: BaselineRegressionRuleOpt
  */
 declare function runTraceChecks(input: TraceCheckInput, options?: RunTraceChecksOptions): TraceCheckResult;
 
-export { type BaselineRegressionRuleOptions, type DecisionRuleOptions, type GuardrailRuleOptions, type LlmUsageRuleOptions, type MaxStepDurationRuleOptions, type RetrievalRuleOptions, type RunDepthRuleOptions, type RunDurationRuleOptions, type RunEventCountRuleOptions, type RunStatusRuleOptions, type RunTraceChecksOptions, type SafetyOversizedAttributeRuleOptions, type SafetyRawContentRuleOptions, type SafetyRedactionRuleOptions, type SafetySecretPattern, type SafetySecretPatternRuleOptions, type StallDetectionRuleOptions, type StructureIncompleteRuleOptions, type StructureOrphanRuleOptions, type StructureParallelWidthRuleOptions, type StructureRelationshipRuleOptions, type ToolFailureRuleOptions, type ToolOrderingRuleOptions, type ToolUsageRuleOptions, type TraceCheckContext, type TraceCheckDiagnostic, type TraceCheckDiagnosticCode, type TraceCheckEvidence, type TraceCheckFacts, type TraceCheckFinding, type TraceCheckFindingStatus, type TraceCheckInput, type TraceCheckResult, type TraceCheckRule, type TraceCheckRuleCategory, type TraceCheckSeverity, type TraceCheckStatus, type TraceCheckSummary, type TraceSignalRuleOptions, createBaselineRegressionRule, createDecisionRule, createGuardrailRule, createLlmUsageRule, createMaxStepDurationRule, createRequireCompletedRule, createRetrievalRule, createRunDepthRule, createRunDurationRule, createRunEventCountRule, createRunStatusRule, createSafetyOversizedAttributeRule, createSafetyRawContentRule, createSafetyRedactionRule, createSafetySecretPatternRule, createStallDetectionRule, createStructureCycleRule, createStructureIncompleteRule, createStructureOrphanRule, createStructureParallelWidthRule, createStructureRelationshipRule, createToolFailureRule, createToolOrderingRule, createToolUsageRule, runTraceChecks };
+export { type BaselineRegressionRuleOptions, type DecisionRuleOptions, type GuardrailRuleOptions, type LlmUsageRuleOptions, type MaxStepDurationRuleOptions, type ObservedOutcomeRuleOptions, type RetrievalRuleOptions, type RunDepthRuleOptions, type RunDurationRuleOptions, type RunEventCountRuleOptions, type RunStatusRuleOptions, type RunTraceChecksOptions, type SafetyOversizedAttributeRuleOptions, type SafetyRawContentRuleOptions, type SafetyRedactionRuleOptions, type SafetySecretPattern, type SafetySecretPatternRuleOptions, type StallDetectionRuleOptions, type StructureIncompleteRuleOptions, type StructureOrphanRuleOptions, type StructureParallelWidthRuleOptions, type StructureRelationshipRuleOptions, type ToolFailureRuleOptions, type ToolOrderingRuleOptions, type ToolUsageRuleOptions, type TraceCheckContext, type TraceCheckDiagnostic, type TraceCheckDiagnosticCode, type TraceCheckEvidence, type TraceCheckFacts, type TraceCheckFinding, type TraceCheckFindingStatus, type TraceCheckInput, type TraceCheckResult, type TraceCheckRule, type TraceCheckRuleCategory, type TraceCheckSeverity, type TraceCheckStatus, type TraceCheckSummary, type TraceSignalRuleOptions, createBaselineRegressionRule, createDecisionRule, createGuardrailRule, createLlmUsageRule, createMaxStepDurationRule, createObservedOutcomeRule, createRequireCompletedRule, createRetrievalRule, createRunDepthRule, createRunDurationRule, createRunEventCountRule, createRunStatusRule, createSafetyOversizedAttributeRule, createSafetyRawContentRule, createSafetyRedactionRule, createSafetySecretPatternRule, createStallDetectionRule, createStructureCycleRule, createStructureIncompleteRule, createStructureOrphanRule, createStructureParallelWidthRule, createStructureRelationshipRule, createToolFailureRule, createToolOrderingRule, createToolUsageRule, runTraceChecks };

package/packages/core/dist/checks.d.ts

@@ -1,6 +1,7 @@
 import { TraceReadResult, TraceReadWarning } from './readers.js';
-import { b as InspectRunTree, c as InspectNode, A as AttributionConfidence } from './inspect-event-CYAV7Hxm.js';
-import { P as PersistedInspectEvent } from './persisted-inspect-event-BLvb0jSX.js';
+import { b as InspectRunTree, c as InspectNode, A as AttributionConfidence } from './inspect-event-IqwOHbGo.js';
+import { P as PersistedInspectEvent } from './persisted-inspect-event-DruwqARZ.js';
+import { a as ObservedOutcomeStatus } from './types-CvsKGweN.js';
 
 /**
  * Experimental trace-check finding severity.
@@ -596,6 +597,13 @@ declare function createSafetyOversizedAttributeRule(options: SafetyOversizedAttr
  * evolve during the v1.x experimental period.
  */
 declare function createBaselineRegressionRule(options: BaselineRegressionRuleOptions): TraceCheckRule;
+/**
+ * Experimental rule for observed real-world outcomes (v4.4+).
+ */
+interface ObservedOutcomeRuleOptions {
+    failOn?: readonly ObservedOutcomeStatus[];
+}
+declare function createObservedOutcomeRule(options?: ObservedOutcomeRuleOptions): TraceCheckRule;
 /**
  * Run experimental deterministic trace checks against normalized reader output.
  *
@@ -607,4 +615,4 @@ declare function createBaselineRegressionRule(options: BaselineRegressionRuleOpt
  */
 declare function runTraceChecks(input: TraceCheckInput, options?: RunTraceChecksOptions): TraceCheckResult;
 
-export { type BaselineRegressionRuleOptions, type DecisionRuleOptions, type GuardrailRuleOptions, type LlmUsageRuleOptions, type MaxStepDurationRuleOptions, type RetrievalRuleOptions, type RunDepthRuleOptions, type RunDurationRuleOptions, type RunEventCountRuleOptions, type RunStatusRuleOptions, type RunTraceChecksOptions, type SafetyOversizedAttributeRuleOptions, type SafetyRawContentRuleOptions, type SafetyRedactionRuleOptions, type SafetySecretPattern, type SafetySecretPatternRuleOptions, type StallDetectionRuleOptions, type StructureIncompleteRuleOptions, type StructureOrphanRuleOptions, type StructureParallelWidthRuleOptions, type StructureRelationshipRuleOptions, type ToolFailureRuleOptions, type ToolOrderingRuleOptions, type ToolUsageRuleOptions, type TraceCheckContext, type TraceCheckDiagnostic, type TraceCheckDiagnosticCode, type TraceCheckEvidence, type TraceCheckFacts, type TraceCheckFinding, type TraceCheckFindingStatus, type TraceCheckInput, type TraceCheckResult, type TraceCheckRule, type TraceCheckRuleCategory, type TraceCheckSeverity, type TraceCheckStatus, type TraceCheckSummary, type TraceSignalRuleOptions, createBaselineRegressionRule, createDecisionRule, createGuardrailRule, createLlmUsageRule, createMaxStepDurationRule, createRequireCompletedRule, createRetrievalRule, createRunDepthRule, createRunDurationRule, createRunEventCountRule, createRunStatusRule, createSafetyOversizedAttributeRule, createSafetyRawContentRule, createSafetyRedactionRule, createSafetySecretPatternRule, createStallDetectionRule, createStructureCycleRule, createStructureIncompleteRule, createStructureOrphanRule, createStructureParallelWidthRule, createStructureRelationshipRule, createToolFailureRule, createToolOrderingRule, createToolUsageRule, runTraceChecks };
+export { type BaselineRegressionRuleOptions, type DecisionRuleOptions, type GuardrailRuleOptions, type LlmUsageRuleOptions, type MaxStepDurationRuleOptions, type ObservedOutcomeRuleOptions, type RetrievalRuleOptions, type RunDepthRuleOptions, type RunDurationRuleOptions, type RunEventCountRuleOptions, type RunStatusRuleOptions, type RunTraceChecksOptions, type SafetyOversizedAttributeRuleOptions, type SafetyRawContentRuleOptions, type SafetyRedactionRuleOptions, type SafetySecretPattern, type SafetySecretPatternRuleOptions, type StallDetectionRuleOptions, type StructureIncompleteRuleOptions, type StructureOrphanRuleOptions, type StructureParallelWidthRuleOptions, type StructureRelationshipRuleOptions, type ToolFailureRuleOptions, type ToolOrderingRuleOptions, type ToolUsageRuleOptions, type TraceCheckContext, type TraceCheckDiagnostic, type TraceCheckDiagnosticCode, type TraceCheckEvidence, type TraceCheckFacts, type TraceCheckFinding, type TraceCheckFindingStatus, type TraceCheckInput, type TraceCheckResult, type TraceCheckRule, type TraceCheckRuleCategory, type TraceCheckSeverity, type TraceCheckStatus, type TraceCheckSummary, type TraceSignalRuleOptions, createBaselineRegressionRule, createDecisionRule, createGuardrailRule, createLlmUsageRule, createMaxStepDurationRule, createObservedOutcomeRule, createRequireCompletedRule, createRetrievalRule, createRunDepthRule, createRunDurationRule, createRunEventCountRule, createRunStatusRule, createSafetyOversizedAttributeRule, createSafetyRawContentRule, createSafetyRedactionRule, createSafetySecretPatternRule, createStallDetectionRule, createStructureCycleRule, createStructureIncompleteRule, createStructureOrphanRule, createStructureParallelWidthRule, createStructureRelationshipRule, createToolFailureRule, createToolOrderingRule, createToolUsageRule, runTraceChecks };

package/packages/core/dist/checks.mjs

@@ -1,3 +1,10 @@
+import { extractOutcomesFromPersistedEvents, outcomesMatchingStatus } from './chunk-VFO76UH3.mjs';
+import './chunk-ULCGIRTC.mjs';
+import './chunk-VU6O5QAH.mjs';
+import './chunk-WRE4CNPF.mjs';
+import './chunk-P366CCL2.mjs';
+import './chunk-7TGZLWEE.mjs';
+
 // packages/core/src/checks/index.ts
 var SEVERITY_RANK = {
   error: 0,
@@ -1564,6 +1571,40 @@ function createBaselineRegressionRule(options) {
     }
   };
 }
+function createObservedOutcomeRule(options = {}) {
+  const failOn = options.failOn ?? ["failed"];
+  return {
+    id: "outcome.status",
+    category: "run",
+    defaultSeverity: "error",
+    evaluate(context) {
+      const outcomes = extractOutcomesFromPersistedEvents(context.events);
+      const matching = outcomesMatchingStatus(outcomes, failOn);
+      if (matching.length === 0) return [];
+      return [
+        failFinding(
+          "outcome.status",
+          `Observed outcome count ${matching.length} matched [${failOn.join(", ")}].`,
+          matching.map((outcome) => ({
+            runId: outcome.runId,
+            eventId: outcome.outcomeId,
+            ...outcome.parentId !== void 0 ? { parentId: outcome.parentId } : {},
+            kind: "OUTCOME",
+            name: outcome.name,
+            status: outcome.status,
+            path: `outcome.${outcome.name}`
+          })),
+          { failOn },
+          matching.map((outcome) => ({
+            name: outcome.name,
+            status: outcome.status,
+            expectation: outcome.expectation
+          }))
+        )
+      ];
+    }
+  };
+}
 function runTraceChecks(input, options = {}) {
   const selected = resolveSelectedRun(input, options.runId);
   if (selected.diagnostics.length > 0) {
@@ -1609,6 +1650,6 @@ function runTraceChecks(input, options = {}) {
   };
 }
 
-export { createBaselineRegressionRule, createDecisionRule, createGuardrailRule, createLlmUsageRule, createMaxStepDurationRule, createRequireCompletedRule, createRetrievalRule, createRunDepthRule, createRunDurationRule, createRunEventCountRule, createRunStatusRule, createSafetyOversizedAttributeRule, createSafetyRawContentRule, createSafetyRedactionRule, createSafetySecretPatternRule, createStallDetectionRule, createStructureCycleRule, createStructureIncompleteRule, createStructureOrphanRule, createStructureParallelWidthRule, createStructureRelationshipRule, createToolFailureRule, createToolOrderingRule, createToolUsageRule, runTraceChecks };
+export { createBaselineRegressionRule, createDecisionRule, createGuardrailRule, createLlmUsageRule, createMaxStepDurationRule, createObservedOutcomeRule, createRequireCompletedRule, createRetrievalRule, createRunDepthRule, createRunDurationRule, createRunEventCountRule, createRunStatusRule, createSafetyOversizedAttributeRule, createSafetyRawContentRule, createSafetyRedactionRule, createSafetySecretPatternRule, createStallDetectionRule, createStructureCycleRule, createStructureIncompleteRule, createStructureOrphanRule, createStructureParallelWidthRule, createStructureRelationshipRule, createToolFailureRule, createToolOrderingRule, createToolUsageRule, runTraceChecks };
 //# sourceMappingURL=checks.mjs.map
 //# sourceMappingURL=checks.mjs.map