@agent-inspect/eval 2.4.0 → 2.6.0

package/dist/index.d.ts

@@ -1,5 +1,89 @@
 import { TraceReadResult } from 'agent-inspect/readers';
 
+type CircuitRuleId = "circuit.same-tool-repetition" | "circuit.same-args-repetition" | "circuit.max-loop-iterations" | "circuit.max-retries" | "circuit.tool-timeout" | "circuit.runaway-llm-loop" | "circuit.excessive-branch-width";
+interface ThresholdRule {
+    maxRepeats: number;
+}
+interface MaxIterationsRule {
+    maxIterations: number;
+}
+interface MaxRetriesRule {
+    maxRetries: number;
+}
+interface ToolTimeoutRule {
+    maxDurationMs: number;
+}
+interface RunawayLlmLoopRule {
+    maxLlmCalls: number;
+}
+interface ExcessiveBranchWidthRule {
+    maxWidth: number;
+}
+interface RunCircuitsOptions {
+    rules?: readonly CircuitRuleId[];
+    sameToolRepetition?: ThresholdRule;
+    sameArgsRepetition?: ThresholdRule;
+    maxLoopIterations?: MaxIterationsRule;
+    maxRetries?: MaxRetriesRule;
+    toolTimeout?: ToolTimeoutRule;
+    runawayLlmLoop?: RunawayLlmLoopRule;
+    excessiveBranchWidth?: ExcessiveBranchWidthRule;
+}
+
+type GuardrailRuleId = "guardrail.banned-phrase" | "guardrail.pii-leak" | "guardrail.unsafe-tool-args" | "guardrail.prompt-injection" | "guardrail.structured-output" | "guardrail.oversize-output" | "guardrail.required-json-shape";
+interface BannedPhraseOptions {
+    phrases: readonly string[];
+    caseInsensitive?: boolean;
+}
+interface PiiLeakOptions {
+    profile?: "local" | "share" | "strict";
+    minSeverity?: "info" | "warning" | "error";
+}
+interface UnsafeToolArgsOptions {
+    blockedTools?: readonly string[];
+    maxDepth?: number;
+    maxStringLength?: number;
+}
+interface PromptInjectionOptions {
+    patterns?: readonly string[];
+}
+interface JsonSchemaField {
+    type?: "string" | "number" | "boolean" | "object" | "array";
+    enum?: readonly unknown[];
+    required?: readonly string[];
+}
+interface StructuredOutputOptions {
+    schema: Record<string, JsonSchemaField>;
+}
+interface OversizeOutputOptions {
+    maxLength?: number;
+    maxSerializedLength?: number;
+}
+interface RequiredJsonShapeOptions {
+    requiredKeys: readonly string[];
+}
+interface GuardrailRuleOptions {
+    bannedPhrase?: BannedPhraseOptions;
+    piiLeak?: PiiLeakOptions;
+    unsafeToolArgs?: UnsafeToolArgsOptions;
+    promptInjection?: PromptInjectionOptions;
+    structuredOutput?: StructuredOutputOptions;
+    oversizeOutput?: OversizeOutputOptions;
+    requiredJsonShape?: RequiredJsonShapeOptions;
+}
+interface RunGuardrailsOptions extends GuardrailRuleOptions {
+    rules?: readonly GuardrailRuleId[];
+}
+
+interface EvalGuardrailRuleOptions extends RunGuardrailsOptions {
+    rules: readonly GuardrailRuleId[];
+}
+interface EvalCircuitRuleOptions extends RunCircuitsOptions {
+    rules: readonly CircuitRuleId[];
+}
+declare function createEvalGuardrailRule(options: EvalGuardrailRuleOptions): EvalRule;
+declare function createEvalCircuitRule(options: EvalCircuitRuleOptions): EvalRule;
+
 type EvalStatus = "pass" | "fail" | "error";
 type EvalSeverity = "error" | "warning" | "info";
 type EvalFindingStatus = "pass" | "fail" | "warning";
@@ -126,4 +210,4 @@ declare const checks: {
 };
 declare function renderEvalMarkdown(result: EvalRunResult): string;
 
-export { type AnswerLengthBoundsOptions, type BannedUnsupportedPhrasesOptions, type CitationPresenceOptions, type ContextOverlapOptions, type EvalContext, type EvalDiagnostic, type EvalDiagnosticCode, type EvalEvidence, type EvalFinding, type EvalFindingStatus, type EvalInput, type EvalRule, type EvalRuleCategory, type EvalRunInput, type EvalRunOptions, type EvalRunResult, type EvalSeverity, type EvalStatus, type EvalSummary, type QuoteOverlapOptions, type RequiredSourceIdsOptions, checks, evalRun, renderEvalMarkdown };
+export { type AnswerLengthBoundsOptions, type BannedUnsupportedPhrasesOptions, type CitationPresenceOptions, type ContextOverlapOptions, type EvalCircuitRuleOptions, type EvalContext, type EvalDiagnostic, type EvalDiagnosticCode, type EvalEvidence, type EvalFinding, type EvalFindingStatus, type EvalGuardrailRuleOptions, type EvalInput, type EvalRule, type EvalRuleCategory, type EvalRunInput, type EvalRunOptions, type EvalRunResult, type EvalSeverity, type EvalStatus, type EvalSummary, type QuoteOverlapOptions, type RequiredSourceIdsOptions, checks, createEvalCircuitRule, createEvalGuardrailRule, evalRun, renderEvalMarkdown };