agent-inspect 1.6.0 → 1.8.0

package/packages/core/dist/checks.d.ts

@@ -0,0 +1,585 @@
+import { TraceReadResult, TraceReadWarning } from './readers.js';
+import { a as InspectRunTree, b as InspectNode, A as AttributionConfidence } from './inspect-event-CevRYp58.js';
+import { P as PersistedInspectEvent } from './persisted-inspect-event-Cw7TeYGr.js';
+
+/**
+ * Experimental trace-check finding severity.
+ *
+ * @experimental Available through `agent-inspect/checks`; the checks API may
+ * evolve during the v1.x experimental period.
+ */
+type TraceCheckSeverity = "error" | "warning" | "info";
+/**
+ * Experimental trace-check rule category.
+ *
+ * @experimental Available through `agent-inspect/checks`; the checks API may
+ * evolve during the v1.x experimental period.
+ */
+type TraceCheckRuleCategory = "run" | "tool" | "llm" | "structure" | "baseline" | "safety" | "reader";
+/**
+ * Experimental trace-check finding status.
+ *
+ * @experimental Available through `agent-inspect/checks`; the checks API may
+ * evolve during the v1.x experimental period.
+ */
+type TraceCheckFindingStatus = "pass" | "fail" | "warning";
+/**
+ * Experimental trace-check aggregate status.
+ *
+ * @experimental Available through `agent-inspect/checks`; the checks API may
+ * evolve during the v1.x experimental period.
+ */
+type TraceCheckStatus = "pass" | "fail" | "error";
+/**
+ * Experimental stable diagnostic code for check execution and input errors.
+ *
+ * @experimental Available through `agent-inspect/checks`; the checks API may
+ * evolve during the v1.x experimental period.
+ */
+type TraceCheckDiagnosticCode = "AI_CHECK_INVALID_ARGUMENTS" | "AI_CHECK_INVALID_CONFIG" | "AI_CHECK_CONFIG_LOAD_FAILED" | "AI_CHECK_TRACE_UNREADABLE" | "AI_CHECK_UNSUPPORTED_FORMAT" | "AI_CHECK_AMBIGUOUS_FORMAT" | "AI_CHECK_RUN_SELECTION_REQUIRED" | "AI_CHECK_BASELINE_UNREADABLE" | "AI_CHECK_BASELINE_INCOMPATIBLE" | "AI_CHECK_RULE_FAILED" | "AI_CHECK_INTERNAL_ERROR";
+/**
+ * Experimental evidence pointing at trace data relevant to a finding.
+ *
+ * Evidence intentionally identifies runs/events/spans and bounded paths rather
+ * than embedding raw prompts, outputs, request bodies, headers, or tool payloads.
+ *
+ * @experimental Available through `agent-inspect/checks`; the checks API may
+ * evolve during the v1.x experimental period.
+ */
+interface TraceCheckEvidence {
+    runId?: string;
+    eventId?: string;
+    parentId?: string;
+    traceId?: string;
+    spanId?: string;
+    kind?: string;
+    name?: string;
+    status?: string;
+    path?: string;
+}
+/**
+ * Experimental finding emitted by a trace-check rule.
+ *
+ * @experimental Available through `agent-inspect/checks`; the checks API may
+ * evolve during the v1.x experimental period.
+ */
+interface TraceCheckFinding {
+    ruleId: string;
+    severity: TraceCheckSeverity;
+    status: TraceCheckFindingStatus;
+    message: string;
+    expected?: unknown;
+    actual?: unknown;
+    evidence: TraceCheckEvidence[];
+}
+/**
+ * Experimental diagnostic emitted when check execution cannot complete normally.
+ *
+ * @experimental Available through `agent-inspect/checks`; the checks API may
+ * evolve during the v1.x experimental period.
+ */
+interface TraceCheckDiagnostic {
+    code: TraceCheckDiagnosticCode;
+    message: string;
+    severity: TraceCheckSeverity;
+    ruleId?: string;
+}
+/**
+ * Experimental input projection for trace checks.
+ *
+ * The `read` value must come from `agent-inspect/readers` or an equivalent
+ * already-normalized `TraceReadResult`. The checks engine does not read files
+ * or reparse source payloads.
+ *
+ * @experimental Available through `agent-inspect/checks`; the checks API may
+ * evolve during the v1.x experimental period.
+ */
+interface TraceCheckInput {
+    read: TraceReadResult;
+    selectedRun?: InspectRunTree;
+    sourceLabel?: string;
+}
+/**
+ * Experimental normalized facts available to trace-check rules.
+ *
+ * @experimental Available through `agent-inspect/checks`; the checks API may
+ * evolve during the v1.x experimental period.
+ */
+interface TraceCheckFacts {
+    format: string;
+    runs: readonly InspectRunTree[];
+    events: readonly PersistedInspectEvent[];
+    readerWarnings: readonly TraceReadWarning[];
+    unsupportedFields: readonly string[];
+    sourceFiles: readonly string[];
+    nodesByEventId: ReadonlyMap<string, InspectNode>;
+    childrenByParentId: ReadonlyMap<string, readonly InspectNode[]>;
+    rootNodes: readonly InspectNode[];
+}
+/**
+ * Experimental rule evaluation context.
+ *
+ * @experimental Available through `agent-inspect/checks`; the checks API may
+ * evolve during the v1.x experimental period.
+ */
+interface TraceCheckContext extends TraceCheckFacts {
+    selectedRun?: InspectRunTree;
+    sourceLabel?: string;
+}
+/**
+ * Experimental trace-check rule definition.
+ *
+ * Rules are synchronous and pure: they receive normalized facts and return
+ * findings without reading files, mutating input, or performing network I/O.
+ *
+ * @experimental Available through `agent-inspect/checks`; the checks API may
+ * evolve during the v1.x experimental period.
+ */
+interface TraceCheckRule {
+    id: string;
+    category: TraceCheckRuleCategory;
+    defaultSeverity: TraceCheckSeverity;
+    evaluate(context: TraceCheckContext): readonly TraceCheckFinding[];
+}
+/**
+ * Experimental options for `runTraceChecks`.
+ *
+ * @experimental Available through `agent-inspect/checks`; the checks API may
+ * evolve during the v1.x experimental period.
+ */
+interface RunTraceChecksOptions {
+    rules?: readonly TraceCheckRule[];
+    select?: readonly string[];
+    runId?: string;
+}
+/**
+ * Experimental options for the built-in run status rule.
+ *
+ * @experimental Available through `agent-inspect/checks`; the checks API may
+ * evolve during the v1.x experimental period.
+ */
+interface RunStatusRuleOptions {
+    expected?: "ok" | "error" | "running";
+    allowIncomplete?: boolean;
+}
+/**
+ * Experimental options for the built-in run duration rule.
+ *
+ * @experimental Available through `agent-inspect/checks`; the checks API may
+ * evolve during the v1.x experimental period.
+ */
+interface RunDurationRuleOptions {
+    maxDurationMs: number;
+}
+/**
+ * Experimental options for the built-in event count rule.
+ *
+ * @experimental Available through `agent-inspect/checks`; the checks API may
+ * evolve during the v1.x experimental period.
+ */
+interface RunEventCountRuleOptions {
+    kind?: PersistedInspectEvent["kind"];
+    min?: number;
+    max?: number;
+}
+/**
+ * Experimental options for the built-in run depth rule.
+ *
+ * @experimental Available through `agent-inspect/checks`; the checks API may
+ * evolve during the v1.x experimental period.
+ */
+interface RunDepthRuleOptions {
+    maxDepth: number;
+}
+/**
+ * Experimental options for the built-in tool usage rule.
+ *
+ * @experimental Available through `agent-inspect/checks`; the checks API may
+ * evolve during the v1.x experimental period.
+ */
+interface ToolUsageRuleOptions {
+    required?: readonly string[];
+    forbidden?: readonly string[];
+    allowed?: readonly string[];
+    minCount?: number;
+    maxCount?: number;
+}
+/**
+ * Experimental options for the built-in tool ordering rule.
+ *
+ * @experimental Available through `agent-inspect/checks`; the checks API may
+ * evolve during the v1.x experimental period.
+ */
+interface ToolOrderingRuleOptions {
+    before: string;
+    after: string;
+}
+/**
+ * Experimental options for the built-in tool failure rule.
+ *
+ * @experimental Available through `agent-inspect/checks`; the checks API may
+ * evolve during the v1.x experimental period.
+ */
+interface ToolFailureRuleOptions {
+    maxFailures?: number;
+    maxRetries?: number;
+}
+/**
+ * Experimental options for the built-in LLM usage rule.
+ *
+ * @experimental Available through `agent-inspect/checks`; the checks API may
+ * evolve during the v1.x experimental period.
+ */
+interface LlmUsageRuleOptions {
+    allowedModels?: readonly string[];
+    allowedProviders?: readonly string[];
+    maxCalls?: number;
+    maxInputTokens?: number;
+    maxOutputTokens?: number;
+    maxTotalTokens?: number;
+    maxCachedTokens?: number;
+    finishReasons?: readonly string[];
+}
+/**
+ * Experimental options for the built-in structure incomplete rule.
+ *
+ * @experimental Available through `agent-inspect/checks`; the checks API may
+ * evolve during the v1.x experimental period.
+ */
+interface StructureIncompleteRuleOptions {
+    allowRunning?: boolean;
+    requireEndedAtForStarted?: boolean;
+}
+/**
+ * Experimental options for the built-in structure orphan rule.
+ *
+ * @experimental Available through `agent-inspect/checks`; the checks API may
+ * evolve during the v1.x experimental period.
+ */
+interface StructureOrphanRuleOptions {
+    allowMarkedUnresolved?: boolean;
+}
+/**
+ * Experimental options for the built-in structure relationship rule.
+ *
+ * @experimental Available through `agent-inspect/checks`; the checks API may
+ * evolve during the v1.x experimental period.
+ */
+interface StructureRelationshipRuleOptions {
+    minConfidence?: AttributionConfidence;
+    requireParentBeforeChild?: boolean;
+    requireTraceParentSpan?: boolean;
+}
+/**
+ * Experimental options for the built-in structure parallel-width rule.
+ *
+ * @experimental Available through `agent-inspect/checks`; the checks API may
+ * evolve during the v1.x experimental period.
+ */
+interface StructureParallelWidthRuleOptions {
+    maxChildren?: number;
+    maxConcurrent?: number;
+}
+/**
+ * Experimental options shared by built-in signal rules.
+ *
+ * @experimental Available through `agent-inspect/checks`; the checks API may
+ * evolve during the v1.x experimental period.
+ */
+interface TraceSignalRuleOptions {
+    required?: readonly string[];
+    forbidden?: readonly string[];
+    allowed?: readonly string[];
+    minCount?: number;
+    maxCount?: number;
+}
+/**
+ * Experimental options for the built-in retrieval signal rule.
+ *
+ * @experimental Available through `agent-inspect/checks`; the checks API may
+ * evolve during the v1.x experimental period.
+ */
+interface RetrievalRuleOptions extends TraceSignalRuleOptions {
+}
+/**
+ * Experimental options for the built-in guardrail signal rule.
+ *
+ * @experimental Available through `agent-inspect/checks`; the checks API may
+ * evolve during the v1.x experimental period.
+ */
+interface GuardrailRuleOptions extends TraceSignalRuleOptions {
+}
+/**
+ * Experimental options for the built-in decision signal rule.
+ *
+ * @experimental Available through `agent-inspect/checks`; the checks API may
+ * evolve during the v1.x experimental period.
+ */
+interface DecisionRuleOptions extends TraceSignalRuleOptions {
+}
+/**
+ * Experimental options for the built-in safety redaction rule.
+ *
+ * @experimental Available through `agent-inspect/checks`; the checks API may
+ * evolve during the v1.x experimental period.
+ */
+interface SafetyRedactionRuleOptions {
+    sensitiveKeys?: readonly string[];
+    redactedMarkers?: readonly string[];
+    maxFindings?: number;
+}
+/**
+ * Experimental options for the built-in raw content path rule.
+ *
+ * @experimental Available through `agent-inspect/checks`; the checks API may
+ * evolve during the v1.x experimental period.
+ */
+interface SafetyRawContentRuleOptions {
+    forbiddenKeys?: readonly string[];
+    includeSummaries?: boolean;
+    maxFindings?: number;
+}
+/**
+ * Experimental secret pattern used by the built-in secret safety rule.
+ *
+ * @experimental Available through `agent-inspect/checks`; the checks API may
+ * evolve during the v1.x experimental period.
+ */
+interface SafetySecretPattern {
+    id: string;
+    pattern: RegExp;
+}
+/**
+ * Experimental options for the built-in secret pattern safety rule.
+ *
+ * @experimental Available through `agent-inspect/checks`; the checks API may
+ * evolve during the v1.x experimental period.
+ */
+interface SafetySecretPatternRuleOptions {
+    patterns?: readonly SafetySecretPattern[];
+    maxStringLength?: number;
+    maxFindings?: number;
+}
+/**
+ * Experimental options for the built-in oversized attribute safety rule.
+ *
+ * @experimental Available through `agent-inspect/checks`; the checks API may
+ * evolve during the v1.x experimental period.
+ */
+interface SafetyOversizedAttributeRuleOptions {
+    maxStringLength?: number;
+    maxArrayLength?: number;
+    maxObjectKeys?: number;
+    maxSerializedBytes?: number;
+    maxFindings?: number;
+}
+/**
+ * Experimental options for the built-in baseline regression rule.
+ *
+ * The baseline must already be normalized through `agent-inspect/readers` or an
+ * equivalent `TraceReadResult`; this rule does not read files or parse traces.
+ * `durationToleranceMs` defaults to `0`, meaning exact duration comparison.
+ *
+ * @experimental Available through `agent-inspect/checks`; the checks API may
+ * evolve during the v1.x experimental period.
+ */
+interface BaselineRegressionRuleOptions {
+    baseline: TraceCheckInput;
+    baselineRunId?: string;
+    durationToleranceMs?: number;
+    compareFormat?: boolean;
+}
+/**
+ * Experimental aggregate counts for trace-check results.
+ *
+ * @experimental Available through `agent-inspect/checks`; the checks API may
+ * evolve during the v1.x experimental period.
+ */
+interface TraceCheckSummary {
+    passed: number;
+    failed: number;
+    warnings: number;
+    errors: number;
+}
+/**
+ * Experimental trace-check result.
+ *
+ * `status: "fail"` means rules ran and at least one error-severity finding
+ * failed. `status: "error"` means execution could not complete because of
+ * invalid input, invalid rule selection, or a thrown rule error.
+ *
+ * @experimental Available through `agent-inspect/checks`; the checks API may
+ * evolve during the v1.x experimental period.
+ */
+interface TraceCheckResult {
+    ok: boolean;
+    status: TraceCheckStatus;
+    format: string;
+    runId?: string;
+    summary: TraceCheckSummary;
+    findings: TraceCheckFinding[];
+    diagnostics: TraceCheckDiagnostic[];
+}
+/**
+ * Create the experimental built-in run status rule.
+ *
+ * @experimental Available through `agent-inspect/checks`; the checks API may
+ * evolve during the v1.x experimental period.
+ */
+declare function createRunStatusRule(options?: RunStatusRuleOptions): TraceCheckRule;
+/**
+ * Create the experimental built-in run duration rule.
+ *
+ * @experimental Available through `agent-inspect/checks`; the checks API may
+ * evolve during the v1.x experimental period.
+ */
+declare function createRunDurationRule(options: RunDurationRuleOptions): TraceCheckRule;
+/**
+ * Create the experimental built-in event count rule.
+ *
+ * @experimental Available through `agent-inspect/checks`; the checks API may
+ * evolve during the v1.x experimental period.
+ */
+declare function createRunEventCountRule(options: RunEventCountRuleOptions): TraceCheckRule;
+/**
+ * Create the experimental built-in run depth rule.
+ *
+ * @experimental Available through `agent-inspect/checks`; the checks API may
+ * evolve during the v1.x experimental period.
+ */
+declare function createRunDepthRule(options: RunDepthRuleOptions): TraceCheckRule;
+/**
+ * Create the experimental built-in tool usage rule.
+ *
+ * @experimental Available through `agent-inspect/checks`; the checks API may
+ * evolve during the v1.x experimental period.
+ */
+declare function createToolUsageRule(options: ToolUsageRuleOptions): TraceCheckRule;
+/**
+ * Create the experimental built-in tool ordering rule.
+ *
+ * @experimental Available through `agent-inspect/checks`; the checks API may
+ * evolve during the v1.x experimental period.
+ */
+declare function createToolOrderingRule(options: ToolOrderingRuleOptions): TraceCheckRule;
+/**
+ * Create the experimental built-in tool failure/retry rule.
+ *
+ * @experimental Available through `agent-inspect/checks`; the checks API may
+ * evolve during the v1.x experimental period.
+ */
+declare function createToolFailureRule(options: ToolFailureRuleOptions): TraceCheckRule;
+/**
+ * Create the experimental built-in LLM usage rule.
+ *
+ * @experimental Available through `agent-inspect/checks`; the checks API may
+ * evolve during the v1.x experimental period.
+ */
+declare function createLlmUsageRule(options: LlmUsageRuleOptions): TraceCheckRule;
+/**
+ * Create the experimental built-in structure incomplete rule.
+ *
+ * @experimental Available through `agent-inspect/checks`; the checks API may
+ * evolve during the v1.x experimental period.
+ */
+declare function createStructureIncompleteRule(options?: StructureIncompleteRuleOptions): TraceCheckRule;
+/**
+ * Create the experimental built-in structure orphan rule.
+ *
+ * @experimental Available through `agent-inspect/checks`; the checks API may
+ * evolve during the v1.x experimental period.
+ */
+declare function createStructureOrphanRule(options?: StructureOrphanRuleOptions): TraceCheckRule;
+/**
+ * Create the experimental built-in structure cycle rule.
+ *
+ * @experimental Available through `agent-inspect/checks`; the checks API may
+ * evolve during the v1.x experimental period.
+ */
+declare function createStructureCycleRule(): TraceCheckRule;
+/**
+ * Create the experimental built-in structure relationship rule.
+ *
+ * @experimental Available through `agent-inspect/checks`; the checks API may
+ * evolve during the v1.x experimental period.
+ */
+declare function createStructureRelationshipRule(options?: StructureRelationshipRuleOptions): TraceCheckRule;
+/**
+ * Create the experimental built-in structure parallel-width rule.
+ *
+ * @experimental Available through `agent-inspect/checks`; the checks API may
+ * evolve during the v1.x experimental period.
+ */
+declare function createStructureParallelWidthRule(options: StructureParallelWidthRuleOptions): TraceCheckRule;
+/**
+ * Create the experimental built-in retrieval signal rule.
+ *
+ * @experimental Available through `agent-inspect/checks`; the checks API may
+ * evolve during the v1.x experimental period.
+ */
+declare function createRetrievalRule(options: RetrievalRuleOptions): TraceCheckRule;
+/**
+ * Create the experimental built-in guardrail signal rule.
+ *
+ * @experimental Available through `agent-inspect/checks`; the checks API may
+ * evolve during the v1.x experimental period.
+ */
+declare function createGuardrailRule(options: GuardrailRuleOptions): TraceCheckRule;
+/**
+ * Create the experimental built-in decision signal rule.
+ *
+ * @experimental Available through `agent-inspect/checks`; the checks API may
+ * evolve during the v1.x experimental period.
+ */
+declare function createDecisionRule(options: DecisionRuleOptions): TraceCheckRule;
+/**
+ * Create the experimental built-in safety redaction rule.
+ *
+ * @experimental Available through `agent-inspect/checks`; the checks API may
+ * evolve during the v1.x experimental period.
+ */
+declare function createSafetyRedactionRule(options?: SafetyRedactionRuleOptions): TraceCheckRule;
+/**
+ * Create the experimental built-in raw content path safety rule.
+ *
+ * @experimental Available through `agent-inspect/checks`; the checks API may
+ * evolve during the v1.x experimental period.
+ */
+declare function createSafetyRawContentRule(options?: SafetyRawContentRuleOptions): TraceCheckRule;
+/**
+ * Create the experimental built-in secret pattern safety rule.
+ *
+ * @experimental Available through `agent-inspect/checks`; the checks API may
+ * evolve during the v1.x experimental period.
+ */
+declare function createSafetySecretPatternRule(options?: SafetySecretPatternRuleOptions): TraceCheckRule;
+/**
+ * Create the experimental built-in oversized attribute safety rule.
+ *
+ * @experimental Available through `agent-inspect/checks`; the checks API may
+ * evolve during the v1.x experimental period.
+ */
+declare function createSafetyOversizedAttributeRule(options: SafetyOversizedAttributeRuleOptions): TraceCheckRule;
+/**
+ * Create the experimental built-in baseline regression rule.
+ *
+ * The rule compares safe structural summaries from a normalized baseline
+ * against the candidate context. It intentionally ignores raw prompt/output,
+ * request/response body, header, and tool payload text.
+ *
+ * @experimental Available through `agent-inspect/checks`; the checks API may
+ * evolve during the v1.x experimental period.
+ */
+declare function createBaselineRegressionRule(options: BaselineRegressionRuleOptions): TraceCheckRule;
+/**
+ * Run experimental deterministic trace checks against normalized reader output.
+ *
+ * The engine is local-only and pure: it does not discover config files, read
+ * trace files, call providers, perform network I/O, or mutate input objects.
+ *
+ * @experimental Available through `agent-inspect/checks`; the checks API may
+ * evolve during the v1.x experimental period.
+ */
+declare function runTraceChecks(input: TraceCheckInput, options?: RunTraceChecksOptions): TraceCheckResult;
+
+export { type BaselineRegressionRuleOptions, type DecisionRuleOptions, type GuardrailRuleOptions, type LlmUsageRuleOptions, type RetrievalRuleOptions, type RunDepthRuleOptions, type RunDurationRuleOptions, type RunEventCountRuleOptions, type RunStatusRuleOptions, type RunTraceChecksOptions, type SafetyOversizedAttributeRuleOptions, type SafetyRawContentRuleOptions, type SafetyRedactionRuleOptions, type SafetySecretPattern, type SafetySecretPatternRuleOptions, type StructureIncompleteRuleOptions, type StructureOrphanRuleOptions, type StructureParallelWidthRuleOptions, type StructureRelationshipRuleOptions, type ToolFailureRuleOptions, type ToolOrderingRuleOptions, type ToolUsageRuleOptions, type TraceCheckContext, type TraceCheckDiagnostic, type TraceCheckDiagnosticCode, type TraceCheckEvidence, type TraceCheckFacts, type TraceCheckFinding, type TraceCheckFindingStatus, type TraceCheckInput, type TraceCheckResult, type TraceCheckRule, type TraceCheckRuleCategory, type TraceCheckSeverity, type TraceCheckStatus, type TraceCheckSummary, type TraceSignalRuleOptions, createBaselineRegressionRule, createDecisionRule, createGuardrailRule, createLlmUsageRule, createRetrievalRule, createRunDepthRule, createRunDurationRule, createRunEventCountRule, createRunStatusRule, createSafetyOversizedAttributeRule, createSafetyRawContentRule, createSafetyRedactionRule, createSafetySecretPatternRule, createStructureCycleRule, createStructureIncompleteRule, createStructureOrphanRule, createStructureParallelWidthRule, createStructureRelationshipRule, createToolFailureRule, createToolOrderingRule, createToolUsageRule, runTraceChecks };