@alis-build/harness-eval 0.1.0

package/dist/index.d.ts

@@ -0,0 +1,725 @@
+import { A as Usage, B as isUserMessage, C as SystemInitEvent, D as TextBlock, E as SystemUnknownEvent, F as isSystemInit, I as isSystemRetry, L as isTextBlock, M as UserMessageEvent, N as isAssistantMessage, O as ToolResultBlock, P as isResult, R as isToolResultBlock, S as SystemCompactBoundaryEvent, T as SystemRetryEvent, _ as ContentBlock, a as HarnessAdapter, b as StopReason, c as AssistantTurn, d as ToolCall, f as TrajectoryView, g as AssistantMessageEvent, h as AssistantMessage, i as BaseAdapterConfig, j as UserMessage, k as ToolUseBlock, l as RetryRecord, m as namespaceOf, n as AdapterError, o as ParseErrorRecord, p as UsageSummary, r as AdapterResult, s as SuiteConfig, t as AdapterDiagnostics, u as SessionMeta, v as McpServerStatus, w as SystemPluginInstallEvent, x as StreamEvent, y as ResultEvent, z as isToolUseBlock } from "./types-B9H4IZtA.js";
+import { n as index_d_exports, o as ClaudeCodeOptions } from "./index-6Z17eKZx.js";
+import { _ as ObjectPredicate, a as ProgressEvent, b as ToolPattern, c as RunSuiteOptions, d as TestSuite, f as Assertion, g as LeafPredicate, h as CompoundPredicate, i as ProgressCallback, l as SuiteReport, m as Cardinality, n as CellReport, o as RepetitionError, p as AssertionResult, r as MatrixCell, s as RepetitionResult, t as AssertionStat, u as TestCase, v as Predicate, y as ThresholdedAssertion } from "./types-BQol062t.js";
+import { i as GradingConfig, r as parseSuite, s as ConfigError, t as loadSuite } from "./loader-DTvoVfN0.js";
+import { t as runSuite } from "./suite-BoOvK_lq.js";
+import { Readable } from "node:stream";
+
+//#region src/types/eval-interchange.d.ts
+/**
+ * TypeScript types for eval interchange output.
+ */
+interface InterchangeToolCall {
+  tool_name: string;
+  tool_input: string;
+}
+interface InterchangeTrajectory {
+  tool_calls: InterchangeToolCall[];
+}
+interface TabularToolCall {
+  tool_name: string;
+  tool_input: unknown;
+}
+interface ContentPart {
+  text?: string;
+  function_call?: {
+    name: string;
+    args: unknown;
+  };
+  function_response?: {
+    name: string;
+    response: unknown;
+  };
+}
+interface AgentEvent {
+  author: string;
+  content: {
+    parts: ContentPart[];
+  };
+  event_time?: string;
+  state_delta?: Record<string, unknown>;
+  active_tools?: Array<{
+    name: string;
+  }>;
+}
+interface ConversationTurn {
+  turn_index: number;
+  turn_id?: string;
+  events: AgentEvent[];
+}
+interface AgentConfig {
+  agent_id: string;
+  agent_type?: string;
+  description?: string;
+  instruction?: string;
+  tools?: Array<{
+    name: string;
+  }>;
+  sub_agents?: string[];
+}
+interface AgentTrace {
+  agents: Record<string, AgentConfig>;
+  turns: ConversationTurn[];
+}
+interface EvalDatasetRow {
+  prompt?: string;
+  response?: string;
+  reference?: string;
+  predicted_trajectory: TabularToolCall[];
+  reference_trajectory?: TabularToolCall[];
+  latency_in_seconds: number;
+  failure: 0 | 1;
+  human_ratings?: Record<string, number>;
+}
+interface ProtoTrajectoryInstance {
+  predicted_trajectory: InterchangeTrajectory;
+  reference_trajectory?: InterchangeTrajectory;
+  prompt?: string;
+  response?: string;
+  reference?: string;
+}
+interface TrajectoryMetrics {
+  trajectory_exact_match: number;
+  trajectory_in_order_match: number;
+  trajectory_any_order_match: number;
+  trajectory_precision: number;
+  trajectory_recall: number;
+  trajectory_single_tool_use: number;
+}
+interface ToolCallMetrics {
+  tool_call_valid: number;
+  tool_name_match: number;
+  tool_parameter_key_match: number;
+  tool_parameter_kv_match: number;
+}
+//#endregion
+//#region src/grader/types.d.ts
+/**
+ * Outcome grading types (LLM-as-judge layer).
+ *
+ * Behavioral assertions live in harness-eval assertions; expectations here
+ * are natural-language outcome checks graded from trajectory transcripts.
+ */
+interface GradedExpectation {
+  text: string;
+  passed: boolean;
+  evidence: string;
+}
+interface GradingSummary {
+  passed: number;
+  failed: number;
+  total: number;
+  passRate: number;
+}
+interface EvalFeedbackSuggestion {
+  assertion?: string;
+  reason: string;
+}
+interface EvalFeedback {
+  suggestions: EvalFeedbackSuggestion[];
+  overall: string;
+}
+/** Grading result for one repetition. */
+interface RepGradingResult {
+  caseId: string;
+  cellLabel: string;
+  repetitionIndex: number;
+  prompt: string;
+  expectations: GradedExpectation[];
+  summary: GradingSummary;
+  evalFeedback?: EvalFeedback;
+  /** Set when the grader subprocess failed or returned unparseable output. */
+  graderError?: string;
+  durationMs: number;
+}
+interface SuiteGradingReport {
+  gradedAt: string;
+  sourceReport: string;
+  /** Grading YAML path when `--config` was used. */
+  gradingConfigPath?: string;
+  results: RepGradingResult[];
+  summary: GradingSummary;
+}
+interface GradeReportOptions {
+  /** Path to the report being graded (stored in output). */
+  sourceReport?: string;
+  /** Path to expectations YAML/JSON sidecar (case id → strings). */
+  expectationsPath?: string;
+  /** Claude binary for grading. Default: `claude`. */
+  binary?: string;
+  /** Model for the grader subprocess. */
+  model?: string;
+  /** Optional judge prompt prefix (maps to upstream system_instruction). */
+  systemInstruction?: string;
+  /** Timeout per grading call (ms). Default 300000 (5 min). */
+  timeoutMs?: number;
+  /** Max concurrent grader subprocesses. Default 2. */
+  maxConcurrent?: number;
+  /** Process env for the judge subprocess (merged over inherited env). */
+  env?: Record<string, string>;
+  /** Working directory for the judge subprocess. */
+  cwd?: string;
+  /** Claude Code options for the judge (nested in grading YAML under `claudeCode`). */
+  claudeCode?: Record<string, unknown>;
+  /** Path to grading YAML when `--config` was used. */
+  gradingConfigPath?: string;
+  /** Inject a custom grader (for tests). */
+  gradeFn?: GraderFn;
+  onProgress?: (event: GradeProgressEvent) => void;
+}
+type GradeProgressEvent = {
+  kind: "grade-start";
+  total: number;
+} | {
+  kind: "grade-complete";
+  caseId: string;
+  cellLabel: string;
+  repetitionIndex: number;
+  passed: number;
+  failed: number;
+  durationMs: number;
+  graderError?: string;
+} | {
+  kind: "grade-done";
+  durationMs: number;
+  totalExpectations: number;
+  passedExpectations: number;
+};
+type GraderFn = (input: GraderInput) => Promise<GraderOutput>;
+interface GraderInput {
+  prompt: string;
+  transcript: string;
+  expectations: string[];
+  systemInstruction?: string;
+}
+interface GraderOutput {
+  expectations: GradedExpectation[];
+  summary: GradingSummary;
+  evalFeedback?: EvalFeedback;
+  error?: string;
+}
+//#endregion
+//#region src/types/eval-record.d.ts
+/** Schema version for {@link EvalRunEnvelope} JSON documents. */
+declare const EVAL_RUN_SCHEMA_VERSION = "1.0";
+/** Schema version embedded in each {@link TrajectoryView} at export time. */
+declare const TRAJECTORY_SCHEMA_VERSION = "1.0";
+/** Link to the suite spec that produced a run. */
+interface SuiteReference {
+  /** Absolute or repo-relative path to the suite YAML. */
+  uri?: string;
+  /** Stable suite identifier when known (e.g. case bundle name). */
+  id?: string;
+  /** SHA-256 or similar hash of suite file contents. */
+  contentHash?: string;
+}
+/** Harness that executed the run. */
+interface HarnessInfo {
+  /** Adapter id from suite YAML, e.g. `claude-code`. */
+  adapter: string;
+  /** harness-eval package version when envelope was built. */
+  frameworkVersion?: string;
+  /** Optional harness binary version (e.g. `claude -v`). */
+  harnessVersion?: string;
+}
+/** CI, git, or runtime provenance for correlation in the DB. */
+interface EvalProvenance {
+  runId?: string;
+  ci?: {
+    provider?: string;
+    jobId?: string;
+    pipelineId?: string;
+    url?: string;
+  };
+  git?: {
+    commit?: string;
+    branch?: string;
+    repository?: string;
+  };
+  pluginVersion?: string;
+  triggeredBy?: string;
+  [key: string]: unknown;
+}
+/** Aggregate behavioral summary for the run. */
+interface EvalRunSummary {
+  cellsTotal: number;
+  cellsPassed: number;
+  /** All cells passed behavioral assertion thresholds. */
+  behavioralPass: boolean;
+  /** All graded expectations passed (when outcome layer present). */
+  outcomePass?: boolean;
+}
+/** Identity of the judge that produced outcome grades. */
+interface JudgeInfo {
+  id: string;
+  model?: string;
+  version?: string;
+}
+/** Outcome grades for one repetition (built-in or external judge). */
+interface OutcomeGrades {
+  judge: JudgeInfo;
+  expectations: GradedExpectation[];
+  summary: GradingSummary;
+  evalFeedback?: {
+    suggestions: Array<{
+      assertion?: string;
+      reason: string;
+    }>;
+    overall: string;
+  };
+  error?: string;
+}
+/** Score from an external eval framework (LangSmith, Braintrust, custom). */
+interface ExternalScore {
+  source: string;
+  metric: string;
+  value: number | boolean | string;
+  metadata?: Record<string, unknown>;
+}
+/** Optional large or vendor-specific blobs (store by reference in DB when possible). */
+interface EvalArtifacts {
+  /** Claude Code `stream-json` lines — debug only, not cross-harness. */
+  rawStreamEvents?: unknown[];
+  /** URI to OTLP JSON (S3, GCS, etc.). */
+  otlpTraceUri?: string;
+  /** Text transcript for judges (`trajectoryToTranscript`). */
+  transcript?: string;
+}
+/**
+ * One harness invocation — the unit external judges and trajectory queries use.
+ */
+interface EvalRepetition {
+  repetitionIndex: number;
+  durationMs: number;
+  /** Normalized harness session. Required when the harness completed with a view. */
+  trajectory?: TrajectoryView & {
+    schemaVersion: string;
+  };
+  diagnostics?: Partial<AdapterDiagnostics>;
+  assertionResults: AssertionResult[];
+  outcomeGrades?: OutcomeGrades;
+  externalScores?: ExternalScore[];
+  artifacts?: EvalArtifacts;
+  /** Interchange-format predicted tool-call trajectory. */
+  predicted_trajectory?: InterchangeToolCall[];
+  /** Full multi-turn agent trace in interchange format. */
+  agent_trace?: AgentTrace;
+  /** Session latency in seconds (interchange field). */
+  latency_in_seconds?: number;
+  /** 1 when the harness run failed, 0 on success (interchange field). */
+  failure?: 0 | 1;
+  /** Trajectory-level metrics when reference_trajectory is provided. */
+  trajectoryMetrics?: TrajectoryMetrics;
+  /** Tool-call-level metrics when reference_trajectory is provided. */
+  toolCallMetrics?: ToolCallMetrics;
+  error?: {
+    message: string;
+    diagnostics?: Partial<AdapterDiagnostics>;
+  };
+}
+/** Behavioral stats for one assertion across repetitions in a cell. */
+interface EvalAssertionStat {
+  description: string;
+  threshold: number;
+  passedCount: number;
+  evaluatedCount: number;
+  passRate: number;
+  meetsThreshold: boolean;
+}
+/** One (test case × matrix cell) result. */
+interface EvalCellResult {
+  caseId: string;
+  category?: string;
+  notes?: string;
+  prompt?: string;
+  expectations?: string[];
+  cellLabel: string;
+  axes?: Record<string, string>;
+  /** Reference tool-call trajectory for metric computation. */
+  reference_trajectory?: TabularToolCall[];
+  /** Human ratings keyed by metric name for judge calibration. */
+  human_ratings?: Record<string, number>;
+  assertionStats: EvalAssertionStat[];
+  adapterErrors: number;
+  /** Passed all behavioral assertion thresholds for this cell. */
+  behavioralPass: boolean;
+  /** Passed all outcome expectations when graded; omitted if not graded. */
+  outcomePass?: boolean;
+  repetitions: EvalRepetition[];
+}
+/**
+ * Top-level document for CI/CD pipelines, APIs, and databases.
+ *
+ * This is the interchange format your storage layer should target — not
+ * {@link import("./stream").StreamEvent} or OTLP traces.
+ */
+interface EvalRunEnvelope {
+  schemaVersion: typeof EVAL_RUN_SCHEMA_VERSION;
+  runId: string;
+  startedAt: string;
+  durationMs: number;
+  suite?: SuiteReference;
+  harness: HarnessInfo;
+  provenance?: EvalProvenance;
+  summary: EvalRunSummary;
+  cells: EvalCellResult[];
+}
+interface BuildEvalRunEnvelopeOptions {
+  /** UUID for this run; generated if omitted. */
+  runId?: string;
+  suite?: SuiteReference;
+  harness?: Partial<HarnessInfo>;
+  provenance?: EvalProvenance;
+  /** Merge outcome grades from `gradeReport()` or compatible structure. */
+  grading?: {
+    gradedAt?: string;
+    sourceReport?: string;
+    results: Array<{
+      caseId: string;
+      cellLabel: string;
+      repetitionIndex: number;
+      expectations: GradedExpectation[];
+      summary: GradingSummary;
+      evalFeedback?: OutcomeGrades["evalFeedback"];
+      graderError?: string;
+      durationMs?: number;
+    }>;
+    judge?: JudgeInfo;
+  };
+  /** Include transcript in each repetition's artifacts. Default true. */
+  includeTranscript?: boolean;
+  /** Include raw stream events when adapter provides them. Default false. */
+  includeRawStreamEvents?: boolean;
+}
+//#endregion
+//#region src/trajectory/builder.d.ts
+declare class TrajectoryBuilder {
+  private meta;
+  private sessionStartTs;
+  private turns;
+  private allToolCalls;
+  /**
+   * tool_use_id → ToolCall, for matching results back to calls.
+   * Entries are removed once a result is observed.
+   */
+  private pendingCalls;
+  private retries;
+  private finalUsage;
+  private finalCostUsd;
+  private finalDurationMs;
+  private finalNumTurns;
+  private finalResultText;
+  private sawResultEvent;
+  private resultIsError;
+  /**
+   * Consume one event. Safe to call with events in stream order.
+   *
+   * Unknown event types are silently ignored — the schema evolves and we
+   * don't want CI to break on a new event type we haven't modelled.
+   */
+  consume(event: StreamEvent): void;
+  /**
+   * Finalize the view. Call after consuming the last event from the stream.
+   *
+   * Throws if no `system/init` was observed — at that point we have no model,
+   * no session id, and no available-tools list, which means assertions like
+   * "called any mcp__api__* tool" can't even be evaluated meaningfully.
+   */
+  build(): TrajectoryView;
+  private handleAssistantMessage;
+  private handleUserMessage;
+}
+/**
+ * Convenience: drain an async iterable of events through a fresh builder.
+ *
+ * Suitable when you have the full event stream and just want the view.
+ * For interactive/incremental scenarios (e.g. surfacing partial state in a UI)
+ * instantiate {@link TrajectoryBuilder} directly and call `consume()` /
+ * `build()` yourself.
+ */
+declare function buildTrajectory(events: AsyncIterable<StreamEvent>): Promise<TrajectoryView>;
+//#endregion
+//#region src/parsers/stream-json.d.ts
+/**
+ * Result of attempting to parse a single line.
+ *
+ * Successful parses yield `{ ok: true }` with the typed event and the raw line
+ * (kept for diagnostics and OTel `events.attributes.raw`). Failed parses yield
+ * `{ ok: false }` with the parse error and the raw line — callers can log,
+ * skip, or fail the run as they see fit.
+ */
+type ParseResult = {
+  ok: true;
+  event: StreamEvent;
+  rawLine: string;
+} | {
+  ok: false;
+  error: Error;
+  rawLine: string;
+};
+/**
+ * Parse a readable stream of NDJSON into a sequence of typed stream-json events.
+ *
+ * @example
+ *   const child = spawn("claude", ["-p", prompt, "--output-format", "stream-json", "--verbose"]);
+ *   for await (const result of parseStreamJson(child.stdout)) {
+ *     if (result.ok) builder.consume(result.event);
+ *     else console.warn("malformed stream line:", result.rawLine, result.error);
+ *   }
+ */
+declare function parseStreamJson(stream: Readable): AsyncGenerator<ParseResult, void, void>;
+//#endregion
+//#region src/assertions/evaluator.d.ts
+/**
+ * Evaluate one assertion against a trajectory view.
+ *
+ * The switch is exhaustive — TypeScript's `never` check at the end will
+ * flag any new variant added to the `Assertion` union that hasn't been
+ * wired up here.
+ */
+declare function evaluate(view: TrajectoryView, assertion: Assertion): AssertionResult;
+/**
+ * Evaluate a list of assertions independently. Used at the test-case level
+ * where each top-level assertion is reported separately (and thresholded
+ * separately, in the runner layer).
+ */
+declare function evaluateAll(view: TrajectoryView, assertions: Assertion[]): AssertionResult[];
+//#endregion
+//#region src/adapters/registry.d.ts
+/**
+ * Register a harness adapter by id.
+ *
+ * Duplicate ids throw — registration is explicit so accidental overrides
+ * surface immediately during startup or test setup.
+ */
+declare function registerAdapter(id: string, adapter: HarnessAdapter): void;
+/** Return all registered adapter ids (built-in and runtime). */
+declare function listAdapters(): string[];
+/** Resolve an adapter by id. Throws if unknown. */
+declare function getAdapter(id: string): HarnessAdapter;
+/** Default adapter when YAML omits `adapter`. */
+declare const DEFAULT_ADAPTER_ID = "claude-code";
+declare function getDefaultAdapter(): HarnessAdapter;
+//#endregion
+//#region src/runner/case.d.ts
+/** Default repetition count when `case.repetitions` is omitted. */
+declare const DEFAULT_REPETITIONS = 5;
+/** Default assertion pass-rate threshold when `threshold` is omitted. */
+declare const DEFAULT_THRESHOLD = 1;
+type AdapterRunFn = (config: BaseAdapterConfig & Record<string, unknown>) => Promise<AdapterResult>;
+/**
+ * Build the effective adapter config for one (suite, case, cell).
+ *
+ * Merge order (later wins): defaultConfig < case.config < cell.config.
+ */
+declare function mergeConfig(suite: TestSuite, testCase: TestCase, cell: MatrixCell): BaseAdapterConfig & Record<string, unknown>;
+declare function getRepetitions(testCase: TestCase): number;
+declare function runRepetition(testCase: TestCase, _cell: MatrixCell, config: BaseAdapterConfig & Record<string, unknown>, repetitionIndex: number, run: AdapterRunFn, signal?: AbortSignal): Promise<RepetitionResult>;
+declare function aggregateCell(testCase: TestCase, cell: MatrixCell, repetitions: RepetitionResult[]): CellReport;
+//#endregion
+//#region src/runner/limit.d.ts
+/**
+ * Promise-based concurrency limiter.
+ *
+ * Functionally equivalent to the `p-limit` package, inlined to avoid an
+ * external dependency for ~20 lines of code.
+ *
+ * Usage:
+ *
+ *   const limit = createLimit(4);
+ *   const results = await Promise.all(tasks.map(t => limit(() => run(t))));
+ *
+ * The limiter is unbounded in queue depth — it doesn't push back on the
+ * caller. If you need bounded enqueue, wrap it.
+ */
+/** A function that runs an async task under the concurrency limit. */
+type LimitedRunner = <T>(fn: () => Promise<T>) => Promise<T>;
+declare function createLimit(max: number): LimitedRunner;
+//#endregion
+//#region src/otel/types.d.ts
+/**
+ * Minimal OTLP JSON types for trace export.
+ *
+ * Shapes follow OTLP/HTTP JSON Protobuf encoding (lowerCamelCase field names).
+ * @see https://opentelemetry.io/docs/specs/otlp/
+ */
+interface ExportTraceServiceRequest {
+  resourceSpans: ResourceSpans[];
+}
+interface ResourceSpans {
+  resource: Resource;
+  scopeSpans: ScopeSpans[];
+}
+interface Resource {
+  attributes: KeyValue[];
+}
+interface ScopeSpans {
+  scope: InstrumentationScope;
+  spans: Span[];
+}
+interface InstrumentationScope {
+  name: string;
+  version?: string;
+}
+interface Span {
+  traceId: string;
+  spanId: string;
+  parentSpanId?: string;
+  name: string;
+  kind: number;
+  startTimeUnixNano: string;
+  endTimeUnixNano: string;
+  attributes: KeyValue[];
+  status?: SpanStatus;
+}
+interface SpanStatus {
+  code: number;
+  message?: string;
+}
+interface KeyValue {
+  key: string;
+  value: AnyValue;
+}
+interface AnyValue {
+  stringValue?: string;
+  boolValue?: boolean;
+  intValue?: string;
+  doubleValue?: number;
+  bytesValue?: string;
+  arrayValue?: ArrayValue;
+  kvlistValue?: KeyValueList;
+}
+interface ArrayValue {
+  values: AnyValue[];
+}
+interface KeyValueList {
+  values: KeyValue[];
+}
+interface EmitOtelOptions {
+  /** User prompt for the first `gen_ai.input.messages` entry. */
+  prompt?: string;
+  /** `gen_ai.agent.name` on the root span. Default: `claude-code`. */
+  agentName?: string;
+  /** `gen_ai.provider.name`. Default: `anthropic`. */
+  providerName?: string;
+  /** Resource `service.name`. Default: `harness-eval`. */
+  serviceName?: string;
+  /** Instrumentation scope name. Default: `@alis-build/harness-eval`. */
+  instrumentationScope?: string;
+  /**
+   * Wall-clock end time for the trace (ms). Defaults to `Date.now()`.
+   * Start is derived from `view.usage.durationMs`.
+   */
+  endTimeMs?: number;
+}
+//#endregion
+//#region src/otel/emitter.d.ts
+/**
+ * Map a {@link TrajectoryView} to OTLP trace JSON.
+ *
+ * Span tree (siblings under `invoke_agent`, not nested):
+ * ```
+ * invoke_agent
+ * ├── chat {model}
+ * ├── execute_tool {name}
+ * ├── chat {model}
+ * └── ...
+ * ```
+ */
+declare function trajectoryToOtlp(view: TrajectoryView, options?: EmitOtelOptions): ExportTraceServiceRequest;
+/** Alias matching the implementation plan naming. */
+declare const emitOtel: typeof trajectoryToOtlp;
+//#endregion
+//#region src/grader/grade-report.d.ts
+declare function gradeReport(report: SuiteReport, options?: GradeReportOptions): Promise<SuiteGradingReport>;
+//#endregion
+//#region src/grader/resolve-grade-options.d.ts
+interface GradeCliOverrides {
+  model?: string;
+  binary?: string;
+  timeoutMs?: number;
+  maxConcurrent?: number;
+  expectationsPath?: string;
+  sourceReport?: string;
+}
+/**
+ * Merge standalone grading YAML with CLI flags (CLI wins).
+ */
+declare function resolveGradeOptions(fileConfig?: GradingConfig, cli?: GradeCliOverrides, configPath?: string): GradeReportOptions;
+//#endregion
+//#region src/grader/transcript.d.ts
+declare function trajectoryToTranscript(view: TrajectoryView, prompt?: string): string;
+//#endregion
+//#region src/grader/claude-grader.d.ts
+interface ClaudeGraderOptions {
+  binary?: string;
+  model?: string;
+  timeoutMs?: number;
+  env?: Record<string, string>;
+  cwd?: string;
+  claudeCode?: ClaudeCodeOptions;
+}
+declare function createClaudeGrader(options?: ClaudeGraderOptions): GraderFn;
+//#endregion
+//#region src/grader/format-console.d.ts
+declare function formatGradingConsole(report: SuiteGradingReport, color?: boolean): string;
+declare function gradingReportPassed(report: SuiteGradingReport): boolean;
+//#endregion
+//#region src/reporter/types.d.ts
+type ReportFormat = "console" | "markdown" | "json";
+interface ReporterOptions {
+  format: ReportFormat;
+  baseline?: SuiteReport;
+  color?: boolean;
+}
+//#endregion
+//#region src/reporter/index.d.ts
+declare function formatReport(report: SuiteReport, options: ReporterOptions): string;
+//#endregion
+//#region src/eval-record/build.d.ts
+/**
+ * Convert a {@link SuiteReport} (and optional grading) into a versioned
+ * {@link EvalRunEnvelope} for storage or API handoff.
+ */
+declare function buildEvalRunEnvelope(report: SuiteReport, options?: BuildEvalRunEnvelopeOptions): EvalRunEnvelope;
+/** Build envelope from on-disk report + optional grading JSON paths. */
+declare function buildEvalRunEnvelopeFromFiles(reportPath: string, options?: BuildEvalRunEnvelopeOptions & {
+  gradingPath?: string;
+  suitePath?: string;
+}): Promise<EvalRunEnvelope>;
+//#endregion
+//#region src/metrics/tool-calls.d.ts
+interface ToolCallMetricOptions {
+  useStrictStringMatch?: boolean;
+}
+type ToolCallInput = InterchangeToolCall | TabularToolCall | {
+  tool_name: string;
+  tool_input: unknown;
+};
+declare function toolCallValid(toolCall: ToolCallInput): number;
+declare function toolNameMatch(predicted: ToolCallInput, reference: ToolCallInput): number;
+declare function toolParameterKeyMatch(predicted: ToolCallInput, reference: ToolCallInput): number;
+declare function toolParameterKvMatch(predicted: ToolCallInput, reference: ToolCallInput, options?: ToolCallMetricOptions): number;
+declare function computeToolCallMetrics(predicted: ToolCallInput[], reference: ToolCallInput[], options?: ToolCallMetricOptions): ToolCallMetrics;
+//#endregion
+//#region src/eval-interchange/projections.d.ts
+declare function toTrajectory(envelope: EvalRunEnvelope): EvalDatasetRow[];
+declare function toProtoInstances(envelope: EvalRunEnvelope): ProtoTrajectoryInstance[];
+declare function toAgentTrace(envelope: EvalRunEnvelope): AgentTrace[];
+declare function enrichRepetitionWithInterchange(repetition: EvalRepetition, referenceTrajectory?: TabularToolCall[]): EvalRepetition;
+//#endregion
+//#region src/metrics/trajectory.d.ts
+type TrajectoryInput = InterchangeToolCall[] | TabularToolCall[] | Array<{
+  tool_name: string;
+  tool_input: unknown;
+}>;
+declare function trajectoryExactMatch(predicted: TrajectoryInput, reference: TrajectoryInput): number;
+declare function trajectoryInOrderMatch(predicted: TrajectoryInput, reference: TrajectoryInput): number;
+declare function trajectoryAnyOrderMatch(predicted: TrajectoryInput, reference: TrajectoryInput): number;
+declare function trajectoryPrecision(predicted: TrajectoryInput, reference: TrajectoryInput): number;
+declare function trajectoryRecall(predicted: TrajectoryInput, reference: TrajectoryInput): number;
+declare function trajectorySingleToolUse(predicted: TrajectoryInput, reference: TrajectoryInput): number;
+declare function computeTrajectoryMetrics(predicted: TrajectoryInput, reference: TrajectoryInput): TrajectoryMetrics;
+//#endregion
+export { type AdapterDiagnostics, AdapterError, type AdapterResult, type AdapterRunFn, AgentConfig, AgentEvent, AgentTrace, Assertion, AssertionResult, AssertionStat, AssistantMessage, AssistantMessageEvent, AssistantTurn, type BaseAdapterConfig, BuildEvalRunEnvelopeOptions, Cardinality, CellReport, CompoundPredicate, ConfigError, ContentBlock, ContentPart, ConversationTurn, DEFAULT_ADAPTER_ID, DEFAULT_REPETITIONS, DEFAULT_THRESHOLD, EVAL_RUN_SCHEMA_VERSION, type EmitOtelOptions, EvalArtifacts, EvalAssertionStat, EvalCellResult, EvalDatasetRow, EvalProvenance, EvalRepetition, EvalRunEnvelope, EvalRunSummary, type ExportTraceServiceRequest, ExternalScore, type GradeReportOptions, type HarnessAdapter, HarnessInfo, InterchangeToolCall, InterchangeTrajectory, JudgeInfo, LeafPredicate, type LimitedRunner, MatrixCell, McpServerStatus, ObjectPredicate, OutcomeGrades, type ParseErrorRecord, type ParseResult, Predicate, ProgressCallback, ProgressEvent, ProtoTrajectoryInstance, type RepGradingResult, RepetitionError, RepetitionResult, type ReporterOptions, ResultEvent, RetryRecord, RunSuiteOptions, SessionMeta, StopReason, StreamEvent, type SuiteConfig, type SuiteGradingReport, SuiteReference, SuiteReport, SystemCompactBoundaryEvent, SystemInitEvent, SystemPluginInstallEvent, SystemRetryEvent, SystemUnknownEvent, TRAJECTORY_SCHEMA_VERSION, TabularToolCall, TestCase, TestSuite, TextBlock, ThresholdedAssertion, ToolCall, type ToolCallMetricOptions, ToolCallMetrics, ToolPattern, ToolResultBlock, ToolUseBlock, TrajectoryBuilder, type TrajectoryInput, TrajectoryMetrics, TrajectoryView, Usage, UsageSummary, UserMessage, UserMessageEvent, aggregateCell, buildEvalRunEnvelope, buildEvalRunEnvelopeFromFiles, buildTrajectory, index_d_exports as claudeCode, computeToolCallMetrics, computeTrajectoryMetrics, createClaudeGrader, createLimit, emitOtel, enrichRepetitionWithInterchange, evaluate, evaluateAll, formatGradingConsole, formatReport, getAdapter, getDefaultAdapter, getRepetitions, gradeReport, gradingReportPassed, isAssistantMessage, isResult, isSystemInit, isSystemRetry, isTextBlock, isToolResultBlock, isToolUseBlock, isUserMessage, listAdapters, loadSuite, mergeConfig, namespaceOf, parseStreamJson, parseSuite, registerAdapter, resolveGradeOptions, runRepetition, runSuite, toAgentTrace, toProtoInstances, toTrajectory, toolCallValid, toolNameMatch, toolParameterKeyMatch, toolParameterKvMatch, trajectoryAnyOrderMatch, trajectoryExactMatch, trajectoryInOrderMatch, trajectoryPrecision, trajectoryRecall, trajectorySingleToolUse, trajectoryToOtlp, trajectoryToTranscript };
+//# sourceMappingURL=index.d.ts.map