@alis-build/harness-eval 0.1.2 → 0.1.4

package/dist/index.d.ts

@@ -1,8 +1,9 @@
-import { A as Usage, B as isUserMessage, C as SystemInitEvent, D as TextBlock, E as SystemUnknownEvent, F as isSystemInit, I as isSystemRetry, L as isTextBlock, M as UserMessageEvent, N as isAssistantMessage, O as ToolResultBlock, P as isResult, R as isToolResultBlock, S as SystemCompactBoundaryEvent, T as SystemRetryEvent, _ as ContentBlock, a as HarnessAdapter, b as StopReason, c as AssistantTurn, d as ToolCall, f as TrajectoryView, g as AssistantMessageEvent, h as AssistantMessage, i as BaseAdapterConfig, j as UserMessage, k as ToolUseBlock, l as RetryRecord, m as namespaceOf, n as AdapterError, o as ParseErrorRecord, p as UsageSummary, r as AdapterResult, s as SuiteConfig, t as AdapterDiagnostics, u as SessionMeta, v as McpServerStatus, w as SystemPluginInstallEvent, x as StreamEvent, y as ResultEvent, z as isToolUseBlock } from "./types-B9H4IZtA.js";
-import { n as index_d_exports, o as ClaudeCodeOptions } from "./index-V22PrR0p.js";
-import { A as ObjectPredicate, C as TrajectoryPairInstanceJson, D as Cardinality, E as AssertionResult, M as ThresholdedAssertion, N as ToolPattern, O as CompoundPredicate, S as TrajectoryInstancesJson, T as Assertion, _ as ProtojsonToolCall, a as ProgressEvent, b as ReferenceTrajectoryConfig, c as RunSuiteOptions, d as TestSuite, f as EvalDatasetRow, g as InstancesJsonlRow, h as InstanceData, i as ProgressCallback, j as Predicate, k as LeafPredicate, l as SuiteReport, m as HarnessMetrics, n as CellReport, o as RepetitionError, p as EvaluationInstanceJson, r as MatrixCell, s as RepetitionResult, t as AssertionStat, u as TestCase, v as ProtojsonTrajectory, w as TrajectorySingleToolUseInstanceJson, x as TrajectoryInstanceMetricKey, y as ReferenceToolNameMode } from "./types-CD3TwOtZ.js";
-import { i as GradingConfig, r as parseSuite, s as ConfigError, t as loadSuite } from "./loader-C9yQHUPC.js";
-import { t as runSuite } from "./suite-DPJMIEbu.js";
+import { A as Usage, B as isUserMessage, C as SystemInitEvent, D as TextBlock, E as SystemUnknownEvent, F as isSystemInit, I as isSystemRetry, L as isTextBlock, M as UserMessageEvent, N as isAssistantMessage, O as ToolResultBlock, P as isResult, R as isToolResultBlock, S as SystemCompactBoundaryEvent, T as SystemRetryEvent, _ as ContentBlock, a as HarnessAdapter, b as StopReason, c as AssistantTurn, d as ToolCall, f as TrajectoryView, g as AssistantMessageEvent, h as AssistantMessage, i as BaseAdapterConfig, j as UserMessage, k as ToolUseBlock, l as RetryRecord, m as namespaceOf, n as AdapterError, o as ParseErrorRecord, p as UsageSummary, r as AdapterResult, s as SuiteConfig, t as AdapterDiagnostics, u as SessionMeta, v as McpServerStatus, w as SystemPluginInstallEvent, x as StreamEvent, y as ResultEvent, z as isToolUseBlock } from "./types-D0HR2WnP.js";
+import { n as index_d_exports, o as ClaudeCodeOptions } from "./index-C56AEDUr.js";
+import { i as CodexOptions } from "./types-DFMpv_HJ.js";
+import { A as ObjectPredicate, C as TrajectoryPairInstanceJson, D as Cardinality, E as AssertionResult, M as ThresholdedAssertion, N as ToolPattern, O as CompoundPredicate, S as TrajectoryInstancesJson, T as Assertion, _ as ProtojsonToolCall, a as ProgressEvent, b as ReferenceTrajectoryConfig, c as RunSuiteOptions, d as TestSuite, f as EvalDatasetRow, g as InstancesJsonlRow, h as InstanceData, i as ProgressCallback, j as Predicate, k as LeafPredicate, l as SuiteReport, m as HarnessMetrics, n as CellReport, o as RepetitionError, p as EvaluationInstanceJson, r as MatrixCell, s as RepetitionResult, t as AssertionStat, u as TestCase, v as ProtojsonTrajectory, w as TrajectorySingleToolUseInstanceJson, x as TrajectoryInstanceMetricKey, y as ReferenceToolNameMode } from "./types-CLt4Yygc.js";
+import { a as loadSuiteDocument, c as GradingConfig, d as ConfigError, n as parseSuite, o as SuiteDocument, s as PipelineConfig, t as loadSuite } from "./loader-CrmzNwkq.js";
+import { t as runSuite } from "./suite-qyOGre2g.js";
 import { Readable } from "node:stream";
 
 //#region src/grader/types.d.ts
@@ -56,6 +57,12 @@ interface SuiteGradingReport {
   sourceReport: string;
   /** Grading YAML path when `--config` was used. */
   gradingConfigPath?: string;
+  /** Judge that produced outcome grades. */
+  judge?: {
+    id: string;
+    model?: string;
+    adapter?: string;
+  };
   results: RepGradingResult[];
   summary: GradingSummary;
 }
@@ -79,8 +86,14 @@ interface GradeReportOptions {
   env?: Record<string, string>;
   /** Working directory for the judge subprocess. */
   cwd?: string;
+  /** Grading adapter id. Default: `claude-code`. */
+  judgeAdapter?: "claude-code" | "codex" | "gemini-cli";
   /** Claude Code options for the judge (nested in grading YAML under `claudeCode`). */
   claudeCode?: Record<string, unknown>;
+  /** Codex CLI options for the judge (nested in grading YAML under `codex`). */
+  codex?: Record<string, unknown>;
+  /** Gemini CLI options for the judge (nested in grading YAML under `geminiCli`). */
+  geminiCli?: Record<string, unknown>;
   /** Path to grading YAML when `--config` was used. */
   gradingConfigPath?: string;
   /** Inject a custom grader (for tests). */
@@ -170,6 +183,8 @@ interface JudgeInfo {
   id: string;
   model?: string;
   version?: string;
+  /** Grading adapter id when known (e.g. `codex`, `claude-code`). */
+  adapter?: string;
 }
 /** Outcome grades for one repetition (built-in or external judge). */
 interface OutcomeGrades {
@@ -451,6 +466,106 @@ declare function aggregateCell(testCase: TestCase, cell: MatrixCell, repetitions
 type LimitedRunner = <T>(fn: () => Promise<T>) => Promise<T>;
 declare function createLimit(max: number): LimitedRunner;
 //#endregion
+//#region src/cli/commands/envelope.d.ts
+/** Supported `--projection` values for envelope output. */
+type EnvelopeProjection = "envelope" | "trajectory" | "instances";
+//#endregion
+//#region src/pipeline/resolve-inputs.d.ts
+type PipelineStepName = "run" | "grade" | "envelope";
+/** CLI overrides for pipeline artifact paths (take precedence over YAML). */
+interface PipelineCliOverrides {
+  run?: {
+    output?: string;
+    maxConcurrent?: number;
+  };
+  grade?: {
+    input?: string;
+    output?: string;
+    maxConcurrent?: number;
+  };
+  envelope?: {
+    report?: string;
+    grading?: string;
+    output?: string;
+    projection?: EnvelopeProjection;
+  };
+}
+/** Resolved paths for the harness run step. */
+interface ResolvedPipelineRun {
+  output: string;
+  maxConcurrent?: number;
+}
+/** Resolved input (suite report) and output (grading JSON) for the grade step. */
+interface ResolvedPipelineGrade {
+  input: string;
+  output: string;
+  maxConcurrent?: number;
+}
+/** Resolved artifact paths for the envelope export step. */
+interface ResolvedPipelineEnvelope {
+  report: string;
+  grading?: string;
+  output: string;
+  projection: EnvelopeProjection;
+  includeRawStreamEvents: boolean;
+  noTranscript: boolean;
+}
+/** Fully resolved pipeline inputs for one or more enabled steps. */
+interface ResolvedPipeline {
+  suitePath: string;
+  run?: ResolvedPipelineRun;
+  grade?: ResolvedPipelineGrade;
+  envelope?: ResolvedPipelineEnvelope;
+}
+/** Inputs for {@link resolvePipelineInputs}. */
+interface ResolvePipelineInputsOptions {
+  suitePath: string;
+  suiteDir: string;
+  pipeline: PipelineConfig;
+  steps: PipelineStepName[];
+  executed?: {
+    run?: {
+      output: string;
+    };
+    grade?: {
+      input: string;
+      output: string;
+    };
+  };
+  overrides?: PipelineCliOverrides;
+}
+/** Resolve absolute paths for enabled pipeline steps. */
+declare function resolvePipelineInputs(options: ResolvePipelineInputsOptions): Promise<ResolvedPipeline>;
+/**
+ * Resolve a grading artifact path from a unified suite's `pipeline:` block.
+ *
+ * Used by `harness-eval envelope --suite` when `--grading` is omitted (spec C-7).
+ * Checks `pipeline.envelope.grading` then default `pipeline.grade.output` on disk.
+ */
+declare function resolveGradingArtifactFromSuite(suitePath: string): Promise<string | undefined>;
+//#endregion
+//#region src/pipeline/run-pipeline.d.ts
+/** Options for {@link runPipeline} (CLI flags and progress callbacks). */
+interface RunPipelineOptions {
+  /** Comma-separated subset of configured steps (e.g. `run,grade`). */
+  steps?: string;
+  maxConcurrent?: number;
+  overrides?: PipelineCliOverrides;
+  onRunProgress?: ProgressCallback;
+  onGradeProgress?: GradeReportOptions["onProgress"];
+  /** Framework version stamped on envelope export. */
+  frameworkVersion?: string;
+}
+/** Outcome of a pipeline run including per-step exit semantics. */
+interface RunPipelineResult {
+  /** 0 pass, 1 eval/grade/envelope failure, 2 load error (thrown before return). */
+  exitCode: number;
+  stepsRun: PipelineStepName[];
+  runReport?: SuiteReport;
+}
+/** Execute configured pipeline steps in order; stop on first failure. */
+declare function runPipeline(doc: SuiteDocument, options?: RunPipelineOptions): Promise<RunPipelineResult>;
+//#endregion
 //#region src/otel/types.d.ts
 /**
  * Minimal OTLP JSON types for trace export.
@@ -601,6 +716,19 @@ interface ClaudeGraderOptions {
 /** Factory returning a {@link GraderFn} bound to subprocess options. */
 declare function createClaudeGrader(options?: ClaudeGraderOptions): GraderFn;
 //#endregion
+//#region src/grader/codex-grader.d.ts
+/** Options for {@link createCodexGrader} / {@link runCodexGrader}. */
+interface CodexGraderOptions {
+  binary?: string;
+  model?: string;
+  timeoutMs?: number;
+  env?: Record<string, string>;
+  cwd?: string;
+  codex?: CodexOptions;
+}
+/** Factory returning a {@link GraderFn} bound to subprocess options. */
+declare function createCodexGrader(options?: CodexGraderOptions): GraderFn;
+//#endregion
 //#region src/grader/format-console.d.ts
 /**
  * Format a {@link SuiteGradingReport} for terminal output.
@@ -839,5 +967,5 @@ declare function toolParameterKvMatch(predicted: ToolCallInput, reference: ToolC
  */
 declare function computeToolCallMetrics(predicted: ToolCallInput[], reference: ToolCallInput[], options?: ToolCallMetricOptions): ToolCallMetrics;
 //#endregion
-export { type AdapterDiagnostics, AdapterError, type AdapterResult, type AdapterRunFn, Assertion, AssertionResult, AssertionStat, AssistantMessage, AssistantMessageEvent, AssistantTurn, type BaseAdapterConfig, BuildEvalRunEnvelopeOptions, Cardinality, CellReport, CompoundPredicate, ConfigError, ContentBlock, DEFAULT_ADAPTER_ID, DEFAULT_REPETITIONS, DEFAULT_THRESHOLD, EVAL_RUN_SCHEMA_VERSION, type EmitOtelOptions, EvalArtifacts, EvalAssertionStat, EvalCellResult, EvalDatasetRow, EvalProvenance, EvalRepetition, EvalRunEnvelope, EvalRunSummary, EvaluationInstanceJson, type ExportTraceServiceRequest, ExternalScore, type GradeReportOptions, type HarnessAdapter, HarnessInfo, HarnessMetrics, InstanceData, InstancesJsonlRow, JudgeInfo, LeafPredicate, type LimitedRunner, MatrixCell, McpServerStatus, ObjectPredicate, OutcomeGrades, type ParseErrorRecord, type ParseResult, Predicate, ProgressCallback, ProgressEvent, ProtojsonToolCall, ProtojsonTrajectory, ReferenceToolNameMode, ReferenceTrajectoryConfig, type RepGradingResult, RepetitionError, RepetitionResult, type ReporterOptions, ResultEvent, RetryRecord, RunSuiteOptions, SessionMeta, StopReason, StreamEvent, type SuiteConfig, type SuiteGradingReport, SuiteReference, SuiteReport, SystemCompactBoundaryEvent, SystemInitEvent, SystemPluginInstallEvent, SystemRetryEvent, SystemUnknownEvent, TRAJECTORY_SCHEMA_VERSION, TestCase, TestSuite, TextBlock, ThresholdedAssertion, ToolCall, type ToolCallMetricOptions, ToolPattern, ToolResultBlock, ToolUseBlock, TrajectoryBuilder, type TrajectoryInput, TrajectoryInstanceMetricKey, TrajectoryInstancesJson, TrajectoryPairInstanceJson, TrajectorySingleToolUseInstanceJson, TrajectoryView, Usage, UsageSummary, UserMessage, UserMessageEvent, aggregateCell, buildEvalRunEnvelope, buildEvalRunEnvelopeFromFiles, buildTrajectory, index_d_exports as claudeCode, computeToolCallMetrics, computeTrajectoryMetrics, createClaudeGrader, createLimit, emitOtel, enrichRepetitionWithProtojson, evaluate, evaluateAll, formatGradingConsole, formatReport, getAdapter, getDefaultAdapter, getRepetitions, gradeReport, gradingReportPassed, isAssistantMessage, isResult, isSystemInit, isSystemRetry, isTextBlock, isToolResultBlock, isToolUseBlock, isUserMessage, listAdapters, loadSuite, mergeConfig, namespaceOf, parseStreamJson, parseSuite, registerAdapter, resolveGradeOptions, runRepetition, runSuite, toEvaluationInstance, toHarnessMetrics, toInstancesJsonl, toTrajectory, toTrajectoryInstances, toolCallValid, toolNameMatch, toolParameterKeyMatch, toolParameterKvMatch, trajectoryAnyOrderMatch, trajectoryExactMatch, trajectoryInOrderMatch, trajectoryPrecision, trajectoryRecall, trajectorySingleToolUse, trajectoryToOtlp, trajectoryToTranscript };
+export { type AdapterDiagnostics, AdapterError, type AdapterResult, type AdapterRunFn, Assertion, AssertionResult, AssertionStat, AssistantMessage, AssistantMessageEvent, AssistantTurn, type BaseAdapterConfig, BuildEvalRunEnvelopeOptions, Cardinality, CellReport, CompoundPredicate, ConfigError, ContentBlock, DEFAULT_ADAPTER_ID, DEFAULT_REPETITIONS, DEFAULT_THRESHOLD, EVAL_RUN_SCHEMA_VERSION, type EmitOtelOptions, EvalArtifacts, EvalAssertionStat, EvalCellResult, EvalDatasetRow, EvalProvenance, EvalRepetition, EvalRunEnvelope, EvalRunSummary, EvaluationInstanceJson, type ExportTraceServiceRequest, ExternalScore, type GradeReportOptions, type HarnessAdapter, HarnessInfo, HarnessMetrics, InstanceData, InstancesJsonlRow, JudgeInfo, LeafPredicate, type LimitedRunner, MatrixCell, McpServerStatus, ObjectPredicate, OutcomeGrades, type ParseErrorRecord, type ParseResult, type PipelineConfig, Predicate, ProgressCallback, ProgressEvent, ProtojsonToolCall, ProtojsonTrajectory, ReferenceToolNameMode, ReferenceTrajectoryConfig, type RepGradingResult, RepetitionError, RepetitionResult, type ReporterOptions, ResultEvent, RetryRecord, RunSuiteOptions, SessionMeta, StopReason, StreamEvent, type SuiteConfig, type SuiteDocument, type SuiteGradingReport, SuiteReference, SuiteReport, SystemCompactBoundaryEvent, SystemInitEvent, SystemPluginInstallEvent, SystemRetryEvent, SystemUnknownEvent, TRAJECTORY_SCHEMA_VERSION, TestCase, TestSuite, TextBlock, ThresholdedAssertion, ToolCall, type ToolCallMetricOptions, ToolPattern, ToolResultBlock, ToolUseBlock, TrajectoryBuilder, type TrajectoryInput, TrajectoryInstanceMetricKey, TrajectoryInstancesJson, TrajectoryPairInstanceJson, TrajectorySingleToolUseInstanceJson, TrajectoryView, Usage, UsageSummary, UserMessage, UserMessageEvent, aggregateCell, buildEvalRunEnvelope, buildEvalRunEnvelopeFromFiles, buildTrajectory, index_d_exports as claudeCode, computeToolCallMetrics, computeTrajectoryMetrics, createClaudeGrader, createCodexGrader, createLimit, emitOtel, enrichRepetitionWithProtojson, evaluate, evaluateAll, formatGradingConsole, formatReport, getAdapter, getDefaultAdapter, getRepetitions, gradeReport, gradingReportPassed, isAssistantMessage, isResult, isSystemInit, isSystemRetry, isTextBlock, isToolResultBlock, isToolUseBlock, isUserMessage, listAdapters, loadSuite, loadSuiteDocument, mergeConfig, namespaceOf, parseStreamJson, parseSuite, registerAdapter, resolveGradeOptions, resolveGradingArtifactFromSuite, resolvePipelineInputs, runPipeline, runRepetition, runSuite, toEvaluationInstance, toHarnessMetrics, toInstancesJsonl, toTrajectory, toTrajectoryInstances, toolCallValid, toolNameMatch, toolParameterKeyMatch, toolParameterKvMatch, trajectoryAnyOrderMatch, trajectoryExactMatch, trajectoryInOrderMatch, trajectoryPrecision, trajectoryRecall, trajectorySingleToolUse, trajectoryToOtlp, trajectoryToTranscript };
 //# sourceMappingURL=index.d.ts.map

package/dist/index.js

@@ -1,7 +1,8 @@
-import { _ as isUserMessage, a as AdapterError, c as buildTrajectory, d as isResult, f as isSystemInit, g as isToolUseBlock, h as isToolResultBlock, l as namespaceOf, m as isTextBlock, n as claude_code_exports, o as parseStreamJson, p as isSystemRetry, s as TrajectoryBuilder, u as isAssistantMessage } from "./claude-code-DZ4Vkgp6.js";
-import { D as trajectoryToOtlp, E as emitOtel, O as EVAL_RUN_SCHEMA_VERSION, S as gradeReport, T as createClaudeGrader, _ as serializeToolInput, a as enrichRepetitionWithProtojson, b as gradingReportPassed, c as parseToolInput, d as trajectoryInOrderMatch, f as trajectoryPrecision, g as toTrajectoryInstances, h as toEvaluationInstance, i as buildEvalRunEnvelopeFromFiles, k as TRAJECTORY_SCHEMA_VERSION, l as trajectoryAnyOrderMatch, m as trajectorySingleToolUse, n as toTrajectory, o as toHarnessMetrics, p as trajectoryRecall, r as buildEvalRunEnvelope, s as computeTrajectoryMetrics, t as toInstancesJsonl, u as trajectoryExactMatch, v as formatReport, w as trajectoryToTranscript, x as resolveGradeOptions, y as formatGradingConsole } from "./projections-BcX7w-f6.js";
-import { a as aggregateCell, c as runRepetition, d as getDefaultAdapter, f as listAdapters, h as evaluateAll, i as DEFAULT_THRESHOLD, l as DEFAULT_ADAPTER_ID, m as evaluate, n as createLimit, o as getRepetitions, p as registerAdapter, r as DEFAULT_REPETITIONS, s as mergeConfig, t as runSuite, u as getAdapter } from "./suite-Dlzl-HI0.js";
-import { o as ConfigError, r as parseSuite, t as loadSuite } from "./loader-DcI0KfRX.js";
+import { a as isAssistantMessage, c as isSystemRetry, d as isToolUseBlock, f as isUserMessage, i as namespaceOf, l as isTextBlock, n as TrajectoryBuilder, o as isResult, r as buildTrajectory, s as isSystemInit, t as AdapterError, u as isToolResultBlock } from "./types-Bac8_Ixb.js";
+import { A as serializeToolInput, B as TRAJECTORY_SCHEMA_VERSION, C as trajectoryExactMatch, D as trajectorySingleToolUse, E as trajectoryRecall, I as trajectoryToTranscript, L as createCodexGrader, M as gradingReportPassed, N as resolveGradeOptions, O as toEvaluationInstance, P as gradeReport, R as createClaudeGrader, S as trajectoryAnyOrderMatch, T as trajectoryPrecision, _ as buildEvalRunEnvelopeFromFiles, b as computeTrajectoryMetrics, d as resolveGradingArtifactFromSuite, f as resolvePipelineInputs, g as buildEvalRunEnvelope, h as toTrajectory, i as runPipeline, j as formatGradingConsole, k as toTrajectoryInstances, m as toInstancesJsonl, n as emitOtel, r as trajectoryToOtlp, t as formatReport, v as enrichRepetitionWithProtojson, w as trajectoryInOrderMatch, x as parseToolInput, y as toHarnessMetrics, z as EVAL_RUN_SCHEMA_VERSION } from "./reporter-BKCJZRYr.js";
+import { a as parseStreamJson, n as claude_code_exports } from "./claude-code-C_7hxC8z.js";
+import { _ as evaluateAll, a as aggregateCell, c as runRepetition, d as getDefaultAdapter, f as listAdapters, g as evaluate, i as DEFAULT_THRESHOLD, l as DEFAULT_ADAPTER_ID, n as createLimit, o as getRepetitions, p as registerAdapter, r as DEFAULT_REPETITIONS, s as mergeConfig, t as runSuite, u as getAdapter } from "./suite-C3-8EjUW.js";
+import { l as ConfigError, n as parseSuite, o as loadSuiteDocument, t as loadSuite } from "./loader-CiBm4Kf6.js";
 //#region src/metrics/tool-calls.ts
 /**
 * Tool-call-level metrics operating on prediction/reference tool-call pairs.
@@ -119,6 +120,6 @@ function computeToolCallMetrics(predicted, reference, options = {}) {
 	};
 }
 //#endregion
-export { AdapterError, ConfigError, DEFAULT_ADAPTER_ID, DEFAULT_REPETITIONS, DEFAULT_THRESHOLD, EVAL_RUN_SCHEMA_VERSION, TRAJECTORY_SCHEMA_VERSION, TrajectoryBuilder, aggregateCell, buildEvalRunEnvelope, buildEvalRunEnvelopeFromFiles, buildTrajectory, claude_code_exports as claudeCode, computeToolCallMetrics, computeTrajectoryMetrics, createClaudeGrader, createLimit, emitOtel, enrichRepetitionWithProtojson, evaluate, evaluateAll, formatGradingConsole, formatReport, getAdapter, getDefaultAdapter, getRepetitions, gradeReport, gradingReportPassed, isAssistantMessage, isResult, isSystemInit, isSystemRetry, isTextBlock, isToolResultBlock, isToolUseBlock, isUserMessage, listAdapters, loadSuite, mergeConfig, namespaceOf, parseStreamJson, parseSuite, registerAdapter, resolveGradeOptions, runRepetition, runSuite, toEvaluationInstance, toHarnessMetrics, toInstancesJsonl, toTrajectory, toTrajectoryInstances, toolCallValid, toolNameMatch, toolParameterKeyMatch, toolParameterKvMatch, trajectoryAnyOrderMatch, trajectoryExactMatch, trajectoryInOrderMatch, trajectoryPrecision, trajectoryRecall, trajectorySingleToolUse, trajectoryToOtlp, trajectoryToTranscript };
+export { AdapterError, ConfigError, DEFAULT_ADAPTER_ID, DEFAULT_REPETITIONS, DEFAULT_THRESHOLD, EVAL_RUN_SCHEMA_VERSION, TRAJECTORY_SCHEMA_VERSION, TrajectoryBuilder, aggregateCell, buildEvalRunEnvelope, buildEvalRunEnvelopeFromFiles, buildTrajectory, claude_code_exports as claudeCode, computeToolCallMetrics, computeTrajectoryMetrics, createClaudeGrader, createCodexGrader, createLimit, emitOtel, enrichRepetitionWithProtojson, evaluate, evaluateAll, formatGradingConsole, formatReport, getAdapter, getDefaultAdapter, getRepetitions, gradeReport, gradingReportPassed, isAssistantMessage, isResult, isSystemInit, isSystemRetry, isTextBlock, isToolResultBlock, isToolUseBlock, isUserMessage, listAdapters, loadSuite, loadSuiteDocument, mergeConfig, namespaceOf, parseStreamJson, parseSuite, registerAdapter, resolveGradeOptions, resolveGradingArtifactFromSuite, resolvePipelineInputs, runPipeline, runRepetition, runSuite, toEvaluationInstance, toHarnessMetrics, toInstancesJsonl, toTrajectory, toTrajectoryInstances, toolCallValid, toolNameMatch, toolParameterKeyMatch, toolParameterKvMatch, trajectoryAnyOrderMatch, trajectoryExactMatch, trajectoryInOrderMatch, trajectoryPrecision, trajectoryRecall, trajectorySingleToolUse, trajectoryToOtlp, trajectoryToTranscript };
 
 //# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","names":[],"sources":["../src/metrics/tool-calls.ts"],"sourcesContent":["/**\n * Tool-call-level metrics operating on prediction/reference tool-call pairs.\n *\n * Implements Vertex-aligned per-call checks: validity, name match, parameter\n * key match, and full key-value match. Used by trajectory metrics and\n * available for custom eval pipelines.\n *\n * Scores are 0 or 1 per call; {@link computeToolCallMetrics} averages across\n * aligned index pairs (max length of predicted vs reference).\n */\n\nimport { parseToolInput, type TrajectoryInput, type WireToolCall } from \"./trajectory\";\nimport { serializeToolInput } from \"../eval-interchange/normalize\";\n\n/** Options for parameter value comparison. */\nexport interface ToolCallMetricOptions {\n  /** When true, compare serialized JSON strictly (reserved for future semantics). */\n  useStrictStringMatch?: boolean;\n}\n\n/** Aggregated tool-call metric scores (each 0..1). */\nexport interface ToolCallMetrics {\n  tool_call_valid: number;\n  tool_name_match: number;\n  tool_parameter_key_match: number;\n  tool_parameter_kv_match: number;\n}\n\ntype ToolCallInput = TrajectoryInput[number];\n\n/** Normalize harness or wire tool call to canonical wire shape for comparison. */\nfunction normalizeToolCall(toolCall: ToolCallInput): WireToolCall {\n  if (typeof toolCall.tool_input === \"string\") {\n    return {\n      tool_name: toolCall.tool_name,\n      tool_input: toolCall.tool_input,\n    };\n  }\n\n  return {\n    tool_name: toolCall.tool_name,\n    tool_input: serializeToolInput(toolCall.tool_input),\n  };\n}\n\n/** Parse tool_input JSON to an object map, or null when not a plain object. */\nfunction parsedArgs(toolCall: WireToolCall): Record<string, unknown> | null {\n  const parsed = parseToolInput(toolCall.tool_input);\n  if (parsed === null || typeof parsed !== \"object\" || Array.isArray(parsed)) {\n    return null;\n  }\n  return parsed as Record<string, unknown>;\n}\n\n/**\n * Whether a predicted tool call is well-formed (non-empty name, parseable JSON input).\n *\n * @returns 1 when valid, 0 otherwise.\n */\nexport function toolCallValid(toolCall: ToolCallInput): number {\n  const normalized = normalizeToolCall(toolCall);\n  if (!normalized.tool_name.trim()) return 0;\n\n  try {\n    JSON.parse(normalized.tool_input);\n    return 1;\n  } catch {\n    return 0;\n  }\n}\n\n/**\n * Whether predicted and reference tool names match exactly.\n *\n * @returns 1 on match, 0 otherwise.\n */\nexport function toolNameMatch(\n  predicted: ToolCallInput,\n  reference: ToolCallInput,\n): number {\n  const predictedNorm = normalizeToolCall(predicted);\n  const referenceNorm = normalizeToolCall(reference);\n  return predictedNorm.tool_name === referenceNorm.tool_name ? 1 : 0;\n}\n\n/**\n * Whether parameter key sets match (same keys, same order after sort).\n *\n * Requires matching tool names first. Returns 0 when args are not objects.\n */\nexport function toolParameterKeyMatch(\n  predicted: ToolCallInput,\n  reference: ToolCallInput,\n): number {\n  if (toolNameMatch(predicted, reference) === 0) return 0;\n\n  const predictedArgs = parsedArgs(normalizeToolCall(predicted));\n  const referenceArgs = parsedArgs(normalizeToolCall(reference));\n  if (predictedArgs === null || referenceArgs === null) return 0;\n\n  const predictedKeys = Object.keys(predictedArgs).sort();\n  const referenceKeys = Object.keys(referenceArgs).sort();\n  if (predictedKeys.length !== referenceKeys.length) return 0;\n\n  return predictedKeys.every((key, index) => key === referenceKeys[index])\n    ? 1\n    : 0;\n}\n\n/** Deep equality via JSON serialization (handles nested objects in args). */\nfunction valuesEqual(\n  left: unknown,\n  right: unknown,\n  useStrictStringMatch: boolean,\n): boolean {\n  if (useStrictStringMatch) {\n    return JSON.stringify(left) === JSON.stringify(right);\n  }\n  return JSON.stringify(left) === JSON.stringify(right);\n}\n\n/**\n * Whether all reference parameter key-value pairs match in the predicted call.\n *\n * Requires {@link toolParameterKeyMatch} first. Only keys present in reference\n * are checked (predicted may have extra keys).\n */\nexport function toolParameterKvMatch(\n  predicted: ToolCallInput,\n  reference: ToolCallInput,\n  options: ToolCallMetricOptions = {},\n): number {\n  if (toolParameterKeyMatch(predicted, reference) === 0) return 0;\n\n  const predictedArgs = parsedArgs(normalizeToolCall(predicted))!;\n  const referenceArgs = parsedArgs(normalizeToolCall(reference))!;\n\n  for (const key of Object.keys(referenceArgs)) {\n    if (\n      !valuesEqual(\n        predictedArgs[key],\n        referenceArgs[key],\n        options.useStrictStringMatch ?? false,\n      )\n    ) {\n      return 0;\n    }\n  }\n\n  return 1;\n}\n\n/**\n * Average tool-call metrics across index-aligned predicted/reference pairs.\n *\n * Denominator is `max(predicted.length, reference.length, 1)`. Missing\n * predicted calls at an index are skipped for pair metrics; validity still\n * counts when a predicted call exists.\n */\nexport function computeToolCallMetrics(\n  predicted: ToolCallInput[],\n  reference: ToolCallInput[],\n  options: ToolCallMetricOptions = {},\n): ToolCallMetrics {\n  const pairCount = Math.max(predicted.length, reference.length, 1);\n  let valid = 0;\n  let nameMatch = 0;\n  let keyMatch = 0;\n  let kvMatch = 0;\n\n  for (let index = 0; index < pairCount; index += 1) {\n    const predictedCall = predicted[index];\n    const referenceCall = reference[index];\n    if (!predictedCall) continue;\n\n    valid += toolCallValid(predictedCall);\n    if (!referenceCall) continue;\n\n    nameMatch += toolNameMatch(predictedCall, referenceCall);\n    keyMatch += toolParameterKeyMatch(predictedCall, referenceCall);\n    kvMatch += toolParameterKvMatch(predictedCall, referenceCall, options);\n  }\n\n  return {\n    tool_call_valid: valid / pairCount,\n    tool_name_match: nameMatch / pairCount,\n    tool_parameter_key_match: keyMatch / pairCount,\n    tool_parameter_kv_match: kvMatch / pairCount,\n  };\n}\n"],"mappings":";;;;;;;;;;;;;;;;AA+BA,SAAS,kBAAkB,UAAuC;CAChE,IAAI,OAAO,SAAS,eAAe,UACjC,OAAO;EACL,WAAW,SAAS;EACpB,YAAY,SAAS;CACvB;CAGF,OAAO;EACL,WAAW,SAAS;EACpB,YAAY,mBAAmB,SAAS,UAAU;CACpD;AACF;;AAGA,SAAS,WAAW,UAAwD;CAC1E,MAAM,SAAS,eAAe,SAAS,UAAU;CACjD,IAAI,WAAW,QAAQ,OAAO,WAAW,YAAY,MAAM,QAAQ,MAAM,GACvE,OAAO;CAET,OAAO;AACT;;;;;;AAOA,SAAgB,cAAc,UAAiC;CAC7D,MAAM,aAAa,kBAAkB,QAAQ;CAC7C,IAAI,CAAC,WAAW,UAAU,KAAK,GAAG,OAAO;CAEzC,IAAI;EACF,KAAK,MAAM,WAAW,UAAU;EAChC,OAAO;CACT,QAAQ;EACN,OAAO;CACT;AACF;;;;;;AAOA,SAAgB,cACd,WACA,WACQ;CACR,MAAM,gBAAgB,kBAAkB,SAAS;CACjD,MAAM,gBAAgB,kBAAkB,SAAS;CACjD,OAAO,cAAc,cAAc,cAAc,YAAY,IAAI;AACnE;;;;;;AAOA,SAAgB,sBACd,WACA,WACQ;CACR,IAAI,cAAc,WAAW,SAAS,MAAM,GAAG,OAAO;CAEtD,MAAM,gBAAgB,WAAW,kBAAkB,SAAS,CAAC;CAC7D,MAAM,gBAAgB,WAAW,kBAAkB,SAAS,CAAC;CAC7D,IAAI,kBAAkB,QAAQ,kBAAkB,MAAM,OAAO;CAE7D,MAAM,gBAAgB,OAAO,KAAK,aAAa,CAAC,CAAC,KAAK;CACtD,MAAM,gBAAgB,OAAO,KAAK,aAAa,CAAC,CAAC,KAAK;CACtD,IAAI,cAAc,WAAW,cAAc,QAAQ,OAAO;CAE1D,OAAO,cAAc,OAAO,KAAK,UAAU,QAAQ,cAAc,MAAM,IACnE,IACA;AACN;;AAGA,SAAS,YACP,MACA,OACA,sBACS;CACT,IAAI,sBACF,OAAO,KAAK,UAAU,IAAI,MAAM,KAAK,UAAU,KAAK;CAEtD,OAAO,KAAK,UAAU,IAAI,MAAM,KAAK,UAAU,KAAK;AACtD;;;;;;;AAQA,SAAgB,qBACd,WACA,WACA,UAAiC,CAAC,GAC1B;CACR,IAAI,sBAAsB,WAAW,SAAS,MAAM,GAAG,OAAO;CAE9D,MAAM,gBAAgB,WAAW,kBAAkB,SAAS,CAAC;CAC7D,MAAM,gBAAgB,WAAW,kBAAkB,SAAS,CAAC;CAE7D,KAAK,MAAM,OAAO,OAAO,KAAK,aAAa,GACzC,IACE,CAAC,YACC,cAAc,MACd,cAAc,MACd,QAAQ,wBAAwB,KAClC,GAEA,OAAO;CAIX,OAAO;AACT;;;;;;;;AASA,SAAgB,uBACd,WACA,WACA,UAAiC,CAAC,GACjB;CACjB,MAAM,YAAY,KAAK,IAAI,UAAU,QAAQ,UAAU,QAAQ,CAAC;CAChE,IAAI,QAAQ;CACZ,IAAI,YAAY;CAChB,IAAI,WAAW;CACf,IAAI,UAAU;CAEd,KAAK,IAAI,QAAQ,GAAG,QAAQ,WAAW,SAAS,GAAG;EACjD,MAAM,gBAAgB,UAAU;EAChC,MAAM,gBAAgB,UAAU;EAChC,IAAI,CAAC,eAAe;EAEpB,SAAS,cAAc,aAAa;EACpC,IAAI,CAAC,eAAe;EAEpB,aAAa,cAAc,eAAe,aAAa;EACvD,YAAY,sBAAsB,eAAe,aAAa;EAC9D,WAAW,qBAAqB,eAAe,eAAe,OAAO;CACvE;CAEA,OAAO;EACL,iBAAiB,QAAQ;EACzB,iBAAiB,YAAY;EAC7B,0BAA0B,WAAW;EACrC,yBAAyB,UAAU;CACrC;AACF"}
+{"version":3,"file":"index.js","names":[],"sources":["../src/metrics/tool-calls.ts"],"sourcesContent":["/**\n * Tool-call-level metrics operating on prediction/reference tool-call pairs.\n *\n * Implements Vertex-aligned per-call checks: validity, name match, parameter\n * key match, and full key-value match. Used by trajectory metrics and\n * available for custom eval pipelines.\n *\n * Scores are 0 or 1 per call; {@link computeToolCallMetrics} averages across\n * aligned index pairs (max length of predicted vs reference).\n */\n\nimport { parseToolInput, type TrajectoryInput, type WireToolCall } from \"./trajectory\";\nimport { serializeToolInput } from \"../eval-interchange/normalize\";\n\n/** Options for parameter value comparison. */\nexport interface ToolCallMetricOptions {\n  /** When true, compare serialized JSON strictly (reserved for future semantics). */\n  useStrictStringMatch?: boolean;\n}\n\n/** Aggregated tool-call metric scores (each 0..1). */\nexport interface ToolCallMetrics {\n  tool_call_valid: number;\n  tool_name_match: number;\n  tool_parameter_key_match: number;\n  tool_parameter_kv_match: number;\n}\n\ntype ToolCallInput = TrajectoryInput[number];\n\n/** Normalize harness or wire tool call to canonical wire shape for comparison. */\nfunction normalizeToolCall(toolCall: ToolCallInput): WireToolCall {\n  if (typeof toolCall.tool_input === \"string\") {\n    return {\n      tool_name: toolCall.tool_name,\n      tool_input: toolCall.tool_input,\n    };\n  }\n\n  return {\n    tool_name: toolCall.tool_name,\n    tool_input: serializeToolInput(toolCall.tool_input),\n  };\n}\n\n/** Parse tool_input JSON to an object map, or null when not a plain object. */\nfunction parsedArgs(toolCall: WireToolCall): Record<string, unknown> | null {\n  const parsed = parseToolInput(toolCall.tool_input);\n  if (parsed === null || typeof parsed !== \"object\" || Array.isArray(parsed)) {\n    return null;\n  }\n  return parsed as Record<string, unknown>;\n}\n\n/**\n * Whether a predicted tool call is well-formed (non-empty name, parseable JSON input).\n *\n * @returns 1 when valid, 0 otherwise.\n */\nexport function toolCallValid(toolCall: ToolCallInput): number {\n  const normalized = normalizeToolCall(toolCall);\n  if (!normalized.tool_name.trim()) return 0;\n\n  try {\n    JSON.parse(normalized.tool_input);\n    return 1;\n  } catch {\n    return 0;\n  }\n}\n\n/**\n * Whether predicted and reference tool names match exactly.\n *\n * @returns 1 on match, 0 otherwise.\n */\nexport function toolNameMatch(\n  predicted: ToolCallInput,\n  reference: ToolCallInput,\n): number {\n  const predictedNorm = normalizeToolCall(predicted);\n  const referenceNorm = normalizeToolCall(reference);\n  return predictedNorm.tool_name === referenceNorm.tool_name ? 1 : 0;\n}\n\n/**\n * Whether parameter key sets match (same keys, same order after sort).\n *\n * Requires matching tool names first. Returns 0 when args are not objects.\n */\nexport function toolParameterKeyMatch(\n  predicted: ToolCallInput,\n  reference: ToolCallInput,\n): number {\n  if (toolNameMatch(predicted, reference) === 0) return 0;\n\n  const predictedArgs = parsedArgs(normalizeToolCall(predicted));\n  const referenceArgs = parsedArgs(normalizeToolCall(reference));\n  if (predictedArgs === null || referenceArgs === null) return 0;\n\n  const predictedKeys = Object.keys(predictedArgs).sort();\n  const referenceKeys = Object.keys(referenceArgs).sort();\n  if (predictedKeys.length !== referenceKeys.length) return 0;\n\n  return predictedKeys.every((key, index) => key === referenceKeys[index])\n    ? 1\n    : 0;\n}\n\n/** Deep equality via JSON serialization (handles nested objects in args). */\nfunction valuesEqual(\n  left: unknown,\n  right: unknown,\n  useStrictStringMatch: boolean,\n): boolean {\n  if (useStrictStringMatch) {\n    return JSON.stringify(left) === JSON.stringify(right);\n  }\n  return JSON.stringify(left) === JSON.stringify(right);\n}\n\n/**\n * Whether all reference parameter key-value pairs match in the predicted call.\n *\n * Requires {@link toolParameterKeyMatch} first. Only keys present in reference\n * are checked (predicted may have extra keys).\n */\nexport function toolParameterKvMatch(\n  predicted: ToolCallInput,\n  reference: ToolCallInput,\n  options: ToolCallMetricOptions = {},\n): number {\n  if (toolParameterKeyMatch(predicted, reference) === 0) return 0;\n\n  const predictedArgs = parsedArgs(normalizeToolCall(predicted))!;\n  const referenceArgs = parsedArgs(normalizeToolCall(reference))!;\n\n  for (const key of Object.keys(referenceArgs)) {\n    if (\n      !valuesEqual(\n        predictedArgs[key],\n        referenceArgs[key],\n        options.useStrictStringMatch ?? false,\n      )\n    ) {\n      return 0;\n    }\n  }\n\n  return 1;\n}\n\n/**\n * Average tool-call metrics across index-aligned predicted/reference pairs.\n *\n * Denominator is `max(predicted.length, reference.length, 1)`. Missing\n * predicted calls at an index are skipped for pair metrics; validity still\n * counts when a predicted call exists.\n */\nexport function computeToolCallMetrics(\n  predicted: ToolCallInput[],\n  reference: ToolCallInput[],\n  options: ToolCallMetricOptions = {},\n): ToolCallMetrics {\n  const pairCount = Math.max(predicted.length, reference.length, 1);\n  let valid = 0;\n  let nameMatch = 0;\n  let keyMatch = 0;\n  let kvMatch = 0;\n\n  for (let index = 0; index < pairCount; index += 1) {\n    const predictedCall = predicted[index];\n    const referenceCall = reference[index];\n    if (!predictedCall) continue;\n\n    valid += toolCallValid(predictedCall);\n    if (!referenceCall) continue;\n\n    nameMatch += toolNameMatch(predictedCall, referenceCall);\n    keyMatch += toolParameterKeyMatch(predictedCall, referenceCall);\n    kvMatch += toolParameterKvMatch(predictedCall, referenceCall, options);\n  }\n\n  return {\n    tool_call_valid: valid / pairCount,\n    tool_name_match: nameMatch / pairCount,\n    tool_parameter_key_match: keyMatch / pairCount,\n    tool_parameter_kv_match: kvMatch / pairCount,\n  };\n}\n"],"mappings":";;;;;;;;;;;;;;;;;AA+BA,SAAS,kBAAkB,UAAuC;CAChE,IAAI,OAAO,SAAS,eAAe,UACjC,OAAO;EACL,WAAW,SAAS;EACpB,YAAY,SAAS;CACvB;CAGF,OAAO;EACL,WAAW,SAAS;EACpB,YAAY,mBAAmB,SAAS,UAAU;CACpD;AACF;;AAGA,SAAS,WAAW,UAAwD;CAC1E,MAAM,SAAS,eAAe,SAAS,UAAU;CACjD,IAAI,WAAW,QAAQ,OAAO,WAAW,YAAY,MAAM,QAAQ,MAAM,GACvE,OAAO;CAET,OAAO;AACT;;;;;;AAOA,SAAgB,cAAc,UAAiC;CAC7D,MAAM,aAAa,kBAAkB,QAAQ;CAC7C,IAAI,CAAC,WAAW,UAAU,KAAK,GAAG,OAAO;CAEzC,IAAI;EACF,KAAK,MAAM,WAAW,UAAU;EAChC,OAAO;CACT,QAAQ;EACN,OAAO;CACT;AACF;;;;;;AAOA,SAAgB,cACd,WACA,WACQ;CACR,MAAM,gBAAgB,kBAAkB,SAAS;CACjD,MAAM,gBAAgB,kBAAkB,SAAS;CACjD,OAAO,cAAc,cAAc,cAAc,YAAY,IAAI;AACnE;;;;;;AAOA,SAAgB,sBACd,WACA,WACQ;CACR,IAAI,cAAc,WAAW,SAAS,MAAM,GAAG,OAAO;CAEtD,MAAM,gBAAgB,WAAW,kBAAkB,SAAS,CAAC;CAC7D,MAAM,gBAAgB,WAAW,kBAAkB,SAAS,CAAC;CAC7D,IAAI,kBAAkB,QAAQ,kBAAkB,MAAM,OAAO;CAE7D,MAAM,gBAAgB,OAAO,KAAK,aAAa,CAAC,CAAC,KAAK;CACtD,MAAM,gBAAgB,OAAO,KAAK,aAAa,CAAC,CAAC,KAAK;CACtD,IAAI,cAAc,WAAW,cAAc,QAAQ,OAAO;CAE1D,OAAO,cAAc,OAAO,KAAK,UAAU,QAAQ,cAAc,MAAM,IACnE,IACA;AACN;;AAGA,SAAS,YACP,MACA,OACA,sBACS;CACT,IAAI,sBACF,OAAO,KAAK,UAAU,IAAI,MAAM,KAAK,UAAU,KAAK;CAEtD,OAAO,KAAK,UAAU,IAAI,MAAM,KAAK,UAAU,KAAK;AACtD;;;;;;;AAQA,SAAgB,qBACd,WACA,WACA,UAAiC,CAAC,GAC1B;CACR,IAAI,sBAAsB,WAAW,SAAS,MAAM,GAAG,OAAO;CAE9D,MAAM,gBAAgB,WAAW,kBAAkB,SAAS,CAAC;CAC7D,MAAM,gBAAgB,WAAW,kBAAkB,SAAS,CAAC;CAE7D,KAAK,MAAM,OAAO,OAAO,KAAK,aAAa,GACzC,IACE,CAAC,YACC,cAAc,MACd,cAAc,MACd,QAAQ,wBAAwB,KAClC,GAEA,OAAO;CAIX,OAAO;AACT;;;;;;;;AASA,SAAgB,uBACd,WACA,WACA,UAAiC,CAAC,GACjB;CACjB,MAAM,YAAY,KAAK,IAAI,UAAU,QAAQ,UAAU,QAAQ,CAAC;CAChE,IAAI,QAAQ;CACZ,IAAI,YAAY;CAChB,IAAI,WAAW;CACf,IAAI,UAAU;CAEd,KAAK,IAAI,QAAQ,GAAG,QAAQ,WAAW,SAAS,GAAG;EACjD,MAAM,gBAAgB,UAAU;EAChC,MAAM,gBAAgB,UAAU;EAChC,IAAI,CAAC,eAAe;EAEpB,SAAS,cAAc,aAAa;EACpC,IAAI,CAAC,eAAe;EAEpB,aAAa,cAAc,eAAe,aAAa;EACvD,YAAY,sBAAsB,eAAe,aAAa;EAC9D,WAAW,qBAAqB,eAAe,eAAe,OAAO;CACvE;CAEA,OAAO;EACL,iBAAiB,QAAQ;EACzB,iBAAiB,YAAY;EAC7B,0BAA0B,WAAW;EACrC,yBAAyB,UAAU;CACrC;AACF"}