@alis-build/harness-eval 0.1.1 → 0.1.3

package/dist/index.js

@@ -1,5 +1,125 @@
-import { _ as isUserMessage, a as AdapterError, c as buildTrajectory, d as isResult, f as isSystemInit, g as isToolUseBlock, h as isToolResultBlock, l as namespaceOf, m as isTextBlock, n as claude_code_exports, o as parseStreamJson, p as isSystemRetry, s as TrajectoryBuilder, u as isAssistantMessage } from "./claude-code-ycT0JQZF.js";
-import { A as TRAJECTORY_SCHEMA_VERSION, C as gradeReport, D as emitOtel, E as createClaudeGrader, O as trajectoryToOtlp, S as resolveGradeOptions, T as trajectoryToTranscript, _ as trajectoryRecall, a as toProtoInstances, b as formatGradingConsole, c as toolCallValid, d as toolParameterKvMatch, f as computeTrajectoryMetrics, g as trajectoryPrecision, h as trajectoryInOrderMatch, i as toAgentTrace, k as EVAL_RUN_SCHEMA_VERSION, l as toolNameMatch, m as trajectoryExactMatch, n as buildEvalRunEnvelopeFromFiles, o as toTrajectory, p as trajectoryAnyOrderMatch, r as enrichRepetitionWithInterchange, s as computeToolCallMetrics, t as buildEvalRunEnvelope, u as toolParameterKeyMatch, v as trajectorySingleToolUse, x as gradingReportPassed, y as formatReport } from "./build-DsVJ_UeU.js";
-import { a as aggregateCell, c as runRepetition, d as getDefaultAdapter, f as listAdapters, h as evaluateAll, i as DEFAULT_THRESHOLD, l as DEFAULT_ADAPTER_ID, m as evaluate, n as createLimit, o as getRepetitions, p as registerAdapter, r as DEFAULT_REPETITIONS, s as mergeConfig, t as runSuite, u as getAdapter } from "./suite-chj0j22j.js";
-import { o as ConfigError, r as parseSuite, t as loadSuite } from "./loader-BCnFJ8rm.js";
-export { AdapterError, ConfigError, DEFAULT_ADAPTER_ID, DEFAULT_REPETITIONS, DEFAULT_THRESHOLD, EVAL_RUN_SCHEMA_VERSION, TRAJECTORY_SCHEMA_VERSION, TrajectoryBuilder, aggregateCell, buildEvalRunEnvelope, buildEvalRunEnvelopeFromFiles, buildTrajectory, claude_code_exports as claudeCode, computeToolCallMetrics, computeTrajectoryMetrics, createClaudeGrader, createLimit, emitOtel, enrichRepetitionWithInterchange, evaluate, evaluateAll, formatGradingConsole, formatReport, getAdapter, getDefaultAdapter, getRepetitions, gradeReport, gradingReportPassed, isAssistantMessage, isResult, isSystemInit, isSystemRetry, isTextBlock, isToolResultBlock, isToolUseBlock, isUserMessage, listAdapters, loadSuite, mergeConfig, namespaceOf, parseStreamJson, parseSuite, registerAdapter, resolveGradeOptions, runRepetition, runSuite, toAgentTrace, toProtoInstances, toTrajectory, toolCallValid, toolNameMatch, toolParameterKeyMatch, toolParameterKvMatch, trajectoryAnyOrderMatch, trajectoryExactMatch, trajectoryInOrderMatch, trajectoryPrecision, trajectoryRecall, trajectorySingleToolUse, trajectoryToOtlp, trajectoryToTranscript };
+import { a as isAssistantMessage, c as isSystemRetry, d as isToolUseBlock, f as isUserMessage, i as namespaceOf, l as isTextBlock, n as TrajectoryBuilder, o as isResult, r as buildTrajectory, s as isSystemInit, t as AdapterError, u as isToolResultBlock } from "./types-Bac8_Ixb.js";
+import { A as serializeToolInput, B as TRAJECTORY_SCHEMA_VERSION, C as trajectoryExactMatch, D as trajectorySingleToolUse, E as trajectoryRecall, I as trajectoryToTranscript, L as createCodexGrader, M as gradingReportPassed, N as resolveGradeOptions, O as toEvaluationInstance, P as gradeReport, R as createClaudeGrader, S as trajectoryAnyOrderMatch, T as trajectoryPrecision, _ as buildEvalRunEnvelopeFromFiles, b as computeTrajectoryMetrics, d as resolveGradingArtifactFromSuite, f as resolvePipelineInputs, g as buildEvalRunEnvelope, h as toTrajectory, i as runPipeline, j as formatGradingConsole, k as toTrajectoryInstances, m as toInstancesJsonl, n as emitOtel, r as trajectoryToOtlp, t as formatReport, v as enrichRepetitionWithProtojson, w as trajectoryInOrderMatch, x as parseToolInput, y as toHarnessMetrics, z as EVAL_RUN_SCHEMA_VERSION } from "./reporter-Biy-5-9M.js";
+import { a as parseStreamJson, n as claude_code_exports } from "./claude-code-C_7hxC8z.js";
+import { a as aggregateCell, c as runRepetition, d as getDefaultAdapter, f as listAdapters, h as evaluateAll, i as DEFAULT_THRESHOLD, l as DEFAULT_ADAPTER_ID, m as evaluate, n as createLimit, o as getRepetitions, p as registerAdapter, r as DEFAULT_REPETITIONS, s as mergeConfig, t as runSuite, u as getAdapter } from "./suite-BcP64nlb.js";
+import { l as ConfigError, n as parseSuite, o as loadSuiteDocument, t as loadSuite } from "./loader-DnQ6Jt0i.js";
+//#region src/metrics/tool-calls.ts
+/**
+* Tool-call-level metrics operating on prediction/reference tool-call pairs.
+*
+* Implements Vertex-aligned per-call checks: validity, name match, parameter
+* key match, and full key-value match. Used by trajectory metrics and
+* available for custom eval pipelines.
+*
+* Scores are 0 or 1 per call; {@link computeToolCallMetrics} averages across
+* aligned index pairs (max length of predicted vs reference).
+*/
+/** Normalize harness or wire tool call to canonical wire shape for comparison. */
+function normalizeToolCall(toolCall) {
+	if (typeof toolCall.tool_input === "string") return {
+		tool_name: toolCall.tool_name,
+		tool_input: toolCall.tool_input
+	};
+	return {
+		tool_name: toolCall.tool_name,
+		tool_input: serializeToolInput(toolCall.tool_input)
+	};
+}
+/** Parse tool_input JSON to an object map, or null when not a plain object. */
+function parsedArgs(toolCall) {
+	const parsed = parseToolInput(toolCall.tool_input);
+	if (parsed === null || typeof parsed !== "object" || Array.isArray(parsed)) return null;
+	return parsed;
+}
+/**
+* Whether a predicted tool call is well-formed (non-empty name, parseable JSON input).
+*
+* @returns 1 when valid, 0 otherwise.
+*/
+function toolCallValid(toolCall) {
+	const normalized = normalizeToolCall(toolCall);
+	if (!normalized.tool_name.trim()) return 0;
+	try {
+		JSON.parse(normalized.tool_input);
+		return 1;
+	} catch {
+		return 0;
+	}
+}
+/**
+* Whether predicted and reference tool names match exactly.
+*
+* @returns 1 on match, 0 otherwise.
+*/
+function toolNameMatch(predicted, reference) {
+	const predictedNorm = normalizeToolCall(predicted);
+	const referenceNorm = normalizeToolCall(reference);
+	return predictedNorm.tool_name === referenceNorm.tool_name ? 1 : 0;
+}
+/**
+* Whether parameter key sets match (same keys, same order after sort).
+*
+* Requires matching tool names first. Returns 0 when args are not objects.
+*/
+function toolParameterKeyMatch(predicted, reference) {
+	if (toolNameMatch(predicted, reference) === 0) return 0;
+	const predictedArgs = parsedArgs(normalizeToolCall(predicted));
+	const referenceArgs = parsedArgs(normalizeToolCall(reference));
+	if (predictedArgs === null || referenceArgs === null) return 0;
+	const predictedKeys = Object.keys(predictedArgs).sort();
+	const referenceKeys = Object.keys(referenceArgs).sort();
+	if (predictedKeys.length !== referenceKeys.length) return 0;
+	return predictedKeys.every((key, index) => key === referenceKeys[index]) ? 1 : 0;
+}
+/** Deep equality via JSON serialization (handles nested objects in args). */
+function valuesEqual(left, right, useStrictStringMatch) {
+	if (useStrictStringMatch) return JSON.stringify(left) === JSON.stringify(right);
+	return JSON.stringify(left) === JSON.stringify(right);
+}
+/**
+* Whether all reference parameter key-value pairs match in the predicted call.
+*
+* Requires {@link toolParameterKeyMatch} first. Only keys present in reference
+* are checked (predicted may have extra keys).
+*/
+function toolParameterKvMatch(predicted, reference, options = {}) {
+	if (toolParameterKeyMatch(predicted, reference) === 0) return 0;
+	const predictedArgs = parsedArgs(normalizeToolCall(predicted));
+	const referenceArgs = parsedArgs(normalizeToolCall(reference));
+	for (const key of Object.keys(referenceArgs)) if (!valuesEqual(predictedArgs[key], referenceArgs[key], options.useStrictStringMatch ?? false)) return 0;
+	return 1;
+}
+/**
+* Average tool-call metrics across index-aligned predicted/reference pairs.
+*
+* Denominator is `max(predicted.length, reference.length, 1)`. Missing
+* predicted calls at an index are skipped for pair metrics; validity still
+* counts when a predicted call exists.
+*/
+function computeToolCallMetrics(predicted, reference, options = {}) {
+	const pairCount = Math.max(predicted.length, reference.length, 1);
+	let valid = 0;
+	let nameMatch = 0;
+	let keyMatch = 0;
+	let kvMatch = 0;
+	for (let index = 0; index < pairCount; index += 1) {
+		const predictedCall = predicted[index];
+		const referenceCall = reference[index];
+		if (!predictedCall) continue;
+		valid += toolCallValid(predictedCall);
+		if (!referenceCall) continue;
+		nameMatch += toolNameMatch(predictedCall, referenceCall);
+		keyMatch += toolParameterKeyMatch(predictedCall, referenceCall);
+		kvMatch += toolParameterKvMatch(predictedCall, referenceCall, options);
+	}
+	return {
+		tool_call_valid: valid / pairCount,
+		tool_name_match: nameMatch / pairCount,
+		tool_parameter_key_match: keyMatch / pairCount,
+		tool_parameter_kv_match: kvMatch / pairCount
+	};
+}
+//#endregion
+export { AdapterError, ConfigError, DEFAULT_ADAPTER_ID, DEFAULT_REPETITIONS, DEFAULT_THRESHOLD, EVAL_RUN_SCHEMA_VERSION, TRAJECTORY_SCHEMA_VERSION, TrajectoryBuilder, aggregateCell, buildEvalRunEnvelope, buildEvalRunEnvelopeFromFiles, buildTrajectory, claude_code_exports as claudeCode, computeToolCallMetrics, computeTrajectoryMetrics, createClaudeGrader, createCodexGrader, createLimit, emitOtel, enrichRepetitionWithProtojson, evaluate, evaluateAll, formatGradingConsole, formatReport, getAdapter, getDefaultAdapter, getRepetitions, gradeReport, gradingReportPassed, isAssistantMessage, isResult, isSystemInit, isSystemRetry, isTextBlock, isToolResultBlock, isToolUseBlock, isUserMessage, listAdapters, loadSuite, loadSuiteDocument, mergeConfig, namespaceOf, parseStreamJson, parseSuite, registerAdapter, resolveGradeOptions, resolveGradingArtifactFromSuite, resolvePipelineInputs, runPipeline, runRepetition, runSuite, toEvaluationInstance, toHarnessMetrics, toInstancesJsonl, toTrajectory, toTrajectoryInstances, toolCallValid, toolNameMatch, toolParameterKeyMatch, toolParameterKvMatch, trajectoryAnyOrderMatch, trajectoryExactMatch, trajectoryInOrderMatch, trajectoryPrecision, trajectoryRecall, trajectorySingleToolUse, trajectoryToOtlp, trajectoryToTranscript };
+
+//# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","names":[],"sources":["../src/metrics/tool-calls.ts"],"sourcesContent":["/**\n * Tool-call-level metrics operating on prediction/reference tool-call pairs.\n *\n * Implements Vertex-aligned per-call checks: validity, name match, parameter\n * key match, and full key-value match. Used by trajectory metrics and\n * available for custom eval pipelines.\n *\n * Scores are 0 or 1 per call; {@link computeToolCallMetrics} averages across\n * aligned index pairs (max length of predicted vs reference).\n */\n\nimport { parseToolInput, type TrajectoryInput, type WireToolCall } from \"./trajectory\";\nimport { serializeToolInput } from \"../eval-interchange/normalize\";\n\n/** Options for parameter value comparison. */\nexport interface ToolCallMetricOptions {\n  /** When true, compare serialized JSON strictly (reserved for future semantics). */\n  useStrictStringMatch?: boolean;\n}\n\n/** Aggregated tool-call metric scores (each 0..1). */\nexport interface ToolCallMetrics {\n  tool_call_valid: number;\n  tool_name_match: number;\n  tool_parameter_key_match: number;\n  tool_parameter_kv_match: number;\n}\n\ntype ToolCallInput = TrajectoryInput[number];\n\n/** Normalize harness or wire tool call to canonical wire shape for comparison. */\nfunction normalizeToolCall(toolCall: ToolCallInput): WireToolCall {\n  if (typeof toolCall.tool_input === \"string\") {\n    return {\n      tool_name: toolCall.tool_name,\n      tool_input: toolCall.tool_input,\n    };\n  }\n\n  return {\n    tool_name: toolCall.tool_name,\n    tool_input: serializeToolInput(toolCall.tool_input),\n  };\n}\n\n/** Parse tool_input JSON to an object map, or null when not a plain object. */\nfunction parsedArgs(toolCall: WireToolCall): Record<string, unknown> | null {\n  const parsed = parseToolInput(toolCall.tool_input);\n  if (parsed === null || typeof parsed !== \"object\" || Array.isArray(parsed)) {\n    return null;\n  }\n  return parsed as Record<string, unknown>;\n}\n\n/**\n * Whether a predicted tool call is well-formed (non-empty name, parseable JSON input).\n *\n * @returns 1 when valid, 0 otherwise.\n */\nexport function toolCallValid(toolCall: ToolCallInput): number {\n  const normalized = normalizeToolCall(toolCall);\n  if (!normalized.tool_name.trim()) return 0;\n\n  try {\n    JSON.parse(normalized.tool_input);\n    return 1;\n  } catch {\n    return 0;\n  }\n}\n\n/**\n * Whether predicted and reference tool names match exactly.\n *\n * @returns 1 on match, 0 otherwise.\n */\nexport function toolNameMatch(\n  predicted: ToolCallInput,\n  reference: ToolCallInput,\n): number {\n  const predictedNorm = normalizeToolCall(predicted);\n  const referenceNorm = normalizeToolCall(reference);\n  return predictedNorm.tool_name === referenceNorm.tool_name ? 1 : 0;\n}\n\n/**\n * Whether parameter key sets match (same keys, same order after sort).\n *\n * Requires matching tool names first. Returns 0 when args are not objects.\n */\nexport function toolParameterKeyMatch(\n  predicted: ToolCallInput,\n  reference: ToolCallInput,\n): number {\n  if (toolNameMatch(predicted, reference) === 0) return 0;\n\n  const predictedArgs = parsedArgs(normalizeToolCall(predicted));\n  const referenceArgs = parsedArgs(normalizeToolCall(reference));\n  if (predictedArgs === null || referenceArgs === null) return 0;\n\n  const predictedKeys = Object.keys(predictedArgs).sort();\n  const referenceKeys = Object.keys(referenceArgs).sort();\n  if (predictedKeys.length !== referenceKeys.length) return 0;\n\n  return predictedKeys.every((key, index) => key === referenceKeys[index])\n    ? 1\n    : 0;\n}\n\n/** Deep equality via JSON serialization (handles nested objects in args). */\nfunction valuesEqual(\n  left: unknown,\n  right: unknown,\n  useStrictStringMatch: boolean,\n): boolean {\n  if (useStrictStringMatch) {\n    return JSON.stringify(left) === JSON.stringify(right);\n  }\n  return JSON.stringify(left) === JSON.stringify(right);\n}\n\n/**\n * Whether all reference parameter key-value pairs match in the predicted call.\n *\n * Requires {@link toolParameterKeyMatch} first. Only keys present in reference\n * are checked (predicted may have extra keys).\n */\nexport function toolParameterKvMatch(\n  predicted: ToolCallInput,\n  reference: ToolCallInput,\n  options: ToolCallMetricOptions = {},\n): number {\n  if (toolParameterKeyMatch(predicted, reference) === 0) return 0;\n\n  const predictedArgs = parsedArgs(normalizeToolCall(predicted))!;\n  const referenceArgs = parsedArgs(normalizeToolCall(reference))!;\n\n  for (const key of Object.keys(referenceArgs)) {\n    if (\n      !valuesEqual(\n        predictedArgs[key],\n        referenceArgs[key],\n        options.useStrictStringMatch ?? false,\n      )\n    ) {\n      return 0;\n    }\n  }\n\n  return 1;\n}\n\n/**\n * Average tool-call metrics across index-aligned predicted/reference pairs.\n *\n * Denominator is `max(predicted.length, reference.length, 1)`. Missing\n * predicted calls at an index are skipped for pair metrics; validity still\n * counts when a predicted call exists.\n */\nexport function computeToolCallMetrics(\n  predicted: ToolCallInput[],\n  reference: ToolCallInput[],\n  options: ToolCallMetricOptions = {},\n): ToolCallMetrics {\n  const pairCount = Math.max(predicted.length, reference.length, 1);\n  let valid = 0;\n  let nameMatch = 0;\n  let keyMatch = 0;\n  let kvMatch = 0;\n\n  for (let index = 0; index < pairCount; index += 1) {\n    const predictedCall = predicted[index];\n    const referenceCall = reference[index];\n    if (!predictedCall) continue;\n\n    valid += toolCallValid(predictedCall);\n    if (!referenceCall) continue;\n\n    nameMatch += toolNameMatch(predictedCall, referenceCall);\n    keyMatch += toolParameterKeyMatch(predictedCall, referenceCall);\n    kvMatch += toolParameterKvMatch(predictedCall, referenceCall, options);\n  }\n\n  return {\n    tool_call_valid: valid / pairCount,\n    tool_name_match: nameMatch / pairCount,\n    tool_parameter_key_match: keyMatch / pairCount,\n    tool_parameter_kv_match: kvMatch / pairCount,\n  };\n}\n"],"mappings":";;;;;;;;;;;;;;;;;AA+BA,SAAS,kBAAkB,UAAuC;CAChE,IAAI,OAAO,SAAS,eAAe,UACjC,OAAO;EACL,WAAW,SAAS;EACpB,YAAY,SAAS;CACvB;CAGF,OAAO;EACL,WAAW,SAAS;EACpB,YAAY,mBAAmB,SAAS,UAAU;CACpD;AACF;;AAGA,SAAS,WAAW,UAAwD;CAC1E,MAAM,SAAS,eAAe,SAAS,UAAU;CACjD,IAAI,WAAW,QAAQ,OAAO,WAAW,YAAY,MAAM,QAAQ,MAAM,GACvE,OAAO;CAET,OAAO;AACT;;;;;;AAOA,SAAgB,cAAc,UAAiC;CAC7D,MAAM,aAAa,kBAAkB,QAAQ;CAC7C,IAAI,CAAC,WAAW,UAAU,KAAK,GAAG,OAAO;CAEzC,IAAI;EACF,KAAK,MAAM,WAAW,UAAU;EAChC,OAAO;CACT,QAAQ;EACN,OAAO;CACT;AACF;;;;;;AAOA,SAAgB,cACd,WACA,WACQ;CACR,MAAM,gBAAgB,kBAAkB,SAAS;CACjD,MAAM,gBAAgB,kBAAkB,SAAS;CACjD,OAAO,cAAc,cAAc,cAAc,YAAY,IAAI;AACnE;;;;;;AAOA,SAAgB,sBACd,WACA,WACQ;CACR,IAAI,cAAc,WAAW,SAAS,MAAM,GAAG,OAAO;CAEtD,MAAM,gBAAgB,WAAW,kBAAkB,SAAS,CAAC;CAC7D,MAAM,gBAAgB,WAAW,kBAAkB,SAAS,CAAC;CAC7D,IAAI,kBAAkB,QAAQ,kBAAkB,MAAM,OAAO;CAE7D,MAAM,gBAAgB,OAAO,KAAK,aAAa,CAAC,CAAC,KAAK;CACtD,MAAM,gBAAgB,OAAO,KAAK,aAAa,CAAC,CAAC,KAAK;CACtD,IAAI,cAAc,WAAW,cAAc,QAAQ,OAAO;CAE1D,OAAO,cAAc,OAAO,KAAK,UAAU,QAAQ,cAAc,MAAM,IACnE,IACA;AACN;;AAGA,SAAS,YACP,MACA,OACA,sBACS;CACT,IAAI,sBACF,OAAO,KAAK,UAAU,IAAI,MAAM,KAAK,UAAU,KAAK;CAEtD,OAAO,KAAK,UAAU,IAAI,MAAM,KAAK,UAAU,KAAK;AACtD;;;;;;;AAQA,SAAgB,qBACd,WACA,WACA,UAAiC,CAAC,GAC1B;CACR,IAAI,sBAAsB,WAAW,SAAS,MAAM,GAAG,OAAO;CAE9D,MAAM,gBAAgB,WAAW,kBAAkB,SAAS,CAAC;CAC7D,MAAM,gBAAgB,WAAW,kBAAkB,SAAS,CAAC;CAE7D,KAAK,MAAM,OAAO,OAAO,KAAK,aAAa,GACzC,IACE,CAAC,YACC,cAAc,MACd,cAAc,MACd,QAAQ,wBAAwB,KAClC,GAEA,OAAO;CAIX,OAAO;AACT;;;;;;;;AASA,SAAgB,uBACd,WACA,WACA,UAAiC,CAAC,GACjB;CACjB,MAAM,YAAY,KAAK,IAAI,UAAU,QAAQ,UAAU,QAAQ,CAAC;CAChE,IAAI,QAAQ;CACZ,IAAI,YAAY;CAChB,IAAI,WAAW;CACf,IAAI,UAAU;CAEd,KAAK,IAAI,QAAQ,GAAG,QAAQ,WAAW,SAAS,GAAG;EACjD,MAAM,gBAAgB,UAAU;EAChC,MAAM,gBAAgB,UAAU;EAChC,IAAI,CAAC,eAAe;EAEpB,SAAS,cAAc,aAAa;EACpC,IAAI,CAAC,eAAe;EAEpB,aAAa,cAAc,eAAe,aAAa;EACvD,YAAY,sBAAsB,eAAe,aAAa;EAC9D,WAAW,qBAAqB,eAAe,eAAe,OAAO;CACvE;CAEA,OAAO;EACL,iBAAiB,QAAQ;EACzB,iBAAiB,YAAY;EAC7B,0BAA0B,WAAW;EACrC,yBAAyB,UAAU;CACrC;AACF"}

package/dist/loader-B1WmGGzf.d.ts

@@ -0,0 +1,107 @@
+import { s as SuiteConfig } from "./types-C0gBkl0-.js";
+import { d as TestSuite, u as TestCase } from "./types-0QkNVyp9.js";
+import { z } from "zod";
+
+//#region src/config/transform.d.ts
+/**
+ * Thrown when a YAML suite fails to validate or transform. Carries a JSON-path-
+ * like trail so users can find the offending node in their config quickly.
+ */
+declare class ConfigError extends Error {
+  readonly path?: string | undefined;
+  constructor(message: string, path?: string | undefined);
+}
+//#endregion
+//#region src/config/grading-loader.d.ts
+/** Runtime shape of a parsed grading config file. */
+interface GradingConfig {
+  judge: SuiteConfig & {
+    adapter?: string;
+    maxConcurrent?: number;
+    system_instruction?: string;
+  };
+}
+/** Load grading YAML from disk and resolve relative paths. */
+declare function loadGradingConfig(filePath: string): Promise<GradingConfig>;
+/**
+ * Parse grading YAML from a string.
+ *
+ * @param sourcePath Optional path for error messages and path resolution.
+ */
+declare function parseGradingConfig(yamlContent: string, sourcePath?: string): GradingConfig;
+//#endregion
+//#region src/config/pipeline-schema.d.ts
+/** Runtime pipeline config with resolved absolute paths. */
+interface PipelineConfig {
+  run?: PipelineRunStep;
+  grade?: PipelineGradeStep;
+  envelope?: PipelineEnvelopeStep;
+}
+/** Resolved `pipeline.run` step — harness eval run output path. */
+interface PipelineRunStep {
+  output: string;
+  maxConcurrent?: number;
+}
+/** Resolved `pipeline.grade` step — LLM outcome grading inputs and output. */
+interface PipelineGradeStep {
+  input?: string;
+  output: string;
+  maxConcurrent?: number;
+}
+/** Resolved `pipeline.envelope` step — EvalRunEnvelope export options. */
+interface PipelineEnvelopeStep {
+  report?: string;
+  grading?: string;
+  output: string;
+  projection: "envelope" | "trajectory" | "instances";
+  includeRawStreamEvents?: boolean;
+  noTranscript?: boolean;
+}
+//#endregion
+//#region src/config/suite-document.d.ts
+/** Parsed suite.yaml including optional orchestration blocks. */
+interface SuiteDocument {
+  /** Absolute path to the suite.yaml file. */
+  suitePath: string;
+  suite: TestSuite;
+  judge?: GradingConfig["judge"];
+  pipeline?: PipelineConfig;
+}
+//#endregion
+//#region src/config/suite-document-loader.d.ts
+interface LoadSuiteDocumentOptions {
+  /**
+   * When true (default), `judge:` and `pipeline:` blocks are validated against
+   * their Zod schemas and returned in the result. When false, they are silently
+   * stripped — used by {@link loadSuite} so callers that only need the
+   * `TestSuite` are not broken by malformed orchestration blocks.
+   */
+  validateOrchestration?: boolean;
+}
+/** Load suite.yaml (or directory) including optional judge and pipeline blocks. */
+declare function loadSuiteDocument(filePath: string, options?: LoadSuiteDocumentOptions): Promise<SuiteDocument>;
+//#endregion
+//#region src/config/loader-internals.d.ts
+/** Parse one case file: single case, array, or `{ cases: [...] }`. */
+declare function parseCasesFile(yamlContent: string, sourcePath?: string): TestCase[];
+//#endregion
+//#region src/config/loader.d.ts
+/**
+ * Load a suite from a file path or directory path (suite portion only).
+ *
+ * Orchestration blocks (`judge:`, `pipeline:`) are silently stripped — callers
+ * that only need the `TestSuite` are not broken by malformed orchestration YAML.
+ * Use {@link loadSuiteDocument} when you need validated orchestration metadata.
+ */
+declare function loadSuite(filePath: string): Promise<TestSuite>;
+/**
+ * Parse suite YAML from a string (single-file layout with inline cases).
+ *
+ * Unknown top-level keys such as `judge` and `pipeline` are stripped.
+ */
+declare function parseSuite(yamlContent: string, sourcePath?: string): TestSuite;
+/** Parse `suite.yaml` for directory layout (cases may be omitted). @internal */
+declare function parseSuiteDirectory(yamlContent: string, sourcePath: string): TestSuite;
+//#endregion
+export { loadSuiteDocument as a, GradingConfig as c, ConfigError as d, parseCasesFile as i, loadGradingConfig as l, parseSuite as n, SuiteDocument as o, parseSuiteDirectory as r, PipelineConfig as s, loadSuite as t, parseGradingConfig as u };
+//# sourceMappingURL=loader-B1WmGGzf.d.ts.map