@alis-build/harness-eval 0.1.0 → 0.1.2

package/dist/index.js

@@ -1,5 +1,124 @@
-import { _ as isUserMessage, a as AdapterError, c as buildTrajectory, d as isResult, f as isSystemInit, g as isToolUseBlock, h as isToolResultBlock, l as namespaceOf, m as isTextBlock, n as claude_code_exports, o as parseStreamJson, p as isSystemRetry, s as TrajectoryBuilder, u as isAssistantMessage } from "./claude-code-ycT0JQZF.js";
-import { A as TRAJECTORY_SCHEMA_VERSION, C as gradeReport, D as emitOtel, E as createClaudeGrader, O as trajectoryToOtlp, S as resolveGradeOptions, T as trajectoryToTranscript, _ as trajectoryRecall, a as toProtoInstances, b as formatGradingConsole, c as toolCallValid, d as toolParameterKvMatch, f as computeTrajectoryMetrics, g as trajectoryPrecision, h as trajectoryInOrderMatch, i as toAgentTrace, k as EVAL_RUN_SCHEMA_VERSION, l as toolNameMatch, m as trajectoryExactMatch, n as buildEvalRunEnvelopeFromFiles, o as toTrajectory, p as trajectoryAnyOrderMatch, r as enrichRepetitionWithInterchange, s as computeToolCallMetrics, t as buildEvalRunEnvelope, u as toolParameterKeyMatch, v as trajectorySingleToolUse, x as gradingReportPassed, y as formatReport } from "./build-DsVJ_UeU.js";
-import { a as aggregateCell, c as runRepetition, d as getDefaultAdapter, f as listAdapters, h as evaluateAll, i as DEFAULT_THRESHOLD, l as DEFAULT_ADAPTER_ID, m as evaluate, n as createLimit, o as getRepetitions, p as registerAdapter, r as DEFAULT_REPETITIONS, s as mergeConfig, t as runSuite, u as getAdapter } from "./suite-chj0j22j.js";
-import { o as ConfigError, r as parseSuite, t as loadSuite } from "./loader-BCnFJ8rm.js";
-export { AdapterError, ConfigError, DEFAULT_ADAPTER_ID, DEFAULT_REPETITIONS, DEFAULT_THRESHOLD, EVAL_RUN_SCHEMA_VERSION, TRAJECTORY_SCHEMA_VERSION, TrajectoryBuilder, aggregateCell, buildEvalRunEnvelope, buildEvalRunEnvelopeFromFiles, buildTrajectory, claude_code_exports as claudeCode, computeToolCallMetrics, computeTrajectoryMetrics, createClaudeGrader, createLimit, emitOtel, enrichRepetitionWithInterchange, evaluate, evaluateAll, formatGradingConsole, formatReport, getAdapter, getDefaultAdapter, getRepetitions, gradeReport, gradingReportPassed, isAssistantMessage, isResult, isSystemInit, isSystemRetry, isTextBlock, isToolResultBlock, isToolUseBlock, isUserMessage, listAdapters, loadSuite, mergeConfig, namespaceOf, parseStreamJson, parseSuite, registerAdapter, resolveGradeOptions, runRepetition, runSuite, toAgentTrace, toProtoInstances, toTrajectory, toolCallValid, toolNameMatch, toolParameterKeyMatch, toolParameterKvMatch, trajectoryAnyOrderMatch, trajectoryExactMatch, trajectoryInOrderMatch, trajectoryPrecision, trajectoryRecall, trajectorySingleToolUse, trajectoryToOtlp, trajectoryToTranscript };
+import { _ as isUserMessage, a as AdapterError, c as buildTrajectory, d as isResult, f as isSystemInit, g as isToolUseBlock, h as isToolResultBlock, l as namespaceOf, m as isTextBlock, n as claude_code_exports, o as parseStreamJson, p as isSystemRetry, s as TrajectoryBuilder, u as isAssistantMessage } from "./claude-code-DZ4Vkgp6.js";
+import { D as trajectoryToOtlp, E as emitOtel, O as EVAL_RUN_SCHEMA_VERSION, S as gradeReport, T as createClaudeGrader, _ as serializeToolInput, a as enrichRepetitionWithProtojson, b as gradingReportPassed, c as parseToolInput, d as trajectoryInOrderMatch, f as trajectoryPrecision, g as toTrajectoryInstances, h as toEvaluationInstance, i as buildEvalRunEnvelopeFromFiles, k as TRAJECTORY_SCHEMA_VERSION, l as trajectoryAnyOrderMatch, m as trajectorySingleToolUse, n as toTrajectory, o as toHarnessMetrics, p as trajectoryRecall, r as buildEvalRunEnvelope, s as computeTrajectoryMetrics, t as toInstancesJsonl, u as trajectoryExactMatch, v as formatReport, w as trajectoryToTranscript, x as resolveGradeOptions, y as formatGradingConsole } from "./projections-BcX7w-f6.js";
+import { a as aggregateCell, c as runRepetition, d as getDefaultAdapter, f as listAdapters, h as evaluateAll, i as DEFAULT_THRESHOLD, l as DEFAULT_ADAPTER_ID, m as evaluate, n as createLimit, o as getRepetitions, p as registerAdapter, r as DEFAULT_REPETITIONS, s as mergeConfig, t as runSuite, u as getAdapter } from "./suite-Dlzl-HI0.js";
+import { o as ConfigError, r as parseSuite, t as loadSuite } from "./loader-DcI0KfRX.js";
+//#region src/metrics/tool-calls.ts
+/**
+* Tool-call-level metrics operating on prediction/reference tool-call pairs.
+*
+* Implements Vertex-aligned per-call checks: validity, name match, parameter
+* key match, and full key-value match. Used by trajectory metrics and
+* available for custom eval pipelines.
+*
+* Scores are 0 or 1 per call; {@link computeToolCallMetrics} averages across
+* aligned index pairs (max length of predicted vs reference).
+*/
+/** Normalize harness or wire tool call to canonical wire shape for comparison. */
+function normalizeToolCall(toolCall) {
+	if (typeof toolCall.tool_input === "string") return {
+		tool_name: toolCall.tool_name,
+		tool_input: toolCall.tool_input
+	};
+	return {
+		tool_name: toolCall.tool_name,
+		tool_input: serializeToolInput(toolCall.tool_input)
+	};
+}
+/** Parse tool_input JSON to an object map, or null when not a plain object. */
+function parsedArgs(toolCall) {
+	const parsed = parseToolInput(toolCall.tool_input);
+	if (parsed === null || typeof parsed !== "object" || Array.isArray(parsed)) return null;
+	return parsed;
+}
+/**
+* Whether a predicted tool call is well-formed (non-empty name, parseable JSON input).
+*
+* @returns 1 when valid, 0 otherwise.
+*/
+function toolCallValid(toolCall) {
+	const normalized = normalizeToolCall(toolCall);
+	if (!normalized.tool_name.trim()) return 0;
+	try {
+		JSON.parse(normalized.tool_input);
+		return 1;
+	} catch {
+		return 0;
+	}
+}
+/**
+* Whether predicted and reference tool names match exactly.
+*
+* @returns 1 on match, 0 otherwise.
+*/
+function toolNameMatch(predicted, reference) {
+	const predictedNorm = normalizeToolCall(predicted);
+	const referenceNorm = normalizeToolCall(reference);
+	return predictedNorm.tool_name === referenceNorm.tool_name ? 1 : 0;
+}
+/**
+* Whether parameter key sets match (same keys, same order after sort).
+*
+* Requires matching tool names first. Returns 0 when args are not objects.
+*/
+function toolParameterKeyMatch(predicted, reference) {
+	if (toolNameMatch(predicted, reference) === 0) return 0;
+	const predictedArgs = parsedArgs(normalizeToolCall(predicted));
+	const referenceArgs = parsedArgs(normalizeToolCall(reference));
+	if (predictedArgs === null || referenceArgs === null) return 0;
+	const predictedKeys = Object.keys(predictedArgs).sort();
+	const referenceKeys = Object.keys(referenceArgs).sort();
+	if (predictedKeys.length !== referenceKeys.length) return 0;
+	return predictedKeys.every((key, index) => key === referenceKeys[index]) ? 1 : 0;
+}
+/** Deep equality via JSON serialization (handles nested objects in args). */
+function valuesEqual(left, right, useStrictStringMatch) {
+	if (useStrictStringMatch) return JSON.stringify(left) === JSON.stringify(right);
+	return JSON.stringify(left) === JSON.stringify(right);
+}
+/**
+* Whether all reference parameter key-value pairs match in the predicted call.
+*
+* Requires {@link toolParameterKeyMatch} first. Only keys present in reference
+* are checked (predicted may have extra keys).
+*/
+function toolParameterKvMatch(predicted, reference, options = {}) {
+	if (toolParameterKeyMatch(predicted, reference) === 0) return 0;
+	const predictedArgs = parsedArgs(normalizeToolCall(predicted));
+	const referenceArgs = parsedArgs(normalizeToolCall(reference));
+	for (const key of Object.keys(referenceArgs)) if (!valuesEqual(predictedArgs[key], referenceArgs[key], options.useStrictStringMatch ?? false)) return 0;
+	return 1;
+}
+/**
+* Average tool-call metrics across index-aligned predicted/reference pairs.
+*
+* Denominator is `max(predicted.length, reference.length, 1)`. Missing
+* predicted calls at an index are skipped for pair metrics; validity still
+* counts when a predicted call exists.
+*/
+function computeToolCallMetrics(predicted, reference, options = {}) {
+	const pairCount = Math.max(predicted.length, reference.length, 1);
+	let valid = 0;
+	let nameMatch = 0;
+	let keyMatch = 0;
+	let kvMatch = 0;
+	for (let index = 0; index < pairCount; index += 1) {
+		const predictedCall = predicted[index];
+		const referenceCall = reference[index];
+		if (!predictedCall) continue;
+		valid += toolCallValid(predictedCall);
+		if (!referenceCall) continue;
+		nameMatch += toolNameMatch(predictedCall, referenceCall);
+		keyMatch += toolParameterKeyMatch(predictedCall, referenceCall);
+		kvMatch += toolParameterKvMatch(predictedCall, referenceCall, options);
+	}
+	return {
+		tool_call_valid: valid / pairCount,
+		tool_name_match: nameMatch / pairCount,
+		tool_parameter_key_match: keyMatch / pairCount,
+		tool_parameter_kv_match: kvMatch / pairCount
+	};
+}
+//#endregion
+export { AdapterError, ConfigError, DEFAULT_ADAPTER_ID, DEFAULT_REPETITIONS, DEFAULT_THRESHOLD, EVAL_RUN_SCHEMA_VERSION, TRAJECTORY_SCHEMA_VERSION, TrajectoryBuilder, aggregateCell, buildEvalRunEnvelope, buildEvalRunEnvelopeFromFiles, buildTrajectory, claude_code_exports as claudeCode, computeToolCallMetrics, computeTrajectoryMetrics, createClaudeGrader, createLimit, emitOtel, enrichRepetitionWithProtojson, evaluate, evaluateAll, formatGradingConsole, formatReport, getAdapter, getDefaultAdapter, getRepetitions, gradeReport, gradingReportPassed, isAssistantMessage, isResult, isSystemInit, isSystemRetry, isTextBlock, isToolResultBlock, isToolUseBlock, isUserMessage, listAdapters, loadSuite, mergeConfig, namespaceOf, parseStreamJson, parseSuite, registerAdapter, resolveGradeOptions, runRepetition, runSuite, toEvaluationInstance, toHarnessMetrics, toInstancesJsonl, toTrajectory, toTrajectoryInstances, toolCallValid, toolNameMatch, toolParameterKeyMatch, toolParameterKvMatch, trajectoryAnyOrderMatch, trajectoryExactMatch, trajectoryInOrderMatch, trajectoryPrecision, trajectoryRecall, trajectorySingleToolUse, trajectoryToOtlp, trajectoryToTranscript };
+
+//# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","names":[],"sources":["../src/metrics/tool-calls.ts"],"sourcesContent":["/**\n * Tool-call-level metrics operating on prediction/reference tool-call pairs.\n *\n * Implements Vertex-aligned per-call checks: validity, name match, parameter\n * key match, and full key-value match. Used by trajectory metrics and\n * available for custom eval pipelines.\n *\n * Scores are 0 or 1 per call; {@link computeToolCallMetrics} averages across\n * aligned index pairs (max length of predicted vs reference).\n */\n\nimport { parseToolInput, type TrajectoryInput, type WireToolCall } from \"./trajectory\";\nimport { serializeToolInput } from \"../eval-interchange/normalize\";\n\n/** Options for parameter value comparison. */\nexport interface ToolCallMetricOptions {\n  /** When true, compare serialized JSON strictly (reserved for future semantics). */\n  useStrictStringMatch?: boolean;\n}\n\n/** Aggregated tool-call metric scores (each 0..1). */\nexport interface ToolCallMetrics {\n  tool_call_valid: number;\n  tool_name_match: number;\n  tool_parameter_key_match: number;\n  tool_parameter_kv_match: number;\n}\n\ntype ToolCallInput = TrajectoryInput[number];\n\n/** Normalize harness or wire tool call to canonical wire shape for comparison. */\nfunction normalizeToolCall(toolCall: ToolCallInput): WireToolCall {\n  if (typeof toolCall.tool_input === \"string\") {\n    return {\n      tool_name: toolCall.tool_name,\n      tool_input: toolCall.tool_input,\n    };\n  }\n\n  return {\n    tool_name: toolCall.tool_name,\n    tool_input: serializeToolInput(toolCall.tool_input),\n  };\n}\n\n/** Parse tool_input JSON to an object map, or null when not a plain object. */\nfunction parsedArgs(toolCall: WireToolCall): Record<string, unknown> | null {\n  const parsed = parseToolInput(toolCall.tool_input);\n  if (parsed === null || typeof parsed !== \"object\" || Array.isArray(parsed)) {\n    return null;\n  }\n  return parsed as Record<string, unknown>;\n}\n\n/**\n * Whether a predicted tool call is well-formed (non-empty name, parseable JSON input).\n *\n * @returns 1 when valid, 0 otherwise.\n */\nexport function toolCallValid(toolCall: ToolCallInput): number {\n  const normalized = normalizeToolCall(toolCall);\n  if (!normalized.tool_name.trim()) return 0;\n\n  try {\n    JSON.parse(normalized.tool_input);\n    return 1;\n  } catch {\n    return 0;\n  }\n}\n\n/**\n * Whether predicted and reference tool names match exactly.\n *\n * @returns 1 on match, 0 otherwise.\n */\nexport function toolNameMatch(\n  predicted: ToolCallInput,\n  reference: ToolCallInput,\n): number {\n  const predictedNorm = normalizeToolCall(predicted);\n  const referenceNorm = normalizeToolCall(reference);\n  return predictedNorm.tool_name === referenceNorm.tool_name ? 1 : 0;\n}\n\n/**\n * Whether parameter key sets match (same keys, same order after sort).\n *\n * Requires matching tool names first. Returns 0 when args are not objects.\n */\nexport function toolParameterKeyMatch(\n  predicted: ToolCallInput,\n  reference: ToolCallInput,\n): number {\n  if (toolNameMatch(predicted, reference) === 0) return 0;\n\n  const predictedArgs = parsedArgs(normalizeToolCall(predicted));\n  const referenceArgs = parsedArgs(normalizeToolCall(reference));\n  if (predictedArgs === null || referenceArgs === null) return 0;\n\n  const predictedKeys = Object.keys(predictedArgs).sort();\n  const referenceKeys = Object.keys(referenceArgs).sort();\n  if (predictedKeys.length !== referenceKeys.length) return 0;\n\n  return predictedKeys.every((key, index) => key === referenceKeys[index])\n    ? 1\n    : 0;\n}\n\n/** Deep equality via JSON serialization (handles nested objects in args). */\nfunction valuesEqual(\n  left: unknown,\n  right: unknown,\n  useStrictStringMatch: boolean,\n): boolean {\n  if (useStrictStringMatch) {\n    return JSON.stringify(left) === JSON.stringify(right);\n  }\n  return JSON.stringify(left) === JSON.stringify(right);\n}\n\n/**\n * Whether all reference parameter key-value pairs match in the predicted call.\n *\n * Requires {@link toolParameterKeyMatch} first. Only keys present in reference\n * are checked (predicted may have extra keys).\n */\nexport function toolParameterKvMatch(\n  predicted: ToolCallInput,\n  reference: ToolCallInput,\n  options: ToolCallMetricOptions = {},\n): number {\n  if (toolParameterKeyMatch(predicted, reference) === 0) return 0;\n\n  const predictedArgs = parsedArgs(normalizeToolCall(predicted))!;\n  const referenceArgs = parsedArgs(normalizeToolCall(reference))!;\n\n  for (const key of Object.keys(referenceArgs)) {\n    if (\n      !valuesEqual(\n        predictedArgs[key],\n        referenceArgs[key],\n        options.useStrictStringMatch ?? false,\n      )\n    ) {\n      return 0;\n    }\n  }\n\n  return 1;\n}\n\n/**\n * Average tool-call metrics across index-aligned predicted/reference pairs.\n *\n * Denominator is `max(predicted.length, reference.length, 1)`. Missing\n * predicted calls at an index are skipped for pair metrics; validity still\n * counts when a predicted call exists.\n */\nexport function computeToolCallMetrics(\n  predicted: ToolCallInput[],\n  reference: ToolCallInput[],\n  options: ToolCallMetricOptions = {},\n): ToolCallMetrics {\n  const pairCount = Math.max(predicted.length, reference.length, 1);\n  let valid = 0;\n  let nameMatch = 0;\n  let keyMatch = 0;\n  let kvMatch = 0;\n\n  for (let index = 0; index < pairCount; index += 1) {\n    const predictedCall = predicted[index];\n    const referenceCall = reference[index];\n    if (!predictedCall) continue;\n\n    valid += toolCallValid(predictedCall);\n    if (!referenceCall) continue;\n\n    nameMatch += toolNameMatch(predictedCall, referenceCall);\n    keyMatch += toolParameterKeyMatch(predictedCall, referenceCall);\n    kvMatch += toolParameterKvMatch(predictedCall, referenceCall, options);\n  }\n\n  return {\n    tool_call_valid: valid / pairCount,\n    tool_name_match: nameMatch / pairCount,\n    tool_parameter_key_match: keyMatch / pairCount,\n    tool_parameter_kv_match: kvMatch / pairCount,\n  };\n}\n"],"mappings":";;;;;;;;;;;;;;;;AA+BA,SAAS,kBAAkB,UAAuC;CAChE,IAAI,OAAO,SAAS,eAAe,UACjC,OAAO;EACL,WAAW,SAAS;EACpB,YAAY,SAAS;CACvB;CAGF,OAAO;EACL,WAAW,SAAS;EACpB,YAAY,mBAAmB,SAAS,UAAU;CACpD;AACF;;AAGA,SAAS,WAAW,UAAwD;CAC1E,MAAM,SAAS,eAAe,SAAS,UAAU;CACjD,IAAI,WAAW,QAAQ,OAAO,WAAW,YAAY,MAAM,QAAQ,MAAM,GACvE,OAAO;CAET,OAAO;AACT;;;;;;AAOA,SAAgB,cAAc,UAAiC;CAC7D,MAAM,aAAa,kBAAkB,QAAQ;CAC7C,IAAI,CAAC,WAAW,UAAU,KAAK,GAAG,OAAO;CAEzC,IAAI;EACF,KAAK,MAAM,WAAW,UAAU;EAChC,OAAO;CACT,QAAQ;EACN,OAAO;CACT;AACF;;;;;;AAOA,SAAgB,cACd,WACA,WACQ;CACR,MAAM,gBAAgB,kBAAkB,SAAS;CACjD,MAAM,gBAAgB,kBAAkB,SAAS;CACjD,OAAO,cAAc,cAAc,cAAc,YAAY,IAAI;AACnE;;;;;;AAOA,SAAgB,sBACd,WACA,WACQ;CACR,IAAI,cAAc,WAAW,SAAS,MAAM,GAAG,OAAO;CAEtD,MAAM,gBAAgB,WAAW,kBAAkB,SAAS,CAAC;CAC7D,MAAM,gBAAgB,WAAW,kBAAkB,SAAS,CAAC;CAC7D,IAAI,kBAAkB,QAAQ,kBAAkB,MAAM,OAAO;CAE7D,MAAM,gBAAgB,OAAO,KAAK,aAAa,CAAC,CAAC,KAAK;CACtD,MAAM,gBAAgB,OAAO,KAAK,aAAa,CAAC,CAAC,KAAK;CACtD,IAAI,cAAc,WAAW,cAAc,QAAQ,OAAO;CAE1D,OAAO,cAAc,OAAO,KAAK,UAAU,QAAQ,cAAc,MAAM,IACnE,IACA;AACN;;AAGA,SAAS,YACP,MACA,OACA,sBACS;CACT,IAAI,sBACF,OAAO,KAAK,UAAU,IAAI,MAAM,KAAK,UAAU,KAAK;CAEtD,OAAO,KAAK,UAAU,IAAI,MAAM,KAAK,UAAU,KAAK;AACtD;;;;;;;AAQA,SAAgB,qBACd,WACA,WACA,UAAiC,CAAC,GAC1B;CACR,IAAI,sBAAsB,WAAW,SAAS,MAAM,GAAG,OAAO;CAE9D,MAAM,gBAAgB,WAAW,kBAAkB,SAAS,CAAC;CAC7D,MAAM,gBAAgB,WAAW,kBAAkB,SAAS,CAAC;CAE7D,KAAK,MAAM,OAAO,OAAO,KAAK,aAAa,GACzC,IACE,CAAC,YACC,cAAc,MACd,cAAc,MACd,QAAQ,wBAAwB,KAClC,GAEA,OAAO;CAIX,OAAO;AACT;;;;;;;;AASA,SAAgB,uBACd,WACA,WACA,UAAiC,CAAC,GACjB;CACjB,MAAM,YAAY,KAAK,IAAI,UAAU,QAAQ,UAAU,QAAQ,CAAC;CAChE,IAAI,QAAQ;CACZ,IAAI,YAAY;CAChB,IAAI,WAAW;CACf,IAAI,UAAU;CAEd,KAAK,IAAI,QAAQ,GAAG,QAAQ,WAAW,SAAS,GAAG;EACjD,MAAM,gBAAgB,UAAU;EAChC,MAAM,gBAAgB,UAAU;EAChC,IAAI,CAAC,eAAe;EAEpB,SAAS,cAAc,aAAa;EACpC,IAAI,CAAC,eAAe;EAEpB,aAAa,cAAc,eAAe,aAAa;EACvD,YAAY,sBAAsB,eAAe,aAAa;EAC9D,WAAW,qBAAqB,eAAe,eAAe,OAAO;CACvE;CAEA,OAAO;EACL,iBAAiB,QAAQ;EACzB,iBAAiB,YAAY;EAC7B,0BAA0B,WAAW;EACrC,yBAAyB,UAAU;CACrC;AACF"}

package/dist/{loader-DTvoVfN0.d.ts → loader-C9yQHUPC.d.ts}

@@ -1,5 +1,5 @@
 import { s as SuiteConfig } from "./types-B9H4IZtA.js";
-import { d as TestSuite, u as TestCase } from "./types-BQol062t.js";
+import { d as TestSuite, u as TestCase } from "./types-CD3TwOtZ.js";
 import { z } from "zod";
 
 //#region src/config/transform.d.ts
@@ -13,6 +13,7 @@ declare class ConfigError extends Error {
 }
 //#endregion
 //#region src/config/grading-loader.d.ts
+/** Runtime shape of a parsed grading config file. */
 interface GradingConfig {
   judge: SuiteConfig & {
     adapter?: string;
@@ -20,14 +21,30 @@ interface GradingConfig {
     system_instruction?: string;
   };
 }
+/** Load grading YAML from disk and resolve relative paths. */
 declare function loadGradingConfig(filePath: string): Promise<GradingConfig>;
+/**
+ * Parse grading YAML from a string.
+ *
+ * @param sourcePath Optional path for error messages and path resolution.
+ */
 declare function parseGradingConfig(yamlContent: string, sourcePath?: string): GradingConfig;
 //#endregion
 //#region src/config/loader.d.ts
+/**
+ * Load a suite from a file path or directory path.
+ *
+ * @throws {@link ConfigError} when the path is unreadable or validation fails.
+ */
 declare function loadSuite(filePath: string): Promise<TestSuite>;
+/**
+ * Parse suite YAML from a string (single-file layout with inline cases).
+ *
+ * @param sourcePath Optional path for error messages and relative path resolution.
+ */
 declare function parseSuite(yamlContent: string, sourcePath?: string): TestSuite;
 /** Parse one case file: single case, array, or `{ cases: [...] }`. */
 declare function parseCasesFile(yamlContent: string, sourcePath?: string): TestCase[];
 //#endregion
 export { loadGradingConfig as a, GradingConfig as i, parseCasesFile as n, parseGradingConfig as o, parseSuite as r, ConfigError as s, loadSuite as t };
-//# sourceMappingURL=loader-DTvoVfN0.d.ts.map
+//# sourceMappingURL=loader-C9yQHUPC.d.ts.map