@alis-build/harness-eval 0.1.0

package/dist/suite-chj0j22j.js

@@ -0,0 +1,684 @@
+import { t as claudeCodeAdapter } from "./claude-code-ycT0JQZF.js";
+import { n as parseCardinality, t as describeCardinality } from "./cardinality-DlE44e-4.js";
+//#region src/assertions/patterns.ts
+/**
+* Test whether a fully-qualified tool name matches a pattern.
+*
+* Literal patterns (no `*`) match by string equality. Glob patterns are
+* compiled to a regex on each call — fine for our scale (dozens of patterns,
+* thousands of calls per run). If this becomes a hot path, memoize.
+*/
+function toolMatches(toolName, pattern) {
+	const p = patternString(pattern);
+	if (!p.includes("*")) return toolName === p;
+	return globToRegex(p).test(toolName);
+}
+/** Extract the underlying string from either pattern form. */
+function patternString(pattern) {
+	return typeof pattern === "string" ? pattern : pattern.pattern;
+}
+/** Human-readable representation for diagnostic messages. */
+function describePattern(pattern) {
+	return patternString(pattern);
+}
+/**
+* Convert a glob (with `*` wildcards only) to an anchored regex.
+* Other regex metacharacters in the input are escaped.
+*/
+function globToRegex(glob) {
+	const escaped = glob.replace(/[.+?^${}()|[\]\\]/g, "\\$&").replace(/\*/g, ".*");
+	return new RegExp(`^${escaped}$`);
+}
+//#endregion
+//#region src/assertions/predicates.ts
+const LEAF_OPS = /* @__PURE__ */ new Set([
+	"equals",
+	"contains",
+	"not_contains",
+	"regex",
+	"gte",
+	"lte",
+	"gt",
+	"lt",
+	"one_of"
+]);
+const COMPOUND_OPS = /* @__PURE__ */ new Set([
+	"any_of",
+	"all_of",
+	"not"
+]);
+/**
+* Apply a predicate to a value. Returns true if the value satisfies the
+* predicate, false otherwise.
+*
+* The `predicate` parameter is typed as `unknown` because YAML deserialization
+* produces unconstrained shapes; runtime dispatch is the validation.
+*/
+function matches(value, predicate) {
+	if (!isPlainObject(predicate)) return deepEquals(value, predicate);
+	const obj = predicate;
+	const keys = Object.keys(obj);
+	if (keys.length === 1) {
+		const key = keys[0];
+		if (COMPOUND_OPS.has(key)) switch (key) {
+			case "any_of": return obj.any_of.some((sub) => matches(value, sub));
+			case "all_of": return obj.all_of.every((sub) => matches(value, sub));
+			case "not": return !matches(value, obj.not);
+		}
+		if (LEAF_OPS.has(key)) return matchesLeaf(value, key, obj[key]);
+	}
+	if (!isPlainObject(value)) return false;
+	const valueObj = value;
+	for (const [field, subPred] of Object.entries(obj)) if (!matches(valueObj[field], subPred)) return false;
+	return true;
+}
+/** Apply a single leaf operator. Caller guarantees `op` is in LEAF_OPS. */
+function matchesLeaf(value, op, target) {
+	switch (op) {
+		case "equals": return deepEquals(value, target);
+		case "contains": return typeof value === "string" && value.includes(target);
+		case "not_contains": return typeof value === "string" && !value.includes(target);
+		case "regex":
+			if (typeof value !== "string" || typeof target !== "string") return false;
+			try {
+				return new RegExp(target).test(value);
+			} catch {
+				return false;
+			}
+		case "gte": return typeof value === "number" && value >= target;
+		case "lte": return typeof value === "number" && value <= target;
+		case "gt": return typeof value === "number" && value > target;
+		case "lt": return typeof value === "number" && value < target;
+		case "one_of": return target.some((t) => deepEquals(value, t));
+		default: throw new Error(`unknown leaf operator: ${op}`);
+	}
+}
+function isPlainObject(x) {
+	return typeof x === "object" && x !== null && !Array.isArray(x);
+}
+/**
+* Structural equality for unknown values. Used by `equals` and `one_of`.
+* Strict — no coercions, no NaN-equals-NaN special case (matches `===`).
+*/
+function deepEquals(a, b) {
+	if (a === b) return true;
+	if (typeof a !== typeof b) return false;
+	if (a === null || b === null) return false;
+	if (typeof a !== "object") return false;
+	if (Array.isArray(a) !== Array.isArray(b)) return false;
+	if (Array.isArray(a) && Array.isArray(b)) {
+		if (a.length !== b.length) return false;
+		return a.every((v, i) => deepEquals(v, b[i]));
+	}
+	const aObj = a;
+	const bObj = b;
+	const aKeys = Object.keys(aObj);
+	const bKeys = Object.keys(bObj);
+	if (aKeys.length !== bKeys.length) return false;
+	return aKeys.every((k) => deepEquals(aObj[k], bObj[k]));
+}
+//#endregion
+//#region src/assertions/tool-calls.ts
+function evaluateCalled(view, assertion) {
+	const matching = view.toolCalls.filter((c) => toolMatches(c.name, assertion.tool));
+	const passed = parseCardinality(assertion.times)(matching.length);
+	return {
+		passed,
+		description: `called(${describePattern(assertion.tool)}, ${describeCardinality(assertion.times)})`,
+		details: passed ? `found ${matching.length} matching call(s)` : `found ${matching.length} call(s), expected ${describeCardinality(assertion.times)}`,
+		matches: matching
+	};
+}
+function evaluateNotCalled(view, assertion) {
+	const matching = view.toolCalls.filter((c) => toolMatches(c.name, assertion.tool));
+	const passed = matching.length === 0;
+	return {
+		passed,
+		description: `not_called(${describePattern(assertion.tool)})`,
+		details: passed ? "no matching calls" : `found ${matching.length} forbidden call(s)`,
+		matches: matching
+	};
+}
+function evaluateCalledAnyOf(view, assertion) {
+	const allMatches = [];
+	for (const pattern of assertion.tools) allMatches.push(...view.toolCalls.filter((c) => toolMatches(c.name, pattern)));
+	const passed = allMatches.length > 0;
+	return {
+		passed,
+		description: `called_any_of(${assertion.tools.map(describePattern).join(", ")})`,
+		details: passed ? `${allMatches.length} matching call(s)` : "no calls matched any pattern",
+		matches: allMatches
+	};
+}
+function evaluateCalledAllOf(view, assertion) {
+	const perPattern = assertion.tools.map((p) => ({
+		pattern: p,
+		matches: view.toolCalls.filter((c) => toolMatches(c.name, p))
+	}));
+	const missing = perPattern.filter((p) => p.matches.length === 0);
+	const passed = missing.length === 0;
+	return {
+		passed,
+		description: `called_all_of(${assertion.tools.map(describePattern).join(", ")})`,
+		details: passed ? "all patterns matched" : `missing: ${missing.map((m) => describePattern(m.pattern)).join(", ")}`,
+		matches: perPattern.flatMap((p) => p.matches)
+	};
+}
+function evaluateCalledBefore(view, assertion) {
+	const firsts = view.toolCalls.filter((c) => toolMatches(c.name, assertion.first));
+	const thens = view.toolCalls.filter((c) => toolMatches(c.name, assertion.then));
+	const desc = `called_before(${describePattern(assertion.first)} → ${describePattern(assertion.then)})`;
+	if (firsts.length === 0) return {
+		passed: false,
+		description: desc,
+		details: `no calls matching first`
+	};
+	if (thens.length === 0) return {
+		passed: false,
+		description: desc,
+		details: `no calls matching then`
+	};
+	const earliestFirst = Math.min(...firsts.map((c) => c.turnIndex));
+	const earliestThen = Math.min(...thens.map((c) => c.turnIndex));
+	const passed = earliestFirst < earliestThen;
+	return {
+		passed,
+		description: desc,
+		details: passed ? `first @ turn ${earliestFirst}, then @ turn ${earliestThen}` : `first @ turn ${earliestFirst}, then @ turn ${earliestThen} (not before)`,
+		matches: [...firsts, ...thens]
+	};
+}
+function evaluateSequence(view, assertion) {
+	const { tools, strict = false } = assertion;
+	const desc = `sequence([${tools.map(describePattern).join(" → ")}]${strict ? ", strict" : ""})`;
+	if (tools.length === 0) return {
+		passed: true,
+		description: desc,
+		details: "empty sequence trivially matches"
+	};
+	if (strict) {
+		if (view.toolCalls.length < tools.length) return {
+			passed: false,
+			description: desc,
+			details: "not enough tool calls"
+		};
+		for (let start = 0; start <= view.toolCalls.length - tools.length; start++) {
+			let ok = true;
+			for (let i = 0; i < tools.length; i++) if (!toolMatches(view.toolCalls[start + i].name, tools[i])) {
+				ok = false;
+				break;
+			}
+			if (ok) return {
+				passed: true,
+				description: desc,
+				details: `matched at positions ${start}..${start + tools.length - 1}`,
+				matches: view.toolCalls.slice(start, start + tools.length)
+			};
+		}
+		return {
+			passed: false,
+			description: desc,
+			details: "no contiguous match"
+		};
+	}
+	let idx = 0;
+	const matched = [];
+	for (const call of view.toolCalls) if (idx < tools.length && toolMatches(call.name, tools[idx])) {
+		matched.push(call);
+		idx++;
+	}
+	const passed = idx === tools.length;
+	return {
+		passed,
+		description: desc,
+		details: passed ? "matched in order" : `matched ${idx}/${tools.length}`,
+		matches: matched
+	};
+}
+function evaluateCalledWith(view, assertion) {
+	const candidates = view.toolCalls.filter((c) => toolMatches(c.name, assertion.tool));
+	const matching = candidates.filter((c) => matches(c.args, assertion.args));
+	const passed = matching.length > 0;
+	let details;
+	if (passed) details = `${matching.length} call(s) with matching args`;
+	else if (candidates.length === 0) details = `no calls to ${describePattern(assertion.tool)} at all`;
+	else details = `${candidates.length} call(s) but none with matching args`;
+	return {
+		passed,
+		description: `called_with(${describePattern(assertion.tool)}, args matching predicate)`,
+		details,
+		matches: matching
+	};
+}
+//#endregion
+//#region src/assertions/behavior.ts
+/**
+* Was the response delivered without using any tool? This is the primary
+* failure mode detector for the skills-loading problem: when the harness
+* ignores the MCP, the trace shows zero tool calls and one terminal
+* assistant turn with finish reason `end_turn`.
+*
+* "Without tool calls" is defined as `toolCalls.length === 0` AND the
+* response text is non-empty (so we don't confuse "answered blind" with
+* "session died before producing anything").
+*/
+function evaluateRespondedWithoutToolCalls(view, _assertion) {
+	const passed = view.toolCalls.length === 0 && view.finalResponse.length > 0;
+	return {
+		passed,
+		description: "responded_without_tool_calls",
+		details: passed ? "no tools called, response non-empty" : view.toolCalls.length > 0 ? `${view.toolCalls.length} tool call(s) made` : "response was empty (session probably aborted)"
+	};
+}
+function evaluateIterationsWithin(view, assertion) {
+	const n = view.usage.numTurns;
+	return {
+		passed: n <= assertion.max,
+		description: `iterations_within(${assertion.max})`,
+		details: `used ${n} turn(s)`
+	};
+}
+function evaluateCostWithinUsd(view, assertion) {
+	const cost = view.usage.totalCostUsd;
+	return {
+		passed: cost <= assertion.max,
+		description: `cost_within_usd(${assertion.max.toFixed(4)})`,
+		details: `used $${cost.toFixed(4)}`
+	};
+}
+function evaluateDurationWithinMs(view, assertion) {
+	const ms = view.usage.durationMs;
+	return {
+		passed: ms <= assertion.max,
+		description: `duration_within_ms(${assertion.max})`,
+		details: `took ${ms}ms`
+	};
+}
+function evaluateFinishedWith(view, assertion) {
+	const allowed = Array.isArray(assertion.reasons) ? assertion.reasons : [assertion.reasons];
+	const actual = view.finalStopReason;
+	return {
+		passed: actual !== null && allowed.includes(actual),
+		description: `finished_with(${allowed.join("|")})`,
+		details: `actual: ${actual ?? "(none)"}`
+	};
+}
+function evaluateResponseContains(view, assertion) {
+	const passed = view.finalResponse.includes(assertion.text);
+	return {
+		passed,
+		description: `response_contains(${JSON.stringify(assertion.text)})`,
+		details: passed ? "text found" : "text not in response"
+	};
+}
+function evaluateResponseNotContains(view, assertion) {
+	const passed = !view.finalResponse.includes(assertion.text);
+	return {
+		passed,
+		description: `response_not_contains(${JSON.stringify(assertion.text)})`,
+		details: passed ? "text absent" : "forbidden text found"
+	};
+}
+function evaluateResponseMatches(view, assertion) {
+	let passed;
+	let details;
+	try {
+		passed = new RegExp(assertion.pattern, assertion.flags).test(view.finalResponse);
+		details = passed ? "pattern matched" : "pattern did not match";
+	} catch (err) {
+		passed = false;
+		details = `invalid regex: ${err instanceof Error ? err.message : String(err)}`;
+	}
+	return {
+		passed,
+		description: `response_matches(/${assertion.pattern}/${assertion.flags ?? ""})`,
+		details
+	};
+}
+/**
+* Run an arbitrary user-supplied predicate against the view.
+*
+* Only available from programmatic test definition (the YAML loader cannot
+* produce functions). Catches thrown errors and reports them as failures so
+* one bad predicate doesn't take down a whole eval run.
+*/
+function evaluatePredicate(view, assertion) {
+	let passed = false;
+	let details;
+	try {
+		passed = assertion.fn(view);
+		details = passed ? "predicate returned true" : "predicate returned false";
+	} catch (err) {
+		details = `predicate threw: ${err instanceof Error ? err.message : String(err)}`;
+	}
+	return {
+		passed,
+		description: assertion.description ?? "predicate(...)",
+		details
+	};
+}
+//#endregion
+//#region src/assertions/compound.ts
+function evaluateAllOf(view, assertion, evaluate) {
+	const children = assertion.assertions.map((a) => evaluate(view, a));
+	const passed = children.every((c) => c.passed);
+	const failedCount = children.filter((c) => !c.passed).length;
+	return {
+		passed,
+		description: `all_of (${children.length} child${children.length === 1 ? "" : "ren"})`,
+		details: passed ? "all passed" : `${failedCount} of ${children.length} failed`,
+		children
+	};
+}
+function evaluateAnyOf(view, assertion, evaluate) {
+	const children = assertion.assertions.map((a) => evaluate(view, a));
+	const passedCount = children.filter((c) => c.passed).length;
+	const passed = passedCount > 0;
+	return {
+		passed,
+		description: `any_of (${children.length} child${children.length === 1 ? "" : "ren"})`,
+		details: passed ? `${passedCount} passed` : "all failed",
+		children
+	};
+}
+function evaluateNot(view, assertion, evaluate) {
+	const child = evaluate(view, assertion.assertion);
+	return {
+		passed: !child.passed,
+		description: `not(${child.description})`,
+		details: child.passed ? "inner passed (so outer fails)" : "inner failed (so outer passes)",
+		children: [child]
+	};
+}
+//#endregion
+//#region src/assertions/evaluator.ts
+/**
+* Evaluate one assertion against a trajectory view.
+*
+* The switch is exhaustive — TypeScript's `never` check at the end will
+* flag any new variant added to the `Assertion` union that hasn't been
+* wired up here.
+*/
+function evaluate(view, assertion) {
+	switch (assertion.type) {
+		case "called": return evaluateCalled(view, assertion);
+		case "not_called": return evaluateNotCalled(view, assertion);
+		case "called_any_of": return evaluateCalledAnyOf(view, assertion);
+		case "called_all_of": return evaluateCalledAllOf(view, assertion);
+		case "called_before": return evaluateCalledBefore(view, assertion);
+		case "sequence": return evaluateSequence(view, assertion);
+		case "called_with": return evaluateCalledWith(view, assertion);
+		case "responded_without_tool_calls": return evaluateRespondedWithoutToolCalls(view, assertion);
+		case "iterations_within": return evaluateIterationsWithin(view, assertion);
+		case "cost_within_usd": return evaluateCostWithinUsd(view, assertion);
+		case "duration_within_ms": return evaluateDurationWithinMs(view, assertion);
+		case "finished_with": return evaluateFinishedWith(view, assertion);
+		case "response_contains": return evaluateResponseContains(view, assertion);
+		case "response_not_contains": return evaluateResponseNotContains(view, assertion);
+		case "response_matches": return evaluateResponseMatches(view, assertion);
+		case "all_of": return evaluateAllOf(view, assertion, evaluate);
+		case "any_of": return evaluateAnyOf(view, assertion, evaluate);
+		case "not": return evaluateNot(view, assertion, evaluate);
+		case "predicate": return evaluatePredicate(view, assertion);
+		default: throw new Error(`unknown assertion: ${JSON.stringify(assertion)}`);
+	}
+}
+/**
+* Evaluate a list of assertions independently. Used at the test-case level
+* where each top-level assertion is reported separately (and thresholded
+* separately, in the runner layer).
+*/
+function evaluateAll(view, assertions) {
+	return assertions.map((a) => evaluate(view, a));
+}
+//#endregion
+//#region src/adapters/registry.ts
+const ADAPTERS = {};
+function registerBuiltIn(id, adapter) {
+	ADAPTERS[id] = adapter;
+}
+registerBuiltIn("claude-code", claudeCodeAdapter);
+/**
+* Register a harness adapter by id.
+*
+* Duplicate ids throw — registration is explicit so accidental overrides
+* surface immediately during startup or test setup.
+*/
+function registerAdapter(id, adapter) {
+	if (ADAPTERS[id]) throw new Error(`adapter "${id}" is already registered`);
+	ADAPTERS[id] = adapter;
+}
+/** Return all registered adapter ids (built-in and runtime). */
+function listAdapters() {
+	return Object.keys(ADAPTERS);
+}
+/** Resolve an adapter by id. Throws if unknown. */
+function getAdapter(id) {
+	const adapter = ADAPTERS[id];
+	if (!adapter) throw new Error(`unknown adapter "${id}". Available: ${listAdapters().join(", ")}`);
+	return adapter;
+}
+/** Default adapter when YAML omits `adapter`. */
+const DEFAULT_ADAPTER_ID = "claude-code";
+function getDefaultAdapter() {
+	return getAdapter(DEFAULT_ADAPTER_ID);
+}
+//#endregion
+//#region src/config/resolve-config.ts
+/**
+* Flatten nested suite config into harness-specific adapter config.
+*/
+/** Merge generic suite config layers into a flat {@link ClaudeCodeAdapterConfig}. */
+function toClaudeCodeConfig(layers, prompt) {
+	const merged = {};
+	for (const layer of layers) {
+		const { claudeCode, ...generic } = layer;
+		Object.assign(merged, generic);
+		if (claudeCode && typeof claudeCode === "object") Object.assign(merged, claudeCode);
+	}
+	merged.prompt = prompt;
+	return merged;
+}
+/**
+* Resolve merged suite layers into the flat config shape expected by the
+* selected harness adapter.
+*/
+function resolveRunConfig(adapterId, layers, prompt) {
+	if (adapterId === "claude-code" || adapterId === "claude-code") return toClaudeCodeConfig(layers, prompt);
+	const merged = {};
+	for (const layer of layers) Object.assign(merged, layer);
+	merged.prompt = prompt;
+	return merged;
+}
+//#endregion
+//#region src/runner/case.ts
+/** Default repetition count when `case.repetitions` is omitted. */
+const DEFAULT_REPETITIONS = 5;
+/** Default assertion pass-rate threshold when `threshold` is omitted. */
+const DEFAULT_THRESHOLD = 1;
+/**
+* Build the effective adapter config for one (suite, case, cell).
+*
+* Merge order (later wins): defaultConfig < case.config < cell.config.
+*/
+function mergeConfig(suite, testCase, cell) {
+	return resolveRunConfig(suite.adapter ?? getDefaultAdapter().id, [
+		suite.defaultConfig ?? {},
+		testCase.config ?? {},
+		cell.config
+	], testCase.prompt);
+}
+function getRepetitions(testCase) {
+	return testCase.repetitions ?? 5;
+}
+async function runRepetition(testCase, _cell, config, repetitionIndex, run, signal) {
+	const startTs = Date.now();
+	try {
+		const adapterResult = await run({
+			...config,
+			signal: signal ?? config.signal
+		});
+		return {
+			repetitionIndex,
+			adapterResult,
+			error: null,
+			assertionResults: evaluateAll(adapterResult.view, testCase.assertions.map((t) => t.assertion)),
+			durationMs: Date.now() - startTs
+		};
+	} catch (err) {
+		return {
+			repetitionIndex,
+			adapterResult: null,
+			error: extractError(err),
+			assertionResults: [],
+			durationMs: Date.now() - startTs
+		};
+	}
+}
+function extractError(err) {
+	const message = err instanceof Error ? err.message : String(err);
+	let diagnostics = {};
+	if (err !== null && typeof err === "object" && "diagnostics" in err) {
+		const d = err.diagnostics;
+		if (d !== null && typeof d === "object") diagnostics = d;
+	}
+	return {
+		message,
+		diagnostics
+	};
+}
+function aggregateCell(testCase, cell, repetitions) {
+	const adapterErrors = repetitions.filter((r) => r.error !== null).length;
+	const evaluatedReps = repetitions.filter((r) => r.error === null);
+	const assertionStats = testCase.assertions.map((thresholded, i) => {
+		const threshold = thresholded.threshold ?? 1;
+		const passedCount = evaluatedReps.filter((r) => r.assertionResults[i]?.passed).length;
+		const evaluatedCount = evaluatedReps.length;
+		const passRate = evaluatedCount === 0 ? 0 : passedCount / evaluatedCount;
+		return {
+			description: evaluatedReps[0]?.assertionResults[i]?.description ?? `(${thresholded.assertion.type})`,
+			threshold,
+			passedCount,
+			evaluatedCount,
+			passRate,
+			meetsThreshold: evaluatedCount > 0 && passRate >= threshold
+		};
+	});
+	const passed = assertionStats.every((s) => s.meetsThreshold);
+	return {
+		caseId: testCase.id,
+		category: testCase.category,
+		notes: testCase.notes,
+		prompt: testCase.prompt,
+		expectations: testCase.expectations,
+		reference_trajectory: testCase.reference_trajectory,
+		human_ratings: testCase.human_ratings,
+		cell,
+		repetitions,
+		assertionStats,
+		adapterErrors,
+		passed
+	};
+}
+//#endregion
+//#region src/runner/limit.ts
+function createLimit(max) {
+	if (!Number.isInteger(max) || max < 1) throw new Error(`createLimit: max must be a positive integer, got ${max}`);
+	let running = 0;
+	/**
+	* FIFO list of resolvers belonging to tasks waiting for a slot. When a
+	* running task finishes, the next resolver is invoked to wake one waiter.
+	*/
+	const waiters = [];
+	return async (fn) => {
+		while (running >= max) await new Promise((resolve) => waiters.push(resolve));
+		running++;
+		try {
+			return await fn();
+		} finally {
+			running--;
+			const next = waiters.shift();
+			if (next) next();
+		}
+	};
+}
+//#endregion
+//#region src/runner/suite.ts
+/**
+* Suite-level runner.
+*/
+const DEFAULT_MAX_CONCURRENT = 4;
+async function runSuite(suite, options = {}) {
+	if (suite.matrix.length === 0) throw new Error("runSuite: suite.matrix must contain at least one cell");
+	if (suite.cases.length === 0) throw new Error("runSuite: suite.cases must contain at least one case");
+	const adapter = options.adapter ?? getAdapter(suite.adapter ?? getDefaultAdapter().id);
+	const run = (config) => adapter.run(config);
+	const limit = createLimit(options.maxConcurrent ?? DEFAULT_MAX_CONCURRENT);
+	const onProgress = options.onProgress;
+	const startTs = Date.now();
+	const startedAt = new Date(startTs).toISOString();
+	const tasks = [];
+	for (const testCase of suite.cases) {
+		const reps = getRepetitions(testCase);
+		for (const cell of suite.matrix) for (let i = 0; i < reps; i++) tasks.push({
+			testCase,
+			cell,
+			repetitionIndex: i
+		});
+	}
+	onProgress?.({
+		kind: "suite-start",
+		totalReps: tasks.length
+	});
+	const buckets = /* @__PURE__ */ new Map();
+	const bucketKey = (caseId, cellLabel) => `${caseId}::${cellLabel}`;
+	for (const testCase of suite.cases) for (const cell of suite.matrix) buckets.set(bucketKey(testCase.id, cell.label), []);
+	await Promise.all(tasks.map((task) => limit(async () => {
+		if (options.signal?.aborted) return;
+		onProgress?.({
+			kind: "rep-start",
+			caseId: task.testCase.id,
+			cellLabel: task.cell.label,
+			repIndex: task.repetitionIndex
+		});
+		const config = mergeConfig(suite, task.testCase, task.cell);
+		const result = await runRepetition(task.testCase, task.cell, config, task.repetitionIndex, run, options.signal);
+		buckets.get(bucketKey(task.testCase.id, task.cell.label)).push(result);
+		onProgress?.({
+			kind: "rep-complete",
+			caseId: task.testCase.id,
+			cellLabel: task.cell.label,
+			repIndex: task.repetitionIndex,
+			ok: result.error === null,
+			durationMs: result.durationMs,
+			toolCallCount: result.adapterResult?.view.toolCalls.length,
+			assertionResults: result.assertionResults,
+			errorMessage: result.error?.message
+		});
+	})));
+	const cells = [];
+	for (const testCase of suite.cases) for (const cell of suite.matrix) {
+		const reps = buckets.get(bucketKey(testCase.id, cell.label)) ?? [];
+		reps.sort((a, b) => a.repetitionIndex - b.repetitionIndex);
+		const cellReport = aggregateCell(testCase, cell, reps);
+		cells.push(cellReport);
+		onProgress?.({
+			kind: "cell-complete",
+			report: cellReport
+		});
+	}
+	const report = {
+		startedAt,
+		durationMs: Date.now() - startTs,
+		cells
+	};
+	onProgress?.({
+		kind: "suite-complete",
+		report
+	});
+	return report;
+}
+//#endregion
+export { aggregateCell as a, runRepetition as c, getDefaultAdapter as d, listAdapters as f, evaluateAll as h, DEFAULT_THRESHOLD as i, DEFAULT_ADAPTER_ID as l, evaluate as m, createLimit as n, getRepetitions as o, registerAdapter as p, DEFAULT_REPETITIONS as r, mergeConfig as s, runSuite as t, getAdapter as u };
+
+//# sourceMappingURL=suite-chj0j22j.js.map